From 451e37df0a24d04b31177c997be55334096cd672 Mon Sep 17 00:00:00 2001 From: tpoisot Date: Thu, 17 Oct 2024 11:44:08 -0400 Subject: [PATCH 01/13] feat(gbif): download API start --- GBIF/Project.toml | 2 +- GBIF/src/GBIF.jl | 6 ++++-- GBIF/src/download.jl | 31 +++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 GBIF/src/download.jl diff --git a/GBIF/Project.toml b/GBIF/Project.toml index 21acb80f0..b3815f861 100644 --- a/GBIF/Project.toml +++ b/GBIF/Project.toml @@ -1,7 +1,7 @@ name = "GBIF" uuid = "ee291a33-5a6c-5552-a3c8-0f29a1181037" authors = ["Timothée Poisot "] -version = "0.5.0" +version = "1.0.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/GBIF/src/GBIF.jl b/GBIF/src/GBIF.jl index ad2156251..68ff0d47b 100644 --- a/GBIF/src/GBIF.jl +++ b/GBIF/src/GBIF.jl @@ -12,12 +12,12 @@ function safeget(endpoint) while !eof(http) append!(body, readavailable(http)) end - close(HTTP.Connections.getrawstream(http)) + return close(HTTP.Connections.getrawstream(http)) end return rsp.status, String(body) end -const gbifurl = "http://api.gbif.org/v1/" +const gbifurl = "https://api.gbif.org/v1/" """ enumerablevalues() @@ -85,6 +85,8 @@ include("paging.jl") export occurrence, occurrences export occurrences! +include("download.jl") + include("occurrencesinterface.jl") end # module diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl new file mode 100644 index 000000000..b8015aff7 --- /dev/null +++ b/GBIF/src/download.jl @@ -0,0 +1,31 @@ +username() = get(ENV, "GBIF_USERNAME", missing) +email() = get(ENV, "GBIF_EMAIL", missing) + +function username!(un::String) + ENV["GBIF_USERNAME"] = un + return GBIF.username() +end + +function email!(em::String) + ENV["GBIF_EMAIL"] = em + return GBIF.email() +end + +""" + download + +Prepares a request for a download through the GBIF API +""" +function download(query::Pair...) + isempty(GBIF.username()) && throw(ErrorException("Use GBIF.username! to login")) + isempty(GBIF.email()) && throw(ErrorException("Use GBIF.email! to login")) +end + +function _predicate(query::Pair...) + querystring = pairs_to_querystring(query...) + predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate/" + pre_s_req = HTTP.get(predicate_url; query=querystring + if pre_s_req.status == 200 + return JSON.parse(String(pre_s_req.body)) + end +end From d302cdb85c691eb27fd3c40e23e507049102502c Mon Sep 17 00:00:00 2001 From: tpoisot Date: Thu, 31 Oct 2024 06:11:07 -0400 Subject: [PATCH 02/13] semver(gbif): v1.1.0 --- GBIF/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GBIF/Project.toml b/GBIF/Project.toml index 0ff9073da..4671f869f 100644 --- a/GBIF/Project.toml +++ b/GBIF/Project.toml @@ -1,7 +1,7 @@ name = "GBIF" uuid = "ee291a33-5a6c-5552-a3c8-0f29a1181037" authors = ["Timothée Poisot "] -version = "1.0.0" +version = "1.1.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From ed9149783ac80851fdcd36aee0791b3bef4f4209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 2 Nov 2024 16:28:10 -0400 Subject: [PATCH 03/13] bug(gbif): missing closing parenthesis --- GBIF/src/download.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index b8015aff7..820e0e064 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -24,7 +24,7 @@ end function _predicate(query::Pair...) querystring = pairs_to_querystring(query...) predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate/" - pre_s_req = HTTP.get(predicate_url; query=querystring + pre_s_req = HTTP.get(predicate_url; query=querystring) if pre_s_req.status == 200 return JSON.parse(String(pre_s_req.body)) end From 70967aa4c2f35bb245041d1f37801780b032366a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 2 Nov 2024 16:29:57 -0400 Subject: [PATCH 04/13] doc(gbif): changelog --- docs/src/reference/gbif/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/reference/gbif/CHANGELOG.md b/docs/src/reference/gbif/CHANGELOG.md index c2bc78a5b..e67d2b289 100644 --- a/docs/src/reference/gbif/CHANGELOG.md +++ b/docs/src/reference/gbif/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## `v1.1.0` + +- **added** support for the download API + ## `v1.0.0` - **added** support for `OccurrencesInterface` at version 1 From b133120c0b43ea78b88f580eedbb6f51c9cd5ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sun, 3 Nov 2024 18:14:16 -0500 Subject: [PATCH 05/13] feat(gbif)?: see own downloads --- GBIF/src/download.jl | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index 820e0e064..4901d4539 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -1,5 +1,6 @@ username() = get(ENV, "GBIF_USERNAME", missing) email() = get(ENV, "GBIF_EMAIL", missing) +password() = get(ENV, "GBIF_PASSWORD", missing) function username!(un::String) ENV["GBIF_USERNAME"] = un @@ -11,19 +12,45 @@ function email!(em::String) return GBIF.email() end +function password!(pw::String) + ENV["GBIF_PASSWORD"] = pw + return GBIF.password() +end + +""" + mydownloads() + +Returns the downloads associated to the currently authenticated user. +""" +function mydownloads() + ismissing(GBIF.username()) && throw(ErrorException("Use GBIF.username! to login")) + ismissing(GBIF.email()) && throw(ErrorException("Use GBIF.email! to login")) + +end + """ download Prepares a request for a download through the GBIF API """ function download(query::Pair...) - isempty(GBIF.username()) && throw(ErrorException("Use GBIF.username! to login")) - isempty(GBIF.email()) && throw(ErrorException("Use GBIF.email! to login")) + ismissing(GBIF.username()) && throw(ErrorException("Use GBIF.username! to login")) + ismissing(GBIF.email()) && throw(ErrorException("Use GBIF.email! to login")) + # Get the predicates + pred = GBIF.predicate(query...) + # end +""" + _predicate(query::Pair...) + +Given the arguments that are accepted by GBIF.occurrences, returns the +corresponding predicates for the download API. +""" function _predicate(query::Pair...) - querystring = pairs_to_querystring(query...) - predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate/" + query = (query..., "format" => "simpleCsv") + querystring = pairs_to_querystring(query...) + predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate" pre_s_req = HTTP.get(predicate_url; query=querystring) if pre_s_req.status == 200 return JSON.parse(String(pre_s_req.body)) From 5fe92e725546cbedf23ebca7077a18212e7cdbd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sun, 3 Nov 2024 18:20:02 -0500 Subject: [PATCH 06/13] feat(gbif): login --- GBIF/src/download.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index 4901d4539..58281df69 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -18,24 +18,24 @@ function password!(pw::String) end """ - mydownloads() - -Returns the downloads associated to the currently authenticated user. +Returns a dict to be used as part of the headers for HTTP functions to do +authentication against the download API """ -function mydownloads() - ismissing(GBIF.username()) && throw(ErrorException("Use GBIF.username! to login")) - ismissing(GBIF.email()) && throw(ErrorException("Use GBIF.email! to login")) - +function apiauth() + uname = GBIF.username() + passwd = GBIF.password() + temp = "Basic " * base64encode("$(uname):$(passwd)") + auth = Dict("Authorization" => temp) + return auth end + """ download Prepares a request for a download through the GBIF API """ function download(query::Pair...) - ismissing(GBIF.username()) && throw(ErrorException("Use GBIF.username! to login")) - ismissing(GBIF.email()) && throw(ErrorException("Use GBIF.email! to login")) # Get the predicates pred = GBIF.predicate(query...) # From 2afe96dd40f005f05ac60611fe0b666f33021a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sun, 3 Nov 2024 18:28:46 -0500 Subject: [PATCH 07/13] chore(gbif)?: TODO --- GBIF/src/download.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index 58281df69..edecf2973 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -22,6 +22,7 @@ Returns a dict to be used as part of the headers for HTTP functions to do authentication against the download API """ function apiauth() + # TODO: errors if username or password are not set uname = GBIF.username() passwd = GBIF.password() temp = "Basic " * base64encode("$(uname):$(passwd)") @@ -29,7 +30,6 @@ function apiauth() return auth end - """ download @@ -38,7 +38,8 @@ Prepares a request for a download through the GBIF API function download(query::Pair...) # Get the predicates pred = GBIF.predicate(query...) - # + # + return nothing end """ @@ -51,7 +52,7 @@ function _predicate(query::Pair...) query = (query..., "format" => "simpleCsv") querystring = pairs_to_querystring(query...) predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate" - pre_s_req = HTTP.get(predicate_url; query=querystring) + pre_s_req = HTTP.get(predicate_url; query = querystring) if pre_s_req.status == 200 return JSON.parse(String(pre_s_req.body)) end From d30d0d35ae3e34fba7dbf58b13b2e3d16c3a60d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Tue, 5 Nov 2024 14:20:05 -0500 Subject: [PATCH 08/13] feat(gbif): check for uname/passwd --- GBIF/src/download.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index edecf2973..6cf1f7a6e 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -25,6 +25,12 @@ function apiauth() # TODO: errors if username or password are not set uname = GBIF.username() passwd = GBIF.password() + if ismissing(uname) + throw(ErrorException("The GBIF username is missing - see the documentation for GBIF.username!")) + end + if ismissing(passwd) + throw(ErrorException("The GBIF password is missing - see the documentation for GBIF.password!")) + end temp = "Basic " * base64encode("$(uname):$(passwd)") auth = Dict("Authorization" => temp) return auth From f3bc1ee8f14c1280b51cba80a6107593df35cb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Thu, 7 Nov 2024 14:48:57 -0500 Subject: [PATCH 09/13] dep(gbif): Base64 --- GBIF/Project.toml | 2 ++ GBIF/src/GBIF.jl | 1 + GBIF/src/download.jl | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/GBIF/Project.toml b/GBIF/Project.toml index 4671f869f..48b9b26f4 100644 --- a/GBIF/Project.toml +++ b/GBIF/Project.toml @@ -4,6 +4,7 @@ authors = ["Timothée Poisot "] version = "1.1.0" [deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" @@ -11,6 +12,7 @@ OccurrencesInterface = "ee6415c8-122a-4855-89a1-90f4bac06ba6" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] +Base64 = "1" HTTP = "1" JSON = "0.21" OccurrencesInterface = "1" diff --git a/GBIF/src/GBIF.jl b/GBIF/src/GBIF.jl index 68ff0d47b..7e73c8703 100644 --- a/GBIF/src/GBIF.jl +++ b/GBIF/src/GBIF.jl @@ -4,6 +4,7 @@ using HTTP using JSON using Dates using Tables +import Base64 import OccurrencesInterface function safeget(endpoint) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index 6cf1f7a6e..0b1380dde 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -19,10 +19,10 @@ end """ Returns a dict to be used as part of the headers for HTTP functions to do -authentication against the download API +authentication against the download API. This returns a dictionary that is used +internally by calls to endpoints for the download API. """ function apiauth() - # TODO: errors if username or password are not set uname = GBIF.username() passwd = GBIF.password() if ismissing(uname) @@ -31,7 +31,7 @@ function apiauth() if ismissing(passwd) throw(ErrorException("The GBIF password is missing - see the documentation for GBIF.password!")) end - temp = "Basic " * base64encode("$(uname):$(passwd)") + temp = "Basic " * Base64.base64encode("$(uname):$(passwd)") auth = Dict("Authorization" => temp) return auth end From 889ad78e0ac3a5164d75cf6ce122f1b603051ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Thu, 7 Nov 2024 14:52:18 -0500 Subject: [PATCH 10/13] feat(gbif): download --- GBIF/src/download.jl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index 0b1380dde..771133ad9 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -41,9 +41,13 @@ end Prepares a request for a download through the GBIF API """ -function download(query::Pair...) +function download(query::Pair...; notification::Bool=false) # Get the predicates - pred = GBIF.predicate(query...) + predicates = GBIF._predicate(query...) + if notification + predicates["sendNotification"] = true + push!(predicates["notificationAddresses"], GBIF.email()) + end # return nothing end @@ -58,7 +62,7 @@ function _predicate(query::Pair...) query = (query..., "format" => "simpleCsv") querystring = pairs_to_querystring(query...) predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate" - pre_s_req = HTTP.get(predicate_url; query = querystring) + pre_s_req = HTTP.get(predicate_url; query = querystring, headers = GBIF.apiauth()) if pre_s_req.status == 200 return JSON.parse(String(pre_s_req.body)) end From 45f5416182cc11e1b7043a2398d68a32f8e10d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Thu, 7 Nov 2024 15:10:35 -0500 Subject: [PATCH 11/13] feat(gbif): list own downloads --- GBIF/src/download.jl | 66 ++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index 771133ad9..bd9708925 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -26,32 +26,24 @@ function apiauth() uname = GBIF.username() passwd = GBIF.password() if ismissing(uname) - throw(ErrorException("The GBIF username is missing - see the documentation for GBIF.username!")) + throw( + ErrorException( + "The GBIF username is missing - see the documentation for GBIF.username!", + ), + ) end if ismissing(passwd) - throw(ErrorException("The GBIF password is missing - see the documentation for GBIF.password!")) + throw( + ErrorException( + "The GBIF password is missing - see the documentation for GBIF.password!", + ), + ) end temp = "Basic " * Base64.base64encode("$(uname):$(passwd)") auth = Dict("Authorization" => temp) return auth end -""" - download - -Prepares a request for a download through the GBIF API -""" -function download(query::Pair...; notification::Bool=false) - # Get the predicates - predicates = GBIF._predicate(query...) - if notification - predicates["sendNotification"] = true - push!(predicates["notificationAddresses"], GBIF.email()) - end - # - return nothing -end - """ _predicate(query::Pair...) @@ -67,3 +59,41 @@ function _predicate(query::Pair...) return JSON.parse(String(pre_s_req.body)) end end + +""" + download + +Prepares a request for a download through the GBIF API +""" +function download(query::Pair...; notification::Bool = false) + # Get the predicates + predicates = GBIF._predicate(query...) + if notification + predicates["sendNotification"] = true + push!(predicates["notificationAddresses"], GBIF.email()) + end + # Make the request + request_url = GBIF.gbifurl * "occurrence/download/request" + @info request_url + request_resp = HTTP.post(request_url; body = predicates, headers = GBIF.apiauth()) + @info request_resp + return nothing +end + +function mydownloads(; preparing=true, running=true, succeeded=true, cancelled=true, killed=true, failed=true, suspended=true, erased=true) + statuses = String[] + preparing && push!(statuses, "PREPARING") + running && push!(statuses, "RUNNING") + succeeded && push!(statuses, "SUCCEEDED") + cancelled && push!(statuses, "CANCELLED") + killed && push!(statuses, "KILLED") + failed && push!(statuses, "FAILED") + suspended && push!(statuses, "SUSPENDED") + erased && push!(statuses, "FILE_ERASED") + # Get the predicates + request_url = GBIF.gbifurl * "occurrence/download/user/$(GBIF.username())" + request_resp = HTTP.get(request_url; query="status=$(join(statuses, ','))", headers = GBIF.apiauth()) + if request_resp.status == 200 + return JSON.parse(String(request_resp.body)) + end +end \ No newline at end of file From f70a629ea34e15f25ea79365f558dcc7b840f65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Thu, 7 Nov 2024 15:19:18 -0500 Subject: [PATCH 12/13] feat(gbif): download function complete --- GBIF/src/download.jl | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index bd9708925..db62d77ef 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -65,7 +65,7 @@ end Prepares a request for a download through the GBIF API """ -function download(query::Pair...; notification::Bool = false) +function request(query::Pair...; notification::Bool = false) # Get the predicates predicates = GBIF._predicate(query...) if notification @@ -80,6 +80,23 @@ function download(query::Pair...; notification::Bool = false) return nothing end +""" + download(key) + +Downloads the zip file associated to a specific query (identified by its key) as +a zip file. +""" +function download(key) + request_url = GBIF.gbifurl * "occurrence/download/request/$(key)" + dl_req = HTTP.get(request_url; headers=GBIF.apiauth()) + if dl_req.status == 200 + # Get that bag + open("$(key).zip", "w") do f + write(f, dl_req.body) + end + end +end + function mydownloads(; preparing=true, running=true, succeeded=true, cancelled=true, killed=true, failed=true, suspended=true, erased=true) statuses = String[] preparing && push!(statuses, "PREPARING") From b57f379b182e56b3760da45e31868f127ae0b98f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Thu, 7 Nov 2024 15:19:54 -0500 Subject: [PATCH 13/13] feat(gbif): return dl file after success --- GBIF/src/download.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl index db62d77ef..60b2e49be 100644 --- a/GBIF/src/download.jl +++ b/GBIF/src/download.jl @@ -94,6 +94,7 @@ function download(key) open("$(key).zip", "w") do f write(f, dl_req.body) end + return "$(key).zip" end end