diff --git a/GBIF/Project.toml b/GBIF/Project.toml index 0ff9073da..48b9b26f4 100644 --- a/GBIF/Project.toml +++ b/GBIF/Project.toml @@ -1,9 +1,10 @@ name = "GBIF" uuid = "ee291a33-5a6c-5552-a3c8-0f29a1181037" authors = ["Timothée Poisot "] -version = "1.0.0" +version = "1.1.0" [deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" @@ -11,6 +12,7 @@ OccurrencesInterface = "ee6415c8-122a-4855-89a1-90f4bac06ba6" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] +Base64 = "1" HTTP = "1" JSON = "0.21" OccurrencesInterface = "1" diff --git a/GBIF/src/GBIF.jl b/GBIF/src/GBIF.jl index ad2156251..7e73c8703 100644 --- a/GBIF/src/GBIF.jl +++ b/GBIF/src/GBIF.jl @@ -4,6 +4,7 @@ using HTTP using JSON using Dates using Tables +import Base64 import OccurrencesInterface function safeget(endpoint) @@ -12,12 +13,12 @@ function safeget(endpoint) while !eof(http) append!(body, readavailable(http)) end - close(HTTP.Connections.getrawstream(http)) + return close(HTTP.Connections.getrawstream(http)) end return rsp.status, String(body) end -const gbifurl = "http://api.gbif.org/v1/" +const gbifurl = "https://api.gbif.org/v1/" """ enumerablevalues() @@ -85,6 +86,8 @@ include("paging.jl") export occurrence, occurrences export occurrences! +include("download.jl") + include("occurrencesinterface.jl") end # module diff --git a/GBIF/src/download.jl b/GBIF/src/download.jl new file mode 100644 index 000000000..60b2e49be --- /dev/null +++ b/GBIF/src/download.jl @@ -0,0 +1,117 @@ +username() = get(ENV, "GBIF_USERNAME", missing) +email() = get(ENV, "GBIF_EMAIL", missing) +password() = get(ENV, "GBIF_PASSWORD", missing) + +function username!(un::String) + ENV["GBIF_USERNAME"] = un + return GBIF.username() +end + +function email!(em::String) + ENV["GBIF_EMAIL"] = em + return GBIF.email() +end + +function password!(pw::String) + ENV["GBIF_PASSWORD"] = pw + return GBIF.password() +end + +""" +Returns a dict to be used as part of the headers for HTTP functions to do +authentication against the download API. This returns a dictionary that is used +internally by calls to endpoints for the download API. +""" +function apiauth() + uname = GBIF.username() + passwd = GBIF.password() + if ismissing(uname) + throw( + ErrorException( + "The GBIF username is missing - see the documentation for GBIF.username!", + ), + ) + end + if ismissing(passwd) + throw( + ErrorException( + "The GBIF password is missing - see the documentation for GBIF.password!", + ), + ) + end + temp = "Basic " * Base64.base64encode("$(uname):$(passwd)") + auth = Dict("Authorization" => temp) + return auth +end + +""" + _predicate(query::Pair...) + +Given the arguments that are accepted by GBIF.occurrences, returns the +corresponding predicates for the download API. +""" +function _predicate(query::Pair...) + query = (query..., "format" => "simpleCsv") + querystring = pairs_to_querystring(query...) + predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate" + pre_s_req = HTTP.get(predicate_url; query = querystring, headers = GBIF.apiauth()) + if pre_s_req.status == 200 + return JSON.parse(String(pre_s_req.body)) + end +end + +""" + download + +Prepares a request for a download through the GBIF API +""" +function request(query::Pair...; notification::Bool = false) + # Get the predicates + predicates = GBIF._predicate(query...) + if notification + predicates["sendNotification"] = true + push!(predicates["notificationAddresses"], GBIF.email()) + end + # Make the request + request_url = GBIF.gbifurl * "occurrence/download/request" + @info request_url + request_resp = HTTP.post(request_url; body = predicates, headers = GBIF.apiauth()) + @info request_resp + return nothing +end + +""" + download(key) + +Downloads the zip file associated to a specific query (identified by its key) as +a zip file. +""" +function download(key) + request_url = GBIF.gbifurl * "occurrence/download/request/$(key)" + dl_req = HTTP.get(request_url; headers=GBIF.apiauth()) + if dl_req.status == 200 + # Get that bag + open("$(key).zip", "w") do f + write(f, dl_req.body) + end + return "$(key).zip" + end +end + +function mydownloads(; preparing=true, running=true, succeeded=true, cancelled=true, killed=true, failed=true, suspended=true, erased=true) + statuses = String[] + preparing && push!(statuses, "PREPARING") + running && push!(statuses, "RUNNING") + succeeded && push!(statuses, "SUCCEEDED") + cancelled && push!(statuses, "CANCELLED") + killed && push!(statuses, "KILLED") + failed && push!(statuses, "FAILED") + suspended && push!(statuses, "SUSPENDED") + erased && push!(statuses, "FILE_ERASED") + # Get the predicates + request_url = GBIF.gbifurl * "occurrence/download/user/$(GBIF.username())" + request_resp = HTTP.get(request_url; query="status=$(join(statuses, ','))", headers = GBIF.apiauth()) + if request_resp.status == 200 + return JSON.parse(String(request_resp.body)) + end +end \ No newline at end of file diff --git a/docs/src/reference/gbif/CHANGELOG.md b/docs/src/reference/gbif/CHANGELOG.md index c2bc78a5b..e67d2b289 100644 --- a/docs/src/reference/gbif/CHANGELOG.md +++ b/docs/src/reference/gbif/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## `v1.1.0` + +- **added** support for the download API + ## `v1.0.0` - **added** support for `OccurrencesInterface` at version 1