Download API for GBIF

GBIF/Project.toml

@@ -1,16 +1,18 @@
 name = "GBIF"
 uuid = "ee291a33-5a6c-5552-a3c8-0f29a1181037"
 authors = ["Timothée Poisot <[email protected]>"]
-version = "1.0.0"
+version = "1.1.0"
 
 [deps]
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 OccurrencesInterface = "ee6415c8-122a-4855-89a1-90f4bac06ba6"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
+Base64 = "1"
 HTTP = "1"
 JSON = "0.21"
 OccurrencesInterface = "1"

GBIF/src/GBIF.jl

@@ -4,6 +4,7 @@ using HTTP
 using JSON
 using Dates
 using Tables
+import Base64
 import OccurrencesInterface
 
 function safeget(endpoint)
@@ -12,12 +13,12 @@ function safeget(endpoint)
         while !eof(http)
             append!(body, readavailable(http))
         end
-        close(HTTP.Connections.getrawstream(http))
+        return close(HTTP.Connections.getrawstream(http))
     end
     return rsp.status, String(body)
 end
 
-const gbifurl = "http://api.gbif.org/v1/"
+const gbifurl = "https://api.gbif.org/v1/"
 
 """
     enumerablevalues()
@@ -85,6 +86,8 @@ include("paging.jl")
 export occurrence, occurrences
 export occurrences!
 
+include("download.jl")
+
 include("occurrencesinterface.jl")
 
 end # module

GBIF/src/download.jl

@@ -0,0 +1,117 @@
+username() = get(ENV, "GBIF_USERNAME", missing)
+email() = get(ENV, "GBIF_EMAIL", missing)
+password() = get(ENV, "GBIF_PASSWORD", missing)
+
+function username!(un::String)
+    ENV["GBIF_USERNAME"] = un
+    return GBIF.username()
+end
+
+function email!(em::String)
+    ENV["GBIF_EMAIL"] = em
+    return GBIF.email()
+end
+
+function password!(pw::String)
+    ENV["GBIF_PASSWORD"] = pw
+    return GBIF.password()
+end
+
+"""
+Returns a dict to be used as part of the headers for HTTP functions to do
+authentication against the download API. This returns a dictionary that is used
+internally by calls to endpoints for the download API.
+"""
+function apiauth()
+    uname = GBIF.username()
+    passwd = GBIF.password()
+    if ismissing(uname)
+        throw(
+            ErrorException(
+                "The GBIF username is missing - see the documentation for GBIF.username!",
+            ),
+        )
+    end
+    if ismissing(passwd)
+        throw(
+            ErrorException(
+                "The GBIF password is missing - see the documentation for GBIF.password!",
+            ),
+        )
+    end
+    temp = "Basic " * Base64.base64encode("$(uname):$(passwd)")
+    auth = Dict("Authorization" => temp)
+    return auth
+end
+
+"""
+    _predicate(query::Pair...)
+
+Given the arguments that are accepted by GBIF.occurrences, returns the
+corresponding predicates for the download API.
+"""
+function _predicate(query::Pair...)
+    query = (query..., "format" => "simpleCsv")
+    querystring = pairs_to_querystring(query...)
+    predicate_url = GBIF.gbifurl * "occurrence/download/request/predicate"
+    pre_s_req = HTTP.get(predicate_url; query = querystring, headers = GBIF.apiauth())
+    if pre_s_req.status == 200
+        return JSON.parse(String(pre_s_req.body))
+    end
+end
+
+"""
+    download
+
+Prepares a request for a download through the GBIF API
+"""
+function request(query::Pair...; notification::Bool = false)
+    # Get the predicates
+    predicates = GBIF._predicate(query...)
+    if notification
+        predicates["sendNotification"] = true
+        push!(predicates["notificationAddresses"], GBIF.email())
+    end
+    # Make the request
+    request_url = GBIF.gbifurl * "occurrence/download/request"
+    @info request_url
+    request_resp = HTTP.post(request_url; body = predicates, headers = GBIF.apiauth())
+    @info request_resp
+    return nothing
+end
+
+"""
+    download(key)
+
+Downloads the zip file associated to a specific query (identified by its key) as
+a zip file.
+"""
+function download(key)
+    request_url = GBIF.gbifurl * "occurrence/download/request/$(key)"
+    dl_req = HTTP.get(request_url; headers=GBIF.apiauth())
+    if dl_req.status == 200
+        # Get that bag
+        open("$(key).zip", "w") do f
+            write(f, dl_req.body)
+        end
+        return "$(key).zip"
+    end
+end
+
+function mydownloads(; preparing=true, running=true, succeeded=true, cancelled=true, killed=true, failed=true, suspended=true, erased=true)
+    statuses = String[]
+    preparing && push!(statuses, "PREPARING")
+    running && push!(statuses, "RUNNING")
+    succeeded && push!(statuses, "SUCCEEDED")
+    cancelled && push!(statuses, "CANCELLED")
+    killed && push!(statuses, "KILLED")
+    failed && push!(statuses, "FAILED")
+    suspended && push!(statuses, "SUSPENDED")
+    erased && push!(statuses, "FILE_ERASED")
+    # Get the predicates
+    request_url = GBIF.gbifurl * "occurrence/download/user/$(GBIF.username())"
+    request_resp = HTTP.get(request_url; query="status=$(join(statuses, ','))", headers = GBIF.apiauth())
+    if request_resp.status == 200
+        return JSON.parse(String(request_resp.body))
+    end
+end

docs/src/reference/gbif/CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## `v1.1.0`
+
+- **added** support for the download API
+
 ## `v1.0.0`
 
 - **added** support for `OccurrencesInterface` at version 1