Merge branch 'master' into time_conversion

.github/workflows/ci.yml

@@ -37,6 +37,7 @@ jobs:
         pip install wheel
         pip install papermill jupyter_client astropy oda-api>=1.1.1 nbconvert scipy --upgrade --ignore-installed
         pip install 'git+https://github.com/integral-observatory/astroquery.git#egg=astroquery'
+        pip install --force-reinstall 'urllib3<2.0' # TODO: to remove after https://github.com/oda-hub/oda_api/issues/198
         export HOME_OVERRRIDE=/tmp/home
         mkdir -pv /tmp/home
         export

.github/workflows/python-publish.yml

@@ -1,28 +1,67 @@
-name: Upload Python Package
+name: Bump version workflow
 
 on:
-  release:
-    types: [created]
+  push:
+    branches:
+    - master
+    - ci
+    # - main
 
 jobs:
-  deploy:
-
-    runs-on: ubuntu-latest
-
+  bump-version:
+    name: Bump package version
+    if: "!contains(github.event.head_commit.message, 'Bump version')"
+    runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
+    - name: actions/checkout
+      uses: actions/checkout@v2
+      # with:
+      #     persist-credentials: false
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        pip install build bump2version
+
+    - name: Bump version
       run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+        git config --global user.email "[email protected]"
+        git config --global user.name "ODA Release Bot"
+        bump2version patch --verbose --commit --tag
+        bump2version release --verbose --commit --tag
+        git push --tags
+
+    - uses: stefanzweifel/git-auto-commit-action@v4
+
+    - name: Build package
+      run: python -m build
+
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        verify_metadata: false
+        password: ${{ secrets.PYPI_API_TOKEN }}        
+
+    #TODO: #211 fix this
+    # - name: Conda publish
+    #   uses: silvxlabs/conda-skeleton-publish@v2
+    #   with:
+    #     # Name of PyPI package
+    #     pypi_package: oda-api # optional
+    #     # Package version on PyPi
+    #     #package_version: # optional, default is latest
+    #     # Python version to build
+    #     #python_version: # optional
+    #     # Conda channels to use for building package
+    #     build_channels: conda-forge
+    #     # Conda channel to upload to
+    #     upload_channel: mmoda# optional
+    #     # Anaconda client access token
+    #     access_token: ${{ secrets.CONDA_ACCESS_TOKEN }}
+    #     # Platforms to publish
+    #     #platforms: # optional, default is win-64 osx-64 osx-arm64 linux-64 linux-aarch64
+    #     # Set this to true if this is a stable release. Passese a `-l beta` arg if false.
+    #     #stable: # optional
+    #     # If true, waits for the PyPi package to become available.
+    #     wait: true # optional

.readthedocs.yaml

@@ -1,2 +1,22 @@
-github:
-  build_pull_requests: true
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: doc/source/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: doc/requirements.txt

README.md

@@ -2,7 +2,11 @@ oda_api
 ==========================================
 
 [![PyPI version](https://badge.fury.io/py/oda-api.svg)](https://badge.fury.io/py/oda-api)
+![](https://anaconda.org/mmoda/oda-api/badges/version.svg)
+![](https://anaconda.org/mmoda/oda-api/badges/latest_release_date.svg)
 [![Documentation Status](https://readthedocs.org/projects/oda-api/badge/?version=latest)](https://oda-api.readthedocs.io/en/latest/?badge=latest)
+
+
 ![Upload Python Package](https://github.com/cdcihub/oda_api/workflows/Upload%20Python%20Package/badge.svg)
 ![Prose Linting](https://github.com/volodymyrss/integral-isgri-rate-meaning/workflows/Prose%20Linting/badge.svg)
 ![Executing Notebook](https://github.com/volodymyrss/integral-isgri-rate-meaning/workflows/Executing%20Notebook/badge.svg)

doc/source/install.rst

@@ -19,7 +19,7 @@ Development version
 
 Download code and contribute
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Source code can be found can be found on github https://github.com/oda-hub/oda_api  (see https://github.com/oda-hub/oda_api/archive/refs/tags/v1.1.36.tar.gz for latest release bundle).
+Source code can be found can be found on github https://github.com/oda-hub/oda_api  (see https://github.com/oda-hub/oda_api/archive/refs/tags/v1.1.43.tar.gz for latest release bundle).
 
 To contribute, please feel free to fork the repository, apply your changes, and prepare a pull request.
 

doc/source/user_guide/TestAPI.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Quick Start (oda api v1.1.36)"
+    "# Quick Start (oda api v1.1.43)"
    ]
   },
   {

doc/source/user_guide/UploadToGallery.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Upload of a product to the gallery (oda api v1.1.36)\n",
+    "# Upload of a product to the gallery (oda api v1.1.43)\n",
     "\n",
     "This notebooks documents the functionality for uploading a data product over the data-product gallery.\n",
     "\n",

oda_api/api.py

@@ -7,6 +7,7 @@
 import pathlib
 
 import rdflib
+from json.decoder import JSONDecodeError 
 
 # NOTE gw is optional for now
 try:
@@ -16,7 +17,14 @@
 except ModuleNotFoundError:
     pass
 
-from .data_products import NumpyDataProduct, BinaryData, ApiCatalog, GWContoursDataProduct, PictureProduct
+from .data_products import (NumpyDataProduct, 
+                            BinaryData, 
+                            BinaryProduct, 
+                            ApiCatalog, 
+                            GWContoursDataProduct, 
+                            PictureProduct,
+                            ODAAstropyTable,
+                            TextLikeProduct)
 from oda_api.token import TokenLocation
 
 from builtins import (bytes, str, open, super, range,
@@ -234,6 +242,7 @@ def __init__(self,
                  n_max_tries=200,
                  session_id=None,
                  use_local_cache=False,
+                 token=None
                  ):
 
         self.setup_loggers()
@@ -282,7 +291,9 @@ def __init__(self,
 
         if session_id is not None:
             self._session_id = session_id
-
+
+        self.token = token if token is not None else self.get_token_from_environment()
+
         self._carriage_return_progress = False
 
         self.run_analysis_handle = run_analysis_handle
@@ -928,13 +939,20 @@ def _decode_res_json(self, res):
 
             raise RemoteException(message=msg)
 
+    def get_token_from_environment(self):
+        token = oda_api.token.discover_token(self.token_discovery_methods)
+        if token is not None:
+            logger.info("discovered token in environment")
+
+        return token
+
     @safe_run
     def get_instrument_description(self, instrument=None):
         if instrument is None:
             instrument = self.instrument
 
         res = requests.get("%s/api/meta-data" % self.url,
-                           params=dict(instrument=instrument), cookies=self.cookies)
+                           params=dict(instrument=instrument, token=self.token), cookies=self.cookies)
 
         if res.status_code != 200:
             raise UnexpectedDispatcherStatusCode(
@@ -945,7 +963,7 @@ def get_instrument_description(self, instrument=None):
     @safe_run
     def get_product_description(self, instrument, product_name):
         res = requests.get("%s/api/meta-data" % self.url, params=dict(
-            instrument=instrument, product_type=product_name), cookies=self.cookies)
+            instrument=instrument, product_type=product_name, token=self.token), cookies=self.cookies)
 
         if res.status_code != 200:
             raise UnexpectedDispatcherStatusCode(
@@ -958,9 +976,8 @@ def get_product_description(self, instrument, product_name):
 
     @safe_run
     def get_instruments_list(self):
-        # print ('instr',self.instrument)
         res = requests.get("%s/api/instr-list" % self.url,
-                           params=dict(instrument=self.instrument), cookies=self.cookies)
+                           params=dict(instrument=self.instrument, token=self.token), cookies=self.cookies)
 
         if res.status_code != 200:
             raise UnexpectedDispatcherStatusCode(
@@ -1104,9 +1121,13 @@ def set_api_code(query_dict, url="www.astro.unige.ch/mmoda/dispatch-data"):
                 if query_dict[k] is not None:
                     _api_dict[n] = query_dict[k]
 
+        python_compatible_par_dict_str = json.dumps(_api_dict, indent=4)
+        python_compatible_par_dict_str = python_compatible_par_dict_str.replace('false', 'False')
+        python_compatible_par_dict_str = python_compatible_par_dict_str.replace('true', 'True')
+
         _cmd_ = f'''{_header}
 
-par_dict={json.dumps(_api_dict, indent=4)}
+par_dict={python_compatible_par_dict_str}
 
 data_collection = disp.get_product(**par_dict)
 '''
@@ -1158,9 +1179,9 @@ def __init__(self, data_list, add_meta_to_name=['src_name', 'product'], instrume
 
             name = ''
             if hasattr(data, 'name'):
-                name = data.name
+                name = data.name    
 
-            if name.strip() == '':
+            if name is None or name.strip() == '':
                 if product is not None:
                     name = '%s' % product
                 elif instrument is not None:
@@ -1250,28 +1271,36 @@ def from_response_json(cls, res_json, instrument, product):
                          for d in res_json['products']['numpy_data_product_list']])
 
         if 'binary_data_product_list' in res_json['products'].keys():
-            data.extend([BinaryData().decode(d)
-                         for d in res_json['products']['binary_data_product_list']])
+            try:
+                data.extend([BinaryProduct.decode(d)
+                             for d in res_json['products']['binary_data_product_list']])
+            except:
+                data.extend([BinaryData().decode(d)
+                             for d in res_json['products']['binary_data_product_list']])
 
         if 'catalog' in res_json['products'].keys():
             data.append(ApiCatalog(
                 res_json['products']['catalog'], name='dispatcher_catalog'))
 
-        if 'astropy_table_product_ascii_list' in res_json['products'].keys():
-            data.extend([ascii.read(table_text['ascii'])
+        if 'astropy_table_product_ascii_list' in res_json['products'].keys():         
+            data.extend([ODAAstropyTable.decode(table_text, use_binary=False)
                          for table_text in res_json['products']['astropy_table_product_ascii_list']])
 
         if 'astropy_table_product_binary_list' in res_json['products'].keys():
-            data.extend([ascii.read(table_binary)
+            data.extend([ODAAstropyTable.decode(table_binary, use_binary=True)
                          for table_binary in res_json['products']['astropy_table_product_binary_list']])
 
         if 'binary_image_product_list' in res_json['products'].keys():
             data.extend([PictureProduct.decode(bin_image_data)
                          for bin_image_data in res_json['products']['binary_image_product_list']])
 
         if 'text_product_list' in res_json['products'].keys():
-            data.extend([text_data
-                         for text_data in res_json['products']['text_product_list']])
+            try:
+                data.extend([TextLikeProduct.decode(text_data)
+                             for text_data in res_json['products']['text_product_list']])
+            except (JSONDecodeError, KeyError):
+                data.extend([text_data
+                             for text_data in res_json['products']['text_product_list']])
 
         if 'gw_strain_product_list' in res_json['products'].keys():
             data.extend([TimeSeries(strain_data['value'],

oda_api/cli.py

@@ -1,15 +1,13 @@
 from datetime import datetime
 from email.policy import default
 import json
-from attr import validate
-from black import out
 import click
 import logging
 import time
 
 import oda_api.api as api
 from oda_api import plot_tools
-from oda_api.token import discover_token, decode_oda_token, format_token, update_token
+from oda_api.token import discover_token, discover_token_and_method, decode_oda_token, format_token, update_token, TokenLocation
 
 logger = logging.getLogger('oda_api')
 
@@ -41,9 +39,16 @@ def cli(obj, debug=False, dispatcher_url=None, test_connection=False, wait=True,
 @click.option("-i", "--instrument", default=None)
 @click.option("-p", "--product", default=None)
 @click.option("-a", "--argument", default=None, multiple=True)
-@click.option("-T", "--discover-token", "_discover_token", is_flag=True)
+@click.option("-T", "--token-discovery-method", "_token_discovery_method", default=None)
 @click.pass_obj
-def get(obj, instrument, product, argument, _discover_token):
+def get(obj, instrument, product, argument, _token_discovery_method):
+    if _token_discovery_method is not None:
+        _token_discovery_method = TokenLocation[str.upper(_token_discovery_method)]
+        discovered_token = discover_token(token_discovery_methods=_token_discovery_method)
+        if discovered_token is None:
+            logger.info("A token could not be found with the desired method, if present, the one automatically discovered will be used")
+        else:
+            obj['dispatcher'].token = discovered_token
     if instrument is None:
         logger.info("found instruments: %s", obj['dispatcher'].get_instruments_list())
     else:
@@ -62,8 +67,7 @@ def get(obj, instrument, product, argument, _discover_token):
 
             logger.debug("request to dispatcher %s", request)
 
-            if _discover_token:
-                request['token'] = discover_token()
+            request['token'] = obj['dispatcher'].token
 
             product = obj['dispatcher'].get_product(**request)