Orion as a service

.github/workflows/build.yml

@@ -170,6 +170,41 @@ jobs:
         env_vars: PLATFORM,PYTHON
         name: codecov-umbrella
         fail_ci_if_error: false
+
+  service:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+          python -m pip install --upgrade pip
+          pip install tox tox-gh-actions
+
+    - name: Install MongoDB
+      run: |
+        wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
+        echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+        sudo apt-get update
+        sudo apt-get install -y mongodb-org
+        mongod --version
+        mongosh --version
+
+    - name: Test with tox
+      run: tox -e service
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml
+        flags: unittests
+        env_vars: PLATFORM,PYTHON
+        name: codecov-umbrella
+        fail_ci_if_error: false
+
   mongodb:
     needs: [pre-commit, pretest]
     runs-on: ubuntu-latest

.gitignore

@@ -88,6 +88,7 @@ docs/src/auto_examples
 docs/src/auto_tutorials
 docs/src/gen_modules
 
+out.txt
 
 # Data and checkpoints downloaded or generated by the Profet tasks.
 profet_data

.pylintrc

@@ -235,7 +235,7 @@ docstring-min-length=-1
 function-rgx=^(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
 
 # Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,v,ex,Run,_,f,e,X,y,a,b,rv
+good-names=i,j,k,v,ex,Run,_,f,e,X,y,a,b,rv,id
 
 # Include a hint for the correct naming format with invalid-name
 include-naming-hint=no

MANIFEST.in

@@ -18,6 +18,7 @@ exclude .pylintrc
 exclude codecov.yml
 exclude .mailmap
 exclude .readthedocs.yml
+exclude scripts/setup.sh
 prune conda/
 prune .github/
 

README.rst

@@ -122,7 +122,7 @@ If you use Oríon for published work, please cite our work using the following b
 
 .. code-block:: bibtex
 
-   @software{xavier_bouthillier_2022_0_2_6,
+   @software{xavier_bouthillier_2023_0_2_7,
      author       = {Xavier Bouthillier and
                      Christos Tsirigotis and
                      François Corneau-Tremblay and
@@ -147,10 +147,10 @@ If you use Oríon for published work, please cite our work using the following b
                      Pascal Lamblin and
                      Christopher Beckham},
      title        = {{Epistimio/orion: Asynchronous Distributed Hyperparameter Optimization}},
-     month        = august,
-     year         = 2022,
+     month        = march,
+     year         = 2023,
      publisher    = {Zenodo},
-     version      = {v0.2.6,
+     version      = {v0.2.7,
      doi          = {10.5281/zenodo.3478592},
      url          = {https://doi.org/10.5281/zenodo.3478592}
    }

ROADMAP.md

@@ -1,8 +1,10 @@
 # Roadmap
-Last update August 18th, 2022
+Last update March 2nd, 2023
 
 ## Next releases - Short-Term
 
+### v0.2.8
+
 ### v0.2.7
 
 ### v0.2.6

scripts/setup.sh

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+# Setup a mongodb server for testing
+# this setup is done my IT when the DB is deployed
+# set -evx
+
+PORT=${MONGO_PORT:-"8123"}
+ADDRESS=${MONGO_ADDRESS:-"localhost"}
+ADMIN=${MONGO_ADMIN:-"god"}
+PASSWORD=${MONGO_PASS:-"god123"}
+DB_PATH=${MONGO_PATH:-"/tmp/db"}
+
+function start_init_mongod {
+    # Start mongodb without Access Control, this is used to insert the admin user
+    # the db is then stopped and started with Access Control
+
+    if test -f "$DB_PATH/pid"; then
+        pid=$(cat $DB_PATH/pid)
+        kill -3 $pid || true
+    fi
+
+    rm -rf $DB_PATH
+    mkdir -p $DB_PATH
+
+    mongod --dbpath $DB_PATH/ --wiredTigerCacheSizeGB 1 --port $PORT --bind_ip localhost --pidfilepath $DB_PATH/pid &
+    sleep 1
+}
+
+function start_mongod {
+    # start mongodb with Access Control
+    mongod --auth --dbpath $DB_PATH/ --wiredTigerCacheSizeGB 1 --port $PORT --bind_ip $ADDRESS --pidfilepath $DB_PATH/pid --fork --logpath $DB_PATH/log.txt
+    sleep 1
+}
+
+function stop_mongod  {
+    # mongo --dbpath $DB_PATH --shutdown
+
+    echo "$(pwd)"
+    pid=$(cat $DB_PATH/pid)
+    kill -s TERM $pid
+    rm -rf $DB_PATH/pid
+    sleep 1
+}
+
+function add_admin_user {
+    # create an admin user
+    # userAdminAnyDatabase: create users, grant & revoke roles, create and modify roles
+
+    CMD=$(cat <<EOM
+    use admin
+    db.createUser({
+        user: "$ADMIN",
+        pwd: "$PASSWORD",
+        roles: [
+            { role: "userAdminAnyDatabase", db: "admin" },
+            { role: "readWriteAnyDatabase", db: "admin" },
+        ]
+    })
+EOM
+    )
+
+    echo "$CMD" | mongosh --port $PORT
+
+    add_user $ADMIN $PASSWORD
+}
+
+function add_user {
+    # Create a user for the orion database
+    #
+    #   How to make a group setup
+    #       - make a Token => Group
+    #       - Token => User => Group
+
+    username=$1
+    password=$2
+    token=$3
+
+    CMD=$(cat << EOM
+    use orion
+    db.createUser({
+        user: "$username",
+        pwd: "$password",
+        roles: [
+            { role: "readWrite", db: "orion" }
+        ]
+    })
+
+    db.users.insertOne({
+        username: "$username",
+        password: "$password",
+        token: "$token",
+    })
+EOM
+    )
+
+    echo "$CMD" | mongosh "mongodb://$ADDRESS:$PORT" --authenticationDatabase "admin" -u $ADMIN -p $PASSWORD
+}
+
+
+ASCENDING=1
+DESCENDING=-1
+
+function ensure_indexes {
+    # User will have limited access to the collection
+    # so orion's client cannot do this
+
+    CMD=$(cat <<EOM
+    use orion
+
+    db.experiments.createIndex(
+        {
+            "name": 1,
+            "version": 1,
+            "owner_id": 1
+        },
+        {
+            unique: true
+        }
+    )
+    db.experiments.createIndex(
+        {
+            "metadata.datetime": 1
+        }
+    )
+
+    db.trials.createIndex(
+        {
+            "experiment": 1,
+            "id": 1
+        },
+        {
+            unique: true
+        }
+    )
+    db.trials.createIndex(
+        {
+            "experiment": 1
+        }
+    )
+    db.trials.createIndex(
+        {
+            "status": 1
+        }
+    )
+    db.trials.createIndex(
+        {
+            "results": 1
+        }
+    )
+    db.trials.createIndex(
+        {
+            "start_time": 1
+        }
+    )
+    db.trials.createIndex(
+        {
+            "end_time": -1
+        }
+    )
+    db.algo.createIndex(
+        {
+            "experiment": 1
+        }
+    )
+EOM
+    )
+
+    echo "$CMD" | mongosh --port $PORT
+}
+
+function launch {
+    # Setup a mongodb for testing
+    start_init_mongod
+    add_admin_user
+    ensure_indexes
+    stop_mongod
+
+    # Start mongodb with access control
+    start_mongod
+
+    add_user User1 Pass1 Tok1
+    add_user User2 Pass2 Tok2
+    add_user User3 Pass3 Tok3
+}
+
+export MONGO_RUNNING="${DB_PATH}"
+launch
+

setup.py

@@ -39,6 +39,9 @@
     "orion.executor",
     "orion.plotting",
     "orion.serving",
+    "orion.service",
+    "orion.service.broker",
+    "orion.service.client",
     "orion.storage",
     "orion.testing",
 ]
@@ -138,6 +141,7 @@
         "BaseStorageProtocol": [
             "track = orion.storage.track:Track",
             "legacy = orion.storage.legacy:Legacy",
+            "rest = orion.storage.rest:RESTStorage",
         ],
         "BaseExecutor": [
             "singleexecutor = orion.executor.single_backend:SingleExecutor",

src/orion/client/__init__.py

@@ -209,6 +209,44 @@ def build_experiment(
         If the algorithm or storage specified is not properly installed.
 
     """
+    return ExperimentClient.create_experiment(
+        name,
+        version=version,
+        space=space,
+        algorithms=algorithms,
+        strategy=strategy,
+        max_trials=max_trials,
+        max_broken=max_broken,
+        storage=storage,
+        branching=branching,
+        max_idle_time=max_idle_time,
+        heartbeat=heartbeat,
+        working_dir=working_dir,
+        debug=debug,
+        knowledge_base=knowledge_base,
+        executor=executor,
+    )
+
+
+# pylint: disable=too-many-arguments
+def _build_experiment(
+    name,
+    version=None,
+    space=None,
+    algorithms=None,
+    strategy=None,
+    max_trials=None,
+    max_broken=None,
+    storage=None,
+    branching=None,
+    max_idle_time=None,
+    heartbeat=None,
+    working_dir=None,
+    debug=False,
+    knowledge_base: KnowledgeBase | dict | None = None,
+    executor=None,
+):
+
     if max_idle_time:
         log.warning(
             "max_idle_time is deprecated. Use experiment.workon(idle_timeout) instead."
@@ -230,6 +268,7 @@ def build_experiment(
             knowledge_base=knowledge_base,
         )
     except RaceCondition:
+        log.debug("Try creating a new experiment again")
         # Try again, but if it fails again, raise. Race conditions due to version increment should
         # only occur once in a short window of time unless code version is changing at a crazy pace.
         try: