Adds ability to set alternate local folder for config/metadata directory

cli.py

@@ -1,10 +1,11 @@
 import ftplib
 import io
 import sys
-import pydantic
-import tum_esm_utils
+
 import click
+import pydantic
 import em27_metadata
+import tum_esm_utils
 
 _RETRIEVAL_ENTRYPOINT = tum_esm_utils.files.rel_to_abs_path("src", "retrieval", "main.py")
 
@@ -16,8 +17,10 @@
 
 def _check_config_validity() -> None:
     import src
+    from src import utils
     try:
-        src.types.Config.load()
+        config_path = utils.environment.get_config_path()
+        src.types.Config.load(config_path)
         click.echo(click.style("Config is valid", fg="green", bold=True))
     except pydantic.ValidationError as e:
         click.echo(
@@ -123,7 +126,9 @@ def request_ginput_status() -> None:
     _check_config_validity()
 
     import src  # import here so that the CLI is more reactive
-    config = src.types.Config.load()
+    from src import utils
+    config_path = utils.environment.get_config_path()
+    config = src.types.Config.load(config_path)
     assert config.profiles is not None, "No profiles config found"
     with ftplib.FTP(
         host="ccycle.gps.caltech.edu",
@@ -151,12 +156,14 @@ def run_bundle() -> None:
 def print_data_report() -> None:
     _check_config_validity()
 
-    import src  # import here so that the CLI is more reactive
     import rich.console
+    import src  # import here so that the CLI is more reactive
+    from src import utils
 
     console = rich.console.Console()
     console.print("Loading config")
-    config = src.types.Config.load()
+    config_path = utils.environment.get_config_path()
+    config = src.types.Config.load(config_path)
 
     # load metadata interface
     console.print("Loading metadata")

docs/pages/guides/configuration.mdx

@@ -6,6 +6,8 @@ import { Callout } from "nextra/components";
 
 The `config/config.json` file configures all steps of the pipeline. You can use the `config/config.template.json` file as a template to create your own configuration file.
 
+For setting up alternate locations for the `config.json` file, see the [metadata section](/guides/metadata).
+
 <Callout type="info" emoji="💡">
 
 The full schema specification and description of each parameter can be found [in the API Reference section](/api-reference/configuration).

docs/pages/guides/metadata.mdx

@@ -18,6 +18,20 @@ To configure the pipeline with the metadata, you have two options: Save them loc
 
 For this option, you can save the three files `locations.json`, `sensors.json`, and `campaigns.json` to the `config/` directory of the pipeline directory. Be sure to include all the files - even if you don't have any campaigns, save an empty list to `campaigns.json`.
 
+Alternatively, you can store these files locally in a location outside of this repository. This is achieved by setting the environment variable `ERP_CONFIG_DIR` to the full path of the alternate location (ERP is short for EM27 Retrieval Pipeline). Note that this should be the same directory that contains the `config.json` file. If this environment variable is not set, the default location the pipeline will look for these files is `config/`. When setting an alternate config directory, it is not necessary to touch the file `config/config.template.json` (this file is version-controlled).
+
+For example, directly setting the environment variable:
+```
+export ERP_CONFIG_DIR=<path-to-local-config-dir>
+```
+
+For example, sourcing a `.env` file:
+```
+set -a && . <path-to-.env-file> && set +a
+```
+
+Check the set environment variables with `export -p`.
+
 ### Option 2: GitHub Repository
 
 **1. Create a repository**

src/bundle/main.py

@@ -19,7 +19,8 @@ def run(
 ) -> None:
     if config is None:
         print("Loading configuration")
-        config = types.Config.load()
+        config_path = utils.environment.get_config_path()
+        config = types.Config.load(config_path)
 
     assert config.bundles is not None, "no bundle targets found"
     assert len(config.bundles) > 0, "no bundle targets found"

src/profiles/main.py

@@ -1,14 +1,16 @@
+import ftplib
 import os
 import sys
+
 import tum_esm_utils
-import ftplib
 
 sys.path.append(tum_esm_utils.files.rel_to_abs_path("../.."))
-from src import types, profiles
+from src import types, profiles, utils
 
 
 def run() -> None:
-    config = types.Config.load()
+    config_path = utils.environment.get_config_path()
+    config = types.Config.load(config_path)
     assert config.profiles is not None, "No profiles config found"
 
     for variant in ["GGG2014", "GGG2020"]:

src/retrieval/main.py

@@ -20,7 +20,8 @@ def run() -> None:
 
     # load config
     try:
-        config = types.Config.load()
+        config_path = utils.environment.get_config_path()
+        config = types.Config.load(config_path)
         main_logger.info("Config is valid")
     except Exception as e:
         main_logger.exception(e, "Config file invalid")

src/utils/__init__.py

@@ -1 +1 @@
-from . import functions, metadata, report, semaphores, text
+from . import environment, functions, metadata, report, semaphores, text

src/utils/environment.py

@@ -0,0 +1,19 @@
+import os
+
+import tum_esm_utils
+
+
+CONFIG_DIR_KEY = "ERP_CONFIG_DIR"
+
+
+def get_config_dir() -> str:
+    default_config_dir = tum_esm_utils.files.rel_to_abs_path("../../config")
+    return os.getenv(CONFIG_DIR_KEY, default_config_dir)
+
+
+def get_config_path() -> str:
+    default_config_dir = get_config_dir()
+    return os.path.join(
+        os.getenv(CONFIG_DIR_KEY, default_config_dir),
+        "config.json"
+    )

src/utils/metadata.py

@@ -1,10 +1,12 @@
-from typing import Optional
 import os
+from typing import Optional
+
 import em27_metadata
-import tum_esm_utils
 
-CONFIG_DIR = tum_esm_utils.files.rel_to_abs_path("../../config")
+from src import utils
+
 
+CONFIG_DIR = utils.environment.get_config_dir()
 
 def load_local_em27_metadata_interface() -> Optional[em27_metadata.EM27MetadataInterface]:
     assert os.path.isdir(CONFIG_DIR)

tests/integration/test_config.py

@@ -1,8 +1,10 @@
 import pytest
-import src
 
+import src
+from src import utils
 
 @pytest.mark.order(2)
 @pytest.mark.integration
 def test_config() -> None:
-    src.types.Config.load()
+    config_path = utils.environment.get_config_path()
+    src.types.Config.load(config_path)

tests/integration/test_metadata_connection.py

@@ -1,12 +1,14 @@
 import pytest
 import em27_metadata
-import src
 
+import src
+from src import utils
 
 @pytest.mark.order(3)
 @pytest.mark.integration
 def test_metadata_connection() -> None:
-    config = src.types.Config.load()
+    config_path = utils.environment.get_config_path()
+    config = src.types.Config.load(config_path)
     if config.general.metadata is None:
         pytest.skip("Remote metadata not configured")
     else:

tests/integration/test_profiles_connection.py

@@ -1,13 +1,14 @@
 import pytest
 import ftplib
-import warnings
-import src
 
+import src
+from src import utils
 
 @pytest.mark.order(3)
 @pytest.mark.integration
 def test_profiles_connection() -> None:
-    config = src.types.Config.load()
+    config_path = utils.environment.get_config_path()
+    config = src.types.Config.load(config_path)
     if config.profiles is None:
         return
     with ftplib.FTP(

tests/repository/test_local_metadata.py

@@ -1,10 +1,12 @@
 from typing import Generator
 import os
 import pytest
-import tum_esm_utils
+
 import src
+from src import utils
+
 
-CONFIG_DIR = tum_esm_utils.files.rel_to_abs_path("../../config")
+CONFIG_DIR = utils.environment.get_config_dir()
 LOCATIONS_PATH = os.path.join(CONFIG_DIR, "locations.json")
 SENSORS_PATH = os.path.join(CONFIG_DIR, "sensors.json")
 CAMPAIGNS_PATH = os.path.join(CONFIG_DIR, "campaigns.json")