Refactor: alternative local config directory

Refinements

cli.py

@@ -17,9 +17,9 @@
 
 def _check_config_validity() -> None:
     import src
-    from src.utils.envutils import get_config_path, config_dir_key
+    from src import utils
     try:
-        config_path = get_config_path(config_dir_key())
+        config_path = utils.environment.get_config_path()
         src.types.Config.load(config_path)
         click.echo(click.style("Config is valid", fg="green", bold=True))
     except pydantic.ValidationError as e:
@@ -126,8 +126,8 @@ def request_ginput_status() -> None:
     _check_config_validity()
 
     import src  # import here so that the CLI is more reactive
-    from src.utils.envutils import get_config_path, config_dir_key
-    config_path = get_config_path(config_dir_key())
+    from src import utils
+    config_path = utils.environment.get_config_path()
     config = src.types.Config.load(config_path)
     assert config.profiles is not None, "No profiles config found"
     with ftplib.FTP(
@@ -158,11 +158,11 @@ def print_data_report() -> None:
 
     import rich.console
     import src  # import here so that the CLI is more reactive
-    from src.utils.envutils import get_config_path, config_dir_key
+    from src import utils
 
     console = rich.console.Console()
     console.print("Loading config")
-    config_path = get_config_path(config_dir_key())
+    config_path = utils.environment.get_config_path()
     config = src.types.Config.load(config_path)
 
     # load metadata interface

docs/pages/guides/metadata.mdx

@@ -18,7 +18,7 @@ To configure the pipeline with the metadata, you have two options: Save them loc
 
 For this option, you can save the three files `locations.json`, `sensors.json`, and `campaigns.json` to the `config/` directory of the pipeline directory. Be sure to include all the files - even if you don't have any campaigns, save an empty list to `campaigns.json`.
 
-Alternatively, you can store these files locally in a location outside of this repository. This is achieved by setting the environment variable `ERP_CONFIG_DIR` to the full path of the alternate location (ERP <- EM27 Retrieval Pipeline). Note that this should be the same directory that contains the `config.json` file. If this environment variable is not set, the default location the pipeline will look for these files is `./config/`. When setting an alternate config directory, leave `./config/config.template.json` for tests (this file is version-controlled).
+Alternatively, you can store these files locally in a location outside of this repository. This is achieved by setting the environment variable `ERP_CONFIG_DIR` to the full path of the alternate location (ERP is short for EM27 Retrieval Pipeline). Note that this should be the same directory that contains the `config.json` file. If this environment variable is not set, the default location the pipeline will look for these files is `config/`. When setting an alternate config directory, it is not necessary to touch the file `config/config.template.json` (this file is version-controlled).
 
 For example, directly setting the environment variable:
 ```

src/bundle/main.py

@@ -10,7 +10,6 @@
 
 sys.path.append(tum_esm_utils.files.rel_to_abs_path("../.."))
 from src import types, utils
-from src.utils.envutils import get_config_path, config_dir_key
 from .load_results import load_results_directory
 
 
@@ -20,7 +19,7 @@ def run(
 ) -> None:
     if config is None:
         print("Loading configuration")
-        config_path = get_config_path(config_dir_key())
+        config_path = utils.environment.get_config_path()
         config = types.Config.load(config_path)
 
     assert config.bundles is not None, "no bundle targets found"

src/profiles/main.py

@@ -5,12 +5,11 @@
 import tum_esm_utils
 
 sys.path.append(tum_esm_utils.files.rel_to_abs_path("../.."))
-from src import types, profiles
-from src.utils.envutils import get_config_path, config_dir_key
+from src import types, profiles, utils
 
 
 def run() -> None:
-    config_path = get_config_path(config_dir_key())
+    config_path = utils.environment.get_config_path()
     config = types.Config.load(config_path)
     assert config.profiles is not None, "No profiles config found"
 

src/retrieval/main.py

@@ -10,7 +10,6 @@
 
 sys.path.append(tum_esm_utils.files.rel_to_abs_path("../.."))
 from src import types, utils, retrieval
-from src.utils.envutils import get_config_path, config_dir_key
 
 
 def run() -> None:
@@ -21,7 +20,7 @@ def run() -> None:
 
     # load config
     try:
-        config_path = get_config_path(config_dir_key())
+        config_path = utils.environment.get_config_path()
         config = types.Config.load(config_path)
         main_logger.info("Config is valid")
     except Exception as e:

src/utils/__init__.py

@@ -1 +1 @@
-from . import functions, metadata, report, semaphores, text
+from . import environment, functions, metadata, report, semaphores, text

src/utils/environment.py

@@ -0,0 +1,19 @@
+import os
+
+import tum_esm_utils
+
+
+CONFIG_DIR_KEY = "ERP_CONFIG_DIR"
+
+
+def get_config_dir() -> str:
+    default_config_dir = tum_esm_utils.files.rel_to_abs_path("../../config")
+    return os.getenv(CONFIG_DIR_KEY, default_config_dir)
+
+
+def get_config_path() -> str:
+    default_config_dir = get_config_dir()
+    return os.path.join(
+        os.getenv(CONFIG_DIR_KEY, default_config_dir),
+        "config.json"
+    )

src/utils/metadata.py

@@ -3,10 +3,10 @@
 
 import em27_metadata
 
-from src.utils.envutils import get_config_dir, config_dir_key
+from src import utils
 
 
-CONFIG_DIR = get_config_dir(config_dir_key())
+CONFIG_DIR = utils.environment.get_config_dir()
 
 def load_local_em27_metadata_interface() -> Optional[em27_metadata.EM27MetadataInterface]:
     assert os.path.isdir(CONFIG_DIR)

tests/integration/test_config.py

@@ -1,10 +1,10 @@
 import pytest
 
 import src
-from src.utils.envutils import get_config_path, config_dir_key
+from src import utils
 
 @pytest.mark.order(2)
 @pytest.mark.integration
 def test_config() -> None:
-    config_path = get_config_path(config_dir_key())
+    config_path = utils.environment.get_config_path()
     src.types.Config.load(config_path)

tests/integration/test_metadata_connection.py

@@ -2,13 +2,12 @@
 import em27_metadata
 
 import src
-from src.utils.envutils import get_config_path, config_dir_key
-
+from src import utils
 
 @pytest.mark.order(3)
 @pytest.mark.integration
 def test_metadata_connection() -> None:
-    config_path = get_config_path(config_dir_key())
+    config_path = utils.environment.get_config_path()
     config = src.types.Config.load(config_path)
     if config.general.metadata is None:
         pytest.skip("Remote metadata not configured")

tests/integration/test_profiles_connection.py

@@ -2,13 +2,12 @@
 import ftplib
 
 import src
-from src.utils.envutils import get_config_path, config_dir_key
-
+from src import utils
 
 @pytest.mark.order(3)
 @pytest.mark.integration
 def test_profiles_connection() -> None:
-    config_path = get_config_path(config_dir_key())
+    config_path = utils.environment.get_config_path()
     config = src.types.Config.load(config_path)
     if config.profiles is None:
         return

tests/repository/test_local_metadata.py

@@ -3,10 +3,10 @@
 import pytest
 
 import src
-from src.utils.envutils import get_config_dir, config_dir_key
+from src import utils
 
 
-CONFIG_DIR = get_config_dir(config_dir_key())
+CONFIG_DIR = utils.environment.get_config_dir()
 LOCATIONS_PATH = os.path.join(CONFIG_DIR, "locations.json")
 SENSORS_PATH = os.path.join(CONFIG_DIR, "sensors.json")
 CAMPAIGNS_PATH = os.path.join(CONFIG_DIR, "campaigns.json")