Add option to skip ifg corruption filter

tum-esm · Jan 31, 2024 · d659576 · d659576
1 parent 39e8093
commit d659576
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 24 deletions.
diff --git a/config/config.template.json b/config/config.template.json
@@ -48,7 +48,8 @@
           "store_binary_spectra": true,
           "dc_min_threshold": 0.05,
           "dc_var_threshold": 0.1,
-          "use_local_pressure_in_pcxs": true
+          "use_local_pressure_in_pcxs": true,
+          "use_ifg_corruption_filter": false
         }
       },
       {

diff --git a/docs/components/config-schema.ts b/docs/components/config-schema.ts
@@ -320,6 +320,12 @@ const CONFIG_SCHEMA: any = {
                                                 "description": "Whether to use the local pressure in the pcxs files. If not used, it will tell PCXS to use the pressure from the atmospheric profiles (set the input value in the `.inp` file to `9999.9`). If used, the pipeline computes the solar noon time using `skyfield` and averages the local pressure over the time period noon-2h to noon+2h.",
                                                 "title": "Use Local Pressure In Pcxs",
                                                 "type": "boolean"
+                                            },
+                                            "use_ifg_corruption_filter": {
+                                                "default": true,
+                                                "description": "Whether to use the ifg corruption filter. This filter is a program based on `preprocess4` and is part of the `tum-esm-utils` library: https://tum-esm-utils.netlify.app/api-reference#tum_esm_utilsinterferograms. If activated, we will only pass the interferograms to the retrieval algorithm that pass the filter - i.e. that won't cause it to crash.",
+                                                "title": "Use Ifg Corruption Filter",
+                                                "type": "boolean"
                                             }
                                         },
                                         "title": "RetrievalJobSettingsConfig",
@@ -328,7 +334,8 @@ const CONFIG_SCHEMA: any = {
                                             "store_binary_spectra": false,
                                             "dc_min_threshold": 0.05,
                                             "dc_var_threshold": 0.1,
-                                            "use_local_pressure_in_pcxs": false
+                                            "use_local_pressure_in_pcxs": false,
+                                            "use_ifg_corruption_filter": true
                                         },
                                         "description": "Advanced settings that only apply to this retrieval job"
                                     }

diff --git a/docs/pages/api-reference/configuration.mdx b/docs/pages/api-reference/configuration.mdx
@@ -68,7 +68,8 @@ import {
                     "store_binary_spectra": true,
                     "dc_min_threshold": 0.05,
                     "dc_var_threshold": 0.1,
-                    "use_local_pressure_in_pcxs": true
+                    "use_local_pressure_in_pcxs": true,
+                    "use_ifg_corruption_filter": false
                 }
             },
             {
@@ -87,7 +88,8 @@ import {
                     "store_binary_spectra": false,
                     "dc_min_threshold": 0.05,
                     "dc_var_threshold": 0.1,
-                    "use_local_pressure_in_pcxs": false
+                    "use_local_pressure_in_pcxs": false,
+                    "use_ifg_corruption_filter": true
                 }
             }
         ]

diff --git a/docs/pages/guides/configuration.mdx b/docs/pages/guides/configuration.mdx
@@ -74,7 +74,8 @@ Template:
                     "store_binary_spectra": true,
                     "dc_min_threshold": 0.05,
                     "dc_var_threshold": 0.1,
-                    "use_local_pressure_in_pcxs": true
+                    "use_local_pressure_in_pcxs": true,
+                    "use_ifg_corruption_filter": false
                 }
             },
             {
@@ -93,7 +94,8 @@ Template:
                     "store_binary_spectra": false,
                     "dc_min_threshold": 0.05,
                     "dc_var_threshold": 0.1,
-                    "use_local_pressure_in_pcxs": false
+                    "use_local_pressure_in_pcxs": false,
+                    "use_ifg_corruption_filter": true
                 }
             }
         ]

diff --git a/src/retrieval/session/move_ifg_files.py b/src/retrieval/session/move_ifg_files.py
@@ -66,24 +66,29 @@ def run(
             os.path.join(dst_date_path, f"{date_string[2:]}SN.{ifg_index + 1}"),
         )
 
-    # EXCLUDE CORRUPT INTERFEROGRAM FILES
+    # OPTIONALLY EXCLUDE CORRUPT INTERFEROGRAM FILES
 
-    try:
-        corrupt_filenames = list(
-            tum_esm_utils.interferograms.detect_corrupt_ifgs(
-                ifg_directory=dst_date_path
-            ).keys()
-        )
-    except subprocess.CalledProcessError:
-        raise AssertionError(
-            "corrupt-files-detection has failed during execution"
-        )
+    if session.job_settings.use_ifg_corruption_filter:
+        logger.info("Using ifg corruption filter")
+        try:
+            corrupt_filenames = list(
+                tum_esm_utils.interferograms.detect_corrupt_ifgs(
+                    ifg_directory=dst_date_path
+                ).keys()
+            )
+        except subprocess.CalledProcessError:
+            raise AssertionError(
+                "corrupt-files-detection has failed during execution"
+            )
 
-    logger.debug(
-        f"Excluding {len(corrupt_filenames)} corrupt file(s) from retrieval" + (
-            f" ({', '.join(corrupt_filenames)})" if len(corrupt_filenames) >
-            0 else ""
+        logger.debug(
+            f"Excluding {len(corrupt_filenames)} corrupt file(s) from retrieval"
+            + (
+                f" ({', '.join(corrupt_filenames)})" if len(corrupt_filenames) >
+                0 else ""
+            )
         )
-    )
-    for f in corrupt_filenames:
-        os.remove(os.path.join(dst_date_path, f))
+        for f in corrupt_filenames:
+            os.remove(os.path.join(dst_date_path, f))
+    else:
+        logger.info("Not using ifg corruption filter")
diff --git a/src/types/config.py b/src/types/config.py
@@ -164,6 +164,11 @@ class RetrievalJobSettingsConfig(pydantic.BaseModel):
         description=
         "Whether to use the local pressure in the pcxs files. If not used, it will tell PCXS to use the pressure from the atmospheric profiles (set the input value in the `.inp` file to `9999.9`). If used, the pipeline computes the solar noon time using `skyfield` and averages the local pressure over the time period noon-2h to noon+2h.",
     )
+    use_ifg_corruption_filter: bool = pydantic.Field(
+        True,
+        description=
+        "Whether to use the ifg corruption filter. This filter is a program based on `preprocess4` and is part of the `tum-esm-utils` library: https://tum-esm-utils.netlify.app/api-reference#tum_esm_utilsinterferograms. If activated, we will only pass the interferograms to the retrieval algorithm that pass the filter - i.e. that won't cause it to crash.",
+    )
 
 
 class RetrievalJobConfig(pydantic.BaseModel):