Configuration parameter for pulse series name (#168)

* documentation * <bot> update requirements-all.txt * <bot> update requirements-client-starter.txt * <bot> update requirements-gcp.txt * <bot> update requirements-nats.txt * <bot> update requirements-pulsar.txt * <bot> update requirements-rabbitmq.txt * <bot> update requirements.txt * disallow overwriting the pulses series * documentation * get rid of nominalPulsesName definition * <bot> update requirements-all.txt * <bot> update requirements-client-starter.txt * <bot> update requirements-gcp.txt * <bot> update requirements-nats.txt * <bot> update requirements-pulsar.txt * <bot> update requirements-rabbitmq.txt * <bot> update requirements.txt * parameterize pulses name * parameterize pulses name * documentation * single source of truth for pulsesName * order of arguments * <bot> update requirements-all.txt * <bot> update requirements-client-starter.txt * <bot> update requirements-gcp.txt * <bot> update requirements-nats.txt * <bot> update requirements-pulsar.txt * <bot> update requirements-rabbitmq.txt * <bot> update requirements.txt * bit of typing * make mypy happy * remove default arguments * keep 3.10 compatibility of type hints * kill stray spaces * <bot> update requirements-all.txt * <bot> update requirements-client-starter.txt * <bot> update requirements-gcp.txt * <bot> update requirements-nats.txt * <bot> update requirements-pulsar.txt * <bot> update requirements-rabbitmq.txt * <bot> update requirements.txt * kill stray spaces/2 --------- Co-authored-by: wipacdevbot <[email protected]>
icecube · Apr 5, 2023 · 5329f6b · 5329f6b
1 parent caffe95
commit 5329f6b
Show file tree

Hide file tree

Showing 13 changed files with 63 additions and 59 deletions.
diff --git a/requirements-all.txt b/requirements-all.txt
@@ -32,15 +32,15 @@ cycler==0.11.0
     # via matplotlib
 ed25519==1.5
     # via nkeys
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
 google-api-core[grpc]==2.11.0
     # via
     #   google-cloud-pubsub
     #   oms-mqclient
-google-auth==2.17.1
+google-auth==2.17.2
     # via
     #   google-api-core
     #   oms-mqclient
@@ -96,7 +96,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient[all]==1.1.0
+oms-mqclient[all]==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/requirements-client-starter.txt b/requirements-client-starter.txt
@@ -26,7 +26,7 @@ cryptography==40.0.1
     # via pyjwt
 cycler==0.11.0
     # via matplotlib
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
@@ -59,7 +59,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient==1.1.0
+oms-mqclient==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/requirements-gcp.txt b/requirements-gcp.txt
@@ -28,15 +28,15 @@ cryptography==40.0.1
     # via pyjwt
 cycler==0.11.0
     # via matplotlib
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
 google-api-core[grpc]==2.11.0
     # via
     #   google-cloud-pubsub
     #   oms-mqclient
-google-auth==2.17.1
+google-auth==2.17.2
     # via
     #   google-api-core
     #   oms-mqclient
@@ -88,7 +88,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient[gcp]==1.1.0
+oms-mqclient[gcp]==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/requirements-nats.txt b/requirements-nats.txt
@@ -28,7 +28,7 @@ cycler==0.11.0
     # via matplotlib
 ed25519==1.5
     # via nkeys
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
@@ -63,7 +63,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient[nats]==1.1.0
+oms-mqclient[nats]==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/requirements-pulsar.txt b/requirements-pulsar.txt
@@ -28,7 +28,7 @@ cryptography==40.0.1
     # via pyjwt
 cycler==0.11.0
     # via matplotlib
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
@@ -59,7 +59,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient[pulsar]==1.1.0
+oms-mqclient[pulsar]==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/requirements-rabbitmq.txt b/requirements-rabbitmq.txt
@@ -26,7 +26,7 @@ cryptography==40.0.1
     # via pyjwt
 cycler==0.11.0
     # via matplotlib
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
@@ -57,7 +57,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient[rabbitmq]==1.1.0
+oms-mqclient[rabbitmq]==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/requirements.txt b/requirements.txt
@@ -26,7 +26,7 @@ cryptography==40.0.1
     # via pyjwt
 cycler==0.11.0
     # via matplotlib
-ewms-pilot==0.5.0
+ewms-pilot==0.6.0
     # via skymap-scanner (setup.py)
 fonttools==4.39.3
     # via matplotlib
@@ -57,7 +57,7 @@ numpy==1.24.2
     #   pyerfa
     #   scipy
     #   skymap-scanner (setup.py)
-oms-mqclient==1.1.0
+oms-mqclient==1.2.0
     # via
     #   ewms-pilot
     #   skymap-scanner (setup.py)

diff --git a/skymap_scanner/config.py b/skymap_scanner/config.py
@@ -17,6 +17,7 @@
 
 
 # physics strings
+INPUT_PULSES_NAME: Final = "SplitUncleanedInIcePulses"
 INPUT_TIME_NAME: Final = "HESE_VHESelfVetoVertexTime"
 INPUT_POS_NAME: Final = "HESE_VHESelfVetoVertexPos"
 OUTPUT_PARTICLE_NAME: Final = "MillipedeSeedParticle"

diff --git a/skymap_scanner/recos/millipede_original.py b/skymap_scanner/recos/millipede_original.py
@@ -35,7 +35,7 @@ class MillipedeOriginal(RecoInterface):
     """Reco logic for millipede."""
     # Constants ########################################################
 
-    pulsesName = "SplitUncleanedInIcePulses"
+    pulsesName = cfg.INPUT_PULSES_NAME
     pulsesName_cleaned = pulsesName+'LatePulseCleaned'
     SPEScale = 0.99
 

diff --git a/skymap_scanner/recos/millipede_wilks.py b/skymap_scanner/recos/millipede_wilks.py
@@ -35,8 +35,8 @@ class MillipedeWilks(RecoInterface):
     """Reco logic for millipede."""
     # Constants ########################################################
 
-    pulsesName_orig = "SplitUncleanedInIcePulses"
-    pulsesName = "SplitUncleanedInIcePulsesIC"
+    pulsesName_orig = cfg.INPUT_PULSES_NAME
+    pulsesName = cfg.INPUT_PULSES_NAME + "IC"
     pulsesName_cleaned = pulsesName+'LatePulseCleaned'
 
     # Load Data ########################################################
@@ -183,6 +183,7 @@ def LatePulseCleaning(frame, Pulses, Residual=1.5e3*I3Units.ns):
     def traysegment(tray, name, logger, seed=None):
         """Perform MillipedeWilks reco."""
         def mask_dc(frame, origpulses, maskedpulses):
+            # Masks DeepCore pulses by selecting string numbers < 79.
             frame[maskedpulses] = dataclasses.I3RecoPulseSeriesMapMask(
                 frame, origpulses, lambda omkey, index, pulse: omkey.string < 79)
         tray.Add(mask_dc, origpulses=MillipedeWilks.pulsesName_orig, maskedpulses=MillipedeWilks.pulsesName)

diff --git a/skymap_scanner/server/start_scan.py b/skymap_scanner/server/start_scan.py
@@ -131,7 +131,9 @@ def __init__(
                 f"({self.event_header.run_id=}, {self.event_header.event_id=})"
             )
 
-        self.pulseseries_hlc = dataclasses.I3RecoPulseSeriesMap.from_frame(p_frame,'SplitUncleanedInIcePulsesHLC')
+        # The HLC pulse mask has been created in prepare_frames().
+        self.pulseseries_hlc = dataclasses.I3RecoPulseSeriesMap.from_frame(p_frame,cfg.INPUT_PULSES_NAME+'HLC')
+
         self.omgeo = g_frame["I3Geometry"].omgeo
 
     @staticmethod
@@ -689,6 +691,7 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]:
         is_real_event=args.real_event,
         cache_dir=str(args.cache_dir),
         GCD_dir=str(args.gcd_dir),
+        pulsesName=cfg.INPUT_PULSES_NAME
     )
 
     # write startup files for client-spawning

diff --git a/skymap_scanner/utils/extract_json_message.py b/skymap_scanner/utils/extract_json_message.py
@@ -52,8 +52,9 @@ def extract_json_message(
     json_data,
     reco_algo: str,
     is_real_event: bool,
-    cache_dir="./cache/",
-    GCD_dir=None,
+    cache_dir: str,
+    GCD_dir: str,
+    pulsesName
 ) -> Tuple[EventMetadata, dict]:
     if not os.path.exists(cache_dir):
         raise RuntimeError("cache directory \"{0}\" does not exist.".format(cache_dir))
@@ -69,6 +70,7 @@ def extract_json_message(
         is_real_event=is_real_event,
         cache_dir=cache_dir,
         GCD_dir=GCD_dir,
+        pulsesName=pulsesName
     )
 
     # try to load existing pixels if there are any
@@ -100,9 +102,9 @@ def __extract_frame_packet(
     frame_packet,
     reco_algo: str,
     is_real_event: bool,
-    cache_dir="./cache/",
-    GCD_dir : str = str(cfg.DEFAULT_GCD_DIR),
-    pulsesName="SplitUncleanedInIcePulses",
+    pulsesName: str,
+    cache_dir: str,
+    GCD_dir: str,
 ) -> Tuple[str, EventMetadata, dict]:
     if not os.path.exists(cache_dir):
         raise RuntimeError("cache directory \"{0}\" does not exist.".format(cache_dir))

diff --git a/skymap_scanner/utils/prepare_frames.py b/skymap_scanner/utils/prepare_frames.py
@@ -12,6 +12,8 @@
 from I3Tray import I3Tray, I3Units  # type: ignore[import]
 from icecube import icetray  # type: ignore[import]
 from icecube.frame_object_diff.segments import uncompress  # type: ignore[import]
+from typing import Union, List
+
 
 from .. import config as cfg
 from . import LOGGER
@@ -24,7 +26,7 @@ def __init__(self, ctx):
             "The frames to push to modules downstream",
             [])
         self.AddOutBox("OutBox")
-        
+
     def Configure(self):
         self.frames = copy.copy(self.GetParameter("Frames"))
 
@@ -39,7 +41,7 @@ def Process(self):
             return
 
         self.PushFrame(self.frames.pop(0)) # push the frontmost item
-        
+
 
 class FrameArraySink(icetray.I3Module):
     def __init__(self, ctx):
@@ -48,24 +50,25 @@ def __init__(self, ctx):
             "Array to which to add frames",
             [])
         self.AddOutBox("OutBox")
-        
+
     def Configure(self):
         self.frame_store = self.GetParameter("FrameStore")
 
     def Process(self):
         frame = self.PopFrame()
         if not frame: return
-        
+
         # ignore potential TrayInfo frames
         if frame.Stop == icetray.I3Frame.TrayInfo:
             self.PushFrame(frame)
             return
-        
+
         self.frame_store.append(frame)
-        
+
         self.PushFrame(frame)
 
-def prepare_frames(frame_array, baseline_GCD, reco_algo, pulsesName="SplitUncleanedInIcePulses"):
+def prepare_frames(frame_array, baseline_GCD: Union[None, str], reco_algo: str, pulsesName: str) -> List[icetray.I3Frame]:
+    # type hint using list available from python 3.11
     from icecube import (
         DomTools,
         VHESelfVeto,
@@ -76,11 +79,9 @@ def prepare_frames(frame_array, baseline_GCD, reco_algo, pulsesName="SplitUnclea
         recclasses,
         simclasses,
     )
-
-    nominalPulsesName = "SplitUncleanedInIcePulses"
-
-    output_frames = []
-
+
+    output_frames: list[icetray.I3Frame] = []
+
     tray = I3Tray()
     tray.AddModule(FrameArraySource, Frames=frame_array)
 
@@ -91,37 +92,31 @@ def prepare_frames(frame_array, baseline_GCD, reco_algo, pulsesName="SplitUnclea
                  base_path=base_GCD_path,
                  base_filename=base_GCD_filename)
 
-    if pulsesName != nominalPulsesName:
-        def copyPulseName(frame, old_name, new_name):
-            mask = dataclasses.I3RecoPulseSeriesMapMask(frame, old_name)
-            if new_name in frame:
-                LOGGER.warning("** WARNING: {0} was already in frame. overwritten".format(new_name))
-                del frame[new_name]
-            frame[new_name] = mask
-            frame[new_name+"TimeRange"] = copy.deepcopy(frame[old_name+"TimeRange"])
-        tray.AddModule(copyPulseName, "copyPulseName",
-            old_name=pulsesName,
-            new_name=nominalPulsesName)
-
+    # Separates pulses in HLC and SLC to obtain the HLC series.
+    # HLC pulses are used for the determination of the vertex.
     tray.AddModule('I3LCPulseCleaning', 'lcclean1',
-        Input=nominalPulsesName,
-        OutputHLC=nominalPulsesName+'HLC',
-        OutputSLC=nominalPulsesName+'SLC',
-        If=lambda frame: nominalPulsesName+'HLC' not in frame)
-
+        Input=pulsesName,
+        OutputHLC=pulsesName+'HLC',
+        OutputSLC=pulsesName+'SLC',
+        If=lambda frame: pulsesName+'HLC' not in frame)
+
+    # Generates the vertex seed for the initial scan. 
+    # Only run if HESE_VHESelfVeto is not present in the frame.
+    # VertexThreshold is 250 in the original HESE analysis (Tianlu)
+    # If HESE_VHESelfVeto is already in the frame, is likely using implicitly a VertexThreshold of 250 already. To be determined when this is not the case.
     if reco_algo.lower() == 'millipede_original':
         # TODO: documentation for this conditional statement
         tray.AddModule('VHESelfVeto', 'selfveto',
                        VertexThreshold=2,
-                       Pulses=nominalPulsesName+'HLC',
+                       Pulses=pulsesName+'HLC',
                        OutputBool='HESE_VHESelfVeto',
                        OutputVertexTime=cfg.INPUT_TIME_NAME,
                        OutputVertexPos=cfg.INPUT_POS_NAME,
                        If=lambda frame: "HESE_VHESelfVeto" not in frame)
     else:
         tray.AddModule('VHESelfVeto', 'selfveto',
             VertexThreshold=250,
-            Pulses=nominalPulsesName+'HLC',
+            Pulses=pulsesName+'HLC',
             OutputBool='HESE_VHESelfVeto',
             OutputVertexTime=cfg.INPUT_TIME_NAME,
             OutputVertexPos=cfg.INPUT_POS_NAME,
@@ -130,12 +125,14 @@ def copyPulseName(frame, old_name, new_name):
         # this only runs if the previous module did not return anything
         tray.AddModule('VHESelfVeto', 'selfveto-emergency-lowen-settings',
                        VertexThreshold=5,
-                       Pulses=nominalPulsesName+'HLC',
+                       Pulses=pulsesName+'HLC',
                        OutputBool='VHESelfVeto_meaningless_lowen',
                        OutputVertexTime=cfg.INPUT_TIME_NAME,
                        OutputVertexPos=cfg.INPUT_POS_NAME,
                        If=lambda frame: not frame.Has("HESE_VHESelfVeto"))
 
+    # If the event has a GCD diff (compressed GCD), only keep the diffs.
+    # The GCD will be reassembled from baseline + diff by the client.
     if baseline_GCD is not None:
         def delFrameObjectsWithDiffsAvailable(frame):
             all_keys = list(frame.keys())
@@ -146,11 +143,11 @@ def delFrameObjectsWithDiffsAvailable(frame):
                     del frame[non_diff_key]
                     LOGGER.debug(f"Deleted {non_diff_key} from frame because a corresponding Diff exists.")
         tray.AddModule(delFrameObjectsWithDiffsAvailable, "delFrameObjectsWithDiffsAvailable", Streams=[icetray.I3Frame.Geometry, icetray.I3Frame.Calibration, icetray.I3Frame.DetectorStatus])
-    
+
     tray.AddModule(FrameArraySink, FrameStore=output_frames)
     tray.AddModule("TrashCan")
     tray.Execute()
     tray.Finish()
     del tray
-    
+
     return output_frames