diff --git a/requirements-all.txt b/requirements-all.txt index 782399fef..3a07bb60b 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -32,7 +32,7 @@ cycler==0.11.0 # via matplotlib ed25519==1.5 # via nkeys -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -40,7 +40,7 @@ google-api-core[grpc]==2.11.0 # via # google-cloud-pubsub # oms-mqclient -google-auth==2.17.1 +google-auth==2.17.2 # via # google-api-core # oms-mqclient @@ -96,7 +96,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient[all]==1.1.0 +oms-mqclient[all]==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/requirements-client-starter.txt b/requirements-client-starter.txt index f4e1087f9..0460f16ee 100644 --- a/requirements-client-starter.txt +++ b/requirements-client-starter.txt @@ -26,7 +26,7 @@ cryptography==40.0.1 # via pyjwt cycler==0.11.0 # via matplotlib -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -59,7 +59,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient==1.1.0 +oms-mqclient==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/requirements-gcp.txt b/requirements-gcp.txt index 446ebe802..cdcc6a1b5 100644 --- a/requirements-gcp.txt +++ b/requirements-gcp.txt @@ -28,7 +28,7 @@ cryptography==40.0.1 # via pyjwt cycler==0.11.0 # via matplotlib -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -36,7 +36,7 @@ google-api-core[grpc]==2.11.0 # via # google-cloud-pubsub # oms-mqclient -google-auth==2.17.1 +google-auth==2.17.2 # via # google-api-core # oms-mqclient @@ -88,7 +88,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient[gcp]==1.1.0 +oms-mqclient[gcp]==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/requirements-nats.txt b/requirements-nats.txt index 0c9c10d99..f8afa8105 100644 --- a/requirements-nats.txt +++ b/requirements-nats.txt @@ -28,7 +28,7 @@ cycler==0.11.0 # via matplotlib ed25519==1.5 # via nkeys -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -63,7 +63,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient[nats]==1.1.0 +oms-mqclient[nats]==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/requirements-pulsar.txt b/requirements-pulsar.txt index 2a55fb963..f1cef2a1e 100644 --- a/requirements-pulsar.txt +++ b/requirements-pulsar.txt @@ -28,7 +28,7 @@ cryptography==40.0.1 # via pyjwt cycler==0.11.0 # via matplotlib -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -59,7 +59,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient[pulsar]==1.1.0 +oms-mqclient[pulsar]==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/requirements-rabbitmq.txt b/requirements-rabbitmq.txt index 3715fd77f..026beca46 100644 --- a/requirements-rabbitmq.txt +++ b/requirements-rabbitmq.txt @@ -26,7 +26,7 @@ cryptography==40.0.1 # via pyjwt cycler==0.11.0 # via matplotlib -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -57,7 +57,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient[rabbitmq]==1.1.0 +oms-mqclient[rabbitmq]==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/requirements.txt b/requirements.txt index aa0d894b5..9c10175d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ cryptography==40.0.1 # via pyjwt cycler==0.11.0 # via matplotlib -ewms-pilot==0.5.0 +ewms-pilot==0.6.0 # via skymap-scanner (setup.py) fonttools==4.39.3 # via matplotlib @@ -57,7 +57,7 @@ numpy==1.24.2 # pyerfa # scipy # skymap-scanner (setup.py) -oms-mqclient==1.1.0 +oms-mqclient==1.2.0 # via # ewms-pilot # skymap-scanner (setup.py) diff --git a/skymap_scanner/config.py b/skymap_scanner/config.py index 3c7ea9187..3a9c8e183 100644 --- a/skymap_scanner/config.py +++ b/skymap_scanner/config.py @@ -17,6 +17,7 @@ # physics strings +INPUT_PULSES_NAME: Final = "SplitUncleanedInIcePulses" INPUT_TIME_NAME: Final = "HESE_VHESelfVetoVertexTime" INPUT_POS_NAME: Final = "HESE_VHESelfVetoVertexPos" OUTPUT_PARTICLE_NAME: Final = "MillipedeSeedParticle" diff --git a/skymap_scanner/recos/millipede_original.py b/skymap_scanner/recos/millipede_original.py index d1e3086a3..c83ff5d2d 100644 --- a/skymap_scanner/recos/millipede_original.py +++ b/skymap_scanner/recos/millipede_original.py @@ -35,7 +35,7 @@ class MillipedeOriginal(RecoInterface): """Reco logic for millipede.""" # Constants ######################################################## - pulsesName = "SplitUncleanedInIcePulses" + pulsesName = cfg.INPUT_PULSES_NAME pulsesName_cleaned = pulsesName+'LatePulseCleaned' SPEScale = 0.99 diff --git a/skymap_scanner/recos/millipede_wilks.py b/skymap_scanner/recos/millipede_wilks.py index 55adfe892..7d39435c4 100644 --- a/skymap_scanner/recos/millipede_wilks.py +++ b/skymap_scanner/recos/millipede_wilks.py @@ -35,8 +35,8 @@ class MillipedeWilks(RecoInterface): """Reco logic for millipede.""" # Constants ######################################################## - pulsesName_orig = "SplitUncleanedInIcePulses" - pulsesName = "SplitUncleanedInIcePulsesIC" + pulsesName_orig = cfg.INPUT_PULSES_NAME + pulsesName = cfg.INPUT_PULSES_NAME + "IC" pulsesName_cleaned = pulsesName+'LatePulseCleaned' # Load Data ######################################################## @@ -183,6 +183,7 @@ def LatePulseCleaning(frame, Pulses, Residual=1.5e3*I3Units.ns): def traysegment(tray, name, logger, seed=None): """Perform MillipedeWilks reco.""" def mask_dc(frame, origpulses, maskedpulses): + # Masks DeepCore pulses by selecting string numbers < 79. frame[maskedpulses] = dataclasses.I3RecoPulseSeriesMapMask( frame, origpulses, lambda omkey, index, pulse: omkey.string < 79) tray.Add(mask_dc, origpulses=MillipedeWilks.pulsesName_orig, maskedpulses=MillipedeWilks.pulsesName) diff --git a/skymap_scanner/server/start_scan.py b/skymap_scanner/server/start_scan.py index a7f14d47c..b465fbe26 100644 --- a/skymap_scanner/server/start_scan.py +++ b/skymap_scanner/server/start_scan.py @@ -131,7 +131,9 @@ def __init__( f"({self.event_header.run_id=}, {self.event_header.event_id=})" ) - self.pulseseries_hlc = dataclasses.I3RecoPulseSeriesMap.from_frame(p_frame,'SplitUncleanedInIcePulsesHLC') + # The HLC pulse mask has been created in prepare_frames(). + self.pulseseries_hlc = dataclasses.I3RecoPulseSeriesMap.from_frame(p_frame,cfg.INPUT_PULSES_NAME+'HLC') + self.omgeo = g_frame["I3Geometry"].omgeo @staticmethod @@ -689,6 +691,7 @@ def _nside_and_pixelextension(val: str) -> Tuple[int, int]: is_real_event=args.real_event, cache_dir=str(args.cache_dir), GCD_dir=str(args.gcd_dir), + pulsesName=cfg.INPUT_PULSES_NAME ) # write startup files for client-spawning diff --git a/skymap_scanner/utils/extract_json_message.py b/skymap_scanner/utils/extract_json_message.py index 8e4974cbd..fa4d62dc1 100644 --- a/skymap_scanner/utils/extract_json_message.py +++ b/skymap_scanner/utils/extract_json_message.py @@ -52,8 +52,9 @@ def extract_json_message( json_data, reco_algo: str, is_real_event: bool, - cache_dir="./cache/", - GCD_dir=None, + cache_dir: str, + GCD_dir: str, + pulsesName ) -> Tuple[EventMetadata, dict]: if not os.path.exists(cache_dir): raise RuntimeError("cache directory \"{0}\" does not exist.".format(cache_dir)) @@ -69,6 +70,7 @@ def extract_json_message( is_real_event=is_real_event, cache_dir=cache_dir, GCD_dir=GCD_dir, + pulsesName=pulsesName ) # try to load existing pixels if there are any @@ -100,9 +102,9 @@ def __extract_frame_packet( frame_packet, reco_algo: str, is_real_event: bool, - cache_dir="./cache/", - GCD_dir : str = str(cfg.DEFAULT_GCD_DIR), - pulsesName="SplitUncleanedInIcePulses", + pulsesName: str, + cache_dir: str, + GCD_dir: str, ) -> Tuple[str, EventMetadata, dict]: if not os.path.exists(cache_dir): raise RuntimeError("cache directory \"{0}\" does not exist.".format(cache_dir)) diff --git a/skymap_scanner/utils/prepare_frames.py b/skymap_scanner/utils/prepare_frames.py index d76f19177..fed4b4559 100644 --- a/skymap_scanner/utils/prepare_frames.py +++ b/skymap_scanner/utils/prepare_frames.py @@ -12,6 +12,8 @@ from I3Tray import I3Tray, I3Units # type: ignore[import] from icecube import icetray # type: ignore[import] from icecube.frame_object_diff.segments import uncompress # type: ignore[import] +from typing import Union, List + from .. import config as cfg from . import LOGGER @@ -24,7 +26,7 @@ def __init__(self, ctx): "The frames to push to modules downstream", []) self.AddOutBox("OutBox") - + def Configure(self): self.frames = copy.copy(self.GetParameter("Frames")) @@ -39,7 +41,7 @@ def Process(self): return self.PushFrame(self.frames.pop(0)) # push the frontmost item - + class FrameArraySink(icetray.I3Module): def __init__(self, ctx): @@ -48,24 +50,25 @@ def __init__(self, ctx): "Array to which to add frames", []) self.AddOutBox("OutBox") - + def Configure(self): self.frame_store = self.GetParameter("FrameStore") def Process(self): frame = self.PopFrame() if not frame: return - + # ignore potential TrayInfo frames if frame.Stop == icetray.I3Frame.TrayInfo: self.PushFrame(frame) return - + self.frame_store.append(frame) - + self.PushFrame(frame) -def prepare_frames(frame_array, baseline_GCD, reco_algo, pulsesName="SplitUncleanedInIcePulses"): +def prepare_frames(frame_array, baseline_GCD: Union[None, str], reco_algo: str, pulsesName: str) -> List[icetray.I3Frame]: + # type hint using list available from python 3.11 from icecube import ( DomTools, VHESelfVeto, @@ -76,11 +79,9 @@ def prepare_frames(frame_array, baseline_GCD, reco_algo, pulsesName="SplitUnclea recclasses, simclasses, ) - - nominalPulsesName = "SplitUncleanedInIcePulses" - - output_frames = [] - + + output_frames: list[icetray.I3Frame] = [] + tray = I3Tray() tray.AddModule(FrameArraySource, Frames=frame_array) @@ -91,29 +92,23 @@ def prepare_frames(frame_array, baseline_GCD, reco_algo, pulsesName="SplitUnclea base_path=base_GCD_path, base_filename=base_GCD_filename) - if pulsesName != nominalPulsesName: - def copyPulseName(frame, old_name, new_name): - mask = dataclasses.I3RecoPulseSeriesMapMask(frame, old_name) - if new_name in frame: - LOGGER.warning("** WARNING: {0} was already in frame. overwritten".format(new_name)) - del frame[new_name] - frame[new_name] = mask - frame[new_name+"TimeRange"] = copy.deepcopy(frame[old_name+"TimeRange"]) - tray.AddModule(copyPulseName, "copyPulseName", - old_name=pulsesName, - new_name=nominalPulsesName) - + # Separates pulses in HLC and SLC to obtain the HLC series. + # HLC pulses are used for the determination of the vertex. tray.AddModule('I3LCPulseCleaning', 'lcclean1', - Input=nominalPulsesName, - OutputHLC=nominalPulsesName+'HLC', - OutputSLC=nominalPulsesName+'SLC', - If=lambda frame: nominalPulsesName+'HLC' not in frame) - + Input=pulsesName, + OutputHLC=pulsesName+'HLC', + OutputSLC=pulsesName+'SLC', + If=lambda frame: pulsesName+'HLC' not in frame) + + # Generates the vertex seed for the initial scan. + # Only run if HESE_VHESelfVeto is not present in the frame. + # VertexThreshold is 250 in the original HESE analysis (Tianlu) + # If HESE_VHESelfVeto is already in the frame, is likely using implicitly a VertexThreshold of 250 already. To be determined when this is not the case. if reco_algo.lower() == 'millipede_original': # TODO: documentation for this conditional statement tray.AddModule('VHESelfVeto', 'selfveto', VertexThreshold=2, - Pulses=nominalPulsesName+'HLC', + Pulses=pulsesName+'HLC', OutputBool='HESE_VHESelfVeto', OutputVertexTime=cfg.INPUT_TIME_NAME, OutputVertexPos=cfg.INPUT_POS_NAME, @@ -121,7 +116,7 @@ def copyPulseName(frame, old_name, new_name): else: tray.AddModule('VHESelfVeto', 'selfveto', VertexThreshold=250, - Pulses=nominalPulsesName+'HLC', + Pulses=pulsesName+'HLC', OutputBool='HESE_VHESelfVeto', OutputVertexTime=cfg.INPUT_TIME_NAME, OutputVertexPos=cfg.INPUT_POS_NAME, @@ -130,12 +125,14 @@ def copyPulseName(frame, old_name, new_name): # this only runs if the previous module did not return anything tray.AddModule('VHESelfVeto', 'selfveto-emergency-lowen-settings', VertexThreshold=5, - Pulses=nominalPulsesName+'HLC', + Pulses=pulsesName+'HLC', OutputBool='VHESelfVeto_meaningless_lowen', OutputVertexTime=cfg.INPUT_TIME_NAME, OutputVertexPos=cfg.INPUT_POS_NAME, If=lambda frame: not frame.Has("HESE_VHESelfVeto")) + # If the event has a GCD diff (compressed GCD), only keep the diffs. + # The GCD will be reassembled from baseline + diff by the client. if baseline_GCD is not None: def delFrameObjectsWithDiffsAvailable(frame): all_keys = list(frame.keys()) @@ -146,11 +143,11 @@ def delFrameObjectsWithDiffsAvailable(frame): del frame[non_diff_key] LOGGER.debug(f"Deleted {non_diff_key} from frame because a corresponding Diff exists.") tray.AddModule(delFrameObjectsWithDiffsAvailable, "delFrameObjectsWithDiffsAvailable", Streams=[icetray.I3Frame.Geometry, icetray.I3Frame.Calibration, icetray.I3Frame.DetectorStatus]) - + tray.AddModule(FrameArraySink, FrameStore=output_frames) tray.AddModule("TrashCan") tray.Execute() tray.Finish() del tray - + return output_frames