From 99a013b234d02fc36239815b62cdde3c7854de21 Mon Sep 17 00:00:00 2001 From: Yifan Chen Date: Wed, 18 Oct 2023 16:18:48 -0700 Subject: [PATCH 1/6] update the geometry script to match with the module0 one --- src/proto_nd_flow/resources/geometry.py | 29 +++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/proto_nd_flow/resources/geometry.py b/src/proto_nd_flow/resources/geometry.py index c5df4024..3675e856 100644 --- a/src/proto_nd_flow/resources/geometry.py +++ b/src/proto_nd_flow/resources/geometry.py @@ -1,6 +1,7 @@ import numpy as np import numpy.ma as ma import logging +import warnings import yaml from h5flow.core import H5FlowResource @@ -16,6 +17,8 @@ class Geometry(H5FlowResource): Parameters: - ``path``: ``str``, path to stored geometry data within file + - ``network_agnostic``: ``bool``, optional, ignore the (io_channel % 4), useful if the io_channel mapping changes run-to-run (``True == ignore``) + - ``n_io_channels_per_tile``: ``int``, optional, only used with ``network_agnostic == True``, sets the number of io channels to have same geometry info - ``crs_geometry_file``: ``str``, path to yaml file describing charge readout system geometry - ``lrs_geometry_file``: ``str``, path to yaml file describing light readout system @@ -52,9 +55,11 @@ class Geometry(H5FlowResource): lrs_geometry_file: 'data/proto_nd_flow/light_module_desc-0.0.0.yaml' ''' - class_version = '0.1.0' + class_version = '0.2.0' default_path = 'geometry_info' + default_network_agnostic = False + default_n_io_channels_per_tile = 4 default_det_geometry_file = '-' default_crs_geometry_file = '-' default_lrs_geometry_file = '-' @@ -64,6 +69,8 @@ def __init__(self, **params): super(Geometry, self).__init__(**params) self.path = params.get('path', self.default_path) + self.network_agnostic = params.get('network_agnostic', self.default_network_agnostic) + self.n_io_channels_per_tile = params.get('n_io_channels_per_tile', self.default_n_io_channels_per_tile) self.det_geometry_file = params.get('det_geometry_file', self.default_crs_geometry_file) self.crs_geometry_file = params.get('crs_geometry_file', self.default_crs_geometry_file) self.lrs_geometry_file = params.get('lrs_geometry_file', self.default_lrs_geometry_file) @@ -86,6 +93,8 @@ def init(self, source_name): classname=self.classname, class_version=self.class_version, pixel_pitch=self.pixel_pitch, + network_agnostic=self.network_agnostic, + n_io_channels_per_tile=self.n_io_channels_per_tile, crs_geometry_file=self.crs_geometry_file ) write_lut(self.data_manager, self.path, self.pixel_xy, 'pixel_xy') @@ -426,7 +435,7 @@ def _load_charge_geometry(self): tiles = np.arange(1,len(geometry_yaml['tile_chip_to_io'])*len(det_geometry_yaml['module_to_io_groups'])+1) io_groups = [ - geometry_yaml['tile_chip_to_io'][tile][chip] // 1000 * (mod-1)*2 + geometry_yaml['tile_chip_to_io'][tile][chip] // 1000 + (mod-1)*2 for tile in geometry_yaml['tile_chip_to_io'] for chip in geometry_yaml['tile_chip_to_io'][tile] for mod in det_geometry_yaml['module_to_io_groups'] @@ -450,7 +459,7 @@ def _load_charge_geometry(self): pixel_xy_min_max = [(min(v), max(v)) for v in (io_groups, io_channels, chip_ids, channel_ids)] self._pixel_xy = LUT('f4', *pixel_xy_min_max, shape=(2,)) - self._pixel_xy.default = 0. + self._pixel_xy.default = np.nan tile_min_max = [(min(v), len(det_geometry_yaml['module_to_io_groups'])*max(v)) for v in (io_groups, io_channels)] self._tile_id = LUT('i4', *tile_min_max) @@ -476,6 +485,13 @@ def _load_charge_geometry(self): io_channel = io_group_io_channel % 1000 self._tile_id[([io_group], [io_channel])] = tile+(module_id-1)*len(tile_chip_to_io) + if self.network_agnostic == True: + # if we don't care about the network configuration, then we + # can just loop over every N io channels and add them to the LUT + start_io_channel = ((io_channel-1)//self.n_io_channels_per_tile)*self.n_io_channels_per_tile + 1 + for io_channel in range(start_io_channel, start_io_channel+self.n_io_channels_per_tile): + self._tile_id[([io_group], [io_channel])] = tile + for chip_channel in chip_channel_to_position: chip = chip_channel // 1000 channel = chip_channel % 1000 @@ -483,7 +499,12 @@ def _load_charge_geometry(self): try: io_group_io_channel = tile_chip_to_io[tile][chip] except KeyError: - continue + if self.network_agnostic == True: + warnings.warn('Encountered an out-of-network chip, but because you enabled ``network_agnostic``, we will carry on with assumptions about the io group and io channel') + # using the info about the first chip on the tile for all others + io_group_io_channel = list(geometry_yaml['tile_chip_to_io'][tile].values())[0] + else: + continue io_group = io_group_io_channel // 1000 + (module_id-1)*len(det_geometry_yaml['module_to_io_groups'][module_id]) io_channel = io_group_io_channel % 1000 From a14d954eecdf3df0aa3481d763e1df862a574141 Mon Sep 17 00:00:00 2001 From: Yifan Chen Date: Wed, 18 Oct 2023 16:39:08 -0700 Subject: [PATCH 2/6] remove the inflation factor of the size estimation --- src/proto_nd_flow/resources/geometry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proto_nd_flow/resources/geometry.py b/src/proto_nd_flow/resources/geometry.py index 3675e856..bf4e0c47 100644 --- a/src/proto_nd_flow/resources/geometry.py +++ b/src/proto_nd_flow/resources/geometry.py @@ -122,7 +122,7 @@ def init(self, source_name): lut_size = (self.pixel_xy.nbytes + self.tile_id.nbytes + self.anode_z.nbytes + self.drift_dir.nbytes + self.tpc_id.nbytes + self.det_id.nbytes - + self.det_bounds.nbytes) * 4 + + self.det_bounds.nbytes) if self.rank == 0: logging.info(f'Geometry LUT(s) size: {lut_size/1024/1024:0.02f}MB') From e6321e01852873b4ca169abfd8db798453acce37 Mon Sep 17 00:00:00 2001 From: Ronan James Doherty Date: Tue, 21 Nov 2023 13:44:48 -0800 Subject: [PATCH 3/6] Remove large initialised arrays --- src/proto_nd_flow/reco/charge/calib_hit_merger.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/proto_nd_flow/reco/charge/calib_hit_merger.py b/src/proto_nd_flow/reco/charge/calib_hit_merger.py index 7b2856e7..5f39302f 100644 --- a/src/proto_nd_flow/reco/charge/calib_hit_merger.py +++ b/src/proto_nd_flow/reco/charge/calib_hit_merger.py @@ -241,7 +241,6 @@ def merge_hits(self,hits, weights, seg_fracs, dt_cut, sum_fields=None, weighted_ # calculate segment contributions for each merged hit if has_mc_truth: - tmp_bt = np.full(shape=new_hits.shape+(2,self.max_contrib_segments),fill_value=0.) back_track = np.full(shape=new_hits.shape,fill_value=0.,dtype=self.hit_frac_dtype) # loop over hits for hit_it, hit in np.ndenumerate(new_hits): @@ -250,12 +249,12 @@ def merge_hits(self,hits, weights, seg_fracs, dt_cut, sum_fields=None, weighted_ # renormalize the fractional contributions given the charge weighted average norm = np.sum(np.multiply(hit_contr[0],hit_contr[1])) if norm == 0.: norm = 1. - tmp_bt[hit_it][0] = np.multiply(hit_contr[0],hit_contr[1])/norm # fractional contributions - tmp_bt[hit_it][1] = hit_contr[2] # segment_ids + tmp_bt_0 = np.multiply(hit_contr[0],hit_contr[1])/norm # fractional contributions + tmp_bt_1 = hit_contr[2] # segment_ids # merge unique track contributions track_dict = defaultdict(lambda:0) - for track in zip(tmp_bt[hit_it][0],tmp_bt[hit_it][1]): + for track in zip(tmp_bt_0,tmp_bt_1): track_dict[track[1]] += track[0] track_dict = dict(track_dict) bt_unique_segs = np.array(list(track_dict.keys())) From cb410c7dcf8941cebce6e5f0afae879d240473a3 Mon Sep 17 00:00:00 2001 From: Matt Kramer Date: Wed, 29 Nov 2023 09:30:31 -0800 Subject: [PATCH 4/6] Make `run_data.py` support .hdf5 as well as .h5 --- src/proto_nd_flow/resources/run_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proto_nd_flow/resources/run_data.py b/src/proto_nd_flow/resources/run_data.py index 17544f32..444c6656 100644 --- a/src/proto_nd_flow/resources/run_data.py +++ b/src/proto_nd_flow/resources/run_data.py @@ -170,7 +170,7 @@ def _lookup_mc_info(self): # mc info has already exists, return return - if self.input_filename[-3:] == '.h5' or '.h5' in self.input_filename: + if any(self.input_filename.endswith(ext) for ext in ['.h5', '.hdf5']): if H5FLOW_MPI: with h5py.File(self.input_filename, 'r', driver='mpio', comm=self.comm) as f: is_mc = 'mc_packets_assn' in f From 8e59ff717b91e83e24f4ee5ba25c3a9509d7050d Mon Sep 17 00:00:00 2001 From: Elise Dianne Hinkle Date: Fri, 8 Dec 2023 07:11:41 -0800 Subject: [PATCH 5/6] Add io_group and tile_id to calib_hits_dtype and calib_prompt_hits dataset. --- src/proto_nd_flow/reco/charge/calib_prompt_hits.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/proto_nd_flow/reco/charge/calib_prompt_hits.py b/src/proto_nd_flow/reco/charge/calib_prompt_hits.py index ed44b71d..454afcce 100644 --- a/src/proto_nd_flow/reco/charge/calib_prompt_hits.py +++ b/src/proto_nd_flow/reco/charge/calib_prompt_hits.py @@ -57,6 +57,8 @@ class CalibHitBuilder(H5FlowStage): z f8, pixel z location [cm] t_drift u8, drift time [ticks???] ts_pps f8, PPS packet timestamp [ns] + io_group u8, io group ID (PACMAN number) + tile_id u8, tile ID (related to PACMAN number & PACMAN UART Number) Q f8, hit charge [ke-] E f8, hit energy [MeV] @@ -81,6 +83,8 @@ class CalibHitBuilder(H5FlowStage): ('z', 'f8'), ('t_drift', 'f8'), ('ts_pps', 'u8'), + ('io_group', 'u8'), + ('tile_id', 'u8'), ('Q', 'f8'), ('E', 'f8') ]) @@ -213,6 +217,8 @@ def run(self, source_name, source_slice, cache): calib_hits_arr['z'] = xy[:,0]/10. calib_hits_arr['ts_pps'] = raw_hits_arr['ts_pps'] calib_hits_arr['t_drift'] = drift_t + calib_hits_arr['io_group'] = packets_arr['io_group'] + calib_hits_arr['tile_id'] = tile_id calib_hits_arr['Q'] = self.charge_from_dataword(packets_arr['dataword'],vref,vcm,ped) calib_hits_arr['E'] = self.charge_from_dataword(packets_arr['dataword'],vref,vcm,ped) * 23.6e-3 # hardcoding W_ion and not accounting for finite electron lifetime From ea5f974c8636affbdbf829c2cb0a789e8d7fbf0a Mon Sep 17 00:00:00 2001 From: Elise Dianne Hinkle Date: Wed, 13 Dec 2023 09:12:31 -0800 Subject: [PATCH 6/6] Changing new saved field in calib_*_hits dsets from tile_id to io_channel. --- src/proto_nd_flow/reco/charge/calib_prompt_hits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/proto_nd_flow/reco/charge/calib_prompt_hits.py b/src/proto_nd_flow/reco/charge/calib_prompt_hits.py index 454afcce..eedc1276 100644 --- a/src/proto_nd_flow/reco/charge/calib_prompt_hits.py +++ b/src/proto_nd_flow/reco/charge/calib_prompt_hits.py @@ -58,7 +58,7 @@ class CalibHitBuilder(H5FlowStage): t_drift u8, drift time [ticks???] ts_pps f8, PPS packet timestamp [ns] io_group u8, io group ID (PACMAN number) - tile_id u8, tile ID (related to PACMAN number & PACMAN UART Number) + io_channel u8, io channel ID (related to PACMAN number & PACMAN UART Number) Q f8, hit charge [ke-] E f8, hit energy [MeV] @@ -84,7 +84,7 @@ class CalibHitBuilder(H5FlowStage): ('t_drift', 'f8'), ('ts_pps', 'u8'), ('io_group', 'u8'), - ('tile_id', 'u8'), + ('io_channel', 'u8'), ('Q', 'f8'), ('E', 'f8') ]) @@ -218,7 +218,7 @@ def run(self, source_name, source_slice, cache): calib_hits_arr['ts_pps'] = raw_hits_arr['ts_pps'] calib_hits_arr['t_drift'] = drift_t calib_hits_arr['io_group'] = packets_arr['io_group'] - calib_hits_arr['tile_id'] = tile_id + calib_hits_arr['io_channel'] = packets_arr['io_channel'] calib_hits_arr['Q'] = self.charge_from_dataword(packets_arr['dataword'],vref,vcm,ped) calib_hits_arr['E'] = self.charge_from_dataword(packets_arr['dataword'],vref,vcm,ped) * 23.6e-3 # hardcoding W_ion and not accounting for finite electron lifetime