Skip to content

Commit

Permalink
Merge pull request datajoint#11 from ttngu207/lulab
Browse files Browse the repository at this point in the history 
fix(processing): make output_dir if trigger
  • Loading branch information
@kushalbakshi
kushalbakshi authored Oct 18, 2024
2 parents 700034c + e149f85 commit 3e87bc1
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 58 deletions.
83 changes: 54 additions & 29 deletions element_calcium_imaging/imaging_no_curation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,9 @@ def _make_taper(size, width):
)

zstack_files = get_zstack_files(key)
assert len(zstack_files) == 1, f"Multiple zstack files found at {zstack_files}. Expected only one."
assert (
len(zstack_files) == 1
), f"Multiple zstack files found at {zstack_files}. Expected only one."

ca_imaging_movie = nd2.imread(movie_file)
zstack = nd2.imread(zstack_files[0])
Expand Down Expand Up @@ -514,7 +516,18 @@ def make(self, key):
ProcessingTask.update1(
{**key, "processing_output_dir": output_dir.as_posix()}
)
output_dir = find_full_path(get_imaging_root_data_dir(), output_dir).as_posix()

try:
output_dir = find_full_path(
get_imaging_root_data_dir(), output_dir
).as_posix()
except FileNotFoundError as e:
if task_mode == "trigger":
processed_dir = pathlib.Path(get_processed_root_data_dir())
output_dir = processed_dir / output_dir
output_dir.mkdir(parents=True, exist_ok=True)
else:
raise e

if task_mode == "load":
method, imaging_dataset = get_loader_result(key, ProcessingTask)
Expand Down Expand Up @@ -547,7 +560,8 @@ def make(self, key):
raw_image_files = (scan.ScanInfo.ScanFile & key).fetch("file_path")
files_to_link = [
find_full_path(get_imaging_root_data_dir(), raw_image_file)
for raw_image_file in raw_image_files if not raw_image_file.endswith("_Z.nd2")
for raw_image_file in raw_image_files
if not raw_image_file.endswith("_Z.nd2")
]
image_files = []
for file in files_to_link:
Expand All @@ -561,7 +575,8 @@ def make(self, key):
image_files = (scan.ScanInfo.ScanFile & key).fetch("file_path")
image_files = [
find_full_path(get_imaging_root_data_dir(), image_file)
for image_file in image_files if not image_file.endswith("_Z.nd2")
for image_file in image_files
if not image_file.endswith("_Z.nd2")
]

method = (ProcessingParamSet * ProcessingTask & key).fetch1(
Expand Down Expand Up @@ -845,7 +860,7 @@ def make(self, key):
f"Unable to load/ingest non-rigid motion correction for plane {plane}."
"Non-rigid motion correction data is not saved by Suite2p for versions above 0.10.*."
)
else:
else:
# -- rigid motion correction --
if idx == 0:
rigid_correction = {
Expand Down Expand Up @@ -1122,30 +1137,40 @@ def make(self, key):
}
for fkey, ref_image, ave_img, corr_img, max_img in zip(
field_keys,
caiman_dataset.motion_correction["reference_image"].transpose(
2, 0, 1
)
if is3D
else caiman_dataset.motion_correction["reference_image"][...][
np.newaxis, ...
],
caiman_dataset.motion_correction["average_image"].transpose(2, 0, 1)
if is3D
else caiman_dataset.motion_correction["average_image"][...][
np.newaxis, ...
],
caiman_dataset.motion_correction["correlation_image"].transpose(
2, 0, 1
)
if is3D
else caiman_dataset.motion_correction["correlation_image"][...][
np.newaxis, ...
],
caiman_dataset.motion_correction["max_image"].transpose(2, 0, 1)
if is3D
else caiman_dataset.motion_correction["max_image"][...][
np.newaxis, ...
],
(
caiman_dataset.motion_correction["reference_image"].transpose(
2, 0, 1
)
if is3D
else caiman_dataset.motion_correction["reference_image"][...][
np.newaxis, ...
]
),
(
caiman_dataset.motion_correction["average_image"].transpose(
2, 0, 1
)
if is3D
else caiman_dataset.motion_correction["average_image"][...][
np.newaxis, ...
]
),
(
caiman_dataset.motion_correction["correlation_image"].transpose(
2, 0, 1
)
if is3D
else caiman_dataset.motion_correction["correlation_image"][...][
np.newaxis, ...
]
),
(
caiman_dataset.motion_correction["max_image"].transpose(2, 0, 1)
if is3D
else caiman_dataset.motion_correction["max_image"][...][
np.newaxis, ...
]
),
)
]
self.Summary.insert(summary_images)
Expand Down
75 changes: 46 additions & 29 deletions element_calcium_imaging/scan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ def get_prairieview_files(scan_key: dict) -> list:
"""
return _linking_module.get_prairieview_files(scan_key)


def get_zstack_files(scan_key: dict) -> list:
"""Retrieve the list of zstack files associated with a given Scan.
Expand Down Expand Up @@ -384,17 +385,21 @@ def make(self, key):
px_width=scan.field_widths[field_id],
um_height=scan.field_heights_in_microns[field_id],
um_width=scan.field_widths_in_microns[field_id],
field_x=x_zero
+ scan._degrees_to_microns(scan.fields[field_id].x)
if x_zero
else None,
field_y=y_zero
+ scan._degrees_to_microns(scan.fields[field_id].y)
if y_zero
else None,
field_z=z_zero + scan.fields[field_id].depth
if z_zero
else None,
field_x=(
x_zero
+ scan._degrees_to_microns(scan.fields[field_id].x)
if x_zero
else None
),
field_y=(
y_zero
+ scan._degrees_to_microns(scan.fields[field_id].y)
if y_zero
else None
),
field_z=(
z_zero + scan.fields[field_id].depth if z_zero else None
),
delay_image=scan.field_offsets[field_id],
roi=scan.field_rois[field_id][0],
)
Expand All @@ -413,9 +418,11 @@ def make(self, key):
um_width=getattr(scan, "image_width_in_microns", None),
field_x=x_zero if x_zero else None,
field_y=y_zero if y_zero else None,
field_z=z_zero + scan.scanning_depths[plane_idx]
if z_zero
else None,
field_z=(
z_zero + scan.scanning_depths[plane_idx]
if z_zero
else None
),
delay_image=scan.field_offsets[plane_idx],
)
for plane_idx in range(scan.num_scanning_depths)
Expand All @@ -442,9 +449,11 @@ def make(self, key):
self.insert1(
dict(
key,
nfields=sbx_meta["num_fields"]
if is_multiROI
else sbx_meta["num_planes"],
nfields=(
sbx_meta["num_fields"]
if is_multiROI
else sbx_meta["num_planes"]
),
nchannels=sbx_meta["num_channels"],
nframes=sbx_meta["num_frames"],
ndepths=sbx_meta["num_planes"],
Expand All @@ -467,12 +476,16 @@ def make(self, key):
field_idx=plane_idx,
px_height=px_height,
px_width=px_width,
um_height=px_height * sbx_meta["um_per_pixel_y"]
if sbx_meta["um_per_pixel_y"]
else None,
um_width=px_width * sbx_meta["um_per_pixel_x"]
if sbx_meta["um_per_pixel_x"]
else None,
um_height=(
px_height * sbx_meta["um_per_pixel_y"]
if sbx_meta["um_per_pixel_y"]
else None
),
um_width=(
px_width * sbx_meta["um_per_pixel_x"]
if sbx_meta["um_per_pixel_x"]
else None
),
field_x=x_zero,
field_y=y_zero,
field_z=z_zero + sbx_meta["etl_pos"][plane_idx],
Expand Down Expand Up @@ -518,9 +531,11 @@ def estimate_nd2_scan_duration(nd2_scan_obj):
scan_datetime = nd2_file.text_info["date"]
scan_datetime = datetime.strptime(
scan_datetime,
"%m/%d/%Y %H:%M:%S %p"
if re.search(("AM|PM"), scan_datetime)
else "%m/%d/%Y %H:%M:%S",
(
"%m/%d/%Y %H:%M:%S %p"
if re.search(("AM|PM"), scan_datetime)
else "%m/%d/%Y %H:%M:%S"
),
)
scan_datetime = datetime.strftime(scan_datetime, "%Y-%m-%d %H:%M:%S")
except: # noqa: E722
Expand Down Expand Up @@ -606,9 +621,11 @@ def estimate_nd2_scan_duration(nd2_scan_obj):
um_width=PVScan_info["width_in_um"],
field_x=PVScan_info["fieldX"],
field_y=PVScan_info["fieldY"],
field_z=PVScan_info["fieldZ"]
if PVScan_info["num_planes"] == 1
else PVScan_info["fieldZ"][plane_idx],
field_z=(
PVScan_info["fieldZ"]
if PVScan_info["num_planes"] == 1
else PVScan_info["fieldZ"][plane_idx]
),
)
for plane_idx in range(PVScan_info["num_planes"])
)
Expand Down
1 change: 1 addition & 0 deletions element_calcium_imaging/version.py
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
"""Package metadata."""

__version__ = "0.5.6"

0 comments on commit 3e87bc1

Please sign in to comment.