Copy data folders on demand to reduce storage use

fmelinscak · Apr 14, 2020 · ab9299e · ab9299e
1 parent 13241b5
commit ab9299e
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 10 deletions.
diff --git a/examples/pspm_pupil/exp1/bench.py b/examples/pspm_pupil/exp1/bench.py
@@ -21,6 +21,7 @@
 CUTOFF_LIST = [30, 50, 70]
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     models = [
         PsPMModel(
             lib_paths=util.LIB_PATHS,

diff --git a/examples/pspm_pupil/exp2/bench.py b/examples/pspm_pupil/exp2/bench.py
@@ -17,6 +17,7 @@
 MISS_PERCS = [10, 20, 30, 40, 50, 60]
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     models = [
         PsPMModel(
             lib_paths=util.LIB_PATHS,

diff --git a/examples/pspm_pupil/exp3/bench.py b/examples/pspm_pupil/exp3/bench.py
@@ -17,6 +17,7 @@
 miss_percs = [10, 20, 30, 40, 50, 60]
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     models = [
         PsPMModel(
             lib_paths=util.LIB_PATHS,

diff --git a/examples/pspm_pupil/exp4/bench.py b/examples/pspm_pupil/exp4/bench.py
@@ -16,6 +16,7 @@
 EXP_OUTPUT_PATH = pathjoin(util.OUT_PATH, MODEL_PATH)
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     # prepare models
     model_list = [
         (

diff --git a/examples/pspm_pupil/exp5/bench.py b/examples/pspm_pupil/exp5/bench.py
@@ -17,6 +17,7 @@
 DISCARD_FACTOR_LIST = [0.0, 0.005, 0.01, 0.015, 0.02]
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     # prepare models
     model_list = [
         (f"blink_saccade {factor:.3f}", {"discard_factor": factor})

diff --git a/examples/pspm_pupil/exp6/bench.py b/examples/pspm_pupil/exp6/bench.py
@@ -17,6 +17,7 @@
 ANGLE_LIST = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     # prepare models
     models = [
         PsPMModel(

diff --git a/examples/pspm_pupil/exp7/bench.py b/examples/pspm_pupil/exp7/bench.py
@@ -18,6 +18,7 @@
 PP_LIST = ["pfe", "valid_fixations"]
 
 if __name__ == "__main__":
+    os.makedirs(EXP_OUTPUT_PATH, exist_ok=True)
     # prepare models
     models = [
         PsPMModel(

diff --git a/examples/pspm_pupil/libcommon/fit_all.m b/examples/pspm_pupil/libcommon/fit_all.m
@@ -1,5 +1,11 @@
 function stats = fit_all(inarg)
-    datapath = inarg.datapath;
+    orig_datapath = inarg.datapath;
+    ds_name = split(orig_datapath, '/');
+    ds_name = ds_name{end};
+    tmp_out_path = fullfile(inarg.tmp_out_path, ds_name);
+    fprintf('Copying %s to temporary location %s', orig_datapath, tmp_out_path);
+    copyfile(orig_datapath, tmp_out_path);
+    inarg.datapath = tmp_out_path;
     subject_ids = inarg.subject_ids;
     stats = [];
     for subj_id = subject_ids
@@ -16,4 +22,6 @@
             warning('ID:fitting_error', getReport(err, 'extended'));
         end
     end
+    fprintf('Deleting folder %s', tmp_out_path);
+    rmdir(tmp_out_path, 's');
 end
diff --git a/examples/pspm_pupil/libcommon/util.py b/examples/pspm_pupil/libcommon/util.py
@@ -33,14 +33,6 @@ def __init__(self, *args, name, subject_ids):
 ]
 
 
-def temp_dataset(tmp_base_path, dataset):
-    dest_path = pathjoin(tmp_base_path, dataset.name)
-    if exists(dest_path):
-        rmtree(dest_path)
-    copytree(dataset.path, dest_path)
-    return dest_path
-
-
 def sm_to_pandas(sm):
     n_rows, n_cols = sm.shape
     arr = np.empty(shape=(n_rows, n_cols), dtype=np.float64)
@@ -55,8 +47,9 @@ def get_obs_dict_list(exp_output_path):
     obs_dict_list = [
         {
             "stimuli": {
-                "datapath": temp_dataset(exp_output_path, ds),
+                "datapath": ds.path,
                 "subject_ids": ds.subject_ids,
+                "tmp_out_path": exp_output_path,
             },
             "actions": [],
         }