From ab9299eb02ae16bd9f7808b7ae83c588032297cf Mon Sep 17 00:00:00 2001 From: Eshref Yozdemir Date: Tue, 14 Apr 2020 17:58:06 +0200 Subject: [PATCH] Copy data folders on demand to reduce storage use --- examples/pspm_pupil/exp1/bench.py | 1 + examples/pspm_pupil/exp2/bench.py | 1 + examples/pspm_pupil/exp3/bench.py | 1 + examples/pspm_pupil/exp4/bench.py | 1 + examples/pspm_pupil/exp5/bench.py | 1 + examples/pspm_pupil/exp6/bench.py | 1 + examples/pspm_pupil/exp7/bench.py | 1 + examples/pspm_pupil/libcommon/fit_all.m | 10 +++++++++- examples/pspm_pupil/libcommon/util.py | 11 ++--------- 9 files changed, 18 insertions(+), 10 deletions(-) diff --git a/examples/pspm_pupil/exp1/bench.py b/examples/pspm_pupil/exp1/bench.py index 41b0937..a3f32ea 100644 --- a/examples/pspm_pupil/exp1/bench.py +++ b/examples/pspm_pupil/exp1/bench.py @@ -21,6 +21,7 @@ CUTOFF_LIST = [30, 50, 70] if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) models = [ PsPMModel( lib_paths=util.LIB_PATHS, diff --git a/examples/pspm_pupil/exp2/bench.py b/examples/pspm_pupil/exp2/bench.py index 0c76088..e277d39 100644 --- a/examples/pspm_pupil/exp2/bench.py +++ b/examples/pspm_pupil/exp2/bench.py @@ -17,6 +17,7 @@ MISS_PERCS = [10, 20, 30, 40, 50, 60] if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) models = [ PsPMModel( lib_paths=util.LIB_PATHS, diff --git a/examples/pspm_pupil/exp3/bench.py b/examples/pspm_pupil/exp3/bench.py index a9e6a42..fd16344 100644 --- a/examples/pspm_pupil/exp3/bench.py +++ b/examples/pspm_pupil/exp3/bench.py @@ -17,6 +17,7 @@ miss_percs = [10, 20, 30, 40, 50, 60] if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) models = [ PsPMModel( lib_paths=util.LIB_PATHS, diff --git a/examples/pspm_pupil/exp4/bench.py b/examples/pspm_pupil/exp4/bench.py index 4c5b17d..5f26912 100644 --- a/examples/pspm_pupil/exp4/bench.py +++ b/examples/pspm_pupil/exp4/bench.py @@ -16,6 +16,7 @@ EXP_OUTPUT_PATH = pathjoin(util.OUT_PATH, MODEL_PATH) if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) # prepare models model_list = [ ( diff --git a/examples/pspm_pupil/exp5/bench.py b/examples/pspm_pupil/exp5/bench.py index 7f4dfe4..b723b89 100644 --- a/examples/pspm_pupil/exp5/bench.py +++ b/examples/pspm_pupil/exp5/bench.py @@ -17,6 +17,7 @@ DISCARD_FACTOR_LIST = [0.0, 0.005, 0.01, 0.015, 0.02] if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) # prepare models model_list = [ (f"blink_saccade {factor:.3f}", {"discard_factor": factor}) diff --git a/examples/pspm_pupil/exp6/bench.py b/examples/pspm_pupil/exp6/bench.py index ad1a36c..736f874 100644 --- a/examples/pspm_pupil/exp6/bench.py +++ b/examples/pspm_pupil/exp6/bench.py @@ -17,6 +17,7 @@ ANGLE_LIST = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) # prepare models models = [ PsPMModel( diff --git a/examples/pspm_pupil/exp7/bench.py b/examples/pspm_pupil/exp7/bench.py index 3769863..8239a28 100644 --- a/examples/pspm_pupil/exp7/bench.py +++ b/examples/pspm_pupil/exp7/bench.py @@ -18,6 +18,7 @@ PP_LIST = ["pfe", "valid_fixations"] if __name__ == "__main__": + os.makedirs(EXP_OUTPUT_PATH, exist_ok=True) # prepare models models = [ PsPMModel( diff --git a/examples/pspm_pupil/libcommon/fit_all.m b/examples/pspm_pupil/libcommon/fit_all.m index 8d93181..1ff9ff3 100644 --- a/examples/pspm_pupil/libcommon/fit_all.m +++ b/examples/pspm_pupil/libcommon/fit_all.m @@ -1,5 +1,11 @@ function stats = fit_all(inarg) - datapath = inarg.datapath; + orig_datapath = inarg.datapath; + ds_name = split(orig_datapath, '/'); + ds_name = ds_name{end}; + tmp_out_path = fullfile(inarg.tmp_out_path, ds_name); + fprintf('Copying %s to temporary location %s', orig_datapath, tmp_out_path); + copyfile(orig_datapath, tmp_out_path); + inarg.datapath = tmp_out_path; subject_ids = inarg.subject_ids; stats = []; for subj_id = subject_ids @@ -16,4 +22,6 @@ warning('ID:fitting_error', getReport(err, 'extended')); end end + fprintf('Deleting folder %s', tmp_out_path); + rmdir(tmp_out_path, 's'); end diff --git a/examples/pspm_pupil/libcommon/util.py b/examples/pspm_pupil/libcommon/util.py index 7530b2a..97cf273 100644 --- a/examples/pspm_pupil/libcommon/util.py +++ b/examples/pspm_pupil/libcommon/util.py @@ -33,14 +33,6 @@ def __init__(self, *args, name, subject_ids): ] -def temp_dataset(tmp_base_path, dataset): - dest_path = pathjoin(tmp_base_path, dataset.name) - if exists(dest_path): - rmtree(dest_path) - copytree(dataset.path, dest_path) - return dest_path - - def sm_to_pandas(sm): n_rows, n_cols = sm.shape arr = np.empty(shape=(n_rows, n_cols), dtype=np.float64) @@ -55,8 +47,9 @@ def get_obs_dict_list(exp_output_path): obs_dict_list = [ { "stimuli": { - "datapath": temp_dataset(exp_output_path, ds), + "datapath": ds.path, "subject_ids": ds.subject_ids, + "tmp_out_path": exp_output_path, }, "actions": [], }