Refactoring tsvtool

.jenkins/Jenkinsfile

@@ -65,21 +65,24 @@ pipeline {
                     '''
                 }
               }
-              stage('TSVTOOL tests Linux') {
+              stage('tsvtools tests Linux') {
                 steps {
-                  echo 'Testing tsvtool tasks...'
-                    sh "echo 'Agent name: ${NODE_NAME}'"
-                    sh '''
-                    source "${CONDA_HOME}/etc/profile.d/conda.sh"
-                    conda activate "${CONDA_ENV}"
-                    cd $WORKSPACE/tests
-                    poetry run pytest \
-                      --junitxml=./test-reports/test_tsvtool_report.xml \
-                      --verbose \
-                      --disable-warnings \
-                      test_tsvtool.py
-                    conda deactivate
-                    '''
+                  catchError(buildResult: 'FAILURE', stageResult: 'UNSTABLE'){
+                    echo 'Testing tsvtool tasks...'
+                      sh "echo 'Agent name: ${NODE_NAME}'"
+                      sh '''
+                      source "${CONDA_HOME}/etc/profile.d/conda.sh"
+                      conda activate "${CONDA_ENV}"
+                      cd $WORKSPACE/tests
+                      poetry run pytest \
+                        --junitxml=./test-reports/test_tsvtool_report.xml \
+                        --verbose \
+                        --disable-warnings \
+                        test_tsvtool.py
+                      conda deactivate
+                      '''
+                  }
+
                 }
                 post {
                   always {
@@ -137,7 +140,7 @@ pipeline {
                   }
                 }
               }
-              stage('Extract tests Linux') {
+              stage('Prepare data tests Linux') {
                 steps {
                   echo 'Testing extract task...'
                     sh "echo 'Agent name: ${NODE_NAME}'"
@@ -151,7 +154,7 @@ pipeline {
                         --junitxml=./test-reports/test_extract_report.xml \
                         --verbose \
                         --disable-warnings \
-                        test_extract.py
+                        test_prepare_data.py
                       conda deactivate
                       '''
                 }

clinicadl/cmdline.py

@@ -2,10 +2,10 @@
 
 import click
 
-from clinicadl.extract.extract_cli import cli as extract_cli
 from clinicadl.generate.generate_cli import cli as generate_cli
 from clinicadl.interpret.interpret_cli import cli as interpret_cli
 from clinicadl.predict.predict_cli import cli as predict_cli
+from clinicadl.prepare_data.prepare_data_cli import cli as prepare_data_cli
 from clinicadl.quality_check.qc_cli import cli as qc_cli
 from clinicadl.random_search.random_search_cli import cli as random_search_cli
 from clinicadl.train.train_cli import cli as train_cli
@@ -39,7 +39,7 @@ def cli(verbosity):
 cli.add_command(tsvtools_cli)
 cli.add_command(train_cli)
 cli.add_command(generate_cli)
-cli.add_command(extract_cli)
+cli.add_command(prepare_data_cli)
 cli.add_command(predict_cli)
 cli.add_command(interpret_cli)
 cli.add_command(qc_cli)

clinicadl/generate/generate.py

@@ -14,7 +14,7 @@
 import torch
 from clinica.utils.inputs import RemoteFileStructure, clinica_file_reader, fetch_file
 
-from clinicadl.extract.extract_utils import compute_extract_json
+from clinicadl.prepare_data.prepare_data_utils import compute_extract_json
 from clinicadl.utils.caps_dataset.data import CapsDataset
 from clinicadl.utils.maps_manager.iotools import check_and_clean, commandline_to_json
 from clinicadl.utils.preprocessing import write_preprocessing

clinicadl/extract/extract.py → clinicadl/prepare_data/prepare_data.py

@@ -14,7 +14,7 @@ def DeepLearningPrepareData(caps_directory, tsv_file, n_proc, parameters):
     from clinicadl.utils.exceptions import ClinicaDLArgumentError
     from clinicadl.utils.preprocessing import write_preprocessing
 
-    from .extract_utils import check_mask_list, compute_folder_and_file_type
+    from .prepare_data_utils import check_mask_list, compute_folder_and_file_type
 
     logger = getLogger("clinicadl")
 
@@ -70,7 +70,7 @@ def write_output_imgs(output_mode, container, subfolder):
     if parameters["mode"] == "image" or not parameters["prepare_dl"]:
 
         def prepare_image(file):
-            from .extract_utils import extract_images
+            from .prepare_data_utils import extract_images
 
             logger.debug(f"  Processing of {file}.")
             container = container_from_filename(file)
@@ -84,7 +84,7 @@ def prepare_image(file):
     elif parameters["prepare_dl"] and parameters["mode"] == "slice":
 
         def prepare_slice(file):
-            from .extract_utils import extract_slices
+            from .prepare_data_utils import extract_slices
 
             logger.debug(f"  Processing of {file}.")
             container = container_from_filename(file)
@@ -103,7 +103,7 @@ def prepare_slice(file):
     elif parameters["prepare_dl"] and parameters["mode"] == "patch":
 
         def prepare_patch(file):
-            from .extract_utils import extract_patches
+            from .prepare_data_utils import extract_patches
 
             logger.debug(f"  Processing of {file}.")
             container = container_from_filename(file)
@@ -121,7 +121,7 @@ def prepare_patch(file):
     elif parameters["prepare_dl"] and parameters["mode"] == "roi":
 
         def prepare_roi(file):
-            from .extract_utils import extract_roi
+            from .prepare_data_utils import extract_roi
 
             logger.debug(f"  Processing of {file}.")
             container = container_from_filename(file)
@@ -134,7 +134,7 @@ def prepare_roi(file):
                 parameters["roi_template"] = parameters["roi_custom_template"]
                 parameters["roi_mask_pattern"] = parameters["roi_custom_mask_pattern"]
             else:
-                from .extract_utils import PATTERN_DICT, TEMPLATE_DICT
+                from .prepare_data_utils import PATTERN_DICT, TEMPLATE_DICT
 
                 parameters["roi_template"] = TEMPLATE_DICT[parameters["preprocessing"]]
                 parameters["roi_mask_pattern"] = PATTERN_DICT[

clinicadl/extract/extract_cli.py → clinicadl/prepare_data/prepare_data_cli.py

@@ -4,8 +4,8 @@
 
 from clinicadl.utils import cli_param
 
-from .extract import DeepLearningPrepareData
-from .extract_utils import get_parameters_dict
+from .prepare_data import DeepLearningPrepareData
+from .prepare_data_utils import get_parameters_dict
 
 
 @click.command(name="image", no_args_is_help=True)
@@ -297,7 +297,7 @@ def list_commands(self, ctx):
         return self.commands.keys()
 
 
-@click.group(cls=RegistrationOrderGroup, name="extract", no_args_is_help=True)
+@click.group(cls=RegistrationOrderGroup, name="prepare-data", no_args_is_help=True)
 def cli() -> None:
     """Extract Pytorch tensors from nifti images."""
     pass

clinicadl/tsvtools/analysis/analysis.py

@@ -1,16 +1,20 @@
 # coding: utf-8
 
 import os
+from copy import copy
 from os import path
 from warnings import warn
 
 import numpy as np
 import pandas as pd
 
+from clinicadl.utils.exceptions import ClinicaDLArgumentError
 from clinicadl.utils.tsvtools_utils import (
     add_demographics,
+    cleaning_nan_diagnoses,
     find_label,
     first_session,
+    merge_tsv_reader,
     next_session,
 )
 
@@ -31,8 +35,9 @@ def demographics_analysis(merged_tsv, formatted_data_path, results_path, diagnos
         demographic analysis of the tsv files in formatted_data_path.
     """
 
-    merged_df = pd.read_csv(merged_tsv, sep="\t")
+    merged_df = merge_tsv_reader(merged_tsv)
     merged_df.set_index(["participant_id", "session_id"], inplace=True)
+    merged_df = cleaning_nan_diagnoses(merged_df)
     parent_directory = path.abspath(path.join(results_path, os.pardir))
     os.makedirs(parent_directory, exist_ok=True)
 
@@ -70,83 +75,90 @@ def demographics_analysis(merged_tsv, formatted_data_path, results_path, diagnos
 
     # Need all values for mean and variance (age, MMSE and scans)
     diagnosis_dict = dict.fromkeys(diagnoses)
+    if not path.exists(formatted_data_path):
+        print(
+            f"getlabels.tsv file with all sessions was not found. "
+            # f"Loads baseline version instead."
+        )
     for diagnosis in diagnoses:
         diagnosis_dict[diagnosis] = {"age": [], "MMSE": [], "scans": []}
-        diagnosis_path = path.join(formatted_data_path, diagnosis + ".tsv")
-        if not path.exists(diagnosis_path):
-            print(
-                f"TSV file with all sessions was not found for diagnosis {diagnosis}. "
-                f"Loads baseline version instead."
+        getlabels_df = pd.read_csv(formatted_data_path, sep="\t")
+        diagnosis_copy_df = copy(getlabels_df)
+        diagnosis_copy_df = diagnosis_copy_df[diagnosis_copy_df["group"] == diagnosis]
+        if not diagnosis_copy_df.empty:
+            diagnosis_demographics_df = add_demographics(
+                diagnosis_copy_df, merged_df, diagnosis
             )
-            diagnosis_path = path.join(formatted_data_path, diagnosis + "_baseline.tsv")
-        diagnosis_df = pd.read_csv(diagnosis_path, sep="\t")
-        diagnosis_demographics_df = add_demographics(diagnosis_df, merged_df, diagnosis)
-        diagnosis_demographics_df.set_index(
-            ["participant_id", "session_id"], inplace=True
-        )
-        diagnosis_df.set_index(["participant_id", "session_id"], inplace=True)
-
-        for subject, subject_df in diagnosis_df.groupby(level=0):
-            first_session_id = first_session(subject_df)
-            feature_absence = isinstance(
-                merged_df.loc[(subject, first_session_id), "diagnosis"], float
+            diagnosis_demographics_df.reset_index()
+            diagnosis_demographics_df.set_index(
+                ["participant_id", "session_id"], inplace=True
             )
-            while feature_absence:
-                first_session_id = next_session(subject_df, first_session_id)
+            diagnosis_copy_df.set_index(["participant_id", "session_id"], inplace=True)
+            for subject, subject_df in diagnosis_copy_df.groupby(level=0):
+                first_session_id = first_session(subject_df)
                 feature_absence = isinstance(
                     merged_df.loc[(subject, first_session_id), "diagnosis"], float
                 )
-            demographics_subject_df = merged_df.loc[subject]
-
-            # Extract features
-            results_df.loc[diagnosis, "n_subjects"] += 1
-            results_df.loc[diagnosis, "n_scans"] += len(subject_df)
-            diagnosis_dict[diagnosis]["age"].append(
-                merged_df.loc[(subject, first_session_id), fields_dict["age"]]
-            )
-            diagnosis_dict[diagnosis]["MMSE"].append(
-                merged_df.loc[(subject, first_session_id), fields_dict["MMSE"]]
-            )
-            diagnosis_dict[diagnosis]["scans"].append(len(subject_df))
-            sexF = (
-                len(
-                    demographics_subject_df[
-                        (demographics_subject_df[fields_dict["sex"]].isin(["F"]))
-                    ]
+                while feature_absence:
+                    first_session_id = next_session(subject_df, first_session_id)
+
+                    feature_absence = isinstance(
+                        merged_df.loc[(subject, first_session_id), "diagnosis"], float
+                    )
+                demographics_subject_df = merged_df.loc[subject]
+
+                # Extract features
+                results_df.loc[diagnosis, "n_subjects"] += 1
+                results_df.loc[diagnosis, "n_scans"] += len(subject_df)
+                diagnosis_dict[diagnosis]["age"].append(
+                    merged_df.loc[(subject, first_session_id), fields_dict["age"]]
                 )
-                > 0
-            )
-            sexM = (
-                len(
-                    demographics_subject_df[
-                        (demographics_subject_df[fields_dict["sex"]].isin(["M"]))
-                    ]
+                diagnosis_dict[diagnosis]["MMSE"].append(
+                    merged_df.loc[(subject, first_session_id), fields_dict["MMSE"]]
                 )
-                > 0
-            )
-            if sexF:
-                results_df.loc[diagnosis, "sexF"] += 1
-            elif sexM:
-                results_df.loc[diagnosis, "sexM"] += 1
-            else:
-                raise ValueError(
-                    f"The field 'sex' for patient {subject} can not be determined"
+                diagnosis_dict[diagnosis]["scans"].append(len(subject_df))
+                sexF = (
+                    len(
+                        demographics_subject_df[
+                            (demographics_subject_df[fields_dict["sex"]].isin(["F"]))
+                        ]
+                    )
+                    > 0
                 )
-
-            cdr = merged_df.at[(subject, first_session_id), fields_dict["CDR"]]
-            if cdr == 0:
-                results_df.loc[diagnosis, "CDR_0"] += 1
-            elif cdr == 0.5:
-                results_df.loc[diagnosis, "CDR_0.5"] += 1
-            elif cdr == 1:
-                results_df.loc[diagnosis, "CDR_1"] += 1
-            elif cdr == 2:
-                results_df.loc[diagnosis, "CDR_2"] += 1
-            elif cdr == 3:
-                results_df.loc[diagnosis, "CDR_3"] += 1
-            else:
-                warn(f"Patient {subject} has CDR {cdr}")
-
+                sexM = (
+                    len(
+                        demographics_subject_df[
+                            (demographics_subject_df[fields_dict["sex"]].isin(["M"]))
+                        ]
+                    )
+                    > 0
+                )
+                if sexF:
+                    results_df.loc[diagnosis, "sexF"] += 1
+                elif sexM:
+                    results_df.loc[diagnosis, "sexM"] += 1
+                else:
+                    raise ValueError(
+                        f"The field 'sex' for patient {subject} can not be determined"
+                    )
+
+                cdr = merged_df.at[(subject, first_session_id), fields_dict["CDR"]]
+                if cdr == 0:
+                    results_df.loc[diagnosis, "CDR_0"] += 1
+                elif cdr == 0.5:
+                    results_df.loc[diagnosis, "CDR_0.5"] += 1
+                elif cdr == 1:
+                    results_df.loc[diagnosis, "CDR_1"] += 1
+                elif cdr == 2:
+                    results_df.loc[diagnosis, "CDR_2"] += 1
+                elif cdr == 3:
+                    results_df.loc[diagnosis, "CDR_3"] += 1
+                else:
+                    tt = 3  # warn(f"Patient {subject} has CDR {cdr}")
+        else:
+            raise ClinicaDLArgumentError(
+                f"There is no subject with diagnosis {diagnosis}"
+            )
     for diagnosis in diagnoses:
         results_df.loc[diagnosis, "mean_age"] = np.nanmean(
             diagnosis_dict[diagnosis]["age"]
@@ -185,6 +197,6 @@ def demographics_analysis(merged_tsv, formatted_data_path, results_path, diagnos
                     f"NaN values were found for {key} values associated to diagnosis {diagnosis}"
                 )
 
-    results_df.index.name = "diagnosis"
+    results_df.index.name = "group"
 
     results_df.to_csv(results_path, sep="\t")

clinicadl/tsvtools/analysis/analysis_cli.py

@@ -5,23 +5,21 @@
 
 @click.command(name="analysis", no_args_is_help=True)
 @cli_param.argument.merged_tsv
-@cli_param.argument.formatted_data_directory
-@cli_param.argument.results_directory
+@cli_param.argument.formatted_data_tsv
+@cli_param.argument.results_tsv
 @cli_param.option.diagnoses
-def cli(merged_tsv, formatted_data_directory, results_directory, diagnoses):
+def cli(merged_tsv, formatted_data_tsv, results_tsv, diagnoses):
     """Demographic analysis of the extracted labels.
 
     MERGED_TSV is the output of `clinica iotools merge-tsv`.
 
-    FORMATTED_DATA_DIRECTORY is the outputs of `clinicadl tsvtool getlabels`.
+    FORMATTED_DATA_TSV is the outputs of `clinicadl tsvtool getlabels`.
 
     Results are stored in RESULTS_DIRECTORY.
     """
     from .analysis import demographics_analysis
 
-    demographics_analysis(
-        merged_tsv, formatted_data_directory, results_directory, diagnoses
-    )
+    demographics_analysis(merged_tsv, formatted_data_tsv, results_tsv, diagnoses)
 
 
 if __name__ == "__main__":