Skip to content

Commit

Permalink
Merge pull request #139 from lucalianas/feat/new_gleason_annotations
Browse files Browse the repository at this point in the history 
Updated support for Gleason pattern annotations
  • Loading branch information
@lucalianas
lucalianas authored Nov 14, 2023
2 parents 86c740d + 930a17e commit 023b893
Show file tree
Hide file tree
Showing 38 changed files with 3,594 additions and 670 deletions.
118 changes: 86 additions & 32 deletions promort/clinical_annotations_manager/management/commands/get_cores_clinical_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import logging

logger = logging.getLogger('promort_commands')
logger = logging.getLogger("promort_commands")


class Command(BaseCommand):
Expand All @@ -34,64 +34,118 @@ class Command(BaseCommand):
"""

def add_arguments(self, parser):
parser.add_argument('--output_file', dest='output', type=str, required=True,
help='path of the output CSV file')
parser.add_argument('--page_size', dest='page_size', type=int, default=0,
help='the number of records retrieved for each page (this will enable pagination)')
parser.add_argument(
"--output_file",
dest="output",
type=str,
required=True,
help="path of the output CSV file",
)
parser.add_argument(
"--page_size",
dest="page_size",
type=int,
default=0,
help="the number of records retrieved for each page (this will enable pagination)",
)

def _dump_data(self, page_size, csv_writer):
if page_size > 0:
logger.info('Pagination enabled (%d records for page)', page_size)
ca_qs = CoreAnnotation.objects.get_queryset().order_by('creation_date')
logger.info("Pagination enabled (%d records for page)", page_size)
ca_qs = CoreAnnotation.objects.get_queryset().order_by("creation_date")
paginator = Paginator(ca_qs, page_size)
for x in paginator.page_range:
logger.info('-- page %d --', x)
logger.info("-- page %d --", x)
page = paginator.page(x)
for ca in page.object_list:
self._dump_row(ca, csv_writer)
else:
logger.info('Loading full batch')
logger.info("Loading full batch")
core_annotations = CoreAnnotation.objects.all()
for ca in core_annotations:
self._dump_row(ca, csv_writer)

def _dump_row(self, core_annotation, csv_writer):
try:
action_start_time = core_annotation.action_start_time.strftime('%Y-%m-%d %H:%M:%S')
action_start_time = core_annotation.action_start_time.strftime(
"%Y-%m-%d %H:%M:%S"
)
except AttributeError:
action_start_time = None
try:
action_complete_time = core_annotation.action_complete_time.strftime('%Y-%m-%d %H:%M:%S')
action_complete_time = core_annotation.action_complete_time.strftime(
"%Y-%m-%d %H:%M:%S"
)
except AttributeError:
action_complete_time = None
csv_writer.writerow(
{
'case_id': core_annotation.core.slice.slide.case.id,
'slide_id': core_annotation.core.slice.slide.id,
'rois_review_step_id': core_annotation.annotation_step.rois_review_step.label,
'clinical_review_step_id': core_annotation.annotation_step.label,
'reviewer': core_annotation.author.username,
'core_id': core_annotation.core.id,
'core_label': core_annotation.core.label,
'action_start_time': action_start_time,
'action_complete_time': action_complete_time,
'creation_date': core_annotation.creation_date.strftime('%Y-%m-%d %H:%M:%S'),
'primary_gleason': core_annotation.primary_gleason,
'secondary_gleason': core_annotation.secondary_gleason,
'gleason_group_who_16': core_annotation.get_grade_group_text()
"case_id": core_annotation.core.slice.slide.case.id,
"slide_id": core_annotation.core.slice.slide.id,
"rois_review_step_id": core_annotation.annotation_step.rois_review_step.label,
"clinical_review_step_id": core_annotation.annotation_step.label,
"reviewer": core_annotation.author.username,
"core_id": core_annotation.core.id,
"core_label": core_annotation.core.label,
"action_start_time": action_start_time,
"action_complete_time": action_complete_time,
"creation_date": core_annotation.creation_date.strftime(
"%Y-%m-%d %H:%M:%S"
),
"primary_gleason": core_annotation.get_primary_gleason(),
"secondary_gleason": core_annotation.get_secondary_gleason(),
"gleason_group_who_16": core_annotation.get_grade_group_text(),
"nuclear_grade_size": core_annotation.nuclear_grade_size,
"intraluminal_acinar_differentiation_grade": core_annotation.intraluminal_acinar_differentiation_grade,
"intraluminal_secretions": core_annotation.intraluminal_secretions,
"central_maturation": core_annotation.central_maturation,
"extra_cribriform_gleason_score": core_annotation.extra_cribriform_gleason_score,
"predominant_rsg": core_annotation.predominant_rsg,
"highest_rsg": core_annotation.highest_rsg,
"rsg_within_highest_grade_area": core_annotation.rsg_within_highest_grade_area,
"perineural_invasion": core_annotation.perineural_invasion,
"perineural_growth_with_cribriform_patterns": core_annotation.perineural_growth_with_cribriform_patterns,
"extrapostatic_extension": core_annotation.extraprostatic_extension,
"largest_confluent_sheet": core_annotation.get_largest_confluent_sheet(),
"total_cribriform_area": core_annotation.get_total_cribriform_area(),
}
)

def _export_data(self, out_file, page_size):
header = ['case_id', 'slide_id', 'rois_review_step_id', 'clinical_review_step_id', 'reviewer',
'core_id', 'core_label', 'action_start_time', 'action_complete_time', 'creation_date',
'primary_gleason', 'secondary_gleason', 'gleason_group_who_16']
with open(out_file, 'w') as ofile:
writer = DictWriter(ofile, delimiter=',', fieldnames=header)
header = [
"case_id",
"slide_id",
"rois_review_step_id",
"clinical_review_step_id",
"reviewer",
"core_id",
"core_label",
"action_start_time",
"action_complete_time",
"creation_date",
"primary_gleason",
"secondary_gleason",
"gleason_group_who_16",
"nuclear_grade_size",
"intraluminal_acinar_differentiation_grade",
"intraluminal_secretions",
"central_maturation",
"extra_cribriform_gleason_score",
"predominant_rsg",
"highest_rsg",
"rsg_within_highest_grade_area",
"perineural_invasion",
"perineural_growth_with_cribriform_patterns",
"extrapostatic_extension",
"largest_confluent_sheet",
"total_cribriform_area",
]
with open(out_file, "w") as ofile:
writer = DictWriter(ofile, delimiter=",", fieldnames=header)
writer.writeheader()
self._dump_data(page_size, writer)

def handle(self, *args, **opts):
logger.info('=== Starting export job ===')
self._export_data(opts['output'], opts['page_size'])
logger.info('=== Data saved to %s ===', opts['output'])
logger.info("=== Starting export job ===")
self._export_data(opts["output"], opts["page_size"])
logger.info("=== Data saved to %s ===", opts["output"])
146 changes: 103 additions & 43 deletions promort/clinical_annotations_manager/management/commands/get_focus_regions_clinical_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import logging

logger = logging.getLogger('promort_commands')
logger = logging.getLogger("promort_commands")


class Command(BaseCommand):
Expand All @@ -34,18 +34,30 @@ class Command(BaseCommand):
"""

def add_arguments(self, parser):
parser.add_argument('--output_file', dest='output', type=str, required=True,
help='path of the output CSV file')
parser.add_argument('--page_size', dest='page_size', type=int, default=0,
help='the number of records retrieved for each page (this will enable pagination)')
parser.add_argument(
"--output_file",
dest="output",
type=str,
required=True,
help="path of the output CSV file",
)
parser.add_argument(
"--page_size",
dest="page_size",
type=int,
default=0,
help="the number of records retrieved for each page (this will enable pagination)",
)

def _dump_data(self, page_size, csv_writer):
if page_size > 0:
logger.info('Pagination enabled (%d records for page)', page_size)
fra_qs = FocusRegionAnnotation.objects.get_queryset().order_by('creation_date')
logger.info("Pagination enabled (%d records for page)", page_size)
fra_qs = FocusRegionAnnotation.objects.get_queryset().order_by(
"creation_date"
)
paginator = Paginator(fra_qs, page_size)
for x in paginator.page_range:
logger.info('-- page %d --', x)
logger.info("-- page %d --", x)
page = paginator.page(x)
for fra in page.object_list:
self._dump_row(fra, csv_writer)
Expand All @@ -56,53 +68,101 @@ def _dump_data(self, page_size, csv_writer):

def _dump_row(self, focus_region_annotation, csv_writer):
try:
action_start_time = focus_region_annotation.action_start_time.strftime('%Y-%m-%d %H:%M:%S')
action_start_time = focus_region_annotation.action_start_time.strftime(
"%Y-%m-%d %H:%M:%S"
)
except AttributeError:
action_start_time = None
try:
action_complete_time = focus_region_annotation.action_complete_time.strftime('%Y-%m-%d %H:%M:%S')
action_complete_time = (
focus_region_annotation.action_complete_time.strftime(
"%Y-%m-%d %H:%M:%S"
)
)
except AttributeError:
action_complete_time = None
csv_writer.writerow(
{
'case_id': focus_region_annotation.focus_region.core.slice.slide.case.id,
'slide_id': focus_region_annotation.focus_region.core.slice.slide.id,
'rois_review_step_id': focus_region_annotation.annotation_step.rois_review_step.label,
'clinical_review_step_id': focus_region_annotation.annotation_step.label,
'reviewer': focus_region_annotation.author.username,
'focus_region_id': focus_region_annotation.focus_region.id,
'focus_region_label': focus_region_annotation.focus_region.label,
'core_id': focus_region_annotation.focus_region.core.id,
'core_label': focus_region_annotation.focus_region.core.label,
'action_start_time': action_start_time,
'action_complete_time': action_complete_time,
'creation_date': focus_region_annotation.creation_date.strftime('%Y-%m-%d %H:%M:%S'),
'perineural_involvement': focus_region_annotation.perineural_involvement,
'intraductal_carcinoma': focus_region_annotation.intraductal_carcinoma,
'ductal_carcinoma': focus_region_annotation.ductal_carcinoma,
'poorly_formed_glands': focus_region_annotation.poorly_formed_glands,
'cribriform_pattern': focus_region_annotation.cribriform_pattern,
'small_cell_signet_ring': focus_region_annotation.small_cell_signet_ring,
'hypernephroid_pattern': focus_region_annotation.hypernephroid_pattern,
'mucinous': focus_region_annotation.mucinous,
'comedo_necrosis': focus_region_annotation.comedo_necrosis,
'total_gleason_4_area': focus_region_annotation.get_total_gleason_4_area(),
'gleason_4_percentage': focus_region_annotation.get_gleason_4_percentage()
"case_id": focus_region_annotation.focus_region.core.slice.slide.case.id,
"slide_id": focus_region_annotation.focus_region.core.slice.slide.id,
"rois_review_step_id": focus_region_annotation.annotation_step.rois_review_step.label,
"clinical_review_step_id": focus_region_annotation.annotation_step.label,
"reviewer": focus_region_annotation.author.username,
"focus_region_id": focus_region_annotation.focus_region.id,
"focus_region_label": focus_region_annotation.focus_region.label,
"core_id": focus_region_annotation.focus_region.core.id,
"core_label": focus_region_annotation.focus_region.core.label,
"action_start_time": action_start_time,
"action_complete_time": action_complete_time,
"creation_date": focus_region_annotation.creation_date.strftime(
"%Y-%m-%d %H:%M:%S"
),
"perineural_involvement": focus_region_annotation.perineural_involvement,
"intraductal_carcinoma": focus_region_annotation.intraductal_carcinoma,
"ductal_carcinoma": focus_region_annotation.ductal_carcinoma,
"poorly_formed_glands": focus_region_annotation.poorly_formed_glands,
"cribriform_pattern": focus_region_annotation.cribriform_pattern,
"small_cell_signet_ring": focus_region_annotation.small_cell_signet_ring,
"hypernephroid_pattern": focus_region_annotation.hypernephroid_pattern,
"mucinous": focus_region_annotation.mucinous,
"comedo_necrosis": focus_region_annotation.comedo_necrosis,
"total_gleason_3_area": focus_region_annotation.get_total_gleason_area(
"G3"
),
"total_gleason_4_area": focus_region_annotation.get_total_gleason_area(
"G4"
),
"total_gleason_5_area": focus_region_annotation.get_total_gleason_area(
"G5"
),
"gleason_3_percentage": focus_region_annotation.get_gleason_percentage(
"G3"
),
"gleason_4_percentage": focus_region_annotation.get_gleason_percentage(
"G4"
),
"gleason_5_percentage": focus_region_annotation.get_gleason_percentage(
"G5"
),
}
)

def _export_data(self, out_file, page_size):
header = ['case_id', 'slide_id', 'rois_review_step_id', 'clinical_review_step_id', 'reviewer',
'focus_region_id', 'focus_region_label', 'core_id', 'core_label', 'action_start_time',
'action_complete_time', 'creation_date', 'perineural_involvement', 'intraductal_carcinoma',
'ductal_carcinoma', 'poorly_formed_glands', 'cribriform_pattern', 'small_cell_signet_ring',
'hypernephroid_pattern', 'mucinous', 'comedo_necrosis', 'total_gleason_4_area', 'gleason_4_percentage']
with open(out_file, 'w') as ofile:
writer = DictWriter(ofile, delimiter=',', fieldnames=header)
header = [
"case_id",
"slide_id",
"rois_review_step_id",
"clinical_review_step_id",
"reviewer",
"focus_region_id",
"focus_region_label",
"core_id",
"core_label",
"action_start_time",
"action_complete_time",
"creation_date",
"perineural_involvement",
"intraductal_carcinoma",
"ductal_carcinoma",
"poorly_formed_glands",
"cribriform_pattern",
"small_cell_signet_ring",
"hypernephroid_pattern",
"mucinous",
"comedo_necrosis",
"total_gleason_3_area",
"gleason_3_percentage",
"total_gleason_4_area",
"gleason_4_percentage",
"total_gleason_5_area",
"gleason_5_percentage",
]
with open(out_file, "w") as ofile:
writer = DictWriter(ofile, delimiter=",", fieldnames=header)
writer.writeheader()
self._dump_data(page_size, writer)

def handle(self, *args, **opts):
logger.info('=== Starting export job ===')
self._export_data(opts['output'], opts['page_size'])
logger.info('=== Data saved to %s ===', opts['output'])
logger.info("=== Starting export job ===")
self._export_data(opts["output"], opts["page_size"])
logger.info("=== Data saved to %s ===", opts["output"])
Loading

0 comments on commit 023b893

Please sign in to comment.