From a132c5968c3bb78d7a6182738c2e5fc7f05003e0 Mon Sep 17 00:00:00 2001 From: Miles Wells Date: Thu, 31 Aug 2023 16:16:45 +0300 Subject: [PATCH 1/5] Remove references to project field --- alyx/actions/views.py | 10 +++++----- scripts/sync_ucl/prune_cortexlab.py | 10 ++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/alyx/actions/views.py b/alyx/actions/views.py index e777ca6e..4a4ff9cc 100644 --- a/alyx/actions/views.py +++ b/alyx/actions/views.py @@ -332,7 +332,7 @@ class SessionAPIList(generics.ListCreateAPIView): - **lab**: lab name (exact) - **task_protocol** (icontains) - **location**: location name (icontains) - - **project**: project name (icontains) + - **projects**: project name (icontains) - **json**: queries on json fields, for example here `tutu` - exact/equal lookup: `/sessions?extended_qc=tutu,True`, - gte lookup: `/sessions/?extended_qc=tutu__gte,0.5`, @@ -354,10 +354,10 @@ class SessionAPIList(generics.ListCreateAPIView): - **histology**: returns sessions for which the subject has an histology session: `/sessions?histology=True` - **django**: generic filter allowing lookups (same syntax as json filter) - `/sessions?django=project__name__icontains,matlab - filters sessions that have matlab in the project name - `/sessions?django=~project__name__icontains,matlab - does the exclusive set: filters sessions that do not have matlab in the project name + `/sessions?django=projects__name__icontains,matlab + filters sessions that have matlab in the project names + `/sessions?django=~projects__name__icontains,matlab + does the exclusive set: filters sessions that do not have matlab in the project names [===> session model reference](/admin/doc/models/actions.session) """ diff --git a/scripts/sync_ucl/prune_cortexlab.py b/scripts/sync_ucl/prune_cortexlab.py index aef757e2..766c818e 100755 --- a/scripts/sync_ucl/prune_cortexlab.py +++ b/scripts/sync_ucl/prune_cortexlab.py @@ -17,15 +17,13 @@ json_file_out = '../scripts/sync_ucl/cortexlab_pruned.json' -# Since we currently still use both the project and the projects field, we need to filter for -# either containing an IBL project -ibl_proj = Q(project__name__icontains='ibl') | Q(projects__name__icontains='ibl') +# Filter for sessions containing an IBL project +ibl_proj = Q(projects__name__icontains='ibl') ses = Session.objects.using('cortexlab').filter(ibl_proj) # remove all subjects that never had anything to do with IBL sub_ibl = list(ses.values_list('subject', flat=True)) sub_ibl += list(Subject.objects.values_list('pk', flat=True)) -sub_ibl += list(Subject.objects.using('cortexlab').filter( - projects__name__icontains='ibl').values_list('pk', flat=True)) +sub_ibl += list(Subject.objects.using('cortexlab').filter(ibl_proj).values_list('pk', flat=True)) Subject.objects.using('cortexlab').exclude(pk__in=sub_ibl).delete() # then remove base Sessions @@ -75,7 +73,7 @@ # import projects from cortexlab. remove those that don't correspond to any session -pk_projs = list(filter(None, flatten(ses_ucl.values_list('project', 'projects').distinct()))) +pk_projs = list(filter(None, flatten(ses_ucl.values_list('projects').distinct()))) pk_projs += list(Project.objects.values_list('pk', flat=True)) Project.objects.using('cortexlab').exclude(pk__in=pk_projs).delete() From 338b2cfbb8728a44e8392888b631399653af2815 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Thu, 14 Mar 2024 16:55:38 +0200 Subject: [PATCH 2/5] Remove project field from Session model --- .../migrations/0022_project_to_projects.py | 39 +++++++++++++++++++ .../migrations/0023_remove_session_project.py | 17 ++++++++ alyx/actions/models.py | 3 -- 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 alyx/actions/migrations/0022_project_to_projects.py create mode 100644 alyx/actions/migrations/0023_remove_session_project.py diff --git a/alyx/actions/migrations/0022_project_to_projects.py b/alyx/actions/migrations/0022_project_to_projects.py new file mode 100644 index 00000000..b0a40011 --- /dev/null +++ b/alyx/actions/migrations/0022_project_to_projects.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.10 on 2024-03-14 14:28 +import logging + +from django.db import migrations +from django.db.models import F, Q + +logger = logging.getLogger(__name__) + +def project2projects(apps, schema_editor): + """ + Find sessions where the project field (singular) value is not in the projects (plural) many-to-many + field and updates them. + + Tested on local instance. + """ + Session = apps.get_model('actions', 'Session') + sessions = Session.objects.exclude(Q(project__isnull=True) | Q(projects=F('project'))) + + # Check query worked + # from one.util import ensure_list + # for session in sessions.values('pk', 'project', 'projects'): + # assert session['project'] not in ensure_list(session['projects']) + + for session in sessions: + session.projects.add(session.project) + # session.project = None + # session.save() # No need to save + + assert Session.objects.exclude(Q(project__isnull=True) | Q(projects=F('project'))).count() == 0 + logger.info(f'project -> projects: {sessions.count():,g} sessions updated') + + +class Migration(migrations.Migration): + + dependencies = [ + ('actions', '0021_alter_session_extended_qc'), + ] + + operations = [migrations.RunPython(project2projects)] diff --git a/alyx/actions/migrations/0023_remove_session_project.py b/alyx/actions/migrations/0023_remove_session_project.py new file mode 100644 index 00000000..c746c337 --- /dev/null +++ b/alyx/actions/migrations/0023_remove_session_project.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.10 on 2024-03-14 14:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('actions', '0022_project_to_projects'), + ] + + operations = [ + migrations.RemoveField( + model_name='session', + name='project', + ), + ] diff --git a/alyx/actions/models.py b/alyx/actions/models.py index fd706e3a..230f9795 100644 --- a/alyx/actions/models.py +++ b/alyx/actions/models.py @@ -242,9 +242,6 @@ class Session(BaseAction): parent_session = models.ForeignKey('Session', null=True, blank=True, on_delete=models.SET_NULL, help_text="Hierarchical parent to this session") - project = models.ForeignKey('subjects.Project', null=True, blank=True, - on_delete=models.SET_NULL, verbose_name='Session Project', - related_name='oldproject') projects = models.ManyToManyField('subjects.Project', blank=True, verbose_name='Session Projects') type = models.CharField(max_length=255, null=True, blank=True, From fe67197a5d9d722e4e984b05eeffab8fdb477418 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Thu, 14 Mar 2024 18:47:51 +0200 Subject: [PATCH 3/5] Set session.projects as Subject.projects on save --- alyx/actions/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/alyx/actions/models.py b/alyx/actions/models.py index 230f9795..44d47c14 100644 --- a/alyx/actions/models.py +++ b/alyx/actions/models.py @@ -264,12 +264,14 @@ class Session(BaseAction): verbose_name='last updated') def save(self, *args, **kwargs): - # Default project is the subject's project. - if not self.project_id: - self.project = self.subject.projects.first() + # Default project is the subject's projects. if not self.lab: self.lab = self.subject.lab - return super(Session, self).save(*args, **kwargs) + obj = super(Session, self).save(*args, **kwargs) + if self.projects.count() == 0 and self.subject.projects.count() > 0: + from subjects.models import Project + self.projects.add(*Project.objects.filter(subject=self.subject)) + return obj def __str__(self): try: From e23ff4864f488a2e749ff066eca083ee195f1416 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Thu, 14 Mar 2024 19:41:53 +0200 Subject: [PATCH 4/5] project -> projects in filters --- alyx/actions/views.py | 2 -- alyx/experiments/tests_rest.py | 2 +- alyx/experiments/views.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/alyx/actions/views.py b/alyx/actions/views.py index bec3461f..abdbe04b 100644 --- a/alyx/actions/views.py +++ b/alyx/actions/views.py @@ -240,8 +240,6 @@ class SessionFilter(BaseActionFilter): extended_qc = django_filters.CharFilter(field_name='extended_qc', method='filter_extended_qc') procedures = django_filters.CharFilter(field_name='procedures__name', lookup_expr='icontains') projects = django_filters.CharFilter(field_name='projects__name', lookup_expr='icontains') - # below is an alias to keep compatibility after moving project FK field to projects M2M - project = django_filters.CharFilter(field_name='projects__name', lookup_expr='icontains') # brain region filters atlas_name = django_filters.CharFilter(field_name='name__icontains', method='atlas') atlas_acronym = django_filters.CharFilter(field_name='acronym__iexact', method='atlas') diff --git a/alyx/experiments/tests_rest.py b/alyx/experiments/tests_rest.py index a717d86f..bf44018e 100644 --- a/alyx/experiments/tests_rest.py +++ b/alyx/experiments/tests_rest.py @@ -113,7 +113,7 @@ def test_probe_insertion_rest(self): self.assertTrue(len(probe_ins) == 0) # test the project filter - urlf = (reverse('probeinsertion-list') + '?&project=brain_wide') + urlf = (reverse('probeinsertion-list') + '?&projects=brain_wide') probe_ins = self.ar(self.client.get(urlf)) self.assertTrue(len(probe_ins) == 0) diff --git a/alyx/experiments/views.py b/alyx/experiments/views.py index 7b329f93..8602fc07 100644 --- a/alyx/experiments/views.py +++ b/alyx/experiments/views.py @@ -76,7 +76,7 @@ class ProbeInsertionFilter(BaseFilterSet): datasets = CharFilter(field_name='datasets', method='filter_datasets') dataset_qc_lte = CharFilter(field_name='dataset_qc', method='filter_dataset_qc_lte') lab = CharFilter(field_name='session__lab__name', lookup_expr='iexact') - project = CharFilter(field_name='session__project__name', lookup_expr='icontains') + projects = CharFilter(field_name='session__projects__name', lookup_expr='icontains') task_protocol = CharFilter(field_name='session__task_protocol', lookup_expr='icontains') tag = CharFilter(field_name='tag', method='filter_tag') # brain region filters @@ -143,7 +143,7 @@ class ProbeInsertionList(generics.ListCreateAPIView): - **session**: session UUID`/insertions?session=aad23144-0e52-4eac-80c5-c4ee2decb198` - **task_protocol** (icontains) - **location**: location name (icontains) - - **project**: project name (icontains) + - **projects**: project name (icontains) - **model**: probe model name `/insertions?model=3A` - **lab**: lab name (exact) - **tag**: tag name (icontains) From 65b4a2ab7218dce6d487c7d1bfd8af235e8cc366 Mon Sep 17 00:00:00 2001 From: k1o0 Date: Fri, 15 Mar 2024 14:32:05 +0200 Subject: [PATCH 5/5] Keep filter as 'project' --- alyx/actions/views.py | 2 ++ alyx/experiments/tests_rest.py | 2 +- alyx/experiments/views.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/alyx/actions/views.py b/alyx/actions/views.py index abdbe04b..bec3461f 100644 --- a/alyx/actions/views.py +++ b/alyx/actions/views.py @@ -240,6 +240,8 @@ class SessionFilter(BaseActionFilter): extended_qc = django_filters.CharFilter(field_name='extended_qc', method='filter_extended_qc') procedures = django_filters.CharFilter(field_name='procedures__name', lookup_expr='icontains') projects = django_filters.CharFilter(field_name='projects__name', lookup_expr='icontains') + # below is an alias to keep compatibility after moving project FK field to projects M2M + project = django_filters.CharFilter(field_name='projects__name', lookup_expr='icontains') # brain region filters atlas_name = django_filters.CharFilter(field_name='name__icontains', method='atlas') atlas_acronym = django_filters.CharFilter(field_name='acronym__iexact', method='atlas') diff --git a/alyx/experiments/tests_rest.py b/alyx/experiments/tests_rest.py index bf44018e..a717d86f 100644 --- a/alyx/experiments/tests_rest.py +++ b/alyx/experiments/tests_rest.py @@ -113,7 +113,7 @@ def test_probe_insertion_rest(self): self.assertTrue(len(probe_ins) == 0) # test the project filter - urlf = (reverse('probeinsertion-list') + '?&projects=brain_wide') + urlf = (reverse('probeinsertion-list') + '?&project=brain_wide') probe_ins = self.ar(self.client.get(urlf)) self.assertTrue(len(probe_ins) == 0) diff --git a/alyx/experiments/views.py b/alyx/experiments/views.py index 8602fc07..233dd8db 100644 --- a/alyx/experiments/views.py +++ b/alyx/experiments/views.py @@ -76,7 +76,7 @@ class ProbeInsertionFilter(BaseFilterSet): datasets = CharFilter(field_name='datasets', method='filter_datasets') dataset_qc_lte = CharFilter(field_name='dataset_qc', method='filter_dataset_qc_lte') lab = CharFilter(field_name='session__lab__name', lookup_expr='iexact') - projects = CharFilter(field_name='session__projects__name', lookup_expr='icontains') + project = CharFilter(field_name='session__projects__name', lookup_expr='icontains') task_protocol = CharFilter(field_name='session__task_protocol', lookup_expr='icontains') tag = CharFilter(field_name='tag', method='filter_tag') # brain region filters @@ -143,7 +143,7 @@ class ProbeInsertionList(generics.ListCreateAPIView): - **session**: session UUID`/insertions?session=aad23144-0e52-4eac-80c5-c4ee2decb198` - **task_protocol** (icontains) - **location**: location name (icontains) - - **projects**: project name (icontains) + - **project**: project name (icontains) - **model**: probe model name `/insertions?model=3A` - **lab**: lab name (exact) - **tag**: tag name (icontains) @@ -436,7 +436,7 @@ class FOVList(generics.ListCreateAPIView): `/fields-of-view?provenance=Estimate` - **atlas**: One or more brain regions covered by a field of view - **subject**: subject nickname: `/fields-of-view?subject=Algernon` - - **project**: the + - **project**: the project name - **date**: session date: `/fields-of-view?date=2020-01-15` - **experiment_number**: session number `/fields-of-view?experiment_number=1` - **session**: `/fields-of-view?session=aad23144-0e52-4eac-80c5-c4ee2decb198`