2.0.0

alyx/actions/admin.py

@@ -390,7 +390,8 @@ def is_water_restricted(self, obj):
 class WeighingForm(BaseActionForm):
     def __init__(self, *args, **kwargs):
         super(WeighingForm, self).__init__(*args, **kwargs)
-        self.fields['subject'].queryset = self.current_user.get_allowed_subjects()
+        if 'subject' in self.fields:
+            self.fields['subject'].queryset = self.current_user.get_allowed_subjects()
         if self.fields.keys():
             self.fields['weight'].widget.attrs.update({'autofocus': 'autofocus'})
 
@@ -455,10 +456,10 @@ class DatasetInline(BaseInlineAdmin):
     show_change_link = True
     model = Dataset
     extra = 1
-    fields = ('name', 'dataset_type', 'collection', '_online', 'version', 'created_by',
-              'created_datetime')
+    fields = ('name', 'dataset_type', 'collection', '_online', 'version', 'qc',
+              'created_by', 'created_datetime')
     readonly_fields = fields
-    ordering = ("name",)
+    ordering = ('name',)
 
     def _online(self, obj):
         return obj.is_online

alyx/actions/migrations/0021_alter_session_extended_qc.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-03-12 13:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('actions', '0020_alter_notification_notification_type_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='session',
+            name='extended_qc',
+            field=models.JSONField(blank=True, help_text='Structured data about session QC, formatted in a user-defined way', null=True),
+        ),
+    ]

alyx/actions/migrations/0022_project_to_projects.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.10 on 2024-03-14 14:28
+import logging
+
+from django.db import migrations
+from django.db.models import F, Q
+
+logger = logging.getLogger(__name__)
+
+def project2projects(apps, schema_editor):
+    """
+    Find sessions where the project field (singular) value is not in the projects (plural) many-to-many
+    field and updates them.
+
+    Tested on local instance.
+    """
+    Session = apps.get_model('actions', 'Session')
+    sessions = Session.objects.exclude(Q(project__isnull=True) | Q(projects=F('project')))
+
+    # Check query worked
+    # from one.util import ensure_list
+    # for session in sessions.values('pk', 'project', 'projects'):
+    #     assert session['project'] not in ensure_list(session['projects'])
+
+    for session in sessions:
+        session.projects.add(session.project)
+        # session.project = None
+        # session.save()  # No need to save
+
+    assert Session.objects.exclude(Q(project__isnull=True) | Q(projects=F('project'))).count() == 0
+    logger.info(f'project -> projects: {sessions.count():,g} sessions updated')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('actions', '0021_alter_session_extended_qc'),
+    ]
+
+    operations = [migrations.RunPython(project2projects)]

alyx/actions/migrations/0023_remove_session_project.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.10 on 2024-03-14 14:54
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('actions', '0022_project_to_projects'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='session',
+            name='project',
+        ),
+    ]

alyx/actions/models.py

@@ -1,7 +1,9 @@
 from datetime import timedelta
-import structlog
 from math import inf
 
+import structlog
+from one.alf.spec import QC
+
 from django.conf import settings
 from django.core.validators import MinValueValidator
 from django.db import models
@@ -240,9 +242,6 @@ class Session(BaseAction):
     parent_session = models.ForeignKey('Session', null=True, blank=True,
                                        on_delete=models.SET_NULL,
                                        help_text="Hierarchical parent to this session")
-    project = models.ForeignKey('subjects.Project', null=True, blank=True,
-                                on_delete=models.SET_NULL, verbose_name='Session Project',
-                                related_name='oldproject')
     projects = models.ManyToManyField('subjects.Project', blank=True,
                                       verbose_name='Session Projects')
     type = models.CharField(max_length=255, null=True, blank=True,
@@ -253,27 +252,19 @@ class Session(BaseAction):
     n_trials = models.IntegerField(blank=True, null=True)
     n_correct_trials = models.IntegerField(blank=True, null=True)
 
-    QC_CHOICES = [
-        (50, 'CRITICAL',),
-        (40, 'FAIL',),
-        (30, 'WARNING',),
-        (0, 'NOT_SET',),
-        (10, 'PASS',),
-    ]
-
-    qc = models.IntegerField(default=0, choices=QC_CHOICES,
+    QC_CHOICES = [(e.value, e.name) for e in QC]
+    qc = models.IntegerField(default=QC.NOT_SET, choices=QC_CHOICES,
                              help_text=' / '.join([str(q[0]) + ': ' + q[1] for q in QC_CHOICES]))
+
     extended_qc = models.JSONField(null=True, blank=True,
-                                   help_text="Structured data about session QC,"
+                                   help_text="Structured data about session QC, "
                                              "formatted in a user-defined way")
 
     auto_datetime = models.DateTimeField(auto_now=True, blank=True, null=True,
                                          verbose_name='last updated')
 
     def save(self, *args, **kwargs):
-        # Default project is the subject's project.
-        if not self.project_id:
-            self.project = self.subject.projects.first()
+        # Default project is the subject's projects.
         if not self.lab:
             self.lab = self.subject.lab
         return super(Session, self).save(*args, **kwargs)

alyx/actions/serializers.py

@@ -98,12 +98,13 @@ class SessionDatasetsSerializer(serializers.ModelSerializer):
         queryset=DatasetType.objects.all(),
     )
     default_revision = serializers.CharField(source='default_dataset')
+    qc = BaseSerializerEnumField(required=False)
 
     class Meta:
         list_serializer_class = FilterDatasetSerializer
         model = Dataset
         fields = ('id', 'name', 'dataset_type', 'data_url', 'url', 'file_size',
-                  'hash', 'version', 'collection', 'revision', 'default_revision')
+                  'hash', 'version', 'collection', 'revision', 'default_revision', 'qc')
 
 
 class SessionWaterAdminSerializer(serializers.ModelSerializer):

alyx/actions/tests_rest.py

@@ -266,8 +266,10 @@ def test_sessions(self):
         # test dataset type filters
         dtype1, _ = DatasetType.objects.get_or_create(name='trials.table')
         dtype2, _ = DatasetType.objects.get_or_create(name='wheel.position')
-        Dataset.objects.create(session=ses, name='_ibl_trials.table.pqt', dataset_type=dtype1)
-        Dataset.objects.create(session=ses, name='_ibl_wheel.position.npy', dataset_type=dtype2)
+        Dataset.objects.create(
+            session=ses, name='_ibl_trials.table.pqt', dataset_type=dtype1, qc=40)
+        Dataset.objects.create(
+            session=ses, name='_ibl_wheel.position.npy', dataset_type=dtype2, qc=30)
         d = self.ar(self.client.get(reverse('session-list') + '?dataset_types=wheel.position'))
         self.assertCountEqual([str(ses.pk)], (x['id'] for x in d))
         q = '?dataset_types=wheel.position,trials.table'  # Check with list
@@ -280,6 +282,22 @@ def test_sessions(self):
         self.assertCountEqual([str(ses.pk)], (x['id'] for x in d))
         q = '?datasets=wheel.position'
         self.assertFalse(self.ar(self.client.get(reverse('session-list') + q)))
+        # multiple datasets
+        q = '?datasets=_ibl_wheel.position.npy,_ibl_trials.table.pqt'
+        d = self.ar(self.client.get(reverse('session-list') + q))
+        self.assertCountEqual([str(ses.pk)], (x['id'] for x in d))
+        # datasets + qc (expect to return sessions where defined datasets have correct QC)
+        q = '?datasets=_ibl_wheel.position.npy,_ibl_trials.table.pqt&dataset_qc_lte=WARNING'
+        self.assertFalse(self.ar(self.client.get(reverse('session-list') + q)))
+        q = '?datasets=_ibl_wheel.position.npy&dataset_qc_lte=WARNING'
+        d = self.ar(self.client.get(reverse('session-list') + q))
+        self.assertCountEqual([str(ses.pk)], (x['id'] for x in d), 'failed to return session')
+        # qc alone (expect to return sessions where any dataset has correct QC)
+        q = '?dataset_qc_lte=WARNING'
+        d = self.ar(self.client.get(reverse('session-list') + q))
+        self.assertCountEqual([str(ses.pk)], (x['id'] for x in d), 'failed to return session')
+        q = '?dataset_qc_lte=10'
+        self.assertFalse(self.ar(self.client.get(reverse('session-list') + q)))
 
     def test_surgeries(self):
         from actions.models import Surgery

alyx/actions/views.py

@@ -1,6 +1,7 @@
 from datetime import timedelta, date
 from operator import itemgetter
 
+from one.alf.spec import QC
 from django.contrib.postgres.fields import JSONField
 from django.db.models import Count, Q, F, ExpressionWrapper, FloatField
 from django.db.models.deletion import Collector
@@ -223,9 +224,11 @@ class ProcedureTypeList(generics.ListCreateAPIView):
 
 
 class SessionFilter(BaseActionFilter):
-    dataset_types = django_filters.CharFilter(field_name='dataset_types',
-                                              method='filter_dataset_types')
+    dataset_types = django_filters.CharFilter(
+        field_name='dataset_types', method='filter_dataset_types')
     datasets = django_filters.CharFilter(field_name='datasets', method='filter_datasets')
+    dataset_qc_lte = django_filters.CharFilter(
+        field_name='dataset_qc', method='filter_dataset_qc_lte')
     performance_gte = django_filters.NumberFilter(field_name='performance',
                                                   method='filter_performance_gte')
     performance_lte = django_filters.NumberFilter(field_name='performance',
@@ -284,13 +287,23 @@ def filter_dataset_types(self, queryset, _, value):
 
     def filter_datasets(self, queryset, _, value):
         # Note this may later be modified to include collections, e.g. ?datasets=alf/obj.attr.ext
+        qc = QC.validate(self.request.query_params.get('dataset_qc_lte', QC.FAIL))
         dsets = value.split(',')
-        queryset = queryset.filter(data_dataset_session_related__name__in=dsets)
+        queryset = queryset.filter(data_dataset_session_related__name__in=dsets,
+                                   data_dataset_session_related__qc__lte=qc)
         queryset = queryset.annotate(
             dsets_count=Count('data_dataset_session_related', distinct=True))
         queryset = queryset.filter(dsets_count__gte=len(dsets))
         return queryset
 
+    def filter_dataset_qc_lte(self, queryset, _, value):
+        # If filtering on datasets too, `filter_datasets` handles both QC and Datasets
+        if 'datasets' in self.request.query_params:
+            return queryset
+        qc = QC.validate(value)
+        queryset = queryset.filter(data_dataset_session_related__qc__lte=qc)
+        return queryset
+
     def filter_performance_gte(self, queryset, name, perf):
         queryset = queryset.exclude(n_trials__isnull=True)
         pf = ExpressionWrapper(100 * F('n_correct_trials') / F('n_trials'),
@@ -326,13 +339,15 @@ class SessionAPIList(generics.ListCreateAPIView):
     -   **subject**: subject nickname `/sessions?subject=Algernon`
     -   **dataset_types**: dataset type(s) `/sessions?dataset_types=trials.table,camera.times`
     -   **datasets**: dataset name(s) `/sessions?datasets=_ibl_leftCamera.times.npy`
+    -   **dataset_qc_lte**: dataset QC values less than or equal to this
+        `/sessions?dataset_qc_lte=WARNING`
     -   **number**: session number
     -   **users**: experimenters (exact)
     -   **date_range**: date `/sessions?date_range=2020-01-12,2020-01-16`
     -   **lab**: lab name (exact)
     -   **task_protocol** (icontains)
     -   **location**: location name (icontains)
-    -   **project**: project name (icontains)
+    -   **projects**: project name (icontains)
     -   **json**: queries on json fields, for example here `tutu`
         -   exact/equal lookup: `/sessions?extended_qc=tutu,True`,
         -   gte lookup: `/sessions/?extended_qc=tutu__gte,0.5`,
@@ -354,10 +369,10 @@ class SessionAPIList(generics.ListCreateAPIView):
     -   **histology**: returns sessions for which the subject has an histology session:
         `/sessions?histology=True`
     -   **django**: generic filter allowing lookups (same syntax as json filter)
-        `/sessions?django=project__name__icontains,matlab
-        filters sessions that have matlab in the project name
-        `/sessions?django=~project__name__icontains,matlab
-        does the exclusive set: filters sessions that do not have matlab in the project name
+        `/sessions?django=projects__name__icontains,matlab`
+        filters sessions that have matlab in the project names
+        `/sessions?django=~projects__name__icontains,matlab`
+        does the exclusive set: filters sessions that do not have matlab in the project names
 
     [===> session model reference](/admin/doc/models/actions.session)
     """

alyx/alyx/__init__.py

@@ -1 +1 @@
-VERSION = __version__ = '1.18.2'
+VERSION = __version__ = '2.0.0'

alyx/alyx/base.py

@@ -6,7 +6,6 @@
 import sys
 import pytz
 import uuid
-from collections import OrderedDict
 import one.alf.spec
 from datetime import datetime
 import traceback
@@ -454,7 +453,7 @@ def ar(self, r, code=200):
         """
         self.assertTrue(r.status_code == code, r.data)
         pkeys = {'count', 'next', 'previous', 'results'}
-        if isinstance(r.data, OrderedDict) and set(r.data.keys()) == pkeys:
+        if isinstance(r.data, dict) and set(r.data.keys()) == pkeys:
             return r.data['results']
         else:
             return r.data

alyx/data/admin.py

@@ -1,7 +1,7 @@
 from django.db.models import Count, ProtectedError
 from django.contrib import admin, messages
 from django.utils.html import format_html
-from django_admin_listfilter_dropdown.filters import RelatedDropdownFilter
+from django_admin_listfilter_dropdown.filters import RelatedDropdownFilter, ChoiceDropdownFilter
 from rangefilter.filters import DateRangeFilter
 
 from .models import (DataRepositoryType, DataRepository, DataFormat, DatasetType,
@@ -84,16 +84,17 @@ class FileRecordInline(BaseInlineAdmin):
 class DatasetAdmin(BaseExperimentalDataAdmin):
     fields = ['name', '_online', 'version', 'dataset_type', 'file_size', 'hash',
               'session_ro', 'collection', 'auto_datetime', 'revision_', 'default_dataset',
-              '_protected', '_public', 'tags']
+              '_protected', '_public', 'tags', 'qc']
     readonly_fields = ['name_', 'session_ro', '_online', 'auto_datetime', 'revision_',
-                       '_protected', '_public', 'tags']
+                       '_protected', '_public', 'tags', 'qc']
     list_display = ['name_', '_online', 'version', 'collection', 'dataset_type_', 'file_size',
-                    'session_ro', 'created_by', 'created_datetime']
+                    'session_ro', 'created_by', 'created_datetime', 'qc']
     inlines = [FileRecordInline]
     list_filter = [('created_by', RelatedDropdownFilter),
                    ('created_datetime', DateRangeFilter),
                    ('dataset_type', RelatedDropdownFilter),
-                   ('tags', RelatedDropdownFilter)
+                   ('tags', RelatedDropdownFilter),
+                   ('qc', ChoiceDropdownFilter)
                    ]
     search_fields = ('session__id', 'name', 'collection', 'dataset_type__name',
                      'dataset_type__filename_pattern', 'version')

alyx/data/fixtures/data.datasettype.json

@@ -2220,5 +2220,16 @@
       "description": "Look up table from photometry ROI, to fiber name registered in the database and Allen brain location",
       "filename_pattern": "*photometryROI.locations*"
     }
+  },
+  {
+    "model": "data.datasettype",
+    "pk": "140cd2a9-91c1-45ee-9d19-77e8d39abb5f",
+    "fields": {
+      "json": null,
+      "name": "laserStimulation.intervals",
+      "created_by": null,
+      "description": "The start and end times of the laser stimulation period.",
+      "filename_pattern": ""
+    }
   }
 ]

alyx/data/migrations/0019_dataset_qc.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-02-13 15:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0018_alter_dataset_collection_alter_revision_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataset',
+            name='qc',
+            field=models.IntegerField(choices=[(50, 'CRITICAL'), (40, 'FAIL'), (30, 'WARNING'), (0, 'NOT_SET'), (10, 'PASS')], default=0, help_text='50: CRITICAL / 40: FAIL / 30: WARNING / 0: NOT_SET / 10: PASS'),
+        ),
+    ]

alyx/data/models.py

@@ -1,4 +1,5 @@
 import structlog
+from one.alf.spec import QC
 
 from django.core.validators import RegexValidator
 from django.db import models
@@ -351,6 +352,10 @@ class Dataset(BaseExperimentalData):
                                           help_text="Whether this dataset is the default "
                                                     "latest revision")
 
+    QC_CHOICES = [(e.value, e.name) for e in QC]
+    qc = models.IntegerField(default=QC.NOT_SET, choices=QC_CHOICES,
+                             help_text=' / '.join([str(q[0]) + ': ' + q[1] for q in QC_CHOICES]))
+
     @property
     def is_online(self):
         fr = self.file_records.filter(data_repository__globus_is_personal=False)