Merge branch '#109' [Closes #109]

CMSTrackerDPG · Mar 23, 2022 · f1132a5 · f1132a5
2 parents 9648b27 + 34993e1
commit f1132a5
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 40 deletions.
diff --git a/shiftleader/exceptions.py b/shiftleader/exceptions.py
@@ -0,0 +1,5 @@
+class CannotAssumeRunTypeException(Exception):
+    """
+    Custom exception to raise when trying to deduct the Run type
+    using the Run Registry class and dataset name
+    """
diff --git a/shiftleader/query.py b/shiftleader/query.py
@@ -17,7 +17,8 @@
 )
 from django.db.models.functions import ExtractWeekDay
 from delete.query import SoftDeletionQuerySet
-from shiftleader.utilities.utilities import convert_run_registry_to_trackercertification
+from shiftleader.utilities.utilities import convert_run_registry_to_trackercertification, _get_run_type_from_run_class_and_dataset_name
+from shiftleader.exceptions import CannotAssumeRunTypeException
 import runregistry
 from operator import itemgetter
 from itertools import groupby
@@ -509,14 +510,11 @@ def compare_with_run_registry(self):
         # to get the run type. If not, do an extra query to get extra oms attributes
         # Issue #100
         for rr_entry in run_registry_entries:
-            if not ('collision' in rr_entry['class'].lower()
-                    or 'collision' in rr_entry['name'].lower()
-                    or 'cosmic' in rr_entry['class'].lower()
-                    or 'cosmic' in rr_entry['name'].lower()):
-                logger.info(
-                    f"Cannot safely assume run_type from class {repr(rr_entry['class'])}"
-                    f" and name {repr(rr_entry['name'])}, getting OMS attributes"
-                )
+            try:
+                _get_run_type_from_run_class_and_dataset_name(
+                    rr_entry['class'], rr_entry['name'])
+            except CannotAssumeRunTypeException as e:
+                logger.info(e)
                 # Run extra query for run info, get oms_attributes and insert
                 # them to the entry
                 rr_entry['oms_attributes'] = runregistry.get_run(

diff --git a/shiftleader/templates/shiftleader/checklist.html b/shiftleader/templates/shiftleader/checklist.html
@@ -26,8 +26,7 @@ <h3>{{ itemgroup.name }}</h3>
     {% else %}
         <div class="alert alert-danger" role="alert">
             No shift leader checklist found. <br/>
-            Please <a href="/admin/certifier/checklist/add/?identifier=shiftleader">create</a> one in the Admin Settings.
-            Make sure to set the Checklist "Identifier" to "shiftleader".
+            Please <a href="{% url 'admin:checklists_checklist_add' %}?identifier=shiftleader">create</a> one in the Admin Settings. Make sure to set the Checklist "Identifier" to "shiftleader".
         </div>
     {% endif %}
 </div>

diff --git a/shiftleader/tests/test_shiftleader_query.py b/shiftleader/tests/test_shiftleader_query.py
@@ -1,8 +1,7 @@
+import os
 import math
 import pytest
-import os
 from mixer.backend.django import mixer
-
 from oms.models import OmsRun
 from certifier.models import TrackerCertification, RunReconstruction
 from shiftleader.utilities.utilities import to_date, to_weekdayname

diff --git a/shiftleader/tests/test_shiftleader_utilities.py b/shiftleader/tests/test_shiftleader_utilities.py
@@ -1,8 +1,10 @@
-from decimal import Decimal
 import datetime
 import pytest
+from decimal import Decimal
 from mixer.backend.django import mixer
-from shiftleader.utilities.utilities import convert_run_registry_to_trackercertification, to_date, chunks
+from oms.models import OmsRun
+from shiftleader.utilities.utilities import convert_run_registry_to_trackercertification, to_date, chunks, _get_run_type_from_run_class_and_dataset_name
+from shiftleader.exceptions import CannotAssumeRunTypeException
 from unittest.mock import MagicMock
 pytestmark = pytest.mark.django_db
 
@@ -30,3 +32,22 @@ def test_convert_run_registry_to_trackercertification_no_type(self):
         input = [{'class': 'Commissioning22', 'state': 'SIGNOFF', 'significant': True, 'stop_reason': 'CSC goes to local', 'run_number': 348409, 'name': '/Express/Commissioning2022/DQM\n', 'dataset_attributes': {'dc_state': 'waiting dqm gui', 'dt_state': 'waiting dqm gui', 'cms_state': 'waiting dqm gui', 'csc_state': 'waiting dqm gui', 'gem_state': 'waiting dqm gui', 'hlt_state': 'waiting dqm gui', 'l1t_state': 'waiting dqm gui', 'rpc_state': 'waiting dqm gui', 'tau_state': 'waiting dqm gui', 'btag_state': 'waiting dqm gui', 'ecal_state': 'waiting dqm gui', 'hcal_state': 'waiting dqm gui', 'lumi_state': 'waiting dqm gui', 'muon_state': 'waiting dqm gui', 'appeared_in': [], 'ctpps_state': 'waiting dqm gui', 'castor_state': 'waiting dqm gui', 'egamma_state': 'waiting dqm gui', 'global_state': 'waiting dqm gui', 'jetmet_state': 'waiting dqm gui', 'tracker_state': 'waiting dqm gui'}, 'datasets_in_gui': [], 'deleted': False, 'version': 45653780, 'lumisections': {'dt-dt': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'csc-tf': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'rpc-hv': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'rpc-lv': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'cms-cms': {'BAD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'csc-csc': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'ecal-es': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'gem-gem': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'hcal-hb': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-he': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-hf': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-hlt': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'hlt-tau': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-l1t': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'rpc-feb': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'rpc-rpc': {'GOOD': 626, 'EMPTY': 0, 'causes': [], 'STANDBY': 2, 'comments': []}, 'tau-tau': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'ecal-ebm': {'EMPTY': 0, 'causes': [], 'STANDBY': 628, 'comments': []}, 'ecal-ebp': {'EMPTY': 0, 'causes': [], 'STANDBY': 628, 'comments': []}, 'ecal-eem': {'EMPTY': 0, 'causes': [], 'STANDBY': 628, 'comments': []}, 'ecal-eep': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'ecal-esm': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'ecal-esp': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'ecal-tpg': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-ho0': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'rpc-elog': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'btag-btag': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'ecal-ecal': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'hcal-hcal': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'hcal-ho12': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-bjets': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-muons': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-l1tmu': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'lumi-lumi': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'muon-muon': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'rpc-noise': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-strips': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-timing': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'dc-lowlumi': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'ecal-laser': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'ecal-noise': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-hb_ls': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-he_ls': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-hf_ls': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-jetmet': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'ctpps-ctpps': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ecal-timing': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hcal-hoO_ls': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-photons': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-e_gamma': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-hf_tech': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-l1tcalo': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'l1t-muon_dt': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'csc-gasgains': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-segments': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'hcal-ho12_ls': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-bcs_tech': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-hg_rings': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-muon_csc': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-muon_rpc': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-rpc_tech': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-software': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'castor-castor': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'csc-integrity': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-occupancy': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-pedestals': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'ecal-analysis': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'egamma-egamma': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'hlt-electrons': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-technical': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'jetmet-jetmet': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'l1t-bptx_tech': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'tracker-pixel': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'tracker-strip': {'GOOD': 628, 'EMPTY': 0, 'causes': [], 'comments': []}, 'tracker-track': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-efficiency': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-resolution': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'csc-triggergpe': {'EMPTY': 0, 'NOTSET': 628, 'causes': [], 'comments': []}, 'ctpps-rp45_210': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ctpps-rp45_220': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ctpps-rp45_cyl': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ctpps-rp56_210': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ctpps-rp56_220': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ctpps-rp56_cyl': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ecal-preshower': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'hlt-hlt_global': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'ecal-collisions': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'l1t-energy_sums': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}, 'ctpps-time45_box': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'ctpps-time56_box': {'EMPTY': 0, 'causes': [], 'EXCLUDED': 628, 'comments': []}, 'cms-infrastructure': {'EMPTY': 0, 'causes': [], 'comments': [], 'NO VALUE FOUND': 628}}}]
         ret = convert_run_registry_to_trackercertification(input)
         assert 'runreconstruction__run__run_type' not in ret[0]
+
+
+    def test_run_type_assumption(self):
+        """
+        Test the function that derives run_type from run class and
+        dataset name
+        """
+        run_type = _get_run_type_from_run_class_and_dataset_name(
+            "Cosmics22", "/PromptReco/Cosmics22/DQM")
+        assert run_type == OmsRun.COSMICS
+
+        run_type = _get_run_type_from_run_class_and_dataset_name(
+            "Collisions22", "/PromptReco/Collisions22/DQM")
+        assert run_type == OmsRun.COLLISIONS
+
+        # Expected to fail
+        with pytest.raises(CannotAssumeRunTypeException):
+            run_type = _get_run_type_from_run_class_and_dataset_name(
+                "Commissioning22", "/Express/Commissioning2022/DQM")
diff --git a/shiftleader/utilities/utilities.py b/shiftleader/utilities/utilities.py
@@ -1,9 +1,8 @@
-from django.utils import timezone
 import datetime
 import logging
-# from certifier.models import RunReconstruction, TrackerCertification # circular import
-
+from django.utils import timezone
 from oms.models import OmsRun
+from shiftleader.exceptions import CannotAssumeRunTypeException
 
 logger = logging.getLogger(__name__)
 
@@ -46,7 +45,29 @@ def get_certification_status(detector_type):
         return "Excluded"
 
 
-def convert_run_registry_to_trackercertification(list_of_dictionaries):
+def _get_run_type_from_run_class_and_dataset_name(run_class: str = "",
+                                                  dataset_name: str = ""
+                                                  ) -> str:
+    """
+    Helper function that, given the "class" and "name" from the data returned
+    from a RunRegistry dataset, returns the "run_type" for the specific dataset
+
+    Raises an Exception if the type cannot be assumed
+    """
+    run_class = run_class.lower()
+    dataset_name = dataset_name.lower()
+
+    if "collision" in run_class or "collision" in dataset_name:
+        return OmsRun.COLLISIONS
+    elif "cosmic" in run_class or "cosmic" in dataset_name:
+        return OmsRun.COSMICS
+
+    raise CannotAssumeRunTypeException(
+        f"Cannot safely assume run_type from class {repr(run_class)}"
+        f" and name {repr(dataset_name)}")
+
+
+def convert_run_registry_to_trackercertification(list_of_dictionaries: list):
     """
     Converts the list of JSON dictionaries into a TrackerCertification 
     compatible format, i.e.:
@@ -60,27 +81,34 @@ def convert_run_registry_to_trackercertification(list_of_dictionaries):
         dataset = entry["dataset"].lower()
         entry["runreconstruction__run__run_number"] = entry.pop("run_number")
 
-        if "collision" in run_class:
-            entry["runreconstruction__run__run_type"] = OmsRun.COLLISIONS
-        elif "cosmic" in run_class:
-            entry["runreconstruction__run__run_type"] = OmsRun.COSMICS
-        elif "collision" in dataset:  # When run_class is e.g. Commissioning18
-            entry["runreconstruction__run__run_type"] = OmsRun.COLLISIONS
-        elif "cosmic" in dataset:
-            entry["runreconstruction__run__run_type"] = OmsRun.COSMICS
-        # Edge case where class is something like "Commissioning22" and
-        # name does not contain "cosmic" nor "collision" (e.g. /Express/Commissioning2022/DQM)
-        elif "oms_attributes" in entry:
-            # Use OMS attributes
-            if "collision" in entry["oms_attributes"]["hlt_key"]:
-                entry["runreconstruction__run__run_type"] = OmsRun.COLLISIONS
-            elif "cosmic" in entry["oms_attributes"]["hlt_key"]:
-                entry["runreconstruction__run__run_type"] = OmsRun.COSMICS
-        else:
-            logger.warning(
-                f"Run {entry['runreconstruction__run__run_number']} (Class:{run_class},"
-                f" Dataset:{dataset}) does not contain enough info"
-                " to assume its run type")
+        # if "collision" in run_class:
+        #     entry["runreconstruction__run__run_type"] = OmsRun.COLLISIONS
+        # elif "cosmic" in run_class:
+        #     entry["runreconstruction__run__run_type"] = OmsRun.COSMICS
+        # elif "collision" in dataset:  # When run_class is e.g. Commissioning18
+        #     entry["runreconstruction__run__run_type"] = OmsRun.COLLISIONS
+        # elif "cosmic" in dataset:
+        #     entry["runreconstruction__run__run_type"] = OmsRun.COSMICS
+
+        try:
+            entry[
+                "runreconstruction__run__run_type"] = _get_run_type_from_run_class_and_dataset_name(
+                    run_class, dataset)
+        except CannotAssumeRunTypeException:
+            # Edge case where class is something like "Commissioning22" and
+            # name does not contain "cosmic" nor "collision" (e.g. /Express/Commissioning2022/DQM)
+            if "oms_attributes" in entry:
+                # Use OMS attributes
+                try:
+                    entry[
+                        "runreconstruction__run__run_type"] = _get_run_type_from_run_class_and_dataset_name(
+                            entry["oms_attributes"]["hlt_key"])
+                except CannotAssumeRunTypeException:
+                    logger.warning(
+                        f"Run {entry['runreconstruction__run__run_number']} (Class:{run_class},"
+                        f" Dataset:{dataset}) does not contain enough info"
+                        " to assume its run type")
+                    raise
 
         if "express" in dataset:
             entry["runreconstruction__reconstruction"] = "express"  # EXPRESS