Merge branch 'main' into sofie_dev

.gitignore

@@ -1,8 +1,10 @@
 # folders
-build/
+build/*
 build*/
+!build/.gitkeep
 log/
 logs/
+generation_logs/
 # ignore all analysis in the config folder apart from the unittest and the template
 analysis_configurations/*
 !analysis_configurations/unittest

build/.gitkeep

@@ -0,0 +1 @@
+build

cmake/ConfigureCrownlib.cmake

@@ -1,11 +1,18 @@
 # build a shared lib from all CROWN functions
 include_directories(${CMAKE_SOURCE_DIR}/src)
 include_directories(${CMAKE_SOURCE_DIR}/include)
+
+include_directories(${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/cpp_addons/src)
+include_directories(${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/cpp_addons/include)
+
 file(GLOB SOURCES_1 ${CMAKE_SOURCE_DIR}/src/*.cxx)
 file(GLOB SOURCES_2 ${CMAKE_SOURCE_DIR}/src/utility/*.cxx
      ${CMAKE_SOURCE_DIR}/src/RecoilCorrections/*.cxx
      ${CMAKE_SOURCE_DIR}/src/SVFit/*.cxx)
-set(SOURCES ${SOURCES_1} ${SOURCES_2})
+
+file(GLOB SOURCES_3 ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/cpp_addons/src/*.cxx)
+
+set(SOURCES ${SOURCES_1} ${SOURCES_2} ${SOURCES_3})
 
 if(BUILD_CROWNLIB_ONLY)
   message(STATUS "Building only the CROWNLIB library")

code_generation/analysis_template.cxx

@@ -33,6 +33,8 @@
 #include "include/triggers.hxx"
 #include "include/fakefactors.hxx"
 
+// {INCLUDE_ANALYSISADDONS}
+
 // {INCLUDES}
 
 int main(int argc, char *argv[]) {

code_generation/analysis_template_friends.cxx

@@ -33,6 +33,9 @@
 #include "include/topreco.hxx"
 #include "include/triggers.hxx"
 #include "include/tripleselection.hxx"
+
+// {INCLUDE_ANALYSISADDONS}
+
 // {INCLUDES}
 
 int validate_rootfile(std::string file, std::string &basetree) {
@@ -60,9 +63,16 @@ int validate_rootfile(std::string file, std::string &basetree) {
         Logger::get("main")->info("CROWN input_file: {} - {} Events", file,
                                   t1->GetEntries());
         return nevents;
+    } else if (list->FindObject("quantities")) {
+        TTree *t1 = (TTree *)f1->Get("quantities");
+        nevents += t1->GetEntries();
+        basetree = "ntuple";
+        Logger::get("main")->critical("CROWN input_file: {} - {} Events", file,
+                                  t1->GetEntries());
+        return nevents;
     } else {
         Logger::get("main")->critical("File {} does not contain a tree "
-                                      "named 'Events' or 'ntuple'",
+                                      "named 'Events' or 'ntuple' or 'quantities'",
                                       file);
         return -1;
     }
@@ -160,11 +170,13 @@ int main(int argc, char *argv[]) {
     }
     // initialize df
     ROOT::RDataFrame df0(dataset);
-    ROOT::RDF::Experimental::AddProgressBar(df0); // add progress bar
     // print all available branches to the log
-    Logger::get("main")->debug("Available branches:");
-    for (auto const &branch : df0.GetColumnNames()) {
-        Logger::get("main")->debug("{}", branch);
+    if (nevents != 0) {
+        ROOT::RDF::Experimental::AddProgressBar(df0); // add progress bar
+        Logger::get("main")->debug("Available branches:");
+        for (auto const &branch : df0.GetColumnNames()) {
+            Logger::get("main")->debug("{}", branch);
+        }
     }
     Logger::get("main")->info(
         "Starting Setup of Dataframe with {} events and {} friends", nevents,

code_generation/code_generation.py

@@ -12,6 +12,32 @@
 log = logging.getLogger(__name__)
 
 
+def addon_includes(analysis_name: str, file_name: str) -> str:
+    """
+    Add the includes all .hxx files from analysis configuration folder:
+    analysis_configurations/{analysis_name}/cpp_addons/include
+    Args:
+        analysis_name: the name of the analysis
+        file_name: Name of file that is templated
+    Returns:
+        str - the include statements for the cpp addons
+    """
+    path = f"analysis_configurations/{analysis_name}/cpp_addons/include"
+    if os.path.exists(path) and os.path.isdir(path) and os.listdir(path):
+        log.debug(
+            f"Adding addons from {path} to {file_name}: {' '.join(os.listdir(path))}"
+        )
+        paths = "\n".join(
+            f'#include "{os.path.abspath(os.path.join(path, item))}"'
+            for item in os.listdir(path)
+            if item.endswith(".hxx")
+        )
+        return paths
+    else:
+        log.debug(f"No addons found in {path}")
+        return ""
+
+
 class CodeSubset(object):
     """
     Class used to generate code for a smaller subset. For each subset, a new object must be created.
@@ -24,6 +50,7 @@ class CodeSubset(object):
         folder: The folder in which the code will be generated.
         parameters: The parameters to be used for the generation.
         name: The name of the code subset.
+        analysis_name: Name of the analysis configuration.
 
     Returns:
         None
@@ -38,6 +65,7 @@ def __init__(
         folder: str,
         configuration_parameters: Dict[str, Any],
         name: str,
+        analysis_name: str,
     ):
         self.file_name = file_name
         self.template = template
@@ -48,6 +76,7 @@ def __init__(
         self.count = 0
         self.folder = folder
         self.commands: List[str] = []
+        self.analysis_name = analysis_name
         self.headerfile = os.path.join(
             self.folder, "include", self.scope, "{}.hxx".format(self.file_name)
         )
@@ -120,8 +149,11 @@ def write(self):
         with open(self.sourcefile + ".new", "w") as f:
             commandstring = "".join(self.commands)
             f.write(
-                self.template.replace("//    { commands }", commandstring).replace(
-                    "{subsetname}", self.name
+                self.template.replace("//    { commands }", commandstring)
+                .replace("{subsetname}", self.name)
+                .replace(
+                    "// {INCLUDE_ANALYSISADDONS}",
+                    addon_includes(self.analysis_name, self.file_name),
                 )
             )
         if os.path.isfile(self.sourcefile):
@@ -350,6 +382,10 @@ def write_code(self, calls: str, includes: str, run_commands: str) -> None:
                     "        // {ZERO_EVENTS_FALLBACK}", self.zero_events_fallback()
                 )
                 .replace("        // {CODE_GENERATION}", calls)
+                .replace(
+                    "// {INCLUDE_ANALYSISADDONS}",
+                    addon_includes(self.analysis_name, self.executable_name + ".cxx"),
+                )
                 .replace("// {INCLUDES}", includes)
                 .replace("        // {RUN_COMMANDS}", run_commands)
                 .replace("// {MULTITHREADING}", threadcall)
@@ -458,6 +494,7 @@ def generate_subsets(self, scope: str) -> None:
                 ),
                 configuration_parameters=self.configuration.config_parameters[scope],
                 name=producer_name + "_" + scope,
+                analysis_name=self.analysis_name,
             )
             subset.create()
             subset.write()

code_generation/configuration.py

@@ -28,6 +28,7 @@
 )
 from code_generation.rules import ProducerRule, RemoveProducer
 from code_generation.systematics import SystematicShift, SystematicShiftByQuantity
+from code_generation.helpers import is_empty
 
 log = logging.getLogger(__name__)
 # type aliases
@@ -257,12 +258,12 @@ def unpack_producergroups(
         """
 
         if isinstance(producers, list):
-            # we always want to know the toplevel producergroup, so if the parent is None, we set it to the first producer.
+            # we always want to know the toplevel producergroup, so if the parent evaluates to false, we set it to the first producer.
             # If a prent is given, we set it to the parent, since this means we are in a producergroup. This is important if we
             # have nested producergroups, this way every producer is assigned to the outermost producergroup, which is important for the
             # potential removal of a single producer.
             for producer in producers:
-                if parent is None:
+                if is_empty(parent):
                     parent_producer = producer
                 else:
                     parent_producer = parent
@@ -276,7 +277,7 @@ def unpack_producergroups(
             if isinstance(producers, ProducerGroup):
                 log.debug("{} Unpacking ".format("    " * depth))
                 for sub_producer in producers.producers[scope]:
-                    if parent is None:
+                    if is_empty(parent):
                         parent_producer = producers
                     else:
                         parent_producer = parent
@@ -287,7 +288,7 @@ def unpack_producergroups(
                             depth=depth + 1,
                         )
             else:
-                if parent is None:
+                if is_empty(parent):
                     log.debug("{} {}".format("    " * depth, producers))
                     self.unpacked_producers[scope][producers] = producers
                 else:
@@ -333,19 +334,19 @@ def add_shift(
         Returns:
             None
         """
-        if exclude_samples is not None and samples is not None:
+        if not is_empty(exclude_samples) and not is_empty(samples):
             raise ConfigurationError(
                 f"You cannot use samples and exclude_samples at the same time -> Shift {shift}, samples {samples}, exclude_samples {exclude_samples}"
             )
-        if samples is not None:
+        if not is_empty(samples):
             if isinstance(samples, str):
                 samples = [samples]
             for sample in samples:
                 if sample not in self.available_sample_types:
                     raise ConfigurationError(
                         f"Sampletype {sample} is not available -> Shift {shift}, available_sample_types {self.available_sample_types}, sample_types {samples}"
                     )
-        if exclude_samples is not None:
+        if not is_empty(exclude_samples):
             if isinstance(exclude_samples, str):
                 exclude_samples = [exclude_samples]
             for excluded_sample in exclude_samples:
@@ -360,7 +361,7 @@ def add_shift(
                 raise TypeError("shift must be of type SystematicShift")
             if isinstance(samples, str):
                 samples = [samples]
-            if samples is None or self.sample in samples:
+            if is_empty(samples) or self.sample in samples:
                 scopes_to_shift = [
                     scope for scope in shift.get_scopes() if scope in self.scopes
                 ]
@@ -513,9 +514,9 @@ def _remove_empty_scopes(self) -> None:
         # we have to use a seperate list, because we cannot modify the list while iterating over it without breaking stuff
         scopes_to_test = [scope for scope in self.scopes]
         for scope in scopes_to_test:
-            if (len(self.producers[scope]) == 0) or (
-                scope not in self.selected_scopes and scope is not self.global_scope
-            ):
+            if (
+                len(self.producers[scope]) == 0 or scope not in self.selected_scopes
+            ) and scope is not self.global_scope:
                 log.warning("Removing unrequested / empty scope {}".format(scope))
                 self.scopes.remove(scope)
                 del self.producers[scope]
@@ -631,12 +632,7 @@ def _remove_empty_configkeys(self, config) -> None:
                     if isinstance(value, dict):
                         self._remove_empty_configkeys(value)
 
-            elif (
-                config[key] is None
-                or config[key] == ""
-                or config[key] == []
-                or config[key] == {}
-            ):
+            elif is_empty(config[key]):
                 log.info(
                     "Removing {} since it is an empty configuration parameter".format(
                         key
@@ -767,9 +763,11 @@ def report(self) -> None:
         total_quantities = [
             sum(
                 [
-                    len(self.config_parameters[scope][output.vec_config])
-                    if isinstance(output, QuantityGroup)
-                    else 1
+                    (
+                        len(self.config_parameters[scope][output.vec_config])
+                        if isinstance(output, QuantityGroup)
+                        else 1
+                    )
                     for output in self.outputs[scope]
                 ]
             )

code_generation/exceptions.py

@@ -1,6 +1,7 @@
 from __future__ import annotations  # needed for type annotations in > python 3.7
 from typing import List, Set, Union
 from code_generation.quantity import Quantity
+from code_generation.helpers import is_empty
 
 
 class ConfigurationError(Exception):
@@ -108,7 +109,7 @@ class InvalidShiftError(ConfigurationError):
     """
 
     def __init__(self, shift: str, sample: str, scope: Union[str, None] = None):
-        if scope is None:
+        if is_empty(scope):
             self.message = "Shift {} is not setup properly or not available for sampletype {}".format(
                 shift, sample
             )

code_generation/helpers.py

@@ -0,0 +1,24 @@
+from __future__ import annotations  # needed for type annotations in > python 3.7
+
+# File with helper functions for the CROWN code generation
+
+
+def is_empty(value):
+    """
+    Check if a value is empty.
+
+    Args:
+        value: The value that should be checked.
+
+    Returns:
+        bool: Whether the input value is considered 'empty'
+    """
+    # List of all values that should be considered empty despite not having a length.
+    empty_values = [None]
+
+    try:
+        length = len(value)
+    except TypeError:
+        length = -1
+    bool_val = value in empty_values or length == 0
+    return bool_val

code_generation/modifiers.py

@@ -4,6 +4,7 @@
     SampleConfigurationError,
     EraConfigurationError,
 )
+from code_generation.helpers import is_empty
 
 ConfigurationParameters = Union[str, int, float, bool]
 
@@ -71,7 +72,7 @@ def apply(self, sample: str) -> ModifierResolved:
         """
         if sample in self.samples:
             return self.modifier_dict[sample]
-        elif self.default is not None:
+        elif not is_empty(self.default):
             return self.default
         else:
             raise SampleConfigurationError(sample, self.samples)
@@ -106,7 +107,7 @@ def apply(self, era: str) -> ModifierResolved:
         """
         if era in self.eras:
             return self.modifier_dict[era]
-        elif self.default is not None:
+        elif not is_empty(self.default):
             return self.default
         else:
             raise EraConfigurationError(era, self.eras)

code_generation/optimizer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations  # needed for type annotations in > python 3.7
 from code_generation.quantity import NanoAODQuantity, Quantity
 from code_generation.producer import Filter, BaseFilter, Producer, ProducerGroup
+from code_generation.helpers import is_empty
 from typing import Set, Tuple, Union, List
 import logging
 
@@ -79,7 +80,7 @@ def get_global_outputs(self) -> List[Quantity]:
         """
         outputs: List[Quantity] = []
         for producer in self.global_producers:
-            if producer.get_outputs("global") is not None:
+            if not is_empty(producer.get_outputs("global")):
                 outputs.extend(
                     [
                         quantity
@@ -155,7 +156,7 @@ def Optimize(self) -> None:
                 log.error("Please check, if all needed producers are activated")
                 raise Exception
             wrongProducer, wrong_inputs = self.check_ordering()
-            if wrongProducer is not None:
+            if not is_empty(wrongProducer):
                 producers_to_relocate = self.find_inputs(wrongProducer, wrong_inputs)
                 # if len(producers_to_relocate) == 0:
                 #     self.optimized = True
@@ -197,7 +198,7 @@ def check_ordering(
             outputs = self.global_outputs
         for producer_to_check in self.ordering:
             temp_outputs = producer_to_check.get_outputs(self.scope)
-            if temp_outputs is not None:
+            if not is_empty(temp_outputs):
                 outputs.extend(
                     [
                         quantity