Introduce a small helper class for tracking observing sessions. (#564)

* Introduce a small helper class for tracking observing sessions.

* Disable totalconvolve unit tests, to be fixed in PR #555

src/toast/data.py

@@ -168,6 +168,7 @@ def split(
         obs_index=False,
         obs_name=False,
         obs_uid=False,
+        obs_session_name=False,
         obs_key=None,
         require_full=False,
     ):
@@ -176,26 +177,33 @@ def split(
         Create new Data objects that have views into unique subsets of the observations
         (the observations are not copied).  Only one "criteria" may be used to perform
         this splitting operation.  The observations may be split by index in the
-        original list, by name, by UID, or by the value of a specified key.
+        original list, by name, by UID, by session, or by the value of a specified key.
 
         The new Data objects are returned in a dictionary whose keys are the value of
         the selection criteria (index, name, uid, or value of the key).  Any observation
-        that cannot be place (because it is missing a name, uid or key) will be ignored
+        that cannot be placed (because it is missing a name, uid or key) will be ignored
         and not added to any of the returned Data objects.  If the `require_full`
         parameter is set to True, such situations will raise an exception.
 
         Args:
             obs_index (bool):  If True, split by index in original list of observations.
             obs_name (bool):  If True, split by observation name.
             obs_uid (bool):  If True, split by observation UID.
+            obs_session_name (bool):  If True, split by session name.
             obs_key (str):  Split by values of this observation key.
 
         Returns:
             (OrderedDict):  The dictionary of new Data objects.
 
         """
         log = Logger.get()
-        check = int(obs_index) + int(obs_name) + int(obs_uid) + int(obs_key is not None)
+        check = (
+            int(obs_index)
+            + int(obs_name)
+            + int(obs_uid)
+            + int(obs_session_name)
+            + int(obs_key is not None)
+        )
         if check == 0 or check > 1:
             raise RuntimeError("You must specify exactly one split criteria")
 
@@ -205,14 +213,14 @@ def split(
         group_comm = self.comm.comm_group
 
         if obs_index:
-            # Splitting by index
+            # Splitting by (unique) index
             for iob, ob in enumerate(self.obs):
                 newdat = Data(comm=self._comm, view=True)
                 newdat._internal = self._internal
                 newdat.obs.append(ob)
                 datasplit[iob] = newdat
         elif obs_name:
-            # Splitting by name
+            # Splitting by (unique) name
             for iob, ob in enumerate(self.obs):
                 if ob.name is None:
                     if require_full:
@@ -237,6 +245,21 @@ def split(
                     newdat._internal = self._internal
                     newdat.obs.append(ob)
                     datasplit[ob.uid] = newdat
+        elif obs_session_name:
+            # Splitting by (non-unique) session name
+            for iob, ob in enumerate(self.obs):
+                if ob.session is None:
+                    if require_full:
+                        msg = f"require_full is True, but observation {iob} has no session"
+                        log.error_rank(msg, comm=group_comm)
+                        raise RuntimeError(msg)
+                else:
+                    sname = ob.session.name
+                    if sname not in datasplit:
+                        newdat = Data(comm=self._comm, view=True)
+                        newdat._internal = self._internal
+                        datasplit[sname] = newdat
+                    datasplit[sname].obs.append(ob)
         elif obs_key is not None:
             # Splitting by arbitrary key.  Unlike name / uid which are built it to the
             # observation class, arbitrary keys might be modified in different ways
@@ -268,7 +291,13 @@ def split(
         return datasplit
 
     def select(
-        self, obs_index=None, obs_name=None, obs_uid=None, obs_key=None, obs_val=None
+        self,
+        obs_index=None,
+        obs_name=None,
+        obs_uid=None,
+        obs_session_name=None,
+        obs_key=None,
+        obs_val=None,
     ):
         """Create a new Data object with a subset of observations.
 
@@ -280,6 +309,7 @@ def select(
             * Index location in the original list of observations
             * Name of the observation
             * UID of the observation
+            * Session of the observation
             * Existence of the specified dictionary key
             * Required value of the specified dictionary key
 
@@ -288,6 +318,7 @@ def select(
             obs_name (str):  The observation name or a compiled regular expression
                 object to use for matching.
             obs_uid (int):  The observation UID to select.
+            obs_session_name (str):  The name of the session.
             obs_key (str):  The observation dictionary key to examine.
             obs_val (str):  The required value of the observation dictionary key or a
                 compiled regular expression object to use for matching.
@@ -312,14 +343,25 @@ def select(
         for iob, ob in enumerate(self.obs):
             if obs_index is not None and obs_index == iob:
                 new_data.obs.append(ob)
+                continue
             if obs_name is not None and ob.name is not None:
                 if isinstance(obs_name, re.Pattern):
                     if obs_name.match(ob.name) is not None:
                         new_data.obs.append(ob)
+                        continue
                 elif obs_name == ob.name:
                     new_data.obs.append(ob)
+                    continue
             if obs_uid is not None and ob.uid is not None and obs_uid == ob.uid:
                 new_data.obs.append(ob)
+                continue
+            if (
+                obs_session_name is not None
+                and ob.session is not None
+                and obs_session_name == ob.session.name
+            ):
+                new_data.obs.append(ob)
+                continue
             if obs_key is not None and obs_key in ob:
                 # Get the values from all processes in the group and check
                 # for consistency.
@@ -336,12 +378,15 @@ def select(
                 if obs_val is None:
                     # We have the key, and are accepting any value
                     new_data.obs.append(ob)
+                    continue
                 elif isinstance(obs_val, re.Pattern):
                     if obs_val.match(ob[obs_key]) is not None:
                         # Matches our regex
                         new_data.obs.append(ob)
+                        continue
                 elif obs_val == ob[obs_key]:
                     new_data.obs.append(ob)
+                    continue
         return new_data
 
     # Accelerator use

src/toast/instrument.py

@@ -2,6 +2,7 @@
 # All rights reserved.  Use of this source code is governed by
 # a BSD-style license that can be found in the LICENSE file.
 
+import datetime
 import os
 import sys
 
@@ -767,6 +768,57 @@ def save_hdf5(self, handle, comm=None, **kwargs):
         table_write_parallel_hdf5(self.detector_data, handle, "focalplane", comm=comm)
 
 
+class Session(object):
+    """Class representing an observing session.
+
+    A session consists of multiple Observation instances with different sets of
+    detectors and possibly different sample rates / times.  However these
+    observations are on the same physical telescope and over the same broad
+    time range.  A session simply tracks that time range and a unique ID which
+    can be used to group the relevant observations.
+
+    Args:
+        name (str):  The name of the session.
+        uid (int):  The Unique ID of the session.  If not specified, it will be
+            constructed from a hash of the name.
+        start (datetime):  The overall start of the session.
+        end (datetime):  The overall end of the session.
+
+    """
+    def __init__(self, name, uid=None, start=None, end=None):
+        self.name = name
+        self.uid = uid
+        if self.uid is None:
+            self.uid = name_UID(name)
+        self.start = start
+        if start is not None and not isinstance(start, datetime.datetime):
+            raise RuntimeError("Session start must be a datetime or None")
+        self.end = end
+        if end is not None and not isinstance(end, datetime.datetime):
+            raise RuntimeError("Session end must be a datetime or None")
+
+    def __repr__(self):
+        value = "<Session '{}': uid = {}, start = {}, end = {}".format(
+            self.name, self.uid, self.start, self.end
+        )
+        value += ">"
+        return value
+
+    def __eq__(self, other):
+        if self.name != other.name:
+            return False
+        if self.uid != other.uid:
+            return False
+        if self.start != other.start:
+            return False
+        if self.end != other.end:
+            return False
+        return True
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
 class Telescope(object):
     """Class representing telescope properties for one observation.
 
@@ -779,7 +831,7 @@ class Telescope(object):
 
     """
 
-    def __init__(self, name, uid=None, focalplane=None, site=None):
+    def __init__(self, name, uid=None, focalplane=None, site=None, session=None):
         self.name = name
         self.uid = uid
         if self.uid is None:
@@ -792,10 +844,12 @@ def __init__(self, name, uid=None, focalplane=None, site=None):
         self.site = site
 
     def __repr__(self):
-        value = "<Telescope '{}': uid = {}, site = {}, focalplane = ".format(
-            self.name, self.uid, self.site
+        value = "<Telescope '{}': uid = {}, site = {}, ".format(
+            self.name, self.uid, self.site,
+        )
+        value += "focalplane = {}".format(
+            self.focalplane.__repr__()
         )
-        value += self.focalplane.__repr__()
         value += ">"
         return value
 

src/toast/io/observation_hdf_load.py

@@ -373,12 +373,12 @@ def load_hdf5(
 
         inst_group = hgroup["instrument"]
         telescope_name = str(inst_group.attrs["telescope_name"])
-        telescope_class_name = str(inst_group.attrs["telescope_class"])
         telescope_uid = int(inst_group.attrs["telescope_uid"])
+        telescope_class = import_from_name(str(inst_group.attrs["telescope_class"]))
 
         site_name = str(inst_group.attrs["site_name"])
-        site_class_name = str(inst_group.attrs["site_class"])
         site_uid = int(inst_group.attrs["site_uid"])
+        site_class = import_from_name(str(inst_group.attrs["site_class"]))
 
         site = None
         if "site_alt_m" in inst_group.attrs:
@@ -405,7 +405,7 @@ def load_hdf5(
                     max_pwv=weather_max_pwv,
                     median_weather=False,
                 )
-            site = GroundSite(
+            site = site_class(
                 site_name,
                 site_lat_deg * u.degree,
                 site_lon_deg * u.degree,
@@ -414,12 +414,37 @@ def load_hdf5(
                 weather=weather,
             )
         else:
-            site = SpaceSite(site_name, uid=site_uid)
+            site = site_class(site_name, uid=site_uid)
+
+        session = None
+        if "session_name" in inst_group.attrs:
+            session_name = str(inst_group.attrs["session_name"])
+            session_uid = int(inst_group.attrs["session_uid"])
+            session_start = inst_group.attrs["session_start"]
+            if str(session_start) == "NONE":
+                session_start = None
+            else:
+                session_start = datetime.datetime.fromtimestamp(
+                    float(inst_group.attrs["session_start"]),
+                    tz=datetime.timezone.utc,
+                )
+            session_end = inst_group.attrs["session_end"]
+            if str(session_end) == "NONE":
+                session_end = None
+            else:
+                session_end = datetime.datetime.fromtimestamp(
+                    float(inst_group.attrs["session_end"]),
+                    tz=datetime.timezone.utc,
+                )
+            session_class = import_from_name(str(inst_group.attrs["session_class"]))
+            session = session_class(
+                session_name, uid=session_uid, start=session_start, end=session_end
+            )
 
         focalplane = Focalplane()
         focalplane.load_hdf5(inst_group, comm=None)
 
-        telescope = Telescope(
+        telescope = telescope_class(
             telescope_name, uid=telescope_uid, focalplane=focalplane, site=site
         )
         del inst_group
@@ -439,6 +464,7 @@ def load_hdf5(
             obs_samples,
             name=obs_name,
             uid=obs_uid,
+            session=session,
             detector_sets=obs_det_sets,
             sample_sets=obs_sample_sets,
             process_rows=process_rows,

src/toast/io/observation_hdf_save.py

@@ -501,6 +501,19 @@ def save_hdf5(
                     inst_group.attrs[
                         "site_weather_time"
                     ] = site.weather.time.timestamp()
+        session = obs.session
+        if session is not None:
+            inst_group.attrs["session_name"] = session.name
+            inst_group.attrs["session_class"] = object_fullname(session.__class__)
+            inst_group.attrs["session_uid"] = session.uid
+            if session.start is None:
+                inst_group.attrs["session_start"] = "NONE"
+            else:
+                inst_group.attrs["session_start"] = session.start.timestamp()
+            if session.end is None:
+                inst_group.attrs["session_end"] = "NONE"
+            else:
+                inst_group.attrs["session_end"] = session.end.timestamp()
     log.verbose_rank(
         f"{log_prefix}  Wrote instrument attributes in",
         comm=comm,