maybe fix t-sqr issue

notebooks_tsqr/NightLog.ipynb

@@ -35,7 +35,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "2",
-   "metadata": {},
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "import datetime as dt\n",
@@ -84,7 +88,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "3",
-   "metadata": {},
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "# Normalize Parameters (both explicit Times Squares params, in implicit ones)\n",
@@ -109,7 +117,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "4",
-   "metadata": {},
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "# Set default env to \"usdf\" and try before PUSH to repo.\n",
@@ -161,7 +173,7 @@
     "    f\"\"\"## Project-Wide Night(s) Report \n",
     "- Run on logs and databases from **{server}/**\n",
     "- Report **{days} observing night(s)** with the last reported night starting on **{date}**.\n",
-    "- This report will include available data from noon **{min_date}** to noon **{max_date}** (inclusive).\n",
+    "- This report will include available data from noon **{min_date}** to noon **{max_date}**.\n",
     "- Using ***Prototype* Logging and Reporting** Version: **{lrversion}**\n",
     "\"\"\"\n",
     ")\n",
@@ -295,7 +307,9 @@
     "exposure_rep.overview\n",
     "\n",
     "# Exposure Tally\n",
-    "md(f\"## Exposure Tally for all Instruments DISABLED\")\n",
+    "md(f\"## Exposure Tally for all Instruments\")\n",
+    "tally = await allsrc.night_tally_observation_gaps()\n",
+    "display(pd.DataFrame(tally))\n",
     "\n",
     "# Observation gaps\n",
     "gaps = allsrc.exp_src.get_observation_gaps()\n",

python/lsst/ts/logging_and_reporting/all_sources.py

@@ -40,6 +40,15 @@ def __init__(
         )
         # This space for rent by ConsDB
 
+        # Get the common min/max date/dayobs from just one source.
+        # They are the same for all of them.
+        self.max_date = self.nig_src.max_date
+        self.min_date = self.nig_src.min_date
+        self.max_dayobs = self.nig_src.max_dayobs
+        self.min_dayobs = self.nig_src.min_dayobs
+
+    # END init
+
     # Our goals are something like this (see DM-46102)
     #
     # Ref                                       Hours
@@ -57,12 +66,22 @@ def __init__(
     # day_obs:: YYYMMDD (int or str)
     # Use almanac begin of night values for day_obs.
     # Use almanac end of night values for day_obs + 1.
-    async def night_tally_observation_gaps(self, dayobs):
+    async def night_tally_observation_gaps(self, verbose=True):
+
         instrument_tally = dict()  # d[instrument] = tally_dict
-        almanac = alm.Almanac(dayobs=dayobs)
-        total_observable_hours = almanac.night_hours
+        almanac = alm.Almanac(dayobs=self.min_dayobs)
+        total_observable_hrs = almanac.night_hours
+
+        targets = await self.efd_src.get_targets()  # a DataFrame
+        if verbose:
+            print(
+                f"AllSources().get_targets() got {len(targets)} targets "
+                f"using date range {self.min_date} to {self.max_date}. "
+            )
+
+        if targets.empty:
+            return dict
 
-        targets = await self.efd_src.get_targets()
         num_slews = targets[["slewTime"]].astype(bool).sum(axis=0).squeeze()
         total_slew_seconds = targets[["slewTime"]].sum().squeeze()
 
@@ -73,24 +92,24 @@ async def night_tally_observation_gaps(self, dayobs):
                 end = dt.datetime.fromisoformat(rec["timespan_end"])
                 exposure_seconds += (end - begin).total_seconds()
             num_exposures = len(records)
-            exposure_hours = exposure_seconds / (60 * 60.0)
-            slew_hours = total_slew_seconds / (60 * 60)
-            idle_hours = (
-                total_observable_hours
-                - exposure_hours
-                # - detector_read_hours
-                - slew_hours
+            exposure_hrs = exposure_seconds / (60 * 60.0)
+            slew_hrs = total_slew_seconds / (60 * 60)
+            idle_hrs = (
+                total_observable_hrs
+                - exposure_hrs
+                # - detector_read_hrs
+                - slew_hrs
             )
             instrument_tally[instrument] = {
-                "Total Night hours": hhmmss(total_observable_hours),  # (a)
-                "Total Exposure hours": hhmmss(exposure_hours),  # (b)
+                "Total Night (HH:MM:SS)": hhmmss(total_observable_hrs),  # (a)
+                "Total Exposure (HH:MM:SS)": hhmmss(exposure_hrs),  # (b)
                 "Number of exposures": num_exposures,  # (c)
                 "Number of slews": num_slews,  # (d)
-                "Total Detector Read hours": "NA",  # (e) UNKNOWN SOURCE
-                "Mean Detector Read hours": "NA",  # (f=e/c)
-                "Total Slew hours": hhmmss(slew_hours),  # (g)
-                "Mean Slew hours": hhmmss(slew_hours / num_slews),  # (h=g/d)
-                "Total Idle hours": hhmmss(idle_hours),  # (i=a-b-e-g)
+                "Total Detector Read (HH:MM:SS)": "NA",  # (e) UNKNOWN SOURCE
+                "Mean Detector Read (HH:MM:SS)": "NA",  # (f=e/c)
+                "Total Slew (HH:MM:SS)": hhmmss(slew_hrs),  # (g)
+                "Mean Slew (HH:MM:SS)": hhmmss(slew_hrs / num_slews),  # (g/d)
+                "Total Idle (HH:MM:SS)": hhmmss(idle_hrs),  # (i=a-b-e-g)
             }
 
         # get_detector_reads()??  # UNKNOWN SOURCE

python/lsst/ts/logging_and_reporting/efd.py

@@ -11,8 +11,8 @@
 """
 
 import asyncio
+import datetime as dt
 import os
-from datetime import datetime, time
 
 import lsst.ts.logging_and_reporting.utils as ut
 from astropy.time import Time, TimeDelta
@@ -75,7 +75,7 @@ async def get_topics(self):
     #      Possibly due to underlying SQL not matching schema.
     async def find_populated_topics(self, days=1, max_topics=None):
         topic_count = 0
-        end = Time(datetime.combine(self.max_date, time()))
+        end = Time(dt.datetime.combine(self.max_date, dt.time()))
         start = end - TimeDelta(days, format="jd")
         errors = dict()  # errors[topic] = error_message
         populated = dict()  # populated[topic] = [field, ...]
@@ -117,9 +117,9 @@ async def find_populated_topics(self, days=1, max_topics=None):
         print(f" DONE in {ut.toc()/60} minutes")
         return populated, errors, topic_count
 
-    async def query_nights(self, topic, fields, days=1, index=301):
-        end = Time(datetime.combine(self.max_date, time()))
-        start = end - TimeDelta(days, format="jd")
+    async def query_nights(self, topic, fields, index=301):
+        start = Time(self.min_date)
+        end = Time(self.max_date)
 
         # TODO resample
         series = await self.client.select_time_series(
@@ -128,29 +128,20 @@ async def query_nights(self, topic, fields, days=1, index=301):
         return series
 
     # slewTime (and probably others) are EXPECTED times, not ACTUAL.
-    async def get_targets(self, days=1):
+    async def get_targets(self):
         topic = "lsst.sal.Scheduler.logevent_target"
         fields_wanted = [
             "blockId",
-            "exposureTimes0",
-            "exposureTimes1",
-            "exposureTimes2",
-            "exposureTimes3",
-            "exposureTimes4",
-            "exposureTimes5",
-            "exposureTimes6",
-            "exposureTimes7",
-            "exposureTimes8",
-            "exposureTimes9",
-            "numExposures",
             "sequenceDuration",
             "sequenceNVisits",
             "sequenceVisits",
             "slewTime",
         ]
         # end = Time(datetime.combine(self.max_date, time()))
         targets = await self.query_nights(
-            topic, fields_wanted, days=days, index=self.salindex
+            topic,
+            fields_wanted,
+            index=self.salindex,
         )
         return targets
 

python/lsst/ts/logging_and_reporting/utils.py

@@ -110,7 +110,7 @@ def get_datetime_from_dayobs_str(dayobs):
         case _:
             no_dash = dayobs.replace("-", "")
             date = dt.datetime.strptime(no_dash, "%Y%m%d").date()
-    return date
+    return dt.datetime.combine(date, dt.time(12))
 
 
 dayobs2dt = get_datetime_from_dayobs_str