diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index c3070ac..14f0636 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -35,7 +35,7 @@ "# Total number of days of data to display (ending on day_obs)\n", "number_of_days = \"1\" # TODO Change to '1' to test with default before push\n", "\n", - "verbose = \"false\" # TODO change to false before push, else true" + "verbose = \"falsex\" # TODO change to false before push, else true" ] }, { @@ -90,21 +90,12 @@ "execution_count": 4, "id": "5d3008ec-1400-48e5-a35c-539869ff3ecf", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'day_obs': 'TODAY', 'number_of_days': 1, 'verbose': False} \n" - ] - } - ], + "outputs": [], "source": [ "# Validate parameters, return usable ones\n", "usable, error = ut.fallback_parameters(day_obs, number_of_days, verbose)\n", "if error:\n", " print(error)\n", - "print(usable, error)\n", "\n", "date = ut.get_datetime_from_dayobs_str(usable['day_obs'])\n", "# date: is EXCLUSIVE (upto, but not including)\n", @@ -158,7 +149,7 @@ { "data": { "text/markdown": [ - "# Showing data for 2024-11-21 to 2024-11-22 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" + "# Showing data for 2024-11-22 to 2024-11-23 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" ], "text/plain": [ "" @@ -170,7 +161,7 @@ { "data": { "text/markdown": [ - "Most recently run on: **2024-11-21 23:57:27**" + "Most recently run on: **2024-11-22 00:16:09**" ], "text/plain": [ "" @@ -230,43 +221,7 @@ { "data": { "text/markdown": [ - "### Telescope: Simonyi" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "```\n", - "====M1M3 work log==\n", - "Bump test done around 16:48pm CLT/19:48pm UTC, 104, 227, 409 was failed initially. However, 104 and 227 passed after the second attempt while FA449 is failed with 2nd attempt. Petr mentioned that 409 is marginal but still fine. \n", - "\n", - "=== Daytime work log===\n", - "We started warmup hexapod around 14pmCLT/17pm UTC, however we had an issue with warmup hexapod scripts. Hexapods moved without issues with EUI and run_command.py. Around 14:43pm CL/17:43pm UTC, we had an issue with GIS, so M1M3, Hexapods in fault (OBS-701). It was cleared then we closed the mirror cover for dome handover. Dome handover was completed without any issue at 15:29 pm UTC/18:29 pm UTC. We found that the even after moving hexapod with run_command.py inPosition for both hexapods seem not working. This can be a blocker for night operations, we pinged people in charge. 17:30pm/20:30 pm, Te-wei roll-backed the fix happened earlier today (see deatils in OBS-702). Then warming-up hexapods worked well without an issue. At 17:53pm CLT/20:53pm CLT we turned on LaserTracker to check it worked mostly fine except one MTPtg fault while tracking. Comcam checkout, Simonyi on Sky checkout (BLOCK-T245), and M2 and Camera alignment (BLOCK-T88) are all done. 20:28 pm CLT/23:23 pm UTC, we had an issue with GIS heartbeat flicker and almost all system were fault due to hard stop engaged (OBS-701). \n", - "\n", - "\n", - "\n", - "We started the TMA checkout around 19:15pm and all passed except that a known issue with turning on the elevation. \n", - "\n", - "Also Chris S. and Paulo L. joined tonight's operation and thanks for supports from Tiago R, Te-wei T, and others.\n", - "```" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "Telescope Status: " + "No nightreport records found 2024-11-22 to 2024-11-23. " ], "text/plain": [ "" @@ -278,7 +233,7 @@ { "data": { "text/markdown": [ - "*Authors: Kevin Fanning, Yijung Kang, Karla Pena, Paulina Venegas, Bryce Kalmbach, Andrew Connolly, Parker Fagrelius, Bruno Quint, Keith Bechtol*" + "Used [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241122&max_day_obs=20241123)" ], "text/plain": [ "" @@ -314,7 +269,7 @@ { "data": { "text/markdown": [ - "**Almanac for the observing night starting 2024-11-21**" + "**Almanac for the observing night starting 2024-11-22**" ], "text/plain": [ "" @@ -328,80 +283,80 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", "
Sun Set2024-11-21 23:18:34Sun Set2024-11-22 23:19:25
Evening Civil Twilight2024-11-21 23:49:23(-6 degrees)Evening Civil Twilight2024-11-22 23:50:18(-6 degrees)
Evening Nautical Twilight2024-11-22 00:21:27(-12 degrees)Evening Nautical Twilight2024-11-23 00:22:27(-12 degrees)
Evening Astronomical Twilight2024-11-22 00:55:19(-18 degrees)Evening Astronomical Twilight2024-11-23 00:56:25(-18 degrees)
Solar Midnight2024-11-22 04:29:01Solar Midnight2024-11-23 04:29:17
Moon Rise2024-11-22 05:01:56Moon Rise2024-11-23 05:32:02
Morning Astronomical Twilight2024-11-22 08:02:43(-18 degrees)Morning Astronomical Twilight2024-11-23 08:02:10(-18 degrees)
Morning Nautical Twilight2024-11-22 08:36:35(-12 degrees)Morning Nautical Twilight2024-11-23 08:36:09(-12 degrees)
Morning Civil Twilight2024-11-22 09:08:39(-6 degrees)Morning Civil Twilight2024-11-23 09:08:18(-6 degrees)
Sun Rise2024-11-22 09:39:28Sun Rise2024-11-23 09:39:11
Moon Set2024-11-22 15:51:00Moon Set2024-11-23 16:47:25
Moon Illumination59%(% illuminated)Moon Illumination49%(% illuminated)
UTCUTC
\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -525,71 +480,26 @@ " \n", " \n", " \n", - " LSSTComCam\n", + " \n", " \n", " \n", " \n", " \n", - " Total Night\n", - " 7:07:23\n", - " \n", - " \n", - " Total Exposure\n", - " 0:00:16\n", - " \n", - " \n", - " Slew time(1)\n", - " 0:00:00\n", - " \n", - " \n", - " Readout time(2)\n", - " 0:00:31\n", - " \n", - " \n", - " Time loss to fault\n", - " 0\n", - " \n", - " \n", - " Time loss to weather\n", - " 0\n", + " Total night\n", + " 7:05:44\n", " \n", " \n", " Idle time\n", - " 7:06:36\n", - " \n", - " \n", - " Number of exposures\n", - " 13\n", - " \n", - " \n", - " Mean readout time\n", - " 0:00:02\n", - " \n", - " \n", - " Number of slews(1)\n", - " 0\n", - " \n", - " \n", - " Mean Slew time(1)\n", - " 0:00:00\n", + " 7:05:44\n", " \n", " \n", "\n", "" ], "text/plain": [ - " LSSTComCam\n", - "Total Night 7:07:23\n", - "Total Exposure 0:00:16\n", - "Slew time(1) 0:00:00\n", - "Readout time(2) 0:00:31\n", - "Time loss to fault 0\n", - "Time loss to weather 0\n", - "Idle time 7:06:36\n", - "Number of exposures 13\n", - "Mean readout time 0:00:02\n", - "Number of slews(1) 0\n", - "Mean Slew time(1) 0:00:00" + " \n", + "Total night 7:05:44\n", + "Idle time 7:05:44" ] }, "metadata": {}, @@ -630,104 +540,7 @@ "execution_count": 12, "id": "18", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "- [obs-662](https://rubinobs.atlassian.net/browse/obs-662)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-701](https://rubinobs.atlassian.net/browse/OBS-701)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [obs-653](https://rubinobs.atlassian.net/browse/obs-653)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-699](https://rubinobs.atlassian.net/browse/OBS-699)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-698](https://rubinobs.atlassian.net/browse/OBS-698)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-700](https://rubinobs.atlassian.net/browse/OBS-700)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "if allsrc.urls:\n", " for url in allsrc.urls:\n", @@ -743,7 +556,7 @@ { "data": { "text/markdown": [ - "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241121&max_day_obs=20241122) in `confluence_url`." + "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241122&max_day_obs=20241123) in `confluence_url`." ], "text/plain": [ "" @@ -819,7 +632,7 @@ { "data": { "text/markdown": [ - "**LSSTComCam: 13 total exposures**" + "**LSSTComCam: 0 total exposures**" ], "text/plain": [ "" @@ -828,147 +641,6 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Type\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Detailgoodquestionablejunkunknown
acq0003
bias00010
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Reason\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Detailgoodquestionablejunkunknown
comcam-usdf-checkout00010
tmacheckout0003
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Science Program\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Detailgoodquestionablejunkunknown
BLOCK-T24600010
BLOCK-T2500003
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "text/markdown": [ @@ -1023,52 +695,9 @@ { "data": { "text/markdown": [ + "- **2024-11-22 00:11** **MainTel** \n", "\n", - "\n", - "h3. Error/Failure explanation:\r\n", - "We had failures of the Camera cable wrap, axes timeouts, and frequent communications errors that impacted our control software. We lost over 2 hours of sky time struggling with these issues. \r\n", - "h3. What were you doing when the failure occurred?:\r\n", - "Tonight we had several more instances of control system crashes due to timeouts. In addition to these incidents, the CCW would not follow the rotator, stall out during short or tracking movements, and then trigger a fault in the rotator following. We also had several recurring axes timeouts that required us to attempt initializing our control software several times before we could move. \r\n", - "h3. Detailed description of the steps followed that caused the error/failure:\r\n", - "Here is a list of some timestamps that indicate the approximate times of communications failures. These demonstrate startTracking command timeouts, CCW timeouts, and axes timeouts. \r\n", - "\r\n", - "Example 1:\r\n", - "2024/11/21 02:11:02 TAI\r\n", - "\r\n", - "Error MTMount replied error for startTracking command (1732828411) Error no:1 message:Failed: Command with id 447721 has timed \r\n", - "\r\n", - "Example 2: 04:32:41\r\n", - "The cable cable wrap faulted the MTMount. The rotator was not commanded anywhere, but the CCW just decided to quit. \r\n", - "\r\n", - "Example 3:\r\n", - "2024/11/21 11:02:09 TAI\r\n", - "\r\n", - "\r\n", - "Command BothAxesResetAlarm(sequence_id=1036635, command_code=, source=, timestamp=1732186929.5137012) failed, waiting 60s and retrying.\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 886, in enable_devices\r\n",
-       "\r\n",
-       "    await self.send_command(command, do_lock=True, timeout=timeout)\r\n",
-       "\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1173, in send_command\r\n",
-       "\r\n",
-       "    return await self._basic_send_command(command=command, timeout=timeout)\r\n",
-       "\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1259, in _basic_send_command\r\n",
-       "\r\n",
-       "    await asyncio.wait_for(\r\n",
-       "\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 489, in wait_for\r\n",
-       "\r\n",
-       "    return fut.result()\r\n",
-       "\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "\r\n",
-       "lsst.ts.salobj.base.ExpectedError: Command with id 1036635 has timed out after not receiving finished
\n" + "
Raising M1M3 again.
\n" ], "text/plain": [ "" @@ -1080,24 +709,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 11:30** **MainTel** \n", + "- **2024-11-22 00:07** **MainTel** \n", "\n", - "
Moving to high elevation to open mirror covers as requested for the daytime. \r\n",
-       "\r\n",
-       "\r\n",
-       "Lol. JK. We didn't do that, because it was likely an old request.
\n" + "
OSS turning on now
\n" ], "text/plain": [ "" @@ -1109,2733 +723,9 @@ { "data": { "text/markdown": [ - "- **2024-11-21 11:06** **MainTel** \n", + "- **2024-11-22 00:05** **MainTel** \n", "\n", - "
Moving TMA in elevation.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 11:05** **MainTel** \n", - "\n", - "
Recovered telescope to perform BLOCK-240.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "TMA timing our frequently while trying to ENABLE MTMount:\r\n", - "2024/11/21 11:02:09 TAI\r\n", - "\r\n", - "\r\n", - "Command BothAxesResetAlarm(sequence_id=1036635, command_code=<CommandCode.BOTH_AXES_RESET_ALARM: 37>, source=<Source.CSC: 1>, timestamp=1732186929.5137012) failed, waiting 60s and retrying.\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 886, in enable_devices\r\n",
-       "    await self.send_command(command, do_lock=True, timeout=timeout)\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1173, in send_command\r\n",
-       "    return await self._basic_send_command(command=command, timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1259, in _basic_send_command\r\n",
-       "    await asyncio.wait_for(\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 489, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "lsst.ts.salobj.base.ExpectedError: Command with id 1036635 has timed out after not receiving finished\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "BLOCK-231 data was collected. Script salindex where GIS was triggered was 104862:\r\n", - "\r\n", - "\r\n", - "2024/11/21 10:42:10 TAI\r\n", - "\r\n", - "\r\n", - "Error in run\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2705, in m1m3_booster_valve\r\n",
-       "    yield\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2315, in move_p2p_azel\r\n",
-       "    await self.process_as_completed(tasks)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1137, in process_as_completed\r\n",
-       "    raise e\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1134, in process_as_completed\r\n",
-       "    ret_val = await res\r\n",
-       "              ^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
-       "    return f.result()  # May raise f.exception().\r\n",
-       "           ^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 510, in check_component_state\r\n",
-       "    raise RuntimeError(\r\n",
-       "RuntimeError: mtm1m3 state is &lt;State.FAULT: 3&gt;, expected &lt;State.ENABLED: 2&gt;\r\n",
-       "\r\n",
-       "\r\n",
-       "During handling of the above exception, another exception occurred:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
-       "    await self.run_block()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
-       "    await self.mtcs.move_p2p_azel(\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2301, in move_p2p_azel\r\n",
-       "    async with self.m1m3_booster_valve():\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/contextlib.py", line 231, in __aexit__\r\n",
-       "    await self.gen.athrow(typ, value, traceback)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2708, in m1m3_booster_valve\r\n",
-       "    await self.close_m1m3_booster_valve()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2616, in close_m1m3_booster_valve\r\n",
-       "    await self._handle_m1m3_booster_valve(open=False)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2657, in _handle_m1m3_booster_valve\r\n",
-       "    await cmd.set_start(timeout=self.fast_timeout)\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=389238772, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=0, result='Failed: The command ClearSlewFlag is not valid in the FaultState.')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:57** **MainTel** \n", - "\n", - "
Raising M1M3.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:44** **MainTel** \n", - "\n", - "
GIS activated during azimuth slew.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "BLOCK-231 failed due to ignore list not being honored. Going to run just a regs p2p.\r\n", - "Terminating with state=7: stop telescope.\r\n", - "\r\n", - "\r\n", - "2024/11/21 10:39:43 TAI\r\n", - "\r\n", - "\r\n", - "Error in run\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
-       "    await self.run_block()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
-       "    await self.mtcs.move_p2p_azel(\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2315, in move_p2p_azel\r\n",
-       "    await self.process_as_completed(tasks)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1137, in process_as_completed\r\n",
-       "    raise e\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1134, in process_as_completed\r\n",
-       "    ret_val = await res\r\n",
-       "              ^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
-       "    return f.result()  # May raise f.exception().\r\n",
-       "           ^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 510, in check_component_state\r\n",
-       "    raise RuntimeError(\r\n",
-       "RuntimeError: mtdome state is &lt;State.DISABLED: 1&gt;, expected &lt;State.ENABLED: 2&gt;\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:34** **MainTel** \n", - "\n", - "
beginning BLOCK-231
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:16** **MainTel** \n", - "\n", - "
M1M3 faulted while trying to recover. Sending to DISABLED state and this happened.\r\n",
-       "2024/11/21 10:14:45 TAI\r\n",
-       "Air controller sensor mismatch
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:13** **MainTel** \n", - "\n", - "
Recovering system from M2 breakaway test.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:07** **MainTel** \n", - "\n", - "
M1M3 faulted during the M2 breakaway test.\r\n",
-       "2024/11/21 10:04:04 TAI\r\n",
-       "Force controller Z Moment Limit - applied -11402.72 N, expected -9000.00 N to 9000.00 N
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 10:03** **MainTel** \n", - "\n", - "
Beginning BLOCK-241.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:49** **MainTel** \n", - "\n", - "
BLOCK-227 completed successfully. Nice job y'all!
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:28** **MainTel** \n", - "\n", - "
Going back to add BLOCK-227 while ignoring MTHexapod.2
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:27** **MainTel** \n", - "\n", - "
MTHexapod.2 faulted upon trying to apply compensation mode. \r\n",
-       "2024/11/21 09:25:16 TAI\r\n",
-       "Fault! errorCode=1, errorReport='Low-level controller went to FAULT state'\r\n",
-       "2024/11/21 09:25:16 TAI\r\n",
-       "max_move_duration=61.74\r\n",
-       "2024/11/21 09:25:16 TAI\r\n",
-       "Applying compensation to the current target position
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "MTHexapod:2 CSC faulted during 10% tests. In the middle of a slew:\r\n", - "SalIndex: 104790\r\n", - "2024/11/21 09:22:10 TAI\r\n", - "\r\n", - "\r\n", - "Fault! errorCode=-2, errorReport='Compensation failed.'\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "2024/11/21 09:22:10 TAI\r\n", - "\r\n", - "\r\n", - "Compensation failed; CSC going to Fault.\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 500, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mthexapod/hexapod_csc.py", line 1105, in wait_stopped\r\n",
-       "    await self.telemetry_event.wait()\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/locks.py", line 213, in wait\r\n",
-       "    await fut\r\n",
-       "asyncio.exceptions.CancelledError\r\n",
-       "\r\n",
-       "\r\n",
-       "The above exception was the direct cause of the following exception:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mthexapod/hexapod_csc.py", line 442, in compensation_loop\r\n",
-       "    await self.compensation_wait()\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mthexapod/hexapod_csc.py", line 471, in compensation_wait\r\n",
-       "    await asyncio.wait_for(self.wait_stopped(), timeout=self.max_move_duration)\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:18** **MainTel** \n", - "\n", - "
Scripts are now loading. BLOCK-227 in progress.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:18** **MainTel** \n", - "\n", - "
Anti-climatically, the block didn't populate scripts. We are enabling the scheduler again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:15** **MainTel** \n", - "\n", - "
Alberto is online, going to begin digging through the trenches that are TMA comms. We wish him the best in the journey.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:15** **MainTel** \n", - "\n", - "
Running BLOCK-227.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:04** **MainTel** \n", - "\n", - "
MTMount would not enable:\r\n",
-       "2024/11/21 09:02:29 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "begin_enable failed; remaining in state <State.DISABLED: 1>: Command BothAxesPower(sequence_id=1006565, command_code=<CommandCode.BOTH_AXES_POWER: 31>, source=<Source.CSC: 1>, timestamp=1732179737.9188824, on=True) failed: ExpectedError('Command failed due to an alarm.')\r\n",
-       "\r\n",
-       "\r\n",
-       "Elevation axis reports a positive adjustable software limit.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:02** **MainTel** \n", - "\n", - "
Preparing TMA for 10% velocity tests with Petr supporting.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 09:02** **MainTel** \n", - "\n", - "
MTDome is closed. No issues in closing completely, although it took several minutes for the ApS left motors to complete the task and exit nominally.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 08:22** **MainTel** \n", - "\n", - "
Finished data taking for the night. Preparing everything to close dome shutters.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 08:16** **MainTel** Time Lost: 0.37; Time Lost Type: fault;\n", - "\n", - "
M1M3 faulted during a closed loop script, around 54 degrees elevation:\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/21 01:28:15 TAI\r\n",
-       "Force Actuator ID 128 (27) Following Error immediate fault
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 08:15** **MainTel** \n", - "\n", - "
MT Air compressor faulted. Going to cycle the CSC and hope connection comes back\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/21 02:35:48 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Fault! errorCode=98, errorReport='While reconnecting:Modbus Error: [Connection] Not connected[AsyncModbusTcpClient m1m3cam-aircomp02.cp.lsst.org:502]'\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/21 02:35:48 TAI\r\n",
-       "While reconnecting:Modbus Error: [Connection] Not connected[AsyncModbusTcpClient m1m3cam-aircomp02.cp.lsst.org:502]
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-700](https://rubinobs.atlassian.net/browse/OBS-700)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 08:07** **MainTel** \n", - "\n", - "
In focus in slightly poor seeing conditions. Grabbing science data.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 07:56** **MainTel** \n", - "\n", - "
Had to re-issue corrections and remove some of previous settings before proceeding with the closed_loop comcam - the previous hexapod state was still applied.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 07:51** **MainTel** \n", - "\n", - "
Taking image on science target.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 07:47** **MainTel** \n", - "\n", - "
Slewing to final science target for the night. Going to try and close the loops on a previously observed target.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 06:21** **MainTel** \n", - "\n", - "
Science team is ceding the telescope back to the AOS team. We will resume AOS testing until ~10 minutes before closing time.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 05:47** **MainTel** \n", - "\n", - "
Resuming the scheduler.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 05:36** **MainTel** \n", - "\n", - "
Recovered the Hexapod, completed the giant donut data, and we returned the rotator to 0. The AOS team is going to take a pause and we are going to collect science data near rot_angle 0 to minimize CCW fault risk.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 05:19** **MainTel** \n", - "\n", - "
MTHexapod failed - compensation failed.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 05:10** **MainTel** \n", - "\n", - "
BLOCK-75 data collection ongoing.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 05:04** **MainTel** \n", - "\n", - "
Going to attempt to track target nearby to get some giant donut data.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:53** **MainTel** \n", - "\n", - "
The CCW axis faulted. It just stopped tracking the rotator.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:37** **MainTel** \n", - "\n", - "
Attempting BLOCK-301.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:33** **MainTel** \n", - "\n", - "
The cable cable wrap faulted the MTMount. The rotator was not commanded anywhere, but the CCW just decided to quit.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:32** **MainTel** \n", - "\n", - "
PSYCH! No it didn't!
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:29** **MainTel** \n", - "\n", - "
We were able to step to 70 degrees in rotation just as the BLOCK does. Hopefully we can run the tests without an additional strain on the CCW following the rotator.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:29** **MainTel** \n", - "\n", - "
Going to attempt the rotator BLOCK-301.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:25** **MainTel** \n", - "\n", - "
Attempting to rotate the rotator in small steps. Trying to identify when the rotator and CCW decouple in steps.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:13** **MainTel** \n", - "\n", - "
Recovered MTRotator, slewed to 10 degrees rotation, and a few degrees into the slew, the CCW stopped following the rotator around -2 degrees. We will attempt now to recover.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 04:08** **MainTel** \n", - "\n", - "
Went down to the first floor to try and find a reset method for the MTAircompressor2. Chiller presents an error on the screen. MTAircompressor reads some pressure.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:36** **MainTel** \n", - "\n", - "
Attempting a 5 degree slew.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:35** **MainTel** \n", - "\n", - "
Continuing to troubleshoot rotator. Perhaps moves were too small to trigger an inposition flag.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:24** **MainTel** \n", - "\n", - "
Attempting to track target and the rotator is timing out.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:15** **MainTel** \n", - "\n", - "
Tiago is helping us try to identify a patch we can deploy tonight so we don't encounter frequent errors with TMA timeouts.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "move_p2p failed \r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "2024/11/21 03:09:44 TAI\r\n", - "\r\n", - "\r\n", - "Error in run\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
-       "    await self.run_block()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
-       "    await self.mtcs.move_p2p_azel(\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2315, in move_p2p_azel\r\n",
-       "    await self.process_as_completed(tasks)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1137, in process_as_completed\r\n",
-       "    raise e\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1134, in process_as_completed\r\n",
-       "    ret_val = await res\r\n",
-       "              ^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
-       "    return f.result()  # May raise f.exception().\r\n",
-       "           ^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=930398849, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: The commands allowed in Tracking are: Stop\\n\\nNot accepted command: 103\\nThe commands allowed in Tracking are: Stop\\n\\nNot accepted command: 403')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:08** **MainTel** \n", - "\n", - "
Unable to track target - continuous timeouts when trying to track target.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:03** **MainTel** \n", - "\n", - "
Going to operate with one compressor until we can find a way to connect MTAirCompressor:2
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 03:02** **MainTel** \n", - "\n", - "
MTPtg faulting frequently. Really impacting operations. Going to stop MTmount and movep2p.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:55** **MainTel** \n", - "\n", - "
Baden confirms that he can ping the MT air compressor 2 machine. As soon as we try to enable the CSC we lose connection. \r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/21 02:52:55 TAI\r\n",
-       "Starting up:Modbus Error: [Connection] Not connected[AsyncModbusTcpClient m1m3cam-aircomp02.cp.lsst.org:502]
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:47** **MainTel** \n", - "\n", - "
IT support is going to dial in and check out some of the network issues we seem to be having.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:47** **MainTel** \n", - "\n", - "
Attempted to cycle compressor CSC, didn't connect. Going to follow Petr's instructions. Troubleshooting.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:29** **MainTel** \n", - "\n", - "
MtPtg fault - just about to initiate a track_target command. \r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/21 02:27:31 TAI\r\n",
-       "Error MTMount replied error for trackTarget command (1734022109) Error no:1 message:Failed: Command with id 474073 has timed ou
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:18** **MainTel** \n", - "\n", - "
MTPtg immediately got grumpy. We're going to have to stop all Mount traffic and move_p2p
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:17** **MainTel** \n", - "\n", - "
Restarting observations after MTPtg fault.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 02:14** **MainTel** \n", - "\n", - "
MTPtg faulted\r\n",
-       "salIndex: 104525\r\n",
-       " 2024/11/21 02:11:12 TAI\r\n",
-       "raise RuntimeError(\r\n",
-       "RuntimeError: mtptg state is <State.FAULT: 3>, expected <State.ENABLED: 2>\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/21 02:11:02 TAI\r\n",
-       "Error MTMount replied error for startTracking command (1732828411) Error no:1 message:Failed: Command with id 447721 has timed
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:57** **MainTel** \n", - "\n", - "
BLOCK-T300 will move the telescope in different elevation angles and run closed loop. We want to understand the focus gradient we saw yesterday.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:56** **MainTel** \n", - "\n", - "
We moved the telescope to the final expected position using move_p2p command. \r\n",
-       "M1M3 behaved fine now. Starting BLOCK-T300 again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:45** **MainTel** \n", - "\n", - "
MTM1M3TS was recovered through Brian's script that automatically recovers it.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:41** **MainTel** \n", - "\n", - "
As part of recovering, we need to recover MTM1M3TS as well. \r\n",
-       "We have the instructions. However, someone connected to MTM1M3TS and recovered it.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:32** **MainTel** \n", - "\n", - "
2024/11/21 01:28:15 TAI\r\n",
-       "MTM1M3 went to a fault: Force Actuator ID 128 (27) Following Error immediate Fault.\r\n",
-       "\r\n",
-       "\r\n",
-       "This happened while running BLOCK-T300. \r\n",
-       "It was slewing the telescope to -100º in Az. It was a big slew and we were at 80º elevation angle. \r\n",
-       "There was no apparent reason for M1M3 to go to a fault.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-699](https://rubinobs.atlassian.net/browse/OBS-699)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:29**\n", - "\n", - "
Running block t-300, MTMount, MTPtg and MTM1M3 faulted
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:23** **MainTel** \n", - "\n", - "
SOAR seeing is ~1.3. We're getting ~1.1 in r at elevation of 71 deg.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:20** **MainTel** \n", - "\n", - "
The values in the Residual AOS FWHM column in RubinTv are plateauing. This means that we are probably ready to move on to the next test case.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 01:18** **MainTel** \n", - "\n", - "
The next test case would be BLOCK-T303. However, it seems that our BLOCK-T249 already accomplished the goal of 303. So we will skip it. We marked it as skipped in Zephyr.  We want to re-evaluate it, so we should keep it in the night plan for tomorrow.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:51** **MainTel** \n", - "\n", - "
Closed loop ongoing. We can see donuts!
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:50** **MainTel** \n", - "\n", - "
We are starting BLOCK-T249 again after all the recovery. \r\n",
-       "Let's start taking a new image.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:48** **MainTel** \n", - "\n", - "
Alright. Now let's try track_target just where we are.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:47** **MainTel** \n", - "\n", - "
This did not work. So we will enable MTPTg and send a stop_tracking script. Then we will try again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:43** **MainTel** \n", - "\n", - "
We are now recovering again. We will home both axes and try a move_p2p. If that does not work, we will need to send a stop command to the mount and try again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:42** **MainTel** \n", - "\n", - "
The TMA EUI went to fault in Elevation. This is a SoftMotion Axis Fault. \r\n",
-       "Set Time 00:39:13.337 21/11/24\r\n",
-       "System: elevation
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-698](https://rubinobs.atlassian.net/browse/OBS-698)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:41**\n", - "\n", - "
Home both axes, track target \r\n",
-       "\r\n",
-       "\r\n",
-       "Then needed to put rotator at zero.  Did that. \r\n",
-       "\r\n",
-       "\r\n",
-       "Then did track target again, and this time both the MTMount and MtPtg faulted
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:33** **MainTel** \n", - "\n", - "
Stop tracking was not enough. When putting M2 into DISABLED state, MTMount went to a FAULT state. We are recovering now.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:31** **MainTel** \n", - "\n", - "
It is an issue with M2.  \r\n",
-       "\r\n",
-       "\r\n",
-       "Stop tracking
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:30** **MainTel** \n", - "\n", - "
Tiago is now debugging what is going on. He says that we might having issues with historical data associated with the crash we had with the DDS systems (yagan13 and yagan17).
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:21** **MainTel** \n", - "\n", - "
AOS Closed loop script faulted with a timeout error. Sal index 301046
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:20** **MainTel** \n", - "\n", - "
We are now running closed loop comcam.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:19** **MainTel** \n", - "\n", - "
When getting this image, we realized that the LaserTracker was powered on. I put LaserTracker to StandBy and powered it off via PDU. We should be ready to go now.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:18** **MainTel** \n", - "\n", - "
Turning off the laser now
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:17** **MainTel** \n", - "\n", - "
We forgot to turn off the laser.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:16** **MainTel** \n", - "\n", - "
We are now starting BLOCK-T249.\r\n",
-       "The telescope is already tracking. So we will start by taking an image first.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 00:10**\n", - "\n", - "
Still taking i-band twilight flats but switching to rotator = 30 deg
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:50** **MainTel** \n", - "\n", - "
Disable MtM2, MTM1M3, Hexapod1,2, rotator, MTM1M3TS etc. to get telemetry to monitor.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:49** **MainTel** \n", - "\n", - "
Still need to support Franco C. Robinson was resetting all the cabinet. He still needs to work on it.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-701](https://rubinobs.atlassian.net/browse/OBS-701)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:48** **MainTel** \n", - "\n", - "
Will try to MTM2 enabled.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:40** **MainTel** \n", - "\n", - "
Robinson cleared up the GIS hard stop. We will recover step by step. First we reset the GIS for M2 and Hexapods (and rotator)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:29** **MainTel** \n", - "\n", - "
GIS heartbeat flicker. The same as what happened today.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:28** **MainTel** \n", - "\n", - "
Hard Stop performed in the OSS and other systems are all fault.   23:23:29.033 21/11/24
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:26** **MainTel** \n", - "\n", - "
(false) hard stop seemed fault almost every system.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:20** **MainTel** \n", - "\n", - "
Opening mirror covers
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:20** **MainTel** \n", - "\n", - "
Enabled dome following
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:15** **MainTel** \n", - "\n", - "
LaserTracker off at PDU
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:15** **MainTel** \n", - "\n", - "
Checked out AOS sequence PAIR mode to obsenv. Reloaded schema to pick up new mode.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:12** **MainTel** \n", - "\n", - "
run_command for close_mirror_cover failed. But seems closed
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:12** **MainTel** \n", - "\n", - "
Mirror covers closing. HVAC off, preparing to open shutter
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:11** **MainTel** \n", - "\n", - "
HVAC configuration is now setup to night time operation with turned on VEC_04 fan.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:11** **MainTel** \n", - "\n", - "
T-88 done.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 23:11** **MainTel** \n", - "\n", - "
Then camera alignment thru align.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:56** **MainTel** \n", - "\n", - "
Block-T88. M2 alignment. running align.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:54** **MainTel** \n", - "\n", - "
We are on BLOCK-T245. Looks okay. Enabling scheduler.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:51** **MainTel** \n", - "\n", - "
Tracking and then take image comcam for checking out. \r\n",
-       "exp_times: 5\r\n",
-       "nimages: 3\r\n",
-       "image_type: \"ACQ\"\r\n",
-       "program: \"BLOCK-T250\"\r\n",
-       "reason: \"TMACheckout\"\r\n",
-       "filter: \"y_04\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "salIndex: 301080 while tracking target, MTPtg failed \r\n", - "2024/11/21 22:48:08 TAI\r\n", - "\r\n", - "\r\n", - "Unexpected exception in stop_tracking.\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_track_target.py", line 551, in cleanup\r\n",
-       "    await asyncio.wait_for(self.tcs.stop_tracking(), timeout=5)\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 489, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_tcs.py", line 1304, in stop_tracking\r\n",
-       "    await getattr(self.rem, self.ptg_name).cmd_stopTracking.start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=2065515254, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=6612, result='Rejected : command not allowed in current state')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:46** **MainTel** \n", - "\n", - "
TMA checkout. \r\n",
-       "track_azel:\r\n",
-       "  el: 65\r\n",
-       "  az: 135\r\n",
-       "rot_value: 0\r\n",
-       "rot_type: PhysicalSky\r\n",
-       "program: \"BLOCK-T250\"\r\n",
-       "reason: \"TMACheckout\"\r\n",
-       "ignore:\r\n",
-       " - mtdome\r\n",
-       " - mtdometrajectory
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:43** **MainTel** \n", - "\n", - "
Tiago R. updated the scheduler.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:39** **MainTel** \n", - "\n", - "
with run_command.py, it works
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "2024/11/21 22:36:56 TAI SALINDEX: salIndex: 104934\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "Error in run\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 967, in open_m1_cover\r\n",
-       "    await self.rem.mtmount.cmd_openMirrorCovers.set_start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1028798034, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: The commands allowed in DiscreteMove are: Stop\\n\\nNot accepted command: 901')\r\n",
-       "\r\n",
-       "\r\n",
-       "During handling of the above exception, another exception occurred:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/open_mirror_covers.py", line 66, in run\r\n",
-       "    await self.mtcs.open_m1_cover()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 972, in open_m1_cover\r\n",
-       "    f"Open mirror cover command failed with {ack.ack!r}::{ack.error}. "\r\n",
-       "                                             ^^^^^^^\r\n",
-       "AttributeError: 'AckError' object has no attribute 'ack'\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-653](https://rubinobs.atlassian.net/browse/obs-653)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:37** **MainTel** \n", - "\n", - "
open_mirror_cover failed.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-662](https://rubinobs.atlassian.net/browse/obs-662)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:31** **MainTel** \n", - "\n", - "
Home both axes then will enable hardpoint correction.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:31** **MainTel** \n", - "\n", - "
Failed to enable MTMount but elevation failed.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:27** **MainTel** \n", - "\n", - "
Changed to 10% without parking!
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:22** **MainTel** \n", - "\n", - "
We are on the parking set. Will move down to 80 deg. and move in Az to check.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:20** **MainTel** \n", - "\n", - "
Will unpark the telescope. First free the locking pins.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 22:20** **MainTel** \n", - "\n", - "
Oh, locking pins are engaged. When it stopped the resetting, will disengage the locking pins.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:15** **MainTel** \n", - "\n", - "
Warm up cam hex also done.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:12** **MainTel** \n", - "\n", - "
We will go to have a dinner soon. Cam hex warming up will run while we are there.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:08** **MainTel** \n", - "\n", - "
Warm up hexapod for Cam hex is running.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:06** **MainTel** \n", - "\n", - "
Warmup hexapod for M2 is done. Will disable M2 hexapod.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:05** **MainTel** \n", - "\n", - "
Raising M1M3!
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:04** **MainTel** \n", - "\n", - "
Kevin F. pulhed the ts_config_mttcs and then reenable LaserTraker1.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:01** **MainTel** \n", - "\n", - "
Pulled update to mttcs for MTAOS CSC. CSC is in standby.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 21:01** **MainTel** \n", - "\n", - "
It didn't work. Guillem MH restarted TS2SA.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:57** **MainTel** \n", - "\n", - "
LaserTracker1 enabled.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:53** **MainTel** \n", - "\n", - "
Per guillem's request, turning on the Lasertracker PDU.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:30** **MainTel** \n", - "\n", - "
Runing BLOCK-T3, warming up for M2 hexapod. It seems working this time.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:29** **MainTel** \n", - "\n", - "
Te-wei roll-backed the hexapod controller versions. The fix of controller's state machine breaks the CSC inPosition event.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:20** **MainTel** \n", - "\n", - "
Petr turned on FCU around 3:30 pm (which was off after GIS activation today).
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:17** **MainTel** \n", - "\n", - "
HVAC setpoint is changed to 9C. Will re-evaluate that after dinner time.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:09** **MainTel** \n", - "\n", - "
Te-wei will add some test result on OBS-702 and then will decide whether roll-back the update or not.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:00** **MainTel** \n", - "\n", - "
Starting check_hardpoint.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 20:00** **MainTel** \n", - "\n", - "
After retest 409 is failing and 227 is failing. 227 looks OK, 409 is not functional on the yForces.Will start hardpoint test, but likely we need to valve off and disable 409. I think we will be able to operate with it disabled. and a (slightly) marginal 227 undershooting on the primary.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 19:49** **MainTel** \n", - "\n", - "
Initially failing actuator list:\r\n",
-       "  - Actuator ID 104: Pri Index 3\r\n",
-       "  - Actuator ID 227: Pri Index 61\r\n",
-       "  - Actuator ID 409: Pri Index 123
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 19:25** **MainTel** \n", - "\n", - "
Not sure hexapods are inposition. We need to ignore the flag to track_target etc for night operation. Kevin F. announce in simonyi channel.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 19:08** **MainTel** \n", - "\n", - "
Turning on PS.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 19:06** **MainTel** \n", - "\n", - "
Increasing sleep time to 100 sec doesn't work.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 19:01** **MainTel** \n", - "\n", - "
warmup_hexapod with \r\n",
-       "hexapod: m2\r\n",
-       "axis: x\r\n",
-       "step_size: 125\r\n",
-       "max_position: 4500\r\n",
-       "sleep_time: 30
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:57** **MainTel** \n", - "\n", - "
Run BLOCK-T3 for warming up hexapod. Checking.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:56** **MainTel** \n", - "\n", - "
We were turning on the OSS. 15:33 pm CLT/18:33pm UTC
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:42** **MainTel** \n", - "\n", - "
Starting BLOCK-T144, M1M3 bump test
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:39** **MainTel** \n", - "\n", - "
run_command.py for both hexapods for x:250 and it worked for both axes. Maybe timeout issue from warmup_hexapod.py issue.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:16** **MainTel** \n", - "\n", - "
No failed again. Will try the Cam hexa for next.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:15** **MainTel** \n", - "\n", - "
TMA lost the heartbeat but recovered. \r\n",
-       "Closing the mirror cover now.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:15** **MainTel** \n", - "\n", - "
Rerun for BLOCK-T3 M2 hexapod.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 18:14** **MainTel** \n", - "\n", - "
Franco said the GIS is cleared. Door is still opened.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "\n", - "We found that there is Safety Modbus Server Error 17:41:52.438 21/11/24 code 66 system Safety Task. \r\n", - "VI chain: ??? in SubVIs.lvlib:TCP Master.lvclass:Protocol Write.vi:6190001->SubVIs.lvlib:IP Data Unit.lvclass:Write ADU Packet.vi:1820001->Modbus Master.lvclass:Read Holding Registers.vi:2420001->ModbusServer.lvclass:UpdateModbusData.vi:3110003->ModbusServer.lvclass:HelperLoop.vi:190002->ModbusServer.lvclass:Process.vi:5310001 Description: LabVIEW: (Hex 0x42) The network connection was closed by the peer. If you are using the Open VI Reference function on a remote VI Server connection, verify that the machine is allowed access by selecting <b>Tools>>Options>>VI Server</b> on the server side.\r\n", - "\r\n", - "\r\n", - "Then 17:41:52.743 21/11/24\r\n", - "VI chain: Description: LabVIEW: (Hex 0x0) Error connecting to GPIB driver or device. The NI-488.2 driver may not be installed.
=========================
VISA: (Hex 0x0) Operation completed successfully.\r\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:51** **MainTel** \n", - "\n", - "
Te-Wei used the hexapod GUIs to test the movements of the camera and M2 hexapods already. It worked well. \r\n",
-       "However really-E-stop seems engagled \r\n",
-       "On TMA EUI hardstop 1 & 2 topple block LS1 seems activated.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:45** **MainTel** \n", - "\n", - "
Found that we lost GIS heartbeat. Kevin F. is going to hit the GIS.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:43** **MainTel** \n", - "\n", - "
Salindex:104906. warmup_hexapod fault ActuallyGIS Emergency stop.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:42** **MainTel** \n", - "\n", - "
Running cam Hex warming. But even Hexapod1 and Hexapod2.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:40** **MainTel** \n", - "\n", - "
salIndex: 104897 Started at: 2024-11-21T17:38:24.829Z\r\n",
-       "\r\n",
-       "\r\n",
-       "warmup_hexapod.py failed waiting for in position event. either not moving of not sending events.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:38** **MainTel** \n", - "\n", - "
Just re-run the block. Will see if there is any issue.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:36** **MainTel** \n", - "\n", - "
salIndex: 104888  - Started at: 2024-11-21T17:33:53.915Z\r\n",
-       "Warmup_hexapod.py failed and timeout. \r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:33** **MainTel** \n", - "\n", - "
Start warm-up M2 hexapod.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:31** **MainTel** \n", - "\n", - "
Hexapod test from Te-wei is done. Will start warm-up hexapods.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:30** **MainTel** \n", - "\n", - "
Mirror cover is still opened by Daycrew (Ian announced it this morning).
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 17:06** **MainTel** \n", - "\n", - "
Start warming up M2 and Camera hexpod. \r\n",
-       "02:05 we stop the warming up as Te-wei need to clearError.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 16:54** **MainTel** \n", - "\n", - "
Start Powering on the Camera Rotator.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 16:08** **MainTel** \n", - "\n", - "
Done for Comcam checkout. Rubin TV and CCS are showing the image
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 16:04** **MainTel** \n", - "\n", - "
BLOCK-T246 Comcam USDF checkout running. Checking Comcam is ready for today's operation.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 16:02** **MainTel** \n", - "\n", - "
Around 8-9pm, 9.8C is expected. Current set point is 8C so we are good
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-21 15:08** **MainTel** \n", - "\n", - "
Daytime observers start shift.
\n" + "
Wiill try to enable M2. Done. We are in closed loop.
\n" ], "text/plain": [ "" @@ -3890,8 +780,8 @@ "text/markdown": [ "## Project-Wide Night(s) Report \n", "- Run on logs and databases from **https://usdf-rsp-dev.slac.stanford.edu/**\n", - "- Report **1 observing night(s)** with the last reported night starting on **2024-11-21 12:00:00**.\n", - "- min_dayobs='2024-11-21', max_dayobs='2024-11-22'\n", + "- Report **1 observing night(s)** with the last reported night starting on **2024-11-22 12:00:00**.\n", + "- min_dayobs='2024-11-22', max_dayobs='2024-11-23'\n", "- Using ***Prototype* Logging and Reporting** Version: **52.dev0+g783922c.d20241115**\n", "- have_consdb = False\n", "- Imported lsst.ts.logging_and_reporting from local packages.\n", @@ -3961,8 +851,8 @@ " \n", " \n", " nightreport/reports\n", - " 1\n", - " https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241121&max_day_obs=20241122\n", + " 0\n", + " https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241122&max_day_obs=20241123\n", " \n", " \n", " exposurelog/instruments\n", @@ -3972,32 +862,32 @@ " \n", " exposurelog/exposures.LSSTComCamSim\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241122&max_day_obs=20241123\n", " \n", " \n", " exposurelog/exposures.LATISS\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241122&max_day_obs=20241123\n", " \n", " \n", " exposurelog/exposures.LSSTComCam\n", - " 13\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", + " 0\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241122&max_day_obs=20241123\n", " \n", " \n", " exposurelog/exposures.LSSTCam\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241122&max_day_obs=20241123\n", " \n", " \n", " exposurelog/messages\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241122&max_day_obs=20241123\n", " \n", " \n", " narrativelog/messages\n", - " 166\n", - " https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-21T00%3A00%3A00&max_date_added=2024-11-22T00%3A00%3A00\n", + " 3\n", + " https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-22T00%3A00%3A00&max_date_added=2024-11-23T00%3A00%3A00\n", " \n", " \n", " efd/targets\n", @@ -4071,7 +961,7 @@ { "data": { "text/markdown": [ - "### Overview for Service: `nightreport` [1]" + "### Overview for Service: `nightreport` [0]" ], "text/plain": [ "" @@ -4090,7 +980,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241121&max_day_obs=20241122" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241122&max_day_obs=20241123" ], "text/plain": [ "" @@ -4104,7 +994,7 @@ "output_type": "stream", "text": [ "None\n", - "- Got 1 records. \n" + "- Got 0 records. \n" ] }, { @@ -4129,7 +1019,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241122&max_day_obs=20241123" ], "text/plain": [ "" @@ -4149,7 +1039,7 @@ { "data": { "text/markdown": [ - "### Overview for Service: `narrativelog` [166]" + "### Overview for Service: `narrativelog` [3]" ], "text/plain": [ "" @@ -4168,7 +1058,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-21T00%3A00%3A00&max_date_added=2024-11-22T00%3A00%3A00" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-22T00%3A00%3A00&max_date_added=2024-11-23T00%3A00%3A00" ], "text/plain": [ "" @@ -4182,7 +1072,7 @@ "output_type": "stream", "text": [ "None\n", - "- Got 166 records. \n" + "- Got 3 records. \n" ] } ], @@ -4238,8 +1128,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Elapsed time (excluding code import): 4.8 seconds\n", - "Finished 2024-11-21 23:57:28 UTC\n" + "Elapsed time (excluding code import): 3.9 seconds\n", + "Finished 2024-11-22 00:16:09 UTC\n" ] } ],