From 79cbe97d1ef35ded9da6a784333bbdb64bbf12cb Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Thu, 21 Nov 2024 17:05:51 -0700 Subject: [PATCH] tweaks to NightLog page. Fallback to default values for passed parameters with bad values --- notebooks_tsqr/ExposureDetail.ipynb | 1290 ++++---- notebooks_tsqr/NightLog.ipynb | 2630 ++++++++++++----- notebooks_tsqr/NightLog.yaml | 2 +- notebooks_tsqr/dashboard.ipynb | 56 +- python/lsst/ts/logging_and_reporting/utils.py | 37 + 5 files changed, 2731 insertions(+), 1284 deletions(-) diff --git a/notebooks_tsqr/ExposureDetail.ipynb b/notebooks_tsqr/ExposureDetail.ipynb index 1a8bf7c..c86217d 100644 --- a/notebooks_tsqr/ExposureDetail.ipynb +++ b/notebooks_tsqr/ExposureDetail.ipynb @@ -152,7 +152,7 @@ { "data": { "text/markdown": [ - "The number of exposures in this filtered result is 124" + "The number of exposures in this filtered result is 150" ], "text/plain": [ "" @@ -180,549 +180,569 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000337\n", - " 2024-11-20T07:22:54.000983\n", - " 337\n", + " CC_O_20241120_000303\n", + " 2024-11-21T08:05:23.895983\n", + " 303\n", " acq\n", " acq\n", - " BLOCK-T155\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000336\n", - " 2024-11-20T07:19:30.564991\n", - " 336\n", + " CC_O_20241120_000302\n", + " 2024-11-21T08:04:03.584007\n", + " 302\n", " acq\n", " infocus\n", - " BLOCK-T155\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000333\n", - " 2024-11-20T07:16:48.884999\n", - " 333\n", + " CC_O_20241120_000299\n", + " 2024-11-21T08:01:42.035992\n", + " 299\n", " acq\n", - " acq\n", - " BLOCK-T155\n", + " infocus\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000295\n", - " 2024-11-20T07:07:14.810986\n", - " 295\n", + " CC_O_20241120_000296\n", + " 2024-11-21T07:59:12.736017\n", + " 296\n", " acq\n", " infocus\n", - " BLOCK-T298\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000292\n", - " 2024-11-20T07:04:44.257990\n", - " 292\n", + " CC_O_20241120_000293\n", + " 2024-11-21T07:56:33.366010\n", + " 293\n", " acq\n", " infocus\n", - " BLOCK-T298\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000289\n", - " 2024-11-20T07:02:13.840017\n", - " 289\n", + " CC_O_20241120_000290\n", + " 2024-11-21T07:55:05.994004\n", + " 290\n", " acq\n", - " infocus\n", - " BLOCK-T298\n", + " acq\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000286\n", - " 2024-11-20T06:59:07.540002\n", - " 286\n", + " CC_O_20241120_000289\n", + " 2024-11-21T07:53:40.926994\n", + " 289\n", " acq\n", " acq\n", - " BLOCK-T286\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000285\n", - " 2024-11-20T06:57:19.744006\n", - " 285\n", + " CC_O_20241120_000287\n", + " 2024-11-21T07:51:28.240005\n", + " 287\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000282\n", - " 2024-11-20T06:54:43.905986\n", - " 282\n", + " CC_O_20241120_000286\n", + " 2024-11-21T07:22:08.822994\n", + " 286\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000279\n", - " 2024-11-20T06:52:08.310008\n", - " 279\n", + " CC_O_20241120_000283\n", + " 2024-11-21T07:19:29.353008\n", + " 283\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000276\n", - " 2024-11-20T06:43:13.312991\n", - " 276\n", + " CC_O_20241120_000280\n", + " 2024-11-21T07:16:54.793999\n", + " 280\n", " acq\n", - " acq\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000275\n", - " 2024-11-20T06:41:57.942005\n", - " 275\n", + " CC_O_20241120_000277\n", + " 2024-11-21T07:14:37.290983\n", + " 277\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000272\n", - " 2024-11-20T06:39:32.559008\n", - " 272\n", + " CC_O_20241120_000274\n", + " 2024-11-21T07:11:57.841999\n", + " 274\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000269\n", - " 2024-11-20T06:37:01.663991\n", - " 269\n", + " CC_O_20241120_000271\n", + " 2024-11-21T07:09:17.925989\n", + " 271\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000266\n", - " 2024-11-20T06:34:23.247992\n", - " 266\n", - " acq\n", + " CC_O_20241120_000268\n", + " 2024-11-21T07:06:42.954993\n", + " 268\n", " acq\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000265\n", - " 2024-11-20T06:33:07.852988\n", + " CC_O_20241120_000265\n", + " 2024-11-21T07:00:27.872014\n", " 265\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000262\n", - " 2024-11-20T06:30:37.300998\n", - " 262\n", + " CC_O_20241120_000264\n", + " 2024-11-21T06:59:02.229992\n", + " 264\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000259\n", - " 2024-11-20T06:28:06.378984\n", - " 259\n", + " CC_O_20241120_000261\n", + " 2024-11-21T06:56:28.132980\n", + " 261\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000256\n", - " 2024-11-20T06:22:15.168980\n", - " 256\n", + " CC_O_20241120_000258\n", + " 2024-11-21T06:53:53.817020\n", + " 258\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000253\n", - " 2024-11-20T06:17:29.144018\n", - " 253\n", + " CC_O_20241120_000255\n", + " 2024-11-21T06:51:19.141013\n", + " 255\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000250\n", - " 2024-11-20T06:14:30.984010\n", - " 250\n", + " CC_O_20241120_000252\n", + " 2024-11-21T06:48:43.774002\n", + " 252\n", " acq\n", - " acq\n", - " BLOCK-T286\n", + " infocus_closed_loop_z4_investigation\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000249\n", - " 2024-11-20T06:13:20.728990\n", - " 249\n", + " CC_O_20241120_000248\n", + " 2024-11-21T06:36:20.760008\n", + " 248\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_sitcom-1147\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000246\n", - " 2024-11-20T06:10:55.054986\n", - " 246\n", + " CC_O_20241120_000245\n", + " 2024-11-21T06:34:36.306984\n", + " 245\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_sitcom-1147\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000243\n", - " 2024-11-20T06:08:29.698986\n", - " 243\n", + " CC_O_20241120_000242\n", + " 2024-11-21T06:32:04.459004\n", + " 242\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_sitcom-1147\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000240\n", - " 2024-11-20T06:05:09.076014\n", - " 240\n", - " acq\n", + " CC_O_20241120_000239\n", + " 2024-11-21T06:28:04.198992\n", + " 239\n", " acq\n", - " BLOCK-T286\n", + " infocus_sitcom-1147\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000239\n", - " 2024-11-20T06:03:53.707000\n", - " 239\n", + " CC_O_20241120_000204\n", + " 2024-11-21T05:42:09.249018\n", + " 204\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_science_refocus\n", + " BLOCK-T155\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000236\n", - " 2024-11-20T06:01:23.260985\n", - " 236\n", + " CC_O_20241120_000201\n", + " 2024-11-21T05:39:33.739019\n", + " 201\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_science_refocus\n", + " BLOCK-T155\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000233\n", - " 2024-11-20T05:58:52.510002\n", - " 233\n", + " CC_O_20241120_000198\n", + " 2024-11-21T05:36:54.229001\n", + " 198\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_closed_loop_science_refocus\n", + " BLOCK-T155\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000228\n", - " 2024-11-20T05:25:00.402014\n", - " 228\n", + " CC_O_20241120_000195\n", + " 2024-11-21T05:20:48.932004\n", + " 195\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000225\n", - " 2024-11-20T05:22:29.952015\n", - " 225\n", + " CC_O_20241120_000194\n", + " 2024-11-21T05:18:40.958005\n", + " 194\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000222\n", - " 2024-11-20T05:19:56.455000\n", - " 222\n", + " CC_O_20241120_000193\n", + " 2024-11-21T05:16:33.014020\n", + " 193\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000219\n", - " 2024-11-20T05:17:31.022999\n", - " 219\n", + " CC_O_20241120_000192\n", + " 2024-11-21T05:13:02.741010\n", + " 192\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000216\n", - " 2024-11-20T05:15:02.164020\n", - " 216\n", + " CC_O_20241120_000191\n", + " 2024-11-21T05:10:54.800002\n", + " 191\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000213\n", - " 2024-11-20T05:03:27.715000\n", - " 213\n", + " CC_O_20241120_000190\n", + " 2024-11-21T05:08:46.842016\n", + " 190\n", " acq\n", " acq\n", - " BLOCK-T286\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000212\n", - " 2024-11-20T05:02:17.950020\n", - " 212\n", + " CC_O_20241120_000189\n", + " 2024-11-21T05:07:17.078995\n", + " 189\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000209\n", - " 2024-11-20T04:59:52.966980\n", - " 209\n", + " CC_O_20241120_000188\n", + " 2024-11-21T05:05:29.555981\n", + " 188\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " acq\n", + " BLOCK-T75\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000206\n", - " 2024-11-20T04:57:23.317017\n", - " 206\n", + " CC_O_20241120_000187\n", + " 2024-11-21T04:49:51.976999\n", + " 187\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_rotator\n", + " BLOCK-T301\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000203\n", - " 2024-11-20T04:54:48.058998\n", - " 203\n", + " CC_O_20241120_000184\n", + " 2024-11-21T04:47:10.732007\n", + " 184\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_rotator\n", + " BLOCK-T301\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000200\n", - " 2024-11-20T04:52:24.793993\n", - " 200\n", + " CC_O_20241120_000181\n", + " 2024-11-21T04:44:35.251016\n", + " 181\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_rotator\n", + " BLOCK-T301\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000197\n", - " 2024-11-20T04:50:00.505015\n", - " 197\n", + " CC_O_20241120_000178\n", + " 2024-11-21T04:41:59.922991\n", + " 178\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_rotator\n", + " BLOCK-T301\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000194\n", - " 2024-11-20T04:47:13.711995\n", - " 194\n", + " CC_O_20241120_000175\n", + " 2024-11-21T04:39:24.321984\n", + " 175\n", " acq\n", + " infocus_z4_gradient_investigation_rotator\n", + " BLOCK-T301\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000172\n", + " 2024-11-21T02:44:00.662013\n", + " 172\n", " acq\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000193\n", - " 2024-11-20T04:46:03.558000\n", - " 193\n", + " CC_O_20241120_000169\n", + " 2024-11-21T02:41:36.015000\n", + " 169\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000190\n", - " 2024-11-20T04:43:38.164985\n", - " 190\n", + " CC_O_20241120_000166\n", + " 2024-11-21T02:39:09.087010\n", + " 166\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000187\n", - " 2024-11-20T04:41:12.469014\n", - " 187\n", + " CC_O_20241120_000163\n", + " 2024-11-21T02:36:43.829983\n", + " 163\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000184\n", - " 2024-11-20T04:38:47.011988\n", - " 184\n", + " CC_O_20241120_000160\n", + " 2024-11-21T02:34:19.778984\n", + " 160\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000181\n", - " 2024-11-20T04:36:14.289018\n", - " 181\n", + " CC_O_20241120_000157\n", + " 2024-11-21T02:26:46.229000\n", + " 157\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000178\n", - " 2024-11-20T04:33:50.479015\n", - " 178\n", + " CC_O_20241120_000154\n", + " 2024-11-21T02:24:21.057991\n", + " 154\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000175\n", - " 2024-11-20T04:31:19.718015\n", - " 175\n", + " CC_O_20241120_000151\n", + " 2024-11-21T02:10:18.498006\n", + " 151\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000172\n", - " 2024-11-20T04:28:37.361020\n", - " 172\n", + " CC_O_20241120_000148\n", + " 2024-11-21T02:07:57.820998\n", + " 148\n", " acq\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000145\n", + " 2024-11-21T02:05:17.019014\n", + " 145\n", " acq\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000171\n", - " 2024-11-20T04:27:27.125995\n", - " 171\n", + " CC_O_20241120_000142\n", + " 2024-11-21T02:02:36.622982\n", + " 142\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000168\n", - " 2024-11-20T04:25:01.753016\n", - " 168\n", + " CC_O_20241120_000139\n", + " 2024-11-21T01:59:56.123993\n", + " 139\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000165\n", - " 2024-11-20T04:22:36.009007\n", - " 165\n", + " CC_O_20241120_000136\n", + " 2024-11-21T01:57:15.330981\n", + " 136\n", " acq\n", - " infocus\n", - " BLOCK-T286\n", + " infocus_z4_gradient_investigation_elevation\n", + " BLOCK-T300\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000162\n", - " 2024-11-20T04:20:08.312000\n", - " 162\n", + " CC_O_20241120_000133\n", + " 2024-11-21T01:25:58.201995\n", + " 133\n", + " acq\n", " acq\n", - " infocus\n", " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000159\n", - " 2024-11-20T04:17:44.456011\n", - " 159\n", + " CC_O_20241120_000132\n", + " 2024-11-21T01:24:37.986981\n", + " 132\n", " acq\n", " infocus\n", " BLOCK-T249\n", " 0.0\n", " \n", " \n", - " unknown\n", - " CC_O_20241119_000156\n", - " 2024-11-20T04:15:20.560996\n", - " 156\n", + " good\n", + " CC_O_20241120_000129\n", + " 2024-11-21T01:22:02.669980\n", + " 129\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -730,29 +750,29 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000153\n", - " 2024-11-20T04:11:45.020986\n", - " 153\n", + " CC_O_20241120_000126\n", + " 2024-11-21T01:19:27.249016\n", + " 126\n", " acq\n", - " acq\n", - " BLOCK-T286\n", + " infocus\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000152\n", - " 2024-11-20T02:55:19.320010\n", - " 152\n", - " acq\n", + " CC_O_20241120_000123\n", + " 2024-11-21T01:16:51.536001\n", + " 123\n", " acq\n", - " BLOCK-T286\n", + " infocus\n", + " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000151\n", - " 2024-11-20T02:54:08.988989\n", - " 151\n", + " CC_O_20241120_000120\n", + " 2024-11-21T01:14:16.072994\n", + " 120\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -760,9 +780,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000148\n", - " 2024-11-20T02:52:26.491013\n", - " 148\n", + " CC_O_20241120_000117\n", + " 2024-11-21T01:11:48.057985\n", + " 117\n", " acq\n", " acq\n", " BLOCK-T249\n", @@ -770,9 +790,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000147\n", - " 2024-11-20T02:51:16.123984\n", - " 147\n", + " CC_O_20241120_000116\n", + " 2024-11-21T01:10:27.814002\n", + " 116\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -780,9 +800,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000144\n", - " 2024-11-20T02:48:50.423990\n", - " 144\n", + " CC_O_20241120_000113\n", + " 2024-11-21T01:07:52.389981\n", + " 113\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -790,9 +810,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000141\n", - " 2024-11-20T02:46:19.914004\n", - " 141\n", + " CC_O_20241120_000110\n", + " 2024-11-21T01:05:16.771996\n", + " 110\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -800,9 +820,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000138\n", - " 2024-11-20T02:43:54.574981\n", - " 138\n", + " CC_O_20241120_000107\n", + " 2024-11-21T01:02:36.295015\n", + " 107\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -810,9 +830,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000135\n", - " 2024-11-20T02:41:29.012986\n", - " 135\n", + " CC_O_20241120_000104\n", + " 2024-11-21T01:00:00.719999\n", + " 104\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -820,19 +840,19 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000132\n", - " 2024-11-20T02:35:58.294000\n", - " 132\n", - " acq\n", + " CC_O_20241120_000101\n", + " 2024-11-21T00:57:34.324012\n", + " 101\n", " acq\n", + " infocus\n", " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000131\n", - " 2024-11-20T02:34:41.992020\n", - " 131\n", + " CC_O_20241120_000098\n", + " 2024-11-21T00:55:00.308995\n", + " 98\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -840,9 +860,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000128\n", - " 2024-11-20T02:32:11.569017\n", - " 128\n", + " CC_O_20241120_000095\n", + " 2024-11-21T00:51:40.388011\n", + " 95\n", " acq\n", " infocus\n", " BLOCK-T249\n", @@ -850,19 +870,19 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000125\n", - " 2024-11-20T02:29:41.249012\n", - " 125\n", + " CC_O_20241120_000092\n", + " 2024-11-21T00:49:43.188003\n", + " 92\n", + " acq\n", " acq\n", - " infocus\n", " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000122\n", - " 2024-11-20T02:28:14.682996\n", - " 122\n", + " CC_O_20241120_000091\n", + " 2024-11-21T00:19:10.341015\n", + " 91\n", " acq\n", " acq\n", " BLOCK-T249\n", @@ -870,279 +890,339 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000121\n", - " 2024-11-20T02:26:59.513981\n", - " 121\n", + " CC_O_20241120_000090\n", + " 2024-11-21T00:16:59.525993\n", + " 90\n", + " acq\n", " acq\n", - " infocus\n", " BLOCK-T249\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000118\n", - " 2024-11-20T02:24:41.175001\n", - " 118\n", + " CC_O_20241120_000089\n", + " 2024-11-21T00:13:35.976986\n", + " 89\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000115\n", - " 2024-11-20T02:22:22.779011\n", - " 115\n", + " CC_O_20241120_000088\n", + " 2024-11-21T00:12:38.866011\n", + " 88\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000112\n", - " 2024-11-20T02:16:42.581982\n", - " 112\n", + " CC_O_20241120_000087\n", + " 2024-11-21T00:11:49.571005\n", + " 87\n", " acq\n", - " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000111\n", - " 2024-11-20T02:15:32.520001\n", - " 111\n", + " CC_O_20241120_000086\n", + " 2024-11-21T00:11:05.565008\n", + " 86\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000108\n", - " 2024-11-20T02:13:13.233007\n", - " 108\n", + " CC_O_20241120_000085\n", + " 2024-11-21T00:09:15.326016\n", + " 85\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000105\n", - " 2024-11-20T02:10:58.198012\n", - " 105\n", + " CC_O_20241120_000084\n", + " 2024-11-21T00:08:29.081005\n", + " 84\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000102\n", - " 2024-11-20T02:09:31.741994\n", - " 102\n", + " CC_O_20241120_000083\n", + " 2024-11-21T00:07:43.521002\n", + " 83\n", " acq\n", - " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", - " junk\n", - " CC_O_20241119_000101\n", - " 2024-11-20T02:04:54.481015\n", - " 101\n", + " unknown\n", + " CC_O_20241120_000082\n", + " 2024-11-21T00:06:55.168016\n", + " 82\n", " acq\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000081\n", + " 2024-11-21T00:06:09.401014\n", + " 81\n", " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000080\n", - " 2024-11-20T01:15:46.564999\n", + " CC_O_20241120_000080\n", + " 2024-11-21T00:05:25.672987\n", " 80\n", " acq\n", - " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000079\n", - " 2024-11-20T01:13:15.540994\n", + " CC_O_20241120_000079\n", + " 2024-11-21T00:04:35.786997\n", " 79\n", " acq\n", - " infocus_closed_loop_science_refocus\n", - " BLOCK-T155\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000076\n", - " 2024-11-20T01:10:50.114988\n", - " 76\n", + " CC_O_20241120_000078\n", + " 2024-11-21T00:03:52.332999\n", + " 78\n", " acq\n", - " infocus_closed_loop_science_refocus\n", - " BLOCK-T155\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000073\n", - " 2024-11-20T01:08:20.268003\n", - " 73\n", + " CC_O_20241120_000077\n", + " 2024-11-21T00:03:08.387994\n", + " 77\n", " acq\n", - " infocus_closed_loop_science_refocus\n", - " BLOCK-T155\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000070\n", - " 2024-11-20T01:04:42.849991\n", - " 70\n", + " CC_O_20241120_000076\n", + " 2024-11-21T00:02:24.926995\n", + " 76\n", " acq\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000075\n", + " 2024-11-21T00:02:03.063987\n", + " 75\n", " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000069\n", - " 2024-11-20T00:57:55.919992\n", - " 69\n", + " CC_O_20241120_000074\n", + " 2024-11-21T00:01:44.014000\n", + " 74\n", " acq\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000073\n", + " 2024-11-21T00:01:24.964013\n", + " 73\n", " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000068\n", - " 2024-11-20T00:55:52.463007\n", - " 68\n", + " CC_O_20241120_000072\n", + " 2024-11-21T00:01:06.016017\n", + " 72\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000065\n", - " 2024-11-20T00:53:22.086997\n", - " 65\n", + " CC_O_20241120_000071\n", + " 2024-11-21T00:00:22.548017\n", + " 71\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000062\n", - " 2024-11-20T00:50:51.545991\n", - " 62\n", + " CC_O_20241120_000070\n", + " 2024-11-21T00:00:03.537016\n", + " 70\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000059\n", - " 2024-11-20T00:48:21.118000\n", - " 59\n", + " CC_O_20241120_000069\n", + " 2024-11-20T23:59:44.754016\n", + " 69\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000056\n", - " 2024-11-20T00:45:50.829014\n", - " 56\n", + " CC_O_20241120_000068\n", + " 2024-11-20T23:59:26.296019\n", + " 68\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000053\n", - " 2024-11-20T00:42:21.622987\n", - " 53\n", + " CC_O_20241120_000067\n", + " 2024-11-20T23:59:07.624987\n", + " 67\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000050\n", - " 2024-11-20T00:40:02.451986\n", - " 50\n", + " CC_O_20241120_000066\n", + " 2024-11-20T23:58:27.798006\n", + " 66\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000047\n", - " 2024-11-20T00:37:42.747011\n", - " 47\n", + " CC_O_20241120_000064\n", + " 2024-11-20T23:56:38.498981\n", + " 64\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000044\n", - " 2024-11-20T00:36:16.352991\n", - " 44\n", + " CC_O_20241120_000063\n", + " 2024-11-20T23:56:21.414020\n", + " 63\n", " acq\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000062\n", + " 2024-11-20T23:56:02.738000\n", + " 62\n", " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000043\n", - " 2024-11-20T00:22:48.950008\n", - " 43\n", + " CC_O_20241120_000061\n", + " 2024-11-20T23:55:45.692991\n", + " 61\n", " acq\n", - " infocus\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000040\n", - " 2024-11-20T00:20:31.603982\n", - " 40\n", + " CC_O_20241120_000060\n", + " 2024-11-20T23:55:28.076993\n", + " 60\n", " acq\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000059\n", + " 2024-11-20T23:54:45.695006\n", + " 59\n", " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000039\n", - " 2024-11-20T00:18:37.336003\n", - " 39\n", + " CC_O_20241120_000058\n", + " 2024-11-20T23:54:27.686011\n", + " 58\n", " acq\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000057\n", + " 2024-11-20T23:54:10.878015\n", + " 57\n", " acq\n", - " BLOCK-T249\n", + " comcam_twilight_flat\n", + " BLOCK-T215\n", " 0.0\n", " \n", " \n", " unknown\n", - " CC_O_20241119_000038\n", - " 2024-11-20T00:08:15.313990\n", - " 38\n", + " CC_O_20241120_000056\n", + " 2024-11-20T23:53:53.971006\n", + " 56\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1150,9 +1230,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000037\n", - " 2024-11-20T00:06:22.913996\n", - " 37\n", + " CC_O_20241120_000055\n", + " 2024-11-20T23:53:35.885004\n", + " 55\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1160,9 +1240,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000036\n", - " 2024-11-20T00:04:38.121002\n", - " 36\n", + " CC_O_20241120_000054\n", + " 2024-11-20T23:52:55.034009\n", + " 54\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1170,9 +1250,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000035\n", - " 2024-11-20T00:03:27.467996\n", - " 35\n", + " CC_O_20241120_000053\n", + " 2024-11-20T23:52:38.325993\n", + " 53\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1180,9 +1260,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000034\n", - " 2024-11-20T00:02:15.770014\n", - " 34\n", + " CC_O_20241120_000052\n", + " 2024-11-20T23:52:20.311003\n", + " 52\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1190,9 +1270,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000033\n", - " 2024-11-20T00:00:41.967988\n", - " 33\n", + " CC_O_20241120_000051\n", + " 2024-11-20T23:52:03.663980\n", + " 51\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1200,9 +1280,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000032\n", - " 2024-11-19T23:58:18.065007\n", - " 32\n", + " CC_O_20241120_000050\n", + " 2024-11-20T23:51:46.625006\n", + " 50\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1210,9 +1290,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000031\n", - " 2024-11-19T23:57:18.418981\n", - " 31\n", + " CC_O_20241120_000049\n", + " 2024-11-20T23:51:07.561005\n", + " 49\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1220,9 +1300,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000030\n", - " 2024-11-19T23:56:25.411987\n", - " 30\n", + " CC_O_20241120_000048\n", + " 2024-11-20T23:50:50.956991\n", + " 48\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1230,9 +1310,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000029\n", - " 2024-11-19T23:55:31.710005\n", - " 29\n", + " CC_O_20241120_000047\n", + " 2024-11-20T23:50:33.510013\n", + " 47\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1240,9 +1320,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000028\n", - " 2024-11-19T23:54:40.464980\n", - " 28\n", + " CC_O_20241120_000046\n", + " 2024-11-20T23:50:14.087990\n", + " 46\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1250,9 +1330,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000027\n", - " 2024-11-19T23:53:52.558018\n", - " 27\n", + " CC_O_20241120_000045\n", + " 2024-11-20T23:49:57.451991\n", + " 45\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1260,9 +1340,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000026\n", - " 2024-11-19T23:53:03.724004\n", - " 26\n", + " CC_O_20241120_000044\n", + " 2024-11-20T23:49:16.451006\n", + " 44\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1270,9 +1350,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000025\n", - " 2024-11-19T23:52:16.213017\n", - " 25\n", + " CC_O_20241120_000043\n", + " 2024-11-20T23:47:25.299004\n", + " 43\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1280,9 +1360,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000024\n", - " 2024-11-19T23:51:23.668984\n", - " 24\n", + " CC_O_20241120_000042\n", + " 2024-11-20T23:46:23.498012\n", + " 42\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1290,9 +1370,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000023\n", - " 2024-11-19T23:50:09.065004\n", - " 23\n", + " CC_O_20241120_000041\n", + " 2024-11-20T23:45:22.695002\n", + " 41\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1300,9 +1380,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000022\n", - " 2024-11-19T23:49:08.415001\n", - " 22\n", + " CC_O_20241120_000040\n", + " 2024-11-20T23:44:21.399981\n", + " 40\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1310,9 +1390,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000021\n", - " 2024-11-19T23:48:25.058004\n", - " 21\n", + " CC_O_20241120_000039\n", + " 2024-11-20T23:43:21.094012\n", + " 39\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1320,9 +1400,9 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000020\n", - " 2024-11-19T23:47:35.469981\n", - " 20\n", + " CC_O_20241120_000038\n", + " 2024-11-20T23:42:18.775983\n", + " 38\n", " acq\n", " comcam_twilight_flat\n", " BLOCK-T215\n", @@ -1330,8 +1410,188 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000019\n", - " 2024-11-19T22:23:21.979990\n", + " CC_O_20241120_000037\n", + " 2024-11-20T22:35:58.032015\n", + " 37\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000036\n", + " 2024-11-20T22:35:40.100992\n", + " 36\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000035\n", + " 2024-11-20T22:35:22.162002\n", + " 35\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000034\n", + " 2024-11-20T22:34:35.098005\n", + " 34\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000033\n", + " 2024-11-20T22:34:17.152015\n", + " 33\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000032\n", + " 2024-11-20T22:33:59.207996\n", + " 32\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000031\n", + " 2024-11-20T22:33:22.273013\n", + " 31\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000030\n", + " 2024-11-20T22:33:04.340018\n", + " 30\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000029\n", + " 2024-11-20T22:32:46.399017\n", + " 29\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000028\n", + " 2024-11-20T22:32:36.135987\n", + " 28\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000027\n", + " 2024-11-20T22:32:28.093987\n", + " 27\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000026\n", + " 2024-11-20T22:32:20.071982\n", + " 26\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000025\n", + " 2024-11-20T22:31:50.197994\n", + " 25\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000024\n", + " 2024-11-20T22:31:42.167983\n", + " 24\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000023\n", + " 2024-11-20T22:31:34.129001\n", + " 23\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000022\n", + " 2024-11-20T22:31:07.118002\n", + " 22\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000021\n", + " 2024-11-20T22:30:59.092981\n", + " 21\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000020\n", + " 2024-11-20T22:30:51.059993\n", + " 20\n", + " acq\n", + " daytime_light_leak\n", + " BLOCK-T296\n", + " 0.0\n", + " \n", + " \n", + " unknown\n", + " CC_O_20241120_000019\n", + " 2024-11-20T21:59:47.581989\n", " 19\n", " acq\n", " tmacheckout\n", @@ -1340,8 +1600,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000018\n", - " 2024-11-19T22:23:09.046003\n", + " CC_O_20241120_000018\n", + " 2024-11-20T21:59:34.657014\n", " 18\n", " acq\n", " tmacheckout\n", @@ -1350,8 +1610,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000017\n", - " 2024-11-19T22:22:56.099020\n", + " CC_O_20241120_000017\n", + " 2024-11-20T21:59:21.707014\n", " 17\n", " acq\n", " tmacheckout\n", @@ -1360,8 +1620,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000016\n", - " 2024-11-19T22:22:08.834984\n", + " CC_O_20241120_000016\n", + " 2024-11-20T21:56:58.217991\n", " 16\n", " acq\n", " tmacheckout\n", @@ -1370,8 +1630,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000015\n", - " 2024-11-19T22:21:55.898019\n", + " CC_O_20241120_000015\n", + " 2024-11-20T21:56:45.271009\n", " 15\n", " acq\n", " tmacheckout\n", @@ -1380,8 +1640,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000014\n", - " 2024-11-19T22:21:42.967009\n", + " CC_O_20241120_000014\n", + " 2024-11-20T21:56:32.337987\n", " 14\n", " acq\n", " tmacheckout\n", @@ -1390,8 +1650,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000013\n", - " 2024-11-19T22:21:08.263999\n", + " CC_O_20241120_000013\n", + " 2024-11-20T21:54:31.906011\n", " 13\n", " acq\n", " tmacheckout\n", @@ -1400,8 +1660,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000012\n", - " 2024-11-19T22:20:55.317982\n", + " CC_O_20241120_000012\n", + " 2024-11-20T21:54:18.964983\n", " 12\n", " acq\n", " tmacheckout\n", @@ -1410,8 +1670,8 @@ " \n", " \n", " unknown\n", - " CC_O_20241119_000011\n", - " 2024-11-19T22:20:42.383994\n", + " CC_O_20241120_000011\n", + " 2024-11-20T21:54:06.018000\n", " 11\n", " acq\n", " tmacheckout\n", @@ -1496,7 +1756,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "https://usdf-rsp-dev.slac.stanford.edu: 2024-11-19 to 2024-11-20 [2500] exposurelog endpoints=['instruments', 'exposures', 'messages']\n", + "https://usdf-rsp-dev.slac.stanford.edu: 2024-11-20 to 2024-11-21 [2500] exposurelog endpoints=['instruments', 'exposures', 'messages']\n", "instrument='LSSTComCam', science_program=None, observation_reason=None, observation_type='acq'\n" ] } @@ -1517,7 +1777,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'LSSTComCamSim': 0, 'LATISS': 0, 'LSSTComCam': 422, 'LSSTCam': 0}\n" + "{'LSSTComCamSim': 0, 'LATISS': 0, 'LSSTComCam': 358, 'LSSTCam': 0}\n" ] } ], diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index e73bfe6..c3070ac 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -6,8 +6,7 @@ "metadata": {}, "source": [ "# About this Page\n", - "See [**slack #ts-logging**](https://rubin-obs.slack.com/archives/C07QJMQKJE6)\n", - "channel for discussion about this page. There you can report problems, questions, and requests for changes.\n", + "See the slack **#ts-logging** channel for discussion about this page. There you can report problems, questions, and requests for changes. *What are we missing?*\n", "\n", "## What is new in this app?(newest change at top of list)\n", "- Do not show Time Loss info unless it is greater than zero\n", @@ -89,26 +88,35 @@ { "cell_type": "code", "execution_count": 4, - "id": "ec883835-42d3-40ae-a014-8b89befe4263", - "metadata": { - "jupyter": { - "source_hidden": true + "id": "5d3008ec-1400-48e5-a35c-539869ff3ecf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'day_obs': 'TODAY', 'number_of_days': 1, 'verbose': False} \n" + ] } - }, - "outputs": [], + ], "source": [ - "date = ut.get_datetime_from_dayobs_str(day_obs)\n", + "# Validate parameters, return usable ones\n", + "usable, error = ut.fallback_parameters(day_obs, number_of_days, verbose)\n", + "if error:\n", + " print(error)\n", + "print(usable, error)\n", + "\n", + "date = ut.get_datetime_from_dayobs_str(usable['day_obs'])\n", "# date: is EXCLUSIVE (upto, but not including)\n", - "days = int(number_of_days)\n", + "days = usable['number_of_days']\n", + "verbose = usable['verbose']\n", "\n", "# Thus: [min_day_obs,max_day_obs)\n", "# Format: string, YYYY-MM-DD\n", "min_date = date - dt.timedelta(days=days - 1)\n", "max_date = date + dt.timedelta(days=1)\n", "min_day_obs = min_date.strftime(\"%Y-%m-%d\") # Inclusive\n", - "max_day_obs = max_date.strftime(\"%Y-%m-%d\") # prep for Exclusive\n", - "\n", - "verbose = (verbose == \"true\") \n" + "max_day_obs = max_date.strftime(\"%Y-%m-%d\") # prep for Exclusive" ] }, { @@ -150,7 +158,7 @@ { "data": { "text/markdown": [ - "# Showing data for 2024-11-20 to 2024-11-21 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" + "# Showing data for 2024-11-21 to 2024-11-22 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" ], "text/plain": [ "" @@ -162,7 +170,7 @@ { "data": { "text/markdown": [ - "Most recently run on: **2024-11-20 22:12:27**" + "Most recently run on: **2024-11-21 23:57:27**" ], "text/plain": [ "" @@ -207,25 +215,58 @@ "metadata": {}, "source": [ "## Night Report BETA \n", - "\n", - "Word wrap instead of chopping words at end of line?\n", - "" + "Possible Changes\n", + "\n", + "- Word wrap instead of chopping words at end of line?\n", + "- (Add summary from AI)" ] }, { "cell_type": "code", "execution_count": 7, "id": "10", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [ { "data": { "text/markdown": [ - "No nightreport records found 2024-11-20 to 2024-11-21. " + "### Telescope: Simonyi" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "```\n", + "====M1M3 work log==\n", + "Bump test done around 16:48pm CLT/19:48pm UTC, 104, 227, 409 was failed initially. However, 104 and 227 passed after the second attempt while FA449 is failed with 2nd attempt. Petr mentioned that 409 is marginal but still fine. \n", + "\n", + "=== Daytime work log===\n", + "We started warmup hexapod around 14pmCLT/17pm UTC, however we had an issue with warmup hexapod scripts. Hexapods moved without issues with EUI and run_command.py. Around 14:43pm CL/17:43pm UTC, we had an issue with GIS, so M1M3, Hexapods in fault (OBS-701). It was cleared then we closed the mirror cover for dome handover. Dome handover was completed without any issue at 15:29 pm UTC/18:29 pm UTC. We found that the even after moving hexapod with run_command.py inPosition for both hexapods seem not working. This can be a blocker for night operations, we pinged people in charge. 17:30pm/20:30 pm, Te-wei roll-backed the fix happened earlier today (see deatils in OBS-702). Then warming-up hexapods worked well without an issue. At 17:53pm CLT/20:53pm CLT we turned on LaserTracker to check it worked mostly fine except one MTPtg fault while tracking. Comcam checkout, Simonyi on Sky checkout (BLOCK-T245), and M2 and Camera alignment (BLOCK-T88) are all done. 20:28 pm CLT/23:23 pm UTC, we had an issue with GIS heartbeat flicker and almost all system were fault due to hard stop engaged (OBS-701). \n", + "\n", + "\n", + "\n", + "We started the TMA checkout around 19:15pm and all passed except that a known issue with turning on the elevation. \n", + "\n", + "Also Chris S. and Paulo L. joined tonight's operation and thanks for supports from Tiago R, Te-wei T, and others.\n", + "```" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "Telescope Status: " ], "text/plain": [ "" @@ -237,7 +278,7 @@ { "data": { "text/markdown": [ - "Used [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241120&max_day_obs=20241121)" + "*Authors: Kevin Fanning, Yijung Kang, Karla Pena, Paulina Venegas, Bryce Kalmbach, Andrew Connolly, Parker Fagrelius, Bruno Quint, Keith Bechtol*" ], "text/plain": [ "" @@ -273,7 +314,7 @@ { "data": { "text/markdown": [ - "**Almanac for the observing night starting 2024-11-20**" + "**Almanac for the observing night starting 2024-11-21**" ], "text/plain": [ "" @@ -287,80 +328,80 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", "
Sun Set2024-11-20 23:17:44Sun Set2024-11-21 23:18:34
Evening Civil Twilight2024-11-20 23:48:28(-6 degrees)Evening Civil Twilight2024-11-21 23:49:23(-6 degrees)
Evening Nautical Twilight2024-11-21 00:20:27(-12 degrees)Evening Nautical Twilight2024-11-22 00:21:27(-12 degrees)
Evening Astronomical Twilight2024-11-21 00:54:12(-18 degrees)Evening Astronomical Twilight2024-11-22 00:55:19(-18 degrees)
Moon Rise2024-11-21 04:26:54Solar Midnight2024-11-22 04:29:01
Solar Midnight2024-11-21 04:28:45Moon Rise2024-11-22 05:01:56
Morning Astronomical Twilight2024-11-21 08:03:18(-18 degrees)Morning Astronomical Twilight2024-11-22 08:02:43(-18 degrees)
Morning Nautical Twilight2024-11-21 08:37:03(-12 degrees)Morning Nautical Twilight2024-11-22 08:36:35(-12 degrees)
Morning Civil Twilight2024-11-21 09:09:02(-6 degrees)Morning Civil Twilight2024-11-22 09:08:39(-6 degrees)
Sun Rise2024-11-21 09:39:47Sun Rise2024-11-22 09:39:28
Moon Set2024-11-21 14:51:23Moon Set2024-11-22 15:51:00
Moon Illumination68%(% illuminated)Moon Illumination59%(% illuminated)
UTCUTC
\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -386,18 +427,18 @@ "id": "14", "metadata": {}, "source": [ - "(content not yet defined in storyboard)" + "(content not yet defined in storyboard)\n", + "\n", + "Possible features\n", + "\n", + "- plot_observation_gap_detail" ] }, { "cell_type": "code", "execution_count": 9, "id": "15", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ "# Plot Observation (Exposure) gaps\n", @@ -410,7 +451,7 @@ "id": "38", "metadata": {}, "source": [ - "## DDV DRAFT" + "## DDV BETA" ] }, { @@ -446,9 +487,9 @@ "id": "9d51f937-37e7-45da-87cb-2319fd76a1cc", "metadata": {}, "source": [ - "## Time Accounting DRAFT\n", + "## Time Accounting BETA\n", "TODO:\n", - "- Time-loss for Fault and Weather\n", + "- Time-loss for Fault and Weather (very rarely in narrativelog/messages)\n", "- SlewTime from TMAEvent (or similar)\n", "\n", "SlewTime (and probably others) from EFD are only EXPECTED (topic=\"lsst.sal.Scheduler.logevent_target\"), not ACTUAL.\n", @@ -490,11 +531,11 @@ " \n", " \n", " Total Night\n", - " 7:09:05\n", + " 7:07:23\n", " \n", " \n", " Total Exposure\n", - " 0:00:49\n", + " 0:00:16\n", " \n", " \n", " Slew time(1)\n", @@ -502,7 +543,7 @@ " \n", " \n", " Readout time(2)\n", - " 0:00:45\n", + " 0:00:31\n", " \n", " \n", " Time loss to fault\n", @@ -514,11 +555,11 @@ " \n", " \n", " Idle time\n", - " 7:07:30\n", + " 7:06:36\n", " \n", " \n", " Number of exposures\n", - " 19\n", + " 13\n", " \n", " \n", " Mean readout time\n", @@ -538,14 +579,14 @@ ], "text/plain": [ " LSSTComCam\n", - "Total Night 7:09:05\n", - "Total Exposure 0:00:49\n", + "Total Night 7:07:23\n", + "Total Exposure 0:00:16\n", "Slew time(1) 0:00:00\n", - "Readout time(2) 0:00:45\n", + "Readout time(2) 0:00:31\n", "Time loss to fault 0\n", "Time loss to weather 0\n", - "Idle time 7:07:30\n", - "Number of exposures 19\n", + "Idle time 7:06:36\n", + "Number of exposures 13\n", "Mean readout time 0:00:02\n", "Number of slews(1) 0\n", "Mean Slew time(1) 0:00:00" @@ -577,8 +618,11 @@ "metadata": {}, "source": [ "## Jira Tickets BETA \n", + "\n", + "Possible Changes\n", + "\n", "- Observing Operations - OBS\n", - "- TODO: better tickets using Jira API" + "- TODO: better tickets using Jira API to get selected fields (which) from tickets" ] }, { @@ -587,6 +631,42 @@ "id": "18", "metadata": {}, "outputs": [ + { + "data": { + "text/markdown": [ + "- [obs-662](https://rubinobs.atlassian.net/browse/obs-662)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- [OBS-701](https://rubinobs.atlassian.net/browse/OBS-701)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- [obs-653](https://rubinobs.atlassian.net/browse/obs-653)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "text/markdown": [ @@ -602,7 +682,19 @@ { "data": { "text/markdown": [ - "- [OBS-695](https://rubinobs.atlassian.net/browse/OBS-695)" + "- [OBS-699](https://rubinobs.atlassian.net/browse/OBS-699)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" ], "text/plain": [ "" @@ -614,7 +706,7 @@ { "data": { "text/markdown": [ - "- [OBS-694](https://rubinobs.atlassian.net/browse/OBS-694)" + "- [OBS-698](https://rubinobs.atlassian.net/browse/OBS-698)" ], "text/plain": [ "" @@ -626,7 +718,7 @@ { "data": { "text/markdown": [ - "- [OBS-696](https://rubinobs.atlassian.net/browse/OBS-696)" + "- [OBS-700](https://rubinobs.atlassian.net/browse/OBS-700)" ], "text/plain": [ "" @@ -646,16 +738,12 @@ "cell_type": "code", "execution_count": 13, "id": "19", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [ { "data": { "text/markdown": [ - "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241120&max_day_obs=20241121) in `confluence_url`." + "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241121&max_day_obs=20241122) in `confluence_url`." ], "text/plain": [ "" @@ -731,7 +819,7 @@ { "data": { "text/markdown": [ - "**LSSTComCam: 19 total exposures**" + "**LSSTComCam: 13 total exposures**" ], "text/plain": [ "" @@ -752,30 +840,30 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
DetailgoodquestionablejunkunknownDetailgoodquestionablejunkunknown
acq0009acq0003
bias00010bias00010
\n" @@ -799,30 +887,30 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
DetailgoodquestionablejunkunknownDetailgoodquestionablejunkunknown
comcam-usdf-checkout00010comcam-usdf-checkout00010
tmacheckout0009tmacheckout0003
\n" @@ -846,30 +934,30 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
DetailgoodquestionablejunkunknownDetailgoodquestionablejunkunknown
BLOCK-T24600010BLOCK-T24600010
BLOCK-T2500009BLOCK-T2500003
\n" @@ -912,10 +1000,12 @@ "metadata": {}, "source": [ "## Narrative Log BETA \n", - "\n", + "\n", + "Possible changes\n", + "\n", + "- Group messages by hour? Might be great when there many simple messages minutes apart. Less so when tracebacks are included.\n", "- Truncate text messages to N chars (ending with \"...\" as link if truncated, link goes to page with full message record including text and all attributes)\n", - "- Detect some kinds of \"blocks\" and render in smaller text (+ other format changes) \n", - "" + "- Detect some kinds of \"blocks\" and render in smaller text (+ other format changes)" ] }, { @@ -933,9 +1023,1343 @@ { "data": { "text/markdown": [ - "- **2024-11-20 10:37** **MainTel** \n", "\n", - "
TMA is parked. M2 recovered. Finishing the remainder of the shut down procedures.
\n" + "\n", + "h3. Error/Failure explanation:\r\n", + "We had failures of the Camera cable wrap, axes timeouts, and frequent communications errors that impacted our control software. We lost over 2 hours of sky time struggling with these issues. \r\n", + "h3. What were you doing when the failure occurred?:\r\n", + "Tonight we had several more instances of control system crashes due to timeouts. In addition to these incidents, the CCW would not follow the rotator, stall out during short or tracking movements, and then trigger a fault in the rotator following. We also had several recurring axes timeouts that required us to attempt initializing our control software several times before we could move. \r\n", + "h3. Detailed description of the steps followed that caused the error/failure:\r\n", + "Here is a list of some timestamps that indicate the approximate times of communications failures. These demonstrate startTracking command timeouts, CCW timeouts, and axes timeouts. \r\n", + "\r\n", + "Example 1:\r\n", + "2024/11/21 02:11:02 TAI\r\n", + "\r\n", + "Error MTMount replied error for startTracking command (1732828411) Error no:1 message:Failed: Command with id 447721 has timed \r\n", + "\r\n", + "Example 2: 04:32:41\r\n", + "The cable cable wrap faulted the MTMount. The rotator was not commanded anywhere, but the CCW just decided to quit. \r\n", + "\r\n", + "Example 3:\r\n", + "2024/11/21 11:02:09 TAI\r\n", + "\r\n", + "\r\n", + "Command BothAxesResetAlarm(sequence_id=1036635, command_code=, source=, timestamp=1732186929.5137012) failed, waiting 60s and retrying.\r\n", + "\r\n", + "
Traceback (most recent call last):\r\n",
+       "\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 886, in enable_devices\r\n",
+       "\r\n",
+       "    await self.send_command(command, do_lock=True, timeout=timeout)\r\n",
+       "\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1173, in send_command\r\n",
+       "\r\n",
+       "    return await self._basic_send_command(command=command, timeout=timeout)\r\n",
+       "\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1259, in _basic_send_command\r\n",
+       "\r\n",
+       "    await asyncio.wait_for(\r\n",
+       "\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 489, in wait_for\r\n",
+       "\r\n",
+       "    return fut.result()\r\n",
+       "\r\n",
+       "           ^^^^^^^^^^^^\r\n",
+       "\r\n",
+       "lsst.ts.salobj.base.ExpectedError: Command with id 1036635 has timed out after not receiving finished
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- Link: [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 11:30** **MainTel** \n", + "\n", + "
Moving to high elevation to open mirror covers as requested for the daytime. \r\n",
+       "\r\n",
+       "\r\n",
+       "Lol. JK. We didn't do that, because it was likely an old request.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 11:06** **MainTel** \n", + "\n", + "
Moving TMA in elevation.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 11:05** **MainTel** \n", + "\n", + "
Recovered telescope to perform BLOCK-240.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "\n", + "\n", + "TMA timing our frequently while trying to ENABLE MTMount:\r\n", + "2024/11/21 11:02:09 TAI\r\n", + "\r\n", + "\r\n", + "Command BothAxesResetAlarm(sequence_id=1036635, command_code=<CommandCode.BOTH_AXES_RESET_ALARM: 37>, source=<Source.CSC: 1>, timestamp=1732186929.5137012) failed, waiting 60s and retrying.\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 886, in enable_devices\r\n",
+       "    await self.send_command(command, do_lock=True, timeout=timeout)\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1173, in send_command\r\n",
+       "    return await self._basic_send_command(command=command, timeout=timeout)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mtmount/mtmount_csc.py", line 1259, in _basic_send_command\r\n",
+       "    await asyncio.wait_for(\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 489, in wait_for\r\n",
+       "    return fut.result()\r\n",
+       "           ^^^^^^^^^^^^\r\n",
+       "lsst.ts.salobj.base.ExpectedError: Command with id 1036635 has timed out after not receiving finished\r\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "\n", + "\n", + "BLOCK-231 data was collected. Script salindex where GIS was triggered was 104862:\r\n", + "\r\n", + "\r\n", + "2024/11/21 10:42:10 TAI\r\n", + "\r\n", + "\r\n", + "Error in run\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2705, in m1m3_booster_valve\r\n",
+       "    yield\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2315, in move_p2p_azel\r\n",
+       "    await self.process_as_completed(tasks)\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1137, in process_as_completed\r\n",
+       "    raise e\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1134, in process_as_completed\r\n",
+       "    ret_val = await res\r\n",
+       "              ^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
+       "    return f.result()  # May raise f.exception().\r\n",
+       "           ^^^^^^^^^^\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 510, in check_component_state\r\n",
+       "    raise RuntimeError(\r\n",
+       "RuntimeError: mtm1m3 state is &lt;State.FAULT: 3&gt;, expected &lt;State.ENABLED: 2&gt;\r\n",
+       "\r\n",
+       "\r\n",
+       "During handling of the above exception, another exception occurred:\r\n",
+       "\r\n",
+       "\r\n",
+       "Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
+       "    await self._run_task\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
+       "    await self.run_block()\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
+       "    await self.mtcs.move_p2p_azel(\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2301, in move_p2p_azel\r\n",
+       "    async with self.m1m3_booster_valve():\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/contextlib.py", line 231, in __aexit__\r\n",
+       "    await self.gen.athrow(typ, value, traceback)\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2708, in m1m3_booster_valve\r\n",
+       "    await self.close_m1m3_booster_valve()\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2616, in close_m1m3_booster_valve\r\n",
+       "    await self._handle_m1m3_booster_valve(open=False)\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2657, in _handle_m1m3_booster_valve\r\n",
+       "    await cmd.set_start(timeout=self.fast_timeout)\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
+       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
+       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
+       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=389238772, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=0, result='Failed: The command ClearSlewFlag is not valid in the FaultState.')\r\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:57** **MainTel** \n", + "\n", + "
Raising M1M3.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:44** **MainTel** \n", + "\n", + "
GIS activated during azimuth slew.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "\n", + "\n", + "BLOCK-231 failed due to ignore list not being honored. Going to run just a regs p2p.\r\n", + "Terminating with state=7: stop telescope.\r\n", + "\r\n", + "\r\n", + "2024/11/21 10:39:43 TAI\r\n", + "\r\n", + "\r\n", + "Error in run\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
+       "    await self._run_task\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
+       "    await self.run_block()\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
+       "    await self.mtcs.move_p2p_azel(\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2315, in move_p2p_azel\r\n",
+       "    await self.process_as_completed(tasks)\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1137, in process_as_completed\r\n",
+       "    raise e\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1134, in process_as_completed\r\n",
+       "    ret_val = await res\r\n",
+       "              ^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
+       "    return f.result()  # May raise f.exception().\r\n",
+       "           ^^^^^^^^^^\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 510, in check_component_state\r\n",
+       "    raise RuntimeError(\r\n",
+       "RuntimeError: mtdome state is &lt;State.DISABLED: 1&gt;, expected &lt;State.ENABLED: 2&gt;\r\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:34** **MainTel** \n", + "\n", + "
beginning BLOCK-231
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:16** **MainTel** \n", + "\n", + "
M1M3 faulted while trying to recover. Sending to DISABLED state and this happened.\r\n",
+       "2024/11/21 10:14:45 TAI\r\n",
+       "Air controller sensor mismatch
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:13** **MainTel** \n", + "\n", + "
Recovering system from M2 breakaway test.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:07** **MainTel** \n", + "\n", + "
M1M3 faulted during the M2 breakaway test.\r\n",
+       "2024/11/21 10:04:04 TAI\r\n",
+       "Force controller Z Moment Limit - applied -11402.72 N, expected -9000.00 N to 9000.00 N
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 10:03** **MainTel** \n", + "\n", + "
Beginning BLOCK-241.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:49** **MainTel** \n", + "\n", + "
BLOCK-227 completed successfully. Nice job y'all!
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:28** **MainTel** \n", + "\n", + "
Going back to add BLOCK-227 while ignoring MTHexapod.2
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:27** **MainTel** \n", + "\n", + "
MTHexapod.2 faulted upon trying to apply compensation mode. \r\n",
+       "2024/11/21 09:25:16 TAI\r\n",
+       "Fault! errorCode=1, errorReport='Low-level controller went to FAULT state'\r\n",
+       "2024/11/21 09:25:16 TAI\r\n",
+       "max_move_duration=61.74\r\n",
+       "2024/11/21 09:25:16 TAI\r\n",
+       "Applying compensation to the current target position
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "\n", + "\n", + "MTHexapod:2 CSC faulted during 10% tests. In the middle of a slew:\r\n", + "SalIndex: 104790\r\n", + "2024/11/21 09:22:10 TAI\r\n", + "\r\n", + "\r\n", + "Fault! errorCode=-2, errorReport='Compensation failed.'\r\n", + "\r\n", + "\r\n", + "\r\n", + "\r\n", + "2024/11/21 09:22:10 TAI\r\n", + "\r\n", + "\r\n", + "Compensation failed; CSC going to Fault.\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 500, in wait_for\r\n",
+       "    return fut.result()\r\n",
+       "           ^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mthexapod/hexapod_csc.py", line 1105, in wait_stopped\r\n",
+       "    await self.telemetry_event.wait()\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/locks.py", line 213, in wait\r\n",
+       "    await fut\r\n",
+       "asyncio.exceptions.CancelledError\r\n",
+       "\r\n",
+       "\r\n",
+       "The above exception was the direct cause of the following exception:\r\n",
+       "\r\n",
+       "\r\n",
+       "Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mthexapod/hexapod_csc.py", line 442, in compensation_loop\r\n",
+       "    await self.compensation_wait()\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/mthexapod/hexapod_csc.py", line 471, in compensation_wait\r\n",
+       "    await asyncio.wait_for(self.wait_stopped(), timeout=self.max_move_duration)\r\n",
+       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/asyncio/tasks.py", line 502, in wait_for\r\n",
+       "    raise exceptions.TimeoutError() from exc\r\n",
+       "TimeoutError\r\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:18** **MainTel** \n", + "\n", + "
Scripts are now loading. BLOCK-227 in progress.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:18** **MainTel** \n", + "\n", + "
Anti-climatically, the block didn't populate scripts. We are enabling the scheduler again.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:15** **MainTel** \n", + "\n", + "
Alberto is online, going to begin digging through the trenches that are TMA comms. We wish him the best in the journey.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:15** **MainTel** \n", + "\n", + "
Running BLOCK-227.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:04** **MainTel** \n", + "\n", + "
MTMount would not enable:\r\n",
+       "2024/11/21 09:02:29 TAI\r\n",
+       "\r\n",
+       "\r\n",
+       "begin_enable failed; remaining in state <State.DISABLED: 1>: Command BothAxesPower(sequence_id=1006565, command_code=<CommandCode.BOTH_AXES_POWER: 31>, source=<Source.CSC: 1>, timestamp=1732179737.9188824, on=True) failed: ExpectedError('Command failed due to an alarm.')\r\n",
+       "\r\n",
+       "\r\n",
+       "Elevation axis reports a positive adjustable software limit.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:02** **MainTel** \n", + "\n", + "
Preparing TMA for 10% velocity tests with Petr supporting.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 09:02** **MainTel** \n", + "\n", + "
MTDome is closed. No issues in closing completely, although it took several minutes for the ApS left motors to complete the task and exit nominally.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 08:22** **MainTel** \n", + "\n", + "
Finished data taking for the night. Preparing everything to close dome shutters.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 08:16** **MainTel** Time Lost: 0.37; Time Lost Type: fault;\n", + "\n", + "
M1M3 faulted during a closed loop script, around 54 degrees elevation:\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/21 01:28:15 TAI\r\n",
+       "Force Actuator ID 128 (27) Following Error immediate fault
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 08:15** **MainTel** \n", + "\n", + "
MT Air compressor faulted. Going to cycle the CSC and hope connection comes back\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/21 02:35:48 TAI\r\n",
+       "\r\n",
+       "\r\n",
+       "Fault! errorCode=98, errorReport='While reconnecting:Modbus Error: [Connection] Not connected[AsyncModbusTcpClient m1m3cam-aircomp02.cp.lsst.org:502]'\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/21 02:35:48 TAI\r\n",
+       "While reconnecting:Modbus Error: [Connection] Not connected[AsyncModbusTcpClient m1m3cam-aircomp02.cp.lsst.org:502]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- Link: [OBS-700](https://rubinobs.atlassian.net/browse/OBS-700)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 08:07** **MainTel** \n", + "\n", + "
In focus in slightly poor seeing conditions. Grabbing science data.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 07:56** **MainTel** \n", + "\n", + "
Had to re-issue corrections and remove some of previous settings before proceeding with the closed_loop comcam - the previous hexapod state was still applied.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 07:51** **MainTel** \n", + "\n", + "
Taking image on science target.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 07:47** **MainTel** \n", + "\n", + "
Slewing to final science target for the night. Going to try and close the loops on a previously observed target.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 06:21** **MainTel** \n", + "\n", + "
Science team is ceding the telescope back to the AOS team. We will resume AOS testing until ~10 minutes before closing time.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 05:47** **MainTel** \n", + "\n", + "
Resuming the scheduler.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 05:36** **MainTel** \n", + "\n", + "
Recovered the Hexapod, completed the giant donut data, and we returned the rotator to 0. The AOS team is going to take a pause and we are going to collect science data near rot_angle 0 to minimize CCW fault risk.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 05:19** **MainTel** \n", + "\n", + "
MTHexapod failed - compensation failed.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 05:10** **MainTel** \n", + "\n", + "
BLOCK-75 data collection ongoing.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 05:04** **MainTel** \n", + "\n", + "
Going to attempt to track target nearby to get some giant donut data.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:53** **MainTel** \n", + "\n", + "
The CCW axis faulted. It just stopped tracking the rotator.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:37** **MainTel** \n", + "\n", + "
Attempting BLOCK-301.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:33** **MainTel** \n", + "\n", + "
The cable cable wrap faulted the MTMount. The rotator was not commanded anywhere, but the CCW just decided to quit.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:32** **MainTel** \n", + "\n", + "
PSYCH! No it didn't!
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:29** **MainTel** \n", + "\n", + "
We were able to step to 70 degrees in rotation just as the BLOCK does. Hopefully we can run the tests without an additional strain on the CCW following the rotator.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:29** **MainTel** \n", + "\n", + "
Going to attempt the rotator BLOCK-301.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:25** **MainTel** \n", + "\n", + "
Attempting to rotate the rotator in small steps. Trying to identify when the rotator and CCW decouple in steps.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:13** **MainTel** \n", + "\n", + "
Recovered MTRotator, slewed to 10 degrees rotation, and a few degrees into the slew, the CCW stopped following the rotator around -2 degrees. We will attempt now to recover.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 04:08** **MainTel** \n", + "\n", + "
Went down to the first floor to try and find a reset method for the MTAircompressor2. Chiller presents an error on the screen. MTAircompressor reads some pressure.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:36** **MainTel** \n", + "\n", + "
Attempting a 5 degree slew.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:35** **MainTel** \n", + "\n", + "
Continuing to troubleshoot rotator. Perhaps moves were too small to trigger an inposition flag.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:24** **MainTel** \n", + "\n", + "
Attempting to track target and the rotator is timing out.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:15** **MainTel** \n", + "\n", + "
Tiago is helping us try to identify a patch we can deploy tonight so we don't encounter frequent errors with TMA timeouts.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "\n", + "\n", + "move_p2p failed \r\n", + "\r\n", + "\r\n", + "\r\n", + "\r\n", + "2024/11/21 03:09:44 TAI\r\n", + "\r\n", + "\r\n", + "Error in run\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
+       "    await self._run_task\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
+       "    await self.run_block()\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
+       "    await self.mtcs.move_p2p_azel(\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2315, in move_p2p_azel\r\n",
+       "    await self.process_as_completed(tasks)\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1137, in process_as_completed\r\n",
+       "    raise e\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1134, in process_as_completed\r\n",
+       "    ret_val = await res\r\n",
+       "              ^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
+       "    return f.result()  # May raise f.exception().\r\n",
+       "           ^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
+       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
+       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
+       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=930398849, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: The commands allowed in Tracking are: Stop\\n\\nNot accepted command: 103\\nThe commands allowed in Tracking are: Stop\\n\\nNot accepted command: 403')\r\n",
+       "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:08** **MainTel** \n", + "\n", + "
Unable to track target - continuous timeouts when trying to track target.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:03** **MainTel** \n", + "\n", + "
Going to operate with one compressor until we can find a way to connect MTAirCompressor:2
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 03:02** **MainTel** \n", + "\n", + "
MTPtg faulting frequently. Really impacting operations. Going to stop MTmount and movep2p.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:55** **MainTel** \n", + "\n", + "
Baden confirms that he can ping the MT air compressor 2 machine. As soon as we try to enable the CSC we lose connection. \r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/21 02:52:55 TAI\r\n",
+       "Starting up:Modbus Error: [Connection] Not connected[AsyncModbusTcpClient m1m3cam-aircomp02.cp.lsst.org:502]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:47** **MainTel** \n", + "\n", + "
IT support is going to dial in and check out some of the network issues we seem to be having.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:47** **MainTel** \n", + "\n", + "
Attempted to cycle compressor CSC, didn't connect. Going to follow Petr's instructions. Troubleshooting.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:29** **MainTel** \n", + "\n", + "
MtPtg fault - just about to initiate a track_target command. \r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/21 02:27:31 TAI\r\n",
+       "Error MTMount replied error for trackTarget command (1734022109) Error no:1 message:Failed: Command with id 474073 has timed ou
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:18** **MainTel** \n", + "\n", + "
MTPtg immediately got grumpy. We're going to have to stop all Mount traffic and move_p2p
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:17** **MainTel** \n", + "\n", + "
Restarting observations after MTPtg fault.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 02:14** **MainTel** \n", + "\n", + "
MTPtg faulted\r\n",
+       "salIndex: 104525\r\n",
+       " 2024/11/21 02:11:12 TAI\r\n",
+       "raise RuntimeError(\r\n",
+       "RuntimeError: mtptg state is <State.FAULT: 3>, expected <State.ENABLED: 2>\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/21 02:11:02 TAI\r\n",
+       "Error MTMount replied error for startTracking command (1732828411) Error no:1 message:Failed: Command with id 447721 has timed
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:57** **MainTel** \n", + "\n", + "
BLOCK-T300 will move the telescope in different elevation angles and run closed loop. We want to understand the focus gradient we saw yesterday.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:56** **MainTel** \n", + "\n", + "
We moved the telescope to the final expected position using move_p2p command. \r\n",
+       "M1M3 behaved fine now. Starting BLOCK-T300 again.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:45** **MainTel** \n", + "\n", + "
MTM1M3TS was recovered through Brian's script that automatically recovers it.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:41** **MainTel** \n", + "\n", + "
As part of recovering, we need to recover MTM1M3TS as well. \r\n",
+       "We have the instructions. However, someone connected to MTM1M3TS and recovered it.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:32** **MainTel** \n", + "\n", + "
2024/11/21 01:28:15 TAI\r\n",
+       "MTM1M3 went to a fault: Force Actuator ID 128 (27) Following Error immediate Fault.\r\n",
+       "\r\n",
+       "\r\n",
+       "This happened while running BLOCK-T300. \r\n",
+       "It was slewing the telescope to -100º in Az. It was a big slew and we were at 80º elevation angle. \r\n",
+       "There was no apparent reason for M1M3 to go to a fault.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- Link: [OBS-699](https://rubinobs.atlassian.net/browse/OBS-699)" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:29**\n", + "\n", + "
Running block t-300, MTMount, MTPtg and MTM1M3 faulted
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:23** **MainTel** \n", + "\n", + "
SOAR seeing is ~1.3. We're getting ~1.1 in r at elevation of 71 deg.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:20** **MainTel** \n", + "\n", + "
The values in the Residual AOS FWHM column in RubinTv are plateauing. This means that we are probably ready to move on to the next test case.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 01:18** **MainTel** \n", + "\n", + "
The next test case would be BLOCK-T303. However, it seems that our BLOCK-T249 already accomplished the goal of 303. So we will skip it. We marked it as skipped in Zephyr.  We want to re-evaluate it, so we should keep it in the night plan for tomorrow.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 00:51** **MainTel** \n", + "\n", + "
Closed loop ongoing. We can see donuts!
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 00:50** **MainTel** \n", + "\n", + "
We are starting BLOCK-T249 again after all the recovery. \r\n",
+       "Let's start taking a new image.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-21 00:48** **MainTel** \n", + "\n", + "
Alright. Now let's try track_target just where we are.
\n" ], "text/plain": [ "" @@ -947,62 +2371,9 @@ { "data": { "text/markdown": [ + "- **2024-11-21 00:47** **MainTel** \n", "\n", - "\n", - "M2 went into fault while trying to park the TMA with elevation parking settings. We had just tried to run BLOCK-284 with no success. \r\n", - "\r\n", - "\r\n", - "M2 is Reporting:\r\n", - "\r\n", - "\r\n", - "2024/11/20 09:46:40 TAI\r\n", - "Inclinometer Difference Error \r\n", - "\r\n", - "\r\n", - "2024/11/20 09:48:08 TAI\r\n", - "\r\n", - "\r\n", - "Error when doing the basic cleanup and power off the motor.\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/m2/csc.py", line 1277, in _basic_cleanup_and_power_off_motor\r\n",
-       "    await self._execute_command(\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/m2/csc.py", line 2052, in _execute_command\r\n",
-       "    await command(*args, timeout=timeout, **kwargs)  # type: ignore[operator]\r\n",
-       "    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/m2com/controller.py", line 952, in set_closed_loop_control_mode\r\n",
-       "    raise RuntimeError(\r\n",
-       "RuntimeError: Closed-loop control mode is &lt;ClosedLoopControlMode.OpenLoop: 3&gt; instead of &lt;ClosedLoopControlMode.TelemetryOnly: 2&gt; in timeout.\r\n",
-       "d\r\n",
-       "2024/11/20 09:46:40 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Fault! errorCode=1, errorReport="Controller's state is Fault."\r\n",
-       "\r\n",
-       "\r\n",
-       "Elevation angle as measured by M2: 70.71. \r\n",
-       "TMA elevation position: 70.88. \r\n",
-       "\r\n",
-       "\r\n",
-       "Is this why there is an inclination error? \r\n",
-       "\r\n",
-       "\r\n",
-       "M2 error codes\r\n",
-       "\r\n",
-       "\r\n",
-       "11/20/2024 06:23:23\r\n",
-       "11/20/2024 06:23:31\r\n",
-       "11/20/2024 06:46:03\r\n",
-       "11/20/2024 06:46:03\r\n",
-       "11/20/2024 06:47:32\r\n",
-       "11/20/2024 06:50:14\r\n",
-       "6053.00\r\n",
-       "cRIO Communication Error\r\n",
-       "0.00\r\n",
-       "1.00\r\n",
-       "Controller's state is Fault.\r\n",
-       "6060.00\r\n",
-       "Inclinometer Difference Error\r\n",
-       "
\n" + "
This did not work. So we will enable MTPTg and send a stop_tracking script. Then we will try again.
\n" ], "text/plain": [ "" @@ -1014,7 +2385,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-696](https://rubinobs.atlassian.net/browse/OBS-696)" + "- **2024-11-21 00:43** **MainTel** \n", + "\n", + "
We are now recovering again. We will home both axes and try a move_p2p. If that does not work, we will need to send a stop command to the mount and try again.
\n" ], "text/plain": [ "" @@ -1026,24 +2399,11 @@ { "data": { "text/markdown": [ - "- **2024-11-20 10:02** **MainTel** \n", + "- **2024-11-21 00:42** **MainTel** \n", "\n", - "
M2 error codes\r\n",
-       "\r\n",
-       "\r\n",
-       "11/20/2024 06:23:23\r\n",
-       "11/20/2024 06:23:31\r\n",
-       "11/20/2024 06:46:03\r\n",
-       "11/20/2024 06:46:03\r\n",
-       "11/20/2024 06:47:32\r\n",
-       "11/20/2024 06:50:14\r\n",
-       "6053.00\r\n",
-       "cRIO Communication Error\r\n",
-       "0.00\r\n",
-       "1.00\r\n",
-       "Controller's state is Fault.\r\n",
-       "6060.00\r\n",
-       "Inclinometer Difference Error
\n" + "
The TMA EUI went to fault in Elevation. This is a SoftMotion Axis Fault. \r\n",
+       "Set Time 00:39:13.337 21/11/24\r\n",
+       "System: elevation
\n" ], "text/plain": [ "" @@ -1055,15 +2415,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 10:01** **MainTel** \n", - "\n", - "
Elevation angle as measured by M2: 70.71. \r\n",
-       "\r\n",
-       "\r\n",
-       "TMA elevation position: 70.88. \r\n",
-       "\r\n",
-       "\r\n",
-       "Is this why there is an inclination error?
\n" + "- Link: [OBS-698](https://rubinobs.atlassian.net/browse/OBS-698)" ], "text/plain": [ "" @@ -1075,34 +2427,15 @@ { "data": { "text/markdown": [ + "- **2024-11-21 00:41**\n", "\n", - "\n", - "M2 is Reporting:\r\n", - "\r\n", - "\r\n", - "2024/11/20 09:46:40 TAI\r\n", - "Inclinometer Difference Error \r\n", - "\r\n", + "
Home both axes, track target \r\n",
        "\r\n",
-       "2024/11/20 09:48:08 TAI\r\n",
        "\r\n",
+       "Then needed to put rotator at zero.  Did that. \r\n",
        "\r\n",
-       "Error when doing the basic cleanup and power off the motor.\r\n",
-       "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/m2/csc.py", line 1277, in _basic_cleanup_and_power_off_motor\r\n",
-       "    await self._execute_command(\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/m2/csc.py", line 2052, in _execute_command\r\n",
-       "    await command(*args, timeout=timeout, **kwargs)  # type: ignore[operator]\r\n",
-       "    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/m2com/controller.py", line 952, in set_closed_loop_control_mode\r\n",
-       "    raise RuntimeError(\r\n",
-       "RuntimeError: Closed-loop control mode is &lt;ClosedLoopControlMode.OpenLoop: 3&gt; instead of &lt;ClosedLoopControlMode.TelemetryOnly: 2&gt; in timeout.\r\n",
-       "d\r\n",
-       "2024/11/20 09:46:40 TAI\r\n",
        "\r\n",
-       "\r\n",
-       "Fault! errorCode=1, errorReport="Controller's state is Fault."\r\n",
-       "
\n"
+       "Then did track target again, and this time both the MTMount and MtPtg faulted
\n" ], "text/plain": [ "" @@ -1114,9 +2447,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 09:48** **MainTel** \n", + "- **2024-11-21 00:33** **MainTel** \n", "\n", - "
M2 went into fault while trying to park the TMA.
\n" + "
Stop tracking was not enough. When putting M2 into DISABLED state, MTMount went to a FAULT state. We are recovering now.
\n" ], "text/plain": [ "" @@ -1128,43 +2461,12 @@ { "data": { "text/markdown": [ + "- **2024-11-21 00:31** **MainTel** \n", "\n", - "\n", - "BLOCK-284 failed again - failing to home TMA, when we literally just did. A bug? \r\n", - "Sal: 104373\r\n", - "\r\n", + "
It is an issue with M2.  \r\n",
        "\r\n",
-       "2024/11/20 09:26:27 TAI\r\n",
        "\r\n",
-       "\r\n",
-       "Error in run\r\n",
-       "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
-       "    await self.run_block()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/move_p2p.py", line 182, in run_block\r\n",
-       "    await self.mtcs.move_p2p_azel(\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2242, in move_p2p_azel\r\n",
-       "    await self.process_as_completed(tasks)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1132, in process_as_completed\r\n",
-       "    raise e\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py", line 1129, in process_as_completed\r\n",
-       "    ret_val = await res\r\n",
-       "              ^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one\r\n",
-       "    return f.result()  # May raise f.exception().\r\n",
-       "           ^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=930398849, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: Homming not done\\nDiscrete motion not available\\nPerform a homming or \\nallow relative movements and relative commands in settings\\nNot accepted command: 103')\r\n",
-       "
\n"
+       "Stop tracking
\n" ], "text/plain": [ "" @@ -1176,9 +2478,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 09:24** **MainTel** \n", + "- **2024-11-21 00:30** **MainTel** \n", "\n", - "
BLOCK-284 failed saying TMA was not homed (it was in the position we left it after a point_azel). We are homing the axes and trying again.
\n" + "
Tiago is now debugging what is going on. He says that we might having issues with historical data associated with the crash we had with the DDS systems (yagan13 and yagan17).
\n" ], "text/plain": [ "" @@ -1190,9 +2492,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 09:17** **MainTel** \n", + "- **2024-11-21 00:21** **MainTel** \n", "\n", - "
MTDome shutters are closed. I went upstairs to close the shutters. I pushed ApS - Right first, and it closed completely. The operation button glowed solid green. I then pushed to close the left shutter. I waited more than 7 minutes. It LOOKED fully closed. The light kept flashing green, however. I waited until 10 minutes. The close button never stopped flashing green. I disabled the switches and left the 8th floor. There appeared to be no light leak.
\n" + "
AOS Closed loop script faulted with a timeout error. Sal index 301046
\n" ], "text/plain": [ "" @@ -1204,7 +2506,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-695](https://rubinobs.atlassian.net/browse/OBS-695)" + "- **2024-11-21 00:20** **MainTel** \n", + "\n", + "
We are now running closed loop comcam.
\n" ], "text/plain": [ "" @@ -1216,9 +2520,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 09:14** **MainTel** \n", + "- **2024-11-21 00:19** **MainTel** \n", "\n", - "
Running BLOCK-284.
\n" + "
When getting this image, we realized that the LaserTracker was powered on. I put LaserTracker to StandBy and powered it off via PDU. We should be ready to go now.
\n" ], "text/plain": [ "" @@ -1230,9 +2534,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 08:35** **MainTel** \n", + "- **2024-11-21 00:18** **MainTel** \n", "\n", - "
CCW and rotator are out of sync again. Using the TMA EUI to realign the two.
\n" + "
Turning off the laser now
\n" ], "text/plain": [ "" @@ -1244,9 +2548,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 08:23** **MainTel** \n", + "- **2024-11-21 00:17** **MainTel** \n", "\n", - "
Raising M1M3 again.
\n" + "
We forgot to turn off the laser.
\n" ], "text/plain": [ "" @@ -1258,14 +2562,10 @@ { "data": { "text/markdown": [ - "- **2024-11-20 08:17** **MainTel** \n", + "- **2024-11-21 00:16** **MainTel** \n", "\n", - "
Attempting to do a move_azel, ignoring Mtdome components. \r\n",
-       "\r\n",
-       "\r\n",
-       "M1M3 faulted immediately. \r\n",
-       "2024/11/20 08:15:51 TAI\r\n",
-       "Force controller Z Moment Limit - applied 12161.35 N, expected -9000.00 N to 9000.00 N
\n" + "
We are now starting BLOCK-T249.\r\n",
+       "The telescope is already tracking. So we will start by taking an image first.
\n" ], "text/plain": [ "" @@ -1277,9 +2577,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 08:10** **MainTel** \n", + "- **2024-11-21 00:10**\n", "\n", - "
Beginning shutdown procedures. Parking dome first.
\n" + "
Still taking i-band twilight flats but switching to rotator = 30 deg
\n" ], "text/plain": [ "" @@ -1291,9 +2591,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 08:05** **MainTel** \n", + "- **2024-11-21 23:50** **MainTel** \n", "\n", - "
We continued on with scheduler tests until about 05:05 and we interrupted observations to close for the night.
\n" + "
Disable MtM2, MTM1M3, Hexapod1,2, rotator, MTM1M3TS etc. to get telemetry to monitor.
\n" ], "text/plain": [ "" @@ -1305,9 +2605,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 07:21** **MainTel** \n", + "- **2024-11-21 23:49** **MainTel** \n", "\n", - "
Close_loop_comcam failed spitting out a bunch of Nan values.
\n" + "
Still need to support Franco C. Robinson was resetting all the cabinet. He still needs to work on it.
\n" ], "text/plain": [ "" @@ -1319,9 +2619,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 07:18** **MainTel** \n", - "\n", - "
AOS team has finished collecting data - going to start the schedule for science driven observations.
\n" + "- Link: [OBS-701](https://rubinobs.atlassian.net/browse/OBS-701)" ], "text/plain": [ "" @@ -1333,9 +2631,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 06:51** **MainTel** \n", + "- **2024-11-21 23:48** **MainTel** \n", "\n", - "
We had a brief MTptg error, but it didn't fault data collection. Still getting data for different elevations for the LUT.
\n" + "
Will try to MTM2 enabled.
\n" ], "text/plain": [ "" @@ -1347,22 +2645,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:59** **MainTel** Time Lost: 0.47; Time Lost Type: fault;\n", + "- **2024-11-21 23:40** **MainTel** \n", "\n", - "
Camera cable wrap stopped following the rotator, probably due to a rotator low-level controller error. We had to enter the Rot EUI and recover, then stand down the MTMount CSCs so we could nudge the CCW back to the rotator's position. Time of troubleshooting was about 30 minutes. \r\n",
-       "\r\n",
-       "\r\n",
-       "MTRotator faulted due to a CCW following error. EUI reports CCW is at 0.24 degrees, and rotator is at 2.2 degrees. \r\n",
-       "2024/11/20 05:27:54 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Fault! errorCode=1, errorReport='Low-level controller went to FAULT state'\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 05:27:51 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Camera cable wrap not following closely enough: error # 3 = 2.2000851374542925 > 2.2 deg
\n" + "
Robinson cleared up the GIS hard stop. We will recover step by step. First we reset the GIS for M2 and Hexapods (and rotator)
\n" ], "text/plain": [ "" @@ -1374,7 +2659,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-694](https://rubinobs.atlassian.net/browse/OBS-694)" + "- **2024-11-21 23:29** **MainTel** \n", + "\n", + "
GIS heartbeat flicker. The same as what happened today.
\n" ], "text/plain": [ "" @@ -1386,9 +2673,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:55** **MainTel** \n", - "\n", - "
Trying to track target at 80 degrees elevation.
\n" + "- Link: [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" ], "text/plain": [ "" @@ -1400,9 +2685,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:55** **MainTel** \n", + "- **2024-11-21 23:28** **MainTel** \n", "\n", - "
Homing axes, MTRotator came back cleanly after nudging the CCW to the rotator's position.
\n" + "
Hard Stop performed in the OSS and other systems are all fault.   23:23:29.033 21/11/24
\n" ], "text/plain": [ "" @@ -1414,19 +2699,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:50** **MainTel** \n", - "\n", - "
MTRotator faulted due to a CCW following error. EUI reports CCW is at 0.24 degrees, and rotator is at 2.2 degrees. \r\n",
-       "2024/11/20 05:27:54 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Fault! errorCode=1, errorReport='Low-level controller went to FAULT state'\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 05:27:51 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Camera cable wrap not following closely enough: error # 3 = 2.2000851374542925 > 2.2 deg
\n" + "- Link: [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" ], "text/plain": [ "" @@ -1438,9 +2711,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:24** **MainTel** Time Lost: 1.0; Time Lost Type: fault;\n", + "- **2024-11-21 23:26** **MainTel** \n", "\n", - "
Logged time loss to OBS-693, but since I added a comment only, the time loss did not propagate in the narrative log. Creating the record here.
\n" + "
(false) hard stop seemed fault almost every system.
\n" ], "text/plain": [ "" @@ -1452,60 +2725,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:21** **MTMount** Time Lost: 0.92; Time Lost Type: fault;\n", + "- **2024-11-21 23:20** **MainTel** \n", "\n", - "
h3. Error/Failure explanation:\r\n",
-       "The MTPtg CSC goes into a fault state when we try to send the telescope to a different position to track a target on the sky using a SAL script: track_target. \r\n",
-       "h3. What were you doing when the failure occurred?:\r\n",
-       "Two different system states have produced this behavior. In one case, the TMA was tracking the sky, and we issued a track_target script to send it to another position. The MTPtg CSC faulted, reporting a time out error that was not associated to anything specific, and we could not slew the telescope. In the other case, the TMA was stopped, not tracking the sky, and had no pending commands. We issued a track target script, and the same errors were produced. \r\n",
-       "h3. Detailed description of the steps followed that caused the error/failure:\r\n",
-       "One example: TMA tracking the sky\r\n",
-       "MTMount reports:\r\n",
-       "\r\n",
-       "2024/11/20 03:15:02 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesEnableTracking(sequence_id=13851778, command_code=, source=, timestamp=1732072501.1088629)=b'13851778\\n38\\n1\\n1732072501.1088629\\r\\n' failed: Command with id 13851778 has timed out after not receiving accept for 500 ms.\r\n",
-       "\r\n",
-       "\r\n",
-       "MTPTg reports:\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 03:15:02 TAI\r\n",
-       "\r\n",
-       "Error MTMount replied error for startTracking command (1732828374) Error no:1 message:Failed: Command with id 13851778 has time\r\n",
-       "\r\n",
-       "Another example - we stopped MTMount, not tracking, no pending commands:\r\n",
-       "MTPtg reports:\r\n",
-       "\r\n",
-       "2024/11/20 03:23:50 TAI\r\n",
-       "\r\n",
-       "Error MTMount replied error for trackTarget command (1733626109) Error no:1 message:Failed: Command with id 13858718 has timed \r\n",
-       "\r\n",
-       "\r\n",
-       "MTMount reports:\r\n",
-       "\r\n",
-       "2024/11/20 03:23:50 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesTrackTarget(sequence_id=13858718, command_code=, source=, timestamp=1732073030.406984, azimuth=-99.97881468391222, elevation=60.02329219282346, azimuth_velocity=-0.0010219022304360321, elevation_velocity=-0.0035535333296652137, tai=1732073030.5560682)=b'13858718\\n35\\n1\\n1732073030.406984\\n-99.97881468391222\\n60.02329219282346\\n-0.0010219022304360321\\n-0.0035535333296652137\\n1732073030.5560682\\r\\n' rejected: Command with id 13858718 has timed out after not receiving accept for 500 ms.\r\n",
-       "\r\n",
-       "Third example:\r\n",
-       "Another MTPtg error. Telescope was tracking after taking a series of closed loop data. \r\n",
-       "\r\n",
-       "\r\n",
-       "Going to slew from elevation 50 -> 80 if that means anything. Although it probably doesn't. \r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 05:05:08 TAI\r\n",
-       "\r\n",
-       "Error MTMount replied error for startTracking command (1732828382) Error no:1 message:Failed: Command with id 13925969 has time\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 05:05:08 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesEnableTracking(sequence_id=13925969, command_code=, source=, timestamp=1732079107.809806)=b'13925969\\n38\\n1\\n1732079107.809806\\r\\n' failed: Command with id 13925969 has timed out after not receiving accept for 500 ms.
\n" + "
Opening mirror covers
\n" ], "text/plain": [ "" @@ -1517,7 +2739,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" + "- **2024-11-21 23:20** **MainTel** \n", + "\n", + "
Enabled dome following
\n" ], "text/plain": [ "" @@ -1529,9 +2753,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:14** **MainTel** \n", + "- **2024-11-21 23:15** **MainTel** \n", "\n", - "
Back on sky
\n" + "
LaserTracker off at PDU
\n" ], "text/plain": [ "" @@ -1543,22 +2767,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 05:07** **MainTel** \n", + "- **2024-11-21 23:15** **MainTel** \n", "\n", - "
Another MTPtg error. Telescope was tracking after taking a series of closed loop data. \r\n",
-       "\r\n",
-       "\r\n",
-       "Going to slew from elevation 50 -> 80 if that means anything. Although it probably doesn't. \r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 05:05:08 TAI\r\n",
-       "Error MTMount replied error for startTracking command (1732828382) Error no:1 message:Failed: Command with id 13925969 has time\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 05:05:08 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesEnableTracking(sequence_id=13925969, command_code=<CommandCode.BOTH_AXES_ENABLE_TRACKING: 38>, source=<Source.CSC: 1>, timestamp=1732079107.809806)=b'13925969\\n38\\n1\\n1732079107.809806\\r\\n' failed: Command with id 13925969 has timed out after not receiving accept for 500 ms.
\n" + "
Checked out AOS sequence PAIR mode to obsenv. Reloaded schema to pick up new mode.
\n" ], "text/plain": [ "" @@ -1570,9 +2781,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 04:15** **MainTel** \n", + "- **2024-11-21 23:12** **MainTel** \n", "\n", - "
Closing the loops to begin BLOCK-286. LUT data collection.
\n" + "
run_command for close_mirror_cover failed. But seems closed
\n" ], "text/plain": [ "" @@ -1584,9 +2795,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 04:12** **MainTel** \n", + "- **2024-11-21 23:12** **MainTel** \n", "\n", - "
Taking baseline image; confirming that we are in a rough focus.
\n" + "
Mirror covers closing. HVAC off, preparing to open shutter
\n" ], "text/plain": [ "" @@ -1598,15 +2809,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 04:11** **MainTel** \n", + "- **2024-11-21 23:11** **MainTel** \n", "\n", - "
In order:\r\n",
-       "\r\n",
-       "\r\n",
-       "stop tracking\r\n",
-       "runcommand MTMount, cmd: stop\r\n",
-       "move_p2p\r\n",
-       "track target
\n" + "
HVAC configuration is now setup to night time operation with turned on VEC_04 fan.
\n" ], "text/plain": [ "" @@ -1618,9 +2823,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 04:10** **MainTel** \n", + "- **2024-11-21 23:11** **MainTel** \n", "\n", - "
Able to track target, returning to sector of el=60, az=-100
\n" + "
T-88 done.
\n" ], "text/plain": [ "" @@ -1632,18 +2837,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 04:00** **MainTel** \n", + "- **2024-11-21 23:11** **MainTel** \n", "\n", - "
MTPtg publishes same timeout error, MtMount publishes:\r\n",
-       "\r\n",
-       "\r\n",
-       "Error\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 03:57:42 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesEnableTracking(sequence_id=13858728, command_code=<CommandCode.BOTH_AXES_ENABLE_TRACKING: 38>, source=<Source.CSC: 1>, timestamp=1732075061.6208801)=b'13858728\\n38\\n1\\n1732075061.6208801\\r\\n' failed: Command with id 13858728 has timed out after not receiving accept for 500 ms.
\n" + "
Then camera alignment thru align.py
\n" ], "text/plain": [ "" @@ -1655,9 +2851,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:59** **MainTel** \n", + "- **2024-11-21 22:56** **MainTel** \n", "\n", - "
MTptg immediately fails when trying to track target.
\n" + "
Block-T88. M2 alignment. running align.py
\n" ], "text/plain": [ "" @@ -1669,9 +2865,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:52** **MainTel** \n", + "- **2024-11-21 22:54** **MainTel** \n", "\n", - "
Finally able to command the dome to a new position. Had to exit fault twice to clear two different errors related to AMCS.
\n" + "
We are on BLOCK-T245. Looks okay. Enabling scheduler.
\n" ], "text/plain": [ "" @@ -1683,9 +2879,15 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:43** **MainTel** \n", + "- **2024-11-21 22:51** **MainTel** \n", "\n", - "
Recovering the azimuth drive fault in MTDome.
\n" + "
Tracking and then take image comcam for checking out. \r\n",
+       "exp_times: 5\r\n",
+       "nimages: 3\r\n",
+       "image_type: \"ACQ\"\r\n",
+       "program: \"BLOCK-T250\"\r\n",
+       "reason: \"TMACheckout\"\r\n",
+       "filter: \"y_04\"
\n" ], "text/plain": [ "" @@ -1697,15 +2899,28 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:38** **MainTel** \n", "\n", - "
Performing another slew to verify that we can slew. MTDome immediately crashes. \r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 03:37:19 TAI\r\n",
+       "\n",
+       "salIndex: 301080 while tracking target, MTPtg failed \r\n",
+       "2024/11/21 22:48:08 TAI\r\n",
        "\r\n",
        "\r\n",
-       "AMCS status message now is 210118=Error of type Automation in subsystem AMCS: Azimuth drives not enabled.
\n" + "Unexpected exception in stop_tracking.\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_track_target.py", line 551, in cleanup\r\n",
+       "    await asyncio.wait_for(self.tcs.stop_tracking(), timeout=5)\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 489, in wait_for\r\n",
+       "    return fut.result()\r\n",
+       "           ^^^^^^^^^^^^\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_tcs.py", line 1304, in stop_tracking\r\n",
+       "    await getattr(self.rem, self.ptg_name).cmd_stopTracking.start(\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
+       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
+       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=2065515254, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=6612, result='Rejected : command not allowed in current state')\r\n",
+       "
\n" ], "text/plain": [ "" @@ -1717,9 +2932,19 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:37** **MainTel** \n", + "- **2024-11-21 22:46** **MainTel** \n", "\n", - "
Tiago recommended a stop mount and movep2p. We were able to slew away successfully.
\n" + "
TMA checkout. \r\n",
+       "track_azel:\r\n",
+       "  el: 65\r\n",
+       "  az: 135\r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T250\"\r\n",
+       "reason: \"TMACheckout\"\r\n",
+       "ignore:\r\n",
+       " - mtdome\r\n",
+       " - mtdometrajectory
\n" ], "text/plain": [ "" @@ -1731,24 +2956,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:28** **MainTel** \n", + "- **2024-11-21 22:43** **MainTel** \n", "\n", - "
Tracking target at 60, -100. Seems to be working. \r\n",
-       "\r\n",
-       "\r\n",
-       "HA! Psych. It does not: repetitive errors \r\n",
-       "\r\n",
-       "\r\n",
-       "MTPtg reports:\r\n",
-       "2024/11/20 03:23:50 TAI\r\n",
-       "Error MTMount replied error for trackTarget command (1733626109) Error no:1 message:Failed: Command with id 13858718 has timed \r\n",
-       "\r\n",
-       "\r\n",
-       "MTMount reports:\r\n",
-       "2024/11/20 03:23:50 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesTrackTarget(sequence_id=13858718, command_code=<CommandCode.BOTH_AXES_TRACK_TARGET: 35>, source=<Source.CSC: 1>, timestamp=1732073030.406984, azimuth=-99.97881468391222, elevation=60.02329219282346, azimuth_velocity=-0.0010219022304360321, elevation_velocity=-0.0035535333296652137, tai=1732073030.5560682)=b'13858718\\n35\\n1\\n1732073030.406984\\n-99.97881468391222\\n60.02329219282346\\n-0.0010219022304360321\\n-0.0035535333296652137\\n1732073030.5560682\\r\\n' rejected: Command with id 13858718 has timed out after not receiving accept for 500 ms.
\n" + "
Tiago R. updated the scheduler.
\n" ], "text/plain": [ "" @@ -1760,9 +2970,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:23** **MainTel** \n", + "- **2024-11-21 22:39** **MainTel** \n", "\n", - "
Stopping MTMount with run_command -> stop
\n" + "
with run_command.py, it works
\n" ], "text/plain": [ "" @@ -1774,20 +2984,41 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:16** **MainTel** \n", "\n", - "
MTMount reports:\r\n",
-       "2024/11/20 03:15:02 TAI\r\n",
+       "\n",
+       "2024/11/21 22:36:56 TAI SALINDEX: salIndex: 104934\r\n",
+       "\r\n",
        "\r\n",
        "\r\n",
-       "Command BothAxesEnableTracking(sequence_id=13851778, command_code=<CommandCode.BOTH_AXES_ENABLE_TRACKING: 38>, source=<Source.CSC: 1>, timestamp=1732072501.1088629)=b'13851778\\n38\\n1\\n1732072501.1088629\\r\\n' failed: Command with id 13851778 has timed out after not receiving accept for 500 ms.\r\n",
+       "\r\n",
+       "Error in run\r\n",
+       "
Traceback (most recent call last):\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 967, in open_m1_cover\r\n",
+       "    await self.rem.mtmount.cmd_openMirrorCovers.set_start(\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
+       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
+       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
+       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1028798034, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: The commands allowed in DiscreteMove are: Stop\\n\\nNot accepted command: 901')\r\n",
        "\r\n",
        "\r\n",
-       "MTPTg reports:\r\n",
+       "During handling of the above exception, another exception occurred:\r\n",
        "\r\n",
        "\r\n",
-       "2024/11/20 03:15:02 TAI\r\n",
-       "Error MTMount replied error for startTracking command (1732828374) Error no:1 message:Failed: Command with id 13851778 has time
\n"
+       "Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
+       "    await self._run_task\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/open_mirror_covers.py", line 66, in run\r\n",
+       "    await self.mtcs.open_m1_cover()\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 972, in open_m1_cover\r\n",
+       "    f"Open mirror cover command failed with {ack.ack!r}::{ack.error}. "\r\n",
+       "                                             ^^^^^^^\r\n",
+       "AttributeError: 'AckError' object has no attribute 'ack'\r\n",
+       "
\n" ], "text/plain": [ "" @@ -1799,9 +3030,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:16** **MainTel** \n", - "\n", - "
Can't even slew without MTPtg faulting
\n" + "- Link: [obs-653](https://rubinobs.atlassian.net/browse/obs-653)" ], "text/plain": [ "" @@ -1813,9 +3042,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:16** **MainTel** \n", + "- **2024-11-21 22:37** **MainTel** \n", "\n", - "
We had not homed the axes after recovering the MTMount so errors accumulated. Slewing to 55, -120
\n" + "
open_mirror_cover failed.
\n" ], "text/plain": [ "" @@ -1827,12 +3056,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:09** **MainTel** \n", - "\n", - "
We are going to disable and re-enable dome following to try and sync the MTptg and dome loop.\r\n",
-       "\r\n",
-       "\r\n",
-       "This was not done due to mis-interpretation of MTPtg error - which was saying the homing was not DONE, not homing DOME.
\n" + "- Link: [obs-662](https://rubinobs.atlassian.net/browse/obs-662)" ], "text/plain": [ "" @@ -1844,16 +3068,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:08** **MainTel** \n", + "- **2024-11-21 22:31** **MainTel** \n", "\n", - "
MTPtg failing to cooperate spitting errors out regarding the \"homing\" not being home\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 03:04:23 TAI\r\n",
-       "Error MTMount replied error for startTracking command (1732828372) Error no:1 message:Failed: Not homming done\r\n",
-       "Not accepted com
\n" + "
Home both axes then will enable hardpoint correction.
\n" ], "text/plain": [ "" @@ -1865,15 +3082,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 03:07** **MainTel** \n", + "- **2024-11-21 22:31** **MainTel** \n", "\n", - "
2024/11/20 03:04:23 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Command BothAxesEnableTracking(sequence_id=13843030, command_code=<CommandCode.BOTH_AXES_ENABLE_TRACKING: 38>, source=<Source.CSC: 1>, timestamp=1732071863.3330677)=b'13843030\\n38\\n1\\n1732071863.3330677\\r\\n' failed: Not homming done\r\n",
-       "Not accepted command: 108\r\n",
-       "Not homming done\r\n",
-       "Not accepted command: 408
\n" + "
Failed to enable MTMount but elevation failed.
\n" ], "text/plain": [ "" @@ -1885,9 +3096,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 02:57** **MainTel** \n", + "- **2024-11-21 22:27** **MainTel** \n", "\n", - "
MTMount and MTPtg faulted, likely due to a long slew. Cycling and continuing on sky.
\n" + "
Changed to 10% without parking!
\n" ], "text/plain": [ "" @@ -1899,9 +3110,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 02:44** **MainTel** \n", + "- **2024-11-21 22:22** **MainTel** \n", "\n", - "
Continuing iterations to converge z corrections so we can begin the LUT run.
\n" + "
We are on the parking set. Will move down to 80 deg. and move in Az to check.
\n" ], "text/plain": [ "" @@ -1913,9 +3124,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 02:25** **MainTel** \n", + "- **2024-11-21 22:20** **MainTel** \n", "\n", - "
The TMA is pointing pretty far north where the pointing model, had to slew back to 100 and re-run BLOCK-292
\n" + "
Will unpark the telescope. First free the locking pins.
\n" ], "text/plain": [ "" @@ -1927,15 +3138,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 02:08** **MainTel** \n", + "- **2024-11-21 22:20** **MainTel** \n", "\n", - "
Another MTPTg error - collecting data and occurrences at this point\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/20 02:07:41 TAI\r\n",
-       "Error MTMount replied error for trackTarget command (1733569250) Error no:1 message:Failed: Command with id 13722734 has timed
\n" + "
Oh, locking pins are engaged. When it stopped the resetting, will disengage the locking pins.
\n" ], "text/plain": [ "" @@ -1947,7 +3152,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" + "- **2024-11-21 21:15** **MainTel** \n", + "\n", + "
Warm up cam hex also done.
\n" ], "text/plain": [ "" @@ -1959,9 +3166,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 02:04** **MainTel** \n", + "- **2024-11-21 21:12** **MainTel** \n", "\n", - "
Repeating BLOCK-249 after restarting the MTAOS CSC.
\n" + "
We will go to have a dinner soon. Cam hex warming up will run while we are there.
\n" ], "text/plain": [ "" @@ -1973,9 +3180,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:54** **MainTel** \n", + "- **2024-11-21 21:08** **MainTel** \n", "\n", - "
Move to az: -25 el:50
\n" + "
Warm up hexapod for Cam hex is running.
\n" ], "text/plain": [ "" @@ -1987,9 +3194,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:54** **MainTel** \n", + "- **2024-11-21 21:06** **MainTel** \n", "\n", - "
We stopped the BLOCK-T155 scheduler because we are below el = 35 deg
\n" + "
Warmup hexapod for M2 is done. Will disable M2 hexapod.
\n" ], "text/plain": [ "" @@ -2001,19 +3208,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:50** **MainTel** \n", + "- **2024-11-21 21:05** **MainTel** \n", "\n", - "
Timeout error again after we resumed the scheduler for BLOCK-T155\r\n",
-       "\r\n",
-       "\r\n",
-       "salIndex: 104138\r\n",
-       " - [STANDARD]\r\n",
-       " - Started at: 2024-11-20T01:40:35.084Z\r\n",
-       "\r\n",
-       "\r\n",
-       "File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError
\n" + "
Raising M1M3!
\n" ], "text/plain": [ "" @@ -2025,7 +3222,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" + "- **2024-11-21 21:04** **MainTel** \n", + "\n", + "
Kevin F. pulhed the ts_config_mttcs and then reenable LaserTraker1.
\n" ], "text/plain": [ "" @@ -2037,23 +3236,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:46** **MainTel** \n", + "- **2024-11-21 21:01** **MainTel** \n", "\n", - "
MTPtg faults with the timeout error whenever: \r\n",
-       "- we first enable the MTMount and try to move \r\n",
-       "- randomly during track target \r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "Here's an example:\r\n",
-       "BLOCK-T155 timed out \r\n",
-       "salIndex: 104136\r\n",
-       " - Started at: 2024-11-20T01:35:29.519Z
\n" + "
Pulled update to mttcs for MTAOS CSC. CSC is in standby.
\n" ], "text/plain": [ "" @@ -2065,7 +3250,9 @@ { "data": { "text/markdown": [ - "- Link: [OBS-693](https://rubinobs.atlassian.net/browse/OBS-693)" + "- **2024-11-21 21:01** **MainTel** \n", + "\n", + "
It didn't work. Guillem MH restarted TS2SA.
\n" ], "text/plain": [ "" @@ -2077,9 +3264,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:43** **MainTel** \n", + "- **2024-11-21 20:57** **MainTel** \n", "\n", - "
Resume the scheduler
\n" + "
LaserTracker1 enabled.
\n" ], "text/plain": [ "" @@ -2091,16 +3278,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:42** **MainTel** \n", + "- **2024-11-21 20:53** **MainTel** \n", "\n", - "
BLOCK-T155 timed out \r\n",
-       "salIndex: 104136\r\n",
-       " - [STANDARD]\r\n",
-       " - Started at: 2024-11-20T01:35:29.519Z\r\n",
-       "\r\n",
-       "\r\n",
-       "MTPTg faulted \r\n",
-       "Cycle the MTPTg
\n" + "
Per guillem's request, turning on the Lasertracker PDU.
\n" ], "text/plain": [ "" @@ -2112,9 +3292,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:04** **MainTel** \n", + "- **2024-11-21 20:30** **MainTel** \n", "\n", - "
Taking image first to see how the focus looks
\n" + "
Runing BLOCK-T3, warming up for M2 hexapod. It seems working this time.
\n" ], "text/plain": [ "" @@ -2126,13 +3306,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:02** **MainTel** \n", + "- **2024-11-21 20:29** **MainTel** \n", "\n", - "
Track target to ECDFS field \r\n",
-       "\r\n",
-       "\r\n",
-       "ra: 3.5416\r\n",
-       "dec: -28.10
\n" + "
Te-wei roll-backed the hexapod controller versions. The fix of controller's state machine breaks the CSC inPosition event.
\n" ], "text/plain": [ "" @@ -2144,9 +3320,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 01:01** **MainTel** \n", + "- **2024-11-21 20:20** **MainTel** \n", "\n", - "
Starting BLOCK-T155.
\n" + "
Petr turned on FCU around 3:30 pm (which was off after GIS activation today).
\n" ], "text/plain": [ "" @@ -2158,9 +3334,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:36** **MainTel** \n", + "- **2024-11-21 20:17** **MainTel** \n", "\n", - "
Restarting BLOCK-T249
\n" + "
HVAC setpoint is changed to 9C. Will re-evaluate that after dinner time.
\n" ], "text/plain": [ "" @@ -2172,9 +3348,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:35** **MainTel** \n", + "- **2024-11-21 20:09** **MainTel** \n", "\n", - "
Set dof again to all zeros
\n" + "
Te-wei will add some test result on OBS-702 and then will decide whether roll-back the update or not.
\n" ], "text/plain": [ "" @@ -2186,9 +3362,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:33** **MainTel** \n", + "- **2024-11-21 20:00** **MainTel** \n", "\n", - "
Sending MTAOS to offline again.
\n" + "
Starting check_hardpoint.
\n" ], "text/plain": [ "" @@ -2200,9 +3376,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:33** **MainTel** \n", + "- **2024-11-21 20:00** **MainTel** \n", "\n", - "
Switching to BLOCK-T298 while AOS Team debugs the BLOCK-T249.   Never mind
\n" + "
After retest 409 is failing and 227 is failing. 227 looks OK, 409 is not functional on the yForces.Will start hardpoint test, but likely we need to valve off and disable 409. I think we will be able to operate with it disabled. and a (slightly) marginal 227 undershooting on the primary.
\n" ], "text/plain": [ "" @@ -2214,12 +3390,12 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:25** **MainTel** \n", + "- **2024-11-21 19:49** **MainTel** \n", "\n", - "
salIndex: 104100 - Started at: 2024-11-20T00:21:42.368Z\r\n",
-       "\r\n",
-       "\r\n",
-       "Guillem is debugging
\n" + "
Initially failing actuator list:\r\n",
+       "  - Actuator ID 104: Pri Index 3\r\n",
+       "  - Actuator ID 227: Pri Index 61\r\n",
+       "  - Actuator ID 409: Pri Index 123
\n" ], "text/plain": [ "" @@ -2231,10 +3407,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:24** **MainTel** \n", + "- **2024-11-21 19:25** **MainTel** \n", "\n", - "
raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1418882047, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: index 18 is out of bounds for axis 1 with size 18')
\n" + "
Not sure hexapods are inposition. We need to ignore the flag to track_target etc for night operation. Kevin F. announce in simonyi channel.
\n" ], "text/plain": [ "" @@ -2246,9 +3421,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:19** **MainTel** \n", + "- **2024-11-21 19:08** **MainTel** \n", "\n", - "
Track target az: 135, el: 65
\n" + "
Turning on PS.
\n" ], "text/plain": [ "" @@ -2260,9 +3435,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:19** **MainTel** \n", + "- **2024-11-21 19:06** **MainTel** \n", "\n", - "
Need to turn on the tracking
\n" + "
Increasing sleep time to 100 sec doesn't work.
\n" ], "text/plain": [ "" @@ -2274,9 +3449,14 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:17** **MainTel** \n", + "- **2024-11-21 19:01** **MainTel** \n", "\n", - "
Starting BLOCK-T249
\n" + "
warmup_hexapod with \r\n",
+       "hexapod: m2\r\n",
+       "axis: x\r\n",
+       "step_size: 125\r\n",
+       "max_position: 4500\r\n",
+       "sleep_time: 30
\n" ], "text/plain": [ "" @@ -2288,9 +3468,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:16** **MainTel** \n", + "- **2024-11-21 18:57** **MainTel** \n", "\n", - "
Setting new degrees of freedom (all zeros)
\n" + "
Run BLOCK-T3 for warming up hexapod. Checking.
\n" ], "text/plain": [ "" @@ -2302,9 +3482,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:14** **MainTel** \n", + "- **2024-11-21 18:56** **MainTel** \n", "\n", - "
Putting MTAOS back in enabled
\n" + "
We were turning on the OSS. 15:33 pm CLT/18:33pm UTC
\n" ], "text/plain": [ "" @@ -2316,12 +3496,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 00:12** **MainTel** \n", + "- **2024-11-21 18:42** **MainTel** \n", "\n", - "
Stop tracking \r\n",
-       "\r\n",
-       "\r\n",
-       "Sending MTAOS to offline for update
\n" + "
Starting BLOCK-T144, M1M3 bump test
\n" ], "text/plain": [ "" @@ -2333,9 +3510,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 21:56** **MainTel** \n", + "- **2024-11-21 18:39** **MainTel** \n", "\n", - "
Just for the record, ComCam has r_03, y_04, and i_06 installed today based on metadata extracted from RubinTV.
\n" + "
run_command.py for both hexapods for x:250 and it worked for both axes. Maybe timeout issue from warmup_hexapod.py issue.
\n" ], "text/plain": [ "" @@ -2347,9 +3524,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 21:36** **MainTel** \n", + "- **2024-11-21 18:16** **MainTel** \n", "\n", - "
The second attempt of `track_target.py` worked fine (sal index 104433).
\n" + "
No failed again. Will try the Cam hexa for next.
\n" ], "text/plain": [ "" @@ -2361,9 +3538,10 @@ { "data": { "text/markdown": [ - "- **2024-11-20 21:35** **MainTel** \n", + "- **2024-11-21 18:15** **MainTel** \n", "\n", - "
I am now running BLOCK-T250 TMA Checkout. When I reached Step 7, which asks to run a track_command. The script faulted with a \"No command acknowledgment seen\" (SAL Index 301036). I will try again (now, using MTQueue).
\n" + "
TMA lost the heartbeat but recovered. \r\n",
+       "Closing the mirror cover now.
\n" ], "text/plain": [ "" @@ -2375,9 +3553,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 20:31**\n", + "- **2024-11-21 18:15** **MainTel** \n", "\n", - "
trying to put MTRotator in enabled (last step of rotator startup), but there is an interlock error
\n" + "
Rerun for BLOCK-T3 M2 hexapod.
\n" ], "text/plain": [ "" @@ -2389,9 +3567,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 20:24**\n", + "- **2024-11-21 18:14** **MainTel** \n", "\n", - "
Working on rotator startup
\n" + "
Franco said the GIS is cleared. Door is still opened.
\n" ], "text/plain": [ "" @@ -2403,9 +3581,15 @@ { "data": { "text/markdown": [ - "- **2024-11-20 20:23**\n", "\n", - "
Running hardpoint breakaway tests
\n" + "\n", + "We found that there is Safety Modbus Server Error 17:41:52.438 21/11/24 code 66 system Safety Task. \r\n", + "VI chain: ??? in SubVIs.lvlib:TCP Master.lvclass:Protocol Write.vi:6190001->SubVIs.lvlib:IP Data Unit.lvclass:Write ADU Packet.vi:1820001->Modbus Master.lvclass:Read Holding Registers.vi:2420001->ModbusServer.lvclass:UpdateModbusData.vi:3110003->ModbusServer.lvclass:HelperLoop.vi:190002->ModbusServer.lvclass:Process.vi:5310001 Description: LabVIEW: (Hex 0x42) The network connection was closed by the peer. If you are using the Open VI Reference function on a remote VI Server connection, verify that the machine is allowed access by selecting <b>Tools>>Options>>VI Server</b> on the server side.\r\n", + "\r\n", + "\r\n", + "Then 17:41:52.743 21/11/24\r\n", + "VI chain: Description: LabVIEW: (Hex 0x0) Error connecting to GPIB driver or device. The NI-488.2 driver may not be installed.
=========================
VISA: (Hex 0x0) Operation completed successfully.\r\n", + "\n" ], "text/plain": [ "" @@ -2417,24 +3601,7 @@ { "data": { "text/markdown": [ - "- **2024-11-20 20:03**\n", - "\n", - "
During dome handoff we lost heartbeat from a lot of the CSCs\r\n",
-       "\r\n",
-       "\r\n",
-       "Tiago said  yagan17 and yagan13 daemons crashed \r\n",
-       "\r\n",
-       "\r\n",
-       "4:37pm the dome is back \r\n",
-       "\r\n",
-       "\r\n",
-       "Tried to send the dome from standby to disabled and faulted \r\n",
-       "\r\n",
-       "\r\n",
-       "We rebooted the cRIO remotely\r\n",
-       "\r\n",
-       "\r\n",
-       "Then we confirmed that we can move the dome in both directions with the CSC
\n" + "- Link: [obs-701](https://rubinobs.atlassian.net/browse/obs-701)" ], "text/plain": [ "" @@ -2446,14 +3613,11 @@ { "data": { "text/markdown": [ - "- **2024-11-20 19:13** **MainTel** \n", + "- **2024-11-21 17:51** **MainTel** \n", "\n", - "
SAA (Single Actuator Axes) Failures:\r\n",
-       "  None\r\n",
-       "DAA (Dual Actuator Axes) Failures:\r\n",
-       "  - Actuator ID 227: Pri Index 61\r\n",
-       "  - Actuator ID 330: Pri Index 107\r\n",
-       "  - Actuator ID 409: Pri Index 123
\n" + "
Te-Wei used the hexapod GUIs to test the movements of the camera and M2 hexapods already. It worked well. \r\n",
+       "However really-E-stop seems engagled \r\n",
+       "On TMA EUI hardstop 1 & 2 topple block LS1 seems activated.
\n" ], "text/plain": [ "" @@ -2465,10 +3629,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 19:00** **MainTel** \n", + "- **2024-11-21 17:45** **MainTel** \n", "\n", - "
Alarm from MTDomeCapacitorBanks.MTDome \r\n",
-       "Second time this has happened today during the day shift.
\n" + "
Found that we lost GIS heartbeat. Kevin F. is going to hit the GIS.
\n" ], "text/plain": [ "" @@ -2480,19 +3643,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 18:51**\n", + "- **2024-11-21 17:43** **MainTel** \n", "\n", - "
These actuators failed the bump test \r\n",
-       "\r\n",
-       "\r\n",
-       "SAA (Single Actuator Axes) Failures:\r\n",
-       "  None\r\n",
-       "DAA (Dual Actuator Axes) Failures:\r\n",
-       "  - Actuator ID 112: Pri Index 11\r\n",
-       "  - Actuator ID 227: Pri Index 61\r\n",
-       "  - Actuator ID 314: Pri Index 91\r\n",
-       "  - Actuator ID 330: Pri Index 107\r\n",
-       "  - Actuator ID 409: Pri Index 123
\n" + "
Salindex:104906. warmup_hexapod fault ActuallyGIS Emergency stop.
\n" ], "text/plain": [ "" @@ -2504,9 +3657,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 18:27** **MainTel** \n", + "- **2024-11-21 17:42** **MainTel** \n", "\n", - "
We can see motor current oscillations in M2Hex for the time it was left in ENABLED. They were low amplitude oscillations. But we need to characterize this data.
\n" + "
Running cam Hex warming. But even Hexapod1 and Hexapod2.
\n" ], "text/plain": [ "" @@ -2518,9 +3671,12 @@ { "data": { "text/markdown": [ - "- **2024-11-20 18:15** **MainTel** \n", + "- **2024-11-21 17:40** **MainTel** \n", "\n", - "
We forgot M2Hex enabled. It is now in a DISABLED state. We checked and there are no runaway events in any of the struts.
\n" + "
salIndex: 104897 Started at: 2024-11-21T17:38:24.829Z\r\n",
+       "\r\n",
+       "\r\n",
+       "warmup_hexapod.py failed waiting for in position event. either not moving of not sending events.
\n" ], "text/plain": [ "" @@ -2532,9 +3688,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 18:06** **MainTel** \n", + "- **2024-11-21 17:38** **MainTel** \n", "\n", - "
Restarting OSS again
\n" + "
Just re-run the block. Will see if there is any issue.
\n" ], "text/plain": [ "" @@ -2546,9 +3702,13 @@ { "data": { "text/markdown": [ - "- **2024-11-20 17:46** **MainTel** \n", + "- **2024-11-21 17:36** **MainTel** \n", "\n", - "
We will stop the OSS soon as part of the OSS microfiltering work lead by Freddy.
\n" + "
salIndex: 104888  - Started at: 2024-11-21T17:33:53.915Z\r\n",
+       "Warmup_hexapod.py failed and timeout. \r\n",
+       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\r\n",
+       "    raise exceptions.TimeoutError() from exc\r\n",
+       "TimeoutError
\n" ], "text/plain": [ "" @@ -2560,12 +3720,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 17:39** **MainTel** \n", + "- **2024-11-21 17:33** **MainTel** \n", "\n", - "
MTM1M3 is now in DISABLED state. Kate noticed that we do not have air inside the cell. Putting M1M3 into ENABLED opened the automatic valve and we are now seeing air inside the cell with ~ 125 PSI. \r\n",
-       "\r\n",
-       "\r\n",
-       "We are getting ready to run bump tests. FA332 is still disabled so we are running via check_actuators.py.
\n" + "
Start warm-up M2 hexapod.
\n" ], "text/plain": [ "" @@ -2577,9 +3734,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 17:33** **MainTel** \n", + "- **2024-11-21 17:31** **MainTel** \n", "\n", - "
Julen connected to perform some configuration changes in TMA EUI.
\n" + "
Hexapod test from Te-wei is done. Will start warm-up hexapods.
\n" ], "text/plain": [ "" @@ -2591,10 +3748,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 17:27** **MainTel** \n", + "- **2024-11-21 17:30** **MainTel** \n", "\n", - "
We are done with BLOCK-T4 and we will start M1M3 Bump Tests.\r\n",
-       "The telescope is still at horizon but we got confirmation that it is safe to run these test.
\n" + "
Mirror cover is still opened by Daycrew (Ian announced it this morning).
\n" ], "text/plain": [ "" @@ -2606,9 +3762,10 @@ { "data": { "text/markdown": [ - "- **2024-11-20 16:45** **MainTel** \n", + "- **2024-11-21 17:06** **MainTel** \n", "\n", - "
Starting BLOCK-T4 now
\n" + "
Start warming up M2 and Camera hexpod. \r\n",
+       "02:05 we stop the warming up as Te-wei need to clearError.
\n" ], "text/plain": [ "" @@ -2620,9 +3777,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 16:43** **MainTel** \n", + "- **2024-11-21 16:54** **MainTel** \n", "\n", - "
We are back from lunch and we will resume warm-up hexapods. This time, we will run M2Hex.
\n" + "
Start Powering on the Camera Rotator.
\n" ], "text/plain": [ "" @@ -2634,9 +3791,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 15:58** **MainTel** \n", + "- **2024-11-21 16:08** **MainTel** \n", "\n", - "
Freddy reports that the mechanical team is working on cleaning the TMA oil. This is transparent for telescope operations and does not block us from moving. I am trying to coordinate TMA handover for today.
\n" + "
Done for Comcam checkout. Rubin TV and CCS are showing the image
\n" ], "text/plain": [ "" @@ -2648,9 +3805,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 15:47**\n", + "- **2024-11-21 16:04** **MainTel** \n", "\n", - "
Warming up camera hexapod
\n" + "
BLOCK-T246 Comcam USDF checkout running. Checking Comcam is ready for today's operation.
\n" ], "text/plain": [ "" @@ -2662,12 +3819,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 15:36** **MainTel** \n", + "- **2024-11-21 16:02** **MainTel** \n", "\n", - "
Meanwhile, we are powering up hexapods. MTHexapod:2 was ENABLED. When trying to transition to DISABLED, it went to a fault. We are now checking the cabinets to see what is their current situation. \r\n",
-       "\r\n",
-       "\r\n",
-       "When checking the hexapod cabinets, we noticed that they were all off. This means that it was only the hexapod summary state that, for whatever reason, was being displayed as ENABLED. This is less concerning, since the hexapod was not energized. We are powering them up now and we will continue with start ups and warm up.
\n" + "
Around 8-9pm, 9.8C is expected. Current set point is 8C so we are good
\n" ], "text/plain": [ "" @@ -2679,9 +3833,9 @@ { "data": { "text/markdown": [ - "- **2024-11-20 15:32** **MainTel** \n", + "- **2024-11-21 15:08** **MainTel** \n", "\n", - "
Bruno and Kate already in the control room. We reviewed the HVAC set point, which is now set to 8ºC. The temperature near the sunset yesterday was 12ºC. However, the forecast temperature near sunset for today is predicted to be 10ºC. Let's leave the set point to 8ºC for now.
\n" + "
Daytime observers start shift.
\n" ], "text/plain": [ "" @@ -2736,8 +3890,8 @@ "text/markdown": [ "## Project-Wide Night(s) Report \n", "- Run on logs and databases from **https://usdf-rsp-dev.slac.stanford.edu/**\n", - "- Report **1 observing night(s)** with the last reported night starting on **2024-11-20 12:00:00**.\n", - "- min_dayobs='2024-11-20', max_dayobs='2024-11-21'\n", + "- Report **1 observing night(s)** with the last reported night starting on **2024-11-21 12:00:00**.\n", + "- min_dayobs='2024-11-21', max_dayobs='2024-11-22'\n", "- Using ***Prototype* Logging and Reporting** Version: **52.dev0+g783922c.d20241115**\n", "- have_consdb = False\n", "- Imported lsst.ts.logging_and_reporting from local packages.\n", @@ -2791,11 +3945,7 @@ "cell_type": "code", "execution_count": 17, "id": "b076e98a-44a2-437c-9342-a180fe2cd829", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [ { "data": { @@ -2811,8 +3961,8 @@ " \n", " \n", " nightreport/reports\n", - " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241120&max_day_obs=20241121\n", + " 1\n", + " https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241121&max_day_obs=20241122\n", " \n", " \n", " exposurelog/instruments\n", @@ -2822,32 +3972,32 @@ " \n", " exposurelog/exposures.LSSTComCamSim\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241120&max_day_obs=20241121\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", " \n", " \n", " exposurelog/exposures.LATISS\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241120&max_day_obs=20241121\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", " \n", " \n", " exposurelog/exposures.LSSTComCam\n", - " 19\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241120&max_day_obs=20241121\n", + " 13\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", " \n", " \n", " exposurelog/exposures.LSSTCam\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241120&max_day_obs=20241121\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", " \n", " \n", " exposurelog/messages\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241120&max_day_obs=20241121\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122\n", " \n", " \n", " narrativelog/messages\n", - " 93\n", - " https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-20T00%3A00%3A00&max_date_added=2024-11-21T00%3A00%3A00\n", + " 166\n", + " https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-21T00%3A00%3A00&max_date_added=2024-11-22T00%3A00%3A00\n", " \n", " \n", " efd/targets\n", @@ -2916,16 +4066,12 @@ "cell_type": "code", "execution_count": 18, "id": "42", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [ { "data": { "text/markdown": [ - "### Overview for Service: `nightreport` [0]" + "### Overview for Service: `nightreport` [1]" ], "text/plain": [ "" @@ -2944,7 +4090,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241120&max_day_obs=20241121" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241121&max_day_obs=20241122" ], "text/plain": [ "" @@ -2958,7 +4104,7 @@ "output_type": "stream", "text": [ "None\n", - "- Got 0 records. \n" + "- Got 1 records. \n" ] }, { @@ -2983,7 +4129,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241120&max_day_obs=20241121" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241121&max_day_obs=20241122" ], "text/plain": [ "" @@ -3003,7 +4149,7 @@ { "data": { "text/markdown": [ - "### Overview for Service: `narrativelog` [93]" + "### Overview for Service: `narrativelog` [166]" ], "text/plain": [ "" @@ -3022,7 +4168,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-20T00%3A00%3A00&max_date_added=2024-11-21T00%3A00%3A00" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-21T00%3A00%3A00&max_date_added=2024-11-22T00%3A00%3A00" ], "text/plain": [ "" @@ -3036,7 +4182,7 @@ "output_type": "stream", "text": [ "None\n", - "- Got 93 records. \n" + "- Got 166 records. \n" ] } ], @@ -3053,11 +4199,7 @@ "cell_type": "code", "execution_count": 19, "id": "43", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ "# Conditionally display our current ability to connect to all needed endpoints.\n", @@ -3096,8 +4238,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Elapsed time (excluding code import): 5.9 seconds\n", - "Finished 2024-11-20 22:12:28 UTC\n" + "Elapsed time (excluding code import): 4.8 seconds\n", + "Finished 2024-11-21 23:57:28 UTC\n" ] } ], diff --git a/notebooks_tsqr/NightLog.yaml b/notebooks_tsqr/NightLog.yaml index de2cbdc..14970e5 100644 --- a/notebooks_tsqr/NightLog.yaml +++ b/notebooks_tsqr/NightLog.yaml @@ -25,7 +25,7 @@ parameters: sunset yesterday). A value of TODAY means the last night shown will be the one that starts tonight (which will have no nightly data before sunset). - default: "YESTERDAY" + default: "YESTERDAYw" number_of_days: type: integer description: > diff --git a/notebooks_tsqr/dashboard.ipynb b/notebooks_tsqr/dashboard.ipynb index 485632e..6ab3467 100644 --- a/notebooks_tsqr/dashboard.ipynb +++ b/notebooks_tsqr/dashboard.ipynb @@ -98,20 +98,20 @@ " 'TMA_CSC',\n", " 'TMA_EUI',\n", " 'Visualization'},\n", - " Counter({'None': 19593,\n", - " 'MainTel': 7159,\n", - " 'AuxTel': 1954,\n", - " 'MTMount': 113,\n", - " 'Other': 98,\n", - " 'Observer Remark': 81,\n", - " 'Mount': 49,\n", + " Counter({'None': 19603,\n", + " 'MainTel': 7180,\n", + " 'AuxTel': 1921,\n", + " 'MTMount': 112,\n", + " 'Other': 97,\n", + " 'Observer Remark': 79,\n", + " 'Mount': 48,\n", " 'CSC level': 39,\n", " 'Environmental Monitoring Systems': 23,\n", - " 'Calibrations': 21,\n", - " 'M1M3': 18,\n", - " 'Dome': 17,\n", + " 'Calibrations': 20,\n", + " 'M1M3': 17,\n", + " 'Dome': 16,\n", " 'Rotator': 13,\n", - " 'Calibration': 10,\n", + " 'Calibration': 9,\n", " 'Component Level (EUI)': 9,\n", " 'LOVE': 8,\n", " 'ATMCS': 8,\n", @@ -172,17 +172,17 @@ "text": [ "Try to connect (self.timeout=(5.05, 20.0)) to each endpoint of https://summit-lsp.lsst.codes/nightreport \n", "DEBUG protected_get(url='https://summit-lsp.lsst.codes/nightreport/reports',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/nightreport/reports (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/nightreport/reports (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "Try to connect (self.timeout=(5.05, 20.0)) to each endpoint of https://summit-lsp.lsst.codes/narrativelog \n", "DEBUG protected_get(url='https://summit-lsp.lsst.codes/narrativelog/messages',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/narrativelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /narrativelog/messages (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/narrativelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /narrativelog/messages (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "Try to connect ({self.timeout=}) to each endpoint of https://summit-lsp.lsst.codes/exposurelog \n", "DEBUG protected_get(url='https://summit-lsp.lsst.codes/exposurelog/instruments',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/exposurelog/instruments (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/instruments (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/exposurelog/instruments (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/instruments (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "DEBUG protected_get(url='https://summit-lsp.lsst.codes/exposurelog/exposures?instrument=na',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/exposurelog/exposures?instrument=na (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/exposures?instrument=na (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/exposurelog/exposures?instrument=na (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/exposures?instrument=na (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "DEBUG protected_get(url='https://summit-lsp.lsst.codes/exposurelog/messages',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/exposurelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/messages (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://summit-lsp.lsst.codes/exposurelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/messages (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "Try to connect (self.timeout=(5.05, 20.0)) to each endpoint of https://usdf-rsp-dev.slac.stanford.edu/nightreport \n", "DEBUG protected_get(url='https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports',timeout=(5.05, 20.0))\n", "Try to connect (self.timeout=(5.05, 20.0)) to each endpoint of https://usdf-rsp-dev.slac.stanford.edu/narrativelog \n", @@ -193,17 +193,17 @@ "DEBUG protected_get(url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages',timeout=(5.05, 20.0))\n", "Try to connect (self.timeout=(5.05, 20.0)) to each endpoint of https://tucson-teststand.lsst.codes/nightreport \n", "DEBUG protected_get(url='https://tucson-teststand.lsst.codes/nightreport/reports',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/nightreport/reports (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/nightreport/reports (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "Try to connect (self.timeout=(5.05, 20.0)) to each endpoint of https://tucson-teststand.lsst.codes/narrativelog \n", "DEBUG protected_get(url='https://tucson-teststand.lsst.codes/narrativelog/messages',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/narrativelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /narrativelog/messages (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/narrativelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /narrativelog/messages (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "Try to connect ({self.timeout=}) to each endpoint of https://tucson-teststand.lsst.codes/exposurelog \n", "DEBUG protected_get(url='https://tucson-teststand.lsst.codes/exposurelog/instruments',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/exposurelog/instruments (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/instruments (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/exposurelog/instruments (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/instruments (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "DEBUG protected_get(url='https://tucson-teststand.lsst.codes/exposurelog/exposures?instrument=na',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/exposurelog/exposures?instrument=na (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/exposures?instrument=na (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/exposurelog/exposures?instrument=na (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/exposures?instrument=na (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "DEBUG protected_get(url='https://tucson-teststand.lsst.codes/exposurelog/messages',timeout=(5.05, 20.0))\n", - "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/exposurelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/messages (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", + "DEBUG protected_get: FAIL: result=\"Error connecting to https://tucson-teststand.lsst.codes/exposurelog/messages (with timeout=(5.05, 20.0)). HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/messages (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=5.05)'))\"\n", "\n", "Connected to 5 out of 15 endpoints.(33%)\n", "Successful connects (5): \n", @@ -301,7 +301,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "7", "metadata": {}, "outputs": [ @@ -529,10 +529,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finished 2024-11-22 00:00:51.113648\n" + ] + } + ], "source": [ "print(f\"Finished {str(datetime.now())}\")" ] diff --git a/python/lsst/ts/logging_and_reporting/utils.py b/python/lsst/ts/logging_and_reporting/utils.py index 4f71286..f345711 100644 --- a/python/lsst/ts/logging_and_reporting/utils.py +++ b/python/lsst/ts/logging_and_reporting/utils.py @@ -33,6 +33,43 @@ # in a Database or API query string. +def fallback_parameters(day_obs, number_of_days, verbose): + """Given parameters from Times Square, return usable versions of + all parameters. If the provide parameters are not usable, return + default usable ones. + """ + day_obs_fb = "YESTERDAY" # Fall Back value + days_fb = 1 + message = "" + + try: + # dayobs(str): YYYY-MM-DD, YYYYMMDD, TODAY, YESTERDAY + get_datetime_from_dayobs_str(day_obs) # ignore result + except Exception as err: + message += f"""\nInvalid day_obs given: {day_obs!r} + Available values are: YYYY-MM-DD, YYYYMMDD, TODAY, YESTERDAY. + Using: {day_obs_fb!r}\n{str(err)!r} + """ + day_obs = day_obs_fb + + try: + days = int(number_of_days) + except Exception as err: + days = days_fb + message += f"""\nInvalid number_of_days given: {number_of_days!r} + Must be an integer. + Using: {days}\n{str(err)!r} + """ + + to_use = dict( + day_obs=day_obs, + number_of_days=days, + verbose=(verbose == "true"), + ) + + return to_use, message + + class DatetimeIter: def __init__(self, start_datetime, stop_datetime, increment=None): """increment:: datetime.timedelta"""