4773 Allow iquery probes to trigger sweep tasks for existing Dockets

cl/corpus_importer/signals.py

@@ -119,11 +119,16 @@ def handle_update_latest_case_id_and_schedule_iquery_sweep(
         # pacer_case_id)
         return None
 
-    # Only call update_latest_case_id_and_schedule_iquery_sweep if this is a
-    # new RECAP district or bankruptcy docket with pacer_case_id not added by
-    # iquery sweep tasks.
+    # Only call update_latest_case_id_and_schedule_iquery_sweep if:
+    # - The docket belongs to a RECAP district or bankruptcy court,
+    # - The docket has a pacer_case_id,
+    # - The docket was newly created (when IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED=True), or
+    # - The docket was created or updated by the last probe iteration from probe_iquery_pages.
+    check_probe_or_created = (
+        not getattr(instance, "avoid_trigger_signal", False) or created
+    )
     if (
-        created
+        check_probe_or_created
         and instance.pacer_case_id
         and instance.court_id
         in list(

cl/corpus_importer/tests.py

@@ -2576,7 +2576,7 @@ def test_update_latest_case_id_and_schedule_iquery_sweep_integration(
             )
             # Probing will add 3 dockets (12, 16, 24) + 2 added for the sweep task (13,18).
             self.assertEqual(
-                dockets.count(), 5, msg="Docket number doesn't match."
+                dockets.count(), 5, msg="Docket count doesn't match."
             )
             # 7 additional PACER HTML files should be stored by now, 3 added by the
             # probing task + 4 added by the sweep task.
@@ -2589,8 +2589,22 @@ def test_update_latest_case_id_and_schedule_iquery_sweep_integration(
 
             ### Integration test probing task + sweep
             # IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED False
+            with override_settings(IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED=False):
+                # Create docket pacer_case_id 12, which is the last docket in
+                # the probe. Even though it already exists, it should trigger
+                # a sweep task.
+                DocketFactory(
+                    court=self.court_txed,
+                    source=Docket.RECAP,
+                    case_name="New Incoming Docket 12",
+                    docket_number="2:10-cv-00602",
+                    pacer_case_id="12",
+                )
+
             dockets = Docket.objects.filter(court_id=self.court_txed.pk)
-            self.assertEqual(dockets.count(), 0)
+            self.assertEqual(
+                dockets.count(), 1, msg="Docket count doesn't match for txed."
+            )
             r = get_redis_interface("CACHE")
             # Simulate a highest_known_pacer_case_id  = 8
             r.hset("iquery:highest_known_pacer_case_id", self.court_txed.pk, 8)
@@ -2615,9 +2629,10 @@ def test_update_latest_case_id_and_schedule_iquery_sweep_integration(
                 1,
                 msg="Wrong number of sweep task called.",
             )
-            # Probing will add 3 dockets (9,10,12) + 1 added for the sweep task (11).
+            # Probing will add 3 dockets (9,10) + 1 added for the sweep task (11).
+            # Docket 12 already exists however, it should still trigger the sweep task that adds 11.
             self.assertEqual(
-                dockets.count(), 4, msg="Docket number doesn't match for txed."
+                dockets.count(), 4, msg="Docket count doesn't match for txed."
             )
         finally:
             # Ensure the signal is disconnected after the test

cl/search/factories.py

@@ -307,7 +307,12 @@ def filepath_local(self, create, extracted, **kwargs):
             self.filepath_local = FileField().evaluate(None, None, kwargs)
 
         if create:
-            self.save(update_fields=["filepath_local"])
+            # Use a Docket queryset to persist filepath_local instead of calling
+            # save(), which can trigger duplicate post_save signals, potentially
+            # causing issues in certain testing scenarios.
+            Docket.objects.filter(pk=self.pk).update(
+                filepath_local=self.filepath_local
+            )
 
 
 class DocketWithChildrenFactory(DocketFactory):

cl/settings/project/corpus_importer.py

@@ -11,7 +11,7 @@
     "IQUERY_COURT_BLOCKED_MAX_ATTEMPTS", default=6
 )
 IQUERY_EMPTY_PROBES_LIMIT = env.int("IQUERY_EMPTY_PROBES_LIMIT", default=15)
-IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED = env(
+IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED = env.bool(
     "IQUERY_SWEEP_UPLOADS_SIGNAL_ENABLED", default=False
 )
 IQUERY_COURT_RATE = env("IQUERY_COURT_RATE", default="100/s")