Merge branch 'main' into remove_viewers_pt3

CHANGELOG.md

@@ -1,3 +1,20 @@
+v10.1.7 (2025-01-14)
+-------------------------
+ * Delete new category counts when deleting a flow
+ * Remove squashing and db triggers used for old category counts
+
+v10.1.6 (2025-01-13)
+-------------------------
+ * More migration tweaking
+
+v10.1.5 (2025-01-13)
+-------------------------
+ * Migrations to backfill Msg.is_android and set not-null
+ * Start using boundary geometry field
+ * Move deps to project dependencies config in pyproject.toml
+ * Ensure that SQL app migrations run after other apps
+ * Adjust ticket flow to allow more change context
+
 v10.1.4 (2025-01-13)
 -------------------------
  * Remove ability to delete runs from the UI

package.json

@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "@nyaruka/flow-editor": "1.37.1",
-    "@nyaruka/temba-components": "0.114.1",
+    "@nyaruka/temba-components": "0.116.0",
     "codemirror": "5.18.2",
     "colorette": "1.2.2",
     "fa-icons": "0.2.0",

pyproject.toml

@@ -1,59 +1,60 @@
 [project]
 name = "temba"
-version = "10.1.4"
+version = "10.1.7"
 description = "Hosted service for visually building interactive messaging applications"
 authors = [
     {"name" = "Nyaruka", "email" = "[email protected]"}
 ]
 readme = "README.md"
 requires-python = ">=3.12,<3.13"
-dependencies = []
+dependencies = [
+    "django ~= 5.1.4",
+    "django-compressor ~= 4.5.0",
+    "django-countries ~= 7.6.1",
+    "django-mptt ~= 0.16.0",
+    "django-redis ~= 4.12.1",
+    "django-storages ~= 1.14.4",
+    "django-timezone-field ~= 6.1.0",
+    "djangorestframework ~= 3.15.1",
+    "dj-database-url ~= 0.5.0",
+    "smartmin ~= 5.1.1",
+    "celery ~= 5.4.0",
+    "redis ~= 5.2.0",
+    "boto3 ~= 1.35.54",
+    "cryptography ~= 43.0.3",
+    "vonage  ~= 3.17.4",
+    "pyotp ~= 2.4.1",
+    "twilio ~= 9.3.7",
+    "geojson ~= 2.5.0",
+    "markdown ~= 3.6.0",
+    "polib ~= 1.2.0",
+    "python-magic ~= 0.4.22",
+    "xlsxlite ~= 0.2.0",
+    "colorama ~= 0.4.6",
+    "gunicorn ~= 22.0.0",
+    "iptools ~= 0.7.0",
+    "iso8601 ~= 0.1.16",
+    "phonenumbers ~= 8.13.49",
+    "pycountry ~= 22.3.5",
+    "python-dateutil ~= 2.9.0",
+    "packaging ~= 22.0",
+    "requests-toolbelt ~= 1.0.0",
+    "chardet ~= 4.0.0",
+    "openpyxl ~= 3.1.5",
+    "ffmpeg-python ~= 0.2.0",
+    "slack-sdk ~= 3.17.0",
+    "django-formtools ~= 2.5.1",
+    "pillow ~= 10.4.0",
+    "django-imagekit ~= 5.0.0",
+    "iso639-lang ~= 2.2.3",
+    "google-auth ~= 2.32.0",
+
+    "psycopg[pool] ~= 3.2.1"
+]
 
 [project.urls]
 repository = "http://github.com/rapidpro/rapidpro"
 
-[tool.poetry.dependencies]
-Django = "^5.1.4"
-django-compressor = "^4.3.1"
-django-countries = "^7.0"
-django-mptt = "^0.16.0"
-django-redis = "^4.12.1"
-django-storages = "^1.11.1"
-django-timezone-field = "^6.1.0"
-djangorestframework = "^3.15.1"
-dj-database-url = "^0.5.0"
-smartmin = "5.1.1"
-celery = "^5.4.0"
-redis = "^5.2.0"
-boto3 = "^1.35.54"
-cryptography = "^43.0.3"
-vonage = "3.17.4"
-pyotp = "2.4.1"
-twilio = "9.3.7"
-geojson = "^2.5.0"
-Markdown = "^3.3.4"
-polib = "^1.1.0"
-python-magic = "^0.4.22"
-xlsxlite = "^0.2.0"
-colorama = "^0.4.6"
-gunicorn = "^22.0.0"
-iptools = "^0.7.0"
-iso8601 = "^0.1.14"
-phonenumbers = "^8.13.49"
-pycountry = "^22.3.5"
-python-dateutil = "^2.9.0"
-packaging = "^22.0"
-requests-toolbelt = "^1.0.0"
-chardet = "^4.0.0"
-openpyxl = "^3.1.5"
-ffmpeg-python = "^0.2.0"
-slack-sdk = "3.17.0"
-django-formtools = "^2.4.1"
-psycopg = { extras = ["pool"], version = "^3.2.1" }
-pillow = "^10.1.0"
-django-imagekit = "^5.0.0"
-iso639-lang = "^2.2.3"
-google-auth = "^2.30.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.4.2"

temba/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "10.1.4"
+__version__ = "10.1.7"
 
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.

temba/api/v2/serializers.py

@@ -135,10 +135,7 @@ def get_aliases(self, obj):
         return [alias.name for alias in obj.aliases.all()]
 
     def get_geometry(self, obj):
-        if self.context["include_geometry"] and obj.simplified_geometry:
-            return json.loads(obj.simplified_geometry.geojson)
-        else:
-            return None
+        return obj.geometry if self.context["include_geometry"] else None
 
     class Meta:
         model = AdminBoundary

temba/api/v2/tests/test_boundaries.py

@@ -1,4 +1,3 @@
-from django.contrib.gis.geos import GEOSGeometry
 from django.urls import reverse
 
 from temba.locations.models import BoundaryAlias
@@ -21,7 +20,7 @@ def test_endpoint(self):
         BoundaryAlias.create(self.org, self.admin, self.state2, "East Prov")
         BoundaryAlias.create(self.org2, self.admin2, self.state1, "Other Org")  # shouldn't be returned
 
-        self.state1.simplified_geometry = GEOSGeometry("MULTIPOLYGON(((1 1, 1 -1, -1 -1, -1 1, 1 1)))")
+        self.state1.geometry = {"type": "MultiPolygon", "coordinates": [[[[1, 1], [1, -1], [-1, -1], [-1, 1], [1, 1]]]]}
         self.state1.save()
 
         # test without geometry

temba/channels/tests.py

@@ -1236,6 +1236,7 @@ def test_msg_counts(self):
                     text="F",
                     direction="O",
                     msg_type="T",
+                    is_android=False,
                     created_on=datetime(2023, 6, 1, 13, 0, 30, 0, tzone.utc),
                     modified_on=datetime(2023, 6, 1, 13, 0, 30, 0, tzone.utc),
                 ),
@@ -1246,6 +1247,7 @@ def test_msg_counts(self):
                     text="G",
                     direction="O",
                     msg_type="T",
+                    is_android=False,
                     created_on=datetime(2023, 6, 1, 13, 0, 30, 0, tzone.utc),
                     modified_on=datetime(2023, 6, 1, 13, 0, 30, 0, tzone.utc),
                 ),
@@ -1256,6 +1258,7 @@ def test_msg_counts(self):
                     text="H",
                     direction="O",
                     msg_type="V",
+                    is_android=False,
                     created_on=datetime(2023, 6, 1, 13, 0, 30, 0, tzone.utc),
                     modified_on=datetime(2023, 6, 1, 13, 0, 30, 0, tzone.utc),
                 ),

temba/flows/migrations/0355_backfill_new_cat_counts.py

@@ -6,7 +6,7 @@
 from django.db.models import Sum
 
 
-def backfill_new_counts(apps, schema_editor):
+def backfill_new_counts(apps, schema_editor):  # pragma: no cover
     Flow = apps.get_model("flows", "Flow")
 
     flow_ids = list(Flow.objects.filter(is_active=True).order_by("id").values_list("id", flat=True))
@@ -24,7 +24,7 @@ def backfill_new_counts(apps, schema_editor):
         print(f"> updated counts for {num_backfilled} of {len(flow_ids)} flows")
 
 
-def backfill_for_flow(apps, flow):
+def backfill_for_flow(apps, flow):  # pragma: no cover
     FlowResultCount = apps.get_model("flows", "FlowResultCount")
 
     to_create = []

temba/flows/migrations/0356_update_triggers.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.4 on 2025-01-13 14:44
+
+from django.db import migrations
+
+SQL = """
+DROP TRIGGER temba_flowrun_delete ON flows_flowrun;
+DROP TRIGGER temba_flowrun_update_flowcategorycount ON flows_flowrun;
+
+DROP FUNCTION temba_flowrun_delete();
+DROP FUNCTION temba_update_flowcategorycount();
+DROP FUNCTION temba_update_category_counts(integer, json, json);
+DROP FUNCTION temba_insert_flowcategorycount(integer, text, json, integer);
+"""
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("flows", "0355_backfill_new_cat_counts"),
+        ("sql", "0007_squashed"),
+    ]
+
+    operations = [migrations.RunSQL(SQL)]

temba/flows/models.py

@@ -991,12 +991,16 @@ def delete(self):
         for start in self.starts.all():
             start.delete()
 
+        self.labels.clear()
+
         delete_in_batches(self.counts.all())
+        delete_in_batches(self.result_counts.all())
+
+        # TODO remove
         delete_in_batches(self.category_counts.all())
         delete_in_batches(self.path_counts.all())
         delete_in_batches(self.node_counts.all())
         delete_in_batches(self.status_counts.all())
-        self.labels.clear()
 
         super().delete()
 
@@ -1493,42 +1497,15 @@ class Meta:
 
 class FlowCategoryCount(BaseSquashableCount):
     """
-    TODO: replace by FlowResultCount
+    TODO: drop
     """
 
-    squash_over = ("flow_id", "node_uuid", "result_key", "result_name", "category_name")
-
     flow = models.ForeignKey(Flow, on_delete=models.PROTECT, related_name="category_counts")
     node_uuid = models.UUIDField(db_index=True)
     result_key = models.CharField(max_length=128)
     result_name = models.CharField(max_length=128)
     category_name = models.CharField(max_length=128)
 
-    @classmethod
-    def get_squash_query(cls, distinct_set: dict) -> tuple:  # pragma: no cover
-        sql = """
-        WITH removed as (
-          DELETE FROM %(table)s WHERE "id" IN (
-            SELECT "id" FROM %(table)s
-              WHERE "flow_id" = %%s AND "node_uuid" = %%s AND "result_key" = %%s AND "result_name" = %%s AND "category_name" = %%s
-              LIMIT 10000
-          ) RETURNING "count"
-        )
-        INSERT INTO %(table)s("flow_id", "node_uuid", "result_key", "result_name", "category_name", "count", "is_squashed")
-        VALUES (%%s, %%s, %%s, %%s, %%s, GREATEST(0, (SELECT SUM("count") FROM removed)), TRUE);
-        """ % {
-            "table": cls._meta.db_table
-        }
-
-        params = (
-            distinct_set["flow_id"],
-            distinct_set["node_uuid"],
-            distinct_set["result_key"],
-            distinct_set["result_name"],
-            distinct_set["category_name"],
-        ) * 2
-        return sql, params
-
     class Meta:
         indexes = [
             models.Index(

temba/flows/tasks.py

@@ -15,16 +15,7 @@
 from temba.utils.crons import cron_task
 from temba.utils.models import delete_in_batches
 
-from .models import (
-    Flow,
-    FlowActivityCount,
-    FlowCategoryCount,
-    FlowResultCount,
-    FlowRevision,
-    FlowRun,
-    FlowSession,
-    FlowStartCount,
-)
+from .models import Flow, FlowActivityCount, FlowResultCount, FlowRevision, FlowRun, FlowSession, FlowStartCount
 
 logger = logging.getLogger(__name__)
 
@@ -44,14 +35,9 @@ def update_session_wait_expires(flow_id):
 
 
 @cron_task(lock_timeout=7200)
-def squash_activity_counts():
+def squash_flow_counts():
     FlowActivityCount.squash()
     FlowResultCount.squash()
-
-
-@cron_task(lock_timeout=7200)
-def squash_flow_counts():
-    FlowCategoryCount.squash()
     FlowStartCount.squash()
 
 

temba/flows/tests/test_counts.py

@@ -4,7 +4,7 @@
 from django.utils import timezone
 
 from temba.flows.models import FlowActivityCount, FlowRun, FlowSession
-from temba.flows.tasks import squash_activity_counts
+from temba.flows.tasks import squash_flow_counts
 from temba.tests import TembaTest
 from temba.utils.uuid import uuid4
 
@@ -130,7 +130,7 @@ def create_runs(flow_status_pairs: tuple) -> list:
         self.assertEqual({"status:W": 2}, flow2.counts.scope_totals())
 
         # no difference after squashing
-        squash_activity_counts()
+        squash_flow_counts()
 
         self.assertEqual({"status:A": 2, "status:W": 1, "status:C": 1}, flow1.counts.scope_totals())
         self.assertEqual({"status:W": 2}, flow2.counts.scope_totals())
@@ -155,7 +155,7 @@ def create_runs(flow_status_pairs: tuple) -> list:
         self.assertEqual({"status:W": 0, "status:X": 1, "status:I": 2}, flow2.counts.scope_totals())
 
         # no difference after squashing except zeros gone
-        squash_activity_counts()
+        squash_flow_counts()
 
         self.assertEqual({"status:A": 2, "status:I": 4}, flow1.counts.scope_totals())
         self.assertEqual({"status:X": 1, "status:I": 2}, flow2.counts.scope_totals())
@@ -242,7 +242,7 @@ def test_squashing(self):
         self.assertEqual(0, flow2.counts.filter(scope="foo:2").sum())
         self.assertEqual(5, flow2.counts.filter(scope="foo:3").sum())
 
-        squash_activity_counts()
+        squash_flow_counts()
 
         self.assertEqual({"foo:1", "foo:2", "foo:3"}, set(flow1.counts.values_list("scope", flat=True)))
 
@@ -258,7 +258,7 @@ def test_squashing(self):
 
         flow2.counts.create(scope="foo:3", count=-5)  # unsquashed zero + squashed zero
 
-        squash_activity_counts()
+        squash_flow_counts()
 
         # flow2/foo:3 should be gone because it squashed to zero
         self.assertEqual({"foo:1"}, set(flow2.counts.values_list("scope", flat=True)))