Add last crawl and subscription status indicators to org list

backend/btrixcloud/basecrawls.py

@@ -1,7 +1,7 @@
 """ base crawl type """
 
 import os
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import Optional, List, Union, Dict, Any, Type, TYPE_CHECKING, cast, Tuple
 from uuid import UUID
 import urllib.parse
@@ -371,6 +371,8 @@ async def delete_crawls(
 
         await self.orgs.inc_org_bytes_stored(org.id, -size, type_)
 
+        await self.orgs.set_last_crawl_finished(org.id)
+
         quota_reached = self.orgs.storage_quota_reached(org)
 
         return res.deleted_count, cids_to_update, quota_reached
@@ -865,6 +867,21 @@ async def calculate_org_crawl_file_storage(
 
         return total_size, crawls_size, uploads_size
 
+    async def get_org_last_crawl_finished(self, oid: UUID) -> Optional[datetime]:
+        """Get last crawl finished time for org"""
+        last_crawl_finished: Optional[datetime] = None
+
+        cursor = (
+            self.crawls.find({"oid": oid, "finished": {"$ne": None}})
+            .sort({"finished": -1})
+            .limit(1)
+        )
+        last_crawl = await cursor.to_list(length=1)
+        if last_crawl:
+            last_crawl_finished = last_crawl[0].get("finished")
+
+        return last_crawl_finished
+
 
 # ============================================================================
 def init_base_crawls_api(app, user_dep, *args):

backend/btrixcloud/db.py

@@ -17,7 +17,7 @@
 from .migrations import BaseMigration
 
 
-CURR_DB_VERSION = "0035"
+CURR_DB_VERSION = "0038"
 
 
 # ============================================================================
@@ -94,7 +94,7 @@ async def update_and_prepare_db(
     """
     await ping_db(mdb)
     print("Database setup started", flush=True)
-    if await run_db_migrations(mdb, user_manager, page_ops):
+    if await run_db_migrations(mdb, user_manager, page_ops, org_ops):
         await drop_indexes(mdb)
     await create_indexes(
         org_ops,
@@ -113,7 +113,7 @@ async def update_and_prepare_db(
 
 
 # ============================================================================
-async def run_db_migrations(mdb, user_manager, page_ops):
+async def run_db_migrations(mdb, user_manager, page_ops, org_ops):
     """Run database migrations."""
 
     # if first run, just set version and exit
@@ -145,7 +145,9 @@ async def run_db_migrations(mdb, user_manager, page_ops):
             assert spec.loader
             migration_module = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(migration_module)
-            migration = migration_module.Migration(mdb, page_ops=page_ops)
+            migration = migration_module.Migration(
+                mdb, page_ops=page_ops, org_ops=org_ops
+            )
             if await migration.run():
                 migrations_run = True
         except ImportError as err:

backend/btrixcloud/migrations/migration_0038_org_last_crawl_finished.py

@@ -0,0 +1,43 @@
+"""
+Migration 0038 - Organization lastCrawlFinished field
+"""
+
+from btrixcloud.migrations import BaseMigration
+
+
+MIGRATION_VERSION = "0038"
+
+
+class Migration(BaseMigration):
+    """Migration class."""
+
+    # pylint: disable=unused-argument
+    def __init__(self, mdb, **kwargs):
+        super().__init__(mdb, migration_version=MIGRATION_VERSION)
+
+        self.org_ops = kwargs.get("org_ops")
+
+    async def migrate_up(self):
+        """Perform migration up. Set lastCrawlFinished for each org."""
+        # pylint: disable=duplicate-code, line-too-long
+        if self.org_ops is None:
+            print(
+                "Unable to set lastCrawlFinished for orgs, missing org_ops", flush=True
+            )
+            return
+
+        orgs_db = self.mdb["organizations"]
+        async for org_dict in orgs_db.find({}):
+            oid = org_dict.get("_id")
+
+            if org_dict.get("lastCrawlFinished"):
+                continue
+
+            try:
+                await self.org_ops.set_last_crawl_finished(oid)
+            # pylint: disable=broad-exception-caught
+            except Exception as err:
+                print(
+                    f"Error setting lastCrawlFinished for org {oid}: {err}",
+                    flush=True,
+                )

backend/btrixcloud/models.py

@@ -1439,6 +1439,8 @@ class OrgOut(BaseMongoModel):
     allowedProxies: list[str] = []
     crawlingDefaults: Optional[CrawlConfigDefaults] = None
 
+    lastCrawlFinished: Optional[datetime] = None
+
 
 # ============================================================================
 class Organization(BaseMongoModel):
@@ -1494,6 +1496,8 @@ class Organization(BaseMongoModel):
     allowedProxies: list[str] = []
     crawlingDefaults: Optional[CrawlConfigDefaults] = None
 
+    lastCrawlFinished: Optional[datetime] = None
+
     def is_owner(self, user):
         """Check if user is owner"""
         return self._is_auth(user, UserRole.OWNER)

backend/btrixcloud/operator/crawls.py

@@ -1506,6 +1506,7 @@ async def do_crawl_finished_tasks(
             await self.org_ops.inc_org_bytes_stored(
                 crawl.oid, status.filesAddedSize, "crawl"
             )
+            await self.org_ops.set_last_crawl_finished(crawl.oid)
             await self.coll_ops.add_successful_crawl_to_collections(crawl.id, crawl.cid)
 
         if state in FAILED_STATES:

backend/btrixcloud/orgs.py

@@ -220,12 +220,25 @@ async def get_orgs_for_user(
         sort_query = {"default": -1}
 
         if sort_by:
-            sort_fields = ("name", "slug", "readOnly")
+            sort_fields = (
+                "name",
+                "slug",
+                "readOnly",
+                "lastCrawlFinished",
+                "subscriptionStatus",
+                "subscriptionPlan",
+            )
             if sort_by not in sort_fields:
                 raise HTTPException(status_code=400, detail="invalid_sort_by")
             if sort_direction not in (1, -1):
                 raise HTTPException(status_code=400, detail="invalid_sort_direction")
 
+            if sort_by == "subscriptionStatus":
+                sort_by = "subscription.status"
+
+            if sort_by == "subscriptionPlan":
+                sort_by = "subscription.planId"
+
             # Do lexical sort of names
             if sort_by == "name":
                 sort_by = "nameLower"
@@ -1358,6 +1371,14 @@ async def recalculate_storage(self, org: Organization) -> dict[str, bool]:
 
         return {"success": True}
 
+    async def set_last_crawl_finished(self, oid: UUID):
+        """Recalculate and set lastCrawlFinished field on org"""
+        last_crawl_finished = await self.base_crawl_ops.get_org_last_crawl_finished(oid)
+        await self.orgs.find_one_and_update(
+            {"_id": oid},
+            {"$set": {"lastCrawlFinished": last_crawl_finished}},
+        )
+
 
 # ============================================================================
 # pylint: disable=too-many-statements, too-many-arguments

backend/test/test_org.py

@@ -755,3 +755,23 @@ def test_sort_orgs(admin_auth_headers):
         if last_name:
             assert org_name_lower < last_name
         last_name = org_name_lower
+
+    # Sort desc by lastCrawlFinished, ensure default org still first
+    r = requests.get(
+        f"{API_PREFIX}/orgs?sortBy=lastCrawlFinished&sortDirection=-1",
+        headers=admin_auth_headers,
+    )
+    data = r.json()
+    orgs = data["items"]
+
+    assert orgs[0]["default"]
+
+    other_orgs = orgs[1:]
+    last_last_crawl_finished = None
+    for org in other_orgs:
+        last_crawl_finished = org.get("lastCrawlFinished")
+        if not last_crawl_finished:
+            continue
+        if last_last_crawl_finished:
+            assert last_crawl_finished <= last_last_crawl_finished
+        last_last_crawl_finished = last_crawl_finished

frontend/src/components/orgs-list.ts

@@ -13,6 +13,7 @@ import { when } from "lit/directives/when.js";
 
 import { BtrixElement } from "@/classes/BtrixElement";
 import type { Dialog } from "@/components/ui/dialog";
+import { SubscriptionStatus } from "@/types/billing";
 import type { ProxiesAPIResponse, Proxy } from "@/types/crawler";
 import type { OrgData } from "@/utils/orgs";
 
@@ -25,7 +26,7 @@ import type { OrgData } from "@/utils/orgs";
 export class OrgsList extends BtrixElement {
   static styles = css`
     btrix-table {
-      grid-template-columns: min-content [clickable-start] 50ch auto auto auto [clickable-end] min-content;
+      grid-template-columns: min-content [clickable-start] minmax(auto, 50ch) auto auto auto [clickable-end] min-content;
     }
   `;
 
@@ -79,6 +80,9 @@ export class OrgsList extends BtrixElement {
           <btrix-table-header-cell class="px-2">
             ${msg("Bytes Stored")}
           </btrix-table-header-cell>
+          <btrix-table-header-cell class="px-2">
+            ${msg("Last Crawl")}
+          </btrix-table-header-cell>
           <btrix-table-header-cell>
             <span class="sr-only">${msg("Actions")}</span>
           </btrix-table-header-cell>
@@ -593,16 +597,82 @@ export class OrgsList extends BtrixElement {
       };
     }
 
+    let subscription = {
+      icon: none,
+      description: msg("No Subscription"),
+    };
+
+    if (org.subscription) {
+      switch (org.subscription.status) {
+        case SubscriptionStatus.Active:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-success"
+              name="nut-fill"
+              label=${msg("Active Subscription")}
+            ></sl-icon>`,
+            description: msg("Active Subscription"),
+          };
+          break;
+        case SubscriptionStatus.Trialing:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-neutral-400"
+              name="nut"
+              label=${msg("Trial")}
+            ></sl-icon>`,
+            description: msg("Trial"),
+          };
+          break;
+        case SubscriptionStatus.TrialingCanceled:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-danger"
+              name="nut"
+              label=${msg("Trial Cancelled")}
+            ></sl-icon>`,
+            description: msg("Trial Canceled"),
+          };
+          break;
+        case SubscriptionStatus.PausedPaymentFailed:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-warning"
+              name="nut-fill"
+              label=${msg("Payment Failed")}
+            ></sl-icon>`,
+            description: msg("Payment Failed"),
+          };
+          break;
+        case SubscriptionStatus.Cancelled:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-danger"
+              name="nut-fill"
+              label=${msg("Canceled")}
+            >
+            </sl-icon>`,
+            description: msg("Canceled"),
+          };
+          break;
+        default:
+          break;
+      }
+    }
+
     return html`
       <btrix-table-row
         class="${isUserOrg
           ? ""
           : "opacity-50"} cursor-pointer select-none border-b bg-neutral-0 transition-colors first-of-type:rounded-t last-of-type:rounded-b last-of-type:border-none focus-within:bg-neutral-50 hover:bg-neutral-50"
       >
-        <btrix-table-cell class="min-w-6 pl-2">
+        <btrix-table-cell class="min-w-6 gap-1 pl-2">
           <sl-tooltip content=${status.description}>
             ${status.icon}
           </sl-tooltip>
+          <sl-tooltip content=${subscription.description}>
+            ${subscription.icon}
+          </sl-tooltip>
         </btrix-table-cell>
         <btrix-table-cell class="p-2" rowClickTarget="a">
           <a
@@ -619,7 +689,6 @@ export class OrgsList extends BtrixElement {
               : org.name}
           </a>
         </btrix-table-cell>
-
         <btrix-table-cell class="p-2">
           ${this.localize.date(org.created, { dateStyle: "short" })}
         </btrix-table-cell>
@@ -631,6 +700,11 @@ export class OrgsList extends BtrixElement {
             ? this.localize.bytes(org.bytesStored, { unitDisplay: "narrow" })
             : none}
         </btrix-table-cell>
+        <btrix-table-cell class="p-2">
+          ${org.lastCrawlFinished
+            ? this.localize.date(org.lastCrawlFinished, { dateStyle: "short" })
+            : none}
+        </btrix-table-cell>
         <btrix-table-cell class="p-1">
           <btrix-overflow-dropdown
             @click=${(e: MouseEvent) => e.stopPropagation()}

frontend/src/types/org.ts

@@ -93,6 +93,7 @@ export const orgDataSchema = z.object({
   crawlingDefaults: crawlingDefaultsSchema.nullable(),
   allowSharedProxies: z.boolean(),
   allowedProxies: z.array(z.string()),
+  lastCrawlFinished: apiDateSchema.nullable(),
 });
 export type OrgData = z.infer<typeof orgDataSchema>;