From e51d93c7df05901d336bed9f7fd41f2cd519b45b Mon Sep 17 00:00:00 2001
From: Alexey Kartashov <alexey.kartashov@scylladb.com>
Date: Mon, 20 Jan 2025 08:46:21 +0100
Subject: [PATCH] improvement(ViewDashboard): Per-widget item filtering

This commit implements per-widget entitity filtering for views,
allowing users to select which tests/groups/releases they want shown on
the widget in a specific view. To facilitate this, stat fetching has
been modified to accept widget id (position) of a view and then filter
tests accordingly. On the frontend side to facilitate conflict-free stat
fetching (for example from two TestDashboards) a hashmap has been
introduced to handle multiple stat objects and to dispatch them
correctly to each widget.

Caveats:
    * Release Planner views in general are not supported and contain
      several unwanted interactions:
        * If test dashboard is modified to filter anything, release
          planner stat bar will be gray (since there's no more anchor
          for it to get the stats from)
        * Anything version related will conflict
    * TestDashboard has a setting for a specific ScyllaVersion filter
      preset - this is not compatible and will make that filter set only
      output that version, which might be undesirable.
    * Aside from TestDashboard, currently no other widgets implement
      stat fetch, so filtering for them might not be interesting.
      However the widget will now receive filtered set of tests, which
      should make the performance summaries automatically support this
      feature.

Fixes #393
---
 argus/backend/controller/view_api.py          | 15 ++-
 argus/backend/service/planner_service.py      |  3 +
 argus/backend/service/stats.py                | 12 ++-
 frontend/AdminPanel/ViewWidget.svelte         | 20 +++-
 frontend/AdminPanel/ViewsManager.svelte       |  2 +-
 frontend/Common/ViewTypes.js                  |  2 +
 .../ReleaseDashboard/TestDashboard.svelte     |  2 +
 frontend/ReleasePlanner/ReleasePlan.svelte    |  5 +-
 frontend/Views/ViewDashboard.svelte           | 98 ++++++++++++++++---
 .../Views/Widgets/ViewTestDashboard.svelte    |  2 +
 10 files changed, 137 insertions(+), 24 deletions(-)

diff --git a/argus/backend/controller/view_api.py b/argus/backend/controller/view_api.py
index d1dca588..d47112bc 100644
--- a/argus/backend/controller/view_api.py
+++ b/argus/backend/controller/view_api.py
@@ -135,8 +135,11 @@ def view_stats():
     version = request.args.get("productVersion", None)
     include_no_version = bool(int(request.args.get("includeNoVersion", True)))
     force = bool(int(request.args.get("force",  0)))
+    widget_id = request.args.get("widgetId", None)
+    if widget_id:
+        widget_id = int(widget_id)
     collector = ViewStatsCollector(view_id=view_id, filter=version)
-    stats = collector.collect(limited=limited, force=force, include_no_version=include_no_version)
+    stats = collector.collect(limited=limited, force=force, include_no_version=include_no_version, widget_id=widget_id)
 
     res = jsonify({
         "status": "ok",
@@ -164,3 +167,13 @@ def view_resolve(view_id: str):
         "status": "ok",
         "response": res
     }
+
+@bp.route("/<string:view_id>/resolve/tests", methods=["GET"])
+@api_login_required
+def view_resolve_tests(view_id: str):
+    service = UserViewService()
+    res = service.resolve_view_tests(view_id)
+    return {
+        "status": "ok",
+        "response": res
+    }
diff --git a/argus/backend/service/planner_service.py b/argus/backend/service/planner_service.py
index 9dc5049e..119c04e0 100644
--- a/argus/backend/service/planner_service.py
+++ b/argus/backend/service/planner_service.py
@@ -81,6 +81,7 @@ class PlanningService:
         {
             "position": 1,
             "type": "githubIssues",
+            "filter": [],
             "settings": {
                 "submitDisabled": True,
                 "aggregateByIssue": True
@@ -89,6 +90,7 @@ class PlanningService:
         {
             "position": 2,
             "type": "releaseStats",
+            "filter": [],
             "settings": {
                 "horizontal": False,
                 "displayExtendedStats": True,
@@ -98,6 +100,7 @@ class PlanningService:
         {
             "position": 3,
             "type": "testDashboard",
+            "filter": [],
             "settings": {
                 "targetVersion": True,
                 "versionsIncludeNoVersion": False,
diff --git a/argus/backend/service/stats.py b/argus/backend/service/stats.py
index ab328284..7c6d3a6e 100644
--- a/argus/backend/service/stats.py
+++ b/argus/backend/service/stats.py
@@ -1,9 +1,10 @@
 from collections import defaultdict
 from functools import reduce
+import json
 import logging
 
 from datetime import datetime
-from typing import TypedDict
+from typing import Any, TypedDict
 from uuid import UUID
 
 from cassandra.cqlengine.models import Model
@@ -531,11 +532,18 @@ def __init__(self, view_id: UUID, filter: str | None = None) -> None:
         self.view_id = view_id
         self.filter = filter
 
-    def collect(self, limited=False, force=False, include_no_version=False) -> dict:
+    def collect(self, limited=False, force=False, include_no_version = False, widget_id: int = None) -> dict:
         self.view: ArgusUserView = ArgusUserView.get(id=self.view_id)
+        widget: dict[str, Any] | None = None
+        if isinstance(widget_id, int): 
+            settings = json.loads(self.view.widget_settings)
+            widget = next((widget for widget in settings if widget["position"] == widget_id), None)
         all_tests: list[ArgusTest] = []
         for slice in chunk(self.view.tests):
             all_tests.extend(ArgusTest.filter(id__in=slice).all())
+
+        if widget and widget.get("filter"):
+            all_tests = [test for test in all_tests if any(str(test[key]) in widget["filter"] for key in ["id", "group_id", "release_id"])]
         build_ids = reduce(lambda acc, test: acc[test.plugin_name or "unknown"].append(test.build_system_id) or acc, all_tests, defaultdict(list))
         self.view_rows = [futures for plugin in all_plugin_models()
                              for futures in plugin.get_stats_for_release(release=self.view, build_ids=build_ids.get(plugin._plugin_name, []))]
diff --git a/frontend/AdminPanel/ViewWidget.svelte b/frontend/AdminPanel/ViewWidget.svelte
index 9f55ce6d..7e7d4ca3 100644
--- a/frontend/AdminPanel/ViewWidget.svelte
+++ b/frontend/AdminPanel/ViewWidget.svelte
@@ -1,14 +1,16 @@
 <script>
     import Fa from "svelte-fa";
     import { WIDGET_TYPES, Widget } from "../Common/ViewTypes";
-    import { faEdit, faList, faTrash } from "@fortawesome/free-solid-svg-icons";
+    import { faEdit, faList, faQuestionCircle, faTrash } from "@fortawesome/free-solid-svg-icons";
     import { createEventDispatcher, onMount } from "svelte";
+    import { titleCase } from "../Common/TextUtils";
 
 
     /**
      * @type {Widget}
      */
     export let widgetSettings = {};
+    export let items = [];
 
     let editingSettings = false;
     const dispatch = createEventDispatcher();
@@ -24,7 +26,10 @@
             });
     };
 
-    onMount(() => populateWidgetSettings());
+    onMount(populateWidgetSettings);
+    onMount(() => {
+        if (!widgetSettings.filter) widgetSettings.filter = [];
+    });
 </script>
 
 <div class="mb-2 rounded bg-white p-2">
@@ -62,6 +67,17 @@
                     </div>
                 </div>
                 <div>
+                    <div>
+                        <div class="mb-2">
+                            Item Filter <span title="Only selected items will be shown in the widget. No selection - Show all"><Fa icon={faQuestionCircle}/></span>
+                            <button class="btn btn-sm btn-outline-danger" on:click={() => (widgetSettings.filter = [])}>Clear All</button>
+                        </div>
+                        <select class="form-select" multiple size=10 bind:value={widgetSettings.filter}>
+                            {#each items as item}
+                                <option value="{item.id}"><span class="fw-bold">[{titleCase(item.type)}]</span> {item.pretty_name || item.name}</option>
+                            {/each}
+                        </select>
+                    </div>
                     {#each Object.entries(WIDGET_DEF.settingDefinitions) as [settingName, definition] (settingName)}
                         {#if typeof definition.type !== "string"}
                             <svelte:component this={definition.type} bind:settings={widgetSettings.settings} {definition} {settingName} />
diff --git a/frontend/AdminPanel/ViewsManager.svelte b/frontend/AdminPanel/ViewsManager.svelte
index eee38236..f5eff61f 100644
--- a/frontend/AdminPanel/ViewsManager.svelte
+++ b/frontend/AdminPanel/ViewsManager.svelte
@@ -447,7 +447,7 @@
                             </button>
                         </div>
                         {#each newWidgets as widget (widget.position)}
-                            <ViewWidget bind:widgetSettings={widget} on:removeWidget={removeWidget}/>
+                            <ViewWidget bind:widgetSettings={widget} items={newView.items} on:removeWidget={removeWidget}/>
                         {/each}
                     </div>
                 </div>
diff --git a/frontend/Common/ViewTypes.js b/frontend/Common/ViewTypes.js
index f0fcd042..9f34c297 100644
--- a/frontend/Common/ViewTypes.js
+++ b/frontend/Common/ViewTypes.js
@@ -16,9 +16,11 @@ export class Widget {
         this.position = position;
         this.type = type;
         this.settings = settings;
+        this.filter = [];
     }
 }
 
+export const GLOBAL_STATS_KEY = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
 
 export const WIDGET_TYPES = {
     UNSUPPORTED: {
diff --git a/frontend/ReleaseDashboard/TestDashboard.svelte b/frontend/ReleaseDashboard/TestDashboard.svelte
index 56f80410..5f342782 100644
--- a/frontend/ReleaseDashboard/TestDashboard.svelte
+++ b/frontend/ReleaseDashboard/TestDashboard.svelte
@@ -32,6 +32,7 @@
     import { timestampToISODate } from "../Common/DateUtils";
     export let dashboardObject;
     export let dashboardObjectType = "release";
+    export let widgetId;
     export let clickedTests = {};
     export let productVersion;
     export let settings = {};
@@ -142,6 +143,7 @@
             viewId: dashboardObject.id,
             limited: new Number(false),
             force: new Number(true),
+            widgetId: widgetId,
             includeNoVersion: new Number(versionsIncludeNoVersion),
             productVersion: productVersion ?? "",
         });
diff --git a/frontend/ReleasePlanner/ReleasePlan.svelte b/frontend/ReleasePlanner/ReleasePlan.svelte
index a4addec2..0825c1d6 100644
--- a/frontend/ReleasePlanner/ReleasePlan.svelte
+++ b/frontend/ReleasePlanner/ReleasePlan.svelte
@@ -8,12 +8,13 @@
     import ViewDashboard from "../Views/ViewDashboard.svelte";
     import ReleaseStats from "../Stats/ReleaseStats.svelte";
     import { faEdit } from "@fortawesome/free-regular-svg-icons";
+    import { GLOBAL_STATS_KEY } from "../Common/ViewTypes";
 
 
     export let plan;
     export let users;
     export let expandedPlans;
-    let planStats;
+    let planStats = {};
 
     let owner;
     $: owner = getUser(plan.owner, users);
@@ -66,7 +67,7 @@
             {plan.target_version}
             {#if planStats}
                 <div>
-                    <ReleaseStats releaseStats={planStats} />
+                    <ReleaseStats releaseStats={planStats[GLOBAL_STATS_KEY]} />
                 </div>
             {/if}
         </div>
diff --git a/frontend/Views/ViewDashboard.svelte b/frontend/Views/ViewDashboard.svelte
index 13218bdf..a17f8740 100644
--- a/frontend/Views/ViewDashboard.svelte
+++ b/frontend/Views/ViewDashboard.svelte
@@ -1,10 +1,17 @@
 <script>
-    import { WIDGET_TYPES } from "../Common/ViewTypes";
+    import { GLOBAL_STATS_KEY, WIDGET_TYPES } from "../Common/ViewTypes";
     import { sendMessage } from "../Stores/AlertStore";
+    import sha1 from "js-sha1";
     export let view;
-    export let stats = undefined;
+    export let stats = {};
     export let productVersion;
     let clickedTests = {};
+    let resolvedTests = [];
+    const versionDispatch = {
+        GLOBAL_STATS_KEY: productVersion,
+    };
+
+    console.log(versionDispatch);
 
     const handleTestClick = function (detail) {
         if (detail.start_time == 0) {
@@ -20,6 +27,40 @@
         }
     };
 
+    const resolveViewTests = async function() {
+        if (resolvedTests.length > 0) {
+            return resolvedTests;
+        }
+        try {
+            let response = await fetch(`/api/v1/views/${view.id}/resolve/tests`);
+            if (response.status !== 200) {
+                throw new Error("Non-200 status code returned from API.");
+            }
+            let json = await response.json();
+            if (json.status === "ok") {
+                resolvedTests = json.response;
+                return resolvedTests;
+            } else {
+                throw json;
+            }
+        } catch (error) {
+            if (error?.status === "error") {
+                sendMessage(
+                    "error",
+                    `API Error resolving view tests.\nMessage: ${error.response.arguments[0]}`,
+                    "ViewDashboard::resolveViewTests"
+                );
+            } else {
+                sendMessage(
+                    "error",
+                    "A backend error occurred during view test resolution.",
+                    "ViewDashboard::resolveViewTests"
+                );
+                console.log(error);
+            }
+        }
+    };
+
     const handleDeleteRequest = function(e) {
         let key = e.detail.id;
         if (clickedTests[key]) {
@@ -45,6 +86,26 @@
         });
     };
 
+    const calculateWidgetStatsKey = function (widget) {
+        return sha1((widget.filter ?? []).join(""));
+    };
+
+    const filterViewForWidget = async function (widget) {
+        let viewCopy = structuredClone(view);
+        if (!widget.filter || widget.filter.length === 0) {
+            return viewCopy;
+        }
+        let tests = await resolveViewTests();
+        tests = tests.reduce((acc, test) => {
+            acc[test.id] = test;
+            return acc;
+        }, {});
+        viewCopy.tests = viewCopy.tests.filter((item) => {
+            const test = tests[item.id] ?? {};
+            return ["id", "release_id", "group_id"].some((key) => widget.filter.includes(test[key]));
+        });
+        return viewCopy;
+    };
 </script>
 
 <div class="rounded bg-white m-2 shadow-sm p-2">
@@ -54,20 +115,25 @@
     <div class="mb-2">
         {#each view.widget_settings as widget}
             <div class="mb-2">
-                <svelte:component
-                    this={WIDGET_TYPES[widget.type]?.type ?? WIDGET_TYPES.UNSUPPORTED.type}
-                    dashboardObject={view}
-                    dashboardObjectType="view"
-                    settings={widget.settings}
-                    bind:stats={stats}
-                    bind:productVersion={productVersion}
-                    bind:clickedTests={clickedTests}
-                    on:statsUpdate
-                    on:testClick={(e) => handleTestClick(e.detail)}
-                    on:versionChange
-                    on:quickSelect={handleQuickSelect}
-                    on:deleteRequest={handleDeleteRequest}
-                />
+                {#await filterViewForWidget(widget)}
+                    <span class="spinner-grow spinner-grow-sm"></span> Loading...
+                {:then view}
+                    <svelte:component
+                        this={WIDGET_TYPES[widget.type]?.type ?? WIDGET_TYPES.UNSUPPORTED.type}
+                        widgetId={widget.position}
+                        dashboardObject={view}
+                        dashboardObjectType="view"
+                        settings={widget.settings}
+                        bind:stats={stats[calculateWidgetStatsKey(widget)]}
+                        bind:productVersion={versionDispatch[calculateWidgetStatsKey(widget)]}
+                        bind:clickedTests={clickedTests}
+                        on:statsUpdate
+                        on:testClick={(e) => handleTestClick(e.detail)}
+                        on:versionChange
+                        on:quickSelect={handleQuickSelect}
+                        on:deleteRequest={handleDeleteRequest}
+                    />
+                {/await}
             </div>
         {:else}
             <div class="alert alert-danger">No widgets defined for view!</div>
diff --git a/frontend/Views/Widgets/ViewTestDashboard.svelte b/frontend/Views/Widgets/ViewTestDashboard.svelte
index 8c3ad7a7..c7742075 100644
--- a/frontend/Views/Widgets/ViewTestDashboard.svelte
+++ b/frontend/Views/Widgets/ViewTestDashboard.svelte
@@ -5,6 +5,7 @@
     export let settings;
     export let productVersion;
     export let clickedTests;
+    export let widgetId;
 
     import TestDashboard from "../../ReleaseDashboard/TestDashboard.svelte";
     import TestPopoutSelector from "../../ReleaseDashboard/TestPopoutSelector.svelte";
@@ -16,6 +17,7 @@
             dashboardObject={dashboardObject}
             {dashboardObjectType}
             {settings}
+            {widgetId}
             bind:productVersion={productVersion}
             bind:stats={stats}
             bind:clickedTests={clickedTests}