Allow filtering samples by compound expressions including multiple scorers

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Inspect View: display tool error messages in transcript when tool errors occur.
 - Inspect View: display any completed samples even if the task fails because of an error
 - Inspect View: don't display the 'input' column heading if there isn't an input
+- Inspect View: allow filtering samples by compound expressions including multiple scorers.
 - Open AI: Handle additional bad request status codes (mapping them to appropriate `StopReason`)
 - Open AI: Use new `max_completion_tokens` option for o1 full.
 - Web Browser: raise error when both `error` and `web_at` fields are present in response.

src/inspect_ai/_view/www/App.css

@@ -650,6 +650,29 @@ table.table.table-sm td {
   height: auto !important;
 }
 
+[data-tooltip] {
+  position: relative;
+}
+[data-tooltip]:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  line-height: 1.25;
+  background: var(--bs-gray-300);
+  color: var(--bs-body-color);
+  padding: 4px 8px;
+  border-radius: 4px;
+  border: 1px solid var(--bs-gray-400);
+  box-shadow: 0 2px 10px 0 rgba(0, 0, 0, 0.25);
+  white-space: pre-wrap;
+  width: max-content;
+  max-width: 400px;
+  z-index: 1000;
+}
+[data-tooltip][data-tooltip-position="bottom-left"]:hover::after {
+  right: 0%;
+  top: 100%;
+}
+
 /* ANSI Coloring */
 .ansi-display {
   font-family: monospace;
@@ -706,6 +729,10 @@ from {
   overflow: unset;
 }
 
+.custom-dropdown-item:hover {
+  background-color: var(--bs-secondary-bg);
+}
+
 pre[class*="language-"].tool-output,
 .tool-output {
   background-color: #f8f8f8;

src/inspect_ai/_view/www/dist/assets/index.css

@@ -14923,6 +14923,29 @@ table.table.table-sm td {
   height: auto !important;
 }
 
+[data-tooltip] {
+  position: relative;
+}
+[data-tooltip]:hover::after {
+  content: attr(data-tooltip);
+  position: absolute;
+  line-height: 1.25;
+  background: var(--bs-gray-300);
+  color: var(--bs-body-color);
+  padding: 4px 8px;
+  border-radius: 4px;
+  border: 1px solid var(--bs-gray-400);
+  box-shadow: 0 2px 10px 0 rgba(0, 0, 0, 0.25);
+  white-space: pre-wrap;
+  width: max-content;
+  max-width: 400px;
+  z-index: 1000;
+}
+[data-tooltip][data-tooltip-position="bottom-left"]:hover::after {
+  right: 0%;
+  top: 100%;
+}
+
 /* ANSI Coloring */
 .ansi-display {
   font-family: monospace;
@@ -14979,6 +15002,10 @@ from {
   overflow: unset;
 }
 
+.custom-dropdown-item:hover {
+  background-color: var(--bs-secondary-bg);
+}
+
 pre[class*="language-"].tool-output,
 .tool-output {
   background-color: #f8f8f8;

src/inspect_ai/_view/www/package.json

@@ -30,8 +30,10 @@
     "bootstrap": "^5.3.3",
     "bootstrap-icons": "^1.11.3",
     "clipboard": "^2.0.11",
+    "codemirror": "^6.0.1",
     "fast-json-patch": "^3.1.1",
     "fflate": "^0.8.2",
+    "filtrex": "^3.1.0",
     "htm": "^3.1.1",
     "json": "^11.0.0",
     "json5": "^2.2.3",

src/inspect_ai/_view/www/src/App.mjs

@@ -38,7 +38,7 @@ import {
 } from "./samples/SamplesDescriptor.mjs";
 import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs";
 import { resolveAttachments } from "./utils/attachments.mjs";
-import { filterFnForType } from "./samples/tools/filters.mjs";
+import { filterSamples } from "./samples/tools/filters.mjs";
 
 import {
   kEvalWorkspaceTabId,
@@ -308,21 +308,19 @@ export function App({
 
   useEffect(() => {
     const samples = selectedLog?.contents?.sampleSummaries || [];
-    const filtered = samples.filter((sample) => {
+    const { result: prefiltered } = filterSamples(
+      evalDescriptor,
+      samples,
+      filter?.value,
+    );
+    const filtered = prefiltered.filter((sample) => {
       // Filter by epoch if specified
       if (epoch && epoch !== "all") {
         if (epoch !== sample.epoch + "") {
           return false;
         }
       }
-
-      // Apply the filter
-      const filterFn = filterFnForType(filter);
-      if (filterFn && filter.value) {
-        return filterFn(samplesDescriptor, sample, filter.value);
-      } else {
-        return true;
-      }
+      return true;
     });
 
     // Sort the samples

src/inspect_ai/_view/www/src/Types.mjs

@@ -25,7 +25,6 @@
 /**
  * @typedef {Object} ScoreFilter
  * @property {string} [value]
- * @property {string} [type]
  */
 
 /**

src/inspect_ai/_view/www/src/navbar/Navbar.mjs

@@ -19,6 +19,7 @@ import { SecondaryBar } from "./SecondaryBar.mjs";
  * @param {import("../types/log").EvalPlan} [props.evalPlan] - The EvalSpec
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats
  * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor
  * @param {string} [props.status] - the status
  * @param {boolean} props.offcanvas - Are we in offcanvas mode?
  * @param {boolean} props.showToggle - Should we show the toggle?
@@ -32,6 +33,7 @@ export const Navbar = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   showToggle,
   offcanvas,
   status,
@@ -182,6 +184,7 @@ export const Navbar = ({
           evalResults=${evalResults}
           evalStats=${evalStats}
           samples=${samples}
+          evalDescriptor=${evalDescriptor}
           status=${status}
           style=${{ gridColumn: "1/-1" }}
         />

src/inspect_ai/_view/www/src/navbar/SecondaryBar.mjs

@@ -3,6 +3,7 @@ import { html } from "htm/preact";
 import { LabeledValue } from "../components/LabeledValue.mjs";
 import { formatDataset, formatDuration } from "../utils/Format.mjs";
 import { ExpandablePanel } from "../components/ExpandablePanel.mjs";
+import { scoreFilterItems } from "../samples/tools/filters.mjs";
 
 /**
  * Renders the Navbar
@@ -13,6 +14,7 @@ import { ExpandablePanel } from "../components/ExpandablePanel.mjs";
  * @param {import("../types/log").EvalResults} [props.evalResults] - The EvalResults
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats
  * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor
  * @param {string} [props.status] - the status
  * @param {Map<string, string>} [props.style] - is this off canvas
  *
@@ -24,6 +26,7 @@ export const SecondaryBar = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   status,
   style,
 }) => {
@@ -56,19 +59,10 @@ export const SecondaryBar = ({
 `,
   });
 
-  const label = evalResults?.scores.length > 1 ? "Scorers" : "Scorer";
-  values.push({
-    size: "minmax(12%, auto)",
-    value: html`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: hasConfig ? "left" : "center" }}>
-    <${ScorerSummary} 
-      scorers=${evalResults?.scores} />
-  </${LabeledValue}>`,
-  });
-
   if (hasConfig) {
     values.push({
       size: "minmax(12%, auto)",
-      value: html`<${LabeledValue} label="Config" style=${{ justifySelf: "right" }}>
+      value: html`<${LabeledValue} label="Config" style=${{ justifySelf: "center" }}>
       <${ParamSummary} params=${hyperparameters}/>
     </${LabeledValue}>`,
     });
@@ -81,11 +75,20 @@ export const SecondaryBar = ({
   values.push({
     size: "minmax(12%, auto)",
     value: html`
-      <${LabeledValue} label="Duration" style=${{ justifySelf: "right" }}>
+      <${LabeledValue} label="Duration" style=${{ justifySelf: "center" }}>
         ${totalDuration}
       </${LabeledValue}>`,
   });
 
+  const label = evalResults?.scores.length > 1 ? "Scorers" : "Scorer";
+  values.push({
+    size: "minmax(12%, auto)",
+    value: html`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: "right" }}>
+    <${ScorerSummary}
+      evalDescriptor=${evalDescriptor} />
+  </${LabeledValue}>`,
+  });
+
   return html`
     <${ExpandablePanel} style=${{ margin: "0", ...style }} collapse=${true} lines=${4}>
     <div
@@ -124,17 +127,23 @@ const DatasetSummary = ({ dataset, samples, epochs, style }) => {
   `;
 };
 
-const ScorerSummary = ({ scorers }) => {
-  if (!scorers) {
+const ScorerSummary = ({ evalDescriptor }) => {
+  if (!evalDescriptor) {
     return "";
   }
 
-  const uniqScorers = new Set();
-  scorers.forEach((scorer) => {
-    uniqScorers.add(scorer.name);
-  });
+  const items = scoreFilterItems(evalDescriptor);
 
-  return Array.from(uniqScorers).join(", ");
+  return html`
+    <span style=${{ position: "relative" }}>
+      ${Array.from(items).map(
+        (item, index) => html`
+          ${index > 0 ? ", " : ""}
+          <span title=${item.tooltip}>${item.canonicalName}</span>
+        `,
+      )}
+    </span>
+  `;
 };
 
 /**

src/inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs

@@ -83,6 +83,29 @@ import {
  * @property {number} normalized.limit - Normalized size of the limit message.
  */
 
+/**
+ * @param {import("../Types.mjs").ScoreLabel | undefined} scoreLabel
+ * @returns {string}
+ */
+export const scoreLabelKey = (scoreLabel) => {
+  if (!scoreLabel) {
+    return "No score key";
+  }
+  return `${scoreLabel.scorer}.${scoreLabel.name}`;
+};
+
+/**
+ * @param {string} key
+ * @returns {import("../Types.mjs").ScoreLabel | undefined}
+ */
+export const parseScoreLabelKey = (key) => {
+  if (key == "No score key") {
+    return undefined;
+  }
+  const [scorer, name] = key.split(".");
+  return { scorer, name };
+};
+
 /**
  * @param {import("../Types.mjs").ScoreLabel[]} scores - the list of available scores
  * @param {import("../api/Types.mjs").SampleSummary[]} samples - the list of sample summaries
@@ -165,17 +188,6 @@ export const createEvalDescriptor = (scores, samples, epochs) => {
     return undefined;
   };
 
-  /**
-   * @param {import("../Types.mjs").ScoreLabel} [scoreLabel]
-   * @returns {string}
-   */
-  const scoreLabelKey = (scoreLabel) => {
-    if (!scoreLabel) {
-      return "No score key";
-    }
-    return `${scoreLabel.scorer}.${scoreLabel.name}`;
-  };
-
   /**
    * The EvalDescriptor is memoized. Compute all descriptors now to avoid duplicate work.
    * @type {Map<string, ScoreDescriptor>}
@@ -462,7 +474,7 @@ const scoreCategorizers = [
      * @returns {ScoreDescriptor} a ScoreDescriptor
      */
     describe: (values, types) => {
-      if (values.length === 2 && types.length === 1 && types[0] === "boolean") {
+      if (types.length === 1 && types[0] === "boolean") {
         return booleanScoreCategorizer();
       }
     },

src/inspect_ai/_view/www/src/samples/SamplesTools.mjs

@@ -23,6 +23,14 @@ export const SampleTools = (props) => {
   const hasEpochs = epochs > 1;
   const tools = [];
 
+  tools.push(
+    html`<${SampleFilter}
+      evalDescriptor=${sampleDescriptor.evalDescriptor}
+      filter=${filter}
+      filterChanged=${filterChanged}
+    />`,
+  );
+
   if (scores.length > 1) {
     tools.push(
       html`<${SelectScorer}
@@ -43,14 +51,6 @@ export const SampleTools = (props) => {
     );
   }
 
-  tools.push(
-    html`<${SampleFilter}
-      filter=${filter}
-      filterChanged=${filterChanged}
-      descriptor=${sampleDescriptor}
-    />`,
-  );
-
   tools.push(
     html`<${SortFilter}
       sampleDescriptor=${sampleDescriptor}