Feature/vscode pyenv (#803)

* Use subdirectory environments when present

- When running or debugging a task, use any discovered environments in subdirectories when present.

- Introduce setting to disable this (on by default).

* Update extension version

---------

Co-authored-by: jjallaire-aisi <[email protected]>

docs/vscode.qmd

@@ -80,6 +80,11 @@ The activity bar has four panels:
 
 -   **Task** provides a way to tweak the CLI arguments passed to `inspect eval` when it is run from the user interface.
 
+## Python Environments
+
+When running and debugging Inspect evaluations, the Inspect extension will attempt to use python environments that it discovers in the task subfolder and its parent folders (all the way to the workspace root). It will use the first environment that it discovers, otherwise it will use the python interpreter configured for the workspace. Note that since the extension will use the sub-environments, Inspect must be installed in any of the environments to be used.
+
+You can control this behavior with the `Use Subdirectory Environments`. If you disable this setting, the globally configured interpreter will always be used when running or debugging evaluations, even when environments are present in subdirectories.
 
 ## Troubleshooting
 

tools/vscode/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.40
+
+- When running and debuggin Inspect evals, the extension will by default use any environments present in the task folder or parent folders (up to the workspace). To always use the workspace environment, change the 'Use Subdirectory Environments` setting.
+
 ## 0.3.39
 
 - Fix an issue that would cause view to be unable to display when using VSCode extension with Inpsect version 0.3.42

tools/vscode/package.json

@@ -7,7 +7,7 @@
   "author": {
     "name": "UK AI Safety Institute"
   },
-  "version": "0.3.39",
+  "version": "0.3.40",
   "license": "MIT",
   "homepage": "https://inspect.ai-safety-institute.org.uk/",
   "repository": {
@@ -242,6 +242,12 @@
           "default": true,
           "description": "Limit evaluation to one sample when debugging.",
           "order": 5
+        },
+        "inspect_ai.useSubdirectoryEnvironments": {
+          "type": "boolean",
+          "default": true,
+          "description": "Run and debug inspect commands using subdirectory environments when present.",
+          "order": 2
         }
       }
     },

tools/vscode/src/core/python/env.ts

@@ -0,0 +1,58 @@
+import * as fs from "fs";
+import * as os from "os";
+import { existsSync } from "node:fs";
+import path, { join } from "path";
+import { AbsolutePath, toAbsolutePath } from "../path";
+
+
+export function findEnvPythonPath(startDir: AbsolutePath, baseDir: AbsolutePath): AbsolutePath | null {
+  let currentDir = startDir;
+  while (currentDir.path !== baseDir.path) {
+
+    // Look for a pythong environment
+    const pythonPath = findEnvPython(currentDir);
+    if (pythonPath) {
+      return toAbsolutePath(pythonPath);
+    }
+
+    // Move to the parent directory
+    currentDir = currentDir.dirname();
+  }
+
+  // No Python environment found
+  return null;
+}
+
+// Helper function to search for Python environment in a given directory
+function findEnvPython(directory: AbsolutePath): string | null {
+  const items = fs.readdirSync(directory.path);
+
+  // Filter only directories and check if any is an environment directory
+  const envDir = items
+    .map((item) => path.join(directory.path, item))
+    .filter((filePath) => fs.statSync(filePath).isDirectory())
+    .find(isEnvDir);
+
+  if (envDir) {
+    return getPythonPath(envDir);
+  }
+
+  return null;
+}
+
+function getPythonPath(dir: string): string | null {
+  const pythonSuffixes = os.platform() === "win32" ? ["Scripts/python.exe", "python.exe"] : ["bin/python3", "bin/python"];
+  for (const suffix of pythonSuffixes) {
+    const fullPath = path.join(dir, suffix);
+    if (fs.existsSync(fullPath)) {
+      return fullPath;
+    }
+  }
+
+  return null;
+}
+
+function isEnvDir(dir: string) {
+  return existsSync(join(dir, "pyvenv.cfg")) ||
+    existsSync(join(dir, "conda-meta"));
+}

tools/vscode/src/core/python/index.ts

@@ -1,4 +1,5 @@
 
 export * from './code';
 export * from './exec';
-export * from './interpreter';
+export * from './interpreter';
+export * from "./env";

tools/vscode/src/providers/inspect/inspect-eval.ts

@@ -18,6 +18,7 @@ import { inspectBinPath } from "../../inspect/props";
 import { activeWorkspaceFolder } from "../../core/workspace";
 import { findOpenPort } from "../../core/port";
 import { log } from "../../core/log";
+import { findEnvPythonPath } from "../../core/python";
 
 export async function activateEvalManager(
   stateManager: WorkspaceStateManager,
@@ -113,6 +114,10 @@ export class InspectEvalManager {
       });
     }
 
+    // Find the python environment
+    const useSubdirectoryEnvironments = workspace.getConfiguration("inspect_ai").get("useSubdirectoryEnvironments");
+    const pythonPath = useSubdirectoryEnvironments ? findEnvPythonPath(file.dirname(), activeWorkspacePath()) : undefined;
+
     // If we're debugging, launch using the debugger
     if (debug) {
       // Handle debugging
@@ -136,16 +141,17 @@ export class InspectEvalManager {
         args,
         workspaceDir.path,
         debugPort,
-        env
+        env,
+        pythonPath ? pythonPath : undefined
       );
     } else {
       // Run the command
-      runEvalCmd(args, workspaceDir.path);
+      runEvalCmd(args, workspaceDir.path, pythonPath ? pythonPath : undefined);
     }
   }
 }
 
-const runEvalCmd = (args: string[], cwd: string) => {
+const runEvalCmd = (args: string[], cwd: string, python?: AbsolutePath) => {
   // See if there a non-busy terminal that we can re-use
   const name = "Inspect Eval";
   let terminal = window.terminals.find((t) => {
@@ -156,15 +162,27 @@ const runEvalCmd = (args: string[], cwd: string) => {
   }
   terminal.show();
   terminal.sendText(`cd ${cwd}`);
-  terminal.sendText(["inspect", ...args].join(" "));
+
+  const cmd = [];
+  if (python) {
+    cmd.push(`${python.path}`);
+    cmd.push("-m");
+    cmd.push("inspect_ai");
+  } else {
+    cmd.push("inspect");
+  }
+  cmd.push(...args);
+
+  terminal.sendText(cmd.join(" "));
 };
 
 const runDebugger = async (
   program: string,
   args: string[],
   cwd: string,
   port: number,
-  env?: Record<string, string>
+  env?: Record<string, string>,
+  pythonPath?: AbsolutePath
 ) => {
   const name = "Inspect Eval";
   const debugConfiguration: DebugConfiguration = {
@@ -178,6 +196,7 @@ const runDebugger = async (
     port,
     env,
     justMyCode: false,
+    pythonPath: pythonPath?.path
   };
   await debug.startDebugging(activeWorkspaceFolder(), debugConfiguration);
 };