Add inline script activation-time discovery (PEP 723 PR 8/16)

src/common/inlineScript/cacheLayout.ts

@@ -204,7 +204,7 @@ export function selectStaleEntries(entries: ReadonlyArray<CacheEntrySummary>, no
 }
 
 /**
- * Verify that a cached env's base interpreter still exists on disk.
+ * Verify that a cached env's launcher and base interpreter still exist on disk.
  */
 export async function verifyBaseInterpreterExists(envDir: Uri): Promise<boolean> {
     return (await getBaseInterpreterStatus(envDir)) === 'available';
@@ -221,6 +221,11 @@ async function getPosixBaseInterpreterStatus(envDir: Uri): Promise<BaseInterpret
 }
 
 async function getWindowsBaseInterpreterStatus(envDir: Uri): Promise<BaseInterpreterStatus> {
+    const launcherStatus = await getRegularFileStatus(getVenvPythonPath(envDir.fsPath), 'cached interpreter launcher');
+    if (launcherStatus !== 'available') {
+        return launcherStatus;
+    }
+
     const pyvenvPath = Uri.joinPath(envDir, 'pyvenv.cfg').fsPath;
     let raw: string;
     try {

src/managers/builtin/inlineScript/envManager.ts

@@ -11,6 +11,7 @@ import {
     DidChangeEnvironmentEventArgs,
     DidChangeEnvironmentsEventArgs,
     EnvironmentManager,
+    EnvironmentChangeKind,
     GetEnvironmentScope,
     GetEnvironmentsScope,
     IconPath,
@@ -49,6 +50,7 @@ import { normalizePath } from '../../../common/utils/pathUtils';
 import { compareReleaseSegments, parseReleaseSegments } from '../../../common/utils/pep440Release';
 import { getVenvPythonPath } from '../../../common/utils/virtualEnvironment';
 import { NativePythonFinder } from '../../common/nativePythonFinder';
+import { sortEnvironments } from '../../common/utils';
 import { resolveSystemPythonEnvironmentPath } from '../utils';
 import * as uvPythonInstaller from '../uvPythonInstaller';
 import { createWithProgress, resolveVenvPythonEnvironmentPath } from '../venvUtils';
@@ -62,6 +64,7 @@ const BASE_INTERPRETER_MANAGER_IDS = new Set([
 const CACHE_LOCK_TIMEOUT_MS = 5 * 60 * 1000;
 const CACHE_LOCK_RETRY_MS = 500;
 const CACHED_ASSOCIATION_VALIDATION_INTERVAL_MS = 5_000;
+const DISCOVERY_RETRY_DELAYS_MS = [1_000, 5_000] as const;
 /** Workspace-state key for PEP 723 script path to environment executable associations. */
 export const INLINE_SCRIPT_ENVS_KEY = `${ENVS_EXTENSION_ID}:inline-script:SCRIPT_ENVIRONMENTS`;
 
@@ -92,13 +95,19 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable {
     private readonly pendingCreations = new Map<string, Promise<PythonEnvironment | undefined>>();
     private readonly directlyResolvedBaseInterpreters = new Map<string, PythonEnvironment>();
     private baseInterpreterInstallationQueue: Promise<void> = Promise.resolve();
+    private collection: PythonEnvironment[] = [];
     private readonly pendingRehydrations = new Map<string, Promise<PythonEnvironment | undefined>>();
     private readonly fsPathToEnv = new Map<string, PythonEnvironment>();
     private readonly fsPathToPersistedEnvPath = new Map<string, string>();
     private readonly cachedAssociationValidatedAt = new Map<string, number>();
     private readonly associationRevisions = new Map<string, number>();
+    private pendingRefresh: Promise<boolean> | undefined;
+    private activationDiscoveryActive = false;
+    private discoveryRetryAttempt = 0;
+    private discoveryRetryTimer: ReturnType<typeof setTimeout> | undefined;
     private persistenceQueue: Promise<void> = Promise.resolve();
     private selectionQueue: Promise<void> = Promise.resolve();
+    private disposed = false;
 
     private readonly _onDidChangeEnvironments = new EventEmitter<DidChangeEnvironmentsEventArgs>();
     public readonly onDidChangeEnvironments: Event<DidChangeEnvironmentsEventArgs> =
@@ -228,10 +237,17 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable {
     }
 
     async refresh(_scope: RefreshEnvironmentsScope): Promise<void> {
-        return;
+        if (this.disposed) {
+            return;
+        }
+        this.stopActivationDiscovery();
+        await this.getOrStartRefreshPass();
     }
 
-    async getEnvironments(_scope: GetEnvironmentsScope): Promise<PythonEnvironment[]> {
+    async getEnvironments(scope: GetEnvironmentsScope): Promise<PythonEnvironment[]> {
+        if (scope === 'all') {
+            return Array.from(this.collection);
+        }
         return [];
     }
 
@@ -247,6 +263,257 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable {
         return undefined;
     }
 
+    public startActivationDiscovery(): void {
+        if (this.disposed || this.activationDiscoveryActive) {
+            return;
+        }
+        this.activationDiscoveryActive = true;
+        this.discoveryRetryAttempt = 0;
+        this.runActivationDiscoveryPass();
+    }
+
+    private async getOrStartRefreshPass(): Promise<boolean> {
+        const pending = this.pendingRefresh;
+        if (pending) {
+            return pending;
+        }
+
+        const refresh = this.refreshDiscoveredEnvironments();
+        this.pendingRefresh = refresh;
+        try {
+            return await refresh;
+        } finally {
+            if (this.pendingRefresh === refresh) {
+                this.pendingRefresh = undefined;
+            }
+        }
+    }
+
+    private runActivationDiscoveryPass(): void {
+        if (this.disposed || !this.activationDiscoveryActive) {
+            return;
+        }
+
+        void this.getOrStartRefreshPass()
+            .then((shouldRetry) => {
+                if (this.disposed || !this.activationDiscoveryActive) {
+                    return;
+                }
+                if (!shouldRetry) {
+                    this.stopActivationDiscovery();
+                    return;
+                }
+                this.scheduleActivationDiscoveryRetry();
+            })
+            .catch((error) => {
+                if (this.disposed || !this.activationDiscoveryActive) {
+                    return;
+                }
+                this.log.warn(`Activation-time inline-script discovery failed: ${getErrorMessage(error)}`);
+                this.stopActivationDiscovery();
+            });
+    }
+
+    private async refreshDiscoveredEnvironments(): Promise<boolean> {
+        const cacheRoot = getScriptEnvCacheRoot(this.globalStorageUri);
+        const previousByKey = new Map(
+            this.collection.map((environment) => [this.getDiscoveredEnvironmentKey(environment), environment]),
+        );
+
+        let entryNames: string[];
+        try {
+            entryNames = await fs.readdir(cacheRoot.fsPath);
+        } catch (error) {
+            if (this.isDefinitivelyStalePathError(error)) {
+                entryNames = [];
+            } else {
+                this.log.warn(
+                    `Unable to inspect the inline-script cache root ${cacheRoot.fsPath}: ${getErrorMessage(error)}`,
+                );
+                return true;
+            }
+        }
+
+        const lockedKeys = new Set<string>();
+        const nextByKey = new Map<string, PythonEnvironment>();
+        let shouldRetry = false;
+        for (const entryName of entryNames.sort()) {
+            if (entryName.endsWith('.lock')) {
+                lockedKeys.add(normalizePath(Uri.joinPath(cacheRoot, entryName.slice(0, -5)).fsPath));
+                shouldRetry = true;
+                continue;
+            }
+
+            if (this.disposed) {
+                return false;
+            }
+
+            const envDir = Uri.joinPath(cacheRoot, entryName);
+            const key = normalizePath(envDir.fsPath);
+            const discovered = await this.inspectDiscoveredCacheEntry(cacheRoot, envDir);
+            if (discovered.kind === 'resolved') {
+                nextByKey.set(key, discovered.environment);
+            } else if (discovered.kind === 'preserve') {
+                shouldRetry = true;
+                const previous = previousByKey.get(key);
+                if (previous) {
+                    nextByKey.set(key, previous);
+                }
+            }
+        }
+        for (const [key, previous] of previousByKey) {
+            if (!nextByKey.has(key) && lockedKeys.has(key)) {
+                nextByKey.set(key, previous);
+            }
+        }
+
+        if (this.disposed) {
+            return false;
+        }
+
+        // Preserve previously known entries when a refresh cannot safely classify
+        // them because a build is in progress or the filesystem is transiently unavailable.
+        this.replaceDiscoveredEnvironments(sortEnvironments(Array.from(nextByKey.values())));
+        return shouldRetry;
+    }
+
+    private async inspectDiscoveredCacheEntry(
+        cacheRoot: Uri,
+        envDir: Uri,
+    ): Promise<DiscoveredCacheEntryResult> {
+        try {
+            const stat = await fs.lstat(envDir.fsPath);
+            if (!stat.isDirectory() || stat.isSymbolicLink()) {
+                return { kind: 'skip' };
+            }
+        } catch (error) {
+            return this.isDefinitivelyStalePathError(error) ? { kind: 'skip' } : { kind: 'preserve' };
+        }
+
+        if (await this.isCacheEntryBusy(envDir.fsPath)) {
+            return { kind: 'preserve' };
+        }
+
+        try {
+            if (!(await resolveCacheEntryPath(cacheRoot, envDir))) {
+                return { kind: 'skip' };
+            }
+        } catch (error) {
+            return this.isDefinitivelyStalePathError(error) ? { kind: 'skip' } : { kind: 'preserve' };
+        }
+
+        const sidecarResult = await inspectMetaJson(envDir);
+        if (sidecarResult.kind !== 'valid') {
+            return { kind: sidecarResult.kind === 'unavailable' ? 'preserve' : 'skip' };
+        }
+
+        const baseInterpreterStatus = await getBaseInterpreterStatus(envDir);
+        if (baseInterpreterStatus !== 'available') {
+            return { kind: baseInterpreterStatus === 'unavailable' ? 'preserve' : 'skip' };
+        }
+
+        let environment: PythonEnvironment | undefined;
+        try {
+            environment = await resolveVenvPythonEnvironmentPath(
+                getVenvPythonPath(envDir.fsPath),
+                this.nativeFinder,
+                this.api,
+                this,
+                this.baseManager,
+            );
+        } catch (error) {
+            this.log.warn(
+                `Unable to resolve inline-script cache entry ${envDir.fsPath}: ${getErrorMessage(error)}`,
+            );
+            return { kind: 'preserve' };
+        }
+        if (!environment) {
+            return { kind: 'preserve' };
+        }
+
+        const ownership = await inspectOwnedCacheEntry(environment, cacheRoot, envDir);
+        if (ownership !== 'expected') {
+            return { kind: ownership === 'uncertain' ? 'preserve' : 'skip' };
+        }
+        if (!this.areEqualPythonReleases(environment.version, sidecarResult.metadata.baseInterpreterVersion)) {
+            return { kind: 'skip' };
+        }
+
+        return { kind: 'resolved', environment };
+    }
+
+    private replaceDiscoveredEnvironments(next: PythonEnvironment[]): void {
+        const previousByKey = new Map(
+            this.collection.map((environment) => [this.getDiscoveredEnvironmentKey(environment), environment]),
+        );
+        const nextByKey = new Map(next.map((environment) => [this.getDiscoveredEnvironmentKey(environment), environment]));
+        const changes: DidChangeEnvironmentsEventArgs = [];
+
+        for (const [key, previous] of previousByKey) {
+            const current = nextByKey.get(key);
+            if (!current || !this.isSameDiscoveredEnvironment(previous, current)) {
+                changes.push({ kind: EnvironmentChangeKind.remove, environment: previous });
+            }
+        }
+        for (const [key, current] of nextByKey) {
+            const previous = previousByKey.get(key);
+            if (!previous || !this.isSameDiscoveredEnvironment(previous, current)) {
+                changes.push({ kind: EnvironmentChangeKind.add, environment: current });
+            }
+        }
+
+        this.collection = next;
+        if (changes.length > 0) {
+            this._onDidChangeEnvironments.fire(changes);
+        }
+    }
+
+    private getDiscoveredEnvironmentKey(environment: PythonEnvironment): string {
+        return normalizePath(environment.sysPrefix);
+    }
+
+    private isSameDiscoveredEnvironment(first: PythonEnvironment, second: PythonEnvironment): boolean {
+        return (
+            first.envId.managerId === second.envId.managerId &&
+            normalizePath(first.environmentPath.fsPath) === normalizePath(second.environmentPath.fsPath) &&
+            first.version === second.version
+        );
+    }
+
+    private scheduleActivationDiscoveryRetry(): void {
+        if (this.discoveryRetryTimer) {
+            return;
+        }
+
+        const delayMs = this.getDiscoveryRetryDelayMs(this.discoveryRetryAttempt);
+        if (delayMs === undefined) {
+            this.stopActivationDiscovery();
+            return;
+        }
+
+        this.discoveryRetryAttempt += 1;
+        this.discoveryRetryTimer = setTimeout(() => {
+            this.discoveryRetryTimer = undefined;
+            if (this.disposed || !this.activationDiscoveryActive) {
+                return;
+            }
+            this.runActivationDiscoveryPass();
+        }, delayMs);
+    }
+
+    private getDiscoveryRetryDelayMs(attempt: number): number | undefined {
+        return DISCOVERY_RETRY_DELAYS_MS[attempt];
+    }
+
+    private stopActivationDiscovery(): void {
+        if (this.discoveryRetryTimer) {
+            clearTimeout(this.discoveryRetryTimer);
+            this.discoveryRetryTimer = undefined;
+        }
+        this.activationDiscoveryActive = false;
+        this.discoveryRetryAttempt = 0;
+    }
+
     private getScriptUri(scope: CreateEnvironmentScope): Uri | undefined {
         const uri = scope instanceof Uri ? scope : Array.isArray(scope) && scope.length === 1 ? scope[0] : undefined;
         return uri?.scheme === 'file' ? uri : undefined;
@@ -1283,6 +1550,8 @@ export class InlineScriptEnvManager implements EnvironmentManager, Disposable {
     }
 
     dispose(): void {
+        this.disposed = true;
+        this.stopActivationDiscovery();
         this._onDidChangeEnvironments.dispose();
         this._onDidChangeEnvironment.dispose();
     }
@@ -1306,3 +1575,7 @@ interface PendingScriptUpdate extends ScriptReference {
     readonly needsPersistence: boolean;
     readonly shouldNotify: boolean;
 }
+
+type DiscoveredCacheEntryResult =
+    | { readonly kind: 'preserve' | 'skip' }
+    | { readonly kind: 'resolved'; readonly environment: PythonEnvironment };

src/managers/builtin/inlineScript/main.ts

@@ -29,5 +29,6 @@ export async function registerInlineScriptFeatures(
     const api: PythonEnvironmentApi = await getPythonApi();
     const mgr = new InlineScriptEnvManager(nativeFinder, api, baseManager, globalStorageUri, log);
     disposables.push(mgr, api.registerEnvironmentManager(mgr));
+    setImmediate(() => mgr.startActivationDiscovery());
     traceInfo('Inline-script env manager: registered (internal flag is on)');
 }