feat(openapi): cache spec fetches (URL index + conditional GET)

.changeset/spec-fetch-cache.md

@@ -0,0 +1,5 @@
+---
+"executor": patch
+---
+
+Stop re-downloading OpenAPI specs the app already has. Spec URLs now resolve through a tenant-shared cache (URL → content hash + ETag/Last-Modified over the existing content-addressed blob store): the add flow's detect → preview → add sequence downloads a spec once instead of per step, and refreshing an integration revalidates with a conditional request — an unchanged upstream costs a bodyless 304 instead of a multi-MB download.

e2e/scenarios/openapi-spec-fetch-cache-ui.test.ts

@@ -0,0 +1,141 @@
+// Cross-target (browser): the spec-fetch cache, driven through the real UI.
+// The API-surface twin (openapi-spec-fetch-cache.test.ts) pins the same
+// contract headlessly; this one exists so the journey is watchable — the
+// session video + trace show the add form analyzing a pasted URL, the
+// integration landing, and an Edit → re-fetch-on-save refresh — while the
+// counting spec server proves what the network actually did underneath:
+//   - the whole add journey (the form's debounced analyze + the submit)
+//     downloads the spec ONCE,
+//   - the refresh consults the server but 304s instead of re-downloading.
+// Skips on targets without a browser surface (selfhost today).
+import { createHash, randomBytes } from "node:crypto";
+import { createServer } from "node:http";
+
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+
+import { scenario } from "../src/scenario";
+import { Browser, Target } from "../src/services";
+
+const pingSpec = JSON.stringify({
+  openapi: "3.0.3",
+  info: { title: "Cached Ping API", version: "1.0.0" },
+  servers: [{ url: "https://api.example.com", description: "Production" }],
+  paths: {
+    "/ping": {
+      get: {
+        operationId: "ping",
+        summary: "Return a pong",
+        responses: { "200": { description: "pong" } },
+      },
+    },
+  },
+});
+
+/** A real 127.0.0.1 spec host with a strong ETag and a request ledger —
+ *  the ground truth the video's UI journey is asserted against. Non-spec
+ *  paths (the add flow's OAuth discovery probes) 404 outside the count. */
+const serveCountingSpec = (body: string) =>
+  Effect.acquireRelease(
+    Effect.callback<{
+      readonly url: string;
+      readonly downloads: () => number;
+      readonly notModified: () => number;
+      readonly close: () => void;
+    }>((resume) => {
+      let downloads = 0;
+      let notModified = 0;
+      const etag = `"${createHash("sha256").update(body).digest("hex")}"`;
+      const server = createServer((request, response) => {
+        if (!request.url?.startsWith("/spec.json")) {
+          response.writeHead(404);
+          response.end();
+          return;
+        }
+        if (request.headers["if-none-match"] === etag) {
+          notModified += 1;
+          response.writeHead(304, { etag });
+          response.end();
+          return;
+        }
+        downloads += 1;
+        response.writeHead(200, { "content-type": "application/json", etag });
+        response.end(body);
+      });
+      server.listen(0, "127.0.0.1", () => {
+        const address = server.address();
+        const port = typeof address === "object" && address ? address.port : 0;
+        resume(
+          Effect.succeed({
+            url: `http://127.0.0.1:${port}/spec.json`,
+            downloads: () => downloads,
+            notModified: () => notModified,
+            close: () => {
+              server.close();
+              server.closeAllConnections();
+            },
+          }),
+        );
+      });
+    }),
+    (server) => Effect.sync(server.close),
+  );
+
+scenario(
+  "OpenAPI · adding a spec by URL in the UI downloads it once and Edit → re-fetch 304s",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const browser = yield* Browser;
+      const identity = yield* target.newIdentity();
+      const specServer = yield* serveCountingSpec(pingSpec);
+      const name = `Cache Demo ${randomBytes(3).toString("hex")}`;
+
+      yield* browser.session(identity, async ({ page, step }) => {
+        await step("Open the Add OpenAPI source form", async () => {
+          await page.goto("/integrations/add/openapi", { waitUntil: "networkidle" });
+          await page.getByPlaceholder("https://api.example.com/openapi.json").waitFor();
+        });
+
+        await step("Paste the spec URL — the form analyzes it (first download)", async () => {
+          await page.getByPlaceholder("https://api.example.com/openapi.json").fill(specServer.url);
+          // The debounced analyze fetches and previews the spec.
+          await page.getByRole("button", { name: "Add integration" }).waitFor({ timeout: 20_000 });
+        });
+
+        await step("Name it and add the integration", async () => {
+          const nameInput = page.getByLabel("Name", { exact: true });
+          if (await nameInput.isVisible().catch(() => false)) {
+            await nameInput.fill(name);
+          }
+          await page.getByRole("button", { name: "Add integration" }).click();
+          await page.waitForURL(/\/integrations\/(?!add\b)[^/?]+$/, { timeout: 30_000 });
+          await page.getByText("Connections").first().waitFor();
+        });
+
+        await step("The whole add journey cost exactly one spec download", async () => {
+          // The form's analyze already fetched it; addSpec on the server reused
+          // the cached copy instead of re-downloading.
+          expect(specServer.downloads(), "one download across analyze + add").toBe(1);
+        });
+
+        await step("Open Edit and stage a re-fetch of the spec", async () => {
+          await page.getByRole("button", { name: "Edit" }).click();
+          await page.getByText("Update spec").waitFor({ timeout: 10_000 });
+          await page.getByText("Re-fetch the spec on save").click();
+        });
+
+        await step("Save — the refresh revalidates (304) instead of re-downloading", async () => {
+          await page.getByRole("button", { name: "Save", exact: true }).click();
+          await page.getByText("Update spec").waitFor({ state: "hidden", timeout: 30_000 });
+        });
+
+        await step("The server saw a conditional request, not a second download", async () => {
+          expect(specServer.notModified(), "the refresh got a bodyless 304").toBe(1);
+          expect(specServer.downloads(), "still exactly one full download").toBe(1);
+        });
+      });
+    }),
+  ),
+);

e2e/scenarios/openapi-spec-fetch-cache.test.ts

@@ -0,0 +1,185 @@
+// Cross-target: the spec-fetch cache — "we don't re-download a spec we already
+// have". A real 127.0.0.1 server serves the spec with a strong ETag and honors
+// If-None-Match, counting every request it sees. The observable contract at
+// the product boundary is that server's request log:
+//   - the add flow (preview, preview again, addSpec) downloads the spec ONCE —
+//     the URL-index cache serves the repeats, across separate HTTP requests
+//     (on cloud each request is its own executor, so this also proves the
+//     index is durable, not in-memory),
+//   - updateSpec on an unchanged upstream still hits the server (an explicit
+//     refresh must revalidate) but gets a bodyless 304, not a re-download,
+//   - updateSpec on a CHANGED upstream busts the cache through the validators
+//     and the new tool catalog lands.
+import { randomBytes } from "node:crypto";
+import { createHash } from "node:crypto";
+import { createServer } from "node:http";
+
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+import { composePluginApi } from "@executor-js/api/server";
+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
+import { IntegrationSlug } from "@executor-js/sdk/shared";
+
+import { scenario } from "../src/scenario";
+import { Api, Target } from "../src/services";
+
+const api = composePluginApi([openApiHttpPlugin()] as const);
+
+const specV1 = JSON.stringify({
+  openapi: "3.0.3",
+  info: { title: "Cached API", version: "1.0.0" },
+  paths: {
+    "/ping": {
+      get: {
+        operationId: "ping",
+        summary: "Return a pong",
+        responses: { "200": { description: "pong" } },
+      },
+    },
+  },
+});
+
+const specV2 = JSON.stringify({
+  openapi: "3.0.3",
+  info: { title: "Cached API", version: "2.0.0" },
+  paths: {
+    "/ping": {
+      get: {
+        operationId: "ping",
+        summary: "Return a pong",
+        responses: { "200": { description: "pong" } },
+      },
+    },
+    "/widgets": {
+      get: {
+        operationId: "listWidgets",
+        summary: "List widgets",
+        responses: { "200": { description: "widgets" } },
+      },
+    },
+  },
+});
+
+/** A real 127.0.0.1 spec host that serves a strong ETag (the body's SHA-256),
+ *  honors `If-None-Match` with a 304, and counts what it saw — the request
+ *  ledger the assertions run against. */
+const serveCountingSpec = (initial: string) =>
+  Effect.acquireRelease(
+    Effect.callback<{
+      readonly url: string;
+      readonly setBody: (body: string) => void;
+      readonly downloads: () => number;
+      readonly notModified: () => number;
+      readonly close: () => void;
+    }>((resume) => {
+      let body = initial;
+      let downloads = 0;
+      let notModified = 0;
+      const server = createServer((request, response) => {
+        // Only /spec.json is the spec. Everything else (the add flow's OAuth
+        // discovery probes .well-known paths on this host) 404s and stays out
+        // of the download count — the assertions are about spec transfers.
+        if (!request.url?.startsWith("/spec.json")) {
+          response.writeHead(404);
+          response.end();
+          return;
+        }
+        const etag = `"${createHash("sha256").update(body).digest("hex")}"`;
+        if (request.headers["if-none-match"] === etag) {
+          notModified += 1;
+          response.writeHead(304, { etag });
+          response.end();
+          return;
+        }
+        downloads += 1;
+        response.writeHead(200, { "content-type": "application/json", etag });
+        response.end(body);
+      });
+      server.listen(0, "127.0.0.1", () => {
+        const address = server.address();
+        const port = typeof address === "object" && address ? address.port : 0;
+        resume(
+          Effect.succeed({
+            url: `http://127.0.0.1:${port}/spec.json`,
+            setBody: (next: string) => {
+              body = next;
+            },
+            downloads: () => downloads,
+            notModified: () => notModified,
+            close: () => {
+              server.close();
+              server.closeAllConnections();
+            },
+          }),
+        );
+      });
+    }),
+    (server) => Effect.sync(server.close),
+  );
+
+scenario(
+  "OpenAPI · the add flow downloads a spec once and refresh revalidates instead of re-downloading",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const { client } = yield* Api;
+      const identity = yield* target.newIdentity();
+      const apiClient = yield* client(api, identity);
+
+      const slug = `spec-cache-${randomBytes(4).toString("hex")}`;
+      const specServer = yield* serveCountingSpec(specV1);
+
+      yield* Effect.ensuring(
+        Effect.gen(function* () {
+          // The add flow as the UI drives it: analyze (debounce can fire it
+          // more than once), then add. Three API calls, one download.
+          const firstPreview = yield* apiClient.openapi.previewSpec({
+            payload: { spec: specServer.url },
+          });
+          expect(firstPreview.operationCount, "preview sees the v1 spec").toBe(1);
+          const secondPreview = yield* apiClient.openapi.previewSpec({
+            payload: { spec: specServer.url },
+          });
+          expect(secondPreview.operationCount, "re-preview still sees v1").toBe(1);
+          const added = yield* apiClient.openapi.addSpec({
+            payload: {
+              spec: { kind: "url", url: specServer.url },
+              slug,
+              baseUrl: "http://127.0.0.1:59999", // tools are never invoked here
+            },
+          });
+          expect(added.toolCount, "v1 spec has one operation").toBe(1);
+          expect(
+            specServer.downloads(),
+            "preview → preview → add downloaded the spec exactly once",
+          ).toBe(1);
+
+          // Explicit refresh with the upstream unchanged: the server must be
+          // consulted (a refresh is a freshness demand), but with the stored
+          // ETag it answers 304 and nothing is re-downloaded.
+          const unchanged = yield* apiClient.openapi.updateSpec({
+            params: { slug },
+            payload: {},
+          });
+          expect(unchanged.addedTools, "no tools appeared").toEqual([]);
+          expect(unchanged.removedTools, "no tools vanished").toEqual([]);
+          expect(specServer.notModified(), "the refresh revalidated and got a 304").toBe(1);
+          expect(specServer.downloads(), "an unchanged spec is not re-downloaded").toBe(1);
+
+          // The upstream ships v2: the validators no longer match, the cache
+          // busts, and the refreshed catalog lands.
+          specServer.setBody(specV2);
+          const updated = yield* apiClient.openapi.updateSpec({
+            params: { slug },
+            payload: {},
+          });
+          expect(updated.addedTools, "the new operation arrived").toEqual(["widgets.listWidgets"]);
+          expect(specServer.downloads(), "the changed spec was downloaded").toBe(2);
+          expect(specServer.notModified(), "no spurious 304 for a changed body").toBe(1);
+        }),
+        apiClient.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore),
+      );
+    }),
+  ),
+);

packages/plugins/openapi/src/sdk/index.ts

@@ -1,4 +1,17 @@
-export { parse, resolveSpecText, fetchSpecText } from "./parse";
+export {
+  parse,
+  resolveSpecText,
+  fetchSpecText,
+  fetchSpecDocument,
+  type SpecDocumentResult,
+  type SpecFetchValidators,
+} from "./parse";
+export {
+  fetchSpecTextCached,
+  SPEC_FETCH_TTL,
+  type ResolvedSpecDocument,
+  type SpecFetchFreshness,
+} from "./spec-cache";
 export { extract, streamOperationBindingsFromStructure } from "./extract";
 export {
   structuralSplit,
@@ -42,7 +55,12 @@ export {
   type OpenApiPluginExtension,
   type OpenApiPluginOptions,
 } from "./plugin";
-export { type OpenapiStore, type StoredOperation, makeDefaultOpenapiStore } from "./store";
+export {
+  type OpenapiStore,
+  type StoredOperation,
+  type SpecSourceEntry,
+  makeDefaultOpenapiStore,
+} from "./store";
 export {
   decodeOpenApiIntegrationConfig,
   renderAuthTemplate,