Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ import {
quarantinePendingTeardown,
} from "../config/pending-teardown";
import { collectStatus, deadProxyRoutingAdviceLines, detectMissingCodexCatalogPath, hubStatusLines, missingCodexCatalogLines, remoteHubBannerLine, remoteHubStatusLines, unusedProxyWarningLines } from "./status";
import { endpointsToProve, everyEndpointProvenDown, sharedTeardownAuthorized, type UninstallObservation } from "./uninstall-plan";
import { endpointsToProve, everyEndpointProvenDownAsync, sharedTeardownAuthorized, type UninstallObservation } from "./uninstall-plan";
import { takeFlag } from "./runtime-api";
import { parseStartOptions, StartArgsError } from "./start-args";

Expand All @@ -85,7 +85,14 @@ import { SpendLedgerOwnerError } from "../lib/spend-ledger-owner";
import { redactUrlForLog } from "../lib/redact";
import { dispatchCommand, decideBusyPreferredPort, decideStartWithLiveOwner } from "./dispatch";
import { AuxiliaryListenerBindError, findAvailablePort, isAddrInUse, PortUnavailableError, shouldPersistSelectedPort, waitForPortAvailable } from "../server/ports";
import { findLiveProxy, probeHostname, probePortOwner, START_OWNERSHIP_LIVENESS, type LiveProxy } from "../server/proxy-liveness";
import {
findLiveProxy,
probeEndpointLiveness,
probeHostname,
probePortOwner,
START_OWNERSHIP_LIVENESS,
type LiveProxy,
} from "../server/proxy-liveness";
import { createReadinessGate } from "../server/readiness";
import { isApiAuthRequired } from "../server/auth-cors";
import { runReady, type ReadyArgs } from "./ready";
Expand Down Expand Up @@ -1021,8 +1028,7 @@ async function handleStopUnlocked() {
// An obligation that cannot name its endpoint cannot be proven discharged.
if (!endpoint) return false;
try {
const { probeProxyLiveness } = await import("../update/proxy-liveness-probe.mjs");
return probeProxyLiveness(endpoint.port, endpoint.hostname) === "dead";
return await probeEndpointLiveness(endpoint) === "dead";
} catch {
// A probe that could not run is not evidence of absence.
return false;
Expand Down Expand Up @@ -1436,11 +1442,10 @@ async function handleUninstall() {
/** Definitive "nothing is answering" on the endpoint this home would serve. */
const proxyEndpointProvenDown = async (): Promise<boolean> => {
try {
const { probeProxyLiveness } = await import("../update/proxy-liveness-probe.mjs");
// Every candidate, not just the preferred one: a stale runtime record pointing at a
// closed port would otherwise "prove" a live proxy on the configured port is gone.
const endpoints = endpointsToProve(readRuntimePort(), loadConfig());
return everyEndpointProvenDown(endpoints, e => probeProxyLiveness(e.port, e.hostname));
return await everyEndpointProvenDownAsync(endpoints, probeEndpointLiveness);
} catch {
return false;
}
Expand Down
23 changes: 12 additions & 11 deletions src/cli/resolve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@ import { readConfigDiagnostics, type ConfigDiagnostics } from "../config";
import { getConfigDir } from "../config/paths";
import { readRuntimePort } from "../config/process-state";
import { packageVersion } from "../lib/package-version";
import { findLiveProxy, START_OWNERSHIP_LIVENESS, type LiveProxy } from "../server/proxy-liveness";
import { endpointsToProve, everyEndpointProvenDown, type ProbeEndpoint } from "./uninstall-plan";
import { probeProxyLiveness } from "../update/proxy-liveness-probe.mjs";
import {
findLiveProxy,
probeEndpointLiveness,
START_OWNERSHIP_LIVENESS,
type EndpointLiveness,
type LiveProxy,
} from "../server/proxy-liveness";
import { endpointsToProve, everyEndpointProvenDownAsync, type ProbeEndpoint } from "./uninstall-plan";
Comment on lines +42 to +47

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Synchronize the owned resolve contract document

This switches ocx resolve to probeEndpointLiveness and everyEndpointProvenDownAsync, but structure/runtime.md:55 still states that the contract uses the updater's probeProxyLiveness and synchronous everyEndpointProvenDown. Update that owned structure document in this change so maintainers do not rely on an obsolete description of this launch-safety path.

AGENTS.md reference: src/AGENTS.md:L11-L11

Useful? React with 👍 / 👎.


/** Wire version of the resolve document. Bump only on an incompatible shape change. */
export const RESOLVE_SCHEMA = "ocx-resolve/1";
Expand Down Expand Up @@ -103,8 +108,8 @@ export interface ResolveIo {
findLive?: () => Promise<LiveProxy | null>;
/** Runtime-port record reader; production default is readRuntimePort. */
readRuntime?: () => { port?: number; hostname?: string } | null;
/** Tri-state endpoint probe; production default is the updater's probeProxyLiveness. */
probeEndpoint?: (endpoint: ProbeEndpoint) => "live" | "dead" | "unknown";
/** Tri-state endpoint probe; production default runs in-process for compiled standalone binaries. */
probeEndpoint?: (endpoint: ProbeEndpoint) => EndpointLiveness | Promise<EndpointLiveness>;
cliVersion?: () => string;
stdout?: { log: (s: string) => void };
stderr?: { error: (s: string) => void };
Expand Down Expand Up @@ -174,11 +179,7 @@ export async function runResolve(args: ResolveArgs, io: ResolveIo = {}): Promise
const readDiagnostics = io.readDiagnostics ?? readConfigDiagnostics;
const findLive = io.findLive ?? (() => findLiveProxy(START_OWNERSHIP_LIVENESS));
const readRuntime = io.readRuntime ?? readRuntimePort;
// The updater's tri-state probe takes (port, hostname) and is plain .mjs (untyped);
// adapt it to the endpoint-shaped seam here. Its own return vocabulary is the
// closed "live" | "dead" | "unknown" set.
const probeEndpoint = io.probeEndpoint
?? ((endpoint: ProbeEndpoint) => probeProxyLiveness(endpoint.port, endpoint.hostname) as "live" | "dead" | "unknown");
const probeEndpoint = io.probeEndpoint ?? probeEndpointLiveness;
const cliVersion = io.cliVersion ?? packageVersion;
const configHome = configDir();
let diagnostics: ConfigDiagnostics;
Expand Down Expand Up @@ -212,7 +213,7 @@ export async function runResolve(args: ResolveArgs, io: ResolveIo = {}): Promise
// authorise starting a second runtime.
let provenDown = false;
try {
provenDown = everyEndpointProvenDown(endpointsToProve(readRuntime(), diagnostics.config), probeEndpoint);
provenDown = await everyEndpointProvenDownAsync(endpointsToProve(readRuntime(), diagnostics.config), probeEndpoint);
} catch {
// A probe that cannot run is not evidence of absence.
provenDown = false;
Expand Down
20 changes: 2 additions & 18 deletions src/cli/status-probes.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { readPidFileValue, readRuntimePort } from "../config/process-state";
import { isOpencodexHealthz, probeHostname } from "../server/proxy-liveness";
import { isConnectionRefused, isOpencodexHealthz, probeHostname } from "../server/proxy-liveness";
import { directLocalHttpFetch } from "../server/direct-local-http";
import { isProcessAlive } from "../lib/process-control";

Expand All @@ -26,23 +26,7 @@ export function proxyHealthFailureReason(error: unknown, signal: AbortSignal): "
: "unreachable";
}

/**
* "Nothing is listening" is narrower than "the probe failed". `unreachable` covers every
* non-abort failure, including a socket that was ACCEPTED and then reset — which is what
* an in-flight start looks like mid-bind. Only a connect-phase refusal proves the port is
* free, so this reads the underlying errno instead of the display string.
*/
export function isConnectionRefused(error: unknown): boolean {
for (let current: unknown = error, depth = 0; current instanceof Error && depth < 4; depth++) {
const code = (current as { code?: unknown }).code;
if (code === "ECONNREFUSED" || code === "ConnectionRefused") return true;
// Bun surfaces the refusal as a plain message on some platforms; the errno name is
// still the discriminator, not a substring of arbitrary prose.
if (typeof code === "string" && code.endsWith("ECONNREFUSED")) return true;
current = (current as { cause?: unknown }).cause;
}
return false;
}
export { isConnectionRefused } from "../server/proxy-liveness";

/**
* A proxy killed by a native trap or SIGKILL never runs the exit cleanup that removes
Expand Down
9 changes: 9 additions & 0 deletions src/cli/uninstall-plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,12 @@ export function everyEndpointProvenDown(
if (endpoints.length === 0) return false;
return endpoints.every(e => probe(e) === "dead");
}

export async function everyEndpointProvenDownAsync(
endpoints: readonly ProbeEndpoint[],
probe: (e: ProbeEndpoint) => Promise<"live" | "dead" | "unknown"> | "live" | "dead" | "unknown",
): Promise<boolean> {
if (endpoints.length === 0) return false;
const results = await Promise.all(endpoints.map(e => probe(e)));
return results.every(result => result === "dead");
}
75 changes: 75 additions & 0 deletions src/server/proxy-liveness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export interface HealthzIdentity {
guiPairCapability?: unknown;
}

export type EndpointLiveness = "live" | "dead" | "unknown";

export interface LivenessIo {
fetchFn?: typeof fetch;
readPidFn?: () => number | null;
Expand Down Expand Up @@ -85,6 +87,11 @@ export const START_OWNERSHIP_LIVENESS: Pick<LivenessIo, "timeoutMs" | "attempts"
attempts: 3,
};

type LivenessFetch = (
input: string | URL | Request,
init?: RequestInit,
) => Promise<Response>;

export interface LiveProxy {
pid: number | null;
port: number;
Expand Down Expand Up @@ -148,6 +155,74 @@ export function isOpencodexHealthz(body: HealthzIdentity | null): boolean {
return body.status === "ok" && typeof body.version === "string" && typeof body.uptime === "number";
}

/**
* "Nothing is listening" is narrower than "the probe failed". Only a connect-phase refusal
* proves the endpoint is free; a timeout, reset, or other transport failure leaves the
* question open.
*/
export function isConnectionRefused(error: unknown): boolean {
const visit = (current: unknown, depth: number): boolean => {
if (depth >= 4) return false;
if (current === null || (typeof current !== "object" && typeof current !== "function")) return false;
const record = current as { code?: unknown; cause?: unknown; errors?: unknown };
if (record.code === "ECONNREFUSED" || record.code === "ConnectionRefused") return true;
if (typeof record.code === "string" && record.code.endsWith("ECONNREFUSED")) return true;
if (Array.isArray(record.errors) && record.errors.length > 0) {
Comment on lines +168 to +170

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Check aggregate members before accepting wrapper refusal

When a configured hostname resolves to multiple addresses, node:net's autoSelectFamily can produce an AggregateError whose top-level code is copied from the first child (for example ECONNREFUSED) even though a later child is ETIMEDOUT or ENETUNREACH. These lines therefore return true before examining errors, causing probeEndpointLiveness to report dead and potentially authorize a duplicate start or shared uninstall teardown despite the mixed result. Inspect a nonempty errors array before accepting the wrapper's code; the added test currently misses this because its synthetic aggregate has no top-level code.

Useful? React with 👍 / 👎.

// One connect attempt fanned out over several addresses reports a single AggregateError.
// Only a unanimous refusal proves the endpoint is free: a bundle that mixes ECONNREFUSED
// with a timeout means one address answered nothing at all, and an address whose state is
// unreadable is unknown, not absence. Collapsing it to "refused" is how a second runtime
// gets started on a port that already has one.
return record.errors.every(error => visit(error, depth + 1));
}
return visit(record.cause, depth + 1);
};
return visit(error, 0);
}

async function classifyHealthz(
url: string,
fetchFn: LivenessFetch,
timeoutMs: number,
): Promise<EndpointLiveness> {
try {
const response = await fetchFn(url, { signal: AbortSignal.timeout(timeoutMs) });
if (response.status !== 200) return "unknown";
const body = (await response.json().catch(() => undefined)) as HealthzIdentity | null | undefined;
if (body === undefined) return "unknown";
return isOpencodexHealthz(body) ? "live" : "dead";
} catch (error) {
return isConnectionRefused(error) ? "dead" : "unknown";
}
}

/**
* Tri-state probe of one endpoint, the in-process counterpart of
* `src/update/proxy-liveness-probe.mjs`. Only a connect-phase refusal or a clean 200 that is
* not ours proves "dead"; a timeout, reset, non-200 or unreadable body leaves the question
* open. Loopback endpoints are checked on both IPv4 and IPv6 because a listener may bind only
* one family. Runs in-process because a compiled standalone binary cannot fork `execPath -e`.
*/
export async function probeEndpointLiveness(
endpoint: { port: number; hostname?: string },
io: Pick<LivenessIo, "fetchFn" | "timeoutMs"> = {},
): Promise<EndpointLiveness> {
if (!Number.isFinite(endpoint.port) || endpoint.port <= 0 || endpoint.port > 65535) return "dead";
const fetchFn = io.fetchFn ?? directLocalHttpFetch;
const timeoutMs = io.timeoutMs ?? 1500;
let sawUnknown = false;
for (const hostname of loopbackProbeHosts(endpoint.hostname)) {
const result = await classifyHealthz(
`http://${hostname}:${endpoint.port}/healthz`,
fetchFn,
timeoutMs,
);
if (result === "live") return "live";
if (result === "unknown") sawUnknown = true;
}
return sawUnknown ? "unknown" : "dead";
}

/** Identity-checked /healthz probe; null when unreachable, non-OK, or not our proxy. */
export async function proxyIdentityAt(
port: number,
Expand Down
19 changes: 17 additions & 2 deletions tests/cli/cli-resolve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,21 @@ describe("runResolve", () => {
expect(parsed.port.effective).toBe(RESOLVE_DEFAULT_PORT);
});

test("accepts async dead probes for every candidate endpoint", async () => {
const lines: string[] = [];
const code = await runResolve({ json: true }, {
configDir: () => "/h",
readDiagnostics: () => ({ config: {}, source: "default", error: null } as ConfigDiagnostics),
findLive: async () => null,
readRuntime: () => ({ port: 10110, hostname: "127.0.0.1" }),
probeEndpoint: async () => "dead",
cliVersion: () => "1.2.3",
stdout: { log: value => lines.push(value) },
});
expect(code).toBe(0);
expect((JSON.parse(lines[0]!) as { liveness: { status: string } }).liveness.status).toBe("absent-proven");
});

test("an undecidable probe is unknown, and unknown is never answered as absent", async () => {
// The launch decision keys on this verdict: a timed-out probe or a listener that
// withholds /healthz must exit 1 rather than let the caller start a second runtime.
Expand All @@ -152,8 +167,8 @@ describe("runResolve", () => {
test("absence requires every endpoint dead, not just the configured one", async () => {
// The runtime record can point at a live port while the configured port refuses;
// answering from the configured port alone would shadow-start over the record.
// everyEndpointProvenDown short-circuits on the first non-dead answer: an unknown
// runtime endpoint defeats the proof without the configured one being probed.
// Every candidate is probed: an unknown runtime endpoint defeats the proof even when the
// configured endpoint is dead.
const seen: string[] = [];
const code = await runResolve({ json: true }, {
configDir: () => "/h",
Expand Down
6 changes: 4 additions & 2 deletions tests/cli/uninstall.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ describe("uninstall gates shared teardown on a proven service stop", () => {
});
});
test("proof covers every distinct endpoint, not just the preferred one", async () => {
const { endpointsToProve, everyEndpointProvenDown } = await import("../../src/cli/uninstall-plan");
const { endpointsToProve, everyEndpointProvenDown, everyEndpointProvenDownAsync } = await import("../../src/cli/uninstall-plan");

// A stale runtime record pointing at a closed port, and the live proxy on the
// configured one. Probing only the runtime candidate reports "dead" for a port nobody
Expand All @@ -267,6 +267,8 @@ describe("uninstall gates shared teardown on a proven service stop", () => {
expect(endpointsToProve(null, {})).toEqual([{ hostname: "127.0.0.1", port: 10100 }]);
// An empty set is not proof of anything.
expect(everyEndpointProvenDown([], () => "dead")).toBe(false);
expect(await everyEndpointProvenDownAsync(endpoints, async () => "dead")).toBe(true);
expect(await everyEndpointProvenDownAsync([], async () => "dead")).toBe(false);
// A nonsense runtime port is skipped rather than probed.
expect(endpointsToProve({ port: 0 }, { port: 10100 })).toEqual([{ hostname: "127.0.0.1", port: 10100 }]);
});
Expand All @@ -284,7 +286,7 @@ describe("uninstall gates shared teardown on a proven service stop", () => {
.toBeLessThan(windowStep.indexOf("observed.respawnWindowVerified = true;"));
// And the proof itself asks every candidate.
expect(fn).toContain("endpointsToProve(readRuntimePort(), loadConfig())");
expect(fn).toContain("everyEndpointProvenDown(endpoints, e => probeProxyLiveness(e.port, e.hostname))");
expect(fn).toContain("everyEndpointProvenDownAsync(endpoints, probeEndpointLiveness)");
});

const safeTeardown: UninstallObservation = {
Expand Down
2 changes: 1 addition & 1 deletion tests/providers/xai/grok-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ describe("Grok fence lifecycle wiring", () => {
expect(noPidBranch).toContain("stopFailed = true;");
expect(noPidBranch).toContain("ownershipBlocked = true;");
const gateFn = sliceFn(CLI_SOURCE, "const abandonedTeardownIsSafeToFinish", "let stopFailed = false;");
expect(gateFn).toContain('probeProxyLiveness(endpoint.port, endpoint.hostname) === "dead"');
expect(gateFn).toContain('probeEndpointLiveness(endpoint) === "dead"');
expect(gateFn).toContain("return false;");
});

Expand Down
Loading
Loading