Skip to content

failover: retry the same host when no fallback origin exists, and cover the Cloud API hosts #723

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
u9g wants to merge 2 commits into
base: main
Choose a base branch
from
Open

failover: retry the same host when no fallback origin exists, and cover the Cloud API hosts #723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bf2ea1b
failover: retry the same host when no fallback origin exists
u9g Sep 11, 2026
3f51ac7
failover: enable retries for the LiveKit Cloud API hosts
u9g Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/cloud-api-failover.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'livekit-server-sdk': patch
---

Retry LiveKit Cloud API (`cloud-api.livekit.io`) requests on transport errors instead of failing after a single attempt.
83 changes: 83 additions & 0 deletions packages/livekit-server-sdk/src/TwirpRPC.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,86 @@ describe('request id', () => {
expect(new Set(ids).size).toBe(1);
});
});

describe('failover without a fallback origin', () => {
afterEach(() => {
vi.restoreAllMocks();
});

// No /settings/regions, so no fallback origin ever exists.
const host = 'https://cloud-api.example.com';

const okResponse = () =>
({ ok: true, status: 200, json: async () => ({}) }) as unknown as Response;

const errorResponse = (status: number) =>
({
ok: false,
status,
statusText: 'Bad Gateway',
headers: { get: () => null },
text: async () => 'bad gateway',
}) as unknown as Response;

const isDiscovery = (input: unknown) => `${input}`.endsWith('/settings/regions');

it('without a fallback origin, a transport error retries the same host', async () => {
let attempt = 0;
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
if (isDiscovery(input)) {
return errorResponse(404);
}
attempt += 1;
if (attempt === 1) {
throw new Error('read: connection reset by peer');
}
return okResponse();
});

const rpc = new TwirpRpc(host, 'livekit', { failoverForce: true, failoverBackoffMs: 0 });
await expect(rpc.request('RoomService', 'CreateRoom', {}, {})).resolves.toEqual({});

const attempts = fetchSpy.mock.calls.filter(([input]) => !isDiscovery(input));
expect(attempts).toHaveLength(2);
expect(attempts.map(([input]) => new URL(`${input}`).host)).toEqual([
'cloud-api.example.com',
'cloud-api.example.com',
]);
});

it('a Cloud API host retries without consulting region discovery', async () => {
let attempt = 0;
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
if (isDiscovery(input)) {
return errorResponse(404);
}
attempt += 1;
if (attempt === 1) {
throw new Error('read: connection reset by peer');
}
return okResponse();
});

const rpc = new TwirpRpc('https://cloud-api.livekit.io', 'livekit', { failoverBackoffMs: 0 });
await expect(rpc.request('RoomService', 'CreateRoom', {}, {})).resolves.toEqual({});

expect(fetchSpy.mock.calls.filter(([input]) => isDiscovery(input))).toHaveLength(0);
expect(fetchSpy.mock.calls.filter(([input]) => !isDiscovery(input))).toHaveLength(2);
});

it('without a fallback origin, a 5xx retries the same host', async () => {
let attempt = 0;
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
if (isDiscovery(input)) {
return errorResponse(404);
}
attempt += 1;
return attempt === 1 ? errorResponse(502) : okResponse();
});

const rpc = new TwirpRpc(host, 'livekit', { failoverForce: true, failoverBackoffMs: 0 });
await expect(rpc.request('RoomService', 'CreateRoom', {}, {})).resolves.toEqual({});

expect(fetchSpy.mock.calls.filter(([input]) => !isDiscovery(input))).toHaveLength(2);
});
});
7 changes: 5 additions & 2 deletions packages/livekit-server-sdk/src/TwirpRPC.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
FAILOVER_BACKOFF_BASE_MS,
failoverAttempts,
hostKey,
isCloudApi,
pickNext,
regionOrigins,
sleep,
Expand Down Expand Up @@ -196,7 +197,8 @@ export class TwirpRpc {
timeout,
);
const attempted = new Set([hostKey(origin)]);
let regions: string[] | undefined;
// A Cloud API host has a single origin; region discovery is never consulted.
let regions: string[] | undefined = isCloudApi(origin.hostname) ? [] : undefined;
let current = this.host;

for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
Expand Down Expand Up @@ -231,7 +233,8 @@ export class TwirpRpc {
if (!regions) {
regions = await regionOrigins(origin, headers);
}
next = pickNext(regions, attempted);
// With no fallback origin, a retryable failure is retried against the same host.
next = pickNext(regions, attempted) ?? current;

@xianshijing-lk xianshijing-lk Sep 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when there is no fallback origin, what will happen to a dead origin ? with this change, it will fail after 30s rather than 10, can we do something better here ?

As minimal, probably we should update the comment

@xianshijing-lk xianshijing-lk Sep 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And my concern is, saying the service is already busy and return a 5xx (indicating busy traffic or whatever), with this change, we will retry 3 times, that might congest the traffic more.

@u9g u9g Sep 17, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A dead origin, ie. connection refused, DNS failure, TLS reject will fail in milliseconds.

Three attempts cost only the backoff, so ~600ms total, not 30s. The 30s worst case (10 + 0.2 + 10 + 0.4 + 10 = 30.6s at the 10s default) only happens when the origin hangs until the per-attempt timeout on every attempt. That's exactly the failure mode this PR exists to fix: on 2026-09-11 the request was acknowledged at Cloudflare's edge and never reached any LiveKit system, and the client burned its full timeout with no response. The only way to recover that is to send it again, and the only signal we have is the timeout itself.

The alternative is a single shared deadline across attempts instead of a per-attempt timeout. I don't think it belongs here: requestTimeout is documented and used as per-request today, so changing it to a total budget is a semantic change to a user-facing option, and it would have to land in server-sdk-go (already merged) and python-sdks at the same time. Happy to open that separately if you want it.

@u9g u9g Sep 17, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The amplification factor isn't new: FAILOVER_MAX_ATTEMPTS is 3, and cross-region failover has always retried 5xx at that same cap for .livekit.cloud hosts. This PR extends the existing 3x ceiling to single-origin hosts rather than raising it.

@u9g u9g Sep 17, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But the real congestion risk you're pointing at is one neither SDK guards against today: the backoff has no jitter. It's failoverBackoffMs * 2 ** attempt here and backoffBase << attempt in Go, both fully deterministic, so under a shared 5xx event every client retries in lockstep at exactly 200ms and 600ms.

Should I add jitter to the backoff?

}

if (!retryable || next === undefined) {
Expand Down
7 changes: 7 additions & 0 deletions packages/livekit-server-sdk/src/failover.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,15 @@ describe('failoverAttempts', () => {
expect(failoverAttempts(true, 'myproject.region.livekit.cloud')).toBe(FAILOVER_MAX_ATTEMPTS);
});

it('fails over for the LiveKit Cloud API hosts', () => {
expect(failoverAttempts(true, 'cloud-api.livekit.io')).toBe(FAILOVER_MAX_ATTEMPTS);
expect(failoverAttempts(true, 'cloud-api.staging.livekit.io')).toBe(FAILOVER_MAX_ATTEMPTS);
expect(failoverAttempts(true, 'CLOUD-API.LIVEKIT.IO')).toBe(FAILOVER_MAX_ATTEMPTS);
});

it('does not fail over for non-cloud hosts', () => {
expect(failoverAttempts(true, 'myproject.livekit.io')).toBe(1);
expect(failoverAttempts(true, 'cloud-api.example.com')).toBe(1);
expect(failoverAttempts(true, 'example.com')).toBe(1);
expect(failoverAttempts(true, '127.0.0.1')).toBe(1);
expect(failoverAttempts(true, 'notlivekit.cloud')).toBe(1);
Expand Down
12 changes: 9 additions & 3 deletions packages/livekit-server-sdk/src/failover.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,16 @@ export const MIN_FAILOVER_TIMEOUT_SECONDS = 5;

/**
* Total request attempts for a host; 1 means no failover. Failover only engages
* when enabled, the host is a LiveKit Cloud domain, and the request timeout is
* long enough to retry. `force` bypasses the cloud-host check (test-only).
* when enabled, the host is a LiveKit Cloud project or Cloud API domain, and the
* request timeout is long enough to retry. `force` bypasses the cloud-host check (test-only).
*/
export function failoverAttempts(
enabled: boolean,
hostname: string,
force = false,
timeoutSeconds = 0,
): number {
if (!enabled || !(force || isCloud(hostname))) {
if (!enabled || !(force || isCloud(hostname) || isCloudApi(hostname))) {
return 1;
}
if (timeoutSeconds > 0 && timeoutSeconds < MIN_FAILOVER_TIMEOUT_SECONDS) {
Expand All @@ -44,6 +44,12 @@ function isCloud(hostname: string): boolean {
return hostname.endsWith('.livekit.cloud');
}

// The LiveKit Cloud API hosts: cloud-api.livekit.io and cloud-api.<env>.livekit.io.
export function isCloudApi(hostname: string): boolean {
const host = hostname.toLowerCase();
return host.startsWith('cloud-api.') && host.endsWith('.livekit.io');
}

/** Normalizes a region URL to an http(s) scheme (ws -> http, wss -> https). */
function toHttp(url: string): string {
return url.startsWith('ws') ? `http${url.slice(2)}` : url;
Expand Down
Loading