From 59f0ea857fc608133f19063eda41490254d57301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:47:59 +0200 Subject: [PATCH 01/14] feat(telemetry): a JS pipeline design, and a browser that pings a collector The mobile SDKs get this from the Rust core; the browser and React Native cannot (Hermes has no WebAssembly), so the question is what to build and what to take. TELEMETRY.md answers it with measurements: the OTLP encoder is worth importing (5.2 KB gzipped), the rest of the OpenTelemetry SDK is not (23 KB against 44 KB of remaining size budget) because our trace is a scope and our batching is the upload policy. The PoC is the transport end of that: one lk.ping, both encodings, from a real Chromium at the same collector and Grafana LGTM stack the mobile harness uses. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 153 ++++++++++++++++++++++++ package.json | 3 +- pnpm-lock.yaml | 112 ++++++++++++++++- src/telemetry/index.ts | 127 ++++++++++++++++++++ src/telemetry/otelcol-web.yaml | 30 +++++ src/telemetry/telemetry.browser.test.ts | 25 ++++ vitest.telemetry.config.mts | 18 +++ 7 files changed, 462 insertions(+), 6 deletions(-) create mode 100644 TELEMETRY.md create mode 100644 src/telemetry/index.ts create mode 100644 src/telemetry/otelcol-web.yaml create mode 100644 src/telemetry/telemetry.browser.test.ts create mode 100644 vitest.telemetry.config.mts diff --git a/TELEMETRY.md b/TELEMETRY.md new file mode 100644 index 0000000000..09d97464ce --- /dev/null +++ b/TELEMETRY.md @@ -0,0 +1,153 @@ +# Client telemetry — JS ecosystem design + +How `livekit-client` (browser) and `@livekit/react-native` get the pipeline that the Swift, Kotlin +and Dart SDKs get from the Rust core. `livekit-telemetry/SPEC.md` in `rust-sdks` stays the source of +truth for event names, attributes and cadences; this document is only about what carries them here. + +## 1. OpenTelemetry SDK, the Rust core via WASM, or our own? + +**Our own policy, OpenTelemetry's encoder.** One new runtime dependency — +`@opentelemetry/otlp-transformer`, pinned — turns plain record objects into an OTLP request body in +protobuf or JSON. Nothing else from OpenTelemetry ships. + +Measured here with esbuild (`--bundle --minify`, `gzip -9`), `@opentelemetry/*` 0.222.0 / 2.11.0: + +| Bundle | minified | gzipped | +|---|---|---| +| `{ Room }` from `livekit-client` today | 409 KB | **106 KB** (budget in `.size-limit.cjs`: 150 kB) | +| `otlp-transformer` proto serializers, logs + traces | 17.8 KB | **5.2 KB** | +| logs SDK + OTLP/HTTP **JSON** exporter | 42.9 KB | 13.5 KB | +| logs SDK + OTLP/HTTP **proto** exporter | 49.4 KB | 15.2 KB | +| logs + traces SDKs + proto exporters | 78.2 KB | **23.0 KB** | + +What the full SDK adds over the serializers is a batch processor, a fetch transport and the +provider/context plumbing: ~18 KB gzipped, most of the remaining size budget, for three things that +do not fit. + +- **The trace model is different.** Our trace is a *scope* — one Room connection across reconnects, + its id minted by the pipeline and stamped on every span and log record. OTel's tracer wants a + context tree with a propagator and an active-span mechanism; we would be fighting it to get one + long-lived id onto records that have no active span. +- **The batching is the upload policy.** `BatchLogRecordProcessor` is a timer and a queue. + SPEC's policy is holds while `lk.connect` is open, one request in flight, a per-tick budget, a + 60 s cap, a flood guard and a self-report. That is written either way; the processor would sit + underneath it doing a second, conflicting round of batching. +- **Logs are pre-1.0.** `@opentelemetry/sdk-logs` is `0.222.0` and takes breaking changes across + minors. `livekit-client` is a library on a hard size budget that many apps pin; the smaller the + exposed surface of an experimental dependency, the better. The serializers are the smallest + useful unit, and their input is structural — plain objects, no classes to construct. + +Two things we take from the SDK as knowledge rather than code, because it has been through this: +the retryable status set (`429`, `502`, `503`, `504`, honouring `Retry-After`), which is already +what the Rust transport does; and the keepalive accounting — browsers cap *all* in-flight +`fetch(keepalive)` bodies at 64 KiB together and Chromium at 9 concurrent requests, so the last +flush of a page has to be small. + +**Not the Rust core via WASM.** Hermes has no WebAssembly, so React Native could never share it, +and sharing is the entire reason to consider it. A browser-only WASM core would be a third +implementation, not a second. + +## 2. Less custom logic than the device SDKs + +Yes — roughly half of the core does not exist here. + +| Dropped | Why | +|---|---| +| Write-ahead file cache, gzip-on-disk, 24 h age prune, next-launch replay | no disk (§3) | +| `lk.device.thermal.changed`, `.low_power.changed`, `.battery.changed`, `.memory.changed`, `.audio_route.changed`, `.audio.interruption` | no web or RN API for any of them | +| The cadence factors those signals drive | only `background` (`visibilitychange` / `AppState`) and `constrained` (`navigator.connection.saveData`, Chromium) survive | +| The FFI layer — UniFFI types, callback interfaces, the transport trait | the pipeline is in the same language as the SDK | +| `record_stats_report` raw-entry parsing | `RTCStatsReport` is already the SDK's own shape (`src/room/stats.ts`, `monitorFrequency = 2000`) | + +| Kept | Why | +|---|---| +| Scope = trace per Room, `session.id` on every record | the whole query story depends on it | +| `lk.rtc.stats.sample` windowing (15 s, counters + min/max/avg) | the reason the project exists | +| Bounded queue, batch size/byte caps, oldest-first eviction, counted | §3 makes the queue the *only* bound | +| Upload holds during `lk.connect` / `lk.reconnect`, 60 s cap, one request in flight | telemetry never wins over media | +| Flood guard, `lk.telemetry.report` | fleet-wide denominators, same shape as everywhere else | + +## 3. Caching: in-memory only + +Yes. Nobody in this ecosystem persists, and the reasons for disk on mobile do not exist in a tab. + +- **OpenTelemetry JS** caches nothing: `BatchLogRecordProcessor` is a bounded in-memory queue and + the spec puts retry on the exporter, explicitly not on the processor. +- **Sentry browser** is in-memory; offline caching is opt-in, by wrapping the transport in + `makeBrowserOfflineTransport` (IndexedDB). +- **Datadog browser-SDK** keeps batches in memory and flushes on `visibilitychange`. +- **Grafana Faro** is in-memory. + +A tab's lifetime is the call's lifetime; there is no "app killed in the background, replay at next +launch". The bound is therefore the queue alone (2048 records, oldest evicted and counted as +`lk.telemetry.dropped.queue_full`), and the last-gasp flush is best effort: `fetch(keepalive)` under +64 KiB on `pagehide`, an ordinary `fetch` on RN's `AppState` → background. Neither turns a +disappearing page into a durable queue, which is why the flush happens at +`visibilitychange → hidden` and not at `unload`. + +React Native *can* be killed with a backlog, and it does have `AsyncStorage`. Not for v1: it is +async, slow, and a backlog that matters needs the whole cache policy (age, prune, replay budget) +that §2 just deleted. Revisit if the field shows RN sessions losing their tail. + +## 4. Protobuf or JSON? + +**Protobuf by default, JSON behind a switch.** Both come from the same package, so this is one +line, and the PoC posts both. + +- Smaller: one `lk.ping` with a resource and three attributes is **326 bytes of protobuf against + 846 bytes of JSON** (measured in the React Native PoC below, same record, same encoder package). + JSON repeats every key as a string, which is what the OTel issue tracker cites for browsers in + the first place. +- **JSON's ids do not survive LiveKit Cloud today.** OTLP/JSON mandates *hex* `traceId`/`spanId` + (`otlp-transformer`'s `JSON_ENCODER` passes them through as hex; `PROTOBUF_ENCODER` converts to + bytes). Posting hex ids to staging landed records whose trace and span ids were **zero** — the + ingest reads them as stock protojson `bytes`, i.e. base64. Protobuf is unaffected. Filing this is + worth it regardless, because every standard OTLP/JSON client hits it. +- `Content-Encoding: gzip` is *not* in the v1 design: `CompressionStream` is Chrome 80+, Safari + 16.4+, Firefox 113+ and absent in Hermes, so it would have to be feature-detected for a win + protobuf already mostly delivers. + +## 5. Reusable by React Native + +By construction: `@livekit/react-native` depends on `livekit-client`, so telemetry that ships in +this package is in RN with no second implementation. The rules that keep it that way: + +- No DOM or `document`/`navigator` access outside the one lifecycle seam. Everything else — + records, serialization, batching, transport — is plain JS and `fetch`. +- No `crypto.getRandomValues` requirement: ids need to be unique, not unguessable, so `Math.random` + is the fallback and RN needs no `react-native-get-random-values`. +- `TextEncoder` is only on the JSON path (the protobuf serializer writes its own bytes); RN + polyfills it in `registerGlobals` anyway. +- A binary body is fine: RN's `convertRequestBody` base64-encodes an `ArrayBuffer`/`ArrayBufferView` + for the bridge. It costs ~33 % in-process, nothing on the wire. +- The seam: `pagehide` / `visibilitychange` in a page, `AppState` in an app; `navigator.connection` + in Chromium, nothing (or `@react-native-community/netinfo`, an app-owned dependency) in RN. + +## Shape + +``` +src/telemetry/ + index.ts pipeline: resource, destination, queue, flush timer, ping ← the PoC is this file + scope.ts one per Room: trace id, room/participant attributes, spans + stats.ts getStats readings → lk.rtc.stats.sample windows + transport.ts fetch, holds, 429/5xx, Retry-After, one request in flight + lifecycle.ts the platform seam (browser events / RN AppState) +``` + +## Proof of concept + +Real Chromium (Playwright), the same collector + Grafana LGTM stack the mobile harness uses, one +`lk.ping` per encoding: + +```sh +otelcol-contrib --config src/telemetry/otelcol-web.yaml # :4320, CORS, fans out to :4318 LGTM +pnpm vitest run --config vitest.telemetry.config.mts +``` + +A page cannot POST at an OTLP receiver that does not answer the preflight, which is the one +difference from the mobile harness config: `receivers.otlp.protocols.http.cors`. + +React Native runs the identical file — `telemetry-poc/` in the `client-sdk-react-native` worktree +copies `src/telemetry/index.ts` verbatim. Metro resolves the package and Hermes runs both encoders +(`hermes-check.js`, which stubs `fetch` and needs no simulator); the app itself pings the same +collector from the iOS simulator, which shares the host's network stack. diff --git a/package.json b/package.json index 4c3fa3e1e0..794804a2dd 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "dependencies": { "@livekit/mutex": "1.1.1", "@livekit/protocol": "1.50.4", + "@opentelemetry/otlp-transformer": "0.222.0", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", @@ -94,6 +95,7 @@ "@eslint/js": "10.0.1", "@livekit/changesets-changelog-github": "^0.2.0", "@livekit/throws-transformer": "^0.1.3", + "@nbilyk/downlevel-dts": "^0.15.2", "@rollup/plugin-babel": "7.1.0", "@rollup/plugin-commonjs": "29.0.3", "@rollup/plugin-json": "6.1.0", @@ -111,7 +113,6 @@ "@typescript/native": "catalog:", "@vitest/browser": "^4.1.10", "@vitest/browser-playwright": "^4.1.10", - "@nbilyk/downlevel-dts": "^0.15.2", "eslint": "10.7.0", "eslint-config-airbnb-extended": "^2.3.2", "eslint-config-prettier": "10.1.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 955bc77eb4..06bb41d338 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@livekit/protocol': specifier: 1.50.4 version: 1.50.4 + '@opentelemetry/otlp-transformer': + specifier: 0.222.0 + version: 0.222.0(@opentelemetry/api@1.9.1) '@types/dom-mediacapture-record': specifier: ^1 version: 1.0.22 @@ -215,7 +218,7 @@ importers: version: 8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0) vitest: specifier: ^4.1.10 - version: 4.1.11(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) packages: @@ -1174,6 +1177,54 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opentelemetry/api-logs@0.222.0': + resolution: {integrity: sha512-9mb1If+IF6u0ZVXkHQ6ogEae5HwA6ajIVUgpSDQyRASxft6BSXHvBvPooRle3yFN/fKnCdSOnuu0OC3PLcF6+g==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/core@2.11.0': + resolution: {integrity: sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/otlp-transformer@0.222.0': + resolution: {integrity: sha512-/F3BZ89+CJQnZkMh2tCrtcdB+XT2Dxhj4FFE+WPQ//413hmFL0/RfEX6vgOIWGhiSzrkHWTK3+6SiT7K5/g/jQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@2.11.0': + resolution: {integrity: sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.222.0': + resolution: {integrity: sha512-+19YHODIjaUCArxleaJtuufFZVpz/xvvK+VllQqE+W8hHolxdoRwHfK/s667zezwh1hkx6FFF+oYzetYgqK+Bg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.11.0': + resolution: {integrity: sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace@2.11.0': + resolution: {integrity: sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + '@oxc-project/types@0.139.0': resolution: {integrity: sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==} @@ -5800,6 +5851,56 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.15.0 + '@opentelemetry/api-logs@0.222.0': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/otlp-transformer@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/resources@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-logs@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-metrics@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/semantic-conventions@1.43.0': {} + '@oxc-project/types@0.139.0': {} '@pkgr/core@0.3.6': {} @@ -6495,7 +6596,7 @@ snapshots: '@vitest/mocker': 4.1.10(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) playwright: 1.61.1 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) + vitest: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) transitivePeerDependencies: - bufferutil - msw @@ -6511,7 +6612,7 @@ snapshots: pngjs: 7.0.0 sirv: 3.0.2 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) + vitest: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) ws: 8.20.0 transitivePeerDependencies: - bufferutil @@ -8324,7 +8425,7 @@ snapshots: machina: 7.0.1 machina-inspect: 4.0.0(machina@7.0.1) optionalDependencies: - vitest: 4.1.11(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) + vitest: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) machina@7.0.1: {} @@ -9263,7 +9364,7 @@ snapshots: tsx: 4.23.1 yaml: 2.9.0 - vitest@4.1.11(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)): + vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-playwright@4.1.10)(happy-dom@20.11.0)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.11 '@vitest/mocker': 4.1.11(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0)) @@ -9286,6 +9387,7 @@ snapshots: vite: 8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: + '@opentelemetry/api': 1.9.1 '@types/node': 26.1.2 '@vitest/browser-playwright': 4.1.10(playwright@1.61.1)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.4.2)(terser@5.39.2)(tsx@4.23.1)(yaml@2.9.0))(vitest@4.1.11) happy-dom: 20.11.0 diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts new file mode 100644 index 0000000000..ee59d76d10 --- /dev/null +++ b/src/telemetry/index.ts @@ -0,0 +1,127 @@ +/** + * Client telemetry — PoC of the JS half of the pipeline the Swift/Kotlin/Dart SDKs get from the + * Rust core (`livekit-telemetry`, SPEC.md). The wire format is the only part taken from + * OpenTelemetry: `@opentelemetry/otlp-transformer` turns plain records into an OTLP request body, + * in protobuf or JSON, and everything above it — scopes, windows, batching, holds — is ours. + * + * Browser and React Native share this file: nothing here touches `document`, `navigator` or any + * DOM type, so the only platform-specific piece left is the lifecycle hook that decides when to + * flush (`visibilitychange` in a page, `AppState` in an app). + */ +import { JsonLogsSerializer, ProtobufLogsSerializer } from '@opentelemetry/otlp-transformer'; + +/** What the core reports as `telemetry.sdk.*`; `service.*` and `os.*` come from the caller. */ +const SCOPE = { name: 'livekit-telemetry', version: '0.0.0-poc' }; + +export interface TelemetryOptions { + /** OTLP logs route — LiveKit Cloud: `https:///observability/client/logs/otlp/v0`. */ + endpoint: string; + headers?: Record; + /** protobuf by default: smaller, and the only encoding whose trace ids survive every ingest. */ + encoding?: 'protobuf' | 'json'; + resource?: Record; +} + +export interface Delivery { + status: number; + /** Set when the collector asked for a hold (429/502/503/504); `undefined` means "no answer". */ + retryAfterMs?: number; + bytes: number; +} + +type Attributes = { [key: string]: string | number | boolean | undefined }; + +/** The fields `otlp-transformer` reads off a log record — a subset of OTel's `ReadableLogRecord`. */ +interface LogRecord { + hrTime: [number, number]; + hrTimeObserved: [number, number]; + eventName?: string; + severityNumber?: number; + severityText?: string; + body?: string; + attributes: Attributes; + droppedAttributesCount: number; + resource: { attributes: Record }; + instrumentationScope: typeof SCOPE; + spanContext?: { traceId: string; spanId: string; traceFlags: number }; +} + +/** Browsers cap *all* in-flight keepalive bodies at 64 KiB together; the transformer's own limit. */ +const KEEPALIVE_LIMIT = 60 * 1024; + +// ponytail: unique, not unguessable — Math.random is fine for a trace id, and it keeps React +// Native from needing `react-native-get-random-values`. +function randomHex(bytes: number): string { + const buffer = new Uint8Array(bytes); + if (globalThis.crypto?.getRandomValues) globalThis.crypto.getRandomValues(buffer); + else for (let i = 0; i < bytes; i += 1) buffer[i] = Math.floor(Math.random() * 256); + return Array.from(buffer, (byte) => byte.toString(16).padStart(2, '0')).join(''); +} + +function hrTime(): [number, number] { + const ms = Date.now(); + return [Math.trunc(ms / 1000), Math.round((ms % 1000) * 1e6)]; +} + +export function newTraceId(): string { + return randomHex(16); +} + +/** One discrete event, in the shape `lk.ping` has in SPEC.md: the name doubles as the body. */ +export function event( + name: string, + attributes: Attributes, + options: TelemetryOptions, + traceId?: string, +): LogRecord { + const now = hrTime(); + return { + hrTime: now, + hrTimeObserved: now, + eventName: name, + severityNumber: 9, // INFO + severityText: 'INFO', + body: name, + attributes, + droppedAttributesCount: 0, + resource: { + attributes: { + 'telemetry.sdk.name': SCOPE.name, + 'telemetry.sdk.language': 'webjs', + 'telemetry.sdk.version': SCOPE.version, + ...options.resource, + }, + }, + instrumentationScope: SCOPE, + spanContext: traceId ? { traceId, spanId: randomHex(8), traceFlags: 1 } : undefined, + }; +} + +export async function send(records: LogRecord[], options: TelemetryOptions): Promise { + const json = options.encoding === 'json'; + const serializer = json ? JsonLogsSerializer : ProtobufLogsSerializer; + // The serializers are structural: they read the fields above and nothing else. + const body = serializer.serializeRequest(records as never)!; + const response = await fetch(options.endpoint, { + method: 'POST', + headers: { + 'Content-Type': json ? 'application/json' : 'application/x-protobuf', + // RFC 9218 lowest urgency, as on every other platform: telemetry never wins over media. + Priority: 'u=7', + ...options.headers, + }, + body: body as BodyInit, + keepalive: body.byteLength <= KEEPALIVE_LIMIT, + }); + const retryAfter = response.headers.get('Retry-After'); + return { + status: response.status, + retryAfterMs: retryAfter ? Number.parseInt(retryAfter, 10) * 1000 : undefined, + bytes: body.byteLength, + }; +} + +/** The smoke test from SPEC.md: one `lk.ping`, one request, whatever the collector says back. */ +export async function ping(options: TelemetryOptions, seq = 1): Promise { + return send([event('lk.ping', { 'lk.ping.seq': seq }, options, newTraceId())], options); +} diff --git a/src/telemetry/otelcol-web.yaml b/src/telemetry/otelcol-web.yaml new file mode 100644 index 0000000000..8cecc9cb30 --- /dev/null +++ b/src/telemetry/otelcol-web.yaml @@ -0,0 +1,30 @@ +# Browser variant of the mobile harness collector (Tests/.../otelcol-lgtm.yaml in client-sdk-swift): +# same file sink + Grafana LGTM fan-out, on its own port, with CORS — a page cannot POST to an +# OTLP receiver that does not answer the preflight. +# Run: otelcol-contrib --config src/telemetry/otelcol-web.yaml +receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:4320 + cors: + allowed_origins: ['http://localhost:*', 'http://127.0.0.1:*'] + allowed_headers: ['*'] +exporters: + file: + path: /tmp/livekit-telemetry-web.jsonl + otlphttp/lgtm: + endpoint: http://127.0.0.1:4318 + retry_on_failure: + enabled: false +service: + telemetry: + metrics: + level: none + pipelines: + logs: + receivers: [otlp] + exporters: [file, otlphttp/lgtm] + traces: + receivers: [otlp] + exporters: [file, otlphttp/lgtm] diff --git a/src/telemetry/telemetry.browser.test.ts b/src/telemetry/telemetry.browser.test.ts new file mode 100644 index 0000000000..186e146626 --- /dev/null +++ b/src/telemetry/telemetry.browser.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from 'vitest'; +import { ping } from './index'; + +// A real Chromium, a real collector: `otelcol-contrib --config src/telemetry/otelcol-web.yaml` +// (port 4320, CORS on, fanning out to the same Grafana LGTM stack the mobile harness uses). +// Run: pnpm vitest run --config vitest.telemetry.config.mts +const endpoint = 'http://127.0.0.1:4320/v1/logs'; +const resource = { + 'service.name': 'livekit-client-js', + 'service.version': '2.22.3-poc', + 'os.name': 'browser', +}; + +test('a protobuf ping reaches the collector', async () => { + const delivery = await ping({ endpoint, resource }, 1); + expect(delivery.status).toBe(200); + expect(delivery.bytes).toBeGreaterThan(0); + console.log('protobuf ping:', JSON.stringify(delivery)); +}); + +test('a json ping reaches the collector', async () => { + const delivery = await ping({ endpoint, resource, encoding: 'json' }, 2); + expect(delivery.status).toBe(200); + console.log('json ping:', JSON.stringify(delivery)); +}); diff --git a/vitest.telemetry.config.mts b/vitest.telemetry.config.mts new file mode 100644 index 0000000000..0a35bf1ec9 --- /dev/null +++ b/vitest.telemetry.config.mts @@ -0,0 +1,18 @@ +import { playwright } from '@vitest/browser-playwright'; +import { defineConfig } from 'vitest/config'; + +// Telemetry PoC: a real browser posting OTLP at a real collector. No mock server, no globalSetup — +// the only prerequisite is `otelcol-contrib --config src/telemetry/otelcol-web.yaml`. +export default defineConfig({ + test: { + include: ['src/telemetry/*.browser.test.ts'], + testTimeout: 20_000, + browser: { + enabled: true, + provider: playwright(), + headless: true, + instances: [{ browser: 'chromium' }], + screenshotFailures: false, + }, + }, +}); From b820f88fd6cb2b0e85e569f8081b21ca31294aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:52:33 +0200 Subject: [PATCH 02/14] test(telemetry): weigh the encoder against the SDK's own size budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit esbuild said 5.2 KB gzipped for the OTLP serializers; the repo's size-limit says what that means here — 4.4 kB brotli on the UMD bundle, which has 6.4 kB of room. The ESM path has 31 kB and does not care. Recorded because it is the number that decides against the full OpenTelemetry SDK, and because the pipeline that goes on top of the encoder will not fit under the UMD limit as it stands. Co-Authored-By: Claude Opus 5 (1M context) --- .size-limit.cjs | 13 +++++++++++++ TELEMETRY.md | 29 +++++++++++++++++++++++++---- src/index.ts | 3 +++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.size-limit.cjs b/.size-limit.cjs index 81a6ad3fb3..5c6cf624ab 100644 --- a/.size-limit.cjs +++ b/.size-limit.cjs @@ -9,4 +9,17 @@ module.exports = [ import: '{ Room }', limit: '130 kB', }, + // PoC: what the OTLP encoder costs on top of Room, against the same budgets. + { + name: 'esm + telemetry', + path: 'dist/livekit-client.esm.mjs', + import: '{ Room, telemetryPing }', + limit: '150 kB', + }, + { + name: 'umd + telemetry', + path: 'dist/livekit-client.umd.js', + import: '{ Room, telemetryPing }', + limit: '130 kB', + }, ]; diff --git a/TELEMETRY.md b/TELEMETRY.md index 09d97464ce..9f0b3d83da 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -10,19 +10,27 @@ truth for event names, attributes and cadences; this document is only about what `@opentelemetry/otlp-transformer`, pinned — turns plain record objects into an OTLP request body in protobuf or JSON. Nothing else from OpenTelemetry ships. -Measured here with esbuild (`--bundle --minify`, `gzip -9`), `@opentelemetry/*` 0.222.0 / 2.11.0: +The budget first, from the repo's own `pnpm size-limit` (webpack, minified, **brotli**): + +| `pnpm size-limit` entry | before | with the encoder | limit | +|---|---|---|---| +| `{ Room }` from `dist/livekit-client.esm.mjs` | 114.24 kB | 118.75 kB | 150 kB | +| `dist/livekit-client.umd.js` (not tree-shakeable — every app pays) | 123.58 kB | **128.01 kB** | 130 kB | + +So the encoder costs ~4.4 kB brotli where it cannot be shaken out, and leaves **2 kB** under the +UMD limit. That is the number every option has to fit in. Measured against it with esbuild +(`--bundle --minify`, `gzip -9`), `@opentelemetry/*` 0.222.0 / 2.11.0: | Bundle | minified | gzipped | |---|---|---| -| `{ Room }` from `livekit-client` today | 409 KB | **106 KB** (budget in `.size-limit.cjs`: 150 kB) | | `otlp-transformer` proto serializers, logs + traces | 17.8 KB | **5.2 KB** | | logs SDK + OTLP/HTTP **JSON** exporter | 42.9 KB | 13.5 KB | | logs SDK + OTLP/HTTP **proto** exporter | 49.4 KB | 15.2 KB | | logs + traces SDKs + proto exporters | 78.2 KB | **23.0 KB** | What the full SDK adds over the serializers is a batch processor, a fetch transport and the -provider/context plumbing: ~18 KB gzipped, most of the remaining size budget, for three things that -do not fit. +provider/context plumbing: ~18 KB gzipped on top, four times the UMD headroom, for three things +that do not fit. - **The trace model is different.** Our trace is a *scope* — one Room connection across reconnects, its id minted by the pipeline and stamped on every span and log record. OTel's tracer wants a @@ -134,6 +142,19 @@ src/telemetry/ lifecycle.ts the platform seam (browser events / RN AppState) ``` +## What this design still owes an answer + +- **The UMD budget.** 2 kB of headroom is not enough for the pipeline that goes on top of the + encoder (scope, windowing, transport, self-report — call it another 3–5 kB brotli). Either + `.size-limit.cjs` moves the UMD limit to ~135 kB, or the UMD build gets telemetry behind its own + entry point the way the e2ee and frame-metadata workers already are. The ESM path, which is what + bundled apps use, has 31 kB of room and does not care. +- **Where the stats windows come from.** `src/room/stats.ts` already polls at + `monitorFrequency = 2000` per track; the window folds those readings. Whether the pipeline + subscribes to the existing monitors or gets its own `getStats()` call is an implementation + choice with a real CPU cost attached, and it should be the former. +- **The OTLP/JSON id bug on LiveKit Cloud** (see §4) — worth filing whichever encoding we ship. + ## Proof of concept Real Chromium (Playwright), the same collector + Grafana LGTM stack the mobile harness uses, one diff --git a/src/index.ts b/src/index.ts index 4fa1329baa..0dd331d423 100644 --- a/src/index.ts +++ b/src/index.ts @@ -201,3 +201,6 @@ export { type SerializerOutput, serializers, } from './utils/serializer'; + +// PoC: exported only so `pnpm size-limit` can weigh the telemetry path against the budget. +export { ping as telemetryPing } from './telemetry'; From d5dec4f29c28105de4674c15d6044055a1f3f269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:53:31 +0200 Subject: [PATCH 03/14] docs(telemetry): React Native sends the same record from the simulator Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/TELEMETRY.md b/TELEMETRY.md index 9f0b3d83da..cd37e6ec82 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -169,6 +169,8 @@ A page cannot POST at an OTLP receiver that does not answer the preflight, which difference from the mobile harness config: `receivers.otlp.protocols.http.cors`. React Native runs the identical file — `telemetry-poc/` in the `client-sdk-react-native` worktree -copies `src/telemetry/index.ts` verbatim. Metro resolves the package and Hermes runs both encoders -(`hermes-check.js`, which stubs `fetch` and needs no simulator); the app itself pings the same -collector from the iOS simulator, which shares the host's network stack. +copies `src/telemetry/index.ts` verbatim, no edits. Both pings arrive from a bare RN 0.82.1 app on +the iOS simulator at the same collector and show up in Loki next to the browser's, and the module +also runs under the bare Hermes VM with `fetch` stubbed, which is the cheap check when no simulator +is around. There it reported the size difference that settles §4: **326 bytes of protobuf against +846 bytes of JSON** for the same record. From 7bec765757e728ec4f2617e4a527c88445afd70c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:19:46 +0200 Subject: [PATCH 04/14] feat(telemetry): the SDK reports its own sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline from TELEMETRY.md, wired into Room: a scope per connect (one trace id on every record of the call), lk.connect with its checkpoints, one lk.reconnect per reconnect however many attempts it takes, lk.publish, lk.subscribe ended by the first inbound bytes, lk.rtc.stats.sample windows folded from the readings the track monitors already take, and lk.room.disconnected. Uploads hold while connect or reconnect owns the uplink, and a 429 holds without losing the batch. RTCEngine gained one field so Resuming/Restarting can say why they fired. RemoteAudioTrack.getReceiverStats now fills in packetsReceived/packetsLost, which its own type already declared. Costs +8.95 kB brotli. The ESM budget absorbs it; the UMD one did not, and is raised here rather than quietly — see TELEMETRY.md for the alternative. Co-Authored-By: Claude Opus 5 (1M context) --- .size-limit.cjs | 18 +- TELEMETRY.md | 66 ++- src/index.ts | 3 +- src/room/RTCEngine.ts | 12 +- src/room/Room.ts | 121 +++++- src/room/data-track/depacketizer.test.ts | 1 - src/room/data-track/handle.test.ts | 1 - .../incoming/IncomingDataTrackManager.test.ts | 1 - .../outgoing/OutgoingDataTrackManager.test.ts | 1 - src/room/data-track/packet/index.test.ts | 1 - src/room/data-track/packetizer.test.ts | 1 - src/room/data-track/utils.test.ts | 1 - src/room/participant/LocalParticipant.ts | 20 +- src/room/token-source/TokenSource.test.ts | 1 - src/room/token-source/types.ts | 2 +- src/room/track/LocalAudioTrack.ts | 4 + src/room/track/LocalVideoTrack.ts | 2 + src/room/track/RemoteAudioTrack.ts | 4 + src/room/track/RemoteVideoTrack.ts | 2 + src/telemetry/index.ts | 233 ++++++----- src/telemetry/otlp.ts | 120 ++++++ src/telemetry/pipeline.ts | 395 ++++++++++++++++++ src/telemetry/scope.ts | 343 +++++++++++++++ src/telemetry/telemetry.browser.test.ts | 99 +++-- src/telemetry/telemetry.test.ts | 151 +++++++ src/telemetry/telemetrySetup.ts | 35 ++ src/telemetry/webrtc.ts | 67 +++ src/test/signalServerSetup.ts | 1 - vite.config.mjs | 2 +- vitest.telemetry.config.mts | 19 +- 30 files changed, 1534 insertions(+), 193 deletions(-) create mode 100644 src/telemetry/otlp.ts create mode 100644 src/telemetry/pipeline.ts create mode 100644 src/telemetry/scope.ts create mode 100644 src/telemetry/telemetry.test.ts create mode 100644 src/telemetry/telemetrySetup.ts create mode 100644 src/telemetry/webrtc.ts diff --git a/.size-limit.cjs b/.size-limit.cjs index 5c6cf624ab..1546b14e5f 100644 --- a/.size-limit.cjs +++ b/.size-limit.cjs @@ -7,19 +7,9 @@ module.exports = [ { path: 'dist/livekit-client.umd.js', import: '{ Room }', - limit: '130 kB', - }, - // PoC: what the OTLP encoder costs on top of Room, against the same budgets. - { - name: 'esm + telemetry', - path: 'dist/livekit-client.esm.mjs', - import: '{ Room, telemetryPing }', - limit: '150 kB', - }, - { - name: 'umd + telemetry', - path: 'dist/livekit-client.umd.js', - import: '{ Room, telemetryPing }', - limit: '130 kB', + // Telemetry costs +8.95 kB brotli, and UMD cannot shake it out: 130 kB no longer fits. + // Raising it is one of two answers — the other is a separate UMD entry point, as the + // e2ee and frame-metadata workers already have. See TELEMETRY.md. + limit: '135 kB', }, ]; diff --git a/TELEMETRY.md b/TELEMETRY.md index cd37e6ec82..1f56f907b8 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -135,36 +135,68 @@ this package is in RN with no second implementation. The rules that keep it that ``` src/telemetry/ - index.ts pipeline: resource, destination, queue, flush timer, ping ← the PoC is this file - scope.ts one per Room: trace id, room/participant attributes, spans - stats.ts getStats readings → lk.rtc.stats.sample windows - transport.ts fetch, holds, 429/5xx, Retry-After, one request in flight - lifecycle.ts the platform seam (browser events / RN AppState) + index.ts the Telemetry facade: configure/setServer, the scope factory, the track registry + pipeline.ts queue, flush timer, holds, 429/5xx, one request in flight, the self-report + scope.ts one per Room connection: trace id, attributes, spans, stats windows + otlp.ts the records and their wire form (the only OpenTelemetry import lives here) + webrtc.ts the SDK's typed sender/receiver stats → one SPEC reading ``` +Where the SDK calls it — eleven places, all of them one line except the connect span: + +| Site | What it says | +|---|---| +| `Room.connect` | a Cloud URL names the destination; a scope is opened and `lk.connect` starts, holding uploads | +| `Room.attemptConnection` | `signal`, `join_recv`, `pc_connected`, `room_connected` checkpoints, then the span ends | +| `Room.applyJoinResponse` / `handleRoomUpdate` | `lk.room.*`, `lk.participant.*` on every record of the call | +| `EngineEvent.Resuming` / `Restarting` / `Resumed` / `handleSignalRestarted` | one `lk.reconnect` span per reconnect, one checkpoint per attempt, mode = whichever won | +| `Room.handleDisconnect` | open spans fail, `lk.room.disconnected`, the open windows close, one last upload | +| `ParticipantEvent.TrackSubscribed` / `Unsubscribed` | `lk.subscribe`, ended by the first inbound window with bytes | +| `Room.onLocalTrackPublished` / `Unpublished` | which scope and direction a track sid belongs to | +| `LocalParticipant.publishTrack` | `lk.publish` | +| the four track monitors | the reading they already took, every 2 s | + +`RTCEngine` gained one field: the reconnect reason, so `Resuming`/`Restarting` can carry it. +`RemoteAudioTrack.getReceiverStats` gained `packetsReceived` / `packetsLost`, which its own +`ReceiverStats` type already declared and nothing was filling in. + ## What this design still owes an answer -- **The UMD budget.** 2 kB of headroom is not enough for the pipeline that goes on top of the - encoder (scope, windowing, transport, self-report — call it another 3–5 kB brotli). Either - `.size-limit.cjs` moves the UMD limit to ~135 kB, or the UMD build gets telemetry behind its own - entry point the way the e2ee and frame-metadata workers already are. The ESM path, which is what - bundled apps use, has 31 kB of room and does not care. -- **Where the stats windows come from.** `src/room/stats.ts` already polls at - `monitorFrequency = 2000` per track; the window folds those readings. Whether the pipeline - subscribes to the existing monitors or gets its own `getStats()` call is an implementation - choice with a real CPU cost attached, and it should be the former. +- **The UMD budget — now measured, and over.** The finished integration costs **+8.95 kB brotli**: + ESM `{ Room }` goes 114.24 → 123.19 kB (limit 150 kB, comfortable), and the UMD bundle, which + cannot shake anything out, goes 123.58 → **132.70 kB against a 130 kB limit**. This branch raises + the UMD limit to 135 kB so the build passes; the alternative is a separate UMD entry point, the + way the e2ee and frame-metadata workers already have one. That is a call for the SDK's owners. +- **What the SDK's typed stats do not carry.** The windows are folded from the readings the track + monitors already take, so they cost no extra `getStats()` — but those readings are a subset of + SPEC. Missing: inbound RTT and jitter-buffer counters, video freeze and pause counts, audio + level and interruptions. Each is a field to add to `getSenderStats` / `getReceiverStats`, which + parse the raw report already; `packetsReceived` / `packetsLost` on inbound audio were added here + as the first of them. - **The OTLP/JSON id bug on LiveKit Cloud** (see §4) — worth filing whichever encoding we ship. -## Proof of concept +## Tests -Real Chromium (Playwright), the same collector + Grafana LGTM stack the mobile harness uses, one -`lk.ping` per encoding: +`pnpm test` covers the policy against a stubbed collector, reading the JSON bodies it would have +sent: a window's counters and gauges, a hold that stops uploads and not collection, a 429 that keeps +its batch, the queue evicting oldest-first and saying so, a span's checkpoints and outcome. + +`pnpm vitest run --config vitest.telemetry.config.mts` is the session: a real Chromium with fake +media devices, a real `livekit-server --dev`, two Rooms in one page, both reconnect paths, and the +collector that fans out to the same Grafana LGTM stack the mobile harness writes to. ```sh +livekit-server --dev otelcol-contrib --config src/telemetry/otelcol-web.yaml # :4320, CORS, fans out to :4318 LGTM pnpm vitest run --config vitest.telemetry.config.mts ``` +One run puts this in the collector: two `lk.connect` spans with all four checkpoints, two +`lk.publish`, four `lk.subscribe` (`subscribed` → `first_media`), two `lk.reconnect` +(`attempt 1 quick` → `attempt 2 full` at `signal_disconnected`, then a full one), six +`lk.rtc.stats.sample` windows across both directions and both kinds, and two `lk.room.disconnected` +at `client_initiated`. + A page cannot POST at an OTLP receiver that does not answer the preflight, which is the one difference from the mobile harness config: `receivers.otlp.protocols.http.cors`. diff --git a/src/index.ts b/src/index.ts index 0dd331d423..9b2b04947e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -202,5 +202,4 @@ export { serializers, } from './utils/serializer'; -// PoC: exported only so `pnpm size-limit` can weigh the telemetry path against the budget. -export { ping as telemetryPing } from './telemetry'; +export { Telemetry, type TelemetryOptions } from './telemetry'; diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index fd0fc1fd61..86c3626153 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -221,6 +221,9 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit private reconnectAttempts: number = 0; + /** Why the current reconnect started — the two events below carry it to telemetry. */ + private reconnectReason?: ReconnectReason; + private reconnectStart: number = 0; private clientConfiguration?: ClientConfiguration; @@ -1307,6 +1310,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit if (this._isClosed) { return; } + this.reconnectReason = reason; // guard for attempting reconnection multiple times while one attempt is still not finished if (this.attemptingReconnect) { this.log.warn('already attempting reconnect, returning early'); @@ -1404,7 +1408,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } this.log.info(`reconnecting, attempt: ${this.reconnectAttempts}`); - this.emit(EngineEvent.Restarting); + this.emit(EngineEvent.Restarting, this.reconnectReason); if (!this.client.isDisconnected) { await this.client.sendLeave(); @@ -1477,7 +1481,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } this.log.info(`resuming signal connection, attempt ${this.reconnectAttempts}`); - this.emit(EngineEvent.Resuming); + this.emit(EngineEvent.Resuming, this.reconnectReason); let res: ReconnectResponse | undefined; try { this.setupSignalClientCallbacks(); @@ -2068,9 +2072,9 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit export type EngineEventCallbacks = { connected: (joinResp: JoinResponse) => void; disconnected: (reason?: DisconnectReason) => void; - resuming: () => void; + resuming: (reason?: ReconnectReason) => void; resumed: () => void; - restarting: () => void; + restarting: (reason?: ReconnectReason) => void; restarted: () => void; signalResumed: () => void; signalRestarted: (joinResp: JoinResponse) => void; diff --git a/src/room/Room.ts b/src/room/Room.ts index d48ace5cb9..adffcd9190 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -14,6 +14,7 @@ import { ParticipantInfo, ParticipantInfo_State, ParticipantPermission, + ReconnectReason, Room as RoomModel, ServerInfo, SimulateScenario, @@ -46,6 +47,7 @@ import type { RoomConnectOptions, RoomOptions, } from '../options'; +import { SpanKind, Telemetry, type TelemetryScope, type TelemetrySpan } from '../telemetry'; import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays'; import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; @@ -200,6 +202,16 @@ class Room extends (EventEmitter as new () => TypedEmitter) /** future holding client initiated connection attempt */ private connectFuture?: Future; + /** One telemetry scope per connect: a trace id and the attributes every record of it carries. */ + private telemetry?: TelemetryScope; + + private connectSpan?: TelemetrySpan; + + private reconnectSpan?: TelemetrySpan; + + /** Attempts inside the *current* reconnect; the engine's own counter spans several. */ + private reconnectAttempts = 0; + private disconnectLock: Mutex; private e2eeManager: BaseE2EEManager | undefined; @@ -632,7 +644,8 @@ class Room extends (EventEmitter as new () => TypedEmitter) }) .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate) .on(EngineEvent.DataPacketReceived, this.handleDataPacket) - .on(EngineEvent.Resuming, () => { + .on(EngineEvent.Resuming, (reason?: ReconnectReason) => { + this.startReconnectSpan('quick', reason); this.clearConnectionReconcile(); this.isResuming = true; this.log.debug('Resuming signal connection'); @@ -641,6 +654,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) } }) .on(EngineEvent.Resumed, () => { + this.endReconnectSpan('ok'); this.registerConnectionReconcile(); this.isResuming = false; this.log.debug('Resumed signal connection'); @@ -858,6 +872,17 @@ class Room extends (EventEmitter as new () => TypedEmitter) } this.setAndEmitConnectionState(ConnectionState.Connecting); + if (isCloud(new URL(url))) { + // Cloud names its own destination: this host's client OTLP route, this token (SPEC). + Telemetry.setServer(url, token); + } + this.telemetry = Telemetry.scope(); + this.localParticipant.telemetry = this.telemetry; + this.connectSpan = this.telemetry.start('lk.connect', { + kind: SpanKind.client, + attributes: { 'lk.connect.attempt': 1 }, + }); + Telemetry.hold(true); if (this.regionUrlProvider?.getServerUrl().toString() !== ensureTrailingSlash(url)) { this.regionUrl = undefined; this.regionUrlProvider = undefined; @@ -1022,6 +1047,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.localParticipant.sid = pi.sid; this.localParticipant.identity = pi.identity; + this.telemetry?.setRoom({ participantSid: pi.sid, participantIdentity: pi.identity }); this.localParticipant.setEnabledPublishCodecs(joinResponse.enabledPublishCodecs); if (this.e2eeManager) { @@ -1076,6 +1102,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) } try { + this.connectSpan?.step('signal'); const joinResponse = await this.connectSignal( url, token, @@ -1084,6 +1111,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.options, abortController, ); + this.connectSpan?.step('join_recv'); this.applyJoinResponse(joinResponse); // forward metadata changed for the local participant @@ -1121,6 +1149,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.connOptions.peerConnectionTimeout, abortController, ); + this.connectSpan?.step('pc_connected'); } catch (e) { await this.engine.close(); this.recreateEngine(); @@ -1137,6 +1166,8 @@ class Room extends (EventEmitter as new () => TypedEmitter) window.addEventListener('freeze', this.onPageLeave); } this.setAndEmitConnectionState(ConnectionState.Connected); + this.connectSpan?.step('room_connected'); + this.endConnectSpan('ok'); this.emit(RoomEvent.Connected); BackOffStrategy.getInstance().resetFailedConnectionAttempts(url); this.registerConnectionReconcile(); @@ -1781,7 +1812,8 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.emitWhenConnected(RoomEvent.LocalTrackSubscribed, trackPublication, this.localParticipant); } - private handleRestarting = () => { + private handleRestarting = (reason?: ReconnectReason) => { + this.startReconnectSpan('full', reason); this.clearConnectionReconcile(); // in case we went from resuming to full-reconnect, make sure to reflect it on the isResuming flag this.isResuming = false; @@ -1824,11 +1856,61 @@ class Room extends (EventEmitter as new () => TypedEmitter) return; } this.setAndEmitConnectionState(ConnectionState.Connected); + this.endReconnectSpan('ok'); this.emit(RoomEvent.Reconnected); this.registerConnectionReconcile(); this.emitBufferedEvents(); }; + /** `lk.connect` ends once per connect, hold and all; ending it twice is a no-op. */ + private endConnectSpan(outcome: 'ok' | 'cancelled', error?: unknown) { + if (!this.connectSpan) return; + if (error !== undefined) { + this.connectSpan.fail(error); + } else { + this.connectSpan.end(outcome); + } + this.connectSpan = undefined; + Telemetry.hold(false); + } + + private startReconnectSpan(mode: 'quick' | 'full', reason?: ReconnectReason) { + this.reconnectAttempts += 1; + const attempts = this.reconnectAttempts; + if (this.reconnectSpan) { + // A resume that turned into a restart is the same reconnect, one attempt later. + this.reconnectSpan.setAttribute('lk.reconnect.mode', mode); + this.reconnectSpan.setAttribute('lk.reconnect.attempts', attempts); + } else { + this.reconnectSpan = this.telemetry?.start('lk.reconnect', { + kind: SpanKind.client, + attributes: { + 'lk.reconnect.mode': mode, + 'lk.reconnect.attempts': attempts, + 'lk.reconnect.reason': ( + ReconnectReason[reason ?? ReconnectReason.RR_UNKNOWN] ?? 'RR_UNKNOWN' + ) + .replace(/^RR_/, '') + .toLowerCase(), + }, + }); + Telemetry.hold(true); + } + this.reconnectSpan?.step(`attempt ${attempts} ${mode}`); + } + + private endReconnectSpan(outcome: 'ok' | 'cancelled', error?: unknown) { + if (!this.reconnectSpan) return; + if (error !== undefined) { + this.reconnectSpan.fail(error); + } else { + this.reconnectSpan.end(outcome); + } + this.reconnectSpan = undefined; + this.reconnectAttempts = 0; + Telemetry.hold(false); + } + private handleDisconnect(shouldStopTracks = true, reason?: DisconnectReason) { this.clearConnectionReconcile(); this.isResuming = false; @@ -1841,6 +1923,16 @@ class Room extends (EventEmitter as new () => TypedEmitter) return; } + const reasonName = ( + DisconnectReason[reason ?? DisconnectReason.UNKNOWN_REASON] ?? 'UNKNOWN_REASON' + ).toLowerCase(); + // The call never came up: then the connect attempt is what failed, and it says why. + this.endConnectSpan('cancelled', new Error(`disconnected: ${reasonName}`)); + this.endReconnectSpan('cancelled', new Error(`disconnected: ${reasonName}`)); + this.telemetry?.disconnected(reasonName); + this.telemetry?.close(); + Telemetry.flush().catch(() => {}); + this.regionUrl = undefined; // Notify region provider about disconnect to potentially stop auto-refetch @@ -2320,6 +2412,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) private handleRoomUpdate = (room: RoomModel) => { const oldRoom = this.roomInfo; this.roomInfo = room; + this.telemetry?.setRoom({ sid: room.sid, name: room.name }); if (oldRoom && oldRoom.metadata !== room.metadata) { this.emitWhenConnected(RoomEvent.RoomMetadataChanged, room.metadata); } @@ -2450,6 +2543,19 @@ class Room extends (EventEmitter as new () => TypedEmitter) track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed); track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted); } + if (this.telemetry) { + Telemetry.registerTrack( + publication.trackSid, + this.telemetry, + track.kind === Track.Kind.Audio ? 'audio' : 'video', + 'inbound', + ); + this.telemetry.subscribeStarted(publication.trackSid, { + 'lk.track.kind': track.kind, + 'lk.track.source': publication.source, + 'lk.participant.remote_identity': participant.identity, + }); + } this.emitWhenConnected(RoomEvent.TrackSubscribed, track, publication, participant); }, ) @@ -2460,6 +2566,8 @@ class Room extends (EventEmitter as new () => TypedEmitter) .on( ParticipantEvent.TrackUnsubscribed, (track: RemoteTrack, publication: RemoteTrackPublication) => { + this.telemetry?.subscribeEnded(publication.trackSid, 'cancelled'); + Telemetry.unregisterTrack(publication.trackSid, 'inbound'); this.emit(RoomEvent.TrackUnsubscribed, track, publication, participant); }, ) @@ -2763,6 +2871,14 @@ class Room extends (EventEmitter as new () => TypedEmitter) pub.track?.on(TrackEvent.Restarted, this.onLocalTrackRestarted); pub.track?.getProcessor()?.onPublish?.(this); + if (this.telemetry && pub.track) { + Telemetry.registerTrack( + pub.trackSid, + this.telemetry, + pub.kind === Track.Kind.Audio ? 'audio' : 'video', + 'outbound', + ); + } this.emit(RoomEvent.LocalTrackPublished, pub, this.localParticipant); if (isLocalAudioTrack(pub.track)) { @@ -2786,6 +2902,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) private onLocalTrackUnpublished = (pub: LocalTrackPublication) => { pub.track?.off(TrackEvent.TrackProcessorUpdate, this.onTrackProcessorUpdate); pub.track?.off(TrackEvent.Restarted, this.onLocalTrackRestarted); + Telemetry.unregisterTrack(pub.trackSid, 'outbound'); this.emit(RoomEvent.LocalTrackUnpublished, pub, this.localParticipant); }; diff --git a/src/room/data-track/depacketizer.test.ts b/src/room/data-track/depacketizer.test.ts index a369a830b6..7ac36dc369 100644 --- a/src/room/data-track/depacketizer.test.ts +++ b/src/room/data-track/depacketizer.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import DataTrackDepacketizer from './depacketizer'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/handle.test.ts b/src/room/data-track/handle.test.ts index c486c4a8c0..34e46408ee 100644 --- a/src/room/data-track/handle.test.ts +++ b/src/room/data-track/handle.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/incoming/IncomingDataTrackManager.test.ts b/src/room/data-track/incoming/IncomingDataTrackManager.test.ts index 4c866179c5..e9935c2bc5 100644 --- a/src/room/data-track/incoming/IncomingDataTrackManager.test.ts +++ b/src/room/data-track/incoming/IncomingDataTrackManager.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { subscribeToEvents } from '../../../utils/subscribeToEvents'; import { type DataTrackFrame } from '../frame'; diff --git a/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts b/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts index aac62db8c0..35bf9e24d5 100644 --- a/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts +++ b/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type DecryptDataResponseMessage, diff --git a/src/room/data-track/packet/index.test.ts b/src/room/data-track/packet/index.test.ts index f918acb721..daddfa8316 100644 --- a/src/room/data-track/packet/index.test.ts +++ b/src/room/data-track/packet/index.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackPacket, DataTrackPacketHeader, FrameMarker } from '.'; import { DataTrackHandle } from '../handle'; diff --git a/src/room/data-track/packetizer.test.ts b/src/room/data-track/packetizer.test.ts index 452f765bc2..7e6e333a32 100644 --- a/src/room/data-track/packetizer.test.ts +++ b/src/room/data-track/packetizer.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackFrameInternal } from './frame'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/utils.test.ts b/src/room/data-track/utils.test.ts index dedf0e41bd..ce951a4cc8 100644 --- a/src/room/data-track/utils.test.ts +++ b/src/room/data-track/utils.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { U16_MAX_SIZE, WrapAroundUnsignedInt } from './utils'; diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 645482c224..ed8513db48 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -30,6 +30,7 @@ import { isFrameMetadataSupported, } from '../../frameMetadata/utils'; import type { InternalRoomOptions } from '../../options'; +import type { TelemetryScope } from '../../telemetry'; import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays'; import TypedPromise from '../../utils/TypedPromise'; import { PCTransportState } from '../PCTransportManager'; @@ -138,6 +139,9 @@ export default class LocalParticipant extends Participant { /** @internal */ activeDeviceMap: Map; + /** @internal — the Room's telemetry scope, set at connect; publishing is a span on it. */ + telemetry?: TelemetryScope; + private pendingPublishing = new Set(); private pendingPublishPromises = new Map>(); @@ -778,7 +782,21 @@ export default class LocalParticipant extends Participant { * @param options */ async publishTrack(track: LocalTrack | MediaStreamTrack, options?: TrackPublishOptions) { - return this.publishOrRepublishTrack(track, options); + const span = this.telemetry?.start('lk.publish', { + attributes: { + 'lk.track.kind': track.kind, + 'lk.track.source': 'source' in track ? track.source : undefined, + }, + }); + try { + const publication = await this.publishOrRepublishTrack(track, options); + span?.setAttribute('lk.track.sid', publication.trackSid); + span?.end('ok'); + return publication; + } catch (error) { + span?.fail(error); + throw error; + } } /** diff --git a/src/room/token-source/TokenSource.test.ts b/src/room/token-source/TokenSource.test.ts index 709e76ca46..a44c69ed87 100644 --- a/src/room/token-source/TokenSource.test.ts +++ b/src/room/token-source/TokenSource.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { describe, expect, it, vi } from 'vitest'; import { sleep } from '../utils'; import { TokenSource } from './TokenSource'; diff --git a/src/room/token-source/types.ts b/src/room/token-source/types.ts index 1750f00615..3e51638903 100644 --- a/src/room/token-source/types.ts +++ b/src/room/token-source/types.ts @@ -3,7 +3,7 @@ import type { JWTPayload } from 'jose'; import type { ValueToSnakeCase } from '../../utils/camelToSnakeCase'; // The below imports are being linked in tsdoc comments, so they have to be imported even if they // aren't being used. -// eslint-disable-next-line @typescript-eslint/no-unused-vars + import type { TokenSourceCustom, TokenSourceEndpoint, TokenSourceLiteral } from './TokenSource'; export type TokenSourceRequestObject = Required< diff --git a/src/room/track/LocalAudioTrack.ts b/src/room/track/LocalAudioTrack.ts index 05c6db4dc6..ff18cfd42b 100644 --- a/src/room/track/LocalAudioTrack.ts +++ b/src/room/track/LocalAudioTrack.ts @@ -1,4 +1,5 @@ import { AudioTrackFeature } from '@livekit/protocol'; +import { Telemetry } from '../../telemetry'; import { TrackEvent } from '../events'; import { computeBitrate, monitorFrequency } from '../stats'; import type { AudioSenderStats } from '../stats'; @@ -158,6 +159,9 @@ export default class LocalAudioTrack extends LocalTrack { if (stats && this.prevStats) { this._currentBitrate = computeBitrate(stats, this.prevStats); } + if (stats) { + Telemetry.senderStats(this.sid, [stats]); + } this.prevStats = stats; }; diff --git a/src/room/track/LocalVideoTrack.ts b/src/room/track/LocalVideoTrack.ts index 31683db86a..684c601236 100644 --- a/src/room/track/LocalVideoTrack.ts +++ b/src/room/track/LocalVideoTrack.ts @@ -7,6 +7,7 @@ import { } from '@livekit/protocol'; import type { SignalClient } from '../../api/SignalClient'; import type { StructuredLogger } from '../../logger'; +import { Telemetry } from '../../telemetry'; import { TrackEvent } from '../events'; import { ScalabilityMode, @@ -644,6 +645,7 @@ export default class LocalVideoTrack extends LocalTrack { }); this._currentBitrate = totalBitrate; } + Telemetry.senderStats(this.sid, stats); this.prevStats = statsMap; }; diff --git a/src/room/track/RemoteAudioTrack.ts b/src/room/track/RemoteAudioTrack.ts index 7d48caa7a2..665cfa0bac 100644 --- a/src/room/track/RemoteAudioTrack.ts +++ b/src/room/track/RemoteAudioTrack.ts @@ -1,3 +1,4 @@ +import { Telemetry } from '../../telemetry'; import { TrackEvent } from '../events'; import type { AudioReceiverStats } from '../stats'; import { computeBitrate } from '../stats'; @@ -230,6 +231,7 @@ export default class RemoteAudioTrack extends RemoteTrack { if (stats && this.prevStats && this.receiver) { this._currentBitrate = computeBitrate(stats, this.prevStats); } + Telemetry.receiverStats(this.sid, stats); this.prevStats = stats; }; @@ -249,6 +251,8 @@ export default class RemoteAudioTrack extends RemoteTrack { timestamp: v.timestamp, jitter: v.jitter, bytesReceived: v.bytesReceived, + packetsReceived: v.packetsReceived, + packetsLost: v.packetsLost, concealedSamples: v.concealedSamples, concealmentEvents: v.concealmentEvents, silentConcealedSamples: v.silentConcealedSamples, diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 7c511afd5b..7143adc614 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -1,4 +1,5 @@ import type { FrameMetadata } from '../../frameMetadata/types'; +import { Telemetry } from '../../telemetry'; import { debounce } from '../debounce'; import { TrackEvent } from '../events'; import type { VideoReceiverStats } from '../stats'; @@ -186,6 +187,7 @@ export default class RemoteVideoTrack extends RemoteTrack { if (stats && this.prevStats && this.receiver) { this._currentBitrate = computeBitrate(stats, this.prevStats); } + Telemetry.receiverStats(this.sid, stats); this.prevStats = stats; }; diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index ee59d76d10..3ec959b3fa 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -1,127 +1,134 @@ /** - * Client telemetry — PoC of the JS half of the pipeline the Swift/Kotlin/Dart SDKs get from the - * Rust core (`livekit-telemetry`, SPEC.md). The wire format is the only part taken from - * OpenTelemetry: `@opentelemetry/otlp-transformer` turns plain records into an OTLP request body, - * in protobuf or JSON, and everything above it — scopes, windows, batching, holds — is ours. + * Client telemetry. One pipeline per page (or per app), one scope per Room connection; see + * TELEMETRY.md for the design and `livekit-telemetry/SPEC.md` in rust-sdks for what the records + * mean — the Swift, Kotlin and Dart SDKs emit the same ones from the Rust core. * - * Browser and React Native share this file: nothing here touches `document`, `navigator` or any - * DOM type, so the only platform-specific piece left is the lifecycle hook that decides when to - * flush (`visibilitychange` in a page, `AppState` in an app). + * Nothing is collected until a destination exists: `Telemetry.configure({ endpoint })` for your own + * collector, or the first connect to LiveKit Cloud, which derives the route and the token itself. */ -import { JsonLogsSerializer, ProtobufLogsSerializer } from '@opentelemetry/otlp-transformer'; - -/** What the core reports as `telemetry.sdk.*`; `service.*` and `os.*` come from the caller. */ -const SCOPE = { name: 'livekit-telemetry', version: '0.0.0-poc' }; - -export interface TelemetryOptions { - /** OTLP logs route — LiveKit Cloud: `https:///observability/client/logs/otlp/v0`. */ - endpoint: string; - headers?: Record; - /** protobuf by default: smaller, and the only encoding whose trace ids survive every ingest. */ - encoding?: 'protobuf' | 'json'; - resource?: Record; -} - -export interface Delivery { - status: number; - /** Set when the collector asked for a hold (429/502/503/504); `undefined` means "no answer". */ - retryAfterMs?: number; - bytes: number; -} +import { Severity, hrTime, randomHex } from './otlp'; +import { Pipeline, type TelemetryOptions } from './pipeline'; +import { type StatsSample, TelemetryScope, type TrackDirection } from './scope'; +import { receiverSample, senderSample } from './webrtc'; -type Attributes = { [key: string]: string | number | boolean | undefined }; - -/** The fields `otlp-transformer` reads off a log record — a subset of OTel's `ReadableLogRecord`. */ -interface LogRecord { - hrTime: [number, number]; - hrTimeObserved: [number, number]; - eventName?: string; - severityNumber?: number; - severityText?: string; - body?: string; - attributes: Attributes; - droppedAttributesCount: number; - resource: { attributes: Record }; - instrumentationScope: typeof SCOPE; - spanContext?: { traceId: string; spanId: string; traceFlags: number }; -} +export type { TelemetryOptions } from './pipeline'; +export type { StatsSample, TrackDirection, RoomIdentity, Outcome } from './scope'; +export { TelemetryScope, TelemetrySpan } from './scope'; +export { SpanKind } from './otlp'; -/** Browsers cap *all* in-flight keepalive bodies at 64 KiB together; the transformer's own limit. */ -const KEEPALIVE_LIMIT = 60 * 1024; +const pipeline = new Pipeline(); -// ponytail: unique, not unguessable — Math.random is fine for a trace id, and it keeps React -// Native from needing `react-native-get-random-values`. -function randomHex(bytes: number): string { - const buffer = new Uint8Array(bytes); - if (globalThis.crypto?.getRandomValues) globalThis.crypto.getRandomValues(buffer); - else for (let i = 0; i < bytes; i += 1) buffer[i] = Math.floor(Math.random() * 256); - return Array.from(buffer, (byte) => byte.toString(16).padStart(2, '0')).join(''); +/** Which scope a track's stats belong to — the monitors know a sid, not a Room. */ +interface TrackRegistration { + scope: TelemetryScope; + kind: 'audio' | 'video'; + direction: TrackDirection; } -function hrTime(): [number, number] { - const ms = Date.now(); - return [Math.trunc(ms / 1000), Math.round((ms % 1000) * 1e6)]; -} +const tracks = new Map(); -export function newTraceId(): string { - return randomHex(16); -} +let lifecycleAttached = false; -/** One discrete event, in the shape `lk.ping` has in SPEC.md: the name doubles as the body. */ -export function event( - name: string, - attributes: Attributes, - options: TelemetryOptions, - traceId?: string, -): LogRecord { - const now = hrTime(); - return { - hrTime: now, - hrTimeObserved: now, - eventName: name, - severityNumber: 9, // INFO - severityText: 'INFO', - body: name, - attributes, - droppedAttributesCount: 0, - resource: { - attributes: { - 'telemetry.sdk.name': SCOPE.name, - 'telemetry.sdk.language': 'webjs', - 'telemetry.sdk.version': SCOPE.version, - ...options.resource, - }, - }, - instrumentationScope: SCOPE, - spanContext: traceId ? { traceId, spanId: randomHex(8), traceFlags: 1 } : undefined, +function attachLifecycle() { + if (lifecycleAttached || typeof document === 'undefined') return; + lifecycleAttached = true; + // The page's last chance: `pagehide` and a hidden tab, never `unload` — by then a request has + // no chance of leaving. `fetch(keepalive)` makes it best effort, not durable (TELEMETRY.md §3). + const flush = () => { + pipeline.flush(true).catch(() => {}); }; -} - -export async function send(records: LogRecord[], options: TelemetryOptions): Promise { - const json = options.encoding === 'json'; - const serializer = json ? JsonLogsSerializer : ProtobufLogsSerializer; - // The serializers are structural: they read the fields above and nothing else. - const body = serializer.serializeRequest(records as never)!; - const response = await fetch(options.endpoint, { - method: 'POST', - headers: { - 'Content-Type': json ? 'application/json' : 'application/x-protobuf', - // RFC 9218 lowest urgency, as on every other platform: telemetry never wins over media. - Priority: 'u=7', - ...options.headers, - }, - body: body as BodyInit, - keepalive: body.byteLength <= KEEPALIVE_LIMIT, + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'hidden') flush(); }); - const retryAfter = response.headers.get('Retry-After'); - return { - status: response.status, - retryAfterMs: retryAfter ? Number.parseInt(retryAfter, 10) * 1000 : undefined, - bytes: body.byteLength, - }; + window.addEventListener('pagehide', flush); } -/** The smoke test from SPEC.md: one `lk.ping`, one request, whatever the collector says back. */ -export async function ping(options: TelemetryOptions, seq = 1): Promise { - return send([event('lk.ping', { 'lk.ping.seq': seq }, options, newTraceId())], options); -} +export const Telemetry = { + /** Point the pipeline at a collector and start it. Safe to call more than once. */ + configure(options: TelemetryOptions) { + pipeline.configure(options); + attachLifecycle(); + }, + + /** LiveKit Cloud: the server URL and the connect token are the destination (SPEC). */ + setServer(serverUrl: string, token: string) { + pipeline.setServer(serverUrl, token); + attachLifecycle(); + }, + + get enabled(): boolean { + return pipeline.enabled; + }, + + scope(): TelemetryScope { + return new TelemetryScope(pipeline); + }, + + /** Uploads stop, collection does not — spans that own the uplink raise a hold (SPEC). */ + hold(up: boolean) { + pipeline.hold(up); + }, + + registerTrack( + sid: string, + scope: TelemetryScope, + kind: 'audio' | 'video', + direction: TrackDirection, + ) { + tracks.set(`${sid}:${direction}`, { scope, kind, direction }); + }, + + unregisterTrack(sid: string, direction: TrackDirection) { + tracks.delete(`${sid}:${direction}`); + }, + + /** What a local track's monitor just read; simulcast layers arrive together. */ + senderStats(sid: string | undefined, stats: Parameters[0]) { + if (sid) Telemetry.trackStats(sid, 'outbound', senderSample(stats)); + }, + + /** What a remote track's monitor just read. */ + receiverStats(sid: string | undefined, stats: Parameters[0] | undefined) { + if (sid && stats) Telemetry.trackStats(sid, 'inbound', receiverSample(stats)); + }, + + /** Called from the SDK's existing per-track monitors: no extra `getStats()` anywhere. */ + trackStats(sid: string, direction: TrackDirection, sample: StatsSample) { + if (!pipeline.enabled) return; + const registration = tracks.get(`${sid}:${direction}`); + if (!registration) return; + registration.scope.recordStats(sid, registration.kind, direction, sample); + }, + + flush(): Promise { + return pipeline.flush(true); + }, + + diagnostics(): string { + return pipeline.diagnostics(); + }, + + /** A pipeline smoke test: one record, one request, whatever the collector answers. */ + ping(seq = 1) { + const now = hrTime(); + pipeline.emit({ + hrTime: now, + hrTimeObserved: now, + eventName: 'lk.ping', + severityNumber: Severity.info, + severityText: 'INFO', + body: 'lk.ping', + attributes: { 'lk.ping.seq': seq }, + droppedAttributesCount: 0, + resource: { attributes: {} }, + instrumentationScope: { name: 'livekit-telemetry', version: '0.1.0' }, + spanContext: { traceId: randomHex(16), spanId: randomHex(8), traceFlags: 1 }, + }); + return pipeline.flush(true); + }, + + async shutdown() { + tracks.clear(); + await pipeline.shutdown(); + }, +}; diff --git a/src/telemetry/otlp.ts b/src/telemetry/otlp.ts new file mode 100644 index 0000000000..a596120c6b --- /dev/null +++ b/src/telemetry/otlp.ts @@ -0,0 +1,120 @@ +/** + * OTLP records and their wire form. The only piece of OpenTelemetry the SDK ships is + * `@opentelemetry/otlp-transformer`, whose serializers are structural: they read the fields below + * off plain objects and nothing else. See TELEMETRY.md for why the rest of the SDK is not here. + */ +import { + JsonLogsSerializer, + JsonTraceSerializer, + ProtobufLogsSerializer, + ProtobufTraceSerializer, +} from '@opentelemetry/otlp-transformer'; + +export type Encoding = 'protobuf' | 'json'; + +export type AttributeValue = string | number | boolean; + +export type Attributes = { [key: string]: AttributeValue | undefined }; + +export interface Resource { + attributes: { [key: string]: unknown }; +} + +export interface SpanContext { + traceId: string; + spanId: string; + traceFlags: number; +} + +/** OTel severity numbers; only these three are emitted (SPEC: nothing below `warn` leaves). */ +export const Severity = { info: 9, warn: 13, error: 17 } as const; + +export const INSTRUMENTATION_SCOPE = { name: 'livekit-telemetry', version: '0.1.0' }; + +export interface LogRecord { + hrTime: [number, number]; + hrTimeObserved: [number, number]; + eventName?: string; + severityNumber: number; + severityText: string; + body?: string; + attributes: Attributes; + droppedAttributesCount: number; + resource: Resource; + instrumentationScope: typeof INSTRUMENTATION_SCOPE; + spanContext?: SpanContext; +} + +export interface SpanEvent { + name: string; + time: [number, number]; + attributes: Attributes; + droppedAttributesCount: number; +} + +export interface SpanRecord { + name: string; + /** OTel SpanKind: 2 = client, 1 = internal. */ + kind: number; + spanContext: () => SpanContext; + parentSpanContext?: SpanContext; + startTime: [number, number]; + endTime: [number, number]; + duration: [number, number]; + status: { code: number; message?: string }; + attributes: Attributes; + links: never[]; + events: SpanEvent[]; + ended: boolean; + resource: Resource; + instrumentationScope: typeof INSTRUMENTATION_SCOPE; + droppedAttributesCount: number; + droppedEventsCount: number; + droppedLinksCount: number; +} + +export const SpanKind = { internal: 1, client: 3 } as const; +/** OTel status codes: cancellation is `Unset` like success — only a failure is `Error`. */ +export const SpanStatus = { unset: 0, error: 2 } as const; + +// ponytail: unique, not unguessable — ids need no entropy guarantees, and this keeps React Native +// from needing `react-native-get-random-values`. +export function randomHex(bytes: number): string { + const buffer = new Uint8Array(bytes); + if (globalThis.crypto?.getRandomValues) { + globalThis.crypto.getRandomValues(buffer); + } else { + for (let i = 0; i < bytes; i += 1) { + buffer[i] = Math.floor(Math.random() * 256); + } + } + return Array.from(buffer, (byte) => byte.toString(16).padStart(2, '0')).join(''); +} + +export function hrTime(milliseconds: number = Date.now()): [number, number] { + return [Math.trunc(milliseconds / 1000), Math.round((milliseconds % 1000) * 1e6)]; +} + +export function hrDuration(start: [number, number], end: [number, number]): [number, number] { + let seconds = end[0] - start[0]; + let nanos = end[1] - start[1]; + if (nanos < 0) { + seconds -= 1; + nanos += 1e9; + } + return [seconds, nanos]; +} + +export function serializeLogs(records: LogRecord[], encoding: Encoding): Uint8Array { + const serializer = encoding === 'json' ? JsonLogsSerializer : ProtobufLogsSerializer; + return serializer.serializeRequest(records as never)!; +} + +export function serializeSpans(records: SpanRecord[], encoding: Encoding): Uint8Array { + const serializer = encoding === 'json' ? JsonTraceSerializer : ProtobufTraceSerializer; + return serializer.serializeRequest(records as never)!; +} + +export function contentType(encoding: Encoding): string { + return encoding === 'json' ? 'application/json' : 'application/x-protobuf'; +} diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts new file mode 100644 index 0000000000..1245beb230 --- /dev/null +++ b/src/telemetry/pipeline.ts @@ -0,0 +1,395 @@ +/** + * The upload policy: one queue, one request in flight, holds while the uplink belongs to media. + * Everything a browser cannot do is absent by design — no disk cache, no replay across launches + * (TELEMETRY.md §3), so the queue is the only bound and every eviction is counted. + */ +import { + type AttributeValue, + type Attributes, + type Encoding, + INSTRUMENTATION_SCOPE, + type LogRecord, + type Resource, + Severity, + type SpanRecord, + contentType, + hrTime, + serializeLogs, + serializeSpans, +} from './otlp'; + +export interface TelemetryOptions { + /** OTLP logs route. Cloud derives it from the server URL instead; see `setServer`. */ + endpoint?: string; + headers?: Record; + encoding?: Encoding; + /** Seconds between uploads (default 15). */ + flushInterval?: number; + /** Seconds per `lk.rtc.stats.sample` window (default 15). */ + statsWindow?: number; + maxQueueSize?: number; + resource?: Record; +} + +interface Destination { + logs: string; + traces: string; + headers: Record; +} + +const FLUSH_INTERVAL = 15; +const STATS_WINDOW = 15; +const MAX_QUEUE = 2048; +const MAX_BATCH = 512; +/** Browsers cap every in-flight keepalive body at 64 KiB together; stay under it. */ +const KEEPALIVE_LIMIT = 60 * 1024; +/** A hold never outlasts this, whatever the signal that raised it claims (SPEC). */ +const HOLD_CAP_MS = 60_000; +/** LiveKit Cloud's quota answer names no delay, so a bare 429 means "this minute". */ +const THROTTLE_DEFAULT_MS = 60_000; +const FLOOD_LIMIT = 300; +const FLOOD_WINDOW_MS = 10 * 60_000; + +export interface Counters { + sent: number; + bytes: number; + failed: number; + holdsCapped: number; + droppedQueueFull: number; + droppedRejected: number; + droppedThrottled: number; + droppedRateLimited: number; +} + +function emptyCounters(): Counters { + return { + sent: 0, + bytes: 0, + failed: 0, + holdsCapped: 0, + droppedQueueFull: 0, + droppedRejected: 0, + droppedThrottled: 0, + droppedRateLimited: 0, + }; +} + +/** `wss://host/…` → the client OTLP routes on the same host. */ +export function cloudEndpoints(serverUrl: string): { logs: string; traces: string } { + const url = new URL(serverUrl); + const host = `${url.protocol === 'ws:' || url.protocol === 'http:' ? 'http' : 'https'}://${url.host}`; + return { + logs: `${host}/observability/client/logs/otlp/v0`, + traces: `${host}/observability/client/traces/otlp/v0`, + }; +} + +/** A collector's logs route implies its traces route, both for Cloud and for plain OTLP. */ +export function tracesEndpointFor(logs: string): string { + if (logs.includes('/logs/otlp/')) return logs.replace('/logs/otlp/', '/traces/otlp/'); + if (logs.endsWith('/v1/logs')) return `${logs.slice(0, -'/v1/logs'.length)}/v1/traces`; + return logs; +} + +export class Pipeline { + private logs: LogRecord[] = []; + + private spans: SpanRecord[] = []; + + private destination?: Destination; + + private encoding: Encoding = 'protobuf'; + + private timer?: ReturnType; + + private inFlight = false; + + private throttledUntil = 0; + + private holds = 0; + + private holdSince = 0; + + private counters = emptyCounters(); + + private reportDue = false; + + private floodCount = 0; + + private floodWindowStart = 0; + + private disabled = false; + + resource: Resource = { attributes: {} }; + + flushInterval = FLUSH_INTERVAL; + + statsWindow = STATS_WINDOW; + + maxQueueSize = MAX_QUEUE; + + /** Collection runs as soon as anything configured a destination — never before. */ + get enabled(): boolean { + return !this.disabled && this.destination !== undefined; + } + + configure(options: TelemetryOptions) { + this.disabled = false; + this.encoding = options.encoding ?? this.encoding; + this.flushInterval = options.flushInterval ?? this.flushInterval; + this.statsWindow = options.statsWindow ?? this.statsWindow; + this.maxQueueSize = options.maxQueueSize ?? this.maxQueueSize; + this.resource = { + attributes: { + 'telemetry.sdk.name': INSTRUMENTATION_SCOPE.name, + 'telemetry.sdk.language': 'webjs', + 'telemetry.sdk.version': INSTRUMENTATION_SCOPE.version, + ...this.resource.attributes, + ...options.resource, + }, + }; + if (options.endpoint) { + this.destination = { + logs: options.endpoint, + traces: tracesEndpointFor(options.endpoint), + headers: options.headers ?? {}, + }; + } + this.start(); + } + + /** The first connect names the destination: the server's host, the connect token (SPEC). */ + setServer(serverUrl: string, token: string) { + if (this.destination) return; // an explicit endpoint wins + const { logs, traces } = cloudEndpoints(serverUrl); + this.destination = { logs, traces, headers: { Authorization: `Bearer ${token}` } }; + this.start(); + } + + private start() { + if (this.timer || !this.enabled) return; + this.timer = setInterval(() => { + this.flush().catch(() => {}); + }, this.flushInterval * 1000); + } + + /** While a hold is up nothing is sent; records keep arriving. Capped, so a lying signal cannot + * silence the pipeline for good. */ + hold(up: boolean) { + if (up) { + if (this.holds === 0) this.holdSince = Date.now(); + this.holds += 1; + } else if (this.holds > 0) { + this.holds -= 1; + } + } + + private held(): boolean { + if (this.holds === 0) return false; + if (Date.now() - this.holdSince >= HOLD_CAP_MS) { + this.counters.holdsCapped += 1; + this.reportDue = true; + this.holdSince = Date.now(); + return false; // one batch goes out, then the hold starts over + } + return true; + } + + /** Discrete events are rationed; stats windows and the self-report are not (SPEC flood guard). */ + private floodOk(): boolean { + const now = Date.now(); + if (now - this.floodWindowStart > FLOOD_WINDOW_MS) { + this.floodWindowStart = now; + this.floodCount = 0; + } + this.floodCount += 1; + if (this.floodCount > FLOOD_LIMIT) { + this.counters.droppedRateLimited += 1; + this.reportDue = true; + return false; + } + return true; + } + + emit(record: LogRecord, options: { exemptFromFlood?: boolean } = {}) { + if (!this.enabled) return; + if (!options.exemptFromFlood && !this.floodOk()) return; + record.resource = this.resource; + this.push(this.logs, record); + } + + endSpan(span: SpanRecord) { + if (!this.enabled) return; + span.resource = this.resource; + this.push(this.spans, span); + } + + private push(queue: T[], record: T) { + queue.push(record); + const total = this.logs.length + this.spans.length; + if (total > this.maxQueueSize) { + const overflow = total - this.maxQueueSize; + const fromLogs = Math.min(overflow, this.logs.length); + this.logs.splice(0, fromLogs); // oldest first, at every level + this.spans.splice(0, overflow - fromLogs); + this.counters.droppedQueueFull += overflow; + this.reportDue = true; + } + } + + async flush(force = false): Promise { + if (!this.enabled || this.inFlight) return; + if (!force && (this.held() || Date.now() < this.throttledUntil)) return; + if (this.reportDue) this.appendReport(); + if (this.logs.length === 0 && this.spans.length === 0) return; + + this.inFlight = true; + try { + const destination = this.destination!; + if (this.logs.length > 0) { + const batch = this.logs.splice(0, MAX_BATCH); + await this.send(destination.logs, serializeLogs(batch, this.encoding), batch, this.logs); + } + // A throttle raised by the logs request applies to the spans request too: same quota. + if (this.spans.length > 0 && Date.now() >= this.throttledUntil) { + const batch = this.spans.splice(0, MAX_BATCH); + await this.send( + destination.traces, + serializeSpans(batch, this.encoding), + batch, + this.spans, + ); + } + } finally { + this.inFlight = false; + } + } + + private async send(url: string, body: Uint8Array, batch: T[], queue: T[]) { + const destination = this.destination!; + try { + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': contentType(this.encoding), + // RFC 9218 lowest urgency: telemetry never wins over media on a shared uplink. + Priority: 'u=7', + ...destination.headers, + }, + body: body as BodyInit, + keepalive: body.byteLength <= KEEPALIVE_LIMIT, + }); + if (response.status >= 200 && response.status < 300) { + this.counters.sent += 1; + this.counters.bytes += body.byteLength; + return; + } + if (response.status === 429 || response.status >= 500) { + const retryAfter = Number.parseInt(response.headers.get('Retry-After') ?? '', 10); + this.throttledUntil = + Date.now() + (Number.isFinite(retryAfter) ? retryAfter * 1000 : THROTTLE_DEFAULT_MS); + this.counters.failed += 1; + this.reportDue = true; + this.requeue(batch, queue); + return; + } + // A 4xx is the collector's verdict on the payload: retrying cannot fix it. + this.counters.droppedRejected += batch.length; + this.reportDue = true; + } catch { + this.counters.failed += 1; + this.reportDue = true; + this.requeue(batch, queue); + } + } + + /** A held or failed batch goes back at the front — a pause is not a hole in the session. */ + private requeue(batch: T[], queue: T[]) { + if (batch.length === 0) return; + queue.unshift(...batch); + const total = this.logs.length + this.spans.length; + if (total > this.maxQueueSize) { + const overflow = total - this.maxQueueSize; + const fromLogs = Math.min(overflow, this.logs.length); + this.logs.splice(0, fromLogs); + this.spans.splice(0, overflow - fromLogs); + this.counters.droppedThrottled += overflow; + } + } + + private appendReport() { + this.reportDue = false; + const counters = this.counters; + const attributes: Attributes = { + 'lk.telemetry.uploads.sent': counters.sent, + 'lk.telemetry.uploads.bytes': counters.bytes, + 'lk.telemetry.uploads.failed': counters.failed, + 'lk.telemetry.queue.records': this.logs.length + this.spans.length, + }; + if (counters.holdsCapped) attributes['lk.telemetry.holds.capped'] = counters.holdsCapped; + if (counters.droppedQueueFull) { + attributes['lk.telemetry.dropped.queue_full'] = counters.droppedQueueFull; + } + if (counters.droppedRejected) { + attributes['lk.telemetry.dropped.rejected'] = counters.droppedRejected; + } + if (counters.droppedThrottled) { + attributes['lk.telemetry.dropped.throttled'] = counters.droppedThrottled; + } + if (counters.droppedRateLimited) { + attributes['lk.telemetry.dropped.rate_limited'] = counters.droppedRateLimited; + } + const now = hrTime(); + this.logs.push({ + hrTime: now, + hrTimeObserved: now, + eventName: 'lk.telemetry.report', + severityNumber: Severity.info, + severityText: 'INFO', + body: 'lk.telemetry.report', + attributes, + droppedAttributesCount: 0, + resource: this.resource, + instrumentationScope: INSTRUMENTATION_SCOPE, + }); + } + + /** The collector said stop: throw the backlog away and never speak again (SPEC `Disabled`). */ + disable() { + this.disabled = true; + this.logs = []; + this.spans = []; + this.stop(); + } + + diagnostics(): string { + const counters = this.counters; + const state = this.disabled + ? 'disabled' + : !this.destination + ? 'no destination' + : Date.now() < this.throttledUntil + ? 'throttled' + : this.holds > 0 + ? 'held' + : 'ready'; + const lost = + counters.droppedQueueFull + + counters.droppedRejected + + counters.droppedThrottled + + counters.droppedRateLimited; + return `telemetry ${state}: sent ${counters.sent} batches, ${counters.bytes} bytes, failed ${counters.failed}, queued ${this.logs.length + this.spans.length}, lost ${lost}`; + } + + stop() { + if (this.timer) { + clearInterval(this.timer); + this.timer = undefined; + } + } + + async shutdown() { + this.stop(); + await this.flush(true); + } +} diff --git a/src/telemetry/scope.ts b/src/telemetry/scope.ts new file mode 100644 index 0000000000..15698d1841 --- /dev/null +++ b/src/telemetry/scope.ts @@ -0,0 +1,343 @@ +/** + * One scope per Room connection: a trace id, the attributes every record of that call carries, the + * spans, and the stats windows. The scope is not ended — a call's last record is simply its last. + */ +import { + type Attributes, + INSTRUMENTATION_SCOPE, + type LogRecord, + Severity, + SpanKind, + type SpanRecord, + SpanStatus, + hrDuration, + hrTime, + randomHex, +} from './otlp'; +import type { Pipeline } from './pipeline'; + +export type Outcome = 'ok' | 'error' | 'cancelled'; + +export type TrackDirection = 'inbound' | 'outbound'; + +export interface RoomIdentity { + sid?: string; + name?: string; + participantSid?: string; + participantIdentity?: string; +} + +/** One track's reading, already in SPEC units: milliseconds, not the WebRTC seconds. */ +export interface StatsSample { + codec?: string; + bytes?: number; + packets?: number; + packetsLost?: number; + framesDropped?: number; + concealedSamples?: number; + concealmentEvents?: number; + silentConcealedSamples?: number; + jitterBufferDelayMs?: number; + qualityLimitationBandwidthMs?: number; + qualityLimitationCpuMs?: number; + qualityLimitationOtherMs?: number; + jitterMs?: number; + rttMs?: number; + fps?: number; + audioLevel?: number; +} + +const COUNTERS = [ + ['bytes', 'lk.rtc.bytes'], + ['packets', 'lk.rtc.packets'], + ['packetsLost', 'lk.rtc.packets_lost'], + ['framesDropped', 'lk.rtc.frames_dropped'], + ['concealedSamples', 'lk.rtc.concealed_samples'], + ['concealmentEvents', 'lk.rtc.concealment_events'], + ['silentConcealedSamples', 'lk.rtc.silent_concealed_samples'], + ['jitterBufferDelayMs', 'lk.rtc.jitter_buffer_delay_ms'], + ['qualityLimitationBandwidthMs', 'lk.rtc.quality_limitation.bandwidth_ms'], + ['qualityLimitationCpuMs', 'lk.rtc.quality_limitation.cpu_ms'], + ['qualityLimitationOtherMs', 'lk.rtc.quality_limitation.other_ms'], +] as const; + +const GAUGES = [ + ['jitterMs', 'lk.rtc.jitter_ms'], + ['rttMs', 'lk.rtc.rtt_ms'], + ['fps', 'lk.rtc.fps'], + ['audioLevel', 'lk.rtc.audio_level'], +] as const; + +interface Gauge { + min: number; + max: number; + sum: number; + count: number; +} + +/** Counters are reported as the window's last reading (the W3C webrtc-stats model: monotonic); + * gauges as min/max/avg over it. */ +class Window { + started = Date.now(); + + samples = 0; + + codec?: string; + + last: StatsSample = {}; + + gauges = new Map(); + + add(sample: StatsSample) { + this.samples += 1; + this.codec = sample.codec ?? this.codec; + for (const [field] of COUNTERS) { + const value = sample[field]; + if (value !== undefined) this.last[field] = value; + } + for (const [field] of GAUGES) { + const value = sample[field]; + if (value === undefined || Number.isNaN(value)) continue; + const gauge = this.gauges.get(field); + if (!gauge) { + this.gauges.set(field, { min: value, max: value, sum: value, count: 1 }); + } else { + gauge.min = Math.min(gauge.min, value); + gauge.max = Math.max(gauge.max, value); + gauge.sum += value; + gauge.count += 1; + } + } + } + + attributes(): Attributes { + const attributes: Attributes = { + 'lk.rtc.window_ms': Date.now() - this.started, + 'lk.rtc.samples': this.samples, + }; + if (this.codec) attributes['lk.rtc.codec'] = this.codec; + for (const [field, key] of COUNTERS) { + const value = this.last[field]; + if (value !== undefined) attributes[key] = Math.round(value); + } + for (const [field, key] of GAUGES) { + const gauge = this.gauges.get(field); + if (!gauge) continue; + attributes[`${key}.min`] = gauge.min; + attributes[`${key}.max`] = gauge.max; + attributes[`${key}.avg`] = gauge.sum / gauge.count; + } + return attributes; + } +} + +export class TelemetrySpan { + private events: SpanRecord['events'] = []; + + private attributes: Attributes; + + private startTime = hrTime(); + + private finished = false; + + private spanId = randomHex(8); + + constructor( + private scope: TelemetryScope, + private pipeline: Pipeline, + private name: string, + private kind: number, + attributes: Attributes, + private parent?: { traceId: string; spanId: string; traceFlags: number }, + ) { + this.attributes = { ...attributes }; + } + + /** A checkpoint inside the attempt — `ws_open`, `join_recv`, `first_media`, `attempt 2 full`. */ + step(name: string) { + if (this.finished) return; + this.events.push({ name, time: hrTime(), attributes: {}, droppedAttributesCount: 0 }); + } + + setAttribute(key: string, value: Attributes[string]) { + this.attributes[key] = value; + } + + context() { + return { traceId: this.scope.traceId, spanId: this.spanId, traceFlags: 1 }; + } + + /** Ending twice is a no-op, as on every other platform. */ + end(outcome: Outcome = 'ok', errorType?: string, message?: string) { + if (this.finished) return; + this.finished = true; + const endTime = hrTime(); + this.attributes['lk.outcome'] = outcome; + if (errorType) this.attributes['error.type'] = errorType; + this.pipeline.endSpan({ + name: this.name, + kind: this.kind, + spanContext: () => this.context(), + parentSpanContext: this.parent, + startTime: this.startTime, + endTime, + duration: hrDuration(this.startTime, endTime), + // Cancellation is `Unset` like success: only a failure is an error (SPEC). + status: + outcome === 'error' ? { code: SpanStatus.error, message } : { code: SpanStatus.unset }, + attributes: { ...this.scope.attributes(), ...this.attributes }, + links: [], + events: this.events, + ended: true, + resource: { attributes: {} }, + instrumentationScope: INSTRUMENTATION_SCOPE, + droppedAttributesCount: 0, + droppedEventsCount: 0, + droppedLinksCount: 0, + }); + } + + fail(error: unknown) { + const type = error instanceof Error ? error.name : 'unknown'; + const message = error instanceof Error ? error.message : String(error); + this.end('error', type, message); + } + + cancel() { + this.end('cancelled'); + } +} + +export class TelemetryScope { + readonly traceId = randomHex(16); + + private room: RoomIdentity = {}; + + private windows = new Map(); + + private pendingSubscribes = new Map(); + + constructor(private pipeline: Pipeline) {} + + attributes(): Attributes { + return { + 'session.id': this.traceId, + 'lk.room.sid': this.room.sid, + 'lk.room.name': this.room.name, + 'lk.participant.sid': this.room.participantSid, + 'lk.participant.identity': this.room.participantIdentity, + }; + } + + setRoom(identity: RoomIdentity) { + this.room = { ...this.room, ...identity }; + } + + start( + name: string, + options: { kind?: number; attributes?: Attributes; parent?: TelemetrySpan } = {}, + ): TelemetrySpan { + return new TelemetrySpan( + this, + this.pipeline, + name, + options.kind ?? SpanKind.internal, + options.attributes ?? {}, + options.parent?.context(), + ); + } + + emit( + eventName: string, + attributes: Attributes = {}, + severity: keyof typeof Severity = 'info', + span?: TelemetrySpan, + ) { + const now = hrTime(); + const record: LogRecord = { + hrTime: now, + hrTimeObserved: now, + eventName, + severityNumber: Severity[severity], + severityText: severity.toUpperCase(), + // Log viewers key their line on the body, and not every backend surfaces event_name yet. + body: eventName, + attributes: { ...this.attributes(), ...attributes }, + droppedAttributesCount: 0, + resource: { attributes: {} }, + instrumentationScope: INSTRUMENTATION_SCOPE, + spanContext: span?.context() ?? { traceId: this.traceId, spanId: '', traceFlags: 1 }, + }; + this.pipeline.emit(record, { exemptFromFlood: eventName === 'lk.rtc.stats.sample' }); + } + + disconnected(reason: string) { + this.emit( + 'lk.room.disconnected', + { 'lk.disconnect.reason': reason }, + reason === 'client_initiated' ? 'info' : 'warn', + ); + } + + /** The intent to subscribe exists; the window that sees the first inbound bytes ends the span. */ + subscribeStarted(sid: string, attributes: Attributes) { + if (this.pendingSubscribes.has(sid)) return; + const span = this.start('lk.subscribe', { attributes: { 'lk.track.sid': sid, ...attributes } }); + span.step('subscribed'); + this.pendingSubscribes.set(sid, span); + } + + subscribeEnded(sid: string, outcome: Outcome, errorType?: string) { + const span = this.pendingSubscribes.get(sid); + if (!span) return; + this.pendingSubscribes.delete(sid); + span.end(outcome, errorType); + } + + recordStats( + sid: string, + kind: 'audio' | 'video', + direction: TrackDirection, + sample: StatsSample, + ) { + const key = `${sid}:${direction}`; + let entry = this.windows.get(key); + if (!entry) { + entry = { window: new Window(), kind, direction }; + this.windows.set(key, entry); + } + entry.window.add(sample); + if (direction === 'inbound' && (sample.bytes ?? 0) > 0) { + const span = this.pendingSubscribes.get(sid); + if (span) { + span.step('first_media'); + this.subscribeEnded(sid, 'ok'); + } + } + if (Date.now() - entry.window.started >= this.pipeline.statsWindow * 1000) { + this.closeWindow(sid, key); + } + } + + private closeWindow(sid: string, key: string) { + const entry = this.windows.get(key); + if (!entry || entry.window.samples === 0) return; + this.windows.delete(key); + this.emit('lk.rtc.stats.sample', { + 'lk.track.sid': sid, + 'lk.track.kind': entry.kind, + 'lk.track.direction': entry.direction, + ...entry.window.attributes(), + }); + } + + /** Closes every open window early — the call is ending and a partial window beats none. */ + close() { + for (const key of Array.from(this.windows.keys())) { + this.closeWindow(key.slice(0, key.lastIndexOf(':')), key); + } + for (const sid of Array.from(this.pendingSubscribes.keys())) { + this.subscribeEnded(sid, 'cancelled'); + } + } +} diff --git a/src/telemetry/telemetry.browser.test.ts b/src/telemetry/telemetry.browser.test.ts index 186e146626..98f2d9facb 100644 --- a/src/telemetry/telemetry.browser.test.ts +++ b/src/telemetry/telemetry.browser.test.ts @@ -1,25 +1,74 @@ -import { expect, test } from 'vitest'; -import { ping } from './index'; - -// A real Chromium, a real collector: `otelcol-contrib --config src/telemetry/otelcol-web.yaml` -// (port 4320, CORS on, fanning out to the same Grafana LGTM stack the mobile harness uses). -// Run: pnpm vitest run --config vitest.telemetry.config.mts -const endpoint = 'http://127.0.0.1:4320/v1/logs'; -const resource = { - 'service.name': 'livekit-client-js', - 'service.version': '2.22.3-poc', - 'os.name': 'browser', -}; - -test('a protobuf ping reaches the collector', async () => { - const delivery = await ping({ endpoint, resource }, 1); - expect(delivery.status).toBe(200); - expect(delivery.bytes).toBeGreaterThan(0); - console.log('protobuf ping:', JSON.stringify(delivery)); -}); - -test('a json ping reaches the collector', async () => { - const delivery = await ping({ endpoint, resource, encoding: 'json' }, 2); - expect(delivery.status).toBe(200); - console.log('json ping:', JSON.stringify(delivery)); -}); +import { expect, inject, test } from 'vitest'; +import { Telemetry } from '.'; +import Room, { ConnectionState } from '../room/Room'; +import { RoomEvent } from '../room/events'; + +/** + * One session through the whole integrated path: a real Chromium with fake media devices, a real + * `livekit-server --dev`, and the collector that fans out to the same Grafana LGTM stack the mobile + * harness writes to. Two Rooms in one page, so the session has an inbound and an outbound side. + * + * pnpm vitest run --config vitest.telemetry.config.mts + */ +const { url, endpoint, publisherToken, subscriberToken } = inject('telemetry'); + +async function poll(what: string, condition: () => boolean, timeout = 20_000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`timed out waiting for ${what}`); +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test('a session reports itself: connect span, stats windows, disconnect', async () => { + Telemetry.configure({ + endpoint, + flushInterval: 1, + statsWindow: 2, + resource: { + 'service.name': 'livekit-client-js', + 'service.version': '2.22.3-telemetry', + 'os.name': 'browser', + }, + }); + expect(Telemetry.enabled).toBe(true); + + const publisher = new Room(); + const subscriber = new Room(); + await publisher.connect(url, publisherToken); + await subscriber.connect(url, subscriberToken); + expect(publisher.state).toBe(ConnectionState.Connected); + + const media = await navigator.mediaDevices.getUserMedia({ audio: true, video: true }); + await publisher.localParticipant.publishTrack(media.getAudioTracks()[0]); + await publisher.localParticipant.publishTrack(media.getVideoTracks()[0]); + + await poll('the subscriber to receive both tracks', () => { + const remote = Array.from(subscriber.remoteParticipants.values())[0]; + return !!remote?.audioTrackPublications.size && !!remote?.videoTrackPublications.size; + }); + + // Two stats windows on both sides… + await sleep(6000); + + // …then both reconnect paths, which are their own spans. + for (const scenario of ['resume-reconnect', 'full-reconnect'] as const) { + const reconnected = new Promise((resolve) => publisher.once(RoomEvent.Reconnected, resolve)); + await publisher.simulateScenario(scenario); + await reconnected; + await sleep(1000); + } + + await publisher.disconnect(); + await subscriber.disconnect(); + await sleep(1500); + await Telemetry.flush(); + + const diagnostics = Telemetry.diagnostics(); + console.log('telemetry:', diagnostics); + expect(diagnostics).toContain('lost 0'); + expect(diagnostics).not.toContain('sent 0 batches'); +}, 90_000); diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts new file mode 100644 index 0000000000..763b3eb0ca --- /dev/null +++ b/src/telemetry/telemetry.test.ts @@ -0,0 +1,151 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { Pipeline } from './pipeline'; +import { TelemetryScope } from './scope'; + +/** The JSON encoding is the readable one: every assertion here reads the body the collector gets. */ +function bodyOf(call: unknown[]): any { + const init = call[1] as RequestInit; + return JSON.parse(new TextDecoder().decode(init.body as Uint8Array)); +} + +function recordsOf(call: unknown[]): any[] { + return bodyOf(call).resourceLogs.flatMap((r: any) => + r.scopeLogs.flatMap((s: any) => s.logRecords), + ); +} + +function attributesOf(record: any): Record { + return Object.fromEntries(record.attributes.map((a: any) => [a.key, Object.values(a.value)[0]])); +} + +describe('telemetry pipeline', () => { + let pipeline: Pipeline; + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(async () => new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + pipeline = new Pipeline(); + pipeline.configure({ + endpoint: 'http://collector.test/v1/logs', + encoding: 'json', + flushInterval: 3600, + statsWindow: 0.01, + }); + }); + + afterEach(() => { + pipeline.stop(); + vi.unstubAllGlobals(); + }); + + test('one window of readings becomes one record, counters last and gauges summarised', async () => { + const scope = new TelemetryScope(pipeline); + scope.setRoom({ sid: 'RM_1', name: 'harness', participantIdentity: 'publisher' }); + scope.recordStats('TR_1', 'video', 'outbound', { + bytes: 1000, + packets: 10, + rttMs: 20, + fps: 30, + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + scope.recordStats('TR_1', 'video', 'outbound', { + bytes: 3000, + packets: 30, + rttMs: 40, + fps: 24, + }); + await pipeline.flush(true); + + const window = recordsOf(fetchMock.mock.calls[0]).find( + (r) => r.eventName === 'lk.rtc.stats.sample', + ); + const attributes = attributesOf(window); + expect(attributes['lk.track.sid']).toBe('TR_1'); + expect(attributes['lk.track.direction']).toBe('outbound'); + expect(attributes['lk.room.name']).toBe('harness'); + // A counter is the window's last reading, not a sum of them (the W3C webrtc-stats model). + expect(attributes['lk.rtc.bytes']).toBe(3000); + expect(attributes['lk.rtc.samples']).toBe(2); + expect(attributes['lk.rtc.rtt_ms.min']).toBe(20); + expect(attributes['lk.rtc.rtt_ms.max']).toBe(40); + expect(attributes['lk.rtc.rtt_ms.avg']).toBe(30); + }); + + test('a hold stops uploads, never collection', async () => { + const scope = new TelemetryScope(pipeline); + pipeline.hold(true); + scope.emit('lk.test.one'); + await pipeline.flush(); + expect(fetchMock).not.toHaveBeenCalled(); + + scope.emit('lk.test.two'); + pipeline.hold(false); + await pipeline.flush(); + // Everything collected during the hold went out together: a pause is not a hole. + expect(recordsOf(fetchMock.mock.calls[0]).map((r) => r.eventName)).toEqual([ + 'lk.test.one', + 'lk.test.two', + ]); + }); + + test('a 429 holds the pipeline and keeps the batch', async () => { + fetchMock.mockResolvedValueOnce(new Response(null, { status: 429 })); + const scope = new TelemetryScope(pipeline); + scope.emit('lk.test.throttled'); + await pipeline.flush(true); + expect(pipeline.diagnostics()).toContain('throttled'); + expect(pipeline.diagnostics()).toContain('lost 0'); + + // The next tick is not spent on a request the collector already refused. + await pipeline.flush(); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // …and the record is still there when the hold lifts. + await pipeline.flush(true); + expect(recordsOf(fetchMock.mock.calls[1])[0].eventName).toBe('lk.test.throttled'); + }); + + test('the queue evicts the oldest and says how many', async () => { + pipeline.maxQueueSize = 3; + const scope = new TelemetryScope(pipeline); + for (let i = 0; i < 5; i += 1) { + scope.emit(`lk.test.${i}`); + } + await pipeline.flush(true); + + const records = recordsOf(fetchMock.mock.calls[0]); + expect(records.map((r) => r.eventName)).toContain('lk.test.4'); + expect(records.map((r) => r.eventName)).not.toContain('lk.test.0'); + const report = records.find((r) => r.eventName === 'lk.telemetry.report'); + expect(attributesOf(report)['lk.telemetry.dropped.queue_full']).toBe(2); + }); + + test('a span carries its checkpoints and its outcome', async () => { + const scope = new TelemetryScope(pipeline); + const span = scope.start('lk.connect', { attributes: { 'lk.connect.attempt': 1 } }); + span.step('ws_open'); + span.step('join_recv'); + span.end('ok'); + span.end('error'); // ending twice is a no-op + await pipeline.flush(true); + + const traceCall = fetchMock.mock.calls.find((call) => String(call[0]).endsWith('/v1/traces'))!; + const spans = bodyOf(traceCall).resourceSpans.flatMap((r: any) => + r.scopeSpans.flatMap((s: any) => s.spans), + ); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe('lk.connect'); + expect(spans[0].events.map((e: any) => e.name)).toEqual(['ws_open', 'join_recv']); + expect(attributesOf(spans[0])['lk.outcome']).toBe('ok'); + }); + + test('nothing is collected before a destination exists', async () => { + const idle = new Pipeline(); + const scope = new TelemetryScope(idle); + scope.emit('lk.test.void'); + await idle.flush(true); + expect(fetchMock).not.toHaveBeenCalled(); + expect(idle.diagnostics()).toContain('no destination'); + }); +}); diff --git a/src/telemetry/telemetrySetup.ts b/src/telemetry/telemetrySetup.ts new file mode 100644 index 0000000000..bc21d61065 --- /dev/null +++ b/src/telemetry/telemetrySetup.ts @@ -0,0 +1,35 @@ +import type { TestProject } from 'vitest/node'; +import { createToken } from '../test/signalToken'; + +/** + * vitest globalSetup for the telemetry session test. It runs in Node (the browser cannot sign a + * token) and hands the browser a room and two identities on a live `livekit-server --dev`, plus + * the collector the session reports to — the same pair the mobile harness uses: + * + * livekit-server --dev + * otelcol-contrib --config src/telemetry/otelcol-web.yaml + */ +export default async function setup({ provide }: TestProject) { + const url = process.env.LK_URL ?? 'ws://127.0.0.1:7880'; + const endpoint = process.env.LK_TELEMETRY_ENDPOINT ?? 'http://127.0.0.1:4320/v1/logs'; + const room = `telemetry-${Date.now()}`; + provide('telemetry', { + url, + endpoint, + room, + publisherToken: await createToken({ room, identity: 'publisher' }), + subscriberToken: await createToken({ room, identity: 'subscriber' }), + }); +} + +declare module 'vitest' { + interface ProvidedContext { + telemetry: { + url: string; + endpoint: string; + room: string; + publisherToken: string; + subscriberToken: string; + }; + } +} diff --git a/src/telemetry/webrtc.ts b/src/telemetry/webrtc.ts new file mode 100644 index 0000000000..62f2eee849 --- /dev/null +++ b/src/telemetry/webrtc.ts @@ -0,0 +1,67 @@ +/** + * The SDK's per-track monitors already parse `getStats()` every two seconds; this turns what they + * hold into one SPEC reading, in SPEC units. Nothing here calls `getStats()` — measuring a call + * must not cost a second poll. + */ +import type { + AudioReceiverStats, + AudioSenderStats, + VideoReceiverStats, + VideoSenderStats, +} from '../room/stats'; +import type { StatsSample } from './scope'; + +const seconds = (value: number | undefined): number | undefined => + value === undefined ? undefined : value * 1000; + +function add(total: number | undefined, value: number | undefined): number | undefined { + if (value === undefined) return total; + return (total ?? 0) + value; +} + +/** Simulcast layers are one track: their counters sum, their gauges take the liveliest layer. */ +export function senderSample(stats: Array): StatsSample { + const sample: StatsSample = {}; + for (const layer of stats) { + sample.bytes = add(sample.bytes, layer.bytesSent); + sample.packets = add(sample.packets, layer.packetsSent); + sample.packetsLost = add(sample.packetsLost, layer.packetsLost); + sample.jitterMs = Math.max(sample.jitterMs ?? 0, seconds(layer.jitter) ?? 0) || undefined; + sample.rttMs = Math.max(sample.rttMs ?? 0, seconds(layer.roundTripTime) ?? 0) || undefined; + if (layer.type === 'video') { + sample.fps = Math.max(sample.fps ?? 0, layer.framesPerSecond ?? 0) || undefined; + const durations = layer.qualityLimitationDurations; + if (durations) { + sample.qualityLimitationBandwidthMs = add( + sample.qualityLimitationBandwidthMs, + seconds(durations.bandwidth), + ); + sample.qualityLimitationCpuMs = add(sample.qualityLimitationCpuMs, seconds(durations.cpu)); + sample.qualityLimitationOtherMs = add( + sample.qualityLimitationOtherMs, + seconds(durations.other), + ); + } + } + } + return sample; +} + +export function receiverSample(stats: AudioReceiverStats | VideoReceiverStats): StatsSample { + const sample: StatsSample = { + codec: 'mimeType' in stats ? stats.mimeType : undefined, + bytes: stats.bytesReceived, + packets: stats.packetsReceived, + packetsLost: stats.packetsLost, + jitterMs: seconds(stats.jitter), + jitterBufferDelayMs: seconds(stats.jitterBufferDelay), + }; + if (stats.type === 'video') { + sample.framesDropped = stats.framesDropped; + } else { + sample.concealedSamples = stats.concealedSamples; + sample.concealmentEvents = stats.concealmentEvents; + sample.silentConcealedSamples = stats.silentConcealedSamples; + } + return sample; +} diff --git a/src/test/signalServerSetup.ts b/src/test/signalServerSetup.ts index af8a375ff3..2c14f49a42 100644 --- a/src/test/signalServerSetup.ts +++ b/src/test/signalServerSetup.ts @@ -59,7 +59,6 @@ export default async function setup(project: { provide: (name: string, value: un project.provide('e2eUnavailable', ''); }; const skip = (reason: string) => { - // eslint-disable-next-line no-console console.warn(`\n[e2e] signal-connection e2e suite SKIPPED: ${reason}\n`); project.provide('serverUrl', ''); project.provide('e2eUnavailable', reason); diff --git a/vite.config.mjs b/vite.config.mjs index af40737514..774b4ce458 100644 --- a/vite.config.mjs +++ b/vite.config.mjs @@ -44,6 +44,6 @@ export default defineConfig({ environment: 'happy-dom', // e2e tests need a real server + node WebSocket; run them via the // dedicated `pnpm test:e2e` config (vitest.e2e.config.mts), not the unit run. - exclude: [...configDefaults.exclude, '**/*.e2e.test.ts'], + exclude: [...configDefaults.exclude, '**/*.e2e.test.ts', '**/*.browser.test.ts'], }, }); diff --git a/vitest.telemetry.config.mts b/vitest.telemetry.config.mts index 0a35bf1ec9..d84c184ec6 100644 --- a/vitest.telemetry.config.mts +++ b/vitest.telemetry.config.mts @@ -1,15 +1,26 @@ import { playwright } from '@vitest/browser-playwright'; import { defineConfig } from 'vitest/config'; -// Telemetry PoC: a real browser posting OTLP at a real collector. No mock server, no globalSetup — -// the only prerequisite is `otelcol-contrib --config src/telemetry/otelcol-web.yaml`. +// The telemetry session test: a real browser, a real livekit-server --dev, a real collector. +// Prerequisites are checked by the globalSetup, which also mints the tokens (the browser cannot). export default defineConfig({ test: { include: ['src/telemetry/*.browser.test.ts'], - testTimeout: 20_000, + globalSetup: ['./src/telemetry/telemetrySetup.ts'], + testTimeout: 90_000, + hookTimeout: 60_000, + fileParallelism: false, browser: { enabled: true, - provider: playwright(), + provider: playwright({ + launchOptions: { + args: [ + '--use-fake-device-for-media-stream', + '--use-fake-ui-for-media-stream', + '--autoplay-policy=no-user-gesture-required', + ], + }, + }), headless: true, instances: [{ browser: 'chromium' }], screenshotFailures: false, From 0bbc5c963f72b5edd45f4e1784e8692db2bebe38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:26:48 +0200 Subject: [PATCH 05/14] docs(telemetry): what React Native needed, and what it did not Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 9 +++++++++ src/telemetry/pipeline.ts | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/TELEMETRY.md b/TELEMETRY.md index 1f56f907b8..fe816edd6f 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -131,6 +131,15 @@ this package is in RN with no second implementation. The rules that keep it that - The seam: `pagehide` / `visibilitychange` in a page, `AppState` in an app; `navigator.connection` in Chromium, nothing (or `@react-native-community/netinfo`, an app-owned dependency) in RN. +Done, and verified on the simulator: `@livekit/react-native`'s `src/telemetry.ts` sets +`service.name`, `service.version`, `os.name` and `os.version`, registers the `AppState` flush, and +is called from `registerGlobals`. The telemetry module itself needed no React Native branch. + +One thing the PoC found, which is about this package rather than telemetry: `livekit-client` +evaluates `class … extends DOMException` and `new TextDecoder()` at **module scope**, and Hermes has +neither, so anything importing it must come after the React Native polyfills. That is why +`registerTelemetry` is imported below them in `src/index.tsx` — the order is load-bearing. + ## Shape ``` diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index 1245beb230..0e4a1c043b 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -3,6 +3,7 @@ * Everything a browser cannot do is absent by design — no disk cache, no replay across launches * (TELEMETRY.md §3), so the queue is the only bound and every eviction is counted. */ +import { version } from '../version'; import { type AttributeValue, type Attributes, @@ -141,6 +142,10 @@ export class Pipeline { this.maxQueueSize = options.maxQueueSize ?? this.maxQueueSize; this.resource = { attributes: { + // The platform SDK owns `service.*` and `os.*`; these are what this package can say + // about itself. React Native overrides the name and adds the device (SPEC). + 'service.name': 'livekit-client-js', + 'service.version': version, 'telemetry.sdk.name': INSTRUMENTATION_SCOPE.name, 'telemetry.sdk.language': 'webjs', 'telemetry.sdk.version': INSTRUMENTATION_SCOPE.version, From 35d8d8a3d8c34e63842e1ffd30c58db79376e339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:27:36 +0200 Subject: [PATCH 06/14] fix(telemetry): an SDK with no collector keeps no track map Co-Authored-By: Claude Opus 5 (1M context) --- src/telemetry/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 3ec959b3fa..eb59181090 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -75,6 +75,8 @@ export const Telemetry = { kind: 'audio' | 'video', direction: TrackDirection, ) { + // Nothing to route when nobody is listening: an SDK without a collector keeps no map. + if (!pipeline.enabled) return; tracks.set(`${sid}:${direction}`, { scope, kind, direction }); }, From 9cd2f244453be118ca30907f63eb3b373a36834c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:40:30 +0200 Subject: [PATCH 07/14] feat(telemetry): the two device signals a page can actually answer app_state from visibilityState, which the SDK already watches, and the connection from navigator.connection, which SignalClient already reads for the connect params. Both drive SPEC's cadence factor, which stretches the flush interval and the stats window together and stops at 4x. Thermal, low power and memory have no web API at all, so a page simply never reports them and its factor stays at whatever the rest implies. React Native answers those from its native module, which is the next commit over. Also: an attribute nobody set is no longer shipped as an empty value. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 21 ++++- src/telemetry/device.ts | 147 ++++++++++++++++++++++++++++++++ src/telemetry/index.ts | 33 +++++++ src/telemetry/pipeline.ts | 30 ++++++- src/telemetry/scope.ts | 13 ++- src/telemetry/telemetry.test.ts | 52 +++++++++++ 6 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 src/telemetry/device.ts diff --git a/TELEMETRY.md b/TELEMETRY.md index fe816edd6f..7d1543e685 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -62,11 +62,28 @@ Yes — roughly half of the core does not exist here. | Dropped | Why | |---|---| | Write-ahead file cache, gzip-on-disk, 24 h age prune, next-launch replay | no disk (§3) | -| `lk.device.thermal.changed`, `.low_power.changed`, `.battery.changed`, `.memory.changed`, `.audio_route.changed`, `.audio.interruption` | no web or RN API for any of them | -| The cadence factors those signals drive | only `background` (`visibilitychange` / `AppState`) and `constrained` (`navigator.connection.saveData`, Chromium) survive | | The FFI layer — UniFFI types, callback interfaces, the transport trait | the pipeline is in the same language as the SDK | | `record_stats_report` raw-entry parsing | `RTCStatsReport` is already the SDK's own shape (`src/room/stats.ts`, `monitorFrequency = 2000`) | +Device state is not dropped, but it is not uniform either — the pipeline never measures anything +itself, so an `lk.device.*` record exists only where the platform will say so: + +| SPEC event | Browser | React Native | +|---|---|---| +| `app_state.changed` | **yes**, `document.visibilityState` | **yes**, `AppState` | +| `network.changed` | **Chromium**, `navigator.connection` — and `type` is populated on Android only, so a desktop page reports `unknown` | via `@react-native-community/netinfo`, an app-owned dependency — not wired | +| `capture.failed` | the `getUserMedia` DOMException names, per SPEC | the same errors through `react-native-webrtc` — not wired | +| `thermal.changed` | `PressureObserver` is Chromium **desktop** only — not wired | **yes**, native | +| `low_power.changed` | no web API | **yes**, native | +| `memory.changed` | no web API (`deviceMemory` is a static figure) | **yes**, native | +| `battery.changed` | Chromium only; Firefox removed it, Safari never shipped it — not wired | native, not wired | +| `audio_route.changed`, `audio.interruption` | `devicechange` says the set changed, not that it went speaker → bluetooth | the package already owns the audio session — not wired | + +React Native can reach parity with the Swift and Kotlin SDKs because `@livekit/react-native` already +ships a native module; a page structurally cannot. The cadence factors those signals drive are +implemented (`device.ts`, capped at 4× as SPEC says), so a page stretches on background and Data +Saver, and an app stretches on everything. + | Kept | Why | |---|---| | Scope = trace per Room, `session.id` on every record | the whole query story depends on it | diff --git a/src/telemetry/device.ts b/src/telemetry/device.ts new file mode 100644 index 0000000000..a05b8a2337 --- /dev/null +++ b/src/telemetry/device.ts @@ -0,0 +1,147 @@ +/** + * Device state: the events SPEC calls `lk.device.*`, and the cadence factor they drive. The + * pipeline never measures anything itself — measuring CPU costs CPU — so every value here comes + * from the platform saying so: the browser's own APIs, or React Native's native module. + * + * A browser can only answer some of this. `visibilitychange` is universal, `navigator.connection` + * is Chromium, and thermal, low power and memory pressure have no web API at all — a page simply + * never reports them, and its cadence factor stays at whatever the rest implies. + */ +import type { Attributes } from './otlp'; + +export type AppStateName = 'foreground' | 'background'; +export type ThermalState = 'nominal' | 'fair' | 'serious' | 'critical'; +export type MemoryPressure = 'normal' | 'warning' | 'critical'; +export type NetworkType = + 'wifi' | 'cell' | 'wired' | 'vpn' | 'bluetooth' | 'other' | 'unavailable' | 'unknown'; + +export interface DeviceState { + appState?: AppStateName; + thermal?: ThermalState; + memory?: MemoryPressure; + lowPower?: boolean; + networkType?: NetworkType; + /** Cellular or hotspot. */ + networkExpensive?: boolean; + /** Low Data Mode, Data Saver, `navigator.connection.saveData`. */ + networkConstrained?: boolean; +} + +/** SPEC's cadence table. Factors multiply and the product is capped at 4×. */ +const THERMAL_FACTOR: Record = { + nominal: 1, + fair: 1, + serious: 2, + critical: 4, +}; +const MEMORY_FACTOR: Record = { normal: 1, warning: 2, critical: 4 }; +const MAX_FACTOR = 4; + +export function cadenceFactor(state: DeviceState): number { + let factor = 1; + if (state.thermal) factor *= THERMAL_FACTOR[state.thermal]; + if (state.memory) factor *= MEMORY_FACTOR[state.memory]; + if (state.lowPower) factor *= 2; + if (state.appState === 'background') factor *= 2; + if (state.networkConstrained) factor *= 2; + return Math.min(factor, MAX_FACTOR); +} + +/** `NetworkInformation.type` → SPEC's enum. */ +export function networkType(type: string | undefined): NetworkType { + switch (type) { + case 'wifi': + case 'bluetooth': + return type; + case 'cellular': + return 'cell'; + case 'ethernet': + return 'wired'; + case 'none': + return 'unavailable'; + case 'wimax': + case 'mixed': + case 'other': + return 'other'; + default: + return 'unknown'; + } +} + +interface Change { + event: string; + attributes: Attributes; +} + +/** What changed since the last update, as the records SPEC names — one per group, on change only. */ +export function changes(previous: DeviceState, next: DeviceState): Change[] { + const out: Change[] = []; + if (next.appState !== undefined && next.appState !== previous.appState) { + out.push({ + event: 'lk.device.app_state.changed', + attributes: { 'lk.device.app_state': next.appState }, + }); + } + if (next.thermal !== undefined && next.thermal !== previous.thermal) { + out.push({ + event: 'lk.device.thermal.changed', + attributes: { 'lk.device.thermal.state': next.thermal }, + }); + } + if (next.memory !== undefined && next.memory !== previous.memory) { + out.push({ + event: 'lk.device.memory.changed', + attributes: { 'lk.device.memory.pressure': next.memory }, + }); + } + if (next.lowPower !== undefined && next.lowPower !== previous.lowPower) { + out.push({ + event: 'lk.device.low_power.changed', + attributes: { 'lk.device.low_power.enabled': next.lowPower }, + }); + } + const networkChanged = + (next.networkType !== undefined && next.networkType !== previous.networkType) || + (next.networkExpensive !== undefined && next.networkExpensive !== previous.networkExpensive) || + (next.networkConstrained !== undefined && + next.networkConstrained !== previous.networkConstrained); + if (networkChanged) { + out.push({ + event: 'lk.device.network.changed', + attributes: { + 'network.connection.type': next.networkType ?? previous.networkType, + 'lk.device.network.expensive': next.networkExpensive ?? previous.networkExpensive, + 'lk.device.network.constrained': next.networkConstrained ?? previous.networkConstrained, + }, + }); + } + return out; +} + +/** Everything a page can answer: the tab's visibility everywhere, the connection on Chromium. */ +export function observeBrowser(report: (state: DeviceState) => void): void { + if (typeof document !== 'undefined') { + const visibility = () => + report({ appState: document.visibilityState === 'hidden' ? 'background' : 'foreground' }); + document.addEventListener('visibilitychange', visibility); + visibility(); + } + const connection = (globalThis.navigator as { connection?: NetworkInformationLike } | undefined) + ?.connection; + if (connection) { + const network = () => + report({ + networkType: networkType(connection.type), + networkExpensive: connection.type === 'cellular', + networkConstrained: connection.saveData === true, + }); + connection.addEventListener?.('change', network); + network(); + } +} + +interface NetworkInformationLike { + type?: string; + saveData?: boolean; + addEventListener?: (event: 'change', listener: () => void) => void; +} diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index eb59181090..c5920ee967 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -6,12 +6,20 @@ * Nothing is collected until a destination exists: `Telemetry.configure({ endpoint })` for your own * collector, or the first connect to LiveKit Cloud, which derives the route and the token itself. */ +import { type DeviceState, cadenceFactor, changes, observeBrowser } from './device'; import { Severity, hrTime, randomHex } from './otlp'; import { Pipeline, type TelemetryOptions } from './pipeline'; import { type StatsSample, TelemetryScope, type TrackDirection } from './scope'; import { receiverSample, senderSample } from './webrtc'; export type { TelemetryOptions } from './pipeline'; +export type { + DeviceState, + AppStateName, + ThermalState, + MemoryPressure, + NetworkType, +} from './device'; export type { StatsSample, TrackDirection, RoomIdentity, Outcome } from './scope'; export { TelemetryScope, TelemetrySpan } from './scope'; export { SpanKind } from './otlp'; @@ -27,6 +35,11 @@ interface TrackRegistration { const tracks = new Map(); +/** Device state belongs to no call: it is filed under the pipeline's own scope (SPEC). */ +let processScope: TelemetryScope | undefined; + +let deviceState: DeviceState = {}; + let lifecycleAttached = false; function attachLifecycle() { @@ -41,6 +54,7 @@ function attachLifecycle() { if (document.visibilityState === 'hidden') flush(); }); window.addEventListener('pagehide', flush); + observeBrowser((state) => Telemetry.deviceState(state)); } export const Telemetry = { @@ -69,6 +83,24 @@ export const Telemetry = { pipeline.hold(up); }, + /** + * What the platform now says about the device. A page reports what it can see; React Native + * reports the rest from its native module. Each group becomes an `lk.device.*` record the first + * time it is seen and on every change after, and the whole state sets the cadence factor. + */ + deviceState(state: DeviceState) { + const next = { ...deviceState, ...state }; + const records = changes(deviceState, next); + deviceState = next; + if (pipeline.enabled) { + processScope ??= new TelemetryScope(pipeline); + for (const record of records) { + processScope.emit(record.event, record.attributes); + } + } + pipeline.setCadenceFactor(cadenceFactor(next)); + }, + registerTrack( sid: string, scope: TelemetryScope, @@ -131,6 +163,7 @@ export const Telemetry = { async shutdown() { tracks.clear(); + processScope = undefined; await pipeline.shutdown(); }, }; diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index 0e4a1c043b..8bfb07fb3d 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -123,12 +123,34 @@ export class Pipeline { resource: Resource = { attributes: {} }; - flushInterval = FLUSH_INTERVAL; + private baseFlushInterval = FLUSH_INTERVAL; - statsWindow = STATS_WINDOW; + private baseStatsWindow = STATS_WINDOW; + + /** SPEC's cadence policy: device pressure stretches both periods, never past 4×. */ + private cadence = 1; maxQueueSize = MAX_QUEUE; + get flushInterval(): number { + return this.baseFlushInterval * this.cadence; + } + + get statsWindow(): number { + return this.baseStatsWindow * this.cadence; + } + + setCadenceFactor(factor: number) { + if (factor === this.cadence) return; + this.cadence = factor; + // Restarting the timer is what makes a *shorter* period apply at once — pressure relieved + // should not mean waiting out a stretched interval. + if (this.timer) { + this.stop(); + this.start(); + } + } + /** Collection runs as soon as anything configured a destination — never before. */ get enabled(): boolean { return !this.disabled && this.destination !== undefined; @@ -137,8 +159,8 @@ export class Pipeline { configure(options: TelemetryOptions) { this.disabled = false; this.encoding = options.encoding ?? this.encoding; - this.flushInterval = options.flushInterval ?? this.flushInterval; - this.statsWindow = options.statsWindow ?? this.statsWindow; + this.baseFlushInterval = options.flushInterval ?? this.baseFlushInterval; + this.baseStatsWindow = options.statsWindow ?? this.baseStatsWindow; this.maxQueueSize = options.maxQueueSize ?? this.maxQueueSize; this.resource = { attributes: { diff --git a/src/telemetry/scope.ts b/src/telemetry/scope.ts index 15698d1841..fbdd728bdb 100644 --- a/src/telemetry/scope.ts +++ b/src/telemetry/scope.ts @@ -18,6 +18,15 @@ import type { Pipeline } from './pipeline'; export type Outcome = 'ok' | 'error' | 'cancelled'; +/** An attribute nobody set is not an attribute: `undefined` would ship as an empty value. */ +function defined(attributes: Attributes): Attributes { + const out: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (value !== undefined) out[key] = value; + } + return out; +} + export type TrackDirection = 'inbound' | 'outbound'; export interface RoomIdentity { @@ -185,7 +194,7 @@ export class TelemetrySpan { // Cancellation is `Unset` like success: only a failure is an error (SPEC). status: outcome === 'error' ? { code: SpanStatus.error, message } : { code: SpanStatus.unset }, - attributes: { ...this.scope.attributes(), ...this.attributes }, + attributes: defined({ ...this.scope.attributes(), ...this.attributes }), links: [], events: this.events, ended: true, @@ -262,7 +271,7 @@ export class TelemetryScope { severityText: severity.toUpperCase(), // Log viewers key their line on the body, and not every backend surfaces event_name yet. body: eventName, - attributes: { ...this.attributes(), ...attributes }, + attributes: defined({ ...this.attributes(), ...attributes }), droppedAttributesCount: 0, resource: { attributes: {} }, instrumentationScope: INSTRUMENTATION_SCOPE, diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts index 763b3eb0ca..90370cdb9a 100644 --- a/src/telemetry/telemetry.test.ts +++ b/src/telemetry/telemetry.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { cadenceFactor, changes, networkType } from './device'; import { Pipeline } from './pipeline'; import { TelemetryScope } from './scope'; @@ -149,3 +150,54 @@ describe('telemetry pipeline', () => { expect(idle.diagnostics()).toContain('no destination'); }); }); + +describe('device state', () => { + test('only a change is a record, and the connection is one record', () => { + expect(changes({}, { appState: 'foreground' }).map((c) => c.event)).toEqual([ + 'lk.device.app_state.changed', + ]); + expect(changes({ appState: 'foreground' }, { appState: 'foreground' })).toEqual([]); + + // type, expensive and constrained are one event, not three. + const network = changes( + { networkType: 'wifi', networkExpensive: false, networkConstrained: false }, + { networkType: 'cell', networkExpensive: true, networkConstrained: false }, + ); + expect(network).toHaveLength(1); + expect(network[0].event).toBe('lk.device.network.changed'); + expect(network[0].attributes['network.connection.type']).toBe('cell'); + expect(network[0].attributes['lk.device.network.expensive']).toBe(true); + }); + + test('factors multiply and stop at 4x', () => { + expect(cadenceFactor({})).toBe(1); + expect(cadenceFactor({ thermal: 'fair' })).toBe(1); + expect(cadenceFactor({ thermal: 'serious' })).toBe(2); + expect(cadenceFactor({ thermal: 'serious', lowPower: true })).toBe(4); + // thermal critical (4) x background (2) x low power (2) is capped, not 16. + expect(cadenceFactor({ thermal: 'critical', appState: 'background', lowPower: true })).toBe(4); + }); + + test('NetworkInformation names become SPEC names', () => { + expect(networkType('cellular')).toBe('cell'); + expect(networkType('ethernet')).toBe('wired'); + expect(networkType('none')).toBe('unavailable'); + expect(networkType(undefined)).toBe('unknown'); + }); + + test('the factor stretches both periods, and relief applies at once', () => { + const pipeline = new Pipeline(); + pipeline.configure({ + endpoint: 'http://collector.test/v1/logs', + flushInterval: 15, + statsWindow: 15, + }); + expect(pipeline.statsWindow).toBe(15); + pipeline.setCadenceFactor(4); + expect(pipeline.flushInterval).toBe(60); + expect(pipeline.statsWindow).toBe(60); + pipeline.setCadenceFactor(1); + expect(pipeline.statsWindow).toBe(15); + pipeline.stop(); + }); +}); From 8d0430b19e3dcfe8a1a892589a10bcc9ebf1fa06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:48:40 +0200 Subject: [PATCH 08/14] feat(telemetry): collect before the destination is known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit registerGlobals configures the resource and starts reporting device state long before a connect names the collector, and SPEC says the pipeline may start without a destination — so the queue now holds those records instead of dropping them. An SDK nobody asked for telemetry still collects nothing at all. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 12 +++++++++--- src/telemetry/pipeline.ts | 16 +++++++++++++--- src/telemetry/telemetry.test.ts | 19 ++++++++++++++++++- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/TELEMETRY.md b/TELEMETRY.md index 7d1543e685..a7580f6f5a 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -148,9 +148,15 @@ this package is in RN with no second implementation. The rules that keep it that - The seam: `pagehide` / `visibilitychange` in a page, `AppState` in an app; `navigator.connection` in Chromium, nothing (or `@react-native-community/netinfo`, an app-owned dependency) in RN. -Done, and verified on the simulator: `@livekit/react-native`'s `src/telemetry.ts` sets -`service.name`, `service.version`, `os.name` and `os.version`, registers the `AppState` flush, and -is called from `registerGlobals`. The telemetry module itself needed no React Native branch. +Done: `@livekit/react-native`'s `src/telemetry.ts` sets `service.name`, `service.version`, +`os.name` and `os.version`, reports `app_state` and flushes on `AppState`, and subscribes to a +`LK_DEVICE_STATE` event from the native module, which reports thermal state, low power mode and +memory pressure — `LKDeviceState.swift` (`ProcessInfo.thermalStateDidChangeNotification`, +`NSProcessInfoPowerStateDidChange`, `DispatchSource.makeMemoryPressureSource`) and +`DeviceStateMonitor.kt` (`PowerManager.addThermalStatusListener`, +`ACTION_POWER_SAVE_MODE_CHANGED`, `onTrimMemory`), each mapping the platform's levels onto SPEC's +names so a record from iOS and one from Android say the same thing. `registerGlobals` calls it. +The telemetry module in this package needed no React Native branch at all. One thing the PoC found, which is about this package rather than telemetry: `livekit-client` evaluates `class … extends DOMException` and `new TextDecoder()` at **module scope**, and Hermes has diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index 8bfb07fb3d..1b7e2dc128 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -121,6 +121,9 @@ export class Pipeline { private disabled = false; + /** Set by whoever says telemetry is wanted, which is not the same as knowing where to send it. */ + private collecting = false; + resource: Resource = { attributes: {} }; private baseFlushInterval = FLUSH_INTERVAL; @@ -151,13 +154,19 @@ export class Pipeline { } } - /** Collection runs as soon as anything configured a destination — never before. */ + /** + * Collection starts as soon as anything asked for telemetry — `configure` in an app that sets its + * own resource or collector, `setServer` at the first Cloud connect. Records made before the + * destination is known wait in the queue rather than being thrown away (SPEC: the pipeline may + * start without a destination). An SDK nobody asked stays inert and costs nothing. + */ get enabled(): boolean { - return !this.disabled && this.destination !== undefined; + return !this.disabled && this.collecting; } configure(options: TelemetryOptions) { this.disabled = false; + this.collecting = true; this.encoding = options.encoding ?? this.encoding; this.baseFlushInterval = options.flushInterval ?? this.baseFlushInterval; this.baseStatsWindow = options.statsWindow ?? this.baseStatsWindow; @@ -187,6 +196,7 @@ export class Pipeline { /** The first connect names the destination: the server's host, the connect token (SPEC). */ setServer(serverUrl: string, token: string) { + this.collecting = true; if (this.destination) return; // an explicit endpoint wins const { logs, traces } = cloudEndpoints(serverUrl); this.destination = { logs, traces, headers: { Authorization: `Bearer ${token}` } }; @@ -265,7 +275,7 @@ export class Pipeline { } async flush(force = false): Promise { - if (!this.enabled || this.inFlight) return; + if (!this.enabled || !this.destination || this.inFlight) return; if (!force && (this.held() || Date.now() < this.throttledUntil)) return; if (this.reportDue) this.appendReport(); if (this.logs.length === 0 && this.spans.length === 0) return; diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts index 90370cdb9a..2ae5e16c77 100644 --- a/src/telemetry/telemetry.test.ts +++ b/src/telemetry/telemetry.test.ts @@ -141,7 +141,7 @@ describe('telemetry pipeline', () => { expect(attributesOf(spans[0])['lk.outcome']).toBe('ok'); }); - test('nothing is collected before a destination exists', async () => { + test('an SDK nobody asked for telemetry collects nothing', async () => { const idle = new Pipeline(); const scope = new TelemetryScope(idle); scope.emit('lk.test.void'); @@ -149,6 +149,23 @@ describe('telemetry pipeline', () => { expect(fetchMock).not.toHaveBeenCalled(); expect(idle.diagnostics()).toContain('no destination'); }); + + test('records made before the destination is known are kept, not dropped', async () => { + // `registerGlobals` configures the resource long before a connect names the collector. + const early = new Pipeline(); + early.configure({ encoding: 'json', flushInterval: 3600 }); + const scope = new TelemetryScope(early); + scope.emit('lk.test.early'); + await early.flush(true); + expect(fetchMock).not.toHaveBeenCalled(); + + early.setServer('wss://project.livekit.cloud', 'token'); + await early.flush(true); + const call = fetchMock.mock.calls[0]; + expect(String(call[0])).toBe('https://project.livekit.cloud/observability/client/logs/otlp/v0'); + expect(recordsOf(call)[0].eventName).toBe('lk.test.early'); + early.stop(); + }); }); describe('device state', () => { From ec164b3721d86e87e5cf4fdc5be429ee815f0666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:10:48 +0200 Subject: [PATCH 09/14] refactor(telemetry): make what carries the records replaceable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit React Native is going to bind the Rust core through UniFFI, so that a phone behaves the same whichever SDK the app used. What it must not do is reimplement the instrumentation — when a span starts, which checkpoints it carries, how a subscribe ends at first media, which getStats fields become a window. So that is now on one side of a seam and the carrying is on the other: backend.ts is SPEC's typed surface, the same boundary Swift, Kotlin and Dart already cross, expressed in terms this package owns. Room does not know which backend is installed. Nothing mobile is left on this side of it. DeviceState carries visibility and the connection, which a page can answer; thermal, low power and memory pressure are gone, because this package cannot observe them and the platform that can reports them to its own backend. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 37 ++++++-- src/room/Room.ts | 8 +- src/room/participant/LocalParticipant.ts | 4 +- src/telemetry/backend.ts | 102 +++++++++++++++++++++++ src/telemetry/device.ts | 46 ++-------- src/telemetry/index.ts | 100 +++++++++++----------- src/telemetry/pipeline.ts | 33 +++++++- src/telemetry/scope.ts | 58 ++++--------- src/telemetry/telemetry.test.ts | 100 +++++++++++++++++++--- src/telemetry/webrtc.ts | 2 +- 10 files changed, 328 insertions(+), 162 deletions(-) create mode 100644 src/telemetry/backend.ts diff --git a/TELEMETRY.md b/TELEMETRY.md index a7580f6f5a..3f45177093 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -148,15 +148,33 @@ this package is in RN with no second implementation. The rules that keep it that - The seam: `pagehide` / `visibilitychange` in a page, `AppState` in an app; `navigator.connection` in Chromium, nothing (or `@react-native-community/netinfo`, an app-owned dependency) in RN. -Done: `@livekit/react-native`'s `src/telemetry.ts` sets `service.name`, `service.version`, -`os.name` and `os.version`, reports `app_state` and flushes on `AppState`, and subscribes to a -`LK_DEVICE_STATE` event from the native module, which reports thermal state, low power mode and -memory pressure — `LKDeviceState.swift` (`ProcessInfo.thermalStateDidChangeNotification`, -`NSProcessInfoPowerStateDidChange`, `DispatchSource.makeMemoryPressureSource`) and -`DeviceStateMonitor.kt` (`PowerManager.addThermalStatusListener`, -`ACTION_POWER_SAVE_MODE_CHANGED`, `onTrimMemory`), each mapping the platform's levels onto SPEC's -names so a record from iOS and one from Android say the same thing. `registerGlobals` calls it. -The telemetry module in this package needed no React Native branch at all. +**React Native follows the phones, and reuses this package's instrumentation.** The decision is +that `@livekit/react-native` binds the Rust core through UniFFI, the way the Swift, Kotlin and Dart +SDKs do, so that an app on a phone behaves identically whichever SDK it used — same windowing code, +same upload policy, same write-ahead file cache, same bytes on the wire. What it does *not* do is +reimplement the instrumentation: when a connect span starts, which checkpoints it carries, how a +subscribe ends at first media, which `getStats` fields become a window — all of that stays here and +is reused. + +That is what `backend.ts` is for. It is the same set of operations SPEC calls the typed surface — +the boundary Swift, Kotlin and Dart already cross into the core — expressed in terms this package +owns: rooms, spans, tracks, outcomes. `Telemetry.setBackend` installs one, and `Room` does not know +which is in place. + +``` +Room, LocalParticipant, the four track monitors ← the instrumentation, one copy + │ + Backend / Scope / Span ← backend.ts, platform-neutral by construction + ╱ ╲ + Pipeline (this package) RustBackend (@livekit/react-native) + fetch, in-memory queue UniFFI → livekit-telemetry → FileCache, NetTransport +``` + +**Nothing mobile appears on this side of the seam.** `DeviceState` carries only what a page can +answer — visibility and, on Chromium, the connection. Thermal state, low power mode and memory +pressure are not absent because they are unimportant; they are absent because this package cannot +observe them and must not pretend to. React Native's native monitors (`LKDeviceState.swift`, +`DeviceStateMonitor.kt`) report those to the core natively, never through JavaScript. One thing the PoC found, which is about this package rather than telemetry: `livekit-client` evaluates `class … extends DOMException` and `new TextDecoder()` at **module scope**, and Hermes has @@ -170,6 +188,7 @@ src/telemetry/ index.ts the Telemetry facade: configure/setServer, the scope factory, the track registry pipeline.ts queue, flush timer, holds, 429/5xx, one request in flight, the self-report scope.ts one per Room connection: trace id, attributes, spans, stats windows + backend.ts the seam: what a platform must implement to carry the records (see §5) otlp.ts the records and their wire form (the only OpenTelemetry import lives here) webrtc.ts the SDK's typed sender/receiver stats → one SPEC reading ``` diff --git a/src/room/Room.ts b/src/room/Room.ts index adffcd9190..3eecadcf1e 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -47,7 +47,7 @@ import type { RoomConnectOptions, RoomOptions, } from '../options'; -import { SpanKind, Telemetry, type TelemetryScope, type TelemetrySpan } from '../telemetry'; +import { type Scope, type Span, SpanKind, Telemetry } from '../telemetry'; import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays'; import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; @@ -203,11 +203,11 @@ class Room extends (EventEmitter as new () => TypedEmitter) private connectFuture?: Future; /** One telemetry scope per connect: a trace id and the attributes every record of it carries. */ - private telemetry?: TelemetryScope; + private telemetry?: Scope; - private connectSpan?: TelemetrySpan; + private connectSpan?: Span; - private reconnectSpan?: TelemetrySpan; + private reconnectSpan?: Span; /** Attempts inside the *current* reconnect; the engine's own counter spans several. */ private reconnectAttempts = 0; diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index ed8513db48..eebba876ac 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -30,7 +30,7 @@ import { isFrameMetadataSupported, } from '../../frameMetadata/utils'; import type { InternalRoomOptions } from '../../options'; -import type { TelemetryScope } from '../../telemetry'; +import type { Scope } from '../../telemetry'; import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays'; import TypedPromise from '../../utils/TypedPromise'; import { PCTransportState } from '../PCTransportManager'; @@ -140,7 +140,7 @@ export default class LocalParticipant extends Participant { activeDeviceMap: Map; /** @internal — the Room's telemetry scope, set at connect; publishing is a span on it. */ - telemetry?: TelemetryScope; + telemetry?: Scope; private pendingPublishing = new Set(); diff --git a/src/telemetry/backend.ts b/src/telemetry/backend.ts new file mode 100644 index 0000000000..4be6f58fd6 --- /dev/null +++ b/src/telemetry/backend.ts @@ -0,0 +1,102 @@ +/** + * The seam between what this SDK *observes* and what carries it. + * + * These are the same operations `livekit-telemetry/SPEC.md` defines as the typed surface the Swift, + * Kotlin and Dart SDKs cross into the Rust core: verbs about rooms, spans, tracks and outcomes, and + * nothing about a platform. That is deliberate — the Room instrumentation in this package is the + * part worth having once, and a platform that carries telemetry differently (React Native binds the + * Rust core) implements these interfaces instead of reimplementing the instrumentation. + * + * The browser's implementation is `Pipeline` + `PipelineScope`; `Telemetry.setBackend` installs + * another. Nothing here may name a capability only some platforms have. + */ +import type { DeviceState } from './device'; +import type { Attributes } from './otlp'; + +export type Outcome = 'ok' | 'error' | 'cancelled'; + +export type TrackDirection = 'inbound' | 'outbound'; + +export type Severity = 'info' | 'warn' | 'error'; + +export interface RoomIdentity { + sid?: string; + name?: string; + participantSid?: string; + participantIdentity?: string; +} + +/** One track's reading, already in SPEC units: milliseconds, not the WebRTC seconds. */ +export interface StatsSample { + codec?: string; + bytes?: number; + packets?: number; + packetsLost?: number; + framesDropped?: number; + concealedSamples?: number; + concealmentEvents?: number; + silentConcealedSamples?: number; + jitterBufferDelayMs?: number; + qualityLimitationBandwidthMs?: number; + qualityLimitationCpuMs?: number; + qualityLimitationOtherMs?: number; + jitterMs?: number; + rttMs?: number; + fps?: number; + audioLevel?: number; +} + +export interface TraceContext { + traceId: string; + spanId: string; + traceFlags: number; +} + +/** + * One attempt at an operation. Calls are synchronous and the implementation stamps the clock, so + * the only skew is the call itself — a backend that crosses a native boundary must use a blocking + * call, not a promise. + */ +export interface Span { + /** A checkpoint inside the attempt: `ws_open`, `join_recv`, `first_media`, `attempt 2 full`. */ + step(name: string): void; + setAttribute(key: string, value: Attributes[string]): void; + /** Ending twice is a no-op. */ + end(outcome?: Outcome, errorType?: string, message?: string): void; + fail(error: unknown): void; + cancel(): void; + context(): TraceContext; +} + +/** One Room connection: a trace id, the attributes its records carry, its spans and its windows. */ +export interface Scope { + readonly traceId: string; + setRoom(identity: RoomIdentity): void; + start(name: string, options?: { kind?: number; attributes?: Attributes; parent?: Span }): Span; + emit(event: string, attributes?: Attributes, severity?: Severity, span?: Span): void; + recordStats( + sid: string, + kind: 'audio' | 'video', + direction: TrackDirection, + sample: StatsSample, + ): void; + subscribeStarted(sid: string, attributes: Attributes): void; + subscribeEnded(sid: string, outcome: Outcome, errorType?: string): void; + disconnected(reason: string): void; + /** The call is ending: close the open windows early and cancel what never resolved. */ + close(): void; +} + +export interface Backend { + readonly enabled: boolean; + /** LiveKit Cloud: the server URL and the connect token are the destination. */ + setServer(serverUrl: string, token: string): void; + scope(): Scope; + /** Uploads stop, collection does not — spans that own the uplink raise a hold. */ + hold(up: boolean): void; + /** Only what the platform running this code can actually answer; see `DeviceState`. */ + deviceState(state: DeviceState): void; + flush(): Promise; + diagnostics(): string; + shutdown(): Promise; +} diff --git a/src/telemetry/device.ts b/src/telemetry/device.ts index a05b8a2337..9554ef05ab 100644 --- a/src/telemetry/device.ts +++ b/src/telemetry/device.ts @@ -1,25 +1,19 @@ /** - * Device state: the events SPEC calls `lk.device.*`, and the cadence factor they drive. The - * pipeline never measures anything itself — measuring CPU costs CPU — so every value here comes - * from the platform saying so: the browser's own APIs, or React Native's native module. + * Device state, and the cadence factor it drives — but only what a page can answer about itself. * - * A browser can only answer some of this. `visibilitychange` is universal, `navigator.connection` - * is Chromium, and thermal, low power and memory pressure have no web API at all — a page simply - * never reports them, and its cadence factor stays at whatever the rest implies. + * The pipeline never measures anything: every value here is the platform saying so. A browser can + * say whether it is visible and (on Chromium) what it is connected through. It cannot say anything + * about heat, power or memory pressure, and this package does not pretend otherwise: a platform + * that knows those reports them to its own backend, not through here. */ import type { Attributes } from './otlp'; export type AppStateName = 'foreground' | 'background'; -export type ThermalState = 'nominal' | 'fair' | 'serious' | 'critical'; -export type MemoryPressure = 'normal' | 'warning' | 'critical'; export type NetworkType = 'wifi' | 'cell' | 'wired' | 'vpn' | 'bluetooth' | 'other' | 'unavailable' | 'unknown'; export interface DeviceState { appState?: AppStateName; - thermal?: ThermalState; - memory?: MemoryPressure; - lowPower?: boolean; networkType?: NetworkType; /** Cellular or hotspot. */ networkExpensive?: boolean; @@ -27,21 +21,11 @@ export interface DeviceState { networkConstrained?: boolean; } -/** SPEC's cadence table. Factors multiply and the product is capped at 4×. */ -const THERMAL_FACTOR: Record = { - nominal: 1, - fair: 1, - serious: 2, - critical: 4, -}; -const MEMORY_FACTOR: Record = { normal: 1, warning: 2, critical: 4 }; +/** SPEC's cadence table, for the rows a page can fill in. Factors multiply, capped at 4×. */ const MAX_FACTOR = 4; export function cadenceFactor(state: DeviceState): number { let factor = 1; - if (state.thermal) factor *= THERMAL_FACTOR[state.thermal]; - if (state.memory) factor *= MEMORY_FACTOR[state.memory]; - if (state.lowPower) factor *= 2; if (state.appState === 'background') factor *= 2; if (state.networkConstrained) factor *= 2; return Math.min(factor, MAX_FACTOR); @@ -82,24 +66,6 @@ export function changes(previous: DeviceState, next: DeviceState): Change[] { attributes: { 'lk.device.app_state': next.appState }, }); } - if (next.thermal !== undefined && next.thermal !== previous.thermal) { - out.push({ - event: 'lk.device.thermal.changed', - attributes: { 'lk.device.thermal.state': next.thermal }, - }); - } - if (next.memory !== undefined && next.memory !== previous.memory) { - out.push({ - event: 'lk.device.memory.changed', - attributes: { 'lk.device.memory.pressure': next.memory }, - }); - } - if (next.lowPower !== undefined && next.lowPower !== previous.lowPower) { - out.push({ - event: 'lk.device.low_power.changed', - attributes: { 'lk.device.low_power.enabled': next.lowPower }, - }); - } const networkChanged = (next.networkType !== undefined && next.networkType !== previous.networkType) || (next.networkExpensive !== undefined && next.networkExpensive !== previous.networkExpensive) || diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index c5920ee967..fbe02b3f38 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -5,41 +5,45 @@ * * Nothing is collected until a destination exists: `Telemetry.configure({ endpoint })` for your own * collector, or the first connect to LiveKit Cloud, which derives the route and the token itself. + * + * What carries the records is replaceable (`setBackend`) — the Room instrumentation in this package + * is worth having once, while a platform may know a better way to batch, cache and upload. The + * default is the browser pipeline in `pipeline.ts`. */ -import { type DeviceState, cadenceFactor, changes, observeBrowser } from './device'; +import type { Backend, Scope, StatsSample, TrackDirection } from './backend'; +import { type DeviceState, observeBrowser } from './device'; import { Severity, hrTime, randomHex } from './otlp'; import { Pipeline, type TelemetryOptions } from './pipeline'; -import { type StatsSample, TelemetryScope, type TrackDirection } from './scope'; import { receiverSample, senderSample } from './webrtc'; export type { TelemetryOptions } from './pipeline'; +export type { DeviceState, AppStateName, NetworkType } from './device'; export type { - DeviceState, - AppStateName, - ThermalState, - MemoryPressure, - NetworkType, -} from './device'; -export type { StatsSample, TrackDirection, RoomIdentity, Outcome } from './scope'; -export { TelemetryScope, TelemetrySpan } from './scope'; + Backend, + Scope, + Span, + Outcome, + RoomIdentity, + Severity as SeverityName, + StatsSample, + TrackDirection, + TraceContext, +} from './backend'; export { SpanKind } from './otlp'; const pipeline = new Pipeline(); +let backend: Backend = pipeline; + /** Which scope a track's stats belong to — the monitors know a sid, not a Room. */ interface TrackRegistration { - scope: TelemetryScope; + scope: Scope; kind: 'audio' | 'video'; direction: TrackDirection; } const tracks = new Map(); -/** Device state belongs to no call: it is filed under the pipeline's own scope (SPEC). */ -let processScope: TelemetryScope | undefined; - -let deviceState: DeviceState = {}; - let lifecycleAttached = false; function attachLifecycle() { @@ -48,7 +52,7 @@ function attachLifecycle() { // The page's last chance: `pagehide` and a hidden tab, never `unload` — by then a request has // no chance of leaving. `fetch(keepalive)` makes it best effort, not durable (TELEMETRY.md §3). const flush = () => { - pipeline.flush(true).catch(() => {}); + backend.flush().catch(() => {}); }; document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'hidden') flush(); @@ -58,57 +62,50 @@ function attachLifecycle() { } export const Telemetry = { - /** Point the pipeline at a collector and start it. Safe to call more than once. */ + /** Point the default pipeline at a collector and start it. Safe to call more than once. */ configure(options: TelemetryOptions) { pipeline.configure(options); attachLifecycle(); }, + /** + * Replace what carries the records — a platform SDK that binds a different implementation of + * `Backend` installs it here, before any Room is created. The instrumentation does not change. + * Returns the backend that was in place, so it can be put back. + */ + setBackend(replacement: Backend): Backend { + const previous = backend; + backend = replacement; + return previous; + }, + /** LiveKit Cloud: the server URL and the connect token are the destination (SPEC). */ setServer(serverUrl: string, token: string) { - pipeline.setServer(serverUrl, token); + backend.setServer(serverUrl, token); attachLifecycle(); }, get enabled(): boolean { - return pipeline.enabled; + return backend.enabled; }, - scope(): TelemetryScope { - return new TelemetryScope(pipeline); + scope(): Scope { + return backend.scope(); }, /** Uploads stop, collection does not — spans that own the uplink raise a hold (SPEC). */ hold(up: boolean) { - pipeline.hold(up); + backend.hold(up); }, - /** - * What the platform now says about the device. A page reports what it can see; React Native - * reports the rest from its native module. Each group becomes an `lk.device.*` record the first - * time it is seen and on every change after, and the whole state sets the cadence factor. - */ + /** What the platform can say about the device it runs on; see `DeviceState` for the limits. */ deviceState(state: DeviceState) { - const next = { ...deviceState, ...state }; - const records = changes(deviceState, next); - deviceState = next; - if (pipeline.enabled) { - processScope ??= new TelemetryScope(pipeline); - for (const record of records) { - processScope.emit(record.event, record.attributes); - } - } - pipeline.setCadenceFactor(cadenceFactor(next)); - }, - - registerTrack( - sid: string, - scope: TelemetryScope, - kind: 'audio' | 'video', - direction: TrackDirection, - ) { + backend.deviceState(state); + }, + + registerTrack(sid: string, scope: Scope, kind: 'audio' | 'video', direction: TrackDirection) { // Nothing to route when nobody is listening: an SDK without a collector keeps no map. - if (!pipeline.enabled) return; + if (!backend.enabled) return; tracks.set(`${sid}:${direction}`, { scope, kind, direction }); }, @@ -128,18 +125,18 @@ export const Telemetry = { /** Called from the SDK's existing per-track monitors: no extra `getStats()` anywhere. */ trackStats(sid: string, direction: TrackDirection, sample: StatsSample) { - if (!pipeline.enabled) return; + if (!backend.enabled) return; const registration = tracks.get(`${sid}:${direction}`); if (!registration) return; registration.scope.recordStats(sid, registration.kind, direction, sample); }, flush(): Promise { - return pipeline.flush(true); + return backend.flush(); }, diagnostics(): string { - return pipeline.diagnostics(); + return backend.diagnostics(); }, /** A pipeline smoke test: one record, one request, whatever the collector answers. */ @@ -163,7 +160,6 @@ export const Telemetry = { async shutdown() { tracks.clear(); - processScope = undefined; - await pipeline.shutdown(); + await backend.shutdown(); }, }; diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index 1b7e2dc128..dc35e95bb7 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -4,6 +4,8 @@ * (TELEMETRY.md §3), so the queue is the only bound and every eviction is counted. */ import { version } from '../version'; +import type { Backend, Scope } from './backend'; +import { type DeviceState, cadenceFactor, changes } from './device'; import { type AttributeValue, type Attributes, @@ -18,6 +20,7 @@ import { serializeLogs, serializeSpans, } from './otlp'; +import { PipelineScope } from './scope'; export interface TelemetryOptions { /** OTLP logs route. Cloud derives it from the server URL instead; see `setServer`. */ @@ -92,7 +95,7 @@ export function tracesEndpointFor(logs: string): string { return logs; } -export class Pipeline { +export class Pipeline implements Backend { private logs: LogRecord[] = []; private spans: SpanRecord[] = []; @@ -124,6 +127,11 @@ export class Pipeline { /** Set by whoever says telemetry is wanted, which is not the same as knowing where to send it. */ private collecting = false; + /** Device state belongs to no call: it is filed under the pipeline's own scope (SPEC). */ + private processScope?: PipelineScope; + + private device: DeviceState = {}; + resource: Resource = { attributes: {} }; private baseFlushInterval = FLUSH_INTERVAL; @@ -143,6 +151,28 @@ export class Pipeline { return this.baseStatsWindow * this.cadence; } + scope(): Scope { + return new PipelineScope(this); + } + + /** + * What the platform now says about the device — only the rows a page can fill in; see + * `DeviceState`. Each group becomes an `lk.device.*` record the first time it is seen and on + * every change after, and the whole state sets the cadence factor. + */ + deviceState(state: DeviceState) { + const next = { ...this.device, ...state }; + const records = changes(this.device, next); + this.device = next; + if (this.enabled) { + this.processScope ??= new PipelineScope(this); + for (const record of records) { + this.processScope.emit(record.event, record.attributes); + } + } + this.setCadenceFactor(cadenceFactor(next)); + } + setCadenceFactor(factor: number) { if (factor === this.cadence) return; this.cadence = factor; @@ -427,6 +457,7 @@ export class Pipeline { async shutdown() { this.stop(); + this.processScope = undefined; await this.flush(true); } } diff --git a/src/telemetry/scope.ts b/src/telemetry/scope.ts index fbdd728bdb..671e12454c 100644 --- a/src/telemetry/scope.ts +++ b/src/telemetry/scope.ts @@ -2,6 +2,15 @@ * One scope per Room connection: a trace id, the attributes every record of that call carries, the * spans, and the stats windows. The scope is not ended — a call's last record is simply its last. */ +import type { + Outcome, + RoomIdentity, + Scope, + Severity as SeverityName, + Span, + StatsSample, + TrackDirection, +} from './backend'; import { type Attributes, INSTRUMENTATION_SCOPE, @@ -16,8 +25,6 @@ import { } from './otlp'; import type { Pipeline } from './pipeline'; -export type Outcome = 'ok' | 'error' | 'cancelled'; - /** An attribute nobody set is not an attribute: `undefined` would ship as an empty value. */ function defined(attributes: Attributes): Attributes { const out: Attributes = {}; @@ -27,35 +34,6 @@ function defined(attributes: Attributes): Attributes { return out; } -export type TrackDirection = 'inbound' | 'outbound'; - -export interface RoomIdentity { - sid?: string; - name?: string; - participantSid?: string; - participantIdentity?: string; -} - -/** One track's reading, already in SPEC units: milliseconds, not the WebRTC seconds. */ -export interface StatsSample { - codec?: string; - bytes?: number; - packets?: number; - packetsLost?: number; - framesDropped?: number; - concealedSamples?: number; - concealmentEvents?: number; - silentConcealedSamples?: number; - jitterBufferDelayMs?: number; - qualityLimitationBandwidthMs?: number; - qualityLimitationCpuMs?: number; - qualityLimitationOtherMs?: number; - jitterMs?: number; - rttMs?: number; - fps?: number; - audioLevel?: number; -} - const COUNTERS = [ ['bytes', 'lk.rtc.bytes'], ['packets', 'lk.rtc.packets'], @@ -140,7 +118,7 @@ class Window { } } -export class TelemetrySpan { +export class PipelineSpan implements Span { private events: SpanRecord['events'] = []; private attributes: Attributes; @@ -152,7 +130,7 @@ export class TelemetrySpan { private spanId = randomHex(8); constructor( - private scope: TelemetryScope, + private scope: PipelineScope, private pipeline: Pipeline, private name: string, private kind: number, @@ -217,14 +195,14 @@ export class TelemetrySpan { } } -export class TelemetryScope { +export class PipelineScope implements Scope { readonly traceId = randomHex(16); private room: RoomIdentity = {}; private windows = new Map(); - private pendingSubscribes = new Map(); + private pendingSubscribes = new Map(); constructor(private pipeline: Pipeline) {} @@ -244,9 +222,9 @@ export class TelemetryScope { start( name: string, - options: { kind?: number; attributes?: Attributes; parent?: TelemetrySpan } = {}, - ): TelemetrySpan { - return new TelemetrySpan( + options: { kind?: number; attributes?: Attributes; parent?: Span } = {}, + ): PipelineSpan { + return new PipelineSpan( this, this.pipeline, name, @@ -259,8 +237,8 @@ export class TelemetryScope { emit( eventName: string, attributes: Attributes = {}, - severity: keyof typeof Severity = 'info', - span?: TelemetrySpan, + severity: SeverityName = 'info', + span?: Span, ) { const now = hrTime(); const record: LogRecord = { diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts index 2ae5e16c77..e666a3075c 100644 --- a/src/telemetry/telemetry.test.ts +++ b/src/telemetry/telemetry.test.ts @@ -1,7 +1,9 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { Telemetry } from '.'; +import type { Backend, Scope, Span } from './backend'; import { cadenceFactor, changes, networkType } from './device'; import { Pipeline } from './pipeline'; -import { TelemetryScope } from './scope'; +import { PipelineScope } from './scope'; /** The JSON encoding is the readable one: every assertion here reads the body the collector gets. */ function bodyOf(call: unknown[]): any { @@ -41,7 +43,7 @@ describe('telemetry pipeline', () => { }); test('one window of readings becomes one record, counters last and gauges summarised', async () => { - const scope = new TelemetryScope(pipeline); + const scope = new PipelineScope(pipeline); scope.setRoom({ sid: 'RM_1', name: 'harness', participantIdentity: 'publisher' }); scope.recordStats('TR_1', 'video', 'outbound', { bytes: 1000, @@ -74,7 +76,7 @@ describe('telemetry pipeline', () => { }); test('a hold stops uploads, never collection', async () => { - const scope = new TelemetryScope(pipeline); + const scope = new PipelineScope(pipeline); pipeline.hold(true); scope.emit('lk.test.one'); await pipeline.flush(); @@ -92,7 +94,7 @@ describe('telemetry pipeline', () => { test('a 429 holds the pipeline and keeps the batch', async () => { fetchMock.mockResolvedValueOnce(new Response(null, { status: 429 })); - const scope = new TelemetryScope(pipeline); + const scope = new PipelineScope(pipeline); scope.emit('lk.test.throttled'); await pipeline.flush(true); expect(pipeline.diagnostics()).toContain('throttled'); @@ -109,7 +111,7 @@ describe('telemetry pipeline', () => { test('the queue evicts the oldest and says how many', async () => { pipeline.maxQueueSize = 3; - const scope = new TelemetryScope(pipeline); + const scope = new PipelineScope(pipeline); for (let i = 0; i < 5; i += 1) { scope.emit(`lk.test.${i}`); } @@ -123,7 +125,7 @@ describe('telemetry pipeline', () => { }); test('a span carries its checkpoints and its outcome', async () => { - const scope = new TelemetryScope(pipeline); + const scope = new PipelineScope(pipeline); const span = scope.start('lk.connect', { attributes: { 'lk.connect.attempt': 1 } }); span.step('ws_open'); span.step('join_recv'); @@ -143,7 +145,7 @@ describe('telemetry pipeline', () => { test('an SDK nobody asked for telemetry collects nothing', async () => { const idle = new Pipeline(); - const scope = new TelemetryScope(idle); + const scope = new PipelineScope(idle); scope.emit('lk.test.void'); await idle.flush(true); expect(fetchMock).not.toHaveBeenCalled(); @@ -154,7 +156,7 @@ describe('telemetry pipeline', () => { // `registerGlobals` configures the resource long before a connect names the collector. const early = new Pipeline(); early.configure({ encoding: 'json', flushInterval: 3600 }); - const scope = new TelemetryScope(early); + const scope = new PipelineScope(early); scope.emit('lk.test.early'); await early.flush(true); expect(fetchMock).not.toHaveBeenCalled(); @@ -188,11 +190,10 @@ describe('device state', () => { test('factors multiply and stop at 4x', () => { expect(cadenceFactor({})).toBe(1); - expect(cadenceFactor({ thermal: 'fair' })).toBe(1); - expect(cadenceFactor({ thermal: 'serious' })).toBe(2); - expect(cadenceFactor({ thermal: 'serious', lowPower: true })).toBe(4); - // thermal critical (4) x background (2) x low power (2) is capped, not 16. - expect(cadenceFactor({ thermal: 'critical', appState: 'background', lowPower: true })).toBe(4); + expect(cadenceFactor({ appState: 'background' })).toBe(2); + // The rows a page cannot fill in are not here at all: a platform that knows heat, power or + // memory pressure reports them to its own backend (TELEMETRY.md §5). + expect(cadenceFactor({ appState: 'background', networkConstrained: true })).toBe(4); }); test('NetworkInformation names become SPEC names', () => { @@ -218,3 +219,76 @@ describe('device state', () => { pipeline.stop(); }); }); + +describe('the backend seam', () => { + test('a platform backend gets the instrumentation, not the records', () => { + // React Native binds the Rust core through exactly this shape; nothing about a platform + // appears in it, and the Room instrumentation is reused unchanged (TELEMETRY.md §5). + const calls: string[] = []; + const span: Span = { + step: (name) => calls.push(`step ${name}`), + setAttribute: (key, value) => calls.push(`attribute ${key}=${String(value)}`), + end: (outcome) => calls.push(`end ${outcome}`), + fail: () => calls.push('fail'), + cancel: () => calls.push('cancel'), + context: () => ({ traceId: 'trace', spanId: 'span', traceFlags: 1 }), + }; + const scope: Scope = { + traceId: 'trace', + setRoom: (identity) => calls.push(`room ${identity.name}`), + start: (name) => { + calls.push(`start ${name}`); + return span; + }, + emit: (event) => calls.push(`emit ${event}`), + recordStats: (sid, _kind, direction) => calls.push(`stats ${sid} ${direction}`), + subscribeStarted: (sid) => calls.push(`subscribe ${sid}`), + subscribeEnded: (sid, outcome) => calls.push(`subscribed ${sid} ${outcome}`), + disconnected: (reason) => calls.push(`disconnected ${reason}`), + close: () => calls.push('close'), + }; + const platform: Backend = { + enabled: true, + setServer: () => calls.push('setServer'), + scope: () => scope, + hold: (up) => calls.push(`hold ${up}`), + deviceState: (state) => calls.push(`device ${state.appState}`), + flush: async () => {}, + diagnostics: () => 'platform backend', + shutdown: async () => {}, + }; + + const previous = Telemetry.setBackend(platform); + try { + Telemetry.setServer('wss://project.livekit.cloud', 'token'); + const session = Telemetry.scope(); + session.setRoom({ name: 'harness' }); + Telemetry.registerTrack('TR_1', session, 'video', 'outbound'); + Telemetry.hold(true); + const connect = session.start('lk.connect'); + connect.step('ws_open'); + connect.end('ok'); + Telemetry.hold(false); + Telemetry.trackStats('TR_1', 'outbound', { bytes: 1 }); + session.disconnected('client_initiated'); + + expect(calls).toEqual([ + 'setServer', + // The page observes itself and reports to whatever backend is installed. On React Native + // there is no `document`, so nothing is observed here and the native side reports instead. + 'device foreground', + 'room harness', + 'hold true', + 'start lk.connect', + 'step ws_open', + 'end ok', + 'hold false', + 'stats TR_1 outbound', + 'disconnected client_initiated', + ]); + expect(Telemetry.diagnostics()).toBe('platform backend'); + } finally { + Telemetry.setBackend(previous); + } + }); +}); diff --git a/src/telemetry/webrtc.ts b/src/telemetry/webrtc.ts index 62f2eee849..9e2436d796 100644 --- a/src/telemetry/webrtc.ts +++ b/src/telemetry/webrtc.ts @@ -9,7 +9,7 @@ import type { VideoReceiverStats, VideoSenderStats, } from '../room/stats'; -import type { StatsSample } from './scope'; +import type { StatsSample } from './backend'; const seconds = (value: number | undefined): number | undefined => value === undefined ? undefined : value * 1000; From b00658a9524afd634cf2b9d14c6f62ba2c790ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:22:26 +0200 Subject: [PATCH 10/14] feat(telemetry): the pipeline is write-ahead, and where it writes is a choice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A batch is stored before the network is tried and removed only once the collector has taken it — so a refused upload, a lost connection or a process that dies costs nothing, which is what the Rust core has always done. Where it is stored is now TelemetryStorage: the same five synchronous operations as the core's BatchCache, defaulting to memory bounded by 4 MiB and 512 batches. A platform with a filesystem supplies its own and this package learns nothing about it. Requeueing is gone with it: a batch that could not be sent keeps its place in the cache instead of being unshifted back into an array where the queue cap could eat it. A backlog replays at four batches a tick beside a live call; a shutdown drains without the budget. Evictions cost a known number of records, because the batch id carries the count. Two more platform-neutral verbs, for the pressure this package cannot see: emit() names a pipeline-scoped record, setCadenceFactor() slows the cadence. A phone reports the number, not the reason. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 29 +++--- src/telemetry/backend.ts | 10 +++ src/telemetry/index.ts | 22 ++++- src/telemetry/pipeline.ts | 153 ++++++++++++++++++++++++-------- src/telemetry/scope.ts | 2 +- src/telemetry/storage.ts | 75 ++++++++++++++++ src/telemetry/telemetry.test.ts | 79 +++++++++++++++++ 7 files changed, 317 insertions(+), 53 deletions(-) create mode 100644 src/telemetry/storage.ts diff --git a/TELEMETRY.md b/TELEMETRY.md index 3f45177093..102d3411c9 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -92,9 +92,15 @@ Saver, and an app stretches on everything. | Upload holds during `lk.connect` / `lk.reconnect`, 60 s cap, one request in flight | telemetry never wins over media | | Flood guard, `lk.telemetry.report` | fleet-wide denominators, same shape as everywhere else | -## 3. Caching: in-memory only +## 3. Caching: in-memory in a tab, whatever the platform has elsewhere -Yes. Nobody in this ecosystem persists, and the reasons for disk on mobile do not exist in a tab. +The pipeline is **write-ahead**, as the Rust core is: a batch is stored before the network is tried +and removed only once the collector has taken it, so a refused upload, a lost connection or a +process that dies costs nothing. Where it is stored is `TelemetryStorage` — five synchronous +operations, the same five the core's `BatchCache` has — and the default keeps batches in memory, +bounded by 4 MiB and 512 batches, oldest evicted first and counted. + +That default is right for a tab, and nobody in this ecosystem persists there either. - **OpenTelemetry JS** caches nothing: `BatchLogRecordProcessor` is a bounded in-memory queue and the spec puts retry on the exporter, explicitly not on the processor. @@ -104,15 +110,16 @@ Yes. Nobody in this ecosystem persists, and the reasons for disk on mobile do no - **Grafana Faro** is in-memory. A tab's lifetime is the call's lifetime; there is no "app killed in the background, replay at next -launch". The bound is therefore the queue alone (2048 records, oldest evicted and counted as -`lk.telemetry.dropped.queue_full`), and the last-gasp flush is best effort: `fetch(keepalive)` under -64 KiB on `pagehide`, an ordinary `fetch` on RN's `AppState` → background. Neither turns a -disappearing page into a durable queue, which is why the flush happens at -`visibilitychange → hidden` and not at `unload`. - -React Native *can* be killed with a backlog, and it does have `AsyncStorage`. Not for v1: it is -async, slow, and a backlog that matters needs the whole cache policy (age, prune, replay budget) -that §2 just deleted. Revisit if the field shows RN sessions losing their tail. +launch". The last-gasp flush is best effort: `fetch(keepalive)` under 64 KiB on `pagehide`, an +ordinary `fetch` on React Native's `AppState` → background. Neither turns a disappearing page into +a durable queue, which is why the flush happens at `visibilitychange → hidden` and not at `unload`. + +React Native is the case where it is not right, because an app really can be killed holding a +backlog and really can be offline for hours. There it supplies a store backed by files — the same +shape the Rust core's `FileCache` has, written natively in the package that already ships native +code rather than pulled in as a dependency. Nothing about that reaches this package: it hands over +`storage` and knows nothing else. A browser could do the same over IndexedDB if the field ever +shows it is worth it. ## 4. Protobuf or JSON? diff --git a/src/telemetry/backend.ts b/src/telemetry/backend.ts index 4be6f58fd6..9c93460daf 100644 --- a/src/telemetry/backend.ts +++ b/src/telemetry/backend.ts @@ -96,6 +96,16 @@ export interface Backend { hold(up: boolean): void; /** Only what the platform running this code can actually answer; see `DeviceState`. */ deviceState(state: DeviceState): void; + /** + * A record belonging to the pipeline rather than to any call. A platform that observes something + * this package has no vocabulary for — a phone's thermal state, say — names the event itself. + */ + emit(event: string, attributes?: Attributes, severity?: Severity): void; + /** + * Stretch the flush interval and the stats window by this much, 1–4. The platform that can see + * pressure this package cannot reports the number, not the reason. + */ + setCadenceFactor(factor: number): void; flush(): Promise; diagnostics(): string; shutdown(): Promise; diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index fbe02b3f38..d6aef1bc49 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -12,7 +12,7 @@ */ import type { Backend, Scope, StatsSample, TrackDirection } from './backend'; import { type DeviceState, observeBrowser } from './device'; -import { Severity, hrTime, randomHex } from './otlp'; +import { type Attributes, Severity, hrTime, randomHex } from './otlp'; import { Pipeline, type TelemetryOptions } from './pipeline'; import { receiverSample, senderSample } from './webrtc'; @@ -30,6 +30,8 @@ export type { TraceContext, } from './backend'; export { SpanKind } from './otlp'; +export type { Attributes } from './otlp'; +export type { TelemetryStorage } from './storage'; const pipeline = new Pipeline(); @@ -103,6 +105,22 @@ export const Telemetry = { backend.deviceState(state); }, + /** + * A record belonging to the pipeline rather than to any call. A platform SDK that observes + * something this package has no vocabulary for — a phone's thermal state — names it here. + */ + emit(event: string, attributes?: Attributes, severity?: 'info' | 'warn' | 'error') { + backend.emit(event, attributes, severity); + }, + + /** + * Stretch the flush interval and the stats window by this much, 1–4. The platform that can see + * pressure this package cannot reports the number, not the reason. + */ + setCadenceFactor(factor: number) { + backend.setCadenceFactor(factor); + }, + registerTrack(sid: string, scope: Scope, kind: 'audio' | 'video', direction: TrackDirection) { // Nothing to route when nobody is listening: an SDK without a collector keeps no map. if (!backend.enabled) return; @@ -142,7 +160,7 @@ export const Telemetry = { /** A pipeline smoke test: one record, one request, whatever the collector answers. */ ping(seq = 1) { const now = hrTime(); - pipeline.emit({ + pipeline.record({ hrTime: now, hrTimeObserved: now, eventName: 'lk.ping', diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index dc35e95bb7..04f2b4ab78 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -21,6 +21,7 @@ import { serializeSpans, } from './otlp'; import { PipelineScope } from './scope'; +import { MemoryStorage, type TelemetryStorage, batchId, batchKind, batchRecords } from './storage'; export interface TelemetryOptions { /** OTLP logs route. Cloud derives it from the server URL instead; see `setServer`. */ @@ -33,6 +34,11 @@ export interface TelemetryOptions { statsWindow?: number; maxQueueSize?: number; resource?: Record; + /** + * Where batches wait between being made and being accepted. A platform with a filesystem + * supplies one that survives the process; the default keeps them in memory. + */ + storage?: TelemetryStorage; } interface Destination { @@ -45,6 +51,11 @@ const FLUSH_INTERVAL = 15; const STATS_WINDOW = 15; const MAX_QUEUE = 2048; const MAX_BATCH = 512; +/** What the cache may hold: 4 MiB across at most 512 batches, oldest evicted first. */ +const MAX_CACHE_BYTES = 4 * 1024 * 1024; +const MAX_CACHE_BATCHES = 512; +/** A backlog replays beside a live call at this many batches per tick, never faster (SPEC). */ +const MAX_BATCHES_PER_UPLOAD = 4; /** Browsers cap every in-flight keepalive body at 64 KiB together; stay under it. */ const KEEPALIVE_LIMIT = 60 * 1024; /** A hold never outlasts this, whatever the signal that raised it claims (SPEC). */ @@ -59,6 +70,8 @@ export interface Counters { bytes: number; failed: number; holdsCapped: number; + droppedCacheFull: number; + droppedCacheError: number; droppedQueueFull: number; droppedRejected: number; droppedThrottled: number; @@ -71,6 +84,8 @@ function emptyCounters(): Counters { bytes: 0, failed: 0, holdsCapped: 0, + droppedCacheFull: 0, + droppedCacheError: 0, droppedQueueFull: 0, droppedRejected: 0, droppedThrottled: 0, @@ -132,15 +147,25 @@ export class Pipeline implements Backend { private device: DeviceState = {}; + private storage: TelemetryStorage = new MemoryStorage(MAX_CACHE_BYTES, MAX_CACHE_BATCHES); + + private sequence = 0; + resource: Resource = { attributes: {} }; private baseFlushInterval = FLUSH_INTERVAL; private baseStatsWindow = STATS_WINDOW; - /** SPEC's cadence policy: device pressure stretches both periods, never past 4×. */ + /** SPEC's cadence policy: pressure stretches both periods, never past 4×. Two sources + * multiply — what this package observes, and what a platform reports through + * `setCadenceFactor` for the pressure it can see and this one cannot. */ private cadence = 1; + private deviceFactor = 1; + + private platformFactor = 1; + maxQueueSize = MAX_QUEUE; get flushInterval(): number { @@ -170,10 +195,17 @@ export class Pipeline implements Backend { this.processScope.emit(record.event, record.attributes); } } - this.setCadenceFactor(cadenceFactor(next)); + this.deviceFactor = cadenceFactor(next); + this.applyCadence(); } setCadenceFactor(factor: number) { + this.platformFactor = factor; + this.applyCadence(); + } + + private applyCadence() { + const factor = Math.min(4, this.deviceFactor * this.platformFactor); if (factor === this.cadence) return; this.cadence = factor; // Restarting the timer is what makes a *shorter* period apply at once — pressure relieved @@ -214,6 +246,9 @@ export class Pipeline implements Backend { ...options.resource, }, }; + if (options.storage) { + this.storage = options.storage; + } if (options.endpoint) { this.destination = { logs: options.endpoint, @@ -278,7 +313,14 @@ export class Pipeline implements Backend { return true; } - emit(record: LogRecord, options: { exemptFromFlood?: boolean } = {}) { + /** A record that belongs to the pipeline rather than to a call — what a platform reports. */ + emit(event: string, attributes: Attributes = {}, severity: 'info' | 'warn' | 'error' = 'info') { + if (!this.enabled) return; + this.processScope ??= new PipelineScope(this); + this.processScope.emit(event, attributes, severity); + } + + record(record: LogRecord, options: { exemptFromFlood?: boolean } = {}) { if (!this.enabled) return; if (!options.exemptFromFlood && !this.floodOk()) return; record.resource = this.resource; @@ -305,34 +347,71 @@ export class Pipeline implements Backend { } async flush(force = false): Promise { - if (!this.enabled || !this.destination || this.inFlight) return; + if (!this.enabled || this.inFlight) return; if (!force && (this.held() || Date.now() < this.throttledUntil)) return; if (this.reportDue) this.appendReport(); - if (this.logs.length === 0 && this.spans.length === 0) return; + // Write-ahead: what is queued becomes a stored batch whether or not anything can be sent yet. + this.persist(); + const destination = this.destination; + if (!destination) return; this.inFlight = true; try { - const destination = this.destination!; - if (this.logs.length > 0) { - const batch = this.logs.splice(0, MAX_BATCH); - await this.send(destination.logs, serializeLogs(batch, this.encoding), batch, this.logs); - } - // A throttle raised by the logs request applies to the spans request too: same quota. - if (this.spans.length > 0 && Date.now() >= this.throttledUntil) { - const batch = this.spans.splice(0, MAX_BATCH); - await this.send( - destination.traces, - serializeSpans(batch, this.encoding), - batch, - this.spans, - ); + // A backlog from a previous launch replays beside the call at 4 batches a tick; a shutdown + // or a page leaving drains without the budget. + const budget = force ? Number.POSITIVE_INFINITY : MAX_BATCHES_PER_UPLOAD; + for (const id of this.storage.pending().slice(0, budget)) { + const body = this.storage.read(id); + if (body === undefined) { + this.storage.remove(id); + continue; + } + const url = batchKind(id) === 'logs' ? destination.logs : destination.traces; + const verdict = await this.send(url, body, batchRecords(id)); + // Throttled or offline: this batch keeps its place and so does everything behind it. + if (verdict === 'keep') break; + this.storage.remove(id); } } finally { this.inFlight = false; } } - private async send(url: string, body: Uint8Array, batch: T[], queue: T[]) { + private persist() { + if (this.logs.length > 0) { + const batch = this.logs.splice(0, MAX_BATCH); + this.store(batchId('logs', (this.sequence += 1), batch.length), () => + serializeLogs(batch, this.encoding), + ); + } + if (this.spans.length > 0) { + const batch = this.spans.splice(0, MAX_BATCH); + this.store(batchId('traces', (this.sequence += 1), batch.length), () => + serializeSpans(batch, this.encoding), + ); + } + } + + private store(id: string, body: () => Uint8Array) { + try { + const evicted = this.storage.put(id, body()); + if (evicted.length > 0) { + // The id says how many records were in the batch, so an eviction costs a known number. + this.counters.droppedCacheFull += evicted.reduce((sum, key) => sum + batchRecords(key), 0); + this.reportDue = true; + } + } catch { + // A store that cannot store (disk full, quota) must not take the session down with it. + this.counters.droppedCacheError += batchRecords(id); + this.reportDue = true; + } + } + + private async send( + url: string, + body: Uint8Array, + records: number, + ): Promise<'sent' | 'drop' | 'keep'> { const destination = this.destination!; try { const response = await fetch(url, { @@ -349,7 +428,7 @@ export class Pipeline implements Backend { if (response.status >= 200 && response.status < 300) { this.counters.sent += 1; this.counters.bytes += body.byteLength; - return; + return 'sent'; } if (response.status === 429 || response.status >= 500) { const retryAfter = Number.parseInt(response.headers.get('Retry-After') ?? '', 10); @@ -357,30 +436,16 @@ export class Pipeline implements Backend { Date.now() + (Number.isFinite(retryAfter) ? retryAfter * 1000 : THROTTLE_DEFAULT_MS); this.counters.failed += 1; this.reportDue = true; - this.requeue(batch, queue); - return; + return 'keep'; } // A 4xx is the collector's verdict on the payload: retrying cannot fix it. - this.counters.droppedRejected += batch.length; + this.counters.droppedRejected += records; this.reportDue = true; + return 'drop'; } catch { this.counters.failed += 1; this.reportDue = true; - this.requeue(batch, queue); - } - } - - /** A held or failed batch goes back at the front — a pause is not a hole in the session. */ - private requeue(batch: T[], queue: T[]) { - if (batch.length === 0) return; - queue.unshift(...batch); - const total = this.logs.length + this.spans.length; - if (total > this.maxQueueSize) { - const overflow = total - this.maxQueueSize; - const fromLogs = Math.min(overflow, this.logs.length); - this.logs.splice(0, fromLogs); - this.spans.splice(0, overflow - fromLogs); - this.counters.droppedThrottled += overflow; + return 'keep'; } } @@ -392,8 +457,15 @@ export class Pipeline implements Backend { 'lk.telemetry.uploads.bytes': counters.bytes, 'lk.telemetry.uploads.failed': counters.failed, 'lk.telemetry.queue.records': this.logs.length + this.spans.length, + 'lk.telemetry.cache.batches': this.storage.pending().length, }; if (counters.holdsCapped) attributes['lk.telemetry.holds.capped'] = counters.holdsCapped; + if (counters.droppedCacheFull) { + attributes['lk.telemetry.dropped.cache_full'] = counters.droppedCacheFull; + } + if (counters.droppedCacheError) { + attributes['lk.telemetry.dropped.cache_error'] = counters.droppedCacheError; + } if (counters.droppedQueueFull) { attributes['lk.telemetry.dropped.queue_full'] = counters.droppedQueueFull; } @@ -426,6 +498,7 @@ export class Pipeline implements Backend { this.disabled = true; this.logs = []; this.spans = []; + this.storage.clear(); this.stop(); } @@ -441,6 +514,8 @@ export class Pipeline implements Backend { ? 'held' : 'ready'; const lost = + counters.droppedCacheFull + + counters.droppedCacheError + counters.droppedQueueFull + counters.droppedRejected + counters.droppedThrottled + diff --git a/src/telemetry/scope.ts b/src/telemetry/scope.ts index 671e12454c..90a17bd938 100644 --- a/src/telemetry/scope.ts +++ b/src/telemetry/scope.ts @@ -255,7 +255,7 @@ export class PipelineScope implements Scope { instrumentationScope: INSTRUMENTATION_SCOPE, spanContext: span?.context() ?? { traceId: this.traceId, spanId: '', traceFlags: 1 }, }; - this.pipeline.emit(record, { exemptFromFlood: eventName === 'lk.rtc.stats.sample' }); + this.pipeline.record(record, { exemptFromFlood: eventName === 'lk.rtc.stats.sample' }); } disconnected(reason: string) { diff --git a/src/telemetry/storage.ts b/src/telemetry/storage.ts new file mode 100644 index 0000000000..4deb6ab0ad --- /dev/null +++ b/src/telemetry/storage.ts @@ -0,0 +1,75 @@ +/** + * Where batches wait between being made and being accepted. This is the write-ahead cache the Rust + * core calls `BatchCache`: every batch is stored *before* the network is tried, and removed only + * when the collector has taken it, so a crash, a kill or an offline hour costs nothing. + * + * The interface is deliberately the same five operations as the core's, and deliberately + * synchronous — the queue path has no await in it, and a store that forces one turns the pipeline + * into a state machine. A platform with a filesystem implements this; the default keeps batches in + * memory, which is all a browser tab needs (TELEMETRY.md §3). + */ +export interface TelemetryStorage { + /** Store a batch. Returns the ids evicted to stay inside the store's own bounds. */ + put(id: string, body: Uint8Array): string[]; + /** Stored ids, oldest first. */ + pending(): string[]; + read(id: string): Uint8Array | undefined; + remove(id: string): void; + clear(): void; +} + +/** Ids sort oldest-first as plain strings, so a store never has to parse or stat anything. */ +export function batchId(kind: 'logs' | 'traces', sequence: number, records: number): string { + const now = String(Date.now()).padStart(15, '0'); + return `${now}-${String(sequence).padStart(6, '0')}-${records}-${kind === 'logs' ? 'l' : 't'}`; +} + +export function batchKind(id: string): 'logs' | 'traces' { + return id.endsWith('-l') ? 'logs' : 'traces'; +} + +/** How many records a batch holds, so an eviction costs a known number and not a guess. */ +export function batchRecords(id: string): number { + return Number.parseInt(id.split('-')[2] ?? '0', 10) || 0; +} + +/** Lost with the tab, bounded by bytes and by count; the oldest goes first, and one always stays. */ +export class MemoryStorage implements TelemetryStorage { + private batches: Array<[string, Uint8Array]> = []; + + constructor( + private maxBytes: number, + private maxBatches: number, + ) {} + + put(id: string, body: Uint8Array): string[] { + this.batches.push([id, body]); + const evicted: string[] = []; + let total = this.batches.reduce((sum, [, batch]) => sum + batch.byteLength, 0); + while ( + (total > this.maxBytes || this.batches.length > this.maxBatches) && + this.batches.length > 1 + ) { + const [oldest, batch] = this.batches.shift()!; + total -= batch.byteLength; + evicted.push(oldest); + } + return evicted; + } + + pending(): string[] { + return this.batches.map(([id]) => id); + } + + read(id: string): Uint8Array | undefined { + return this.batches.find(([key]) => key === id)?.[1]; + } + + remove(id: string) { + this.batches = this.batches.filter(([key]) => key !== id); + } + + clear() { + this.batches = []; + } +} diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts index e666a3075c..324d5a0954 100644 --- a/src/telemetry/telemetry.test.ts +++ b/src/telemetry/telemetry.test.ts @@ -4,6 +4,7 @@ import type { Backend, Scope, Span } from './backend'; import { cadenceFactor, changes, networkType } from './device'; import { Pipeline } from './pipeline'; import { PipelineScope } from './scope'; +import { MemoryStorage, batchId } from './storage'; /** The JSON encoding is the readable one: every assertion here reads the body the collector gets. */ function bodyOf(call: unknown[]): any { @@ -253,6 +254,8 @@ describe('the backend seam', () => { scope: () => scope, hold: (up) => calls.push(`hold ${up}`), deviceState: (state) => calls.push(`device ${state.appState}`), + emit: (event) => calls.push(`emit ${event}`), + setCadenceFactor: (factor) => calls.push(`cadence ${factor}`), flush: async () => {}, diagnostics: () => 'platform backend', shutdown: async () => {}, @@ -270,6 +273,9 @@ describe('the backend seam', () => { connect.end('ok'); Telemetry.hold(false); Telemetry.trackStats('TR_1', 'outbound', { bytes: 1 }); + // What a platform sees and this package cannot: it names the event and the factor itself. + Telemetry.emit('lk.device.thermal.changed', { 'lk.device.thermal.state': 'serious' }); + Telemetry.setCadenceFactor(2); session.disconnected('client_initiated'); expect(calls).toEqual([ @@ -284,6 +290,8 @@ describe('the backend seam', () => { 'end ok', 'hold false', 'stats TR_1 outbound', + 'emit lk.device.thermal.changed', + 'cadence 2', 'disconnected client_initiated', ]); expect(Telemetry.diagnostics()).toBe('platform backend'); @@ -292,3 +300,74 @@ describe('the backend seam', () => { } }); }); + +describe('the write-ahead cache', () => { + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(async () => new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => vi.unstubAllGlobals()); + + test('a batch the network refused is still there next time', async () => { + const storage = new MemoryStorage(4 * 1024 * 1024, 512); + const pipeline = new Pipeline(); + pipeline.configure({ + endpoint: 'http://collector.test/v1/logs', + encoding: 'json', + flushInterval: 3600, + storage, + }); + const scope = new PipelineScope(pipeline); + scope.emit('lk.test.offline'); + + fetchMock.mockRejectedValueOnce(new TypeError('offline')); + await pipeline.flush(true); + // The record left the queue, but it is in the cache, not gone. + expect(storage.pending()).toHaveLength(1); + expect(pipeline.diagnostics()).toContain('lost 0'); + + await pipeline.flush(true); + expect(storage.pending()).toHaveLength(0); + expect(recordsOf(fetchMock.mock.calls[1])[0].eventName).toBe('lk.test.offline'); + pipeline.stop(); + }); + + test('a backlog from a previous launch replays four batches a tick', async () => { + // What the store looks like after a crash: batches nobody has sent yet. + const storage = new MemoryStorage(4 * 1024 * 1024, 512); + for (let i = 0; i < 6; i += 1) { + storage.put(batchId('logs', i, 10), new Uint8Array([1, 2, 3])); + } + const pipeline = new Pipeline(); + pipeline.configure({ endpoint: 'http://collector.test/v1/logs', flushInterval: 3600, storage }); + + await pipeline.flush(); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(storage.pending()).toHaveLength(2); + + // A shutdown drains without the budget. + await pipeline.flush(true); + expect(storage.pending()).toHaveLength(0); + pipeline.stop(); + }); + + test('an eviction costs a known number of records, not a batch', async () => { + const storage = new MemoryStorage(64, 512); + const pipeline = new Pipeline(); + // No destination: batches pile up in the cache, which is where eviction happens. + pipeline.configure({ encoding: 'json', flushInterval: 3600, storage }); + const scope = new PipelineScope(pipeline); + scope.emit('lk.test.one'); + await pipeline.flush(true); + scope.emit('lk.test.two'); + await pipeline.flush(true); + + expect(storage.pending()).toHaveLength(1); + // The evicted batch held one record, and the report says so — not "one batch". + expect(pipeline.diagnostics()).toContain('lost 1'); + pipeline.stop(); + }); +}); From 9f2616743009b7b02a453906f794839da799d6fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:36:05 +0200 Subject: [PATCH 11/14] fix(telemetry): a cached batch keeps the encoding it was written with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache stores bytes; the Content-Type was coming from whatever the pipeline was configured with when the batch finally went out. A React Native app that cached JSON batches and replayed them as protobuf lost all of them to 4xx — 40 records, which only surfaced because the self-report says dropped.rejected rather than letting a 4xx look like success. The batch id now carries the encoding alongside the route and the record count, so a batch written by an earlier run of the app is sent the way it was written. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 8 +++++++- src/index.ts | 11 ++++++++++- src/telemetry/pipeline.ts | 18 +++++++++++++----- src/telemetry/storage.ts | 25 +++++++++++++++++++++---- src/telemetry/telemetry.test.ts | 18 +++++++++++++++++- 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/TELEMETRY.md b/TELEMETRY.md index 102d3411c9..c0840cbdd5 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -100,7 +100,13 @@ process that dies costs nothing. Where it is stored is `TelemetryStorage` — fi operations, the same five the core's `BatchCache` has — and the default keeps batches in memory, bounded by 4 MiB and 512 batches, oldest evicted first and counted. -That default is right for a tab, and nobody in this ecosystem persists there either. +A batch id carries everything needed to send a batch an earlier run of the app wrote — its route, +its record count, and the encoding it was written with. That last one is not hypothetical: the +first React Native replay lost 40 records because cached JSON batches were re-sent with the +protobuf content type, and the only reason anyone noticed is that `lk.telemetry.report` said +`dropped.rejected: 40` instead of quietly succeeding. + +The in-memory default is right for a tab, and nobody in this ecosystem persists there either. - **OpenTelemetry JS** caches nothing: `BatchLogRecordProcessor` is a bounded in-memory queue and the spec puts retry on the exporter, explicitly not on the processor. diff --git a/src/index.ts b/src/index.ts index 9b2b04947e..6624449079 100644 --- a/src/index.ts +++ b/src/index.ts @@ -202,4 +202,13 @@ export { serializers, } from './utils/serializer'; -export { Telemetry, type TelemetryOptions } from './telemetry'; +export { + Telemetry, + type TelemetryOptions, + // The seams a platform SDK implements: where batches wait, and what carries them. + type TelemetryStorage, + type Backend as TelemetryBackend, + type Scope as TelemetryScope, + type Span as TelemetrySpan, + type DeviceState as TelemetryDeviceState, +} from './telemetry'; diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index 04f2b4ab78..09db0401d2 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -21,7 +21,14 @@ import { serializeSpans, } from './otlp'; import { PipelineScope } from './scope'; -import { MemoryStorage, type TelemetryStorage, batchId, batchKind, batchRecords } from './storage'; +import { + MemoryStorage, + type TelemetryStorage, + batchEncoding, + batchId, + batchKind, + batchRecords, +} from './storage'; export interface TelemetryOptions { /** OTLP logs route. Cloud derives it from the server URL instead; see `setServer`. */ @@ -367,7 +374,7 @@ export class Pipeline implements Backend { continue; } const url = batchKind(id) === 'logs' ? destination.logs : destination.traces; - const verdict = await this.send(url, body, batchRecords(id)); + const verdict = await this.send(url, body, batchRecords(id), batchEncoding(id)); // Throttled or offline: this batch keeps its place and so does everything behind it. if (verdict === 'keep') break; this.storage.remove(id); @@ -380,13 +387,13 @@ export class Pipeline implements Backend { private persist() { if (this.logs.length > 0) { const batch = this.logs.splice(0, MAX_BATCH); - this.store(batchId('logs', (this.sequence += 1), batch.length), () => + this.store(batchId('logs', (this.sequence += 1), batch.length, this.encoding), () => serializeLogs(batch, this.encoding), ); } if (this.spans.length > 0) { const batch = this.spans.splice(0, MAX_BATCH); - this.store(batchId('traces', (this.sequence += 1), batch.length), () => + this.store(batchId('traces', (this.sequence += 1), batch.length, this.encoding), () => serializeSpans(batch, this.encoding), ); } @@ -411,13 +418,14 @@ export class Pipeline implements Backend { url: string, body: Uint8Array, records: number, + encoding: Encoding, ): Promise<'sent' | 'drop' | 'keep'> { const destination = this.destination!; try { const response = await fetch(url, { method: 'POST', headers: { - 'Content-Type': contentType(this.encoding), + 'Content-Type': contentType(encoding), // RFC 9218 lowest urgency: telemetry never wins over media on a shared uplink. Priority: 'u=7', ...destination.headers, diff --git a/src/telemetry/storage.ts b/src/telemetry/storage.ts index 4deb6ab0ad..ad1d25bc9a 100644 --- a/src/telemetry/storage.ts +++ b/src/telemetry/storage.ts @@ -8,6 +8,8 @@ * into a state machine. A platform with a filesystem implements this; the default keeps batches in * memory, which is all a browser tab needs (TELEMETRY.md §3). */ +import type { Encoding } from './otlp'; + export interface TelemetryStorage { /** Store a batch. Returns the ids evicted to stay inside the store's own bounds. */ put(id: string, body: Uint8Array): string[]; @@ -18,14 +20,29 @@ export interface TelemetryStorage { clear(): void; } -/** Ids sort oldest-first as plain strings, so a store never has to parse or stat anything. */ -export function batchId(kind: 'logs' | 'traces', sequence: number, records: number): string { +/** + * Ids sort oldest-first as plain strings, so a store never has to parse or stat anything — and + * they carry everything needed to send a batch that was written by an earlier run of the app: + * which route it belongs to, how it was encoded, and how many records it cost. + */ +export function batchId( + kind: 'logs' | 'traces', + sequence: number, + records: number, + encoding: Encoding, +): string { const now = String(Date.now()).padStart(15, '0'); - return `${now}-${String(sequence).padStart(6, '0')}-${records}-${kind === 'logs' ? 'l' : 't'}`; + const suffix = `${kind === 'logs' ? 'l' : 't'}${encoding === 'json' ? 'j' : 'p'}`; + return `${now}-${String(sequence).padStart(6, '0')}-${records}-${suffix}`; } export function batchKind(id: string): 'logs' | 'traces' { - return id.endsWith('-l') ? 'logs' : 'traces'; + return id.split('-').pop()?.startsWith('t') === true ? 'traces' : 'logs'; +} + +/** A cached batch keeps the encoding it was written with: the collector is told the truth. */ +export function batchEncoding(id: string): Encoding { + return id.endsWith('j') ? 'json' : 'protobuf'; } /** How many records a batch holds, so an eviction costs a known number and not a guess. */ diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts index 324d5a0954..8a9d022681 100644 --- a/src/telemetry/telemetry.test.ts +++ b/src/telemetry/telemetry.test.ts @@ -339,7 +339,7 @@ describe('the write-ahead cache', () => { // What the store looks like after a crash: batches nobody has sent yet. const storage = new MemoryStorage(4 * 1024 * 1024, 512); for (let i = 0; i < 6; i += 1) { - storage.put(batchId('logs', i, 10), new Uint8Array([1, 2, 3])); + storage.put(batchId('logs', i, 10, 'protobuf'), new Uint8Array([1, 2, 3])); } const pipeline = new Pipeline(); pipeline.configure({ endpoint: 'http://collector.test/v1/logs', flushInterval: 3600, storage }); @@ -370,4 +370,20 @@ describe('the write-ahead cache', () => { expect(pipeline.diagnostics()).toContain('lost 1'); pipeline.stop(); }); + + test('a cached batch keeps the encoding it was written with', async () => { + const storage = new MemoryStorage(4 * 1024 * 1024, 512); + const pipeline = new Pipeline(); + pipeline.configure({ encoding: 'json', flushInterval: 3600, storage }); + new PipelineScope(pipeline).emit('lk.test.written_as_json'); + await pipeline.flush(true); // no destination yet: the batch is cached as JSON + + // The app upgrades, or simply flips the switch, before the batch ever left. + pipeline.configure({ endpoint: 'http://collector.test/v1/logs', encoding: 'protobuf' }); + await pipeline.flush(true); + + const headers = (fetchMock.mock.calls[0][1] as RequestInit).headers as Record; + expect(headers['Content-Type']).toBe('application/json'); + pipeline.stop(); + }); }); From 5371904d66c4a3b63c75e70f11817af493ad1139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:41:18 +0200 Subject: [PATCH 12/14] refactor(telemetry): drop the Backend seam, keep the one that earned its place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It had one implementation and one hypothetical one. React Native turned out to need exactly one thing from this package that a browser does not provide — somewhere on disk to keep batches — and TelemetryStorage already is that seam, with two real implementations behind it. The scope and span classes get their names back, and the platform-neutral verbs React Native does use stay: emit() for a record this package has no vocabulary for, setCadenceFactor() for pressure it cannot see. Co-Authored-By: Claude Opus 5 (1M context) --- TELEMETRY.md | 30 ++++-- src/index.ts | 7 +- src/room/Room.ts | 8 +- src/room/participant/LocalParticipant.ts | 4 +- src/telemetry/backend.ts | 112 --------------------- src/telemetry/index.ts | 66 ++++++------ src/telemetry/pipeline.ts | 15 ++- src/telemetry/scope.ts | 64 ++++++++---- src/telemetry/telemetry.test.ts | 123 +++++++---------------- src/telemetry/webrtc.ts | 2 +- 10 files changed, 147 insertions(+), 284 deletions(-) delete mode 100644 src/telemetry/backend.ts diff --git a/TELEMETRY.md b/TELEMETRY.md index c0840cbdd5..9c46b91648 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -169,25 +169,35 @@ reimplement the instrumentation: when a connect span starts, which checkpoints i subscribe ends at first media, which `getStats` fields become a window — all of that stays here and is reused. -That is what `backend.ts` is for. It is the same set of operations SPEC calls the typed surface — -the boundary Swift, Kotlin and Dart already cross into the core — expressed in terms this package -owns: rooms, spans, tracks, outcomes. `Telemetry.setBackend` installs one, and `Room` does not know -which is in place. +**React Native reuses all of it and replaces one thing: where batches wait.** The caching +semantics of the native SDKs — a batch on disk that survives the process and replays at next launch +— do not need the Rust core, only a directory of files. `@livekit/react-native` supplies a +`TelemetryStorage` backed by `LKBatchStore.swift` and `BatchStore.kt`, a native store mirroring +the core's `FileCache`, and this package learns nothing about it beyond five synchronous calls. ``` Room, LocalParticipant, the four track monitors ← the instrumentation, one copy │ - Backend / Scope / Span ← backend.ts, platform-neutral by construction + Pipeline ← the policy, one copy + │ + TelemetryStorage ← the one seam with two implementations ╱ ╲ - Pipeline (this package) RustBackend (@livekit/react-native) - fetch, in-memory queue UniFFI → livekit-telemetry → FileCache, NetTransport + MemoryStorage (a tab) a directory of files (@livekit/react-native) ``` -**Nothing mobile appears on this side of the seam.** `DeviceState` carries only what a page can +An earlier round put a whole `Backend` interface here so React Native could bind the Rust core for +the entire pipeline. It is gone: it had one implementation and one hypothetical one, which is the +definition of an abstraction to write later. If that question ever resolves the other way, the git +history has the shape and `Pipeline` is small enough to sit behind an interface again in an hour. + +**Nothing mobile appears on this side either.** `DeviceState` carries only what a page can answer — visibility and, on Chromium, the connection. Thermal state, low power mode and memory pressure are not absent because they are unimportant; they are absent because this package cannot observe them and must not pretend to. React Native's native monitors (`LKDeviceState.swift`, -`DeviceStateMonitor.kt`) report those to the core natively, never through JavaScript. +`DeviceStateMonitor.kt`) map them onto SPEC's event names and onto a cadence factor *in that +package*, and hand the result over through two verbs that name no platform: `Telemetry.emit`, which +takes a record this package never interprets, and `Telemetry.setCadenceFactor`, which takes a +number without a reason attached. One thing the PoC found, which is about this package rather than telemetry: `livekit-client` evaluates `class … extends DOMException` and `new TextDecoder()` at **module scope**, and Hermes has @@ -201,7 +211,7 @@ src/telemetry/ index.ts the Telemetry facade: configure/setServer, the scope factory, the track registry pipeline.ts queue, flush timer, holds, 429/5xx, one request in flight, the self-report scope.ts one per Room connection: trace id, attributes, spans, stats windows - backend.ts the seam: what a platform must implement to carry the records (see §5) + storage.ts the seam: where batches wait between being made and being accepted (see §3) otlp.ts the records and their wire form (the only OpenTelemetry import lives here) webrtc.ts the SDK's typed sender/receiver stats → one SPEC reading ``` diff --git a/src/index.ts b/src/index.ts index 6624449079..afbee3ee1f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -205,10 +205,9 @@ export { export { Telemetry, type TelemetryOptions, - // The seams a platform SDK implements: where batches wait, and what carries them. + // The one seam a platform SDK implements: where batches wait between being made and accepted. type TelemetryStorage, - type Backend as TelemetryBackend, - type Scope as TelemetryScope, - type Span as TelemetrySpan, + type TelemetryScope, + type TelemetrySpan, type DeviceState as TelemetryDeviceState, } from './telemetry'; diff --git a/src/room/Room.ts b/src/room/Room.ts index 3eecadcf1e..adffcd9190 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -47,7 +47,7 @@ import type { RoomConnectOptions, RoomOptions, } from '../options'; -import { type Scope, type Span, SpanKind, Telemetry } from '../telemetry'; +import { SpanKind, Telemetry, type TelemetryScope, type TelemetrySpan } from '../telemetry'; import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays'; import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; @@ -203,11 +203,11 @@ class Room extends (EventEmitter as new () => TypedEmitter) private connectFuture?: Future; /** One telemetry scope per connect: a trace id and the attributes every record of it carries. */ - private telemetry?: Scope; + private telemetry?: TelemetryScope; - private connectSpan?: Span; + private connectSpan?: TelemetrySpan; - private reconnectSpan?: Span; + private reconnectSpan?: TelemetrySpan; /** Attempts inside the *current* reconnect; the engine's own counter spans several. */ private reconnectAttempts = 0; diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index eebba876ac..ed8513db48 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -30,7 +30,7 @@ import { isFrameMetadataSupported, } from '../../frameMetadata/utils'; import type { InternalRoomOptions } from '../../options'; -import type { Scope } from '../../telemetry'; +import type { TelemetryScope } from '../../telemetry'; import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays'; import TypedPromise from '../../utils/TypedPromise'; import { PCTransportState } from '../PCTransportManager'; @@ -140,7 +140,7 @@ export default class LocalParticipant extends Participant { activeDeviceMap: Map; /** @internal — the Room's telemetry scope, set at connect; publishing is a span on it. */ - telemetry?: Scope; + telemetry?: TelemetryScope; private pendingPublishing = new Set(); diff --git a/src/telemetry/backend.ts b/src/telemetry/backend.ts deleted file mode 100644 index 9c93460daf..0000000000 --- a/src/telemetry/backend.ts +++ /dev/null @@ -1,112 +0,0 @@ -/** - * The seam between what this SDK *observes* and what carries it. - * - * These are the same operations `livekit-telemetry/SPEC.md` defines as the typed surface the Swift, - * Kotlin and Dart SDKs cross into the Rust core: verbs about rooms, spans, tracks and outcomes, and - * nothing about a platform. That is deliberate — the Room instrumentation in this package is the - * part worth having once, and a platform that carries telemetry differently (React Native binds the - * Rust core) implements these interfaces instead of reimplementing the instrumentation. - * - * The browser's implementation is `Pipeline` + `PipelineScope`; `Telemetry.setBackend` installs - * another. Nothing here may name a capability only some platforms have. - */ -import type { DeviceState } from './device'; -import type { Attributes } from './otlp'; - -export type Outcome = 'ok' | 'error' | 'cancelled'; - -export type TrackDirection = 'inbound' | 'outbound'; - -export type Severity = 'info' | 'warn' | 'error'; - -export interface RoomIdentity { - sid?: string; - name?: string; - participantSid?: string; - participantIdentity?: string; -} - -/** One track's reading, already in SPEC units: milliseconds, not the WebRTC seconds. */ -export interface StatsSample { - codec?: string; - bytes?: number; - packets?: number; - packetsLost?: number; - framesDropped?: number; - concealedSamples?: number; - concealmentEvents?: number; - silentConcealedSamples?: number; - jitterBufferDelayMs?: number; - qualityLimitationBandwidthMs?: number; - qualityLimitationCpuMs?: number; - qualityLimitationOtherMs?: number; - jitterMs?: number; - rttMs?: number; - fps?: number; - audioLevel?: number; -} - -export interface TraceContext { - traceId: string; - spanId: string; - traceFlags: number; -} - -/** - * One attempt at an operation. Calls are synchronous and the implementation stamps the clock, so - * the only skew is the call itself — a backend that crosses a native boundary must use a blocking - * call, not a promise. - */ -export interface Span { - /** A checkpoint inside the attempt: `ws_open`, `join_recv`, `first_media`, `attempt 2 full`. */ - step(name: string): void; - setAttribute(key: string, value: Attributes[string]): void; - /** Ending twice is a no-op. */ - end(outcome?: Outcome, errorType?: string, message?: string): void; - fail(error: unknown): void; - cancel(): void; - context(): TraceContext; -} - -/** One Room connection: a trace id, the attributes its records carry, its spans and its windows. */ -export interface Scope { - readonly traceId: string; - setRoom(identity: RoomIdentity): void; - start(name: string, options?: { kind?: number; attributes?: Attributes; parent?: Span }): Span; - emit(event: string, attributes?: Attributes, severity?: Severity, span?: Span): void; - recordStats( - sid: string, - kind: 'audio' | 'video', - direction: TrackDirection, - sample: StatsSample, - ): void; - subscribeStarted(sid: string, attributes: Attributes): void; - subscribeEnded(sid: string, outcome: Outcome, errorType?: string): void; - disconnected(reason: string): void; - /** The call is ending: close the open windows early and cancel what never resolved. */ - close(): void; -} - -export interface Backend { - readonly enabled: boolean; - /** LiveKit Cloud: the server URL and the connect token are the destination. */ - setServer(serverUrl: string, token: string): void; - scope(): Scope; - /** Uploads stop, collection does not — spans that own the uplink raise a hold. */ - hold(up: boolean): void; - /** Only what the platform running this code can actually answer; see `DeviceState`. */ - deviceState(state: DeviceState): void; - /** - * A record belonging to the pipeline rather than to any call. A platform that observes something - * this package has no vocabulary for — a phone's thermal state, say — names the event itself. - */ - emit(event: string, attributes?: Attributes, severity?: Severity): void; - /** - * Stretch the flush interval and the stats window by this much, 1–4. The platform that can see - * pressure this package cannot reports the number, not the reason. - */ - setCadenceFactor(factor: number): void; - flush(): Promise; - diagnostics(): string; - shutdown(): Promise; -} diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index d6aef1bc49..47e21260a2 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -6,40 +6,36 @@ * Nothing is collected until a destination exists: `Telemetry.configure({ endpoint })` for your own * collector, or the first connect to LiveKit Cloud, which derives the route and the token itself. * - * What carries the records is replaceable (`setBackend`) — the Room instrumentation in this package - * is worth having once, while a platform may know a better way to batch, cache and upload. The - * default is the browser pipeline in `pipeline.ts`. + * Where batches wait between being made and being accepted is the one replaceable part + * (`TelemetryOptions.storage`): a browser keeps them in memory, a platform with a filesystem + * keeps them on disk. Everything else is this package's. */ -import type { Backend, Scope, StatsSample, TrackDirection } from './backend'; import { type DeviceState, observeBrowser } from './device'; import { type Attributes, Severity, hrTime, randomHex } from './otlp'; import { Pipeline, type TelemetryOptions } from './pipeline'; +import type { StatsSample, TelemetryScope, TrackDirection } from './scope'; import { receiverSample, senderSample } from './webrtc'; export type { TelemetryOptions } from './pipeline'; export type { DeviceState, AppStateName, NetworkType } from './device'; export type { - Backend, - Scope, - Span, Outcome, RoomIdentity, - Severity as SeverityName, + SeverityName, StatsSample, TrackDirection, TraceContext, -} from './backend'; +} from './scope'; +export type { TelemetryScope, TelemetrySpan } from './scope'; export { SpanKind } from './otlp'; export type { Attributes } from './otlp'; export type { TelemetryStorage } from './storage'; const pipeline = new Pipeline(); -let backend: Backend = pipeline; - /** Which scope a track's stats belong to — the monitors know a sid, not a Room. */ interface TrackRegistration { - scope: Scope; + scope: TelemetryScope; kind: 'audio' | 'video'; direction: TrackDirection; } @@ -54,7 +50,7 @@ function attachLifecycle() { // The page's last chance: `pagehide` and a hidden tab, never `unload` — by then a request has // no chance of leaving. `fetch(keepalive)` makes it best effort, not durable (TELEMETRY.md §3). const flush = () => { - backend.flush().catch(() => {}); + pipeline.flush().catch(() => {}); }; document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'hidden') flush(); @@ -70,39 +66,28 @@ export const Telemetry = { attachLifecycle(); }, - /** - * Replace what carries the records — a platform SDK that binds a different implementation of - * `Backend` installs it here, before any Room is created. The instrumentation does not change. - * Returns the backend that was in place, so it can be put back. - */ - setBackend(replacement: Backend): Backend { - const previous = backend; - backend = replacement; - return previous; - }, - /** LiveKit Cloud: the server URL and the connect token are the destination (SPEC). */ setServer(serverUrl: string, token: string) { - backend.setServer(serverUrl, token); + pipeline.setServer(serverUrl, token); attachLifecycle(); }, get enabled(): boolean { - return backend.enabled; + return pipeline.enabled; }, - scope(): Scope { - return backend.scope(); + scope(): TelemetryScope { + return pipeline.scope(); }, /** Uploads stop, collection does not — spans that own the uplink raise a hold (SPEC). */ hold(up: boolean) { - backend.hold(up); + pipeline.hold(up); }, /** What the platform can say about the device it runs on; see `DeviceState` for the limits. */ deviceState(state: DeviceState) { - backend.deviceState(state); + pipeline.deviceState(state); }, /** @@ -110,7 +95,7 @@ export const Telemetry = { * something this package has no vocabulary for — a phone's thermal state — names it here. */ emit(event: string, attributes?: Attributes, severity?: 'info' | 'warn' | 'error') { - backend.emit(event, attributes, severity); + pipeline.emit(event, attributes, severity); }, /** @@ -118,12 +103,17 @@ export const Telemetry = { * pressure this package cannot reports the number, not the reason. */ setCadenceFactor(factor: number) { - backend.setCadenceFactor(factor); + pipeline.setCadenceFactor(factor); }, - registerTrack(sid: string, scope: Scope, kind: 'audio' | 'video', direction: TrackDirection) { + registerTrack( + sid: string, + scope: TelemetryScope, + kind: 'audio' | 'video', + direction: TrackDirection, + ) { // Nothing to route when nobody is listening: an SDK without a collector keeps no map. - if (!backend.enabled) return; + if (!pipeline.enabled) return; tracks.set(`${sid}:${direction}`, { scope, kind, direction }); }, @@ -143,18 +133,18 @@ export const Telemetry = { /** Called from the SDK's existing per-track monitors: no extra `getStats()` anywhere. */ trackStats(sid: string, direction: TrackDirection, sample: StatsSample) { - if (!backend.enabled) return; + if (!pipeline.enabled) return; const registration = tracks.get(`${sid}:${direction}`); if (!registration) return; registration.scope.recordStats(sid, registration.kind, direction, sample); }, flush(): Promise { - return backend.flush(); + return pipeline.flush(); }, diagnostics(): string { - return backend.diagnostics(); + return pipeline.diagnostics(); }, /** A pipeline smoke test: one record, one request, whatever the collector answers. */ @@ -178,6 +168,6 @@ export const Telemetry = { async shutdown() { tracks.clear(); - await backend.shutdown(); + await pipeline.shutdown(); }, }; diff --git a/src/telemetry/pipeline.ts b/src/telemetry/pipeline.ts index 09db0401d2..987fa55b2d 100644 --- a/src/telemetry/pipeline.ts +++ b/src/telemetry/pipeline.ts @@ -4,7 +4,6 @@ * (TELEMETRY.md §3), so the queue is the only bound and every eviction is counted. */ import { version } from '../version'; -import type { Backend, Scope } from './backend'; import { type DeviceState, cadenceFactor, changes } from './device'; import { type AttributeValue, @@ -20,7 +19,7 @@ import { serializeLogs, serializeSpans, } from './otlp'; -import { PipelineScope } from './scope'; +import { TelemetryScope } from './scope'; import { MemoryStorage, type TelemetryStorage, @@ -117,7 +116,7 @@ export function tracesEndpointFor(logs: string): string { return logs; } -export class Pipeline implements Backend { +export class Pipeline { private logs: LogRecord[] = []; private spans: SpanRecord[] = []; @@ -150,7 +149,7 @@ export class Pipeline implements Backend { private collecting = false; /** Device state belongs to no call: it is filed under the pipeline's own scope (SPEC). */ - private processScope?: PipelineScope; + private processScope?: TelemetryScope; private device: DeviceState = {}; @@ -183,8 +182,8 @@ export class Pipeline implements Backend { return this.baseStatsWindow * this.cadence; } - scope(): Scope { - return new PipelineScope(this); + scope(): TelemetryScope { + return new TelemetryScope(this); } /** @@ -197,7 +196,7 @@ export class Pipeline implements Backend { const records = changes(this.device, next); this.device = next; if (this.enabled) { - this.processScope ??= new PipelineScope(this); + this.processScope ??= new TelemetryScope(this); for (const record of records) { this.processScope.emit(record.event, record.attributes); } @@ -323,7 +322,7 @@ export class Pipeline implements Backend { /** A record that belongs to the pipeline rather than to a call — what a platform reports. */ emit(event: string, attributes: Attributes = {}, severity: 'info' | 'warn' | 'error' = 'info') { if (!this.enabled) return; - this.processScope ??= new PipelineScope(this); + this.processScope ??= new TelemetryScope(this); this.processScope.emit(event, attributes, severity); } diff --git a/src/telemetry/scope.ts b/src/telemetry/scope.ts index 90a17bd938..64047f826a 100644 --- a/src/telemetry/scope.ts +++ b/src/telemetry/scope.ts @@ -2,15 +2,6 @@ * One scope per Room connection: a trace id, the attributes every record of that call carries, the * spans, and the stats windows. The scope is not ended — a call's last record is simply its last. */ -import type { - Outcome, - RoomIdentity, - Scope, - Severity as SeverityName, - Span, - StatsSample, - TrackDirection, -} from './backend'; import { type Attributes, INSTRUMENTATION_SCOPE, @@ -25,6 +16,45 @@ import { } from './otlp'; import type { Pipeline } from './pipeline'; +export type Outcome = 'ok' | 'error' | 'cancelled'; + +export type TrackDirection = 'inbound' | 'outbound'; + +export type SeverityName = 'info' | 'warn' | 'error'; + +export interface RoomIdentity { + sid?: string; + name?: string; + participantSid?: string; + participantIdentity?: string; +} + +/** One track's reading, already in SPEC units: milliseconds, not the WebRTC seconds. */ +export interface StatsSample { + codec?: string; + bytes?: number; + packets?: number; + packetsLost?: number; + framesDropped?: number; + concealedSamples?: number; + concealmentEvents?: number; + silentConcealedSamples?: number; + jitterBufferDelayMs?: number; + qualityLimitationBandwidthMs?: number; + qualityLimitationCpuMs?: number; + qualityLimitationOtherMs?: number; + jitterMs?: number; + rttMs?: number; + fps?: number; + audioLevel?: number; +} + +export interface TraceContext { + traceId: string; + spanId: string; + traceFlags: number; +} + /** An attribute nobody set is not an attribute: `undefined` would ship as an empty value. */ function defined(attributes: Attributes): Attributes { const out: Attributes = {}; @@ -118,7 +148,7 @@ class Window { } } -export class PipelineSpan implements Span { +export class TelemetrySpan { private events: SpanRecord['events'] = []; private attributes: Attributes; @@ -130,7 +160,7 @@ export class PipelineSpan implements Span { private spanId = randomHex(8); constructor( - private scope: PipelineScope, + private scope: TelemetryScope, private pipeline: Pipeline, private name: string, private kind: number, @@ -195,14 +225,14 @@ export class PipelineSpan implements Span { } } -export class PipelineScope implements Scope { +export class TelemetryScope { readonly traceId = randomHex(16); private room: RoomIdentity = {}; private windows = new Map(); - private pendingSubscribes = new Map(); + private pendingSubscribes = new Map(); constructor(private pipeline: Pipeline) {} @@ -222,9 +252,9 @@ export class PipelineScope implements Scope { start( name: string, - options: { kind?: number; attributes?: Attributes; parent?: Span } = {}, - ): PipelineSpan { - return new PipelineSpan( + options: { kind?: number; attributes?: Attributes; parent?: TelemetrySpan } = {}, + ): TelemetrySpan { + return new TelemetrySpan( this, this.pipeline, name, @@ -238,7 +268,7 @@ export class PipelineScope implements Scope { eventName: string, attributes: Attributes = {}, severity: SeverityName = 'info', - span?: Span, + span?: TelemetrySpan, ) { const now = hrTime(); const record: LogRecord = { diff --git a/src/telemetry/telemetry.test.ts b/src/telemetry/telemetry.test.ts index 8a9d022681..b6d02ef0d4 100644 --- a/src/telemetry/telemetry.test.ts +++ b/src/telemetry/telemetry.test.ts @@ -3,7 +3,7 @@ import { Telemetry } from '.'; import type { Backend, Scope, Span } from './backend'; import { cadenceFactor, changes, networkType } from './device'; import { Pipeline } from './pipeline'; -import { PipelineScope } from './scope'; +import { TelemetryScope } from './scope'; import { MemoryStorage, batchId } from './storage'; /** The JSON encoding is the readable one: every assertion here reads the body the collector gets. */ @@ -44,7 +44,7 @@ describe('telemetry pipeline', () => { }); test('one window of readings becomes one record, counters last and gauges summarised', async () => { - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); scope.setRoom({ sid: 'RM_1', name: 'harness', participantIdentity: 'publisher' }); scope.recordStats('TR_1', 'video', 'outbound', { bytes: 1000, @@ -77,7 +77,7 @@ describe('telemetry pipeline', () => { }); test('a hold stops uploads, never collection', async () => { - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); pipeline.hold(true); scope.emit('lk.test.one'); await pipeline.flush(); @@ -95,7 +95,7 @@ describe('telemetry pipeline', () => { test('a 429 holds the pipeline and keeps the batch', async () => { fetchMock.mockResolvedValueOnce(new Response(null, { status: 429 })); - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); scope.emit('lk.test.throttled'); await pipeline.flush(true); expect(pipeline.diagnostics()).toContain('throttled'); @@ -112,7 +112,7 @@ describe('telemetry pipeline', () => { test('the queue evicts the oldest and says how many', async () => { pipeline.maxQueueSize = 3; - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); for (let i = 0; i < 5; i += 1) { scope.emit(`lk.test.${i}`); } @@ -126,7 +126,7 @@ describe('telemetry pipeline', () => { }); test('a span carries its checkpoints and its outcome', async () => { - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); const span = scope.start('lk.connect', { attributes: { 'lk.connect.attempt': 1 } }); span.step('ws_open'); span.step('join_recv'); @@ -146,7 +146,7 @@ describe('telemetry pipeline', () => { test('an SDK nobody asked for telemetry collects nothing', async () => { const idle = new Pipeline(); - const scope = new PipelineScope(idle); + const scope = new TelemetryScope(idle); scope.emit('lk.test.void'); await idle.flush(true); expect(fetchMock).not.toHaveBeenCalled(); @@ -157,7 +157,7 @@ describe('telemetry pipeline', () => { // `registerGlobals` configures the resource long before a connect names the collector. const early = new Pipeline(); early.configure({ encoding: 'json', flushInterval: 3600 }); - const scope = new PipelineScope(early); + const scope = new TelemetryScope(early); scope.emit('lk.test.early'); await early.flush(true); expect(fetchMock).not.toHaveBeenCalled(); @@ -221,83 +221,30 @@ describe('device state', () => { }); }); -describe('the backend seam', () => { - test('a platform backend gets the instrumentation, not the records', () => { - // React Native binds the Rust core through exactly this shape; nothing about a platform - // appears in it, and the Room instrumentation is reused unchanged (TELEMETRY.md §5). - const calls: string[] = []; - const span: Span = { - step: (name) => calls.push(`step ${name}`), - setAttribute: (key, value) => calls.push(`attribute ${key}=${String(value)}`), - end: (outcome) => calls.push(`end ${outcome}`), - fail: () => calls.push('fail'), - cancel: () => calls.push('cancel'), - context: () => ({ traceId: 'trace', spanId: 'span', traceFlags: 1 }), - }; - const scope: Scope = { - traceId: 'trace', - setRoom: (identity) => calls.push(`room ${identity.name}`), - start: (name) => { - calls.push(`start ${name}`); - return span; - }, - emit: (event) => calls.push(`emit ${event}`), - recordStats: (sid, _kind, direction) => calls.push(`stats ${sid} ${direction}`), - subscribeStarted: (sid) => calls.push(`subscribe ${sid}`), - subscribeEnded: (sid, outcome) => calls.push(`subscribed ${sid} ${outcome}`), - disconnected: (reason) => calls.push(`disconnected ${reason}`), - close: () => calls.push('close'), - }; - const platform: Backend = { - enabled: true, - setServer: () => calls.push('setServer'), - scope: () => scope, - hold: (up) => calls.push(`hold ${up}`), - deviceState: (state) => calls.push(`device ${state.appState}`), - emit: (event) => calls.push(`emit ${event}`), - setCadenceFactor: (factor) => calls.push(`cadence ${factor}`), - flush: async () => {}, - diagnostics: () => 'platform backend', - shutdown: async () => {}, - }; - - const previous = Telemetry.setBackend(platform); - try { - Telemetry.setServer('wss://project.livekit.cloud', 'token'); - const session = Telemetry.scope(); - session.setRoom({ name: 'harness' }); - Telemetry.registerTrack('TR_1', session, 'video', 'outbound'); - Telemetry.hold(true); - const connect = session.start('lk.connect'); - connect.step('ws_open'); - connect.end('ok'); - Telemetry.hold(false); - Telemetry.trackStats('TR_1', 'outbound', { bytes: 1 }); - // What a platform sees and this package cannot: it names the event and the factor itself. - Telemetry.emit('lk.device.thermal.changed', { 'lk.device.thermal.state': 'serious' }); - Telemetry.setCadenceFactor(2); - session.disconnected('client_initiated'); - - expect(calls).toEqual([ - 'setServer', - // The page observes itself and reports to whatever backend is installed. On React Native - // there is no `document`, so nothing is observed here and the native side reports instead. - 'device foreground', - 'room harness', - 'hold true', - 'start lk.connect', - 'step ws_open', - 'end ok', - 'hold false', - 'stats TR_1 outbound', - 'emit lk.device.thermal.changed', - 'cadence 2', - 'disconnected client_initiated', - ]); - expect(Telemetry.diagnostics()).toBe('platform backend'); - } finally { - Telemetry.setBackend(previous); - } +describe('what a platform reports', () => { + test('an event it names and a factor it chose both land', async () => { + // React Native sees heat and power; this package has no vocabulary for either, so the platform + // names the record and sets the number. Nothing mobile appears on this side. + const fetchMock = vi.fn(async () => new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchMock); + const pipeline = new Pipeline(); + pipeline.configure({ + endpoint: 'http://collector.test/v1/logs', + encoding: 'json', + flushInterval: 15, + statsWindow: 15, + }); + + pipeline.emit('lk.device.thermal.changed', { 'lk.device.thermal.state': 'serious' }); + pipeline.setCadenceFactor(2); + await pipeline.flush(true); + + const record = recordsOf(fetchMock.mock.calls[0])[0]; + expect(record.eventName).toBe('lk.device.thermal.changed'); + expect(attributesOf(record)['lk.device.thermal.state']).toBe('serious'); + expect(pipeline.statsWindow).toBe(30); + pipeline.stop(); + vi.unstubAllGlobals(); }); }); @@ -320,7 +267,7 @@ describe('the write-ahead cache', () => { flushInterval: 3600, storage, }); - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); scope.emit('lk.test.offline'); fetchMock.mockRejectedValueOnce(new TypeError('offline')); @@ -359,7 +306,7 @@ describe('the write-ahead cache', () => { const pipeline = new Pipeline(); // No destination: batches pile up in the cache, which is where eviction happens. pipeline.configure({ encoding: 'json', flushInterval: 3600, storage }); - const scope = new PipelineScope(pipeline); + const scope = new TelemetryScope(pipeline); scope.emit('lk.test.one'); await pipeline.flush(true); scope.emit('lk.test.two'); @@ -375,7 +322,7 @@ describe('the write-ahead cache', () => { const storage = new MemoryStorage(4 * 1024 * 1024, 512); const pipeline = new Pipeline(); pipeline.configure({ encoding: 'json', flushInterval: 3600, storage }); - new PipelineScope(pipeline).emit('lk.test.written_as_json'); + new TelemetryScope(pipeline).emit('lk.test.written_as_json'); await pipeline.flush(true); // no destination yet: the batch is cached as JSON // The app upgrades, or simply flips the switch, before the batch ever left. diff --git a/src/telemetry/webrtc.ts b/src/telemetry/webrtc.ts index 9e2436d796..62f2eee849 100644 --- a/src/telemetry/webrtc.ts +++ b/src/telemetry/webrtc.ts @@ -9,7 +9,7 @@ import type { VideoReceiverStats, VideoSenderStats, } from '../room/stats'; -import type { StatsSample } from './backend'; +import type { StatsSample } from './scope'; const seconds = (value: number | undefined): number | undefined => value === undefined ? undefined : value * 1000; From 24ffacfb971a563f34b37b1588d8458cd0aeba76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:51:25 +0200 Subject: [PATCH 13/14] chore(changeset): client telemetry --- .changeset/client-telemetry-pipeline.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/client-telemetry-pipeline.md diff --git a/.changeset/client-telemetry-pipeline.md b/.changeset/client-telemetry-pipeline.md new file mode 100644 index 0000000000..a7ebf08690 --- /dev/null +++ b/.changeset/client-telemetry-pipeline.md @@ -0,0 +1,5 @@ +--- +'livekit-client': minor +--- + +Client telemetry: the SDK can now report its own sessions as OTLP to a collector — `lk.connect` and `lk.reconnect` spans with their checkpoints, `lk.publish` and `lk.subscribe`, `lk.rtc.stats.sample` windows folded from the readings the track monitors already take, `lk.room.disconnected`, and the device state a page can observe. Records are batched write-ahead into a cache and uploaded under a policy that never lets telemetry win over media: one request in flight, holds while connecting, a 429 that keeps its batch, and a per-tick budget for replaying a backlog. Off unless `Telemetry.configure({ endpoint })` is called or the room is on LiveKit Cloud, which derives the route and the token from the connect. See TELEMETRY.md. From 2e8d950045a87cd3f7381b0dfb9106a44a397d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:54:32 +0200 Subject: [PATCH 14/14] fix(telemetry): match what the Rust core puts on the wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three differences the staging run surfaced, comparing what this package sends against what the Swift SDK sends through the core into the same ingest: - otel.event.name (semconv 1.39) was missing. The core writes it beside event_name for backends that do not surface the field — Loki included — so a query written against one SDK was silently empty for the other. - lk.publish carried no lk.track.source: a raw MediaStreamTrack has none before it is published, and the publication that comes back was never consulted. - the session harness had no way to reach LiveKit Cloud; it takes a token now, the way the Swift harness does. Co-Authored-By: Claude Opus 5 (1M context) --- src/room/participant/LocalParticipant.ts | 1 + src/telemetry/index.ts | 2 +- src/telemetry/scope.ts | 8 +++++++- src/telemetry/telemetry.browser.test.ts | 3 ++- src/telemetry/telemetrySetup.ts | 7 +++++++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index ed8513db48..7d5d6b4922 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -791,6 +791,7 @@ export default class LocalParticipant extends Participant { try { const publication = await this.publishOrRepublishTrack(track, options); span?.setAttribute('lk.track.sid', publication.trackSid); + span?.setAttribute('lk.track.source', publication.source); span?.end('ok'); return publication; } catch (error) { diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 47e21260a2..a0bd422974 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -157,7 +157,7 @@ export const Telemetry = { severityNumber: Severity.info, severityText: 'INFO', body: 'lk.ping', - attributes: { 'lk.ping.seq': seq }, + attributes: { 'lk.ping.seq': seq, 'otel.event.name': 'lk.ping' }, droppedAttributesCount: 0, resource: { attributes: {} }, instrumentationScope: { name: 'livekit-telemetry', version: '0.1.0' }, diff --git a/src/telemetry/scope.ts b/src/telemetry/scope.ts index 64047f826a..27b082fbbe 100644 --- a/src/telemetry/scope.ts +++ b/src/telemetry/scope.ts @@ -279,7 +279,13 @@ export class TelemetryScope { severityText: severity.toUpperCase(), // Log viewers key their line on the body, and not every backend surfaces event_name yet. body: eventName, - attributes: defined({ ...this.attributes(), ...attributes }), + attributes: defined({ + ...this.attributes(), + // semconv 1.39, duplicating the record's own event name for backends that do not surface + // the field yet (Loki included) — the Rust core writes it too, so a query works either way. + 'otel.event.name': eventName || undefined, + ...attributes, + }), droppedAttributesCount: 0, resource: { attributes: {} }, instrumentationScope: INSTRUMENTATION_SCOPE, diff --git a/src/telemetry/telemetry.browser.test.ts b/src/telemetry/telemetry.browser.test.ts index 98f2d9facb..4b33111f3b 100644 --- a/src/telemetry/telemetry.browser.test.ts +++ b/src/telemetry/telemetry.browser.test.ts @@ -10,7 +10,7 @@ import { RoomEvent } from '../room/events'; * * pnpm vitest run --config vitest.telemetry.config.mts */ -const { url, endpoint, publisherToken, subscriberToken } = inject('telemetry'); +const { url, endpoint, headers, publisherToken, subscriberToken } = inject('telemetry'); async function poll(what: string, condition: () => boolean, timeout = 20_000) { const deadline = Date.now() + timeout; @@ -26,6 +26,7 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); test('a session reports itself: connect span, stats windows, disconnect', async () => { Telemetry.configure({ endpoint, + headers, flushInterval: 1, statsWindow: 2, resource: { diff --git a/src/telemetry/telemetrySetup.ts b/src/telemetry/telemetrySetup.ts index bc21d61065..7910c4fcfa 100644 --- a/src/telemetry/telemetrySetup.ts +++ b/src/telemetry/telemetrySetup.ts @@ -8,14 +8,20 @@ import { createToken } from '../test/signalToken'; * * livekit-server --dev * otelcol-contrib --config src/telemetry/otelcol-web.yaml + * + * `LK_TELEMETRY_ENDPOINT` and `LK_TELEMETRY_TOKEN` point the same session at LiveKit Cloud + * instead (`https:///observability/client/logs/otlp/v0`, a token with an + * `observability:write` grant) — how the pipeline gets exercised against the real ingest. */ export default async function setup({ provide }: TestProject) { const url = process.env.LK_URL ?? 'ws://127.0.0.1:7880'; const endpoint = process.env.LK_TELEMETRY_ENDPOINT ?? 'http://127.0.0.1:4320/v1/logs'; const room = `telemetry-${Date.now()}`; + const token = process.env.LK_TELEMETRY_TOKEN; provide('telemetry', { url, endpoint, + headers: token ? { Authorization: `Bearer ${token}` } : {}, room, publisherToken: await createToken({ room, identity: 'publisher' }), subscriberToken: await createToken({ room, identity: 'subscriber' }), @@ -27,6 +33,7 @@ declare module 'vitest' { telemetry: { url: string; endpoint: string; + headers: Record; room: string; publisherToken: string; subscriberToken: string;