From 757f94af35d73995be7393d9afa07af1bb3f3d10 Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Thu, 13 Aug 2026 09:36:41 +0000
Subject: [PATCH 1/8] feat(abot): add TurboServe-style interactive serving
baseline
---
docs/en/abot_world.md | 107 ++-
docs/en/stream_server.md | 47 +-
examples/abot_world/README.md | 32 +-
examples/abot_world/_loader.py | 18 +-
.../abot_world/abot_world_interactive_web.py | 28 +-
.../abot_world/abot_world_livekit_service.py | 21 +-
.../summary.md | 39 +
.../summary.md | 21 +
.../summary.md | 18 +
.../abot_batched_lf3_4gpu_20260813/summary.md | 31 +
.../summary.md | 45 +
.../summary.md | 31 +
.../summary.md | 20 +
.../summary.md | 24 +
telefuser/entrypoints/cli/main.py | 24 +-
telefuser/models/taew2_2.py | 478 ++++++++++
telefuser/models/wan22_video_vae.py | 116 ++-
telefuser/orchestrator/__init__.py | 2 +
telefuser/orchestrator/batched_stage_actor.py | 266 ++++++
telefuser/pipelines/abot_world/denoising.py | 21 +-
telefuser/pipelines/abot_world/interactive.py | 572 +++++++++--
telefuser/pipelines/abot_world/pipeline.py | 4 +-
telefuser/pipelines/abot_world/service.py | 888 +++++++++++++++---
telefuser/pipelines/abot_world/taew_vae.py | 52 +
telefuser/pipelines/wan_video/vae.py | 23 +
.../service/core/stream_pipeline_service.py | 25 +-
telefuser/service/livekit/__init__.py | 17 +-
telefuser/service/livekit/config.py | 42 +-
telefuser/service/livekit/main.py | 12 +
.../service/livekit/multi_session_worker.py | 5 +-
.../livekit/nccl_process_worker_pool.py | 371 ++++++++
telefuser/service/livekit/nccl_transfer.py | 86 ++
telefuser/service/livekit/pipeline_adapter.py | 21 +-
telefuser/service/livekit/pipeline_router.py | 257 +++++
.../service/livekit/process_worker_pool.py | 639 +++++++++++++
telefuser/service/livekit/runtime.py | 408 +++++++-
telefuser/service/livekit/scheduler.py | 73 +-
telefuser/service/livekit/turboserve.py | 662 +++++++++++++
telefuser/service/livekit/worker_pool.py | 115 ++-
.../models/test_wan22_vae_streaming_state.py | 71 ++
.../orchestrator/test_batched_stage_actor.py | 88 ++
.../pipelines/abot_world/test_interactive.py | 70 +-
.../abot_world/test_interactive_web.py | 11 +
.../abot_world/test_livekit_examples.py | 15 +-
.../abot_world/test_livekit_service.py | 413 +++++---
.../pipelines/abot_world/test_migration.py | 65 ++
tests/unit/service/livekit/test_cli.py | 11 +
.../livekit/test_multi_session_worker.py | 11 +-
.../service/livekit/test_nccl_transfer.py | 42 +
.../service/livekit/test_pipeline_router.py | 136 +++
.../livekit/test_process_worker_failures.py | 169 ++++
.../livekit/test_process_worker_pool.py | 239 +++++
tests/unit/service/livekit/test_runtime.py | 195 ++++
tests/unit/service/livekit/test_scheduler.py | 46 +
tests/unit/service/livekit/test_turboserve.py | 85 ++
.../unit/service/livekit/test_worker_pool.py | 42 +
tools/validation/benchmark_abot_microbatch.py | 187 ++++
tools/validation/benchmark_abot_turboserve.py | 124 +++
.../benchmark_abot_turboserve_concurrent.py | 265 ++++++
tools/validation/run_abot_batch_scaling.py | 264 ++++++
.../validate_abot_nccl_migration.py | 182 ++++
61 files changed, 7912 insertions(+), 480 deletions(-)
create mode 100644 results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md
create mode 100644 results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md
create mode 100644 results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md
create mode 100644 results/experiments/abot_batched_lf3_4gpu_20260813/summary.md
create mode 100644 results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md
create mode 100644 results/experiments/abot_h100_microbatch_lf3_20260813/summary.md
create mode 100644 results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md
create mode 100644 results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md
create mode 100644 telefuser/models/taew2_2.py
create mode 100644 telefuser/orchestrator/batched_stage_actor.py
create mode 100644 telefuser/pipelines/abot_world/taew_vae.py
create mode 100644 telefuser/service/livekit/nccl_process_worker_pool.py
create mode 100644 telefuser/service/livekit/nccl_transfer.py
create mode 100644 telefuser/service/livekit/pipeline_router.py
create mode 100644 telefuser/service/livekit/process_worker_pool.py
create mode 100644 telefuser/service/livekit/turboserve.py
create mode 100644 tests/unit/models/test_wan22_vae_streaming_state.py
create mode 100644 tests/unit/orchestrator/test_batched_stage_actor.py
create mode 100644 tests/unit/pipelines/abot_world/test_migration.py
create mode 100644 tests/unit/service/livekit/test_nccl_transfer.py
create mode 100644 tests/unit/service/livekit/test_pipeline_router.py
create mode 100644 tests/unit/service/livekit/test_process_worker_failures.py
create mode 100644 tests/unit/service/livekit/test_process_worker_pool.py
create mode 100644 tests/unit/service/livekit/test_turboserve.py
create mode 100644 tools/validation/benchmark_abot_microbatch.py
create mode 100644 tools/validation/benchmark_abot_turboserve.py
create mode 100644 tools/validation/benchmark_abot_turboserve_concurrent.py
create mode 100644 tools/validation/run_abot_batch_scaling.py
create mode 100644 tools/validation/validate_abot_nccl_migration.py
diff --git a/docs/en/abot_world.md b/docs/en/abot_world.md
index 12a62632..ed1e487c 100644
--- a/docs/en/abot_world.md
+++ b/docs/en/abot_world.md
@@ -1,7 +1,9 @@
# ABot-World-0-5B-LF
-TeleFuser provides a single-GPU integration for the public ABot-World-0-5B-LF
-long-forcing checkpoint. There are two supported transport entry points:
+TeleFuser provides a TurboServe-style concurrent integration for the public
+ABot-World-0-5B-LF long-forcing checkpoint. Each model replica remains single-GPU,
+while a LiveKit deployment can run one replica per configured GPU and continuously
+batch compatible retained sessions. There are two supported transport entry points:
* Native HTTP controller for local debugging:
@@ -71,10 +73,21 @@ VAE and text stages plus the model-specific `ABotWorldDenoisingStage`.
`ABotWorldDiT` uses the public TeleFuser attention operations and the official
four-step x0-prediction causal sampler.
-`ABotWorldInteractivePipeline` retains the prompt embedding, initial image
-latent, self/cross KV caches, scheduler, RNG, and VAE temporal cache between
-control blocks. The initial integration supports one GPU and one retained
-causal session.
+`ABotWorldInteractivePipeline` retains the prompt embedding, initial-image
+latent, self/cross KV caches, scheduler, RNG, and VAE temporal cache per session.
+The scheduler owns one GPU execution thread per replica and batches compatible
+sessions through DiT and cached VAE decode. Per-session RNG draws, KV/VAE cache
+scatter, relative RoPE positions, and chunk counters remain isolated.
+
+The service performs a real one-session warmup before admission, separates retained
+state from temporary workspace memory, and applies the resulting capacity ceiling. The
+planner treats the measured one-item workspace as batch-scaled: for each candidate
+capacity it budgets every retained session plus `min(candidate, max_batch_size)`
+workspace items under a 10% free-memory reserve. This avoids advertising a capacity
+that is safe only for batch size one.
+Idle state can be suspended to CPU. A two-phase chunk-boundary migration snapshot
+contains prompt state, RNG, KV, VAE caches, counters, and ownership epoch; the
+in-process LiveKit router changes model ownership only after target import succeeds.
## Controls And Idle Behavior
@@ -84,9 +97,11 @@ not advance the DiT with an empty action state. A non-empty control snapshot
starts the next three-latent causal block. Releasing all keys stops new model
execution without discarding frames already queued for playback.
-The browser consumes decoded frames in order at 12 FPS. The bounded FIFO
-applies producer backpressure when playback is behind, so normal playback does
-not drop generated blocks.
+The browser consumes decoded frames in order at 12 FPS. Every session has a
+bounded output queue. The default `latest` delivery mode evicts the oldest complete
+video block when a slow client fills that queue and increments drop metrics, keeping
+control latency bounded. `delivery_mode=lossless` instead applies per-session
+scheduling backpressure; it does not block other ready sessions.
## KV And RoPE
@@ -100,7 +115,46 @@ This fixed logical position policy is an intentional difference from the
original non-sink ABot baseline and must be evaluated as part of any future
long-horizon quality claim.
-## Tests
+## Multi-GPU and autoscaling
+
+Assign exactly one numeric GPU ID to each ABot worker. For example, four warm
+replicas with hardware-sized retained-session capacity use:
+
+```bash
+telefuser stream-serve examples/abot_world/abot_world_livekit_service.py \
+ --livekit-url ws://127.0.0.1:7880 \
+ --livekit-api-key devkey --livekit-api-secret secret \
+ --num-workers 4 --worker-gpu-map '0;1;2;3' \
+ --worker-mode process-nccl \
+ --max-sessions-per-worker auto --queue-size 32 \
+ --port 8088 --skip-validation
+```
+
+`process-nccl` is the cross-process TurboServe baseline. The parent keeps each
+LiveKit room, ingress, and egress alive; a source GPU quiesces at a chunk boundary,
+the target GPU receives retained CUDA tensors directly with NCCL P2P, and routing
+changes only after source release and ownership commit. Controls received during
+that window are buffered in the parent and replayed on the committed owner.
+
+It deliberately uses a fixed one-GPU-per-worker NCCL group, so it does not combine
+with process autoscaling. Plain `--worker-mode process` remains independent-replica
+batching and reports `migration_supported: false`; it is not a TurboServe migration
+baseline. In-process migration remains useful for debugging, but stages state via CPU.
+
+For plain `process` mode, optional cold-replica autoscaling starts only the requested
+minimum and scales within the GPUs declared above. For example:
+
+```bash
+ --enable-autoscaling --autoscaling-min-workers 1 \
+ --autoscaling-target-utilization 0.75 \
+ --autoscaling-hysteresis 0.10 \
+ --autoscaling-cooldown-seconds 30 --autoscaling-interval-seconds 5
+```
+
+Because scale-out loads a checkpoint, autoscaling with multiple workers requires
+a non-zero session queue.
+
+## Tests and benchmark
CPU contract tests cover model conversion, sink KV rolling, RoPE boundaries,
session cleanup, idle behavior, FIFO backpressure, and action layout:
@@ -121,12 +175,31 @@ python -m pytest -m "gpu and slow" \
tests/integration/test_abot_world_smoke.py -v -s
```
-The smoke is a generation and cache contract test. It does not establish
-visual quality, prompt fidelity, or parity over an unbounded session.
+The deterministic continuous-batching benchmark runs multiple sessions for
+30 blocks and writes stage/batch latency plus throughput JSON:
+
+```bash
+python tools/validation/benchmark_abot_turboserve.py \
+ --model-root /path/to/ABot-World-0-5B-LF \
+ --image /path/to/initial.png \
+ --sessions 2 --chunks 30 --batch-size 2 \
+ --output /tmp/abot-turboserve.json
+```
+
+The smoke and benchmark are generation, cache-isolation, batching, and ordering
+contract tests. They do not establish visual quality, prompt fidelity, or parity
+over an unbounded session. Every ABot replica remains single-GPU; multi-GPU
+deployments scale with independent replicas rather than tensor parallelism.
-## Scope
+For a service-level workload with bursty arrivals, independent keyboard activity,
+and playback-paced consumers, run:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python tools/validation/benchmark_abot_turboserve_concurrent.py \
+ --model-root /path/to/ABot-World-0-5B-LF --image /path/to/initial.png \
+ --sessions 4 --duration-seconds 12 --arrival-window-seconds 1.5 \
+ --max-batch-size 4 --output /tmp/abot-concurrent.json
+```
-The integration is intentionally single-GPU and advertises one retained causal
-session per worker. Both transports use the same interactive pipeline and
-fixed six-sink/twelve-tail KV policy; LiveKit adds only the shared TeleFuser
-transport and room lifecycle.
+This reports delivered FPS, first-chunk latency, scheduler queue wait, model compute
+time, observed batch-size distribution, and per-session latest-queue drops.
diff --git a/docs/en/stream_server.md b/docs/en/stream_server.md
index 9137cb2b..1b1a73e4 100644
--- a/docs/en/stream_server.md
+++ b/docs/en/stream_server.md
@@ -22,16 +22,16 @@ flowchart LR
C <-->|WebRTC| LK[LiveKit signaling + SFU]
V <-->|WebRTC| LK
API --> A[Registry + admission]
- A --> W[One in-process model worker]
+ A --> W[Model worker pool]
W <-->|one room runner per session| LK
- W --> S[One shared service instance]
+ W --> S[One service instance per worker]
S --> P1[Pipeline session A]
S --> P2[Pipeline session B]
```
| Term | Meaning and ownership |
|---|---|
-| Service process | One `telefuser stream-serve` process containing the HTTP API, registry, admission scheduler, and current in-process worker. |
+| Service process | One `telefuser stream-serve` parent containing the HTTP API, registry, admission scheduler, and either in-process or spawned model workers. |
| Model worker | Loads the pipeline file once, owns one service instance, and accounts for retained-session capacity. |
| Service instance | The single object returned by `get_service()`; model weights and its pipeline actor graph are loaded once. |
| HTTP session | TeleFuser's public admission and lifecycle record. It maps one-to-one to a room name and, after admission, a room runner. |
@@ -39,9 +39,16 @@ flowchart LR
| Pipeline session | Per-user state returned by `BidirectionalService.create_session()`, such as control, noise, VAE, and model-cache state. |
| Stage actor | An internal pipeline execution owner. It is not the model worker that owns retained-session capacity. |
-The current runtime supports exactly one `in-process` model worker and calls `get_service()` once. Multiple users do
-not load multiple model replicas. Additional replicas require separate `stream-serve` processes and external
-request routing; their registries, queues, health, and session state are independent.
+`worker_mode=in-process` loads every configured replica in the API process. `worker_mode=process` uses the
+multiprocessing `spawn` context and loads exactly one service instance in each model-worker process. The parent
+process remains model-free and owns admission, health, and lifecycle state; capacity and session events cross a
+IPC control plane rather than executing model work on the API event loop. Each process worker retains and
+batches its own sessions, so one slow or failed GPU worker does not serialize the other GPU workers.
+
+`worker_mode=process-nccl` keeps the LiveKit room transport in the parent and puts only the model session in
+fixed, one-GPU child processes. At a chunk boundary it moves retained model tensors with NCCL point-to-point
+operations, then switches the parent route after the ownership commit. Plain `process` mode continues to report
+`migration_supported=false`.
## Service contracts and capacity
@@ -355,8 +362,8 @@ Use `telefuser stream-serve --help` for the complete option list. The options wi
| Option | Default | Semantics |
|---|---:|---|
| `--host`, `--port` | `0.0.0.0`, `8088` | HTTP bind address |
-| `--num-workers` | `1` | Must remain `1` in the current runtime |
-| `--worker-gpu-map` | unset | One logical GPU group for the current worker, for example `0,1,2,3` |
+| `--num-workers` | `1` | Number of model replicas; process mode starts one child per worker |
+| `--worker-gpu-map` | unset | Semicolon-separated GPU group per worker, for example `0;1;2;3` |
| `--max-sessions-per-worker` | `auto` | Hardware-calculated retained sessions; an integer is a safety ceiling |
| `--queue-size` | `0` | HTTP admission FIFO length; zero rejects at capacity |
| `--control-idle-timeout` | `10` | LingBot lease idle threshold when another session waits |
@@ -364,7 +371,7 @@ Use `telefuser stream-serve --help` for the complete option list. The options wi
| `--token-ttl` | `3600` | Join-token lifetime |
| `--controller-timeout` | `60` | Reserved; not currently enforced |
| `--room-empty-timeout` | `30` | Reserved; not currently enforced |
-| `--worker-mode` | `in-process` | `process` is accepted by the CLI but not implemented by the runtime |
+| `--worker-mode` | `in-process` | Use `process` for independent multi-GPU model executors |
The CLI can fall back to `TELEFUSER_LIVEKIT_URL`, `TELEFUSER_LIVEKIT_API_KEY`,
`TELEFUSER_LIVEKIT_API_SECRET`, `TELEFUSER_LIVEKIT_WORKER_GPU_MAP`,
@@ -376,24 +383,30 @@ value is unset. Environment-only settings include `TELEFUSER_LIVEKIT_DEFAULT_FPS
Other Click options currently pass their displayed defaults explicitly, so use the CLI option rather than a
same-named environment variable for those fields.
-In the current in-process runtime, `worker_gpu_map` records scheduler topology and its group size becomes the
-`gpu_num` passed to `get_service()`. It does not set `CUDA_VISIBLE_DEVICES`, isolate devices, or rewrite
-`ModelRuntimeConfig`. Select physical GPUs with `CUDA_VISIBLE_DEVICES` and ensure that the pipeline uses the
-corresponding process-local device indices. For example:
+In process mode every worker group is passed to the child pipeline as explicit device IDs. Multiple process workers
+require `worker_gpu_map`; duplicate GPU IDs are rejected before models load. Process isolation separates Python,
+asyncio, CUDA contexts, and model executors, but it does not rewrite `CUDA_VISIBLE_DEVICES`. IDs in the map are
+logical within the parent's visible-device set. For four one-GPU worker processes on physical GPUs 4-7, use:
```bash
CUDA_VISIBLE_DEVICES=4,5,6,7 \
-telefuser stream-serve PIPE_PATH --worker-gpu-map 0,1,2,3
+telefuser stream-serve PIPE_PATH \
+ --num-workers 4 \
+ --worker-gpu-map '0;1;2;3' \
+ --worker-mode process
```
-This exposes physical GPUs 4-7 as local devices 0-3 and passes `gpu_num=4`; it still loads one service instance.
+This exposes physical GPUs 4-7 as local devices 0-3 and loads four independent service instances. A group may
+contain multiple device IDs for a model replica that itself uses tensor, sequence, or pipeline parallelism. In
+`in-process` mode the same map still binds adapters explicitly, but all replicas share the API process and its
+Python runtime.
## Observability
| Signal | Exact interpretation |
|---|---|
-| `workers_busy` | Model workers retaining at least one session; with the current runtime this is `0` or `1`. |
-| `workers_idle` | Non-failed model workers retaining no sessions. |
+| `workers_busy` | Model workers retaining at least one session. |
+| `workers_idle` | Active model workers in the `idle` state; stopped autoscaling replicas are excluded. |
| `workers_failed` | Workers whose aggregate state is failed. |
| `queued_sessions` | HTTP admission queue depth only; it excludes LingBot lease and pipeline artifact waits. |
| `livekit_connected` | Derived from aggregate worker status being `starting_pipeline`, `running`, or `draining`; it is not a direct LiveKit server probe. |
diff --git a/examples/abot_world/README.md b/examples/abot_world/README.md
index 17c88234..2234edb2 100644
--- a/examples/abot_world/README.md
+++ b/examples/abot_world/README.md
@@ -1,7 +1,7 @@
# ABot-World-0-5B-LF
-This example exposes a local single-GPU HTTP entry point and a LiveKit entry
-point. The HTTP controller is useful for model debugging:
+This example exposes a local single-GPU HTTP entry point and a concurrent
+TurboServe-style LiveKit entry point. The HTTP controller is useful for model debugging:
```bash
python examples/abot_world/abot_world_interactive_web.py \
@@ -12,8 +12,9 @@ python examples/abot_world/abot_world_interactive_web.py \
The browser controls WASD/arrow movement and IJKL camera rotation. Connecting
creates the image-conditioned causal session but does not advance the DiT
-until a non-empty control state is received. Generated blocks remain ordered
-in a bounded FIFO and the producer waits when the browser is behind.
+until a non-empty control state is received. Generated blocks remain ordered in a bounded per-session queue. The default
+`latest` mode drops the oldest complete block, with metrics, only when a slow
+browser fills the queue; `lossless` mode applies scheduling backpressure instead.
The six sink latents and rolling tail use fixed logical RoPE positions, so the
global session frame number does not index beyond the trained local window.
@@ -48,6 +49,15 @@ telefuser stream-serve examples/abot_world/abot_world_livekit_service.py \
--port 8088 --skip-validation
```
+For multiple GPUs, use one worker per GPU, for example
+`--num-workers 4 --worker-gpu-map '0;1;2;3' --worker-mode process-nccl`. This mode loads each
+GPU replica in a spawned child so Python, asyncio, CUDA contexts, and model execution are isolated across GPUs.
+It keeps room transport in the parent and enables NCCL session migration; its NCCL group is fixed, so do not
+enable process autoscaling. Use plain `--worker-mode process` plus a non-zero queue and
+`--enable-autoscaling --autoscaling-min-workers 1` for on-demand independent replicas.
+Each worker continuously batches compatible retained sessions through both DiT
+and cached VAE decode; GPU IDs are passed explicitly to the ABot model factory.
+
Serve the reused browser page in another terminal:
```bash
@@ -79,7 +89,13 @@ ABOT_WORLD_TEST_IMAGE=/path/to/initial.png \
pytest -m "gpu and slow" tests/integration/test_abot_world_smoke.py -v
```
-The smoke uses the public 480x832 shape, a fixed seed, and a fixed control
-state. It checks that every block decodes frames and that the session's
-emitted-frame counter matches the observed count. It is a generation contract
-test, not a visual-quality or long-horizon parity claim.
+The multi-session benchmark exercises 30 continuously batched blocks:
+
+```bash
+python tools/validation/benchmark_abot_turboserve.py \
+ --model-root /path/to/ABot-World-0-5B-LF --image /path/to/initial.png \
+ --sessions 2 --chunks 30 --batch-size 2 --output /tmp/abot-turboserve.json
+```
+
+The smoke and benchmark check generation, session-state isolation, block ordering,
+and batching; they are not visual-quality or long-horizon parity claims.
diff --git a/examples/abot_world/_loader.py b/examples/abot_world/_loader.py
index 3cc3e2cc..a77d9c42 100644
--- a/examples/abot_world/_loader.py
+++ b/examples/abot_world/_loader.py
@@ -16,6 +16,7 @@
)
from telefuser.core.module_manager import ModuleManager
from telefuser.models.abot_world_dit import ABotWorldDiT
+from telefuser.models.taew2_2 import TAEHV
from telefuser.models.wan22_video_vae import Wan22VideoVAE
from telefuser.models.wan_video_text_encoder import WanTextEncoder
from telefuser.ops.attention.backends import FLASH_ATTN_3_AVAILABLE, FLASH_ATTN_4_AVAILABLE
@@ -42,11 +43,12 @@ def get_pipeline(
height: int = 480,
width: int = 832,
latent_frames: int = 31,
+ device_id: int = 0,
pipeline_class: type[ABotWorldPipeline] = ABotWorldPipeline,
) -> ABotWorldPipeline:
"""Load the downloaded ABot checkpoint with VAE/T5 model CPU offload."""
root = Path(model_root).expanduser()
- required = ("diffusion_pytorch_model.safetensors", "Wan2.2_VAE.pth", "models_t5_umt5-xxl-enc-bf16.pth")
+ required = ("diffusion_pytorch_model.safetensors", "Wan2.2_VAE.pth", "taew2_2.pth", "models_t5_umt5-xxl-enc-bf16.pth")
missing = [name for name in required if not (root / name).is_file()]
if missing:
raise FileNotFoundError(f"ABot model root {root} is missing: {', '.join(missing)}")
@@ -59,6 +61,11 @@ def get_pipeline(
torch_dtype=torch.float32,
low_cpu_mem_usage=True,
)
+ model_manager.add_module(
+ TAEHV(str(root / "taew2_2.pth")).eval().requires_grad_(False),
+ name="abot_world_taew_decoder",
+ path=str(root / "taew2_2.pth"),
+ )
model_manager.load_model(
str(root / "models_t5_umt5-xxl-enc-bf16.pth"),
name="wan_video_text_encoder",
@@ -75,19 +82,20 @@ def get_pipeline(
)
cpu_offload = OffloadConfig(offload_type=WeightOffloadType.MODEL_CPU_OFFLOAD)
- pipeline = pipeline_class(device="cuda", torch_dtype=torch.bfloat16)
+ device = f"cuda:{device_id}"
+ pipeline = pipeline_class(device=device, torch_dtype=torch.bfloat16)
pipeline.init(
model_manager,
ABotWorldPipelineConfig(
vae_config=ModelRuntimeConfig(
- device_type="cuda", device_id=0, torch_dtype=torch.float32, offload_config=cpu_offload
+ device_type="cuda", device_id=device_id, torch_dtype=torch.float32, offload_config=cpu_offload
),
text_encoding_config=ModelRuntimeConfig(
- device_type="cuda", device_id=0, torch_dtype=torch.bfloat16, offload_config=cpu_offload
+ device_type="cuda", device_id=device_id, torch_dtype=torch.bfloat16, offload_config=cpu_offload
),
dit_config=ModelRuntimeConfig(
device_type="cuda",
- device_id=0,
+ device_id=device_id,
torch_dtype=torch.bfloat16,
attention_config=AttentionConfig.dense_attention(_attention_backend()),
),
diff --git a/examples/abot_world/abot_world_interactive_web.py b/examples/abot_world/abot_world_interactive_web.py
index a977f666..74d63577 100644
--- a/examples/abot_world/abot_world_interactive_web.py
+++ b/examples/abot_world/abot_world_interactive_web.py
@@ -49,8 +49,8 @@ def __init__(
control_latent_frames: int,
output_queue_size: int = _DEFAULT_OUTPUT_QUEUE_SIZE,
) -> None:
- if control_latent_frames not in {1, 3}:
- raise ValueError("control_latent_frames must be 1 or 3")
+ if control_latent_frames not in {1, 2, 3}:
+ raise ValueError("control_latent_frames must be 1, 2, or 3")
if output_queue_size <= 0:
raise ValueError("output_queue_size must be positive")
self.pipeline = pipeline
@@ -154,7 +154,7 @@ def _enqueue_video_output(self, payload: dict[str, Any]) -> bool:
This is deliberate backpressure: normal streaming never discards a
generated ABot block merely because the browser is temporarily ahead
- of its 12 FPS playback clock.
+ of its configured playback clock.
"""
blocked_started_at: float | None = None
while not self._stop_event.is_set():
@@ -431,7 +431,7 @@ def video_bytes(self) -> bytes | None:
↓
- Hold a key or mouse button to light it. Release it to stop that action. While a control is held, the background producer fills a lossless bounded FIFO. With no control held, no idle chunk is sent. The browser waits for at least 12 predecoded frames, then consumes them in order at 12 FPS. A full FIFO applies producer backpressure; normal playback never drops generated frames.
+ Hold a key or mouse button to light it. Release it to stop that action. While a control is held, the background producer fills a lossless bounded FIFO. With no control held, no idle chunk is sent. The browser waits for one configured playback second of predecoded frames, then consumes them in order at the configured FPS. A full FIFO applies producer backpressure; normal playback never drops generated frames.
Download stopped session video
@@ -441,9 +441,10 @@ def video_bytes(self) -> bytes | None:
const DEFAULT_PROMPT = __DEFAULT_PROMPT__;
const pressedControls = new Set();
const keyToControl = { ArrowUp:"w", ArrowDown:"s", ArrowLeft:"a", ArrowRight:"d", KeyW:"w", KeyA:"a", KeyS:"s", KeyD:"d", KeyI:"i", KeyJ:"j", KeyK:"k", KeyL:"l" };
-const FRAME_INTERVAL_MS = 1000 / 12;
-const PLAYBACK_JITTER_BUFFER_FRAMES = 12;
-const MAX_CLIENT_BUFFERED_FRAMES = 18;
+const PLAYBACK_FPS = __PLAYBACK_FPS__;
+const FRAME_INTERVAL_MS = 1000 / PLAYBACK_FPS;
+const PLAYBACK_JITTER_BUFFER_FRAMES = PLAYBACK_FPS;
+const MAX_CLIENT_BUFFERED_FRAMES = 2 * PLAYBACK_FPS;
const playbackQueue = [];
let running = false;
let requestInFlight = false;
@@ -696,10 +697,11 @@ def video_bytes(self) -> bytes | None:
"""
-def _render_html() -> bytes:
+def _render_html(runtime: InteractiveRuntime) -> bytes:
return (
_HTML.replace("__DEFAULT_IMAGE_PATH__", json.dumps(str(_OFFICIAL_SAMPLE)))
.replace("__DEFAULT_PROMPT__", json.dumps(DEFAULT_PROMPT))
+ .replace("__PLAYBACK_FPS__", json.dumps(runtime.fps))
.encode("utf-8")
)
@@ -729,7 +731,7 @@ def _read_json(self) -> dict[str, Any]:
def do_GET(self) -> None: # noqa: N802
path = urlparse(self.path).path
if path == "/":
- self._send(HTTPStatus.OK, _render_html(), "text/html; charset=utf-8")
+ self._send(HTTPStatus.OK, _render_html(runtime), "text/html; charset=utf-8")
elif path == "/sample-image":
self._send(HTTPStatus.OK, _OFFICIAL_SAMPLE.read_bytes(), "image/jpeg")
elif path.startswith("/api/block-frame/"):
@@ -804,13 +806,13 @@ def main() -> None:
parser.add_argument("--height", type=int, default=480)
parser.add_argument("--width", type=int, default=832)
parser.add_argument("--latent-frames", type=int, default=31)
- parser.add_argument("--fps", type=int, default=12)
+ parser.add_argument("--fps", type=int, default=8, help="Playback and downloaded-video FPS; 8 is the real-time target.")
parser.add_argument(
"--control-latent-frames",
type=int,
- choices=(1, 3),
- default=3,
- help="Causal latents per control update: 3 matches the official ABot streaming checkpoint; 1 is experimental.",
+ choices=(1, 2, 3),
+ default=2,
+ help="Causal latents per control update: 3 matches the official ABot streaming checkpoint; 2 is the 8-FPS experimental target and 1 is experimental.",
)
parser.add_argument("--output-queue-size", type=int, default=_DEFAULT_OUTPUT_QUEUE_SIZE)
parser.add_argument("--host", default="127.0.0.1")
diff --git a/examples/abot_world/abot_world_livekit_service.py b/examples/abot_world/abot_world_livekit_service.py
index 5a647bd5..a5a3fc6b 100644
--- a/examples/abot_world/abot_world_livekit_service.py
+++ b/examples/abot_world/abot_world_livekit_service.py
@@ -22,19 +22,24 @@
get_pipeline = _LOADER.get_pipeline
-def get_service(gpu_num: int = 1) -> ABotWorldLiveKitService:
- """Load one ABot model copy for the shared TeleFuser LiveKit worker."""
- if gpu_num != 1:
- raise ValueError("ABot-World-0-5B-LF currently supports exactly one GPU")
- pipeline = get_pipeline(pipeline_class=ABotWorldInteractivePipeline)
+def get_service(gpu_num: int = 1, gpu_ids: list[str] | None = None) -> ABotWorldLiveKitService:
+ """Load one ABot replica on the single GPU assigned to this worker."""
+ assigned = list(gpu_ids) if gpu_ids else ["0"]
+ if gpu_num != 1 or len(assigned) != 1:
+ raise ValueError("Each ABot worker owns exactly one GPU; use multiple workers for multiple GPUs")
+ try:
+ device_id = int(assigned[0])
+ except ValueError as exc:
+ raise ValueError(f"ABot worker GPU id must be numeric, got {assigned[0]!r}") from exc
+ pipeline = get_pipeline(device_id=device_id, pipeline_class=ABotWorldInteractivePipeline)
return ABotWorldLiveKitService(
pipeline,
- default_fps=12,
+ default_fps=8,
default_session_config={
"image_path": str(_DEFAULT_IMAGE_PATH),
"prompt": DEFAULT_PROMPT,
- "fps": 12,
- "control_latent_frames": 3,
+ "fps": 8,
+ "control_latent_frames": 2,
"seed": 42,
},
)
diff --git a/results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md b/results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md
new file mode 100644
index 00000000..a7c5a750
--- /dev/null
+++ b/results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md
@@ -0,0 +1,39 @@
+# ABot-World four-GPU concurrent-user baseline (LF=3)
+
+Date: 2026-08-13. GPUs 4--7 are four independent single-GPU service replicas;
+this is a per-replica capacity baseline, not a global multi-GPU TurboServe result.
+
+## Fixed workload
+
+- Model: `ABot-World-0-5B-LF`; input: `84b90ad568b693d2.png` at the default 832x480.
+- `control_latent_frames=3` (the original ABot-World streaming setting).
+- Four replicas, continuous active controls every 0.3 s, 30 s per run, no idle periods.
+- Consumer pulls immediately (`consumer_playback_fps=0`), lossless delivery, batch window 2 ms,
+ and `max_batch_size=4`. Reported FPS is consumer-visible end-to-end FPS in the local
+ service harness; it excludes browser/WebRTC encode and network transport.
+
+| Users / GPU | Total users | Per-user FPS | Aggregate / GPU | Approx. cluster FPS | Mean batch | Mean compute / batch (s) | Mean queue wait (ms) | Mean first frame (s) | Outcome |
+|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|
+| 1 | 4 | 14.361 | 14.361 | 57.444 | 1.000 | 0.811 | 0.000 | 0.664 | admitted |
+| 2 | 8 | 7.189 | 14.378 | 57.510 | 1.649 | 1.304 | 0.065 | 1.067 | admitted |
+| 3 | 12 | 4.715 | 14.144 | 56.576 | 2.243 | 1.722 | 1.957 | 1.425 | admitted; each H100 reached about 81 GB during the run |
+| 4 | 16 | -- | -- | -- | -- | -- | -- | -- | rejected: `capacity=3` |
+
+`users_per_gpu_4` does not produce JSON because the fourth retained session is rejected by
+the service's admission controller (`ABot retained-session capacity is exhausted (capacity=3)`).
+The consumer-close timeout subsequently printed by the harness is a cleanup artifact, not a
+model-inference latency measurement.
+
+## Interpretation
+
+For LF=3, the single-user result is 14.24--14.51 FPS across the four cards (mean 14.36),
+consistent with the previously matched direct single-GPU result (about 15 FPS). Increasing
+the number of active sessions does form batches, but raises batch compute time almost
+proportionally, leaving per-GPU throughput flat at about 14.1--14.4 FPS. Thus the current
+baseline's limiting factor in this workload is model/state memory and batched compute scaling,
+not scheduler queueing. This is a useful pre-experiment gap for a workload-aware world-model
+scheduler: it should avoid admitting a fourth retained LF=3 state locally and should use global
+placement/migration or state offload rather than merely increasing the local batch.
+
+Raw files: `users_per_gpu_{1,2,3}/gpu{4,5,6,7}.json`; logs, including the four admission
+rejections, are co-located in `users_per_gpu_4/`.
diff --git a/results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md b/results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md
new file mode 100644
index 00000000..e9ddc5c4
--- /dev/null
+++ b/results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md
@@ -0,0 +1,21 @@
+# ABot steady-state batch scaling: one latent control frame
+
+Hardware: one NVIDIA H100 80 GB (GPU 0). The ABot-World-0-5B-LF service was
+preloaded, then each point used continuously active retained sessions, one warmup
+chunk per session, and two measured chunks per session. Values below are from the
+LiveKit service scheduler, not a synthetic model loop.
+
+| Sessions | Batch cap | Observed batch | Aggregate FPS | Per-session FPS | p95 chunk latency (s) | p95 queue wait (s) | Peak allocated GiB | Result |
+|---:|---:|---:|---:|---:|---:|---:|---:|---|
+| 1 | 1 | 1.0 | 11.00 | 11.00 | 0.376 | 0.000 | 39.05 | OK |
+| 2 | 1 | 1.0 | 11.73 | 5.86 | 0.692 | 0.347 | 44.33 | OK |
+| 2 | 2 | 2.0 | 14.31 | 7.16 | 0.572 | 0.000 | 54.89 | OK |
+| 4 | 1 | 1.0 | 11.49 | 2.87 | 1.422 | 1.070 | 54.91 | OK |
+| 4 | 2 | 2.0 | 14.31 | 3.58 | 1.137 | 0.570 | 65.47 | OK |
+| 4 | 4 | -- | -- | -- | -- | -- | -- | OOM in VAE temporal decode (requested 3.81 GiB) |
+
+The valid batch-2 points increase aggregate throughput by 22.0% (two sessions)
+and 24.6% (four sessions) over batch cap 1. However, the four-session latency
+remains above one second and batch 4 is infeasible despite 80 GB device memory.
+
+Raw data: [results.csv](results.csv) and [results.json](results.json).
diff --git a/results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md b/results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md
new file mode 100644
index 00000000..55c56339
--- /dev/null
+++ b/results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md
@@ -0,0 +1,18 @@
+# ABot steady-state batch scaling: three latent control frames
+
+Hardware and warmup are the same as the LF=1 experiment. Each scheduled chunk
+generates three latent frames and was measured through the LiveKit scheduler.
+
+| Sessions | Batch cap | Observed batch | Aggregate FPS | Per-session FPS | p95 chunk latency (s) | p95 queue wait (s) | Peak allocated GiB | Result |
+|---:|---:|---:|---:|---:|---:|---:|---:|---|
+| 1 | 1 | 1.0 | 15.66 | 15.66 | 0.776 | 0.000 | 39.14 | OK |
+| 2 | 1 | 1.0 | 15.54 | 7.77 | 1.546 | 0.774 | 44.42 | OK |
+| 2 | 2 | 2.0 | 16.14 | 8.07 | 1.508 | 0.001 | 55.07 | OK |
+
+Batching two long-control sessions improves aggregate FPS only 3.8% while the
+batch compute time rises from about 0.77 s to 1.51 s. VAE decode consumes about
+20% of each batch, and DiT about 28-32%; the remaining time is cache collation,
+output conversion, and scheduler-side work. This differs sharply from the LF=1
+case and motivates a workload-aware policy rather than a fixed batch cap.
+
+Raw data: [results.csv](results.csv) and [results.json](results.json).
diff --git a/results/experiments/abot_batched_lf3_4gpu_20260813/summary.md b/results/experiments/abot_batched_lf3_4gpu_20260813/summary.md
new file mode 100644
index 00000000..7bbd84f2
--- /dev/null
+++ b/results/experiments/abot_batched_lf3_4gpu_20260813/summary.md
@@ -0,0 +1,31 @@
+# ABot-World explicit cross-session batching experiment (LF=3)
+
+Date: 2026-08-13. This experiment uses four independent single-GPU replicas
+(GPUs 4--7) and **explicitly selects** `scheduler_mode=batched`. It is not
+the TurboServe baseline: TurboServe's per-worker open-source loop is
+single-session round-robin. The purpose here is to evaluate the experimental
+TeleFuser cross-session model-batching path against that baseline.
+
+## Fixed workload
+
+- ABot-World-0-5B-LF, default 832x480 image, `control_latent_frames=3`.
+- Four replicas; simultaneous active clients, control heartbeat every 0.3 s,
+ 30 s run, no idle intervals; immediate consumer and lossless delivery.
+- `max_batch_size=4`, batching window 2 ms. FPS is local consumer-visible
+ end-to-end FPS, excluding browser/WebRTC encode and network transport.
+
+| Users / GPU | Total users | Per-user FPS | Aggregate / GPU | Approx. cluster FPS | Mean observed batch | Mean compute / batch (s) | Mean queue wait (ms) | Mean first frame (s) |
+|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| 1 | 4 | 14.340 | 14.340 | 57.360 | 1.000 | 0.809 | 0.000 | 0.665 |
+| 2 | 8 | 7.188 | 14.376 | 57.504 | 1.649 | 1.304 | 0.065 | 1.067 |
+| 3 | 12 | 4.694 | 14.082 | 56.328 | 2.243 | 1.730 | 1.943 | 1.438 |
+
+Observed batch histograms per GPU were respectively `{1:37}`, `{1:13,2:24}`
+and `{1:9,2:10,3:18}`. Thus batching does form after warmup, but it does not
+produce throughput scaling: batch=2 has a roughly 1.58 s steady batch time,
+close to twice a single-session 0.81 s step. Batch=3 reaches about 1.73 s and
+requires about 81 GB per H100. The bottleneck is therefore the current
+batched execution/state layout, not waiting for the scheduler to collect
+requests.
+
+Raw per-GPU JSON and logs are in `users_per_gpu_{1,2,3}/`.
diff --git a/results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md b/results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md
new file mode 100644
index 00000000..e02977c4
--- /dev/null
+++ b/results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md
@@ -0,0 +1,45 @@
+# ABot-World 8-FPS / two-latent concurrent baseline
+
+## Target and workload
+
+- Model: `ABot-World-0-5B-LF`, real public checkpoint, 832x480.
+- Target: 8 FPS per user; one continuation chunk contains 2 latent frames and decodes to 8 RGB frames.
+- Controls: every active user holds a valid control snapshot; it is refreshed once per second.
+- Consumer metric: lossless consumer displays frames at 8 FPS. `consumer_end_to_end_fps` includes startup and final queued-frame drain, so it is deliberately a user-visible, conservative metric.
+- GPU: one NVIDIA H100 80 GB (physical GPU 4), one model replica.
+- Each run uses 18 seconds of sustained input, no intentional idle intervals, and synchronized session arrivals.
+
+## Results
+
+| Active users | Scheduler | Batch cap | Mean observed batch | Mean displayed FPS/user | p95 compute per scheduled chunk (s) | p95 queue wait (s) | p95 inter-chunk interval (s) | 8-FPS target met? |
+|---:|---|---:|---:|---:|---:|---:|---:|---|
+| 1 | strict round-robin | 1 | 1.00 | 7.736 | 0.608 | 0.089 | 1.003 | Steady-state yes; end-to-end aggregate is conservative |
+| 2 | strict round-robin | 1 | 1.00 | 6.392 | 0.603 | 0.381 | 1.212 | No |
+| 3 | strict round-robin | 1 | 1.00 | 4.262 | 0.602 | 0.387 | 1.802 | No |
+| 2 | coalesced batch | 2 | 1.406 | 6.347 | 1.142 | 0.407 | 1.741 | No |
+
+## Interpretation
+
+A one-user continuation chunk stabilizes at about 0.60 seconds. Strict round-robin therefore needs about 1.20 seconds for two continuously active users and about 1.80 seconds for three, while the playback/control period is one second. This is the primary pre-improvement bottleneck: a per-GPU 8-FPS deadline miss caused by serial session scheduling, not video delivery or dropped frames.
+
+Coalesced batch size two is not a sufficient fix in the current ABot path. Its actual batch-2 compute is about 1.12 seconds, so it too misses the one-second deadline. The result is only a small end-to-end change (6.392 to 6.347 FPS/user) and uses substantially more memory (about 66.5 GiB while loaded in this run). This motivates improving both model-stage batching efficiency and workload-aware placement/admission rather than merely turning on batching.
+
+## Reproduce
+
+```bash
+cd /public/fanyk1/lwb/TeleFuser-abot-world
+
+CUDA_VISIBLE_DEVICES=4 PYTHONPATH=. \
+/public/fanyk1/lwb/envs/telefuser_sage291/bin/python \
+tools/validation/benchmark_abot_turboserve_concurrent.py \
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \
+ --sessions 2 --duration-seconds 18 --arrival-window-seconds 0 \
+ --fps 8 --consumer-playback-fps 8 --control-latent-frames 2 \
+ --scheduler-mode round_robin --max-batch-size 1 --batching-window-ms 0 \
+ --delivery-mode lossless --control-update-min-seconds 1 \
+ --control-update-max-seconds 1 --idle-probability 0 \
+ --output results/experiments/abot_concurrent_8fps_lf2_20260813/sessions_2_round_robin.json
+```
+
+For the batching comparison, change `--scheduler-mode batched --max-batch-size 2 --batching-window-ms 2`.
diff --git a/results/experiments/abot_h100_microbatch_lf3_20260813/summary.md b/results/experiments/abot_h100_microbatch_lf3_20260813/summary.md
new file mode 100644
index 00000000..1b5d8ce5
--- /dev/null
+++ b/results/experiments/abot_h100_microbatch_lf3_20260813/summary.md
@@ -0,0 +1,31 @@
+# Single-H100 ABot-World retained-session microbatch benchmark (LF=3)
+
+Date: 2026-08-13. One NVIDIA H100 80 GB (GPU 4), ABot-World-0-5B-LF,
+832x480, `control_latent_frames=3`.
+
+For each batch size B, the benchmark creates B independent retained sessions.
+It discards the special 9-frame seed chunk, warms three 12-frame continuation
+chunks, then measures eight synchronized calls to
+`generate_next_blocks(B sessions)`. Every timed call generates exactly 12
+frames for every active session. `T(B)` below is the mean timed batch-call
+duration. It excludes the service scheduler, client delivery, browser/WebRTC,
+and initial-session creation.
+
+| Batch B | Chunk Time T(B) | Aggregate FPS = 12B/T(B) | FPS/session = 12/T(B) |
+|---:|---:|---:|---:|
+| 1 | 0.7979 s | 15.04 | 15.04 |
+| 2 | 1.5656 s | 15.33 | 7.66 |
+| 3 | 2.2802 s | 15.79 | 5.26 |
+| 4 | OOM | OOM | OOM |
+
+Measurement variation (standard deviation over eight samples): B=1 6.9 ms,
+B=2 13.3 ms, B=3 23.9 ms. Peak PyTorch allocated memory was 39.1 GiB,
+55.0 GiB, and 71.1 GiB for B=1,2,3 respectively. B=4 failed while attempting
+to allocate a further 4.63 GiB, with only 0.86 GiB free.
+
+The aggregate gain from B=1 to B=3 is only 5.0%, so this ABot implementation's
+current native model batch path is close to linear-time in B. This is a model
+execution/state-layout result, not a TurboServe scheduling artifact.
+
+Raw machine-readable results: `results.json` and `results.csv`. The benchmark
+implementation is `tools/validation/benchmark_abot_microbatch.py`.
diff --git a/results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md b/results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md
new file mode 100644
index 00000000..f2180333
--- /dev/null
+++ b/results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md
@@ -0,0 +1,20 @@
+# ABot-World LF=3 microbatch stage profile (one H100)
+
+The setup matches the synchronous retained-session microbenchmark: B independent
+sessions, 9-frame seed chunk excluded, three continuation warmups, then six
+timed 12-frame continuation batches. Stage times use CUDA events.
+
+| B | End-to-end chunk | DiT denoise | VAE decode | Other state/Python/tensor work | Aggregate FPS |
+|---:|---:|---:|---:|---:|---:|
+| 1 | 807.0 ms | 289.2 ms | 415.9 ms | 101.8 ms | 14.87 |
+| 2 | 1567.3 ms | 505.9 ms | 865.3 ms | 196.0 ms | 15.31 |
+| 3 | 2279.1 ms | 736.9 ms | 1260.1 ms | 282.0 ms | 15.80 |
+
+DiT scales sublinearly (B=3 is 2.55x B=1), demonstrating some GPU batch
+parallelism. VAE decode is nearly linear (B=2: 2.08x, B=3: 3.03x); it is the
+largest stage and is the primary reason aggregate FPS remains nearly flat.
+The remainder also grows near-linearly because the current retained-session
+implementation collates KV/VAE state before a batch and scatters it afterward.
+
+This profile is not a scheduler measurement: it invokes the native batched
+model path directly, after state creation and before any service delivery.
diff --git a/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md b/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md
new file mode 100644
index 00000000..55ab659d
--- /dev/null
+++ b/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md
@@ -0,0 +1,24 @@
+# ABot-World LightVAE LF=3 single-GPU microbatch capacity
+
+## Configuration
+
+- GPU: one NVIDIA H100 80 GiB (CUDA device 5)
+- Checkpoint: `ABot-World-0-5B-LF` with official `taew2_2` lightweight decoder
+- `control_latent_frames=3`; each continuation chunk delivers 12 display frames per independent session
+- 2 warmup chunks, then 5 synchronized steady-state samples per batch size
+- Input: `examples/data/1.png`; fixed prompt and deterministic session seeds
+
+## Results
+
+| Concurrent sessions / DiT batch | Mean chunk time (s) | Aggregate FPS | FPS/session | Peak allocated GiB | 8 FPS deadline (1.5 s) |
+|---:|---:|---:|---:|---:|---:|
+| 1 | 0.387 | 30.98 | 30.98 | 28.91 | pass |
+| 2 | 0.703 | 34.13 | 17.07 | 34.56 | pass |
+| 3 | 1.021 | 35.26 | 11.75 | 40.28 | pass |
+| 4 | 1.297 | 37.01 | 9.25 | 45.95 | pass |
+| 5 | 1.599 | 37.53 | 7.51 | 51.73 | fail |
+| 6 | 1.961 | 36.71 | 6.12 | 57.36 | fail |
+
+The 8 FPS service limit is four simultaneous sessions: B=4 p95 is 1.371 s, while B=5 mean latency already exceeds the 1.5 s deadline. Aggregate throughput saturates at about 37 FPS; this is an SLO limit, not an HBM OOM limit. At B=6, mean DiT time is 1.402 s and LightVAE decode is 0.056 s.
+
+Raw local artifacts (`results.json`, `results.csv`, `run.log`) are intentionally ignored by repository rules.
diff --git a/telefuser/entrypoints/cli/main.py b/telefuser/entrypoints/cli/main.py
index bf69be6d..0a2f40ee 100644
--- a/telefuser/entrypoints/cli/main.py
+++ b/telefuser/entrypoints/cli/main.py
@@ -158,6 +158,14 @@ def serve(
@click.option(
"--queue-size", default=0, type=int, help="Maximum queued sessions; 0 rejects when retained slots are full"
)
+@click.option("--enable-autoscaling", is_flag=True, help="Dynamically load workers from the configured GPU map")
+@click.option("--autoscaling-min-workers", default=1, type=int, help="Initially loaded worker replicas")
+@click.option(
+ "--autoscaling-target-utilization", default=0.75, type=float, help="Target retained-session utilization"
+)
+@click.option("--autoscaling-hysteresis", default=0.10, type=float, help="Scale decision hysteresis band")
+@click.option("--autoscaling-cooldown-seconds", default=30.0, type=float, help="Minimum time between scales")
+@click.option("--autoscaling-interval-seconds", default=5.0, type=float, help="Autoscaling control interval")
@click.option(
"--control-idle-timeout",
default=None,
@@ -170,9 +178,9 @@ def serve(
@click.option("--room-empty-timeout", default=30, type=int, help="Seconds to keep a session after room becomes empty")
@click.option(
"--worker-mode",
- type=click.Choice(["in-process", "process"], case_sensitive=False),
+ type=click.Choice(["in-process", "process", "process-nccl"], case_sensitive=False),
default="in-process",
- help="Worker isolation mode; the current runtime supports in-process only",
+ help="Worker isolation mode; process-nccl keeps LiveKit transport in the parent and migrates model state over NCCL",
)
@click.option(
"--security-level",
@@ -197,6 +205,12 @@ def stream_serve(
max_sessions_per_worker: int | None,
worker_gpu_map: str | None,
queue_size: int,
+ enable_autoscaling: bool,
+ autoscaling_min_workers: int,
+ autoscaling_target_utilization: float,
+ autoscaling_hysteresis: float,
+ autoscaling_cooldown_seconds: float,
+ autoscaling_interval_seconds: float,
control_idle_timeout: float | None,
session_timeout: int,
token_ttl: int,
@@ -235,6 +249,12 @@ def stream_serve(
max_sessions_per_worker=max_sessions_per_worker,
worker_gpu_map=worker_gpu_map,
queue_size=queue_size,
+ autoscaling_enabled=enable_autoscaling,
+ autoscaling_min_workers=autoscaling_min_workers,
+ autoscaling_target_utilization=autoscaling_target_utilization,
+ autoscaling_hysteresis=autoscaling_hysteresis,
+ autoscaling_cooldown_seconds=autoscaling_cooldown_seconds,
+ autoscaling_interval_seconds=autoscaling_interval_seconds,
control_idle_timeout=control_idle_timeout,
session_timeout=session_timeout,
token_ttl=token_ttl,
diff --git a/telefuser/models/taew2_2.py b/telefuser/models/taew2_2.py
new file mode 100644
index 00000000..f7bc1165
--- /dev/null
+++ b/telefuser/models/taew2_2.py
@@ -0,0 +1,478 @@
+"""TAeW2.2 lightweight streaming VAE used by official ABot-World.
+
+Imported from the Apache-2.0 licensed official ABot-World release and kept
+model-local so the ABot integration does not depend on an external package.
+"""
+
+"""
+Tiny AutoEncoder for Hunyuan Video
+(DNN for encoding / decoding videos to Hunyuan Video's latent space)
+"""
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from tqdm.auto import tqdm
+from collections import namedtuple
+
+TWorkItem = namedtuple("TWorkItem", ("input_tensor", "block_index"))
+
+def conv(n_in, n_out, **kwargs):
+ return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs)
+
+class Clamp(nn.Module):
+ def forward(self, x):
+ return torch.tanh(x / 3) * 3
+
+class MemBlock(nn.Module):
+ def __init__(self, n_in, n_out):
+ super().__init__()
+ self.conv = nn.Sequential(conv(n_in * 2, n_out), nn.ReLU(inplace=True), conv(n_out, n_out), nn.ReLU(inplace=True), conv(n_out, n_out))
+ self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
+ self.act = nn.ReLU(inplace=True)
+ def forward(self, x, past):
+ return self.act(self.conv(torch.cat([x, past], 1)) + self.skip(x))
+
+class TPool(nn.Module):
+ def __init__(self, n_f, stride):
+ super().__init__()
+ self.stride = stride
+ self.conv = nn.Conv2d(n_f*stride,n_f, 1, bias=False)
+ def forward(self, x):
+ _NT, C, H, W = x.shape
+ return self.conv(x.reshape(-1, self.stride * C, H, W))
+
+class TGrow(nn.Module):
+ def __init__(self, n_f, stride):
+ super().__init__()
+ self.stride = stride
+ self.conv = nn.Conv2d(n_f, n_f*stride, 1, bias=False)
+ def forward(self, x):
+ _NT, C, H, W = x.shape
+ x = self.conv(x)
+ return x.reshape(-1, C, H, W)
+
+def apply_model_with_memblocks_parallel(model, x, show_progress_bar):
+ """
+ Apply a sequential model with memblocks to the given input,
+ with parallelization over the time axis and iteration over blocks.
+
+ Args:
+ - model: nn.Sequential of blocks to apply
+ - x: input data, of dimensions NTCHW
+ - show_progress_bar: if True, enables tqdm progressbar display
+
+ Returns NTCHW tensor of output data.
+ """
+ assert x.ndim == 5, f"TAEHV operates on NTCHW tensors, but got {x.ndim}-dim tensor"
+ N, T, C, H, W = x.shape
+ x = x.reshape(N*T, C, H, W)
+
+ # parallel over input timesteps, iterate over blocks
+ for b in tqdm(model, disable=not show_progress_bar):
+ if isinstance(b, MemBlock):
+ NT, C, H, W = x.shape
+ T = NT // N
+ _x = x.reshape(N, T, C, H, W)
+ # pad with zeros along time axis (i.e. empty memory), slice
+ block_memory = F.pad(_x, (0,0,0,0,0,0,1,0), value=0)[:,:T].reshape(x.shape)
+ x = b(x, block_memory)
+ else:
+ x = b(x)
+ NT, C, H, W = x.shape
+ T = NT // N
+ return x.view(N, T, C, H, W)
+
+def apply_model_with_memblocks_sequential_single_step(model, memory, work_queue, progress_bar=None):
+ """
+ Process the work queue (a graph traversal over blocks and timesteps)
+ until an output frame is produced or the queue is empty.
+ Mutates memory and work_queue in place.
+
+ Returns N1CHW output tensor, or None if the queue needs more input.
+ """
+ while work_queue:
+ xt, i = work_queue.pop(0)
+ if progress_bar is not None and i == 0:
+ progress_bar.update(1)
+ if i == len(model):
+ return xt.unsqueeze(1)
+ b = model[i]
+ if isinstance(b, MemBlock):
+ # mem blocks are simple since we're visiting the graph in causal order
+ if memory[i] is None:
+ xt_new = b(xt, xt * 0)
+ else:
+ xt_new = b(xt, memory[i])
+ memory[i] = xt
+ work_queue.insert(0, TWorkItem(xt_new, i+1))
+ elif isinstance(b, TPool):
+ # pool blocks accumulate inputs until they have enough to pool
+ if memory[i] is None:
+ memory[i] = []
+ memory[i].append(xt)
+ if len(memory[i]) > b.stride:
+ raise ValueError(f"TPool memory overflow: {len(memory[i])} items for stride {b.stride}")
+ elif len(memory[i]) == b.stride:
+ N, C, H, W = xt.shape
+ xt = b(torch.cat(memory[i], 1).view(N*b.stride, C, H, W))
+ memory[i] = []
+ work_queue.insert(0, TWorkItem(xt, i+1))
+ elif isinstance(b, TGrow):
+ xt = b(xt)
+ NT, C, H, W = xt.shape
+ for xt_next in reversed(xt.view(NT//b.stride, b.stride*C, H, W).chunk(b.stride, 1)):
+ work_queue.insert(0, TWorkItem(xt_next, i+1))
+ else:
+ xt = b(xt)
+ work_queue.insert(0, TWorkItem(xt, i+1))
+ return None
+
+def apply_model_with_memblocks_sequential(model, x, show_progress_bar):
+ """
+ Apply a sequential model with memblocks to the given input,
+ with iteration over timesteps as well as blocks.
+
+ Args:
+ - model: nn.Sequential of blocks to apply
+ - x: input data, of dimensions NTCHW
+ - show_progress_bar: if True, enables tqdm progressbar display
+
+ Returns NTCHW tensor of output data.
+ """
+ assert x.ndim == 5, f"TAEHV operates on NTCHW tensors, but got {x.ndim}-dim tensor"
+ work_queue = [TWorkItem(xt, 0) for xt in x.unbind(1)]
+ memory = [None] * len(model)
+ progress_bar = tqdm(range(len(work_queue)), disable=not show_progress_bar)
+ out = []
+ while work_queue:
+ xt = apply_model_with_memblocks_sequential_single_step(model, memory, work_queue, progress_bar)
+ if xt is not None:
+ out.append(xt)
+ progress_bar.close()
+ return torch.cat(out, 1)
+
+def apply_model_with_memblocks(model, x, parallel, show_progress_bar):
+ """
+ Apply a sequential model with memblocks to the given input.
+ Args:
+ - model: nn.Sequential of blocks to apply
+ - x: input data, of dimensions NTCHW
+ - parallel: if True, parallelize over timesteps (fast but uses O(T) memory)
+ if False, each timestep will be processed sequentially (slow but uses O(1) memory)
+ - show_progress_bar: if True, enables tqdm progressbar display
+
+ Returns NTCHW tensor of output data.
+ """
+ if parallel:
+ return apply_model_with_memblocks_parallel(model, x, show_progress_bar)
+ else:
+ return apply_model_with_memblocks_sequential(model, x, show_progress_bar)
+
+class TAEHV(nn.Module):
+ def __init__(self, checkpoint_path="taehv.pth", encoder_time_downscale=(True, True, False), decoder_time_upscale=(False, True, True), decoder_space_upscale=(True, True, True), patch_size=1, latent_channels=16):
+ """Initialize pretrained TAEHV from the given checkpoint.
+
+ Arg:
+ checkpoint_path: path to weight file to load. taehv.pth for Hunyuan, taew2_1.pth for Wan 2.1.
+ encoder_time_downscale: whether temporal downsampling is enabled for each block.
+ decoder_time_upscale: whether temporal upsampling is enabled for each block. upsampling can be disabled for a cheaper preview.
+ decoder_space_upscale: whether spatial upsampling is enabled for each block. upsampling can be disabled for a cheaper preview.
+ patch_size: input/output pixelshuffle patch-size for this model.
+ latent_channels: number of latent channels (z dim) for this model.
+ """
+ super().__init__()
+ self.patch_size = patch_size
+ self.latent_channels = latent_channels
+ self.image_channels = 3
+ if len(decoder_time_upscale) == 2:
+ decoder_time_upscale = (False, *decoder_time_upscale)
+ self.is_cogvideox = checkpoint_path is not None and "taecvx" in checkpoint_path
+ if checkpoint_path is not None and "taew2_2" in checkpoint_path:
+ self.patch_size, self.latent_channels = 2, 48
+ if checkpoint_path is not None and "taehv1_5" in checkpoint_path:
+ self.patch_size, self.latent_channels = 2, 32
+ if checkpoint_path is not None and "taeltx" in checkpoint_path: # same for both 2 and 2.3
+ self.patch_size, self.latent_channels, encoder_time_downscale, decoder_time_upscale = 4, 128, (True, True, True), (True, True, True)
+ self.encoder = nn.Sequential(
+ conv(self.image_channels*self.patch_size**2, 64), nn.ReLU(inplace=True),
+ TPool(64, 2 if encoder_time_downscale[0] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
+ TPool(64, 2 if encoder_time_downscale[1] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
+ TPool(64, 2 if encoder_time_downscale[2] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
+ conv(64, self.latent_channels),
+ )
+ n_f = [256, 128, 64, 64]
+ self.decoder = nn.Sequential(
+ Clamp(), conv(self.latent_channels, n_f[0]), nn.ReLU(inplace=True),
+ MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), TGrow(n_f[0], 2 if decoder_time_upscale[0] else 1), conv(n_f[0], n_f[1], bias=False),
+ MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1), TGrow(n_f[1], 2 if decoder_time_upscale[1] else 1), conv(n_f[1], n_f[2], bias=False),
+ MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Upsample(scale_factor=2 if decoder_space_upscale[2] else 1), TGrow(n_f[2], 2 if decoder_time_upscale[2] else 1), conv(n_f[2], n_f[3], bias=False),
+ nn.ReLU(inplace=True), conv(n_f[3], self.image_channels*self.patch_size**2),
+ )
+ # computed properties
+ self.t_downscale = 2**sum(t.stride == 2 for t in self.encoder if isinstance(t, TPool))
+ self.t_upscale = 2**sum(t.stride == 2 for t in self.decoder if isinstance(t, TGrow))
+ self.frames_to_trim = self.t_upscale - 1
+
+ if checkpoint_path is not None:
+ self.load_state_dict(self.patch_tgrow_layers(torch.load(checkpoint_path, map_location="cpu", weights_only=True)))
+
+ def patch_tgrow_layers(self, sd):
+ """Patch TGrow layers to use a smaller kernel if needed.
+
+ Args:
+ sd: state dict to patch
+ """
+ new_sd = self.state_dict()
+ for i, layer in enumerate(self.decoder):
+ if isinstance(layer, TGrow):
+ key = f"decoder.{i}.conv.weight"
+ if sd[key].shape[0] > new_sd[key].shape[0]:
+ # take the last-timestep output channels
+ sd[key] = sd[key][-new_sd[key].shape[0]:]
+ return sd
+
+ def preprocess_input_frames(self, x):
+ """Preprocess RGB input frames prior to the main encoder sequence."""
+ if self.patch_size > 1: x = F.pixel_unshuffle(x, self.patch_size)
+ return x
+
+ def encode_video(self, x, parallel=True, show_progress_bar=True):
+ """Encode a sequence of frames.
+
+ Args:
+ x: input NTCHW RGB (C=3) tensor with values in [0, 1].
+ parallel: if True, all frames will be processed at once.
+ (this is faster but may require more memory).
+ if False, frames will be processed sequentially.
+ Returns NTCHW latent tensor with ~Gaussian values.
+ """
+ x = self.preprocess_input_frames(x)
+ if x.shape[1] % self.t_downscale != 0:
+ # pad at end to multiple of self.t_downscale
+ n_pad = self.t_downscale - x.shape[1] % self.t_downscale
+ padding = x[:, -1:].repeat_interleave(n_pad, dim=1)
+ x = torch.cat([x, padding], 1)
+ return apply_model_with_memblocks(self.encoder, x, parallel, show_progress_bar)
+
+ def postprocess_output_frames(self, x):
+ """Postprocess RGB frames after the main decoder sequence."""
+ if self.patch_size > 1: x = F.pixel_shuffle(x, self.patch_size)
+ return x.clamp_(0, 1)
+
+ def decode_video(self, x, parallel=True, show_progress_bar=True):
+ """Decode a sequence of frames.
+
+ Args:
+ x: input NTCHW latent (C=self.latent_channels) tensor with ~Gaussian values.
+ parallel: if True, all frames will be processed at once.
+ (this is faster but may require more memory).
+ if False, frames will be processed sequentially.
+ Returns NTCHW RGB tensor with ~[0, 1] values.
+ """
+ skip_trim = self.is_cogvideox and x.shape[1] % 2 == 0
+ x = apply_model_with_memblocks(self.decoder, x, parallel, show_progress_bar)
+ x = self.postprocess_output_frames(x)
+ if skip_trim:
+ # skip trimming for cogvideox to make frame counts match.
+ # this still doesn't have correct temporal alignment for certain frame counts
+ # (cogvideox seems to pad at the start?), but for multiple-of-4 it's fine.
+ return x
+ return x[:, self.frames_to_trim:]
+
+class StreamingTAEHV(nn.Module):
+ def __init__(self, taehv):
+ """Streaming wrapper around TAEHV for real-time use-cases (where not all inputs are available immediately).
+
+ Encode-decode (video-to-video) usage:
+ streaming = StreamingTAEHV(taehv)
+ for frame in video_frames:
+ latent = streaming.encode(frame_tensor)
+ decoded = streaming.decode(latent) # feeds latent if not None, then returns next frame
+ if decoded is not None:
+ display(decoded)
+ for frame in streaming.flush():
+ display(frame)
+
+ Decode-only (world model) usage:
+ streaming = StreamingTAEHV(taehv)
+ while running:
+ latent = world_model.step() # latent represents t_upscale frames
+ frame = streaming.decode(latent) # returns first frame immediately
+ while frame is not None: # retrieve remaining frames from this latent
+ display(frame)
+ frame = streaming.decode()
+ """
+ super().__init__()
+ self.taehv = taehv
+ self.reset()
+
+ def reset(self):
+ """Reset all internal state. Call this to start encoding/decoding a new stream."""
+ self.encoder_work_queue, self.encoder_memory = [], [None] * len(self.taehv.encoder)
+ self.decoder_work_queue, self.decoder_memory = [], [None] * len(self.taehv.decoder)
+ self.n_frames_encoded, self.n_frames_decoded = 0, 0
+ self._last_encoder_input_frame = None
+
+ def encode(self, x=None):
+ """Feed an input frame (optional) and try to produce an encoder output.
+
+ The encoder accumulates t_downscale input frames before producing one latent,
+ so most calls will return None. Use flush_encoder() at end-of-stream to pad and
+ drain any remaining latents.
+
+ Args:
+ x: NTCHW RGB frame tensor with values in [0, 1], or None to just process pending work.
+ Returns: N1CHW latent tensor, or None if not enough input has been accumulated.
+ """
+ if x is not None:
+ assert x.ndim == 5 and x.shape[2] == self.taehv.image_channels, f"Expected NTCHW frames but got {x.shape=}"
+ self._last_encoder_input_frame = x[:, -1:]
+ x = self.taehv.preprocess_input_frames(x)
+ self.encoder_work_queue.extend(TWorkItem(xt, 0) for xt in x.unbind(1))
+ self.n_frames_encoded += x.shape[1]
+ xt = apply_model_with_memblocks_sequential_single_step(
+ self.taehv.encoder, self.encoder_memory, self.encoder_work_queue)
+ return xt
+
+ def decode(self, x=None):
+ """Feed a latent (optional) and try to produce a decoded frame.
+
+ Each latent produces t_upscale output frames due to temporal upscaling. The first
+ decode(latent) call returns the first of these frames; call decode() with no argument
+ to retrieve the rest, one at a time. Each call does the minimum decoder work needed to
+ produce one frame.
+
+ Startup frames (the first frames_to_trim raw decoder outputs, used for causal alignment
+ with the reference VAE) are consumed internally and never returned.
+
+ Args:
+ x: NTCHW latent tensor, or None to retrieve the next pending frame.
+ Returns: N1CHW decoded RGB frame tensor, or None if the queue needs more input.
+ """
+ if x is not None:
+ assert x.ndim == 5 and x.shape[2] == self.taehv.latent_channels, f"Expected NTCHW latents but got {x.shape=}"
+ self.decoder_work_queue.extend(TWorkItem(xt, 0) for xt in x.unbind(1))
+
+ imgs = []
+ if self.n_frames_decoded == 0:
+ first_chunk = True
+ else:
+ first_chunk = False
+ while True:
+ xt = apply_model_with_memblocks_sequential_single_step(
+ self.taehv.decoder, self.decoder_memory, self.decoder_work_queue)
+ if xt is not None:
+ imgs.append(self.taehv.postprocess_output_frames(xt))
+ else:
+ if first_chunk:
+ return torch.cat(imgs, 1)[:, self.taehv.frames_to_trim:]
+ else:
+ return torch.cat(imgs, 1)
+ self.n_frames_decoded += 1
+ if not self.taehv.is_cogvideox:
+ continue
+
+ def flush_encoder(self):
+ """Pad (if needed) and drain all remaining latents from the encoder.
+
+ Returns list of N1CHW latent tensors.
+ """
+ latents = []
+ if self._last_encoder_input_frame is not None and self.n_frames_encoded % self.taehv.t_downscale != 0:
+ n_pad = self.taehv.t_downscale - self.n_frames_encoded % self.taehv.t_downscale
+ for _ in range(n_pad):
+ lat = self.encode(self._last_encoder_input_frame)
+ if lat is not None:
+ latents.append(lat)
+ while (lat := self.encode()) is not None:
+ latents.append(lat)
+ return latents
+
+ def flush_decoder(self):
+ """Drain all remaining decoded frames from the decoder.
+
+ Returns list of N1CHW decoded RGB frame tensors.
+ """
+ frames = []
+ while (frame := self.decode()) is not None:
+ frames.append(frame)
+ return frames
+
+ def flush(self):
+ """Flush encoder (with padding) and decoder, returning all remaining decoded frames.
+
+ Returns list of N1CHW decoded RGB frame tensors.
+ """
+ frames = []
+ for latent in self.flush_encoder():
+ frame = self.decode(latent)
+ if frame is not None:
+ frames.append(frame)
+ frames.extend(self.flush_decoder())
+ return frames
+
+@torch.no_grad()
+def main():
+ """Run TAEHV roundtrip reconstruction on the given video paths."""
+ import os
+ import sys
+ import cv2 # no highly esteemed deed is commemorated here
+
+ class VideoTensorReader:
+ def __init__(self, video_file_path):
+ self.cap = cv2.VideoCapture(video_file_path)
+ assert self.cap.isOpened(), f"Could not load {video_file_path}"
+ self.fps = self.cap.get(cv2.CAP_PROP_FPS)
+ def __iter__(self):
+ return self
+ def __next__(self):
+ ret, frame = self.cap.read()
+ if not ret:
+ self.cap.release()
+ raise StopIteration # End of video or error
+ return torch.from_numpy(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)).permute(2, 0, 1) # BGR HWC -> RGB CHW
+
+ class VideoTensorWriter:
+ def __init__(self, video_file_path, width_height, fps=30):
+ self.writer = cv2.VideoWriter(video_file_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, width_height)
+ assert self.writer.isOpened(), f"Could not create writer for {video_file_path}"
+ def write(self, frame_tensor):
+ assert frame_tensor.ndim == 3 and frame_tensor.shape[0] == 3, f"{frame_tensor.shape}??"
+ self.writer.write(cv2.cvtColor(frame_tensor.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR)) # RGB CHW -> BGR HWC
+ def __del__(self):
+ if hasattr(self, 'writer'): self.writer.release()
+
+ dev = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu")
+ dtype = torch.float16
+ checkpoint_path = os.getenv("TAEHV_CHECKPOINT_PATH", "taehv.pth")
+ checkpoint_name = os.path.splitext(os.path.basename(checkpoint_path))[0]
+ print(f"Using device \033[31m{dev}\033[0m, dtype \033[32m{dtype}\033[0m, checkpoint \033[34m{checkpoint_name}\033[0m ({checkpoint_path})")
+ taehv = TAEHV(checkpoint_path=checkpoint_path).to(dev, dtype)
+ for video_path in sys.argv[1:]:
+ print(f"Processing {video_path}...")
+ video_in = VideoTensorReader(video_path)
+ video = torch.stack(list(video_in), 0)[None]
+ vid_dev = video.to(dev, dtype).div_(255.0)
+ # convert to device tensor
+ if video.numel() < 100_000_000:
+ print(f" {video_path} seems small enough, will process all frames in parallel")
+ # convert to device tensor
+ vid_enc = taehv.encode_video(vid_dev)
+ print(f" Encoded {video_path} -> {vid_enc.shape}. Decoding...")
+ vid_dec = taehv.decode_video(vid_enc)
+ print(f" Decoded {video_path} -> {vid_dec.shape}")
+ else:
+ print(f" {video_path} seems large, will process each frame sequentially")
+ # convert to device tensor
+ vid_enc = taehv.encode_video(vid_dev, parallel=False)
+ print(f" Encoded {video_path} -> {vid_enc.shape}. Decoding...")
+ vid_dec = taehv.decode_video(vid_enc, parallel=False)
+ print(f" Decoded {video_path} -> {vid_dec.shape}")
+ video_out_path = video_path + f".reconstructed_by_{checkpoint_name}.mp4"
+ video_out = VideoTensorWriter(video_out_path, (vid_dec.shape[-1], vid_dec.shape[-2]), fps=int(round(video_in.fps)))
+ for frame in vid_dec.clamp_(0, 1).mul_(255).round_().byte().cpu()[0]:
+ video_out.write(frame)
+ print(f" Saved to {video_out_path}")
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/telefuser/models/wan22_video_vae.py b/telefuser/models/wan22_video_vae.py
index 35e722e2..bb15d021 100644
--- a/telefuser/models/wan22_video_vae.py
+++ b/telefuser/models/wan22_video_vae.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from dataclasses import dataclass, field
+
import torch
import torch.nn as nn
import torch.nn.functional as F
@@ -22,6 +24,14 @@
)
+@dataclass
+class Wan22VideoVAEStreamingDecodeState:
+ """Session-owned temporal feature cache for incremental Wan2.2 decoding."""
+
+ feat_cache: list[object] = field(default_factory=list)
+ feat_idx: list[int] = field(default_factory=lambda: [0])
+
+
class Resample(nn.Module):
"""2D/3D resampling module for Wan2.2 VAE.
@@ -1416,6 +1426,7 @@ def cached_decode_withflag(
device: torch.device,
is_first_clip: bool,
is_last_clip: bool,
+ decode_state: Wan22VideoVAEStreamingDecodeState | None = None,
) -> torch.Tensor:
"""Decode with persistent feature cache for streaming generation.
@@ -1431,11 +1442,22 @@ def cached_decode_withflag(
Returns:
Decoded video tensor [C, T_out, H_out, W_out]
"""
+ feat_cache = self._feat_cache if decode_state is None else decode_state.feat_cache
+ feat_idx = self._feat_idx if decode_state is None else decode_state.feat_idx
+
# Initialize cache on first clip
if is_first_clip:
conv_num = _count_conv3d(self.model.decoder)
- self._feat_cache = [None] * conv_num
- self._feat_idx = [0]
+ feat_cache = [None] * conv_num
+ feat_idx = [0]
+ if decode_state is None:
+ self._feat_cache = feat_cache
+ self._feat_idx = feat_idx
+ else:
+ decode_state.feat_cache = feat_cache
+ decode_state.feat_idx = feat_idx
+ elif not feat_cache:
+ raise RuntimeError("Wan2.2 VAE decode cache must be initialized by the first clip")
# Add batch dimension if needed
if hidden_state.dim() == 4:
@@ -1452,26 +1474,30 @@ def cached_decode_withflag(
x = self.model.conv2(z)
for i in range(iter_):
- self._feat_idx[0] = 0 # Reset index for each frame
+ feat_idx[0] = 0 # Reset index for each frame
if i == 0:
out = self.model.decoder(
x[:, :, i : i + 1, :, :],
- feat_cache=self._feat_cache,
- feat_idx=self._feat_idx,
+ feat_cache=feat_cache,
+ feat_idx=feat_idx,
first_chunk=is_first_clip and i == 0,
)
else:
out_ = self.model.decoder(
x[:, :, i : i + 1, :, :],
- feat_cache=self._feat_cache,
- feat_idx=self._feat_idx,
+ feat_cache=feat_cache,
+ feat_idx=feat_idx,
)
out = torch.cat([out, out_], 2)
# Clear cache on last clip
if is_last_clip:
- self._feat_cache = []
- self._feat_idx = [0]
+ if decode_state is None:
+ self._feat_cache = []
+ self._feat_idx = [0]
+ else:
+ decode_state.feat_cache = []
+ decode_state.feat_idx = [0]
video = out.clamp_(-1, 1)
@@ -1484,6 +1510,78 @@ def cached_decode_withflag(
return video
+ def cached_decode_batch_withflag(
+ self,
+ hidden_states: torch.Tensor,
+ device: torch.device,
+ is_first_clip: bool,
+ is_last_clip: bool,
+ decode_states: list[Wan22VideoVAEStreamingDecodeState],
+ ) -> torch.Tensor:
+ """Decode one compatible latent chunk per session in one VAE batch."""
+ if hidden_states.ndim != 5:
+ raise ValueError("Batched Wan2.2 VAE latents must be [B,C,T,H,W]")
+ if hidden_states.shape[0] != len(decode_states) or not decode_states:
+ raise ValueError("decode_states must contain one entry per latent batch item")
+
+ conv_num = _count_conv3d(self.model.decoder)
+ if is_first_clip:
+ for state in decode_states:
+ state.feat_cache = [None] * conv_num
+ state.feat_idx = [0]
+ feat_cache: list[object] = [None] * conv_num
+ else:
+ if any(len(state.feat_cache) != conv_num for state in decode_states):
+ raise RuntimeError("Every Wan2.2 VAE batch state must have an initialized decode cache")
+ feat_cache = []
+ for cache_index in range(conv_num):
+ values = [state.feat_cache[cache_index] for state in decode_states]
+ if all(value is None for value in values):
+ feat_cache.append(None)
+ elif all(isinstance(value, torch.Tensor) for value in values):
+ feat_cache.append(torch.cat(values, dim=0))
+ elif all(not isinstance(value, torch.Tensor) and value == values[0] for value in values):
+ feat_cache.append(values[0])
+ else:
+ raise RuntimeError("Wan2.2 VAE batch states have incompatible temporal caches")
+
+ hidden_states = hidden_states.to(device)
+ scale = self._get_scale_on_device(device, hidden_states.dtype)
+ z = hidden_states / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(
+ 1, self.z_dim, 1, 1, 1
+ )
+ x = self.model.conv2(z)
+ feat_idx = [0]
+ outputs: list[torch.Tensor] = []
+ for frame_index in range(z.shape[2]):
+ feat_idx[0] = 0
+ outputs.append(
+ self.model.decoder(
+ x[:, :, frame_index : frame_index + 1],
+ feat_cache=feat_cache,
+ feat_idx=feat_idx,
+ first_chunk=is_first_clip and frame_index == 0,
+ )
+ )
+ output = torch.cat(outputs, dim=2).clamp_(-1, 1)
+
+ if is_last_clip:
+ for state in decode_states:
+ state.feat_cache = []
+ state.feat_idx = [0]
+ else:
+ for cache_index, value in enumerate(feat_cache):
+ if isinstance(value, torch.Tensor):
+ for batch_index, state in enumerate(decode_states):
+ state.feat_cache[cache_index] = value[batch_index : batch_index + 1].detach()
+ else:
+ for state in decode_states:
+ state.feat_cache[cache_index] = value
+ for state in decode_states:
+ state.feat_idx[0] = 0
+
+ return unpatchify(output, patch_size=2)
+
@staticmethod
def state_dict_converter():
return Wan22VideoVAEStateDictConverter()
diff --git a/telefuser/orchestrator/__init__.py b/telefuser/orchestrator/__init__.py
index ed9e6c8a..7dc4c23a 100644
--- a/telefuser/orchestrator/__init__.py
+++ b/telefuser/orchestrator/__init__.py
@@ -7,6 +7,7 @@
from __future__ import annotations
from .artifact_save_stage import ArtifactSaveConfig, ArtifactSaveStage
+from .batched_stage_actor import BatchedLocalStageActor
from .parallel_worker_stage_actor import ParallelWorkerStageActor
from .pipeline_orchestrator import FlexiblePipelineOrchestrator, RequestState
from .stage_wrapper import EnhancedPipelineStageWrapper, StageConfig, StageResult, StageTask
@@ -38,6 +39,7 @@
__all__ = [
"ArtifactSaveConfig",
"ArtifactSaveStage",
+ "BatchedLocalStageActor",
"FlexiblePipelineOrchestrator",
"RequestState",
"EnhancedPipelineStageWrapper",
diff --git a/telefuser/orchestrator/batched_stage_actor.py b/telefuser/orchestrator/batched_stage_actor.py
new file mode 100644
index 00000000..18a22061
--- /dev/null
+++ b/telefuser/orchestrator/batched_stage_actor.py
@@ -0,0 +1,266 @@
+"""Continuous-batching stage actor for stateful streaming pipelines."""
+
+from __future__ import annotations
+
+import queue
+import threading
+import time
+from collections import deque
+from collections.abc import Callable, Mapping, Sequence
+from concurrent.futures import Future, InvalidStateError
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from dataclasses import dataclass
+
+from .streaming_pipeline_orchestrator import (
+ StreamingActorBusyError,
+ StreamingActorHealth,
+ StreamingActorState,
+ StreamingSessionCloseReason,
+ StreamingSessionContext,
+ StreamingStageInvocation,
+)
+
+
+@dataclass
+class _BatchInvocationMessage:
+ invocation: StreamingStageInvocation
+ future: Future[Mapping[str, object]]
+
+
+@dataclass
+class _BatchSessionCloseMessage:
+ context: StreamingSessionContext
+ reason: StreamingSessionCloseReason
+ future: Future[None]
+
+
+class BatchedLocalStageActor:
+ """Own one stage and coalesce compatible cross-session invocations."""
+
+ def __init__(
+ self,
+ batch_handler: Callable[
+ [Sequence[StreamingStageInvocation]],
+ Sequence[Mapping[str, object]],
+ ],
+ *,
+ batch_key: Callable[[StreamingStageInvocation], object] | None = None,
+ max_batch_size: int = 8,
+ batching_window_seconds: float = 0.002,
+ mailbox_capacity: int = 64,
+ name: str = "batched-local-stage-actor",
+ session_closer: Callable[[StreamingSessionContext, StreamingSessionCloseReason], None] | None = None,
+ ) -> None:
+ if max_batch_size < 1 or mailbox_capacity < 1 or batching_window_seconds < 0:
+ raise ValueError("Invalid batched actor capacity or batching window")
+ self._batch_handler = batch_handler
+ self._batch_key = batch_key or (lambda _invocation: None)
+ self._max_batch_size = int(max_batch_size)
+ self._batching_window_seconds = float(batching_window_seconds)
+ self._session_closer = session_closer
+ self._mailbox: queue.Queue[_BatchInvocationMessage | _BatchSessionCloseMessage | None] = queue.Queue(
+ maxsize=mailbox_capacity
+ )
+ self._backlog: deque[_BatchInvocationMessage | _BatchSessionCloseMessage | None] = deque()
+ self._closed = False
+ self._failure_reason: str | None = None
+ self._pending_invocations = 0
+ self._pending_session_closes = 0
+ self._batch_count = 0
+ self._batch_item_count = 0
+ self._max_observed_batch_size = 0
+ self._lock = threading.Lock()
+ self._idle = threading.Condition(self._lock)
+ self._thread = threading.Thread(target=self._run, daemon=True, name=name)
+ self._thread.start()
+
+ def submit(self, invocation: StreamingStageInvocation) -> Future[Mapping[str, object]]:
+ with self._idle:
+ if self._closed:
+ raise RuntimeError("Stage actor is closed")
+ if self._failure_reason is not None:
+ raise RuntimeError(f"Stage actor has failed: {self._failure_reason}")
+ future: Future[Mapping[str, object]] = Future()
+ try:
+ self._mailbox.put_nowait(_BatchInvocationMessage(invocation, future))
+ except queue.Full as exc:
+ raise StreamingActorBusyError("Stage actor mailbox is full") from exc
+ self._pending_invocations += 1
+ return future
+
+ def health(self) -> StreamingActorHealth:
+ with self._idle:
+ if self._failure_reason is not None:
+ state = StreamingActorState.FAILED
+ elif self._closed:
+ state = StreamingActorState.CLOSED
+ else:
+ state = StreamingActorState.RUNNING
+ return StreamingActorHealth(state, self._pending_invocations, self._failure_reason)
+
+ def batch_metrics(self) -> dict[str, int | float]:
+ with self._idle:
+ mean = self._batch_item_count / self._batch_count if self._batch_count else 0.0
+ return {
+ "batch_count": self._batch_count,
+ "batch_items": self._batch_item_count,
+ "max_batch_size": self._max_observed_batch_size,
+ "mean_batch_size": mean,
+ }
+
+ def barrier(self, timeout: float = 5.0) -> None:
+ if timeout < 0:
+ raise ValueError("timeout must be non-negative")
+ deadline = time.monotonic() + timeout
+ with self._idle:
+ while self._pending_invocations or self._pending_session_closes:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("Timed out waiting for batched stage actor quiescence")
+ self._idle.wait(remaining)
+
+ def close_session(
+ self,
+ context: StreamingSessionContext,
+ reason: StreamingSessionCloseReason,
+ timeout: float = 5.0,
+ ) -> None:
+ if timeout < 0:
+ raise ValueError("timeout must be non-negative")
+ deadline = time.monotonic() + timeout
+ future: Future[None] = Future()
+ with self._idle:
+ if self._closed:
+ raise RuntimeError("Stage actor is closed")
+ if self._failure_reason is not None:
+ raise RuntimeError(f"Stage actor has failed: {self._failure_reason}")
+ self._pending_session_closes += 1
+ try:
+ self._mailbox.put(_BatchSessionCloseMessage(context, reason, future), timeout=timeout)
+ except queue.Full as exc:
+ with self._idle:
+ self._pending_session_closes -= 1
+ self._idle.notify_all()
+ raise TimeoutError("Timed out submitting stage session cleanup") from exc
+ try:
+ future.result(timeout=max(0.0, deadline - time.monotonic()))
+ except FutureTimeoutError as exc:
+ raise TimeoutError("Timed out waiting for stage session cleanup") from exc
+
+ def close(self) -> None:
+ with self._idle:
+ if self._closed:
+ return
+ self._closed = True
+ self._mailbox.put(None)
+ self._thread.join()
+
+ def _next_message(
+ self,
+ timeout: float | None = None,
+ ) -> _BatchInvocationMessage | _BatchSessionCloseMessage | None:
+ if self._backlog:
+ return self._backlog.popleft()
+ return self._mailbox.get(timeout=timeout)
+
+ def _run(self) -> None:
+ try:
+ while True:
+ message = self._next_message()
+ if message is None:
+ return
+ if isinstance(message, _BatchSessionCloseMessage):
+ self._run_close(message)
+ continue
+ batch = self._collect_batch(message)
+ self._run_batch(batch)
+ except BaseException as exc:
+ self._fail(exc)
+
+ def _collect_batch(self, first: _BatchInvocationMessage) -> list[_BatchInvocationMessage]:
+ batch = [first]
+ key = self._batch_key(first.invocation)
+ sessions = {first.invocation.key.session_id}
+ deadline = time.monotonic() + self._batching_window_seconds
+ while len(batch) < self._max_batch_size:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+ try:
+ candidate = self._next_message(timeout=remaining)
+ except queue.Empty:
+ break
+ if candidate is None or isinstance(candidate, _BatchSessionCloseMessage):
+ self._backlog.appendleft(candidate)
+ break
+ candidate_session = candidate.invocation.key.session_id
+ if self._batch_key(candidate.invocation) != key or candidate_session in sessions:
+ self._backlog.append(candidate)
+ break
+ batch.append(candidate)
+ sessions.add(candidate_session)
+ return batch
+
+ def _run_batch(self, batch: Sequence[_BatchInvocationMessage]) -> None:
+ runnable = [message for message in batch if message.future.set_running_or_notify_cancel()]
+ if not runnable:
+ self._finish_invocations(len(batch))
+ return
+ try:
+ results = list(self._batch_handler([message.invocation for message in runnable]))
+ if len(results) != len(runnable):
+ raise RuntimeError("Batched stage handler returned the wrong number of results")
+ except BaseException as exc:
+ for message in runnable:
+ message.future.set_exception(exc)
+ else:
+ for message, result in zip(runnable, results):
+ message.future.set_result(result)
+ with self._idle:
+ self._batch_count += 1
+ self._batch_item_count += len(runnable)
+ self._max_observed_batch_size = max(self._max_observed_batch_size, len(runnable))
+ finally:
+ self._finish_invocations(len(batch))
+
+ def _run_close(self, message: _BatchSessionCloseMessage) -> None:
+ try:
+ if self._session_closer is not None:
+ self._session_closer(message.context, message.reason)
+ except BaseException as exc:
+ message.future.set_exception(exc)
+ else:
+ message.future.set_result(None)
+ finally:
+ with self._idle:
+ self._pending_session_closes -= 1
+ self._idle.notify_all()
+
+ def _finish_invocations(self, count: int) -> None:
+ with self._idle:
+ self._pending_invocations -= count
+ self._idle.notify_all()
+
+ def _fail(self, exc: BaseException) -> None:
+ failure_reason = f"{type(exc).__name__}: {exc}"
+ with self._idle:
+ self._failure_reason = failure_reason
+ pending = list(self._backlog)
+ self._backlog.clear()
+ while True:
+ try:
+ pending.append(self._mailbox.get_nowait())
+ except queue.Empty:
+ break
+ for message in pending:
+ if message is None or message.future.done():
+ continue
+ try:
+ message.future.set_exception(RuntimeError(f"Stage actor failed: {failure_reason}"))
+ except InvalidStateError:
+ pass
+ if isinstance(message, _BatchSessionCloseMessage):
+ self._pending_session_closes -= 1
+ else:
+ self._pending_invocations -= 1
+ self._idle.notify_all()
diff --git a/telefuser/pipelines/abot_world/denoising.py b/telefuser/pipelines/abot_world/denoising.py
index 514dfcb3..9d044551 100644
--- a/telefuser/pipelines/abot_world/denoising.py
+++ b/telefuser/pipelines/abot_world/denoising.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from collections.abc import Sequence
from typing import Any
import torch
@@ -106,7 +107,7 @@ def _denoise_block(
self_cache: list[dict[str, Any]],
cross_cache: list[dict[str, Any]],
current_start: int,
- generator: torch.Generator,
+ generator: torch.Generator | Sequence[torch.Generator],
scheduler: FlowMatchScheduler,
) -> torch.Tensor:
current = latent
@@ -133,7 +134,23 @@ def _denoise_block(
)
x0 = self._x0_prediction(flow_prediction, current, timestep, scheduler)
if index < len(timesteps) - 1:
- noise = torch.randn(x0.shape, generator=generator, dtype=x0.dtype, device=self.device)
+ if isinstance(generator, Sequence):
+ if len(generator) != x0.shape[0]:
+ raise ValueError("ABot batched denoising requires one generator per session")
+ noise = torch.cat(
+ [
+ torch.randn(
+ (1, *x0.shape[1:]),
+ generator=item_generator,
+ dtype=x0.dtype,
+ device=self.device,
+ )
+ for item_generator in generator
+ ],
+ dim=0,
+ )
+ else:
+ noise = torch.randn(x0.shape, generator=generator, dtype=x0.dtype, device=self.device)
current = scheduler.add_noise(x0, noise, timesteps[index + 1])
else:
current = x0
diff --git a/telefuser/pipelines/abot_world/interactive.py b/telefuser/pipelines/abot_world/interactive.py
index e9acbca3..0766058c 100644
--- a/telefuser/pipelines/abot_world/interactive.py
+++ b/telefuser/pipelines/abot_world/interactive.py
@@ -1,24 +1,36 @@
-"""Persistent single-session interaction for ABot-World on one GPU.
-
-The runtime mirrors LingBot's important session invariant: text embeddings,
-causal DiT KV caches, scheduler state, RNG state, and VAE temporal decode
-cache all remain resident between control blocks. It intentionally supports
-one local session; LiveKit admission and multi-session scheduling are a later
-transport/service layer rather than a prerequisite for browser testing.
-"""
+"""Persistent multi-session interaction and batching for ABot-World."""
from __future__ import annotations
import threading
-from dataclasses import dataclass
-from typing import Any, Mapping
+import time
+import uuid
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
import torch
from PIL import Image
from telefuser.core.config import WeightOffloadType
+from telefuser.models.wan22_video_vae import Wan22VideoVAEStreamingDecodeState
from .pipeline import ABotWorldPipeline
+from .taew_vae import ABotWorldTAEWDecodeState
+
+
+class ABotWorldSessionLifecycle(str, Enum):
+ """Residency and execution lifecycle for one retained ABot session."""
+
+ READY = "ready"
+ ACTIVE = "active"
+ IDLE = "idle"
+ SUSPENDED = "suspended"
+ MIGRATING = "migrating"
+ CLOSING = "closing"
+ CLOSED = "closed"
+ FAILED = "failed"
@dataclass
@@ -31,23 +43,57 @@ class ABotWorldInteractiveSession:
cross_cache: list[dict[str, Any]]
scheduler: Any
generator: torch.Generator
+ vae_decode_state: Wan22VideoVAEStreamingDecodeState = field(
+ default_factory=Wan22VideoVAEStreamingDecodeState
+ )
+ taew_decode_state: ABotWorldTAEWDecodeState | None = None
+ session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
next_latent_frame: int = 0
emitted_frames: int = 0
+ lifecycle: ABotWorldSessionLifecycle = ABotWorldSessionLifecycle.READY
+ last_activity_at: float = field(default_factory=time.monotonic)
+ owner_worker_id: str | None = None
+ ownership_epoch: int = 0
closed: bool = False
+ lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
+
+ @property
+ def is_resident(self) -> bool:
+ """Return whether tensors are currently resident on the execution device."""
+ return self.lifecycle != ABotWorldSessionLifecycle.SUSPENDED
+
+
+@dataclass(frozen=True)
+class ABotWorldSessionSnapshot:
+ """CPU-owned state transferred between ABot workers at a chunk boundary."""
+
+ session_id: str
+ prompt_emb: torch.Tensor
+ first_frame_latent: torch.Tensor
+ self_cache: tuple[dict[str, Any], ...]
+ cross_cache: tuple[dict[str, Any], ...]
+ vae_feat_cache: tuple[object, ...]
+ vae_feat_idx: tuple[int, ...]
+ generator_state: torch.Tensor
+ next_latent_frame: int
+ emitted_frames: int
+ ownership_epoch: int
class ABotWorldInteractivePipeline(ABotWorldPipeline):
- """ABot pipeline whose model weights and one generation session stay on GPU."""
+ """ABot pipeline with shared weights and isolated retained sessions."""
def __init__(self, *args: object, **kwargs: object) -> None:
super().__init__(*args, **kwargs)
- self._interactive_lock = threading.RLock()
- self._interactive_session: ABotWorldInteractiveSession | None = None
+ self._lifecycle_lock = threading.RLock()
+ self._execution_lock = threading.RLock()
+ self._interactive_sessions: dict[str, ABotWorldInteractiveSession] = {}
self._models_preloaded = False
+ self._last_stage_metrics: dict[str, float | int] = {}
def preload_models(self) -> None:
"""Place VAE, T5, and DiT on the configured GPU before accepting controls."""
- with self._interactive_lock:
+ with self._lifecycle_lock:
if self._models_preloaded:
return
for stage in self._get_stages():
@@ -63,31 +109,50 @@ def create_interactive_session(
prompt: str,
*,
seed: int = 42,
+ session_id: str | None = None,
) -> ABotWorldInteractiveSession:
"""Encode the start image and allocate session-owned causal caches."""
if not isinstance(image, Image.Image):
raise TypeError("image must be a PIL Image")
- with self._interactive_lock:
- if self._interactive_session is not None:
- self.close_interactive_session(self._interactive_session)
- self.preload_models()
+ self.preload_models()
+ with self._execution_lock:
pixels = self.preprocess_image(image.convert("RGB"), self.config.height, self.config.width)
+ encode_started_at = time.monotonic()
start_latent, _ = self.vae_stage.process("encode_image", pixels, None, 1, concat_mask=False)
+ encode_seconds = time.monotonic() - encode_started_at
first_frame_latent = start_latent.unsqueeze(0).to(device=self.device, dtype=self.torch_dtype)
- prompt_emb = self.text_encoding_stage.process([prompt])[0].to(device=self.device, dtype=self.torch_dtype)
- self_cache, cross_cache = self.denoise_stage._new_cache(
- first_frame_latent.shape[0], first_frame_latent.shape[-2], first_frame_latent.shape[-1]
+ text_started_at = time.monotonic()
+ prompt_emb = self.text_encoding_stage.process([prompt])[0].to(
+ device=self.device,
+ dtype=self.torch_dtype,
)
- session = ABotWorldInteractiveSession(
- prompt_emb=prompt_emb,
- first_frame_latent=first_frame_latent,
- self_cache=self_cache,
- cross_cache=cross_cache,
- scheduler=self.denoise_stage._scheduler(),
- generator=torch.Generator(device=self.device).manual_seed(seed),
+ text_seconds = time.monotonic() - text_started_at
+ self._last_stage_metrics = {
+ "batch_size": 1,
+ "vae_encode_seconds": encode_seconds,
+ "text_encode_seconds": text_seconds,
+ }
+ self_cache, cross_cache = self.denoise_stage._new_cache(
+ first_frame_latent.shape[0],
+ first_frame_latent.shape[-2],
+ first_frame_latent.shape[-1],
)
- self._interactive_session = session
- return session
+ session = ABotWorldInteractiveSession(
+ session_id=session_id or str(uuid.uuid4()),
+ prompt_emb=prompt_emb,
+ first_frame_latent=first_frame_latent,
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ scheduler=self.denoise_stage._scheduler(),
+ generator=torch.Generator(device=self.device).manual_seed(seed),
+ )
+ session.taew_decode_state = self.taew_decode_stage.create_decode_state()
+ self.taew_decode_stage.warmup_first_frame(session.taew_decode_state, first_frame_latent)
+ with self._lifecycle_lock:
+ if session.session_id in self._interactive_sessions:
+ raise ValueError(f"ABot interactive session {session.session_id!r} already exists")
+ self._interactive_sessions[session.session_id] = session
+ return session
@torch.inference_mode()
def generate_next_block(
@@ -96,67 +161,406 @@ def generate_next_block(
actions: Mapping[str, bool] | None = None,
control_latent_frames: int = 3,
) -> list[Image.Image]:
- """Generate one causal action-controlled latent group."""
- if control_latent_frames not in {1, 3}:
- raise ValueError("control_latent_frames must be 1 or 3")
- with self._interactive_lock:
- if session is not self._interactive_session or session.closed:
- raise RuntimeError("ABot interactive session is no longer active")
+ """Generate one block through the same batch path used by concurrent serving."""
+ return self.generate_next_blocks(
+ [session],
+ [actions],
+ control_latent_frames=control_latent_frames,
+ )[0]
+
+ @torch.inference_mode()
+ def generate_next_blocks(
+ self,
+ sessions: Sequence[ABotWorldInteractiveSession],
+ actions: Sequence[Mapping[str, bool] | None],
+ *,
+ control_latent_frames: int = 3,
+ ) -> list[list[Image.Image]]:
+ """Generate one compatible causal block for every session in one model batch."""
+ if not sessions or len(sessions) != len(actions):
+ raise ValueError("sessions and actions must be non-empty and have equal length")
+ if control_latent_frames not in {1, 2, 3}:
+ raise ValueError("control_latent_frames must be 1, 2, or 3")
+ first_flags = {session.next_latent_frame == 0 for session in sessions}
+ if len(first_flags) != 1:
+ raise ValueError("ABot first chunks must be batched separately from continuation chunks")
+ relative_rope = bool(self.denoise_stage.dit.use_relative_rope)
+ if not relative_rope and len({session.next_latent_frame for session in sessions}) != 1:
+ raise ValueError("Absolute-RoPE ABot sessions must share next_latent_frame")
+ if len({tuple(session.first_frame_latent.shape) for session in sessions}) != 1:
+ raise ValueError("ABot batch sessions must have compatible latent shapes")
+
+ with self._lifecycle_lock:
+ for session in sessions:
+ if self._interactive_sessions.get(session.session_id) is not session or session.closed:
+ raise RuntimeError("ABot interactive session is no longer active")
+ if not session.is_resident:
+ raise RuntimeError("ABot interactive session must be restored before generation")
+
+ with self._execution_lock:
+ batch_started_at = time.monotonic()
+ for session in sessions:
+ session.lifecycle = ABotWorldSessionLifecycle.ACTIVE
+ session.last_activity_at = time.monotonic()
frame_count = control_latent_frames
- latent_shape = session.first_frame_latent.shape
- noise = torch.randn(
- (latent_shape[0], latent_shape[1], frame_count, latent_shape[3], latent_shape[4]),
- generator=session.generator,
- device=self.device,
- dtype=torch.float32,
+ noises = []
+ action_contexts = []
+ for session, session_actions in zip(sessions, actions):
+ latent_shape = session.first_frame_latent.shape
+ noises.append(
+ torch.randn(
+ (1, latent_shape[1], frame_count, latent_shape[3], latent_shape[4]),
+ generator=session.generator,
+ device=self.device,
+ dtype=torch.float32,
+ )
+ )
+ action_contexts.append(
+ self.build_action_context(
+ session_actions,
+ latent_frames=frame_count,
+ height=self.config.height,
+ width=self.config.width,
+ device=self.device,
+ dtype=self.torch_dtype,
+ )
+ )
+
+ input_prepare_seconds = time.monotonic() - batch_started_at
+ cache_collate_started_at = time.monotonic()
+ original_global_ends = [
+ [int(layer["global_end_index"].item()) for layer in session.self_cache]
+ for session in sessions
+ ]
+ self_cache = self._collate_caches(sessions, "self_cache")
+ cross_cache = self._collate_caches(sessions, "cross_cache")
+ cache_collate_seconds = time.monotonic() - cache_collate_started_at
+ start = sessions[0].next_latent_frame
+ # CUDA events provide stage time without treating asynchronous kernel
+ # launch latency as DiT runtime. The final VAE event is synchronized
+ # before metrics are read, while the normal stream ordering remains
+ # unchanged.
+ use_cuda_events = torch.device(self.device).type == "cuda"
+ if use_cuda_events:
+ denoise_started = torch.cuda.Event(enable_timing=True)
+ denoise_finished = torch.cuda.Event(enable_timing=True)
+ vae_started = torch.cuda.Event(enable_timing=True)
+ vae_finished = torch.cuda.Event(enable_timing=True)
+ denoise_started.record()
+ else:
+ denoise_started_at = time.monotonic()
+ latents = self.denoise_stage._denoise_block(
+ torch.cat(noises, dim=0).to(dtype=self.torch_dtype),
+ torch.cat([session.prompt_emb for session in sessions], dim=0),
+ torch.cat(action_contexts, dim=0),
+ torch.cat([session.first_frame_latent for session in sessions], dim=0) if start == 0 else None,
+ self_cache,
+ cross_cache,
+ start,
+ [session.generator for session in sessions],
+ sessions[0].scheduler,
)
- action_context = self.build_action_context(
- actions,
- latent_frames=frame_count,
- height=self.config.height,
- width=self.config.width,
- device=self.device,
- dtype=self.torch_dtype,
+ if use_cuda_events:
+ denoise_finished.record()
+ else:
+ denoise_seconds = time.monotonic() - denoise_started_at
+ global_deltas = [
+ int(layer["global_end_index"].item()) - original_global_ends[0][layer_index]
+ for layer_index, layer in enumerate(self_cache)
+ ]
+ cache_scatter_started_at = time.monotonic()
+ self._scatter_caches(sessions, "self_cache", self_cache)
+ for session_index, session in enumerate(sessions):
+ for layer_index, delta in enumerate(global_deltas):
+ session.self_cache[layer_index]["global_end_index"].fill_(
+ original_global_ends[session_index][layer_index] + delta
+ )
+ self._scatter_caches(sessions, "cross_cache", cross_cache)
+ cache_scatter_seconds = time.monotonic() - cache_scatter_started_at
+ if use_cuda_events:
+ vae_started.record()
+ else:
+ decode_started_at = time.monotonic()
+ if any(session.taew_decode_state is None for session in sessions):
+ raise RuntimeError("ABot session is missing its TAeW2.2 decode state")
+ decoded = torch.cat(
+ [
+ self.taew_decode_stage.decode_chunk(latents[index : index + 1], session.taew_decode_state)
+ for index, session in enumerate(sessions)
+ ],
+ dim=0,
)
- latents = self.denoise_stage._denoise_block(
- noise.to(dtype=self.torch_dtype),
- session.prompt_emb,
- action_context,
- session.first_frame_latent if session.next_latent_frame == 0 else None,
- session.self_cache,
- session.cross_cache,
- session.next_latent_frame,
- session.generator,
- session.scheduler,
+ if use_cuda_events:
+ vae_finished.record()
+ vae_finished.synchronize()
+ denoise_seconds = denoise_started.elapsed_time(denoise_finished) / 1000.0
+ decode_seconds = vae_started.elapsed_time(vae_finished) / 1000.0
+ else:
+ decode_seconds = time.monotonic() - decode_started_at
+ postprocess_started_at = time.monotonic()
+ results: list[list[Image.Image]] = []
+ for batch_index, session in enumerate(sessions):
+ frames = self.tensor2video(decoded[batch_index])
+ session.next_latent_frame += frame_count
+ session.emitted_frames += len(frames)
+ results.append(frames)
+ self._last_stage_metrics = {
+ "batch_size": len(sessions),
+ "input_prepare_seconds": input_prepare_seconds,
+ "cache_collate_seconds": cache_collate_seconds,
+ "denoise_seconds": denoise_seconds,
+ "cache_scatter_seconds": cache_scatter_seconds,
+ "vae_decode_seconds": decode_seconds,
+ "postprocess_seconds": time.monotonic() - postprocess_started_at,
+ "total_seconds": time.monotonic() - batch_started_at,
+ }
+ return results
+
+ @staticmethod
+ def _collate_caches(
+ sessions: Sequence[ABotWorldInteractiveSession],
+ attribute: str,
+ ) -> list[dict[str, Any]]:
+ cache_lists = [getattr(session, attribute) for session in sessions]
+ if len({len(cache) for cache in cache_lists}) != 1:
+ raise ValueError(f"ABot {attribute} layer counts do not match")
+ collated: list[dict[str, Any]] = []
+ for layer_index in range(len(cache_lists[0])):
+ entries = [cache[layer_index] for cache in cache_lists]
+ layer: dict[str, Any] = {}
+ for key in entries[0]:
+ values = [entry[key] for entry in entries]
+ if key in {"k", "v"}:
+ layer[key] = torch.cat(values, dim=0)
+ elif isinstance(values[0], torch.Tensor):
+ scalar_values = [int(value.item()) for value in values]
+ if key != "global_end_index" and len(set(scalar_values)) != 1:
+ raise ValueError(f"ABot batch cache cursor {key!r} must match")
+ layer[key] = values[0].clone()
+ else:
+ if len(set(values)) != 1:
+ raise ValueError(f"ABot batch cache metadata {key!r} must match")
+ layer[key] = values[0]
+ collated.append(layer)
+ return collated
+
+ @staticmethod
+ def _scatter_caches(
+ sessions: Sequence[ABotWorldInteractiveSession],
+ attribute: str,
+ collated: list[dict[str, Any]],
+ ) -> None:
+ for batch_index, session in enumerate(sessions):
+ cache_list = getattr(session, attribute)
+ for layer_index, layer in enumerate(collated):
+ for key, value in layer.items():
+ if key in {"k", "v"}:
+ cache_list[layer_index][key] = value[batch_index : batch_index + 1].detach().clone()
+ elif isinstance(value, torch.Tensor):
+ cache_list[layer_index][key] = value.detach().clone()
+ else:
+ cache_list[layer_index][key] = value
+
+ def snapshot_interactive_session(
+ self,
+ session: ABotWorldInteractiveSession,
+ ) -> ABotWorldSessionSnapshot:
+ """Clone a quiescent session to CPU for suspend or cross-worker migration."""
+ with self._execution_lock, session.lock:
+ self._require_session(session)
+ session.lifecycle = ABotWorldSessionLifecycle.MIGRATING
+ return ABotWorldSessionSnapshot(
+ session_id=session.session_id,
+ prompt_emb=session.prompt_emb.detach().to("cpu").clone(),
+ first_frame_latent=session.first_frame_latent.detach().to("cpu").clone(),
+ self_cache=tuple(self._clone_cache_to_cpu(session.self_cache)),
+ cross_cache=tuple(self._clone_cache_to_cpu(session.cross_cache)),
+ vae_feat_cache=tuple(
+ value.detach().to("cpu").clone() if isinstance(value, torch.Tensor) else value
+ for value in session.vae_decode_state.feat_cache
+ ),
+ vae_feat_idx=tuple(session.vae_decode_state.feat_idx),
+ generator_state=session.generator.get_state().to("cpu").clone(),
+ next_latent_frame=session.next_latent_frame,
+ emitted_frames=session.emitted_frames,
+ ownership_epoch=session.ownership_epoch,
)
- decoded = self.vae_stage.process(
- "decode_video_cached",
- latents[0],
- session.next_latent_frame == 0,
- False,
+
+ def restore_interactive_snapshot(
+ self,
+ snapshot: ABotWorldSessionSnapshot,
+ *,
+ owner_worker_id: str | None = None,
+ ownership_epoch: int | None = None,
+ ) -> ABotWorldInteractiveSession:
+ """Install a transferred CPU snapshot as a new resident session."""
+ return self._restore_snapshot(
+ snapshot,
+ owner_worker_id=owner_worker_id,
+ ownership_epoch=ownership_epoch,
+ direct_device_tensors=False,
+ )
+
+ def restore_interactive_device_snapshot(
+ self,
+ snapshot: ABotWorldSessionSnapshot,
+ *,
+ owner_worker_id: str | None = None,
+ ownership_epoch: int | None = None,
+ ) -> ABotWorldInteractiveSession:
+ """Adopt a snapshot already received on this pipeline's CUDA device.
+
+ This is the NCCL migration path. It preserves received target-GPU
+ allocations rather than cloning them again after the direct transfer.
+ """
+ return self._restore_snapshot(
+ snapshot,
+ owner_worker_id=owner_worker_id,
+ ownership_epoch=ownership_epoch,
+ direct_device_tensors=True,
+ )
+
+ def _restore_snapshot(
+ self,
+ snapshot: ABotWorldSessionSnapshot,
+ *,
+ owner_worker_id: str | None,
+ ownership_epoch: int | None,
+ direct_device_tensors: bool,
+ ) -> ABotWorldInteractiveSession:
+ with self._execution_lock:
+ generator = torch.Generator(device=self.device)
+ generator.set_state(snapshot.generator_state)
+ if direct_device_tensors:
+ expected_device = torch.device(self.device)
+ tensors = [snapshot.prompt_emb, snapshot.first_frame_latent]
+ tensors.extend(
+ value
+ for cache in (*snapshot.self_cache, *snapshot.cross_cache)
+ for value in cache.values()
+ if isinstance(value, torch.Tensor)
+ )
+ tensors.extend(value for value in snapshot.vae_feat_cache if isinstance(value, torch.Tensor))
+ if any(tensor.device != expected_device for tensor in tensors):
+ raise ValueError("NCCL migration tensors must already reside on the target pipeline device")
+ prompt_emb = snapshot.prompt_emb
+ first_frame_latent = snapshot.first_frame_latent
+ self_cache = [dict(cache) for cache in snapshot.self_cache]
+ cross_cache = [dict(cache) for cache in snapshot.cross_cache]
+ vae_feat_cache = list(snapshot.vae_feat_cache)
+ else:
+ prompt_emb = snapshot.prompt_emb.to(self.device, dtype=self.torch_dtype)
+ first_frame_latent = snapshot.first_frame_latent.to(self.device, dtype=self.torch_dtype)
+ self_cache = self._clone_cache_to_device(snapshot.self_cache, self.device)
+ cross_cache = self._clone_cache_to_device(snapshot.cross_cache, self.device)
+ vae_feat_cache = [
+ value.to(self.device).clone() if isinstance(value, torch.Tensor) else value
+ for value in snapshot.vae_feat_cache
+ ]
+ session = ABotWorldInteractiveSession(
+ session_id=snapshot.session_id,
+ prompt_emb=prompt_emb,
+ first_frame_latent=first_frame_latent,
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ scheduler=self.denoise_stage._scheduler(),
+ generator=generator,
+ vae_decode_state=Wan22VideoVAEStreamingDecodeState(
+ feat_cache=vae_feat_cache,
+ feat_idx=list(snapshot.vae_feat_idx),
+ ),
+ next_latent_frame=snapshot.next_latent_frame,
+ emitted_frames=snapshot.emitted_frames,
+ owner_worker_id=owner_worker_id,
+ ownership_epoch=snapshot.ownership_epoch + 1 if ownership_epoch is None else ownership_epoch,
)
- if decoded.ndim == 5:
- decoded = decoded[0]
- frames = self.tensor2video(decoded)
- session.next_latent_frame += frame_count
- session.emitted_frames += len(frames)
- return frames
+ with self._lifecycle_lock:
+ if session.session_id in self._interactive_sessions:
+ raise ValueError(f"ABot interactive session {session.session_id!r} already exists")
+ self._interactive_sessions[session.session_id] = session
+ return session
- def close_interactive_session(self, session: ABotWorldInteractiveSession | None = None) -> None:
- """Release retained cache references and reset the model-specific VAE stream cache."""
- with self._interactive_lock:
- target = self._interactive_session if session is None else session
- if target is None or target.closed:
+ @staticmethod
+ def _clone_cache_to_cpu(caches: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
+ return ABotWorldInteractivePipeline._clone_cache_to_device(caches, "cpu")
+
+ @staticmethod
+ def _clone_cache_to_device(
+ caches: Sequence[dict[str, Any]],
+ device: str | torch.device,
+ ) -> list[dict[str, Any]]:
+ return [
+ {
+ key: value.detach().to(device).clone() if isinstance(value, torch.Tensor) else value
+ for key, value in layer.items()
+ }
+ for layer in caches
+ ]
+
+ def last_stage_metrics(self) -> dict[str, float | int]:
+ """Return raw timings for the most recently completed model batch."""
+ with self._execution_lock:
+ return dict(self._last_stage_metrics)
+
+ def suspend_interactive_session(self, session: ABotWorldInteractiveSession) -> None:
+ """Move all material session tensors to CPU at a chunk boundary."""
+ with self._execution_lock, session.lock:
+ self._require_session(session)
+ if session.lifecycle == ABotWorldSessionLifecycle.SUSPENDED:
+ return
+ session.prompt_emb = session.prompt_emb.to("cpu")
+ session.first_frame_latent = session.first_frame_latent.to("cpu")
+ self._move_cache_tensors(session.self_cache, "cpu")
+ self._move_cache_tensors(session.cross_cache, "cpu")
+ session.vae_decode_state.feat_cache = [
+ value.to("cpu") if isinstance(value, torch.Tensor) else value
+ for value in session.vae_decode_state.feat_cache
+ ]
+ session.lifecycle = ABotWorldSessionLifecycle.SUSPENDED
+
+ def restore_interactive_session(self, session: ABotWorldInteractiveSession) -> None:
+ """Restore a suspended session to the pipeline execution device."""
+ with self._execution_lock, session.lock:
+ self._require_session(session)
+ if session.lifecycle != ABotWorldSessionLifecycle.SUSPENDED:
return
- target.closed = True
- target.self_cache.clear()
- target.cross_cache.clear()
- vae = self.vae_stage.vae
- if hasattr(vae, "_feat_cache"):
- vae._feat_cache = []
- vae._feat_idx = [0]
- if target is self._interactive_session:
- self._interactive_session = None
+ session.prompt_emb = session.prompt_emb.to(self.device, dtype=self.torch_dtype)
+ session.first_frame_latent = session.first_frame_latent.to(self.device, dtype=self.torch_dtype)
+ self._move_cache_tensors(session.self_cache, self.device)
+ self._move_cache_tensors(session.cross_cache, self.device)
+ session.vae_decode_state.feat_cache = [
+ value.to(self.device) if isinstance(value, torch.Tensor) else value
+ for value in session.vae_decode_state.feat_cache
+ ]
+ session.lifecycle = ABotWorldSessionLifecycle.READY
+
+ @staticmethod
+ def _move_cache_tensors(caches: list[dict[str, Any]], device: str | torch.device) -> None:
+ for layer in caches:
+ for key, value in tuple(layer.items()):
+ if isinstance(value, torch.Tensor):
+ layer[key] = value.to(device)
+
+ def _require_session(self, session: ABotWorldInteractiveSession) -> None:
+ with self._lifecycle_lock:
+ if self._interactive_sessions.get(session.session_id) is not session or session.closed:
+ raise RuntimeError("ABot interactive session is no longer active")
+
+ def close_interactive_session(self, session: ABotWorldInteractiveSession | None = None) -> None:
+ """Release only the requested session's retained state."""
+ with self._lifecycle_lock:
+ targets = list(self._interactive_sessions.values()) if session is None else [session]
+ for target in targets:
+ if target.closed:
+ continue
+ target.lifecycle = ABotWorldSessionLifecycle.CLOSING
+ target.closed = True
+ target.self_cache.clear()
+ target.cross_cache.clear()
+ target.vae_decode_state.feat_cache.clear()
+ target.vae_decode_state.feat_idx = [0]
+ target.lifecycle = ABotWorldSessionLifecycle.CLOSED
+ self._interactive_sessions.pop(target.session_id, None)
def close(self) -> None:
self.close_interactive_session()
diff --git a/telefuser/pipelines/abot_world/pipeline.py b/telefuser/pipelines/abot_world/pipeline.py
index 6b987187..873789f0 100644
--- a/telefuser/pipelines/abot_world/pipeline.py
+++ b/telefuser/pipelines/abot_world/pipeline.py
@@ -15,6 +15,7 @@
from telefuser.pipelines.wan_video.vae import VAEStage
from .denoising import ABotWorldDenoisingStage
+from .taew_vae import ABotWorldTAEWDecodeStage
@dataclass
@@ -50,7 +51,7 @@ def __init__(self, device: str | torch.device = "cuda", torch_dtype: torch.dtype
self.width_division_factor = 32
def _get_stages(self) -> list:
- return [self.vae_stage, self.text_encoding_stage, self.denoise_stage]
+ return [self.vae_stage, self.text_encoding_stage, self.denoise_stage, self.taew_decode_stage]
def init(self, module_manager: ModuleManager, config: ABotWorldPipelineConfig) -> None:
if config.dit_config.parallel_config.world_size != 1:
@@ -67,6 +68,7 @@ def init(self, module_manager: ModuleManager, config: ABotWorldPipelineConfig) -
self._model_info = module_manager.get_model_info()
self.config = config
self.vae_stage = VAEStage("abot_world_vae", module_manager, config.vae_config)
+ self.taew_decode_stage = ABotWorldTAEWDecodeStage("abot_world_taew_decode", module_manager, config.vae_config)
self.text_encoding_stage = TextEncodingStage(
"abot_world_text_encoding", module_manager, config.text_encoding_config
)
diff --git a/telefuser/pipelines/abot_world/service.py b/telefuser/pipelines/abot_world/service.py
index 1076b8c4..acd7c08a 100644
--- a/telefuser/pipelines/abot_world/service.py
+++ b/telefuser/pipelines/abot_world/service.py
@@ -1,26 +1,34 @@
-"""LiveKit stream service for the ABot-World-0-5B-LF interactive pipeline."""
+"""TurboServe-style LiveKit service for ABot-World-0-5B-LF."""
from __future__ import annotations
import asyncio
import base64
import binascii
+import gc
import io
+import math
import queue
import threading
import time
import uuid
-from collections.abc import AsyncGenerator, Mapping
+from collections import deque
+from collections.abc import AsyncGenerator, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
+import torch
from PIL import Image
from telefuser.pipelines.abot_world.interactive import (
ABotWorldInteractivePipeline,
ABotWorldInteractiveSession,
+ ABotWorldSessionLifecycle,
+ ABotWorldSessionSnapshot,
)
+from telefuser.service.livekit.nccl_transfer import flatten_tensor_tree, rebuild_tensor_tree
+from telefuser.service.livekit.turboserve import TurboServeWorkloadDetector
from telefuser.utils.logging import logger
_CONTROL_ALIASES = {
@@ -54,6 +62,8 @@
_VALID_CONTROLS = frozenset("WASDIJKL")
_MAX_INPUT_IMAGE_BYTES = 10 * 1024 * 1024
_DEFAULT_OUTPUT_QUEUE_SIZE = 4
+_VIDEO_OUTPUT_TYPES = frozenset({"preview", "chunk"})
+_TERMINAL_OUTPUT_TYPES = frozenset({"error", "done"})
@dataclass
@@ -62,7 +72,7 @@ class _ABotWorldLiveKitSession:
pipeline_session: ABotWorldInteractiveSession
output_queue: queue.Queue[dict[str, Any]]
control_event: threading.Event
- config: dict[str, int]
+ config: dict[str, Any]
control_idle_timeout: float = 10.0
controls: set[str] = field(default_factory=set)
last_control_at: float = field(default_factory=time.monotonic)
@@ -70,15 +80,42 @@ class _ABotWorldLiveKitSession:
active: bool = True
worker: threading.Thread | None = None
lock: threading.RLock = field(default_factory=threading.RLock)
+ in_flight: bool = False
+ ready_since: float | None = None
+ next_playout_deadline: float = field(default_factory=time.monotonic)
+ created_at: float = field(default_factory=time.monotonic)
+ output_queue_high_watermark: int = 0
+ dropped_video_payloads: int = 0
+ dropped_status_payloads: int = 0
+ scheduled_chunks: int = 0
+ batch_items: int = 0
+ total_queue_wait_seconds: float = 0.0
+ total_compute_seconds: float = 0.0
+ last_error: str | None = None
+ migrating: bool = False
+
+
+@dataclass(frozen=True)
+class ABotWorldMigrationBundle:
+ """Quiescent service and pipeline state transferred to another ABot worker."""
+
+ snapshot: ABotWorldSessionSnapshot
+ config: dict[str, Any]
+ controls: frozenset[str]
+ control_idle_timeout: float
+ last_control_at: float
+ next_chunk_index: int
+ next_playout_deadline: float
class ABotWorldLiveKitService:
- """Expose one ABot causal session through TeleFuser's LiveKit contract.
+ """One-GPU retained-session owner with TurboServe-style round-robin stepping.
- The service intentionally advertises capacity one because the current
- ABot interactive pipeline owns one retained session at a time. The shared
- TeleFuser LiveKit worker still owns room admission, tokens, pacing, and
- media publication.
+ ``round_robin`` is the default and mirrors TurboServe: every scheduler turn
+ selects exactly one runnable session and advances it by one causal block.
+ Session KV/VAE state stays resident and is never collated across sessions.
+ ``batched`` retains the former experimental path for research comparisons;
+ it is deliberately opt-in because it is not the TurboServe execution model.
"""
def __init__(
@@ -90,55 +127,184 @@ def __init__(
output_queue_size: int = _DEFAULT_OUTPUT_QUEUE_SIZE,
control_idle_timeout: float = 10.0,
close_timeout: float = 300.0,
+ max_batch_size: int = 8,
+ batching_window_ms: float = 2.0,
+ idle_suspension_seconds: float = 5.0,
+ scheduler_mode: str = "round_robin",
) -> None:
if default_fps < 1:
raise ValueError(f"default_fps must be positive, got {default_fps}")
if output_queue_size < 1:
raise ValueError(f"output_queue_size must be positive, got {output_queue_size}")
- if control_idle_timeout <= 0:
- raise ValueError(f"control_idle_timeout must be positive, got {control_idle_timeout}")
- if close_timeout <= 0:
- raise ValueError(f"close_timeout must be positive, got {close_timeout}")
+ if control_idle_timeout <= 0 or close_timeout <= 0:
+ raise ValueError("control_idle_timeout and close_timeout must be positive")
+ if max_batch_size < 1:
+ raise ValueError("max_batch_size must be positive")
+ if batching_window_ms < 0 or idle_suspension_seconds <= 0:
+ raise ValueError("batching_window_ms must be non-negative and idle_suspension_seconds positive")
+ if scheduler_mode not in {"round_robin", "batched"}:
+ raise ValueError("scheduler_mode must be 'round_robin' or 'batched'")
self.pipeline = pipeline
self.default_fps = int(default_fps)
self.default_session_config = dict(default_session_config or {})
self.output_queue_size = int(output_queue_size)
self.control_idle_timeout = float(control_idle_timeout)
self.close_timeout = float(close_timeout)
+ self.max_batch_size = int(max_batch_size)
+ self.batching_window_seconds = float(batching_window_ms) / 1000.0
+ self.idle_suspension_seconds = float(idle_suspension_seconds)
+ self.scheduler_mode = scheduler_mode
self._sessions: dict[str, _ABotWorldLiveKitSession] = {}
+ self._round_robin_order: deque[str] = deque()
self._sessions_lock = threading.RLock()
+ self._scheduler_condition = threading.Condition(self._sessions_lock)
+ self._scheduler_thread: threading.Thread | None = None
+ self._scheduler_stopping = False
self._capacity_profile: dict[str, object] | None = None
+ self._scheduler_paused = False
+ self._batch_count = 0
+ self._batch_item_count = 0
+ self._maximum_batch_size = 0
+ self._last_stage_metrics: dict[str, float | int] = {}
+ self._workload_detector = TurboServeWorkloadDetector()
def start(self) -> None:
- """Preload ABot weights before the LiveKit worker accepts sessions."""
+ """Preload weights and start the sole GPU scheduling thread."""
self.pipeline.preload_models()
- logger.info("ABotWorldLiveKitService started")
+ self._ensure_scheduler_started()
+ logger.info("ABotWorldLiveKitService TurboServe scheduler started")
+
+ def stop(self, *, close_pipeline: bool = True) -> None:
+ """Stop admission and release retained sessions.
- def stop(self) -> None:
- """Close all retained sessions and release the loaded pipeline."""
+ Offline experiment suites may keep a preloaded pipeline alive across
+ independent scheduler instances by passing ``close_pipeline=False``.
+ Production callers retain the original full teardown by default.
+ """
+ with self._scheduler_condition:
+ self._scheduler_stopping = True
+ self._scheduler_condition.notify_all()
+ scheduler = self._scheduler_thread
+ if scheduler is not None and scheduler.is_alive() and scheduler is not threading.current_thread():
+ scheduler.join(timeout=self.close_timeout)
with self._sessions_lock:
session_ids = list(self._sessions)
for session_id in session_ids:
self.close_session(session_id)
- self.pipeline.close()
+ if close_pipeline:
+ self.pipeline.close()
def configure_session_capacity(self, max_sessions: int | None = None) -> dict[str, object]:
- """Report the single retained-session capacity required by ABot."""
+ """Estimate retained-session capacity from free memory and ABot cache geometry."""
if max_sessions is not None and max_sessions < 1:
raise ValueError(f"max_sessions must be positive when provided, got {max_sessions}")
- if max_sessions is not None and max_sessions != 1:
- raise ValueError("ABot-World supports one retained LiveKit session per worker")
- profile = {
+ with self._sessions_lock:
+ if self._sessions:
+ raise RuntimeError("cannot configure retained-session capacity while sessions are active")
+ if self._capacity_profile is not None:
+ if self._capacity_profile["configured_limit"] != max_sessions:
+ raise RuntimeError("ABot retained-session capacity is already configured with another limit")
+ return dict(self._capacity_profile)
+
+ per_session_bytes = self._estimate_session_bytes()
+ workspace_peak_bytes = 0
+ profiled_session_bytes = 0
+ free_bytes = 0
+ pipeline_device = torch.device(getattr(self.pipeline, "device", "cpu"))
+ if torch.cuda.is_available() and pipeline_device.type == "cuda":
+ profile = self._profile_session_memory()
+ profiled_session_bytes = int(profile["profiled_session_bytes"])
+ workspace_peak_bytes = int(profile["workspace_peak_bytes"])
+ per_session_bytes = max(per_session_bytes, profiled_session_bytes)
+ free_bytes, _ = torch.cuda.mem_get_info(pipeline_device)
+ memory_budget = max(0, int(free_bytes * 0.90))
+ if free_bytes:
+ computed_capacity = 0
+ for candidate in range(1, 65):
+ active_batch = min(candidate, self.max_batch_size) if self.scheduler_mode == "batched" else 1
+ required_bytes = candidate * per_session_bytes + active_batch * workspace_peak_bytes
+ if required_bytes > memory_budget:
+ break
+ computed_capacity = candidate
+ # A successful real-session warmup proves that one session can run even when the
+ # conservative 10% allocator reserve makes the arithmetic round below one.
+ computed_capacity = max(1, computed_capacity)
+ else:
+ computed_capacity = max_sessions or 1
+ effective_capacity = min(computed_capacity, max_sessions) if max_sessions is not None else computed_capacity
+ effective_batch_size = min(self.max_batch_size, effective_capacity) if self.scheduler_mode == "batched" else 1
+ profile: dict[str, object] = {
"configured_limit": max_sessions,
- "effective_capacity": 1,
- "computed_capacity": 1,
+ "effective_capacity": effective_capacity,
+ "computed_capacity": computed_capacity,
"model": "ABot-World-0-5B-LF",
+ "free_device_bytes": int(free_bytes),
+ "memory_budget_bytes": memory_budget,
+ "estimated_session_bytes": per_session_bytes,
+ "profiled_session_bytes": profiled_session_bytes,
+ "workspace_peak_bytes": workspace_peak_bytes,
+ "estimated_batch_workspace_bytes": effective_batch_size * workspace_peak_bytes,
+ "max_batch_size": self.max_batch_size,
+ "effective_max_batch_size": effective_batch_size,
+ "scheduler_mode": self.scheduler_mode,
}
self._capacity_profile = profile
return dict(profile)
+ def _estimate_session_bytes(self) -> int:
+ dit = self.pipeline.denoise_stage.dit
+ latent_height = self.pipeline.config.height // 16
+ latent_width = self.pipeline.config.width // 16
+ frame_tokens = (latent_height // dit.patch_size[1]) * (latent_width // dit.patch_size[2])
+ head_dim = dit.dim // dit.num_heads
+ element_size = torch.empty((), dtype=self.pipeline.torch_dtype).element_size()
+ self_cache = 2 * dit.num_layers * dit.local_attn_size * frame_tokens * dit.num_heads * head_dim * element_size
+ cross_cache = 2 * dit.num_layers * dit.text_len * dit.num_heads * head_dim * element_size
+ prompt = dit.text_len * dit.text_dim * element_size if hasattr(dit, "text_dim") else 0
+ # Reserve another 35% for VAE temporal state, latents, allocator fragmentation, and runtime workspaces.
+ return max(1, math.ceil((self_cache + cross_cache + prompt) * 1.35))
+
+ def _profile_session_memory(self) -> dict[str, int]:
+ """Warm one real chunk and measure retained state separately from workspace peaks."""
+ device = torch.device(self.pipeline.device)
+ torch.cuda.synchronize(device)
+ torch.cuda.empty_cache()
+ baseline = torch.cuda.memory_allocated(device)
+ torch.cuda.reset_peak_memory_stats(device)
+ session: ABotWorldInteractiveSession | None = None
+ try:
+ image = self._load_image(self.default_session_config)
+ prompt = str(self.default_session_config.get("prompt", "")).strip()
+ if not prompt:
+ raise ValueError("ABot capacity warmup requires the default prompt")
+ session = self.pipeline.create_interactive_session(
+ image,
+ prompt,
+ seed=int(self.default_session_config.get("seed", 42)),
+ session_id="__abot_capacity_warmup__",
+ )
+ self.pipeline.generate_next_block(
+ session,
+ {"W": True},
+ control_latent_frames=int(self.default_session_config.get("control_latent_frames", 3)),
+ )
+ torch.cuda.synchronize(device)
+ retained = max(1, torch.cuda.memory_allocated(device) - baseline)
+ peak = max(retained, torch.cuda.max_memory_allocated(device) - baseline)
+ return {
+ "profiled_session_bytes": int(retained),
+ "workspace_peak_bytes": int(max(0, peak - retained)),
+ }
+ except Exception:
+ logger.exception("ABot session capacity warmup failed; using analytical memory estimate")
+ return {"profiled_session_bytes": 0, "workspace_peak_bytes": 0}
+ finally:
+ if session is not None:
+ self.pipeline.close_interactive_session(session)
+ gc.collect()
+ torch.cuda.empty_cache()
+
def session_capacity_profile(self) -> dict[str, object] | None:
- """Return the startup capacity facts for service metadata."""
return dict(self._capacity_profile) if self._capacity_profile is not None else None
def has_session(self, session_id: str) -> bool:
@@ -146,10 +312,12 @@ def has_session(self, session_id: str) -> bool:
return session_id in self._sessions
def create_session(self, config: dict) -> str:
- """Create a preview-only session; non-empty controls start generation."""
+ """Create a preview-only retained session; controls make it scheduler-ready."""
+ self._ensure_scheduler_started()
with self._sessions_lock:
- if self._sessions:
- raise RuntimeError("ABot-World supports one retained session per worker")
+ capacity = int(self._capacity_profile["effective_capacity"]) if self._capacity_profile else 1
+ if len(self._sessions) >= capacity:
+ raise RuntimeError(f"ABot retained-session capacity is exhausted (capacity={capacity})")
session_id = str(config.get("session_id") or uuid.uuid4())
image = self._load_image(config)
@@ -163,28 +331,38 @@ def create_session(self, config: dict) -> str:
if session_idle_timeout <= 0:
raise ValueError(f"control_idle_timeout must be positive, got {session_idle_timeout}")
control_latent_frames = int(
- config.get(
- "control_latent_frames",
- self.default_session_config.get("control_latent_frames", 3),
- )
+ config.get("control_latent_frames", self.default_session_config.get("control_latent_frames", 3))
)
- if control_latent_frames not in {1, 3}:
- raise ValueError("control_latent_frames must be 1 or 3")
+ if control_latent_frames not in {1, 2, 3}:
+ raise ValueError("control_latent_frames must be 1, 2, or 3")
+ delivery_mode = str(config.get("delivery_mode", self.default_session_config.get("delivery_mode", "latest")))
+ if delivery_mode not in {"latest", "lossless"}:
+ raise ValueError("delivery_mode must be 'latest' or 'lossless'")
seed = int(config.get("seed", self.default_session_config.get("seed", 42)))
- pipeline_session = self.pipeline.create_interactive_session(image, prompt, seed=seed)
+ pipeline_session = self.pipeline.create_interactive_session(image, prompt, seed=seed, session_id=session_id)
state = _ABotWorldLiveKitSession(
session_id=session_id,
pipeline_session=pipeline_session,
output_queue=queue.Queue(maxsize=self.output_queue_size),
control_event=threading.Event(),
- config={"fps": fps, "control_latent_frames": control_latent_frames},
+ config={
+ "fps": fps,
+ "control_latent_frames": control_latent_frames,
+ "delivery_mode": delivery_mode,
+ },
control_idle_timeout=session_idle_timeout,
)
- with self._sessions_lock:
- if self._sessions:
+ with self._scheduler_condition:
+ if session_id in self._sessions:
self.pipeline.close_interactive_session(pipeline_session)
- raise RuntimeError("ABot-World supports one retained session per worker")
+ raise ValueError(f"ABot session {session_id!r} already exists")
+ capacity = int(self._capacity_profile["effective_capacity"]) if self._capacity_profile else 1
+ if len(self._sessions) >= capacity:
+ self.pipeline.close_interactive_session(pipeline_session)
+ raise RuntimeError(f"ABot retained-session capacity is exhausted (capacity={capacity})")
self._sessions[session_id] = state
+ self._round_robin_order.append(session_id)
+ self._workload_detector.record_arrival(session_id)
preview = image.convert("RGB").resize(
(self.pipeline.config.width, self.pipeline.config.height),
@@ -192,36 +370,24 @@ def create_session(self, config: dict) -> str:
)
self._put_output(
state,
- {
- "type": "preview",
- "index": -1,
- "fps": fps,
- "timestamp": time.time(),
- "frames": [preview],
- },
- )
- state.worker = threading.Thread(
- target=self._generation_loop,
- args=(state,),
- daemon=True,
- name=f"abot-world-livekit-{session_id[:8]}",
+ {"type": "preview", "index": -1, "fps": fps, "timestamp": time.time(), "frames": [preview]},
)
- state.worker.start()
return session_id
def push_chunk(self, session_id: str, chunk: dict) -> None:
- """Apply a normalized TeleFuser control message to one ABot session."""
+ """Apply a control message and wake the event-driven scheduler."""
state = self._session(session_id)
if state is None:
return
message_type = str(chunk.get("type", ""))
- with state.lock:
+ with self._scheduler_condition, state.lock:
if not state.active:
return
if message_type == "stop":
state.active = False
state.controls.clear()
- state.control_event.set()
+ state.pipeline_session.lifecycle = ABotWorldSessionLifecycle.CLOSING
+ self._scheduler_condition.notify_all()
return
if message_type == "control_state":
raw_controls = chunk.get("controls", [])
@@ -238,15 +404,21 @@ def push_chunk(self, session_id: str, chunk: dict) -> None:
else:
state.controls.discard(control)
elif message_type in {"reset", "prompt"}:
- # ABot prompt/image state is fixed for a causal session.
state.controls.clear()
else:
raise ValueError(f"Unsupported ABot control message type: {message_type}")
- state.last_control_at = time.monotonic()
+ now = time.monotonic()
+ state.last_control_at = now
+ state.ready_since = now if state.controls else None
state.control_event.set()
+ if state.controls:
+ self._workload_detector.record_active(session_id, now)
+ else:
+ self._workload_detector.record_idle(session_id, now)
+ self._scheduler_condition.notify_all()
async def pull_chunks(self, session_id: str) -> AsyncGenerator[dict, None]:
- """Yield preview and generated frames in order until the session closes."""
+ """Yield preview and generated frames in per-session sequence order."""
state = self._session(session_id)
if state is None:
return
@@ -258,86 +430,558 @@ async def pull_chunks(self, session_id: str) -> AsyncGenerator[dict, None]:
if not state.active:
return
continue
+ with self._scheduler_condition:
+ self._scheduler_condition.notify_all()
yield payload
def close_session(self, session_id: str, timeout: float | None = None) -> None:
- """Stop generation, wait for the producer, and release the ABot session state."""
+ """Wait for a chunk boundary, then release only this session's state."""
effective_timeout = self.close_timeout if timeout is None else timeout
- with self._sessions_lock:
- state = self._sessions.pop(session_id, None)
- if state is None:
- return
- with state.lock:
- state.active = False
- state.controls.clear()
- state.control_event.set()
- worker = state.worker
- if worker is not None and worker.is_alive() and worker is not threading.current_thread():
- worker.join(timeout=effective_timeout)
- if worker is not None and worker.is_alive():
- logger.warning("ABot session producer did not stop before timeout: session=%s", session_id)
- return
+ deadline = time.monotonic() + effective_timeout
+ with self._scheduler_condition:
+ state = self._sessions.get(session_id)
+ if state is None:
+ return
+ with state.lock:
+ state.active = False
+ state.controls.clear()
+ state.ready_since = None
+ self._scheduler_condition.notify_all()
+ while state.in_flight:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ logger.warning("ABot session did not reach a chunk boundary before close timeout: %s", session_id)
+ return
+ self._scheduler_condition.wait(remaining)
+ self._sessions.pop(session_id, None)
+ self._discard_from_round_robin(session_id)
+ self._workload_detector.record_departure(session_id)
self.pipeline.close_interactive_session(state.pipeline_session)
- def _generation_loop(self, state: _ABotWorldLiveKitSession) -> None:
- fps = int(state.config["fps"])
- control_latent_frames = int(state.config["control_latent_frames"])
- while True:
- state.control_event.wait(timeout=0.25)
- state.control_event.clear()
+ def prepare_migration(
+ self,
+ session_id: str,
+ timeout: float | None = None,
+ ) -> ABotWorldMigrationBundle:
+ """Quiesce one session at a chunk boundary and return a CPU snapshot."""
+ state = self._quiesce_migration(session_id, timeout)
+ with self._scheduler_condition:
+ snapshot = self.pipeline.snapshot_interactive_session(state.pipeline_session)
+ return ABotWorldMigrationBundle(
+ snapshot=snapshot,
+ config=dict(state.config),
+ controls=frozenset(state.controls),
+ control_idle_timeout=state.control_idle_timeout,
+ last_control_at=state.last_control_at,
+ next_chunk_index=state.next_chunk_index,
+ next_playout_deadline=state.next_playout_deadline,
+ )
+
+ def prepare_migration_nccl_metadata(self, session_id: str, timeout: float | None = None) -> dict[str, Any]:
+ """Quiesce a session and describe its resident tensors for direct NCCL transfer."""
+ state = self._quiesce_migration(session_id, timeout)
+ session = state.pipeline_session
+ payload = {
+ "prompt_emb": session.prompt_emb,
+ "first_frame_latent": session.first_frame_latent,
+ "self_cache": session.self_cache,
+ "cross_cache": session.cross_cache,
+ "vae_feat_cache": session.vae_decode_state.feat_cache,
+ }
+ skeleton, manifest, leaves = flatten_tensor_tree(payload)
+ return {
+ "session_id": session_id,
+ "tensor_skeleton": skeleton,
+ "tensor_manifest": manifest,
+ "generator_state": session.generator.get_state().detach().cpu(),
+ "vae_feat_idx": list(session.vae_decode_state.feat_idx),
+ "next_latent_frame": session.next_latent_frame,
+ "emitted_frames": session.emitted_frames,
+ "ownership_epoch": session.ownership_epoch,
+ "config": dict(state.config),
+ "controls": sorted(state.controls),
+ "control_idle_timeout": state.control_idle_timeout,
+ "last_control_at": state.last_control_at,
+ "next_chunk_index": state.next_chunk_index,
+ "next_playout_deadline": state.next_playout_deadline,
+ "state_bytes": sum(value.numel() * value.element_size() for value in leaves.values()),
+ "_nccl_tensor_leaves": leaves,
+ }
+
+ def import_migration_nccl(
+ self,
+ metadata: Mapping[str, Any],
+ tensor_leaves: Mapping[tuple[Any, ...], torch.Tensor],
+ *,
+ owner_worker_id: str | None = None,
+ ownership_epoch: int | None = None,
+ ) -> str:
+ """Install target-GPU tensors received by NCCL without a CPU snapshot copy."""
+ payload = rebuild_tensor_tree(metadata["tensor_skeleton"], dict(tensor_leaves))
+ snapshot = ABotWorldSessionSnapshot(
+ session_id=str(metadata["session_id"]),
+ prompt_emb=payload["prompt_emb"],
+ first_frame_latent=payload["first_frame_latent"],
+ self_cache=tuple(payload["self_cache"]),
+ cross_cache=tuple(payload["cross_cache"]),
+ vae_feat_cache=tuple(payload["vae_feat_cache"]),
+ vae_feat_idx=tuple(int(value) for value in metadata["vae_feat_idx"]),
+ generator_state=metadata["generator_state"],
+ next_latent_frame=int(metadata["next_latent_frame"]),
+ emitted_frames=int(metadata["emitted_frames"]),
+ ownership_epoch=int(metadata["ownership_epoch"]),
+ )
+ bundle = ABotWorldMigrationBundle(
+ snapshot=snapshot,
+ config=dict(metadata["config"]),
+ controls=frozenset(metadata["controls"]),
+ control_idle_timeout=float(metadata["control_idle_timeout"]),
+ last_control_at=float(metadata["last_control_at"]),
+ next_chunk_index=int(metadata["next_chunk_index"]),
+ next_playout_deadline=float(metadata["next_playout_deadline"]),
+ )
+ self._ensure_scheduler_started()
+ with self._scheduler_condition:
+ capacity = int(self._capacity_profile["effective_capacity"]) if self._capacity_profile else 1
+ if len(self._sessions) >= capacity:
+ raise RuntimeError(f"ABot retained-session capacity is exhausted (capacity={capacity})")
+ if snapshot.session_id in self._sessions:
+ raise ValueError(f"ABot session {snapshot.session_id!r} already exists")
+ pipeline_session = self.pipeline.restore_interactive_device_snapshot(
+ bundle.snapshot,
+ owner_worker_id=owner_worker_id,
+ ownership_epoch=ownership_epoch,
+ )
+ state = _ABotWorldLiveKitSession(
+ session_id=snapshot.session_id,
+ pipeline_session=pipeline_session,
+ output_queue=queue.Queue(maxsize=self.output_queue_size),
+ control_event=threading.Event(),
+ config=dict(bundle.config),
+ control_idle_timeout=bundle.control_idle_timeout,
+ controls=set(bundle.controls),
+ last_control_at=bundle.last_control_at,
+ next_chunk_index=bundle.next_chunk_index,
+ next_playout_deadline=bundle.next_playout_deadline,
+ ready_since=time.monotonic() if bundle.controls else None,
+ )
+ with self._scheduler_condition:
+ if snapshot.session_id in self._sessions:
+ self.pipeline.close_interactive_session(pipeline_session)
+ raise ValueError(f"ABot session {snapshot.session_id!r} already exists")
+ self._sessions[state.session_id] = state
+ self._round_robin_order.append(state.session_id)
+ self._scheduler_condition.notify_all()
+ return state.session_id
+
+ def _quiesce_migration(self, session_id: str, timeout: float | None) -> _ABotWorldLiveKitSession:
+ effective_timeout = self.close_timeout if timeout is None else timeout
+ deadline = time.monotonic() + effective_timeout
+ with self._scheduler_condition:
+ state = self._sessions.get(session_id)
+ if state is None:
+ raise KeyError(f"Unknown ABot session {session_id!r}")
+ state.migrating = True
+ self._scheduler_condition.notify_all()
+ while state.in_flight or not state.output_queue.empty():
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ state.migrating = False
+ raise TimeoutError("Timed out waiting for ABot migration chunk boundary and output drain")
+ self._scheduler_condition.wait(remaining)
+ return state
+
+ def import_migration(
+ self,
+ bundle: ABotWorldMigrationBundle,
+ *,
+ owner_worker_id: str | None = None,
+ ownership_epoch: int | None = None,
+ ) -> str:
+ """Install a prepared migration bundle without emitting another preview."""
+ self._ensure_scheduler_started()
+ with self._scheduler_condition:
+ capacity = int(self._capacity_profile["effective_capacity"]) if self._capacity_profile else 1
+ if len(self._sessions) >= capacity:
+ raise RuntimeError(f"ABot retained-session capacity is exhausted (capacity={capacity})")
+ if bundle.snapshot.session_id in self._sessions:
+ raise ValueError(f"ABot session {bundle.snapshot.session_id!r} already exists")
+ pipeline_session = self.pipeline.restore_interactive_snapshot(
+ bundle.snapshot,
+ owner_worker_id=owner_worker_id,
+ ownership_epoch=ownership_epoch,
+ )
+ state = _ABotWorldLiveKitSession(
+ session_id=bundle.snapshot.session_id,
+ pipeline_session=pipeline_session,
+ output_queue=queue.Queue(maxsize=self.output_queue_size),
+ control_event=threading.Event(),
+ config=dict(bundle.config),
+ control_idle_timeout=bundle.control_idle_timeout,
+ controls=set(bundle.controls),
+ last_control_at=bundle.last_control_at,
+ next_chunk_index=bundle.next_chunk_index,
+ next_playout_deadline=bundle.next_playout_deadline,
+ ready_since=time.monotonic() if bundle.controls else None,
+ )
+ with self._scheduler_condition:
+ if bundle.snapshot.session_id in self._sessions:
+ self.pipeline.close_interactive_session(pipeline_session)
+ raise ValueError(f"ABot session {bundle.snapshot.session_id!r} already exists")
+ self._sessions[state.session_id] = state
+ self._round_robin_order.append(state.session_id)
+ self._scheduler_condition.notify_all()
+ return state.session_id
+
+ def commit_migration(self, session_id: str) -> None:
+ """Release a source session after target installation and ownership commit."""
+ with self._scheduler_condition:
+ state = self._sessions.get(session_id)
+ if state is None:
+ return
+ if not state.migrating or state.in_flight:
+ raise RuntimeError("ABot source session is not quiescent for migration commit")
+ self._sessions.pop(session_id)
+ self._discard_from_round_robin(session_id)
+ self.pipeline.close_interactive_session(state.pipeline_session)
+
+ def abort_migration(self, session_id: str) -> None:
+ """Resume source scheduling when target installation or ownership commit fails."""
+ with self._scheduler_condition:
+ state = self._sessions.get(session_id)
+ if state is None:
+ return
+ state.migrating = False
+ if state.pipeline_session.lifecycle == ABotWorldSessionLifecycle.MIGRATING:
+ state.pipeline_session.lifecycle = ABotWorldSessionLifecycle.READY
+ state.ready_since = time.monotonic() if state.controls else None
+ self._scheduler_condition.notify_all()
+
+ def pause_scheduler(self, timeout: float | None = None) -> None:
+ """Stop selecting new batches and wait for current work to reach a boundary."""
+ deadline = time.monotonic() + (self.close_timeout if timeout is None else timeout)
+ with self._scheduler_condition:
+ self._scheduler_paused = True
+ self._scheduler_condition.notify_all()
+ while any(state.in_flight for state in self._sessions.values()):
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ self._scheduler_paused = False
+ self._scheduler_condition.notify_all()
+ raise TimeoutError("Timed out pausing ABot scheduler at a chunk boundary")
+ self._scheduler_condition.wait(remaining)
+
+ def resume_scheduler(self) -> None:
+ """Resume admission of ready ABot session batches after a control transaction."""
+ with self._scheduler_condition:
+ self._scheduler_paused = False
+ self._scheduler_condition.notify_all()
+
+ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | int]:
+ """Return raw scheduler facts for service metadata and benchmarks."""
+ with self._sessions_lock:
+ if session_id is None:
+ workload = self._workload_detector.snapshot()
+ return {
+ "scheduler_mode": self.scheduler_mode,
+ "sessions": len(self._sessions),
+ "batches": self._batch_count,
+ "batch_items": self._batch_item_count,
+ "maximum_batch_size": self._maximum_batch_size,
+ "active_sessions": workload.active_sessions,
+ "arrivals_per_second": round(workload.arrivals_per_second, 6),
+ "activation_volatility": round(workload.activation_volatility, 6),
+ "mean_chunk_seconds": round(workload.mean_chunk_seconds, 6),
+ "p95_chunk_seconds": round(workload.p95_chunk_seconds, 6),
+ **self._last_stage_metrics,
+ }
+ state = self._sessions[session_id]
with state.lock:
- if not state.active:
+ return {
+ "scheduler_mode": self.scheduler_mode,
+ "scheduled_chunks": state.scheduled_chunks,
+ "batch_items": state.batch_items,
+ "output_queue_high_watermark": state.output_queue_high_watermark,
+ "dropped_video_payloads": state.dropped_video_payloads,
+ "dropped_status_payloads": state.dropped_status_payloads,
+ "active": int(bool(state.controls) and state.active),
+ "in_flight": int(state.in_flight),
+ "resident": int(state.pipeline_session.is_resident),
+ "emitted_frames": int(getattr(state.pipeline_session, "emitted_frames", 0)),
+ "total_queue_wait_seconds": round(state.total_queue_wait_seconds, 6),
+ "total_compute_seconds": round(state.total_compute_seconds, 6),
+ }
+
+ def _ensure_scheduler_started(self) -> None:
+ with self._scheduler_condition:
+ if self._scheduler_thread is not None and self._scheduler_thread.is_alive():
+ return
+ if self._scheduler_stopping:
+ raise RuntimeError("ABot scheduler is stopping")
+ self._scheduler_thread = threading.Thread(
+ target=self._scheduler_loop,
+ daemon=True,
+ name="abot-world-turboserve",
+ )
+ self._scheduler_thread.start()
+
+ def _scheduler_loop(self) -> None:
+ while True:
+ suspend_candidate: _ABotWorldLiveKitSession | None = None
+ with self._scheduler_condition:
+ if self._scheduler_stopping:
return
- controls = set(state.controls)
- idle_seconds = time.monotonic() - state.last_control_at
- if not controls:
- continue
- if idle_seconds >= state.control_idle_timeout:
- with state.lock:
- state.controls.clear()
+ now = time.monotonic()
+ if self._scheduler_paused:
+ self._scheduler_condition.wait(timeout=0.05)
+ continue
+ for state in self._sessions.values():
+ with state.lock:
+ if state.controls and now - state.last_control_at >= state.control_idle_timeout:
+ state.controls.clear()
+ state.ready_since = None
+ state.pipeline_session.lifecycle = ABotWorldSessionLifecycle.IDLE
+ if (
+ not state.controls
+ and not state.in_flight
+ and state.pipeline_session.is_resident
+ and now - state.last_control_at >= self.idle_suspension_seconds
+ ):
+ suspend_candidate = state
+ break
+ ready = self._ready_sessions(now)
+ if not ready and suspend_candidate is None:
+ self._scheduler_condition.wait(timeout=0.05)
+ continue
+ if (
+ self.scheduler_mode == "batched"
+ and ready
+ and len(ready) < self.max_batch_size
+ and self.batching_window_seconds
+ ):
+ self._scheduler_condition.wait(timeout=self.batching_window_seconds)
+ now = time.monotonic()
+ ready = self._ready_sessions(now)
+ batch = self._select_batch(ready)
+ controls: list[dict[str, bool]] = []
+ if batch:
+ for state in batch:
+ with state.lock:
+ state.in_flight = True
+ controls.append({key: True for key in state.controls})
+
+ if not batch:
+ if suspend_candidate is not None:
+ try:
+ self.pipeline.suspend_interactive_session(suspend_candidate.pipeline_session)
+ except Exception:
+ logger.exception("Failed to suspend ABot session %s", suspend_candidate.session_id)
continue
- try:
- frames = self.pipeline.generate_next_block(
- state.pipeline_session,
- {key: True for key in controls},
- control_latent_frames=control_latent_frames,
+ self._execute_batch(batch, controls)
+
+ def _ready_sessions(self, now: float) -> list[_ABotWorldLiveKitSession]:
+ ready: list[_ABotWorldLiveKitSession] = []
+ for state in self._sessions.values():
+ with state.lock:
+ lossless_blocked = (
+ state.config["delivery_mode"] == "lossless" and state.output_queue.full()
)
- except Exception as exc:
- logger.exception("ABot LiveKit generation failed: session=%s", state.session_id)
- self._put_output(
- state,
- {"type": "error", "error": str(exc), "timestamp": time.time()},
+ if (
+ state.active
+ and state.controls
+ and not state.in_flight
+ and not state.migrating
+ and not lossless_blocked
+ ):
+ if state.ready_since is None:
+ state.ready_since = now
+ ready.append(state)
+ ready.sort(key=lambda state: (state.next_playout_deadline, state.ready_since or now, state.session_id))
+ return ready
+
+ def _select_batch(
+ self,
+ ready: Sequence[_ABotWorldLiveKitSession],
+ ) -> list[_ABotWorldLiveKitSession]:
+ if not ready:
+ return []
+ if self.scheduler_mode == "round_robin":
+ return self._select_round_robin_session(ready)
+ pivot_key = self._batch_key(ready[0])
+ return [state for state in ready if self._batch_key(state) == pivot_key][: self.max_batch_size]
+
+ def _select_round_robin_session(
+ self,
+ ready: Sequence[_ABotWorldLiveKitSession],
+ ) -> list[_ABotWorldLiveKitSession]:
+ """Pick one runnable session, rotating ownership after every block.
+
+ This follows TurboServe's ``LocalRoundRobinStepScheduler`` rather than
+ sorting by a deadline or waiting to form a micro-batch.
+ """
+ ready_by_id = {state.session_id: state for state in ready}
+ for _ in range(len(self._round_robin_order)):
+ session_id = self._round_robin_order.popleft()
+ state = self._sessions.get(session_id)
+ if state is None:
+ continue
+ self._round_robin_order.append(session_id)
+ if session_id in ready_by_id:
+ return [ready_by_id[session_id]]
+ return []
+
+ def _discard_from_round_robin(self, session_id: str) -> None:
+ self._round_robin_order = deque(value for value in self._round_robin_order if value != session_id)
+
+ @staticmethod
+ def _batch_key(state: _ABotWorldLiveKitSession) -> tuple[object, ...]:
+ session = state.pipeline_session
+ local_end = 0
+ if session.self_cache:
+ value = session.self_cache[0]["local_end_index"]
+ local_end = int(value.item()) if isinstance(value, torch.Tensor) else int(value)
+ return (
+ int(state.config["control_latent_frames"]),
+ session.next_latent_frame == 0,
+ local_end,
+ tuple(session.first_frame_latent.shape),
+ session.lifecycle == ABotWorldSessionLifecycle.SUSPENDED,
+ )
+
+ def _execute_batch(
+ self,
+ batch: Sequence[_ABotWorldLiveKitSession],
+ controls: Sequence[dict[str, bool]],
+ ) -> None:
+ started_at = time.monotonic()
+ try:
+ for state in batch:
+ if not state.pipeline_session.is_resident:
+ self.pipeline.restore_interactive_session(state.pipeline_session)
+ frame_counts = {int(state.config["control_latent_frames"]) for state in batch}
+ if len(frame_counts) != 1:
+ raise RuntimeError("ABot scheduler selected an incompatible latent-frame batch")
+ if len(batch) == 1:
+ results = [
+ self.pipeline.generate_next_block(
+ batch[0].pipeline_session,
+ controls[0],
+ control_latent_frames=frame_counts.pop(),
+ )
+ ]
+ else:
+ results = self.pipeline.generate_next_blocks(
+ [state.pipeline_session for state in batch],
+ list(controls),
+ control_latent_frames=frame_counts.pop(),
)
+ except Exception as exc:
+ logger.exception(
+ "ABot TurboServe batch generation failed: sessions=%s",
+ [item.session_id for item in batch],
+ )
+ for state in batch:
with state.lock:
+ state.last_error = str(exc)
state.active = False
- return
- if not frames:
- continue
- if not self._put_output(
- state,
- {
- "type": "chunk",
- "index": state.next_chunk_index,
- "fps": fps,
- "timestamp": time.time(),
- "controls": sorted(controls),
- "frames": frames,
- },
- ):
- return
- state.next_chunk_index += 1
+ state.pipeline_session.lifecycle = ABotWorldSessionLifecycle.FAILED
+ self._put_output(state, {"type": "error", "error": str(exc), "timestamp": time.time()})
+ else:
+ completed_at = time.monotonic()
+ stage_metrics_callback = getattr(self.pipeline, "last_stage_metrics", None)
+ self._last_stage_metrics = dict(stage_metrics_callback()) if callable(stage_metrics_callback) else {}
+ self._workload_detector.record_chunk(completed_at - started_at, completed_at)
+ self._batch_count += 1
+ self._batch_item_count += len(batch)
+ self._maximum_batch_size = max(self._maximum_batch_size, len(batch))
+ for state, frames, applied_controls in zip(batch, results, controls):
+ with state.lock:
+ queue_wait = max(0.0, started_at - (state.ready_since or started_at))
+ state.total_queue_wait_seconds += queue_wait
+ state.total_compute_seconds += completed_at - started_at
+ state.scheduled_chunks += 1
+ state.batch_items += len(batch)
+ payload = {
+ "type": "chunk",
+ "index": state.next_chunk_index,
+ "fps": int(state.config["fps"]),
+ "timestamp": time.time(),
+ "controls": sorted(applied_controls),
+ "frames": frames,
+ "scheduler": {
+ "batch_size": len(batch),
+ "queue_wait_seconds": round(queue_wait, 6),
+ "compute_seconds": round(completed_at - started_at, 6),
+ **self._last_stage_metrics,
+ },
+ }
+ state.next_chunk_index += 1
+ state.next_playout_deadline = max(state.next_playout_deadline, completed_at) + len(frames) / int(
+ state.config["fps"]
+ )
+ state.ready_since = completed_at if state.controls else None
+ self._put_output(state, payload)
+ finally:
+ with self._scheduler_condition:
+ for state in batch:
+ with state.lock:
+ state.in_flight = False
+ self._scheduler_condition.notify_all()
def _put_output(self, state: _ABotWorldLiveKitSession, payload: dict[str, Any]) -> bool:
- """Queue an ordered payload, blocking the producer when playback is behind."""
- while True:
- with state.lock:
- if not state.active:
- return False
- try:
- state.output_queue.put(payload, timeout=0.25)
+ """Enqueue one payload without letting a slow latest-mode client retain stale video."""
+ payload_type = str(payload.get("type", ""))
+ with state.lock:
+ if not state.active and payload_type not in _TERMINAL_OUTPUT_TYPES:
+ return False
+ if not state.output_queue.full():
+ state.output_queue.put_nowait(payload)
+ state.output_queue_high_watermark = max(
+ state.output_queue_high_watermark,
+ state.output_queue.qsize(),
+ )
return True
- except queue.Full:
- continue
+ if state.config.get("delivery_mode", "latest") == "lossless":
+ return False
+
+ discarded = False
+ # Queue consumers run on a different thread; manipulate the backing
+ # deque only while Queue's mutex is held so latest-mode eviction
+ # cannot race with get()/put().
+ with state.output_queue.mutex:
+ queued = state.output_queue.queue
+ if payload_type in _VIDEO_OUTPUT_TYPES:
+ for item in tuple(queued):
+ if item.get("type") in _VIDEO_OUTPUT_TYPES:
+ queued.remove(item)
+ state.output_queue.unfinished_tasks = max(0, state.output_queue.unfinished_tasks - 1)
+ state.output_queue.not_full.notify()
+ state.dropped_video_payloads += 1
+ discarded = True
+ break
+ if not discarded:
+ state.dropped_video_payloads += 1
+ return False
+ elif payload_type in _TERMINAL_OUTPUT_TYPES:
+ for item in tuple(queued):
+ if item.get("type") in _VIDEO_OUTPUT_TYPES:
+ queued.remove(item)
+ state.output_queue.unfinished_tasks = max(0, state.output_queue.unfinished_tasks - 1)
+ state.output_queue.not_full.notify()
+ state.dropped_video_payloads += 1
+ discarded = True
+ break
+ if not discarded and queued:
+ queued.popleft()
+ state.output_queue.unfinished_tasks = max(0, state.output_queue.unfinished_tasks - 1)
+ state.output_queue.not_full.notify()
+ state.dropped_status_payloads += 1
+ else:
+ state.dropped_status_payloads += 1
+ return False
+ state.output_queue.put_nowait(payload)
+ state.output_queue_high_watermark = max(state.output_queue_high_watermark, state.output_queue.qsize())
+ return True
def _session(self, session_id: str) -> _ABotWorldLiveKitSession | None:
with self._sessions_lock:
diff --git a/telefuser/pipelines/abot_world/taew_vae.py b/telefuser/pipelines/abot_world/taew_vae.py
new file mode 100644
index 00000000..78b9a8b9
--- /dev/null
+++ b/telefuser/pipelines/abot_world/taew_vae.py
@@ -0,0 +1,52 @@
+"""Official TAeW2.2 lightweight streaming decoder stage for ABot-World."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+
+from telefuser.core.base_stage import BaseStage, with_model_offload
+from telefuser.core.config import ModelRuntimeConfig
+from telefuser.core.module_manager import ModuleManager
+from telefuser.models.taew2_2 import StreamingTAEHV, TAEHV
+
+
+@dataclass
+class ABotWorldTAEWDecodeState:
+ """Session-owned TAeW streaming queues and temporal MemBlock state."""
+
+ stream: StreamingTAEHV
+
+
+class ABotWorldTAEWDecodeStage(BaseStage):
+ """Decode ABot latent chunks with the official TAeW2.2 streaming decoder."""
+
+ def __init__(self, name: str, module_manager: ModuleManager, model_runtime_config: ModelRuntimeConfig) -> None:
+ super().__init__(name, model_runtime_config)
+ taew = module_manager.fetch_module("abot_world_taew_decoder")
+ if taew is None or not isinstance(taew, TAEHV):
+ raise ValueError("ABot-World requires a loaded abot_world_taew_decoder module")
+ self.taew = taew
+ self.model_names = ["taew"]
+
+ def create_decode_state(self) -> ABotWorldTAEWDecodeState:
+ """Create an isolated stream state while sharing immutable decoder weights."""
+ return ABotWorldTAEWDecodeState(stream=StreamingTAEHV(self.taew))
+
+ @with_model_offload(["taew"])
+ @torch.inference_mode()
+ def warmup_first_frame(self, state: ABotWorldTAEWDecodeState, first_frame_latent: torch.Tensor) -> None:
+ """Populate official TAeW temporal memory from the conditioning latent."""
+ state.stream.reset()
+ latent = first_frame_latent.permute(0, 2, 1, 3, 4).to(self.device, dtype=self.torch_dtype)
+ state.stream.decode(latent)
+
+ @with_model_offload(["taew"])
+ @torch.inference_mode()
+ def decode_chunk(self, latents: torch.Tensor, state: ABotWorldTAEWDecodeState) -> torch.Tensor:
+ """Decode one causal latent chunk to RGB frames in [-1, 1]."""
+ decoded = state.stream.decode(latents.permute(0, 2, 1, 3, 4).to(self.device, dtype=self.torch_dtype))
+ if decoded is None:
+ return latents.new_empty((latents.shape[0], 0, 3, 0, 0))
+ return decoded.mul(2).sub(1).clamp(-1, 1).permute(0, 2, 1, 3, 4).contiguous()
diff --git a/telefuser/pipelines/wan_video/vae.py b/telefuser/pipelines/wan_video/vae.py
index c79077f9..78c70748 100644
--- a/telefuser/pipelines/wan_video/vae.py
+++ b/telefuser/pipelines/wan_video/vae.py
@@ -193,6 +193,7 @@ def decode_video_cached(
latents: torch.Tensor,
is_first_clip: bool,
is_last_clip: bool,
+ decode_state: object | None = None,
) -> torch.Tensor:
"""Decode latents to video frames with persistent feature cache.
@@ -213,9 +214,31 @@ def decode_video_cached(
device=self.device,
is_first_clip=is_first_clip,
is_last_clip=is_last_clip,
+ decode_state=decode_state,
)
return frames
+ @ProfilingContext4Debug("vae decode video cached batch")
+ def decode_video_cached_batch(
+ self,
+ latents: torch.Tensor,
+ is_first_clip: bool,
+ is_last_clip: bool,
+ decode_states: list[object],
+ ) -> torch.Tensor:
+ """Decode compatible session chunks in one model invocation."""
+ method = getattr(self.vae, "cached_decode_batch_withflag", None)
+ if not callable(method):
+ raise NotImplementedError("The loaded VAE does not support session-batched cached decode")
+ with torch.autocast(device_type=self.device_type, dtype=self.torch_dtype):
+ return method(
+ latents,
+ device=self.device,
+ is_first_clip=is_first_clip,
+ is_last_clip=is_last_clip,
+ decode_states=decode_states,
+ )
+
def parallel_models(self):
"""Configure tensor parallelism for VAE."""
self.vae.set_parallelism(self.model_runtime_config.parallel_config.world_size)
diff --git a/telefuser/service/core/stream_pipeline_service.py b/telefuser/service/core/stream_pipeline_service.py
index 175510f8..f898176c 100644
--- a/telefuser/service/core/stream_pipeline_service.py
+++ b/telefuser/service/core/stream_pipeline_service.py
@@ -124,15 +124,25 @@ def __init__(
# -- lifecycle -----------------------------------------------------------
- def start_service(self, ppl_file: str, skip_validation: bool = False, gpu_num: int = 1) -> bool:
- """Load module, call get_service(), detect mode, and start."""
+ def start_service(
+ self,
+ ppl_file: str,
+ skip_validation: bool = False,
+ gpu_num: int = 1,
+ gpu_ids: list[str] | None = None,
+ ) -> bool:
+ """Load a stream factory with the worker's explicit CUDA device assignment."""
if self.is_running:
logger.warning("Stream service is already running")
return True
self._startup_measurement = None
self._runtime_environment = {}
- devices = visible_cuda_devices()
+ devices = (
+ [gpu_id if str(gpu_id).startswith("cuda") else f"cuda:{gpu_id}" for gpu_id in gpu_ids]
+ if gpu_ids is not None
+ else visible_cuda_devices()
+ )
started_at = time.perf_counter()
measurement: RuntimeMeasurement | None = None
try:
@@ -162,10 +172,15 @@ def start_service(self, ppl_file: str, skip_validation: bool = False, gpu_num: i
get_service = self._module.get_service
signature = inspect.signature(get_service)
- accepts_gpu_num = "gpu_num" in signature.parameters or any(
+ accepts_kwargs = any(
parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in signature.parameters.values()
)
- self.service = get_service(gpu_num=gpu_num) if accepts_gpu_num else get_service()
+ factory_kwargs: dict[str, object] = {}
+ if "gpu_num" in signature.parameters or accepts_kwargs:
+ factory_kwargs["gpu_num"] = gpu_num
+ if "gpu_ids" in signature.parameters or accepts_kwargs:
+ factory_kwargs["gpu_ids"] = list(gpu_ids) if gpu_ids is not None else None
+ self.service = get_service(**factory_kwargs)
self.stream_mode = self._detect_mode(self.service)
self.service.start()
self.is_running = True
diff --git a/telefuser/service/livekit/__init__.py b/telefuser/service/livekit/__init__.py
index f6b681ac..c7e176e0 100644
--- a/telefuser/service/livekit/__init__.py
+++ b/telefuser/service/livekit/__init__.py
@@ -3,5 +3,20 @@
from __future__ import annotations
from .config import LiveKitServeConfig
+from .pipeline_router import TurboServePipelineRouter, TurboServeWorkerPipelineView
+from .turboserve import (
+ TurboServeAutoscalingController,
+ TurboServeOwnershipTable,
+ TurboServePlacementController,
+ TurboServeWorkloadDetector,
+)
-__all__ = ["LiveKitServeConfig"]
+__all__ = [
+ "LiveKitServeConfig",
+ "TurboServePipelineRouter",
+ "TurboServeWorkerPipelineView",
+ "TurboServeAutoscalingController",
+ "TurboServeOwnershipTable",
+ "TurboServePlacementController",
+ "TurboServeWorkloadDetector",
+]
diff --git a/telefuser/service/livekit/config.py b/telefuser/service/livekit/config.py
index c58ddb1b..795440d2 100644
--- a/telefuser/service/livekit/config.py
+++ b/telefuser/service/livekit/config.py
@@ -4,7 +4,7 @@
from typing import Literal
-from pydantic import Field, field_validator
+from pydantic import Field, field_validator, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -33,12 +33,36 @@ class LiveKitServeConfig(BaseSettings):
default=None,
description="Semicolon-separated worker GPU groups, for example '0,1;2,3'",
)
- worker_mode: Literal["in-process", "process"] = Field(
+ worker_mode: Literal["in-process", "process", "process-nccl"] = Field(
default="in-process",
description="Worker isolation mode",
)
queue_size: int = Field(default=0, ge=0, le=10000, description="Maximum queued sessions")
+ autoscaling_enabled: bool = Field(default=False, description="Dynamically load configured GPU workers")
+ autoscaling_min_workers: int = Field(default=1, ge=1, le=64)
+ autoscaling_target_utilization: float = Field(default=0.75, gt=0, le=1)
+ autoscaling_hysteresis: float = Field(default=0.10, ge=0, lt=1)
+ autoscaling_cooldown_seconds: float = Field(default=30.0, ge=0)
+ autoscaling_interval_seconds: float = Field(default=5.0, gt=0)
+ turboserve_rebalance_enabled: bool = Field(
+ default=True,
+ description="Rebalance compatible in-process TurboServe sessions at chunk boundaries",
+ )
+ turboserve_migration_bandwidth_gbps: float = Field(
+ default=24.0,
+ gt=0,
+ description="Conservative effective bandwidth used by the migration-aware placement model",
+ )
+ turboserve_migration_penalty: float = Field(
+ default=1.0,
+ ge=0,
+ description="Relative penalty applied to estimated model-session migration time",
+ )
+ turboserve_scale_in_hold_seconds: float = Field(default=5.0, ge=0)
+ turboserve_migration_eta: float = Field(default=0.35, ge=0)
+ turboserve_min_migration_gain_ms: float = Field(default=40.0, ge=0)
+ turboserve_rebalance_iteration_limit: int = Field(default=3, ge=1, le=64)
control_idle_timeout: float = Field(
default=10.0,
gt=0,
@@ -90,6 +114,20 @@ def validate_max_sessions_per_worker(cls: type[LiveKitServeConfig], value: objec
raise ValueError("max_sessions_per_worker must be 'auto' or an integer in [1, 64]")
return value
+ @model_validator(mode="after")
+ def validate_autoscaling_bounds(self) -> LiveKitServeConfig:
+ if self.autoscaling_min_workers > self.num_workers:
+ raise ValueError("autoscaling_min_workers cannot exceed num_workers")
+ if self.autoscaling_enabled and self.num_workers > 1 and self.queue_size == 0:
+ raise ValueError("autoscaling with multiple workers requires queue_size > 0 for cold-start admission")
+ if self.worker_mode == "process-nccl":
+ if self.num_workers < 2:
+ raise ValueError("process-nccl requires at least two model workers")
+ groups = self.worker_gpu_groups()
+ if any(len(group) != 1 for group in groups):
+ raise ValueError("process-nccl requires exactly one GPU id per model worker")
+ return self
+
def session_capacity_limit(self) -> int | None:
"""Return the operator ceiling, or ``None`` for hardware auto-sizing."""
return self.max_sessions_per_worker if isinstance(self.max_sessions_per_worker, int) else None
diff --git a/telefuser/service/livekit/main.py b/telefuser/service/livekit/main.py
index 81b86968..035cb403 100644
--- a/telefuser/service/livekit/main.py
+++ b/telefuser/service/livekit/main.py
@@ -28,6 +28,12 @@ def run_stream_server(
max_sessions_per_worker: int | str | None = None,
worker_gpu_map: str | None = None,
queue_size: int | None = None,
+ autoscaling_enabled: bool | None = None,
+ autoscaling_min_workers: int | None = None,
+ autoscaling_target_utilization: float | None = None,
+ autoscaling_hysteresis: float | None = None,
+ autoscaling_cooldown_seconds: float | None = None,
+ autoscaling_interval_seconds: float | None = None,
control_idle_timeout: float | None = None,
session_timeout: int | None = None,
token_ttl: int | None = None,
@@ -49,6 +55,12 @@ def run_stream_server(
"max_sessions_per_worker": max_sessions_per_worker,
"worker_gpu_map": worker_gpu_map,
"queue_size": queue_size,
+ "autoscaling_enabled": autoscaling_enabled,
+ "autoscaling_min_workers": autoscaling_min_workers,
+ "autoscaling_target_utilization": autoscaling_target_utilization,
+ "autoscaling_hysteresis": autoscaling_hysteresis,
+ "autoscaling_cooldown_seconds": autoscaling_cooldown_seconds,
+ "autoscaling_interval_seconds": autoscaling_interval_seconds,
"control_idle_timeout": control_idle_timeout,
"session_timeout": session_timeout,
"token_ttl": token_ttl,
diff --git a/telefuser/service/livekit/multi_session_worker.py b/telefuser/service/livekit/multi_session_worker.py
index 4f95b697..1f334f26 100644
--- a/telefuser/service/livekit/multi_session_worker.py
+++ b/telefuser/service/livekit/multi_session_worker.py
@@ -51,6 +51,7 @@ def __init__(
pipeline_adapter: LiveKitPipelineAdapter | None = None,
room_client_factory: Callable[[], RoomClient] | None = None,
gpu_num: int = 1,
+ gpu_ids: list[str] | None = None,
) -> None:
self.worker_id = worker_id
self.config = config
@@ -59,7 +60,8 @@ def __init__(
self.event_sink = event_sink or NullWorkerEventSink()
self.pipeline_adapter = pipeline_adapter or LiveKitPipelineAdapter()
self.room_client_factory = room_client_factory or LiveKitRoomClient
- self.gpu_num = gpu_num
+ self.gpu_ids = list(gpu_ids) if gpu_ids is not None else None
+ self.gpu_num = len(self.gpu_ids) if self.gpu_ids else gpu_num
self._sessions: dict[str, LiveKitSessionRunner] = {}
self._session_worker_statuses: dict[str, str] = {}
self._started = False
@@ -73,6 +75,7 @@ async def start(self, *, skip_validation: bool = False) -> None:
self.pipeline_file,
skip_validation=skip_validation,
gpu_num=self.gpu_num,
+ gpu_ids=self.gpu_ids,
)
profile = None
configure_capacity = getattr(self.pipeline_adapter, "configure_session_capacity", None)
diff --git a/telefuser/service/livekit/nccl_process_worker_pool.py b/telefuser/service/livekit/nccl_process_worker_pool.py
new file mode 100644
index 00000000..c82d91ef
--- /dev/null
+++ b/telefuser/service/livekit/nccl_process_worker_pool.py
@@ -0,0 +1,371 @@
+"""Process-isolated ABot model workers with NCCL state migration.
+
+The parent retains LiveKit transport ownership. Child processes retain model
+state, so a committed migration changes only the model route, not the room.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import socket
+import time
+from typing import Any
+
+import torch
+import torch.distributed as dist
+
+from telefuser.service.core.stream_pipeline_service import STREAM_MODE_BIDIRECTIONAL
+from telefuser.service.security.security_validator import SecurityLevel
+from telefuser.utils.logging import logger
+
+from .nccl_transfer import allocate_tensor_tree_leaves, transfer_tensor_leaves_nccl
+from .pipeline_adapter import LiveKitPipelineAdapter
+from .process_worker_pool import ProcessLiveKitWorkerPool, ProcessWorkerSpec, _close_queue
+from .session_registry import SessionRecord
+from .token_service import LiveKitTokenService
+from .turboserve import TurboServeOwnership, TurboServeOwnershipTable
+from .worker import LiveKitWorker
+
+
+class _ProcessPipelineAdapter:
+ stream_mode = STREAM_MODE_BIDIRECTIONAL
+
+ def __init__(self, pool: "NCCLProcessLiveKitWorkerPool", initial_worker_id: str) -> None:
+ self._pool = pool
+ self._initial_worker_id = initial_worker_id
+
+ def create_session(self, config: dict) -> str:
+ session_id = str(config["session_id"])
+ self._pool.create_model_session(self._initial_worker_id, session_id, config)
+ return session_id
+
+ def push_chunk(self, session_id: str, chunk: dict) -> None:
+ self._pool.push_model_chunk(session_id, chunk)
+
+ async def pull_chunks(self, session_id: str):
+ async for chunk in self._pool.pull_model_chunks(session_id):
+ yield chunk
+
+ def close_session(self, session_id: str) -> None:
+ self._pool.close_model_session(session_id)
+
+
+class _ParentTransportSink:
+ def __init__(self, pool: "NCCLProcessLiveKitWorkerPool") -> None:
+ self.pool = pool
+
+ def on_worker_status(self, worker_id: str, status: str) -> None:
+ del worker_id, status
+
+ def on_worker_capacity(self, worker_id: str, capacity: int, profile: dict[str, object] | None = None) -> None:
+ self.pool._event_sink.on_worker_capacity(worker_id, capacity, profile)
+
+ def on_session_status(self, session_id: str, status: str, error: str | None = None) -> None:
+ self.pool._event_sink.on_session_status(session_id, status, error)
+
+ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None:
+ self.pool._event_sink.on_pipeline_session(session_id, pipeline_session_id)
+
+ def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
+ self.pool._transport_finished(session_id)
+ self.pool._event_sink.on_session_finished(worker_id, session_id, error)
+
+
+class NCCLProcessLiveKitWorkerPool(ProcessLiveKitWorkerPool):
+ """TurboServe-compatible parent transport / GPU model-process pool."""
+
+ def __init__(self, specs: list[ProcessWorkerSpec], **kwargs: Any) -> None:
+ super().__init__(specs, **kwargs)
+ self._worker_target = _nccl_model_worker_main
+ self._ownership = TurboServeOwnershipTable()
+ self._model_outputs: dict[str, asyncio.Queue[dict | None]] = {}
+ self._transport_workers: dict[str, LiveKitWorker] = {}
+ self._transport_tasks: dict[str, asyncio.Task[None]] = {}
+ self._migrating_controls: dict[str, list[dict]] = {}
+ self._worker_runtime_metrics: dict[str, dict[str, float | int]] = {}
+ self._session_runtime_metrics: dict[str, dict[str, float | int]] = {}
+ self._migration_total_ms: list[float] = []
+ self._nccl_ranks: dict[str, int] = {}
+ self._migration_lock = asyncio.Lock()
+
+ async def start(self, *, skip_validation: bool = False) -> None:
+ await super().start(skip_validation=skip_validation)
+ if len(self._active_workers) > 1:
+ await self._init_nccl()
+
+ async def scale_to(self, target_workers: int) -> int:
+ """Rebuild the static NCCL communicator around a new replica set."""
+ async with self._migration_lock:
+ if len(self._active_workers) == target_workers:
+ return target_workers
+ if self._nccl_ranks:
+ await asyncio.gather(
+ *(self._request(worker_id, "nccl_destroy") for worker_id in self._nccl_ranks),
+ return_exceptions=True,
+ )
+ self._nccl_ranks.clear()
+ actual = await super().scale_to(target_workers)
+ if actual > 1:
+ await self._init_nccl()
+ return actual
+
+ def start_session(self, record: SessionRecord) -> None:
+ if record.worker_id is None or record.worker_id not in self._active_workers:
+ raise RuntimeError("Model worker is not active")
+ runner = LiveKitWorker(
+ worker_id=record.worker_id,
+ config=self._config,
+ pipeline_file=self._pipeline_file,
+ token_service=LiveKitTokenService(
+ api_key=self._config.livekit_api_key,
+ api_secret=self._config.livekit_api_secret,
+ token_ttl=self._config.token_ttl,
+ ),
+ event_sink=_ParentTransportSink(self),
+ pipeline_adapter=_ProcessPipelineAdapter(self, record.worker_id),
+ )
+ task = asyncio.create_task(runner.run_session(record), name=f"livekit-transport-{record.session_id}")
+ self._transport_workers[record.session_id] = runner
+ self._transport_tasks[record.session_id] = task
+ task.add_done_callback(lambda done, sid=record.session_id: self._transport_task_done(sid, done))
+
+ async def stop_session(self, session_id: str) -> None:
+ runner = self._transport_workers.get(session_id)
+ task = self._transport_tasks.get(session_id)
+ if runner is not None:
+ await runner.stop_session(session_id)
+ if task is not None:
+ with contextlib.suppress(asyncio.TimeoutError, asyncio.CancelledError):
+ await asyncio.wait_for(asyncio.shield(task), timeout=15.0)
+
+ def create_model_session(self, worker_id: str, session_id: str, config: dict) -> None:
+ self._model_outputs[session_id] = asyncio.Queue()
+ self._pipeline_routes[session_id] = worker_id
+ self._session_workers[session_id] = worker_id
+ self._ownership.register(session_id, worker_id)
+ self._send(worker_id, {"type": "model_create", "session_id": session_id, "config": dict(config)})
+
+ def push_model_chunk(self, session_id: str, chunk: dict) -> None:
+ if session_id in self._migrating_controls:
+ self._migrating_controls[session_id].append(dict(chunk))
+ return
+ self._send(self._pipeline_routes[session_id], {"type": "model_push", "session_id": session_id, "chunk": dict(chunk)})
+
+ def close_model_session(self, session_id: str) -> None:
+ worker_id = self._pipeline_routes.pop(session_id, None)
+ self._session_workers.pop(session_id, None)
+ self._ownership.release(session_id)
+ self._migrating_controls.pop(session_id, None)
+ self._session_runtime_metrics.pop(session_id, None)
+ if worker_id in self._active_workers:
+ self._send(worker_id, {"type": "model_close", "session_id": session_id})
+ if (output := self._model_outputs.pop(session_id, None)) is not None:
+ output.put_nowait(None)
+
+ async def pull_model_chunks(self, session_id: str):
+ output = self._model_outputs[session_id]
+ while (payload := await output.get()) is not None:
+ yield payload
+
+ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
+ async with self._migration_lock:
+ source_worker_id = self._pipeline_routes[pipeline_session_id]
+ if source_worker_id == target_worker_id:
+ return self._ownership.owner(pipeline_session_id)
+ if source_worker_id not in self._nccl_ranks or target_worker_id not in self._nccl_ranks:
+ raise RuntimeError("NCCL migration requires initialized source and target workers")
+ token = self._ownership.prepare_migration(pipeline_session_id, source_worker_id, target_worker_id)
+ self._migrating_controls[pipeline_session_id] = []
+ started = time.monotonic()
+ try:
+ await asyncio.gather(
+ self._request(source_worker_id, "scheduler_pause", timeout=300.0),
+ self._request(target_worker_id, "scheduler_pause", timeout=300.0),
+ )
+ exported = await self._request(source_worker_id, "nccl_export", session_id=pipeline_session_id, transfer_id=token.token_id, timeout=300.0)
+ metadata = dict(exported["result"])
+ await self._request(target_worker_id, "nccl_prepare_recv", transfer_id=token.token_id, metadata=metadata, source_rank=self._nccl_ranks[source_worker_id], owner_worker_id=target_worker_id, ownership_epoch=token.source_epoch + 1, timeout=300.0)
+ await asyncio.gather(
+ self._request(source_worker_id, "nccl_send", transfer_id=token.token_id, target_rank=self._nccl_ranks[target_worker_id], timeout=300.0),
+ self._request(target_worker_id, "nccl_recv", transfer_id=token.token_id, source_rank=self._nccl_ranks[source_worker_id], timeout=300.0),
+ )
+ await self._request(source_worker_id, "nccl_commit_source", session_id=pipeline_session_id, timeout=300.0)
+ ownership = self._ownership.commit_migration(token)
+ except Exception:
+ with contextlib.suppress(Exception):
+ await self._request(target_worker_id, "nccl_discard", transfer_id=token.token_id, session_id=pipeline_session_id)
+ with contextlib.suppress(Exception):
+ await self._request(source_worker_id, "nccl_abort_source", session_id=pipeline_session_id, transfer_id=token.token_id)
+ self._ownership.abort_migration(token)
+ for chunk in self._migrating_controls.pop(pipeline_session_id, []):
+ self._send(source_worker_id, {"type": "model_push", "session_id": pipeline_session_id, "chunk": chunk})
+ raise
+ await asyncio.gather(
+ self._request(source_worker_id, "scheduler_resume"),
+ self._request(target_worker_id, "scheduler_resume"),
+ return_exceptions=True,
+ )
+ self._pipeline_routes[pipeline_session_id] = target_worker_id
+ self._session_workers[pipeline_session_id] = target_worker_id
+ for chunk in self._migrating_controls.pop(pipeline_session_id, []):
+ self._send(target_worker_id, {"type": "model_push", "session_id": pipeline_session_id, "chunk": chunk})
+ self._migration_total_ms.append((time.monotonic() - started) * 1000.0)
+ await asyncio.gather(
+ self._request(source_worker_id, "scheduler_resume"),
+ self._request(target_worker_id, "scheduler_resume"),
+ return_exceptions=True,
+ )
+ return ownership
+
+ def turboserve_snapshot(self) -> dict[str, object]:
+ snapshot = super().turboserve_snapshot()
+ snapshot.update({
+ "migration_supported": bool(self._nccl_ranks),
+ "migration_backend": "process_nccl" if self._nccl_ranks else None,
+ "nccl_ranks": dict(self._nccl_ranks),
+ "worker_runtime_metrics": {worker_id: dict(self._worker_runtime_metrics.get(worker_id, {})) for worker_id in self._specs},
+ "session_runtime_metrics": dict(self._session_runtime_metrics),
+ "migration_calibration": {"average_total_ms": sum(self._migration_total_ms) / len(self._migration_total_ms) if self._migration_total_ms else 0.0},
+ })
+ return snapshot
+
+ async def aclose(self) -> None:
+ for session_id in tuple(self._transport_workers):
+ with contextlib.suppress(Exception):
+ await self.stop_session(session_id)
+ if self._nccl_ranks:
+ await asyncio.gather(*(self._request(worker_id, "nccl_destroy") for worker_id in self._nccl_ranks), return_exceptions=True)
+ self._nccl_ranks.clear()
+ await super().aclose()
+
+ async def _init_nccl(self) -> None:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.bind(("127.0.0.1", 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ workers = sorted(self._active_workers)
+ await asyncio.gather(*(self._request(worker_id, "nccl_init", rank=rank, world_size=len(workers), init_method=f"tcp://127.0.0.1:{port}") for rank, worker_id in enumerate(workers)))
+ self._nccl_ranks = {worker_id: rank for rank, worker_id in enumerate(workers)}
+
+ def _transport_finished(self, session_id: str) -> None:
+ self.close_model_session(session_id)
+
+ def _transport_task_done(self, session_id: str, task: asyncio.Task[None]) -> None:
+ self._transport_tasks.pop(session_id, None)
+ self._transport_workers.pop(session_id, None)
+ if not task.cancelled() and task.exception() is not None:
+ logger.warning("LiveKit transport failed: session=%s error=%s", session_id, task.exception())
+
+ def _dispatch_event(self, event: dict[str, Any]) -> None:
+ if event.get("type") == "model_output":
+ metrics = event.get("runtime_metrics")
+ if isinstance(metrics, dict):
+ self._worker_runtime_metrics[event["worker_id"]] = {key: value for key, value in metrics.items() if isinstance(value, int | float)}
+ session_metrics = event.get("session_runtime_metrics")
+ if isinstance(session_metrics, dict):
+ self._session_runtime_metrics[event["session_id"]] = {key: value for key, value in session_metrics.items() if isinstance(value, int | float)}
+ if (output := self._model_outputs.get(event["session_id"])) is not None:
+ output.put_nowait(event["payload"])
+ return
+ super()._dispatch_event(event)
+
+
+def _nccl_model_worker_main(spec: ProcessWorkerSpec, config_values: dict[str, Any], pipeline_file: str, skip_validation: bool, security_name: str | None, commands: Any, events: Any) -> None:
+ try:
+ asyncio.run(_run_nccl_model_worker(spec, pipeline_file, skip_validation, security_name, commands, events))
+ finally:
+ _close_queue(commands, join=False)
+ _close_queue(events)
+
+
+async def _run_nccl_model_worker(spec: ProcessWorkerSpec, pipeline_file: str, skip_validation: bool, security_name: str | None, commands: Any, events: Any) -> None:
+ if not spec.gpu_ids:
+ raise RuntimeError("process-nccl requires one CUDA GPU per worker")
+ torch.cuda.set_device(int(spec.gpu_ids[0]))
+ adapter = LiveKitPipelineAdapter(security_level=SecurityLevel[security_name] if security_name else None)
+ adapter.start(pipeline_file, skip_validation=skip_validation, gpu_num=1, gpu_ids=spec.gpu_ids)
+ if adapter.stream_mode != STREAM_MODE_BIDIRECTIONAL:
+ raise RuntimeError("process-nccl requires a bidirectional pipeline")
+ profile = adapter.configure_session_capacity(None)
+ events.put({"type": "worker_capacity", "worker_id": spec.worker_id, "capacity": int((profile or {}).get("effective_capacity", 1)), "profile": profile})
+ events.put({"type": "worker_status", "worker_id": spec.worker_id, "status": "idle"})
+ events.put({"type": "worker_ready", "worker_id": spec.worker_id})
+ service = adapter.stream_service.service
+ outputs: dict[str, asyncio.Task[None]] = {}
+ outgoing: dict[str, dict[tuple[Any, ...], torch.Tensor]] = {}
+ incoming: dict[str, tuple[dict[str, Any], dict[tuple[Any, ...], torch.Tensor], str, int]] = {}
+
+ async def pump(session_id: str) -> None:
+ async for payload in adapter.pull_chunks(session_id):
+ events.put({"type": "model_output", "worker_id": spec.worker_id, "session_id": session_id, "payload": payload, "runtime_metrics": adapter.runtime_metrics() or {}, "session_runtime_metrics": service.runtime_metrics(session_id)})
+
+ async def result(request_id: str | None, value: Any = True, error: Exception | None = None) -> None:
+ if request_id is not None:
+ events.put({"type": "command_result", "worker_id": spec.worker_id, "request_id": request_id, "result": value, "error": repr(error) if error else None})
+
+ try:
+ while True:
+ command = await asyncio.to_thread(commands.get)
+ request_id, kind = command.get("request_id"), command["type"]
+ try:
+ if kind == "model_create":
+ session_id = adapter.create_session(command["config"])
+ outputs[session_id] = asyncio.create_task(pump(session_id))
+ elif kind == "model_push":
+ adapter.push_chunk(command["session_id"], command["chunk"])
+ elif kind == "model_close":
+ adapter.close_session(command["session_id"])
+ if (task := outputs.pop(command["session_id"], None)):
+ task.cancel()
+ elif kind == "nccl_init":
+ await asyncio.to_thread(dist.init_process_group, "nccl", init_method=command["init_method"], rank=command["rank"], world_size=command["world_size"])
+ elif kind == "scheduler_pause":
+ await asyncio.to_thread(service.pause_scheduler)
+ elif kind == "scheduler_resume":
+ service.resume_scheduler()
+ elif kind == "nccl_export":
+ metadata = service.prepare_migration_nccl_metadata(command["session_id"])
+ outgoing[command["transfer_id"]] = metadata.pop("_nccl_tensor_leaves")
+ await result(request_id, metadata)
+ continue
+ elif kind == "nccl_prepare_recv":
+ metadata = command["metadata"]
+ leaves = allocate_tensor_tree_leaves(metadata["tensor_manifest"], torch.device(f"cuda:{spec.gpu_ids[0]}"))
+ incoming[command["transfer_id"]] = (metadata, leaves, command["owner_worker_id"], command["ownership_epoch"])
+ elif kind == "nccl_send":
+ transfer_tensor_leaves_nccl(outgoing.pop(command["transfer_id"]), peer_rank=command["target_rank"], send=True)
+ elif kind == "nccl_recv":
+ metadata, leaves, owner, epoch = incoming.pop(command["transfer_id"])
+ transfer_tensor_leaves_nccl(leaves, peer_rank=command["source_rank"], send=False)
+ session_id = service.import_migration_nccl(metadata, leaves, owner_worker_id=owner, ownership_epoch=epoch)
+ outputs[session_id] = asyncio.create_task(pump(session_id))
+ elif kind == "nccl_commit_source":
+ service.commit_migration(command["session_id"])
+ if (task := outputs.pop(command["session_id"], None)):
+ task.cancel()
+ elif kind == "nccl_abort_source":
+ service.abort_migration(command["session_id"])
+ outgoing.pop(command.get("transfer_id", ""), None)
+ elif kind == "nccl_discard":
+ incoming.pop(command["transfer_id"], None)
+ if service.has_session(command["session_id"]):
+ service.close_session(command["session_id"])
+ elif kind == "nccl_destroy":
+ if dist.is_initialized():
+ dist.destroy_process_group()
+ elif kind == "shutdown":
+ break
+ else:
+ raise ValueError(f"Unknown process-nccl command {kind!r}")
+ await result(request_id)
+ except Exception as exc:
+ await result(request_id, error=exc)
+ finally:
+ for task in outputs.values():
+ task.cancel()
+ await asyncio.gather(*outputs.values(), return_exceptions=True)
+ if dist.is_initialized():
+ with contextlib.suppress(Exception):
+ dist.destroy_process_group()
+ await adapter.aclose()
diff --git a/telefuser/service/livekit/nccl_transfer.py b/telefuser/service/livekit/nccl_transfer.py
new file mode 100644
index 00000000..d12cf912
--- /dev/null
+++ b/telefuser/service/livekit/nccl_transfer.py
@@ -0,0 +1,86 @@
+"""Tensor-manifest helpers for chunk-boundary NCCL session migration.
+
+The control plane transports only a small Python metadata object. All retained
+model tensors are described by that object, allocated on the target GPU, and
+then copied directly with ``torch.distributed`` point-to-point NCCL operations.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+import torch.distributed as dist
+
+
+def flatten_tensor_tree(value: Any, *, path: tuple[Any, ...] = ()) -> tuple[Any, list[dict[str, Any]], dict[tuple[Any, ...], torch.Tensor]]:
+ """Separate a nested tree into scalar skeleton, tensor manifest, and leaves."""
+ manifest: list[dict[str, Any]] = []
+ leaves: dict[tuple[Any, ...], torch.Tensor] = {}
+
+ def visit(item: Any, item_path: tuple[Any, ...]) -> Any:
+ if isinstance(item, torch.Tensor):
+ tensor = item.detach()
+ manifest.append(
+ {
+ "path": list(item_path),
+ "shape": list(tensor.shape),
+ "dtype": str(tensor.dtype).removeprefix("torch."),
+ }
+ )
+ leaves[item_path] = tensor
+ return {"__tensor__": list(item_path)}
+ if isinstance(item, dict):
+ return {"__dict__": {key: visit(child, item_path + (key,)) for key, child in item.items()}}
+ if isinstance(item, list):
+ return {"__list__": [visit(child, item_path + (index,)) for index, child in enumerate(item)]}
+ if isinstance(item, tuple):
+ return {"__tuple__": [visit(child, item_path + (index,)) for index, child in enumerate(item)]}
+ return {"__value__": item}
+
+ return visit(value, path), manifest, leaves
+
+
+def allocate_tensor_tree_leaves(manifest: list[dict[str, Any]], device: torch.device) -> dict[tuple[Any, ...], torch.Tensor]:
+ """Allocate target GPU tensors from a source manifest."""
+ dtype_table = {name.removeprefix("torch."): value for name, value in vars(torch).items() if isinstance(value, torch.dtype)}
+ leaves: dict[tuple[Any, ...], torch.Tensor] = {}
+ for entry in manifest:
+ dtype_name = str(entry["dtype"])
+ if dtype_name not in dtype_table:
+ raise ValueError(f"Unsupported NCCL tensor dtype {dtype_name!r}")
+ leaves[tuple(entry["path"])] = torch.empty(tuple(entry["shape"]), dtype=dtype_table[dtype_name], device=device)
+ return leaves
+
+
+def rebuild_tensor_tree(skeleton: Any, leaves: dict[tuple[Any, ...], torch.Tensor]) -> Any:
+ """Rebuild a nested value produced by :func:`flatten_tensor_tree`."""
+ if "__tensor__" in skeleton:
+ return leaves[tuple(skeleton["__tensor__"])]
+ if "__dict__" in skeleton:
+ return {key: rebuild_tensor_tree(value, leaves) for key, value in skeleton["__dict__"].items()}
+ if "__list__" in skeleton:
+ return [rebuild_tensor_tree(value, leaves) for value in skeleton["__list__"]]
+ if "__tuple__" in skeleton:
+ return tuple(rebuild_tensor_tree(value, leaves) for value in skeleton["__tuple__"])
+ if "__value__" in skeleton:
+ return skeleton["__value__"]
+ raise ValueError("Invalid NCCL tensor-tree skeleton")
+
+
+def transfer_tensor_leaves_nccl(
+ leaves: dict[tuple[Any, ...], torch.Tensor],
+ *,
+ peer_rank: int,
+ send: bool,
+) -> int:
+ """Synchronously send or receive a manifest's GPU leaves over NCCL."""
+ if not dist.is_available() or not dist.is_initialized():
+ raise RuntimeError("NCCL process group is not initialized")
+ ordered = [leaves[path].contiguous() for path in sorted(leaves, key=lambda value: tuple(map(str, value)))]
+ ops = [dist.P2POp(dist.isend if send else dist.irecv, tensor, peer_rank) for tensor in ordered]
+ if ops:
+ requests = dist.batch_isend_irecv(ops)
+ for request in requests:
+ request.wait()
+ return sum(tensor.numel() * tensor.element_size() for tensor in ordered)
diff --git a/telefuser/service/livekit/pipeline_adapter.py b/telefuser/service/livekit/pipeline_adapter.py
index d9fbd674..54c7090d 100644
--- a/telefuser/service/livekit/pipeline_adapter.py
+++ b/telefuser/service/livekit/pipeline_adapter.py
@@ -15,9 +15,18 @@ class LiveKitPipelineAdapter:
def __init__(self, *, security_level: SecurityLevel | None = None, config: ServerConfig | None = None) -> None:
self.stream_service = StreamPipelineService(security_level=security_level, config=config)
- def start(self, pipeline_file: str, *, skip_validation: bool = False, gpu_num: int = 1) -> None:
- """Load and start a stream pipeline."""
- if not self.stream_service.start_service(pipeline_file, skip_validation=skip_validation, gpu_num=gpu_num):
+ def start(
+ self,
+ pipeline_file: str,
+ *,
+ skip_validation: bool = False,
+ gpu_num: int = 1,
+ gpu_ids: list[str] | None = None,
+ ) -> None:
+ """Load and start a stream pipeline on the assigned CUDA devices."""
+ if not self.stream_service.start_service(
+ pipeline_file, skip_validation=skip_validation, gpu_num=gpu_num, gpu_ids=gpu_ids
+ ):
raise RuntimeError(f"Failed to start LiveKit stream pipeline: {pipeline_file}")
@property
@@ -50,3 +59,9 @@ def close_session(self, session_id: str) -> None:
def configure_session_capacity(self, max_sessions: int | None) -> dict[str, object] | None:
"""Configure and return the loaded pipeline's optional capacity profile."""
return self.stream_service.configure_session_capacity(max_sessions)
+
+ def runtime_metrics(self) -> dict[str, float | int] | None:
+ """Return optional model-service scheduling measurements for placement."""
+ service = getattr(self.stream_service, "service", None)
+ metrics = getattr(service, "runtime_metrics", None)
+ return dict(metrics()) if callable(metrics) else None
diff --git a/telefuser/service/livekit/pipeline_router.py b/telefuser/service/livekit/pipeline_router.py
new file mode 100644
index 00000000..00d99de8
--- /dev/null
+++ b/telefuser/service/livekit/pipeline_router.py
@@ -0,0 +1,257 @@
+"""Chunk-boundary routing and two-phase migration for TurboServe stream sessions."""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from collections.abc import AsyncGenerator
+from dataclasses import asdict
+from typing import Any
+
+from telefuser.utils.logging import logger
+
+from .pipeline_adapter import LiveKitPipelineAdapter
+from .turboserve import TurboServeOwnership, TurboServeOwnershipTable
+
+
+class TurboServePipelineRouter:
+ """Keep LiveKit transport stable while model-session ownership moves between workers."""
+
+ def __init__(self, backends: dict[str, LiveKitPipelineAdapter]) -> None:
+ if not backends:
+ raise ValueError("TurboServe router requires at least one backend")
+ self._backends = dict(backends)
+ self._routes: dict[str, str] = {}
+ self._ownership = TurboServeOwnershipTable()
+ self._pending_chunks: dict[str, list[dict[str, Any]]] = {}
+ self._migration_cleanup_failures = 0
+ self._lock = threading.RLock()
+
+ def worker_view(self, worker_id: str, *, gpu_ids: list[str] | None = None) -> TurboServeWorkerPipelineView:
+ if worker_id not in self._backends:
+ raise KeyError(worker_id)
+ return TurboServeWorkerPipelineView(self, worker_id, gpu_ids=gpu_ids)
+
+ def create_session(self, worker_id: str, config: dict[str, Any]) -> str:
+ backend = self._backends[worker_id]
+ pipeline_session_id = backend.create_session(config)
+ try:
+ with self._lock:
+ if pipeline_session_id in self._routes:
+ raise ValueError(f"Pipeline session {pipeline_session_id!r} is already routed")
+ self._ownership.register(pipeline_session_id, worker_id)
+ self._routes[pipeline_session_id] = worker_id
+ except Exception:
+ backend.close_session(pipeline_session_id)
+ raise
+ return pipeline_session_id
+
+ def push_chunk(self, pipeline_session_id: str, chunk: dict[str, Any]) -> None:
+ with self._lock:
+ pending = self._pending_chunks.get(pipeline_session_id)
+ if pending is not None:
+ pending.append(dict(chunk))
+ return
+ worker_id = self._routes.get(pipeline_session_id)
+ if worker_id is None:
+ raise KeyError(f"Unknown routed pipeline session {pipeline_session_id!r}")
+ self._backends[worker_id].push_chunk(pipeline_session_id, chunk)
+
+ async def pull_chunks(self, pipeline_session_id: str) -> AsyncGenerator[dict, None]:
+ """Continue on a new backend after the source generator closes during commit."""
+ while True:
+ with self._lock:
+ worker_id = self._routes.get(pipeline_session_id)
+ if worker_id is None:
+ return
+ backend = self._backends[worker_id]
+ async for chunk in backend.pull_chunks(pipeline_session_id):
+ yield chunk
+ with self._lock:
+ next_worker_id = self._routes.get(pipeline_session_id)
+ if next_worker_id is None or next_worker_id == worker_id:
+ return
+
+ def close_session(self, pipeline_session_id: str) -> None:
+ with self._lock:
+ worker_id = self._routes.pop(pipeline_session_id, None)
+ if worker_id is None:
+ return
+ self._ownership.release(pipeline_session_id)
+ self._backends[worker_id].close_session(pipeline_session_id)
+
+ def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
+ """Atomically switch ownership while buffering controls received during transfer."""
+ with self._lock:
+ source_worker_id = self._routes[pipeline_session_id]
+ if source_worker_id == target_worker_id:
+ return self._ownership.owner(pipeline_session_id)
+ if target_worker_id not in self._backends:
+ raise KeyError(target_worker_id)
+ token = self._ownership.prepare_migration(
+ pipeline_session_id,
+ source_worker_id,
+ target_worker_id,
+ )
+ self._pending_chunks[pipeline_session_id] = []
+
+ imported = False
+ prepared = False
+ source = None
+ target = None
+ try:
+ source = self._migration_service(source_worker_id)
+ target = self._migration_service(target_worker_id)
+ bundle = source.prepare_migration(pipeline_session_id)
+ prepared = True
+ target.import_migration(
+ bundle,
+ owner_worker_id=target_worker_id,
+ ownership_epoch=token.source_epoch + 1,
+ )
+ imported = True
+ except Exception:
+ if imported and target is not None:
+ target.close_session(pipeline_session_id)
+ if prepared and source is not None:
+ source.abort_migration(pipeline_session_id)
+ with self._lock:
+ pending = self._pending_chunks.pop(pipeline_session_id, [])
+ self._ownership.abort_migration(token)
+ for chunk in pending:
+ self._backends[source_worker_id].push_chunk(pipeline_session_id, chunk)
+ raise
+
+ with self._lock:
+ ownership = self._ownership.commit_migration(token)
+ self._routes[pipeline_session_id] = target_worker_id
+ pending = self._pending_chunks.pop(pipeline_session_id)
+ for chunk in pending:
+ self._backends[target_worker_id].push_chunk(pipeline_session_id, chunk)
+
+ try:
+ source.commit_migration(pipeline_session_id)
+ except Exception as exc:
+ # Ownership is already committed. Preserve the live target and report a source cleanup leak.
+ with self._lock:
+ self._migration_cleanup_failures += 1
+ logger.warning(
+ f"TurboServe source cleanup failed after committed migration: "
+ f"session={pipeline_session_id} source={source_worker_id} error={exc}"
+ )
+ return ownership
+
+ def snapshot(self) -> dict[str, object]:
+ with self._lock:
+ routes = dict(self._routes)
+ ownership = {session_id: asdict(self._ownership.owner(session_id)) for session_id in routes}
+ retained_by_worker = {worker_id: 0 for worker_id in self._backends}
+ for worker_id in routes.values():
+ retained_by_worker[worker_id] += 1
+ runtime_metrics: dict[str, dict[str, float | int]] = {}
+ for worker_id, backend in self._backends.items():
+ metrics = getattr(backend, "runtime_metrics", None)
+ if not callable(metrics):
+ continue
+ try:
+ value = metrics()
+ except Exception as exc:
+ logger.warning(f"Unable to read TurboServe runtime metrics: worker={worker_id} error={exc}")
+ continue
+ if value is not None:
+ runtime_metrics[worker_id] = dict(value)
+ return {
+ "routes": routes,
+ "ownership": ownership,
+ "retained_sessions_by_worker": retained_by_worker,
+ "worker_runtime_metrics": runtime_metrics,
+ "migration_supported": True,
+ "migration_cleanup_failures": self._migration_cleanup_failures,
+ }
+
+ def _backend_for(self, pipeline_session_id: str) -> LiveKitPipelineAdapter:
+ with self._lock:
+ worker_id = self._routes.get(pipeline_session_id)
+ if worker_id is None:
+ raise KeyError(f"Unknown routed pipeline session {pipeline_session_id!r}")
+ return self._backends[worker_id]
+
+ def _migration_service(self, worker_id: str) -> object:
+ backend = self._backends[worker_id]
+ stream_service = getattr(backend, "stream_service", None)
+ service = getattr(stream_service, "service", None)
+ required = ("prepare_migration", "import_migration", "commit_migration", "abort_migration", "close_session")
+ if service is None or any(not callable(getattr(service, name, None)) for name in required):
+ raise RuntimeError(f"Worker {worker_id} pipeline does not implement TurboServe migration")
+ return service
+
+
+class TurboServeWorkerPipelineView:
+ """Worker-scoped facade backed by a shared session ownership router."""
+
+ def __init__(
+ self,
+ router: TurboServePipelineRouter,
+ worker_id: str,
+ *,
+ gpu_ids: list[str] | None = None,
+ ) -> None:
+ self._router = router
+ self.worker_id = worker_id
+ self.gpu_ids = list(gpu_ids) if gpu_ids is not None else None
+
+ @property
+ def _backend(self) -> LiveKitPipelineAdapter:
+ return self._router._backends[self.worker_id]
+
+ @property
+ def stream_mode(self) -> str | None:
+ return self._backend.stream_mode
+
+ def start(
+ self,
+ pipeline_file: str,
+ *,
+ skip_validation: bool = False,
+ gpu_num: int = 1,
+ gpu_ids: list[str] | None = None,
+ ) -> None:
+ assigned = self.gpu_ids if gpu_ids is None else gpu_ids
+ self._backend.start(
+ pipeline_file,
+ skip_validation=skip_validation,
+ gpu_num=gpu_num,
+ gpu_ids=assigned,
+ )
+
+ async def aclose(self) -> None:
+ await self._backend.aclose()
+
+ def configure_session_capacity(self, max_sessions: int | None) -> dict[str, object] | None:
+ return self._backend.configure_session_capacity(max_sessions)
+
+ def create_session(self, config: dict[str, Any]) -> str:
+ return self._router.create_session(self.worker_id, config)
+
+ def push_chunk(self, pipeline_session_id: str, chunk: dict[str, Any]) -> None:
+ self._router.push_chunk(pipeline_session_id, chunk)
+
+ async def pull_chunks(self, pipeline_session_id: str) -> AsyncGenerator[dict, None]:
+ async for chunk in self._router.pull_chunks(pipeline_session_id):
+ yield chunk
+
+ def close_session(self, pipeline_session_id: str) -> None:
+ self._router.close_session(pipeline_session_id)
+
+ async def stream_task(self, config: dict[str, Any]) -> AsyncGenerator[dict, None]:
+ async for chunk in self._backend.stream_task(config):
+ yield chunk
+
+
+class TurboServeRoutedWorkerPoolMixin:
+ """Small protocol helper used only for runtime feature detection."""
+
+ router: TurboServePipelineRouter
+
+ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
+ return await asyncio.to_thread(self.router.migrate_session, pipeline_session_id, target_worker_id)
diff --git a/telefuser/service/livekit/process_worker_pool.py b/telefuser/service/livekit/process_worker_pool.py
new file mode 100644
index 00000000..20bcc4ee
--- /dev/null
+++ b/telefuser/service/livekit/process_worker_pool.py
@@ -0,0 +1,639 @@
+"""Process-isolated LiveKit model workers."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import multiprocessing
+import uuid
+from dataclasses import dataclass
+from multiprocessing.context import BaseContext
+from multiprocessing.process import BaseProcess
+from typing import Any
+
+from telefuser.service.security.security_validator import SecurityLevel
+from telefuser.utils.logging import logger
+
+from .config import LiveKitServeConfig
+from .session_registry import SessionRecord
+from .worker import WorkerEventSink
+
+_WORKER_SHUTDOWN_TIMEOUT_SECONDS = 60.0
+_WORKER_START_TIMEOUT_SECONDS = 600.0
+_COMMAND_TIMEOUT_SECONDS = 15.0
+_PROCESS_JOIN_TIMEOUT_SECONDS = 10.0
+_PROCESS_MONITOR_INTERVAL_SECONDS = 0.5
+
+
+@dataclass(frozen=True)
+class ProcessWorkerSpec:
+ """Serializable configuration for one isolated model worker."""
+
+ worker_id: str
+ gpu_ids: list[str]
+
+
+@dataclass
+class _ProcessHandle:
+ spec: ProcessWorkerSpec
+ commands: Any
+ process: BaseProcess
+
+
+class ProcessLiveKitWorkerPool:
+ """Run one model replica per spawned process and keep the API process model-free."""
+
+ def __init__(
+ self,
+ specs: list[ProcessWorkerSpec],
+ *,
+ config: LiveKitServeConfig,
+ pipeline_file: str,
+ event_sink: WorkerEventSink,
+ security_level: SecurityLevel | None = None,
+ initial_workers: int | None = None,
+ context: BaseContext | None = None,
+ worker_target: Any = None,
+ ) -> None:
+ if not specs:
+ raise ValueError("Process worker pool requires at least one worker")
+ if initial_workers is not None and not 1 <= initial_workers <= len(specs):
+ raise ValueError("initial_workers must be within the configured worker pool")
+ self._specs = {spec.worker_id: spec for spec in specs}
+ self._config = config
+ self._pipeline_file = pipeline_file
+ self._event_sink = event_sink
+ self._security_level = security_level
+ self._initial_workers = initial_workers
+ self._context = context or multiprocessing.get_context("spawn")
+ self._worker_target = worker_target or _process_worker_main
+ self._events = self._context.Queue()
+ self._handles: dict[str, _ProcessHandle] = {}
+ self._active_workers: set[str] = set()
+ self._stopping_workers: set[str] = set()
+ self._session_workers: dict[str, str] = {}
+ self._pipeline_routes: dict[str, str] = {}
+ self._pending: dict[str, asyncio.Future[dict[str, Any]]] = {}
+ self._pending_workers: dict[str, str] = {}
+ self._startup: dict[str, asyncio.Future[None]] = {}
+ self._event_task: asyncio.Task | None = None
+ self._monitor_task: asyncio.Task | None = None
+ self._scale_lock = asyncio.Lock()
+ self._started = False
+ self._closing = False
+ self._skip_validation = False
+
+ async def start(self, *, skip_validation: bool = False) -> None:
+ """Spawn and wait for the configured initial replica set."""
+ if self._started:
+ return
+ self._started = True
+ self._skip_validation = skip_validation
+ self._event_task = asyncio.create_task(self._event_loop(), name="livekit-process-events")
+ self._monitor_task = asyncio.create_task(self._monitor_loop(), name="livekit-process-monitor")
+ target = self._initial_workers or len(self._specs)
+ try:
+ await self.scale_to(target)
+ except Exception:
+ await self.aclose()
+ raise
+ for worker_id in self._specs:
+ if worker_id not in self._active_workers:
+ self._event_sink.on_worker_status(worker_id, "stopped")
+
+ def start_session(self, record: SessionRecord) -> None:
+ """Submit one retained LiveKit session to its owning process."""
+ if not self._started or self._closing:
+ raise RuntimeError("LiveKit process worker pool is not accepting sessions")
+ if record.worker_id is None:
+ raise RuntimeError(f"Session {record.session_id} has no assigned worker")
+ if record.worker_id not in self._active_workers:
+ raise RuntimeError(f"Worker {record.worker_id} is not active")
+ if record.session_id in self._session_workers:
+ raise RuntimeError(f"Session {record.session_id} is already running")
+ self._session_workers[record.session_id] = record.worker_id
+ try:
+ self._send(
+ record.worker_id,
+ {
+ "type": "start_session",
+ "record": record.model_dump(mode="python"),
+ },
+ )
+ except Exception:
+ self._session_workers.pop(record.session_id, None)
+ raise
+
+ async def stop_session(self, session_id: str) -> None:
+ """Stop a child-owned room and wait for model-state cleanup."""
+ worker_id = self._session_workers.get(session_id)
+ if worker_id is None or worker_id not in self._active_workers:
+ return
+ await self._request(worker_id, "stop_session", session_id=session_id)
+
+ async def scale_to(self, target_workers: int) -> int:
+ """Spawn replicas or retire idle replicas until the target is reached."""
+ if not self._started:
+ raise RuntimeError("LiveKit process worker pool is not started")
+ if not 1 <= target_workers <= len(self._specs):
+ raise ValueError("target_workers must be within the configured worker pool")
+ async with self._scale_lock:
+ while len(self._active_workers) < target_workers:
+ worker_id = next(worker_id for worker_id in self._specs if worker_id not in self._active_workers)
+ await self._start_worker(worker_id)
+ while len(self._active_workers) > target_workers:
+ candidate = self._scale_in_candidate()
+ if candidate is None:
+ break
+ await self._stop_worker(candidate)
+ return len(self._active_workers)
+
+ def active_worker_count(self) -> int:
+ return len(self._active_workers)
+
+ def turboserve_snapshot(self) -> dict[str, object]:
+ retained = {worker_id: 0 for worker_id in self._specs}
+ for worker_id in self._session_workers.values():
+ retained[worker_id] += 1
+ return {
+ "routes": dict(self._pipeline_routes),
+ "retained_sessions_by_worker": retained,
+ "active_workers": sorted(self._active_workers),
+ "configured_workers": len(self._specs),
+ "migration_supported": False,
+ }
+
+ async def aclose(self) -> None:
+ """Stop children, terminate unresponsive processes, and close IPC resources."""
+ if self._closing:
+ return
+ self._closing = True
+ for worker_id in tuple(self._active_workers):
+ with contextlib.suppress(Exception):
+ await self._stop_worker(worker_id)
+ if self._monitor_task is not None:
+ self._monitor_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._monitor_task
+ if self._event_task is not None:
+ self._events.put({"type": "pool_stop"})
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._event_task
+ self._monitor_task = None
+ self._event_task = None
+ for future in self._pending.values():
+ if not future.done():
+ future.set_exception(RuntimeError("LiveKit process worker pool closed"))
+ self._pending.clear()
+ self._pending_workers.clear()
+ self._startup.clear()
+ _close_queue(self._events)
+ self._started = False
+ self._closing = False
+
+ async def _start_worker(self, worker_id: str) -> None:
+ spec = self._specs[worker_id]
+ commands = self._context.Queue()
+ security_name = self._security_level.name if self._security_level is not None else None
+ process = self._context.Process(
+ target=self._worker_target,
+ name=f"telefuser-{worker_id}",
+ args=(
+ spec,
+ self._config.model_dump(mode="python"),
+ self._pipeline_file,
+ self._skip_validation,
+ security_name,
+ commands,
+ self._events,
+ ),
+ )
+ future = asyncio.get_running_loop().create_future()
+ self._startup[worker_id] = future
+ self._handles[worker_id] = _ProcessHandle(spec, commands, process)
+ try:
+ process.start()
+ except Exception:
+ self._startup.pop(worker_id, None)
+ self._terminate_handle(worker_id)
+ raise
+ try:
+ await asyncio.wait_for(future, timeout=_WORKER_START_TIMEOUT_SECONDS)
+ except Exception:
+ self._terminate_handle(worker_id)
+ raise
+ finally:
+ self._startup.pop(worker_id, None)
+ self._active_workers.add(worker_id)
+
+ async def _stop_worker(self, worker_id: str) -> None:
+ if worker_id not in self._handles:
+ return
+ self._stopping_workers.add(worker_id)
+ try:
+ try:
+ await self._request(worker_id, "shutdown", timeout=_WORKER_SHUTDOWN_TIMEOUT_SECONDS)
+ except Exception as exc:
+ logger.warning(f"Process worker did not shut down cleanly: worker={worker_id} error={exc}")
+ handle = self._handles.get(worker_id)
+ if handle is not None:
+ await asyncio.to_thread(handle.process.join, _PROCESS_JOIN_TIMEOUT_SECONDS)
+ if handle.process.is_alive():
+ self._terminate_handle(worker_id)
+ else:
+ self._discard_handle(worker_id)
+ finally:
+ self._active_workers.discard(worker_id)
+ self._stopping_workers.discard(worker_id)
+
+ async def _request(
+ self,
+ worker_id: str,
+ command_type: str,
+ *,
+ timeout: float = _COMMAND_TIMEOUT_SECONDS,
+ **payload: Any,
+ ) -> dict[str, Any]:
+ request_id = str(uuid.uuid4())
+ future = asyncio.get_running_loop().create_future()
+ self._pending[request_id] = future
+ self._pending_workers[request_id] = worker_id
+ try:
+ self._send(worker_id, {"type": command_type, "request_id": request_id, **payload})
+ return await asyncio.wait_for(future, timeout=timeout)
+ finally:
+ self._pending.pop(request_id, None)
+ self._pending_workers.pop(request_id, None)
+
+ def _send(self, worker_id: str, command: dict[str, Any]) -> None:
+ handle = self._handles.get(worker_id)
+ if handle is None or not handle.process.is_alive():
+ raise RuntimeError(f"Worker process {worker_id} is not alive")
+ handle.commands.put(command)
+
+ async def _event_loop(self) -> None:
+ while True:
+ event = await asyncio.to_thread(self._events.get)
+ if event.get("type") == "pool_stop":
+ return
+ self._dispatch_event(event)
+
+ def _dispatch_event(self, event: dict[str, Any]) -> None:
+ event_type = event.get("type")
+ worker_id = event.get("worker_id")
+ if event_type == "worker_ready":
+ future = self._startup.get(worker_id)
+ if future is not None and not future.done():
+ future.set_result(None)
+ return
+ if event_type == "worker_start_failed":
+ error = RuntimeError(str(event.get("error", "worker startup failed")))
+ future = self._startup.get(worker_id)
+ if future is not None and not future.done():
+ future.set_exception(error)
+ self._event_sink.on_worker_status(worker_id, "failed")
+ return
+ if event_type == "command_result":
+ future = self._pending.get(event.get("request_id"))
+ if future is not None and not future.done():
+ if event.get("error") is not None:
+ future.set_exception(RuntimeError(str(event["error"])))
+ else:
+ future.set_result(event)
+ return
+ if event_type == "worker_status":
+ self._event_sink.on_worker_status(worker_id, event["status"])
+ elif event_type == "worker_capacity":
+ self._event_sink.on_worker_capacity(worker_id, int(event["capacity"]), event.get("profile"))
+ elif event_type == "session_status":
+ self._event_sink.on_session_status(event["session_id"], event["status"], event.get("error"))
+ elif event_type == "pipeline_session":
+ session_id = event["session_id"]
+ pipeline_session_id = event["pipeline_session_id"]
+ self._pipeline_routes[pipeline_session_id] = worker_id
+ self._event_sink.on_pipeline_session(session_id, pipeline_session_id)
+ elif event_type == "session_finished":
+ session_id = event["session_id"]
+ self._session_workers.pop(session_id, None)
+ for pipeline_session_id, owner in tuple(self._pipeline_routes.items()):
+ if owner == worker_id and pipeline_session_id == event.get("pipeline_session_id"):
+ self._pipeline_routes.pop(pipeline_session_id, None)
+ self._event_sink.on_session_finished(worker_id, session_id, event.get("error"))
+
+ async def _monitor_loop(self) -> None:
+ while True:
+ await asyncio.sleep(_PROCESS_MONITOR_INTERVAL_SECONDS)
+ for worker_id, handle in tuple(self._handles.items()):
+ if handle.process.is_alive():
+ continue
+ startup = self._startup.get(worker_id)
+ if startup is not None:
+ if not startup.done():
+ startup.set_exception(
+ RuntimeError(f"Worker process exited during startup with code {handle.process.exitcode}")
+ )
+ self._discard_handle(worker_id)
+ continue
+ if worker_id in self._stopping_workers:
+ error = None
+ if handle.process.exitcode != 0:
+ error = f"Worker process exited during shutdown with code {handle.process.exitcode}"
+ self._resolve_pending_requests(worker_id, error)
+ self._discard_handle(worker_id)
+ continue
+ if worker_id not in self._active_workers:
+ continue
+ self._handle_unexpected_exit(worker_id, handle.process.exitcode)
+
+ def _handle_unexpected_exit(self, worker_id: str, exitcode: int | None) -> None:
+ self._active_workers.discard(worker_id)
+ self._event_sink.on_worker_status(worker_id, "failed")
+ error = f"Worker process exited unexpectedly with code {exitcode}"
+ self._resolve_pending_requests(worker_id, error)
+ for session_id, owner in tuple(self._session_workers.items()):
+ if owner == worker_id:
+ self._session_workers.pop(session_id, None)
+ self._event_sink.on_session_finished(worker_id, session_id, error)
+ for pipeline_session_id, owner in tuple(self._pipeline_routes.items()):
+ if owner == worker_id:
+ self._pipeline_routes.pop(pipeline_session_id, None)
+ self._discard_handle(worker_id)
+
+ def _scale_in_candidate(self) -> str | None:
+ busy = set(self._session_workers.values())
+ return next(
+ (
+ worker_id
+ for worker_id in reversed(tuple(self._specs))
+ if worker_id in self._active_workers and worker_id not in busy
+ ),
+ None,
+ )
+
+ def _resolve_pending_requests(self, worker_id: str, error: str | None) -> None:
+ for request_id, owner in tuple(self._pending_workers.items()):
+ if owner != worker_id:
+ continue
+ future = self._pending.get(request_id)
+ if future is None or future.done():
+ continue
+ if error is None:
+ future.set_result({"type": "command_result", "worker_id": worker_id})
+ else:
+ future.set_exception(RuntimeError(error))
+
+ def _terminate_handle(self, worker_id: str) -> None:
+ handle = self._handles.get(worker_id)
+ if handle is None:
+ return
+ if handle.process.is_alive():
+ handle.process.terminate()
+ handle.process.join(_PROCESS_JOIN_TIMEOUT_SECONDS)
+ if handle.process.is_alive() and hasattr(handle.process, "kill"):
+ handle.process.kill()
+ handle.process.join(_PROCESS_JOIN_TIMEOUT_SECONDS)
+ self._discard_handle(worker_id)
+
+ def _discard_handle(self, worker_id: str) -> None:
+ handle = self._handles.pop(worker_id, None)
+ if handle is not None:
+ _close_queue(handle.commands, join=False)
+
+
+def _process_worker_main(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
+ """Child entrypoint; imports model-facing modules only after process spawn."""
+ try:
+ asyncio.run(
+ _run_process_worker(
+ spec,
+ config_values,
+ pipeline_file,
+ skip_validation,
+ security_name,
+ commands,
+ events,
+ )
+ )
+ except BaseException as exc:
+ events.put({"type": "worker_start_failed", "worker_id": spec.worker_id, "error": repr(exc)})
+ raise
+ finally:
+ _close_queue(commands, join=False)
+ _close_queue(events)
+
+
+def _close_queue(ipc_queue: Any, *, join: bool = True) -> None:
+ """Close one multiprocessing queue and wait for its local feeder thread."""
+ if not join:
+ with contextlib.suppress(Exception):
+ ipc_queue.cancel_join_thread()
+ with contextlib.suppress(Exception):
+ ipc_queue.close()
+ if join:
+ with contextlib.suppress(Exception):
+ ipc_queue.join_thread()
+
+
+async def _run_process_worker(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
+ from .multi_session_worker import MultiSessionLiveKitWorker
+ from .pipeline_adapter import LiveKitPipelineAdapter
+ from .token_service import LiveKitTokenService
+
+ config = LiveKitServeConfig(**config_values)
+ security_level = SecurityLevel[security_name] if security_name is not None else None
+ sink = _ProcessEventSink(spec.worker_id, events)
+ token_service = LiveKitTokenService(
+ api_key=config.livekit_api_key,
+ api_secret=config.livekit_api_secret,
+ token_ttl=config.token_ttl,
+ )
+ worker = MultiSessionLiveKitWorker(
+ worker_id=spec.worker_id,
+ config=config,
+ pipeline_file=pipeline_file,
+ token_service=token_service,
+ event_sink=sink,
+ pipeline_adapter=LiveKitPipelineAdapter(security_level=security_level),
+ gpu_num=max(1, len(spec.gpu_ids)),
+ gpu_ids=spec.gpu_ids or None,
+ )
+ tasks: dict[str, asyncio.Task[None]] = {}
+ await worker.start(skip_validation=skip_validation)
+ events.put({"type": "worker_ready", "worker_id": spec.worker_id})
+ try:
+ while True:
+ command = await asyncio.to_thread(commands.get)
+ command_type = command["type"]
+ request_id = command.get("request_id")
+ try:
+ if command_type == "start_session":
+ record = SessionRecord.model_validate(command["record"])
+ task = asyncio.create_task(worker.run_session(record), name=f"room-{record.session_id}")
+ tasks[record.session_id] = task
+ task.add_done_callback(
+ lambda done, sid=record.session_id: _consume_session_task(
+ tasks,
+ sid,
+ done,
+ worker_id=spec.worker_id,
+ events=events,
+ )
+ )
+ elif command_type == "stop_session":
+ session_id = command["session_id"]
+ await worker.stop_session(session_id)
+ task = tasks.get(session_id)
+ if task is not None:
+ try:
+ await asyncio.wait_for(asyncio.shield(task), timeout=_COMMAND_TIMEOUT_SECONDS)
+ except asyncio.TimeoutError:
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+ elif command_type == "shutdown":
+ for session_id in tuple(tasks):
+ await worker.stop_session(session_id)
+ pending = set()
+ if tasks:
+ _, pending = await asyncio.wait(tasks.values(), timeout=_COMMAND_TIMEOUT_SECONDS)
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
+ await worker.stop()
+ else:
+ raise ValueError(f"Unknown process-worker command {command_type!r}")
+ except Exception as exc:
+ if request_id is not None:
+ events.put(
+ {
+ "type": "command_result",
+ "worker_id": spec.worker_id,
+ "request_id": request_id,
+ "error": repr(exc),
+ }
+ )
+ if command_type == "shutdown":
+ raise
+ else:
+ if request_id is not None:
+ events.put(
+ {
+ "type": "command_result",
+ "worker_id": spec.worker_id,
+ "request_id": request_id,
+ }
+ )
+ if command_type == "shutdown":
+ return
+ finally:
+ for task in tasks.values():
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*tasks.values(), return_exceptions=True)
+ with contextlib.suppress(Exception):
+ await worker.stop()
+
+
+def _consume_session_task(
+ tasks: dict[str, asyncio.Task[None]],
+ session_id: str,
+ task: asyncio.Task[None],
+ *,
+ worker_id: str,
+ events: Any,
+) -> None:
+ tasks.pop(session_id, None)
+ if task.cancelled():
+ return
+ error = task.exception()
+ if error is None:
+ return
+ events.put(
+ {
+ "type": "session_status",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ "status": "failed",
+ "error": repr(error),
+ }
+ )
+ events.put(
+ {
+ "type": "session_finished",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ "pipeline_session_id": None,
+ "error": repr(error),
+ }
+ )
+
+
+class _ProcessEventSink:
+ """Serialize child lifecycle callbacks onto the parent event queue."""
+
+ def __init__(self, worker_id: str, events: Any) -> None:
+ self.worker_id = worker_id
+ self.events = events
+ self._pipeline_sessions: dict[str, str] = {}
+
+ def on_worker_status(self, worker_id: str, status: str) -> None:
+ self.events.put({"type": "worker_status", "worker_id": worker_id, "status": status})
+
+ def on_worker_capacity(self, worker_id: str, capacity: int, profile: dict[str, object] | None = None) -> None:
+ self.events.put(
+ {
+ "type": "worker_capacity",
+ "worker_id": worker_id,
+ "capacity": capacity,
+ "profile": profile,
+ }
+ )
+
+ def on_session_status(self, session_id: str, status: str, error: str | None = None) -> None:
+ self.events.put(
+ {
+ "type": "session_status",
+ "worker_id": self.worker_id,
+ "session_id": session_id,
+ "status": status,
+ "error": error,
+ }
+ )
+
+ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None:
+ self._pipeline_sessions[session_id] = pipeline_session_id
+ self.events.put(
+ {
+ "type": "pipeline_session",
+ "worker_id": self.worker_id,
+ "session_id": session_id,
+ "pipeline_session_id": pipeline_session_id,
+ }
+ )
+
+ def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
+ self.events.put(
+ {
+ "type": "session_finished",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ "pipeline_session_id": self._pipeline_sessions.pop(session_id, None),
+ "error": error,
+ }
+ )
diff --git a/telefuser/service/livekit/runtime.py b/telefuser/service/livekit/runtime.py
index 8466c6be..d7edd45c 100644
--- a/telefuser/service/livekit/runtime.py
+++ b/telefuser/service/livekit/runtime.py
@@ -2,14 +2,20 @@
from __future__ import annotations
+import asyncio
+import contextlib
import threading
from dataclasses import dataclass
from telefuser.service.security.security_validator import SecurityLevel
+from telefuser.utils.logging import logger
from .config import LiveKitServeConfig
from .multi_session_worker import MultiSessionLiveKitWorker as LiveKitWorker
from .pipeline_adapter import LiveKitPipelineAdapter
+from .pipeline_router import TurboServePipelineRouter
+from .process_worker_pool import ProcessLiveKitWorkerPool, ProcessWorkerSpec
+from .nccl_process_worker_pool import NCCLProcessLiveKitWorkerPool
from .scheduler import LiveKitScheduler, SchedulerAdmission
from .schemas import (
LiveKitHealthResponse,
@@ -21,6 +27,20 @@
)
from .session_registry import TERMINAL_SESSION_STATUSES, SessionRecord, SessionRegistry
from .token_service import LiveKitTokenService
+from .turboserve import (
+ TurboServeClusterScheduler,
+ TurboServeRuntimeCalibration,
+ TurboServeSchedulerConfig,
+ TurboServeSchedulingSnapshot,
+ TurboServeSessionView,
+ TurboServeAutoscalingController,
+ TurboServeMigrationPlan,
+ TurboServeOwnership,
+ TurboServePlacementController,
+ TurboServeScaleDecision,
+ TurboServeSessionDemand,
+ TurboServeWorkerLoad,
+)
from .worker_pool import InProcessLiveKitWorkerPool, WorkerPool
@@ -69,7 +89,38 @@ def __init__(
self._closing = False
self._closed = False
self._finished_sessions: set[str] = set()
+ self._reported_worker_capacities: dict[str, int] = {}
self._worker_capacity_profiles: dict[str, dict[str, object]] = {}
+ self._autoscaling_controller = TurboServeAutoscalingController(
+ sessions_per_worker=config.session_capacity_limit() or 1,
+ target_utilization=config.autoscaling_target_utilization,
+ hysteresis=config.autoscaling_hysteresis,
+ cooldown_seconds=config.autoscaling_cooldown_seconds,
+ min_workers=config.autoscaling_min_workers,
+ max_workers=config.num_workers,
+ )
+ self._placement_controller = TurboServePlacementController(
+ migration_bandwidth_bytes_per_second=config.turboserve_migration_bandwidth_gbps * 1_000_000_000,
+ migration_penalty=config.turboserve_migration_penalty,
+ )
+ self._autoscale_task: asyncio.Task | None = None
+ self._cluster_scheduler = TurboServeClusterScheduler(
+ TurboServeSchedulerConfig(
+ enable_autoscaling=config.autoscaling_enabled,
+ enable_migration=config.turboserve_rebalance_enabled,
+ min_workers=config.autoscaling_min_workers,
+ max_workers=config.num_workers,
+ capacity_per_worker=config.session_capacity_limit() or 1,
+ target_utilization=config.autoscaling_target_utilization,
+ scale_in_hold_seconds=config.turboserve_scale_in_hold_seconds,
+ migration_eta=config.turboserve_migration_eta,
+ min_gain_ms=config.turboserve_min_migration_gain_ms,
+ rebalance_iteration_limit=config.turboserve_rebalance_iteration_limit,
+ )
+ )
+ self._last_scale_decision: TurboServeScaleDecision | None = None
+ self._last_migration_plan: TurboServeMigrationPlan | None = None
+ self._last_migration_error: str | None = None
self._lock = threading.RLock()
@property
@@ -84,13 +135,19 @@ async def start(self) -> None:
return
if self._closed:
raise RuntimeError("LiveKit runtime is already closed")
- if self.config.worker_mode != "in-process":
- raise NotImplementedError("stream-serve currently supports only worker_mode='in-process'")
- if self.config.num_workers != 1:
- raise NotImplementedError("stream-serve currently supports exactly one in-process worker")
+ worker_groups = self.config.worker_gpu_groups()
+ if self.config.worker_mode == "process" and self.config.num_workers > 1 and any(
+ not group for group in worker_groups
+ ):
+ raise ValueError("worker_gpu_map is required for multiple process workers")
+ groups = [gpu_id for group in worker_groups for gpu_id in group]
+ if len(groups) != len(set(groups)):
+ raise ValueError("worker_gpu_map assigns a GPU to more than one worker")
await self.worker_pool.start(skip_validation=self.skip_validation)
with self._lock:
self._started = True
+ if self.config.autoscaling_enabled or self.config.turboserve_rebalance_enabled:
+ self._autoscale_task = asyncio.create_task(self._autoscale_loop(), name="livekit-turboserve-control")
def create_session(self, request: SessionCreateRequest) -> CreateSessionResult:
"""Create a session record, mint a controller token, and reserve capacity."""
@@ -173,8 +230,21 @@ def on_worker_status(self, worker_id: str, status: str) -> None:
def on_worker_capacity(self, worker_id: str, capacity: int, profile: dict[str, object] | None = None) -> None:
"""Apply worker-local hardware capacity before the runtime becomes ready."""
self.scheduler.update_worker_capacity(worker_id, capacity)
- if profile is not None:
- self._worker_capacity_profiles[worker_id] = dict(profile)
+ with self._lock:
+ self._reported_worker_capacities[worker_id] = capacity
+ if profile is not None:
+ self._worker_capacity_profiles[worker_id] = dict(profile)
+ effective_capacity = min(self._reported_worker_capacities.values())
+ self._autoscaling_controller.sessions_per_worker = effective_capacity
+ # Keep the source-aligned controller on the hardware-measured
+ # capacity too. Otherwise an auto-capacity deployment silently
+ # schedules every worker as if its capacity were one.
+ self._cluster_scheduler.config = TurboServeSchedulerConfig(
+ **{
+ **self._cluster_scheduler.config.__dict__,
+ "capacity_per_worker": effective_capacity,
+ }
+ )
def on_session_status(self, session_id: str, status: SessionStatus, error: str | None = None) -> None:
"""Apply a worker-reported public session state."""
@@ -193,40 +263,95 @@ def on_session_finished(self, worker_id: str, session_id: str, error: str | None
def health(self) -> LiveKitHealthResponse:
"""Return service health based on current scheduler state."""
snapshot = self.scheduler.health_snapshot()
- workers_total = snapshot["workers_total"]
workers_failed = snapshot["workers_failed"]
+ workers = self.scheduler.workers()
+ serving_workers = [worker for worker in workers if worker.status not in {"failed", "stopped"}]
status = "healthy"
- if workers_total and workers_failed == workers_total:
+ if not serving_workers:
status = "unhealthy"
elif workers_failed:
status = "degraded"
connected_statuses = {"starting_pipeline", "running", "draining"}
return LiveKitHealthResponse(
status=status,
- livekit_connected=any(worker.status in connected_statuses for worker in self.scheduler.workers()),
+ livekit_connected=any(worker.status in connected_statuses for worker in workers),
**snapshot,
)
def metadata(self) -> dict:
"""Return runtime metadata for `/v1/service/metadata`."""
health = self.health()
+ with self._lock:
+ reported_capacities = tuple(self._reported_worker_capacities.values())
+ capacity_profiles = dict(self._worker_capacity_profiles)
+ max_sessions_per_worker = (
+ min(reported_capacities)
+ if reported_capacities
+ else min(worker.session_capacity for worker in self.scheduler.workers())
+ )
metadata = {
"service_type": "stream",
"transport": "livekit",
"pipeline_file": self.pipeline_file,
"livekit_url": self.config.livekit_url,
"num_workers": self.config.num_workers,
- "max_sessions_per_worker": min(worker.session_capacity for worker in self.scheduler.workers()),
+ "max_sessions_per_worker": max_sessions_per_worker,
"configured_max_sessions_per_worker": self.config.max_sessions_per_worker,
"control_idle_timeout": self.config.control_idle_timeout,
"worker_mode": self.config.worker_mode,
"queue_size": self.config.queue_size,
+ "autoscaling_enabled": self.config.autoscaling_enabled,
**health.model_dump(),
}
- if self._worker_capacity_profiles:
- metadata["session_capacity"] = dict(self._worker_capacity_profiles)
+ if capacity_profiles:
+ metadata["session_capacity"] = capacity_profiles
+ snapshot = getattr(self.worker_pool, "turboserve_snapshot", None)
+ if callable(snapshot) and (routing := snapshot()) is not None:
+ metadata["turboserve_routing"] = routing
+ if self._last_scale_decision is not None:
+ metadata["autoscaling"] = {
+ "current_workers": self._last_scale_decision.current_workers,
+ "target_workers": self._last_scale_decision.target_workers,
+ "action": self._last_scale_decision.action,
+ "target_utilization": self._last_scale_decision.target_utilization,
+ "reason": self._last_scale_decision.reason,
+ }
+ if self._last_migration_plan is not None or self._last_migration_error is not None:
+ metadata["turboserve_rebalance"] = {
+ "enabled": self.config.turboserve_rebalance_enabled,
+ "last_plan": (
+ {
+ "session_id": self._last_migration_plan.session_id,
+ "source_worker_id": self._last_migration_plan.source_worker_id,
+ "target_worker_id": self._last_migration_plan.target_worker_id,
+ "gain_seconds": self._last_migration_plan.gain_seconds,
+ "migration_cost_seconds": self._last_migration_plan.migration_cost_seconds,
+ }
+ if self._last_migration_plan is not None
+ else None
+ ),
+ "last_error": self._last_migration_error,
+ }
return metadata
+ async def migrate_session(self, session_id: str, target_worker_id: str) -> TurboServeOwnership:
+ """Migrate ABot model state without reconnecting the LiveKit room."""
+ record = self.registry.require(session_id)
+ if record.pipeline_session_id is None:
+ raise RuntimeError(f"Session {session_id} has not created its pipeline state yet")
+ target = next((worker for worker in self.scheduler.workers() if worker.worker_id == target_worker_id), None)
+ if target is None:
+ raise KeyError(f"Unknown migration target {target_worker_id}")
+ if target_worker_id != record.worker_id and len(target.session_ids) >= target.session_capacity:
+ raise RuntimeError(f"Migration target {target_worker_id} has no retained-session capacity")
+ migrate = getattr(self.worker_pool, "migrate_session", None)
+ if not callable(migrate):
+ raise RuntimeError("Configured worker pool does not support TurboServe migration")
+ ownership = await migrate(record.pipeline_session_id, target_worker_id)
+ self.scheduler.reassign_session(session_id, target_worker_id)
+ self.registry.assign_worker(session_id, target_worker_id)
+ return ownership
+
async def aclose(self) -> None:
"""Stop runtime-owned background resources."""
with self._lock:
@@ -234,6 +359,11 @@ async def aclose(self) -> None:
return
self._closing = True
try:
+ if self._autoscale_task is not None:
+ self._autoscale_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._autoscale_task
+ self._autoscale_task = None
await self.worker_pool.aclose()
for record in self.registry.list_records():
if record.status not in TERMINAL_SESSION_STATUSES:
@@ -248,18 +378,266 @@ def _create_worker_pool(self) -> WorkerPool:
security_level = self.security_level
if isinstance(security_level, str):
security_level = SecurityLevel[security_level.upper()]
+ initial_workers = self.config.autoscaling_min_workers if self.config.autoscaling_enabled else None
+ if self.config.worker_mode in {"process", "process-nccl"}:
+ specs = [
+ ProcessWorkerSpec(worker_id=state.worker_id, gpu_ids=list(state.gpu_ids))
+ for state in self.scheduler.workers()
+ ]
+ pool_type = NCCLProcessLiveKitWorkerPool if self.config.worker_mode == "process-nccl" else ProcessLiveKitWorkerPool
+ return pool_type(
+ specs,
+ config=self.config,
+ pipeline_file=self.pipeline_file,
+ event_sink=self,
+ security_level=security_level,
+ initial_workers=initial_workers,
+ )
+ worker_states = self.scheduler.workers()
+ backends = {
+ state.worker_id: LiveKitPipelineAdapter(security_level=security_level) for state in worker_states
+ }
+ router = TurboServePipelineRouter(backends)
workers: dict[str, LiveKitWorker] = {}
- for worker_state in self.scheduler.workers():
+ for worker_state in worker_states:
workers[worker_state.worker_id] = LiveKitWorker(
worker_id=worker_state.worker_id,
config=self.config,
pipeline_file=self.pipeline_file,
token_service=self.token_service,
event_sink=self,
- pipeline_adapter=LiveKitPipelineAdapter(security_level=security_level),
+ pipeline_adapter=router.worker_view(
+ worker_state.worker_id, gpu_ids=worker_state.gpu_ids or None
+ ),
gpu_num=max(1, len(worker_state.gpu_ids)),
+ gpu_ids=worker_state.gpu_ids or None,
)
- return InProcessLiveKitWorkerPool(workers)
+ return InProcessLiveKitWorkerPool(workers, router=router, initial_workers=initial_workers)
+
+ async def _autoscale_loop(self) -> None:
+ while True:
+ await asyncio.sleep(self.config.autoscaling_interval_seconds)
+ try:
+ await self._turboserve_control_once()
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.exception(f"LiveKit autoscaling iteration failed: {exc}")
+
+ async def _autoscale_once(self) -> TurboServeScaleDecision:
+ active_count = getattr(self.worker_pool, "active_worker_count", None)
+ scale_to = getattr(self.worker_pool, "scale_to", None)
+ if not callable(active_count) or not callable(scale_to):
+ raise RuntimeError("Configured worker pool does not support autoscaling")
+ snapshot = self.scheduler.health_snapshot()
+ workload = self._workload_snapshot()
+ retained = sum(len(worker.session_ids) for worker in self.scheduler.workers())
+ demand = max(retained + snapshot["queued_sessions"], workload["active_sessions"] + snapshot["queued_sessions"])
+ decision = self._autoscaling_controller.decide(
+ demand,
+ active_count(),
+ activation_volatility=workload["activation_volatility"],
+ )
+ actual = await scale_to(decision.target_workers)
+ if actual > decision.current_workers:
+ for admission in self.scheduler.drain_queue():
+ self._start_queued_session(admission)
+ await self._rebalance_once()
+ self._last_scale_decision = TurboServeScaleDecision(
+ current_workers=decision.current_workers,
+ target_workers=actual,
+ action=decision.action if actual != decision.current_workers else "hold",
+ target_utilization=decision.target_utilization,
+ reason=decision.reason,
+ )
+ return self._last_scale_decision
+
+ async def _turboserve_control_once(self) -> None:
+ """Run one source-compatible closed-loop scheduling decision."""
+
+ snapshot_fn = getattr(self.worker_pool, "turboserve_snapshot", None)
+ active_count = getattr(self.worker_pool, "active_worker_count", None)
+ scale_to = getattr(self.worker_pool, "scale_to", None)
+ if not callable(snapshot_fn) or not callable(active_count) or not callable(scale_to):
+ if self.config.autoscaling_enabled:
+ await self._autoscale_once()
+ else:
+ await self._rebalance_once()
+ return
+ routing = snapshot_fn()
+ if not isinstance(routing, dict):
+ return
+ worker_order = tuple(worker.worker_id for worker in self.scheduler.workers())
+ if not worker_order:
+ return
+ capacity = min(
+ self._reported_worker_capacities.values(),
+ default=min(worker.session_capacity for worker in self.scheduler.workers()),
+ )
+ session_metrics = routing.get("session_runtime_metrics", {})
+ if not isinstance(session_metrics, dict):
+ session_metrics = {}
+ sessions: dict[str, TurboServeSessionView] = {}
+ placement: dict[str, str | None] = {}
+ for record in self.registry.list_records():
+ if record.status in TERMINAL_SESSION_STATUSES or record.pipeline_session_id is None:
+ continue
+ metrics = session_metrics.get(record.pipeline_session_id, {})
+ if not isinstance(metrics, dict):
+ metrics = {}
+ profile = self._worker_capacity_profiles.get(record.worker_id or "", {})
+ bytes_ = int(profile.get("estimated_session_bytes", 1)) if isinstance(profile, dict) else 1
+ sessions[record.session_id] = TurboServeSessionView(
+ session_id=record.session_id,
+ active=bool(metrics.get("active", record.status == "running")),
+ state_size_mb=max(1.0, bytes_ / (1024 * 1024)),
+ frame_count=int(metrics.get("emitted_frames", 9)),
+ )
+ placement[record.session_id] = record.worker_id
+ calibration = routing.get("migration_calibration", {})
+ if not isinstance(calibration, dict):
+ calibration = {}
+ worker_metrics = routing.get("worker_runtime_metrics", {})
+ if not isinstance(worker_metrics, dict):
+ worker_metrics = {}
+ base_latency_ms = max(
+ (float(values.get("p95_chunk_seconds", 0.0)) * 1000 for values in worker_metrics.values() if isinstance(values, dict)),
+ default=0.0,
+ )
+ decision = self._cluster_scheduler.decide(
+ TurboServeSchedulingSnapshot(
+ time_seconds=asyncio.get_running_loop().time(),
+ sessions=sessions,
+ placement=placement,
+ current_workers=active_count(),
+ worker_order=worker_order,
+ capacity_per_worker=max(1, capacity),
+ runtime_calibration=TurboServeRuntimeCalibration(
+ average_migration_total_ms=float(calibration.get("average_total_ms", 0.0)),
+ base_chunk_latency_ms=base_latency_ms,
+ ),
+ )
+ )
+ current_workers = active_count()
+ if decision.worker_budget > current_workers:
+ actual = await scale_to(decision.worker_budget)
+ for admission in self.scheduler.drain_queue():
+ self._start_queued_session(admission)
+ else:
+ actual = current_workers
+ migrations = 0
+ for session_id, target_worker_id in decision.placement.items():
+ record = self.registry.require(session_id)
+ if (
+ target_worker_id is None
+ or target_worker_id == record.worker_id
+ or record.pipeline_session_id is None
+ ):
+ continue
+ try:
+ await self.migrate_session(session_id, target_worker_id)
+ migrations += 1
+ except Exception as exc:
+ self._last_migration_error = str(exc)
+ logger.warning(
+ "TurboServe placement move failed: session=%s target=%s error=%s",
+ session_id,
+ target_worker_id,
+ exc,
+ )
+ if decision.worker_budget < actual:
+ actual = await scale_to(decision.worker_budget)
+ action = str(decision.metadata["autoscale_action"])
+ self._last_scale_decision = TurboServeScaleDecision(
+ current_workers=current_workers,
+ target_workers=actual,
+ action=action if action in {"scale_out", "scale_in"} else "hold",
+ target_utilization=self.config.autoscaling_target_utilization,
+ reason=action,
+ )
+ if migrations:
+ self._last_migration_error = None
+
+ """Aggregate live ABot control activity when the pool can expose it."""
+ def _workload_snapshot(self) -> dict[str, float | int]:
+ snapshot = getattr(self.worker_pool, "turboserve_snapshot", None)
+ routing = snapshot() if callable(snapshot) else None
+ per_worker = routing.get("worker_runtime_metrics", {}) if isinstance(routing, dict) else {}
+ if not isinstance(per_worker, dict) or not per_worker:
+ return {"active_sessions": 0, "activation_volatility": 0.0}
+ active = 0
+ volatility = 0.0
+ for values in per_worker.values():
+ if not isinstance(values, dict):
+ continue
+ active += int(values.get("active_sessions", 0))
+ volatility = max(volatility, float(values.get("activation_volatility", 0.0)))
+ return {"active_sessions": active, "activation_volatility": volatility}
+
+ async def _rebalance_once(self) -> None:
+ """Commit at most one profitable chunk-boundary migration per control tick."""
+ if not self.config.turboserve_rebalance_enabled:
+ return
+ migrate = getattr(self.worker_pool, "migrate_session", None)
+ snapshot_fn = getattr(self.worker_pool, "turboserve_snapshot", None)
+ if not callable(migrate) or not callable(snapshot_fn):
+ return
+ routing = snapshot_fn()
+ if not isinstance(routing, dict) or routing.get("migration_supported") is False:
+ return
+ metrics = routing.get("worker_runtime_metrics", {})
+ if not isinstance(metrics, dict):
+ return
+ workers: list[TurboServeWorkerLoad] = []
+ for state in self.scheduler.workers():
+ values = metrics.get(state.worker_id, {})
+ if not isinstance(values, dict):
+ values = {}
+ p95 = float(values.get("p95_chunk_seconds", 0.0))
+ mean = float(values.get("mean_chunk_seconds", 0.0))
+ workers.append(
+ TurboServeWorkerLoad(
+ worker_id=state.worker_id,
+ capacity=state.session_capacity,
+ active_sessions=int(values.get("active_sessions", 0)),
+ retained_sessions=len(state.session_ids),
+ predicted_chunk_latency_seconds=max(p95, mean, 1e-6),
+ ready=state.status not in {"failed", "stopped"},
+ draining=state.status == "draining",
+ )
+ )
+ if len(workers) < 2:
+ return
+ profiles = self._worker_capacity_profiles
+ sessions: list[TurboServeSessionDemand] = []
+ for record in self.registry.list_records():
+ if record.status in TERMINAL_SESSION_STATUSES or record.pipeline_session_id is None or record.worker_id is None:
+ continue
+ profile = profiles.get(record.worker_id, {})
+ state_bytes = int(profile.get("estimated_session_bytes", 1)) if isinstance(profile, dict) else 1
+ sessions.append(
+ TurboServeSessionDemand(
+ session_id=record.session_id,
+ active=True,
+ state_bytes=max(1, state_bytes),
+ owner_worker_id=record.worker_id,
+ )
+ )
+ plans = self._placement_controller.plan_rebalance(sessions, workers)
+ if not plans:
+ return
+ plan = plans[0]
+ try:
+ await self.migrate_session(plan.session_id, plan.target_worker_id)
+ except Exception as exc:
+ self._last_migration_error = str(exc)
+ logger.warning(
+ f"TurboServe rebalance migration failed: session={plan.session_id} "
+ f"source={plan.source_worker_id} target={plan.target_worker_id} error={exc}"
+ )
+ return
+ self._last_migration_plan = plan
+ self._last_migration_error = None
def _finish_session(self, session_id: str, *, error: str | None = None) -> SessionRecord:
with self._lock:
diff --git a/telefuser/service/livekit/scheduler.py b/telefuser/service/livekit/scheduler.py
index 9e8b112e..6ee5bf4a 100644
--- a/telefuser/service/livekit/scheduler.py
+++ b/telefuser/service/livekit/scheduler.py
@@ -94,7 +94,7 @@ def __init__(
def assign(self, *, session_id: str, room_name: str) -> SchedulerAdmission:
"""Assign a worker with capacity or enqueue/reject the session."""
with self._lock:
- worker = self._first_available_worker()
+ worker = self._least_loaded_worker()
if worker is not None:
worker.status = "assigned"
worker.session_ids.append(session_id)
@@ -143,6 +143,53 @@ def release_session(self, session_id: str) -> SchedulerAdmission | None:
worker.last_heartbeat_at = utc_timestamp()
return SchedulerAdmission(status="assigned", worker_id=worker.worker_id, session_id=queued.session_id)
+ def drain_queue(self) -> list[SchedulerAdmission]:
+ """Assign as many queued sessions as newly active capacity permits."""
+ admissions: list[SchedulerAdmission] = []
+ with self._lock:
+ while self._queue and (worker := self._least_loaded_worker()) is not None:
+ queued = self._queue.popleft()
+ worker.status = "assigned"
+ worker.session_ids.append(queued.session_id)
+ self._room_names[queued.session_id] = queued.room_name
+ worker.session_id = queued.session_id
+ worker.room_name = queued.room_name
+ worker.last_heartbeat_at = utc_timestamp()
+ admissions.append(
+ SchedulerAdmission(
+ status="assigned", worker_id=worker.worker_id, session_id=queued.session_id
+ )
+ )
+ return admissions
+
+ def reassign_session(self, session_id: str, target_worker_id: str) -> SchedulerAdmission:
+ """Move admission ownership after the pipeline router commits migration."""
+ with self._lock:
+ source = self._worker_for_session(session_id)
+ if source is None:
+ raise KeyError(f"Session {session_id} is not assigned")
+ target = self._workers[target_worker_id]
+ if source.worker_id == target.worker_id:
+ return SchedulerAdmission(status="assigned", worker_id=target.worker_id, session_id=session_id)
+ if target.status in {"failed", "stopped", "draining"}:
+ raise RuntimeError(f"Migration target {target_worker_id} is not available")
+ if len(target.session_ids) >= target.session_capacity:
+ raise RuntimeError(f"Migration target {target_worker_id} has no retained-session capacity")
+ source.session_ids.remove(session_id)
+ source.session_id = source.session_ids[-1] if source.session_ids else None
+ source.room_name = self._room_names.get(source.session_id) if source.session_id is not None else None
+ if source.status not in {"failed", "stopped"}:
+ source.status = "assigned" if source.session_ids else "idle"
+ source.error = None
+ source.last_heartbeat_at = utc_timestamp()
+ target.session_ids.append(session_id)
+ target.session_id = session_id
+ target.room_name = self._room_names.get(session_id)
+ target.status = "assigned"
+ target.error = None
+ target.last_heartbeat_at = utc_timestamp()
+ return SchedulerAdmission(status="assigned", worker_id=target.worker_id, session_id=session_id)
+
def update_worker_status(self, worker_id: str, status: WorkerStatus) -> WorkerState:
"""Update a worker lifecycle status."""
with self._lock:
@@ -193,17 +240,29 @@ def health_snapshot(self) -> dict[str, int]:
workers = list(self._workers.values())
return {
"workers_total": len(workers),
- "workers_idle": sum(1 for worker in workers if worker.status != "failed" and not worker.session_ids),
+ "workers_idle": sum(1 for worker in workers if worker.status == "idle"),
"workers_busy": sum(1 for worker in workers if bool(worker.session_ids)),
"workers_failed": sum(1 for worker in workers if worker.status == "failed"),
"queued_sessions": len(self._queue),
}
- def _first_available_worker(self) -> WorkerState | None:
- for worker in self._workers.values():
- if worker.status not in {"failed", "stopped"} and len(worker.session_ids) < worker.session_capacity:
- return worker
- return None
+ def _least_loaded_worker(self) -> WorkerState | None:
+ candidates = [
+ worker
+ for worker in self._workers.values()
+ if worker.status not in {"failed", "stopped", "draining"}
+ and len(worker.session_ids) < worker.session_capacity
+ ]
+ if not candidates:
+ return None
+ return min(
+ candidates,
+ key=lambda worker: (
+ len(worker.session_ids) / worker.session_capacity,
+ len(worker.session_ids),
+ worker.worker_id,
+ ),
+ )
def _worker_for_session(self, session_id: str) -> WorkerState | None:
for worker in self._workers.values():
diff --git a/telefuser/service/livekit/turboserve.py b/telefuser/service/livekit/turboserve.py
new file mode 100644
index 00000000..5e45c854
--- /dev/null
+++ b/telefuser/service/livekit/turboserve.py
@@ -0,0 +1,662 @@
+"""TurboServe workload, placement, migration, and autoscaling controllers."""
+
+from __future__ import annotations
+
+import math
+import statistics
+import threading
+import time
+import uuid
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Literal
+
+
+@dataclass(frozen=True)
+class TurboServeSessionDemand:
+ """Scheduler-visible demand and residency facts for one streaming session."""
+
+ session_id: str
+ active: bool
+ state_bytes: int
+ owner_worker_id: str | None = None
+ migration_bytes: int | None = None
+
+
+@dataclass(frozen=True)
+class TurboServeWorkerLoad:
+ """One worker's measured and predicted load."""
+
+ worker_id: str
+ capacity: int
+ active_sessions: int
+ retained_sessions: int
+ predicted_chunk_latency_seconds: float
+ ready: bool = True
+ draining: bool = False
+
+
+@dataclass(frozen=True)
+class TurboServePlacementDecision:
+ """Placement result with its predicted global bottleneck latency."""
+
+ worker_id: str
+ predicted_bottleneck_seconds: float
+
+
+@dataclass(frozen=True)
+class TurboServeMigrationPlan:
+ """A beneficial chunk-boundary state movement."""
+
+ session_id: str
+ source_worker_id: str
+ target_worker_id: str
+ gain_seconds: float
+ migration_cost_seconds: float
+
+
+@dataclass(frozen=True)
+class TurboServeWorkloadSnapshot:
+ """Sliding-window demand statistics used by placement and autoscaling."""
+
+ active_sessions: int
+ arrivals_per_second: float
+ activation_volatility: float
+ mean_chunk_seconds: float
+ p95_chunk_seconds: float
+ observed_at: float
+
+
+@dataclass(frozen=True)
+class TurboServeScaleDecision:
+ """Desired worker count and the reason for changing or retaining it."""
+
+ current_workers: int
+ target_workers: int
+ action: Literal["scale_out", "scale_in", "hold"]
+ target_utilization: float
+ reason: str
+
+
+class TurboServeWorkloadDetector:
+ """Track long-lived arrivals, active/idle transitions, and chunk service time."""
+
+ def __init__(self, window_seconds: float = 60.0, volatility_bins: int = 10) -> None:
+ if window_seconds <= 0 or volatility_bins < 2:
+ raise ValueError("window_seconds must be positive and volatility_bins at least two")
+ self.window_seconds = float(window_seconds)
+ self.volatility_bins = int(volatility_bins)
+ self._events: deque[tuple[float, str, str]] = deque()
+ self._chunks: deque[tuple[float, float]] = deque()
+ self._active: set[str] = set()
+ self._lock = threading.Lock()
+
+ def record_arrival(self, session_id: str, now: float | None = None) -> None:
+ self._record(session_id, "arrival", now)
+
+ def record_active(self, session_id: str, now: float | None = None) -> None:
+ observed_at = time.monotonic() if now is None else now
+ with self._lock:
+ self._active.add(session_id)
+ self._events.append((observed_at, "active", session_id))
+ self._trim(observed_at)
+
+ def record_idle(self, session_id: str, now: float | None = None) -> None:
+ observed_at = time.monotonic() if now is None else now
+ with self._lock:
+ self._active.discard(session_id)
+ self._events.append((observed_at, "idle", session_id))
+ self._trim(observed_at)
+
+ def record_departure(self, session_id: str, now: float | None = None) -> None:
+ observed_at = time.monotonic() if now is None else now
+ with self._lock:
+ self._active.discard(session_id)
+ self._events.append((observed_at, "departure", session_id))
+ self._trim(observed_at)
+
+ def record_chunk(self, duration_seconds: float, now: float | None = None) -> None:
+ if duration_seconds < 0:
+ raise ValueError("duration_seconds must be non-negative")
+ observed_at = time.monotonic() if now is None else now
+ with self._lock:
+ self._chunks.append((observed_at, float(duration_seconds)))
+ self._trim(observed_at)
+
+ def snapshot(self, now: float | None = None) -> TurboServeWorkloadSnapshot:
+ observed_at = time.monotonic() if now is None else now
+ with self._lock:
+ self._trim(observed_at)
+ arrivals = sum(1 for _, event, _ in self._events if event == "arrival")
+ durations = sorted(duration for _, duration in self._chunks)
+ mean_chunk = statistics.fmean(durations) if durations else 0.0
+ p95_index = max(0, math.ceil(len(durations) * 0.95) - 1)
+ p95_chunk = durations[p95_index] if durations else 0.0
+ volatility = self._volatility(observed_at)
+ return TurboServeWorkloadSnapshot(
+ active_sessions=len(self._active),
+ arrivals_per_second=arrivals / self.window_seconds,
+ activation_volatility=volatility,
+ mean_chunk_seconds=mean_chunk,
+ p95_chunk_seconds=p95_chunk,
+ observed_at=observed_at,
+ )
+
+ def _record(self, session_id: str, event: str, now: float | None) -> None:
+ observed_at = time.monotonic() if now is None else now
+ with self._lock:
+ self._events.append((observed_at, event, session_id))
+ self._trim(observed_at)
+
+ def _trim(self, now: float) -> None:
+ cutoff = now - self.window_seconds
+ while self._events and self._events[0][0] < cutoff:
+ self._events.popleft()
+ while self._chunks and self._chunks[0][0] < cutoff:
+ self._chunks.popleft()
+
+ def _volatility(self, now: float) -> float:
+ bin_width = self.window_seconds / self.volatility_bins
+ counts = [0] * self.volatility_bins
+ cutoff = now - self.window_seconds
+ for observed_at, event, _ in self._events:
+ if event not in {"active", "idle"}:
+ continue
+ index = min(self.volatility_bins - 1, max(0, int((observed_at - cutoff) / bin_width)))
+ counts[index] += 1
+ mean = statistics.fmean(counts)
+ return statistics.pstdev(counts) / mean if mean > 0 else 0.0
+
+
+class TurboServePlacementController:
+ """Minimize predicted bottleneck latency and migration-aware rebalance cost."""
+
+ def __init__(self, migration_bandwidth_bytes_per_second: float, migration_penalty: float = 1.0) -> None:
+ if migration_bandwidth_bytes_per_second <= 0 or migration_penalty < 0:
+ raise ValueError("migration bandwidth must be positive and migration_penalty non-negative")
+ self.migration_bandwidth_bytes_per_second = float(migration_bandwidth_bytes_per_second)
+ self.migration_penalty = float(migration_penalty)
+
+ def place(
+ self,
+ demand: TurboServeSessionDemand,
+ workers: list[TurboServeWorkerLoad],
+ ) -> TurboServePlacementDecision:
+ candidates = [
+ worker
+ for worker in workers
+ if worker.ready and not worker.draining and worker.retained_sessions < worker.capacity
+ ]
+ if not candidates:
+ raise RuntimeError("No TurboServe worker has retained-session capacity")
+ if demand.owner_worker_id is not None:
+ retained = next((item for item in candidates if item.worker_id == demand.owner_worker_id), None)
+ if retained is not None:
+ return TurboServePlacementDecision(
+ retained.worker_id,
+ max(item.predicted_chunk_latency_seconds for item in workers),
+ )
+ best = min(
+ candidates,
+ key=lambda candidate: (
+ self._predicted_bottleneck_after_add(candidate.worker_id, workers),
+ candidate.active_sessions,
+ candidate.retained_sessions,
+ candidate.worker_id,
+ ),
+ )
+ return TurboServePlacementDecision(
+ best.worker_id,
+ self._predicted_bottleneck_after_add(best.worker_id, workers),
+ )
+
+ def plan_rebalance(
+ self,
+ sessions: list[TurboServeSessionDemand],
+ workers: list[TurboServeWorkerLoad],
+ ) -> list[TurboServeMigrationPlan]:
+ if len(workers) < 2:
+ return []
+ bottleneck = max(workers, key=lambda worker: worker.predicted_chunk_latency_seconds)
+ old_max = bottleneck.predicted_chunk_latency_seconds
+ best: TurboServeMigrationPlan | None = None
+ for session in sessions:
+ if session.owner_worker_id != bottleneck.worker_id:
+ continue
+ for target in workers:
+ if (
+ target.worker_id == bottleneck.worker_id
+ or not target.ready
+ or target.draining
+ or target.retained_sessions >= target.capacity
+ ):
+ continue
+ new_max = self._predicted_bottleneck_after_move(bottleneck, target, workers)
+ migration_bytes = session.migration_bytes or session.state_bytes
+ migration_cost = migration_bytes / self.migration_bandwidth_bytes_per_second
+ gain = old_max - new_max - self.migration_penalty * migration_cost
+ candidate = TurboServeMigrationPlan(
+ session.session_id,
+ bottleneck.worker_id,
+ target.worker_id,
+ gain,
+ migration_cost,
+ )
+ if gain > 0 and (best is None or candidate.gain_seconds > best.gain_seconds):
+ best = candidate
+ return [best] if best is not None else []
+
+ @staticmethod
+ def _predicted_bottleneck_after_add(worker_id: str, workers: list[TurboServeWorkerLoad]) -> float:
+ predictions = []
+ for worker in workers:
+ latency = worker.predicted_chunk_latency_seconds
+ if worker.worker_id == worker_id:
+ latency *= (worker.active_sessions + 1) / max(1, worker.active_sessions)
+ predictions.append(latency)
+ return max(predictions)
+
+ @staticmethod
+ def _predicted_bottleneck_after_move(
+ source: TurboServeWorkerLoad,
+ target: TurboServeWorkerLoad,
+ workers: list[TurboServeWorkerLoad],
+ ) -> float:
+ predictions = []
+ for worker in workers:
+ latency = worker.predicted_chunk_latency_seconds
+ if worker.worker_id == source.worker_id:
+ latency *= max(0, worker.active_sessions - 1) / max(1, worker.active_sessions)
+ elif worker.worker_id == target.worker_id:
+ latency *= (worker.active_sessions + 1) / max(1, worker.active_sessions)
+ predictions.append(latency)
+ return max(predictions)
+
+
+class TurboServeAutoscalingController:
+ """Compute a hysteretic worker target from active demand and profiled capacity."""
+
+ def __init__(
+ self,
+ sessions_per_worker: int,
+ target_utilization: float = 0.75,
+ hysteresis: float = 0.10,
+ cooldown_seconds: float = 30.0,
+ min_workers: int = 1,
+ max_workers: int = 64,
+ ) -> None:
+ if sessions_per_worker < 1 or not 0 < target_utilization <= 1 or not 0 <= hysteresis < 1:
+ raise ValueError("Invalid TurboServe autoscaling capacity or utilization")
+ if cooldown_seconds < 0 or not 1 <= min_workers <= max_workers:
+ raise ValueError("Invalid TurboServe autoscaling cooldown or worker bounds")
+ self.sessions_per_worker = int(sessions_per_worker)
+ self.target_utilization = float(target_utilization)
+ self.hysteresis = float(hysteresis)
+ self.cooldown_seconds = float(cooldown_seconds)
+ self.min_workers = int(min_workers)
+ self.max_workers = int(max_workers)
+ self._last_scale_at: float | None = None
+
+ def decide(
+ self,
+ active_sessions: int,
+ current_workers: int,
+ *,
+ activation_volatility: float = 0.0,
+ now: float | None = None,
+ ) -> TurboServeScaleDecision:
+ if active_sessions < 0 or current_workers < 1:
+ raise ValueError("active_sessions must be non-negative and current_workers positive")
+ observed_at = time.monotonic() if now is None else now
+ volatility_margin = min(0.25, max(0.0, activation_volatility) * 0.05)
+ utilization = max(0.25, self.target_utilization - volatility_margin)
+ raw_target = math.ceil(active_sessions / (self.sessions_per_worker * utilization)) if active_sessions else 1
+ raw_target = min(self.max_workers, max(self.min_workers, raw_target))
+ current_capacity = current_workers * self.sessions_per_worker
+ current_utilization = active_sessions / current_capacity
+ upper = min(1.0, utilization + self.hysteresis)
+ lower = max(0.0, utilization - self.hysteresis)
+ if self._last_scale_at is not None and observed_at - self._last_scale_at < self.cooldown_seconds:
+ return TurboServeScaleDecision(current_workers, current_workers, "hold", utilization, "cooldown")
+ if raw_target > current_workers and current_utilization > upper:
+ self._last_scale_at = observed_at
+ return TurboServeScaleDecision(current_workers, raw_target, "scale_out", utilization, "above_upper_band")
+ if raw_target < current_workers and current_utilization < lower:
+ self._last_scale_at = observed_at
+ return TurboServeScaleDecision(current_workers, raw_target, "scale_in", utilization, "below_lower_band")
+ return TurboServeScaleDecision(current_workers, current_workers, "hold", utilization, "within_hysteresis")
+
+
+@dataclass(frozen=True)
+class TurboServeOwnership:
+ """Committed owner and monotonically increasing epoch for one session."""
+
+ session_id: str
+ worker_id: str
+ epoch: int
+
+
+@dataclass(frozen=True)
+class TurboServeMigrationToken:
+ """Opaque prepare record used to commit or abort a migration transaction."""
+
+ token_id: str
+ session_id: str
+ source_worker_id: str
+ target_worker_id: str
+ source_epoch: int
+
+
+class TurboServeOwnershipTable:
+ """Serialize prepare/commit migration ownership at chunk boundaries."""
+
+ def __init__(self) -> None:
+ self._owners: dict[str, TurboServeOwnership] = {}
+ self._pending: dict[str, TurboServeMigrationToken] = {}
+ self._lock = threading.RLock()
+
+ def register(self, session_id: str, worker_id: str) -> TurboServeOwnership:
+ with self._lock:
+ if session_id in self._owners:
+ raise ValueError(f"TurboServe session {session_id!r} already has an owner")
+ ownership = TurboServeOwnership(session_id, worker_id, 1)
+ self._owners[session_id] = ownership
+ return ownership
+
+ def owner(self, session_id: str) -> TurboServeOwnership:
+ with self._lock:
+ return self._owners[session_id]
+
+ def prepare_migration(
+ self,
+ session_id: str,
+ source_worker_id: str,
+ target_worker_id: str,
+ ) -> TurboServeMigrationToken:
+ with self._lock:
+ owner = self._owners[session_id]
+ if owner.worker_id != source_worker_id:
+ raise RuntimeError("Migration source does not own the session")
+ if session_id in self._pending:
+ raise RuntimeError("Session already has a pending migration")
+ token = TurboServeMigrationToken(
+ str(uuid.uuid4()),
+ session_id,
+ source_worker_id,
+ target_worker_id,
+ owner.epoch,
+ )
+ self._pending[session_id] = token
+ return token
+
+ def commit_migration(self, token: TurboServeMigrationToken) -> TurboServeOwnership:
+ with self._lock:
+ if self._pending.get(token.session_id) != token:
+ raise RuntimeError("Migration token is stale or already completed")
+ owner = self._owners[token.session_id]
+ if owner.worker_id != token.source_worker_id or owner.epoch != token.source_epoch:
+ raise RuntimeError("Session ownership changed while migration was prepared")
+ committed = TurboServeOwnership(token.session_id, token.target_worker_id, owner.epoch + 1)
+ self._owners[token.session_id] = committed
+ del self._pending[token.session_id]
+ return committed
+
+ def abort_migration(self, token: TurboServeMigrationToken) -> None:
+ with self._lock:
+ if self._pending.get(token.session_id) == token:
+ del self._pending[token.session_id]
+
+ def release(self, session_id: str) -> None:
+ with self._lock:
+ self._pending.pop(session_id, None)
+ self._owners.pop(session_id, None)
+
+# The classes below intentionally mirror the closed-loop scheduler in the
+# TurboServe reference implementation. The older controllers above are kept
+# for API compatibility with the first TeleFuser prototype.
+
+
+@dataclass(frozen=True)
+class TurboServeSchedulerConfig:
+ """Knobs of TurboServe's budget sizing and migration-aware placement."""
+
+ enable_autoscaling: bool = True
+ enable_migration: bool = True
+ min_workers: int = 1
+ max_workers: int = 64
+ capacity_per_worker: int = 1
+ target_utilization: float = 0.9
+ scale_in_hold_seconds: float = 5.0
+ migration_eta: float = 0.35
+ min_gain_ms: float = 40.0
+ rebalance_iteration_limit: int = 3
+
+
+@dataclass(frozen=True)
+class TurboServeSessionView:
+ """One session as seen by the cluster controller at a control boundary."""
+
+ session_id: str
+ active: bool
+ state_size_mb: float
+ chunk_compute_units: float = 1.0
+ prompt_tokens: int = 256
+ resolution: str = "480p"
+ frame_count: int = 9
+
+
+@dataclass(frozen=True)
+class TurboServeRuntimeCalibration:
+ """Measured values that replace the migration-model cold estimate."""
+
+ average_migration_total_ms: float = 0.0
+ base_chunk_latency_ms: float = 0.0
+
+
+@dataclass(frozen=True)
+class TurboServeSchedulingSnapshot:
+ """Complete input to one source-compatible scheduling decision."""
+
+ time_seconds: float
+ sessions: dict[str, TurboServeSessionView]
+ placement: dict[str, str | None]
+ current_workers: int
+ worker_order: tuple[str, ...]
+ capacity_per_worker: int
+ runtime_calibration: TurboServeRuntimeCalibration = field(
+ default_factory=TurboServeRuntimeCalibration
+ )
+
+
+@dataclass(frozen=True)
+class TurboServeSchedulingDecision:
+ """Worker budget and requested active-session placement for one tick."""
+
+ worker_budget: int
+ placement: dict[str, str | None]
+ metadata: dict[str, object]
+
+
+@dataclass
+class TurboServeLatencyModel:
+ """The reference analytic chunk and migration latency model, in ms."""
+
+ migration_alpha_ms: float = 8.0
+ migration_bandwidth_mb_per_ms: float = 32.0
+ base_chunk_latency_ms: float = 180.0
+ load_penalty_ms: float = 55.0
+ quadratic_load_penalty_ms: float = 18.0
+ prompt_token_penalty_ms: float = 0.015
+ frame_reference_count: float = 64.0
+ frame_exponent: float = 0.5
+ min_frame_factor: float = 0.5
+ resolution_factors: dict[str, float] = field(
+ default_factory=lambda: {
+ "360p": 0.45,
+ "480p": 0.65,
+ "720p": 1.0,
+ "1080p": 1.7,
+ "4k": 3.2,
+ }
+ )
+
+ def migration_cost_ms(
+ self, session: TurboServeSessionView, calibration: TurboServeRuntimeCalibration
+ ) -> float:
+ if calibration.average_migration_total_ms > 0:
+ return calibration.average_migration_total_ms
+ return self.migration_alpha_ms + session.state_size_mb / max(
+ 1e-9, self.migration_bandwidth_mb_per_ms
+ )
+
+ def session_latency_ms(
+ self,
+ session: TurboServeSessionView,
+ colocated_sessions: int,
+ capacity_per_worker: int,
+ calibration: TurboServeRuntimeCalibration,
+ ) -> float:
+ base = calibration.base_chunk_latency_ms or self.base_chunk_latency_ms
+ load = max(1, colocated_sessions)
+ normalized = load / max(1, capacity_per_worker)
+ compute = base * session.chunk_compute_units
+ compute *= self.resolution_factors.get(session.resolution, 1.0)
+ compute *= max(
+ self.min_frame_factor,
+
+ (max(1, session.frame_count) / self.frame_reference_count) ** self.frame_exponent,
+ )
+ return (
+ compute
+ + self.prompt_token_penalty_ms * session.prompt_tokens
+ + self.load_penalty_ms * (load - 1)
+ + self.quadratic_load_penalty_ms * normalized * normalized
+ )
+
+
+class TurboServeClusterScheduler:
+ """Source-aligned closed-loop budget and migration-aware placement."""
+
+ def __init__(self, config: TurboServeSchedulerConfig | None = None, latency_model: TurboServeLatencyModel | None = None) -> None:
+ self.config = config or TurboServeSchedulerConfig()
+ self.latency_model = latency_model or TurboServeLatencyModel()
+ self._scale_in_target: int | None = None
+ self._scale_in_deadline_seconds: float | None = None
+
+ def decide(self, snapshot: TurboServeSchedulingSnapshot) -> TurboServeSchedulingDecision:
+ budget, action = self._autoscale_budget(snapshot)
+ placement, metadata = self._place_at_budget(snapshot, budget)
+ metadata.update({"scheduler": "turboserve", "autoscale_action": action, "worker_budget": budget, "active_sessions": sum(session.active for session in snapshot.sessions.values()), "target_utilization": self.config.target_utilization})
+ return TurboServeSchedulingDecision(budget, placement, metadata)
+
+ def _autoscale_budget(self, snapshot: TurboServeSchedulingSnapshot) -> tuple[int, str]:
+ current = self._clamp(snapshot.current_workers, len(snapshot.worker_order))
+ if not self.config.enable_autoscaling:
+ self._scale_in_target = self._scale_in_deadline_seconds = None
+ return current, "disabled"
+ active = sum(session.active for session in snapshot.sessions.values())
+ capacity = max(1, min(snapshot.capacity_per_worker, self.config.capacity_per_worker))
+ target = self._target_budget(active, capacity, len(snapshot.worker_order))
+ if not self.config.enable_migration:
+ for session_id, session in snapshot.sessions.items():
+ owner = snapshot.placement.get(session_id)
+ if session.active and owner in snapshot.worker_order:
+ target = max(target, snapshot.worker_order.index(owner) + 1)
+ if target > current:
+ self._scale_in_target = self._scale_in_deadline_seconds = None
+ return target, "scale_out"
+ if target < current:
+ if self._scale_in_target != target or self._scale_in_deadline_seconds is None:
+ self._scale_in_target = target
+ self._scale_in_deadline_seconds = snapshot.time_seconds + max(0.0, self.config.scale_in_hold_seconds)
+ if snapshot.time_seconds >= self._scale_in_deadline_seconds:
+ self._scale_in_target = self._scale_in_deadline_seconds = None
+ return target, "scale_in"
+ return current, "hold_scale_in"
+ self._scale_in_target = self._scale_in_deadline_seconds = None
+ return current, "hold"
+
+ def _target_budget(self, active: int, capacity: int, maximum: int) -> int:
+ if active <= 0:
+ return self._clamp(self.config.min_workers, maximum)
+ utilization = min(1.0, max(0.01, self.config.target_utilization))
+ hard = math.ceil(active / capacity)
+ target = math.ceil(active / (capacity * utilization))
+ return self._clamp(max(hard, target), maximum)
+
+ def _clamp(self, value: int, maximum: int) -> int:
+ return min(maximum, self.config.max_workers, max(self.config.min_workers, int(value)))
+
+ def _place_at_budget(self, snapshot: TurboServeSchedulingSnapshot, budget: int) -> tuple[dict[str, str | None], dict[str, object]]:
+ workers = tuple(snapshot.worker_order[:budget])
+ capacity = max(1, min(snapshot.capacity_per_worker, self.config.capacity_per_worker))
+ loads: dict[str, list[str]] = {worker: [] for worker in workers}
+ placement: dict[str, str | None] = {}
+ pending: list[str] = []
+ for session_id in sorted(snapshot.sessions):
+ session = snapshot.sessions[session_id]
+ owner = snapshot.placement.get(session_id)
+ if not session.active:
+ placement[session_id] = None
+ elif owner in loads and (not self.config.enable_migration or len(loads[owner]) < capacity):
+ placement[session_id] = owner
+ loads[owner].append(session_id)
+ else:
+ placement[session_id] = None
+ pending.append(session_id)
+ for session_id in pending:
+ feasible = [worker for worker, sessions in loads.items() if len(sessions) < capacity]
+ if feasible:
+ target = min(feasible, key=lambda worker: (len(loads[worker]), worker))
+ placement[session_id] = target
+ loads[target].append(session_id)
+ before = self._bottleneck(snapshot, loads, capacity)
+ moves = evaluations = 0
+ if self.config.enable_migration:
+ moves, evaluations = self._rebalance(snapshot, loads, placement, capacity)
+ after = self._bottleneck(snapshot, loads, capacity)
+ unplaced = sum(session.active and placement.get(session_id) is None for session_id, session in snapshot.sessions.items())
+ rho_max = max((len(items) / capacity for items in loads.values()), default=0.0)
+ return placement, {"algorithm": "least_load_with_optional_rebalance", "capacity_per_worker": capacity, "rebalance_moves": moves, "candidate_evaluations": evaluations, "unplaced_active": unplaced, "bottleneck_before_ms": round(before, 3), "bottleneck_after_ms": round(after, 3), "rho_max": round(rho_max, 4)}
+
+ def _rebalance(self, snapshot: TurboServeSchedulingSnapshot, loads: dict[str, list[str]], placement: dict[str, str | None], capacity: int) -> tuple[int, int]:
+ moves = evaluations = 0
+ for _ in range(self.config.rebalance_iteration_limit):
+ if not loads:
+ break
+ source = max(loads, key=lambda worker: (self._worker_worst(snapshot, loads[worker], capacity), len(loads[worker]), worker))
+ if not loads[source]:
+ break
+ current = self._bottleneck(snapshot, loads, capacity)
+ best: tuple[tuple[float, float, str, str], str, str] | None = None
+ for session_id in tuple(loads[source]):
+ session = snapshot.sessions[session_id]
+ migration = self.latency_model.migration_cost_ms(session, snapshot.runtime_calibration)
+ for target, target_load in loads.items():
+ if target == source or len(target_load) >= capacity:
+ continue
+ evaluations += 1
+ candidate = {key: list(value) for key, value in loads.items()}
+ candidate[source].remove(session_id)
+ candidate[target].append(session_id)
+ gain = current - self._bottleneck(snapshot, candidate, capacity)
+ gain -= self.config.migration_eta * migration
+ score = (gain, -session.state_size_mb, target, session_id)
+ if best is None or score > best[0]:
+ best = (score, session_id, target)
+ if best is None or best[0][0] <= self.config.min_gain_ms:
+ break
+ _, session_id, target = best
+ loads[source].remove(session_id)
+ loads[target].append(session_id)
+ placement[session_id] = target
+ moves += 1
+ return moves, evaluations
+
+ def _bottleneck(self, snapshot: TurboServeSchedulingSnapshot, loads: dict[str, list[str]], capacity: int) -> float:
+ return max((self._worker_worst(snapshot, sessions, capacity) for sessions in loads.values()), default=0.0)
+
+ def _worker_worst(self, snapshot: TurboServeSchedulingSnapshot, sessions: list[str], capacity: int) -> float:
+ if not sessions:
+ return 0.0
+ return max(self.latency_model.session_latency_ms(snapshot.sessions[session_id], len(sessions), capacity, snapshot.runtime_calibration) for session_id in sessions)
diff --git a/telefuser/service/livekit/worker_pool.py b/telefuser/service/livekit/worker_pool.py
index 0ac00647..a4122847 100644
--- a/telefuser/service/livekit/worker_pool.py
+++ b/telefuser/service/livekit/worker_pool.py
@@ -7,7 +7,9 @@
from telefuser.utils.logging import logger
+from .pipeline_router import TurboServePipelineRouter
from .session_registry import SessionRecord
+from .turboserve import TurboServeOwnership
from .worker import LiveKitWorker
_SESSION_STOP_GRACE_SECONDS = 8.0
@@ -25,34 +27,58 @@ async def aclose(self) -> None: ...
class InProcessLiveKitWorkerPool:
- """Run LiveKit workers as asyncio tasks in the API server process."""
-
- def __init__(self, workers: dict[str, LiveKitWorker]) -> None:
+ """Run independently device-bound LiveKit workers in the API process."""
+
+ def __init__(
+ self,
+ workers: dict[str, LiveKitWorker],
+ *,
+ router: TurboServePipelineRouter | None = None,
+ initial_workers: int | None = None,
+ ) -> None:
+ if initial_workers is not None and not 1 <= initial_workers <= len(workers):
+ raise ValueError("initial_workers must be within the configured worker pool")
self._workers = workers
+ self.router = router
+ self._initial_workers = initial_workers
self._started = False
+ self._skip_validation = False
self._tasks: dict[str, asyncio.Task] = {}
+ self._task_workers: dict[str, str] = {}
+ self._active_workers: set[str] = set()
+ self._scale_lock = asyncio.Lock()
async def start(self, *, skip_validation: bool = False) -> None:
- """Load all worker-owned pipelines."""
+ """Load the configured initial replica set."""
if self._started:
return
- for worker in self._workers.values():
- await worker.start(skip_validation=skip_validation)
-
self._started = True
+ self._skip_validation = skip_validation
+ target = self._initial_workers or len(self._workers)
+ try:
+ await self.scale_to(target)
+ except Exception:
+ self._started = False
+ raise
+ for worker_id, worker in self._workers.items():
+ if worker_id not in self._active_workers:
+ worker.event_sink.on_worker_status(worker_id, "stopped")
def start_session(self, record: SessionRecord) -> None:
- """Start a worker task for an assigned session."""
+ """Start a room runner on its assigned active worker."""
if not self._started:
raise RuntimeError("LiveKit worker pool is not started")
if record.worker_id is None:
raise RuntimeError(f"Session {record.session_id} has no assigned worker")
+ if record.worker_id not in self._active_workers:
+ raise RuntimeError(f"Worker {record.worker_id} is not active")
if record.session_id in self._tasks:
raise RuntimeError(f"Session {record.session_id} is already running")
worker = self._workers[record.worker_id]
task = asyncio.create_task(worker.run_session(record), name=f"livekit-worker-{record.worker_id}")
self._tasks[record.session_id] = task
+ self._task_workers[record.session_id] = record.worker_id
task.add_done_callback(lambda done: self._on_task_done(record.session_id, done))
async def stop_session(self, session_id: str) -> None:
@@ -60,8 +86,9 @@ async def stop_session(self, session_id: str) -> None:
task = self._tasks.get(session_id)
if task is None:
return
- for worker in self._workers.values():
- await worker.stop_session(session_id)
+ worker_id = self._task_workers.get(session_id)
+ if worker_id is not None:
+ await self._workers[worker_id].stop_session(session_id)
try:
await asyncio.wait_for(asyncio.shield(task), timeout=_SESSION_STOP_GRACE_SECONDS)
@@ -84,17 +111,75 @@ async def stop_session(self, session_id: str) -> None:
f"{_SESSION_CANCEL_GRACE_SECONDS:g}s: session={session_id}"
)
+ async def migrate_session(
+ self, pipeline_session_id: str, target_worker_id: str
+ ) -> TurboServeOwnership:
+ """Move model state while the existing LiveKit runner keeps publishing."""
+ if self.router is None:
+ raise RuntimeError("Worker pool was created without TurboServe routing")
+ if target_worker_id not in self._active_workers:
+ raise RuntimeError(f"Migration target {target_worker_id} is not active")
+ return await asyncio.to_thread(self.router.migrate_session, pipeline_session_id, target_worker_id)
+
+ async def scale_to(self, target_workers: int) -> int:
+ """Start replicas or retire idle replicas until the requested count is reached."""
+ if not self._started:
+ raise RuntimeError("LiveKit worker pool is not started")
+ if not 1 <= target_workers <= len(self._workers):
+ raise ValueError("target_workers must be within the configured worker pool")
+ async with self._scale_lock:
+ while len(self._active_workers) < target_workers:
+ worker_id = next(worker_id for worker_id in self._workers if worker_id not in self._active_workers)
+ worker = self._workers[worker_id]
+ await worker.start(skip_validation=self._skip_validation)
+ self._active_workers.add(worker_id)
+
+ while len(self._active_workers) > target_workers:
+ candidate = self._scale_in_candidate()
+ if candidate is None:
+ break
+ await self._workers[candidate].stop()
+ self._active_workers.remove(candidate)
+ return len(self._active_workers)
+
+ def active_worker_count(self) -> int:
+ return len(self._active_workers)
+
+ def turboserve_snapshot(self) -> dict[str, object] | None:
+ snapshot = self.router.snapshot() if self.router is not None else {}
+ return {
+ **snapshot,
+ "active_workers": sorted(self._active_workers),
+ "configured_workers": len(self._workers),
+ }
+
async def aclose(self) -> None:
- """Stop every active session and worker."""
- session_ids = list(self._tasks.keys())
- self._started = False
+ """Stop every active session and loaded worker."""
+ session_ids = list(self._tasks)
for session_id in session_ids:
await self.stop_session(session_id)
- for worker in self._workers.values():
- await worker.stop()
+ for worker_id in tuple(self._active_workers):
+ await self._workers[worker_id].stop()
+ self._active_workers.clear()
+ self._started = False
+
+ def _scale_in_candidate(self) -> str | None:
+ busy_transport_workers = set(self._task_workers.values())
+ retained_by_worker: dict[str, int] = {}
+ if self.router is not None:
+ retained_by_worker = self.router.snapshot()["retained_sessions_by_worker"]
+ candidates = [
+ worker_id
+ for worker_id in reversed(tuple(self._workers))
+ if worker_id in self._active_workers
+ and worker_id not in busy_transport_workers
+ and retained_by_worker.get(worker_id, 0) == 0
+ ]
+ return candidates[0] if candidates else None
def _on_task_done(self, session_id: str, task: asyncio.Task) -> None:
self._tasks.pop(session_id, None)
+ self._task_workers.pop(session_id, None)
if task.cancelled():
return
exc = task.exception()
diff --git a/tests/unit/models/test_wan22_vae_streaming_state.py b/tests/unit/models/test_wan22_vae_streaming_state.py
new file mode 100644
index 00000000..5a4fc637
--- /dev/null
+++ b/tests/unit/models/test_wan22_vae_streaming_state.py
@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import torch
+import torch.nn as nn
+
+from telefuser.models import wan22_video_vae
+
+
+class _StatefulRecordingDecoder(nn.Module):
+ def forward(self, x, feat_cache, feat_idx, first_chunk: bool = False):
+ del first_chunk
+ feat_idx[0] += 1
+ feat_cache[0] = x.detach().clone()
+ return x
+
+
+def test_cached_decode_uses_explicit_session_owned_state(monkeypatch) -> None:
+ fake_vae = SimpleNamespace(
+ model=SimpleNamespace(conv2=lambda value: value, decoder=_StatefulRecordingDecoder()),
+ z_dim=1,
+ _feat_cache=[],
+ _feat_idx=[0],
+ _get_scale_on_device=lambda _device, _dtype: [torch.zeros(1), torch.ones(1)],
+ )
+ monkeypatch.setattr(wan22_video_vae, "_count_conv3d", lambda _decoder: 1)
+ monkeypatch.setattr(wan22_video_vae, "unpatchify", lambda video, patch_size: video)
+ state = wan22_video_vae.Wan22VideoVAEStreamingDecodeState()
+
+ wan22_video_vae.Wan22VideoVAE.cached_decode_withflag(
+ fake_vae,
+ torch.ones(1, 1, 1, 1, 1),
+ torch.device("cpu"),
+ True,
+ False,
+ state,
+ )
+
+ assert len(state.feat_cache) == 1
+ assert state.feat_cache[0].item() == 1
+ assert fake_vae._feat_cache == []
+
+
+def test_cached_decode_batches_and_scatters_temporal_state(monkeypatch) -> None:
+ fake_vae = SimpleNamespace(
+ model=SimpleNamespace(conv2=lambda value: value, decoder=_StatefulRecordingDecoder()),
+ z_dim=1,
+ _get_scale_on_device=lambda _device, _dtype: [torch.zeros(1), torch.ones(1)],
+ )
+ monkeypatch.setattr(wan22_video_vae, "_count_conv3d", lambda _decoder: 1)
+ monkeypatch.setattr(wan22_video_vae, "unpatchify", lambda video, patch_size: video)
+ states = [
+ wan22_video_vae.Wan22VideoVAEStreamingDecodeState(),
+ wan22_video_vae.Wan22VideoVAEStreamingDecodeState(),
+ ]
+
+ output = wan22_video_vae.Wan22VideoVAE.cached_decode_batch_withflag(
+ fake_vae,
+ torch.tensor([1.0, 2.0]).view(2, 1, 1, 1, 1),
+ torch.device("cpu"),
+ True,
+ False,
+ states,
+ )
+
+ assert output.shape == (2, 1, 1, 1, 1)
+ assert states[0].feat_cache[0].item() == 1
+ assert states[1].feat_cache[0].item() == 2
+ states[0].feat_cache[0].zero_()
+ assert states[1].feat_cache[0].item() == 2
diff --git a/tests/unit/orchestrator/test_batched_stage_actor.py b/tests/unit/orchestrator/test_batched_stage_actor.py
new file mode 100644
index 00000000..1251dcca
--- /dev/null
+++ b/tests/unit/orchestrator/test_batched_stage_actor.py
@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import threading
+
+from telefuser.orchestrator import (
+ BatchedLocalStageActor,
+ StreamingSessionCloseReason,
+ StreamingSessionContext,
+ StreamingStageInvocation,
+ StreamingTaskKey,
+)
+
+
+def _invocation(session_id: str, sequence_id: int, bucket: str = "same") -> StreamingStageInvocation:
+ return StreamingStageInvocation(
+ key=StreamingTaskKey(session_id, 1, sequence_id, "stage", f"{session_id}-{sequence_id}"),
+ inputs={"value": sequence_id, "bucket": bucket},
+ is_first=sequence_id == 0,
+ is_last=False,
+ )
+
+
+def test_actor_coalesces_compatible_cross_session_invocations() -> None:
+ observed_batches: list[list[str]] = []
+ release = threading.Event()
+
+ def handler(invocations: list[StreamingStageInvocation]) -> list[dict[str, object]]:
+ observed_batches.append([item.key.session_id for item in invocations])
+ release.wait(timeout=1)
+ return [{"output": item.inputs["value"]} for item in invocations]
+
+ actor = BatchedLocalStageActor(
+ handler,
+ batch_key=lambda item: item.inputs["bucket"],
+ max_batch_size=4,
+ batching_window_seconds=0.05,
+ )
+ first = actor.submit(_invocation("a", 0))
+ second = actor.submit(_invocation("b", 0))
+ release.set()
+ try:
+ assert first.result(timeout=1) == {"output": 0}
+ assert second.result(timeout=1) == {"output": 0}
+ assert observed_batches == [["a", "b"]]
+ assert actor.batch_metrics()["max_batch_size"] == 2
+ finally:
+ actor.close()
+
+
+def test_actor_does_not_batch_two_strict_items_from_the_same_session() -> None:
+ batch_sizes: list[int] = []
+
+ def handler(invocations: list[StreamingStageInvocation]) -> list[dict[str, object]]:
+ batch_sizes.append(len(invocations))
+ return [{"output": item.key.sequence_id} for item in invocations]
+
+ actor = BatchedLocalStageActor(handler, batching_window_seconds=0.01)
+ first = actor.submit(_invocation("a", 0))
+ second = actor.submit(_invocation("a", 1))
+ try:
+ assert first.result(timeout=1) == {"output": 0}
+ assert second.result(timeout=1) == {"output": 1}
+ assert batch_sizes == [1, 1]
+ finally:
+ actor.close()
+
+
+def test_session_cleanup_runs_after_preceding_actor_work() -> None:
+ events: list[str] = []
+
+ def handler(invocations: list[StreamingStageInvocation]) -> list[dict[str, object]]:
+ events.append("batch")
+ return [{"output": 1} for _ in invocations]
+
+ def close_session(context: StreamingSessionContext, reason: StreamingSessionCloseReason) -> None:
+ events.append(f"close:{context.session_id}:{reason.value}")
+
+ actor = BatchedLocalStageActor(handler, session_closer=close_session)
+ future = actor.submit(_invocation("a", 0))
+ try:
+ assert future.result(timeout=1) == {"output": 1}
+ actor.close_session(
+ StreamingSessionContext("a", 1),
+ StreamingSessionCloseReason.CLOSED,
+ )
+ assert events == ["batch", "close:a:closed"]
+ finally:
+ actor.close()
diff --git a/tests/unit/pipelines/abot_world/test_interactive.py b/tests/unit/pipelines/abot_world/test_interactive.py
index 3588d50a..48fd5cf5 100644
--- a/tests/unit/pipelines/abot_world/test_interactive.py
+++ b/tests/unit/pipelines/abot_world/test_interactive.py
@@ -4,31 +4,75 @@
import torch
+from telefuser.models.wan22_video_vae import Wan22VideoVAEStreamingDecodeState
from telefuser.pipelines.abot_world.interactive import (
ABotWorldInteractivePipeline,
ABotWorldInteractiveSession,
+ ABotWorldSessionLifecycle,
)
-def test_close_interactive_session_clears_all_retained_state() -> None:
- pipeline = ABotWorldInteractivePipeline(device="cpu")
- vae = SimpleNamespace(_feat_cache=[torch.ones(1)], _feat_idx=[3])
- pipeline.vae_stage = SimpleNamespace(vae=vae)
- session = ABotWorldInteractiveSession(
+def _session(session_id: str) -> ABotWorldInteractiveSession:
+ return ABotWorldInteractiveSession(
+ session_id=session_id,
prompt_emb=torch.ones(1),
first_frame_latent=torch.ones(1),
self_cache=[{"k": torch.ones(1)}],
cross_cache=[{"k": torch.ones(1)}],
scheduler=object(),
generator=torch.Generator(device="cpu"),
+ vae_decode_state=Wan22VideoVAEStreamingDecodeState(feat_cache=[torch.ones(1)]),
)
- pipeline._interactive_session = session
- pipeline.close_interactive_session(session)
- assert session.closed
- assert session.self_cache == []
- assert session.cross_cache == []
- assert vae._feat_cache == []
- assert vae._feat_idx == [0]
- assert pipeline._interactive_session is None
+def test_close_interactive_session_clears_only_target_state() -> None:
+ pipeline = ABotWorldInteractivePipeline(device="cpu")
+ pipeline.vae_stage = SimpleNamespace(vae=SimpleNamespace(_feat_cache=[torch.ones(1)], _feat_idx=[3]))
+ first = _session("first")
+ second = _session("second")
+ pipeline._interactive_sessions = {"first": first, "second": second}
+
+ pipeline.close_interactive_session(first)
+
+ assert first.closed
+ assert first.lifecycle == ABotWorldSessionLifecycle.CLOSED
+ assert first.self_cache == []
+ assert first.cross_cache == []
+ assert first.vae_decode_state.feat_cache == []
+ assert second.self_cache
+ assert second.cross_cache
+ assert second.vae_decode_state.feat_cache
+ assert pipeline._interactive_sessions == {"second": second}
+ # Session cleanup must not mutate the legacy model-owned cache.
+ assert pipeline.vae_stage.vae._feat_idx == [3]
+
+
+def test_cache_collation_and_scatter_preserve_session_isolation() -> None:
+ first = _session("first")
+ second = _session("second")
+ first.self_cache = [
+ {
+ "k": torch.tensor([[[[1.0]]]]),
+ "v": torch.tensor([[[[2.0]]]]),
+ "global_end_index": torch.tensor([3]),
+ "local_end_index": torch.tensor([3]),
+ }
+ ]
+ second.self_cache = [
+ {
+ "k": torch.tensor([[[[4.0]]]]),
+ "v": torch.tensor([[[[5.0]]]]),
+ "global_end_index": torch.tensor([12]),
+ "local_end_index": torch.tensor([3]),
+ }
+ ]
+
+ collated = ABotWorldInteractivePipeline._collate_caches([first, second], "self_cache")
+ assert collated[0]["k"].shape[0] == 2
+ collated[0]["k"].add_(10)
+ ABotWorldInteractivePipeline._scatter_caches([first, second], "self_cache", collated)
+
+ assert first.self_cache[0]["k"].item() == 11
+ assert second.self_cache[0]["k"].item() == 14
+ first.self_cache[0]["k"].zero_()
+ assert second.self_cache[0]["k"].item() == 14
diff --git a/tests/unit/pipelines/abot_world/test_interactive_web.py b/tests/unit/pipelines/abot_world/test_interactive_web.py
index 13efb8cf..6b3380e4 100644
--- a/tests/unit/pipelines/abot_world/test_interactive_web.py
+++ b/tests/unit/pipelines/abot_world/test_interactive_web.py
@@ -48,6 +48,17 @@ def test_connect_with_empty_controls_does_not_advance_dit(tmp_path: Path) -> Non
runtime.stop()
+def test_runtime_accepts_two_latent_experimental_chunk(tmp_path: Path) -> None:
+ image_path = tmp_path / "initial.png"
+ _write_image(image_path)
+ runtime = InteractiveRuntime(_FakePipeline(), fps=8, control_latent_frames=2, output_queue_size=2)
+ try:
+ assert runtime.control_latent_frames == 2
+ assert runtime.fps == 8
+ finally:
+ runtime.stop()
+
+
def test_full_fifo_applies_backpressure_without_reordering() -> None:
pipeline = _FakePipeline()
runtime = InteractiveRuntime(pipeline, fps=12, control_latent_frames=3, output_queue_size=1)
diff --git a/tests/unit/pipelines/abot_world/test_livekit_examples.py b/tests/unit/pipelines/abot_world/test_livekit_examples.py
index e8c22f07..548e8dd4 100644
--- a/tests/unit/pipelines/abot_world/test_livekit_examples.py
+++ b/tests/unit/pipelines/abot_world/test_livekit_examples.py
@@ -17,19 +17,24 @@ def fake_get_pipeline(**kwargs: object) -> object:
return pipeline
monkeypatch.setattr(service_example, "get_pipeline", fake_get_pipeline)
- service = service_example.get_service(gpu_num=1)
+ service = service_example.get_service(gpu_num=1, gpu_ids=["3"])
assert isinstance(service, ABotWorldLiveKitService)
assert service.pipeline is pipeline
- assert captured == {"pipeline_class": ABotWorldInteractivePipeline}
- assert service.default_fps == 12
- assert service.default_session_config["fps"] == 12
- assert service.default_session_config["control_latent_frames"] == 3
+ assert captured == {"device_id": 3, "pipeline_class": ABotWorldInteractivePipeline}
+ assert service.default_fps == 8
+ assert service.default_session_config["fps"] == 8
+ assert service.default_session_config["control_latent_frames"] == 2
assert service.default_session_config["seed"] == 42
assert service.default_session_config["prompt"] == service_example.DEFAULT_PROMPT
assert str(service.default_session_config["image_path"]).endswith("84b90ad568b693d2.png")
+def test_livekit_service_entrypoint_rejects_non_numeric_gpu_id() -> None:
+ with pytest.raises(ValueError, match="must be numeric"):
+ service_example.get_service(gpu_num=1, gpu_ids=["GPU-deadbeef"])
+
+
@pytest.mark.parametrize("gpu_num", [0, 2])
def test_livekit_service_entrypoint_rejects_unsupported_gpu_counts(gpu_num: int) -> None:
with pytest.raises(ValueError, match="exactly one GPU"):
diff --git a/tests/unit/pipelines/abot_world/test_livekit_service.py b/tests/unit/pipelines/abot_world/test_livekit_service.py
index 4f5cdeed..4b19597f 100644
--- a/tests/unit/pipelines/abot_world/test_livekit_service.py
+++ b/tests/unit/pipelines/abot_world/test_livekit_service.py
@@ -7,37 +7,111 @@
from types import SimpleNamespace
import pytest
+import torch
from PIL import Image
-from telefuser.pipelines.abot_world.service import (
- ABotWorldLiveKitService,
- _ABotWorldLiveKitSession,
-)
+from telefuser.pipelines.abot_world.interactive import ABotWorldSessionLifecycle
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService, _ABotWorldLiveKitSession
from telefuser.service.core.stream_pipeline_service import BidirectionalService
+class _FakePipelineSession:
+ def __init__(self, session_id: str) -> None:
+ self.session_id = session_id
+ self.next_latent_frame = 0
+ self.first_frame_latent = torch.zeros(1, 1, 1, 1, 1)
+ self.self_cache = [
+ {
+ "local_end_index": torch.zeros(1, dtype=torch.long),
+ "global_end_index": torch.zeros(1, dtype=torch.long),
+ }
+ ]
+ self.lifecycle = ABotWorldSessionLifecycle.READY
+ self.closed = False
+
+ @property
+ def is_resident(self) -> bool:
+ return self.lifecycle != ABotWorldSessionLifecycle.SUSPENDED
+
+
class _FakePipeline:
def __init__(self) -> None:
self.config = SimpleNamespace(width=8, height=8)
- self.generate_calls: list[dict[str, bool]] = []
- self.closed_sessions: list[object] = []
+ self.device = torch.device("cpu")
+ self.torch_dtype = torch.float32
+ self.denoise_stage = SimpleNamespace(
+ dit=SimpleNamespace(
+ patch_size=(1, 2, 2),
+ dim=8,
+ num_heads=2,
+ num_layers=2,
+ local_attn_size=18,
+ text_len=8,
+ )
+ )
+ self.generate_calls: list[tuple[str, dict[str, bool]]] = []
+ self.batch_sizes: list[int] = []
+ self.closed_sessions: list[str] = []
+ self.suspended_sessions: list[str] = []
+ self.restored_sessions: list[str] = []
self.closed = False
def preload_models(self) -> None:
return None
- def create_interactive_session(self, image: Image.Image, prompt: str, *, seed: int) -> object:
+ def create_interactive_session(
+ self,
+ image: Image.Image,
+ prompt: str,
+ *,
+ seed: int,
+ session_id: str | None = None,
+ ) -> _FakePipelineSession:
+ del seed
assert image.mode == "RGB"
assert prompt
- return object()
-
- def generate_next_block(self, session: object, controls: dict[str, bool], *, control_latent_frames: int) -> list:
+ assert session_id is not None
+ return _FakePipelineSession(session_id)
+
+ def generate_next_block(
+ self,
+ session: _FakePipelineSession,
+ controls: dict[str, bool],
+ *,
+ control_latent_frames: int,
+ ) -> list[Image.Image]:
assert control_latent_frames == 3
- self.generate_calls.append(controls)
- return [Image.new("RGB", (8, 8), color=(20, len(self.generate_calls), 40))]
-
- def close_interactive_session(self, session: object) -> None:
- self.closed_sessions.append(session)
+ self.generate_calls.append((session.session_id, controls))
+ self.batch_sizes.append(1)
+ session.next_latent_frame += control_latent_frames
+ return [Image.new("RGB", (8, 8), color=(20, len(self.generate_calls) % 255, 40))]
+
+ def generate_next_blocks(
+ self,
+ sessions: list[_FakePipelineSession],
+ controls: list[dict[str, bool]],
+ *,
+ control_latent_frames: int,
+ ) -> list[list[Image.Image]]:
+ self.batch_sizes.append(len(sessions))
+ results = []
+ for session, state in zip(sessions, controls):
+ self.generate_calls.append((session.session_id, state))
+ session.next_latent_frame += control_latent_frames
+ results.append([Image.new("RGB", (8, 8), color=(20, len(self.generate_calls) % 255, 40))])
+ return results
+
+ def suspend_interactive_session(self, session: _FakePipelineSession) -> None:
+ session.lifecycle = ABotWorldSessionLifecycle.SUSPENDED
+ self.suspended_sessions.append(session.session_id)
+
+ def restore_interactive_session(self, session: _FakePipelineSession) -> None:
+ session.lifecycle = ABotWorldSessionLifecycle.READY
+ self.restored_sessions.append(session.session_id)
+
+ def close_interactive_session(self, session: _FakePipelineSession) -> None:
+ session.closed = True
+ self.closed_sessions.append(session.session_id)
def close(self) -> None:
self.closed = True
@@ -53,136 +127,228 @@ def _service(**kwargs: object) -> tuple[ABotWorldLiveKitService, _FakePipeline]:
return service, pipeline
-def test_service_matches_shared_bidirectional_contract() -> None:
+def _create(service: ABotWorldLiveKitService, session_id: str, **config: object) -> str:
+ return service.create_session(
+ {
+ "session_id": session_id,
+ "image": Image.new("RGB", (8, 8)),
+ **config,
+ }
+ )
+
+
+def test_service_matches_shared_multi_session_bidirectional_contract() -> None:
service, _ = _service()
assert isinstance(service, BidirectionalService)
- assert service.configure_session_capacity(1)["effective_capacity"] == 1
- with pytest.raises(ValueError, match="one retained"):
- service.configure_session_capacity(2)
+ profile = service.configure_session_capacity(3)
+ assert profile["effective_capacity"] == 3
+ assert profile["max_batch_size"] == 8
+ service.stop()
-def test_session_is_preview_only_until_control_and_preserves_chunk_order() -> None:
- service, pipeline = _service(output_queue_size=2)
- session_id = service.create_session({"image": Image.new("RGB", (10, 10)), "prompt": "test"})
- state = service._session(session_id)
- assert state is not None
- preview = state.output_queue.get(timeout=1)
- assert preview["type"] == "preview"
- assert pipeline.generate_calls == []
-
- service.push_chunk(session_id, {"type": "control_state", "controls": ["ArrowUp"]})
- generated = state.output_queue.get(timeout=1)
- assert generated["type"] == "chunk"
- assert generated["index"] == 0
- assert generated["controls"] == ["W"]
- assert pipeline.generate_calls == [{"W": True}]
- service.close_session(session_id)
- assert len(pipeline.closed_sessions) == 1
+def test_capacity_profile_accepts_explicit_cuda_device_string(monkeypatch) -> None:
+ service, pipeline = _service()
+ pipeline.device = "cuda:3"
+ monkeypatch.setattr("telefuser.pipelines.abot_world.service.torch.cuda.is_available", lambda: True)
+ monkeypatch.setattr(service, "_profile_session_memory", lambda: {
+ "profiled_session_bytes": 100,
+ "workspace_peak_bytes": 200,
+ })
+ observed = {}
+
+ def fake_mem_get_info(device):
+ observed["device"] = device
+ return 10_000, 20_000
+
+ monkeypatch.setattr("telefuser.pipelines.abot_world.service.torch.cuda.mem_get_info", fake_mem_get_info)
+
+ profile = service.configure_session_capacity(2)
+
+ assert observed["device"] == torch.device("cuda:3")
+ assert profile["effective_capacity"] == 2
+ service.stop()
+
+def test_round_robin_capacity_uses_one_active_workspace(monkeypatch) -> None:
+ service, pipeline = _service(max_batch_size=8)
+ pipeline.device = "cuda:0"
+ monkeypatch.setattr("telefuser.pipelines.abot_world.service.torch.cuda.is_available", lambda: True)
+ monkeypatch.setattr(service, "_estimate_session_bytes", lambda: 100)
+ monkeypatch.setattr(service, "_profile_session_memory", lambda: {
+ "profiled_session_bytes": 100,
+ "workspace_peak_bytes": 200,
+ })
+ monkeypatch.setattr(
+ "telefuser.pipelines.abot_world.service.torch.cuda.mem_get_info",
+ lambda device: (1_000, 2_000),
+ )
+ profile = service.configure_session_capacity(10)
-def test_control_aliases_and_release_stop_generation() -> None:
- service, pipeline = _service(output_queue_size=4)
- session_id = service.create_session({"image": Image.new("RGB", (8, 8))})
- state = service._session(session_id)
- assert state is not None
- state.output_queue.get(timeout=1)
- service.push_chunk(session_id, {"type": "control", "control": "KeyJ", "event": "press"})
- assert state.output_queue.get(timeout=1)["controls"] == ["J"]
- service.push_chunk(session_id, {"type": "control", "control": "KeyJ", "event": "release"})
- calls_after_release = len(pipeline.generate_calls)
- time.sleep(0.15)
- assert len(pipeline.generate_calls) == calls_after_release
- service.close_session(session_id)
+ assert profile["computed_capacity"] == 7
+ assert profile["effective_capacity"] == 7
+ assert profile["estimated_batch_workspace_bytes"] == 200
+ assert profile["scheduler_mode"] == "round_robin"
+ service.stop()
-def test_bounded_output_queue_applies_backpressure_without_dropping() -> None:
- service, _ = _service(output_queue_size=1)
- state = _ABotWorldLiveKitSession(
- session_id="test",
- pipeline_session=object(),
- output_queue=queue.Queue(maxsize=1),
- control_event=threading.Event(),
- config={"fps": 12, "control_latent_frames": 3},
- )
- first = {"type": "chunk", "index": 0}
- second = {"type": "chunk", "index": 1}
- state.output_queue.put(first)
- completed = threading.Event()
-
- def produce() -> None:
- assert service._put_output(state, second)
- completed.set()
-
- producer = threading.Thread(target=produce)
- producer.start()
- time.sleep(0.1)
- assert not completed.is_set()
- assert state.output_queue.get(timeout=1) is first
- producer.join(timeout=1)
- assert completed.is_set()
- assert state.output_queue.get(timeout=1) is second
-
-
-def test_public_pull_stream_yields_thirty_complete_blocks_in_order() -> None:
- service, pipeline = _service(output_queue_size=1, control_idle_timeout=30.0)
- session_id = service.create_session({"image": Image.new("RGB", (8, 8))})
- state = service._session(session_id)
- assert state is not None
- async def collect() -> list[dict]:
- chunks: list[dict] = []
- preview_seen = False
+def test_two_ready_sessions_are_generated_in_one_batch_and_keep_order() -> None:
+ service, pipeline = _service(output_queue_size=4, batching_window_ms=30, scheduler_mode="batched")
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ second = _create(service, "second")
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ assert first_state.output_queue.get(timeout=1)["type"] == "preview"
+ assert second_state.output_queue.get(timeout=1)["type"] == "preview"
+
+ service.push_chunk(first, {"type": "control_state", "controls": ["KeyW"]})
+ service.push_chunk(second, {"type": "control_state", "controls": ["KeyD"]})
+ first_chunk = first_state.output_queue.get(timeout=2)
+ second_chunk = second_state.output_queue.get(timeout=2)
+
+ assert first_chunk["index"] == 0
+ assert second_chunk["index"] == 0
+ assert first_chunk["scheduler"]["batch_size"] == 2
+ assert second_chunk["scheduler"]["batch_size"] == 2
+ assert 2 in pipeline.batch_sizes
+ service.stop()
+
+
+def test_service_accepts_two_latent_experimental_chunk() -> None:
+ service, _ = _service()
+ service.configure_session_capacity(1)
+ session_id = _create(service, "two-latent", fps=8, control_latent_frames=2)
+ try:
+ state = service._session(session_id)
+ assert state is not None
+ assert state.config["fps"] == 8
+ assert state.config["control_latent_frames"] == 2
+ finally:
+ service.close_session(session_id)
+ service.stop()
+
+
+def test_default_scheduler_round_robins_single_session_steps() -> None:
+ service, pipeline = _service(output_queue_size=4)
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ second = _create(service, "second")
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ assert first_state.output_queue.get(timeout=1)["type"] == "preview"
+ assert second_state.output_queue.get(timeout=1)["type"] == "preview"
+
+ service.push_chunk(first, {"type": "control_state", "controls": ["KeyW"]})
+ service.push_chunk(second, {"type": "control_state", "controls": ["KeyD"]})
+ first_chunk = first_state.output_queue.get(timeout=2)
+ second_chunk = second_state.output_queue.get(timeout=2)
+
+ assert first_chunk["scheduler"]["batch_size"] == 1
+ assert second_chunk["scheduler"]["batch_size"] == 1
+ assert pipeline.batch_sizes[:2] == [1, 1]
+ assert service.runtime_metrics()["scheduler_mode"] == "round_robin"
+ service.stop()
+
+
+def test_lossless_sessions_each_stream_thirty_chunks_without_drops() -> None:
+ service, _ = _service(output_queue_size=2, batching_window_ms=10, control_idle_timeout=30)
+ service.configure_session_capacity(2)
+ session_ids = [_create(service, value, delivery_mode="lossless") for value in ("a", "b")]
+
+ async def collect(session_id: str) -> list[int]:
+ indexes: list[int] = []
async for payload in service.pull_chunks(session_id):
if payload["type"] == "preview":
- preview_seen = True
service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
continue
- assert preview_seen
- assert payload["type"] == "chunk"
- chunks.append(payload)
- if len(chunks) == 30:
+ indexes.append(payload["index"])
+ if len(indexes) == 30:
service.push_chunk(session_id, {"type": "control", "control": "KeyW", "event": "release"})
- break
- state.control_event.set()
- return chunks
+ return indexes
+ return indexes
+
+ async def run() -> list[list[int]]:
+ return await asyncio.gather(*(collect(session_id) for session_id in session_ids))
try:
- chunks = asyncio.run(collect())
- assert [chunk["index"] for chunk in chunks] == list(range(30))
- assert all(len(chunk["frames"]) == 1 for chunk in chunks)
- assert pipeline.generate_calls == [{"W": True}] * 30
+ indexes = asyncio.run(run())
+ assert indexes == [list(range(30)), list(range(30))]
+ for session_id in session_ids:
+ assert service.runtime_metrics(session_id)["dropped_video_payloads"] == 0
finally:
- service.close_session(session_id)
-
+ service.stop()
-def test_public_pull_stream_drains_preview_then_finishes_after_stop() -> None:
- service, pipeline = _service(output_queue_size=1)
- session_id = service.create_session({"image": Image.new("RGB", (8, 8))})
- service.push_chunk(session_id, {"type": "stop"})
- async def collect() -> list[dict]:
- return [payload async for payload in service.pull_chunks(session_id)]
+def test_latest_queue_discards_oldest_video_and_records_metric() -> None:
+ service, _ = _service(output_queue_size=1)
+ pipeline_session = _FakePipelineSession("test")
+ state = _ABotWorldLiveKitSession(
+ session_id="test",
+ pipeline_session=pipeline_session,
+ output_queue=queue.Queue(maxsize=1),
+ control_event=threading.Event(),
+ config={"fps": 12, "control_latent_frames": 3, "delivery_mode": "latest"},
+ )
+ state.output_queue.put({"type": "chunk", "index": 0})
- try:
- payloads = asyncio.run(collect())
- assert [payload["type"] for payload in payloads] == ["preview"]
- assert pipeline.generate_calls == []
- finally:
- service.close_session(session_id)
+ assert service._put_output(state, {"type": "chunk", "index": 1})
+ assert state.output_queue.get_nowait()["index"] == 1
+ assert state.dropped_video_payloads == 1
+ service.stop()
-def test_second_livekit_session_is_rejected_until_first_is_closed() -> None:
+def test_migration_waits_for_already_generated_output_to_drain() -> None:
service, pipeline = _service()
- first_session_id = service.create_session({"image": Image.new("RGB", (8, 8))})
- try:
- with pytest.raises(RuntimeError, match="one retained session"):
- service.create_session({"image": Image.new("RGB", (8, 8))})
- finally:
- service.close_session(first_session_id)
+ service.configure_session_capacity(1)
+ session_id = _create(service, "migrating")
+ state = service._session(session_id)
+ assert state is not None
+ pipeline.snapshot_interactive_session = lambda session: SimpleNamespace(session_id=session.session_id)
+
+ def drain_preview() -> None:
+ time.sleep(0.05)
+ state.output_queue.get_nowait()
+ with service._scheduler_condition:
+ service._scheduler_condition.notify_all()
+
+ thread = threading.Thread(target=drain_preview)
+ thread.start()
+ started_at = time.monotonic()
+ bundle = service.prepare_migration(session_id, timeout=1)
+ thread.join()
+
+ assert bundle.snapshot.session_id == session_id
+ assert time.monotonic() - started_at >= 0.04
+ service.abort_migration(session_id)
+ service.close_session(session_id)
+ service.stop()
- second_session_id = service.create_session({"image": Image.new("RGB", (8, 8))})
- service.close_session(second_session_id)
- assert len(pipeline.closed_sessions) == 2
+
+def test_idle_session_suspends_and_restores_on_new_control() -> None:
+ service, pipeline = _service(
+ output_queue_size=2,
+ batching_window_ms=0,
+ idle_suspension_seconds=0.02,
+ control_idle_timeout=30,
+ )
+ service.configure_session_capacity(1)
+ session_id = _create(service, "idle")
+ state = service._session(session_id)
+ assert state is not None
+ state.output_queue.get(timeout=1)
+
+ deadline = time.monotonic() + 1
+ while session_id not in pipeline.suspended_sessions and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert session_id in pipeline.suspended_sessions
+
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
+ assert state.output_queue.get(timeout=1)["type"] == "chunk"
+ assert session_id in pipeline.restored_sessions
+ service.stop()
@pytest.mark.parametrize(
@@ -195,9 +361,10 @@ def test_second_livekit_session_is_rejected_until_first_is_closed() -> None:
)
def test_invalid_livekit_control_payloads_are_rejected(payload: dict) -> None:
service, _ = _service()
- session_id = service.create_session({"image": Image.new("RGB", (8, 8))})
+ service.configure_session_capacity(1)
+ session_id = _create(service, "invalid")
try:
with pytest.raises(ValueError):
service.push_chunk(session_id, payload)
finally:
- service.close_session(session_id)
+ service.stop()
diff --git a/tests/unit/pipelines/abot_world/test_migration.py b/tests/unit/pipelines/abot_world/test_migration.py
new file mode 100644
index 00000000..13c88d03
--- /dev/null
+++ b/tests/unit/pipelines/abot_world/test_migration.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import torch
+
+from telefuser.models.wan22_video_vae import Wan22VideoVAEStreamingDecodeState
+from telefuser.pipelines.abot_world.interactive import (
+ ABotWorldInteractivePipeline,
+ ABotWorldInteractiveSession,
+ ABotWorldSessionLifecycle,
+)
+
+
+def test_session_snapshot_round_trip_preserves_causal_and_rng_state() -> None:
+ source = ABotWorldInteractivePipeline(device="cpu", torch_dtype=torch.float32)
+ source.denoise_stage = SimpleNamespace(_scheduler=lambda: object())
+ generator = torch.Generator(device="cpu").manual_seed(123)
+ session = ABotWorldInteractiveSession(
+ session_id="migrating",
+ prompt_emb=torch.tensor([1.0]),
+ first_frame_latent=torch.tensor([2.0]),
+ self_cache=[
+ {
+ "k": torch.tensor([[[[3.0]]]]),
+ "v": torch.tensor([[[[4.0]]]]),
+ "global_end_index": torch.tensor([12]),
+ "local_end_index": torch.tensor([6]),
+ }
+ ],
+ cross_cache=[
+ {
+ "k": torch.tensor([[[[5.0]]]]),
+ "v": torch.tensor([[[[6.0]]]]),
+ "is_init": True,
+ "sequence_length": 1,
+ }
+ ],
+ scheduler=object(),
+ generator=generator,
+ vae_decode_state=Wan22VideoVAEStreamingDecodeState(feat_cache=[torch.tensor([7.0])]),
+ next_latent_frame=12,
+ emitted_frames=45,
+ ownership_epoch=4,
+ )
+ source._interactive_sessions[session.session_id] = session
+ generator_state = generator.get_state()
+ expected_next_random = torch.randn(1, generator=generator)
+ generator.set_state(generator_state)
+
+ snapshot = source.snapshot_interactive_session(session)
+ source.close_interactive_session(session)
+ target = ABotWorldInteractivePipeline(device="cpu", torch_dtype=torch.float32)
+ target.denoise_stage = SimpleNamespace(_scheduler=lambda: object())
+ restored = target.restore_interactive_snapshot(snapshot, owner_worker_id="gpu-1")
+
+ assert restored.lifecycle == ABotWorldSessionLifecycle.READY
+ assert restored.owner_worker_id == "gpu-1"
+ assert restored.ownership_epoch == 5
+ assert restored.next_latent_frame == 12
+ assert restored.emitted_frames == 45
+ assert restored.self_cache[0]["k"].item() == 3
+ assert restored.cross_cache[0]["v"].item() == 6
+ assert restored.vae_decode_state.feat_cache[0].item() == 7
+ assert torch.equal(torch.randn(1, generator=restored.generator), expected_next_random)
diff --git a/tests/unit/service/livekit/test_cli.py b/tests/unit/service/livekit/test_cli.py
index 1495e76c..b9031877 100644
--- a/tests/unit/service/livekit/test_cli.py
+++ b/tests/unit/service/livekit/test_cli.py
@@ -35,6 +35,13 @@ def fake_run_stream_server(**kwargs):
"12.5",
"--queue-size",
"3",
+ "--enable-autoscaling",
+ "--autoscaling-min-workers",
+ "1",
+ "--autoscaling-target-utilization",
+ "0.7",
+ "--autoscaling-cooldown-seconds",
+ "10",
],
)
@@ -48,4 +55,8 @@ def fake_run_stream_server(**kwargs):
assert captured["max_sessions_per_worker"] == 4
assert captured["control_idle_timeout"] == 12.5
assert captured["queue_size"] == 3
+ assert captured["autoscaling_enabled"] is True
+ assert captured["autoscaling_min_workers"] == 1
+ assert captured["autoscaling_target_utilization"] == 0.7
+ assert captured["autoscaling_cooldown_seconds"] == 10
assert captured["skip_validation"] is True
diff --git a/tests/unit/service/livekit/test_multi_session_worker.py b/tests/unit/service/livekit/test_multi_session_worker.py
index c70f0718..bb4b50d9 100644
--- a/tests/unit/service/livekit/test_multi_session_worker.py
+++ b/tests/unit/service/livekit/test_multi_session_worker.py
@@ -28,8 +28,15 @@ def __init__(self) -> None:
self.queues: dict[str, asyncio.Queue[dict | None]] = {}
self.capacity_profile: dict[str, object] | None = None
- def start(self, pipeline_file: str, *, skip_validation: bool = False, gpu_num: int = 1) -> None:
- del pipeline_file, skip_validation, gpu_num
+ def start(
+ self,
+ pipeline_file: str,
+ *,
+ skip_validation: bool = False,
+ gpu_num: int = 1,
+ gpu_ids: list[str] | None = None,
+ ) -> None:
+ del pipeline_file, skip_validation, gpu_num, gpu_ids
self.start_calls += 1
async def aclose(self) -> None:
diff --git a/tests/unit/service/livekit/test_nccl_transfer.py b/tests/unit/service/livekit/test_nccl_transfer.py
new file mode 100644
index 00000000..19ad4aed
--- /dev/null
+++ b/tests/unit/service/livekit/test_nccl_transfer.py
@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import torch
+import pytest
+
+from telefuser.service.livekit.nccl_transfer import flatten_tensor_tree, rebuild_tensor_tree
+from telefuser.service.livekit.config import LiveKitServeConfig
+
+
+def test_tensor_manifest_round_trip_preserves_nested_structure() -> None:
+ source = {
+ "cache": [{"k": torch.ones((1, 2)), "cursor": 3}],
+ "latent": torch.zeros((1, 3)),
+ "flags": (True, None),
+ }
+ skeleton, manifest, leaves = flatten_tensor_tree(source)
+
+ restored = rebuild_tensor_tree(skeleton, leaves)
+
+ assert len(manifest) == 2
+ assert restored["cache"][0]["cursor"] == 3
+ assert restored["flags"] == (True, None)
+ assert torch.equal(restored["cache"][0]["k"], source["cache"][0]["k"])
+
+
+def test_process_nccl_requires_fixed_two_gpu_group() -> None:
+ config = LiveKitServeConfig(
+ worker_mode="process-nccl",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ )
+
+ assert config.worker_mode == "process-nccl"
+
+ autoscaling = LiveKitServeConfig(
+ worker_mode="process-nccl",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ autoscaling_enabled=True,
+ queue_size=1,
+ )
+ assert autoscaling.autoscaling_enabled is True
diff --git a/tests/unit/service/livekit/test_pipeline_router.py b/tests/unit/service/livekit/test_pipeline_router.py
new file mode 100644
index 00000000..ceb05808
--- /dev/null
+++ b/tests/unit/service/livekit/test_pipeline_router.py
@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from telefuser.service.core.stream_pipeline_service import STREAM_MODE_BIDIRECTIONAL
+from telefuser.service.livekit.pipeline_router import TurboServePipelineRouter
+
+
+class _MigratableService:
+ def __init__(self, *, fail_import: bool = False) -> None:
+ self.fail_import = fail_import
+ self.sessions: set[str] = set()
+ self.queues: dict[str, asyncio.Queue[dict | None]] = {}
+ self.prepared: set[str] = set()
+ self.aborted: set[str] = set()
+ self.on_import = None
+
+ def create(self, session_id: str) -> str:
+ self.sessions.add(session_id)
+ self.queues[session_id] = asyncio.Queue()
+ return session_id
+
+ def prepare_migration(self, session_id: str) -> dict[str, str]:
+ self.prepared.add(session_id)
+ return {"session_id": session_id}
+
+ def import_migration(self, bundle: dict[str, str], **_: object) -> str:
+ if self.fail_import:
+ raise RuntimeError("target import failed")
+ session_id = self.create(bundle["session_id"])
+ if self.on_import is not None:
+ self.on_import()
+ return session_id
+
+ def commit_migration(self, session_id: str) -> None:
+ self.close_session(session_id)
+
+ def abort_migration(self, session_id: str) -> None:
+ self.prepared.discard(session_id)
+ self.aborted.add(session_id)
+
+ def close_session(self, session_id: str) -> None:
+ self.sessions.discard(session_id)
+ queue = self.queues.get(session_id)
+ if queue is not None:
+ queue.put_nowait(None)
+
+
+class _Backend:
+ stream_mode = STREAM_MODE_BIDIRECTIONAL
+
+ def __init__(self, *, fail_import: bool = False) -> None:
+ self.service = _MigratableService(fail_import=fail_import)
+ self.stream_service = SimpleNamespace(service=self.service)
+ self.started: list[dict[str, object]] = []
+ self.pushed: list[tuple[str, dict]] = []
+
+ def start(self, pipeline_file: str, **kwargs: object) -> None:
+ self.started.append({"pipeline_file": pipeline_file, **kwargs})
+
+ async def aclose(self) -> None:
+ return None
+
+ def configure_session_capacity(self, max_sessions: int | None) -> dict[str, object]:
+ return {"effective_capacity": max_sessions or 1}
+
+ def create_session(self, config: dict) -> str:
+ return self.service.create(str(config["session_id"]))
+
+ def push_chunk(self, session_id: str, chunk: dict) -> None:
+ self.pushed.append((session_id, chunk))
+
+ async def pull_chunks(self, session_id: str):
+ queue = self.service.queues[session_id]
+ while True:
+ chunk = await queue.get()
+ if chunk is None:
+ return
+ yield chunk
+
+ async def stream_task(self, config: dict):
+ if False:
+ yield config
+
+ def close_session(self, session_id: str) -> None:
+ self.service.close_session(session_id)
+
+
+def test_router_switches_pull_and_push_without_recreating_transport() -> None:
+ async def _run() -> None:
+ source = _Backend()
+ target = _Backend()
+ router = TurboServePipelineRouter({"worker-0": source, "worker-1": target})
+ view = router.worker_view("worker-0", gpu_ids=["0"])
+ view.start("pipeline.py", skip_validation=True, gpu_num=1)
+ session_id = view.create_session({"session_id": "session-a"})
+ chunks = router.pull_chunks(session_id)
+
+ source.service.queues[session_id].put_nowait({"index": 0})
+ assert await anext(chunks) == {"index": 0}
+ target.service.on_import = lambda: router.push_chunk(session_id, {"type": "during_migration"})
+ ownership = router.migrate_session(session_id, "worker-1")
+ target.service.queues[session_id].put_nowait({"index": 1})
+ router.push_chunk(session_id, {"type": "control_state"})
+
+ assert await anext(chunks) == {"index": 1}
+ assert ownership.worker_id == "worker-1"
+ assert ownership.epoch == 2
+ assert target.pushed == [
+ (session_id, {"type": "during_migration"}),
+ (session_id, {"type": "control_state"}),
+ ]
+ assert router.snapshot()["routes"] == {session_id: "worker-1"}
+ assert source.started[0]["gpu_ids"] == ["0"]
+
+ router.close_session(session_id)
+ with pytest.raises(StopAsyncIteration):
+ await anext(chunks)
+
+ asyncio.run(_run())
+
+
+def test_router_aborts_source_when_target_import_fails() -> None:
+ source = _Backend()
+ target = _Backend(fail_import=True)
+ router = TurboServePipelineRouter({"worker-0": source, "worker-1": target})
+ session_id = router.create_session("worker-0", {"session_id": "session-a"})
+
+ with pytest.raises(RuntimeError, match="target import failed"):
+ router.migrate_session(session_id, "worker-1")
+
+ assert router.snapshot()["routes"] == {session_id: "worker-0"}
+ assert source.service.aborted == {session_id}
diff --git a/tests/unit/service/livekit/test_process_worker_failures.py b/tests/unit/service/livekit/test_process_worker_failures.py
new file mode 100644
index 00000000..dc1c1513
--- /dev/null
+++ b/tests/unit/service/livekit/test_process_worker_failures.py
@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import asyncio
+import multiprocessing
+from typing import Any
+
+import pytest
+
+from telefuser.service.livekit.config import LiveKitServeConfig
+from telefuser.service.livekit.process_worker_pool import ProcessLiveKitWorkerPool, ProcessWorkerSpec
+from telefuser.service.livekit.runtime import LiveKitServeRuntime
+from telefuser.service.livekit.session_registry import SessionRecord
+
+
+def _exit_during_start(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
+ del spec, config_values, pipeline_file, skip_validation, security_name, commands, events
+ raise SystemExit(9)
+
+
+def _exit_during_stop(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
+ del config_values, pipeline_file, skip_validation, security_name
+ events.put({"type": "worker_ready", "worker_id": spec.worker_id})
+ while True:
+ command = commands.get()
+ if command["type"] == "stop_session":
+ raise SystemExit(11)
+
+
+def _exit_before_shutdown_ack(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
+ del config_values, pipeline_file, skip_validation, security_name
+ events.put({"type": "worker_ready", "worker_id": spec.worker_id})
+ while commands.get()["type"] != "shutdown":
+ pass
+
+
+class _Sink:
+ def __init__(self) -> None:
+ self.worker_statuses: list[tuple[str, str]] = []
+
+ def on_worker_status(self, worker_id: str, status: str) -> None:
+ self.worker_statuses.append((worker_id, status))
+
+ def on_worker_capacity(self, worker_id: str, capacity: int, profile: dict[str, object] | None = None) -> None:
+ return None
+
+ def on_session_status(self, session_id: str, status: str, error: str | None = None) -> None:
+ return None
+
+ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None:
+ return None
+
+ def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
+ return None
+
+
+def _config(**updates: object) -> LiveKitServeConfig:
+ values: dict[str, object] = {
+ "livekit_url": "wss://livekit.example",
+ "livekit_api_key": "key",
+ "livekit_api_secret": "secret",
+ "worker_mode": "process",
+ **updates,
+ }
+ return LiveKitServeConfig(**values)
+
+
+def test_process_worker_start_fails_immediately_when_child_exits() -> None:
+ async def _run() -> None:
+ pool = ProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"])],
+ config=_config(),
+ pipeline_file="pipeline.py",
+ event_sink=_Sink(),
+ context=multiprocessing.get_context("spawn"),
+ worker_target=_exit_during_start,
+ )
+
+ with pytest.raises(RuntimeError, match="exited during startup with code 9"):
+ await asyncio.wait_for(pool.start(), timeout=10)
+
+ asyncio.run(_run())
+
+
+def test_process_runtime_requires_explicit_gpu_map_for_multiple_workers() -> None:
+ async def _run() -> None:
+ runtime = LiveKitServeRuntime(
+ config=_config(num_workers=2),
+ pipeline_file="pipeline.py",
+ )
+ with pytest.raises(ValueError, match="worker_gpu_map is required"):
+ await runtime.start()
+ await runtime.aclose()
+
+ asyncio.run(_run())
+
+
+def test_pending_command_fails_when_child_exits() -> None:
+ async def _run() -> None:
+ pool = ProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"])],
+ config=_config(),
+ pipeline_file="pipeline.py",
+ event_sink=_Sink(),
+ context=multiprocessing.get_context("spawn"),
+ worker_target=_exit_during_stop,
+ )
+ await pool.start()
+ pool.start_session(
+ SessionRecord(
+ session_id="session-1",
+ room_name="room-1",
+ controller_identity="controller-1",
+ status="assigned",
+ worker_id="worker-0",
+ config={},
+ created_at=0,
+ updated_at=0,
+ )
+ )
+
+ with pytest.raises(RuntimeError, match="exited unexpectedly with code 11"):
+ await asyncio.wait_for(pool.stop_session("session-1"), timeout=3)
+ await pool.aclose()
+
+ asyncio.run(_run())
+
+
+def test_clean_child_exit_completes_shutdown_without_ack() -> None:
+ async def _run() -> None:
+ sink = _Sink()
+ pool = ProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"])],
+ config=_config(),
+ pipeline_file="pipeline.py",
+ event_sink=sink,
+ context=multiprocessing.get_context("spawn"),
+ worker_target=_exit_before_shutdown_ack,
+ )
+ await pool.start()
+
+ await asyncio.wait_for(pool.aclose(), timeout=3)
+
+ assert ("worker-0", "failed") not in sink.worker_statuses
+
+ asyncio.run(_run())
diff --git a/tests/unit/service/livekit/test_process_worker_pool.py b/tests/unit/service/livekit/test_process_worker_pool.py
new file mode 100644
index 00000000..f141ff3c
--- /dev/null
+++ b/tests/unit/service/livekit/test_process_worker_pool.py
@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import asyncio
+import multiprocessing
+import time
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+from telefuser.service.livekit.config import LiveKitServeConfig
+from telefuser.service.livekit.process_worker_pool import ProcessLiveKitWorkerPool, ProcessWorkerSpec
+from telefuser.service.livekit.runtime import LiveKitServeRuntime
+from telefuser.service.livekit.session_registry import SessionRecord
+
+
+def _fake_process_worker(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
+ del config_values, pipeline_file, skip_validation, security_name
+ events.put(
+ {
+ "type": "worker_capacity",
+ "worker_id": spec.worker_id,
+ "capacity": 2,
+ "profile": {"effective_capacity": 2},
+ }
+ )
+ events.put({"type": "worker_status", "worker_id": spec.worker_id, "status": "idle"})
+ events.put({"type": "worker_ready", "worker_id": spec.worker_id})
+ pipeline_sessions: dict[str, str] = {}
+ while True:
+ command = commands.get()
+ command_type = command["type"]
+ request_id = command.get("request_id")
+ if command_type == "start_session":
+ record = command["record"]
+ session_id = record["session_id"]
+ pipeline_session_id = f"pipeline-{session_id}"
+ pipeline_sessions[session_id] = pipeline_session_id
+ events.put(
+ {
+ "type": "pipeline_session",
+ "worker_id": spec.worker_id,
+ "session_id": session_id,
+ "pipeline_session_id": pipeline_session_id,
+ }
+ )
+ events.put(
+ {
+ "type": "session_status",
+ "worker_id": spec.worker_id,
+ "session_id": session_id,
+ "status": "running",
+ "error": None,
+ }
+ )
+ continue
+ if command_type == "stop_session":
+ session_id = command["session_id"]
+ events.put(
+ {
+ "type": "session_finished",
+ "worker_id": spec.worker_id,
+ "session_id": session_id,
+ "pipeline_session_id": pipeline_sessions.pop(session_id, None),
+ "error": None,
+ }
+ )
+ if request_id is not None:
+ events.put(
+ {
+ "type": "command_result",
+ "worker_id": spec.worker_id,
+ "request_id": request_id,
+ }
+ )
+ if command_type == "shutdown":
+ return
+
+
+class _RecordingSink:
+ def __init__(self) -> None:
+ self.worker_statuses: list[tuple[str, str]] = []
+ self.capacities: list[tuple[str, int, dict[str, object] | None]] = []
+ self.session_statuses: list[tuple[str, str, str | None]] = []
+ self.pipeline_sessions: list[tuple[str, str]] = []
+ self.finished: list[tuple[str, str, str | None]] = []
+
+ def on_worker_status(self, worker_id: str, status: str) -> None:
+ self.worker_statuses.append((worker_id, status))
+
+ def on_worker_capacity(self, worker_id: str, capacity: int, profile: dict[str, object] | None = None) -> None:
+ self.capacities.append((worker_id, capacity, profile))
+
+ def on_session_status(self, session_id: str, status: str, error: str | None = None) -> None:
+ self.session_statuses.append((session_id, status, error))
+
+ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None:
+ self.pipeline_sessions.append((session_id, pipeline_session_id))
+
+ def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
+ self.finished.append((worker_id, session_id, error))
+
+
+def _record() -> SessionRecord:
+ return SessionRecord(
+ session_id="session-1",
+ room_name="room-1",
+ controller_identity="controller-1",
+ status="assigned",
+ worker_id="worker-0",
+ config={},
+ created_at=0,
+ updated_at=0,
+ )
+
+
+async def _wait_until(predicate: Any, timeout: float = 5.0) -> None:
+ deadline = time.monotonic() + timeout
+ while not predicate():
+ if time.monotonic() >= deadline:
+ raise TimeoutError("Timed out waiting for process-worker event")
+ await asyncio.sleep(0.01)
+
+
+def test_process_worker_pool_forwards_lifecycle_over_spawn_ipc() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ worker_mode="process",
+ )
+ sink = _RecordingSink()
+ pool = ProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"])],
+ config=config,
+ pipeline_file="pipeline.py",
+ event_sink=sink,
+ context=multiprocessing.get_context("spawn"),
+ worker_target=_fake_process_worker,
+ )
+ await pool.start(skip_validation=True)
+ assert pool.active_worker_count() == 1
+
+ pool.start_session(_record())
+ assert sink.capacities == [("worker-0", 2, {"effective_capacity": 2})]
+ await _wait_until(lambda: bool(sink.pipeline_sessions))
+ assert sink.pipeline_sessions == [("session-1", "pipeline-session-1")]
+ assert pool.turboserve_snapshot()["retained_sessions_by_worker"] == {"worker-0": 1}
+
+ await pool.stop_session("session-1")
+ await _wait_until(lambda: bool(sink.finished))
+ assert sink.finished == [("worker-0", "session-1", None)]
+ assert pool.turboserve_snapshot()["retained_sessions_by_worker"] == {"worker-0": 0}
+ await pool.aclose()
+
+ asyncio.run(_run())
+
+
+def test_process_worker_pool_scales_spawned_replicas() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ worker_mode="process",
+ )
+ sink = _RecordingSink()
+ pool = ProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"]), ProcessWorkerSpec("worker-1", ["1"])],
+ config=config,
+ pipeline_file="pipeline.py",
+ event_sink=sink,
+ initial_workers=1,
+ context=multiprocessing.get_context("spawn"),
+ worker_target=_fake_process_worker,
+ )
+ await pool.start(skip_validation=True)
+ assert pool.active_worker_count() == 1
+
+ assert await pool.scale_to(2) == 2
+ assert await pool.scale_to(1) == 1
+ assert pool.turboserve_snapshot()["active_workers"] == ["worker-0"]
+ await pool.aclose()
+
+ asyncio.run(_run())
+
+
+def test_process_worker_pool_rolls_back_route_when_session_command_fails() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ worker_mode="process",
+ )
+ pool = ProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"])],
+ config=config,
+ pipeline_file="pipeline.py",
+ event_sink=_RecordingSink(),
+ context=multiprocessing.get_context("spawn"),
+ worker_target=_fake_process_worker,
+ )
+ await pool.start(skip_validation=True)
+
+ with (
+ patch.object(pool, "_send", side_effect=RuntimeError("command send failed")),
+ pytest.raises(RuntimeError, match="command send failed"),
+ ):
+ pool.start_session(_record())
+
+ assert pool.turboserve_snapshot()["retained_sessions_by_worker"] == {"worker-0": 0}
+ await pool.aclose()
+
+ asyncio.run(_run())
+
+
+def test_runtime_selects_process_worker_pool() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ worker_mode="process",
+ )
+ runtime = LiveKitServeRuntime(config=config, pipeline_file="pipeline.py")
+
+ assert isinstance(runtime.worker_pool, ProcessLiveKitWorkerPool)
+ assert runtime.worker_pool.turboserve_snapshot()["configured_workers"] == 2
diff --git a/tests/unit/service/livekit/test_runtime.py b/tests/unit/service/livekit/test_runtime.py
index 300f0808..3ddd2a65 100644
--- a/tests/unit/service/livekit/test_runtime.py
+++ b/tests/unit/service/livekit/test_runtime.py
@@ -5,6 +5,7 @@
from telefuser.service.livekit.config import LiveKitServeConfig
from telefuser.service.livekit.runtime import LiveKitServeRuntime
from telefuser.service.livekit.schemas import SessionCreateRequest
+from telefuser.service.livekit.turboserve import TurboServeOwnership
class FakeTokenService:
@@ -19,6 +20,9 @@ def __init__(self) -> None:
self.closed = False
self.start_options: list[bool] = []
self.close_calls = 0
+ self.active_workers = 1
+ self.scale_targets: list[int] = []
+ self.on_scale = None
async def start(self, *, skip_validation: bool = False) -> None:
self.start_options.append(skip_validation)
@@ -29,11 +33,93 @@ def start_session(self, record) -> None:
async def stop_session(self, session_id: str) -> None:
self.stopped.append(session_id)
+ def active_worker_count(self) -> int:
+ return self.active_workers
+
+ async def scale_to(self, target_workers: int) -> int:
+ self.scale_targets.append(target_workers)
+ self.active_workers = target_workers
+ if self.on_scale is not None:
+ self.on_scale()
+ return target_workers
+
async def aclose(self) -> None:
self.closed = True
self.close_calls += 1
+class MigratingWorkerPool(FakeWorkerPool):
+ def __init__(self) -> None:
+ super().__init__()
+ self.migrations: list[tuple[str, str]] = []
+
+ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
+ self.migrations.append((pipeline_session_id, target_worker_id))
+ return TurboServeOwnership(pipeline_session_id, target_worker_id, len(self.migrations) + 1)
+
+ def turboserve_snapshot(self) -> dict[str, object]:
+ return {
+ "migration_supported": True,
+ "worker_runtime_metrics": {
+ "worker-0": {"active_sessions": 2, "mean_chunk_seconds": 2.0, "p95_chunk_seconds": 4.0},
+ "worker-1": {"active_sessions": 0, "mean_chunk_seconds": 0.5, "p95_chunk_seconds": 1.0},
+ },
+ }
+
+
+def test_runtime_autoscaling_scales_out_and_drains_capacity() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ queue_size=2,
+ autoscaling_enabled=True,
+ autoscaling_min_workers=1,
+ autoscaling_cooldown_seconds=0,
+ )
+ pool = FakeWorkerPool()
+ runtime = LiveKitServeRuntime(
+ config=config, pipeline_file="pipeline.py", token_service=FakeTokenService(), worker_pool=pool
+ )
+ runtime.scheduler.update_worker_status("worker-1", "stopped")
+ pool.on_scale = lambda: runtime.scheduler.update_worker_status("worker-1", "idle")
+ first = runtime.create_session(SessionCreateRequest(identity="controller-1"))
+ second = runtime.create_session(SessionCreateRequest(identity="controller-2"))
+ assert first.admission.status == "assigned"
+ assert second.admission.status == "queued"
+
+ decision = await runtime._autoscale_once()
+
+ assert decision.target_workers == 2
+ assert pool.scale_targets == [2]
+ assert runtime.registry.require(second.record.session_id).status == "assigned"
+
+ asyncio.run(_run())
+
+
+def test_runtime_allows_multiple_in_process_gpu_workers() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ )
+ pool = FakeWorkerPool()
+ runtime = LiveKitServeRuntime(
+ config=config, pipeline_file="pipeline.py", token_service=FakeTokenService(), worker_pool=pool
+ )
+ await runtime.start()
+ assert runtime.is_ready is True
+ await runtime.aclose()
+
+ asyncio.run(_run())
+
+
def test_runtime_starts_queued_session_when_worker_is_released() -> None:
async def _run() -> None:
config = LiveKitServeConfig(
@@ -87,6 +173,68 @@ def test_runtime_worker_callbacks_release_capacity() -> None:
assert runtime.scheduler.health_snapshot()["workers_idle"] == 1
+def test_runtime_migration_updates_registry_and_admission_owner() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ max_sessions_per_worker=2,
+ )
+ pool = MigratingWorkerPool()
+ runtime = LiveKitServeRuntime(
+ config=config, pipeline_file="pipeline.py", token_service=FakeTokenService(), worker_pool=pool
+ )
+ created = runtime.create_session(SessionCreateRequest(identity="controller-1"))
+ runtime.on_pipeline_session(created.record.session_id, "pipeline-1")
+
+ ownership = await runtime.migrate_session(created.record.session_id, "worker-1")
+
+ assert ownership.worker_id == "worker-1"
+ assert pool.migrations == [("pipeline-1", "worker-1")]
+ assert runtime.registry.require(created.record.session_id).worker_id == "worker-1"
+ workers = {worker.worker_id: worker for worker in runtime.scheduler.workers()}
+ assert workers["worker-0"].session_ids == []
+ assert workers["worker-1"].session_ids == [created.record.session_id]
+
+ asyncio.run(_run())
+
+
+def test_runtime_rebalances_one_profitable_migration_from_measured_load() -> None:
+ async def _run() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ max_sessions_per_worker=3,
+ turboserve_migration_bandwidth_gbps=1000,
+ )
+ pool = MigratingWorkerPool()
+ runtime = LiveKitServeRuntime(
+ config=config, pipeline_file="pipeline.py", token_service=FakeTokenService(), worker_pool=pool
+ )
+ first = runtime.create_session(SessionCreateRequest(identity="controller-1"))
+ second = runtime.create_session(SessionCreateRequest(identity="controller-2"))
+ runtime.on_pipeline_session(first.record.session_id, "pipeline-1")
+ runtime.on_pipeline_session(second.record.session_id, "pipeline-2")
+ runtime.scheduler.reassign_session(second.record.session_id, "worker-0")
+ runtime.registry.assign_worker(second.record.session_id, "worker-0")
+ runtime.on_worker_capacity("worker-0", 3, {"estimated_session_bytes": 1})
+ runtime.on_worker_capacity("worker-1", 3, {"estimated_session_bytes": 1})
+
+ await runtime._rebalance_once()
+
+ assert pool.migrations == [("pipeline-1", "worker-1")]
+ assert runtime.registry.require(first.record.session_id).worker_id == "worker-1"
+ assert runtime.metadata()["turboserve_rebalance"]["last_plan"]["source_worker_id"] == "worker-0"
+
+ asyncio.run(_run())
+
+
def test_runtime_reports_livekit_connected_only_after_room_connection() -> None:
config = LiveKitServeConfig(livekit_url="wss://livekit.example", livekit_api_key="key", livekit_api_secret="secret")
runtime = LiveKitServeRuntime(
@@ -121,6 +269,53 @@ def test_runtime_exposes_worker_calculated_capacity() -> None:
assert metadata["session_capacity"] == {"worker-0": {"effective_capacity": 3, "limiting_device": 1}}
+def test_runtime_uses_only_reported_capacities_for_autoscaling() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ autoscaling_enabled=True,
+ queue_size=1,
+ autoscaling_min_workers=1,
+ )
+ runtime = LiveKitServeRuntime(
+ config=config,
+ pipeline_file="pipeline.py",
+ token_service=FakeTokenService(),
+ worker_pool=FakeWorkerPool(),
+ )
+ runtime.scheduler.update_worker_status("worker-1", "stopped")
+
+ runtime.on_worker_capacity("worker-0", 3, {"effective_capacity": 3})
+
+ assert runtime._autoscaling_controller.sessions_per_worker == 3
+ assert runtime.metadata()["max_sessions_per_worker"] == 3
+
+
+def test_runtime_is_unhealthy_when_all_configured_workers_are_failed_or_stopped() -> None:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=2,
+ )
+ runtime = LiveKitServeRuntime(
+ config=config,
+ pipeline_file="pipeline.py",
+ token_service=FakeTokenService(),
+ worker_pool=FakeWorkerPool(),
+ )
+ runtime.scheduler.update_worker_status("worker-0", "failed")
+ runtime.scheduler.update_worker_status("worker-1", "stopped")
+
+ health = runtime.health()
+
+ assert health.status == "unhealthy"
+ assert health.workers_idle == 0
+
+
def test_runtime_start_and_close_are_idempotent() -> None:
async def _run() -> None:
config = LiveKitServeConfig(
diff --git a/tests/unit/service/livekit/test_scheduler.py b/tests/unit/service/livekit/test_scheduler.py
index b988cb12..216b988d 100644
--- a/tests/unit/service/livekit/test_scheduler.py
+++ b/tests/unit/service/livekit/test_scheduler.py
@@ -15,6 +15,15 @@ def test_scheduler_assigns_first_idle_worker() -> None:
assert worker.gpu_ids == ["0"]
+def test_scheduler_balances_by_normalized_retained_load() -> None:
+ scheduler = LiveKitScheduler(num_workers=2, max_sessions_per_worker=2)
+ first = scheduler.assign(session_id="session-1", room_name="room-1")
+ second = scheduler.assign(session_id="session-2", room_name="room-2")
+
+ assert first.worker_id == "worker-0"
+ assert second.worker_id == "worker-1"
+
+
def test_scheduler_rejects_when_busy_and_queue_disabled() -> None:
scheduler = LiveKitScheduler(num_workers=1, queue_size=0)
scheduler.assign(session_id="session-1", room_name="room-1")
@@ -40,6 +49,18 @@ def test_scheduler_queues_and_assigns_on_release() -> None:
assert worker.session_id == "session-2"
+def test_scheduler_drains_queue_across_new_capacity() -> None:
+ scheduler = LiveKitScheduler(num_workers=2, queue_size=2)
+ scheduler.update_worker_status("worker-1", "stopped")
+ scheduler.assign(session_id="session-1", room_name="room-1")
+ assert scheduler.assign(session_id="session-2", room_name="room-2").status == "queued"
+ scheduler.update_worker_status("worker-1", "idle")
+
+ admissions = scheduler.drain_queue()
+
+ assert [(item.session_id, item.worker_id) for item in admissions] == [("session-2", "worker-1")]
+
+
def test_scheduler_health_counts_failed_workers() -> None:
scheduler = LiveKitScheduler(num_workers=2)
scheduler.assign(session_id="session-1", room_name="room-1")
@@ -54,6 +75,19 @@ def test_scheduler_health_counts_failed_workers() -> None:
}
+def test_scheduler_does_not_count_stopped_workers_as_idle() -> None:
+ scheduler = LiveKitScheduler(num_workers=2)
+ scheduler.update_worker_status("worker-1", "stopped")
+
+ assert scheduler.health_snapshot() == {
+ "workers_total": 2,
+ "workers_idle": 1,
+ "workers_busy": 0,
+ "workers_failed": 0,
+ "queued_sessions": 0,
+ }
+
+
def test_scheduler_updates_worker_capacity_before_admission() -> None:
scheduler = LiveKitScheduler(num_workers=1, max_sessions_per_worker=1)
@@ -61,3 +95,15 @@ def test_scheduler_updates_worker_capacity_before_admission() -> None:
assert worker.session_capacity == 3
assert scheduler.workers()[0].session_capacity == 3
+
+
+def test_scheduler_reassigns_an_admitted_session_after_migration() -> None:
+ scheduler = LiveKitScheduler(num_workers=2, max_sessions_per_worker=2)
+ scheduler.assign(session_id="session-1", room_name="room-1")
+
+ admission = scheduler.reassign_session("session-1", "worker-1")
+
+ assert admission.worker_id == "worker-1"
+ workers = {worker.worker_id: worker for worker in scheduler.workers()}
+ assert workers["worker-0"].session_ids == []
+ assert workers["worker-1"].session_ids == ["session-1"]
diff --git a/tests/unit/service/livekit/test_turboserve.py b/tests/unit/service/livekit/test_turboserve.py
new file mode 100644
index 00000000..be69e112
--- /dev/null
+++ b/tests/unit/service/livekit/test_turboserve.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import pytest
+
+from telefuser.service.livekit.turboserve import (
+ TurboServeAutoscalingController,
+ TurboServeOwnershipTable,
+ TurboServePlacementController,
+ TurboServeSessionDemand,
+ TurboServeWorkerLoad,
+ TurboServeWorkloadDetector,
+)
+
+
+def test_workload_detector_reports_activity_volatility_and_chunk_latency() -> None:
+ detector = TurboServeWorkloadDetector(window_seconds=10.0, volatility_bins=5)
+ detector.record_arrival("a", now=1.0)
+ detector.record_active("a", now=2.0)
+ detector.record_chunk(0.2, now=2.0)
+ detector.record_chunk(0.4, now=3.0)
+ detector.record_idle("a", now=4.0)
+
+ snapshot = detector.snapshot(now=5.0)
+
+ assert snapshot.active_sessions == 0
+ assert snapshot.arrivals_per_second == pytest.approx(0.1)
+ assert snapshot.mean_chunk_seconds == pytest.approx(0.3)
+ assert snapshot.p95_chunk_seconds == pytest.approx(0.4)
+ assert snapshot.activation_volatility > 0
+
+
+def test_placement_retains_owner_and_rebalance_accounts_for_migration_cost() -> None:
+ controller = TurboServePlacementController(
+ migration_bandwidth_bytes_per_second=1_000_000_000,
+ migration_penalty=0.1,
+ )
+ workers = [
+ TurboServeWorkerLoad("gpu-0", 8, 4, 4, 4.0),
+ TurboServeWorkerLoad("gpu-1", 8, 1, 1, 1.0),
+ ]
+ retained = controller.place(
+ TurboServeSessionDemand("session-a", True, 100, owner_worker_id="gpu-0"),
+ workers,
+ )
+ assert retained.worker_id == "gpu-0"
+
+ plans = controller.plan_rebalance(
+ [TurboServeSessionDemand("session-a", True, 100, owner_worker_id="gpu-0")],
+ workers,
+ )
+ assert len(plans) == 1
+ assert plans[0].source_worker_id == "gpu-0"
+ assert plans[0].target_worker_id == "gpu-1"
+ assert plans[0].gain_seconds > 0
+
+
+def test_autoscaler_applies_capacity_hysteresis_and_cooldown() -> None:
+ controller = TurboServeAutoscalingController(
+ sessions_per_worker=4,
+ target_utilization=0.75,
+ hysteresis=0.05,
+ cooldown_seconds=10.0,
+ max_workers=8,
+ )
+ scale_out = controller.decide(8, 1, now=20.0)
+ assert scale_out.action == "scale_out"
+ assert scale_out.target_workers == 3
+ assert controller.decide(8, 1, now=21.0).reason == "cooldown"
+
+ scale_in = controller.decide(1, 4, now=31.0)
+ assert scale_in.action == "scale_in"
+ assert scale_in.target_workers == 1
+
+
+def test_ownership_migration_commit_is_atomic_and_epoch_guarded() -> None:
+ table = TurboServeOwnershipTable()
+ assert table.register("session-a", "gpu-0").epoch == 1
+ token = table.prepare_migration("session-a", "gpu-0", "gpu-1")
+
+ committed = table.commit_migration(token)
+
+ assert committed.worker_id == "gpu-1"
+ assert committed.epoch == 2
+ with pytest.raises(RuntimeError, match="stale"):
+ table.commit_migration(token)
diff --git a/tests/unit/service/livekit/test_worker_pool.py b/tests/unit/service/livekit/test_worker_pool.py
index 130a7272..124a7bea 100644
--- a/tests/unit/service/livekit/test_worker_pool.py
+++ b/tests/unit/service/livekit/test_worker_pool.py
@@ -81,3 +81,45 @@ async def _run() -> None:
assert worker.cancelled is True
asyncio.run(_run())
+
+
+class _ScaleSink:
+ def __init__(self) -> None:
+ self.statuses: list[tuple[str, str]] = []
+
+ def on_worker_status(self, worker_id: str, status: str) -> None:
+ self.statuses.append((worker_id, status))
+
+
+class _ScalableWorker(_CooperativeWorker):
+ def __init__(self, worker_id: str) -> None:
+ super().__init__(complete_on_stop=True)
+ self.worker_id = worker_id
+ self.event_sink = _ScaleSink()
+ self.start_calls = 0
+ self.stop_calls = 0
+
+ async def start(self, *, skip_validation: bool = False) -> None:
+ del skip_validation
+ self.start_calls += 1
+
+ async def stop(self) -> None:
+ self.stop_calls += 1
+
+
+def test_worker_pool_scales_configured_idle_replicas() -> None:
+ async def _run() -> None:
+ workers = {worker_id: _ScalableWorker(worker_id) for worker_id in ("worker-0", "worker-1")}
+ pool = InProcessLiveKitWorkerPool(workers, initial_workers=1)
+ await pool.start(skip_validation=True)
+
+ assert pool.active_worker_count() == 1
+ assert workers["worker-0"].start_calls == 1
+ assert workers["worker-1"].start_calls == 0
+
+ assert await pool.scale_to(2) == 2
+ assert workers["worker-1"].start_calls == 1
+ assert await pool.scale_to(1) == 1
+ assert workers["worker-1"].stop_calls == 1
+
+ asyncio.run(_run())
diff --git a/tools/validation/benchmark_abot_microbatch.py b/tools/validation/benchmark_abot_microbatch.py
new file mode 100644
index 00000000..5c01706f
--- /dev/null
+++ b/tools/validation/benchmark_abot_microbatch.py
@@ -0,0 +1,187 @@
+"""Measure synchronous ABot-World retained-session microbatch scaling on one GPU.
+
+This intentionally bypasses the service scheduler. For every requested batch
+size B it creates B independent retained sessions, warms them up, and then
+executes one ``generate_next_blocks`` call per sample. Each measured call
+generates a continuation chunk of ``4 * control_latent_frames`` video frames for every session.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import importlib.util
+import json
+import statistics
+import time
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+
+
+def _load_example_loader() -> Any:
+ path = Path(__file__).resolve().parents[2] / "examples/abot_world/_loader.py"
+ spec = importlib.util.spec_from_file_location("abot_microbatch_loader", path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot loader: {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _parse_batch_sizes(value: str) -> list[int]:
+ values = [int(item) for item in value.split(",") if item.strip()]
+ if not values or any(item < 1 for item in values):
+ raise argparse.ArgumentTypeError("batch sizes must be positive integers")
+ return values
+
+
+def _sample_stats(samples: list[float]) -> dict[str, float]:
+ ordered = sorted(samples)
+ return {
+ "mean": statistics.fmean(samples),
+ "stdev": statistics.stdev(samples) if len(samples) > 1 else 0.0,
+ "min": ordered[0],
+ "max": ordered[-1],
+ "p50": ordered[(len(ordered) - 1) // 2],
+ "p95": ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))],
+ }
+
+
+def _run_point(
+ pipeline: ABotWorldInteractivePipeline,
+ image: Image.Image,
+ args: argparse.Namespace,
+ batch_size: int,
+) -> dict[str, Any]:
+ sessions = []
+ device = torch.device(pipeline.device)
+ try:
+ for index in range(batch_size):
+ sessions.append(
+ pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=args.seed + index,
+ session_id=f"microbatch-b{batch_size}-s{index}",
+ )
+ )
+ controls = [{"W": True} for _ in sessions]
+
+ # Exclude the first generation from warmup and timing. It exercises a
+ # distinct first-frame path, but it still emits one configured chunk.
+ # Every chunk therefore emits exactly 4 pixel frames per requested
+ # continuation latent, per session.
+ initial_frames = pipeline.generate_next_blocks(
+ sessions, controls, control_latent_frames=args.control_latent_frames
+ )
+ expected_frames = 4 * args.control_latent_frames
+ if any(len(item) != expected_frames for item in initial_frames):
+ raise RuntimeError(
+ f"initial chunk did not emit {expected_frames} frames per session: "
+ f"{[len(item) for item in initial_frames]}"
+ )
+ for _ in range(args.warmup_chunks):
+ frames = pipeline.generate_next_blocks(
+ sessions, controls, control_latent_frames=args.control_latent_frames
+ )
+ expected_frames = 4 * args.control_latent_frames
+ if any(len(item) != expected_frames for item in frames):
+ raise RuntimeError(f"warmup did not emit {expected_frames} frames per session: {[len(item) for item in frames]}")
+
+ torch.cuda.synchronize(device)
+ torch.cuda.reset_peak_memory_stats(device)
+ samples: list[float] = []
+ denoise_samples: list[float] = []
+ vae_samples: list[float] = []
+ for _ in range(args.repeats):
+ torch.cuda.synchronize(device)
+ started_at = time.perf_counter()
+ frames = pipeline.generate_next_blocks(
+ sessions, controls, control_latent_frames=args.control_latent_frames
+ )
+ torch.cuda.synchronize(device)
+ elapsed = time.perf_counter() - started_at
+ expected_frames = 4 * args.control_latent_frames
+ if any(len(item) != expected_frames for item in frames):
+ raise RuntimeError(f"sample did not emit {expected_frames} frames per session: {[len(item) for item in frames]}")
+ samples.append(elapsed)
+ stage_metrics = pipeline.last_stage_metrics()
+ denoise_samples.append(float(stage_metrics.get("denoise_seconds", 0.0)))
+ vae_samples.append(float(stage_metrics.get("vae_decode_seconds", 0.0)))
+
+ timing = _sample_stats(samples)
+ chunk_time = timing["mean"]
+ return {
+ "status": "ok",
+ "batch": batch_size,
+ "warmup_chunks": args.warmup_chunks,
+ "repeats": args.repeats,
+ "control_latent_frames": args.control_latent_frames,
+ "frames_per_session_per_chunk": 4 * args.control_latent_frames,
+ "chunk_time_seconds": chunk_time,
+ "chunk_time_stats_seconds": timing,
+ "mean_denoise_seconds": statistics.fmean(denoise_samples),
+ "mean_vae_decode_seconds": statistics.fmean(vae_samples),
+ "mean_other_seconds": chunk_time - statistics.fmean(denoise_samples) - statistics.fmean(vae_samples),
+ "aggregate_fps": float(4 * args.control_latent_frames * batch_size) / chunk_time,
+ "fps_per_session": float(4 * args.control_latent_frames) / chunk_time,
+ "gpu_peak_memory_bytes": int(torch.cuda.max_memory_allocated(device)),
+ }
+ finally:
+ for session in sessions:
+ pipeline.close_interactive_session(session)
+ torch.cuda.empty_cache()
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--batch-sizes", type=_parse_batch_sizes, default=[1, 2, 3, 4])
+ parser.add_argument("--warmup-chunks", type=int, default=3)
+ parser.add_argument("--repeats", type=int, default=8)
+ parser.add_argument("--control-latent-frames", type=int, choices=(1, 2, 3), default=3)
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ args = parser.parse_args()
+ if args.warmup_chunks < 1 or args.repeats < 2:
+ parser.error("warmup-chunks must be at least 1 and repeats at least 2")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ loader = _load_example_loader()
+ pipeline = loader.get_pipeline(model_root=args.model_root, pipeline_class=ABotWorldInteractivePipeline)
+ image = Image.open(args.image).convert("RGB")
+ results: list[dict[str, Any]] = []
+ try:
+ for batch_size in args.batch_sizes:
+ print(f"running batch={batch_size}", flush=True)
+ try:
+ row = _run_point(pipeline, image, args, batch_size)
+ except torch.OutOfMemoryError as exc:
+ torch.cuda.empty_cache()
+ row = {"status": "oom", "batch": batch_size, "error": str(exc).splitlines()[0]}
+ results.append(row)
+ (args.output_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n")
+ finally:
+ pipeline.close()
+
+ fields = ["batch", "status", "chunk_time_seconds", "aggregate_fps", "fps_per_session", "gpu_peak_memory_bytes"]
+ with (args.output_dir / "results.csv").open("w", newline="") as handle:
+ writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
+ writer.writeheader()
+ writer.writerows(results)
+ print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/benchmark_abot_turboserve.py b/tools/validation/benchmark_abot_turboserve.py
new file mode 100644
index 00000000..7fc258ac
--- /dev/null
+++ b/tools/validation/benchmark_abot_turboserve.py
@@ -0,0 +1,124 @@
+"""Profile ABot TurboServe continuous batching with retained causal sessions."""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import statistics
+import time
+from pathlib import Path
+
+from PIL import Image
+
+from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+
+
+def _loader_module():
+ loader_path = Path(__file__).resolve().parents[2] / "examples/abot_world/_loader.py"
+ spec = importlib.util.spec_from_file_location("abot_turboserve_benchmark_loader", loader_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot example loader: {loader_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _percentile(values: list[float], quantile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ return ordered[max(0, min(len(ordered) - 1, int(len(ordered) * quantile + 0.999999) - 1))]
+
+
+def benchmark(args: argparse.Namespace) -> dict[str, object]:
+ loader = _loader_module()
+ pipeline = loader.get_pipeline(
+ model_root=args.model_root,
+ pipeline_class=ABotWorldInteractivePipeline,
+ )
+ pipeline.preload_models()
+ image = Image.open(args.image).convert("RGB")
+ sessions = [
+ pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=args.seed + index,
+ session_id=f"benchmark-{index}",
+ )
+ for index in range(args.sessions)
+ ]
+ chunk_latencies: list[float] = []
+ stage_samples: list[dict[str, float | int]] = []
+ total_frames = 0
+ started_at = time.monotonic()
+ try:
+ for _chunk_index in range(args.chunks):
+ for offset in range(0, len(sessions), args.batch_size):
+ batch = sessions[offset : offset + args.batch_size]
+ batch_started_at = time.monotonic()
+ outputs = pipeline.generate_next_blocks(
+ batch,
+ [{"W": True} for _ in batch],
+ control_latent_frames=args.control_latent_frames,
+ )
+ chunk_latencies.append(time.monotonic() - batch_started_at)
+ stage_samples.append(pipeline.last_stage_metrics())
+ total_frames += sum(len(frames) for frames in outputs)
+ elapsed = time.monotonic() - started_at
+ expected_latents = args.chunks * args.control_latent_frames
+ if any(session.next_latent_frame != expected_latents for session in sessions):
+ raise RuntimeError("One or more ABot sessions did not advance through every requested chunk")
+ return {
+ "sessions": args.sessions,
+ "chunks_per_session": args.chunks,
+ "configured_batch_size": args.batch_size,
+ "control_latent_frames": args.control_latent_frames,
+ "total_frames": total_frames,
+ "elapsed_seconds": elapsed,
+ "frames_per_second": total_frames / elapsed if elapsed else 0.0,
+ "batch_latency_seconds": {
+ "count": len(chunk_latencies),
+ "mean": statistics.fmean(chunk_latencies),
+ "p50": _percentile(chunk_latencies, 0.50),
+ "p95": _percentile(chunk_latencies, 0.95),
+ "p99": _percentile(chunk_latencies, 0.99),
+ "maximum": max(chunk_latencies),
+ },
+ "stage_samples": stage_samples,
+ }
+ finally:
+ for session in sessions:
+ pipeline.close_interactive_session(session)
+ pipeline.close()
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--sessions", type=int, default=2)
+ parser.add_argument("--chunks", type=int, default=30)
+ parser.add_argument("--batch-size", type=int, default=2)
+ parser.add_argument("--control-latent-frames", type=int, choices=(1, 2, 3), default=3)
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--output", type=Path)
+ args = parser.parse_args()
+ if args.sessions < 1 or args.chunks < 1 or not 1 <= args.batch_size <= args.sessions:
+ parser.error("sessions/chunks must be positive and batch-size in [1, sessions]")
+ return args
+
+
+def main() -> None:
+ args = parse_args()
+ result = benchmark(args)
+ rendered = json.dumps(result, indent=2, sort_keys=True)
+ print(rendered)
+ if args.output is not None:
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ args.output.write_text(rendered + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/benchmark_abot_turboserve_concurrent.py b/tools/validation/benchmark_abot_turboserve_concurrent.py
new file mode 100644
index 00000000..85489442
--- /dev/null
+++ b/tools/validation/benchmark_abot_turboserve_concurrent.py
@@ -0,0 +1,265 @@
+"""Drive the ABot continuous-batching service with bursty interactive clients.
+
+Unlike ``benchmark_abot_turboserve.py``, this exercises the same service
+scheduler used by the LiveKit adapter: sessions arrive over time, controls are
+updated independently, and each client consumes frames at its requested FPS.
+Run one process per GPU for multi-replica measurements.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import random
+import statistics
+import time
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from examples.abot_world._loader import DEFAULT_PROMPT, get_pipeline
+from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+
+
+def _percentile(values: list[float], quantile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ return ordered[max(0, min(len(ordered) - 1, int(len(ordered) * quantile + 0.999999) - 1))]
+
+
+async def _client(
+ service: ABotWorldLiveKitService,
+ *,
+ session_index: int,
+ args: argparse.Namespace,
+ started_at: float,
+) -> dict[str, Any]:
+ rng = random.Random(args.seed + session_index)
+ await asyncio.sleep(rng.uniform(0.0, args.arrival_window_seconds))
+ session_id = f"concurrent-{session_index}"
+ service.create_session(
+ {
+ "session_id": session_id,
+ "image_path": str(args.image),
+ "prompt": args.prompt,
+ "seed": args.seed + session_index,
+ "fps": args.fps,
+ "control_latent_frames": args.control_latent_frames,
+ "delivery_mode": args.delivery_mode,
+ }
+ )
+ created_at = time.monotonic()
+ produced_chunks = 0
+ produced_frames = 0
+ scheduler_waits: list[float] = []
+ compute_times: list[float] = []
+ batch_sizes: list[int] = []
+ first_chunk_at: float | None = None
+ received_at: list[float] = []
+ displayed_at: list[float] = []
+ first_frame_at: float | None = None
+ displayed_frames = 0
+ consumer_done = asyncio.Event()
+
+ async def consume() -> None:
+ nonlocal first_chunk_at, first_frame_at, produced_chunks, produced_frames, displayed_frames
+ async for payload in service.pull_chunks(session_id):
+ if payload.get("type") != "chunk":
+ continue
+ received = time.monotonic()
+ received_at.append(received)
+ first_chunk_at = received if first_chunk_at is None else first_chunk_at
+ frames = payload.get("frames", [])
+ produced_chunks += 1
+ produced_frames += len(frames)
+ scheduler = payload.get("scheduler", {})
+ scheduler_waits.append(float(scheduler.get("queue_wait_seconds", 0.0)))
+ compute_times.append(float(scheduler.get("compute_seconds", 0.0)))
+ batch_sizes.append(int(scheduler.get("batch_size", 1)))
+ # Count a frame only after the client has consumed and displayed it.
+ # This is the user-visible end-to-end measurement, not model output.
+ for _ in frames:
+ if args.consumer_playback_fps > 0:
+ await asyncio.sleep(1.0 / args.consumer_playback_fps)
+ displayed = time.monotonic()
+ displayed_at.append(displayed)
+ first_frame_at = displayed if first_frame_at is None else first_frame_at
+ displayed_frames += 1
+ consumer_done.set()
+
+ consumer = asyncio.create_task(consume(), name=f"abot-consumer-{session_id}")
+ deadline = started_at + args.duration_seconds
+ controls = ["W"]
+ try:
+ while time.monotonic() < deadline:
+ service.push_chunk(session_id, {"type": "control_state", "controls": controls})
+ await asyncio.sleep(rng.uniform(args.control_update_min_seconds, args.control_update_max_seconds))
+ # Brief releases are common in real keyboard input and create an
+ # independently changing active set for the scheduler.
+ if rng.random() < args.idle_probability:
+ service.push_chunk(session_id, {"type": "control_state", "controls": []})
+ await asyncio.sleep(rng.uniform(args.idle_min_seconds, args.idle_max_seconds))
+ controls = rng.choice((["W"], ["W", "A"], ["W", "D"], ["S"], ["I"], ["W", "J"]))
+ metrics = service.runtime_metrics(session_id)
+ finally:
+ service.close_session(session_id)
+ await asyncio.wait_for(consumer_done.wait(), timeout=args.close_timeout_seconds)
+ await consumer
+ consumer_completed_at = time.monotonic()
+ intervals = [later - earlier for earlier, later in zip(received_at, received_at[1:])]
+ return {
+ "session_id": session_id,
+ "arrival_offset_seconds": created_at - started_at,
+ "first_chunk_seconds": (first_chunk_at - created_at) if first_chunk_at is not None else None,
+ "received_chunks": produced_chunks,
+ "received_frames": produced_frames,
+ "consumer_displayed_frames": displayed_frames,
+ "consumer_end_to_end_seconds": consumer_completed_at - created_at,
+ "consumer_end_to_end_fps": (
+ displayed_frames / (consumer_completed_at - created_at)
+ if consumer_completed_at > created_at
+ else 0.0
+ ),
+ "consumer_first_frame_seconds": (first_frame_at - created_at) if first_frame_at is not None else None,
+ "scheduler_queue_wait_seconds": scheduler_waits,
+ "compute_seconds": compute_times,
+ "batch_sizes": batch_sizes,
+ "chunk_interarrival_seconds": intervals,
+ "service_metrics": metrics,
+ }
+
+
+async def _benchmark(args: argparse.Namespace) -> dict[str, Any]:
+ pipeline = get_pipeline(model_root=args.model_root, pipeline_class=ABotWorldInteractivePipeline)
+ service = ABotWorldLiveKitService(
+ pipeline,
+ default_fps=args.fps,
+ default_session_config={
+ "image_path": str(args.image),
+ "prompt": args.prompt,
+ "seed": args.seed,
+ "control_latent_frames": args.control_latent_frames,
+ },
+ max_batch_size=args.max_batch_size,
+ batching_window_ms=args.batching_window_ms,
+ scheduler_mode=args.scheduler_mode,
+ output_queue_size=args.output_queue_size,
+ idle_suspension_seconds=args.idle_suspension_seconds,
+ )
+ try:
+ service.start()
+ service.configure_session_capacity(args.sessions)
+ started_at = time.monotonic()
+ results = await asyncio.gather(
+ *(_client(service, session_index=index, args=args, started_at=started_at) for index in range(args.sessions))
+ )
+ elapsed = time.monotonic() - started_at
+ finally:
+ service.stop()
+ waits = [value for result in results for value in result["scheduler_queue_wait_seconds"]]
+ computes = [value for result in results for value in result["compute_seconds"]]
+ batches = [value for result in results for value in result["batch_sizes"]]
+ interarrivals = [value for result in results for value in result["chunk_interarrival_seconds"]]
+ first_chunks = [result["first_chunk_seconds"] for result in results if result["first_chunk_seconds"] is not None]
+ total_frames = sum(result["received_frames"] for result in results)
+ displayed_frames = sum(result["consumer_displayed_frames"] for result in results)
+ consumer_fps = [result["consumer_end_to_end_fps"] for result in results]
+ consumer_first_frames = [
+ result["consumer_first_frame_seconds"]
+ for result in results
+ if result["consumer_first_frame_seconds"] is not None
+ ]
+ return {
+ "scenario": {
+ "kind": "interactive_continuous_service",
+ "scheduler_mode": args.scheduler_mode,
+ "sessions": args.sessions,
+ "arrival_window_seconds": args.arrival_window_seconds,
+ "duration_seconds": args.duration_seconds,
+ "target_fps_per_session": args.fps,
+ "consumer_playback_fps": args.consumer_playback_fps,
+ "control_latent_frames": args.control_latent_frames,
+ "delivery_mode": args.delivery_mode,
+ "max_batch_size": args.max_batch_size,
+ "batching_window_ms": args.batching_window_ms,
+ },
+ "elapsed_seconds": elapsed,
+ "received_frames": total_frames,
+ "consumer_displayed_frames": displayed_frames,
+ "consumer_end_to_end_fps": _summary(consumer_fps),
+ "consumer_first_frame_seconds": _summary(consumer_first_frames),
+ "first_chunk_seconds": _summary(first_chunks),
+ "scheduler_queue_wait_seconds": _summary(waits),
+ "model_compute_seconds": _summary(computes),
+ "chunk_interarrival_seconds": _summary(interarrivals),
+ "observed_batch_sizes": dict(sorted(Counter(batches).items())),
+ "mean_observed_batch_size": statistics.fmean(batches) if batches else 0.0,
+ "sessions_detail": results,
+ }
+
+
+def _summary(values: list[float]) -> dict[str, float | int]:
+ return {
+ "count": len(values),
+ "mean": statistics.fmean(values) if values else 0.0,
+ "p50": _percentile(values, 0.50),
+ "p95": _percentile(values, 0.95),
+ "p99": _percentile(values, 0.99),
+ "maximum": max(values) if values else 0.0,
+ }
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--prompt", default=DEFAULT_PROMPT)
+ parser.add_argument("--sessions", type=int, default=4)
+ parser.add_argument("--duration-seconds", type=float, default=8.0)
+ parser.add_argument("--arrival-window-seconds", type=float, default=1.5)
+ parser.add_argument("--fps", type=float, default=8.0)
+ parser.add_argument(
+ "--consumer-playback-fps",
+ type=float,
+ default=8.0,
+ help="Client display cadence used for the end-to-end FPS metric; defaults to the 8-FPS target.",
+ )
+ parser.add_argument("--control-latent-frames", type=int, choices=(1, 2, 3), default=2)
+ parser.add_argument("--scheduler-mode", choices=("round_robin", "batched"), default="round_robin")
+ parser.add_argument("--max-batch-size", type=int, default=4)
+ parser.add_argument("--batching-window-ms", type=float, default=2.0)
+ parser.add_argument("--output-queue-size", type=int, default=4)
+ parser.add_argument("--delivery-mode", choices=("latest", "lossless"), default="latest")
+ parser.add_argument("--control-update-min-seconds", type=float, default=1.0)
+ parser.add_argument("--control-update-max-seconds", type=float, default=1.0)
+ parser.add_argument("--idle-probability", type=float, default=0.0)
+ parser.add_argument("--idle-min-seconds", type=float, default=0.05)
+ parser.add_argument("--idle-max-seconds", type=float, default=0.25)
+ parser.add_argument("--idle-suspension-seconds", type=float, default=5.0)
+ parser.add_argument("--close-timeout-seconds", type=float, default=30.0)
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--output", type=Path, required=True)
+ args = parser.parse_args()
+ if args.sessions < 1 or args.duration_seconds <= 0 or args.arrival_window_seconds < 0:
+ parser.error("sessions and duration must be positive; arrival window must be non-negative")
+ if args.consumer_playback_fps < 0:
+ parser.error("consumer playback FPS must be non-negative")
+ if args.control_update_min_seconds <= 0 or args.control_update_max_seconds < args.control_update_min_seconds:
+ parser.error("control update range must be positive and ordered")
+ if not 0 <= args.idle_probability <= 1 or args.idle_min_seconds < 0 or args.idle_max_seconds < args.idle_min_seconds:
+ parser.error("invalid idle burst configuration")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ result = asyncio.run(_benchmark(args))
+ args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
+ print(json.dumps({key: value for key, value in result.items() if key != "sessions_detail"}, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/run_abot_batch_scaling.py b/tools/validation/run_abot_batch_scaling.py
new file mode 100644
index 00000000..d1817c1a
--- /dev/null
+++ b/tools/validation/run_abot_batch_scaling.py
@@ -0,0 +1,264 @@
+"""Run a reproducible single-GPU ABot continuous-batching scaling sweep.
+
+This is the first motivation experiment for world-model serving. It uses the
+same ``ABotWorldLiveKitService`` scheduler as LiveKit, keeps all sessions
+continuously active, and measures the steady-state cost of different retained
+session counts and batch caps after a warm-up round.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import importlib.util
+import json
+import statistics
+import time
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+
+
+def _loader_module() -> Any:
+ loader_path = Path(__file__).resolve().parents[2] / "examples/abot_world/_loader.py"
+ spec = importlib.util.spec_from_file_location("abot_batch_scaling_loader", loader_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot loader: {loader_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _percentile(values: list[float], q: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ return ordered[max(0, min(len(ordered) - 1, int(len(ordered) * q + 0.999999) - 1))]
+
+
+def _mean(values: Iterable[float | int]) -> float:
+ materialized = [float(value) for value in values]
+ return statistics.fmean(materialized) if materialized else 0.0
+
+
+def _drain_initial_previews(service: ABotWorldLiveKitService, session_ids: list[str]) -> None:
+ for session_id in session_ids:
+ state = service._session(session_id) # Session service boundary, kept local for an offline benchmark.
+ if state is None:
+ raise KeyError(session_id)
+ preview = state.output_queue.get(timeout=30.0)
+ if preview.get("type") != "preview":
+ raise RuntimeError(f"Expected preview output for {session_id}, got {preview.get('type')!r}")
+
+
+def _run_point(
+ *,
+ pipeline: ABotWorldInteractivePipeline,
+ image: Image.Image,
+ args: argparse.Namespace,
+ sessions: int,
+ max_batch_size: int,
+) -> dict[str, Any]:
+ service = ABotWorldLiveKitService(
+ pipeline,
+ default_fps=args.fps,
+ default_session_config={"image_path": str(args.image), "prompt": args.prompt, "seed": args.seed},
+ max_batch_size=max_batch_size,
+ batching_window_ms=args.batching_window_ms,
+ output_queue_size=max(128, args.chunks_per_session + args.warmup_chunks + 8),
+ control_idle_timeout=args.control_idle_timeout_seconds,
+ idle_suspension_seconds=max(args.idle_suspension_seconds, args.duration_hint_seconds),
+ )
+ # The offline sweep explicitly controls admission and never calls the
+ # runtime capacity profiler; its warmup is measured separately below.
+ service._capacity_profile = {"effective_capacity": sessions} # noqa: SLF001
+ session_ids: list[str] = []
+ try:
+ service.start()
+ for index in range(sessions):
+ session_ids.append(
+ service.create_session(
+ {
+ "session_id": f"s{sessions}-b{max_batch_size}-{index}",
+ "image_path": str(args.image),
+ "prompt": args.prompt,
+ "seed": args.seed + index,
+ "fps": args.fps,
+ "control_latent_frames": args.control_latent_frames,
+ "delivery_mode": "lossless",
+ }
+ )
+ )
+ _drain_initial_previews(service, session_ids)
+ for session_id in session_ids:
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["W"]})
+
+ outputs: dict[str, list[dict[str, Any]]] = {session_id: [] for session_id in session_ids}
+ warmup_remaining = {session_id: args.warmup_chunks for session_id in session_ids}
+ measured_remaining = {session_id: args.chunks_per_session for session_id in session_ids}
+ measurement_started_at: float | None = None
+ while any(value > 0 for value in measured_remaining.values()):
+ made_progress = False
+ for session_id in session_ids:
+ if measured_remaining[session_id] <= 0 and warmup_remaining[session_id] <= 0:
+ continue
+ state = service._session(session_id) # noqa: SLF001
+ if state is None:
+ raise RuntimeError(f"Benchmark session disappeared: {session_id}")
+ payload = state.output_queue.get(timeout=args.chunk_timeout_seconds)
+ if payload.get("type") == "error":
+ error = str(payload.get("error", "ABot scheduler failed"))
+ if "out of memory" in error.lower():
+ raise torch.OutOfMemoryError(error)
+ raise RuntimeError(error)
+ if payload.get("type") != "chunk":
+ continue
+ made_progress = True
+ if warmup_remaining[session_id] > 0:
+ warmup_remaining[session_id] -= 1
+ if all(value == 0 for value in warmup_remaining.values()):
+ measurement_started_at = time.monotonic()
+ continue
+ outputs[session_id].append(payload)
+ measured_remaining[session_id] -= 1
+ if not made_progress:
+ raise RuntimeError("No ABot chunks were produced during benchmark")
+ ended_at = time.monotonic()
+ if measurement_started_at is None:
+ measurement_started_at = ended_at
+
+ samples = [payload for payloads in outputs.values() for payload in payloads]
+ scheduler = [payload.get("scheduler", {}) for payload in samples]
+ compute = [float(item.get("compute_seconds", 0.0)) for item in scheduler]
+ queue_wait = [float(item.get("queue_wait_seconds", 0.0)) for item in scheduler]
+ batch_sizes = [int(item.get("batch_size", 1)) for item in scheduler]
+ denoise = [float(item.get("denoise_seconds", 0.0)) for item in scheduler]
+ vae_decode = [float(item.get("vae_decode_seconds", 0.0)) for item in scheduler]
+ total_frames = sum(len(payload.get("frames", [])) for payload in samples)
+ elapsed = ended_at - measurement_started_at
+ per_session_frames = [sum(len(payload.get("frames", [])) for payload in outputs[session_id]) for session_id in session_ids]
+ return {
+ "sessions": sessions,
+ "max_batch_size": max_batch_size,
+ "control_latent_frames": args.control_latent_frames,
+ "chunks_per_session": args.chunks_per_session,
+ "warmup_chunks": args.warmup_chunks,
+ "elapsed_seconds": elapsed,
+ "total_frames": total_frames,
+ "aggregate_fps": total_frames / elapsed if elapsed else 0.0,
+ "per_session_fps": _mean(frame / elapsed for frame in per_session_frames) if elapsed else 0.0,
+ "mean_batch_size": _mean(batch_sizes),
+ "max_observed_batch_size": max(batch_sizes, default=0),
+ "p50_compute_seconds": _percentile(compute, 0.50),
+ "p95_compute_seconds": _percentile(compute, 0.95),
+ "p50_queue_wait_seconds": _percentile(queue_wait, 0.50),
+ "p95_queue_wait_seconds": _percentile(queue_wait, 0.95),
+ "p95_chunk_latency_seconds": _percentile([left + right for left, right in zip(queue_wait, compute)], 0.95),
+ "mean_denoise_seconds": _mean(denoise),
+ "mean_vae_decode_seconds": _mean(vae_decode),
+ "denoise_share": _mean(denoise) / _mean(compute) if _mean(compute) else 0.0,
+ "vae_decode_share": _mean(vae_decode) / _mean(compute) if _mean(compute) else 0.0,
+ "gpu_peak_memory_bytes": int(torch.cuda.max_memory_allocated(pipeline.device)),
+ "service_runtime_metrics": service.runtime_metrics(),
+ }
+ finally:
+ service.stop(close_pipeline=False)
+ torch.cuda.empty_cache()
+
+
+def _parse_ints(value: str) -> list[int]:
+ parsed = [int(item) for item in value.split(",") if item.strip()]
+ if not parsed or any(item < 1 for item in parsed):
+ raise argparse.ArgumentTypeError("expected a comma-separated list of positive integers")
+ return parsed
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--sessions", type=_parse_ints, default=[1, 2, 4])
+ parser.add_argument("--max-batch-sizes", type=_parse_ints, default=[1, 2, 4])
+ parser.add_argument("--chunks-per-session", type=int, default=4)
+ parser.add_argument("--warmup-chunks", type=int, default=1)
+ parser.add_argument("--control-latent-frames", choices=(1, 2, 3), type=int, default=2)
+ parser.add_argument("--fps", type=int, default=8)
+ parser.add_argument("--batching-window-ms", type=float, default=2.0)
+ parser.add_argument("--idle-suspension-seconds", type=float, default=600.0)
+ parser.add_argument("--control-idle-timeout-seconds", type=float, default=3600.0)
+ parser.add_argument("--duration-hint-seconds", type=float, default=600.0)
+ parser.add_argument("--chunk-timeout-seconds", type=float, default=600.0)
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ args = parser.parse_args()
+ if args.chunks_per_session < 1 or args.warmup_chunks < 0:
+ parser.error("chunks-per-session must be positive and warmup-chunks non-negative")
+ if args.control_idle_timeout_seconds <= 0:
+ parser.error("control-idle-timeout-seconds must be positive")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ loader = _loader_module()
+ pipeline = loader.get_pipeline(
+ model_root=args.model_root,
+ pipeline_class=ABotWorldInteractivePipeline,
+ )
+ image = Image.open(args.image).convert("RGB")
+ results: list[dict[str, Any]] = []
+ json_path = args.output_dir / "results.json"
+ csv_path = args.output_dir / "results.csv"
+
+ def write_results() -> None:
+ json_path.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ if not results:
+ return
+ fieldnames = sorted({key for row in results for key in row if key != "service_runtime_metrics"})
+ with csv_path.open("w", newline="", encoding="utf-8") as handle:
+ writer = csv.DictWriter(handle, fieldnames=fieldnames)
+ writer.writeheader()
+ writer.writerows({key: value for key, value in row.items() if key in fieldnames} for row in results)
+
+ try:
+ for sessions in args.sessions:
+ for max_batch_size in args.max_batch_sizes:
+ if max_batch_size > sessions:
+ continue
+ print(f"running sessions={sessions} max_batch_size={max_batch_size}", flush=True)
+ try:
+ row = _run_point(
+ pipeline=pipeline,
+ image=image,
+ args=args,
+ sessions=sessions,
+ max_batch_size=max_batch_size,
+ )
+ except torch.OutOfMemoryError as exc:
+ torch.cuda.empty_cache()
+ row = {
+ "sessions": sessions,
+ "max_batch_size": max_batch_size,
+ "control_latent_frames": args.control_latent_frames,
+ "status": "oom",
+ "error": str(exc).splitlines()[0],
+ }
+ print(f"OOM sessions={sessions} max_batch_size={max_batch_size}", flush=True)
+ results.append(row)
+ write_results()
+ finally:
+ pipeline.close()
+ write_results()
+ print(json.dumps(results, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/validate_abot_nccl_migration.py b/tools/validation/validate_abot_nccl_migration.py
new file mode 100644
index 00000000..df9e576f
--- /dev/null
+++ b/tools/validation/validate_abot_nccl_migration.py
@@ -0,0 +1,182 @@
+"""Validate one ABot session migration across two GPU processes with NCCL.
+
+This is intentionally model-service level: each rank owns an independent ABot
+replica, rank 0 generates one causal chunk, the retained tensors are moved with
+the same manifest/P2P helpers used by ``--worker-mode process-nccl``, and rank
+1 continues that exact session. No CPU tensor snapshot is used for model
+state.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import socket
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+from telefuser.service.livekit.nccl_transfer import allocate_tensor_tree_leaves, transfer_tensor_leaves_nccl
+
+
+def _loader_module() -> Any:
+ loader_path = Path(__file__).resolve().parents[2] / "examples/abot_world/_loader.py"
+ spec = importlib.util.spec_from_file_location("abot_nccl_validation_loader", loader_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot loader: {loader_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _free_port() -> int:
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
+def _take_output(service: ABotWorldLiveKitService, session_id: str, timeout: float = 120.0) -> dict:
+ state = service._session(session_id) # Validation intentionally observes the service output boundary.
+ if state is None:
+ raise KeyError(session_id)
+ return state.output_queue.get(timeout=timeout)
+
+
+def _drain_outputs(service: ABotWorldLiveKitService, session_id: str, stop: threading.Event) -> None:
+ """Mirror the parent LiveKit transport's continuous model-output pull."""
+ state = service._session(session_id)
+ if state is None:
+ return
+ while not stop.is_set():
+ try:
+ state.output_queue.get(timeout=0.05)
+ except Exception:
+ continue
+
+
+def _rank_main(rank: int, args: argparse.Namespace, port: int) -> None:
+ torch.cuda.set_device(rank)
+ dist.init_process_group(
+ backend="nccl",
+ init_method=f"tcp://127.0.0.1:{port}",
+ rank=rank,
+ world_size=2,
+ )
+ service: ABotWorldLiveKitService | None = None
+ try:
+ loader = _loader_module()
+ pipeline = loader.get_pipeline(
+ model_root=args.model_root,
+ pipeline_class=ABotWorldInteractivePipeline,
+ device_id=rank,
+ )
+ service = ABotWorldLiveKitService(
+ pipeline,
+ max_batch_size=1,
+ default_session_config={
+ "image_path": str(args.image),
+ "prompt": args.prompt,
+ "fps": 12,
+ "control_latent_frames": args.control_latent_frames,
+ "seed": args.seed,
+ },
+ )
+ # A process-nccl child preloads its replica at worker startup. The
+ # target must do the same before it adopts CUDA session tensors.
+ if rank == 1:
+ service.start()
+ session_id = "nccl-validation-session"
+ metadata: dict[str, Any] | None = None
+ leaves: dict[tuple[Any, ...], torch.Tensor] | None = None
+ if rank == 0:
+ service.create_session({"session_id": session_id})
+ preview = _take_output(service, session_id)
+ assert preview["type"] == "preview"
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["W"]})
+ source_chunk = _take_output(service, session_id)
+ if source_chunk.get("type") not in {"chunk", "video"}:
+ raise RuntimeError(f"Expected source generated output, got {source_chunk.get('type')!r}")
+ # Stop new scheduling and emulate the parent transport draining any
+ # in-flight output while the source reaches a migration boundary.
+ service.push_chunk(session_id, {"type": "control_state", "controls": []})
+ drain_stop = threading.Event()
+ drain_thread = threading.Thread(
+ target=_drain_outputs,
+ args=(service, session_id, drain_stop),
+ daemon=True,
+ )
+ drain_thread.start()
+ metadata = service.prepare_migration_nccl_metadata(session_id, timeout=120.0)
+ drain_stop.set()
+ drain_thread.join(timeout=1.0)
+ leaves = metadata.pop("_nccl_tensor_leaves")
+ print(
+ f"source_chunk={source_chunk.get('index')} state_bytes={metadata['state_bytes']} "
+ f"tensor_leaves={len(leaves)}",
+ flush=True,
+ )
+
+ object_list: list[Any] = [metadata]
+ dist.broadcast_object_list(object_list, src=0, device=torch.device(f"cuda:{rank}"))
+ metadata = object_list[0]
+ assert isinstance(metadata, dict)
+ if rank == 1:
+ leaves = allocate_tensor_tree_leaves(metadata["tensor_manifest"], torch.device(f"cuda:{rank}"))
+ assert leaves is not None
+
+ started = time.monotonic()
+ transferred = transfer_tensor_leaves_nccl(leaves, peer_rank=1 - rank, send=rank == 0)
+ torch.cuda.synchronize(rank)
+ elapsed = time.monotonic() - started
+ if rank == 0:
+ print(f"nccl_copy_bytes={transferred} nccl_copy_seconds={elapsed:.6f}", flush=True)
+ if rank == 1:
+ installed = service.import_migration_nccl(metadata, leaves, owner_worker_id="worker-1", ownership_epoch=1)
+ assert installed == session_id
+ dist.barrier()
+ if rank == 0:
+ service.commit_migration(session_id)
+ else:
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["W"]})
+ target_chunk = _take_output(service, session_id)
+ if target_chunk.get("type") not in {"chunk", "video"}:
+ raise RuntimeError(f"Expected target generated output, got {target_chunk.get('type')!r}")
+ session = service._session(session_id)
+ assert session is not None
+ print(
+ f"target_chunk={target_chunk.get('index')} next_latent_frame={session.pipeline_session.next_latent_frame} "
+ f"emitted_frames={session.pipeline_session.emitted_frames}",
+ flush=True,
+ )
+ dist.barrier()
+ finally:
+ if service is not None:
+ service.stop()
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--control-latent-frames", choices=(1, 2, 3), type=int, default=1)
+ parser.add_argument("--seed", type=int, default=42)
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = parse_args()
+ mp.spawn(_rank_main, args=(args, _free_port()), nprocs=2, join=True)
+
+
+if __name__ == "__main__":
+ main()
From 14a378f0092f6a5f9e91201611a699b4e21bcea4 Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Fri, 14 Aug 2026 08:01:20 +0000
Subject: [PATCH 2/8] feat(abot): add multi-session serving, observability, and
workload traces
---
.gitignore | 5 +
deploy/observability/README.md | 129 ++
deploy/observability/dcgm-metrics.csv | 15 +
deploy/observability/docker-compose.yml | 73 +
.../dashboards/telefuser-abot-serving.json | 456 ++++++
.../provisioning/dashboards/dashboards.yml | 11 +
.../provisioning/datasources/prometheus.yml | 9 +
deploy/observability/prometheus.yml | 21 +
docs/en/abot_world.md | 160 +-
docs/en/stream_server.md | 19 +
docs/zh/stream_server.md | 16 +
examples/abot_world/README.md | 6 +
.../abot_world/abot_world_livekit_service.py | 60 +-
telefuser/metrics/collector.py | 10 +-
telefuser/pipelines/abot_world/interactive.py | 42 +-
telefuser/pipelines/abot_world/service.py | 253 ++-
telefuser/pipelines/abot_world/taew_vae.py | 418 ++++-
telefuser/service/livekit/app.py | 17 +-
telefuser/service/livekit/metrics.py | 693 +++++++++
.../service/livekit/multi_session_worker.py | 30 +
.../livekit/nccl_process_worker_pool.py | 572 ++++++-
telefuser/service/livekit/pipeline_adapter.py | 2 +-
telefuser/service/livekit/pipeline_router.py | 2 +-
.../service/livekit/process_worker_pool.py | 60 +
telefuser/service/livekit/runtime.py | 92 +-
telefuser/service/livekit/worker.py | 55 +
.../pipelines/abot_world/test_interactive.py | 60 +
.../abot_world/test_livekit_examples.py | 49 +-
.../abot_world/test_livekit_service.py | 295 +++-
.../pipelines/abot_world/test_migration.py | 303 +++-
tests/unit/service/livekit/test_app.py | 17 +
.../livekit/test_multi_session_capacity.py | 35 +
.../livekit/test_nccl_process_worker_pool.py | 192 +++
.../service/livekit/test_serving_metrics.py | 290 ++++
tests/unit/service/livekit/test_worker.py | 29 +
tests/unit/test_metrics.py | 12 +
.../validation/test_abot_livekit_burst.py | 228 +++
.../test_capture_abot_serving_metrics.py | 85 +
.../test_capture_gpu_nvml_metrics.py | 19 +
.../benchmark_abot_livekit_burst.py | 1383 +++++++++++++++++
.../capture_abot_serving_metrics.py | 275 ++++
tools/validation/capture_gpu_nvml_metrics.py | 269 ++++
.../validate_abot_nccl_migration.py | 65 +-
...4gpu_lf3_12fps_all_active_peak16_wave.json | 79 +
...u_lf3_12fps_intermittent_input_peak16.json | 106 ++
.../abot_livekit_4gpu_lf3_12fps_wave.json | 62 +
46 files changed, 6893 insertions(+), 186 deletions(-)
create mode 100644 deploy/observability/README.md
create mode 100644 deploy/observability/dcgm-metrics.csv
create mode 100644 deploy/observability/docker-compose.yml
create mode 100644 deploy/observability/grafana/dashboards/telefuser-abot-serving.json
create mode 100644 deploy/observability/grafana/provisioning/dashboards/dashboards.yml
create mode 100644 deploy/observability/grafana/provisioning/datasources/prometheus.yml
create mode 100644 deploy/observability/prometheus.yml
create mode 100644 telefuser/service/livekit/metrics.py
create mode 100644 tests/unit/service/livekit/test_nccl_process_worker_pool.py
create mode 100644 tests/unit/service/livekit/test_serving_metrics.py
create mode 100644 tests/unit/validation/test_abot_livekit_burst.py
create mode 100644 tests/unit/validation/test_capture_abot_serving_metrics.py
create mode 100644 tests/unit/validation/test_capture_gpu_nvml_metrics.py
create mode 100644 tools/validation/benchmark_abot_livekit_burst.py
create mode 100644 tools/validation/capture_abot_serving_metrics.py
create mode 100755 tools/validation/capture_gpu_nvml_metrics.py
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
diff --git a/.gitignore b/.gitignore
index 256048bc..75344814 100755
--- a/.gitignore
+++ b/.gitignore
@@ -148,6 +148,11 @@ docs/**/modelzoo.md
!benchmarks/telefuser_aiperf/configs/*.json
!benchmarks/telefuser_aiperf/data/stream_lingbot_controls.json
!benchmarks/baseline/sglang_lingbot_stream/configs/*.json
+!tools/validation/workloads/*.json
+
+# Version the reproducible ABot LiveKit user-wave scenario, not generated result JSON.
+!tools/validation/workloads/
+!tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
project
output
diff --git a/deploy/observability/README.md b/deploy/observability/README.md
new file mode 100644
index 00000000..1e2cd3ba
--- /dev/null
+++ b/deploy/observability/README.md
@@ -0,0 +1,129 @@
+# TeleFuser serving observability
+
+This stack scrapes the LiveKit serving API, NVIDIA DCGM Exporter, and Node
+Exporter into Prometheus, then provisions a Grafana dashboard automatically.
+It is intended for the four-GPU ABot-World experiment, but has no model-specific
+runtime dependency.
+
+## Start the serving API first
+
+Expose the TeleFuser metrics endpoint on the host port used below (default
+`8088`). The endpoint includes low-cardinality scheduler/session/batch/pipeline
+metrics in addition to generic TeleFuser metrics:
+
+```bash
+curl -fsS http://127.0.0.1:8088/metrics | grep '^telefuser_serving_'
+```
+
+The compose file maps `host.docker.internal` to Docker's host gateway. On an
+older Docker engine that does not support `host-gateway`, replace
+`host.docker.internal:8088` in `prometheus.yml` with the host's reachable IP.
+The serving API must be reachable from the Prometheus container.
+
+## Start the monitoring stack
+
+```bash
+cd deploy/observability
+docker compose up -d
+```
+
+Open Grafana at `http://:3000` (default credentials are `admin` / `admin`; set
+`GRAFANA_ADMIN_PASSWORD` before starting in any shared environment). Prometheus
+is available at port 9090.
+
+DCGM Exporter needs the NVIDIA Container Toolkit. By default it observes
+**physical GPUs 0--3**, through NVIDIA Container Toolkit's
+`NVIDIA_VISIBLE_DEVICES` allowlist. Select a different physical GPU set without
+editing Compose:
+
+```bash
+# The four GPUs used by the current ABot experiment (the default).
+TELEFUSER_MONITOR_GPU_IDS=0,1,2,3 docker compose up -d
+
+# Observe all eight physical GPUs from one DCGM Exporter.
+TELEFUSER_MONITOR_GPU_IDS=0,1,2,3,4,5,6,7 docker compose up -d
+```
+
+These are host/physical GPU indices. `CUDA_VISIBLE_DEVICES` remaps only the
+serving process to logical IDs; it does **not** remap an independent Docker
+container. The custom `dcgm-metrics.csv` requests GPU/HBM, power/temperature,
+PCIe, NVLink, DRAM, and tensor-core telemetry. Some profiling fields are
+conditionally omitted when the driver, GPU, or DCGM version cannot expose them.
+
+## Capture an experiment without Docker
+
+When Docker, DCGM Exporter, or Grafana cannot run on the experiment host, save
+the serving metrics as a durable artifact with the lightweight collector. It
+queries the same `/metrics` and `/v1/service/metrics/json` APIs used by
+Prometheus, requires only the Python standard library, and explicitly bypasses
+all proxy environment variables (including for `127.0.0.1`).
+
+```bash
+python tools/validation/capture_abot_serving_metrics.py \
+ --server-url http://127.0.0.1:8088 \
+ --duration 420 \
+ --interval 1 \
+ --output-dir results/experiments/abot_livekit_realistic_peak16/metrics
+```
+
+For the physical GPUs used by this experiment, pair it with the direct NVML
+fallback. It does not shell out to `nvidia-smi`:
+
+```bash
+python tools/validation/capture_gpu_nvml_metrics.py \
+ --gpu-indices 0,1,2,3 \
+ --duration 420 \
+ --interval 1 \
+ --output-dir results/experiments/abot_livekit_realistic_peak16/gpu_metrics
+```
+
+Run both collectors in parallel with the workload, then wait for them before
+stopping the serving API. The service collector writes raw Prometheus snapshots,
+an aggregate-only JSONL time series, and a manifest even if interrupted. The
+NVML collector writes one JSONL row per sample with GPU utilization, framebuffer
+memory, power, and temperature. It is a local fallback, not a replacement for
+DCGM: PCIe/NVLink, DRAM, tensor-core profiling, host CPU/RAM, and network
+counters still require the full DCGM/Node Exporter stack.
+
+
+## Metric groups
+
+- `telefuser_serving_worker_*`: worker/GPU retained sessions, capacity, busy
+ ratio, latest model stage timing, and model-reported chunk latency.
+- `telefuser_serving_sessions`, `*_queue_depth`, `*_session_status`: admission,
+ active/idle/waiting session state without session-id labels.
+- `telefuser_serving_batch_*`: observed coalesced batch distribution and mean.
+- `telefuser_serving_pipeline_stage_latency_seconds`: cumulative stage
+ histograms for VAE encode/decode, DiT, cache operations, and postprocessing.
+- `telefuser_serving_action_to_first_frame_seconds`: validated action ingress
+ to the first frame accepted by the LiveKit publisher. It is a server-side
+ A2F measurement; network and browser decode are intentionally excluded.
+- `telefuser_serving_published_fps`: trailing 30-second aggregate and
+ per-active-session published FPS. This is not model compute FPS.
+- `telefuser_serving_slo_*`: each chunk is judged against
+ `frames / configured_fps`, using scheduler queue wait plus compute time.
+- DCGM metrics cover GPU utilization, framebuffer/HBM use, power, temperature,
+ PCIe, NVLink, DRAM, and tensor-core fields supported by the installed driver.
+ Node Exporter provides CPU, RAM, disk, and host-network counters.
+
+OpenTelemetry is deliberately optional: use it only when per-action trace IDs
+are needed. Prometheus remains the experiment source of truth because it avoids
+high-cardinality session/action labels.
+
+## Useful PromQL
+
+```promql
+# P95 model chunk time
+histogram_quantile(0.95, sum by (le) (rate(telefuser_serving_chunk_latency_seconds_bucket[1m])))
+
+# Per-stage P95
+histogram_quantile(0.95, sum by (stage, le) (rate(telefuser_serving_pipeline_stage_latency_seconds_bucket[1m])))
+
+# SLO attainment over the last minute
+sum(rate(telefuser_serving_slo_chunks_total{result="met"}[1m]))
+/
+sum(rate(telefuser_serving_slo_chunks_total[1m]))
+
+# Published user-visible aggregate FPS
+telefuser_serving_published_fps{scope="aggregate"}
+```
diff --git a/deploy/observability/dcgm-metrics.csv b/deploy/observability/dcgm-metrics.csv
new file mode 100644
index 00000000..d1c8b708
--- /dev/null
+++ b/deploy/observability/dcgm-metrics.csv
@@ -0,0 +1,15 @@
+# TeleFuser ABot-World serving collector set.
+# DCGM field, Prometheus type, help text. Profiling fields are emitted only
+# when the installed driver/DCGM version supports them.
+DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).
+DCGM_FI_DEV_MEM_COPY_UTIL, gauge, GPU memory-copy utilization (in %).
+DCGM_FI_DEV_FB_USED, gauge, Framebuffer or HBM memory used (in MiB).
+DCGM_FI_DEV_FB_FREE, gauge, Framebuffer or HBM memory free (in MiB).
+DCGM_FI_DEV_POWER_USAGE, gauge, GPU board power draw (in W).
+DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C).
+DCGM_FI_DEV_MEMORY_TEMP, gauge, GPU memory temperature (in C).
+DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, gauge, Aggregate NVLink throughput.
+DCGM_FI_PROF_PCIE_TX_BYTES, gauge, Active PCIe transmit bandwidth in bytes per second.
+DCGM_FI_PROF_PCIE_RX_BYTES, gauge, Active PCIe receive bandwidth in bytes per second.
+DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles with DRAM active.
+DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles with tensor cores active.
diff --git a/deploy/observability/docker-compose.yml b/deploy/observability/docker-compose.yml
new file mode 100644
index 00000000..37978c01
--- /dev/null
+++ b/deploy/observability/docker-compose.yml
@@ -0,0 +1,73 @@
+services:
+ prometheus:
+ image: prom/prometheus:v2.55.1
+ command:
+ - --config.file=/etc/prometheus/prometheus.yml
+ - --storage.tsdb.path=/prometheus
+ - --web.enable-lifecycle
+ ports:
+ - "9090:9090"
+ volumes:
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+ - prometheus-data:/prometheus
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ restart: unless-stopped
+
+ grafana:
+ image: grafana/grafana:11.2.2
+ ports:
+ - "3000:3000"
+ environment:
+ GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+ GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+ GF_USERS_ALLOW_SIGN_UP: "false"
+ volumes:
+ - grafana-data:/var/lib/grafana
+ - ./grafana/provisioning:/etc/grafana/provisioning:ro
+ - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+ depends_on:
+ - prometheus
+ restart: unless-stopped
+
+ dcgm-exporter:
+ image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
+ command:
+ - --collectors
+ - /etc/dcgm-exporter/telefuser.csv
+ - --collect-interval
+ - "5000"
+ ports:
+ - "9400:9400"
+ runtime: nvidia
+ cap_add:
+ - SYS_ADMIN
+ # NVIDIA Container Toolkit applies this physical-GPU allowlist before the
+ # container starts. Do not use the serving process's logical CUDA IDs here.
+ # Override it at launch, for example:
+ # TELEFUSER_MONITOR_GPU_IDS=0,1,2,3,4,5,6,7 docker compose up -d
+ environment:
+ NVIDIA_VISIBLE_DEVICES: "${TELEFUSER_MONITOR_GPU_IDS:-0,1,2,3}"
+ NVIDIA_DRIVER_CAPABILITIES: utility
+ volumes:
+ - ./dcgm-metrics.csv:/etc/dcgm-exporter/telefuser.csv:ro
+ restart: unless-stopped
+
+ node-exporter:
+ image: prom/node-exporter:v1.8.2
+ command:
+ - --path.rootfs=/host
+ - --path.procfs=/host/proc
+ - --path.sysfs=/host/sys
+ ports:
+ - "9100:9100"
+ pid: host
+ volumes:
+ - /:/host:ro,rslave
+ - /proc:/host/proc:ro
+ - /sys:/host/sys:ro
+ restart: unless-stopped
+
+volumes:
+ prometheus-data:
+ grafana-data:
diff --git a/deploy/observability/grafana/dashboards/telefuser-abot-serving.json b/deploy/observability/grafana/dashboards/telefuser-abot-serving.json
new file mode 100644
index 00000000..8337015c
--- /dev/null
+++ b/deploy/observability/grafana/dashboards/telefuser-abot-serving.json
@@ -0,0 +1,456 @@
+{
+ "annotations": {
+ "list": []
+ },
+ "editable": true,
+ "graphTooltip": 1,
+ "links": [],
+ "panels": [
+ {
+ "id": 1,
+ "title": "Published FPS",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "telefuser_serving_published_fps",
+ "legendFormat": "{{scope}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "fps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ }
+ },
+ {
+ "id": 2,
+ "title": "Sessions and admission queue",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "telefuser_serving_sessions",
+ "legendFormat": "sessions {{state}}",
+ "refId": "A"
+ },
+ {
+ "expr": "telefuser_serving_queue_depth",
+ "legendFormat": "queue {{queue}}",
+ "refId": "B"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ }
+ },
+ {
+ "id": 3,
+ "title": "P95 chunk / A2F latency",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(telefuser_serving_chunk_latency_seconds_bucket[$__rate_interval])))",
+ "legendFormat": "chunk p95",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(telefuser_serving_action_to_first_frame_seconds_bucket[$__rate_interval])))",
+ "legendFormat": "A2F p95",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ }
+ },
+ {
+ "id": 4,
+ "title": "P95 pipeline stages",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(telefuser_serving_pipeline_stage_latency_seconds_bucket[$__rate_interval])))",
+ "legendFormat": "{{stage}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ }
+ },
+ {
+ "id": 5,
+ "title": "Per-worker sessions and capacity",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "telefuser_serving_worker_sessions",
+ "legendFormat": "{{worker_id}} GPU {{gpu}} sessions",
+ "refId": "A"
+ },
+ {
+ "expr": "telefuser_serving_worker_capacity",
+ "legendFormat": "{{worker_id}} GPU {{gpu}} capacity",
+ "refId": "B"
+ },
+ {
+ "expr": "telefuser_serving_worker_busy_ratio",
+ "legendFormat": "{{worker_id}} GPU {{gpu}} busy ratio",
+ "refId": "C"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 16
+ }
+ },
+ {
+ "id": 6,
+ "title": "GPU utilization",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "DCGM_FI_DEV_GPU_UTIL",
+ "legendFormat": "GPU {{gpu}} util",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 16
+ }
+ },
+ {
+ "id": 7,
+ "title": "GPU framebuffer / HBM",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "DCGM_FI_DEV_FB_USED",
+ "legendFormat": "GPU {{gpu}} used",
+ "refId": "A"
+ },
+ {
+ "expr": "DCGM_FI_DEV_FB_FREE",
+ "legendFormat": "GPU {{gpu}} free",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "mbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 24
+ }
+ },
+ {
+ "id": 8,
+ "title": "GPU power / temperature",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "DCGM_FI_DEV_POWER_USAGE",
+ "legendFormat": "GPU {{gpu}} power W",
+ "refId": "A"
+ },
+ {
+ "expr": "DCGM_FI_DEV_GPU_TEMP",
+ "legendFormat": "GPU {{gpu}} temp C",
+ "refId": "B"
+ },
+ {
+ "expr": "DCGM_FI_DEV_MEMORY_TEMP",
+ "legendFormat": "GPU {{gpu}} HBM temp C",
+ "refId": "C"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 24
+ }
+ },
+ {
+ "id": 9,
+ "title": "GPU PCIe / NVLink throughput",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "DCGM_FI_PROF_PCIE_TX_BYTES",
+ "legendFormat": "GPU {{gpu}} PCIe TX",
+ "refId": "A"
+ },
+ {
+ "expr": "DCGM_FI_PROF_PCIE_RX_BYTES",
+ "legendFormat": "GPU {{gpu}} PCIe RX",
+ "refId": "B"
+ },
+ {
+ "expr": "DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL",
+ "legendFormat": "GPU {{gpu}} NVLink",
+ "refId": "C"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 32
+ }
+ },
+ {
+ "id": 10,
+ "title": "Node CPU utilization",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
+ "legendFormat": "{{instance}} CPU",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 32
+ }
+ },
+ {
+ "id": 11,
+ "title": "Node memory available",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "node_memory_MemAvailable_bytes",
+ "legendFormat": "{{instance}} available",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 40
+ }
+ },
+ {
+ "id": 12,
+ "title": "Node network traffic",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[$__rate_interval]))",
+ "legendFormat": "{{instance}} RX",
+ "refId": "A"
+ },
+ {
+ "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo\"}[$__rate_interval]))",
+ "legendFormat": "{{instance}} TX",
+ "refId": "B"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ }
+ },
+ {
+ "id": 13,
+ "title": "Batching and SLO attainment",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "telefuser_serving_mean_batch_size",
+ "legendFormat": "mean batch",
+ "refId": "A"
+ },
+ {
+ "expr": "telefuser_serving_slo_attainment_ratio",
+ "legendFormat": "SLO attainment",
+ "refId": "B"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ }
+ },
+ {
+ "id": 14,
+ "title": "Migrations and serving errors",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "targets": [
+ {
+ "expr": "sum by (result) (rate(telefuser_serving_migrations_total[$__rate_interval]))",
+ "legendFormat": "migration {{result}}",
+ "refId": "A"
+ },
+ {
+ "expr": "sum by (kind) (rate(telefuser_serving_errors_total[$__rate_interval]))",
+ "legendFormat": "error {{kind}}",
+ "refId": "B"
+ }
+ ],
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ }
+ }
+ ],
+ "refresh": "5s",
+ "schemaVersion": 39,
+ "tags": [
+ "telefuser",
+ "serving",
+ "abot-world"
+ ],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timezone": "browser",
+ "title": "TeleFuser ABot-World Serving",
+ "uid": "telefuser-abot-serving",
+ "version": 2
+}
diff --git a/deploy/observability/grafana/provisioning/dashboards/dashboards.yml b/deploy/observability/grafana/provisioning/dashboards/dashboards.yml
new file mode 100644
index 00000000..5c91f8f7
--- /dev/null
+++ b/deploy/observability/grafana/provisioning/dashboards/dashboards.yml
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+ - name: TeleFuser
+ orgId: 1
+ folder: TeleFuser
+ type: file
+ disableDeletion: false
+ editable: true
+ options:
+ path: /var/lib/grafana/dashboards
diff --git a/deploy/observability/grafana/provisioning/datasources/prometheus.yml b/deploy/observability/grafana/provisioning/datasources/prometheus.yml
new file mode 100644
index 00000000..bb009bb2
--- /dev/null
+++ b/deploy/observability/grafana/provisioning/datasources/prometheus.yml
@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+ - name: Prometheus
+ type: prometheus
+ access: proxy
+ url: http://prometheus:9090
+ isDefault: true
+ editable: false
diff --git a/deploy/observability/prometheus.yml b/deploy/observability/prometheus.yml
new file mode 100644
index 00000000..d795c9c7
--- /dev/null
+++ b/deploy/observability/prometheus.yml
@@ -0,0 +1,21 @@
+global:
+ scrape_interval: 5s
+ evaluation_interval: 5s
+
+scrape_configs:
+ - job_name: telefuser-serving
+ metrics_path: /metrics
+ static_configs:
+ # The serving process runs on the host. On Linux, replace this with the
+ # host IP or start this compose stack with network_mode: host.
+ - targets: ["host.docker.internal:8088"]
+ labels:
+ service: telefuser-abot-world
+
+ - job_name: dcgm
+ static_configs:
+ - targets: ["dcgm-exporter:9400"]
+
+ - job_name: node
+ static_configs:
+ - targets: ["node-exporter:9100"]
diff --git a/docs/en/abot_world.md b/docs/en/abot_world.md
index ed1e487c..20d9c598 100644
--- a/docs/en/abot_world.md
+++ b/docs/en/abot_world.md
@@ -115,31 +115,161 @@ This fixed logical position policy is an intentional difference from the
original non-sink ABot baseline and must be evaluated as part of any future
long-horizon quality claim.
-## Multi-GPU and autoscaling
+## Four-GPU Black-Box User-Wave Baseline
+
+The checked-in ABot factory fixes the real-time baseline at **12 FPS**,
+**three control latents per chunk** (12 decoded frames), `scheduler_mode=batched`,
+and `max_batch_size=2`. The following launch uses four physical GPUs (4--7),
+one spawned model worker per GPU, two retained sessions per worker, and a bounded
+HTTP admission queue. It is a black-box deployment: clients call only the public
+HTTP/LiveKit interfaces and never select a GPU.
-Assign exactly one numeric GPU ID to each ABot worker. For example, four warm
-replicas with hardware-sized retained-session capacity use:
+Start a local LiveKit server in a separate terminal. A loopback experiment does
+not need the public TURN setup; retain the TURN setup above when clients are remote.
```bash
-telefuser stream-serve examples/abot_world/abot_world_livekit_service.py \
+livekit-server --dev --bind 127.0.0.1
+```
+
+Use the source-tree CLI below. The currently installed `telefuser` console script
+may be older than this checkout and omit `process-nccl`; `python -m` with
+`PYTHONPATH=$PWD` is therefore intentional.
+
+```bash
+cd /public/fanyk1/lwb/TeleFuser-abot-world
+
+export PYTHONPATH=$PWD
+export TF_MODEL_ZOO_PATH=/public/fanyk1/lwb/model_zoo
+export CUDA_VISIBLE_DEVICES=4,5,6,7
+unset http_proxy https_proxy all_proxy HTTP_PROXY HTTPS_PROXY ALL_PROXY
+
+/public/fanyk1/lwb/envs/telefuser_sage291/bin/python -m telefuser.entrypoints.cli.main \
+ stream-serve examples/abot_world/abot_world_livekit_service.py \
+ --host 127.0.0.1 --port 8088 \
--livekit-url ws://127.0.0.1:7880 \
--livekit-api-key devkey --livekit-api-secret secret \
--num-workers 4 --worker-gpu-map '0;1;2;3' \
--worker-mode process-nccl \
- --max-sessions-per-worker auto --queue-size 32 \
- --port 8088 --skip-validation
+ --max-sessions-per-worker 2 --queue-size 16 \
+ --skip-validation
+```
+
+`CUDA_VISIBLE_DEVICES=4,5,6,7` remaps physical GPUs 4--7 to logical IDs 0--3,
+which is why the worker map is exactly `'0;1;2;3'` rather than `'4;5;6;7'`.
+The parent admission scheduler assigns each incoming session to the least-loaded
+worker; no browser or workload-client GPU argument exists. `process-nccl` keeps
+LiveKit transport in the parent, holds model state in the children, batches each
+worker's compatible ready sessions, and can migrate retained state at a chunk
+boundary. Rebalancing is enabled by default.
+
+Wait for readiness and record the public configuration before the load starts:
+
+```bash
+curl --noproxy '*' --fail --silent http://127.0.0.1:8088/v1/service/ready
+curl --noproxy '*' --fail --silent http://127.0.0.1:8088/v1/service/metadata | python -m json.tool
+```
+
+The metadata must report `worker_mode: process-nccl`, `num_workers: 4`, and
+`configured_max_sessions_per_worker: 2`. It also reports the routing snapshot and
+any rebalance decision. Do not claim a migration result merely because
+`migration_supported` is true: first pass the focused ABot TAeW migration validation
+after its state-snapshot patch has landed. A balanced user wave normally exercises
+placement, local batching, admission queueing, and recovery; it need not create an
+imbalanced placement worth migrating.
+
+### Run the arrival/burst/recovery workload
+
+The tracked scenario at
+`tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json` drives real
+LiveKit controller clients. Each client creates a session through
+`POST /v1/stream/sessions`, joins its room, sends reliable `tf.control` states,
+counts received WebRTC video frames, and deletes its public session on departure.
+The workload has four phases: 4-user warmup, 8-user SLO-capacity ramp, 12-user
+burst (four requests should queue under the 4 x 2 admission cap), then recovery to
+4 users. Scale-down removes the newest sessions first, so long-lived session state
+is preserved; a client stays counted until its scheduled shutdown actually runs.
+
+```bash
+cd /public/fanyk1/lwb/TeleFuser-abot-world
+
+PYTHONPATH=$PWD /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \
+ tools/validation/benchmark_abot_livekit_burst.py \
+ --scenario tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json \
+ --output /public/fanyk1/lwb/results/experiments/abot_livekit_4gpu_lf3_12fps_wave/result.json
```
-`process-nccl` is the cross-process TurboServe baseline. The parent keeps each
-LiveKit room, ingress, and egress alive; a source GPU quiesces at a chunk boundary,
-the target GPU receives retained CUDA tensors directly with NCCL P2P, and routing
-changes only after source release and ownership commit. Controls received during
-that window are buffered in the parent and replayed on the committed owner.
+Use `--dry-run` first to validate the JSON without contacting the service. The
+result artifact keeps raw one-second delivery samples, session admissions, phase
+events, server metadata snapshots, and a concise phase table. `Aggregate FPS` is
+all generated frames actually received by the WebRTC clients. `FPS/all-user` and SLO
+attainment use every requested session after its 15-second grace interval; an
+unassigned, disconnected, or stalled request contributes zero FPS. The legacy
+`FPS/controlled` field remains available for comparison with active media clients.
+
+This is an end-to-end client-delivery experiment, not a model-only benchmark.
+Correlate its artifact with `/metrics` or the Prometheus/Grafana stack for GPU,
+queue, batching, and stage telemetry; do not infer GPU utilization from delivery
+FPS alone.
+
+### All-active peak-16 capacity trace
+
+Use the following trace to measure fixed four-GPU overload behavior without hiding
+users behind an admission queue. It requires four warm workers, four retained
+sessions per worker, batch cap four, and `--queue-size 0`: the 16 requested users
+must all receive an immediate `assigned` response. A queued, rejected, disconnected,
+or zero-output user remains in the `FPS/all-user` and SLO denominator after grace;
+there is no per-client GPU selection.
+
+| Phase | Duration | Arrival/departure window | Target users |
+| --- | ---: | ---: | ---: |
+| warmup | 45 s | arrivals over 10 s | 4 |
+| ramps | 55 s + 55 s | arrivals over 15 s | 8, 12 |
+| peak | 80 s | arrivals over 15 s | 16 |
+| recovery | 50 s + 45 s | departures over 15 s | 8, 4 |
+
+Start the service on physical GPUs 0--3 with the all-active profile:
+
+```bash
+cd /public/fanyk1/lwb/TeleFuser-abot-world
+export PYTHONPATH=$PWD
+export TF_MODEL_ZOO_PATH=/public/fanyk1/lwb/model_zoo
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export TELEFUSER_ABOT_SCHEDULER_MODE=batched
+export TELEFUSER_ABOT_MAX_BATCH_SIZE=4
+export TELEFUSER_ABOT_BATCHING_WINDOW_MS=2
+
+/public/fanyk1/lwb/envs/telefuser_sage291/bin/python -m telefuser.entrypoints.cli.main \
+ stream-serve examples/abot_world/abot_world_livekit_service.py \
+ --host 127.0.0.1 --port 8088 \
+ --livekit-url ws://127.0.0.1:7880 \
+ --livekit-api-key devkey --livekit-api-secret secret \
+ --num-workers 4 --worker-gpu-map '0;1;2;3' \
+ --worker-mode process-nccl \
+ --max-sessions-per-worker 4 --queue-size 0 \
+ --skip-validation
+```
+
+Then run the black-box client trace:
+
+```bash
+PYTHONPATH=$PWD /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \
+ tools/validation/benchmark_abot_livekit_burst.py \
+ --scenario tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json \
+ --output /public/fanyk1/lwb/results/experiments/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave/result.json
+```
+
+The printed `admitted` column must be `16/16` in the peak phase. The JSON phase
+summary also records `max_requested_users`, all-user FPS, and immediate-assignment
+contract status, so a queue or failed admission invalidates rather than improves the
+reported serving result.
+
+## Multi-GPU and autoscaling
-It deliberately uses a fixed one-GPU-per-worker NCCL group, so it does not combine
-with process autoscaling. Plain `--worker-mode process` remains independent-replica
-batching and reports `migration_supported: false`; it is not a TurboServe migration
-baseline. In-process migration remains useful for debugging, but stages state via CPU.
+`process-nccl` deliberately uses a fixed one-GPU-per-worker NCCL group, so it does
+not combine with process autoscaling. Plain `--worker-mode process` remains
+independent-replica batching and reports `migration_supported: false`; it is not a
+TurboServe migration baseline. In-process migration remains useful for debugging,
+but stages state via CPU.
For plain `process` mode, optional cold-replica autoscaling starts only the requested
minimum and scales within the GPUs declared above. For example:
diff --git a/docs/en/stream_server.md b/docs/en/stream_server.md
index 1b1a73e4..61eacaa9 100644
--- a/docs/en/stream_server.md
+++ b/docs/en/stream_server.md
@@ -294,6 +294,7 @@ The current runtime has these deliberate documentation-visible limitations:
| `/v1/service/ready` | GET | Readiness probe |
| `/v1/service/metadata` | GET | Runtime topology and service metadata |
| `/v1/service/metrics` | GET | Prometheus text metrics |
+| `/metrics` | GET | Prometheus-compatible alias of `/v1/service/metrics` |
| `/v1/service/metrics/json` | GET | JSON service and LiveKit health metrics |
Create a controller session:
@@ -327,6 +328,24 @@ A direct admission returns HTTP 200. A bounded wait returns HTTP 202 with `queue
returns HTTP 429. The one-minute LingBot-World v2 workload and observed four-H100 results are documented in
[TeleFuser and AIPerf](benchmark_aiperf.md).
+
+## Serving observability
+
+`/metrics` is the conventional Prometheus scrape alias for `/v1/service/metrics`.
+For ABot-World it includes bounded worker/GPU, scheduler, batch, queue, pipeline
+stage, SLO, migration, action-to-first-frame, and published-FPS series; it never
+uses a session ID as a Prometheus label. The companion JSON endpoint contains only
+aggregate serving summaries.
+
+For the four-GPU experiment, launch the checked-in [Prometheus, Grafana, DCGM
+Exporter, and Node Exporter stack](../../deploy/observability/README.md). The
+checked-in compose file monitors physical GPUs 0--3 by default; override
+`TELEFUSER_MONITOR_GPU_IDS` to select another physical set. Keep this distinct
+from the serving process's logical `CUDA_VISIBLE_DEVICES` view. If Docker is not
+available, use `tools/validation/capture_abot_serving_metrics.py` to save the
+same serving metrics as an experiment artifact (it does not collect DCGM GPU
+hardware counters).
+
## LiveKit data protocol
| Topic | Direction | Delivery | Current use |
diff --git a/docs/zh/stream_server.md b/docs/zh/stream_server.md
index 6c1ef682..0f1bd46d 100644
--- a/docs/zh/stream_server.md
+++ b/docs/zh/stream_server.md
@@ -280,6 +280,7 @@ stateDiagram-v2
| `/v1/service/ready` | GET | Readiness probe |
| `/v1/service/metadata` | GET | Runtime 拓扑与服务 metadata |
| `/v1/service/metrics` | GET | Prometheus 文本指标 |
+| `/metrics` | GET | `/v1/service/metrics` 的 Prometheus 兼容别名 |
| `/v1/service/metrics/json` | GET | JSON 服务与 LiveKit 健康指标 |
创建 controller session：
@@ -313,6 +314,21 @@ curl -X DELETE http://127.0.0.1:8088/v1/stream/sessions/
一分钟 LingBot-World v2 workload 与四张 H100 的实测结果见
[TeleFuser 与 AIPerf](benchmark_aiperf.md)。
+
+## Serving 可观测性
+
+`/metrics` 是 `/v1/service/metrics` 的标准 Prometheus scrape 别名。对
+ABot-World，它导出有界的 worker/GPU、scheduler、batch、队列、pipeline stage、SLO、
+migration、action-to-first-frame 和 published-FPS 指标；不会把 session ID 作为
+Prometheus label。JSON 端点只返回聚合后的 serving 摘要。
+
+四卡实验可直接启动仓库内的 [Prometheus、Grafana、DCGM Exporter 和 Node
+Exporter 栈](../../deploy/observability/README.md)。其中 compose 监控物理 GPU
+0--3（默认值）；可通过 `TELEFUSER_MONITOR_GPU_IDS` 选择其他物理卡。它与
+serving 进程的逻辑 `CUDA_VISIBLE_DEVICES` 视图不同。若实验机无法运行 Docker，
+可运行 `tools/validation/capture_abot_serving_metrics.py` 保存同一 serving
+指标作为实验 artifact；该方式不包含 DCGM 的 GPU 硬件计数器。
+
## LiveKit 数据协议
| Topic | 方向 | 传输 | 当前用途 |
diff --git a/examples/abot_world/README.md b/examples/abot_world/README.md
index 2234edb2..6585f3a4 100644
--- a/examples/abot_world/README.md
+++ b/examples/abot_world/README.md
@@ -58,6 +58,12 @@ enable process autoscaling. Use plain `--worker-mode process` plus a non-zero qu
Each worker continuously batches compatible retained sessions through both DiT
and cached VAE decode; GPU IDs are passed explicitly to the ABot model factory.
+For the reproducible four-GPU black-box baseline (physical GPUs 4--7, logical
+worker map `'0;1;2;3'`, `process-nccl`, retained capacity 2 per worker, and
+12 FPS / three-latent chunks), use the exact source-tree launch and LiveKit
+arrival/burst/recovery workload in the [ABot serving guide](../../docs/en/abot_world.md#four-gpu-black-box-user-wave-baseline).
+Clients do not choose a GPU; the parent scheduler assigns each public session.
+
Serve the reused browser page in another terminal:
```bash
diff --git a/examples/abot_world/abot_world_livekit_service.py b/examples/abot_world/abot_world_livekit_service.py
index a5a3fc6b..62461d3d 100644
--- a/examples/abot_world/abot_world_livekit_service.py
+++ b/examples/abot_world/abot_world_livekit_service.py
@@ -3,6 +3,8 @@
from __future__ import annotations
import importlib.util
+import math
+import os
from pathlib import Path
from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
@@ -21,6 +23,51 @@
DEFAULT_PROMPT = _LOADER.DEFAULT_PROMPT
get_pipeline = _LOADER.get_pipeline
+_DEFAULT_SCHEDULER_MODE = "batched"
+_DEFAULT_MAX_BATCH_SIZE = 2
+_DEFAULT_BATCHING_WINDOW_MS = 2.0
+_SCHEDULER_MODE_ENV = "TELEFUSER_ABOT_SCHEDULER_MODE"
+_MAX_BATCH_SIZE_ENV = "TELEFUSER_ABOT_MAX_BATCH_SIZE"
+_BATCHING_WINDOW_MS_ENV = "TELEFUSER_ABOT_BATCHING_WINDOW_MS"
+
+
+def _serving_schedule_from_environment() -> tuple[str, int, float]:
+ """Return the worker-local ABot scheduling settings selected by the operator.
+
+ The retained-session admission limit is intentionally configured by
+ ``telefuser stream-serve --max-sessions-per-worker`` instead of here.
+ Keeping these knobs separate makes it possible to run a fixed-capacity
+ all-active trace with four retained sessions and one DiT batch of four on
+ each GPU, without changing the conservative B=2 default profile.
+ """
+ scheduler_mode = os.getenv(_SCHEDULER_MODE_ENV, _DEFAULT_SCHEDULER_MODE).strip().lower()
+ if scheduler_mode not in {"batched", "round_robin"}:
+ raise ValueError(f"{_SCHEDULER_MODE_ENV} must be 'batched' or 'round_robin'")
+
+ raw_max_batch_size = os.getenv(_MAX_BATCH_SIZE_ENV)
+ if raw_max_batch_size is None:
+ max_batch_size = _DEFAULT_MAX_BATCH_SIZE
+ else:
+ try:
+ max_batch_size = int(raw_max_batch_size)
+ except ValueError as exc:
+ raise ValueError(f"{_MAX_BATCH_SIZE_ENV} must be a positive integer") from exc
+ if max_batch_size < 1:
+ raise ValueError(f"{_MAX_BATCH_SIZE_ENV} must be a positive integer")
+
+ raw_batching_window_ms = os.getenv(_BATCHING_WINDOW_MS_ENV)
+ if raw_batching_window_ms is None:
+ batching_window_ms = _DEFAULT_BATCHING_WINDOW_MS
+ else:
+ try:
+ batching_window_ms = float(raw_batching_window_ms)
+ except ValueError as exc:
+ raise ValueError(f"{_BATCHING_WINDOW_MS_ENV} must be a non-negative finite number") from exc
+ if not math.isfinite(batching_window_ms) or batching_window_ms < 0:
+ raise ValueError(f"{_BATCHING_WINDOW_MS_ENV} must be a non-negative finite number")
+
+ return scheduler_mode, max_batch_size, batching_window_ms
+
def get_service(gpu_num: int = 1, gpu_ids: list[str] | None = None) -> ABotWorldLiveKitService:
"""Load one ABot replica on the single GPU assigned to this worker."""
@@ -31,15 +78,22 @@ def get_service(gpu_num: int = 1, gpu_ids: list[str] | None = None) -> ABotWorld
device_id = int(assigned[0])
except ValueError as exc:
raise ValueError(f"ABot worker GPU id must be numeric, got {assigned[0]!r}") from exc
+ scheduler_mode, max_batch_size, batching_window_ms = _serving_schedule_from_environment()
pipeline = get_pipeline(device_id=device_id, pipeline_class=ABotWorldInteractivePipeline)
return ABotWorldLiveKitService(
pipeline,
- default_fps=8,
+ # The default is the previously measured B=2 baseline. The all-active
+ # 4-GPU/16-session trace explicitly selects B=4 through
+ # TELEFUSER_ABOT_MAX_BATCH_SIZE=4 on every model worker.
+ default_fps=12,
default_session_config={
"image_path": str(_DEFAULT_IMAGE_PATH),
"prompt": DEFAULT_PROMPT,
- "fps": 8,
- "control_latent_frames": 2,
+ "fps": 12,
+ "control_latent_frames": 3,
"seed": 42,
},
+ scheduler_mode=scheduler_mode,
+ max_batch_size=max_batch_size,
+ batching_window_ms=batching_window_ms,
)
diff --git a/telefuser/metrics/collector.py b/telefuser/metrics/collector.py
index a14d1b13..e29f152a 100644
--- a/telefuser/metrics/collector.py
+++ b/telefuser/metrics/collector.py
@@ -282,15 +282,15 @@ def to_prometheus(self) -> list[str]:
label_str = self._labels_to_prometheus()
- # Cumulative bucket counts
- cumulative = 0
+ # ``observe`` already increments every matching bucket, so the stored
+ # counts are cumulative. Adding them again here turns a valid
+ # Prometheus histogram into a double-cumulative one.
for bucket in self._buckets:
- cumulative += bucket.count
- le = "inf" if bucket.upper_bound == float("inf") else str(bucket.upper_bound)
+ le = "+Inf" if bucket.upper_bound == float("inf") else str(bucket.upper_bound)
bucket_label = f'{{le="{le}"}}'
if label_str:
bucket_label = "{" + f'le="{le}", ' + label_str[1:]
- lines.append(f"{self.name}_bucket{bucket_label} {cumulative}")
+ lines.append(f"{self.name}_bucket{bucket_label} {bucket.count}")
# Sum and count
lines.append(f"{self.name}_sum{label_str} {self._sum}")
diff --git a/telefuser/pipelines/abot_world/interactive.py b/telefuser/pipelines/abot_world/interactive.py
index 0766058c..28c226f8 100644
--- a/telefuser/pipelines/abot_world/interactive.py
+++ b/telefuser/pipelines/abot_world/interactive.py
@@ -74,6 +74,7 @@ class ABotWorldSessionSnapshot:
cross_cache: tuple[dict[str, Any], ...]
vae_feat_cache: tuple[object, ...]
vae_feat_idx: tuple[int, ...]
+ taew_decode_state: dict[str, Any]
generator_state: torch.Tensor
next_latent_frame: int
emitted_frames: int
@@ -283,12 +284,9 @@ def generate_next_blocks(
decode_started_at = time.monotonic()
if any(session.taew_decode_state is None for session in sessions):
raise RuntimeError("ABot session is missing its TAeW2.2 decode state")
- decoded = torch.cat(
- [
- self.taew_decode_stage.decode_chunk(latents[index : index + 1], session.taew_decode_state)
- for index, session in enumerate(sessions)
- ],
- dim=0,
+ decoded = self.taew_decode_stage.decode_chunks(
+ latents,
+ [session.taew_decode_state for session in sessions],
)
if use_cuda_events:
vae_finished.record()
@@ -311,6 +309,7 @@ def generate_next_blocks(
"denoise_seconds": denoise_seconds,
"cache_scatter_seconds": cache_scatter_seconds,
"vae_decode_seconds": decode_seconds,
+ **self.taew_decode_stage.last_decode_metrics(),
"postprocess_seconds": time.monotonic() - postprocess_started_at,
"total_seconds": time.monotonic() - batch_started_at,
}
@@ -380,6 +379,7 @@ def snapshot_interactive_session(
for value in session.vae_decode_state.feat_cache
),
vae_feat_idx=tuple(session.vae_decode_state.feat_idx),
+ taew_decode_state=self._snapshot_taew_decode_state(session),
generator_state=session.generator.get_state().to("cpu").clone(),
next_latent_frame=session.next_latent_frame,
emitted_frames=session.emitted_frames,
@@ -441,7 +441,7 @@ def _restore_snapshot(
if isinstance(value, torch.Tensor)
)
tensors.extend(value for value in snapshot.vae_feat_cache if isinstance(value, torch.Tensor))
- if any(tensor.device != expected_device for tensor in tensors):
+ if any(not self._matches_pipeline_device(tensor.device, expected_device) for tensor in tensors):
raise ValueError("NCCL migration tensors must already reside on the target pipeline device")
prompt_emb = snapshot.prompt_emb
first_frame_latent = snapshot.first_frame_latent
@@ -457,6 +457,10 @@ def _restore_snapshot(
value.to(self.device).clone() if isinstance(value, torch.Tensor) else value
for value in snapshot.vae_feat_cache
]
+ taew_decode_state = self.taew_decode_stage.restore_decode_state(
+ snapshot.taew_decode_state,
+ direct_device_tensors=direct_device_tensors,
+ )
session = ABotWorldInteractiveSession(
session_id=snapshot.session_id,
prompt_emb=prompt_emb,
@@ -469,6 +473,7 @@ def _restore_snapshot(
feat_cache=vae_feat_cache,
feat_idx=list(snapshot.vae_feat_idx),
),
+ taew_decode_state=taew_decode_state,
next_latent_frame=snapshot.next_latent_frame,
emitted_frames=snapshot.emitted_frames,
owner_worker_id=owner_worker_id,
@@ -480,6 +485,24 @@ def _restore_snapshot(
self._interactive_sessions[session.session_id] = session
return session
+ def _snapshot_taew_decode_state(self, session: ABotWorldInteractiveSession) -> dict[str, Any]:
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW2.2 decode state")
+ return self.taew_decode_stage.snapshot_decode_state(session.taew_decode_state)
+
+ def _move_taew_decode_state(
+ self,
+ session: ABotWorldInteractiveSession,
+ device: str | torch.device,
+ ) -> None:
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW2.2 decode state")
+ self.taew_decode_stage.move_decode_state(session.taew_decode_state, device)
+
+ @staticmethod
+ def _matches_pipeline_device(actual: torch.device, expected: torch.device) -> bool:
+ return actual.type == expected.type and (expected.index is None or actual.index == expected.index)
+
@staticmethod
def _clone_cache_to_cpu(caches: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
return ABotWorldInteractivePipeline._clone_cache_to_device(caches, "cpu")
@@ -516,6 +539,7 @@ def suspend_interactive_session(self, session: ABotWorldInteractiveSession) -> N
value.to("cpu") if isinstance(value, torch.Tensor) else value
for value in session.vae_decode_state.feat_cache
]
+ self._move_taew_decode_state(session, "cpu")
session.lifecycle = ABotWorldSessionLifecycle.SUSPENDED
def restore_interactive_session(self, session: ABotWorldInteractiveSession) -> None:
@@ -532,6 +556,7 @@ def restore_interactive_session(self, session: ABotWorldInteractiveSession) -> N
value.to(self.device) if isinstance(value, torch.Tensor) else value
for value in session.vae_decode_state.feat_cache
]
+ self._move_taew_decode_state(session, self.device)
session.lifecycle = ABotWorldSessionLifecycle.READY
@staticmethod
@@ -559,6 +584,9 @@ def close_interactive_session(self, session: ABotWorldInteractiveSession | None
target.cross_cache.clear()
target.vae_decode_state.feat_cache.clear()
target.vae_decode_state.feat_idx = [0]
+ if target.taew_decode_state is not None:
+ self.taew_decode_stage.clear_decode_state(target.taew_decode_state)
+ target.taew_decode_state = None
target.lifecycle = ABotWorldSessionLifecycle.CLOSED
self._interactive_sessions.pop(target.session_id, None)
diff --git a/telefuser/pipelines/abot_world/service.py b/telefuser/pipelines/abot_world/service.py
index acd7c08a..bdf7556d 100644
--- a/telefuser/pipelines/abot_world/service.py
+++ b/telefuser/pipelines/abot_world/service.py
@@ -64,6 +64,9 @@
_DEFAULT_OUTPUT_QUEUE_SIZE = 4
_VIDEO_OUTPUT_TYPES = frozenset({"preview", "chunk"})
_TERMINAL_OUTPUT_TYPES = frozenset({"error", "done"})
+_PACING_SAFETY_FACTOR = 1.10
+_PACING_MAX_COALESCING_SECONDS = 0.010
+_PACING_RENDEZVOUS_WAKE_GUARD_SECONDS = 0.001
@dataclass
@@ -91,6 +94,9 @@ class _ABotWorldLiveKitSession:
batch_items: int = 0
total_queue_wait_seconds: float = 0.0
total_compute_seconds: float = 0.0
+ last_compute_seconds: float = 0.0
+ last_chunk_duration_seconds: float = 0.0
+ pacing_ready_at: float = field(default_factory=time.monotonic)
last_error: str | None = None
migrating: bool = False
@@ -109,13 +115,13 @@ class ABotWorldMigrationBundle:
class ABotWorldLiveKitService:
- """One-GPU retained-session owner with TurboServe-style round-robin stepping.
+ """One-GPU retained-session owner with coalesced causal-block scheduling.
- ``round_robin`` is the default and mirrors TurboServe: every scheduler turn
- selects exactly one runnable session and advances it by one causal block.
- Session KV/VAE state stays resident and is never collated across sessions.
- ``batched`` retains the former experimental path for research comparisons;
- it is deliberately opt-in because it is not the TurboServe execution model.
+ By default, compatible ready sessions are coalesced into one DiT invocation
+ and their independent KV/RNG state is scattered back afterwards. This is
+ the execution model described by TurboServe's paper and is the production
+ ABot serving baseline. ``round_robin`` remains available as an ablation: it
+ advances exactly one runnable session per scheduler turn.
"""
def __init__(
@@ -130,7 +136,7 @@ def __init__(
max_batch_size: int = 8,
batching_window_ms: float = 2.0,
idle_suspension_seconds: float = 5.0,
- scheduler_mode: str = "round_robin",
+ scheduler_mode: str = "batched",
) -> None:
if default_fps < 1:
raise ValueError(f"default_fps must be positive, got {default_fps}")
@@ -166,6 +172,11 @@ def __init__(
self._batch_item_count = 0
self._maximum_batch_size = 0
self._last_stage_metrics: dict[str, float | int] = {}
+ self._pacing_eligible_sessions = 0
+ self._pacing_throttled_sessions = 0
+ self._pacing_buffered_sessions = 0
+ # Observed wall-clock runtimes make deadline rendezvous conservative.
+ self._batch_compute_estimates: dict[int, float] = {}
self._workload_detector = TurboServeWorkloadDetector()
def start(self) -> None:
@@ -383,6 +394,7 @@ def push_chunk(self, session_id: str, chunk: dict) -> None:
with self._scheduler_condition, state.lock:
if not state.active:
return
+ was_controlled = bool(state.controls)
if message_type == "stop":
state.active = False
state.controls.clear()
@@ -410,6 +422,11 @@ def push_chunk(self, session_id: str, chunk: dict) -> None:
now = time.monotonic()
state.last_control_at = now
state.ready_since = now if state.controls else None
+ # A newly reactivated session should not wait behind an old playout
+ # prediction. Existing active sessions keep their pacing deadline so
+ # frequent control updates cannot force the scheduler to free-run.
+ if state.controls and not was_controlled:
+ state.pacing_ready_at = now
state.control_event.set()
if state.controls:
self._workload_detector.record_active(session_id, now)
@@ -481,12 +498,17 @@ def prepare_migration_nccl_metadata(self, session_id: str, timeout: float | None
"""Quiesce a session and describe its resident tensors for direct NCCL transfer."""
state = self._quiesce_migration(session_id, timeout)
session = state.pipeline_session
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW2.2 decode state")
payload = {
"prompt_emb": session.prompt_emb,
"first_frame_latent": session.first_frame_latent,
"self_cache": session.self_cache,
"cross_cache": session.cross_cache,
"vae_feat_cache": session.vae_decode_state.feat_cache,
+ "taew_decode_state": self.pipeline.taew_decode_stage.export_decode_state_for_nccl(
+ session.taew_decode_state
+ ),
}
skeleton, manifest, leaves = flatten_tensor_tree(payload)
return {
@@ -526,6 +548,7 @@ def import_migration_nccl(
cross_cache=tuple(payload["cross_cache"]),
vae_feat_cache=tuple(payload["vae_feat_cache"]),
vae_feat_idx=tuple(int(value) for value in metadata["vae_feat_idx"]),
+ taew_decode_state=dict(payload["taew_decode_state"]),
generator_state=metadata["generator_state"],
next_latent_frame=int(metadata["next_latent_frame"]),
emitted_frames=int(metadata["emitted_frames"]),
@@ -688,6 +711,9 @@ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | in
"batches": self._batch_count,
"batch_items": self._batch_item_count,
"maximum_batch_size": self._maximum_batch_size,
+ "pacing_eligible_sessions": self._pacing_eligible_sessions,
+ "pacing_throttled_sessions": self._pacing_throttled_sessions,
+ "pacing_buffered_sessions": self._pacing_buffered_sessions,
"active_sessions": workload.active_sessions,
"arrivals_per_second": round(workload.arrivals_per_second, 6),
"activation_volatility": round(workload.activation_volatility, 6),
@@ -710,6 +736,8 @@ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | in
"emitted_frames": int(getattr(state.pipeline_session, "emitted_frames", 0)),
"total_queue_wait_seconds": round(state.total_queue_wait_seconds, 6),
"total_compute_seconds": round(state.total_compute_seconds, 6),
+ "pacing_ready_in_seconds": round(max(0.0, state.pacing_ready_at - time.monotonic()), 6),
+ "pacing_buffered_video_payloads": self._queued_video_payloads(state),
}
def _ensure_scheduler_started(self) -> None:
@@ -751,18 +779,15 @@ def _scheduler_loop(self) -> None:
break
ready = self._ready_sessions(now)
if not ready and suspend_candidate is None:
- self._scheduler_condition.wait(timeout=0.05)
+ self._scheduler_condition.wait(timeout=self._next_scheduler_wake_seconds(now))
continue
- if (
- self.scheduler_mode == "batched"
- and ready
- and len(ready) < self.max_batch_size
- and self.batching_window_seconds
- ):
- self._scheduler_condition.wait(timeout=self.batching_window_seconds)
+ if self.scheduler_mode == "batched" and ready and len(ready) < self.max_batch_size:
+ wait_seconds = self._batch_formation_wait_seconds(ready, now)
+ if wait_seconds > 0:
+ self._scheduler_condition.wait(timeout=wait_seconds)
now = time.monotonic()
ready = self._ready_sessions(now)
- batch = self._select_batch(ready)
+ batch = self._select_batch(ready, now=now)
controls: list[dict[str, bool]] = []
if batch:
for state in batch:
@@ -781,6 +806,9 @@ def _scheduler_loop(self) -> None:
def _ready_sessions(self, now: float) -> list[_ABotWorldLiveKitSession]:
ready: list[_ABotWorldLiveKitSession] = []
+ pacing_eligible = 0
+ pacing_throttled = 0
+ pacing_buffered = 0
for state in self._sessions.values():
with state.lock:
lossless_blocked = (
@@ -795,20 +823,176 @@ def _ready_sessions(self, now: float) -> list[_ABotWorldLiveKitSession]:
):
if state.ready_since is None:
state.ready_since = now
+ if state.config["delivery_mode"] == "latest":
+ buffered_video_payloads = self._queued_video_payloads(state)
+ pacing_ready = now + self._pacing_coalescing_slack_seconds(state) >= state.pacing_ready_at
+ if buffered_video_payloads or not pacing_ready:
+ pacing_throttled += 1
+ pacing_buffered += int(bool(buffered_video_payloads))
+ continue
+ pacing_eligible += 1
ready.append(state)
+ self._pacing_eligible_sessions = pacing_eligible
+ self._pacing_throttled_sessions = pacing_throttled
+ self._pacing_buffered_sessions = pacing_buffered
ready.sort(key=lambda state: (state.next_playout_deadline, state.ready_since or now, state.session_id))
return ready
+ def _next_scheduler_wake_seconds(self, now: float) -> float:
+ """Wake promptly for a pacing deadline while retaining event-driven idling."""
+ next_wake_at: float | None = None
+ for state in self._sessions.values():
+ with state.lock:
+ if state.active and state.controls:
+ control_expiry = state.last_control_at + state.control_idle_timeout
+ next_wake_at = control_expiry if next_wake_at is None else min(next_wake_at, control_expiry)
+ if (
+ state.config["delivery_mode"] == "latest"
+ and not state.in_flight
+ and not state.migrating
+ and not self._queued_video_payloads(state)
+ ):
+ pacing_wake = state.pacing_ready_at - self._pacing_coalescing_slack_seconds(state)
+ next_wake_at = pacing_wake if next_wake_at is None else min(next_wake_at, pacing_wake)
+ elif not state.in_flight and state.pipeline_session.is_resident and not state.controls:
+ suspension_at = state.last_control_at + self.idle_suspension_seconds
+ next_wake_at = suspension_at if next_wake_at is None else min(next_wake_at, suspension_at)
+ if next_wake_at is None:
+ return 0.05
+ return max(0.001, next_wake_at - now)
+
+ def _pacing_coalescing_slack_seconds(self, state: _ABotWorldLiveKitSession) -> float:
+ """Allow a small, bounded early window so near-aligned sessions still batch."""
+ if state.last_chunk_duration_seconds <= 0:
+ return self.batching_window_seconds
+ return max(
+ self.batching_window_seconds,
+ min(_PACING_MAX_COALESCING_SECONDS, state.last_chunk_duration_seconds * 0.05),
+ )
+
+ def _batch_formation_wait_seconds(
+ self,
+ ready: Sequence[_ABotWorldLiveKitSession],
+ now: float,
+ ) -> float:
+ """Wait for a compatible continuation only while all playout deadlines are safe."""
+ batch = self._select_batch(ready, now=now)
+ if not batch or len(batch) >= self.max_batch_size:
+ return 0.0
+
+ # First generated chunks stay latency-critical. They may coalesce when
+ # peers are ready in the same turn, but never wait for a future peer.
+ if any(state.scheduled_chunks == 0 for state in batch):
+ return 0.0
+ # Lossless queues are governed by consumer backpressure rather than a
+ # playout deadline, so preserve their original micro-batching behavior.
+ if any(state.config["delivery_mode"] != "latest" for state in batch):
+ return self.batching_window_seconds
+ legacy_wait = min(
+ self.batching_window_seconds,
+ max(0.0, self._latest_safe_batch_start(batch) - now),
+ )
+
+ pivot_key = self._batch_key(batch[0])
+ selected_ids = {state.session_id for state in batch}
+ rendezvous_waits: list[float] = []
+ for candidate in self._sessions.values():
+ if candidate.session_id in selected_ids:
+ continue
+ with candidate.lock:
+ if (
+ not candidate.active
+ or not candidate.controls
+ or candidate.in_flight
+ or candidate.migrating
+ or candidate.config["delivery_mode"] != "latest"
+ or candidate.scheduled_chunks == 0
+ or self._batch_key(candidate) != pivot_key
+ # A generated chunk owned by the publisher has an external
+ # dequeue time, so it cannot be a rendezvous promise.
+ or self._queued_video_payloads(candidate)
+ ):
+ continue
+
+ release_at = candidate.pacing_ready_at - self._pacing_coalescing_slack_seconds(candidate)
+ if release_at <= now:
+ # An already-eligible session should be in ready. Avoid
+ # turning a state race into an extra scheduler delay.
+ continue
+ proposed_batch = [*batch, candidate]
+ latest_safe_start = self._latest_safe_batch_start(proposed_batch)
+ if release_at + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS <= latest_safe_start:
+ rendezvous_waits.append(release_at - now + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS)
+
+ if rendezvous_waits:
+ # Earliest compatible release minimizes queueing; the condition
+ # wait is followed by a full readiness/deadline revalidation.
+ return min(rendezvous_waits)
+ return legacy_wait
+
+ def _latest_safe_batch_start(self, batch: Sequence[_ABotWorldLiveKitSession]) -> float:
+ """Return the latest launch time that still meets every playout deadline."""
+ if not batch:
+ return float("-inf")
+ predicted_compute_seconds = self._estimated_batch_compute_seconds(batch)
+ return min(state.next_playout_deadline for state in batch) - predicted_compute_seconds
+
+ def _estimated_batch_compute_seconds(self, batch: Sequence[_ABotWorldLiveKitSession]) -> float:
+ """Conservatively estimate batch wall time from observed service work."""
+ batch_size = len(batch)
+ observed = self._batch_compute_estimates.get(batch_size)
+ if observed is None:
+ singleton_seconds = max((state.last_compute_seconds for state in batch), default=0.0)
+ observed = singleton_seconds * batch_size
+ return observed * _PACING_SAFETY_FACTOR
+
+ @staticmethod
+ def _queued_video_payloads(state: _ABotWorldLiveKitSession) -> int:
+ """Return generated chunks not yet taken by the real-time publisher.
+
+ ``latest`` output intentionally evicts stale payloads, so Queue.full()
+ cannot express a prefetch bound. The preview is not generated video and
+ must not delay the latency-critical first chunk. Keeping at most one
+ queued chunk makes it the only chunk ahead of the one being played.
+ """
+ with state.output_queue.mutex:
+ return sum(item.get("type") == "chunk" for item in state.output_queue.queue)
+
def _select_batch(
self,
ready: Sequence[_ABotWorldLiveKitSession],
+ *,
+ now: float | None = None,
) -> list[_ABotWorldLiveKitSession]:
if not ready:
return []
if self.scheduler_mode == "round_robin":
return self._select_round_robin_session(ready)
pivot_key = self._batch_key(ready[0])
- return [state for state in ready if self._batch_key(state) == pivot_key][: self.max_batch_size]
+ batch = [state for state in ready if self._batch_key(state) == pivot_key][: self.max_batch_size]
+ if len(batch) <= 1:
+ return batch
+
+ # First chunks are latency-critical but intentionally retain their
+ # same-turn coalescing behavior. Lossless sessions are paced by their
+ # consumer queues rather than a playout deadline. Only a batch made
+ # entirely of already-playing latest-mode sessions has a deadline that
+ # makes a larger batch potentially worse than two singleton turns.
+ if any(
+ state.scheduled_chunks == 0 or state.config["delivery_mode"] != "latest"
+ for state in batch
+ ):
+ return batch
+
+ selected_at = time.monotonic() if now is None else now
+ if selected_at <= self._latest_safe_batch_start(batch):
+ return batch
+
+ # The earliest state owns the earliest playout deadline because ready
+ # is deadline-sorted. Do not let an observed slow B>1 execution turn
+ # an otherwise feasible pair of B=1 continuations into an avoidable
+ # playout miss.
+ return [batch[0]]
def _select_round_robin_session(
self,
@@ -890,7 +1074,15 @@ def _execute_batch(
completed_at = time.monotonic()
stage_metrics_callback = getattr(self.pipeline, "last_stage_metrics", None)
self._last_stage_metrics = dict(stage_metrics_callback()) if callable(stage_metrics_callback) else {}
- self._workload_detector.record_chunk(completed_at - started_at, completed_at)
+ observed_compute_seconds = completed_at - started_at
+ previous_estimate = self._batch_compute_estimates.get(len(batch), 0.0)
+ # Keep a service-run high-water mark so a transient fast batch
+ # cannot make a later rendezvous overrun a playout deadline.
+ self._batch_compute_estimates[len(batch)] = max(
+ observed_compute_seconds,
+ previous_estimate,
+ )
+ self._workload_detector.record_chunk(observed_compute_seconds, completed_at)
self._batch_count += 1
self._batch_item_count += len(batch)
self._maximum_batch_size = max(self._maximum_batch_size, len(batch))
@@ -915,10 +1107,31 @@ def _execute_batch(
**self._last_stage_metrics,
},
}
+ chunk_duration_seconds = max(1, len(frames)) / int(state.config["fps"])
+ previous_compute_seconds = state.last_compute_seconds
state.next_chunk_index += 1
- state.next_playout_deadline = max(state.next_playout_deadline, completed_at) + len(frames) / int(
- state.config["fps"]
+ state.next_playout_deadline = (
+ max(state.next_playout_deadline, completed_at) + chunk_duration_seconds
)
+ state.last_chunk_duration_seconds = chunk_duration_seconds
+ state.last_compute_seconds = observed_compute_seconds
+ if state.config["delivery_mode"] == "latest":
+ if state.scheduled_chunks <= 1:
+ # The first chunk is latency-critical; immediately allow one
+ # successor once the publisher has taken this chunk.
+ state.pacing_ready_at = completed_at
+ else:
+ # Start the next block early enough to meet the start of the
+ # sole prefetched block, but never run before this block ends.
+ predicted_compute_seconds = max(
+ observed_compute_seconds, previous_compute_seconds
+ ) * _PACING_SAFETY_FACTOR
+ next_chunk_playout_start = state.next_playout_deadline - chunk_duration_seconds
+ state.pacing_ready_at = max(
+ completed_at, next_chunk_playout_start - predicted_compute_seconds
+ )
+ else:
+ state.pacing_ready_at = completed_at
state.ready_since = completed_at if state.controls else None
self._put_output(state, payload)
finally:
diff --git a/telefuser/pipelines/abot_world/taew_vae.py b/telefuser/pipelines/abot_world/taew_vae.py
index 78b9a8b9..36fb651b 100644
--- a/telefuser/pipelines/abot_world/taew_vae.py
+++ b/telefuser/pipelines/abot_world/taew_vae.py
@@ -2,14 +2,24 @@
from __future__ import annotations
+from collections.abc import Mapping, Sequence
from dataclasses import dataclass
+from typing import Any
import torch
from telefuser.core.base_stage import BaseStage, with_model_offload
from telefuser.core.config import ModelRuntimeConfig
from telefuser.core.module_manager import ModuleManager
-from telefuser.models.taew2_2 import StreamingTAEHV, TAEHV
+from telefuser.models.taew2_2 import TAEHV, StreamingTAEHV, TWorkItem
+
+# This is deliberately a numeric enum: stage metrics are forwarded through
+# process boundaries and exported as low-cardinality Prometheus facts. Keep it
+# separate from the DiT scheduler batch size, which can be greater than one
+# even when causal TAeW state requires serial decode calls.
+_TAEW_DECODE_SINGLETON = 0
+_TAEW_DECODE_SYNCHRONIZED_BATCH = 1
+_TAEW_DECODE_SERIAL_FALLBACK = 2
@dataclass
@@ -29,11 +39,85 @@ def __init__(self, name: str, module_manager: ModuleManager, model_runtime_confi
raise ValueError("ABot-World requires a loaded abot_world_taew_decoder module")
self.taew = taew
self.model_names = ["taew"]
+ self._last_decode_metrics: dict[str, int] = self._empty_decode_metrics()
+
+ @staticmethod
+ def _empty_decode_metrics() -> dict[str, int]:
+ return {
+ "taew_decode_items": 0,
+ "taew_decode_batch_size": 0,
+ "taew_decode_invocations": 0,
+ "taew_decode_mode": _TAEW_DECODE_SINGLETON,
+ }
+
+ def last_decode_metrics(self) -> dict[str, int]:
+ """Return facts for the most recently successful TAeW decode.
+
+ ``taew_decode_mode`` is numeric so it remains low-cardinality across
+ worker IPC and Prometheus export: ``0`` = singleton, ``1`` = one
+ synchronized native LightVAE batch, and ``2`` = safe serial fallback.
+ ``taew_decode_batch_size`` is the *effective native decoder* batch
+ size, not the enclosing DiT scheduler batch size.
+ """
+ return dict(getattr(self, "_last_decode_metrics", self._empty_decode_metrics()))
def create_decode_state(self) -> ABotWorldTAEWDecodeState:
"""Create an isolated stream state while sharing immutable decoder weights."""
return ABotWorldTAEWDecodeState(stream=StreamingTAEHV(self.taew))
+ def snapshot_decode_state(self, state: ABotWorldTAEWDecodeState) -> dict[str, Any]:
+ """Clone decode-only streaming state to a CPU-owned tensor tree.
+
+ ABot only calls :meth:`StreamingTAEHV.decode`, so its continuation is
+ fully determined by the decoder queue, temporal decoder memory, and
+ decoded-frame counter. Encoder-side state intentionally does not belong
+ to an ABot session snapshot.
+ """
+ return self._decode_state_tensor_tree(state, device="cpu", clone_tensors=True)
+
+ def export_decode_state_for_nccl(self, state: ABotWorldTAEWDecodeState) -> dict[str, Any]:
+ """Return a tensor-tree payload whose leaves remain on the source GPU.
+
+ The returned structure contains only scalars, mappings, sequences, and
+ tensors, so ``flatten_tensor_tree`` can describe it for direct NCCL
+ transfer without materializing a CPU copy.
+ """
+ return self._decode_state_tensor_tree(state, device=None, clone_tensors=False)
+
+ def restore_decode_state(
+ self,
+ snapshot: Mapping[str, Any],
+ *,
+ direct_device_tensors: bool = False,
+ ) -> ABotWorldTAEWDecodeState:
+ """Restore a decoder stream from a CPU or direct-NCCL tensor tree."""
+ target_device = None if direct_device_tensors else self.device
+ tree = self._normalise_decode_state_tree(
+ snapshot,
+ device=target_device,
+ clone_tensors=not direct_device_tensors,
+ )
+ if direct_device_tensors:
+ expected_device = torch.device(self.device)
+ if any(
+ not self._matches_device(tensor.device, expected_device)
+ for tensor in self._iter_tensors(tree)
+ ):
+ raise ValueError("NCCL TAeW migration tensors must already reside on the target decoder device")
+ state = self.create_decode_state()
+ self._apply_decode_state_tensor_tree(state, tree)
+ return state
+
+ def move_decode_state(self, state: ABotWorldTAEWDecodeState, device: torch.device | str) -> None:
+ """Move every session-owned decoder tensor while retaining causal state."""
+ tree = self._decode_state_tensor_tree(state, device=device, clone_tensors=False)
+ self._apply_decode_state_tensor_tree(state, tree)
+
+ @staticmethod
+ def clear_decode_state(state: ABotWorldTAEWDecodeState) -> None:
+ """Release queued tensors and temporal memory for a closed session."""
+ state.stream.reset()
+
@with_model_offload(["taew"])
@torch.inference_mode()
def warmup_first_frame(self, state: ABotWorldTAEWDecodeState, first_frame_latent: torch.Tensor) -> None:
@@ -42,11 +126,337 @@ def warmup_first_frame(self, state: ABotWorldTAEWDecodeState, first_frame_latent
latent = first_frame_latent.permute(0, 2, 1, 3, 4).to(self.device, dtype=self.torch_dtype)
state.stream.decode(latent)
- @with_model_offload(["taew"])
- @torch.inference_mode()
def decode_chunk(self, latents: torch.Tensor, state: ABotWorldTAEWDecodeState) -> torch.Tensor:
"""Decode one causal latent chunk to RGB frames in [-1, 1]."""
- decoded = state.stream.decode(latents.permute(0, 2, 1, 3, 4).to(self.device, dtype=self.torch_dtype))
+ return self.decode_chunks(latents, [state])
+
+ @with_model_offload(["taew"])
+ @torch.inference_mode()
+ def decode_chunks(
+ self,
+ latents: torch.Tensor,
+ states: Sequence[ABotWorldTAEWDecodeState],
+ ) -> torch.Tensor:
+ """Decode synchronized session chunks in one LightVAE batch when safe.
+
+ State is merged only when its causal queues and temporal-memory layout
+ are identical. Incompatible states retain correct per-session decoding
+ instead of forcing an invalid batch.
+ """
+ return self._decode_chunks_impl(latents, states)
+
+ def _decode_chunks_impl(
+ self,
+ latents: torch.Tensor,
+ states: Sequence[ABotWorldTAEWDecodeState],
+ ) -> torch.Tensor:
+ if latents.ndim != 5:
+ raise ValueError(f"TAeW decode expects BCTHW latents, got shape {tuple(latents.shape)}")
+ if not states or latents.shape[0] != len(states):
+ raise ValueError("TAeW decode states must be non-empty and match the latent batch size")
+ decoder_latents = latents.permute(0, 2, 1, 3, 4).to(self.device, dtype=self.torch_dtype)
+ item_count = len(states)
+ if item_count == 1:
+ decoded = states[0].stream.decode(decoder_latents)
+ decode_metrics = {
+ "taew_decode_items": item_count,
+ "taew_decode_batch_size": 1,
+ "taew_decode_invocations": 1,
+ "taew_decode_mode": _TAEW_DECODE_SINGLETON,
+ }
+ elif self._states_are_batch_compatible(states):
+ decoded = self._decode_synchronized_batch(decoder_latents, states)
+ decode_metrics = {
+ "taew_decode_items": item_count,
+ "taew_decode_batch_size": item_count,
+ "taew_decode_invocations": 1,
+ "taew_decode_mode": _TAEW_DECODE_SYNCHRONIZED_BATCH,
+ }
+ else:
+ decoded = self._decode_serial_batch(decoder_latents, states)
+ decode_metrics = {
+ "taew_decode_items": item_count,
+ "taew_decode_batch_size": 1,
+ "taew_decode_invocations": item_count,
+ "taew_decode_mode": _TAEW_DECODE_SERIAL_FALLBACK,
+ }
+ # Only commit the telemetry after the native calls have completed. A
+ # failed decode must not be misreported as a successful batch/fallback.
+ self._last_decode_metrics = decode_metrics
if decoded is None:
return latents.new_empty((latents.shape[0], 0, 3, 0, 0))
return decoded.mul(2).sub(1).clamp(-1, 1).permute(0, 2, 1, 3, 4).contiguous()
+
+ def _decode_synchronized_batch(
+ self,
+ decoder_latents: torch.Tensor,
+ states: Sequence[ABotWorldTAEWDecodeState],
+ ) -> torch.Tensor | None:
+ combined = self._combine_decode_states(states)
+ decoded = combined.stream.decode(decoder_latents)
+ self._scatter_decode_state(combined, states)
+ return decoded
+
+ @staticmethod
+ def _decode_serial_batch(
+ decoder_latents: torch.Tensor,
+ states: Sequence[ABotWorldTAEWDecodeState],
+ ) -> torch.Tensor | None:
+ decoded_parts = [
+ state.stream.decode(decoder_latents[index : index + 1])
+ for index, state in enumerate(states)
+ ]
+ if all(item is None for item in decoded_parts):
+ return None
+ if any(item is None for item in decoded_parts):
+ raise RuntimeError("TAeW decoder states produced incompatible output frame counts")
+ return torch.cat([item for item in decoded_parts if item is not None], dim=0)
+
+ @classmethod
+ def _states_are_batch_compatible(cls, states: Sequence[ABotWorldTAEWDecodeState]) -> bool:
+ reference = cls._decode_state_batch_signature(states[0])
+ return all(cls._decode_state_batch_signature(state) == reference for state in states[1:])
+
+ @classmethod
+ def _decode_state_batch_signature(cls, state: ABotWorldTAEWDecodeState) -> tuple[Any, ...]:
+ stream = state.stream
+ return (
+ int(stream.n_frames_decoded),
+ tuple(
+ (int(item.block_index), cls._state_value_batch_signature(item.input_tensor))
+ for item in stream.decoder_work_queue
+ ),
+ cls._state_value_batch_signature(stream.decoder_memory),
+ )
+
+ @classmethod
+ def _state_value_batch_signature(cls, value: Any) -> Any:
+ if isinstance(value, torch.Tensor):
+ if value.ndim < 1 or value.shape[0] != 1:
+ return ("invalid-tensor", tuple(value.shape), str(value.dtype), str(value.device))
+ return ("tensor", tuple(value.shape[1:]), str(value.dtype), str(value.device))
+ if value is None:
+ return ("none",)
+ if isinstance(value, list):
+ return ("list", tuple(cls._state_value_batch_signature(item) for item in value))
+ if isinstance(value, tuple):
+ return ("tuple", tuple(cls._state_value_batch_signature(item) for item in value))
+ return ("value", value)
+
+ def _combine_decode_states(self, states: Sequence[ABotWorldTAEWDecodeState]) -> ABotWorldTAEWDecodeState:
+ combined = self.create_decode_state()
+ reference = states[0].stream
+ combined.stream.decoder_work_queue = [
+ TWorkItem(
+ torch.cat([state.stream.decoder_work_queue[index].input_tensor for state in states], dim=0),
+ int(item.block_index),
+ )
+ for index, item in enumerate(reference.decoder_work_queue)
+ ]
+ combined.stream.decoder_memory = self._collate_state_values(
+ [state.stream.decoder_memory for state in states]
+ )
+ combined.stream.n_frames_decoded = int(reference.n_frames_decoded)
+ return combined
+
+ @classmethod
+ def _collate_state_values(cls, values: Sequence[Any]) -> Any:
+ first = values[0]
+ if isinstance(first, torch.Tensor):
+ return torch.cat(list(values), dim=0)
+ if first is None:
+ return None
+ if isinstance(first, list):
+ return [
+ cls._collate_state_values([value[index] for value in values])
+ for index in range(len(first))
+ ]
+ if isinstance(first, tuple):
+ return tuple(
+ cls._collate_state_values([value[index] for value in values])
+ for index in range(len(first))
+ )
+ if all(value == first for value in values[1:]):
+ return first
+ raise ValueError("TAeW decoder states cannot be collated")
+
+ @classmethod
+ def _scatter_decode_state(
+ cls,
+ combined: ABotWorldTAEWDecodeState,
+ states: Sequence[ABotWorldTAEWDecodeState],
+ ) -> None:
+ batch_size = len(states)
+ stream = combined.stream
+ queue_items = [
+ [
+ TWorkItem(
+ item.input_tensor[index : index + 1].detach().clone(),
+ int(item.block_index),
+ )
+ for item in stream.decoder_work_queue
+ ]
+ for index in range(batch_size)
+ ]
+ memory_items = cls._split_state_value(stream.decoder_memory, batch_size)
+ for index, state in enumerate(states):
+ state.stream.decoder_work_queue = queue_items[index]
+ state.stream.decoder_memory = memory_items[index]
+ state.stream.n_frames_decoded = int(stream.n_frames_decoded)
+
+ @classmethod
+ def _split_state_value(cls, value: Any, batch_size: int) -> list[Any]:
+ if isinstance(value, torch.Tensor):
+ if value.ndim < 1 or value.shape[0] != batch_size:
+ raise ValueError("Batched TAeW decoder tensor has an invalid leading batch dimension")
+ return [value[index : index + 1].detach().clone() for index in range(batch_size)]
+ if value is None:
+ return [None] * batch_size
+ if isinstance(value, list):
+ children = [cls._split_state_value(item, batch_size) for item in value]
+ return [[child[index] for child in children] for index in range(batch_size)]
+ if isinstance(value, tuple):
+ children = [cls._split_state_value(item, batch_size) for item in value]
+ return [tuple(child[index] for child in children) for index in range(batch_size)]
+ return [value] * batch_size
+
+ def _decode_state_tensor_tree(
+ self,
+ state: ABotWorldTAEWDecodeState,
+ *,
+ device: torch.device | str | None,
+ clone_tensors: bool,
+ ) -> dict[str, Any]:
+ stream = state.stream
+ return {
+ "decoder_work_queue": [
+ {
+ "input_tensor": self._copy_tensor_tree(
+ item.input_tensor,
+ device=device,
+ clone_tensors=clone_tensors,
+ ),
+ "block_index": int(item.block_index),
+ }
+ for item in stream.decoder_work_queue
+ ],
+ "decoder_memory": self._copy_tensor_tree(
+ stream.decoder_memory,
+ device=device,
+ clone_tensors=clone_tensors,
+ ),
+ "n_frames_decoded": int(stream.n_frames_decoded),
+ }
+
+ def _normalise_decode_state_tree(
+ self,
+ snapshot: Mapping[str, Any],
+ *,
+ device: torch.device | str | None,
+ clone_tensors: bool,
+ ) -> dict[str, Any]:
+ required = {"decoder_work_queue", "decoder_memory", "n_frames_decoded"}
+ missing = required.difference(snapshot)
+ if missing:
+ raise ValueError(f"TAeW decode-state snapshot is missing fields: {sorted(missing)}")
+ work_queue = snapshot["decoder_work_queue"]
+ if not isinstance(work_queue, (list, tuple)):
+ raise TypeError("TAeW decoder_work_queue snapshot must be a sequence")
+ restored_queue: list[dict[str, Any]] = []
+ for item in work_queue:
+ if not isinstance(item, Mapping):
+ raise TypeError("TAeW decoder_work_queue entries must be mappings")
+ if "input_tensor" not in item or "block_index" not in item:
+ raise ValueError("TAeW decoder_work_queue entry is incomplete")
+ input_tensor = item["input_tensor"]
+ if not isinstance(input_tensor, torch.Tensor):
+ raise TypeError("TAeW decoder_work_queue input_tensor must be a tensor")
+ block_index = int(item["block_index"])
+ if not 0 <= block_index <= len(self.taew.decoder):
+ raise ValueError(f"TAeW decoder work item has invalid block index {block_index}")
+ restored_queue.append(
+ {
+ "input_tensor": self._copy_tensor_tree(
+ input_tensor,
+ device=device,
+ clone_tensors=clone_tensors,
+ ),
+ "block_index": block_index,
+ }
+ )
+ decoder_memory = snapshot["decoder_memory"]
+ if not isinstance(decoder_memory, (list, tuple)):
+ raise TypeError("TAeW decoder_memory snapshot must be a sequence")
+ if len(decoder_memory) != len(self.taew.decoder):
+ raise ValueError(
+ "TAeW decoder_memory snapshot does not match the loaded decoder architecture"
+ )
+ n_frames_decoded = int(snapshot["n_frames_decoded"])
+ if n_frames_decoded < 0:
+ raise ValueError("TAeW n_frames_decoded must be non-negative")
+ return {
+ "decoder_work_queue": restored_queue,
+ "decoder_memory": self._copy_tensor_tree(
+ list(decoder_memory),
+ device=device,
+ clone_tensors=clone_tensors,
+ ),
+ "n_frames_decoded": n_frames_decoded,
+ }
+
+ @staticmethod
+ def _apply_decode_state_tensor_tree(state: ABotWorldTAEWDecodeState, tree: Mapping[str, Any]) -> None:
+ stream = state.stream
+ stream.decoder_work_queue = [
+ TWorkItem(item["input_tensor"], int(item["block_index"]))
+ for item in tree["decoder_work_queue"]
+ ]
+ stream.decoder_memory = list(tree["decoder_memory"])
+ stream.n_frames_decoded = int(tree["n_frames_decoded"])
+
+ @classmethod
+ def _copy_tensor_tree(
+ cls,
+ value: Any,
+ *,
+ device: torch.device | str | None,
+ clone_tensors: bool,
+ ) -> Any:
+ if isinstance(value, torch.Tensor):
+ tensor = value.detach()
+ if device is not None:
+ tensor = tensor.to(device)
+ return tensor.clone() if clone_tensors else tensor
+ if isinstance(value, list):
+ return [
+ cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors)
+ for item in value
+ ]
+ if isinstance(value, tuple):
+ return tuple(
+ cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors)
+ for item in value
+ )
+ if isinstance(value, dict):
+ return {
+ key: cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors)
+ for key, item in value.items()
+ }
+ if value is None or isinstance(value, (bool, float, int, str)):
+ return value
+ raise TypeError(f"Unsupported TAeW decoder state value: {type(value)!r}")
+
+ @classmethod
+ def _iter_tensors(cls, value: Any):
+ if isinstance(value, torch.Tensor):
+ yield value
+ elif isinstance(value, Mapping):
+ for item in value.values():
+ yield from cls._iter_tensors(item)
+ elif isinstance(value, (list, tuple)):
+ for item in value:
+ yield from cls._iter_tensors(item)
+
+ @staticmethod
+ def _matches_device(actual: torch.device, expected: torch.device) -> bool:
+ return actual.type == expected.type and (expected.index is None or actual.index == expected.index)
diff --git a/telefuser/service/livekit/app.py b/telefuser/service/livekit/app.py
index f6b506be..8b005ced 100644
--- a/telefuser/service/livekit/app.py
+++ b/telefuser/service/livekit/app.py
@@ -145,14 +145,24 @@ async def service_ready() -> JSONResponse:
async def service_metadata() -> dict:
return runtime.metadata()
- @app.get("/v1/service/metrics")
- async def service_metrics() -> Response:
+ def render_prometheus_metrics() -> Response:
service_metrics_obj = get_service_metrics()
+ generic = service_metrics_obj.get_prometheus_format().rstrip()
+ serving = runtime.prometheus_metrics().rstrip()
return Response(
- content=service_metrics_obj.get_prometheus_format(),
+ content="\n\n".join(part for part in (generic, serving) if part) + "\n",
media_type="text/plain; charset=utf-8",
)
+ @app.get("/metrics", include_in_schema=False)
+ async def metrics_alias() -> Response:
+ """Prometheus conventional alias for the LiveKit serving endpoint."""
+ return render_prometheus_metrics()
+
+ @app.get("/v1/service/metrics")
+ async def service_metrics() -> Response:
+ return render_prometheus_metrics()
+
@app.get("/v1/service/metrics/json")
async def service_metrics_json() -> dict:
service_metrics_obj = get_service_metrics()
@@ -162,6 +172,7 @@ async def service_metrics_json() -> dict:
"service_type": "stream",
"transport": "livekit",
"livekit": health.model_dump(),
+ "serving": runtime.serving_metrics_snapshot(),
}
return app
diff --git a/telefuser/service/livekit/metrics.py b/telefuser/service/livekit/metrics.py
new file mode 100644
index 00000000..b7c2199c
--- /dev/null
+++ b/telefuser/service/livekit/metrics.py
@@ -0,0 +1,693 @@
+"""Low-cardinality Prometheus metrics for LiveKit stream serving.
+
+The API process is the Prometheus scrape target in a process-NCCL deployment.
+This collector consumes scheduler state and model-output events forwarded to that
+process. It deliberately never exposes session IDs as labels: per-session
+performance is exported as a distribution instead.
+"""
+
+from __future__ import annotations
+
+import math
+import threading
+import time
+from collections import defaultdict, deque
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from .runtime import LiveKitServeRuntime
+
+
+_LATENCY_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.5, 5.0, 10.0)
+_BATCH_BUCKETS = (1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 16.0, 32.0)
+_FPS_WINDOW_SECONDS = 30.0
+_TERMINAL_STATUSES = frozenset({"closed", "failed", "expired"})
+
+
+def _labels(labels: dict[str, object] | None = None) -> tuple[tuple[str, str], ...]:
+ return tuple(sorted((str(key), str(value)) for key, value in (labels or {}).items()))
+
+
+def _label_text(labels: tuple[tuple[str, str], ...]) -> str:
+ if not labels:
+ return ""
+
+ def escape(value: str) -> str:
+ return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+ return "{" + ",".join(f'{key}="{escape(value)}"' for key, value in labels) + "}"
+
+
+def _number(value: float | int) -> str:
+ number = float(value)
+ return f"{number:.12g}" if math.isfinite(number) else "0"
+
+
+def _quantile(values: list[float], q: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ return ordered[min(len(ordered) - 1, max(0, math.ceil(len(ordered) * q) - 1))]
+
+
+@dataclass
+class _Histogram:
+ buckets: tuple[float, ...]
+ counts: list[int] = field(init=False)
+ count: int = 0
+ total: float = 0.0
+
+ def __post_init__(self) -> None:
+ self.counts = [0 for _ in (*self.buckets, float("inf"))]
+
+ def observe(self, value: float) -> None:
+ if not math.isfinite(value) or value < 0:
+ return
+ self.count += 1
+ self.total += value
+ for index, upper_bound in enumerate((*self.buckets, float("inf"))):
+ if value <= upper_bound:
+ self.counts[index] += 1
+
+
+class LiveKitServingMetrics:
+ """Cumulative serving measurements plus scrape-time scheduler gauges."""
+
+ def __init__(self) -> None:
+ self._started_at = time.monotonic()
+ self._lock = threading.RLock()
+ self._counters: dict[tuple[str, tuple[tuple[str, str], ...]], float] = defaultdict(float)
+ self._histograms: dict[tuple[str, tuple[tuple[str, str], ...]], _Histogram] = {}
+ self._pending_action_at: dict[str, float] = {}
+ self._frame_events: deque[tuple[float, str, int]] = deque()
+
+ def record_admission(self, result: str) -> None:
+ self._inc("telefuser_serving_session_admissions_total", {"result": result})
+
+ def record_session_finished(self, status: str, error: str | None = None) -> None:
+ result = "failed" if status == "failed" or error else "closed"
+ self._inc("telefuser_serving_sessions_finished_total", {"result": result})
+ if error:
+ self._record_error(error)
+
+ def record_migration(
+ self,
+ *,
+ success: bool,
+ duration_seconds: float | None = None,
+ error: str | None = None,
+ ) -> None:
+ self._inc("telefuser_serving_migrations_total", {"result": "success" if success else "error"})
+ if duration_seconds is not None:
+ self._observe("telefuser_serving_migration_duration_seconds", {}, duration_seconds)
+ if error:
+ self._record_error(error)
+
+ def on_control_received(self, worker_id: str, session_id: str, received_at: float | None = None) -> None:
+ """Anchor A2F at validated controller-action ingress."""
+ del worker_id
+ with self._lock:
+ self._pending_action_at[session_id] = time.monotonic() if received_at is None else float(received_at)
+ self._inc_locked("telefuser_serving_actions_total", {})
+
+ def on_model_output(
+ self,
+ runtime: LiveKitServeRuntime,
+ *,
+ worker_id: str,
+ pipeline_session_id: str,
+ payload: dict[str, Any],
+ runtime_metrics: dict[str, Any] | None = None,
+ session_runtime_metrics: dict[str, Any] | None = None,
+ ) -> None:
+ """Ingest a model output before network transport and frame pacing."""
+ del worker_id, runtime_metrics, session_runtime_metrics
+ if not isinstance(payload, dict):
+ return
+ if payload.get("type") == "error":
+ self._inc("telefuser_serving_model_outputs_total", {"result": "error"})
+ self._record_error(str(payload.get("error", "model output error")))
+ return
+ if payload.get("type") != "chunk":
+ return
+
+ scheduler = payload.get("scheduler")
+ if not isinstance(scheduler, dict):
+ nested = payload.get("data")
+ scheduler = (
+ nested.get("scheduler")
+ if isinstance(nested, dict) and isinstance(nested.get("scheduler"), dict)
+ else {}
+ )
+ frames = payload.get("frames")
+ if not isinstance(frames, (list, tuple)):
+ nested = payload.get("data")
+ frames = nested.get("frames") if isinstance(nested, dict) else None
+ if isinstance(frames, (list, tuple)):
+ frame_count = len(frames)
+ else:
+ frame_count = int(self._nonnegative(payload.get("frame_count")) or 0)
+ batch_size = self._positive(scheduler.get("batch_size"), default=1.0)
+ compute_seconds = self._nonnegative(scheduler.get("compute_seconds"))
+ queue_wait_seconds = self._nonnegative(scheduler.get("queue_wait_seconds"))
+ taew_decode = self._taew_decode_measurement(scheduler)
+
+ with self._lock:
+ self._inc_locked("telefuser_serving_model_outputs_total", {"result": "chunk"})
+ self._inc_locked("telefuser_serving_chunks_total", {"result": "processed"})
+ # Every member observes the same batch. Fractional counting makes
+ # the total exactly one per coalesced execution after all members.
+ self._inc_locked("telefuser_serving_batches_total", {}, amount=1.0 / batch_size)
+ self._inc_locked("telefuser_serving_batch_items_total", {})
+ self._observe_locked("telefuser_serving_batch_size", {}, batch_size, buckets=_BATCH_BUCKETS)
+ if taew_decode is not None:
+ mode, items, invocations = taew_decode
+ # Each session output carries the same batch-level facts. Count
+ # one logical item per output and split native decoder calls
+ # across those items, so a DiT B=2 serial fallback contributes
+ # two TAeW invocations while a synchronized decode contributes one.
+ self._inc_locked(f"telefuser_serving_taew_decode_{mode}_items_total", {})
+ self._inc_locked(
+ f"telefuser_serving_taew_decode_{mode}_executions_total",
+ {},
+ amount=invocations / items,
+ )
+ if compute_seconds is not None:
+ self._observe_locked("telefuser_serving_chunk_latency_seconds", {}, compute_seconds)
+ if queue_wait_seconds is not None:
+ self._observe_locked("telefuser_serving_queue_wait_seconds", {}, queue_wait_seconds)
+ for key, stage in (
+ ("input_prepare_seconds", "input_prepare"),
+ ("cache_collate_seconds", "cache_collate"),
+ ("denoise_seconds", "dit"),
+ ("cache_scatter_seconds", "cache_scatter"),
+ ("vae_encode_seconds", "vae_encode"),
+ ("vae_decode_seconds", "vae_decode"),
+ ("postprocess_seconds", "postprocess"),
+ ):
+ value = self._nonnegative(scheduler.get(key))
+ if value is not None:
+ self._observe_locked("telefuser_serving_pipeline_stage_latency_seconds", {"stage": stage}, value)
+
+ fps = self._session_fps(runtime, pipeline_session_id, payload)
+ elapsed = (compute_seconds or 0.0) + (queue_wait_seconds or 0.0)
+ if frame_count and fps is not None and elapsed > 0:
+ budget = frame_count / fps
+ self._observe_locked("telefuser_serving_slo_budget_seconds", {}, budget)
+ self._inc_locked(
+ "telefuser_serving_slo_chunks_total",
+ {"result": "met" if elapsed <= budget else "missed"},
+ )
+
+ def on_chunk_published(
+ self,
+ *,
+ worker_id: str,
+ session_id: str,
+ frames: int,
+ first_frame_at: float | None = None,
+ ) -> None:
+ """Record frames handed to LiveKit's video publisher.
+
+ This intentionally happens after model completion and video pacing. It
+ is the server-side measure closest to client-visible end-to-end FPS;
+ receiver decode and network jitter remain client-side observability.
+ """
+ del worker_id
+ if frames <= 0:
+ return
+ published_at = time.monotonic()
+ first_at = published_at if first_frame_at is None else float(first_frame_at)
+ with self._lock:
+ self._inc_locked("telefuser_serving_chunks_total", {"result": "published"})
+ self._inc_locked("telefuser_serving_frames_total", {"result": "published"}, amount=float(frames))
+ action_at = self._pending_action_at.pop(session_id, None)
+ if action_at is not None and first_at >= action_at:
+ self._observe_locked("telefuser_serving_action_to_first_frame_seconds", {}, first_at - action_at)
+ self._frame_events.append((published_at, session_id, int(frames)))
+ self._trim_frame_events_locked(published_at)
+
+ def render_prometheus(self, runtime: LiveKitServeRuntime) -> str:
+ """Render a low-cardinality Prometheus exposition fragment."""
+ state = self._state(runtime)
+ lines: list[str] = []
+ self._render_gauges(lines, state["gauges"])
+ self._render_counters(lines, state["counters"])
+ self._render_histograms(lines, state["histograms"])
+ return "\n".join(lines)
+
+ def json_snapshot(self, runtime: LiveKitServeRuntime) -> dict[str, Any]:
+ """Return a compact aggregate-only JSON view for experiment tooling."""
+ state = self._state(runtime)
+ return {
+ "summary": state["summary"],
+ "counters": {name + _label_text(labels): value for (name, labels), value in state["counters"].items()},
+ }
+
+ def _state(self, runtime: LiveKitServeRuntime) -> dict[str, Any]:
+ now = time.monotonic()
+ with self._lock:
+ self._trim_frame_events_locked(now)
+ counters = dict(self._counters)
+ histograms = dict(self._histograms)
+ frame_events = tuple(self._frame_events)
+
+ health = runtime.health().model_dump()
+ workers = runtime.scheduler.workers()
+ records = runtime.registry.list_records()
+ snapshot_fn = getattr(runtime.worker_pool, "turboserve_snapshot", None)
+ routing = snapshot_fn() if callable(snapshot_fn) else {}
+ routing = routing if isinstance(routing, dict) else {}
+ worker_metrics = routing.get("worker_runtime_metrics", {})
+ worker_metrics = worker_metrics if isinstance(worker_metrics, dict) else {}
+ session_metrics = routing.get("session_runtime_metrics", {})
+ session_metrics = session_metrics if isinstance(session_metrics, dict) else {}
+
+ active = 0
+ retained = 0
+ status_counts: dict[str, int] = defaultdict(int)
+ for record in records:
+ status_counts[str(record.status)] += 1
+ if record.status in _TERMINAL_STATUSES:
+ continue
+ if record.worker_id is not None:
+ retained += 1
+ metrics = session_metrics.get(record.pipeline_session_id, {}) if record.pipeline_session_id else {}
+ if isinstance(metrics, dict) and bool(metrics.get("active", 0)):
+ active += 1
+ elif record.status == "running":
+ active += 1
+ idle = max(0, retained - active)
+
+ gauges: list[tuple[str, str, tuple[tuple[str, str], ...], float]] = []
+
+ def gauge(name: str, description: str, value: float | int, labels: dict[str, object] | None = None) -> None:
+ gauges.append((name, description, _labels(labels), float(value)))
+
+ gauge(
+ "telefuser_serving_uptime_seconds",
+ "Seconds since the LiveKit serving metrics collector started",
+ now - self._started_at,
+ )
+ for state, value in (
+ ("retained", retained),
+ ("active", active),
+ ("idle", idle),
+ ("waiting", health["queued_sessions"]),
+ ):
+ gauge(
+ "telefuser_serving_sessions",
+ "Current sessions grouped by scheduler state",
+ int(value),
+ {"state": state},
+ )
+ for status, count in sorted(status_counts.items()):
+ gauge(
+ "telefuser_serving_session_status",
+ "Current sessions grouped by public lifecycle status",
+ count,
+ {"status": status},
+ )
+ gauge(
+ "telefuser_serving_queue_depth",
+ "Current scheduler queue depth",
+ health["queued_sessions"],
+ {"queue": "admission"},
+ )
+ for state, value in (
+ ("configured", health["workers_total"]),
+ ("busy", health["workers_busy"]),
+ ("idle", health["workers_idle"]),
+ ("failed", health["workers_failed"]),
+ ):
+ gauge("telefuser_serving_workers", "Workers grouped by current state", value, {"state": state})
+
+ scheduler_mode = "unknown"
+ for worker in workers:
+ values = worker_metrics.get(worker.worker_id, {})
+ values = values if isinstance(values, dict) else {}
+ scheduler_mode = str(values.get("scheduler_mode", scheduler_mode))
+ gpu_ids = worker.gpu_ids or ["unassigned"]
+ for gpu_id in gpu_ids:
+ labels = {"worker_id": worker.worker_id, "gpu": gpu_id}
+ gauge(
+ "telefuser_serving_worker_sessions",
+ "Retained sessions assigned to a worker/GPU group",
+ len(worker.session_ids),
+ labels,
+ )
+ gauge(
+ "telefuser_serving_worker_capacity",
+ "Retained-session capacity for a worker/GPU group",
+ worker.session_capacity,
+ labels,
+ )
+ gauge(
+ "telefuser_serving_worker_up",
+ "Whether a configured worker is available",
+ int(worker.status not in {"failed", "stopped"}),
+ labels,
+ )
+ if worker.session_capacity:
+ gauge(
+ "telefuser_serving_worker_busy_ratio",
+ "Scheduler retained-session occupancy for a worker/GPU group",
+ len(worker.session_ids) / worker.session_capacity,
+ labels,
+ )
+ for metric_key, metric_name, description, extra_labels in (
+ (
+ "active_sessions",
+ "telefuser_serving_worker_active_sessions",
+ "Active model sessions reported by a worker",
+ {},
+ ),
+ (
+ "maximum_batch_size",
+ "telefuser_serving_worker_maximum_batch_size",
+ "Largest batch observed by a worker",
+ {},
+ ),
+ (
+ "mean_chunk_seconds",
+ "telefuser_serving_worker_chunk_latency_seconds",
+ "Worker-reported chunk latency",
+ {"stat": "mean"},
+ ),
+ (
+ "p95_chunk_seconds",
+ "telefuser_serving_worker_chunk_latency_seconds",
+ "Worker-reported chunk latency",
+ {"stat": "p95"},
+ ),
+ ):
+ value = self._nonnegative(values.get(metric_key))
+ if value is not None:
+ gauge(metric_name, description, value, {**labels, **extra_labels})
+ for metric_key, metric_name, description in (
+ (
+ "taew_decode_items",
+ "telefuser_serving_worker_taew_decode_items",
+ "Logical chunks in the latest TAeW LightVAE decode",
+ ),
+ (
+ "taew_decode_batch_size",
+ "telefuser_serving_worker_taew_decode_batch_size",
+ "Effective native batch size in the latest TAeW LightVAE decode",
+ ),
+ (
+ "taew_decode_invocations",
+ "telefuser_serving_worker_taew_decode_invocations",
+ "Native TAeW LightVAE decode calls in the latest model batch",
+ ),
+ (
+ "taew_decode_mode",
+ "telefuser_serving_worker_taew_decode_mode",
+ "TAeW decode mode: 0 singleton, 1 synchronized batch, 2 safe serial fallback",
+ ),
+ ):
+ value = self._nonnegative(values.get(metric_key))
+ if value is not None:
+ gauge(metric_name, description, value, labels)
+ for stage_key, stage in (
+ ("input_prepare_seconds", "input_prepare"),
+ ("cache_collate_seconds", "cache_collate"),
+ ("denoise_seconds", "dit"),
+ ("cache_scatter_seconds", "cache_scatter"),
+ ("vae_encode_seconds", "vae_encode"),
+ ("vae_decode_seconds", "vae_decode"),
+ ("postprocess_seconds", "postprocess"),
+ ):
+ value = self._nonnegative(values.get(stage_key))
+ if value is not None:
+ gauge(
+ "telefuser_serving_worker_pipeline_stage_last_latency_seconds",
+ "Latest worker-reported pipeline stage latency",
+ value,
+ {**labels, "stage": stage},
+ )
+ gauge(
+ "telefuser_serving_scheduler_mode_info",
+ "One for the ABot scheduler mode selected by each worker",
+ 1,
+ {"mode": scheduler_mode},
+ )
+
+ batches = counters.get(("telefuser_serving_batches_total", ()), 0.0)
+ batch_items = counters.get(("telefuser_serving_batch_items_total", ()), 0.0)
+ if batches:
+ gauge("telefuser_serving_mean_batch_size", "Mean observed coalesced batch size", batch_items / batches)
+ taew_items = sum(
+ counters.get((f"telefuser_serving_taew_decode_{mode}_items_total", ()), 0.0)
+ for mode in ("singleton", "synchronized", "serial_fallback")
+ )
+ taew_executions = sum(
+ counters.get((f"telefuser_serving_taew_decode_{mode}_executions_total", ()), 0.0)
+ for mode in ("singleton", "synchronized", "serial_fallback")
+ )
+ if taew_executions:
+ gauge(
+ "telefuser_serving_taew_decode_mean_native_batch_size",
+ "Mean effective native TAeW LightVAE batch size across observed decoder calls",
+ taew_items / taew_executions,
+ )
+
+ met = counters.get(("telefuser_serving_slo_chunks_total", _labels({"result": "met"})), 0.0)
+ missed = counters.get(("telefuser_serving_slo_chunks_total", _labels({"result": "missed"})), 0.0)
+ if met + missed:
+ gauge(
+ "telefuser_serving_slo_attainment_ratio",
+ "Fraction of chunks meeting their FPS-derived queue-plus-compute budget",
+ met / (met + missed),
+ )
+
+ fps = self._fps_summary(frame_events, now)
+ for scope, value in fps.items():
+ gauge(
+ "telefuser_serving_published_fps",
+ "Published video frame rate over the trailing 30 seconds",
+ value,
+ {"scope": scope},
+ )
+
+ return {
+ "gauges": gauges,
+ "counters": counters,
+ "histograms": histograms,
+ "summary": {
+ "sessions": {
+ "retained": retained,
+ "active": active,
+ "idle": idle,
+ "waiting": int(health["queued_sessions"]),
+ },
+ "published_fps": fps,
+ "scheduler_mode": scheduler_mode,
+ "worker_runtime_metrics": {
+ str(worker_id): dict(values)
+ for worker_id, values in worker_metrics.items()
+ if isinstance(values, dict)
+ },
+ },
+ }
+
+ def _session_fps(
+ self, runtime: LiveKitServeRuntime, pipeline_session_id: str, payload: dict[str, Any]
+ ) -> float | None:
+ for record in runtime.registry.list_records():
+ if record.pipeline_session_id == pipeline_session_id:
+ value = record.config.get("fps", payload.get("fps", runtime.config.default_fps))
+ break
+ else:
+ value = payload.get("fps", runtime.config.default_fps)
+ try:
+ fps = float(value)
+ except (TypeError, ValueError):
+ return None
+ return fps if fps > 0 and math.isfinite(fps) else None
+
+ def _fps_summary(self, events: tuple[tuple[float, str, int], ...], now: float) -> dict[str, float]:
+ window_start = max(self._started_at, now - _FPS_WINDOW_SECONDS)
+ duration = max(1e-6, now - window_start)
+ total = 0
+ per_session: dict[str, int] = defaultdict(int)
+ for timestamp, session_id, frames in events:
+ if timestamp >= window_start:
+ total += frames
+ per_session[session_id] += frames
+ rates = [frames / duration for frames in per_session.values()]
+ return {
+ "aggregate": total / duration,
+ "per_active_session_mean": sum(rates) / len(rates) if rates else 0.0,
+ "per_active_session_p50": _quantile(rates, 0.50),
+ "per_active_session_p95": _quantile(rates, 0.95),
+ "per_active_session_min": min(rates) if rates else 0.0,
+ }
+
+ def _record_error(self, error: str) -> None:
+ normalized = error.lower()
+ kind = "oom" if "out of memory" in normalized or "cuda oom" in normalized else "model"
+ self._inc("telefuser_serving_errors_total", {"kind": kind})
+
+ def _inc(self, name: str, labels: dict[str, object], amount: float = 1.0) -> None:
+ with self._lock:
+ self._inc_locked(name, labels, amount)
+
+ def _inc_locked(self, name: str, labels: dict[str, object], amount: float = 1.0) -> None:
+ if math.isfinite(amount) and amount >= 0:
+ self._counters[(name, _labels(labels))] += amount
+
+ def _observe(
+ self,
+ name: str,
+ labels: dict[str, object],
+ value: float,
+ *,
+ buckets: tuple[float, ...] = _LATENCY_BUCKETS,
+ ) -> None:
+ with self._lock:
+ self._observe_locked(name, labels, value, buckets=buckets)
+
+ def _observe_locked(
+ self,
+ name: str,
+ labels: dict[str, object],
+ value: float,
+ *,
+ buckets: tuple[float, ...] = _LATENCY_BUCKETS,
+ ) -> None:
+ key = (name, _labels(labels))
+ series = self._histograms.get(key)
+ if series is None:
+ series = self._histograms[key] = _Histogram(buckets=buckets)
+ series.observe(float(value))
+
+ def _trim_frame_events_locked(self, now: float) -> None:
+ cutoff = now - _FPS_WINDOW_SECONDS
+ while self._frame_events and self._frame_events[0][0] < cutoff:
+ self._frame_events.popleft()
+
+ @staticmethod
+ def _nonnegative(value: object) -> float | None:
+ try:
+ number = float(value)
+ except (TypeError, ValueError):
+ return None
+ return number if math.isfinite(number) and number >= 0 else None
+
+ @classmethod
+ def _positive(cls, value: object, *, default: float) -> float:
+ number = cls._nonnegative(value)
+ return number if number is not None and number > 0 else default
+
+ @classmethod
+ def _taew_decode_measurement(cls, scheduler: dict[str, Any]) -> tuple[str, int, int] | None:
+ """Validate batch-local TAeW facts before exporting cumulative counters."""
+ items = cls._positive_integer(scheduler.get("taew_decode_items"))
+ native_batch_size = cls._positive_integer(scheduler.get("taew_decode_batch_size"))
+ invocations = cls._positive_integer(scheduler.get("taew_decode_invocations"))
+ mode = cls._nonnegative(scheduler.get("taew_decode_mode"))
+ if items is None or native_batch_size is None or invocations is None or mode is None:
+ return None
+ if not mode.is_integer():
+ return None
+ if int(mode) == 0 and (items, native_batch_size, invocations) == (1, 1, 1):
+ return ("singleton", items, invocations)
+ if int(mode) == 1 and items > 1 and (native_batch_size, invocations) == (items, 1):
+ return ("synchronized", items, invocations)
+ if int(mode) == 2 and items > 1 and (native_batch_size, invocations) == (1, items):
+ return ("serial_fallback", items, invocations)
+ return None
+
+ @classmethod
+ def _positive_integer(cls, value: object) -> int | None:
+ number = cls._nonnegative(value)
+ if number is None or number <= 0 or not number.is_integer():
+ return None
+ return int(number)
+
+ @staticmethod
+ def _render_gauges(
+ lines: list[str],
+ gauges: list[tuple[str, str, tuple[tuple[str, str], ...], float]],
+ ) -> None:
+ grouped: dict[str, tuple[str, list[tuple[tuple[tuple[str, str], ...], float]]]] = {}
+ for name, description, labels, value in gauges:
+ if name not in grouped:
+ grouped[name] = (description, [])
+ grouped[name][1].append((labels, value))
+ for name, (description, samples) in sorted(grouped.items()):
+ lines.extend((f"# HELP {name} {description}", f"# TYPE {name} gauge"))
+ lines.extend(f"{name}{_label_text(labels)} {_number(value)}" for labels, value in sorted(samples))
+
+ @staticmethod
+ def _render_counters(
+ lines: list[str],
+ counters: dict[tuple[str, tuple[tuple[str, str], ...]], float],
+ ) -> None:
+ descriptions = {
+ "telefuser_serving_actions_total": "Validated control actions accepted by the serving transport",
+ "telefuser_serving_batch_items_total": "Session chunks included in coalesced batches",
+ "telefuser_serving_batches_total": "Coalesced model batch executions",
+ "telefuser_serving_chunks_total": "Processed or published chunks",
+ "telefuser_serving_errors_total": "Serving errors grouped by bounded error class",
+ "telefuser_serving_frames_total": "Frames handed to the LiveKit publisher",
+ "telefuser_serving_migrations_total": "Session migration attempts grouped by outcome",
+ "telefuser_serving_model_outputs_total": "Model output messages grouped by result",
+ "telefuser_serving_session_admissions_total": "Session admissions grouped by scheduler result",
+ "telefuser_serving_sessions_finished_total": "Terminal sessions grouped by outcome",
+ "telefuser_serving_slo_chunks_total": "Chunks grouped by whether their FPS-derived budget was met",
+ "telefuser_serving_taew_decode_serial_fallback_executions_total": (
+ "Native TAeW LightVAE decode calls made after safe serial fallback"
+ ),
+ "telefuser_serving_taew_decode_serial_fallback_items_total": (
+ "Logical session chunks decoded through safe TAeW serial fallback"
+ ),
+ "telefuser_serving_taew_decode_singleton_executions_total": (
+ "Native TAeW LightVAE decode calls for singleton chunks"
+ ),
+ "telefuser_serving_taew_decode_singleton_items_total": (
+ "Logical singleton session chunks decoded by TAeW LightVAE"
+ ),
+ "telefuser_serving_taew_decode_synchronized_executions_total": (
+ "Native synchronized TAeW LightVAE decode calls"
+ ),
+ "telefuser_serving_taew_decode_synchronized_items_total": (
+ "Logical session chunks decoded in synchronized native TAeW batches"
+ ),
+ }
+ grouped: dict[str, list[tuple[tuple[tuple[str, str], ...], float]]] = defaultdict(list)
+ for (name, labels), value in counters.items():
+ grouped[name].append((labels, value))
+ for name, samples in sorted(grouped.items()):
+ lines.extend((f"# HELP {name} {descriptions.get(name, name)}", f"# TYPE {name} counter"))
+ lines.extend(f"{name}{_label_text(labels)} {_number(value)}" for labels, value in sorted(samples))
+
+ @staticmethod
+ def _render_histograms(
+ lines: list[str],
+ histograms: dict[tuple[str, tuple[tuple[str, str], ...]], _Histogram],
+ ) -> None:
+ descriptions = {
+ "telefuser_serving_action_to_first_frame_seconds": (
+ "Validated action ingress to first frame handed to LiveKit"
+ ),
+ "telefuser_serving_batch_size": "Observed coalesced model batch size",
+ "telefuser_serving_chunk_latency_seconds": "Scheduler compute time for one session chunk",
+ "telefuser_serving_migration_duration_seconds": "End-to-end migration time",
+ "telefuser_serving_pipeline_stage_latency_seconds": "Model pipeline stage latency",
+ "telefuser_serving_queue_wait_seconds": "Time a ready session waits before a chunk starts",
+ "telefuser_serving_slo_budget_seconds": "FPS-derived queue-plus-compute budget",
+ }
+ grouped: dict[str, list[tuple[tuple[tuple[str, str], ...], _Histogram]]] = defaultdict(list)
+ for (name, labels), series in histograms.items():
+ grouped[name].append((labels, series))
+ for name, series_list in sorted(grouped.items()):
+ lines.extend((f"# HELP {name} {descriptions.get(name, name)}", f"# TYPE {name} histogram"))
+ for labels, series in sorted(series_list):
+ for bound, count in zip((*series.buckets, float("inf")), series.counts):
+ bucket_labels = _labels({**dict(labels), "le": "+Inf" if math.isinf(bound) else str(bound)})
+ lines.append(f"{name}_bucket{_label_text(bucket_labels)} {count}")
+ lines.append(f"{name}_sum{_label_text(labels)} {_number(series.total)}")
+ lines.append(f"{name}_count{_label_text(labels)} {series.count}")
diff --git a/telefuser/service/livekit/multi_session_worker.py b/telefuser/service/livekit/multi_session_worker.py
index 1f334f26..7995f64e 100644
--- a/telefuser/service/livekit/multi_session_worker.py
+++ b/telefuser/service/livekit/multi_session_worker.py
@@ -36,6 +36,36 @@ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None
def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
self._owner.event_sink.on_session_finished(worker_id, session_id, error)
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ callback = getattr(self._owner.event_sink, "on_control_received", None)
+ if callable(callback):
+ callback(worker_id, session_id)
+
+ def on_chunk_published(
+ self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
+ ) -> None:
+ callback = getattr(self._owner.event_sink, "on_chunk_published", None)
+ if callable(callback):
+ callback(worker_id, session_id, frames, first_frame_at)
+
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None:
+ callback = getattr(self._owner.event_sink, "on_model_output", None)
+ if callable(callback):
+ callback(
+ worker_id,
+ session_id,
+ payload,
+ runtime_metrics=runtime_metrics,
+ session_runtime_metrics=session_runtime_metrics,
+ )
+
class MultiSessionLiveKitWorker:
"""Load one model pipeline and retain multiple independent room sessions."""
diff --git a/telefuser/service/livekit/nccl_process_worker_pool.py b/telefuser/service/livekit/nccl_process_worker_pool.py
index c82d91ef..bcc20912 100644
--- a/telefuser/service/livekit/nccl_process_worker_pool.py
+++ b/telefuser/service/livekit/nccl_process_worker_pool.py
@@ -10,6 +10,7 @@
import contextlib
import socket
import time
+from dataclasses import dataclass
from typing import Any
import torch
@@ -27,6 +28,80 @@
from .turboserve import TurboServeOwnership, TurboServeOwnershipTable
from .worker import LiveKitWorker
+# The parent retains at most one decoded payload waiting for the LiveKit
+# transport. The transport acknowledges a payload as soon as it dequeues it,
+# which permits exactly one next payload to be prefetched while the current
+# one is paced onto WebRTC. Consequently, at most two fully-materialized
+# payloads per session live outside ABot's own bounded/latest output queue:
+# one being published and one in this parent queue.
+_MODEL_OUTPUT_PARENT_QUEUE_SIZE = 1
+_VIDEO_OUTPUT_TYPES = frozenset({"preview", "chunk"})
+_TERMINAL_OUTPUT_TYPES = frozenset({"error", "done"})
+
+
+@dataclass(frozen=True)
+class _ModelOutput:
+ """One child-model payload plus the worker that owns its output credit."""
+
+ worker_id: str
+ payload: dict[str, Any]
+
+
+async def _pump_model_outputs(
+ adapter: Any,
+ service: Any,
+ *,
+ worker_id: str,
+ session_id: str,
+ credits: asyncio.BoundedSemaphore,
+ events: Any,
+) -> None:
+ """Pull only after a parent credit, so IPC cannot outrun WebRTC playback."""
+ chunks = adapter.pull_chunks(session_id)
+ iterator = chunks.__aiter__()
+ credit_held = False
+ try:
+ while True:
+ await credits.acquire()
+ credit_held = True
+ try:
+ payload = await iterator.__anext__()
+ except StopAsyncIteration:
+ credits.release()
+ credit_held = False
+ return
+ except asyncio.CancelledError:
+ credits.release()
+ credit_held = False
+ raise
+ except Exception:
+ credits.release()
+ credit_held = False
+ raise
+ # No await separates dequeue from IPC submission. If migration
+ # pauses this task, a payload is either sent exactly once or has
+ # not been removed from the ABot generator.
+ events.put(
+ {
+ "type": "model_output",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ "payload": payload,
+ "runtime_metrics": adapter.runtime_metrics() or {},
+ "session_runtime_metrics": service.runtime_metrics(session_id),
+ }
+ )
+ # The credit now belongs to the parent queue and transport path.
+ credit_held = False
+ finally:
+ if credit_held:
+ with contextlib.suppress(ValueError):
+ credits.release()
+ aclose = getattr(chunks, "aclose", None)
+ if callable(aclose):
+ with contextlib.suppress(Exception):
+ await aclose()
+
class _ProcessPipelineAdapter:
stream_mode = STREAM_MODE_BIDIRECTIONAL
@@ -71,6 +146,18 @@ def on_session_finished(self, worker_id: str, session_id: str, error: str | None
self.pool._transport_finished(session_id)
self.pool._event_sink.on_session_finished(worker_id, session_id, error)
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ callback = getattr(self.pool._event_sink, "on_control_received", None)
+ if callable(callback):
+ callback(worker_id, session_id)
+
+ def on_chunk_published(
+ self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
+ ) -> None:
+ callback = getattr(self.pool._event_sink, "on_chunk_published", None)
+ if callable(callback):
+ callback(worker_id, session_id, frames, first_frame_at)
+
class NCCLProcessLiveKitWorkerPool(ProcessLiveKitWorkerPool):
"""TurboServe-compatible parent transport / GPU model-process pool."""
@@ -79,19 +166,32 @@ def __init__(self, specs: list[ProcessWorkerSpec], **kwargs: Any) -> None:
super().__init__(specs, **kwargs)
self._worker_target = _nccl_model_worker_main
self._ownership = TurboServeOwnershipTable()
- self._model_outputs: dict[str, asyncio.Queue[dict | None]] = {}
+ self._model_outputs: dict[str, asyncio.Queue[_ModelOutput | None]] = {}
+ self._model_output_inflight: set[str] = set()
+ self._model_output_drained: dict[str, asyncio.Event] = {}
+ self._model_output_dropped: dict[str, int] = {}
self._transport_workers: dict[str, LiveKitWorker] = {}
self._transport_tasks: dict[str, asyncio.Task[None]] = {}
self._migrating_controls: dict[str, list[dict]] = {}
- self._worker_runtime_metrics: dict[str, dict[str, float | int]] = {}
+ # Worker snapshots include scalar timings/counters plus the bounded
+ # scheduler mode string (``batched`` or ``round_robin``).
+ self._worker_runtime_metrics: dict[str, dict[str, float | int | str]] = {}
self._session_runtime_metrics: dict[str, dict[str, float | int]] = {}
self._migration_total_ms: list[float] = []
self._nccl_ranks: dict[str, int] = {}
self._migration_lock = asyncio.Lock()
+ self._initializing_workers = False
async def start(self, *, skip_validation: bool = False) -> None:
- await super().start(skip_validation=skip_validation)
- if len(self._active_workers) > 1:
+ # ``ProcessLiveKitWorkerPool.start`` calls this class's ``scale_to``
+ # once per replica. Defer communicator construction until all initial
+ # workers have completed their sequential checkpoint load.
+ self._initializing_workers = True
+ try:
+ await super().start(skip_validation=skip_validation)
+ finally:
+ self._initializing_workers = False
+ if len(self._active_workers) > 1 and not self._nccl_ranks:
await self._init_nccl()
async def scale_to(self, target_workers: int) -> int:
@@ -106,7 +206,7 @@ async def scale_to(self, target_workers: int) -> int:
)
self._nccl_ranks.clear()
actual = await super().scale_to(target_workers)
- if actual > 1:
+ if actual > 1 and not self._initializing_workers:
await self._init_nccl()
return actual
@@ -140,17 +240,33 @@ async def stop_session(self, session_id: str) -> None:
await asyncio.wait_for(asyncio.shield(task), timeout=15.0)
def create_model_session(self, worker_id: str, session_id: str, config: dict) -> None:
- self._model_outputs[session_id] = asyncio.Queue()
+ output: asyncio.Queue[_ModelOutput | None] = asyncio.Queue(maxsize=_MODEL_OUTPUT_PARENT_QUEUE_SIZE)
+ drained = asyncio.Event()
+ drained.set()
+ self._model_outputs[session_id] = output
+ self._model_output_drained[session_id] = drained
+ self._model_output_dropped[session_id] = 0
self._pipeline_routes[session_id] = worker_id
self._session_workers[session_id] = worker_id
self._ownership.register(session_id, worker_id)
- self._send(worker_id, {"type": "model_create", "session_id": session_id, "config": dict(config)})
+ self._send(
+ worker_id,
+ {
+ "type": "model_create",
+ "session_id": session_id,
+ "config": dict(config),
+ "model_output_credit_window": _MODEL_OUTPUT_PARENT_QUEUE_SIZE,
+ },
+ )
def push_model_chunk(self, session_id: str, chunk: dict) -> None:
if session_id in self._migrating_controls:
self._migrating_controls[session_id].append(dict(chunk))
return
- self._send(self._pipeline_routes[session_id], {"type": "model_push", "session_id": session_id, "chunk": dict(chunk)})
+ self._send(
+ self._pipeline_routes[session_id],
+ {"type": "model_push", "session_id": session_id, "chunk": dict(chunk)},
+ )
def close_model_session(self, session_id: str) -> None:
worker_id = self._pipeline_routes.pop(session_id, None)
@@ -160,13 +276,122 @@ def close_model_session(self, session_id: str) -> None:
self._session_runtime_metrics.pop(session_id, None)
if worker_id in self._active_workers:
self._send(worker_id, {"type": "model_close", "session_id": session_id})
- if (output := self._model_outputs.pop(session_id, None)) is not None:
- output.put_nowait(None)
+ self._close_model_output(session_id)
async def pull_model_chunks(self, session_id: str):
- output = self._model_outputs[session_id]
- while (payload := await output.get()) is not None:
- yield payload
+ output = self._model_outputs.get(session_id)
+ if output is None:
+ return
+ while True:
+ item = await output.get()
+ if item is None:
+ return
+ # This is deliberately before ``yield``: it allows one queued
+ # prefetch while the transport paces the just-dequeued payload.
+ self._model_output_inflight.add(session_id)
+ self._update_model_output_drained(session_id)
+ self._ack_model_output(session_id, item)
+ try:
+ yield item.payload
+ finally:
+ self._model_output_inflight.discard(session_id)
+ self._update_model_output_drained(session_id)
+
+ def _close_model_output(self, session_id: str) -> None:
+ output = self._model_outputs.pop(session_id, None)
+ self._model_output_inflight.discard(session_id)
+ drained = self._model_output_drained.pop(session_id, None)
+ self._model_output_dropped.pop(session_id, None)
+ if isinstance(output, asyncio.Queue):
+ while True:
+ try:
+ output.get_nowait()
+ except asyncio.QueueEmpty:
+ break
+ with contextlib.suppress(asyncio.QueueFull):
+ output.put_nowait(None)
+ elif output is not None:
+ # Compatibility for simple queue doubles used by isolated tests.
+ output.put_nowait(None)
+ if drained is not None:
+ drained.set()
+
+ def _enqueue_model_output(self, session_id: str, item: _ModelOutput) -> None:
+ output = self._model_outputs.get(session_id)
+ if output is None:
+ return
+ if not isinstance(output, asyncio.Queue):
+ # Keep legacy light-weight test doubles usable while production
+ # always takes the bounded branch below.
+ output.put_nowait(item.payload)
+ return
+ try:
+ output.put_nowait(item)
+ except asyncio.QueueFull:
+ queued = output.get_nowait()
+ if queued is None:
+ output.put_nowait(None)
+ self._record_dropped_model_output(session_id, item)
+ elif self._should_replace_queued_output(queued, item):
+ output.put_nowait(item)
+ self._record_dropped_model_output(session_id, queued)
+ else:
+ output.put_nowait(queued)
+ self._record_dropped_model_output(session_id, item)
+ self._update_model_output_drained(session_id)
+
+ @staticmethod
+ def _should_replace_queued_output(queued: _ModelOutput, incoming: _ModelOutput) -> bool:
+ queued_type = str(queued.payload.get("type", ""))
+ incoming_type = str(incoming.payload.get("type", ""))
+ if queued_type in _TERMINAL_OUTPUT_TYPES:
+ return False
+ if incoming_type in _TERMINAL_OUTPUT_TYPES:
+ return True
+ # Preserve an initial preview until it reaches the transport. Later
+ # generated chunks are latest-wins, matching ABot's own queue.
+ if queued_type == "preview":
+ return False
+ if incoming_type == "preview":
+ return queued_type in _VIDEO_OUTPUT_TYPES
+ return queued_type == "chunk" and incoming_type == "chunk"
+
+ def _record_dropped_model_output(self, session_id: str, item: _ModelOutput, *, acknowledge: bool = True) -> None:
+ dropped = getattr(self, "_model_output_dropped", None)
+ if isinstance(dropped, dict):
+ dropped[session_id] = int(dropped.get(session_id, 0)) + 1
+ if acknowledge:
+ self._ack_model_output(session_id, item)
+
+ def _ack_model_output(self, session_id: str, item: _ModelOutput) -> None:
+ # Never wait for a child response from the parent event loop. The
+ # command only releases that child session's bounded semaphore.
+ try:
+ if item.worker_id in self._active_workers:
+ self._send(
+ item.worker_id,
+ {"type": "model_output_credit", "session_id": session_id},
+ )
+ except Exception:
+ # The owning child can disappear during close/migration; there is
+ # then no useful credit to return.
+ return
+
+ def _update_model_output_drained(self, session_id: str) -> None:
+ drained = self._model_output_drained.get(session_id)
+ output = self._model_outputs.get(session_id)
+ if drained is None or output is None or not isinstance(output, asyncio.Queue):
+ return
+ if session_id not in self._model_output_inflight and output.empty():
+ drained.set()
+ else:
+ drained.clear()
+
+ async def _wait_for_model_output_drain(self, session_id: str, *, timeout: float) -> None:
+ self._update_model_output_drained(session_id)
+ drained = self._model_output_drained.get(session_id)
+ if drained is not None:
+ await asyncio.wait_for(drained.wait(), timeout=timeout)
async def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
async with self._migration_lock:
@@ -178,64 +403,153 @@ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str)
token = self._ownership.prepare_migration(pipeline_session_id, source_worker_id, target_worker_id)
self._migrating_controls[pipeline_session_id] = []
started = time.monotonic()
+ source_output_paused = False
try:
await asyncio.gather(
self._request(source_worker_id, "scheduler_pause", timeout=300.0),
self._request(target_worker_id, "scheduler_pause", timeout=300.0),
)
- exported = await self._request(source_worker_id, "nccl_export", session_id=pipeline_session_id, transfer_id=token.token_id, timeout=300.0)
+ exported = await self._request(
+ source_worker_id,
+ "nccl_export",
+ session_id=pipeline_session_id,
+ transfer_id=token.token_id,
+ timeout=300.0,
+ )
+ await self._request(
+ source_worker_id,
+ "model_output_pause",
+ session_id=pipeline_session_id,
+ timeout=300.0,
+ )
+ source_output_paused = True
+ await self._wait_for_model_output_drain(pipeline_session_id, timeout=300.0)
metadata = dict(exported["result"])
- await self._request(target_worker_id, "nccl_prepare_recv", transfer_id=token.token_id, metadata=metadata, source_rank=self._nccl_ranks[source_worker_id], owner_worker_id=target_worker_id, ownership_epoch=token.source_epoch + 1, timeout=300.0)
+ await self._request(
+ target_worker_id,
+ "nccl_prepare_recv",
+ transfer_id=token.token_id,
+ metadata=metadata,
+ source_rank=self._nccl_ranks[source_worker_id],
+ owner_worker_id=target_worker_id,
+ ownership_epoch=token.source_epoch + 1,
+ model_output_credit_window=_MODEL_OUTPUT_PARENT_QUEUE_SIZE,
+ timeout=300.0,
+ )
await asyncio.gather(
- self._request(source_worker_id, "nccl_send", transfer_id=token.token_id, target_rank=self._nccl_ranks[target_worker_id], timeout=300.0),
- self._request(target_worker_id, "nccl_recv", transfer_id=token.token_id, source_rank=self._nccl_ranks[source_worker_id], timeout=300.0),
+ self._request(
+ source_worker_id,
+ "nccl_send",
+ transfer_id=token.token_id,
+ target_rank=self._nccl_ranks[target_worker_id],
+ timeout=300.0,
+ ),
+ self._request(
+ target_worker_id,
+ "nccl_recv",
+ transfer_id=token.token_id,
+ source_rank=self._nccl_ranks[source_worker_id],
+ timeout=300.0,
+ ),
+ )
+ await self._request(
+ source_worker_id, "nccl_commit_source", session_id=pipeline_session_id, timeout=300.0
)
- await self._request(source_worker_id, "nccl_commit_source", session_id=pipeline_session_id, timeout=300.0)
ownership = self._ownership.commit_migration(token)
+ self._pipeline_routes[pipeline_session_id] = target_worker_id
+ self._session_workers[pipeline_session_id] = target_worker_id
+ for chunk in self._migrating_controls.pop(pipeline_session_id, []):
+ self._send(
+ target_worker_id, {"type": "model_push", "session_id": pipeline_session_id, "chunk": chunk}
+ )
+ self._migration_total_ms.append((time.monotonic() - started) * 1000.0)
+ return ownership
except Exception:
with contextlib.suppress(Exception):
- await self._request(target_worker_id, "nccl_discard", transfer_id=token.token_id, session_id=pipeline_session_id)
+ await self._request(
+ target_worker_id,
+ "nccl_discard",
+ transfer_id=token.token_id,
+ session_id=pipeline_session_id,
+ )
with contextlib.suppress(Exception):
- await self._request(source_worker_id, "nccl_abort_source", session_id=pipeline_session_id, transfer_id=token.token_id)
+ await self._request(
+ source_worker_id,
+ "nccl_abort_source",
+ session_id=pipeline_session_id,
+ transfer_id=token.token_id,
+ )
+ if source_output_paused:
+ with contextlib.suppress(Exception):
+ await self._request(
+ source_worker_id,
+ "model_output_resume",
+ session_id=pipeline_session_id,
+ )
self._ownership.abort_migration(token)
for chunk in self._migrating_controls.pop(pipeline_session_id, []):
- self._send(source_worker_id, {"type": "model_push", "session_id": pipeline_session_id, "chunk": chunk})
+ self._send(
+ source_worker_id, {"type": "model_push", "session_id": pipeline_session_id, "chunk": chunk}
+ )
raise
+ finally:
+ # A failed state copy must not leave either GPU permanently
+ # paused. Resume is deliberately best-effort: the original
+ # migration exception is the meaningful caller-visible error.
await asyncio.gather(
self._request(source_worker_id, "scheduler_resume"),
self._request(target_worker_id, "scheduler_resume"),
return_exceptions=True,
)
- self._pipeline_routes[pipeline_session_id] = target_worker_id
- self._session_workers[pipeline_session_id] = target_worker_id
- for chunk in self._migrating_controls.pop(pipeline_session_id, []):
- self._send(target_worker_id, {"type": "model_push", "session_id": pipeline_session_id, "chunk": chunk})
- self._migration_total_ms.append((time.monotonic() - started) * 1000.0)
- await asyncio.gather(
- self._request(source_worker_id, "scheduler_resume"),
- self._request(target_worker_id, "scheduler_resume"),
- return_exceptions=True,
- )
- return ownership
def turboserve_snapshot(self) -> dict[str, object]:
snapshot = super().turboserve_snapshot()
- snapshot.update({
- "migration_supported": bool(self._nccl_ranks),
- "migration_backend": "process_nccl" if self._nccl_ranks else None,
- "nccl_ranks": dict(self._nccl_ranks),
- "worker_runtime_metrics": {worker_id: dict(self._worker_runtime_metrics.get(worker_id, {})) for worker_id in self._specs},
- "session_runtime_metrics": dict(self._session_runtime_metrics),
- "migration_calibration": {"average_total_ms": sum(self._migration_total_ms) / len(self._migration_total_ms) if self._migration_total_ms else 0.0},
- })
+ snapshot.update(
+ {
+ "migration_supported": bool(self._nccl_ranks),
+ "migration_backend": "process_nccl" if self._nccl_ranks else None,
+ "nccl_ranks": dict(self._nccl_ranks),
+ "worker_runtime_metrics": {
+ worker_id: dict(self._worker_runtime_metrics.get(worker_id, {})) for worker_id in self._specs
+ },
+ "session_runtime_metrics": dict(self._session_runtime_metrics),
+ "migration_calibration": {
+ "average_total_ms": sum(self._migration_total_ms) / len(self._migration_total_ms)
+ if self._migration_total_ms
+ else 0.0
+ },
+ "model_output_flow_control": self._model_output_flow_snapshot(),
+ }
+ )
return snapshot
+ def _model_output_flow_snapshot(self) -> dict[str, object]:
+ outputs = getattr(self, "_model_outputs", {})
+ inflight = getattr(self, "_model_output_inflight", set())
+ backlog: dict[str, int] = {}
+ for session_id, output in outputs.items():
+ qsize = getattr(output, "qsize", None)
+ if callable(qsize):
+ queued = int(qsize())
+ else:
+ queued = len(getattr(output, "items", ()))
+ backlog[session_id] = queued + int(session_id in inflight)
+ return {
+ "parent_queue_capacity": _MODEL_OUTPUT_PARENT_QUEUE_SIZE,
+ "ack_on_dequeue": True,
+ "max_materialized_payloads_per_session": _MODEL_OUTPUT_PARENT_QUEUE_SIZE + 1,
+ "backlog": backlog,
+ "dropped_payloads": dict(getattr(self, "_model_output_dropped", {})),
+ }
+
async def aclose(self) -> None:
for session_id in tuple(self._transport_workers):
with contextlib.suppress(Exception):
await self.stop_session(session_id)
if self._nccl_ranks:
- await asyncio.gather(*(self._request(worker_id, "nccl_destroy") for worker_id in self._nccl_ranks), return_exceptions=True)
+ await asyncio.gather(
+ *(self._request(worker_id, "nccl_destroy") for worker_id in self._nccl_ranks), return_exceptions=True
+ )
self._nccl_ranks.clear()
await super().aclose()
@@ -245,7 +559,14 @@ async def _init_nccl(self) -> None:
port = sock.getsockname()[1]
sock.close()
workers = sorted(self._active_workers)
- await asyncio.gather(*(self._request(worker_id, "nccl_init", rank=rank, world_size=len(workers), init_method=f"tcp://127.0.0.1:{port}") for rank, worker_id in enumerate(workers)))
+ await asyncio.gather(
+ *(
+ self._request(
+ worker_id, "nccl_init", rank=rank, world_size=len(workers), init_method=f"tcp://127.0.0.1:{port}"
+ )
+ for rank, worker_id in enumerate(workers)
+ )
+ )
self._nccl_ranks = {worker_id: rank for rank, worker_id in enumerate(workers)}
def _transport_finished(self, session_id: str) -> None:
@@ -261,25 +582,58 @@ def _dispatch_event(self, event: dict[str, Any]) -> None:
if event.get("type") == "model_output":
metrics = event.get("runtime_metrics")
if isinstance(metrics, dict):
- self._worker_runtime_metrics[event["worker_id"]] = {key: value for key, value in metrics.items() if isinstance(value, int | float)}
+ self._worker_runtime_metrics[event["worker_id"]] = {
+ key: value for key, value in metrics.items() if isinstance(value, int | float | str)
+ }
session_metrics = event.get("session_runtime_metrics")
if isinstance(session_metrics, dict):
- self._session_runtime_metrics[event["session_id"]] = {key: value for key, value in session_metrics.items() if isinstance(value, int | float)}
- if (output := self._model_outputs.get(event["session_id"])) is not None:
- output.put_nowait(event["payload"])
+ self._session_runtime_metrics[event["session_id"]] = {
+ key: value for key, value in session_metrics.items() if isinstance(value, int | float)
+ }
+ output_callback = getattr(self._event_sink, "on_model_output", None)
+ if callable(output_callback):
+ output_callback(
+ event["worker_id"],
+ event["session_id"],
+ event["payload"],
+ runtime_metrics=metrics if isinstance(metrics, dict) else None,
+ session_runtime_metrics=session_metrics if isinstance(session_metrics, dict) else None,
+ )
+ self._enqueue_model_output(
+ event["session_id"],
+ _ModelOutput(worker_id=event["worker_id"], payload=event["payload"]),
+ )
return
super()._dispatch_event(event)
-def _nccl_model_worker_main(spec: ProcessWorkerSpec, config_values: dict[str, Any], pipeline_file: str, skip_validation: bool, security_name: str | None, commands: Any, events: Any) -> None:
+def _nccl_model_worker_main(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
try:
- asyncio.run(_run_nccl_model_worker(spec, pipeline_file, skip_validation, security_name, commands, events))
+ asyncio.run(
+ _run_nccl_model_worker(spec, config_values, pipeline_file, skip_validation, security_name, commands, events)
+ )
finally:
_close_queue(commands, join=False)
_close_queue(events)
-async def _run_nccl_model_worker(spec: ProcessWorkerSpec, pipeline_file: str, skip_validation: bool, security_name: str | None, commands: Any, events: Any) -> None:
+async def _run_nccl_model_worker(
+ spec: ProcessWorkerSpec,
+ config_values: dict[str, Any],
+ pipeline_file: str,
+ skip_validation: bool,
+ security_name: str | None,
+ commands: Any,
+ events: Any,
+) -> None:
if not spec.gpu_ids:
raise RuntimeError("process-nccl requires one CUDA GPU per worker")
torch.cuda.set_device(int(spec.gpu_ids[0]))
@@ -287,22 +641,66 @@ async def _run_nccl_model_worker(spec: ProcessWorkerSpec, pipeline_file: str, sk
adapter.start(pipeline_file, skip_validation=skip_validation, gpu_num=1, gpu_ids=spec.gpu_ids)
if adapter.stream_mode != STREAM_MODE_BIDIRECTIONAL:
raise RuntimeError("process-nccl requires a bidirectional pipeline")
- profile = adapter.configure_session_capacity(None)
- events.put({"type": "worker_capacity", "worker_id": spec.worker_id, "capacity": int((profile or {}).get("effective_capacity", 1)), "profile": profile})
+ # Honour the operator ceiling in process-NCCL too. Previously this path
+ # always auto-sized and then overwrote --max-sessions-per-worker at the
+ # parent scheduler, which can violate a measured per-session FPS SLO.
+ from .config import LiveKitServeConfig
+
+ config = LiveKitServeConfig(**config_values)
+ profile = adapter.configure_session_capacity(config.session_capacity_limit())
+ events.put(
+ {
+ "type": "worker_capacity",
+ "worker_id": spec.worker_id,
+ "capacity": int((profile or {}).get("effective_capacity", 1)),
+ "profile": profile,
+ }
+ )
events.put({"type": "worker_status", "worker_id": spec.worker_id, "status": "idle"})
events.put({"type": "worker_ready", "worker_id": spec.worker_id})
service = adapter.stream_service.service
outputs: dict[str, asyncio.Task[None]] = {}
+ output_credits: dict[str, asyncio.BoundedSemaphore] = {}
outgoing: dict[str, dict[tuple[Any, ...], torch.Tensor]] = {}
- incoming: dict[str, tuple[dict[str, Any], dict[tuple[Any, ...], torch.Tensor], str, int]] = {}
+ incoming: dict[str, tuple[dict[str, Any], dict[tuple[Any, ...], torch.Tensor], str, int, int]] = {}
+
+ def start_pump(session_id: str, *, credit_window: int = _MODEL_OUTPUT_PARENT_QUEUE_SIZE) -> None:
+ credits = output_credits.get(session_id)
+ if credits is None:
+ credits = asyncio.BoundedSemaphore(max(1, int(credit_window)))
+ output_credits[session_id] = credits
+ if session_id not in outputs:
+ outputs[session_id] = asyncio.create_task(
+ _pump_model_outputs(
+ adapter,
+ service,
+ worker_id=spec.worker_id,
+ session_id=session_id,
+ credits=credits,
+ events=events,
+ ),
+ name=f"model-output-{session_id}",
+ )
- async def pump(session_id: str) -> None:
- async for payload in adapter.pull_chunks(session_id):
- events.put({"type": "model_output", "worker_id": spec.worker_id, "session_id": session_id, "payload": payload, "runtime_metrics": adapter.runtime_metrics() or {}, "session_runtime_metrics": service.runtime_metrics(session_id)})
+ async def stop_pump(session_id: str, *, drop_credit_state: bool = False) -> None:
+ task = outputs.pop(session_id, None)
+ if task is not None:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ if drop_credit_state:
+ output_credits.pop(session_id, None)
async def result(request_id: str | None, value: Any = True, error: Exception | None = None) -> None:
if request_id is not None:
- events.put({"type": "command_result", "worker_id": spec.worker_id, "request_id": request_id, "result": value, "error": repr(error) if error else None})
+ events.put(
+ {
+ "type": "command_result",
+ "worker_id": spec.worker_id,
+ "request_id": request_id,
+ "result": value,
+ "error": repr(error) if error else None,
+ }
+ )
try:
while True:
@@ -311,44 +709,75 @@ async def result(request_id: str | None, value: Any = True, error: Exception | N
try:
if kind == "model_create":
session_id = adapter.create_session(command["config"])
- outputs[session_id] = asyncio.create_task(pump(session_id))
+ start_pump(
+ session_id,
+ credit_window=int(command.get("model_output_credit_window", _MODEL_OUTPUT_PARENT_QUEUE_SIZE)),
+ )
elif kind == "model_push":
adapter.push_chunk(command["session_id"], command["chunk"])
elif kind == "model_close":
+ await stop_pump(command["session_id"], drop_credit_state=True)
adapter.close_session(command["session_id"])
- if (task := outputs.pop(command["session_id"], None)):
- task.cancel()
+ elif kind == "model_output_credit":
+ credits = output_credits.get(command["session_id"])
+ if credits is not None:
+ with contextlib.suppress(ValueError):
+ credits.release()
+ elif kind == "model_output_pause":
+ await stop_pump(command["session_id"])
+ elif kind == "model_output_resume":
+ has_session = getattr(service, "has_session", None)
+ if not callable(has_session) or has_session(command["session_id"]):
+ start_pump(command["session_id"])
elif kind == "nccl_init":
- await asyncio.to_thread(dist.init_process_group, "nccl", init_method=command["init_method"], rank=command["rank"], world_size=command["world_size"])
+ await asyncio.to_thread(
+ dist.init_process_group,
+ "nccl",
+ init_method=command["init_method"],
+ rank=command["rank"],
+ world_size=command["world_size"],
+ )
elif kind == "scheduler_pause":
await asyncio.to_thread(service.pause_scheduler)
elif kind == "scheduler_resume":
service.resume_scheduler()
elif kind == "nccl_export":
- metadata = service.prepare_migration_nccl_metadata(command["session_id"])
+ metadata = await asyncio.to_thread(service.prepare_migration_nccl_metadata, command["session_id"])
outgoing[command["transfer_id"]] = metadata.pop("_nccl_tensor_leaves")
await result(request_id, metadata)
continue
elif kind == "nccl_prepare_recv":
metadata = command["metadata"]
- leaves = allocate_tensor_tree_leaves(metadata["tensor_manifest"], torch.device(f"cuda:{spec.gpu_ids[0]}"))
- incoming[command["transfer_id"]] = (metadata, leaves, command["owner_worker_id"], command["ownership_epoch"])
+ leaves = allocate_tensor_tree_leaves(
+ metadata["tensor_manifest"], torch.device(f"cuda:{spec.gpu_ids[0]}")
+ )
+ incoming[command["transfer_id"]] = (
+ metadata,
+ leaves,
+ command["owner_worker_id"],
+ command["ownership_epoch"],
+ int(command.get("model_output_credit_window", _MODEL_OUTPUT_PARENT_QUEUE_SIZE)),
+ )
elif kind == "nccl_send":
- transfer_tensor_leaves_nccl(outgoing.pop(command["transfer_id"]), peer_rank=command["target_rank"], send=True)
+ transfer_tensor_leaves_nccl(
+ outgoing.pop(command["transfer_id"]), peer_rank=command["target_rank"], send=True
+ )
elif kind == "nccl_recv":
- metadata, leaves, owner, epoch = incoming.pop(command["transfer_id"])
+ metadata, leaves, owner, epoch, credit_window = incoming.pop(command["transfer_id"])
transfer_tensor_leaves_nccl(leaves, peer_rank=command["source_rank"], send=False)
- session_id = service.import_migration_nccl(metadata, leaves, owner_worker_id=owner, ownership_epoch=epoch)
- outputs[session_id] = asyncio.create_task(pump(session_id))
+ session_id = service.import_migration_nccl(
+ metadata, leaves, owner_worker_id=owner, ownership_epoch=epoch
+ )
+ start_pump(session_id, credit_window=credit_window)
elif kind == "nccl_commit_source":
+ await stop_pump(command["session_id"], drop_credit_state=True)
service.commit_migration(command["session_id"])
- if (task := outputs.pop(command["session_id"], None)):
- task.cancel()
elif kind == "nccl_abort_source":
service.abort_migration(command["session_id"])
outgoing.pop(command.get("transfer_id", ""), None)
elif kind == "nccl_discard":
incoming.pop(command["transfer_id"], None)
+ await stop_pump(command["session_id"], drop_credit_state=True)
if service.has_session(command["session_id"]):
service.close_session(command["session_id"])
elif kind == "nccl_destroy":
@@ -362,9 +791,8 @@ async def result(request_id: str | None, value: Any = True, error: Exception | N
except Exception as exc:
await result(request_id, error=exc)
finally:
- for task in outputs.values():
- task.cancel()
- await asyncio.gather(*outputs.values(), return_exceptions=True)
+ for session_id in tuple(outputs):
+ await stop_pump(session_id, drop_credit_state=True)
if dist.is_initialized():
with contextlib.suppress(Exception):
dist.destroy_process_group()
diff --git a/telefuser/service/livekit/pipeline_adapter.py b/telefuser/service/livekit/pipeline_adapter.py
index 54c7090d..e1cbdd22 100644
--- a/telefuser/service/livekit/pipeline_adapter.py
+++ b/telefuser/service/livekit/pipeline_adapter.py
@@ -60,7 +60,7 @@ def configure_session_capacity(self, max_sessions: int | None) -> dict[str, obje
"""Configure and return the loaded pipeline's optional capacity profile."""
return self.stream_service.configure_session_capacity(max_sessions)
- def runtime_metrics(self) -> dict[str, float | int] | None:
+ def runtime_metrics(self) -> dict[str, float | int | str] | None:
"""Return optional model-service scheduling measurements for placement."""
service = getattr(self.stream_service, "service", None)
metrics = getattr(service, "runtime_metrics", None)
diff --git a/telefuser/service/livekit/pipeline_router.py b/telefuser/service/livekit/pipeline_router.py
index 00d99de8..1664c4ce 100644
--- a/telefuser/service/livekit/pipeline_router.py
+++ b/telefuser/service/livekit/pipeline_router.py
@@ -148,7 +148,7 @@ def snapshot(self) -> dict[str, object]:
retained_by_worker = {worker_id: 0 for worker_id in self._backends}
for worker_id in routes.values():
retained_by_worker[worker_id] += 1
- runtime_metrics: dict[str, dict[str, float | int]] = {}
+ runtime_metrics: dict[str, dict[str, float | int | str]] = {}
for worker_id, backend in self._backends.items():
metrics = getattr(backend, "runtime_metrics", None)
if not callable(metrics):
diff --git a/telefuser/service/livekit/process_worker_pool.py b/telefuser/service/livekit/process_worker_pool.py
index 20bcc4ee..7eb0b144 100644
--- a/telefuser/service/livekit/process_worker_pool.py
+++ b/telefuser/service/livekit/process_worker_pool.py
@@ -319,6 +319,29 @@ def _dispatch_event(self, event: dict[str, Any]) -> None:
if owner == worker_id and pipeline_session_id == event.get("pipeline_session_id"):
self._pipeline_routes.pop(pipeline_session_id, None)
self._event_sink.on_session_finished(worker_id, session_id, event.get("error"))
+ elif event_type == "control_received":
+ callback = getattr(self._event_sink, "on_control_received", None)
+ if callable(callback):
+ callback(worker_id, event["session_id"])
+ elif event_type == "chunk_published":
+ callback = getattr(self._event_sink, "on_chunk_published", None)
+ if callable(callback):
+ callback(
+ worker_id,
+ event["session_id"],
+ int(event.get("frames", 0)),
+ event.get("first_frame_at"),
+ )
+ elif event_type == "model_output":
+ callback = getattr(self._event_sink, "on_model_output", None)
+ if callable(callback):
+ callback(
+ worker_id,
+ event["session_id"],
+ event["payload"],
+ runtime_metrics=event.get("runtime_metrics"),
+ session_runtime_metrics=event.get("session_runtime_metrics"),
+ )
async def _monitor_loop(self) -> None:
while True:
@@ -637,3 +660,40 @@ def on_session_finished(self, worker_id: str, session_id: str, error: str | None
"error": error,
}
)
+
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ self.events.put(
+ {"type": "control_received", "worker_id": worker_id, "session_id": session_id}
+ )
+
+ def on_chunk_published(
+ self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
+ ) -> None:
+ self.events.put(
+ {
+ "type": "chunk_published",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ "frames": frames,
+ "first_frame_at": first_frame_at,
+ }
+ )
+
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None:
+ self.events.put(
+ {
+ "type": "model_output",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ "payload": payload,
+ "runtime_metrics": runtime_metrics,
+ "session_runtime_metrics": session_runtime_metrics,
+ }
+ )
diff --git a/telefuser/service/livekit/runtime.py b/telefuser/service/livekit/runtime.py
index d7edd45c..186bc0f4 100644
--- a/telefuser/service/livekit/runtime.py
+++ b/telefuser/service/livekit/runtime.py
@@ -11,11 +11,12 @@
from telefuser.utils.logging import logger
from .config import LiveKitServeConfig
+from .metrics import LiveKitServingMetrics
from .multi_session_worker import MultiSessionLiveKitWorker as LiveKitWorker
+from .nccl_process_worker_pool import NCCLProcessLiveKitWorkerPool
from .pipeline_adapter import LiveKitPipelineAdapter
from .pipeline_router import TurboServePipelineRouter
from .process_worker_pool import ProcessLiveKitWorkerPool, ProcessWorkerSpec
-from .nccl_process_worker_pool import NCCLProcessLiveKitWorkerPool
from .scheduler import LiveKitScheduler, SchedulerAdmission
from .schemas import (
LiveKitHealthResponse,
@@ -28,17 +29,17 @@
from .session_registry import TERMINAL_SESSION_STATUSES, SessionRecord, SessionRegistry
from .token_service import LiveKitTokenService
from .turboserve import (
- TurboServeClusterScheduler,
- TurboServeRuntimeCalibration,
- TurboServeSchedulerConfig,
- TurboServeSchedulingSnapshot,
- TurboServeSessionView,
TurboServeAutoscalingController,
+ TurboServeClusterScheduler,
TurboServeMigrationPlan,
TurboServeOwnership,
TurboServePlacementController,
+ TurboServeRuntimeCalibration,
TurboServeScaleDecision,
+ TurboServeSchedulerConfig,
+ TurboServeSchedulingSnapshot,
TurboServeSessionDemand,
+ TurboServeSessionView,
TurboServeWorkerLoad,
)
from .worker_pool import InProcessLiveKitWorkerPool, WorkerPool
@@ -121,6 +122,7 @@ def __init__(
self._last_scale_decision: TurboServeScaleDecision | None = None
self._last_migration_plan: TurboServeMigrationPlan | None = None
self._last_migration_error: str | None = None
+ self._serving_metrics = LiveKitServingMetrics()
self._lock = threading.RLock()
@property
@@ -170,6 +172,7 @@ def create_session(self, request: SessionCreateRequest) -> CreateSessionResult:
timeout_s=self.config.session_timeout,
)
admission = self.scheduler.assign(session_id=session_id, room_name=room_name)
+ self._serving_metrics.record_admission(admission.status)
if admission.status == "rejected":
self.registry.delete(session_id)
return CreateSessionResult(record=record, token="", admission=admission)
@@ -258,7 +261,53 @@ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None
def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
"""Release capacity after a worker session exits."""
del worker_id
- self._finish_session(session_id, error=error)
+ record = self._finish_session(session_id, error=error)
+ self._serving_metrics.record_session_finished(record.status, error=error)
+
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ """Record a validated controller action entering the serving pipeline."""
+ self._serving_metrics.on_control_received(worker_id, session_id)
+
+ def on_chunk_published(
+ self,
+ worker_id: str,
+ session_id: str,
+ frames: int,
+ first_frame_at: float | None = None,
+ ) -> None:
+ """Record frames accepted by the LiveKit video publisher."""
+ self._serving_metrics.on_chunk_published(
+ worker_id=worker_id,
+ session_id=session_id,
+ frames=frames,
+ first_frame_at=first_frame_at,
+ )
+
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None:
+ """Ingest a child-model output forwarded by the process-NCCL pool."""
+ self._serving_metrics.on_model_output(
+ self,
+ worker_id=worker_id,
+ pipeline_session_id=session_id,
+ payload=payload,
+ runtime_metrics=runtime_metrics,
+ session_runtime_metrics=session_runtime_metrics,
+ )
+
+ def prometheus_metrics(self) -> str:
+ """Render runtime, scheduler, session, and pipeline serving metrics."""
+ return self._serving_metrics.render_prometheus(self)
+
+ def serving_metrics_snapshot(self) -> dict:
+ """Return aggregate serving metrics for the JSON endpoint and experiments."""
+ return self._serving_metrics.json_snapshot(self)
def health(self) -> LiveKitHealthResponse:
"""Return service health based on current scheduler state."""
@@ -347,7 +396,16 @@ async def migrate_session(self, session_id: str, target_worker_id: str) -> Turbo
migrate = getattr(self.worker_pool, "migrate_session", None)
if not callable(migrate):
raise RuntimeError("Configured worker pool does not support TurboServe migration")
- ownership = await migrate(record.pipeline_session_id, target_worker_id)
+ migration_started_at = asyncio.get_running_loop().time()
+ try:
+ ownership = await migrate(record.pipeline_session_id, target_worker_id)
+ except Exception as exc:
+ self._serving_metrics.record_migration(success=False, error=str(exc))
+ raise
+ self._serving_metrics.record_migration(
+ success=True,
+ duration_seconds=asyncio.get_running_loop().time() - migration_started_at,
+ )
self.scheduler.reassign_session(session_id, target_worker_id)
self.registry.assign_worker(session_id, target_worker_id)
return ownership
@@ -384,7 +442,11 @@ def _create_worker_pool(self) -> WorkerPool:
ProcessWorkerSpec(worker_id=state.worker_id, gpu_ids=list(state.gpu_ids))
for state in self.scheduler.workers()
]
- pool_type = NCCLProcessLiveKitWorkerPool if self.config.worker_mode == "process-nccl" else ProcessLiveKitWorkerPool
+ pool_type = (
+ NCCLProcessLiveKitWorkerPool
+ if self.config.worker_mode == "process-nccl"
+ else ProcessLiveKitWorkerPool
+ )
return pool_type(
specs,
config=self.config,
@@ -501,7 +563,11 @@ async def _turboserve_control_once(self) -> None:
if not isinstance(worker_metrics, dict):
worker_metrics = {}
base_latency_ms = max(
- (float(values.get("p95_chunk_seconds", 0.0)) * 1000 for values in worker_metrics.values() if isinstance(values, dict)),
+ (
+ float(values.get("p95_chunk_seconds", 0.0)) * 1000
+ for values in worker_metrics.values()
+ if isinstance(values, dict)
+ ),
default=0.0,
)
decision = self._cluster_scheduler.decide(
@@ -611,7 +677,11 @@ async def _rebalance_once(self) -> None:
profiles = self._worker_capacity_profiles
sessions: list[TurboServeSessionDemand] = []
for record in self.registry.list_records():
- if record.status in TERMINAL_SESSION_STATUSES or record.pipeline_session_id is None or record.worker_id is None:
+ if (
+ record.status in TERMINAL_SESSION_STATUSES
+ or record.pipeline_session_id is None
+ or record.worker_id is None
+ ):
continue
profile = profiles.get(record.worker_id, {})
state_bytes = int(profile.get("estimated_session_bytes", 1)) if isinstance(profile, dict) else 1
diff --git a/telefuser/service/livekit/worker.py b/telefuser/service/livekit/worker.py
index 8ee208e7..18d03258 100644
--- a/telefuser/service/livekit/worker.py
+++ b/telefuser/service/livekit/worker.py
@@ -36,6 +36,18 @@ def on_worker_capacity(self, worker_id: str, capacity: int, profile: dict[str, o
def on_session_status(self, session_id: str, status: SessionStatus, error: str | None = None) -> None: ...
def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None: ...
def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None: ...
+ def on_control_received(self, worker_id: str, session_id: str) -> None: ...
+ def on_chunk_published(
+ self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
+ ) -> None: ...
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None: ...
class NullWorkerEventSink:
@@ -56,6 +68,24 @@ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None
def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
return None
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ return None
+
+ def on_chunk_published(
+ self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
+ ) -> None:
+ return None
+
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None:
+ return None
+
class LiveKitWorker:
"""Owns one active LiveKit room and one active TeleFuser pipeline session."""
@@ -208,6 +238,9 @@ def _on_data_message(
self._delivery_ack_event.set()
return
+ control_callback = getattr(self.event_sink, "on_control_received", None)
+ if callable(control_callback):
+ control_callback(self.worker_id, record.session_id)
self.pipeline_adapter.push_chunk(self._pipeline_session_id, chunk)
if chunk.get("type") == "stop":
self._stop_event.set()
@@ -227,6 +260,22 @@ async def _publish_pipeline_chunks(
break
frames, audio, metadata = split_chunk_media(chunk)
+ model_output_callback = getattr(self.event_sink, "on_model_output", None)
+ if callable(model_output_callback) and self._pipeline_session_id is not None:
+ chunk_data_for_metrics = chunk.get("data") if isinstance(chunk.get("data"), dict) else chunk
+ scheduler = (
+ chunk_data_for_metrics.get("scheduler") if isinstance(chunk_data_for_metrics, dict) else None
+ )
+ model_output_callback(
+ self.worker_id,
+ self._pipeline_session_id,
+ {
+ "type": chunk.get("type"),
+ "fps": chunk_data_for_metrics.get("fps", chunk.get("fps", self.config.default_fps)),
+ "scheduler": dict(scheduler) if isinstance(scheduler, dict) else {},
+ "frame_count": len(frames),
+ },
+ )
decoded_ready_at = chunk.get("timestamp")
publish_started_at = time.time()
publish_started_monotonic = time.monotonic()
@@ -244,6 +293,7 @@ async def _publish_pipeline_chunks(
await self.room_client.publish_video_track("telefuser-output", width, height, fps=fps)
await asyncio.sleep(_VIDEO_TRACK_SUBSCRIPTION_GRACE_SECONDS)
+ first_frame_at: float | None = None
for frame in frames:
if self._stop_event.is_set():
break
@@ -254,6 +304,8 @@ async def _publish_pipeline_chunks(
if delay > 0:
await asyncio.sleep(delay)
await self.room_client.publish_video_frame(frame, fps=fps)
+ if first_frame_at is None:
+ first_frame_at = time.monotonic()
next_frame_at += frame_interval
published_frames += 1
@@ -267,6 +319,9 @@ async def _publish_pipeline_chunks(
if self._stop_event.is_set():
break
if frames:
+ published_callback = getattr(self.event_sink, "on_chunk_published", None)
+ if callable(published_callback):
+ published_callback(self.worker_id, session_id, len(frames), first_frame_at)
chunk_count += 1
metadata["transport_measurement"] = {
"decoded_ready_at": decoded_ready_at if isinstance(decoded_ready_at, int | float) else None,
diff --git a/tests/unit/pipelines/abot_world/test_interactive.py b/tests/unit/pipelines/abot_world/test_interactive.py
index 48fd5cf5..75760f84 100644
--- a/tests/unit/pipelines/abot_world/test_interactive.py
+++ b/tests/unit/pipelines/abot_world/test_interactive.py
@@ -76,3 +76,63 @@ def test_cache_collation_and_scatter_preserve_session_isolation() -> None:
assert second.self_cache[0]["k"].item() == 14
first.self_cache[0]["k"].zero_()
assert second.self_cache[0]["k"].item() == 14
+
+
+class _DecodeMetricsStage:
+ def __init__(self) -> None:
+ self.states: list[object] = []
+
+ def decode_chunks(self, latents: torch.Tensor, states: list[object]) -> torch.Tensor:
+ self.states = list(states)
+ return latents
+
+ @staticmethod
+ def last_decode_metrics() -> dict[str, int]:
+ return {
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 1,
+ "taew_decode_invocations": 2,
+ "taew_decode_mode": 2,
+ }
+
+
+def test_generate_next_blocks_surfaces_effective_taew_decode_metrics() -> None:
+ pipeline = ABotWorldInteractivePipeline(device="cpu", torch_dtype=torch.float32)
+ pipeline.config = SimpleNamespace(height=32, width=32)
+
+ def denoise(noise: torch.Tensor, *_: object) -> torch.Tensor:
+ return noise
+
+ pipeline.denoise_stage = SimpleNamespace(
+ dit=SimpleNamespace(use_relative_rope=True),
+ _denoise_block=denoise,
+ )
+ decode_stage = _DecodeMetricsStage()
+ pipeline.taew_decode_stage = decode_stage
+ pipeline.tensor2video = lambda decoded: [object() for _ in range(decoded.shape[1])]
+
+ def make_session(session_id: str) -> ABotWorldInteractiveSession:
+ return ABotWorldInteractiveSession(
+ session_id=session_id,
+ prompt_emb=torch.ones(1, 1),
+ first_frame_latent=torch.ones(1, 1, 1, 1, 1),
+ self_cache=[],
+ cross_cache=[],
+ scheduler=object(),
+ generator=torch.Generator(device="cpu").manual_seed(3),
+ taew_decode_state=object(),
+ )
+
+ first = make_session("first")
+ second = make_session("second")
+ pipeline._interactive_sessions = {"first": first, "second": second}
+
+ output = pipeline.generate_next_blocks([first, second], [{"W": True}, {"D": True}])
+
+ assert [len(frames) for frames in output] == [3, 3]
+ assert decode_stage.states == [first.taew_decode_state, second.taew_decode_state]
+ assert pipeline.last_stage_metrics()["batch_size"] == 2
+ assert pipeline.last_stage_metrics()["taew_decode_items"] == 2
+ assert pipeline.last_stage_metrics()["taew_decode_batch_size"] == 1
+ assert pipeline.last_stage_metrics()["taew_decode_invocations"] == 2
+ assert pipeline.last_stage_metrics()["taew_decode_mode"] == 2
diff --git a/tests/unit/pipelines/abot_world/test_livekit_examples.py b/tests/unit/pipelines/abot_world/test_livekit_examples.py
index 548e8dd4..6eb0d7de 100644
--- a/tests/unit/pipelines/abot_world/test_livekit_examples.py
+++ b/tests/unit/pipelines/abot_world/test_livekit_examples.py
@@ -12,6 +12,10 @@ def test_livekit_service_entrypoint_builds_single_gpu_abot_service(monkeypatch:
pipeline = object()
captured: dict[str, object] = {}
+ monkeypatch.delenv("TELEFUSER_ABOT_SCHEDULER_MODE", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_MAX_BATCH_SIZE", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_BATCHING_WINDOW_MS", raising=False)
+
def fake_get_pipeline(**kwargs: object) -> object:
captured.update(kwargs)
return pipeline
@@ -22,14 +26,53 @@ def fake_get_pipeline(**kwargs: object) -> object:
assert isinstance(service, ABotWorldLiveKitService)
assert service.pipeline is pipeline
assert captured == {"device_id": 3, "pipeline_class": ABotWorldInteractivePipeline}
- assert service.default_fps == 8
- assert service.default_session_config["fps"] == 8
- assert service.default_session_config["control_latent_frames"] == 2
+ assert service.default_fps == 12
+ assert service.default_session_config["fps"] == 12
+ assert service.default_session_config["control_latent_frames"] == 3
+ assert service.scheduler_mode == "batched"
+ assert service.max_batch_size == 2
assert service.default_session_config["seed"] == 42
assert service.default_session_config["prompt"] == service_example.DEFAULT_PROMPT
assert str(service.default_session_config["image_path"]).endswith("84b90ad568b693d2.png")
+def test_livekit_service_entrypoint_selects_batched_four_session_schedule_from_environment(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ pipeline = object()
+ monkeypatch.setattr(service_example, "get_pipeline", lambda **_kwargs: pipeline)
+ monkeypatch.setenv("TELEFUSER_ABOT_SCHEDULER_MODE", "batched")
+ monkeypatch.setenv("TELEFUSER_ABOT_MAX_BATCH_SIZE", "4")
+ monkeypatch.setenv("TELEFUSER_ABOT_BATCHING_WINDOW_MS", "2")
+
+ service = service_example.get_service(gpu_num=1, gpu_ids=["0"])
+
+ assert service.pipeline is pipeline
+ assert service.scheduler_mode == "batched"
+ assert service.max_batch_size == 4
+ assert service.batching_window_seconds == pytest.approx(0.002)
+
+
+@pytest.mark.parametrize(
+ ("environment", "expected"),
+ [
+ ({"TELEFUSER_ABOT_SCHEDULER_MODE": "unknown"}, "SCHEDULER_MODE"),
+ ({"TELEFUSER_ABOT_MAX_BATCH_SIZE": "0"}, "MAX_BATCH_SIZE"),
+ ({"TELEFUSER_ABOT_BATCHING_WINDOW_MS": "nan"}, "BATCHING_WINDOW_MS"),
+ ],
+)
+def test_livekit_service_entrypoint_rejects_invalid_schedule_environment(
+ monkeypatch: pytest.MonkeyPatch,
+ environment: dict[str, str],
+ expected: str,
+) -> None:
+ for key, value in environment.items():
+ monkeypatch.setenv(key, value)
+
+ with pytest.raises(ValueError, match=expected):
+ service_example.get_service(gpu_num=1, gpu_ids=["0"])
+
+
def test_livekit_service_entrypoint_rejects_non_numeric_gpu_id() -> None:
with pytest.raises(ValueError, match="must be numeric"):
service_example.get_service(gpu_num=1, gpu_ids=["GPU-deadbeef"])
diff --git a/tests/unit/pipelines/abot_world/test_livekit_service.py b/tests/unit/pipelines/abot_world/test_livekit_service.py
index 4b19597f..0e4b9a80 100644
--- a/tests/unit/pipelines/abot_world/test_livekit_service.py
+++ b/tests/unit/pipelines/abot_world/test_livekit_service.py
@@ -50,6 +50,8 @@ def __init__(self) -> None:
)
)
self.generate_calls: list[tuple[str, dict[str, bool]]] = []
+ self.call_times: list[float] = []
+ self.frames_per_chunk = 1
self.batch_sizes: list[int] = []
self.closed_sessions: list[str] = []
self.suspended_sessions: list[str] = []
@@ -82,9 +84,13 @@ def generate_next_block(
) -> list[Image.Image]:
assert control_latent_frames == 3
self.generate_calls.append((session.session_id, controls))
+ self.call_times.append(time.monotonic())
self.batch_sizes.append(1)
session.next_latent_frame += control_latent_frames
- return [Image.new("RGB", (8, 8), color=(20, len(self.generate_calls) % 255, 40))]
+ return [
+ Image.new("RGB", (8, 8), color=(20, len(self.generate_calls) % 255, 40))
+ for _ in range(self.frames_per_chunk)
+ ]
def generate_next_blocks(
self,
@@ -97,8 +103,14 @@ def generate_next_blocks(
results = []
for session, state in zip(sessions, controls):
self.generate_calls.append((session.session_id, state))
+ self.call_times.append(time.monotonic())
session.next_latent_frame += control_latent_frames
- results.append([Image.new("RGB", (8, 8), color=(20, len(self.generate_calls) % 255, 40))])
+ results.append(
+ [
+ Image.new("RGB", (8, 8), color=(20, len(self.generate_calls) % 255, 40))
+ for _ in range(self.frames_per_chunk)
+ ]
+ )
return results
def suspend_interactive_session(self, session: _FakePipelineSession) -> None:
@@ -137,6 +149,27 @@ def _create(service: ABotWorldLiveKitService, session_id: str, **config: object)
)
+def _take_and_notify(
+ service: ABotWorldLiveKitService,
+ state: _ABotWorldLiveKitSession,
+ *,
+ timeout: float = 1.0,
+) -> dict[str, object]:
+ payload = state.output_queue.get(timeout=timeout)
+ with service._scheduler_condition:
+ service._scheduler_condition.notify_all()
+ return payload
+
+
+def _wait_for(predicate, *, timeout: float = 2.0) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if predicate():
+ return
+ time.sleep(0.002)
+ assert predicate()
+
+
def test_service_matches_shared_multi_session_bidirectional_contract() -> None:
service, _ = _service()
assert isinstance(service, BidirectionalService)
@@ -168,7 +201,7 @@ def fake_mem_get_info(device):
assert profile["effective_capacity"] == 2
service.stop()
-def test_round_robin_capacity_uses_one_active_workspace(monkeypatch) -> None:
+def test_batched_capacity_accounts_for_active_batch_workspace(monkeypatch) -> None:
service, pipeline = _service(max_batch_size=8)
pipeline.device = "cuda:0"
monkeypatch.setattr("telefuser.pipelines.abot_world.service.torch.cuda.is_available", lambda: True)
@@ -184,16 +217,16 @@ def test_round_robin_capacity_uses_one_active_workspace(monkeypatch) -> None:
profile = service.configure_session_capacity(10)
- assert profile["computed_capacity"] == 7
- assert profile["effective_capacity"] == 7
- assert profile["estimated_batch_workspace_bytes"] == 200
- assert profile["scheduler_mode"] == "round_robin"
+ assert profile["computed_capacity"] == 3
+ assert profile["effective_capacity"] == 3
+ assert profile["estimated_batch_workspace_bytes"] == 600
+ assert profile["scheduler_mode"] == "batched"
service.stop()
def test_two_ready_sessions_are_generated_in_one_batch_and_keep_order() -> None:
- service, pipeline = _service(output_queue_size=4, batching_window_ms=30, scheduler_mode="batched")
+ service, pipeline = _service(output_queue_size=4, batching_window_ms=30)
service.configure_session_capacity(2)
first = _create(service, "first")
second = _create(service, "second")
@@ -230,7 +263,7 @@ def test_service_accepts_two_latent_experimental_chunk() -> None:
service.stop()
-def test_default_scheduler_round_robins_single_session_steps() -> None:
+def test_default_scheduler_coalesces_compatible_sessions() -> None:
service, pipeline = _service(output_queue_size=4)
service.configure_session_capacity(2)
first = _create(service, "first")
@@ -246,13 +279,249 @@ def test_default_scheduler_round_robins_single_session_steps() -> None:
first_chunk = first_state.output_queue.get(timeout=2)
second_chunk = second_state.output_queue.get(timeout=2)
- assert first_chunk["scheduler"]["batch_size"] == 1
- assert second_chunk["scheduler"]["batch_size"] == 1
- assert pipeline.batch_sizes[:2] == [1, 1]
- assert service.runtime_metrics()["scheduler_mode"] == "round_robin"
+ assert first_chunk["scheduler"]["batch_size"] == 2
+ assert second_chunk["scheduler"]["batch_size"] == 2
+ assert pipeline.batch_sizes[:1] == [2]
+ assert service.runtime_metrics()["scheduler_mode"] == "batched"
service.stop()
+def test_round_robin_remains_single_session_ablation() -> None:
+ service, pipeline = _service(output_queue_size=4, scheduler_mode="round_robin")
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ second = _create(service, "second")
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ try:
+ assert _take_and_notify(service, first_state)["type"] == "preview"
+ assert _take_and_notify(service, second_state)["type"] == "preview"
+ service.push_chunk(first, {"type": "control_state", "controls": ["KeyW"]})
+ service.push_chunk(second, {"type": "control_state", "controls": ["KeyD"]})
+ assert _take_and_notify(service, first_state, timeout=2)["type"] == "chunk"
+ assert _take_and_notify(service, second_state, timeout=2)["type"] == "chunk"
+ assert pipeline.batch_sizes[:2] == [1, 1]
+ finally:
+ service.stop()
+
+
+def test_latest_mode_bounds_prefetch_and_resumes_at_playout_deadline() -> None:
+ service, pipeline = _service(output_queue_size=4, batching_window_ms=0, control_idle_timeout=30)
+ pipeline.frames_per_chunk = 12
+ service.configure_session_capacity(1)
+ session_id = _create(service, "paced", fps=12, control_latent_frames=3)
+ state = service._session(session_id)
+ assert state is not None
+ try:
+ assert _take_and_notify(service, state)["type"] == "preview"
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
+ _wait_for(lambda: len(pipeline.generate_calls) >= 1)
+ assert _take_and_notify(service, state)["type"] == "chunk"
+ _wait_for(lambda: len(pipeline.generate_calls) >= 2)
+
+ with state.lock:
+ expected_resume_at = state.next_playout_deadline - state.last_chunk_duration_seconds
+ assert state.pacing_ready_at <= expected_resume_at
+ remaining = expected_resume_at - time.monotonic()
+ if remaining > 0.05:
+ time.sleep(remaining - 0.05)
+ # The second chunk is the sole prefetch. It remains queued while the
+ # first 12-frame chunk is being played, so there is no free-running c3.
+ assert len(pipeline.generate_calls) == 2
+ metrics = service.runtime_metrics(session_id)
+ assert metrics["pacing_buffered_video_payloads"] == 1
+ assert service.runtime_metrics()["pacing_throttled_sessions"] == 1
+
+ remaining = expected_resume_at - time.monotonic()
+ if remaining > 0:
+ time.sleep(remaining)
+ assert _take_and_notify(service, state)["type"] == "chunk"
+ dequeued_at = time.monotonic()
+ _wait_for(lambda: len(pipeline.generate_calls) >= 3)
+ third_started_at = pipeline.call_times[2]
+ assert third_started_at >= expected_resume_at - 0.05
+ assert third_started_at <= dequeued_at + 0.15
+ finally:
+ service.stop()
+
+
+def test_latest_mode_batches_mildly_staggered_playout_consumers() -> None:
+ service, pipeline = _service(output_queue_size=4, batching_window_ms=20, control_idle_timeout=30)
+ pipeline.frames_per_chunk = 12
+ service.configure_session_capacity(2)
+ first = _create(service, "first", fps=12, control_latent_frames=3)
+ second = _create(service, "second", fps=12, control_latent_frames=3)
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ try:
+ assert _take_and_notify(service, first_state)["type"] == "preview"
+ assert _take_and_notify(service, second_state)["type"] == "preview"
+ service.push_chunk(first, {"type": "control_state", "controls": ["KeyW"]})
+ service.push_chunk(second, {"type": "control_state", "controls": ["KeyD"]})
+ _wait_for(lambda: len(pipeline.batch_sizes) >= 1)
+ assert pipeline.batch_sizes[0] == 2
+
+ assert _take_and_notify(service, first_state)["type"] == "chunk"
+ time.sleep(0.003)
+ assert _take_and_notify(service, second_state)["type"] == "chunk"
+ _wait_for(lambda: len(pipeline.batch_sizes) >= 2)
+ assert pipeline.batch_sizes[1] == 2
+
+ with first_state.lock:
+ continuation_at = first_state.next_playout_deadline - first_state.last_chunk_duration_seconds
+ remaining = continuation_at - time.monotonic()
+ if remaining > 0:
+ time.sleep(remaining)
+ assert _take_and_notify(service, first_state)["type"] == "chunk"
+ time.sleep(0.003)
+ assert _take_and_notify(service, second_state)["type"] == "chunk"
+ _wait_for(lambda: len(pipeline.batch_sizes) >= 3)
+ assert pipeline.batch_sizes[2] == 2
+ finally:
+ service.stop()
+
+
+def _prepare_latest_continuation(
+ state: _ABotWorldLiveKitSession,
+ *,
+ now: float,
+ pacing_ready_at: float,
+ next_playout_deadline: float,
+) -> None:
+ with state.lock:
+ state.controls = {"W"}
+ state.ready_since = now
+ state.scheduled_chunks = 2
+ state.last_chunk_duration_seconds = 1.0
+ state.last_compute_seconds = 0.05
+ state.pacing_ready_at = pacing_ready_at
+ state.next_playout_deadline = next_playout_deadline
+ state.pipeline_session.next_latent_frame = 6
+
+
+def test_latest_mode_rendezvouses_staggered_continuations_within_deadline_slack() -> None:
+ service, pipeline = _service(output_queue_size=4, batching_window_ms=2, control_idle_timeout=30)
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ second = _create(service, "second")
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ try:
+ now = 10_000.0
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ _prepare_latest_continuation(
+ second_state,
+ now=now,
+ pacing_ready_at=now + 0.12,
+ next_playout_deadline=now + 1.12,
+ )
+
+ ready = service._ready_sessions(now)
+ assert [state.session_id for state in ready] == [first]
+ wait_seconds = service._batch_formation_wait_seconds(ready, now)
+ # 10 ms pacing slack makes the second continuation eligible at +110 ms.
+ assert wait_seconds == pytest.approx(0.11, abs=0.002)
+
+ ready = service._ready_sessions(now + wait_seconds + 0.001)
+ batch = service._select_batch(ready, now=now + wait_seconds + 0.001)
+ assert [state.session_id for state in batch] == [first, second]
+ service._execute_batch(batch, [{"W": True}, {"D": True}])
+ assert pipeline.batch_sizes[-1] == 2
+ finally:
+ service.stop()
+
+
+def test_latest_mode_rendezvous_never_waits_past_a_playout_deadline() -> None:
+ service, _ = _service(output_queue_size=4, batching_window_ms=2, control_idle_timeout=30)
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ second = _create(service, "second")
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ try:
+ now = 20_000.0
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 0.20,
+ )
+ _prepare_latest_continuation(
+ second_state,
+ now=now,
+ pacing_ready_at=now + 0.16,
+ next_playout_deadline=now + 0.36,
+ )
+
+ ready = service._ready_sessions(now)
+ assert [state.session_id for state in ready] == [first]
+ wait_seconds = service._batch_formation_wait_seconds(ready, now)
+ # A B=2 estimate is 2 * 50 ms * 1.1, leaving only 90 ms for first.
+ # The peer releases at 150 ms, so use only the legacy 2 ms window.
+ assert wait_seconds == pytest.approx(service.batching_window_seconds)
+ assert now + wait_seconds < service._latest_safe_batch_start([first_state, second_state])
+ ready_after_window = service._ready_sessions(now + wait_seconds)
+ assert [state.session_id for state in service._select_batch(ready_after_window, now=now + wait_seconds)] == [first]
+ finally:
+ service.stop()
+
+
+def test_latest_mode_falls_back_to_singleton_when_observed_batch_misses_deadline() -> None:
+ service, _ = _service(output_queue_size=4, batching_window_ms=2, control_idle_timeout=30)
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ second = _create(service, "second")
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ try:
+ now = 30_000.0
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 0.50,
+ )
+ _prepare_latest_continuation(
+ second_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.00,
+ )
+ # A measured B=2 takes 1.01 seconds, while each B=1 takes 0.40s.
+ # With the 10% safety margin, B=2 misses the first deadline, but B=1
+ # followed by B=1 still fits the two respective deadlines.
+ service._batch_compute_estimates.update({1: 0.40, 2: 1.01})
+ ready = service._ready_sessions(now)
+ assert [state.session_id for state in ready] == [first, second]
+ assert service._latest_safe_batch_start(ready) < now
+ assert now <= service._latest_safe_batch_start([first_state])
+ assert now + service._estimated_batch_compute_seconds([first_state]) <= service._latest_safe_batch_start(
+ [second_state]
+ )
+
+ batch = service._select_batch(ready, now=now)
+
+ assert [state.session_id for state in batch] == [first]
+ finally:
+ service.stop()
+
+
def test_lossless_sessions_each_stream_thirty_chunks_without_drops() -> None:
service, _ = _service(output_queue_size=2, batching_window_ms=10, control_idle_timeout=30)
service.configure_session_capacity(2)
diff --git a/tests/unit/pipelines/abot_world/test_migration.py b/tests/unit/pipelines/abot_world/test_migration.py
index 13c88d03..ce15705c 100644
--- a/tests/unit/pipelines/abot_world/test_migration.py
+++ b/tests/unit/pipelines/abot_world/test_migration.py
@@ -1,20 +1,63 @@
from __future__ import annotations
-from types import SimpleNamespace
+import threading
+from collections import deque
+from types import MethodType, SimpleNamespace
+from typing import Any
import torch
+import torch.nn as nn
+from telefuser.models.taew2_2 import TAEHV, TWorkItem
from telefuser.models.wan22_video_vae import Wan22VideoVAEStreamingDecodeState
from telefuser.pipelines.abot_world.interactive import (
ABotWorldInteractivePipeline,
ABotWorldInteractiveSession,
ABotWorldSessionLifecycle,
)
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+from telefuser.pipelines.abot_world.taew_vae import ABotWorldTAEWDecodeStage
+from telefuser.service.livekit.nccl_transfer import flatten_tensor_tree, rebuild_tensor_tree
+
+
+class _TinyTAEW(nn.Module):
+ """Minimal decoder topology used only to exercise session-state transport."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.encoder = nn.Sequential(nn.Identity())
+ self.decoder = nn.Sequential(*(nn.Identity() for _ in range(4)))
+
+
+def _taew_stage() -> ABotWorldTAEWDecodeStage:
+ stage = object.__new__(ABotWorldTAEWDecodeStage)
+ stage.taew = _TinyTAEW()
+ stage.device = torch.device("cpu")
+ stage.torch_dtype = torch.float32
+ return stage
+
+
+def _taew_state(stage: ABotWorldTAEWDecodeStage) -> Any:
+ state = stage.create_decode_state()
+ state.stream.decoder_work_queue = [TWorkItem(torch.tensor([[[[8.0]]]]), 2)]
+ state.stream.decoder_memory[1] = torch.tensor([[[[9.0]]]])
+ state.stream.decoder_memory[2] = [torch.tensor([[[[10.0]]]])]
+ state.stream.n_frames_decoded = 11
+ return state
+
+
+def _assert_taew_state(state: Any) -> None:
+ assert state.stream.decoder_work_queue[0].block_index == 2
+ assert state.stream.decoder_work_queue[0].input_tensor.item() == 8
+ assert state.stream.decoder_memory[1].item() == 9
+ assert state.stream.decoder_memory[2][0].item() == 10
+ assert state.stream.n_frames_decoded == 11
def test_session_snapshot_round_trip_preserves_causal_and_rng_state() -> None:
source = ABotWorldInteractivePipeline(device="cpu", torch_dtype=torch.float32)
source.denoise_stage = SimpleNamespace(_scheduler=lambda: object())
+ source.taew_decode_stage = _taew_stage()
generator = torch.Generator(device="cpu").manual_seed(123)
session = ABotWorldInteractiveSession(
session_id="migrating",
@@ -39,6 +82,7 @@ def test_session_snapshot_round_trip_preserves_causal_and_rng_state() -> None:
scheduler=object(),
generator=generator,
vae_decode_state=Wan22VideoVAEStreamingDecodeState(feat_cache=[torch.tensor([7.0])]),
+ taew_decode_state=_taew_state(source.taew_decode_stage),
next_latent_frame=12,
emitted_frames=45,
ownership_epoch=4,
@@ -49,9 +93,13 @@ def test_session_snapshot_round_trip_preserves_causal_and_rng_state() -> None:
generator.set_state(generator_state)
snapshot = source.snapshot_interactive_session(session)
+ assert snapshot.taew_decode_state["decoder_work_queue"][0]["input_tensor"].device.type == "cpu"
+ session.taew_decode_state.stream.decoder_work_queue[0].input_tensor.fill_(99)
+ assert snapshot.taew_decode_state["decoder_work_queue"][0]["input_tensor"].item() == 8
source.close_interactive_session(session)
target = ABotWorldInteractivePipeline(device="cpu", torch_dtype=torch.float32)
target.denoise_stage = SimpleNamespace(_scheduler=lambda: object())
+ target.taew_decode_stage = _taew_stage()
restored = target.restore_interactive_snapshot(snapshot, owner_worker_id="gpu-1")
assert restored.lifecycle == ABotWorldSessionLifecycle.READY
@@ -62,4 +110,257 @@ def test_session_snapshot_round_trip_preserves_causal_and_rng_state() -> None:
assert restored.self_cache[0]["k"].item() == 3
assert restored.cross_cache[0]["v"].item() == 6
assert restored.vae_decode_state.feat_cache[0].item() == 7
+ assert restored.taew_decode_state is not None
+ _assert_taew_state(restored.taew_decode_state)
+ target.suspend_interactive_session(restored)
+ assert restored.taew_decode_state.stream.decoder_memory[1].device.type == "cpu"
+ target.restore_interactive_session(restored)
+ _assert_taew_state(restored.taew_decode_state)
assert torch.equal(torch.randn(1, generator=restored.generator), expected_next_random)
+ target.close_interactive_session(restored)
+ assert restored.taew_decode_state is None
+
+
+def _real_taew_stage() -> ABotWorldTAEWDecodeStage:
+ model = TAEHV(
+ checkpoint_path=None,
+ encoder_time_downscale=(False, False, False),
+ decoder_time_upscale=(False, False, False),
+ decoder_space_upscale=(False, False, False),
+ latent_channels=2,
+ ).eval()
+ stage = object.__new__(ABotWorldTAEWDecodeStage)
+ stage.taew = model
+ stage.device = torch.device("cpu")
+ stage.torch_dtype = torch.float32
+ return stage
+
+
+def test_taew_batched_decode_matches_independent_streams() -> None:
+ stage = _real_taew_stage()
+ serial_states = [stage.create_decode_state(), stage.create_decode_state()]
+ batched_states = [stage.create_decode_state(), stage.create_decode_state()]
+ first = [torch.randn(1, 2, 1, 1, 1), torch.randn(1, 2, 1, 1, 1)]
+ continuation = [torch.randn(1, 2, 3, 1, 1), torch.randn(1, 2, 3, 1, 1)]
+
+ expected_first = torch.cat(
+ [stage._decode_chunks_impl(latents, [state]) for latents, state in zip(first, serial_states)],
+ dim=0,
+ )
+ actual_first = stage._decode_chunks_impl(torch.cat(first), batched_states)
+ torch.testing.assert_close(actual_first, expected_first)
+
+ expected_continuation = torch.cat(
+ [
+ stage._decode_chunks_impl(latents, [state])
+ for latents, state in zip(continuation, serial_states)
+ ],
+ dim=0,
+ )
+ actual_continuation = stage._decode_chunks_impl(torch.cat(continuation), batched_states)
+ torch.testing.assert_close(actual_continuation, expected_continuation)
+ assert [state.stream.n_frames_decoded for state in batched_states] == [
+ state.stream.n_frames_decoded for state in serial_states
+ ]
+
+
+def test_taew_snapshot_restore_preserves_real_causal_decode() -> None:
+ source_stage = _real_taew_stage()
+ original = source_stage.create_decode_state()
+ transferred = source_stage.create_decode_state()
+ first = torch.randn(1, 2, 1, 1, 1)
+ continuation = torch.randn(1, 2, 3, 1, 1)
+
+ source_stage._decode_chunks_impl(first, [original])
+ source_stage._decode_chunks_impl(first, [transferred])
+ snapshot = source_stage.snapshot_decode_state(transferred)
+ restored = source_stage.restore_decode_state(snapshot)
+
+ expected = source_stage._decode_chunks_impl(continuation, [original])
+ actual = source_stage._decode_chunks_impl(continuation, [restored])
+ torch.testing.assert_close(actual, expected)
+
+
+def test_taew_decoder_state_nccl_tensor_tree_round_trip() -> None:
+ source_stage = _taew_stage()
+ source_state = _taew_state(source_stage)
+
+ payload = source_stage.export_decode_state_for_nccl(source_state)
+ skeleton, manifest, leaves = flatten_tensor_tree(payload)
+ assert manifest
+ assert any(path[:2] == ("decoder_work_queue", 0) for path in leaves)
+ target_leaves = {path: tensor.clone() for path, tensor in leaves.items()}
+ rebuilt = rebuild_tensor_tree(skeleton, target_leaves)
+
+ target_stage = _taew_stage()
+ restored = target_stage.restore_decode_state(rebuilt, direct_device_tensors=True)
+ _assert_taew_state(restored)
+ source_state.stream.decoder_memory[1].fill_(99)
+ assert restored.stream.decoder_memory[1].item() == 9
+
+
+def test_nccl_migration_metadata_includes_taew_decoder_state() -> None:
+ stage = _taew_stage()
+ generator = torch.Generator(device="cpu").manual_seed(7)
+ session = SimpleNamespace(
+ prompt_emb=torch.tensor([1.0]),
+ first_frame_latent=torch.tensor([2.0]),
+ self_cache=[],
+ cross_cache=[],
+ vae_decode_state=Wan22VideoVAEStreamingDecodeState(),
+ taew_decode_state=_taew_state(stage),
+ generator=generator,
+ next_latent_frame=3,
+ emitted_frames=12,
+ ownership_epoch=2,
+ )
+ service = object.__new__(ABotWorldLiveKitService)
+ service.pipeline = SimpleNamespace(taew_decode_stage=stage)
+ service._quiesce_migration = lambda session_id, timeout: SimpleNamespace(
+ pipeline_session=session,
+ config={"fps": 12},
+ controls={"W"},
+ control_idle_timeout=10.0,
+ last_control_at=1.0,
+ next_chunk_index=1,
+ next_playout_deadline=2.0,
+ )
+
+ metadata = service.prepare_migration_nccl_metadata("migrating", timeout=1)
+ payload = rebuild_tensor_tree(metadata["tensor_skeleton"], metadata["_nccl_tensor_leaves"])
+
+ assert metadata["state_bytes"] > 0
+ assert "taew_decode_state" in payload
+ restored = stage.restore_decode_state(payload["taew_decode_state"], direct_device_tensors=True)
+ _assert_taew_state(restored)
+
+
+def test_import_migration_nccl_restores_taew_state_without_cpu_copy() -> None:
+ source_stage = _taew_stage()
+ source_state = _taew_state(source_stage)
+ payload = {
+ "prompt_emb": torch.tensor([1.0]),
+ "first_frame_latent": torch.tensor([2.0]),
+ "self_cache": [],
+ "cross_cache": [],
+ "vae_feat_cache": [],
+ "taew_decode_state": source_stage.export_decode_state_for_nccl(source_state),
+ }
+ skeleton, _, leaves = flatten_tensor_tree(payload)
+ target_leaves = {path: tensor.clone() for path, tensor in leaves.items()}
+ restored_pipeline_session = SimpleNamespace(session_id="migrating")
+ observed: dict[str, Any] = {}
+
+ def restore_interactive_device_snapshot(snapshot, **kwargs):
+ observed["snapshot"] = snapshot
+ observed["kwargs"] = kwargs
+ return restored_pipeline_session
+
+ pipeline = SimpleNamespace(
+ restore_interactive_device_snapshot=restore_interactive_device_snapshot,
+ close_interactive_session=lambda session: None,
+ )
+ service = object.__new__(ABotWorldLiveKitService)
+ service.pipeline = pipeline
+ service._ensure_scheduler_started = lambda: None
+ service._capacity_profile = {"effective_capacity": 1}
+ service._sessions = {}
+ service._round_robin_order = deque()
+ service._scheduler_condition = threading.Condition(threading.RLock())
+ service.output_queue_size = 1
+
+ session_id = service.import_migration_nccl(
+ {
+ "session_id": "migrating",
+ "tensor_skeleton": skeleton,
+ "vae_feat_idx": [],
+ "generator_state": torch.Generator(device="cpu").get_state(),
+ "next_latent_frame": 3,
+ "emitted_frames": 12,
+ "ownership_epoch": 2,
+ "config": {"fps": 12},
+ "controls": ["W"],
+ "control_idle_timeout": 10.0,
+ "last_control_at": 1.0,
+ "next_chunk_index": 1,
+ "next_playout_deadline": 2.0,
+ },
+ target_leaves,
+ owner_worker_id="gpu-1",
+ ownership_epoch=3,
+ )
+
+ assert session_id == "migrating"
+ snapshot = observed["snapshot"]
+ assert observed["kwargs"] == {"owner_worker_id": "gpu-1", "ownership_epoch": 3}
+ restored = source_stage.restore_decode_state(snapshot.taew_decode_state, direct_device_tensors=True)
+ _assert_taew_state(restored)
+ assert service._sessions[session_id].pipeline_session is restored_pipeline_session
+
+
+class _FakeTAEWStream:
+ """CPU-only stream double that exposes compatibility and call count."""
+
+ def __init__(self, cursor: int) -> None:
+ self.decoder_work_queue: list[object] = []
+ self.decoder_memory = None
+ self.n_frames_decoded = cursor
+ self.decode_shapes: list[tuple[int, ...]] = []
+
+ def decode(self, latents: torch.Tensor) -> torch.Tensor:
+ self.decode_shapes.append(tuple(latents.shape))
+ return latents
+
+
+def _fake_taew_telemetry_stage() -> ABotWorldTAEWDecodeStage:
+ stage = object.__new__(ABotWorldTAEWDecodeStage)
+ stage.device = torch.device("cpu")
+ stage.torch_dtype = torch.float32
+ stage.synchronized_decode_shapes = []
+
+ def synchronized_decode(self, decoder_latents, states):
+ del states
+ self.synchronized_decode_shapes.append(tuple(decoder_latents.shape))
+ return decoder_latents
+
+ stage._decode_synchronized_batch = MethodType(synchronized_decode, stage)
+ return stage
+
+
+def test_taew_decode_telemetry_distinguishes_native_batch_from_serial_fallback() -> None:
+ stage = _fake_taew_telemetry_stage()
+ latents = torch.ones(2, 1, 3, 1, 1)
+
+ synchronized_states = [
+ SimpleNamespace(stream=_FakeTAEWStream(cursor=0)),
+ SimpleNamespace(stream=_FakeTAEWStream(cursor=0)),
+ ]
+ synchronized = stage._decode_chunks_impl(latents, synchronized_states)
+
+ assert synchronized.shape == latents.shape
+ assert stage.synchronized_decode_shapes == [(2, 3, 1, 1, 1)]
+ assert [state.stream.decode_shapes for state in synchronized_states] == [[], []]
+ assert stage.last_decode_metrics() == {
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 2,
+ "taew_decode_invocations": 1,
+ "taew_decode_mode": 1,
+ }
+
+ fallback_states = [
+ SimpleNamespace(stream=_FakeTAEWStream(cursor=0)),
+ SimpleNamespace(stream=_FakeTAEWStream(cursor=1)),
+ ]
+ fallback = stage._decode_chunks_impl(latents, fallback_states)
+
+ assert fallback.shape == latents.shape
+ assert [state.stream.decode_shapes for state in fallback_states] == [
+ [(1, 3, 1, 1, 1)],
+ [(1, 3, 1, 1, 1)],
+ ]
+ assert stage.last_decode_metrics() == {
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 1,
+ "taew_decode_invocations": 2,
+ "taew_decode_mode": 2,
+ }
diff --git a/tests/unit/service/livekit/test_app.py b/tests/unit/service/livekit/test_app.py
index 687313a9..cf0efd9d 100644
--- a/tests/unit/service/livekit/test_app.py
+++ b/tests/unit/service/livekit/test_app.py
@@ -109,3 +109,20 @@ def test_livekit_health_and_service_metadata_routes() -> None:
assert metadata.status_code == 200
assert metadata.json()["service_type"] == "stream"
assert metadata.json()["transport"] == "livekit"
+
+
+def test_livekit_metrics_alias_matches_versioned_endpoint() -> None:
+ runtime = _make_runtime()
+ app = create_livekit_app(runtime)
+
+ with ASGITestClient(app) as client:
+ alias = client.get("/metrics")
+ versioned = client.get("/v1/service/metrics")
+ json_metrics = client.get("/v1/service/metrics/json")
+
+ assert alias.status_code == 200
+ assert versioned.status_code == 200
+ assert "telefuser_serving_sessions" in alias.text
+ assert "telefuser_serving_sessions" in versioned.text
+ assert json_metrics.status_code == 200
+ assert json_metrics.json()["serving"]["summary"]["sessions"]["waiting"] == 0
diff --git a/tests/unit/service/livekit/test_multi_session_capacity.py b/tests/unit/service/livekit/test_multi_session_capacity.py
index 20972e9e..a11f94ab 100644
--- a/tests/unit/service/livekit/test_multi_session_capacity.py
+++ b/tests/unit/service/livekit/test_multi_session_capacity.py
@@ -105,3 +105,38 @@ def test_runtime_starts_two_sessions_on_one_model_worker() -> None:
assert worker_pool.started == [first.record.session_id, second.record.session_id]
assert runtime.registry.require(first.record.session_id).config["control_idle_timeout"] == 8.0
assert runtime.registry.require(second.record.session_id).worker_id == "worker-0"
+
+
+def test_runtime_assigns_all_peak16_sessions_without_waiting_when_capacity_is_four_per_worker() -> None:
+ """A fixed 4x4 deployment admits its complete peak before a queue exists."""
+ worker_pool = _WorkerPool()
+ runtime = LiveKitServeRuntime(
+ config=LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ num_workers=4,
+ worker_gpu_map="0;1;2;3",
+ worker_mode="process-nccl",
+ max_sessions_per_worker=4,
+ # A peak-16 all-active trace must not convert capacity pressure
+ # into invisible zero-FPS queued sessions.
+ queue_size=0,
+ ),
+ pipeline_file="pipeline.py",
+ token_service=_TokenService(),
+ worker_pool=worker_pool,
+ )
+
+ admissions = [
+ runtime.create_session(SessionCreateRequest(identity=f"controller-{index}")) for index in range(16)
+ ]
+
+ assert all(result.admission.status == "assigned" for result in admissions)
+ assert len(worker_pool.started) == 16
+ assert [len(worker.session_ids) for worker in runtime.scheduler.workers()] == [4, 4, 4, 4]
+ assert runtime.scheduler.health_snapshot()["queued_sessions"] == 0
+
+ overflow = runtime.create_session(SessionCreateRequest(identity="controller-overflow"))
+ assert overflow.admission.status == "rejected"
+ assert runtime.scheduler.health_snapshot()["queued_sessions"] == 0
diff --git a/tests/unit/service/livekit/test_nccl_process_worker_pool.py b/tests/unit/service/livekit/test_nccl_process_worker_pool.py
new file mode 100644
index 00000000..b24640bc
--- /dev/null
+++ b/tests/unit/service/livekit/test_nccl_process_worker_pool.py
@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from telefuser.service.livekit.config import LiveKitServeConfig
+from telefuser.service.livekit.nccl_process_worker_pool import (
+ _MODEL_OUTPUT_PARENT_QUEUE_SIZE,
+ NCCLProcessLiveKitWorkerPool,
+ _pump_model_outputs,
+)
+from telefuser.service.livekit.process_worker_pool import (
+ ProcessLiveKitWorkerPool,
+ ProcessWorkerSpec,
+)
+from telefuser.service.livekit.worker import NullWorkerEventSink
+
+
+class _EventCollector:
+ def __init__(self) -> None:
+ self.items: list[dict[str, Any]] = []
+ self.updated = asyncio.Event()
+
+ def put(self, item: dict[str, Any]) -> None:
+ self.items.append(item)
+ self.updated.set()
+
+
+class _PumpAdapter:
+ def __init__(self, payloads: list[dict[str, Any]]) -> None:
+ self.payloads = payloads
+ self.pull_count = 0
+
+ async def pull_chunks(self, session_id: str):
+ del session_id
+ for payload in self.payloads:
+ self.pull_count += 1
+ yield payload
+
+ def runtime_metrics(self) -> dict[str, int]:
+ return {"active_sessions": 1}
+
+
+class _PumpService:
+ def runtime_metrics(self, session_id: str) -> dict[str, int]:
+ del session_id
+ return {"active": 1}
+
+
+def _pool() -> NCCLProcessLiveKitWorkerPool:
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ worker_mode="process-nccl",
+ num_workers=2,
+ worker_gpu_map="0;1",
+ )
+ pool = NCCLProcessLiveKitWorkerPool(
+ [ProcessWorkerSpec("worker-0", ["0"]), ProcessWorkerSpec("worker-1", ["1"])],
+ config=config,
+ pipeline_file="pipeline.py",
+ event_sink=NullWorkerEventSink(),
+ )
+ pool._active_workers = {"worker-0"}
+ return pool
+
+
+def _model_output(session_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "type": "model_output",
+ "worker_id": "worker-0",
+ "session_id": session_id,
+ "payload": payload,
+ }
+
+
+async def _wait_for_count(events: _EventCollector, count: int) -> None:
+ while len(events.items) < count:
+ events.updated.clear()
+ await asyncio.wait_for(events.updated.wait(), timeout=1.0)
+
+
+def test_child_pump_waits_for_credit_before_reading_next_abot_payload() -> None:
+ async def run() -> None:
+ events = _EventCollector()
+ adapter = _PumpAdapter([{"type": "chunk", "index": 0}, {"type": "chunk", "index": 1}])
+ credits = asyncio.BoundedSemaphore(_MODEL_OUTPUT_PARENT_QUEUE_SIZE)
+ task = asyncio.create_task(
+ _pump_model_outputs(
+ adapter,
+ _PumpService(),
+ worker_id="worker-0",
+ session_id="pipeline-1",
+ credits=credits,
+ events=events,
+ )
+ )
+ await _wait_for_count(events, 1)
+ await asyncio.sleep(0)
+ assert adapter.pull_count == 1
+ assert [item["payload"]["index"] for item in events.items] == [0]
+
+ credits.release()
+ await _wait_for_count(events, 2)
+ assert adapter.pull_count == 2
+ assert [item["payload"]["index"] for item in events.items] == [0, 1]
+
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+
+ asyncio.run(run())
+
+
+def test_parent_queue_preserves_preview_then_replaces_stale_video_and_returns_credit() -> None:
+ async def run() -> None:
+ pool = _pool()
+ sent: list[tuple[str, dict[str, Any]]] = []
+ pool._send = lambda worker_id, command: sent.append((worker_id, command))
+ pool.create_model_session("worker-0", "pipeline-1", {})
+ sent.clear()
+
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "preview", "index": -1}))
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "chunk", "index": 0}))
+ assert pool._model_outputs["pipeline-1"].qsize() == 1
+ assert pool._model_output_dropped["pipeline-1"] == 1
+
+ chunks = pool.pull_model_chunks("pipeline-1")
+ assert (await chunks.__anext__())["type"] == "preview"
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "chunk", "index": 1}))
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "chunk", "index": 2}))
+ assert (await chunks.__anext__())["index"] == 2
+ snapshot = pool.turboserve_snapshot()["model_output_flow_control"]
+ assert snapshot["parent_queue_capacity"] == 1
+ assert snapshot["max_materialized_payloads_per_session"] == 2
+ assert snapshot["dropped_payloads"] == {"pipeline-1": 2}
+ credits = [command for _, command in sent if command["type"] == "model_output_credit"]
+ assert [command["session_id"] for command in credits] == ["pipeline-1"] * 4
+ await chunks.aclose()
+
+ asyncio.run(run())
+
+
+def test_parent_queue_prioritizes_terminal_payload_over_queued_video() -> None:
+ async def run() -> None:
+ pool = _pool()
+ pool._send = lambda worker_id, command: None
+ pool.create_model_session("worker-0", "pipeline-1", {})
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "preview"}))
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "error", "error": "model failed"}))
+
+ chunks = pool.pull_model_chunks("pipeline-1")
+ payload = await chunks.__anext__()
+ assert payload == {"type": "error", "error": "model failed"}
+ assert pool._model_output_dropped["pipeline-1"] == 1
+ await chunks.aclose()
+
+ asyncio.run(run())
+
+
+def test_initial_start_builds_one_nccl_group_after_all_workers(monkeypatch) -> None:
+ async def run() -> None:
+ pool = _pool()
+ pool._active_workers = set()
+ init_sizes: list[int] = []
+
+ async def fake_parent_scale_to(self, target_workers: int) -> int:
+ self._active_workers = set(list(self._specs)[:target_workers])
+ return len(self._active_workers)
+
+ async def fake_parent_start(self, *, skip_validation: bool = False) -> None:
+ assert skip_validation
+ # This mirrors ProcessLiveKitWorkerPool.start: its virtual
+ # scale_to calls occur once for each sequential worker startup.
+ await self.scale_to(1)
+ await self.scale_to(2)
+
+ async def fake_init_nccl() -> None:
+ init_sizes.append(len(pool._active_workers))
+ pool._nccl_ranks = {worker_id: index for index, worker_id in enumerate(sorted(pool._active_workers))}
+
+ monkeypatch.setattr(ProcessLiveKitWorkerPool, "scale_to", fake_parent_scale_to)
+ monkeypatch.setattr(ProcessLiveKitWorkerPool, "start", fake_parent_start)
+ pool._init_nccl = fake_init_nccl
+
+ await pool.start(skip_validation=True)
+
+ assert init_sizes == [2]
+ assert not pool._initializing_workers
+ assert pool._nccl_ranks == {"worker-0": 0, "worker-1": 1}
+
+ asyncio.run(run())
diff --git a/tests/unit/service/livekit/test_serving_metrics.py b/tests/unit/service/livekit/test_serving_metrics.py
new file mode 100644
index 00000000..93a7ed72
--- /dev/null
+++ b/tests/unit/service/livekit/test_serving_metrics.py
@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+from queue import SimpleQueue
+
+from telefuser.service.livekit.config import LiveKitServeConfig
+from telefuser.service.livekit.nccl_process_worker_pool import (
+ NCCLProcessLiveKitWorkerPool,
+ _ParentTransportSink,
+)
+from telefuser.service.livekit.process_worker_pool import ProcessLiveKitWorkerPool, _ProcessEventSink
+from telefuser.service.livekit.runtime import LiveKitServeRuntime
+from telefuser.service.livekit.schemas import SessionCreateRequest
+
+
+class _TokenService:
+ def create_token(self, *, identity: str, room_name: str, role: str, **kwargs: object) -> str:
+ del kwargs
+ return f"{role}:{identity}:{room_name}"
+
+
+class _WorkerPool:
+ def __init__(self) -> None:
+ self.snapshot = {
+ "worker_runtime_metrics": {
+ "worker-0": {
+ "scheduler_mode": "batched",
+ "active_sessions": 1,
+ "mean_chunk_seconds": 0.5,
+ "p95_chunk_seconds": 0.75,
+ "maximum_batch_size": 2,
+ "denoise_seconds": 0.42,
+ "vae_decode_seconds": 0.06,
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 1,
+ "taew_decode_invocations": 2,
+ "taew_decode_mode": 2,
+ }
+ },
+ "session_runtime_metrics": {
+ "pipeline-session-1": {
+ "active": 1,
+ "emitted_frames": 12,
+ }
+ },
+ }
+
+ async def start(self, *, skip_validation: bool = False) -> None:
+ del skip_validation
+
+ def start_session(self, record) -> None:
+ del record
+
+ async def stop_session(self, session_id: str) -> None:
+ del session_id
+
+ async def aclose(self) -> None:
+ return None
+
+ def turboserve_snapshot(self) -> dict:
+ return self.snapshot
+
+
+def _runtime() -> LiveKitServeRuntime:
+ return LiveKitServeRuntime(
+ config=LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ worker_gpu_map="4",
+ max_sessions_per_worker=2,
+ default_fps=12,
+ ),
+ pipeline_file="pipeline.py",
+ token_service=_TokenService(),
+ worker_pool=_WorkerPool(),
+ )
+
+
+def test_serving_metrics_render_scheduler_pipeline_slo_and_no_session_id_labels() -> None:
+ runtime = _runtime()
+ created = runtime.create_session(SessionCreateRequest(identity="controller", config={"fps": 12}))
+ runtime.on_pipeline_session(created.record.session_id, "pipeline-session-1")
+ runtime.on_session_status(created.record.session_id, "running")
+
+ runtime.on_control_received("worker-0", created.record.session_id)
+ runtime.on_model_output(
+ "worker-0",
+ "pipeline-session-1",
+ {
+ "type": "chunk",
+ "fps": 12,
+ "frame_count": 12,
+ "scheduler": {
+ "batch_size": 2,
+ "queue_wait_seconds": 0.05,
+ "compute_seconds": 0.55,
+ "denoise_seconds": 0.42,
+ "vae_decode_seconds": 0.06,
+ },
+ },
+ )
+ runtime.on_chunk_published("worker-0", created.record.session_id, 12)
+
+ rendered = runtime.prometheus_metrics()
+
+ assert 'telefuser_serving_worker_sessions{gpu="4",worker_id="worker-0"} 1' in rendered
+ assert 'telefuser_serving_scheduler_mode_info{mode="batched"} 1' in rendered
+ assert 'telefuser_serving_pipeline_stage_latency_seconds_bucket{le="0.5",stage="dit"} 1' in rendered
+ assert 'telefuser_serving_slo_chunks_total{result="met"} 1' in rendered
+ assert "telefuser_serving_action_to_first_frame_seconds_count 1" in rendered
+ assert 'telefuser_serving_published_fps{scope="aggregate"}' in rendered
+ assert created.record.session_id not in rendered
+ assert "pipeline-session-1" not in rendered
+
+ summary = runtime.serving_metrics_snapshot()["summary"]
+ assert summary["sessions"] == {"retained": 1, "active": 1, "idle": 0, "waiting": 0}
+ assert summary["scheduler_mode"] == "batched"
+
+
+def test_serving_metrics_records_migration_errors() -> None:
+ runtime = _runtime()
+ runtime._serving_metrics.record_migration(success=False, error="CUDA out of memory")
+ rendered = runtime.prometheus_metrics()
+
+ assert 'telefuser_serving_migrations_total{result="error"} 1' in rendered
+ assert 'telefuser_serving_errors_total{kind="oom"} 1' in rendered
+
+
+class _ForwardedEventSink:
+ def __init__(self) -> None:
+ self.controls: list[tuple[str, str]] = []
+ self.published: list[tuple[str, str, int, float | None]] = []
+ self.outputs: list[tuple[str, str, dict, dict | None, dict | None]] = []
+
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ self.controls.append((worker_id, session_id))
+
+ def on_chunk_published(
+ self,
+ worker_id: str,
+ session_id: str,
+ frames: int,
+ first_frame_at: float | None = None,
+ ) -> None:
+ self.published.append((worker_id, session_id, frames, first_frame_at))
+
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None:
+ self.outputs.append((worker_id, session_id, payload, runtime_metrics, session_runtime_metrics))
+
+
+class _OutputQueue:
+ def __init__(self) -> None:
+ self.items: list[dict | None] = []
+
+ def put_nowait(self, item: dict | None) -> None:
+ self.items.append(item)
+
+
+class _NCCLTransportPool:
+ def __init__(self, event_sink: _ForwardedEventSink) -> None:
+ self._event_sink = event_sink
+
+
+def test_process_worker_ipc_events_reach_runtime_sink() -> None:
+ child_events = SimpleQueue()
+ child = _ProcessEventSink("worker-0", child_events)
+ payload = {"type": "chunk", "frame_count": 12}
+ child.on_control_received("worker-0", "http-session-1")
+ child.on_chunk_published("worker-0", "http-session-1", 12, 123.0)
+ child.on_model_output(
+ "worker-0",
+ "pipeline-session-1",
+ payload,
+ runtime_metrics={"scheduler_mode": "batched"},
+ session_runtime_metrics={"active": 1},
+ )
+
+ forwarded = _ForwardedEventSink()
+ parent = object.__new__(ProcessLiveKitWorkerPool)
+ parent._event_sink = forwarded
+ for _ in range(3):
+ ProcessLiveKitWorkerPool._dispatch_event(parent, child_events.get())
+
+ assert forwarded.controls == [("worker-0", "http-session-1")]
+ assert forwarded.published == [("worker-0", "http-session-1", 12, 123.0)]
+ assert forwarded.outputs == [
+ (
+ "worker-0",
+ "pipeline-session-1",
+ payload,
+ {"scheduler_mode": "batched"},
+ {"active": 1},
+ )
+ ]
+
+
+def test_nccl_parent_transport_and_model_event_hooks_preserve_scheduler_mode() -> None:
+ forwarded = _ForwardedEventSink()
+ output = _OutputQueue()
+ parent = object.__new__(NCCLProcessLiveKitWorkerPool)
+ parent._event_sink = forwarded
+ parent._worker_runtime_metrics = {}
+ parent._session_runtime_metrics = {}
+ parent._model_outputs = {"pipeline-session-1": output}
+ parent._specs = {"worker-0": object()}
+ parent._session_workers = {}
+ parent._pipeline_routes = {}
+ parent._active_workers = set()
+ parent._nccl_ranks = {}
+ parent._migration_total_ms = []
+
+ payload = {"type": "chunk", "frame_count": 12}
+ NCCLProcessLiveKitWorkerPool._dispatch_event(
+ parent,
+ {
+ "type": "model_output",
+ "worker_id": "worker-0",
+ "session_id": "pipeline-session-1",
+ "payload": payload,
+ "runtime_metrics": {"scheduler_mode": "batched", "maximum_batch_size": 2},
+ "session_runtime_metrics": {"active": 1},
+ },
+ )
+
+ transport = _ParentTransportSink(_NCCLTransportPool(forwarded))
+ transport.on_control_received("worker-0", "http-session-1")
+ transport.on_chunk_published("worker-0", "http-session-1", 12, 234.0)
+
+ snapshot = NCCLProcessLiveKitWorkerPool.turboserve_snapshot(parent)
+ assert snapshot["worker_runtime_metrics"] == {"worker-0": {"scheduler_mode": "batched", "maximum_batch_size": 2}}
+ assert output.items == [payload]
+ assert forwarded.controls == [("worker-0", "http-session-1")]
+ assert forwarded.published == [("worker-0", "http-session-1", 12, 234.0)]
+ assert forwarded.outputs == [
+ (
+ "worker-0",
+ "pipeline-session-1",
+ payload,
+ {"scheduler_mode": "batched", "maximum_batch_size": 2},
+ {"active": 1},
+ )
+ ]
+
+
+
+def test_serving_metrics_distinguish_native_taew_batch_from_dit_batch() -> None:
+ runtime = _runtime()
+ synchronized_scheduler = {
+ "batch_size": 2,
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 2,
+ "taew_decode_invocations": 1,
+ "taew_decode_mode": 1,
+ }
+ serial_fallback_scheduler = {
+ "batch_size": 2,
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 1,
+ "taew_decode_invocations": 2,
+ "taew_decode_mode": 2,
+ }
+ for scheduler in (
+ synchronized_scheduler,
+ synchronized_scheduler,
+ serial_fallback_scheduler,
+ serial_fallback_scheduler,
+ ):
+ runtime.on_model_output(
+ "worker-0",
+ "pipeline-session-1",
+ {"type": "chunk", "frame_count": 12, "scheduler": scheduler},
+ )
+
+ rendered = runtime.prometheus_metrics()
+
+ assert "telefuser_serving_taew_decode_synchronized_items_total 2" in rendered
+ assert "telefuser_serving_taew_decode_synchronized_executions_total 1" in rendered
+ assert "telefuser_serving_taew_decode_serial_fallback_items_total 2" in rendered
+ assert "telefuser_serving_taew_decode_serial_fallback_executions_total 2" in rendered
+ assert "telefuser_serving_taew_decode_mean_native_batch_size 1.33333333333" in rendered
+ assert "telefuser_serving_worker_taew_decode_mode" in rendered
+ assert "telefuser_serving_taew_decode_synchronized_items_total{mode=" not in rendered
+ assert "telefuser_serving_taew_decode_serial_fallback_items_total{mode=" not in rendered
diff --git a/tests/unit/service/livekit/test_worker.py b/tests/unit/service/livekit/test_worker.py
index 5208ceb9..e42c5f2a 100644
--- a/tests/unit/service/livekit/test_worker.py
+++ b/tests/unit/service/livekit/test_worker.py
@@ -121,6 +121,9 @@ def __init__(self) -> None:
self.session_statuses: list[tuple[str, str, str | None]] = []
self.pipeline_sessions: list[tuple[str, str]] = []
self.finished: list[tuple[str, str, str | None]] = []
+ self.controls: list[tuple[str, str]] = []
+ self.published: list[tuple[str, str, int, float | None]] = []
+ self.model_outputs: list[tuple[str, str, dict]] = []
def on_worker_status(self, worker_id: str, status: str) -> None:
self.worker_statuses.append((worker_id, status))
@@ -134,6 +137,25 @@ def on_pipeline_session(self, session_id: str, pipeline_session_id: str) -> None
def on_session_finished(self, worker_id: str, session_id: str, error: str | None = None) -> None:
self.finished.append((worker_id, session_id, error))
+ def on_control_received(self, worker_id: str, session_id: str) -> None:
+ self.controls.append((worker_id, session_id))
+
+ def on_chunk_published(
+ self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
+ ) -> None:
+ self.published.append((worker_id, session_id, frames, first_frame_at))
+
+ def on_model_output(
+ self,
+ worker_id: str,
+ session_id: str,
+ payload: dict,
+ runtime_metrics: dict | None = None,
+ session_runtime_metrics: dict | None = None,
+ ) -> None:
+ del runtime_metrics, session_runtime_metrics
+ self.model_outputs.append((worker_id, session_id, payload))
+
def _jpeg_chunk() -> dict:
frame = np.zeros((8, 8, 3), dtype=np.uint8)
@@ -241,6 +263,13 @@ async def _run() -> None:
assert room.statuses[-1]["type"] == "done"
assert room.disconnected is True
assert sink.pipeline_sessions == [("session-1", "pipeline-session-1")]
+ assert sink.controls == [("worker-0", "session-1")]
+ assert [output[2]["frame_count"] for output in sink.model_outputs] == [0, 1, 1, 0]
+ assert [(item[0], item[1], item[2]) for item in sink.published] == [
+ ("worker-0", "session-1", 1),
+ ("worker-0", "session-1", 1),
+ ]
+ assert all(item[3] is not None for item in sink.published)
assert room.statuses[-1]["total_chunks"] == 2
assert room.statuses[-1]["published_frames"] == 2
assert sink.finished == [("worker-0", "session-1", None)]
diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py
index 3d9bae72..3d71e483 100644
--- a/tests/unit/test_metrics.py
+++ b/tests/unit/test_metrics.py
@@ -217,6 +217,18 @@ def test_histogram_custom_buckets(self) -> None:
assert any('le="0.1"' in line for line in lines)
assert any('le="0.5"' in line for line in lines)
+ def test_histogram_prometheus_buckets_are_cumulative(self) -> None:
+ """Prometheus buckets must not be cumulatively added a second time."""
+ hist = Histogram("request_duration", "Request duration", buckets=[0.1, 0.5])
+ hist.observe(0.05)
+ hist.observe(0.2)
+
+ lines = hist.to_prometheus()
+
+ assert 'request_duration_bucket{le="0.1"} 1' in lines
+ assert 'request_duration_bucket{le="0.5"} 2' in lines
+ assert 'request_duration_bucket{le="+Inf"} 2' in lines
+
def test_histogram_reset(self) -> None:
"""Test resetting a histogram."""
hist = Histogram("request_duration", "Request duration")
diff --git a/tests/unit/validation/test_abot_livekit_burst.py b/tests/unit/validation/test_abot_livekit_burst.py
new file mode 100644
index 00000000..01bced1d
--- /dev/null
+++ b/tests/unit/validation/test_abot_livekit_burst.py
@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from tools.validation import benchmark_abot_livekit_burst as wave
+
+
+def _scenario_payload(image_path: Path) -> dict[str, Any]:
+ return {
+ "name": "unit-wave",
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "seed": 7,
+ "session": {
+ "prompt": "unit-test prompt",
+ "image_path": str(image_path),
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {"action_states": [["KeyW"]]},
+ },
+ "measurement": {
+ "sample_interval_seconds": 1,
+ "connect_timeout_seconds": 90,
+ "http_timeout_seconds": 30,
+ "shutdown_timeout_seconds": 20,
+ "first_generation_grace_seconds": 15,
+ },
+ "phases": [
+ {"name": "warmup", "duration_seconds": 2, "target_users": 4, "arrival_window_seconds": 1},
+ {"name": "recovery", "duration_seconds": 2, "target_users": 2, "departure_window_seconds": 1},
+ ],
+ }
+
+
+def _load_scenario(tmp_path: Path) -> wave.Scenario:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ scenario_path = tmp_path / "scenario.json"
+ scenario_path.write_text(json.dumps(_scenario_payload(image)), encoding="utf-8")
+ return wave.load_scenario(scenario_path)
+
+
+def _runner_for_scheduling(scenario: wave.Scenario) -> tuple[wave.LiveKitWaveRunner, list[Any]]:
+ runner = object.__new__(wave.LiveKitWaveRunner)
+ runner.scenario = scenario
+ runner._sessions = []
+ runner._background_tasks = set()
+ runner._http = object()
+ runner.rtc = object()
+ runner.started_at = 0.0
+ runner.record_event = lambda *args, **kwargs: None
+ scheduled: list[Any] = []
+ runner._spawn_background = scheduled.append
+ return runner, scheduled
+
+
+def _session(index: int, scenario: wave.Scenario) -> wave.LiveKitWaveSession:
+ return wave.LiveKitWaveSession(
+ index=index,
+ scenario=scenario,
+ http=object(),
+ rtc=object(),
+ record_event=lambda *args, **kwargs: None,
+ started_at=0.0,
+ )
+
+
+def test_load_scenario_validates_lf3_process_nccl_wave(tmp_path: Path) -> None:
+ scenario = _load_scenario(tmp_path)
+
+ assert scenario.expected_worker_mode == "process-nccl"
+ assert scenario.expected_num_workers == 4
+ assert scenario.session.fps == 12
+ assert scenario.session.control_latent_frames == 3
+ assert scenario.first_generation_grace_seconds == 15
+ assert scenario.slo_fps_tolerance == 0.25
+ assert [phase.target_users for phase in scenario.phases] == [4, 2]
+
+
+def test_load_scenario_rejects_arrival_window_longer_than_phase(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ payload = _scenario_payload(image)
+ payload["phases"][0]["arrival_window_seconds"] = 3
+ scenario_path = tmp_path / "scenario.json"
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+
+ with pytest.raises(wave.ScenarioError, match="cannot exceed duration"):
+ wave.load_scenario(scenario_path)
+
+
+def test_scale_down_marks_newest_sessions_and_keeps_them_counted_until_stop(tmp_path: Path) -> None:
+ scenario = _load_scenario(tmp_path)
+ runner, scheduled = _runner_for_scheduling(scenario)
+ runner._sessions = [_session(index, scenario) for index in range(4)]
+
+ runner._schedule_transition(wave.Phase("down", 2, target_users=2, departure_window_seconds=1))
+
+ assert [session.index for session in runner._sessions if session.departure_scheduled] == [2, 3]
+ assert all(not session.stop_requested for session in runner._sessions)
+ assert len(scheduled) == 2
+ for coroutine in scheduled:
+ coroutine.close()
+
+
+def test_scale_up_spreads_only_new_arrivals_across_its_window(tmp_path: Path) -> None:
+ scenario = _load_scenario(tmp_path)
+ runner, scheduled = _runner_for_scheduling(scenario)
+ runner._sessions = [_session(index, scenario) for index in range(4)]
+
+ runner._schedule_transition(wave.Phase("up", 2, target_users=8, arrival_window_seconds=3))
+
+ assert [session.index for session in runner._sessions] == list(range(8))
+ assert len(scheduled) == 4
+ for coroutine in scheduled:
+ coroutine.close()
+
+
+def test_slo_includes_zero_fps_after_first_generation_grace(tmp_path: Path) -> None:
+ scenario = _load_scenario(tmp_path)
+ runner, _ = _runner_for_scheduling(scenario)
+ session = _session(0, scenario)
+ session.connected = True
+ session.current_controls = ("KeyW",)
+ session.first_active_control_at = 0.0
+
+ assert runner._session_delivery_fps(session, now=14.9, interval=1.0, delta=0) == (None, None)
+ assert runner._session_delivery_fps(session, now=15.0, interval=1.0, delta=0) == (0.0, 0.0)
+
+ session.first_generated_frame_at = 1.0
+ assert runner._session_delivery_fps(session, now=15.0, interval=2.0, delta=24) == (12.0, 12.0)
+
+
+def test_load_scenario_parses_all_active_admission_contract(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ payload = _scenario_payload(image)
+ payload["admission"] = {
+ "require_immediate_assignment": True,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0,
+ }
+ scenario_path = tmp_path / "scenario.json"
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+
+ scenario = wave.load_scenario(scenario_path)
+
+ assert scenario.admission.require_immediate_assignment is True
+ assert scenario.admission.expected_max_sessions_per_worker == 4
+ assert scenario.admission.expected_queue_size == 0
+
+
+def test_requested_user_fps_counts_unserved_request_as_zero_after_grace(tmp_path: Path) -> None:
+ scenario = _load_scenario(tmp_path)
+ runner, _ = _runner_for_scheduling(scenario)
+ session = _session(0, scenario)
+ session.create_started_at = 0.0
+
+ assert runner._session_requested_delivery_fps(session, now=14.9, interval=1.0, delta=0) is None
+ assert runner._session_requested_delivery_fps(session, now=15.0, interval=1.0, delta=0) == 0.0
+
+ session.first_generated_frame_at = 1.0
+ assert runner._session_requested_delivery_fps(session, now=15.0, interval=2.0, delta=24) == 12.0
+
+
+def test_peak16_trace_requires_all_active_users() -> None:
+ scenario_path = (
+ wave._REPO_ROOT / "tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json"
+ )
+ payload = json.loads(scenario_path.read_text(encoding="utf-8"))
+
+ assert payload["admission"] == {
+ "require_immediate_assignment": True,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0,
+ }
+ assert [phase["target_users"] for phase in payload["phases"]] == [4, 8, 12, 16, 8, 4]
+ assert sum(phase["duration_seconds"] for phase in payload["phases"]) == 330.0
+
+
+def test_phase_input_activity_pauses_and_resumes_without_departure(tmp_path: Path) -> None:
+ scenario = _load_scenario(tmp_path)
+ runner, scheduled = _runner_for_scheduling(scenario)
+ runner._sessions = [_session(index, scenario) for index in range(8)]
+
+ runner._schedule_input_activity(wave.Phase("pause", 2, target_users=8, active_input_fraction=0.5))
+
+ assert len(scheduled) == 4
+ for coroutine in scheduled:
+ asyncio.run(coroutine)
+ assert sum(session.input_enabled for session in runner._sessions) == 4
+ assert all(not session.stop_requested for session in runner._sessions)
+ assert all(not session.departure_scheduled for session in runner._sessions)
+
+ scheduled.clear()
+ runner._schedule_input_activity(wave.Phase("resume", 2, target_users=8, active_input_fraction=1.0))
+ assert len(scheduled) == 4
+ for coroutine in scheduled:
+ asyncio.run(coroutine)
+ assert all(session.input_enabled for session in runner._sessions)
+
+
+def test_intermittent_peak16_trace_models_pauses_and_reengagement() -> None:
+ scenario_path = (
+ wave._REPO_ROOT / "tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json"
+ )
+ payload = json.loads(scenario_path.read_text(encoding="utf-8"))
+ phases = payload["phases"]
+
+ assert payload["admission"] == {
+ "require_immediate_assignment": True,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0,
+ }
+ assert [phase["target_users"] for phase in phases] == [4, 8, 8, 16, 16, 16, 16, 8, 4]
+ assert max(phase["target_users"] for phase in phases) == 16
+ assert sum(phase["duration_seconds"] for phase in phases) == 385.0
+ assert phases[2]["active_input_fraction"] == 0.5
+ assert phases[5]["active_input_fraction"] == 0.5
+ assert phases[6]["active_input_fraction"] == 1.0
diff --git a/tests/unit/validation/test_capture_abot_serving_metrics.py b/tests/unit/validation/test_capture_abot_serving_metrics.py
new file mode 100644
index 00000000..c22490fa
--- /dev/null
+++ b/tests/unit/validation/test_capture_abot_serving_metrics.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from tools.validation import capture_abot_serving_metrics as capture
+
+
+class _MetricsHandler(BaseHTTPRequestHandler):
+ paths: list[str] = []
+
+ def do_GET(self) -> None: # noqa: N802
+ type(self).paths.append(self.path)
+ if self.path == "/metrics":
+ body = b"telefuser_serving_sessions{state=\"active\"} 4\n"
+ self.send_response(200)
+ self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+ elif self.path == "/v1/service/metrics/json":
+ body = json.dumps(
+ {
+ "serving": {
+ "summary": {"sessions": {"active": 4}},
+ "counters": {"telefuser_serving_chunks_total{result=\"processed\"}": 12},
+ }
+ }
+ ).encode("utf-8")
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json; charset=utf-8")
+ else:
+ body = b"not found"
+ self.send_response(404)
+ self.send_header("Content-Type", "text/plain; charset=utf-8")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, format: str, *args: object) -> None:
+ del format, args
+
+
+def test_capture_writes_prometheus_jsonl_and_manifest_without_proxy(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ _MetricsHandler.paths = []
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _MetricsHandler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ monkeypatch.setenv("http_proxy", "http://127.0.0.1:1")
+ monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:1")
+
+ try:
+ output_dir = tmp_path / "metrics"
+ config = capture.CaptureConfig(
+ server_url=f"http://127.0.0.1:{server.server_port}",
+ duration_seconds=0.04,
+ interval_seconds=0.01,
+ timeout_seconds=1.0,
+ output_dir=output_dir,
+ )
+ manifest = capture.capture(config)
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=3)
+
+ assert manifest["status"] == "completed"
+ assert manifest["configuration"]["proxy_mode"] == "direct_no_proxy"
+ assert manifest["samples"]["complete"] >= 1
+ assert _MetricsHandler.paths.count("/metrics") >= 1
+ assert _MetricsHandler.paths.count("/v1/service/metrics/json") >= 1
+
+ records = [
+ json.loads(line)
+ for line in (output_dir / "serving-metrics.jsonl").read_text(encoding="utf-8").splitlines()
+ ]
+ assert len(records) == manifest["samples"]["attempted"]
+ assert records[0]["serving"]["snapshot"]["summary"]["sessions"]["active"] == 4
+ first_prometheus = output_dir / records[0]["prometheus"]["path"]
+ assert "telefuser_serving_sessions" in first_prometheus.read_text(encoding="utf-8")
+
+ stored_manifest = json.loads((output_dir / "manifest.json").read_text(encoding="utf-8"))
+ assert stored_manifest == manifest
diff --git a/tests/unit/validation/test_capture_gpu_nvml_metrics.py b/tests/unit/validation/test_capture_gpu_nvml_metrics.py
new file mode 100644
index 00000000..c7e89e15
--- /dev/null
+++ b/tests/unit/validation/test_capture_gpu_nvml_metrics.py
@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import pytest
+
+from tools.validation import capture_gpu_nvml_metrics as capture
+
+
+def test_parse_args_defaults_to_the_four_experiment_gpus(tmp_path) -> None:
+ config = capture.parse_args(["--duration", "2", "--output-dir", str(tmp_path / "metrics")])
+
+ assert config.gpu_indices == (0, 1, 2, 3)
+ assert config.duration_seconds == 2.0
+ assert config.interval_seconds == 1.0
+
+
+@pytest.mark.parametrize("value", ["", "0,,1", "0,0", "-1", "one"])
+def test_gpu_indices_reject_malformed_values(value: str) -> None:
+ with pytest.raises(SystemExit):
+ capture.parse_args(["--gpu-indices", value, "--duration", "1", "--output-dir", "/tmp/unused"])
diff --git a/tools/validation/benchmark_abot_livekit_burst.py b/tools/validation/benchmark_abot_livekit_burst.py
new file mode 100644
index 00000000..847c023c
--- /dev/null
+++ b/tools/validation/benchmark_abot_livekit_burst.py
@@ -0,0 +1,1383 @@
+#!/usr/bin/env python3
+"""Run a phase-based, black-box ABot-World LiveKit workload.
+
+The driver intentionally talks only to the public serving interfaces:
+
+* ``POST /v1/stream/sessions`` for admission;
+* a real LiveKit/WebRTC room for keyboard controls and video consumption; and
+* ``DELETE /v1/stream/sessions/{session_id}`` for departure.
+
+It never selects a GPU, reaches into a worker process, or calls an ABot
+pipeline object. It is therefore suitable for measuring the four-GPU serving
+system as a black box. The LiveKit client flow is the same one used by the
+TeleFuser AIPerf adapter, but this tool adds phase-based user arrivals and
+departures for long-lived world-model sessions.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import contextlib
+import ipaddress
+import json
+import os
+import random
+import statistics
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpx
+
+_CONTROL_TOPIC = "tf.control"
+_METRICS_TOPIC = "tf.metrics"
+_STATUS_TOPIC = "tf.status"
+_PROXY_ENV_NAMES = (
+ "http_proxy",
+ "https_proxy",
+ "all_proxy",
+ "HTTP_PROXY",
+ "HTTPS_PROXY",
+ "ALL_PROXY",
+)
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+class ScenarioError(ValueError):
+ """Raised when a user-wave scenario is malformed."""
+
+
+@dataclass(frozen=True)
+class Phase:
+ """One target-concurrency interval in a user-wave experiment."""
+
+ name: str
+ duration_seconds: float
+ target_users: int
+ arrival_window_seconds: float = 0.0
+ departure_window_seconds: float = 0.0
+ active_input_fraction: float = 1.0
+ input_transition_window_seconds: float = 0.0
+
+
+@dataclass(frozen=True)
+class ControlConfig:
+ """Independent keyboard activity generated for each connected client."""
+
+ interval_seconds: float
+ jitter_seconds: float
+ idle_probability: float
+ idle_min_seconds: float
+ idle_max_seconds: float
+ action_states: tuple[tuple[str, ...], ...]
+
+
+@dataclass(frozen=True)
+class SessionConfig:
+ """ABot request and user-visible playback settings."""
+
+ prompt: str
+ image_path: str
+ fps: float
+ control_latent_frames: int
+ delivery_mode: str
+ expected_preview_frames: int
+ control: ControlConfig
+
+
+@dataclass(frozen=True)
+class AdmissionExpectation:
+ """Public-admission contract required for a workload to be valid."""
+
+ require_immediate_assignment: bool = False
+ expected_max_sessions_per_worker: int | None = None
+ expected_queue_size: int | None = None
+
+
+@dataclass(frozen=True)
+class Scenario:
+ """Validated configuration of one complete black-box experiment."""
+
+ name: str
+ server_url: str
+ session: SessionConfig
+ phases: tuple[Phase, ...]
+ sample_interval_seconds: float
+ connect_timeout_seconds: float
+ http_timeout_seconds: float
+ shutdown_timeout_seconds: float
+ first_generation_grace_seconds: float
+ slo_fps_tolerance: float
+ seed: int
+ expected_worker_mode: str | None
+ admission: AdmissionExpectation
+ expected_num_workers: int | None
+ raw: dict[str, Any]
+
+
+def _percentile(values: Sequence[float], quantile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ index = max(0, min(len(ordered) - 1, int(len(ordered) * quantile + 0.999999) - 1))
+ return float(ordered[index])
+
+
+def _summary(values: Sequence[float]) -> dict[str, float | int]:
+ return {
+ "count": len(values),
+ "mean": round(statistics.fmean(values), 6) if values else 0.0,
+ "p50": round(_percentile(values, 0.50), 6),
+ "p95": round(_percentile(values, 0.95), 6),
+ "p99": round(_percentile(values, 0.99), 6),
+ "maximum": round(max(values), 6) if values else 0.0,
+ }
+
+
+def _require_mapping(value: object, label: str) -> dict[str, Any]:
+ if not isinstance(value, Mapping):
+ raise ScenarioError(f"{label} must be an object")
+ return dict(value)
+
+
+def _require_positive_float(value: object, label: str, *, allow_zero: bool = False) -> float:
+ if not isinstance(value, int | float) or isinstance(value, bool):
+ raise ScenarioError(f"{label} must be a number")
+ parsed = float(value)
+ if parsed < 0 or (not allow_zero and parsed == 0):
+ comparison = "non-negative" if allow_zero else "positive"
+ raise ScenarioError(f"{label} must be {comparison}")
+ return parsed
+
+
+def _require_non_negative_int(value: object, label: str) -> int:
+ if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+ raise ScenarioError(f"{label} must be a non-negative integer")
+ return int(value)
+
+
+def _resolve_image_path(value: object) -> str:
+ if not isinstance(value, str) or not value.strip():
+ raise ScenarioError("session.image_path must be a non-empty path")
+ path = Path(value).expanduser()
+ if not path.is_absolute():
+ path = (_REPO_ROOT / path).resolve()
+ if not path.is_file():
+ raise ScenarioError(f"session.image_path does not exist: {path}")
+ return str(path)
+
+
+def _parse_control(value: object) -> ControlConfig:
+ raw = _require_mapping(value, "session.control")
+ interval = _require_positive_float(raw.get("interval_seconds", 0.5), "session.control.interval_seconds")
+ jitter = _require_positive_float(raw.get("jitter_seconds", 0.0), "session.control.jitter_seconds", allow_zero=True)
+ idle_probability = _require_positive_float(
+ raw.get("idle_probability", 0.0), "session.control.idle_probability", allow_zero=True
+ )
+ if idle_probability > 1:
+ raise ScenarioError("session.control.idle_probability must be at most one")
+ idle_min = _require_positive_float(
+ raw.get("idle_min_seconds", 0.2), "session.control.idle_min_seconds", allow_zero=True
+ )
+ idle_max = _require_positive_float(
+ raw.get("idle_max_seconds", 1.0), "session.control.idle_max_seconds", allow_zero=True
+ )
+ if idle_max < idle_min:
+ raise ScenarioError("session.control.idle_max_seconds must be at least idle_min_seconds")
+ raw_states = raw.get("action_states", [["KeyW"], ["KeyW", "KeyA"], ["KeyW", "KeyD"], ["KeyI"]])
+ if not isinstance(raw_states, list) or not raw_states:
+ raise ScenarioError("session.control.action_states must be a non-empty list of key lists")
+ action_states: list[tuple[str, ...]] = []
+ for index, state in enumerate(raw_states):
+ if not isinstance(state, list) or not state or not all(isinstance(key, str) and key for key in state):
+ raise ScenarioError(f"session.control.action_states[{index}] must be a non-empty list of keys")
+ action_states.append(tuple(state))
+ return ControlConfig(
+ interval_seconds=interval,
+ jitter_seconds=jitter,
+ idle_probability=idle_probability,
+ idle_min_seconds=idle_min,
+ idle_max_seconds=idle_max,
+ action_states=tuple(action_states),
+ )
+
+
+def _parse_admission(value: object) -> AdmissionExpectation:
+ """Parse optional public-admission expectations for a user wave."""
+ raw = _require_mapping(value, "admission")
+ require_immediate_assignment = raw.get("require_immediate_assignment", False)
+ if not isinstance(require_immediate_assignment, bool):
+ raise ScenarioError("admission.require_immediate_assignment must be a boolean")
+
+ def optional_positive_int(key: str) -> int | None:
+ field = raw.get(key)
+ if field is None:
+ return None
+ parsed = _require_non_negative_int(field, f"admission.{key}")
+ if parsed < 1:
+ raise ScenarioError(f"admission.{key} must be positive when supplied")
+ return parsed
+
+ expected_queue_size = raw.get("expected_queue_size")
+ if expected_queue_size is not None:
+ expected_queue_size = _require_non_negative_int(expected_queue_size, "admission.expected_queue_size")
+
+ return AdmissionExpectation(
+ require_immediate_assignment=require_immediate_assignment,
+ expected_max_sessions_per_worker=optional_positive_int("expected_max_sessions_per_worker"),
+ expected_queue_size=expected_queue_size,
+ )
+
+
+def load_scenario(path: Path, *, server_url_override: str | None = None) -> Scenario:
+ """Load and validate a JSON user-wave scenario."""
+ try:
+ raw_value = json.loads(path.read_text())
+ except OSError as exc:
+ raise ScenarioError(f"Could not read scenario {path}: {exc}") from exc
+ except json.JSONDecodeError as exc:
+ raise ScenarioError(f"Scenario is not valid JSON: {exc}") from exc
+ raw = _require_mapping(raw_value, "scenario")
+ name = raw.get("name", path.stem)
+ if not isinstance(name, str) or not name.strip():
+ raise ScenarioError("scenario.name must be a non-empty string")
+ server_url = server_url_override or raw.get("server_url", "http://127.0.0.1:8088")
+ if not isinstance(server_url, str) or not server_url.startswith(("http://", "https://")):
+ raise ScenarioError("server_url must be an http(s) URL")
+ server_url = server_url.rstrip("/")
+
+ session_raw = _require_mapping(raw.get("session"), "session")
+ prompt = session_raw.get("prompt")
+ if not isinstance(prompt, str) or not prompt.strip():
+ raise ScenarioError("session.prompt must be a non-empty string")
+ fps = _require_positive_float(session_raw.get("fps", 12), "session.fps")
+ control_latent_frames = _require_non_negative_int(
+ session_raw.get("control_latent_frames", 3), "session.control_latent_frames"
+ )
+ if control_latent_frames not in {1, 2, 3}:
+ raise ScenarioError("session.control_latent_frames must be 1, 2, or 3")
+ delivery_mode = session_raw.get("delivery_mode", "latest")
+ if delivery_mode not in {"latest", "lossless"}:
+ raise ScenarioError("session.delivery_mode must be 'latest' or 'lossless'")
+ expected_preview_frames = _require_non_negative_int(
+ session_raw.get("expected_preview_frames", 1), "session.expected_preview_frames"
+ )
+ admission = _parse_admission(raw.get("admission", {}))
+ session = SessionConfig(
+ prompt=prompt,
+ image_path=_resolve_image_path(session_raw.get("image_path")),
+ fps=fps,
+ control_latent_frames=control_latent_frames,
+ delivery_mode=delivery_mode,
+ expected_preview_frames=expected_preview_frames,
+ control=_parse_control(session_raw.get("control", {})),
+ )
+
+ phases_raw = raw.get("phases")
+ if not isinstance(phases_raw, list) or not phases_raw:
+ raise ScenarioError("phases must be a non-empty list")
+ phases: list[Phase] = []
+ phase_names: set[str] = set()
+ for index, phase_value in enumerate(phases_raw):
+ phase_raw = _require_mapping(phase_value, f"phases[{index}]")
+ phase_name = phase_raw.get("name", f"phase-{index}")
+ if not isinstance(phase_name, str) or not phase_name.strip():
+ raise ScenarioError(f"phases[{index}].name must be a non-empty string")
+ if phase_name in phase_names:
+ raise ScenarioError(f"Duplicate phase name: {phase_name}")
+ phase_names.add(phase_name)
+ duration = _require_positive_float(phase_raw.get("duration_seconds"), f"phases[{index}].duration_seconds")
+ arrival_window = _require_positive_float(
+ phase_raw.get("arrival_window_seconds", 0.0),
+ f"phases[{index}].arrival_window_seconds",
+ allow_zero=True,
+ )
+ departure_window = _require_positive_float(
+ phase_raw.get("departure_window_seconds", 0.0),
+ f"phases[{index}].departure_window_seconds",
+ allow_zero=True,
+ )
+ active_input_fraction = _require_positive_float(
+ phase_raw.get("active_input_fraction", 1.0),
+ f"phases[{index}].active_input_fraction",
+ allow_zero=True,
+ )
+ if active_input_fraction > 1:
+ raise ScenarioError(f"phases[{index}].active_input_fraction must be at most one")
+ input_transition_window = _require_positive_float(
+ phase_raw.get("input_transition_window_seconds", 0.0),
+ f"phases[{index}].input_transition_window_seconds",
+ allow_zero=True,
+ )
+ if (
+ arrival_window > duration
+ or departure_window > duration
+ or input_transition_window > duration
+ ):
+ raise ScenarioError(f"phases[{index}] transition windows cannot exceed duration")
+ phases.append(
+ Phase(
+ name=phase_name,
+ duration_seconds=duration,
+ target_users=_require_non_negative_int(phase_raw.get("target_users"), f"phases[{index}].target_users"),
+ arrival_window_seconds=arrival_window,
+ departure_window_seconds=departure_window,
+ active_input_fraction=active_input_fraction,
+ input_transition_window_seconds=input_transition_window,
+ )
+ )
+
+ measurement = _require_mapping(raw.get("measurement", {}), "measurement")
+ expected_worker_mode = raw.get("expected_worker_mode")
+ if expected_worker_mode is not None and not isinstance(expected_worker_mode, str):
+ raise ScenarioError("expected_worker_mode must be a string when supplied")
+ expected_num_workers = raw.get("expected_num_workers")
+ if expected_num_workers is not None:
+ expected_num_workers = _require_non_negative_int(expected_num_workers, "expected_num_workers")
+ if expected_num_workers < 1:
+ raise ScenarioError("expected_num_workers must be positive")
+ seed = raw.get("seed", 42)
+ if not isinstance(seed, int) or isinstance(seed, bool):
+ raise ScenarioError("seed must be an integer")
+ slo_fps_tolerance = _require_positive_float(
+ measurement.get("slo_fps_tolerance", 0.25),
+ "measurement.slo_fps_tolerance",
+ allow_zero=True,
+ )
+ if slo_fps_tolerance >= session.fps:
+ raise ScenarioError("measurement.slo_fps_tolerance must be smaller than session.fps")
+ return Scenario(
+ name=name,
+ server_url=server_url,
+ session=session,
+ phases=tuple(phases),
+ sample_interval_seconds=_require_positive_float(
+ measurement.get("sample_interval_seconds", 1.0), "measurement.sample_interval_seconds"
+ ),
+ connect_timeout_seconds=_require_positive_float(
+ measurement.get("connect_timeout_seconds", 60.0), "measurement.connect_timeout_seconds"
+ ),
+ http_timeout_seconds=_require_positive_float(
+ measurement.get("http_timeout_seconds", 30.0), "measurement.http_timeout_seconds"
+ ),
+ shutdown_timeout_seconds=_require_positive_float(
+ measurement.get("shutdown_timeout_seconds", 20.0), "measurement.shutdown_timeout_seconds"
+ ),
+ admission=admission,
+ first_generation_grace_seconds=_require_positive_float(
+ measurement.get("first_generation_grace_seconds", 15.0),
+ "measurement.first_generation_grace_seconds",
+ allow_zero=True,
+ ),
+ slo_fps_tolerance=slo_fps_tolerance,
+ seed=seed,
+ expected_worker_mode=expected_worker_mode,
+ expected_num_workers=expected_num_workers,
+ raw=raw,
+ )
+
+
+def _disable_proxy_for_loopback(url: str) -> None:
+ """Avoid accidentally sending local LiveKit traffic through a proxy."""
+ host = urlsplit(url).hostname
+ if host is None:
+ return
+ try:
+ loopback = host.lower() == "localhost" or ipaddress.ip_address(host).is_loopback
+ except ValueError:
+ loopback = False
+ if loopback:
+ for name in _PROXY_ENV_NAMES:
+ os.environ.pop(name, None)
+
+
+def _load_livekit_rtc() -> Any:
+ try:
+ from livekit import rtc
+ except ModuleNotFoundError as exc:
+ raise RuntimeError(
+ "The black-box LiveKit workload requires the TeleFuser 'livekit' runtime dependency. "
+ "Use the Telefuser serving environment."
+ ) from exc
+ return rtc
+
+
+def _safe_json_message(payload: bytes | str) -> dict[str, Any] | None:
+ try:
+ decoded = json.loads(payload.decode("utf-8") if isinstance(payload, bytes) else payload)
+ except (UnicodeDecodeError, json.JSONDecodeError):
+ return None
+ return decoded if isinstance(decoded, dict) else None
+
+
+@dataclass
+class LiveKitWaveSession:
+ """One real browser-equivalent ABot client in a user-wave experiment."""
+
+ index: int
+ scenario: Scenario
+ http: httpx.AsyncClient
+ rtc: Any
+ record_event: Any
+ started_at: float
+ _room: Any | None = field(default=None, init=False, repr=False)
+ _video_streams: list[Any] = field(default_factory=list, init=False, repr=False)
+ _video_tasks: list[asyncio.Task[None]] = field(default_factory=list, init=False, repr=False)
+ _control_task: asyncio.Task[None] | None = field(default=None, init=False, repr=False)
+ _rng: random.Random = field(init=False, repr=False)
+ scheduled_at: float | None = field(default=None, init=False)
+ create_started_at: float | None = field(default=None, init=False)
+ created_at: float | None = field(default=None, init=False)
+ connected_at: float | None = field(default=None, init=False)
+ first_media_frame_at: float | None = field(default=None, init=False)
+ first_generated_frame_at: float | None = field(default=None, init=False)
+ last_generated_frame_at: float | None = field(default=None, init=False)
+ first_active_control_at: float | None = field(default=None, init=False)
+ stopped_at: float | None = field(default=None, init=False)
+ server_session_id: str | None = field(default=None, init=False)
+ worker_id: str | None = field(default=None, init=False)
+ admission_status: str | None = field(default=None, init=False)
+ queue_position: int | None = field(default=None, init=False)
+ admission_violation: str | None = field(default=None, init=False)
+ frames_received: int = field(default=0, init=False)
+ generated_frames_received: int = field(default=0, init=False)
+ control_messages_sent: int = field(default=0, init=False)
+ status_messages_received: int = field(default=0, init=False)
+ error: str | None = field(default=None, init=False)
+ stop_requested: bool = field(default=False, init=False)
+ departure_scheduled: bool = field(default=False, init=False)
+ remote_session_deleted: bool = field(default=False, init=False)
+ connected: bool = field(default=False, init=False)
+ current_controls: tuple[str, ...] = field(default_factory=tuple, init=False)
+ input_enabled: bool = field(default=True, init=False)
+ input_pauses: int = field(default=0, init=False)
+ input_resumes: int = field(default=0, init=False)
+ input_pause_started_at: float | None = field(default=None, init=False)
+
+ def __post_init__(self) -> None:
+ self._rng = random.Random(self.scenario.seed + self.index)
+
+ @property
+ def logical_id(self) -> str:
+ return f"wave-{self.index:03d}"
+
+ @property
+ def active_controls(self) -> bool:
+ return bool(self.current_controls) and self.input_enabled and not self.stop_requested
+
+ async def set_input_enabled(self, enabled: bool, *, reason: str) -> None:
+ """Pause or resume controller input without dropping the LiveKit session.
+
+ This models a user temporarily releasing all keys or switching away from
+ the browser. It preserves the public serving session, KV/decoder state,
+ and WebRTC subscription; actual session departure still uses ``stop``.
+ """
+ if self.stop_requested or self.input_enabled == enabled:
+ return
+ now = time.perf_counter()
+ self.input_enabled = enabled
+ if enabled:
+ self.input_resumes += 1
+ paused_for = (
+ max(0.0, now - self.input_pause_started_at)
+ if self.input_pause_started_at is not None
+ else None
+ )
+ self.input_pause_started_at = None
+ self.record_event(
+ "input_resumed",
+ session=self.logical_id,
+ reason=reason,
+ paused_seconds=round(paused_for, 6) if paused_for is not None else None,
+ )
+ controls = self._rng.choice(self.scenario.session.control.action_states)
+ else:
+ self.input_pauses += 1
+ self.input_pause_started_at = now
+ self.record_event("input_paused", session=self.logical_id, reason=reason)
+ controls = ()
+
+ # Do not wait for the next heartbeat to clear a stale key state.
+ if self.connected and self._room is not None:
+ try:
+ await self._publish_control_state(controls)
+ except Exception as exc: # noqa: BLE001 - a transition failure is a workload fact
+ detail = f"{type(exc).__name__}: {exc}"
+ if self.error is None:
+ self.error = f"InputTransitionPublishError: {detail}"
+ self.record_event("input_transition_publish_error", session=self.logical_id, error=detail)
+
+ async def start(self) -> None:
+ """Create a public session, join its room, and start keyboard heartbeats."""
+ if self.stop_requested:
+ return
+ self.create_started_at = time.perf_counter()
+ identity = f"abot-wave-{self.index}-{self.scenario.seed}"
+ request = {
+ "identity": identity,
+ "role": "controller",
+ "prompt": self.scenario.session.prompt,
+ "image_path": self.scenario.session.image_path,
+ "config": {
+ "fps": self.scenario.session.fps,
+ "control_latent_frames": self.scenario.session.control_latent_frames,
+ "delivery_mode": self.scenario.session.delivery_mode,
+ "seed": self.scenario.seed + self.index,
+ },
+ }
+ self.record_event("session_create_started", session=self.logical_id)
+ try:
+ response = await self.http.post(
+ f"{self.scenario.server_url}/v1/stream/sessions",
+ json=request,
+ )
+ response.raise_for_status()
+ created = response.json()
+ if not isinstance(created, Mapping):
+ raise RuntimeError("session-create response is not an object")
+ required = ("session_id", "livekit_url", "token")
+ missing = [key for key in required if not isinstance(created.get(key), str)]
+ if missing:
+ raise RuntimeError("session-create response is missing " + ", ".join(missing))
+ self.server_session_id = str(created["session_id"])
+ worker_id = created.get("worker_id")
+ self.worker_id = worker_id if isinstance(worker_id, str) else None
+ status = created.get("status")
+ self.admission_status = status if isinstance(status, str) else None
+ queue_position = created.get("queue_position")
+ self.queue_position = queue_position if isinstance(queue_position, int) else None
+ if self.scenario.admission.require_immediate_assignment and self.admission_status != "assigned":
+ self.admission_violation = f"Expected immediate assignment, got {self.admission_status!r}"
+ self.record_event(
+ "admission_contract_violation",
+ session=self.logical_id,
+ violation=self.admission_violation,
+ )
+ self.created_at = time.perf_counter()
+ self.record_event(
+ "session_created",
+ session=self.logical_id,
+ server_session_id=self.server_session_id,
+ worker_id=self.worker_id,
+ status=self.admission_status,
+ queue_position=self.queue_position,
+ offer_rtt_seconds=round(self.created_at - self.create_started_at, 6),
+ )
+ if self.stop_requested:
+ await self._delete_remote_session()
+ return
+ await self._connect_room(str(created["livekit_url"]), str(created["token"]))
+ except Exception as exc: # noqa: BLE001 - externally visible workload outcome
+ self.error = f"{type(exc).__name__}: {exc}"
+ self.record_event("session_start_failed", session=self.logical_id, error=self.error)
+ await self._delete_remote_session()
+
+ async def _connect_room(self, livekit_url: str, token: str) -> None:
+ _disable_proxy_for_loopback(livekit_url)
+ room = self.rtc.Room()
+ self._room = room
+
+ @room.on("data_received")
+ def on_data_received(packet: Any) -> None:
+ topic = getattr(packet, "topic", "") or ""
+ if topic not in {_STATUS_TOPIC, _METRICS_TOPIC}:
+ return
+ self.status_messages_received += 1
+ payload = _safe_json_message(getattr(packet, "data", b""))
+ if payload is None:
+ return
+ data = payload.get("data") if isinstance(payload.get("data"), Mapping) else payload
+ stage = data.get("stage") if isinstance(data, Mapping) else None
+ if stage in {"worker_running", "runtime_ready"}:
+ self.record_event("worker_ready", session=self.logical_id, stage=stage)
+
+ @room.on("track_subscribed")
+ def on_track_subscribed(track: Any, _publication: Any, _participant: Any) -> None:
+ if getattr(track, "kind", None) != self.rtc.TrackKind.KIND_VIDEO:
+ return
+ stream = self.rtc.VideoStream(track)
+ self._video_streams.append(stream)
+ self._video_tasks.append(asyncio.create_task(self._consume_video(stream)))
+ self.record_event("video_track_subscribed", session=self.logical_id)
+
+ @room.on("disconnected")
+ def on_disconnected(reason: Any) -> None:
+ if not self.stop_requested and self.error is None:
+ self.error = f"LiveKit disconnected: {reason}"
+ self.record_event("room_disconnected", session=self.logical_id, reason=str(reason))
+
+ options = self.rtc.RoomOptions(auto_subscribe=True, connect_timeout=self.scenario.connect_timeout_seconds)
+ await room.connect(livekit_url, token, options)
+ self.connected = True
+ self.connected_at = time.perf_counter()
+ self.record_event(
+ "room_connected",
+ session=self.logical_id,
+ connected_seconds=round(self.connected_at - (self.create_started_at or self.connected_at), 6),
+ )
+ if not self.stop_requested:
+ self._control_task = asyncio.create_task(self._send_controls(), name=f"abot-controls-{self.logical_id}")
+
+ async def _consume_video(self, stream: Any) -> None:
+ try:
+ async for _ in stream:
+ now = time.perf_counter()
+ self.frames_received += 1
+ if self.first_media_frame_at is None:
+ self.first_media_frame_at = now
+ self.record_event("first_media_frame", session=self.logical_id)
+ if self.frames_received > self.scenario.session.expected_preview_frames:
+ self.generated_frames_received += 1
+ if self.first_generated_frame_at is None:
+ self.first_generated_frame_at = now
+ self.record_event(
+ "first_generated_frame",
+ session=self.logical_id,
+ action_to_first_generated_seconds=(
+ round(now - self.first_active_control_at, 6)
+ if self.first_active_control_at is not None
+ else None
+ ),
+ )
+ self.last_generated_frame_at = now
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # noqa: BLE001 - WebRTC termination is a workload fact
+ if not self.stop_requested and self.error is None:
+ self.error = f"VideoStreamError: {type(exc).__name__}: {exc}"
+ self.record_event("video_stream_error", session=self.logical_id, error=self.error)
+
+ async def _send_controls(self) -> None:
+ assert self._room is not None
+ control = self.scenario.session.control
+ idle_until = 0.0
+ sent_non_idle_control = False
+ try:
+ while not self.stop_requested:
+ now = time.perf_counter()
+ if not self.input_enabled or now < idle_until:
+ controls: tuple[str, ...] = ()
+ else:
+ if sent_non_idle_control and self._rng.random() < control.idle_probability:
+ idle_until = now + self._rng.uniform(control.idle_min_seconds, control.idle_max_seconds)
+ controls = ()
+ else:
+ controls = self._rng.choice(control.action_states)
+ try:
+ await self._publish_control_state(controls)
+ sent_non_idle_control = sent_non_idle_control or bool(controls)
+ except Exception as exc: # noqa: BLE001 - network write error is data
+ if not self.stop_requested and self.error is None:
+ self.error = f"ControlPublishError: {type(exc).__name__}: {exc}"
+ self.record_event("control_publish_error", session=self.logical_id, error=self.error)
+ interval = control.interval_seconds + self._rng.uniform(-control.jitter_seconds, control.jitter_seconds)
+ await asyncio.sleep(max(interval, 0.01))
+ except asyncio.CancelledError:
+ raise
+
+ async def _publish_control_state(self, controls: tuple[str, ...]) -> None:
+ """Publish one reliable control heartbeat and retain its public state."""
+ if self._room is None:
+ return
+ payload = {"type": "control_state", "controls": list(controls)}
+ await self._room.local_participant.publish_data(
+ json.dumps(payload, separators=(",", ":")).encode("utf-8"),
+ topic=_CONTROL_TOPIC,
+ reliable=True,
+ )
+ self.control_messages_sent += 1
+ self.current_controls = controls
+ if controls and self.first_active_control_at is None:
+ self.first_active_control_at = time.perf_counter()
+ self.record_event("first_active_control", session=self.logical_id, controls=list(controls))
+
+ async def stop(self) -> None:
+ """Stop only this client/session; sibling sessions keep running."""
+ if self.stop_requested and self.stopped_at is not None:
+ return
+ self.stop_requested = True
+ self.departure_scheduled = False
+ self.current_controls = ()
+ control_task = self._control_task
+ if control_task is not None and not control_task.done():
+ control_task.cancel()
+ await asyncio.gather(control_task, return_exceptions=True)
+ room = self._room
+ if room is not None and self.connected:
+ with contextlib.suppress(Exception):
+ await room.local_participant.publish_data(b'{"type":"stop"}', topic=_CONTROL_TOPIC, reliable=True)
+ await self._delete_remote_session()
+ for stream in self._video_streams:
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(stream.aclose(), timeout=self.scenario.shutdown_timeout_seconds)
+ for task in self._video_tasks:
+ if not task.done():
+ task.cancel()
+ if self._video_tasks:
+ await asyncio.gather(*self._video_tasks, return_exceptions=True)
+ if room is not None:
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(room.disconnect(), timeout=self.scenario.shutdown_timeout_seconds)
+ self.connected = False
+ self.stopped_at = time.perf_counter()
+ self.record_event("session_stopped", session=self.logical_id)
+
+ async def _delete_remote_session(self) -> None:
+ if self.server_session_id is None or self.remote_session_deleted:
+ return
+ session_id = self.server_session_id
+ self.remote_session_deleted = True
+ try:
+ await self.http.delete(
+ f"{self.scenario.server_url}/v1/stream/sessions/{session_id}",
+ )
+ self.record_event("session_deleted", session=self.logical_id, server_session_id=session_id)
+ except Exception as exc: # noqa: BLE001 - cleanup is best effort
+ self.record_event(
+ "session_delete_error",
+ session=self.logical_id,
+ server_session_id=session_id,
+ error=f"{type(exc).__name__}: {exc}",
+ )
+
+ def snapshot(self, now: float) -> dict[str, Any]:
+ """Return bounded client-side facts; no model-internal state is read."""
+ action_to_first = (
+ self.first_generated_frame_at - self.first_active_control_at
+ if self.first_generated_frame_at is not None and self.first_active_control_at is not None
+ else None
+ )
+ creation_to_first = (
+ self.first_generated_frame_at - self.create_started_at
+ if self.first_generated_frame_at is not None and self.create_started_at is not None
+ else None
+ )
+ return {
+ "logical_session_id": self.logical_id,
+ "server_session_id": self.server_session_id,
+ "worker_id_at_admission": self.worker_id,
+ "admission_status": self.admission_status,
+ "queue_position": self.queue_position,
+ "scheduled": self.scheduled_at is not None,
+ "admission_contract_violation": self.admission_violation,
+ "connected": self.connected,
+ "stop_requested": self.stop_requested,
+ "departure_scheduled": self.departure_scheduled,
+ "remote_session_deleted": self.remote_session_deleted,
+ "input_enabled": self.input_enabled,
+ "active_controls": self.active_controls,
+ "input_pauses": self.input_pauses,
+ "input_resumes": self.input_resumes,
+ "frames_received": self.frames_received,
+ "generated_frames_received": self.generated_frames_received,
+ "control_messages_sent": self.control_messages_sent,
+ "status_messages_received": self.status_messages_received,
+ "offer_rtt_seconds": (
+ round(self.created_at - self.create_started_at, 6)
+ if self.created_at is not None and self.create_started_at is not None
+ else None
+ ),
+ "connected_seconds": (
+ round(self.connected_at - self.create_started_at, 6)
+ if self.connected_at is not None and self.create_started_at is not None
+ else None
+ ),
+ "action_to_first_generated_seconds": round(action_to_first, 6) if action_to_first is not None else None,
+ "creation_to_first_generated_seconds": (
+ round(creation_to_first, 6) if creation_to_first is not None else None
+ ),
+ "last_generated_frame_age_seconds": (
+ round(now - self.last_generated_frame_at, 6) if self.last_generated_frame_at is not None else None
+ ),
+ "error": self.error,
+ }
+
+
+class LiveKitWaveRunner:
+ """Coordinate a black-box user wave and persist client-delivery facts."""
+
+ def __init__(self, scenario: Scenario) -> None:
+ self.scenario = scenario
+ self.rtc = _load_livekit_rtc()
+ self.started_at = 0.0
+ self._sessions: list[LiveKitWaveSession] = []
+ self._background_tasks: set[asyncio.Task[None]] = set()
+ self._monitor_task: asyncio.Task[None] | None = None
+ self._monitoring = False
+ self._phase_name = "startup"
+ self._phase_target_users = 0
+ self._phase_active_input_fraction = 1.0
+ self._samples: list[dict[str, Any]] = []
+ self._events: list[dict[str, Any]] = []
+ self._phase_results: list[dict[str, Any]] = []
+ self._previous_generated_frames: dict[str, int] = {}
+ self._previous_sample_at: float | None = None
+ self._server_metadata: list[dict[str, Any]] = []
+ self._warnings: list[str] = []
+
+ def record_event(self, event: str, **values: Any) -> None:
+ now = time.perf_counter()
+ self._events.append(
+ {
+ "offset_seconds": round(max(0.0, now - self.started_at), 6),
+ "event": event,
+ **values,
+ }
+ )
+
+ async def run(self) -> dict[str, Any]:
+ """Run all phases and return a complete JSON-serializable artifact."""
+ _disable_proxy_for_loopback(self.scenario.server_url)
+ timeout = httpx.Timeout(self.scenario.http_timeout_seconds)
+ # User arrivals in the supplied wave are intentionally several seconds
+ # apart. Uvicorn can close an idle HTTP/1.1 keep-alive socket at the
+ # same boundary, which otherwise turns an unrelated client-side stale
+ # connection into a false "session_start_failed" observation. Session
+ # admission is not idempotent, so retrying a POST after a read failure
+ # could create a duplicate world state. Use a fresh loopback HTTP
+ # connection for each public API request instead.
+ limits = httpx.Limits(max_keepalive_connections=0)
+ async with httpx.AsyncClient(timeout=timeout, limits=limits, trust_env=False) as http:
+ self._http = http
+ self.started_at = time.perf_counter()
+ await self._capture_server_metadata("before_workload")
+ self._monitoring = True
+ self._monitor_task = asyncio.create_task(self._monitor(), name="abot-livekit-wave-monitor")
+ try:
+ for phase in self.scenario.phases:
+ await self._run_phase(phase)
+ finally:
+ self._monitoring = False
+ if self._monitor_task is not None:
+ self._monitor_task.cancel()
+ await asyncio.gather(self._monitor_task, return_exceptions=True)
+ await self._stop_all_sessions()
+ await self._capture_server_metadata("after_workload")
+ completed_at = time.perf_counter()
+ return {
+ "schema_version": "abot_livekit_user_wave_v1",
+ "scenario": self.scenario.raw,
+ "effective_scenario": {
+ "name": self.scenario.name,
+ "server_url": self.scenario.server_url,
+ "target_fps_per_active_session": self.scenario.session.fps,
+ "control_latent_frames": self.scenario.session.control_latent_frames,
+ "delivery_mode": self.scenario.session.delivery_mode,
+ "slo_fps_tolerance": self.scenario.slo_fps_tolerance,
+ "admission": {
+ "require_immediate_assignment": self.scenario.admission.require_immediate_assignment,
+ "expected_max_sessions_per_worker": self.scenario.admission.expected_max_sessions_per_worker,
+ "expected_queue_size": self.scenario.admission.expected_queue_size,
+ },
+ "phases": [
+ {
+ "name": phase.name,
+ "duration_seconds": phase.duration_seconds,
+ "target_users": phase.target_users,
+ "arrival_window_seconds": phase.arrival_window_seconds,
+ "departure_window_seconds": phase.departure_window_seconds,
+ "active_input_fraction": phase.active_input_fraction,
+ "input_transition_window_seconds": phase.input_transition_window_seconds,
+ }
+ for phase in self.scenario.phases
+ ],
+ },
+ "started_at_unix_seconds": time.time() - max(0.0, completed_at - self.started_at),
+ "elapsed_seconds": round(completed_at - self.started_at, 6),
+ "server_metadata": self._server_metadata,
+ "warnings": self._warnings,
+ "phase_results": self._phase_results,
+ "sessions": [session.snapshot(completed_at) for session in self._sessions],
+ "samples": self._samples,
+ "events": self._events,
+ }
+
+ async def _run_phase(self, phase: Phase) -> None:
+ phase_started = time.perf_counter()
+ sample_start = len(self._samples)
+ self._phase_name = phase.name
+ self._phase_target_users = phase.target_users
+ self._phase_active_input_fraction = phase.active_input_fraction
+ self.record_event(
+ "phase_started",
+ phase=phase.name,
+ target_users=phase.target_users,
+ active_input_fraction=phase.active_input_fraction,
+ )
+ await self._capture_server_metadata(f"phase_start:{phase.name}")
+ self._schedule_transition(phase)
+ self._schedule_input_activity(phase)
+ await asyncio.sleep(phase.duration_seconds)
+ phase_completed = time.perf_counter()
+ await self._capture_server_metadata(f"phase_end:{phase.name}")
+ result = self._summarize_phase(
+ phase,
+ phase_started=phase_started,
+ phase_completed=phase_completed,
+ samples=self._samples[sample_start:],
+ )
+ self._phase_results.append(result)
+ self.record_event("phase_completed", phase=phase.name, summary=result["summary"])
+
+ def _schedule_transition(self, phase: Phase) -> None:
+ present = [
+ session for session in self._sessions if not session.stop_requested and not session.departure_scheduled
+ ]
+ difference = phase.target_users - len(present)
+ if difference > 0:
+ for ordinal in range(difference):
+ session = LiveKitWaveSession(
+ index=len(self._sessions),
+ scenario=self.scenario,
+ http=self._http,
+ rtc=self.rtc,
+ record_event=self.record_event,
+ started_at=self.started_at,
+ )
+ session.scheduled_at = time.perf_counter()
+ self._sessions.append(session)
+ offset = self._spread_offset(ordinal + 1, difference, phase.arrival_window_seconds)
+ self._spawn_background(self._delayed_start(session, offset))
+ return
+ if difference < 0:
+ # Newest users leave first. This also removes still-queued arrivals before
+ # disrupting sessions that have already accumulated a world state.
+ departing = present[-(-difference):]
+ for ordinal, session in enumerate(reversed(departing)):
+ session.departure_scheduled = True
+ offset = self._spread_offset(ordinal + 1, -difference, phase.departure_window_seconds)
+ self._spawn_background(self._delayed_stop(session, offset))
+
+ def _schedule_input_activity(self, phase: Phase) -> None:
+ """Schedule long input pauses/resumes for sessions present in this phase.
+
+ Arrivals default to active input. The supplied real-user trace uses input
+ fractions only after its target population has already arrived; this keeps
+ an arrival's first action deterministic and observable.
+ """
+ present = [
+ session for session in self._sessions if not session.stop_requested and not session.departure_scheduled
+ ]
+ if not present:
+ return
+ active_count = round(len(present) * phase.active_input_fraction)
+ phase_offset = sum(ord(character) for character in phase.name)
+ ordered = sorted(
+ present,
+ key=lambda session: (((session.index + 1) * 17 + phase_offset) % len(present), session.index),
+ )
+ active_ids = {session.logical_id for session in ordered[:active_count]}
+ changes = [session for session in present if session.input_enabled != (session.logical_id in active_ids)]
+ for ordinal, session in enumerate(changes):
+ offset = self._spread_offset(ordinal + 1, len(changes), phase.input_transition_window_seconds)
+ self._spawn_background(
+ self._delayed_set_input_enabled(
+ session,
+ session.logical_id in active_ids,
+ offset,
+ reason=f"phase:{phase.name}",
+ )
+ )
+
+ @staticmethod
+ def _spread_offset(position: int, count: int, window_seconds: float) -> float:
+ if count <= 1 or window_seconds <= 0:
+ return 0.0
+ return window_seconds * (position - 1) / (count - 1)
+
+ def _spawn_background(self, coroutine: Any) -> None:
+ task = asyncio.create_task(coroutine)
+ self._background_tasks.add(task)
+ task.add_done_callback(self._background_tasks.discard)
+
+ async def _delayed_start(self, session: LiveKitWaveSession, offset_seconds: float) -> None:
+ await asyncio.sleep(offset_seconds)
+ if session.stop_requested or session.departure_scheduled:
+ return
+ try:
+ await asyncio.wait_for(session.start(), timeout=self.scenario.connect_timeout_seconds)
+ except asyncio.TimeoutError:
+ session.error = f"SessionStartTimeout after {self.scenario.connect_timeout_seconds:g}s"
+ self.record_event("session_start_timeout", session=session.logical_id, error=session.error)
+
+ async def _delayed_stop(self, session: LiveKitWaveSession, offset_seconds: float) -> None:
+ await asyncio.sleep(offset_seconds)
+ await session.stop()
+
+ async def _delayed_set_input_enabled(
+ self, session: LiveKitWaveSession, enabled: bool, offset_seconds: float, *, reason: str
+ ) -> None:
+ await asyncio.sleep(offset_seconds)
+ await session.set_input_enabled(enabled, reason=reason)
+
+ def _session_delivery_fps(
+ self, session: LiveKitWaveSession, *, now: float, interval: float, delta: int
+ ) -> tuple[float | None, float | None]:
+ """Return visible delivery FPS and the SLO observation, if either is eligible.
+
+ The SLO path deliberately emits a zero for a connected, continuously
+ controlled user that is still waiting beyond the first-generation grace
+ period. The first return is restricted to active controls because it is
+ the user-facing per-session FPS shown in phase summaries.
+ """
+ if interval <= 0 or session.stop_requested:
+ return None, None
+ controlled_after_grace = (
+ session.connected
+ and session.active_controls
+ and session.first_active_control_at is not None
+ and now - session.first_active_control_at >= self.scenario.first_generation_grace_seconds
+ )
+ if session.first_generated_frame_at is not None and session.active_controls:
+ fps = delta / interval
+ return fps, fps if controlled_after_grace else None
+ if controlled_after_grace:
+ return 0.0, 0.0
+ return None, None
+
+ def _session_requested_delivery_fps(
+ self, session: LiveKitWaveSession, *, now: float, interval: float, delta: int
+ ) -> float | None:
+ """Return FPS for every requested user once its grace interval expires."""
+ if interval <= 0 or session.stop_requested or session.create_started_at is None:
+ return None
+ service_started_at = session.first_active_control_at or session.create_started_at
+ if now - service_started_at < self.scenario.first_generation_grace_seconds:
+ return None
+ if session.first_generated_frame_at is None:
+ return 0.0
+ return delta / interval
+
+ async def _monitor(self) -> None:
+ while self._monitoring:
+ now = time.perf_counter()
+ elapsed = max(0.0, now - self.started_at)
+ previous_at = self._previous_sample_at
+ interval = now - previous_at if previous_at is not None else 0.0
+ session_fps: dict[str, float] = {}
+ slo_session_fps: dict[str, float] = {}
+ demand_slo_session_fps: dict[str, float] = {}
+ active_session_fps: list[float] = []
+ requested_session_fps: dict[str, float] = {}
+ requested_session_values: list[float] = []
+ connected_sessions = 0
+ active_control_sessions = 0
+ input_enabled_sessions = 0
+ requested_sessions = 0
+ generated_frames_delta = 0
+ for session in self._sessions:
+ snapshot = session.snapshot(now)
+ logical_id = session.logical_id
+ previous_frames = self._previous_generated_frames.get(logical_id, session.generated_frames_received)
+ delta = max(0, session.generated_frames_received - previous_frames)
+ self._previous_generated_frames[logical_id] = session.generated_frames_received
+ if session.connected:
+ connected_sessions += 1
+ if session.active_controls:
+ active_control_sessions += 1
+ if session.input_enabled and not session.stop_requested:
+ input_enabled_sessions += 1
+ if session.create_started_at is not None and not session.stop_requested:
+ requested_sessions += 1
+ if interval > 0 and not session.stop_requested:
+ # Aggregate delivery counts every generated video frame, including
+ # frames from users that are temporarily idle between controls.
+ if session.first_generated_frame_at is not None:
+ generated_frames_delta += delta
+ delivery_fps, demand_slo_fps = self._session_delivery_fps(
+ session, now=now, interval=interval, delta=delta
+ )
+ if delivery_fps is not None:
+ session_fps[logical_id] = round(delivery_fps, 6)
+ active_session_fps.append(delivery_fps)
+ if demand_slo_fps is not None:
+ demand_slo_session_fps[logical_id] = round(demand_slo_fps, 6)
+ requested_fps = self._session_requested_delivery_fps(session, now=now, interval=interval, delta=delta)
+ if requested_fps is not None:
+ requested_session_fps[logical_id] = round(requested_fps, 6)
+ requested_session_values.append(requested_fps)
+ slo_session_fps[logical_id] = round(requested_fps, 6)
+ snapshot["generated_frames_delta"] = delta
+ snapshot["delivery_fps"] = session_fps.get(logical_id)
+ snapshot["requested_delivery_fps"] = requested_session_fps.get(logical_id)
+ aggregate_fps = generated_frames_delta / interval if interval > 0 else 0.0
+ self._samples.append(
+ {
+ "offset_seconds": round(elapsed, 6),
+ "phase": self._phase_name,
+ "target_users": self._phase_target_users,
+ "target_active_input_fraction": self._phase_active_input_fraction,
+ "requested_users": requested_sessions,
+ "connected_users": connected_sessions,
+ "input_enabled_users": input_enabled_sessions,
+ "active_control_users": active_control_sessions,
+ "aggregate_delivery_fps": round(aggregate_fps, 6),
+ "per_requested_session_delivery_fps": round(statistics.fmean(requested_session_values), 6)
+ if requested_session_values
+ else None,
+ "per_active_session_delivery_fps": round(statistics.fmean(active_session_fps), 6)
+ if active_session_fps
+ else None,
+ "requested_session_delivery_fps": requested_session_fps,
+ "session_delivery_fps": session_fps,
+ "slo_session_delivery_fps": slo_session_fps,
+ "slo_observation_sessions": len(slo_session_fps),
+ "demand_slo_session_delivery_fps": demand_slo_session_fps,
+ "demand_slo_observation_sessions": len(demand_slo_session_fps),
+ "sessions": [session.snapshot(now) for session in self._sessions],
+ }
+ )
+ self._previous_sample_at = now
+ await asyncio.sleep(self.scenario.sample_interval_seconds)
+
+ def _summarize_phase(
+ self,
+ phase: Phase,
+ *,
+ phase_started: float,
+ phase_completed: float,
+ samples: Sequence[dict[str, Any]],
+ ) -> dict[str, Any]:
+ aggregate_fps = [float(sample["aggregate_delivery_fps"]) for sample in samples if sample["offset_seconds"] > 0]
+ per_session_fps = [float(fps) for sample in samples for fps in sample["session_delivery_fps"].values()]
+ active_session_fps = [
+ float(sample["per_active_session_delivery_fps"])
+ for sample in samples
+ if sample["per_active_session_delivery_fps"] is not None
+ ]
+ requested_session_fps = [
+ float(fps) for sample in samples for fps in sample["requested_session_delivery_fps"].values()
+ ]
+ requested_active_fps = [
+ float(sample["per_requested_session_delivery_fps"])
+ for sample in samples
+ if sample["per_requested_session_delivery_fps"] is not None
+ ]
+ phase_sessions = [
+ session
+ for session in self._sessions
+ if session.create_started_at is not None and phase_started <= session.create_started_at <= phase_completed
+ ]
+ phase_population = [
+ session
+ for session in self._sessions
+ if session.create_started_at is not None
+ and session.create_started_at <= phase_completed
+ and (session.stopped_at is None or session.stopped_at >= phase_started)
+ ]
+ action_to_first = [
+ session.first_generated_frame_at - session.first_active_control_at
+ for session in phase_sessions
+ if session.first_generated_frame_at is not None and session.first_active_control_at is not None
+ ]
+ # The all-user SLO denominator includes every requested session after
+ # its grace interval, including rejected, disconnected, or stalled users
+ # as zero-FPS observations.
+ slo_observations = [float(fps) for sample in samples for fps in sample["slo_session_delivery_fps"].values()]
+ demand_slo_observations = [
+ float(fps) for sample in samples for fps in sample["demand_slo_session_delivery_fps"].values()
+ ]
+ slo_threshold = self.scenario.session.fps - self.scenario.slo_fps_tolerance
+ slo_hits = sum(value >= slo_threshold for value in slo_observations)
+ demand_slo_hits = sum(value >= slo_threshold for value in demand_slo_observations)
+ assigned_sessions = sum(session.admission_status == "assigned" for session in phase_population)
+ queued_sessions = sum(session.admission_status == "queued" for session in phase_population)
+ unassigned_sessions = len(phase_population) - assigned_sessions - queued_sessions
+ admission_contract_satisfied = not self.scenario.admission.require_immediate_assignment or all(
+ session.admission_status == "assigned" for session in phase_population
+ )
+ summary = {
+ "duration_seconds": round(phase_completed - phase_started, 6),
+ "target_users": phase.target_users,
+ "max_requested_users": max((int(sample["requested_users"]) for sample in samples), default=0),
+ "max_connected_users": max((int(sample["connected_users"]) for sample in samples), default=0),
+ "max_input_enabled_users": max((int(sample["input_enabled_users"]) for sample in samples), default=0),
+ "max_active_control_users": max((int(sample["active_control_users"]) for sample in samples), default=0),
+ "admission": {
+ "require_immediate_assignment": self.scenario.admission.require_immediate_assignment,
+ "phase_population_sessions": len(phase_population),
+ "assigned_sessions": assigned_sessions,
+ "queued_sessions": queued_sessions,
+ "unassigned_or_failed_sessions": unassigned_sessions,
+ "immediate_assignment_satisfied": admission_contract_satisfied,
+ },
+ "aggregate_delivery_fps": _summary(aggregate_fps),
+ "per_requested_session_delivery_fps": _summary(requested_active_fps),
+ "per_requested_user_delivery_fps": _summary(requested_session_fps),
+ "per_active_session_delivery_fps": _summary(active_session_fps),
+ "per_session_delivery_fps": _summary(per_session_fps),
+ "action_to_first_generated_seconds": _summary(action_to_first),
+ "slo_target_fps": self.scenario.session.fps,
+ "slo_tolerance_fps": self.scenario.slo_fps_tolerance,
+ "slo_threshold_fps": round(slo_threshold, 6),
+ "slo_first_generation_grace_seconds": self.scenario.first_generation_grace_seconds,
+ "slo_observation_samples": len(slo_observations),
+ "slo_satisfied_samples": slo_hits,
+ "slo_sample_attainment": (round(slo_hits / len(slo_observations), 6) if slo_observations else 0.0),
+ "demand_slo_observation_samples": len(demand_slo_observations),
+ "demand_slo_satisfied_samples": demand_slo_hits,
+ "demand_slo_sample_attainment": (
+ round(demand_slo_hits / len(demand_slo_observations), 6) if demand_slo_observations else 0.0
+ ),
+ "started_sessions": len(phase_sessions),
+ "failed_sessions": sum(1 for session in phase_sessions if session.error is not None),
+ }
+ return {
+ "phase": phase.name,
+ "started_offset_seconds": round(phase_started - self.started_at, 6),
+ "completed_offset_seconds": round(phase_completed - self.started_at, 6),
+ "summary": summary,
+ }
+
+ async def _capture_server_metadata(self, label: str) -> None:
+ try:
+ response = await self._http.get(f"{self.scenario.server_url}/v1/service/metadata")
+ response.raise_for_status()
+ metadata = response.json()
+ if not isinstance(metadata, Mapping):
+ raise RuntimeError("metadata response is not an object")
+ metadata_dict = dict(metadata)
+ self._server_metadata.append(
+ {
+ "label": label,
+ "offset_seconds": round(max(0.0, time.perf_counter() - self.started_at), 6),
+ "metadata": metadata_dict,
+ }
+ )
+ self._validate_server_metadata(metadata_dict, label)
+ except Exception as exc: # noqa: BLE001 - metadata availability is an experiment fact
+ warning = f"Could not collect server metadata at {label}: {type(exc).__name__}: {exc}"
+ self._warnings.append(warning)
+ self.record_event("metadata_error", label=label, warning=warning)
+
+ def _validate_server_metadata(self, metadata: Mapping[str, Any], label: str) -> None:
+ expected_mode = self.scenario.expected_worker_mode
+ if expected_mode is not None and metadata.get("worker_mode") != expected_mode:
+ warning = (
+ f"Expected worker_mode={expected_mode!r}, got {metadata.get('worker_mode')!r} at {label}. "
+ "This is not the intended four-GPU process-NCCL baseline."
+ )
+ self._warnings.append(warning)
+ expected_workers = self.scenario.expected_num_workers
+ if expected_workers is not None and metadata.get("num_workers") != expected_workers:
+ warning = f"Expected num_workers={expected_workers}, got {metadata.get('num_workers')!r} at {label}."
+ self._warnings.append(warning)
+ admission = self.scenario.admission
+ if (
+ admission.expected_max_sessions_per_worker is not None
+ and metadata.get("configured_max_sessions_per_worker") != admission.expected_max_sessions_per_worker
+ ):
+ warning = (
+ "Expected configured_max_sessions_per_worker="
+ f"{admission.expected_max_sessions_per_worker}, got "
+ f"{metadata.get('configured_max_sessions_per_worker')!r} at {label}."
+ )
+ self._warnings.append(warning)
+ if admission.expected_queue_size is not None and metadata.get("queue_size") != admission.expected_queue_size:
+ warning = (
+ f"Expected queue_size={admission.expected_queue_size}, got {metadata.get('queue_size')!r} at {label}."
+ )
+ self._warnings.append(warning)
+
+ async def _stop_all_sessions(self) -> None:
+ for session in self._sessions:
+ session.stop_requested = True
+ for task in tuple(self._background_tasks):
+ if not task.done():
+ task.cancel()
+ if self._background_tasks:
+ await asyncio.gather(*tuple(self._background_tasks), return_exceptions=True)
+ await asyncio.gather(*(session.stop() for session in self._sessions), return_exceptions=True)
+
+
+def _print_summary(result: Mapping[str, Any]) -> None:
+ """Print a compact table suitable for an experiment log."""
+ print("\\nABot LiveKit black-box user-wave results")
+ print(
+ "phase target max-req max-input agg-FPS FPS/demand FPS/all-user "
+ "admitted demand-SLO all-user-SLO A2F-p95(s)"
+ )
+ for phase in result["phase_results"]:
+ summary = phase["summary"]
+ aggregate = summary["aggregate_delivery_fps"]
+ per_user = summary["per_requested_session_delivery_fps"]
+ admission = summary["admission"]
+ a2f = summary["action_to_first_generated_seconds"]
+ print(
+ f"{phase['phase'][:28]:28} "
+ f"{summary['target_users']:>6} "
+ f"{summary['max_requested_users']:>8} "
+ f"{summary['max_input_enabled_users']:>10} "
+ f"{aggregate['mean']:>8.3f} "
+ f"{summary['per_active_session_delivery_fps']['mean']:>10.3f} "
+ f"{per_user['mean']:>13.3f} "
+ f"{admission['assigned_sessions']:>3}/{admission['phase_population_sessions']:<3} "
+ f"{summary['demand_slo_sample_attainment'] * 100:>10.1f}% "
+ f"{summary['slo_sample_attainment'] * 100:>12.1f}% "
+ f"{a2f['p95']:>11.3f} "
+ )
+ warnings = result.get("warnings", [])
+ if warnings:
+ print("\nWarnings:")
+ for warning in warnings:
+ print(f"- {warning}")
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--scenario",
+ type=Path,
+ default=Path("tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json"),
+ help="JSON phase/user-wave scenario, relative to the repository root by default.",
+ )
+ parser.add_argument("--server-url", help="Override scenario.server_url, for example http://127.0.0.1:8088")
+ parser.add_argument("--output", type=Path, help="Write the complete JSON artifact to this path.")
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Validate and print the scenario without contacting the service.",
+ )
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = _parse_args()
+ scenario_path = args.scenario.expanduser()
+ if not scenario_path.is_absolute():
+ scenario_path = (_REPO_ROOT / scenario_path).resolve()
+ scenario = load_scenario(scenario_path, server_url_override=args.server_url)
+ if args.dry_run:
+ print(json.dumps(scenario.raw, indent=2, sort_keys=True))
+ print(f"\nValidated scenario: {scenario.name}")
+ return
+ if args.output is None:
+ raise SystemExit("--output is required unless --dry-run is used")
+ output = args.output.expanduser()
+ if not output.is_absolute():
+ output = (_REPO_ROOT / output).resolve()
+ output.parent.mkdir(parents=True, exist_ok=True)
+ result = asyncio.run(LiveKitWaveRunner(scenario).run())
+ output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
+ _print_summary(result)
+ print(f"\nWrote complete artifact: {output}")
+ if scenario.admission.require_immediate_assignment:
+ invalid_phases = [
+ str(phase["phase"])
+ for phase in result["phase_results"]
+ if not phase["summary"]["admission"]["immediate_assignment_satisfied"]
+ ]
+ if invalid_phases:
+ raise SystemExit(
+ "Immediate-assignment contract failed in "
+ + ", ".join(invalid_phases)
+ + f"; artifact was preserved at {output}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/capture_abot_serving_metrics.py b/tools/validation/capture_abot_serving_metrics.py
new file mode 100644
index 00000000..ba6217c5
--- /dev/null
+++ b/tools/validation/capture_abot_serving_metrics.py
@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""Persist TeleFuser ABot serving metrics when Prometheus is unavailable.
+
+This collector samples the public LiveKit service endpoints only. It is useful
+on experiment nodes without Docker/DCGM/Grafana and intentionally uses a direct
+urllib opener so inherited HTTP proxy environment variables cannot intercept
+loopback requests.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit
+from urllib.request import OpenerDirector, ProxyHandler, Request, build_opener
+
+_SCHEMA_VERSION = 1
+_USER_AGENT = "TeleFuserServingMetricsCapture/1.0"
+
+
+class CaptureError(RuntimeError):
+ """Raised when one HTTP endpoint cannot be captured."""
+
+
+@dataclass(frozen=True)
+class CaptureConfig:
+ """Validated command-line configuration for one capture run."""
+
+ server_url: str
+ duration_seconds: float
+ interval_seconds: float
+ timeout_seconds: float
+ output_dir: Path
+
+
+def _positive_seconds(value: str) -> float:
+ try:
+ parsed = float(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must be a number of seconds") from exc
+ if parsed <= 0:
+ raise argparse.ArgumentTypeError("must be greater than zero")
+ return parsed
+
+
+def _server_url(value: str) -> str:
+ normalized = value.strip().rstrip("/")
+ parsed = urlsplit(normalized)
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ raise argparse.ArgumentTypeError("must be an http(s) URL with a host")
+ if parsed.query or parsed.fragment:
+ raise argparse.ArgumentTypeError("must not include a query string or fragment")
+ return normalized
+
+
+def parse_args(argv: Sequence[str] | None = None) -> CaptureConfig:
+ """Parse the standalone collector interface."""
+
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--server-url", type=_server_url, default="http://127.0.0.1:8088")
+ parser.add_argument("--duration", type=_positive_seconds, required=True, help="Capture duration in seconds.")
+ parser.add_argument("--interval", type=_positive_seconds, default=1.0, help="Sample interval in seconds.")
+ parser.add_argument("--timeout", type=_positive_seconds, default=3.0, help="Per-request timeout in seconds.")
+ parser.add_argument("--output-dir", type=Path, required=True, help="New or empty directory for capture artifacts.")
+ args = parser.parse_args(argv)
+
+ output_dir = args.output_dir.expanduser().resolve()
+ return CaptureConfig(
+ server_url=args.server_url,
+ duration_seconds=args.duration,
+ interval_seconds=args.interval,
+ timeout_seconds=args.timeout,
+ output_dir=output_dir,
+ )
+
+
+def _utc_timestamp() -> str:
+ return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def build_direct_opener() -> OpenerDirector:
+ """Build an opener that ignores HTTP(S)_PROXY and all other proxy variables."""
+
+ return build_opener(ProxyHandler({}))
+
+
+def _request_text(opener: OpenerDirector, url: str, timeout_seconds: float) -> str:
+ request = Request(url, headers={"Accept": "application/json, text/plain; q=0.9", "User-Agent": _USER_AGENT})
+ try:
+ with opener.open(request, timeout=timeout_seconds) as response:
+ charset = response.headers.get_content_charset() or "utf-8"
+ return response.read().decode(charset)
+ except (HTTPError, URLError, OSError, TimeoutError, UnicodeDecodeError) as exc:
+ raise CaptureError(f"{type(exc).__name__}: {exc}") from exc
+
+
+def _prepare_output_dir(output_dir: Path) -> tuple[Path, Path, Path]:
+ if output_dir.exists() and not output_dir.is_dir():
+ raise ValueError(f"--output-dir is not a directory: {output_dir}")
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ manifest_path = output_dir / "manifest.json"
+ jsonl_path = output_dir / "serving-metrics.jsonl"
+ prometheus_dir = output_dir / "prometheus"
+ if manifest_path.exists() or jsonl_path.exists():
+ raise ValueError(f"Refusing to overwrite an existing capture artifact in {output_dir}")
+ if prometheus_dir.exists() and not prometheus_dir.is_dir():
+ raise ValueError(f"Prometheus artifact path is not a directory: {prometheus_dir}")
+ if prometheus_dir.exists() and any(prometheus_dir.iterdir()):
+ raise ValueError(f"Refusing to reuse non-empty Prometheus artifact directory: {prometheus_dir}")
+ prometheus_dir.mkdir(exist_ok=True)
+ return manifest_path, jsonl_path, prometheus_dir
+
+
+def _write_json(path: Path, value: dict[str, Any]) -> None:
+ path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _record_error(record: dict[str, Any], endpoint: str, error: Exception) -> None:
+ record["errors"].append({"endpoint": endpoint, "error": f"{type(error).__name__}: {error}"})
+
+
+def _capture_once(
+ *,
+ config: CaptureConfig,
+ opener: OpenerDirector,
+ sequence: int,
+ started_monotonic: float,
+ prometheus_dir: Path,
+) -> dict[str, Any]:
+ prometheus_url = f"{config.server_url}/metrics"
+ json_url = f"{config.server_url}/v1/service/metrics/json"
+ record: dict[str, Any] = {
+ "schema_version": _SCHEMA_VERSION,
+ "sequence": sequence,
+ "observed_at_utc": _utc_timestamp(),
+ "offset_seconds": round(time.monotonic() - started_monotonic, 6),
+ "prometheus": {"url": prometheus_url},
+ "serving": {"url": json_url},
+ "errors": [],
+ }
+
+ try:
+ prometheus_text = _request_text(opener, prometheus_url, config.timeout_seconds)
+ prometheus_name = f"{sequence:06d}.prom"
+ (prometheus_dir / prometheus_name).write_text(prometheus_text, encoding="utf-8")
+ record["prometheus"].update(
+ {
+ "path": str(Path("prometheus") / prometheus_name),
+ "bytes": len(prometheus_text.encode("utf-8")),
+ }
+ )
+ except CaptureError as exc:
+ _record_error(record, "metrics", exc)
+
+ try:
+ json_text = _request_text(opener, json_url, config.timeout_seconds)
+ response = json.loads(json_text)
+ if not isinstance(response, dict) or not isinstance(response.get("serving"), dict):
+ raise CaptureError("response does not contain an object-valued 'serving' field")
+ record["serving"]["snapshot"] = response["serving"]
+ except (CaptureError, json.JSONDecodeError) as exc:
+ _record_error(record, "metrics_json", exc)
+
+ return record
+
+
+def capture(config: CaptureConfig) -> dict[str, Any]:
+ """Capture service metrics and always write a terminal manifest after start."""
+
+ manifest_path, jsonl_path, prometheus_dir = _prepare_output_dir(config.output_dir)
+ started_monotonic = time.monotonic()
+ started_at_utc = _utc_timestamp()
+ deadline = started_monotonic + config.duration_seconds
+ statistics = {
+ "attempted": 0,
+ "prometheus_saved": 0,
+ "serving_json_saved": 0,
+ "complete": 0,
+ "errors": 0,
+ }
+ status = "completed"
+ failure: Exception | None = None
+ opener = build_direct_opener()
+
+ try:
+ with jsonl_path.open("x", encoding="utf-8") as jsonl_file:
+ while True:
+ if statistics["attempted"] and time.monotonic() >= deadline:
+ break
+ sequence = statistics["attempted"] + 1
+ record = _capture_once(
+ config=config,
+ opener=opener,
+ sequence=sequence,
+ started_monotonic=started_monotonic,
+ prometheus_dir=prometheus_dir,
+ )
+ jsonl_file.write(json.dumps(record, sort_keys=True) + "\n")
+ jsonl_file.flush()
+
+ statistics["attempted"] += 1
+ prometheus_saved = "path" in record["prometheus"]
+ serving_json_saved = "snapshot" in record["serving"]
+ statistics["prometheus_saved"] += int(prometheus_saved)
+ statistics["serving_json_saved"] += int(serving_json_saved)
+ statistics["complete"] += int(prometheus_saved and serving_json_saved)
+ statistics["errors"] += len(record["errors"])
+
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+ time.sleep(min(config.interval_seconds, remaining))
+ except KeyboardInterrupt:
+ status = "interrupted"
+ except Exception as exc:
+ status = "failed"
+ failure = exc
+ finally:
+ manifest = {
+ "schema_version": _SCHEMA_VERSION,
+ "status": status,
+ "started_at_utc": started_at_utc,
+ "completed_at_utc": _utc_timestamp(),
+ "elapsed_seconds": round(time.monotonic() - started_monotonic, 6),
+ "configuration": {
+ "server_url": config.server_url,
+ "duration_seconds": config.duration_seconds,
+ "interval_seconds": config.interval_seconds,
+ "timeout_seconds": config.timeout_seconds,
+ "proxy_mode": "direct_no_proxy",
+ },
+ "samples": statistics,
+ "artifacts": {
+ "serving_metrics_jsonl": jsonl_path.name,
+ "prometheus_snapshots_directory": prometheus_dir.name,
+ },
+ }
+ _write_json(manifest_path, manifest)
+
+ if failure is not None:
+ raise failure
+ return manifest
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """Run the collector and report the durable artifact path."""
+
+ try:
+ config = parse_args(argv)
+ manifest = capture(config)
+ except (CaptureError, OSError, ValueError) as exc:
+ print(f"Metrics capture failed: {exc}", file=sys.stderr)
+ return 2
+
+ samples = manifest["samples"]
+ print(
+ f"Metrics capture {manifest['status']}: {samples['complete']}/{samples['attempted']} complete samples; "
+ f"manifest: {config.output_dir / 'manifest.json'}"
+ )
+ if manifest["status"] == "interrupted":
+ return 130
+ return 0 if samples["complete"] else 2
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/validation/capture_gpu_nvml_metrics.py b/tools/validation/capture_gpu_nvml_metrics.py
new file mode 100755
index 00000000..62b9b57a
--- /dev/null
+++ b/tools/validation/capture_gpu_nvml_metrics.py
@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""Capture GPU metrics through NVML without Docker, DCGM, or ``nvidia-smi``.
+
+This is a deliberately small fallback for experiment nodes where the full
+Prometheus/DCGM stack cannot run. Pair it with
+``capture_abot_serving_metrics.py``: both artifacts use UTC and monotonic
+offset timestamps, so serving, scheduling, and physical-GPU time series can be
+correlated without parsing logs by hand.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+import json
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+_SCHEMA_VERSION = 1
+_NVML_SUCCESS = 0
+_NVML_TEMPERATURE_GPU = 0
+
+
+class NvmlError(RuntimeError):
+ """Raised for an NVML initialization or device-query failure."""
+
+
+class _NvmlMemory(ctypes.Structure):
+ _fields_ = [
+ ("total", ctypes.c_ulonglong),
+ ("free", ctypes.c_ulonglong),
+ ("used", ctypes.c_ulonglong),
+ ]
+
+
+class _NvmlUtilization(ctypes.Structure):
+ _fields_ = [("gpu", ctypes.c_uint), ("memory", ctypes.c_uint)]
+
+
+@dataclass(frozen=True)
+class CaptureConfig:
+ gpu_indices: tuple[int, ...]
+ duration_seconds: float
+ interval_seconds: float
+ output_dir: Path
+
+
+def _positive_seconds(value: str) -> float:
+ try:
+ parsed = float(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must be a number of seconds") from exc
+ if parsed <= 0:
+ raise argparse.ArgumentTypeError("must be greater than zero")
+ return parsed
+
+
+def _gpu_indices(value: str) -> tuple[int, ...]:
+ parts = [part.strip() for part in value.split(",")]
+ if not parts or any(not part for part in parts):
+ raise argparse.ArgumentTypeError("must be a comma-separated non-empty list of GPU indices")
+ try:
+ indices = tuple(int(part) for part in parts)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must contain only integer GPU indices") from exc
+ if any(index < 0 for index in indices) or len(set(indices)) != len(indices):
+ raise argparse.ArgumentTypeError("GPU indices must be unique non-negative integers")
+ return indices
+
+
+def parse_args(argv: list[str] | None = None) -> CaptureConfig:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--gpu-indices", type=_gpu_indices, default=(0, 1, 2, 3))
+ parser.add_argument("--duration", required=True, type=_positive_seconds)
+ parser.add_argument("--interval", type=_positive_seconds, default=1.0)
+ parser.add_argument("--output-dir", required=True, type=Path)
+ args = parser.parse_args(argv)
+ return CaptureConfig(
+ gpu_indices=args.gpu_indices,
+ duration_seconds=args.duration,
+ interval_seconds=args.interval,
+ output_dir=args.output_dir.expanduser().resolve(),
+ )
+
+
+def _utc_timestamp() -> str:
+ return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+class NvmlSampler:
+ """Minimal ctypes wrapper for the NVML fields used in serving experiments."""
+
+ def __init__(self, gpu_indices: tuple[int, ...]) -> None:
+ try:
+ library = ctypes.CDLL("libnvidia-ml.so.1")
+ except OSError as exc:
+ raise NvmlError(f"could not load libnvidia-ml.so.1: {exc}") from exc
+ self._library = library
+ self._init = library.nvmlInit_v2
+ self._init.restype = ctypes.c_int
+ self._shutdown = library.nvmlShutdown
+ self._shutdown.restype = ctypes.c_int
+ self._get_count = library.nvmlDeviceGetCount_v2
+ self._get_count.argtypes = [ctypes.POINTER(ctypes.c_uint)]
+ self._get_count.restype = ctypes.c_int
+ self._get_handle = library.nvmlDeviceGetHandleByIndex_v2
+ self._get_handle.argtypes = [ctypes.c_uint, ctypes.POINTER(ctypes.c_void_p)]
+ self._get_handle.restype = ctypes.c_int
+ self._get_name = library.nvmlDeviceGetName
+ self._get_name.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_char), ctypes.c_uint]
+ self._get_name.restype = ctypes.c_int
+ self._get_utilization = library.nvmlDeviceGetUtilizationRates
+ self._get_utilization.argtypes = [ctypes.c_void_p, ctypes.POINTER(_NvmlUtilization)]
+ self._get_utilization.restype = ctypes.c_int
+ self._get_memory = library.nvmlDeviceGetMemoryInfo
+ self._get_memory.argtypes = [ctypes.c_void_p, ctypes.POINTER(_NvmlMemory)]
+ self._get_memory.restype = ctypes.c_int
+ self._get_power = library.nvmlDeviceGetPowerUsage
+ self._get_power.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint)]
+ self._get_power.restype = ctypes.c_int
+ self._get_temperature = library.nvmlDeviceGetTemperature
+ self._get_temperature.argtypes = [ctypes.c_void_p, ctypes.c_uint, ctypes.POINTER(ctypes.c_uint)]
+ self._get_temperature.restype = ctypes.c_int
+
+ self._check(self._init(), "nvmlInit_v2")
+ try:
+ count = ctypes.c_uint()
+ self._check(self._get_count(ctypes.byref(count)), "nvmlDeviceGetCount_v2")
+ if any(index >= count.value for index in gpu_indices):
+ raise NvmlError(f"requested GPU indices {gpu_indices} exceed detected GPU count {count.value}")
+ self._handles = {index: self._handle(index) for index in gpu_indices}
+ except Exception:
+ self.close()
+ raise
+
+ @staticmethod
+ def _check(status: int, operation: str) -> None:
+ if status != _NVML_SUCCESS:
+ raise NvmlError(f"{operation} returned NVML status {status}")
+
+ def _handle(self, index: int) -> ctypes.c_void_p:
+ handle = ctypes.c_void_p()
+ self._check(self._get_handle(index, ctypes.byref(handle)), f"nvmlDeviceGetHandleByIndex_v2({index})")
+ return handle
+
+ def _name(self, handle: ctypes.c_void_p) -> str:
+ buffer = ctypes.create_string_buffer(96)
+ self._check(self._get_name(handle, buffer, len(buffer)), "nvmlDeviceGetName")
+ return buffer.value.decode("utf-8", errors="replace")
+
+ def sample(self) -> list[dict[str, Any]]:
+ rows: list[dict[str, Any]] = []
+ for index, handle in self._handles.items():
+ utilization = _NvmlUtilization()
+ memory = _NvmlMemory()
+ power_milliwatts = ctypes.c_uint()
+ temperature = ctypes.c_uint()
+ self._check(self._get_utilization(handle, ctypes.byref(utilization)), "nvmlDeviceGetUtilizationRates")
+ self._check(self._get_memory(handle, ctypes.byref(memory)), "nvmlDeviceGetMemoryInfo")
+ self._check(self._get_power(handle, ctypes.byref(power_milliwatts)), "nvmlDeviceGetPowerUsage")
+ self._check(
+ self._get_temperature(handle, _NVML_TEMPERATURE_GPU, ctypes.byref(temperature)),
+ "nvmlDeviceGetTemperature",
+ )
+ rows.append(
+ {
+ "gpu_index": index,
+ "gpu_name": self._name(handle),
+ "gpu_utilization_percent": int(utilization.gpu),
+ "memory_utilization_percent": int(utilization.memory),
+ "memory_total_bytes": int(memory.total),
+ "memory_used_bytes": int(memory.used),
+ "memory_free_bytes": int(memory.free),
+ "power_watts": round(float(power_milliwatts.value) / 1000.0, 3),
+ "temperature_celsius": int(temperature.value),
+ }
+ )
+ return rows
+
+ def close(self) -> None:
+ if getattr(self, "_library", None) is not None:
+ self._shutdown()
+ self._library = None
+
+
+def capture(config: CaptureConfig) -> dict[str, Any]:
+ config.output_dir.mkdir(parents=True, exist_ok=True)
+ jsonl_path = config.output_dir / "gpu-metrics.jsonl"
+ manifest_path = config.output_dir / "manifest.json"
+ if jsonl_path.exists() or manifest_path.exists():
+ raise ValueError(f"refusing to overwrite an existing capture artifact in {config.output_dir}")
+
+ sampler = NvmlSampler(config.gpu_indices)
+ started = time.monotonic()
+ started_at_utc = _utc_timestamp()
+ attempted = 0
+ complete = 0
+ status = "completed"
+ try:
+ with jsonl_path.open("x", encoding="utf-8") as output:
+ while True:
+ if attempted and time.monotonic() >= started + config.duration_seconds:
+ break
+ attempted += 1
+ record: dict[str, Any] = {
+ "schema_version": _SCHEMA_VERSION,
+ "source": "nvml",
+ "sequence": attempted,
+ "observed_at_utc": _utc_timestamp(),
+ "offset_seconds": round(time.monotonic() - started, 6),
+ "gpu_indices": list(config.gpu_indices),
+ "gpus": [],
+ "error": None,
+ }
+ try:
+ record["gpus"] = sampler.sample()
+ complete += 1
+ except NvmlError as exc:
+ record["error"] = str(exc)
+ output.write(json.dumps(record, sort_keys=True) + "\n")
+ output.flush()
+ remaining = started + config.duration_seconds - time.monotonic()
+ if remaining <= 0:
+ break
+ time.sleep(min(config.interval_seconds, remaining))
+ except KeyboardInterrupt:
+ status = "interrupted"
+ finally:
+ sampler.close()
+
+ manifest = {
+ "schema_version": _SCHEMA_VERSION,
+ "status": status,
+ "source": "nvml",
+ "started_at_utc": started_at_utc,
+ "completed_at_utc": _utc_timestamp(),
+ "elapsed_seconds": round(time.monotonic() - started, 6),
+ "configuration": {
+ "gpu_indices": list(config.gpu_indices),
+ "duration_seconds": config.duration_seconds,
+ "interval_seconds": config.interval_seconds,
+ },
+ "samples": {"attempted": attempted, "complete": complete, "errors": attempted - complete},
+ "artifact": jsonl_path.name,
+ }
+ manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ return manifest
+
+
+def main(argv: list[str] | None = None) -> int:
+ try:
+ config = parse_args(argv)
+ manifest = capture(config)
+ except (NvmlError, OSError, ValueError) as exc:
+ print(f"GPU metrics capture failed: {exc}", file=sys.stderr)
+ return 2
+ print(
+ f"GPU metrics capture {manifest['status']}: {manifest['samples']['complete']}/"
+ f"{manifest['samples']['attempted']} complete samples; manifest: {config.output_dir / 'manifest.json'}"
+ )
+ return 0 if manifest["samples"]["complete"] else 2
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/validation/validate_abot_nccl_migration.py b/tools/validation/validate_abot_nccl_migration.py
index df9e576f..a8908d53 100644
--- a/tools/validation/validate_abot_nccl_migration.py
+++ b/tools/validation/validate_abot_nccl_migration.py
@@ -63,35 +63,45 @@ def _drain_outputs(service: ABotWorldLiveKitService, session_id: str, stop: thre
def _rank_main(rank: int, args: argparse.Namespace, port: int) -> None:
torch.cuda.set_device(rank)
- dist.init_process_group(
- backend="nccl",
- init_method=f"tcp://127.0.0.1:{port}",
- rank=rank,
- world_size=2,
- )
service: ABotWorldLiveKitService | None = None
try:
- loader = _loader_module()
- pipeline = loader.get_pipeline(
- model_root=args.model_root,
- pipeline_class=ABotWorldInteractivePipeline,
- device_id=rank,
+ dist.init_process_group(
+ backend="nccl",
+ init_method=f"tcp://127.0.0.1:{port}",
+ rank=rank,
+ world_size=2,
)
- service = ABotWorldLiveKitService(
- pipeline,
- max_batch_size=1,
- default_session_config={
- "image_path": str(args.image),
- "prompt": args.prompt,
- "fps": 12,
- "control_latent_frames": args.control_latent_frames,
- "seed": args.seed,
- },
- )
- # A process-nccl child preloads its replica at worker startup. The
- # target must do the same before it adopts CUDA session tensors.
- if rank == 1:
- service.start()
+ print(f"rank={rank} phase=nccl_ready", flush=True)
+
+ # Process workers are started serially in the real serving pool. Do
+ # the same here: concurrently materializing two 25-GB checkpoints on
+ # this shared filesystem can make a migration smoke test look like an
+ # NCCL deadlock before either rank reaches the communicator.
+ for loading_rank in range(2):
+ if rank == loading_rank:
+ print(f"rank={rank} phase=load_replica", flush=True)
+ loader = _loader_module()
+ pipeline = loader.get_pipeline(
+ model_root=args.model_root,
+ pipeline_class=ABotWorldInteractivePipeline,
+ device_id=rank,
+ )
+ service = ABotWorldLiveKitService(
+ pipeline,
+ max_batch_size=1,
+ default_session_config={
+ "image_path": str(args.image),
+ "prompt": args.prompt,
+ "fps": 12,
+ "control_latent_frames": args.control_latent_frames,
+ "seed": args.seed,
+ },
+ )
+ service.start()
+ torch.cuda.synchronize(rank)
+ print(f"rank={rank} phase=replica_ready", flush=True)
+ dist.barrier()
+ assert service is not None
session_id = "nccl-validation-session"
metadata: dict[str, Any] | None = None
leaves: dict[tuple[Any, ...], torch.Tensor] | None = None
@@ -151,7 +161,8 @@ def _rank_main(rank: int, args: argparse.Namespace, port: int) -> None:
session = service._session(session_id)
assert session is not None
print(
- f"target_chunk={target_chunk.get('index')} next_latent_frame={session.pipeline_session.next_latent_frame} "
+ f"target_chunk={target_chunk.get('index')} "
+ f"next_latent_frame={session.pipeline_session.next_latent_frame} "
f"emitted_frames={session.pipeline_session.emitted_frames}",
flush=True,
)
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json
new file mode 100644
index 00000000..b1ff7d91
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json
@@ -0,0 +1,79 @@
+{
+ "name": "abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave",
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "admission": {
+ "require_immediate_assignment": true,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0
+ },
+ "seed": 20260814,
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.1,
+ "idle_probability": 0.0,
+ "idle_min_seconds": 0.2,
+ "idle_max_seconds": 1.0,
+ "action_states": [
+ ["KeyW"],
+ ["KeyW", "KeyA"],
+ ["KeyW", "KeyD"],
+ ["KeyI"]
+ ]
+ }
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "phases": [
+ {
+ "name": "warmup_4_users",
+ "duration_seconds": 45.0,
+ "target_users": 4,
+ "arrival_window_seconds": 10.0
+ },
+ {
+ "name": "ramp_to_8_users",
+ "duration_seconds": 55.0,
+ "target_users": 8,
+ "arrival_window_seconds": 15.0
+ },
+ {
+ "name": "ramp_to_12_users",
+ "duration_seconds": 55.0,
+ "target_users": 12,
+ "arrival_window_seconds": 15.0
+ },
+ {
+ "name": "peak_16_all_active_users",
+ "duration_seconds": 80.0,
+ "target_users": 16,
+ "arrival_window_seconds": 15.0
+ },
+ {
+ "name": "recovery_to_8_users",
+ "duration_seconds": 50.0,
+ "target_users": 8,
+ "departure_window_seconds": 15.0
+ },
+ {
+ "name": "recovery_to_4_users",
+ "duration_seconds": 45.0,
+ "target_users": 4,
+ "departure_window_seconds": 15.0
+ }
+ ]
+}
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json
new file mode 100644
index 00000000..1abb0db1
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json
@@ -0,0 +1,106 @@
+{
+ "name": "abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16",
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "admission": {
+ "require_immediate_assignment": true,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0
+ },
+ "seed": 20260815,
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.1,
+ "idle_probability": 0.03,
+ "idle_min_seconds": 1.5,
+ "idle_max_seconds": 6.0,
+ "action_states": [
+ ["KeyW"],
+ ["KeyW", "KeyA"],
+ ["KeyW", "KeyD"],
+ ["KeyI"]
+ ]
+ }
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "phases": [
+ {
+ "name": "warmup_4_continuous",
+ "duration_seconds": 35.0,
+ "target_users": 4,
+ "arrival_window_seconds": 10.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "ramp_8_continuous",
+ "duration_seconds": 45.0,
+ "target_users": 8,
+ "arrival_window_seconds": 15.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "input_lull_8_half_paused",
+ "duration_seconds": 35.0,
+ "target_users": 8,
+ "active_input_fraction": 0.5,
+ "input_transition_window_seconds": 8.0
+ },
+ {
+ "name": "ramp_16_resume_and_arrive",
+ "duration_seconds": 55.0,
+ "target_users": 16,
+ "arrival_window_seconds": 20.0,
+ "active_input_fraction": 1.0,
+ "input_transition_window_seconds": 8.0
+ },
+ {
+ "name": "peak_16_continuous",
+ "duration_seconds": 45.0,
+ "target_users": 16,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "peak_16_half_paused",
+ "duration_seconds": 45.0,
+ "target_users": 16,
+ "active_input_fraction": 0.5,
+ "input_transition_window_seconds": 15.0
+ },
+ {
+ "name": "peak_16_reengage",
+ "duration_seconds": 50.0,
+ "target_users": 16,
+ "active_input_fraction": 1.0,
+ "input_transition_window_seconds": 10.0
+ },
+ {
+ "name": "recovery_8_departures",
+ "duration_seconds": 40.0,
+ "target_users": 8,
+ "departure_window_seconds": 15.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "recovery_4_departures",
+ "duration_seconds": 35.0,
+ "target_users": 4,
+ "departure_window_seconds": 12.0,
+ "active_input_fraction": 1.0
+ }
+ ]
+}
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
new file mode 100644
index 00000000..17bad1ed
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
@@ -0,0 +1,62 @@
+{
+ "name": "abot_livekit_4gpu_lf3_12fps_user_wave",
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "seed": 20260813,
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.1,
+ "idle_probability": 0.0,
+ "idle_min_seconds": 0.2,
+ "idle_max_seconds": 1.0,
+ "action_states": [
+ ["KeyW"],
+ ["KeyW", "KeyA"],
+ ["KeyW", "KeyD"],
+ ["KeyI"]
+ ]
+ }
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "phases": [
+ {
+ "name": "warmup_4_users",
+ "duration_seconds": 45.0,
+ "target_users": 4,
+ "arrival_window_seconds": 15.0
+ },
+ {
+ "name": "ramp_to_slo_capacity_8_users",
+ "duration_seconds": 75.0,
+ "target_users": 8,
+ "arrival_window_seconds": 25.0
+ },
+ {
+ "name": "burst_above_admission_capacity_12_users",
+ "duration_seconds": 75.0,
+ "target_users": 12,
+ "arrival_window_seconds": 15.0
+ },
+ {
+ "name": "recovery_to_4_users",
+ "duration_seconds": 60.0,
+ "target_users": 4,
+ "departure_window_seconds": 15.0
+ }
+ ]
+}
From 51039fcd4a4578d0e73ae705c3bde9d5c296e48c Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Fri, 14 Aug 2026 08:11:26 +0000
Subject: [PATCH 3/8] docs(abot): replace stale pre-LightVAE results
---
.../summary.md | 39 ----------------
.../summary.md | 21 ---------
.../summary.md | 18 --------
.../abot_batched_lf3_4gpu_20260813/summary.md | 31 -------------
.../summary.md | 45 -------------------
.../summary.md | 31 -------------
.../summary.md | 20 ---------
.../summary.md | 23 +++++++++-
8 files changed, 22 insertions(+), 206 deletions(-)
delete mode 100644 results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md
delete mode 100644 results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md
delete mode 100644 results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md
delete mode 100644 results/experiments/abot_batched_lf3_4gpu_20260813/summary.md
delete mode 100644 results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md
delete mode 100644 results/experiments/abot_h100_microbatch_lf3_20260813/summary.md
delete mode 100644 results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md
diff --git a/results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md b/results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md
deleted file mode 100644
index a7c5a750..00000000
--- a/results/experiments/abot_4gpu_lf3_user_sweep_20260813/summary.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# ABot-World four-GPU concurrent-user baseline (LF=3)
-
-Date: 2026-08-13. GPUs 4--7 are four independent single-GPU service replicas;
-this is a per-replica capacity baseline, not a global multi-GPU TurboServe result.
-
-## Fixed workload
-
-- Model: `ABot-World-0-5B-LF`; input: `84b90ad568b693d2.png` at the default 832x480.
-- `control_latent_frames=3` (the original ABot-World streaming setting).
-- Four replicas, continuous active controls every 0.3 s, 30 s per run, no idle periods.
-- Consumer pulls immediately (`consumer_playback_fps=0`), lossless delivery, batch window 2 ms,
- and `max_batch_size=4`. Reported FPS is consumer-visible end-to-end FPS in the local
- service harness; it excludes browser/WebRTC encode and network transport.
-
-| Users / GPU | Total users | Per-user FPS | Aggregate / GPU | Approx. cluster FPS | Mean batch | Mean compute / batch (s) | Mean queue wait (ms) | Mean first frame (s) | Outcome |
-|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|
-| 1 | 4 | 14.361 | 14.361 | 57.444 | 1.000 | 0.811 | 0.000 | 0.664 | admitted |
-| 2 | 8 | 7.189 | 14.378 | 57.510 | 1.649 | 1.304 | 0.065 | 1.067 | admitted |
-| 3 | 12 | 4.715 | 14.144 | 56.576 | 2.243 | 1.722 | 1.957 | 1.425 | admitted; each H100 reached about 81 GB during the run |
-| 4 | 16 | -- | -- | -- | -- | -- | -- | -- | rejected: `capacity=3` |
-
-`users_per_gpu_4` does not produce JSON because the fourth retained session is rejected by
-the service's admission controller (`ABot retained-session capacity is exhausted (capacity=3)`).
-The consumer-close timeout subsequently printed by the harness is a cleanup artifact, not a
-model-inference latency measurement.
-
-## Interpretation
-
-For LF=3, the single-user result is 14.24--14.51 FPS across the four cards (mean 14.36),
-consistent with the previously matched direct single-GPU result (about 15 FPS). Increasing
-the number of active sessions does form batches, but raises batch compute time almost
-proportionally, leaving per-GPU throughput flat at about 14.1--14.4 FPS. Thus the current
-baseline's limiting factor in this workload is model/state memory and batched compute scaling,
-not scheduler queueing. This is a useful pre-experiment gap for a workload-aware world-model
-scheduler: it should avoid admitting a fourth retained LF=3 state locally and should use global
-placement/migration or state offload rather than merely increasing the local batch.
-
-Raw files: `users_per_gpu_{1,2,3}/gpu{4,5,6,7}.json`; logs, including the four admission
-rejections, are co-located in `users_per_gpu_4/`.
diff --git a/results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md b/results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md
deleted file mode 100644
index e9ddc5c4..00000000
--- a/results/experiments/abot_batch_scaling_20260812_steady_lf1/summary.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# ABot steady-state batch scaling: one latent control frame
-
-Hardware: one NVIDIA H100 80 GB (GPU 0). The ABot-World-0-5B-LF service was
-preloaded, then each point used continuously active retained sessions, one warmup
-chunk per session, and two measured chunks per session. Values below are from the
-LiveKit service scheduler, not a synthetic model loop.
-
-| Sessions | Batch cap | Observed batch | Aggregate FPS | Per-session FPS | p95 chunk latency (s) | p95 queue wait (s) | Peak allocated GiB | Result |
-|---:|---:|---:|---:|---:|---:|---:|---:|---|
-| 1 | 1 | 1.0 | 11.00 | 11.00 | 0.376 | 0.000 | 39.05 | OK |
-| 2 | 1 | 1.0 | 11.73 | 5.86 | 0.692 | 0.347 | 44.33 | OK |
-| 2 | 2 | 2.0 | 14.31 | 7.16 | 0.572 | 0.000 | 54.89 | OK |
-| 4 | 1 | 1.0 | 11.49 | 2.87 | 1.422 | 1.070 | 54.91 | OK |
-| 4 | 2 | 2.0 | 14.31 | 3.58 | 1.137 | 0.570 | 65.47 | OK |
-| 4 | 4 | -- | -- | -- | -- | -- | -- | OOM in VAE temporal decode (requested 3.81 GiB) |
-
-The valid batch-2 points increase aggregate throughput by 22.0% (two sessions)
-and 24.6% (four sessions) over batch cap 1. However, the four-session latency
-remains above one second and batch 4 is infeasible despite 80 GB device memory.
-
-Raw data: [results.csv](results.csv) and [results.json](results.json).
diff --git a/results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md b/results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md
deleted file mode 100644
index 55c56339..00000000
--- a/results/experiments/abot_batch_scaling_20260812_steady_lf3/summary.md
+++ /dev/null
@@ -1,18 +0,0 @@
-# ABot steady-state batch scaling: three latent control frames
-
-Hardware and warmup are the same as the LF=1 experiment. Each scheduled chunk
-generates three latent frames and was measured through the LiveKit scheduler.
-
-| Sessions | Batch cap | Observed batch | Aggregate FPS | Per-session FPS | p95 chunk latency (s) | p95 queue wait (s) | Peak allocated GiB | Result |
-|---:|---:|---:|---:|---:|---:|---:|---:|---|
-| 1 | 1 | 1.0 | 15.66 | 15.66 | 0.776 | 0.000 | 39.14 | OK |
-| 2 | 1 | 1.0 | 15.54 | 7.77 | 1.546 | 0.774 | 44.42 | OK |
-| 2 | 2 | 2.0 | 16.14 | 8.07 | 1.508 | 0.001 | 55.07 | OK |
-
-Batching two long-control sessions improves aggregate FPS only 3.8% while the
-batch compute time rises from about 0.77 s to 1.51 s. VAE decode consumes about
-20% of each batch, and DiT about 28-32%; the remaining time is cache collation,
-output conversion, and scheduler-side work. This differs sharply from the LF=1
-case and motivates a workload-aware policy rather than a fixed batch cap.
-
-Raw data: [results.csv](results.csv) and [results.json](results.json).
diff --git a/results/experiments/abot_batched_lf3_4gpu_20260813/summary.md b/results/experiments/abot_batched_lf3_4gpu_20260813/summary.md
deleted file mode 100644
index 7bbd84f2..00000000
--- a/results/experiments/abot_batched_lf3_4gpu_20260813/summary.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# ABot-World explicit cross-session batching experiment (LF=3)
-
-Date: 2026-08-13. This experiment uses four independent single-GPU replicas
-(GPUs 4--7) and **explicitly selects** `scheduler_mode=batched`. It is not
-the TurboServe baseline: TurboServe's per-worker open-source loop is
-single-session round-robin. The purpose here is to evaluate the experimental
-TeleFuser cross-session model-batching path against that baseline.
-
-## Fixed workload
-
-- ABot-World-0-5B-LF, default 832x480 image, `control_latent_frames=3`.
-- Four replicas; simultaneous active clients, control heartbeat every 0.3 s,
- 30 s run, no idle intervals; immediate consumer and lossless delivery.
-- `max_batch_size=4`, batching window 2 ms. FPS is local consumer-visible
- end-to-end FPS, excluding browser/WebRTC encode and network transport.
-
-| Users / GPU | Total users | Per-user FPS | Aggregate / GPU | Approx. cluster FPS | Mean observed batch | Mean compute / batch (s) | Mean queue wait (ms) | Mean first frame (s) |
-|---:|---:|---:|---:|---:|---:|---:|---:|---:|
-| 1 | 4 | 14.340 | 14.340 | 57.360 | 1.000 | 0.809 | 0.000 | 0.665 |
-| 2 | 8 | 7.188 | 14.376 | 57.504 | 1.649 | 1.304 | 0.065 | 1.067 |
-| 3 | 12 | 4.694 | 14.082 | 56.328 | 2.243 | 1.730 | 1.943 | 1.438 |
-
-Observed batch histograms per GPU were respectively `{1:37}`, `{1:13,2:24}`
-and `{1:9,2:10,3:18}`. Thus batching does form after warmup, but it does not
-produce throughput scaling: batch=2 has a roughly 1.58 s steady batch time,
-close to twice a single-session 0.81 s step. Batch=3 reaches about 1.73 s and
-requires about 81 GB per H100. The bottleneck is therefore the current
-batched execution/state layout, not waiting for the scheduler to collect
-requests.
-
-Raw per-GPU JSON and logs are in `users_per_gpu_{1,2,3}/`.
diff --git a/results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md b/results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md
deleted file mode 100644
index e02977c4..00000000
--- a/results/experiments/abot_concurrent_8fps_lf2_20260813/summary.md
+++ /dev/null
@@ -1,45 +0,0 @@
-# ABot-World 8-FPS / two-latent concurrent baseline
-
-## Target and workload
-
-- Model: `ABot-World-0-5B-LF`, real public checkpoint, 832x480.
-- Target: 8 FPS per user; one continuation chunk contains 2 latent frames and decodes to 8 RGB frames.
-- Controls: every active user holds a valid control snapshot; it is refreshed once per second.
-- Consumer metric: lossless consumer displays frames at 8 FPS. `consumer_end_to_end_fps` includes startup and final queued-frame drain, so it is deliberately a user-visible, conservative metric.
-- GPU: one NVIDIA H100 80 GB (physical GPU 4), one model replica.
-- Each run uses 18 seconds of sustained input, no intentional idle intervals, and synchronized session arrivals.
-
-## Results
-
-| Active users | Scheduler | Batch cap | Mean observed batch | Mean displayed FPS/user | p95 compute per scheduled chunk (s) | p95 queue wait (s) | p95 inter-chunk interval (s) | 8-FPS target met? |
-|---:|---|---:|---:|---:|---:|---:|---:|---|
-| 1 | strict round-robin | 1 | 1.00 | 7.736 | 0.608 | 0.089 | 1.003 | Steady-state yes; end-to-end aggregate is conservative |
-| 2 | strict round-robin | 1 | 1.00 | 6.392 | 0.603 | 0.381 | 1.212 | No |
-| 3 | strict round-robin | 1 | 1.00 | 4.262 | 0.602 | 0.387 | 1.802 | No |
-| 2 | coalesced batch | 2 | 1.406 | 6.347 | 1.142 | 0.407 | 1.741 | No |
-
-## Interpretation
-
-A one-user continuation chunk stabilizes at about 0.60 seconds. Strict round-robin therefore needs about 1.20 seconds for two continuously active users and about 1.80 seconds for three, while the playback/control period is one second. This is the primary pre-improvement bottleneck: a per-GPU 8-FPS deadline miss caused by serial session scheduling, not video delivery or dropped frames.
-
-Coalesced batch size two is not a sufficient fix in the current ABot path. Its actual batch-2 compute is about 1.12 seconds, so it too misses the one-second deadline. The result is only a small end-to-end change (6.392 to 6.347 FPS/user) and uses substantially more memory (about 66.5 GiB while loaded in this run). This motivates improving both model-stage batching efficiency and workload-aware placement/admission rather than merely turning on batching.
-
-## Reproduce
-
-```bash
-cd /public/fanyk1/lwb/TeleFuser-abot-world
-
-CUDA_VISIBLE_DEVICES=4 PYTHONPATH=. \
-/public/fanyk1/lwb/envs/telefuser_sage291/bin/python \
-tools/validation/benchmark_abot_turboserve_concurrent.py \
- --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \
- --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \
- --sessions 2 --duration-seconds 18 --arrival-window-seconds 0 \
- --fps 8 --consumer-playback-fps 8 --control-latent-frames 2 \
- --scheduler-mode round_robin --max-batch-size 1 --batching-window-ms 0 \
- --delivery-mode lossless --control-update-min-seconds 1 \
- --control-update-max-seconds 1 --idle-probability 0 \
- --output results/experiments/abot_concurrent_8fps_lf2_20260813/sessions_2_round_robin.json
-```
-
-For the batching comparison, change `--scheduler-mode batched --max-batch-size 2 --batching-window-ms 2`.
diff --git a/results/experiments/abot_h100_microbatch_lf3_20260813/summary.md b/results/experiments/abot_h100_microbatch_lf3_20260813/summary.md
deleted file mode 100644
index 1b5d8ce5..00000000
--- a/results/experiments/abot_h100_microbatch_lf3_20260813/summary.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# Single-H100 ABot-World retained-session microbatch benchmark (LF=3)
-
-Date: 2026-08-13. One NVIDIA H100 80 GB (GPU 4), ABot-World-0-5B-LF,
-832x480, `control_latent_frames=3`.
-
-For each batch size B, the benchmark creates B independent retained sessions.
-It discards the special 9-frame seed chunk, warms three 12-frame continuation
-chunks, then measures eight synchronized calls to
-`generate_next_blocks(B sessions)`. Every timed call generates exactly 12
-frames for every active session. `T(B)` below is the mean timed batch-call
-duration. It excludes the service scheduler, client delivery, browser/WebRTC,
-and initial-session creation.
-
-| Batch B | Chunk Time T(B) | Aggregate FPS = 12B/T(B) | FPS/session = 12/T(B) |
-|---:|---:|---:|---:|
-| 1 | 0.7979 s | 15.04 | 15.04 |
-| 2 | 1.5656 s | 15.33 | 7.66 |
-| 3 | 2.2802 s | 15.79 | 5.26 |
-| 4 | OOM | OOM | OOM |
-
-Measurement variation (standard deviation over eight samples): B=1 6.9 ms,
-B=2 13.3 ms, B=3 23.9 ms. Peak PyTorch allocated memory was 39.1 GiB,
-55.0 GiB, and 71.1 GiB for B=1,2,3 respectively. B=4 failed while attempting
-to allocate a further 4.63 GiB, with only 0.86 GiB free.
-
-The aggregate gain from B=1 to B=3 is only 5.0%, so this ABot implementation's
-current native model batch path is close to linear-time in B. This is a model
-execution/state-layout result, not a TurboServe scheduling artifact.
-
-Raw machine-readable results: `results.json` and `results.csv`. The benchmark
-implementation is `tools/validation/benchmark_abot_microbatch.py`.
diff --git a/results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md b/results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md
deleted file mode 100644
index f2180333..00000000
--- a/results/experiments/abot_h100_microbatch_lf3_stage_profile_20260813/summary.md
+++ /dev/null
@@ -1,20 +0,0 @@
-# ABot-World LF=3 microbatch stage profile (one H100)
-
-The setup matches the synchronous retained-session microbenchmark: B independent
-sessions, 9-frame seed chunk excluded, three continuation warmups, then six
-timed 12-frame continuation batches. Stage times use CUDA events.
-
-| B | End-to-end chunk | DiT denoise | VAE decode | Other state/Python/tensor work | Aggregate FPS |
-|---:|---:|---:|---:|---:|---:|
-| 1 | 807.0 ms | 289.2 ms | 415.9 ms | 101.8 ms | 14.87 |
-| 2 | 1567.3 ms | 505.9 ms | 865.3 ms | 196.0 ms | 15.31 |
-| 3 | 2279.1 ms | 736.9 ms | 1260.1 ms | 282.0 ms | 15.80 |
-
-DiT scales sublinearly (B=3 is 2.55x B=1), demonstrating some GPU batch
-parallelism. VAE decode is nearly linear (B=2: 2.08x, B=3: 3.03x); it is the
-largest stage and is the primary reason aggregate FPS remains nearly flat.
-The remainder also grows near-linearly because the current retained-session
-implementation collates KV/VAE state before a batch and scatters it afterward.
-
-This profile is not a scheduler measurement: it invokes the native batched
-model path directly, after state creation and before any service delivery.
diff --git a/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md b/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md
index 55ab659d..72db9b7d 100644
--- a/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md
+++ b/results/experiments/abot_taew_lf3_microbatch_capacity_20260813/summary.md
@@ -1,5 +1,9 @@
# ABot-World LightVAE LF=3 single-GPU microbatch capacity
+> Scope: controlled native-model microbenchmark, not an end-to-end serving
+> result. It supersedes the pre-LightVAE summaries that were previously kept in
+> this branch. Raw JSON/CSV/log artifacts remain outside Git.
+
## Configuration
- GPU: one NVIDIA H100 80 GiB (CUDA device 5)
@@ -19,6 +23,23 @@
| 5 | 1.599 | 37.53 | 7.51 | 51.73 | fail |
| 6 | 1.961 | 36.71 | 6.12 | 57.36 | fail |
-The 8 FPS service limit is four simultaneous sessions: B=4 p95 is 1.371 s, while B=5 mean latency already exceeds the 1.5 s deadline. Aggregate throughput saturates at about 37 FPS; this is an SLO limit, not an HBM OOM limit. At B=6, mean DiT time is 1.402 s and LightVAE decode is 0.056 s.
+## Current conclusion
+
+This measurement uses the official `taew2_2` LightVAE path and directly refutes
+the old claim that one H100 is saturated at `B=1`: aggregate model throughput
+increases from **30.98 FPS** at `B=1` to **37.01 FPS** at synchronized `B=4`
+(+19.5%). The 8-FPS model-side limit is four synchronized sessions: `B=4` p95
+chunk time is 1.371 s, while `B=5` mean latency already exceeds the 1.5 s
+deadline. The approximately 37-FPS plateau is an 8-FPS SLO/model-execution
+limit, not an HBM OOM limit; at `B=6`, mean DiT time is 1.402 s and LightVAE
+decode is 0.056 s.
+
+The benchmark deliberately synchronizes identical session histories before each
+native call. It therefore measures the *available* model batch efficiency, not
+whether a real serving trace forms `B=4`. In the public 16-user trace, the
+observed DiT batch distribution and native LightVAE batch distribution must be
+read from serving telemetry; session readiness, playout deadlines, and causal
+decoder-state compatibility can keep those batches near `B=1` even though this
+controlled microbenchmark can execute `B=4`.
Raw local artifacts (`results.json`, `results.csv`, `run.log`) are intentionally ignored by repository rules.
From a10b89e92fb11bd2b403d46270895a9715d78e0d Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Wed, 19 Aug 2026 03:47:48 +0000
Subject: [PATCH 4/8] feat(abot): add credit-aware batched serving and trace
tooling
---
docs/en/abot_world.md | 78 +
examples/abot_world/_loader.py | 41 +-
.../abot_world/abot_world_livekit_service.py | 136 +-
telefuser/models/abot_world_dit.py | 260 +
telefuser/pipelines/abot_world/denoising.py | 785 +-
telefuser/pipelines/abot_world/interactive.py | 173 +-
telefuser/pipelines/abot_world/pipeline.py | 5 +
telefuser/pipelines/abot_world/service.py | 797 +-
.../service/core/stream_pipeline_service.py | 34 +
telefuser/service/livekit/config.py | 11 +
telefuser/service/livekit/metrics.py | 33 +
.../livekit/nccl_process_worker_pool.py | 417 +-
telefuser/service/livekit/pipeline_adapter.py | 18 +
telefuser/service/livekit/pipeline_router.py | 43 +
.../service/livekit/process_worker_pool.py | 211 +-
telefuser/service/livekit/worker.py | 105 +-
.../pipelines/abot_world/test_denoising.py | 147 +-
.../abot_world/test_livekit_examples.py | 38 +
.../abot_world/test_livekit_service.py | 751 +-
tests/unit/pipelines/abot_world/test_model.py | 131 +
.../pipelines/abot_world/test_pipeline.py | 5 +
.../service/livekit/test_dispatch_trace.py | 178 +
.../livekit/test_nccl_process_worker_pool.py | 197 +
.../service/livekit/test_serving_metrics.py | 13 +-
tests/unit/service/livekit/test_worker.py | 96 +
.../validation/test_abot_livekit_burst.py | 237 +
.../test_abot_scheduler_timeline.py | 51 +
.../test_abot_turboserve_trace_adapter.py | 108 +
.../test_analyze_abot_serving_trace.py | 60 +
tools/validation/abot_steady_eager.py | 307 +
.../validation/analyze_abot_serving_trace.py | 453 +
tools/validation/benchmark_abot_cuda_graph.py | 840 +
.../benchmark_abot_livekit_burst.py | 317 +-
.../derive_abot_turboserve_trace.py | 592 +
...se_abot_cuda_graph_persistent_three_way.py | 807 +
.../diagnose_abot_cuda_graph_three_way.py | 625 +
...calize_abot_steady_state_cache_mismatch.py | 570 +
.../validation/render_abot_demand_scatter.py | 287 +
.../render_abot_dispatch_timeline.py | 1001 ++
.../render_abot_scheduler_summary.py | 455 +
.../replay_abot_livekit_lifecycle_trace.py | 418 +
tools/validation/run_abot_4gpu_30min_trace.sh | 180 +
.../trace_abot_scheduler_timeline.py | 736 +
.../trace_abot_scheduler_timeline_native.py | 321 +
.../validate_abot_cuda_graph_batch_parity.py | 663 +
.../validate_abot_cuda_graph_parity.py | 512 +
.../validate_abot_public_vs_steady_state.py | 462 +
.../README-turboserve-public-demo-trace.md | 50 +
...ps_turboserve_public_demo_trace_peak4.json | 2999 ++++
...lf3_12fps_diagnostic_phase_aligned_16.json | 67 +
..._12fps_intermittent_input_peak16_5min.json | 106 +
...4gpu_lf3_12fps_realistic_async_peak16.json | 122 +
...s_turboserve_public_demo_trace_peak16.json | 13666 ++++++++++++++++
53 files changed, 31563 insertions(+), 152 deletions(-)
create mode 100644 tests/unit/service/livekit/test_dispatch_trace.py
create mode 100644 tests/unit/validation/test_abot_scheduler_timeline.py
create mode 100644 tests/unit/validation/test_abot_turboserve_trace_adapter.py
create mode 100644 tests/unit/validation/test_analyze_abot_serving_trace.py
create mode 100644 tools/validation/abot_steady_eager.py
create mode 100644 tools/validation/analyze_abot_serving_trace.py
create mode 100644 tools/validation/benchmark_abot_cuda_graph.py
create mode 100644 tools/validation/derive_abot_turboserve_trace.py
create mode 100644 tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py
create mode 100644 tools/validation/diagnose_abot_cuda_graph_three_way.py
create mode 100644 tools/validation/localize_abot_steady_state_cache_mismatch.py
create mode 100644 tools/validation/render_abot_demand_scatter.py
create mode 100644 tools/validation/render_abot_dispatch_timeline.py
create mode 100644 tools/validation/render_abot_scheduler_summary.py
create mode 100644 tools/validation/replay_abot_livekit_lifecycle_trace.py
create mode 100644 tools/validation/run_abot_4gpu_30min_trace.sh
create mode 100644 tools/validation/trace_abot_scheduler_timeline.py
create mode 100755 tools/validation/trace_abot_scheduler_timeline_native.py
create mode 100644 tools/validation/validate_abot_cuda_graph_batch_parity.py
create mode 100644 tools/validation/validate_abot_cuda_graph_parity.py
create mode 100644 tools/validation/validate_abot_public_vs_steady_state.py
create mode 100644 tools/validation/workloads/README-turboserve-public-demo-trace.md
create mode 100644 tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json
create mode 100644 tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json
diff --git a/docs/en/abot_world.md b/docs/en/abot_world.md
index 20d9c598..167e5e1a 100644
--- a/docs/en/abot_world.md
+++ b/docs/en/abot_world.md
@@ -177,6 +177,84 @@ after its state-snapshot patch has landed. A balanced user wave normally exercis
placement, local batching, admission queueing, and recovery; it need not create an
imbalanced placement worth migrating.
+### Experimental EDF deadline-aware micro-batching
+
+`TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS` is a separate, opt-in upper bound for
+waiting to form a compatible continuation batch. It does **not** replace
+`TELEFUSER_ABOT_BATCHING_WINDOW_MS`, which remains the scheduler's legacy/pacing
+coalescing window. Its default is `0`, preserving the baseline behavior.
+
+For an initial B=2 experiment, use a conservative 100 ms cap:
+
+```bash
+export TELEFUSER_ABOT_SCHEDULER_MODE=batched
+export TELEFUSER_ABOT_MAX_BATCH_SIZE=2
+export TELEFUSER_ABOT_BATCHING_WINDOW_MS=2
+export TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS=100
+```
+
+### Offline H100 batch-time priors
+
+Before the first real B=2 dispatch, the generic scheduler would otherwise estimate
+it as two B=1 calls. For the measured ABot-World-0.5B-LF, LF=3, H100 full-pipeline
+profile, select a named, explicit prior table instead:
+
+```bash
+# Eager P95 raw seconds: B2=0.7405, B3=1.0691, B4=1.4073.
+export TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE=h100_lf3_eager_full_pipeline_v1
+
+# Or use the separately validated CUDA-Graph profile: B2=0.6923, B3=1.0319.
+# export TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE=h100_lf3_cuda_graph_v1
+
+# Optional: use raw P95 × 1.05 for this run (default: ×1.10).
+export TELEFUSER_ABOT_BATCH_COMPUTE_SAFETY_FACTOR=1.05
+```
+
+The scheduler applies this factor once to raw profile/observed timing, then retains the
+maximum of the selected prior and every observed runtime. It must be finite and at least
+`1.0`; the default is `1.10`. The default profile is `none`; do not select an H100
+profile for a mismatched GPU, model shape, LF, or execution backend.
+
+When the GPU becomes free, the scheduler holds the EDF-earliest compatible
+continuation only until the earlier of this cap and its deadline-safe B=2 start.
+With `MAX_BATCH_SIZE=3`, once a compatible B arrives, it waits for C only when C's
+predicted release is before both the retained B=2 fallback start and the B=3
+latest-safe-start; otherwise it dispatches B=2 immediately. If no peer arrives, the
+held session falls back to B=1. First chunks never wait. If another EDF job is
+ready during the hold, it runs first only when both its own deadline and the held
+session's B=1 fallback deadline remain safe. Inspect `deadline_batch_waits_started`,
+`deadline_batch_wait_timeouts`, and `deadline_batch_filler_dispatches` in
+`/v1/service/metrics/json` when evaluating the policy.
+
+### Experimental publisher-frame-credit EDF
+
+To make EDF use playout slack rather than only ABot's local output queue, enable
+the following **opt-in** policy:
+
+```bash
+export TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_ENABLED=true
+# Default ABot FPS is 12, so 3 seconds yields F=36 frames.
+export TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS=3.0
+# Optional exact override; it takes precedence over TARGET_SECONDS.
+export TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES=36
+export TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES=4
+export TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_GUARD_MS=50
+```
+
+The service tracks `F = queued chunk frames + frames dequeued by the publisher but
+not yet accepted by LiveKit's `capture_frame`. `capture_frame` is a server-side
+transport handoff, not a browser render acknowledgement. For latest-mode
+continuations, EDF computes its safe start from `F`, keeps the configured reserve and
+guard, and waits for a compatible peer only while the B=2 fallback remains safe.
+First chunks and lossless sessions retain their existing behavior.
+`TARGET_FRAMES` changes only the low-watermark at which a latest-mode session becomes
+eligible again; it does not replace the reserve used to calculate the deadline. With
+`F=36`, 12 FPS, a 4-frame reserve, and a 50 ms guard, the maximum modeled completion
+slack is `(36 - 4) / 12 - 0.05 = 2.617s`.
+
+Inspect `queued_video_frames`, `publisher_unsubmitted_frames`, `frame_credit_frames`, and
+`frame_credit_deadline_in_seconds` in `/v1/service/metrics/json` or dispatch JSONL.
+
### Run the arrival/burst/recovery workload
The tracked scenario at
diff --git a/examples/abot_world/_loader.py b/examples/abot_world/_loader.py
index a77d9c42..31c61eba 100644
--- a/examples/abot_world/_loader.py
+++ b/examples/abot_world/_loader.py
@@ -19,7 +19,12 @@
from telefuser.models.taew2_2 import TAEHV
from telefuser.models.wan22_video_vae import Wan22VideoVAE
from telefuser.models.wan_video_text_encoder import WanTextEncoder
-from telefuser.ops.attention.backends import FLASH_ATTN_3_AVAILABLE, FLASH_ATTN_4_AVAILABLE
+from telefuser.ops.attention.backends import (
+ FLASH_ATTN_3_AVAILABLE,
+ FLASH_ATTN_4_AVAILABLE,
+ SAGE_ATTN_AVAILABLE,
+ sageattention,
+)
from telefuser.pipelines.abot_world import ABotWorldPipeline, ABotWorldPipelineConfig
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
@@ -29,7 +34,29 @@
DEFAULT_PROMPT = "A smooth first-person exploration through a vivid natural landscape."
-def _attention_backend() -> AttnImplType:
+def _env_flag(name: str, default: bool = False) -> bool:
+ """Read an explicit boolean environment override for an experimental path."""
+ value = os.environ.get(name)
+ if value is None:
+ return default
+ return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _attention_backend(device_id: int) -> AttnImplType:
+ """Choose the explicitly requested ABot attention backend when safe."""
+ requested = os.environ.get("TELEFUSER_ABOT_ATTENTION", "auto").strip().lower()
+ if requested in {"sage_sm90", "sageattn_sm90", "sage_attn_sm90"}:
+ if not torch.cuda.is_available() or torch.cuda.get_device_capability(device_id) != (9, 0):
+ raise RuntimeError("TELEFUSER_ABOT_ATTENTION=sage_sm90 requires an SM90 H100-class CUDA device")
+ if (
+ not SAGE_ATTN_AVAILABLE
+ or sageattention is None
+ or not hasattr(sageattention, "sageattn_qk_int8_pv_fp8_cuda_sm90")
+ ):
+ raise RuntimeError("sage_sm90 requires the tf_kernel SM90 SageAttention extension")
+ return AttnImplType.SAGE_ATTN_2_8_8_SM90
+ if requested not in {"", "auto"}:
+ raise ValueError(f"Unsupported TELEFUSER_ABOT_ATTENTION value {requested!r}; use 'auto' or 'sage_sm90'.")
if FLASH_ATTN_4_AVAILABLE:
return AttnImplType.FLASH_ATTN_4
if FLASH_ATTN_3_AVAILABLE:
@@ -48,7 +75,12 @@ def get_pipeline(
) -> ABotWorldPipeline:
"""Load the downloaded ABot checkpoint with VAE/T5 model CPU offload."""
root = Path(model_root).expanduser()
- required = ("diffusion_pytorch_model.safetensors", "Wan2.2_VAE.pth", "taew2_2.pth", "models_t5_umt5-xxl-enc-bf16.pth")
+ required = (
+ "diffusion_pytorch_model.safetensors",
+ "Wan2.2_VAE.pth",
+ "taew2_2.pth",
+ "models_t5_umt5-xxl-enc-bf16.pth",
+ )
missing = [name for name in required if not (root / name).is_file()]
if missing:
raise FileNotFoundError(f"ABot model root {root} is missing: {', '.join(missing)}")
@@ -97,13 +129,14 @@ def get_pipeline(
device_type="cuda",
device_id=device_id,
torch_dtype=torch.bfloat16,
- attention_config=AttentionConfig.dense_attention(_attention_backend()),
+ attention_config=AttentionConfig.dense_attention(_attention_backend(device_id)),
),
height=height,
width=width,
latent_frames=latent_frames,
local_attn_size=18,
sink_size=6,
+ cuda_graph_enabled=_env_flag("TELEFUSER_ABOT_CUDA_GRAPH_ENABLED"),
),
)
return pipeline
diff --git a/examples/abot_world/abot_world_livekit_service.py b/examples/abot_world/abot_world_livekit_service.py
index 62461d3d..0e6c900f 100644
--- a/examples/abot_world/abot_world_livekit_service.py
+++ b/examples/abot_world/abot_world_livekit_service.py
@@ -26,12 +26,41 @@
_DEFAULT_SCHEDULER_MODE = "batched"
_DEFAULT_MAX_BATCH_SIZE = 2
_DEFAULT_BATCHING_WINDOW_MS = 2.0
+_DEFAULT_MAX_DEADLINE_BATCH_WAIT_MS = 0.0
_SCHEDULER_MODE_ENV = "TELEFUSER_ABOT_SCHEDULER_MODE"
_MAX_BATCH_SIZE_ENV = "TELEFUSER_ABOT_MAX_BATCH_SIZE"
_BATCHING_WINDOW_MS_ENV = "TELEFUSER_ABOT_BATCHING_WINDOW_MS"
+_MAX_DEADLINE_BATCH_WAIT_MS_ENV = "TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS"
+_DEFAULT_PUBLISHER_FRAME_CREDIT_ENABLED = False
+_DEFAULT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS = 3.0
+_DEFAULT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES = 4
+_DEFAULT_PUBLISHER_FRAME_CREDIT_GUARD_MS = 50.0
+_PUBLISHER_FRAME_CREDIT_ENABLED_ENV = "TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_ENABLED"
+_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS_ENV = "TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS"
+_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES_ENV = "TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES"
+_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES_ENV = "TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES"
+_PUBLISHER_FRAME_CREDIT_GUARD_MS_ENV = "TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_GUARD_MS"
+_DEFAULT_BATCH_COMPUTE_PROFILE = "none"
+_BATCH_COMPUTE_PROFILE_ENV = "TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE"
+_DEFAULT_BATCH_COMPUTE_SAFETY_FACTOR = 1.10
+_BATCH_COMPUTE_SAFETY_FACTOR_ENV = "TELEFUSER_ABOT_BATCH_COMPUTE_SAFETY_FACTOR"
+# Raw P95 full-chunk wall times from
+# results/experiments/abot_h100_batch_vs_serial_lf3_20260814/batched/results.json.
+# They are deliberately opt-in: a timing profile from an H100 must never be
+# assumed safe on another GPU, model shape, LF, or execution backend.
+_BATCH_COMPUTE_PRIOR_PROFILES_SECONDS: dict[str, dict[int, float]] = {
+ "none": {},
+ "h100_lf3_eager_full_pipeline_v1": {
+ 2: 0.7404982000589371,
+ 3: 1.0691392589360476,
+ 4: 1.407263021916151,
+ },
+ "h100_lf3_cuda_graph_v1": {2: 0.6922691259533167, 3: 1.0319155678153038},
+}
-def _serving_schedule_from_environment() -> tuple[str, int, float]:
+
+def _serving_schedule_from_environment() -> tuple[str, int, float, float]:
"""Return the worker-local ABot scheduling settings selected by the operator.
The retained-session admission limit is intentionally configured by
@@ -66,7 +95,90 @@ def _serving_schedule_from_environment() -> tuple[str, int, float]:
if not math.isfinite(batching_window_ms) or batching_window_ms < 0:
raise ValueError(f"{_BATCHING_WINDOW_MS_ENV} must be a non-negative finite number")
- return scheduler_mode, max_batch_size, batching_window_ms
+ raw_deadline_batch_wait_ms = os.getenv(_MAX_DEADLINE_BATCH_WAIT_MS_ENV)
+ if raw_deadline_batch_wait_ms is None:
+ deadline_batch_wait_ms = _DEFAULT_MAX_DEADLINE_BATCH_WAIT_MS
+ else:
+ try:
+ deadline_batch_wait_ms = float(raw_deadline_batch_wait_ms)
+ except ValueError as exc:
+ raise ValueError(f"{_MAX_DEADLINE_BATCH_WAIT_MS_ENV} must be a non-negative finite number") from exc
+ if not math.isfinite(deadline_batch_wait_ms) or deadline_batch_wait_ms < 0:
+ raise ValueError(f"{_MAX_DEADLINE_BATCH_WAIT_MS_ENV} must be a non-negative finite number")
+
+ return scheduler_mode, max_batch_size, batching_window_ms, deadline_batch_wait_ms
+
+
+def _publisher_frame_credit_from_environment() -> tuple[bool, float, int | None, int, float]:
+ raw_enabled = (
+ os.getenv(_PUBLISHER_FRAME_CREDIT_ENABLED_ENV, str(_DEFAULT_PUBLISHER_FRAME_CREDIT_ENABLED)).strip().lower()
+ )
+ if raw_enabled in {"1", "true", "yes", "on"}:
+ enabled = True
+ elif raw_enabled in {"0", "false", "no", "off"}:
+ enabled = False
+ else:
+ raise ValueError(f"{_PUBLISHER_FRAME_CREDIT_ENABLED_ENV} must be a boolean")
+
+ def finite_float(name: str, default: float, *, positive: bool) -> float:
+ raw = os.getenv(name, str(default))
+ try:
+ value = float(raw)
+ except ValueError as exc:
+ raise ValueError(f"{name} must be a finite number") from exc
+ if not math.isfinite(value) or (value <= 0 if positive else value < 0):
+ raise ValueError(f"{name} must be a {'positive' if positive else 'non-negative'} finite number")
+ return value
+
+ target_seconds = finite_float(
+ _PUBLISHER_FRAME_CREDIT_TARGET_SECONDS_ENV, _DEFAULT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS, positive=True
+ )
+ raw_target_frames = os.getenv(_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES_ENV)
+ target_frames: int | None = None
+ if raw_target_frames is not None:
+ try:
+ target_frames = int(raw_target_frames)
+ except ValueError as exc:
+ raise ValueError(f"{_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES_ENV} must be a positive integer") from exc
+ if target_frames <= 0:
+ raise ValueError(f"{_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES_ENV} must be a positive integer")
+ raw_reserve_frames = os.getenv(
+ _PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES_ENV, str(_DEFAULT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES)
+ )
+ try:
+ reserve_frames = int(raw_reserve_frames)
+ except ValueError as exc:
+ raise ValueError(f"{_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES_ENV} must be a non-negative integer") from exc
+ if reserve_frames < 0:
+ raise ValueError(f"{_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES_ENV} must be a non-negative integer")
+ guard_ms = finite_float(
+ _PUBLISHER_FRAME_CREDIT_GUARD_MS_ENV, _DEFAULT_PUBLISHER_FRAME_CREDIT_GUARD_MS, positive=False
+ )
+ return enabled, target_seconds, target_frames, reserve_frames, guard_ms
+
+
+def _batch_compute_profile_from_environment() -> tuple[str, dict[int, float]]:
+ """Return an explicit, hardware-specific cold-start batch timing profile."""
+ name = os.getenv(_BATCH_COMPUTE_PROFILE_ENV, _DEFAULT_BATCH_COMPUTE_PROFILE).strip().lower()
+ profile = _BATCH_COMPUTE_PRIOR_PROFILES_SECONDS.get(name)
+ if profile is None:
+ choices = ", ".join(sorted(_BATCH_COMPUTE_PRIOR_PROFILES_SECONDS))
+ raise ValueError(f"{_BATCH_COMPUTE_PROFILE_ENV} must be one of: {choices}")
+ return name, dict(profile)
+
+
+def _batch_compute_safety_factor_from_environment() -> float:
+ """Return the conservative multiplier for offline and online batch timings."""
+ raw = os.getenv(_BATCH_COMPUTE_SAFETY_FACTOR_ENV, str(_DEFAULT_BATCH_COMPUTE_SAFETY_FACTOR))
+ try:
+ value = float(raw)
+ except ValueError as exc:
+ raise ValueError(
+ f"{_BATCH_COMPUTE_SAFETY_FACTOR_ENV} must be a finite number greater than or equal to 1"
+ ) from exc
+ if not math.isfinite(value) or value < 1.0:
+ raise ValueError(f"{_BATCH_COMPUTE_SAFETY_FACTOR_ENV} must be a finite number greater than or equal to 1")
+ return value
def get_service(gpu_num: int = 1, gpu_ids: list[str] | None = None) -> ABotWorldLiveKitService:
@@ -78,7 +190,16 @@ def get_service(gpu_num: int = 1, gpu_ids: list[str] | None = None) -> ABotWorld
device_id = int(assigned[0])
except ValueError as exc:
raise ValueError(f"ABot worker GPU id must be numeric, got {assigned[0]!r}") from exc
- scheduler_mode, max_batch_size, batching_window_ms = _serving_schedule_from_environment()
+ scheduler_mode, max_batch_size, batching_window_ms, deadline_batch_wait_ms = _serving_schedule_from_environment()
+ (
+ publisher_frame_credit_enabled,
+ publisher_frame_credit_target_seconds,
+ publisher_frame_credit_target_frames,
+ publisher_frame_credit_reserve_frames,
+ publisher_frame_credit_guard_ms,
+ ) = _publisher_frame_credit_from_environment()
+ batch_compute_profile_name, batch_compute_prior_seconds = _batch_compute_profile_from_environment()
+ batch_compute_safety_factor = _batch_compute_safety_factor_from_environment()
pipeline = get_pipeline(device_id=device_id, pipeline_class=ABotWorldInteractivePipeline)
return ABotWorldLiveKitService(
pipeline,
@@ -96,4 +217,13 @@ def get_service(gpu_num: int = 1, gpu_ids: list[str] | None = None) -> ABotWorld
scheduler_mode=scheduler_mode,
max_batch_size=max_batch_size,
batching_window_ms=batching_window_ms,
+ max_deadline_batch_wait_ms=deadline_batch_wait_ms,
+ batch_compute_safety_factor=batch_compute_safety_factor,
+ publisher_frame_credit_enabled=publisher_frame_credit_enabled,
+ batch_compute_profile_name=batch_compute_profile_name,
+ batch_compute_prior_seconds=batch_compute_prior_seconds,
+ publisher_frame_credit_target_seconds=publisher_frame_credit_target_seconds,
+ publisher_frame_credit_target_frames=publisher_frame_credit_target_frames,
+ publisher_frame_credit_reserve_frames=publisher_frame_credit_reserve_frames,
+ publisher_frame_credit_guard_ms=publisher_frame_credit_guard_ms,
)
diff --git a/telefuser/models/abot_world_dit.py b/telefuser/models/abot_world_dit.py
index 8e5e8220..392f8158 100644
--- a/telefuser/models/abot_world_dit.py
+++ b/telefuser/models/abot_world_dit.py
@@ -59,6 +59,37 @@ def _rope_apply(
return rotated.to(dtype=x.dtype)
+def _rope_apply_static(
+ x: torch.Tensor,
+ grid_size: tuple[int, int, int],
+ freqs: torch.Tensor,
+ frame_indices: torch.Tensor,
+) -> torch.Tensor:
+ """Apply fixed, prevalidated RoPE indices without host scalar reads."""
+ frames, height, width = grid_size
+ sequence_length = frames * height * width
+ if x.shape[1] != sequence_length:
+ raise ValueError(f"RoPE expected {sequence_length} tokens, got {x.shape[1]}")
+ half_dim = x.shape[-1] // 2
+ time_dim = half_dim - 2 * (half_dim // 3)
+ height_dim = half_dim // 3
+ width_dim = half_dim // 3
+ freq_t, freq_h, freq_w = freqs.split([time_dim, height_dim, width_dim], dim=1)
+ indices = frame_indices.to(device=x.device, dtype=torch.long)
+ expanded = torch.cat(
+ [
+ freq_t[indices].view(frames, 1, 1, -1).expand(frames, height, width, -1),
+ freq_h[:height].view(1, height, 1, -1).expand(frames, height, width, -1),
+ freq_w[:width].view(1, 1, width, -1).expand(frames, height, width, -1),
+ ],
+ dim=-1,
+ ).reshape(sequence_length, 1, -1)
+ rotated = torch.view_as_real(
+ torch.view_as_complex(x.float().reshape(x.shape[0], sequence_length, x.shape[2], -1, 2)) * expanded
+ ).flatten(3)
+ return rotated.to(dtype=x.dtype)
+
+
class _ResidualBlock(nn.Module):
def __init__(self, dim: int) -> None:
super().__init__()
@@ -177,6 +208,95 @@ def _update_cache(
self._set_cursor(cache, "local_end_index", local_end)
return local_start, local_end
+ def forward_steady_state(
+ self,
+ x: torch.Tensor,
+ grid_size: tuple[int, int, int],
+ freqs: torch.Tensor,
+ kv_cache: dict[str, Any],
+ current_end: torch.Tensor,
+ roll_scratch_k: torch.Tensor,
+ roll_scratch_v: torch.Tensor,
+ *,
+ update_cache: bool,
+ ) -> torch.Tensor:
+ """Run a full, Relative-RoPE KV window without reading cache cursors.
+
+ This is a specialized continuation path for a cache already filled by
+ the regular forward method. It keeps the sink prefix, rolls the tail
+ by exactly one fixed input block when update_cache is true, and always
+ writes the current block at the fixed tail position. Cursor tensors are
+ only written with current_end; they are never read on the host, which
+ makes this method suitable for CUDA graph capture.
+
+ The caller must ensure every layer has a full local cache and that the
+ input grid, batch, and block length stay fixed for a graph slot.
+ """
+ batch, tokens, _ = x.shape
+ frames, height, width = grid_size
+ frame_tokens = height * width
+ if tokens != frames * frame_tokens:
+ raise ValueError("ABot causal attention received inconsistent grid size")
+ if not self.use_relative_rope:
+ raise ValueError("ABot steady-state attention requires Relative-RoPE")
+ if self.local_attn_size < 1:
+ raise ValueError("ABot steady-state attention requires a finite local window")
+ capacity = kv_cache["k"].shape[1]
+ if capacity != self.local_attn_size * frame_tokens:
+ raise ValueError("ABot steady-state cache capacity does not match its local window")
+ if kv_cache["v"].shape != kv_cache["k"].shape:
+ raise ValueError("ABot steady-state key and value cache shapes must match")
+ if current_end.numel() != 1:
+ raise ValueError("ABot steady-state current_end must be a scalar tensor")
+ sink_tokens = self.sink_size * frame_tokens
+ local_start = capacity - tokens
+ rolled_tokens = local_start - sink_tokens
+ if local_start < sink_tokens or rolled_tokens < 0:
+ raise ValueError("ABot steady-state block does not fit in the rolling cache tail")
+ scratch_shape = (batch, rolled_tokens, self.num_heads, self.head_dim)
+ if roll_scratch_k.shape != scratch_shape or roll_scratch_v.shape != scratch_shape:
+ raise ValueError("ABot steady-state rolling scratch has an incompatible shape")
+
+ query = rearrange(self.norm_q(self.q(x)), "b s (h d) -> b s h d", h=self.num_heads)
+ key = rearrange(self.norm_k(self.k(x)), "b s (h d) -> b s h d", h=self.num_heads)
+ value = rearrange(self.v(x), "b s (h d) -> b s h d", h=self.num_heads)
+ if update_cache:
+ if rolled_tokens:
+ roll_scratch_k.copy_(kv_cache["k"][:, sink_tokens + tokens : capacity])
+ roll_scratch_v.copy_(kv_cache["v"][:, sink_tokens + tokens : capacity])
+ kv_cache["k"][:, sink_tokens:local_start].copy_(roll_scratch_k)
+ kv_cache["v"][:, sink_tokens:local_start].copy_(roll_scratch_v)
+ kv_cache["global_end_index"].copy_(current_end.reshape_as(kv_cache["global_end_index"]))
+ kv_cache["local_end_index"].fill_(capacity)
+ kv_cache["k"][:, local_start:capacity].copy_(key.detach())
+ kv_cache["v"][:, local_start:capacity].copy_(value)
+
+ cache_indices = torch.arange(self.local_attn_size, device=x.device)
+ query_start = self.local_attn_size - frames
+ if query_start < 0:
+ raise ValueError("ABot steady-state query block is larger than its local window")
+ cached_key = _rope_apply_static(
+ kv_cache["k"],
+ (self.local_attn_size, height, width),
+ freqs,
+ cache_indices,
+ )
+ query = _rope_apply_static(
+ query,
+ grid_size,
+ freqs,
+ torch.arange(query_start, self.local_attn_size, device=x.device),
+ )
+ output = attention_fn(
+ query,
+ cached_key,
+ kv_cache["v"],
+ attention_config=self.attention_config,
+ input_layout="BSND",
+ output_layout="BSND",
+ )
+ return self.o(rearrange(output, "b s h d -> b s (h d)"))
+
def forward(
self,
x: torch.Tensor,
@@ -284,6 +404,27 @@ def forward(self, x: torch.Tensor, context: torch.Tensor, cache: dict[str, Any])
)
return self.o(rearrange(output, "b s h d -> b s (h d)"))
+ def forward_steady_state(
+ self,
+ x: torch.Tensor,
+ cache: dict[str, Any],
+ *,
+ context_length: int,
+ ) -> torch.Tensor:
+ """Attend to an already-initialized fixed text cache without host reads."""
+ if context_length < 1 or context_length > cache["k"].shape[1]:
+ raise ValueError("ABot steady-state cross-attention context length is invalid")
+ query = rearrange(self.norm_q(self.q(x)), "b s (h d) -> b s h d", h=self.num_heads)
+ output = attention_fn(
+ query,
+ cache["k"][:, :context_length],
+ cache["v"][:, :context_length],
+ attention_config=self.attention_config,
+ input_layout="BSND",
+ output_layout="BSND",
+ )
+ return self.o(rearrange(output, "b s h d -> b s (h d)"))
+
class CausalWanAttentionBlock(nn.Module):
def __init__(
@@ -337,6 +478,50 @@ def forward(
ffn_output = self.ffn(ffn_input)
return x + (ffn_output.unflatten(1, (frames, frame_tokens)) * gate_mlp).flatten(1, 2)
+ def forward_steady_state(
+ self,
+ x: torch.Tensor,
+ time_modulation: torch.Tensor,
+ grid_size: tuple[int, int, int],
+ freqs: torch.Tensor,
+ kv_cache: dict[str, Any],
+ crossattn_cache: dict[str, Any],
+ current_end: torch.Tensor,
+ roll_scratch_k: torch.Tensor,
+ roll_scratch_v: torch.Tensor,
+ *,
+ context_length: int,
+ update_cache: bool,
+ ) -> torch.Tensor:
+ """Compose the fixed-window self and already-cached cross attention."""
+ frames, height, width = grid_size
+ frame_tokens = height * width
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+ self.modulation.to(device=x.device, dtype=x.dtype).unsqueeze(0) + time_modulation
+ ).chunk(6, dim=2)
+ normed = self.norm1(x).unflatten(1, (frames, frame_tokens))
+ attention_input = (normed * (1 + scale_msa) + shift_msa).flatten(1, 2)
+ attention = self.self_attn.forward_steady_state(
+ attention_input,
+ grid_size,
+ freqs,
+ kv_cache,
+ current_end,
+ roll_scratch_k,
+ roll_scratch_v,
+ update_cache=update_cache,
+ )
+ x = x + (attention.unflatten(1, (frames, frame_tokens)) * gate_msa).flatten(1, 2)
+ x = x + self.cross_attn.forward_steady_state(
+ self.norm3(x),
+ crossattn_cache,
+ context_length=context_length,
+ )
+ normed = self.norm2(x).unflatten(1, (frames, frame_tokens))
+ ffn_input = (normed * (1 + scale_mlp) + shift_mlp).flatten(1, 2)
+ ffn_output = self.ffn(ffn_input)
+ return x + (ffn_output.unflatten(1, (frames, frame_tokens)) * gate_mlp).flatten(1, 2)
+
class CausalHead(nn.Module):
def __init__(self, dim: int, out_dim: int, patch_size: tuple[int, int, int], eps: float) -> None:
@@ -453,6 +638,81 @@ def _unpatchify(self, x: torch.Tensor, grid_size: tuple[int, int, int]) -> torch
c=self.out_dim,
)
+ def forward_steady_state(
+ self,
+ x: torch.Tensor,
+ timestep: torch.Tensor,
+ context: torch.Tensor,
+ act_context: torch.Tensor,
+ kv_cache: list[dict[str, Any]],
+ crossattn_cache: list[dict[str, Any]],
+ current_end: torch.Tensor,
+ roll_scratch_k: torch.Tensor,
+ roll_scratch_v: torch.Tensor,
+ *,
+ update_cache: bool,
+ act_context_scale: float = 1.0,
+ ) -> torch.Tensor:
+ """Run a fixed-shape, full-cache Relative-RoPE continuation.
+
+ This method is intentionally narrower than forward: callers must prime
+ every self-attention cache to its full local window and initialize each
+ text cross-attention cache through the regular path. It then avoids
+ cache-cursor scalar reads and only writes the supplied device-resident
+ current_end scalar. The normal dynamic forward path remains the
+ fallback for first chunks, changing shapes, or stale caches.
+
+ context is retained in the API to make equivalence explicit; only its
+ fixed sequence length is used because cross-attention keys/values are
+ already cached.
+ """
+ if x.ndim != 5 or timestep.ndim != 2:
+ raise ValueError("ABot expects x=[B,C,F,H,W] and timestep=[B,F]")
+ if x.shape[2] != timestep.shape[1] or x.shape[0] != timestep.shape[0]:
+ raise ValueError("ABot timestep shape must match the latent batch and frame dimensions")
+ if context.ndim != 3 or context.shape[0] != x.shape[0]:
+ raise ValueError("ABot steady-state context must match the latent batch")
+ if len(kv_cache) != self.num_layers or len(crossattn_cache) != self.num_layers:
+ raise ValueError("ABot cache lists must contain one entry per transformer layer")
+ if not self.use_relative_rope:
+ raise ValueError("ABot steady-state forward requires Relative-RoPE")
+ if current_end.device != x.device or current_end.dtype != torch.long:
+ raise ValueError("ABot steady-state current_end must be a device-resident int64 tensor")
+ if roll_scratch_k.device != x.device or roll_scratch_v.device != x.device:
+ raise ValueError("ABot steady-state rolling scratch must share the latent device")
+
+ embedded = self.patch_embedding(x)
+ action = self.act_control_adapter(act_context.to(device=x.device, dtype=embedded.dtype))
+ if action.shape != embedded.shape:
+ raise ValueError(
+ f"ABot action adapter output {tuple(action.shape)} does not match latent tokens {tuple(embedded.shape)}"
+ )
+ embedded = embedded + action * act_context_scale
+ grid_size = tuple(int(value) for value in embedded.shape[2:])
+ tokens = rearrange(embedded, "b c f h w -> b (f h w) c")
+ time_embedding = self.time_embedding(
+ sinusoidal_embedding_1d(self.freq_dim, timestep.flatten()).to(tokens.dtype)
+ )
+ time_embedding = time_embedding.unflatten(0, timestep.shape)
+ time_modulation = self.time_projection(time_embedding).unflatten(2, (6, self.dim))
+ freqs = self._frequencies(tokens.device)
+ context_length = context.shape[1]
+ for index, block in enumerate(self.blocks):
+ tokens = block.forward_steady_state(
+ tokens,
+ time_modulation,
+ grid_size,
+ freqs,
+ kv_cache[index],
+ crossattn_cache[index],
+ current_end,
+ roll_scratch_k,
+ roll_scratch_v,
+ context_length=context_length,
+ update_cache=update_cache,
+ )
+ return self._unpatchify(self.head(tokens, time_embedding.unsqueeze(2)), grid_size)
+
def forward(
self,
x: torch.Tensor,
diff --git a/telefuser/pipelines/abot_world/denoising.py b/telefuser/pipelines/abot_world/denoising.py
index 9d044551..ebb907cf 100644
--- a/telefuser/pipelines/abot_world/denoising.py
+++ b/telefuser/pipelines/abot_world/denoising.py
@@ -3,15 +3,307 @@
from __future__ import annotations
from collections.abc import Sequence
+from dataclasses import dataclass
from typing import Any
import torch
from telefuser.core.base_stage import BaseStage, with_model_offload
-from telefuser.core.config import ModelRuntimeConfig
+from telefuser.core.config import AttnImplType, ModelRuntimeConfig
from telefuser.core.module_manager import ModuleManager
from telefuser.models.abot_world_dit import ABotWorldDiT
from telefuser.schedulers.flow_match import FlowMatchScheduler
+from telefuser.utils.logging import logger
+
+
+@dataclass
+class _CudaGraphSlot:
+ """One replayable fixed-shape DiT call and its static output."""
+
+ graph: torch.cuda.CUDAGraph
+ output: torch.Tensor
+
+
+@dataclass
+class _BatchedCudaGraphState:
+ """Persistent B=2/3 cache arena and graph for one ordered session cohort.
+
+ Each member session's K/V tensors are rebound to a row view into this
+ arena after the initial successful capture. Subsequent singleton work on
+ a member therefore updates the same storage, while a later replay of the
+ exact cohort can directly use the contiguous batched cache without a
+ multi-gigabyte collate/scatter copy.
+ """
+
+ session_ids: tuple[str, ...]
+ self_cache: list[dict[str, Any]]
+ cross_cache: list[dict[str, Any]]
+ graph: "_ABotSteadyCudaGraph"
+
+ def matches_members(
+ self,
+ session_ids: Sequence[str],
+ self_caches: Sequence[Sequence[dict[str, Any]]],
+ cross_caches: Sequence[Sequence[dict[str, Any]]],
+ ) -> bool:
+ if tuple(session_ids) != self.session_ids:
+ return False
+ for row, (member_self, member_cross) in enumerate(zip(self_caches, cross_caches, strict=True)):
+ if len(member_self) != len(self.self_cache) or len(member_cross) != len(self.cross_cache):
+ return False
+ for source, arena in zip(member_self, self.self_cache, strict=True):
+ if source["k"].data_ptr() != arena["k"][row : row + 1].data_ptr():
+ return False
+ if source["v"].data_ptr() != arena["v"][row : row + 1].data_ptr():
+ return False
+ for source, arena in zip(member_cross, self.cross_cache, strict=True):
+ if source["k"].data_ptr() != arena["k"][row : row + 1].data_ptr():
+ return False
+ if source["v"].data_ptr() != arena["v"][row : row + 1].data_ptr():
+ return False
+ return True
+
+
+class _ABotSteadyCudaGraph:
+ """CUDA-Graph state for one full-window Relative-RoPE ABot session.
+
+ The graph owns only persistent input/scratch tensors; the session's KV and
+ cross-attention caches deliberately stay in place. That keeps graph replay
+ compatible with the existing retained-session lifecycle without copying a
+ multi-gigabyte KV cache for every continuation.
+ """
+
+ def __init__(
+ self,
+ dit: ABotWorldDiT,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ *,
+ torch_dtype: torch.dtype,
+ ) -> None:
+ self.dit = dit
+ self.device = latent.device
+ self.torch_dtype = torch_dtype
+ self.frames = latent.shape[2]
+ self.frame_tokens = (latent.shape[-2] // dit.patch_size[1]) * (latent.shape[-1] // dit.patch_size[2])
+ self.static_x = torch.empty_like(latent)
+ self.static_action = torch.empty_like(action_context)
+ self.static_timestep = torch.empty((latent.shape[0], self.frames), dtype=torch.float32, device=self.device)
+ self.static_context = prompt_emb.detach().clone()
+ self.current_end = torch.empty(1, dtype=torch.long, device=self.device)
+ capacity = self_cache[0]["k"].shape[1]
+ sink_tokens = dit.sink_size * self.frame_tokens
+ rolled_tokens = capacity - sink_tokens - latent.shape[2] * self.frame_tokens
+ if rolled_tokens < 0:
+ raise ValueError("ABot CUDA Graph block does not fit in its rolling cache tail")
+ scratch_shape = (
+ latent.shape[0],
+ rolled_tokens,
+ dit.num_heads,
+ dit.dim // dit.num_heads,
+ )
+ self.roll_scratch_k = torch.empty(scratch_shape, dtype=latent.dtype, device=self.device)
+ self.roll_scratch_v = torch.empty_like(self.roll_scratch_k)
+ self.entry: _CudaGraphSlot | None = None
+ self.refinement: _CudaGraphSlot | None = None
+ self._self_cache_signature = self._cache_signature(self_cache)
+ self._cross_cache_signature = self._cache_signature(cross_cache)
+
+ @staticmethod
+ def _cache_signature(caches: Sequence[dict[str, Any]]) -> tuple[tuple[int, int, tuple[int, ...]], ...]:
+ return tuple(
+ (
+ int(layer["k"].data_ptr()),
+ int(layer["v"].data_ptr()),
+ tuple(layer["k"].shape),
+ )
+ for layer in caches
+ )
+
+ def matches(
+ self,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: Sequence[dict[str, Any]],
+ cross_cache: Sequence[dict[str, Any]],
+ ) -> bool:
+ return (
+ self.entry is not None
+ and self.refinement is not None
+ and tuple(latent.shape) == tuple(self.static_x.shape)
+ and latent.dtype == self.static_x.dtype
+ and tuple(prompt_emb.shape) == tuple(self.static_context.shape)
+ and prompt_emb.dtype == self.static_context.dtype
+ and tuple(action_context.shape) == tuple(self.static_action.shape)
+ and action_context.dtype == self.static_action.dtype
+ and self._cache_signature(self_cache) == self._self_cache_signature
+ and self._cache_signature(cross_cache) == self._cross_cache_signature
+ )
+
+ @staticmethod
+ def backup_caches(caches: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
+ return [
+ {key: value.detach().clone() if isinstance(value, torch.Tensor) else value for key, value in layer.items()}
+ for layer in caches
+ ]
+
+ @staticmethod
+ def restore_caches(caches: Sequence[dict[str, Any]], backup: Sequence[dict[str, Any]]) -> None:
+ for current, saved in zip(caches, backup, strict=True):
+ for key, saved_value in saved.items():
+ current_value = current[key]
+ if isinstance(saved_value, torch.Tensor):
+ if not isinstance(current_value, torch.Tensor):
+ raise RuntimeError("ABot CUDA Graph cache metadata changed during capture")
+ current_value.copy_(saved_value)
+ else:
+ current[key] = saved_value
+
+ def _set_inputs(
+ self,
+ latent: torch.Tensor,
+ action_context: torch.Tensor,
+ timestep: torch.Tensor | None,
+ *,
+ current_end: int,
+ ) -> None:
+ self.static_x.copy_(latent)
+ self.static_action.copy_(action_context)
+ if timestep is None:
+ self.static_timestep.zero_()
+ else:
+ self.static_timestep.copy_(timestep.reshape(1, 1).expand_as(self.static_timestep))
+ self.current_end.fill_(current_end)
+
+ def _capture_slot(
+ self,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ *,
+ update_cache: bool,
+ ) -> _CudaGraphSlot:
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph, capture_error_mode="thread_local"):
+ with torch.autocast(self.device.type, dtype=self.torch_dtype, enabled=self.device.type == "cuda"):
+ output = self.dit.forward_steady_state(
+ x=self.static_x,
+ timestep=self.static_timestep,
+ context=self.static_context,
+ act_context=self.static_action,
+ kv_cache=self_cache,
+ crossattn_cache=cross_cache,
+ current_end=self.current_end,
+ roll_scratch_k=self.roll_scratch_k,
+ roll_scratch_v=self.roll_scratch_v,
+ update_cache=update_cache,
+ )
+ return _CudaGraphSlot(graph=graph, output=output)
+
+ @staticmethod
+ def _draw_noise(
+ current: torch.Tensor,
+ generator: torch.Generator | Sequence[torch.Generator],
+ ) -> torch.Tensor:
+ if isinstance(generator, Sequence):
+ if len(generator) != current.shape[0]:
+ raise ValueError("ABot CUDA Graph batch needs one generator per session")
+ return torch.cat(
+ [
+ torch.randn(
+ (1, *current.shape[1:]),
+ generator=item_generator,
+ dtype=current.dtype,
+ device=current.device,
+ )
+ for item_generator in generator
+ ],
+ dim=0,
+ )
+ return torch.randn(current.shape, generator=generator, dtype=current.dtype, device=current.device)
+
+ def run(
+ self,
+ stage: "ABotWorldDenoisingStage",
+ latent: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ *,
+ current_start: int,
+ generator: torch.Generator | Sequence[torch.Generator],
+ scheduler: FlowMatchScheduler,
+ capture: bool,
+ ) -> tuple[torch.Tensor, int]:
+ """Execute the four fixed sampler calls plus the context-cache update."""
+ if capture and (self.entry is not None or self.refinement is not None):
+ raise RuntimeError("ABot CUDA Graph capture was attempted twice for one session")
+ if not capture and (self.entry is None or self.refinement is None):
+ raise RuntimeError("ABot CUDA Graph replay was requested before capture")
+ timesteps = stage._official_denoising_timesteps(scheduler).to(device=self.device)
+ current_end = (current_start + self.frames) * self.frame_tokens
+ current = latent
+ replays = 0
+ for index, current_timestep in enumerate(timesteps):
+ self._set_inputs(current, action_context, current_timestep, current_end=current_end)
+ if index == 0:
+ if capture:
+ self.entry = self._capture_slot(self_cache, cross_cache, update_cache=True)
+ # CUDA stream capture records operations but does not run
+ # them. Replay once before consuming the static output or
+ # relying on its externally-owned KV-cache writes.
+ self.entry.graph.replay()
+ replays += 1
+ else:
+ assert self.entry is not None
+ self.entry.graph.replay()
+ replays += 1
+ assert self.entry is not None
+ flow_prediction = self.entry.output
+ elif index == 1 and capture:
+ self.refinement = self._capture_slot(self_cache, cross_cache, update_cache=False)
+ self.refinement.graph.replay()
+ replays += 1
+ flow_prediction = self.refinement.output
+ else:
+ assert self.refinement is not None
+ self.refinement.graph.replay()
+ replays += 1
+ flow_prediction = self.refinement.output
+ x0 = stage._x0_prediction(flow_prediction, current, self.static_timestep, scheduler)
+ if index < len(timesteps) - 1:
+ current = scheduler.add_noise(x0, self._draw_noise(x0, generator), timesteps[index + 1])
+ else:
+ # The cache-only context pass is dynamic, so it does not
+ # overwrite the refinement graph's output buffer. Retain the
+ # original x0 layout: Conv3D may select a layout-sensitive
+ # kernel for the final cache write.
+ current = x0
+ # The final forward exists only to commit the denoised x0 into the
+ # retained KV cache. Keep it on the original dynamic path. Its input
+ # must not alias a graph slot's static output: although the values are
+ # equal, reuse of that storage can alter the cache-write behavior of
+ # the subsequent eager DiT call. The small x0 clone is therefore a
+ # correctness boundary; the four denoising calls remain graphed.
+ context_input = current.clone()
+ self.static_timestep.zero_()
+ # Match _denoise_block exactly: its final cache-only public forward is
+ # deliberately outside the sampler autocast scope. Changing that
+ # precision scope leaves the generated x0 intact but changes retained
+ # tail K/V values, which then diverge on the next continuation.
+ self.dit(
+ x=context_input.to(dtype=self.torch_dtype),
+ timestep=self.static_timestep,
+ context=self.static_context,
+ act_context=action_context,
+ kv_cache=self_cache,
+ crossattn_cache=cross_cache,
+ current_start=current_start * self.frame_tokens,
+ )
+ return current, replays
class ABotWorldDenoisingStage(BaseStage):
@@ -24,6 +316,91 @@ def __init__(self, name: str, module_manager: ModuleManager, model_runtime_confi
raise ValueError("ABot-World requires a loaded abot_world_dit module")
self.dit = dit
self.model_names = ["dit"]
+ self._cuda_graph_enabled = False
+ self._cuda_graph_states: dict[str, _ABotSteadyCudaGraph] = {}
+ self._cuda_graph_batch_states: dict[tuple[str, ...], _BatchedCudaGraphState] = {}
+ self._cuda_graph_captures = 0
+ self._cuda_graph_replays = 0
+ self._cuda_graph_capture_failures = 0
+ self._last_cuda_graph_metrics: dict[str, int] = {
+ "cuda_graph_enabled": 0,
+ "cuda_graph_eligible": 0,
+ "cuda_graph_captured": 0,
+ "cuda_graph_replays": 0,
+ "cuda_graph_fallback": 0,
+ "cuda_graph_batch_size": 0,
+ "cuda_graph_batched": 0,
+ }
+
+ def configure_cuda_graph(self, enabled: bool) -> None:
+ """Enable the experimental fixed-shape CUDA Graph continuation path."""
+ self._cuda_graph_enabled = bool(enabled)
+ self._cuda_graph_states.clear()
+ self._cuda_graph_batch_states.clear()
+
+ def release_cuda_graph(self, session_id: str) -> None:
+ """Drop a graph whose cache pointers are about to move or be released."""
+ self._cuda_graph_states.pop(session_id, None)
+ for cohort_key in tuple(self._cuda_graph_batch_states):
+ if session_id in cohort_key:
+ self._cuda_graph_batch_states.pop(cohort_key, None)
+
+ def cuda_graph_metrics(self) -> dict[str, int]:
+ """Return process-local, low-cardinality CUDA Graph accounting."""
+ return {
+ "enabled": int(self._cuda_graph_enabled),
+ "resident_sessions": len(self._cuda_graph_states),
+ "resident_batch_cohorts": len(self._cuda_graph_batch_states),
+ "resident_batched_sessions": sum(
+ len(state.session_ids) for state in self._cuda_graph_batch_states.values()
+ ),
+ "captures": self._cuda_graph_captures,
+ "replays": self._cuda_graph_replays,
+ "capture_failures": self._cuda_graph_capture_failures,
+ }
+
+ def last_cuda_graph_metrics(self) -> dict[str, int]:
+ """Return graph status for the most recently generated interactive block."""
+ return dict(self._last_cuda_graph_metrics)
+
+ def record_cuda_graph_not_used(self) -> None:
+ """Mark a non-eligible batch without exposing a stale previous hit."""
+ self._set_cuda_graph_last_metrics(eligible=False)
+
+ def _cuda_graph_backend_is_supported(self) -> bool:
+ """Return whether the active attention backend passed graph parity.
+
+ The public SageAttention wrappers allocate and quantize temporary
+ tensors on every call. Their current SM90 implementation can enter a
+ CUDA graph without an exception but does not yet replay equivalently,
+ so keep Sage eager-only until it has a static-buffer adapter.
+ """
+ blocked = {
+ AttnImplType.SAGE_ATTN_2_8_8,
+ AttnImplType.SAGE_ATTN_2_8_16,
+ AttnImplType.SAGE_ATTN_2_8_8_SM90,
+ }
+ return self.dit.blocks[0].self_attn.attention_config.attn_impl not in blocked
+
+ def _set_cuda_graph_last_metrics(
+ self,
+ *,
+ eligible: bool,
+ captured: bool = False,
+ replays: int = 0,
+ fallback: bool = False,
+ batch_size: int = 0,
+ batched: bool = False,
+ ) -> None:
+ self._last_cuda_graph_metrics = {
+ "cuda_graph_enabled": int(self._cuda_graph_enabled),
+ "cuda_graph_eligible": int(eligible),
+ "cuda_graph_captured": int(captured),
+ "cuda_graph_replays": replays,
+ "cuda_graph_fallback": int(fallback),
+ "cuda_graph_batch_size": batch_size,
+ "cuda_graph_batched": int(batched),
+ }
def parallel_models(self) -> None:
if self.model_runtime_config.parallel_config.world_size != 1:
@@ -98,6 +475,412 @@ def _official_denoising_timesteps(scheduler: FlowMatchScheduler) -> torch.Tensor
# schedule with ``1000 - [1000, 750, 500, 250]``.
return scheduler.timesteps[torch.tensor((0, 250, 500, 750), dtype=torch.long)]
+ def _is_cuda_graph_eligible(
+ self,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: Sequence[dict[str, Any]],
+ cross_cache: Sequence[dict[str, Any]],
+ *,
+ current_start: int,
+ generator: torch.Generator,
+ ) -> bool:
+ """Check static continuation invariants only before first capture."""
+ if not self._cuda_graph_enabled or torch.device(self.device).type != "cuda":
+ return False
+ if not self._cuda_graph_backend_is_supported():
+ return False
+ if latent.shape[0] != 1 or latent.shape[2] != 3 or action_context.shape[0] != 1:
+ return False
+ if not self.dit.use_relative_rope or not isinstance(generator, torch.Generator):
+ return False
+ if len(self_cache) != self.dit.num_layers or len(cross_cache) != self.dit.num_layers:
+ return False
+ frame_tokens = (latent.shape[-2] // self.dit.patch_size[1]) * (latent.shape[-1] // self.dit.patch_size[2])
+ expected_global_end = current_start * frame_tokens
+ capacity = self.dit.local_attn_size * frame_tokens
+ if self.dit.local_attn_size <= latent.shape[2] or self.dit.sink_size < 0:
+ return False
+ for self_layer, cross_layer in zip(self_cache, cross_cache, strict=True):
+ if self_layer["k"].shape[1] != capacity or self_layer["v"].shape != self_layer["k"].shape:
+ return False
+ if int(self_layer["local_end_index"].item()) != capacity:
+ return False
+ if int(self_layer["global_end_index"].item()) != expected_global_end:
+ return False
+ if not bool(cross_layer["is_init"]) or int(cross_layer["sequence_length"]) != prompt_emb.shape[1]:
+ return False
+ return True
+
+ def _is_cuda_graph_batched_eligible(
+ self,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_caches: Sequence[Sequence[dict[str, Any]]],
+ cross_caches: Sequence[Sequence[dict[str, Any]]],
+ *,
+ current_starts: Sequence[int],
+ generators: Sequence[torch.Generator],
+ ) -> bool:
+ """Check B=2/3 fixed-window Relative-RoPE continuation invariants."""
+ batch_size = latent.shape[0]
+ if not self._cuda_graph_enabled or torch.device(self.device).type != "cuda":
+ return False
+ if not self._cuda_graph_backend_is_supported():
+ return False
+ if batch_size not in {2, 3} or latent.shape[2] != 3 or action_context.shape[0] != batch_size:
+ return False
+ if prompt_emb.shape[0] != batch_size or not self.dit.use_relative_rope:
+ return False
+ if len(self_caches) != batch_size or len(cross_caches) != batch_size:
+ return False
+ if len(current_starts) != batch_size or len(generators) != batch_size:
+ return False
+ # The graph owns one internal cursor tensor. Relative-RoPE permits
+ # generic eager batches at different global positions, but a graph
+ # cohort must stay position-aligned until it owns per-row cursors.
+ if len(set(current_starts)) != 1:
+ return False
+ if not all(isinstance(generator, torch.Generator) for generator in generators):
+ return False
+ frame_tokens = (latent.shape[-2] // self.dit.patch_size[1]) * (latent.shape[-1] // self.dit.patch_size[2])
+ capacity = self.dit.local_attn_size * frame_tokens
+ if self.dit.local_attn_size <= latent.shape[2] or self.dit.sink_size < 0:
+ return False
+ for current_start, self_cache, cross_cache in zip(current_starts, self_caches, cross_caches, strict=True):
+ if len(self_cache) != self.dit.num_layers or len(cross_cache) != self.dit.num_layers:
+ return False
+ expected_global_end = current_start * frame_tokens
+ for self_layer, cross_layer in zip(self_cache, cross_cache, strict=True):
+ if self_layer["k"].shape[0] != 1 or self_layer["k"].shape[1] != capacity:
+ return False
+ if self_layer["v"].shape != self_layer["k"].shape:
+ return False
+ if int(self_layer["local_end_index"].item()) != capacity:
+ return False
+ if int(self_layer["global_end_index"].item()) != expected_global_end:
+ return False
+ if not bool(cross_layer["is_init"]) or int(cross_layer["sequence_length"]) != prompt_emb.shape[1]:
+ return False
+ return True
+
+ def _drop_conflicting_batched_cuda_graphs(self, cohort_key: tuple[str, ...]) -> None:
+ members = set(cohort_key)
+ for existing_key in tuple(self._cuda_graph_batch_states):
+ if existing_key != cohort_key and members.intersection(existing_key):
+ self._cuda_graph_batch_states.pop(existing_key, None)
+
+ def _create_batched_cuda_graph_state(
+ self,
+ session_ids: tuple[str, ...],
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_caches: Sequence[Sequence[dict[str, Any]]],
+ cross_caches: Sequence[Sequence[dict[str, Any]]],
+ *,
+ current_starts: Sequence[int],
+ ) -> _BatchedCudaGraphState:
+ """Create an unbound persistent cache arena for one graph cohort."""
+ batch_size = len(session_ids)
+ arena_self, arena_cross = self._new_cache(batch_size, latent.shape[-2], latent.shape[-1])
+ for row, (source_self, source_cross) in enumerate(zip(self_caches, cross_caches, strict=True)):
+ for source, arena in zip(source_self, arena_self, strict=True):
+ arena["k"][row : row + 1].copy_(source["k"])
+ arena["v"][row : row + 1].copy_(source["v"])
+ for source, arena in zip(source_cross, arena_cross, strict=True):
+ arena["k"][row : row + 1].copy_(source["k"])
+ arena["v"][row : row + 1].copy_(source["v"])
+ frame_tokens = (latent.shape[-2] // self.dit.patch_size[1]) * (latent.shape[-1] // self.dit.patch_size[2])
+ capacity = self.dit.local_attn_size * frame_tokens
+ for self_layer, cross_layer in zip(arena_self, arena_cross, strict=True):
+ self_layer["global_end_index"].fill_(current_starts[0] * frame_tokens)
+ self_layer["local_end_index"].fill_(capacity)
+ cross_layer["is_init"] = True
+ cross_layer["sequence_length"] = prompt_emb.shape[1]
+ return _BatchedCudaGraphState(
+ session_ids=session_ids,
+ self_cache=arena_self,
+ cross_cache=arena_cross,
+ graph=_ABotSteadyCudaGraph(
+ self.dit,
+ latent,
+ prompt_emb,
+ action_context,
+ arena_self,
+ arena_cross,
+ torch_dtype=self.torch_dtype,
+ ),
+ )
+
+ @staticmethod
+ def _bind_batched_cache_arena(
+ state: _BatchedCudaGraphState,
+ self_caches: Sequence[Sequence[dict[str, Any]]],
+ cross_caches: Sequence[Sequence[dict[str, Any]]],
+ ) -> None:
+ """Make each member cache's K/V tensors a view of its arena row."""
+ for row, (member_self, member_cross) in enumerate(zip(self_caches, cross_caches, strict=True)):
+ for source, arena in zip(member_self, state.self_cache, strict=True):
+ source["k"] = arena["k"][row : row + 1]
+ source["v"] = arena["v"][row : row + 1]
+ for source, arena in zip(member_cross, state.cross_cache, strict=True):
+ source["k"] = arena["k"][row : row + 1]
+ source["v"] = arena["v"][row : row + 1]
+
+ def _advance_batched_cache_cursors(
+ self,
+ self_caches: Sequence[Sequence[dict[str, Any]]],
+ *,
+ current_starts: Sequence[int],
+ latent: torch.Tensor,
+ ) -> None:
+ """Update independent per-session cursor metadata after arena replay."""
+ frame_tokens = (latent.shape[-2] // self.dit.patch_size[1]) * (latent.shape[-1] // self.dit.patch_size[2])
+ capacity = self.dit.local_attn_size * frame_tokens
+ for current_start, self_cache in zip(current_starts, self_caches, strict=True):
+ expected_global_end = (current_start + latent.shape[2]) * frame_tokens
+ for layer in self_cache:
+ layer["global_end_index"].fill_(expected_global_end)
+ layer["local_end_index"].fill_(capacity)
+
+ def denoise_interactive_blocks(
+ self,
+ *,
+ session_ids: Sequence[str],
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_caches: Sequence[Sequence[dict[str, Any]]],
+ cross_caches: Sequence[Sequence[dict[str, Any]]],
+ current_starts: Sequence[int],
+ generators: Sequence[torch.Generator],
+ scheduler: FlowMatchScheduler,
+ ) -> torch.Tensor | None:
+ """Run an exact B=2/3 cohort through a persistent CUDA-Graph arena.
+
+ ``None`` means the caller must use its existing generic eager batch
+ path. The arena is only bound to sessions after its first capture has
+ successfully produced a real continuation, so a failed capture cannot
+ corrupt their retained caches.
+ """
+ if not self._is_cuda_graph_batched_eligible(
+ latent,
+ prompt_emb,
+ action_context,
+ self_caches,
+ cross_caches,
+ current_starts=current_starts,
+ generators=generators,
+ ):
+ self._set_cuda_graph_last_metrics(eligible=False)
+ return None
+
+ cohort_key = tuple(session_ids)
+ state = self._cuda_graph_batch_states.get(cohort_key)
+ if state is not None and (
+ not state.matches_members(session_ids, self_caches, cross_caches)
+ or not state.graph.matches(latent, prompt_emb, action_context, state.self_cache, state.cross_cache)
+ ):
+ self._cuda_graph_batch_states.pop(cohort_key, None)
+ state = None
+
+ if state is not None:
+ output, replays = state.graph.run(
+ self,
+ latent,
+ action_context,
+ state.self_cache,
+ state.cross_cache,
+ current_start=current_starts[0],
+ generator=generators,
+ scheduler=scheduler,
+ capture=False,
+ )
+ self._advance_batched_cache_cursors(self_caches, current_starts=current_starts, latent=latent)
+ self._cuda_graph_replays += replays
+ self._set_cuda_graph_last_metrics(
+ eligible=True,
+ replays=replays,
+ batch_size=latent.shape[0],
+ batched=True,
+ )
+ return output
+
+ self._drop_conflicting_batched_cuda_graphs(cohort_key)
+ generator_states = [generator.get_state().clone() for generator in generators]
+ try:
+ # The cohort's arena is private until a successful capture. This
+ # avoids cloning multi-gigabyte session KV caches just to recover
+ # from an unsupported graph backend.
+ torch.cuda.synchronize(self.device)
+ captured = self._create_batched_cuda_graph_state(
+ cohort_key,
+ latent,
+ prompt_emb,
+ action_context,
+ self_caches,
+ cross_caches,
+ current_starts=current_starts,
+ )
+ output, replays = captured.graph.run(
+ self,
+ latent,
+ action_context,
+ captured.self_cache,
+ captured.cross_cache,
+ current_start=current_starts[0],
+ generator=generators,
+ scheduler=scheduler,
+ capture=True,
+ )
+ except (RuntimeError, ValueError) as exc:
+ for generator, saved_state in zip(generators, generator_states, strict=True):
+ generator.set_state(saved_state)
+ self._cuda_graph_capture_failures += 1
+ self._set_cuda_graph_last_metrics(
+ eligible=True,
+ fallback=True,
+ batch_size=latent.shape[0],
+ batched=True,
+ )
+ logger.warning("ABot CUDA Graph capture for cohort {} failed; using eager: {}", cohort_key, exc)
+ return None
+
+ for session_id in cohort_key:
+ self._cuda_graph_states.pop(session_id, None)
+ self._bind_batched_cache_arena(captured, self_caches, cross_caches)
+ self._advance_batched_cache_cursors(self_caches, current_starts=current_starts, latent=latent)
+ self._cuda_graph_batch_states[cohort_key] = captured
+ self._cuda_graph_captures += 1
+ self._cuda_graph_replays += replays
+ self._set_cuda_graph_last_metrics(
+ eligible=True,
+ captured=True,
+ replays=replays,
+ batch_size=latent.shape[0],
+ batched=True,
+ )
+ return output
+
+ def denoise_interactive_block(
+ self,
+ *,
+ session_id: str,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ current_start: int,
+ generator: torch.Generator,
+ scheduler: FlowMatchScheduler,
+ ) -> torch.Tensor:
+ """Use a graph replay for an eligible retained-session continuation.
+
+ The first eligible chunk captures the two static DiT call shapes while
+ producing a valid real result. Capture has a full cache/RNG rollback;
+ therefore an unsupported PyTorch or attention backend safely falls
+ back to the existing eager implementation.
+ """
+ state = self._cuda_graph_states.get(session_id)
+ if state is not None and state.matches(latent, prompt_emb, action_context, self_cache, cross_cache):
+ output, replays = state.run(
+ self,
+ latent,
+ action_context,
+ self_cache,
+ cross_cache,
+ current_start=current_start,
+ generator=generator,
+ scheduler=scheduler,
+ capture=False,
+ )
+ self._cuda_graph_replays += replays
+ self._set_cuda_graph_last_metrics(eligible=True, replays=replays)
+ return output
+ if state is not None:
+ self.release_cuda_graph(session_id)
+ if not self._is_cuda_graph_eligible(
+ latent,
+ prompt_emb,
+ action_context,
+ self_cache,
+ cross_cache,
+ current_start=current_start,
+ generator=generator,
+ ):
+ self._set_cuda_graph_last_metrics(eligible=False)
+ return self._denoise_block(
+ latent,
+ prompt_emb,
+ action_context,
+ None,
+ self_cache,
+ cross_cache,
+ current_start,
+ generator,
+ scheduler,
+ )
+
+ self_backup: list[dict[str, Any]] | None = None
+ cross_backup: list[dict[str, Any]] | None = None
+ generator_state = generator.get_state().clone()
+ try:
+ # Capture is a rare event. Synchronizing here prevents a previous
+ # eager chunk from leaving a lazy kernel initialization on the
+ # capture stream; normal replays remain fully asynchronous.
+ torch.cuda.synchronize(self.device)
+ self_backup = _ABotSteadyCudaGraph.backup_caches(self_cache)
+ cross_backup = _ABotSteadyCudaGraph.backup_caches(cross_cache)
+ captured = _ABotSteadyCudaGraph(
+ self.dit,
+ latent,
+ prompt_emb,
+ action_context,
+ self_cache,
+ cross_cache,
+ torch_dtype=self.torch_dtype,
+ )
+ output, replays = captured.run(
+ self,
+ latent,
+ action_context,
+ self_cache,
+ cross_cache,
+ current_start=current_start,
+ generator=generator,
+ scheduler=scheduler,
+ capture=True,
+ )
+ except (RuntimeError, ValueError) as exc:
+ if self_backup is not None:
+ _ABotSteadyCudaGraph.restore_caches(self_cache, self_backup)
+ if cross_backup is not None:
+ _ABotSteadyCudaGraph.restore_caches(cross_cache, cross_backup)
+ generator.set_state(generator_state)
+ self._cuda_graph_capture_failures += 1
+ self._set_cuda_graph_last_metrics(eligible=True, fallback=True)
+ logger.warning("ABot CUDA Graph capture for session {} failed; falling back to eager: {}", session_id, exc)
+ return self._denoise_block(
+ latent,
+ prompt_emb,
+ action_context,
+ None,
+ self_cache,
+ cross_cache,
+ current_start,
+ generator,
+ scheduler,
+ )
+ self._cuda_graph_states[session_id] = captured
+ self._cuda_graph_captures += 1
+ self._cuda_graph_replays += replays
+ self._set_cuda_graph_last_metrics(eligible=True, captured=True, replays=replays)
+ return output
+
def _denoise_block(
self,
latent: torch.Tensor,
diff --git a/telefuser/pipelines/abot_world/interactive.py b/telefuser/pipelines/abot_world/interactive.py
index 28c226f8..f433c852 100644
--- a/telefuser/pipelines/abot_world/interactive.py
+++ b/telefuser/pipelines/abot_world/interactive.py
@@ -43,9 +43,7 @@ class ABotWorldInteractiveSession:
cross_cache: list[dict[str, Any]]
scheduler: Any
generator: torch.Generator
- vae_decode_state: Wan22VideoVAEStreamingDecodeState = field(
- default_factory=Wan22VideoVAEStreamingDecodeState
- )
+ vae_decode_state: Wan22VideoVAEStreamingDecodeState = field(default_factory=Wan22VideoVAEStreamingDecodeState)
taew_decode_state: ABotWorldTAEWDecodeState | None = None
session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
next_latent_frame: int = 0
@@ -227,16 +225,39 @@ def generate_next_blocks(
)
)
- input_prepare_seconds = time.monotonic() - batch_started_at
- cache_collate_started_at = time.monotonic()
- original_global_ends = [
- [int(layer["global_end_index"].item()) for layer in session.self_cache]
- for session in sessions
- ]
- self_cache = self._collate_caches(sessions, "self_cache")
- cross_cache = self._collate_caches(sessions, "cross_cache")
- cache_collate_seconds = time.monotonic() - cache_collate_started_at
+ # Keep a singleton session's cache allocation in place. Besides
+ # avoiding a needless cat/scatter clone, this gives the optional
+ # CUDA Graph path stable KV-cache pointers across continuations.
+ direct_session_cache = len(sessions) == 1
start = sessions[0].next_latent_frame
+ batched_latent: torch.Tensor | None = None
+ batched_prompt: torch.Tensor | None = None
+ batched_action: torch.Tensor | None = None
+ if not direct_session_cache:
+ batched_latent = torch.cat(noises, dim=0).to(dtype=self.torch_dtype)
+ batched_prompt = torch.cat([session.prompt_emb for session in sessions], dim=0)
+ batched_action = torch.cat(action_contexts, dim=0)
+ input_prepare_seconds = time.monotonic() - batch_started_at
+ cache_collate_seconds = 0.0
+ original_global_ends: list[list[int]] | None = None
+ self_cache: list[dict[str, Any]] | None = None
+ cross_cache: list[dict[str, Any]] | None = None
+ graph_batched_cache = False
+ if direct_session_cache:
+ self_cache = sessions[0].self_cache
+ cross_cache = sessions[0].cross_cache
+ elif start == 0:
+ # First chunks initialize per-session caches and stay eager.
+ for session in sessions:
+ self._release_cuda_graph(session.session_id)
+ self.denoise_stage.record_cuda_graph_not_used()
+ cache_collate_started_at = time.monotonic()
+ original_global_ends = [
+ [int(layer["global_end_index"].item()) for layer in session.self_cache] for session in sessions
+ ]
+ self_cache = self._collate_caches(sessions, "self_cache")
+ cross_cache = self._collate_caches(sessions, "cross_cache")
+ cache_collate_seconds = time.monotonic() - cache_collate_started_at
# CUDA events provide stage time without treating asynchronous kernel
# launch latency as DiT runtime. The final VAE event is synchronized
# before metrics are read, while the normal stream ordering remains
@@ -250,33 +271,105 @@ def generate_next_blocks(
denoise_started.record()
else:
denoise_started_at = time.monotonic()
- latents = self.denoise_stage._denoise_block(
- torch.cat(noises, dim=0).to(dtype=self.torch_dtype),
- torch.cat([session.prompt_emb for session in sessions], dim=0),
- torch.cat(action_contexts, dim=0),
- torch.cat([session.first_frame_latent for session in sessions], dim=0) if start == 0 else None,
- self_cache,
- cross_cache,
- start,
- [session.generator for session in sessions],
- sessions[0].scheduler,
- )
+ if direct_session_cache and start > 0:
+ assert self_cache is not None and cross_cache is not None
+ latents = self.denoise_stage.denoise_interactive_block(
+ session_id=sessions[0].session_id,
+ latent=noises[0].to(dtype=self.torch_dtype),
+ prompt_emb=sessions[0].prompt_emb,
+ action_context=action_contexts[0],
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ current_start=start,
+ generator=sessions[0].generator,
+ scheduler=sessions[0].scheduler,
+ )
+ elif start > 0:
+ assert batched_latent is not None and batched_prompt is not None and batched_action is not None
+ latents = self.denoise_stage.denoise_interactive_blocks(
+ session_ids=[session.session_id for session in sessions],
+ latent=batched_latent,
+ prompt_emb=batched_prompt,
+ action_context=batched_action,
+ self_caches=[session.self_cache for session in sessions],
+ cross_caches=[session.cross_cache for session in sessions],
+ current_starts=[session.next_latent_frame for session in sessions],
+ generators=[session.generator for session in sessions],
+ scheduler=sessions[0].scheduler,
+ )
+ if latents is None:
+ # Generic eager collate/scatter replaces cache tensors;
+ # invalidate any graph using those pointers first.
+ for session in sessions:
+ self._release_cuda_graph(session.session_id)
+ cache_collate_started_at = time.monotonic()
+ original_global_ends = [
+ [int(layer["global_end_index"].item()) for layer in session.self_cache] for session in sessions
+ ]
+ self_cache = self._collate_caches(sessions, "self_cache")
+ cross_cache = self._collate_caches(sessions, "cross_cache")
+ cache_collate_seconds = time.monotonic() - cache_collate_started_at
+ latents = self.denoise_stage._denoise_block(
+ batched_latent,
+ batched_prompt,
+ batched_action,
+ None,
+ self_cache,
+ cross_cache,
+ start,
+ [session.generator for session in sessions],
+ sessions[0].scheduler,
+ )
+ else:
+ graph_batched_cache = True
+ else:
+ # The singleton first chunk is dynamic as well; its cache is
+ # deliberately not eligible until the full local window exists.
+ assert self_cache is not None and cross_cache is not None
+ if direct_session_cache:
+ self.denoise_stage.record_cuda_graph_not_used()
+ latents = self.denoise_stage._denoise_block(
+ noises[0].to(dtype=self.torch_dtype),
+ sessions[0].prompt_emb,
+ action_contexts[0],
+ sessions[0].first_frame_latent,
+ self_cache,
+ cross_cache,
+ start,
+ sessions[0].generator,
+ sessions[0].scheduler,
+ )
+ else:
+ assert batched_latent is not None and batched_prompt is not None and batched_action is not None
+ latents = self.denoise_stage._denoise_block(
+ batched_latent,
+ batched_prompt,
+ batched_action,
+ torch.cat([session.first_frame_latent for session in sessions], dim=0),
+ self_cache,
+ cross_cache,
+ start,
+ [session.generator for session in sessions],
+ sessions[0].scheduler,
+ )
if use_cuda_events:
denoise_finished.record()
else:
denoise_seconds = time.monotonic() - denoise_started_at
- global_deltas = [
- int(layer["global_end_index"].item()) - original_global_ends[0][layer_index]
- for layer_index, layer in enumerate(self_cache)
- ]
cache_scatter_started_at = time.monotonic()
- self._scatter_caches(sessions, "self_cache", self_cache)
- for session_index, session in enumerate(sessions):
- for layer_index, delta in enumerate(global_deltas):
- session.self_cache[layer_index]["global_end_index"].fill_(
- original_global_ends[session_index][layer_index] + delta
- )
- self._scatter_caches(sessions, "cross_cache", cross_cache)
+ if not direct_session_cache and not graph_batched_cache:
+ assert self_cache is not None and cross_cache is not None and original_global_ends is not None
+ global_deltas = [
+ int(layer["global_end_index"].item()) - original_global_ends[0][layer_index]
+ for layer_index, layer in enumerate(self_cache)
+ ]
+ self._scatter_caches(sessions, "self_cache", self_cache)
+ for session_index, session in enumerate(sessions):
+ for layer_index, delta in enumerate(global_deltas):
+ session.self_cache[layer_index]["global_end_index"].fill_(
+ original_global_ends[session_index][layer_index] + delta
+ )
+ self._scatter_caches(sessions, "cross_cache", cross_cache)
cache_scatter_seconds = time.monotonic() - cache_scatter_started_at
if use_cuda_events:
vae_started.record()
@@ -310,6 +403,7 @@ def generate_next_blocks(
"cache_scatter_seconds": cache_scatter_seconds,
"vae_decode_seconds": decode_seconds,
**self.taew_decode_stage.last_decode_metrics(),
+ **self.denoise_stage.last_cuda_graph_metrics(),
"postprocess_seconds": time.monotonic() - postprocess_started_at,
"total_seconds": time.monotonic() - batch_started_at,
}
@@ -367,6 +461,7 @@ def snapshot_interactive_session(
"""Clone a quiescent session to CPU for suspend or cross-worker migration."""
with self._execution_lock, session.lock:
self._require_session(session)
+ self._release_cuda_graph(session.session_id)
session.lifecycle = ABotWorldSessionLifecycle.MIGRATING
return ABotWorldSessionSnapshot(
session_id=session.session_id,
@@ -429,6 +524,7 @@ def _restore_snapshot(
direct_device_tensors: bool,
) -> ABotWorldInteractiveSession:
with self._execution_lock:
+ self._release_cuda_graph(snapshot.session_id)
generator = torch.Generator(device=self.device)
generator.set_state(snapshot.generator_state)
if direct_device_tensors:
@@ -525,10 +621,17 @@ def last_stage_metrics(self) -> dict[str, float | int]:
with self._execution_lock:
return dict(self._last_stage_metrics)
+ def _release_cuda_graph(self, session_id: str) -> None:
+ """Drop optional graph state without coupling test/minimal stages to it."""
+ release = getattr(self.denoise_stage, "release_cuda_graph", None)
+ if callable(release):
+ release(session_id)
+
def suspend_interactive_session(self, session: ABotWorldInteractiveSession) -> None:
"""Move all material session tensors to CPU at a chunk boundary."""
with self._execution_lock, session.lock:
self._require_session(session)
+ self._release_cuda_graph(session.session_id)
if session.lifecycle == ABotWorldSessionLifecycle.SUSPENDED:
return
session.prompt_emb = session.prompt_emb.to("cpu")
@@ -546,6 +649,7 @@ def restore_interactive_session(self, session: ABotWorldInteractiveSession) -> N
"""Restore a suspended session to the pipeline execution device."""
with self._execution_lock, session.lock:
self._require_session(session)
+ self._release_cuda_graph(session.session_id)
if session.lifecycle != ABotWorldSessionLifecycle.SUSPENDED:
return
session.prompt_emb = session.prompt_emb.to(self.device, dtype=self.torch_dtype)
@@ -576,6 +680,7 @@ def close_interactive_session(self, session: ABotWorldInteractiveSession | None
with self._lifecycle_lock:
targets = list(self._interactive_sessions.values()) if session is None else [session]
for target in targets:
+ self._release_cuda_graph(target.session_id)
if target.closed:
continue
target.lifecycle = ABotWorldSessionLifecycle.CLOSING
diff --git a/telefuser/pipelines/abot_world/pipeline.py b/telefuser/pipelines/abot_world/pipeline.py
index 873789f0..c7431ab1 100644
--- a/telefuser/pipelines/abot_world/pipeline.py
+++ b/telefuser/pipelines/abot_world/pipeline.py
@@ -31,6 +31,10 @@ class ABotWorldPipelineConfig:
# Match LingBot-World v2: six fixed sink latents plus a twelve-latent rolling tail.
local_attn_size: int = 18
sink_size: int = 6
+ # Opt-in only: capture the fixed-shape, steady-state Relative-RoPE DiT
+ # continuation path. Dynamic first chunks, cache warmup, VAE and output
+ # postprocessing remain eager.
+ cuda_graph_enabled: bool = False
class ABotWorldPipeline(BasePipeline):
@@ -74,6 +78,7 @@ def init(self, module_manager: ModuleManager, config: ABotWorldPipelineConfig) -
)
self.denoise_stage = ABotWorldDenoisingStage("abot_world_denoise", module_manager, config.dit_config)
self.denoise_stage.parallel_models()
+ self.denoise_stage.configure_cuda_graph(config.cuda_graph_enabled)
self.denoise_stage.dit.set_causal_attention_window(config.local_attn_size, config.sink_size)
@classmethod
diff --git a/telefuser/pipelines/abot_world/service.py b/telefuser/pipelines/abot_world/service.py
index bdf7556d..a3dccfb2 100644
--- a/telefuser/pipelines/abot_world/service.py
+++ b/telefuser/pipelines/abot_world/service.py
@@ -13,7 +13,7 @@
import time
import uuid
from collections import deque
-from collections.abc import AsyncGenerator, Mapping, Sequence
+from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@@ -64,9 +64,23 @@
_DEFAULT_OUTPUT_QUEUE_SIZE = 4
_VIDEO_OUTPUT_TYPES = frozenset({"preview", "chunk"})
_TERMINAL_OUTPUT_TYPES = frozenset({"error", "done"})
-_PACING_SAFETY_FACTOR = 1.10
+_DEFAULT_BATCH_COMPUTE_SAFETY_FACTOR = 1.10
_PACING_MAX_COALESCING_SECONDS = 0.010
_PACING_RENDEZVOUS_WAKE_GUARD_SECONDS = 0.001
+_TRACE_STAGE_FIELDS = {
+ "input_prepare": "input_prepare_seconds",
+ "cache_collate": "cache_collate_seconds",
+ "denoise": "denoise_seconds",
+ "cache_scatter": "cache_scatter_seconds",
+ "vae_decode": "vae_decode_seconds",
+ "postprocess": "postprocess_seconds",
+ "total": "total_seconds",
+}
+_TAEW_DECODE_MODE_NAMES = {
+ 0: "singleton",
+ 1: "synchronized_batch",
+ 2: "serial_fallback",
+}
@dataclass
@@ -76,6 +90,7 @@ class _ABotWorldLiveKitSession:
output_queue: queue.Queue[dict[str, Any]]
control_event: threading.Event
config: dict[str, Any]
+ output_available_event: threading.Event = field(default_factory=threading.Event)
control_idle_timeout: float = 10.0
controls: set[str] = field(default_factory=set)
last_control_at: float = field(default_factory=time.monotonic)
@@ -97,6 +112,24 @@ class _ABotWorldLiveKitSession:
last_compute_seconds: float = 0.0
last_chunk_duration_seconds: float = 0.0
pacing_ready_at: float = field(default_factory=time.monotonic)
+ # Frame credit is enabled only when a real-time publisher explicitly opts
+ # in. ``output_queue`` accounts for Fq (model chunks not yet dequeued); the
+ # publisher owns Fp after dequeue and reports successful capture_frame()
+ # calls back here. Keeping the two quantities separate lets the scheduler
+ # make EDF decisions from actual playout slack instead of a virtual chunk
+ # deadline alone.
+ publisher_frame_tracking_enabled: bool = False
+ publisher_unsubmitted_frames: int = 0
+ publisher_progress_sequence: int = -1
+ publisher_progress_updated_at: float | None = None
+ publisher_frames_submitted: int = 0
+ publisher_frames_abandoned: int = 0
+ # A continuation may be held briefly for a compatible peer. This is
+ # deliberately per-session so scheduler wakeups caused by unrelated
+ # controls cannot repeatedly restart the same batching timeout.
+ deadline_batch_wait_until: float | None = None
+ deadline_batch_wait_started_at: float | None = None
+ deadline_batch_force_singleton: bool = False
last_error: str | None = None
migrating: bool = False
@@ -135,6 +168,15 @@ def __init__(
close_timeout: float = 300.0,
max_batch_size: int = 8,
batching_window_ms: float = 2.0,
+ max_deadline_batch_wait_ms: float = 0.0,
+ batch_compute_prior_seconds: Mapping[int, float] | None = None,
+ batch_compute_profile_name: str = "none",
+ batch_compute_safety_factor: float = _DEFAULT_BATCH_COMPUTE_SAFETY_FACTOR,
+ publisher_frame_credit_enabled: bool = False,
+ publisher_frame_credit_target_seconds: float = 3.0,
+ publisher_frame_credit_target_frames: int | None = None,
+ publisher_frame_credit_reserve_frames: int = 4,
+ publisher_frame_credit_guard_ms: float = 50.0,
idle_suspension_seconds: float = 5.0,
scheduler_mode: str = "batched",
) -> None:
@@ -146,18 +188,82 @@ def __init__(
raise ValueError("control_idle_timeout and close_timeout must be positive")
if max_batch_size < 1:
raise ValueError("max_batch_size must be positive")
- if batching_window_ms < 0 or idle_suspension_seconds <= 0:
- raise ValueError("batching_window_ms must be non-negative and idle_suspension_seconds positive")
+ if publisher_frame_credit_target_frames is not None and (
+ isinstance(publisher_frame_credit_target_frames, bool)
+ or not isinstance(publisher_frame_credit_target_frames, int)
+ or publisher_frame_credit_target_frames <= 0
+ ):
+ raise ValueError("publisher_frame_credit_target_frames must be a positive integer or None")
+ if (
+ not math.isfinite(batching_window_ms)
+ or batching_window_ms < 0
+ or not math.isfinite(max_deadline_batch_wait_ms)
+ or max_deadline_batch_wait_ms < 0
+ or not math.isfinite(publisher_frame_credit_target_seconds)
+ or publisher_frame_credit_target_seconds <= 0
+ or publisher_frame_credit_reserve_frames < 0
+ or not math.isfinite(publisher_frame_credit_guard_ms)
+ or publisher_frame_credit_guard_ms < 0
+ or idle_suspension_seconds <= 0
+ ):
+ raise ValueError(
+ "batching_window_ms, max_deadline_batch_wait_ms, and publisher_frame_credit_guard_ms must be "
+ "non-negative finite values; publisher_frame_credit_target_seconds and "
+ "idle_suspension_seconds must be positive; publisher_frame_credit_reserve_frames must be "
+ "non-negative"
+ )
if scheduler_mode not in {"round_robin", "batched"}:
raise ValueError("scheduler_mode must be 'round_robin' or 'batched'")
+ normalized_batch_compute_priors: dict[int, float] = {}
+ for raw_batch_size, raw_seconds in (batch_compute_prior_seconds or {}).items():
+ if isinstance(raw_batch_size, bool) or not isinstance(raw_batch_size, int) or raw_batch_size < 1:
+ raise ValueError("batch_compute_prior_seconds keys must be positive integer batch sizes")
+ try:
+ seconds = float(raw_seconds)
+ except (TypeError, ValueError) as exc:
+ raise ValueError("batch_compute_prior_seconds values must be positive finite seconds") from exc
+ if not math.isfinite(seconds) or seconds <= 0:
+ raise ValueError("batch_compute_prior_seconds values must be positive finite seconds")
+ normalized_batch_compute_priors[raw_batch_size] = seconds
+ if not isinstance(batch_compute_profile_name, str) or not batch_compute_profile_name.strip():
+ raise ValueError("batch_compute_profile_name must be a non-empty string")
+ try:
+ normalized_batch_compute_safety_factor = float(batch_compute_safety_factor)
+ except (TypeError, ValueError) as exc:
+ raise ValueError("batch_compute_safety_factor must be a finite number greater than or equal to 1") from exc
+ if not math.isfinite(normalized_batch_compute_safety_factor) or normalized_batch_compute_safety_factor < 1.0:
+ raise ValueError("batch_compute_safety_factor must be a finite number greater than or equal to 1")
self.pipeline = pipeline
self.default_fps = int(default_fps)
self.default_session_config = dict(default_session_config or {})
self.output_queue_size = int(output_queue_size)
self.control_idle_timeout = float(control_idle_timeout)
+ # This is independent of batching_window_seconds: the latter is also
+ # an early-pacing slack, while this is a bounded timeout used to wait
+ # for an otherwise absent compatible peer.
+ self.max_deadline_batch_wait_seconds = float(max_deadline_batch_wait_ms) / 1000.0
self.close_timeout = float(close_timeout)
self.max_batch_size = int(max_batch_size)
self.batching_window_seconds = float(batching_window_ms) / 1000.0
+ # Offline profiles seed B>1 estimates before the first coalesced
+ # dispatch. Without this, the generic fallback assumes B2 is two B1
+ # calls and can make a deadline-aware rendezvous unable to bootstrap.
+ # Values are raw measured wall seconds; the per-service safety factor is
+ # applied only in _estimated_batch_compute_seconds.
+ self.batch_compute_profile_name = batch_compute_profile_name.strip()
+ self._batch_compute_priors = normalized_batch_compute_priors
+ self.batch_compute_safety_factor = normalized_batch_compute_safety_factor
+ # This is an opt-in experimental policy. Existing generic
+ # ``pull_chunks`` consumers do not report capture progress and should
+ # preserve the previous virtual-pacing behavior until a LiveKit
+ # publisher explicitly enables tracking for its session.
+ self.publisher_frame_credit_enabled = bool(publisher_frame_credit_enabled)
+ self.publisher_frame_credit_target_seconds = float(publisher_frame_credit_target_seconds)
+ self.publisher_frame_credit_target_frames = (
+ None if publisher_frame_credit_target_frames is None else int(publisher_frame_credit_target_frames)
+ )
+ self.publisher_frame_credit_reserve_frames = int(publisher_frame_credit_reserve_frames)
+ self.publisher_frame_credit_guard_seconds = float(publisher_frame_credit_guard_ms) / 1000.0
self.idle_suspension_seconds = float(idle_suspension_seconds)
self.scheduler_mode = scheduler_mode
self._sessions: dict[str, _ABotWorldLiveKitSession] = {}
@@ -172,12 +278,20 @@ def __init__(
self._batch_item_count = 0
self._maximum_batch_size = 0
self._last_stage_metrics: dict[str, float | int] = {}
+ self._deadline_batch_waits_started = 0
+ self._deadline_batch_wait_timeouts = 0
+ self._deadline_batch_filler_dispatches = 0
self._pacing_eligible_sessions = 0
self._pacing_throttled_sessions = 0
self._pacing_buffered_sessions = 0
# Observed wall-clock runtimes make deadline rendezvous conservative.
- self._batch_compute_estimates: dict[int, float] = {}
+ # Start from any selected offline profile and only move upward online.
+ self._batch_compute_estimates: dict[int, float] = dict(self._batch_compute_priors)
self._workload_detector = TurboServeWorkloadDetector()
+ # The process-NCCL child installs a small callback which forwards one
+ # record per actual model dispatch to the parent process.
+ self._dispatch_trace_callback: Callable[[dict[str, Any]], None] | None = None
+ self._dispatch_trace_sequence = 0
def start(self) -> None:
"""Preload weights and start the sole GPU scheduling thread."""
@@ -440,15 +554,25 @@ async def pull_chunks(self, session_id: str) -> AsyncGenerator[dict, None]:
if state is None:
return
while True:
- try:
- payload = await asyncio.to_thread(state.output_queue.get, True, 0.25)
- except queue.Empty:
- with state.lock:
- if not state.active:
- return
+ payload: dict[str, Any] | None = None
+ with self._scheduler_condition, state.lock:
+ try:
+ payload = state.output_queue.get_nowait()
+ except queue.Empty:
+ state.output_available_event.clear()
+ active = state.active
+ else:
+ if state.output_queue.empty():
+ state.output_available_event.clear()
+ if state.publisher_frame_tracking_enabled and payload.get("type") == "chunk":
+ state.publisher_unsubmitted_frames += self._payload_frame_count(payload)
+ active = True
+ self._scheduler_condition.notify_all()
+ if payload is None:
+ if not active:
+ return
+ await asyncio.to_thread(state.output_available_event.wait, 0.25)
continue
- with self._scheduler_condition:
- self._scheduler_condition.notify_all()
yield payload
def close_session(self, session_id: str, timeout: float | None = None) -> None:
@@ -463,6 +587,7 @@ def close_session(self, session_id: str, timeout: float | None = None) -> None:
state.active = False
state.controls.clear()
state.ready_since = None
+ state.output_available_event.set()
self._scheduler_condition.notify_all()
while state.in_flight:
remaining = deadline - time.monotonic()
@@ -606,7 +731,11 @@ def _quiesce_migration(self, session_id: str, timeout: float | None) -> _ABotWor
raise KeyError(f"Unknown ABot session {session_id!r}")
state.migrating = True
self._scheduler_condition.notify_all()
- while state.in_flight or not state.output_queue.empty():
+ while (
+ state.in_flight
+ or not state.output_queue.empty()
+ or (state.publisher_frame_tracking_enabled and state.publisher_unsubmitted_frames > 0)
+ ):
remaining = deadline - time.monotonic()
if remaining <= 0:
state.migrating = False
@@ -665,7 +794,13 @@ def commit_migration(self, session_id: str) -> None:
if not state.migrating or state.in_flight:
raise RuntimeError("ABot source session is not quiescent for migration commit")
self._sessions.pop(session_id)
+ # Wake an existing pull_chunks() generator. The router will then
+ # observe its source iterator close and continue on the target.
+ state.active = False
+ state.controls.clear()
+ state.output_available_event.set()
self._discard_from_round_robin(session_id)
+ self._scheduler_condition.notify_all()
self.pipeline.close_interactive_session(state.pipeline_session)
def abort_migration(self, session_id: str) -> None:
@@ -700,7 +835,7 @@ def resume_scheduler(self) -> None:
self._scheduler_paused = False
self._scheduler_condition.notify_all()
- def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | int]:
+ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | int | str]:
"""Return raw scheduler facts for service metadata and benchmarks."""
with self._sessions_lock:
if session_id is None:
@@ -711,6 +846,24 @@ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | in
"batches": self._batch_count,
"batch_items": self._batch_item_count,
"maximum_batch_size": self._maximum_batch_size,
+ "batch_compute_profile_name": self.batch_compute_profile_name,
+ "batch_compute_prior_2_seconds": round(self._batch_compute_priors.get(2, 0.0), 6),
+ "batch_compute_prior_3_seconds": round(self._batch_compute_priors.get(3, 0.0), 6),
+ "batch_compute_prior_4_seconds": round(self._batch_compute_priors.get(4, 0.0), 6),
+ "batch_compute_safety_factor": round(self.batch_compute_safety_factor, 6),
+ "max_deadline_batch_wait_seconds": round(self.max_deadline_batch_wait_seconds, 6),
+ "deadline_batch_waits_started": self._deadline_batch_waits_started,
+ "deadline_batch_wait_timeouts": self._deadline_batch_wait_timeouts,
+ "deadline_batch_filler_dispatches": self._deadline_batch_filler_dispatches,
+ "publisher_frame_credit_enabled": int(self.publisher_frame_credit_enabled),
+ "publisher_frame_credit_target_seconds": round(self.publisher_frame_credit_target_seconds, 6),
+ "publisher_frame_credit_target_frames": (
+ self.publisher_frame_credit_target_frames
+ if self.publisher_frame_credit_target_frames is not None
+ else 0
+ ),
+ "publisher_frame_credit_reserve_frames": self.publisher_frame_credit_reserve_frames,
+ "publisher_frame_credit_guard_seconds": round(self.publisher_frame_credit_guard_seconds, 6),
"pacing_eligible_sessions": self._pacing_eligible_sessions,
"pacing_throttled_sessions": self._pacing_throttled_sessions,
"pacing_buffered_sessions": self._pacing_buffered_sessions,
@@ -723,6 +876,9 @@ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | in
}
state = self._sessions[session_id]
with state.lock:
+ now = time.monotonic()
+ queued_video_frames = self._queued_video_frames(state)
+ frame_credit_frames = queued_video_frames + state.publisher_unsubmitted_frames
return {
"scheduler_mode": self.scheduler_mode,
"scheduled_chunks": state.scheduled_chunks,
@@ -737,9 +893,151 @@ def runtime_metrics(self, session_id: str | None = None) -> dict[str, float | in
"total_queue_wait_seconds": round(state.total_queue_wait_seconds, 6),
"total_compute_seconds": round(state.total_compute_seconds, 6),
"pacing_ready_in_seconds": round(max(0.0, state.pacing_ready_at - time.monotonic()), 6),
+ "deadline_batch_wait_remaining_seconds": round(
+ max(0.0, (state.deadline_batch_wait_until or 0.0) - time.monotonic()), 6
+ ),
"pacing_buffered_video_payloads": self._queued_video_payloads(state),
+ "publisher_frame_tracking_enabled": int(state.publisher_frame_tracking_enabled),
+ "queued_video_frames": queued_video_frames,
+ "frame_credit_target_frames": self._frame_credit_target_frames(state),
+ "publisher_unsubmitted_frames": state.publisher_unsubmitted_frames,
+ "frame_credit_frames": frame_credit_frames,
+ "frame_credit_seconds": round(frame_credit_frames / max(1, int(state.config["fps"])), 6),
+ "frame_credit_deadline_in_seconds": round(self._session_deadline(state, now) - now, 6),
+ "publisher_progress_sequence": state.publisher_progress_sequence,
+ "publisher_frames_submitted": state.publisher_frames_submitted,
+ "publisher_frames_abandoned": state.publisher_frames_abandoned,
}
+ def set_dispatch_trace_callback(self, callback: Callable[[dict[str, Any]], None] | None) -> None:
+ """Install an optional sink for one record per model batch dispatch."""
+ self._dispatch_trace_callback = callback
+
+ @staticmethod
+ def _trace_number(value: object) -> int | float | None:
+ """Convert a scalar metric or scalar tensor into JSON-safe telemetry."""
+ if isinstance(value, torch.Tensor):
+ if value.numel() != 1:
+ return None
+ value = value.item()
+ if isinstance(value, bool):
+ return int(value)
+ if isinstance(value, int):
+ return int(value)
+ if isinstance(value, float):
+ return float(value) if math.isfinite(value) else None
+ return None
+
+ def _new_dispatch_session_trace(
+ self,
+ state: _ABotWorldLiveKitSession,
+ controls: Mapping[str, bool],
+ *,
+ selected_at: float,
+ ) -> dict[str, Any]:
+ """Capture one session's position before a model invocation mutates it."""
+ with state.lock:
+ session = state.pipeline_session
+ ready_at = state.ready_since or selected_at
+ queued_video_frames = self._queued_video_frames(state)
+ frame_credit_frames = queued_video_frames + state.publisher_unsubmitted_frames
+ frame_credit_deadline = self._session_deadline(state, selected_at)
+ return {
+ "session_id": str(state.session_id),
+ "chunk_index": int(state.next_chunk_index),
+ "next_latent_frame_before": self._trace_number(getattr(session, "next_latent_frame", None)),
+ "next_latent_frame_after": None,
+ "emitted_frames_before": self._trace_number(getattr(session, "emitted_frames", None)),
+ "emitted_frames_after": None,
+ "frames": None,
+ "controls": sorted(str(key) for key, enabled in controls.items() if enabled),
+ "queue_wait_seconds": max(0.0, selected_at - ready_at),
+ "frame_credit_enabled": int(self._uses_publisher_frame_credit(state)),
+ "queued_video_frames": queued_video_frames,
+ "frame_credit_target_frames": self._frame_credit_target_frames(state),
+ "publisher_unsubmitted_frames": state.publisher_unsubmitted_frames,
+ "frame_credit_frames": frame_credit_frames,
+ "frame_credit_deadline_in_seconds": frame_credit_deadline - selected_at,
+ }
+
+ def _finish_dispatch_session_trace(
+ self,
+ trace: dict[str, Any],
+ state: _ABotWorldLiveKitSession,
+ frames: Sequence[Image.Image],
+ ) -> None:
+ """Fill output facts after the corresponding session's chunk commits."""
+ session = state.pipeline_session
+ trace.update(
+ {
+ "next_latent_frame_after": self._trace_number(getattr(session, "next_latent_frame", None)),
+ "emitted_frames_after": self._trace_number(getattr(session, "emitted_frames", None)),
+ "frames": int(len(frames)),
+ }
+ )
+
+ def _emit_dispatch_trace(
+ self,
+ *,
+ selected_at: float,
+ selected_wall_time: float,
+ model_started_at: float | None,
+ model_started_wall_time: float | None,
+ completed_at: float,
+ completed_wall_time: float,
+ session_traces: Sequence[dict[str, Any]],
+ control_latent_frames: int | None,
+ stage_metrics: Mapping[str, object],
+ outcome: str,
+ error: str | None = None,
+ ) -> None:
+ """Forward an audit record without ever changing serving behavior."""
+ callback = getattr(self, "_dispatch_trace_callback", None)
+ if not callable(callback):
+ return
+ sequence = int(getattr(self, "_dispatch_trace_sequence", 0)) + 1
+ self._dispatch_trace_sequence = sequence
+ mode_value = self._trace_number(stage_metrics.get("taew_decode_mode"))
+ mode = int(mode_value) if mode_value is not None else None
+ record = {
+ "schema_version": 1,
+ "event_type": "model_dispatch",
+ "trace_sequence": sequence,
+ "scheduler_mode": self.scheduler_mode,
+ "selected_monotonic_seconds": selected_at,
+ "selected_unix_seconds": selected_wall_time,
+ "model_started_monotonic_seconds": model_started_at,
+ "model_started_unix_seconds": model_started_wall_time,
+ "model_completed_monotonic_seconds": completed_at,
+ "model_completed_unix_seconds": completed_wall_time,
+ "model_duration_seconds": (
+ max(0.0, completed_at - model_started_at) if model_started_at is not None else None
+ ),
+ "pre_model_overhead_seconds": (
+ max(0.0, model_started_at - selected_at) if model_started_at is not None else None
+ ),
+ "batch_size": len(session_traces),
+ "control_latent_frames": control_latent_frames,
+ "sessions": [dict(trace) for trace in session_traces],
+ "stages_seconds": {
+ name: self._trace_number(stage_metrics.get(metric_name))
+ for name, metric_name in _TRACE_STAGE_FIELDS.items()
+ },
+ "vae_decode": {
+ "mode": mode,
+ "mode_name": _TAEW_DECODE_MODE_NAMES.get(mode) if mode is not None else None,
+ "items": self._trace_number(stage_metrics.get("taew_decode_items")),
+ "effective_batch_size": self._trace_number(stage_metrics.get("taew_decode_batch_size")),
+ "invocations": self._trace_number(stage_metrics.get("taew_decode_invocations")),
+ },
+ "outcome": outcome,
+ "error": error,
+ }
+ try:
+ callback(record)
+ except Exception:
+ logger.exception("ABot dispatch-trace callback failed")
+
def _ensure_scheduler_started(self) -> None:
with self._scheduler_condition:
if self._scheduler_thread is not None and self._scheduler_thread.is_alive():
@@ -768,6 +1066,8 @@ def _scheduler_loop(self) -> None:
if state.controls and now - state.last_control_at >= state.control_idle_timeout:
state.controls.clear()
state.ready_since = None
+ self._clear_deadline_batch_wait(state)
+ state.deadline_batch_force_singleton = False
state.pipeline_session.lifecycle = ABotWorldSessionLifecycle.IDLE
if (
not state.controls
@@ -783,8 +1083,15 @@ def _scheduler_loop(self) -> None:
continue
if self.scheduler_mode == "batched" and ready and len(ready) < self.max_batch_size:
wait_seconds = self._batch_formation_wait_seconds(ready, now)
+ deadline_wait_active = any(state.deadline_batch_wait_until is not None for state in ready)
if wait_seconds > 0:
self._scheduler_condition.wait(timeout=wait_seconds)
+ if deadline_wait_active:
+ # A new control or peer readiness wakes this
+ # condition. Re-evaluate from the EDF head instead
+ # of dispatching the old singleton and losing its
+ # persistent dynamic-batching hold.
+ continue
now = time.monotonic()
ready = self._ready_sessions(now)
batch = self._select_batch(ready, now=now)
@@ -792,6 +1099,8 @@ def _scheduler_loop(self) -> None:
if batch:
for state in batch:
with state.lock:
+ self._clear_deadline_batch_wait(state)
+ state.deadline_batch_force_singleton = False
state.in_flight = True
controls.append({key: True for key in state.controls})
@@ -811,9 +1120,7 @@ def _ready_sessions(self, now: float) -> list[_ABotWorldLiveKitSession]:
pacing_buffered = 0
for state in self._sessions.values():
with state.lock:
- lossless_blocked = (
- state.config["delivery_mode"] == "lossless" and state.output_queue.full()
- )
+ lossless_blocked = state.config["delivery_mode"] == "lossless" and state.output_queue.full()
if (
state.active
and state.controls
@@ -824,8 +1131,12 @@ def _ready_sessions(self, now: float) -> list[_ABotWorldLiveKitSession]:
if state.ready_since is None:
state.ready_since = now
if state.config["delivery_mode"] == "latest":
- buffered_video_payloads = self._queued_video_payloads(state)
- pacing_ready = now + self._pacing_coalescing_slack_seconds(state) >= state.pacing_ready_at
+ frame_credit_enabled = self._uses_publisher_frame_credit(state)
+ buffered_video_payloads = 0 if frame_credit_enabled else self._queued_video_payloads(state)
+ pacing_ready_at = (
+ self._frame_credit_ready_at(state, now) if frame_credit_enabled else state.pacing_ready_at
+ )
+ pacing_ready = now + self._pacing_coalescing_slack_seconds(state) >= pacing_ready_at
if buffered_video_payloads or not pacing_ready:
pacing_throttled += 1
pacing_buffered += int(bool(buffered_video_payloads))
@@ -835,7 +1146,7 @@ def _ready_sessions(self, now: float) -> list[_ABotWorldLiveKitSession]:
self._pacing_eligible_sessions = pacing_eligible
self._pacing_throttled_sessions = pacing_throttled
self._pacing_buffered_sessions = pacing_buffered
- ready.sort(key=lambda state: (state.next_playout_deadline, state.ready_since or now, state.session_id))
+ ready.sort(key=lambda state: (self._session_deadline(state, now), state.ready_since or now, state.session_id))
return ready
def _next_scheduler_wake_seconds(self, now: float) -> float:
@@ -850,9 +1161,13 @@ def _next_scheduler_wake_seconds(self, now: float) -> float:
state.config["delivery_mode"] == "latest"
and not state.in_flight
and not state.migrating
- and not self._queued_video_payloads(state)
+ and (self._uses_publisher_frame_credit(state) or not self._queued_video_payloads(state))
):
- pacing_wake = state.pacing_ready_at - self._pacing_coalescing_slack_seconds(state)
+ pacing_wake = (
+ self._frame_credit_ready_at(state, now)
+ if self._uses_publisher_frame_credit(state)
+ else state.pacing_ready_at - self._pacing_coalescing_slack_seconds(state)
+ )
next_wake_at = pacing_wake if next_wake_at is None else min(next_wake_at, pacing_wake)
elif not state.in_flight and state.pipeline_session.is_resident and not state.controls:
suspension_at = state.last_control_at + self.idle_suspension_seconds
@@ -870,6 +1185,116 @@ def _pacing_coalescing_slack_seconds(self, state: _ABotWorldLiveKitSession) -> f
min(_PACING_MAX_COALESCING_SECONDS, state.last_chunk_duration_seconds * 0.05),
)
+ def _clear_deadline_batch_wait(
+ self,
+ state: _ABotWorldLiveKitSession,
+ *,
+ timed_out: bool = False,
+ ) -> None:
+ """Clear an in-progress deadline-aware batching hold.
+
+ The scheduler owns these transient fields. Callers hold ``state.lock``
+ whenever the service is running concurrently.
+ """
+ if state.deadline_batch_wait_until is None:
+ return
+ if timed_out:
+ self._deadline_batch_wait_timeouts += 1
+ state.deadline_batch_wait_until = None
+ state.deadline_batch_wait_started_at = None
+
+ def _deadline_batch_wait_seconds(
+ self,
+ state: _ABotWorldLiveKitSession,
+ now: float,
+ ) -> tuple[float, bool]:
+ """Return remaining peer-wait budget and whether it just expired.
+
+ A timeout is created once for the EDF head rather than on every
+ scheduler wakeup. The hypothetical second item gives a conservative
+ B=2 latest-start boundary even before a peer is known.
+ """
+ if (
+ self.max_deadline_batch_wait_seconds <= 0
+ or self.max_batch_size < 2
+ or state.scheduled_chunks == 0
+ or state.config["delivery_mode"] != "latest"
+ or (not self._uses_publisher_frame_credit(state) and self._queued_video_payloads(state))
+ ):
+ return 0.0, False
+
+ if state.deadline_batch_wait_until is not None:
+ remaining = state.deadline_batch_wait_until - now
+ if remaining <= 0:
+ self._clear_deadline_batch_wait(state, timed_out=True)
+ return 0.0, True
+ return remaining, False
+
+ latest_start = self._session_deadline(state, now) - self._estimated_batch_compute_seconds((state, state))
+ wait_until = min(now + self.max_deadline_batch_wait_seconds, latest_start)
+ if wait_until <= now:
+ return 0.0, False
+ state.deadline_batch_wait_started_at = now
+ state.deadline_batch_wait_until = wait_until
+ self._deadline_batch_waits_started += 1
+ return wait_until - now, False
+
+ def _active_deadline_batch_waiter(
+ self,
+ ready: Sequence[_ABotWorldLiveKitSession],
+ now: float,
+ ) -> _ABotWorldLiveKitSession | None:
+ """Return the held ready session with the earliest playout deadline."""
+ waiting: list[_ABotWorldLiveKitSession] = []
+ for state in ready:
+ with state.lock:
+ if (
+ state.deadline_batch_wait_until is not None
+ and state.deadline_batch_wait_until > now
+ and not state.deadline_batch_force_singleton
+ ):
+ waiting.append(state)
+ if not waiting:
+ return None
+ return min(
+ waiting,
+ key=lambda state: (
+ self._session_deadline(state, now),
+ state.deadline_batch_wait_until or float("inf"),
+ state.session_id,
+ ),
+ )
+
+ def _can_dispatch_before_waiter(
+ self,
+ batch: Sequence[_ABotWorldLiveKitSession],
+ waiter: _ABotWorldLiveKitSession,
+ now: float,
+ ) -> bool:
+ """Whether an EDF batch may run before a held singleton safely.
+
+ This reserves enough time for the held session to fall back to B=1.
+ First chunks and lossless work remain latency/consumer critical and are
+ allowed through immediately; they are not part of the optional wait.
+ """
+ if any(state is waiter for state in batch):
+ return True
+ if any(state.scheduled_chunks == 0 or state.config["delivery_mode"] != "latest" for state in batch):
+ return True
+ batch_finish = now + self._estimated_batch_compute_seconds(batch)
+ if batch_finish > min(self._session_deadline(state, now) for state in batch):
+ return False
+ waiter_finish = batch_finish + self._estimated_batch_compute_seconds((waiter,))
+ return waiter_finish <= self._session_deadline(waiter, now)
+
+ def _has_compatible_ready_peer(
+ self,
+ state: _ABotWorldLiveKitSession,
+ ready: Sequence[_ABotWorldLiveKitSession],
+ ) -> bool:
+ pivot_key = self._batch_key(state)
+ return any(candidate is not state and self._batch_key(candidate) == pivot_key for candidate in ready)
+
def _batch_formation_wait_seconds(
self,
ready: Sequence[_ABotWorldLiveKitSession],
@@ -877,6 +1302,42 @@ def _batch_formation_wait_seconds(
) -> float:
"""Wait for a compatible continuation only while all playout deadlines are safe."""
batch = self._select_batch(ready, now=now)
+ if batch and batch[0].deadline_batch_force_singleton:
+ return 0.0
+ held_batch_waiter: _ABotWorldLiveKitSession | None = None
+ if len(batch) >= 2 and any(state.deadline_batch_wait_until is not None for state in batch):
+ # A B=1 hold may grow to B=3, but never by consuming the B=2
+ # fallback budget. Larger already-ready batches still launch now.
+ if len(batch) != 2 or self.max_batch_size < 3:
+ return 0.0
+ held_batch_waiter = next(
+ (state for state in batch if state.deadline_batch_wait_until is not None),
+ None,
+ )
+ if held_batch_waiter is None:
+ return 0.0
+
+ # Preserve the original configured cap, then tighten it to the
+ # latest safe B=2 start for both ready members.
+ b2_latest_safe_start = self._latest_safe_batch_start(batch, now=now)
+ with held_batch_waiter.lock:
+ wait_until = held_batch_waiter.deadline_batch_wait_until
+ if wait_until is None:
+ return 0.0
+ held_batch_waiter.deadline_batch_wait_until = min(wait_until, b2_latest_safe_start)
+ if held_batch_waiter.deadline_batch_wait_until <= now:
+ return 0.0
+ waiter = self._active_deadline_batch_waiter(ready, now)
+ if waiter is not None and all(state is not waiter for state in batch):
+ if self._can_dispatch_before_waiter(batch, waiter, now):
+ self._deadline_batch_filler_dispatches += 1
+ return 0.0
+ # The EDF candidate would consume the singleton fallback budget of
+ # the held request. Dispatch the held request now rather than
+ # allowing a speculative wait to become an avoidable deadline miss.
+ with waiter.lock:
+ waiter.deadline_batch_force_singleton = True
+ return 0.0
if not batch or len(batch) >= self.max_batch_size:
return 0.0
@@ -890,8 +1351,17 @@ def _batch_formation_wait_seconds(
return self.batching_window_seconds
legacy_wait = min(
self.batching_window_seconds,
- max(0.0, self._latest_safe_batch_start(batch) - now),
+ max(0.0, self._latest_safe_batch_start(batch, now=now) - now),
)
+ if len(batch) == 1 and self._has_compatible_ready_peer(batch[0], ready):
+ # A ready peer was considered but did not make a deadline-safe
+ # batch. Waiting for another peer cannot improve this EDF turn.
+ return 0.0
+ wait_state = held_batch_waiter or batch[0]
+ with wait_state.lock:
+ deadline_wait, deadline_wait_expired = self._deadline_batch_wait_seconds(wait_state, now)
+ if deadline_wait_expired:
+ return 0.0
pivot_key = self._batch_key(batch[0])
selected_ids = {state.session_id for state in batch}
@@ -910,41 +1380,59 @@ def _batch_formation_wait_seconds(
or self._batch_key(candidate) != pivot_key
# A generated chunk owned by the publisher has an external
# dequeue time, so it cannot be a rendezvous promise.
- or self._queued_video_payloads(candidate)
+ or (not self._uses_publisher_frame_credit(candidate) and self._queued_video_payloads(candidate))
):
continue
- release_at = candidate.pacing_ready_at - self._pacing_coalescing_slack_seconds(candidate)
+ release_at = (
+ self._frame_credit_ready_at(candidate, now)
+ if self._uses_publisher_frame_credit(candidate)
+ else candidate.pacing_ready_at - self._pacing_coalescing_slack_seconds(candidate)
+ )
if release_at <= now:
# An already-eligible session should be in ready. Avoid
# turning a state race into an extra scheduler delay.
continue
proposed_batch = [*batch, candidate]
- latest_safe_start = self._latest_safe_batch_start(proposed_batch)
- if release_at + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS <= latest_safe_start:
+ latest_safe_start = self._latest_safe_batch_start(proposed_batch, now=now)
+ if release_at + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS <= latest_safe_start and (
+ held_batch_waiter is None
+ or release_at - now + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS <= deadline_wait
+ ):
rendezvous_waits.append(release_at - now + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS)
+ if held_batch_waiter is not None:
+ # A held B=2 may only wait for a specifically viable third peer;
+ # otherwise launch the safe pair without consuming more slack.
+ if deadline_wait <= 0 or not rendezvous_waits:
+ return 0.0
+ return min(deadline_wait, *rendezvous_waits)
+ if deadline_wait > 0:
+ # The timeout is persistent across condition wakeups. A known peer
+ # may wake us sooner, but never extends the caller's max wait.
+ return min(deadline_wait, *rendezvous_waits) if rendezvous_waits else deadline_wait
if rendezvous_waits:
# Earliest compatible release minimizes queueing; the condition
# wait is followed by a full readiness/deadline revalidation.
return min(rendezvous_waits)
return legacy_wait
- def _latest_safe_batch_start(self, batch: Sequence[_ABotWorldLiveKitSession]) -> float:
+ def _latest_safe_batch_start(self, batch: Sequence[_ABotWorldLiveKitSession], *, now: float | None = None) -> float:
"""Return the latest launch time that still meets every playout deadline."""
if not batch:
return float("-inf")
+ effective_now = time.monotonic() if now is None else now
predicted_compute_seconds = self._estimated_batch_compute_seconds(batch)
- return min(state.next_playout_deadline for state in batch) - predicted_compute_seconds
+ return min(self._session_deadline(state, effective_now) for state in batch) - predicted_compute_seconds
def _estimated_batch_compute_seconds(self, batch: Sequence[_ABotWorldLiveKitSession]) -> float:
- """Conservatively estimate batch wall time from observed service work."""
+ """Conservatively estimate batch wall time from offline priors or observed work."""
batch_size = len(batch)
observed = self._batch_compute_estimates.get(batch_size)
if observed is None:
singleton_seconds = max((state.last_compute_seconds for state in batch), default=0.0)
observed = singleton_seconds * batch_size
- return observed * _PACING_SAFETY_FACTOR
+ return observed * self.batch_compute_safety_factor
@staticmethod
def _queued_video_payloads(state: _ABotWorldLiveKitSession) -> int:
@@ -968,6 +1456,10 @@ def _select_batch(
return []
if self.scheduler_mode == "round_robin":
return self._select_round_robin_session(ready)
+ for state in ready:
+ with state.lock:
+ if state.deadline_batch_force_singleton:
+ return [state]
pivot_key = self._batch_key(ready[0])
batch = [state for state in ready if self._batch_key(state) == pivot_key][: self.max_batch_size]
if len(batch) <= 1:
@@ -978,14 +1470,11 @@ def _select_batch(
# consumer queues rather than a playout deadline. Only a batch made
# entirely of already-playing latest-mode sessions has a deadline that
# makes a larger batch potentially worse than two singleton turns.
- if any(
- state.scheduled_chunks == 0 or state.config["delivery_mode"] != "latest"
- for state in batch
- ):
+ if any(state.scheduled_chunks == 0 or state.config["delivery_mode"] != "latest" for state in batch):
return batch
selected_at = time.monotonic() if now is None else now
- if selected_at <= self._latest_safe_batch_start(batch):
+ if selected_at <= self._latest_safe_batch_start(batch, now=selected_at):
return batch
# The earliest state owns the earliest playout deadline because ready
@@ -1017,16 +1506,37 @@ def _select_round_robin_session(
def _discard_from_round_robin(self, session_id: str) -> None:
self._round_robin_order = deque(value for value in self._round_robin_order if value != session_id)
- @staticmethod
- def _batch_key(state: _ABotWorldLiveKitSession) -> tuple[object, ...]:
+ def _uses_relative_rope(self) -> bool:
+ """Return the DiT RoPE mode, failing closed for unknown pipeline adapters.
+
+ ``generate_next_blocks`` accepts sessions with different global frame
+ cursors only for Relative-RoPE. A third-party pipeline adapter that
+ does not expose this capability is therefore treated as Absolute-RoPE
+ rather than risking an invalid mixed-position batch.
+ """
+ denoise_stage = getattr(self.pipeline, "denoise_stage", None)
+ dit = getattr(denoise_stage, "dit", None)
+ return bool(getattr(dit, "use_relative_rope", False))
+
+ def _batch_key(self, state: _ABotWorldLiveKitSession) -> tuple[object, ...]:
session = state.pipeline_session
local_end = 0
if session.self_cache:
value = session.self_cache[0]["local_end_index"]
local_end = int(value.item()) if isinstance(value, torch.Tensor) else int(value)
+ # The native batch path collates K/V rows and permits per-session
+ # ``global_end_index`` values; it restores each cursor after applying
+ # the common update delta. Relative-RoPE reindexes the retained KV
+ # window locally, so equal local layout is sufficient here. Absolute
+ # RoPE instead receives one scalar ``current_start`` for the entire
+ # model call and must retain exact global frame alignment.
+ position_key: int | None = None
+ if not self._uses_relative_rope():
+ position_key = int(session.next_latent_frame)
return (
int(state.config["control_latent_frames"]),
session.next_latent_frame == 0,
+ position_key,
local_end,
tuple(session.first_frame_latent.shape),
session.lifecycle == ABotWorldSessionLifecycle.SUSPENDED,
@@ -1037,7 +1547,19 @@ def _execute_batch(
batch: Sequence[_ABotWorldLiveKitSession],
controls: Sequence[dict[str, bool]],
) -> None:
- started_at = time.monotonic()
+ # ``selected_*`` measures the scheduler boundary. The model interval is
+ # intentionally narrower: it begins immediately before the real
+ # generate_next_block(s) call, so a timeline is a truthful GPU-work
+ # interval rather than an IPC/queue approximation.
+ selected_at = time.monotonic()
+ selected_wall_time = time.time()
+ model_started_at: float | None = None
+ model_started_wall_time: float | None = None
+ control_latent_frames: int | None = None
+ session_traces = [
+ self._new_dispatch_session_trace(state, applied_controls, selected_at=selected_at)
+ for state, applied_controls in zip(batch, controls)
+ ]
try:
for state in batch:
if not state.pipeline_session.is_resident:
@@ -1045,21 +1567,39 @@ def _execute_batch(
frame_counts = {int(state.config["control_latent_frames"]) for state in batch}
if len(frame_counts) != 1:
raise RuntimeError("ABot scheduler selected an incompatible latent-frame batch")
+ control_latent_frames = next(iter(frame_counts))
+ model_started_at = time.monotonic()
+ model_started_wall_time = time.time()
if len(batch) == 1:
results = [
self.pipeline.generate_next_block(
batch[0].pipeline_session,
controls[0],
- control_latent_frames=frame_counts.pop(),
+ control_latent_frames=control_latent_frames,
)
]
else:
results = self.pipeline.generate_next_blocks(
[state.pipeline_session for state in batch],
list(controls),
- control_latent_frames=frame_counts.pop(),
+ control_latent_frames=control_latent_frames,
)
except Exception as exc:
+ completed_at = time.monotonic()
+ completed_wall_time = time.time()
+ self._emit_dispatch_trace(
+ selected_at=selected_at,
+ selected_wall_time=selected_wall_time,
+ model_started_at=model_started_at,
+ model_started_wall_time=model_started_wall_time,
+ completed_at=completed_at,
+ completed_wall_time=completed_wall_time,
+ session_traces=session_traces,
+ control_latent_frames=control_latent_frames,
+ stage_metrics={},
+ outcome="error",
+ error=repr(exc),
+ )
logger.exception(
"ABot TurboServe batch generation failed: sessions=%s",
[item.session_id for item in batch],
@@ -1072,9 +1612,10 @@ def _execute_batch(
self._put_output(state, {"type": "error", "error": str(exc), "timestamp": time.time()})
else:
completed_at = time.monotonic()
+ completed_wall_time = time.time()
stage_metrics_callback = getattr(self.pipeline, "last_stage_metrics", None)
self._last_stage_metrics = dict(stage_metrics_callback()) if callable(stage_metrics_callback) else {}
- observed_compute_seconds = completed_at - started_at
+ observed_compute_seconds = completed_at - selected_at
previous_estimate = self._batch_compute_estimates.get(len(batch), 0.0)
# Keep a service-run high-water mark so a transient fast batch
# cannot make a later rendezvous overrun a playout deadline.
@@ -1086,11 +1627,11 @@ def _execute_batch(
self._batch_count += 1
self._batch_item_count += len(batch)
self._maximum_batch_size = max(self._maximum_batch_size, len(batch))
- for state, frames, applied_controls in zip(batch, results, controls):
+ for trace, state, frames, applied_controls in zip(session_traces, batch, results, controls):
with state.lock:
- queue_wait = max(0.0, started_at - (state.ready_since or started_at))
+ queue_wait = max(0.0, selected_at - (state.ready_since or selected_at))
state.total_queue_wait_seconds += queue_wait
- state.total_compute_seconds += completed_at - started_at
+ state.total_compute_seconds += completed_at - selected_at
state.scheduled_chunks += 1
state.batch_items += len(batch)
payload = {
@@ -1103,7 +1644,7 @@ def _execute_batch(
"scheduler": {
"batch_size": len(batch),
"queue_wait_seconds": round(queue_wait, 6),
- "compute_seconds": round(completed_at - started_at, 6),
+ "compute_seconds": round(completed_at - selected_at, 6),
**self._last_stage_metrics,
},
}
@@ -1123,9 +1664,10 @@ def _execute_batch(
else:
# Start the next block early enough to meet the start of the
# sole prefetched block, but never run before this block ends.
- predicted_compute_seconds = max(
- observed_compute_seconds, previous_compute_seconds
- ) * _PACING_SAFETY_FACTOR
+ predicted_compute_seconds = (
+ max(observed_compute_seconds, previous_compute_seconds)
+ * self.batch_compute_safety_factor
+ )
next_chunk_playout_start = state.next_playout_deadline - chunk_duration_seconds
state.pacing_ready_at = max(
completed_at, next_chunk_playout_start - predicted_compute_seconds
@@ -1133,7 +1675,20 @@ def _execute_batch(
else:
state.pacing_ready_at = completed_at
state.ready_since = completed_at if state.controls else None
+ self._finish_dispatch_session_trace(trace, state, frames)
self._put_output(state, payload)
+ self._emit_dispatch_trace(
+ selected_at=selected_at,
+ selected_wall_time=selected_wall_time,
+ model_started_at=model_started_at,
+ model_started_wall_time=model_started_wall_time,
+ completed_at=completed_at,
+ completed_wall_time=completed_wall_time,
+ session_traces=session_traces,
+ control_latent_frames=control_latent_frames,
+ stage_metrics=self._last_stage_metrics,
+ outcome="ok",
+ )
finally:
with self._scheduler_condition:
for state in batch:
@@ -1149,6 +1704,7 @@ def _put_output(self, state: _ABotWorldLiveKitSession, payload: dict[str, Any])
return False
if not state.output_queue.full():
state.output_queue.put_nowait(payload)
+ state.output_available_event.set()
state.output_queue_high_watermark = max(
state.output_queue_high_watermark,
state.output_queue.qsize(),
@@ -1193,6 +1749,7 @@ def _put_output(self, state: _ABotWorldLiveKitSession, payload: dict[str, Any])
state.dropped_status_payloads += 1
return False
state.output_queue.put_nowait(payload)
+ state.output_available_event.set()
state.output_queue_high_watermark = max(state.output_queue_high_watermark, state.output_queue.qsize())
return True
@@ -1239,3 +1796,143 @@ def _canonical_control(value: object) -> str:
@classmethod
def _canonical_controls(cls, values: list[object]) -> set[str]:
return {cls._canonical_control(value) for value in values}
+
+ @staticmethod
+ def _payload_frame_count(payload: Mapping[str, object]) -> int:
+ """Return video-frame count without treating previews/status as playout."""
+
+ frames = payload.get("frames")
+ if isinstance(frames, Sequence) and not isinstance(frames, str | bytes | bytearray):
+ return len(frames)
+ return 0
+
+ @classmethod
+ def _queued_video_frames(cls, state: _ABotWorldLiveKitSession) -> int:
+ """Return Fq: generated chunk frames not yet dequeued by the publisher."""
+
+ with state.output_queue.mutex:
+ return sum(
+ cls._payload_frame_count(item)
+ for item in state.output_queue.queue
+ if isinstance(item, Mapping) and item.get("type") == "chunk"
+ )
+
+ def _uses_publisher_frame_credit(self, state: _ABotWorldLiveKitSession) -> bool:
+ """Whether this continuation has an authoritative publisher credit."""
+
+ return bool(
+ self.publisher_frame_credit_enabled
+ and state.publisher_frame_tracking_enabled
+ and state.config["delivery_mode"] == "latest"
+ and state.scheduled_chunks > 0
+ )
+
+ def _publisher_frame_credit(self, state: _ABotWorldLiveKitSession) -> int:
+ """Return Fi = Fq + Fp, frames not yet accepted by LiveKit."""
+
+ return self._queued_video_frames(state) + state.publisher_unsubmitted_frames
+
+ def _frame_credit_target_frames(self, state: _ABotWorldLiveKitSession) -> int:
+ fps = max(1, int(state.config["fps"]))
+ target_frames = self.publisher_frame_credit_target_frames
+ if target_frames is None:
+ target_frames = math.ceil(fps * self.publisher_frame_credit_target_seconds)
+ return max(self.publisher_frame_credit_reserve_frames, target_frames)
+
+ def _frame_credit_ready_at(self, state: _ABotWorldLiveKitSession, now: float) -> float:
+ """Predict when an overfilled publisher buffer reaches its low watermark."""
+
+ if not self._uses_publisher_frame_credit(state):
+ return now
+ frames_over_target = max(0, self._publisher_frame_credit(state) - self._frame_credit_target_frames(state))
+ return now + frames_over_target / max(1, int(state.config["fps"]))
+
+ def _session_deadline(self, state: _ABotWorldLiveKitSession, now: float) -> float:
+ """Return completion deadline from real publisher credit or legacy pacing."""
+
+ if not self._uses_publisher_frame_credit(state):
+ return state.next_playout_deadline
+ usable_frames = max(0, self._publisher_frame_credit(state) - self.publisher_frame_credit_reserve_frames)
+ return now + usable_frames / max(1, int(state.config["fps"])) - self.publisher_frame_credit_guard_seconds
+
+ def enable_publisher_frame_tracking(self, session_id: str) -> bool:
+ """Enable publisher handoff tracking for a real-time transport.
+
+ A generic pull consumer may never call capture_frame(). Tracking is
+ therefore explicitly enabled by the LiveKit worker. The scheduler only
+ consumes the resulting credit when ``publisher_frame_credit_enabled``
+ is set, but migration always needs the handoff state to drain safely.
+ """
+ state = self._session(session_id)
+ if state is None:
+ return False
+ with self._scheduler_condition, state.lock:
+ if not state.active:
+ return False
+ if not state.publisher_frame_tracking_enabled:
+ state.publisher_frame_tracking_enabled = True
+ state.publisher_unsubmitted_frames = 0
+ state.publisher_progress_sequence = -1
+ state.publisher_progress_updated_at = None
+ self._scheduler_condition.notify_all()
+ return True
+
+ def migration_drain_status(self, session_id: str) -> dict[str, int | bool]:
+ """Return a source-side barrier snapshot for transport-safe migration.
+
+ A caller uses this only after scheduler pause. It deliberately exposes
+ queue emptiness separately from the parent transport's own drain state;
+ together they prove that no generated payload or publisher-owned frame
+ can be copied into a migrated session.
+ """
+ state = self._session(session_id)
+ if state is None:
+ raise KeyError(f"Unknown ABot session {session_id!r}")
+ with self._scheduler_condition, state.lock:
+ return {
+ "in_flight": bool(state.in_flight),
+ "output_queue_empty": state.output_queue.empty(),
+ "publisher_unsubmitted_frames": int(state.publisher_unsubmitted_frames),
+ }
+
+ def report_publisher_frame_progress(
+ self,
+ session_id: str,
+ *,
+ event: str,
+ frames_delta: int,
+ sequence: int,
+ observed_monotonic_seconds: float | None = None,
+ ) -> bool:
+ """Apply one monotonic publisher progress update.
+
+ frames_delta is negative for both a frame accepted by LiveKit and an
+ abandoned/dropped frame. It is deliberately a delta: dequeue is
+ accounted atomically in pull_chunks(), so no empty-queue race can make
+ the scheduler believe that the publisher has no remaining video.
+ """
+ if event not in {"submitted", "dropped", "abandoned"}:
+ raise ValueError(f"Unsupported publisher progress event: {event!r}")
+ if frames_delta > 0:
+ raise ValueError("publisher progress frames_delta must be non-positive")
+ state = self._session(session_id)
+ if state is None:
+ return False
+ with self._scheduler_condition, state.lock:
+ if not state.active or not state.publisher_frame_tracking_enabled:
+ return False
+ if sequence <= state.publisher_progress_sequence:
+ return False
+ state.publisher_progress_sequence = int(sequence)
+ state.publisher_progress_updated_at = (
+ time.monotonic() if observed_monotonic_seconds is None else float(observed_monotonic_seconds)
+ )
+ previous = state.publisher_unsubmitted_frames
+ state.publisher_unsubmitted_frames = max(0, previous + int(frames_delta))
+ applied = previous - state.publisher_unsubmitted_frames
+ if event == "submitted":
+ state.publisher_frames_submitted += applied
+ else:
+ state.publisher_frames_abandoned += applied
+ self._scheduler_condition.notify_all()
+ return True
diff --git a/telefuser/service/core/stream_pipeline_service.py b/telefuser/service/core/stream_pipeline_service.py
index f898176c..ff3d8258 100644
--- a/telefuser/service/core/stream_pipeline_service.py
+++ b/telefuser/service/core/stream_pipeline_service.py
@@ -299,6 +299,40 @@ async def pull_chunks(self, session_id: str) -> AsyncGenerator[dict, None]:
async for chunk in svc.pull_chunks(session_id):
yield chunk
+ def enable_publisher_frame_tracking(self, session_id: str) -> bool:
+ """Enable optional publisher feedback on a bidirectional service."""
+
+ service = self._ensure_bidirectional()
+ enable = getattr(service, "enable_publisher_frame_tracking", None)
+ if not callable(enable):
+ return False
+ return bool(enable(session_id))
+
+ def report_publisher_frame_progress(
+ self,
+ session_id: str,
+ *,
+ event: str,
+ frames_delta: int,
+ sequence: int,
+ observed_monotonic_seconds: float,
+ ) -> bool:
+ """Forward optional publisher progress without extending the core protocol."""
+
+ service = self._ensure_bidirectional()
+ report = getattr(service, "report_publisher_frame_progress", None)
+ if not callable(report):
+ return False
+ return bool(
+ report(
+ session_id,
+ event=event,
+ frames_delta=frames_delta,
+ sequence=sequence,
+ observed_monotonic_seconds=observed_monotonic_seconds,
+ )
+ )
+
def close_session(self, session_id: str) -> None:
self._ensure_bidirectional().close_session(session_id)
diff --git a/telefuser/service/livekit/config.py b/telefuser/service/livekit/config.py
index 795440d2..d1f5228f 100644
--- a/telefuser/service/livekit/config.py
+++ b/telefuser/service/livekit/config.py
@@ -38,6 +38,17 @@ class LiveKitServeConfig(BaseSettings):
description="Worker isolation mode",
)
+ dispatch_trace_path: str | None = Field(
+ default=None,
+ description="Fresh parent-process JSONL path for bounded model-dispatch audit records",
+ )
+ dispatch_trace_max_events: int = Field(
+ default=10_000,
+ ge=1,
+ le=1_000_000,
+ description="Maximum model-dispatch records written to the optional JSONL audit trace",
+ )
+
queue_size: int = Field(default=0, ge=0, le=10000, description="Maximum queued sessions")
autoscaling_enabled: bool = Field(default=False, description="Dynamically load configured GPU workers")
autoscaling_min_workers: int = Field(default=1, ge=1, le=64)
diff --git a/telefuser/service/livekit/metrics.py b/telefuser/service/livekit/metrics.py
index b7c2199c..0480a9bd 100644
--- a/telefuser/service/livekit/metrics.py
+++ b/telefuser/service/livekit/metrics.py
@@ -263,6 +263,21 @@ def _state(self, runtime: LiveKitServeRuntime) -> dict[str, Any]:
worker_metrics = worker_metrics if isinstance(worker_metrics, dict) else {}
session_metrics = routing.get("session_runtime_metrics", {})
session_metrics = session_metrics if isinstance(session_metrics, dict) else {}
+ frame_credit_sessions = tuple(values for values in session_metrics.values() if isinstance(values, dict))
+ frame_credit = {
+ "tracked_sessions": int(
+ sum(bool(values.get("publisher_frame_tracking_enabled", 0)) for values in frame_credit_sessions)
+ ),
+ "queued_frames": sum(
+ self._nonnegative(values.get("queued_video_frames")) or 0.0 for values in frame_credit_sessions
+ ),
+ "publisher_unsubmitted_frames": sum(
+ self._nonnegative(values.get("publisher_unsubmitted_frames")) or 0.0 for values in frame_credit_sessions
+ ),
+ "total_frames": sum(
+ self._nonnegative(values.get("frame_credit_frames")) or 0.0 for values in frame_credit_sessions
+ ),
+ }
active = 0
retained = 0
@@ -315,6 +330,23 @@ def gauge(name: str, description: str, value: float | int, labels: dict[str, obj
health["queued_sessions"],
{"queue": "admission"},
)
+ for state, value in (
+ ("queued", frame_credit["queued_frames"]),
+ ("publisher", frame_credit["publisher_unsubmitted_frames"]),
+ ("total", frame_credit["total_frames"]),
+ ):
+ gauge(
+ "telefuser_serving_frame_credit_frames",
+ "Frames retained between ABot output and LiveKit capture_frame",
+ value,
+ {"state": state},
+ )
+ gauge(
+ "telefuser_serving_frame_credit_sessions",
+ "Sessions with publisher frame-credit tracking enabled",
+ frame_credit["tracked_sessions"],
+ )
+
for state, value in (
("configured", health["workers_total"]),
("busy", health["workers_busy"]),
@@ -484,6 +516,7 @@ def gauge(name: str, description: str, value: float | int, labels: dict[str, obj
},
"published_fps": fps,
"scheduler_mode": scheduler_mode,
+ "frame_credit": frame_credit,
"worker_runtime_metrics": {
str(worker_id): dict(values)
for worker_id, values in worker_metrics.items()
diff --git a/telefuser/service/livekit/nccl_process_worker_pool.py b/telefuser/service/livekit/nccl_process_worker_pool.py
index bcc20912..a7e399f6 100644
--- a/telefuser/service/livekit/nccl_process_worker_pool.py
+++ b/telefuser/service/livekit/nccl_process_worker_pool.py
@@ -8,9 +8,12 @@
import asyncio
import contextlib
+import json
import socket
import time
from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
from typing import Any
import torch
@@ -22,7 +25,12 @@
from .nccl_transfer import allocate_tensor_tree_leaves, transfer_tensor_leaves_nccl
from .pipeline_adapter import LiveKitPipelineAdapter
-from .process_worker_pool import ProcessLiveKitWorkerPool, ProcessWorkerSpec, _close_queue
+from .process_worker_pool import (
+ ProcessLiveKitWorkerPool,
+ ProcessWorkerSpec,
+ _close_queue,
+ _process_dispatch_trace_gpu_metadata,
+)
from .session_registry import SessionRecord
from .token_service import LiveKitTokenService
from .turboserve import TurboServeOwnership, TurboServeOwnershipTable
@@ -37,6 +45,12 @@
_MODEL_OUTPUT_PARENT_QUEUE_SIZE = 1
_VIDEO_OUTPUT_TYPES = frozenset({"preview", "chunk"})
_TERMINAL_OUTPUT_TYPES = frozenset({"error", "done"})
+# Process-group creation happens only after every worker has loaded the model,
+# so it needs a dedicated budget rather than the normal 15-second IPC timeout.
+# Keep the parent request longer than the child process-group timeout so a
+# child can return a useful failure instead of being torn down mid-initialization.
+_NCCL_INIT_GROUP_TIMEOUT_SECONDS = 180.0
+_NCCL_INIT_PARENT_TIMEOUT_SECONDS = 210.0
@dataclass(frozen=True)
@@ -47,6 +61,82 @@ class _ModelOutput:
payload: dict[str, Any]
+class _DispatchTraceWriter:
+ """Bounded, parent-owned JSONL writer for experiment audit records."""
+
+ def __init__(self, path: str, *, max_events: int, workers: dict[str, list[str]]) -> None:
+ self.path = Path(path).expanduser().resolve()
+ self.max_events = int(max_events)
+ self.received_events = 0
+ self.written_events = 0
+ self.dropped_events = 0
+ self.write_errors = 0
+ self._write_error_logged = False
+ self._handle: Any | None = None
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ if self.path.exists():
+ raise FileExistsError(f"dispatch trace path already exists; choose a fresh run-scoped path: {self.path}")
+ self._handle = self.path.open("x", encoding="utf-8")
+ self._write_line(
+ {
+ "schema_version": 1,
+ "event_type": "trace_metadata",
+ "trace_started_monotonic_seconds": time.monotonic(),
+ "trace_started_unix_seconds": time.time(),
+ "trace_started_utc": datetime.now(timezone.utc).isoformat(),
+ "max_dispatch_events": self.max_events,
+ "configured_workers": workers,
+ }
+ )
+
+ def _write_line(self, record: dict[str, Any]) -> bool:
+ handle = self._handle
+ if handle is None:
+ return False
+ try:
+ handle.write(json.dumps(record, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n")
+ handle.flush()
+ return True
+ except (OSError, TypeError, ValueError) as exc:
+ self.write_errors += 1
+ if not self._write_error_logged:
+ self._write_error_logged = True
+ logger.warning("Failed to write ABot dispatch trace %s: %s", self.path, exc)
+ return False
+
+ def append(self, record: dict[str, Any]) -> None:
+ self.received_events += 1
+ if self.received_events > self.max_events:
+ self.dropped_events += 1
+ return
+ enriched = dict(record)
+ enriched["parent_sequence"] = self.received_events
+ enriched["parent_received_monotonic_seconds"] = time.monotonic()
+ enriched["parent_received_unix_seconds"] = time.time()
+ if self._write_line(enriched):
+ self.written_events += 1
+ else:
+ self.dropped_events += 1
+
+ def snapshot(self) -> dict[str, object]:
+ return {
+ "enabled": True,
+ "path": str(self.path),
+ "max_events": self.max_events,
+ "received_events": self.received_events,
+ "written_events": self.written_events,
+ "dropped_events": self.dropped_events,
+ "write_errors": self.write_errors,
+ }
+
+ def close(self) -> None:
+ handle = self._handle
+ self._handle = None
+ if handle is not None:
+ with contextlib.suppress(OSError):
+ handle.close()
+
+
async def _pump_model_outputs(
adapter: Any,
service: Any,
@@ -69,6 +159,16 @@ async def _pump_model_outputs(
except StopAsyncIteration:
credits.release()
credit_held = False
+ # An inactive model session naturally ends its generator. The
+ # parent transport must see this EOF and release its route;
+ # otherwise it waits forever in ``pull_model_chunks``.
+ events.put(
+ {
+ "type": "model_output_eos",
+ "worker_id": worker_id,
+ "session_id": session_id,
+ }
+ )
return
except asyncio.CancelledError:
credits.release()
@@ -122,6 +222,20 @@ async def pull_chunks(self, session_id: str):
async for chunk in self._pool.pull_model_chunks(session_id):
yield chunk
+ def enable_publisher_frame_tracking(self, session_id: str) -> bool:
+ return self._pool.enable_publisher_frame_tracking(session_id)
+
+ def report_publisher_frame_progress(
+ self, session_id: str, *, event: str, frames_delta: int, sequence: int, observed_monotonic_seconds: float
+ ) -> bool:
+ del sequence
+ return self._pool.report_publisher_frame_progress(
+ session_id,
+ event=event,
+ frames_delta=frames_delta,
+ observed_monotonic_seconds=observed_monotonic_seconds,
+ )
+
def close_session(self, session_id: str) -> None:
self._pool.close_model_session(session_id)
@@ -169,6 +283,9 @@ def __init__(self, specs: list[ProcessWorkerSpec], **kwargs: Any) -> None:
self._model_outputs: dict[str, asyncio.Queue[_ModelOutput | None]] = {}
self._model_output_inflight: set[str] = set()
self._model_output_drained: dict[str, asyncio.Event] = {}
+ self._model_output_inflight_owner: dict[str, str] = {}
+ self._publisher_progress_sequences: dict[str, int] = {}
+ self._publisher_frame_tracking: dict[str, bool] = {}
self._model_output_dropped: dict[str, int] = {}
self._transport_workers: dict[str, LiveKitWorker] = {}
self._transport_tasks: dict[str, asyncio.Task[None]] = {}
@@ -181,6 +298,9 @@ def __init__(self, specs: list[ProcessWorkerSpec], **kwargs: Any) -> None:
self._nccl_ranks: dict[str, int] = {}
self._migration_lock = asyncio.Lock()
self._initializing_workers = False
+ # ``ProcessLiveKitWorkerPool`` owns the parent JSONL writer for both
+ # isolated-worker modes. Recreating it here would reject the path it
+ # has just created, preventing a traced process-NCCL run from starting.
async def start(self, *, skip_validation: bool = False) -> None:
# ``ProcessLiveKitWorkerPool.start`` calls this class's ``scale_to``
@@ -247,6 +367,8 @@ def create_model_session(self, worker_id: str, session_id: str, config: dict) ->
self._model_output_drained[session_id] = drained
self._model_output_dropped[session_id] = 0
self._pipeline_routes[session_id] = worker_id
+ self._publisher_progress_sequences[session_id] = -1
+ self._publisher_frame_tracking[session_id] = True
self._session_workers[session_id] = worker_id
self._ownership.register(session_id, worker_id)
self._send(
@@ -268,12 +390,62 @@ def push_model_chunk(self, session_id: str, chunk: dict) -> None:
{"type": "model_push", "session_id": session_id, "chunk": dict(chunk)},
)
+ def enable_publisher_frame_tracking(self, session_id: str) -> bool:
+ return bool(self._publisher_frame_tracking.get(session_id, False))
+
+ def report_publisher_frame_progress(
+ self, session_id: str, *, event: str, frames_delta: int, observed_monotonic_seconds: float
+ ) -> bool:
+ worker_id = self._model_output_inflight_owner.get(session_id) or self._pipeline_routes.get(session_id)
+ return self._send_publisher_frame_progress(
+ session_id,
+ worker_id=worker_id,
+ event=event,
+ frames_delta=frames_delta,
+ observed_monotonic_seconds=observed_monotonic_seconds,
+ )
+
+ def _send_publisher_frame_progress(
+ self,
+ session_id: str,
+ *,
+ worker_id: str | None,
+ event: str,
+ frames_delta: int,
+ observed_monotonic_seconds: float,
+ ) -> bool:
+ if (
+ worker_id is None
+ or worker_id not in self._active_workers
+ or not self._publisher_frame_tracking.get(session_id, False)
+ ):
+ return False
+ sequence = int(self._publisher_progress_sequences.get(session_id, -1)) + 1
+ self._publisher_progress_sequences[session_id] = sequence
+ try:
+ self._send(
+ worker_id,
+ {
+ "type": "model_publisher_frame_progress",
+ "session_id": session_id,
+ "event": event,
+ "frames_delta": int(frames_delta),
+ "sequence": sequence,
+ "observed_monotonic_seconds": float(observed_monotonic_seconds),
+ },
+ )
+ except Exception:
+ return False
+ return True
+
def close_model_session(self, session_id: str) -> None:
worker_id = self._pipeline_routes.pop(session_id, None)
self._session_workers.pop(session_id, None)
self._ownership.release(session_id)
self._migrating_controls.pop(session_id, None)
self._session_runtime_metrics.pop(session_id, None)
+ self._publisher_progress_sequences.pop(session_id, None)
+ self._publisher_frame_tracking.pop(session_id, None)
if worker_id in self._active_workers:
self._send(worker_id, {"type": "model_close", "session_id": session_id})
self._close_model_output(session_id)
@@ -289,17 +461,20 @@ async def pull_model_chunks(self, session_id: str):
# This is deliberately before ``yield``: it allows one queued
# prefetch while the transport paces the just-dequeued payload.
self._model_output_inflight.add(session_id)
+ self._model_output_inflight_owner[session_id] = item.worker_id
self._update_model_output_drained(session_id)
self._ack_model_output(session_id, item)
try:
yield item.payload
finally:
self._model_output_inflight.discard(session_id)
+ self._model_output_inflight_owner.pop(session_id, None)
self._update_model_output_drained(session_id)
def _close_model_output(self, session_id: str) -> None:
output = self._model_outputs.pop(session_id, None)
self._model_output_inflight.discard(session_id)
+ self._model_output_inflight_owner.pop(session_id, None)
drained = self._model_output_drained.pop(session_id, None)
self._model_output_dropped.pop(session_id, None)
if isinstance(output, asyncio.Queue):
@@ -360,6 +535,16 @@ def _record_dropped_model_output(self, session_id: str, item: _ModelOutput, *, a
dropped = getattr(self, "_model_output_dropped", None)
if isinstance(dropped, dict):
dropped[session_id] = int(dropped.get(session_id, 0)) + 1
+ payload = item.payload
+ frames = payload.get("frames")
+ if payload.get("type") == "chunk" and isinstance(frames, list) and frames:
+ self._send_publisher_frame_progress(
+ session_id,
+ worker_id=item.worker_id,
+ event="dropped",
+ frames_delta=-len(frames),
+ observed_monotonic_seconds=time.monotonic(),
+ )
if acknowledge:
self._ack_model_output(session_id, item)
@@ -393,6 +578,72 @@ async def _wait_for_model_output_drain(self, session_id: str, *, timeout: float)
if drained is not None:
await asyncio.wait_for(drained.wait(), timeout=timeout)
+ @staticmethod
+ def _source_model_output_drain_complete(status: object) -> bool:
+ if not isinstance(status, dict):
+ return False
+ return bool(
+ not status.get("in_flight", True)
+ and status.get("output_queue_empty", False)
+ and int(status.get("publisher_unsubmitted_frames", 1)) == 0
+ )
+
+ async def _drain_model_outputs_for_migration(
+ self,
+ session_id: str,
+ *,
+ source_worker_id: str,
+ timeout: float,
+ ) -> None:
+ """Drain child Fq and parent publisher Fp before copying model state.
+
+ The child pump intentionally remains active until the source service
+ reports no queued model output and no publisher-owned frames. Each
+ status request is a command-queue barrier, so it follows all earlier
+ publisher-progress commands for this child. A parent drain after the
+ barrier accounts for model-output events emitted before that barrier.
+ """
+ deadline = time.monotonic() + timeout
+ while True:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("Timed out draining model output before NCCL migration")
+ await self._wait_for_model_output_drain(session_id, timeout=remaining)
+
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("Timed out waiting for source drain status before NCCL migration")
+ status_event = await self._request(
+ source_worker_id,
+ "model_output_drain_status",
+ session_id=session_id,
+ timeout=remaining,
+ )
+ status = status_event.get("result")
+ if not self._source_model_output_drain_complete(status):
+ await asyncio.sleep(0.001)
+ continue
+
+ # The source barrier follows all of its model-output events; drain
+ # those parent transport payloads before accepting the snapshot.
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("Timed out draining parent transport before NCCL migration")
+ await self._wait_for_model_output_drain(session_id, timeout=remaining)
+
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("Timed out confirming source drain before NCCL migration")
+ final_event = await self._request(
+ source_worker_id,
+ "model_output_drain_status",
+ session_id=session_id,
+ timeout=remaining,
+ )
+ if self._source_model_output_drain_complete(final_event.get("result")):
+ return
+ await asyncio.sleep(0.001)
+
async def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
async with self._migration_lock:
source_worker_id = self._pipeline_routes[pipeline_session_id]
@@ -409,11 +660,9 @@ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str)
self._request(source_worker_id, "scheduler_pause", timeout=300.0),
self._request(target_worker_id, "scheduler_pause", timeout=300.0),
)
- exported = await self._request(
- source_worker_id,
- "nccl_export",
- session_id=pipeline_session_id,
- transfer_id=token.token_id,
+ await self._drain_model_outputs_for_migration(
+ pipeline_session_id,
+ source_worker_id=source_worker_id,
timeout=300.0,
)
await self._request(
@@ -423,7 +672,21 @@ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str)
timeout=300.0,
)
source_output_paused = True
- await self._wait_for_model_output_drain(pipeline_session_id, timeout=300.0)
+ paused_status_event = await self._request(
+ source_worker_id,
+ "model_output_drain_status",
+ session_id=pipeline_session_id,
+ timeout=300.0,
+ )
+ if not self._source_model_output_drain_complete(paused_status_event.get("result")):
+ raise RuntimeError("Source output changed while preparing NCCL migration")
+ exported = await self._request(
+ source_worker_id,
+ "nccl_export",
+ session_id=pipeline_session_id,
+ transfer_id=token.token_id,
+ timeout=300.0,
+ )
metadata = dict(exported["result"])
await self._request(
target_worker_id,
@@ -519,6 +782,11 @@ def turboserve_snapshot(self) -> dict[str, object]:
else 0.0
},
"model_output_flow_control": self._model_output_flow_snapshot(),
+ "dispatch_trace": (
+ self._dispatch_trace.snapshot()
+ if getattr(self, "_dispatch_trace", None) is not None
+ else {"enabled": False}
+ ),
}
)
return snapshot
@@ -543,15 +811,21 @@ def _model_output_flow_snapshot(self) -> dict[str, object]:
}
async def aclose(self) -> None:
- for session_id in tuple(self._transport_workers):
- with contextlib.suppress(Exception):
- await self.stop_session(session_id)
- if self._nccl_ranks:
- await asyncio.gather(
- *(self._request(worker_id, "nccl_destroy") for worker_id in self._nccl_ranks), return_exceptions=True
- )
- self._nccl_ranks.clear()
- await super().aclose()
+ try:
+ for session_id in tuple(self._transport_workers):
+ with contextlib.suppress(Exception):
+ await self.stop_session(session_id)
+ if self._nccl_ranks:
+ await asyncio.gather(
+ *(self._request(worker_id, "nccl_destroy") for worker_id in self._nccl_ranks),
+ return_exceptions=True,
+ )
+ self._nccl_ranks.clear()
+ await super().aclose()
+ finally:
+ trace = getattr(self, "_dispatch_trace", None)
+ if trace is not None:
+ trace.close()
async def _init_nccl(self) -> None:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -559,14 +833,27 @@ async def _init_nccl(self) -> None:
port = sock.getsockname()[1]
sock.close()
workers = sorted(self._active_workers)
- await asyncio.gather(
+ results = await asyncio.gather(
*(
self._request(
- worker_id, "nccl_init", rank=rank, world_size=len(workers), init_method=f"tcp://127.0.0.1:{port}"
+ worker_id,
+ "nccl_init",
+ rank=rank,
+ world_size=len(workers),
+ init_method=f"tcp://127.0.0.1:{port}",
+ timeout=_NCCL_INIT_PARENT_TIMEOUT_SECONDS,
)
for rank, worker_id in enumerate(workers)
- )
+ ),
+ return_exceptions=True,
)
+ failures = [
+ f"{worker_id}: {result}"
+ for worker_id, result in zip(workers, results, strict=True)
+ if isinstance(result, BaseException)
+ ]
+ if failures:
+ raise RuntimeError(f"NCCL process-group initialization failed: {'; '.join(failures)}")
self._nccl_ranks = {worker_id: rank for rank, worker_id in enumerate(workers)}
def _transport_finished(self, session_id: str) -> None:
@@ -578,7 +865,33 @@ def _transport_task_done(self, session_id: str, task: asyncio.Task[None]) -> Non
if not task.cancelled() and task.exception() is not None:
logger.warning("LiveKit transport failed: session=%s error=%s", session_id, task.exception())
+ def _record_dispatch_trace(self, event: dict[str, Any]) -> None:
+ """Persist one child-reported model invocation in the parent process."""
+ trace = getattr(self, "_dispatch_trace", None)
+ raw = event.get("trace")
+ if trace is None or not isinstance(raw, dict):
+ return
+ gpu = event.get("gpu")
+ record = dict(raw)
+ record["worker_id"] = str(event.get("worker_id", "unknown"))
+ record["gpu"] = dict(gpu) if isinstance(gpu, dict) else {}
+ trace.append(record)
+
def _dispatch_event(self, event: dict[str, Any]) -> None:
+ if event.get("type") == "model_publisher_frame_tracking":
+ session_id = str(event.get("session_id", ""))
+ if session_id in self._publisher_frame_tracking:
+ self._publisher_frame_tracking[session_id] = bool(event.get("enabled", False))
+ return
+ if event.get("type") == "model_dispatch_trace":
+ self._record_dispatch_trace(event)
+ return
+ if event.get("type") == "model_output_eos":
+ # ``close_model_session`` installs the parent queue sentinel and
+ # asks the child to release retained state. It is deliberately
+ # idempotent because the transport finally block also calls back.
+ self.close_model_session(str(event["session_id"]))
+ return
if event.get("type") == "model_output":
metrics = event.get("runtime_metrics")
if isinstance(metrics, dict):
@@ -659,6 +972,29 @@ async def _run_nccl_model_worker(
events.put({"type": "worker_status", "worker_id": spec.worker_id, "status": "idle"})
events.put({"type": "worker_ready", "worker_id": spec.worker_id})
service = adapter.stream_service.service
+ set_dispatch_trace_callback = getattr(service, "set_dispatch_trace_callback", None)
+ trace_path_value = config.dispatch_trace_path
+ if isinstance(trace_path_value, str) and trace_path_value.strip() and callable(set_dispatch_trace_callback):
+ try:
+ logical_cuda_device: int | None = int(torch.cuda.current_device())
+ except Exception:
+ logical_cuda_device = None
+ trace_gpu = _process_dispatch_trace_gpu_metadata(
+ spec,
+ logical_cuda_device=logical_cuda_device,
+ )
+
+ def forward_dispatch_trace(record: dict[str, Any]) -> None:
+ events.put(
+ {
+ "type": "model_dispatch_trace",
+ "worker_id": spec.worker_id,
+ "gpu": trace_gpu,
+ "trace": record,
+ }
+ )
+
+ set_dispatch_trace_callback(forward_dispatch_trace)
outputs: dict[str, asyncio.Task[None]] = {}
output_credits: dict[str, asyncio.BoundedSemaphore] = {}
outgoing: dict[str, dict[tuple[Any, ...], torch.Tensor]] = {}
@@ -709,12 +1045,32 @@ async def result(request_id: str | None, value: Any = True, error: Exception | N
try:
if kind == "model_create":
session_id = adapter.create_session(command["config"])
+ try:
+ publisher_tracking_enabled = bool(adapter.enable_publisher_frame_tracking(session_id))
+ except Exception:
+ publisher_tracking_enabled = False
+ events.put(
+ {
+ "type": "model_publisher_frame_tracking",
+ "worker_id": spec.worker_id,
+ "session_id": session_id,
+ "enabled": publisher_tracking_enabled,
+ }
+ )
start_pump(
session_id,
credit_window=int(command.get("model_output_credit_window", _MODEL_OUTPUT_PARENT_QUEUE_SIZE)),
)
elif kind == "model_push":
adapter.push_chunk(command["session_id"], command["chunk"])
+ elif kind == "model_publisher_frame_progress":
+ adapter.report_publisher_frame_progress(
+ command["session_id"],
+ event=str(command["event"]),
+ frames_delta=int(command["frames_delta"]),
+ sequence=int(command["sequence"]),
+ observed_monotonic_seconds=float(command["observed_monotonic_seconds"]),
+ )
elif kind == "model_close":
await stop_pump(command["session_id"], drop_credit_state=True)
adapter.close_session(command["session_id"])
@@ -725,17 +1081,28 @@ async def result(request_id: str | None, value: Any = True, error: Exception | N
credits.release()
elif kind == "model_output_pause":
await stop_pump(command["session_id"])
+ elif kind == "model_output_drain_status":
+ migration_drain_status = getattr(service, "migration_drain_status", None)
+ if not callable(migration_drain_status):
+ raise RuntimeError("Pipeline service does not expose migration drain status")
+ status = await asyncio.to_thread(migration_drain_status, command["session_id"])
+ await result(request_id, status)
+ continue
+
elif kind == "model_output_resume":
has_session = getattr(service, "has_session", None)
if not callable(has_session) or has_session(command["session_id"]):
start_pump(command["session_id"])
elif kind == "nccl_init":
+ device_id = torch.device("cuda", torch.cuda.current_device())
await asyncio.to_thread(
dist.init_process_group,
"nccl",
init_method=command["init_method"],
rank=command["rank"],
world_size=command["world_size"],
+ timeout=timedelta(seconds=_NCCL_INIT_GROUP_TIMEOUT_SECONDS),
+ device_id=device_id,
)
elif kind == "scheduler_pause":
await asyncio.to_thread(service.pause_scheduler)
@@ -768,6 +1135,18 @@ async def result(request_id: str | None, value: Any = True, error: Exception | N
session_id = service.import_migration_nccl(
metadata, leaves, owner_worker_id=owner, ownership_epoch=epoch
)
+ try:
+ publisher_tracking_enabled = bool(adapter.enable_publisher_frame_tracking(session_id))
+ except Exception:
+ publisher_tracking_enabled = False
+ events.put(
+ {
+ "type": "model_publisher_frame_tracking",
+ "worker_id": spec.worker_id,
+ "session_id": session_id,
+ "enabled": publisher_tracking_enabled,
+ }
+ )
start_pump(session_id, credit_window=credit_window)
elif kind == "nccl_commit_source":
await stop_pump(command["session_id"], drop_credit_state=True)
diff --git a/telefuser/service/livekit/pipeline_adapter.py b/telefuser/service/livekit/pipeline_adapter.py
index e1cbdd22..1167d615 100644
--- a/telefuser/service/livekit/pipeline_adapter.py
+++ b/telefuser/service/livekit/pipeline_adapter.py
@@ -48,6 +48,24 @@ async def pull_chunks(self, session_id: str) -> AsyncGenerator[dict, None]:
async for chunk in self.stream_service.pull_chunks(session_id):
yield chunk
+ def enable_publisher_frame_tracking(self, session_id: str) -> bool:
+ """Enable the wrapped service's optional real-time frame feedback."""
+
+ return self.stream_service.enable_publisher_frame_tracking(session_id)
+
+ def report_publisher_frame_progress(
+ self, session_id: str, *, event: str, frames_delta: int, sequence: int, observed_monotonic_seconds: float
+ ) -> bool:
+ """Forward one idempotent publisher progress update when supported."""
+
+ return self.stream_service.report_publisher_frame_progress(
+ session_id,
+ event=event,
+ frames_delta=frames_delta,
+ sequence=sequence,
+ observed_monotonic_seconds=observed_monotonic_seconds,
+ )
+
async def stream_task(self, config: dict) -> AsyncGenerator[dict, None]:
"""Yield chunks from a server-push service."""
async for chunk in self.stream_service.stream_task(config):
diff --git a/telefuser/service/livekit/pipeline_router.py b/telefuser/service/livekit/pipeline_router.py
index 1664c4ce..51bf31af 100644
--- a/telefuser/service/livekit/pipeline_router.py
+++ b/telefuser/service/livekit/pipeline_router.py
@@ -72,6 +72,26 @@ async def pull_chunks(self, pipeline_session_id: str) -> AsyncGenerator[dict, No
if next_worker_id is None or next_worker_id == worker_id:
return
+ def enable_publisher_frame_tracking(self, pipeline_session_id: str) -> bool:
+ return self._backend_for(pipeline_session_id).enable_publisher_frame_tracking(pipeline_session_id)
+
+ def report_publisher_frame_progress(
+ self,
+ pipeline_session_id: str,
+ *,
+ event: str,
+ frames_delta: int,
+ sequence: int,
+ observed_monotonic_seconds: float,
+ ) -> bool:
+ return self._backend_for(pipeline_session_id).report_publisher_frame_progress(
+ pipeline_session_id,
+ event=event,
+ frames_delta=frames_delta,
+ sequence=sequence,
+ observed_monotonic_seconds=observed_monotonic_seconds,
+ )
+
def close_session(self, pipeline_session_id: str) -> None:
with self._lock:
worker_id = self._routes.pop(pipeline_session_id, None)
@@ -109,6 +129,9 @@ def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> Tu
owner_worker_id=target_worker_id,
ownership_epoch=token.source_epoch + 1,
)
+ enable_tracking = getattr(target, "enable_publisher_frame_tracking", None)
+ if callable(enable_tracking):
+ enable_tracking(pipeline_session_id)
imported = True
except Exception:
if imported and target is not None:
@@ -240,6 +263,26 @@ async def pull_chunks(self, pipeline_session_id: str) -> AsyncGenerator[dict, No
async for chunk in self._router.pull_chunks(pipeline_session_id):
yield chunk
+ def enable_publisher_frame_tracking(self, pipeline_session_id: str) -> bool:
+ return self._router.enable_publisher_frame_tracking(pipeline_session_id)
+
+ def report_publisher_frame_progress(
+ self,
+ pipeline_session_id: str,
+ *,
+ event: str,
+ frames_delta: int,
+ sequence: int,
+ observed_monotonic_seconds: float,
+ ) -> bool:
+ return self._router.report_publisher_frame_progress(
+ pipeline_session_id,
+ event=event,
+ frames_delta=frames_delta,
+ sequence=sequence,
+ observed_monotonic_seconds=observed_monotonic_seconds,
+ )
+
def close_session(self, pipeline_session_id: str) -> None:
self._router.close_session(pipeline_session_id)
diff --git a/telefuser/service/livekit/process_worker_pool.py b/telefuser/service/livekit/process_worker_pool.py
index 7eb0b144..0231fe74 100644
--- a/telefuser/service/livekit/process_worker_pool.py
+++ b/telefuser/service/livekit/process_worker_pool.py
@@ -4,11 +4,16 @@
import asyncio
import contextlib
+import json
import multiprocessing
+import os
+import time
import uuid
from dataclasses import dataclass
+from datetime import datetime, timezone
from multiprocessing.context import BaseContext
from multiprocessing.process import BaseProcess
+from pathlib import Path
from typing import Any
from telefuser.service.security.security_validator import SecurityLevel
@@ -40,6 +45,88 @@ class _ProcessHandle:
process: BaseProcess
+class _DispatchTraceWriter:
+ """Bounded, parent-owned JSONL writer for experiment audit records.
+
+ Both isolated-worker modes forward model-dispatch callbacks over their
+ existing child-to-parent event queue. Keeping the file descriptor in the
+ parent makes the single-GPU ``process`` schema identical to
+ ``process-nccl``.
+ """
+
+ def __init__(self, path: str, *, max_events: int, workers: dict[str, list[str]]) -> None:
+ self.path = Path(path).expanduser().resolve()
+ self.max_events = int(max_events)
+ self.received_events = 0
+ self.written_events = 0
+ self.dropped_events = 0
+ self.write_errors = 0
+ self._write_error_logged = False
+ self._handle: Any | None = None
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ if self.path.exists():
+ raise FileExistsError(f"dispatch trace path already exists; choose a fresh run-scoped path: {self.path}")
+ self._handle = self.path.open("x", encoding="utf-8")
+ self._write_line(
+ {
+ "schema_version": 1,
+ "event_type": "trace_metadata",
+ "trace_started_monotonic_seconds": time.monotonic(),
+ "trace_started_unix_seconds": time.time(),
+ "trace_started_utc": datetime.now(timezone.utc).isoformat(),
+ "max_dispatch_events": self.max_events,
+ "configured_workers": workers,
+ }
+ )
+
+ def _write_line(self, record: dict[str, Any]) -> bool:
+ handle = self._handle
+ if handle is None:
+ return False
+ try:
+ handle.write(json.dumps(record, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n")
+ handle.flush()
+ return True
+ except (OSError, TypeError, ValueError) as exc:
+ self.write_errors += 1
+ if not self._write_error_logged:
+ self._write_error_logged = True
+ logger.warning("Failed to write ABot dispatch trace %s: %s", self.path, exc)
+ return False
+
+ def append(self, record: dict[str, Any]) -> None:
+ self.received_events += 1
+ if self.received_events > self.max_events:
+ self.dropped_events += 1
+ return
+ enriched = dict(record)
+ enriched["parent_sequence"] = self.received_events
+ enriched["parent_received_monotonic_seconds"] = time.monotonic()
+ enriched["parent_received_unix_seconds"] = time.time()
+ if self._write_line(enriched):
+ self.written_events += 1
+ else:
+ self.dropped_events += 1
+
+ def snapshot(self) -> dict[str, object]:
+ return {
+ "enabled": True,
+ "path": str(self.path),
+ "max_events": self.max_events,
+ "received_events": self.received_events,
+ "written_events": self.written_events,
+ "dropped_events": self.dropped_events,
+ "write_errors": self.write_errors,
+ }
+
+ def close(self) -> None:
+ handle = self._handle
+ self._handle = None
+ if handle is not None:
+ with contextlib.suppress(OSError):
+ handle.close()
+
+
class ProcessLiveKitWorkerPool:
"""Run one model replica per spawned process and keep the API process model-free."""
@@ -82,6 +169,17 @@ def __init__(
self._started = False
self._closing = False
self._skip_validation = False
+ trace_path_value = getattr(self._config, "dispatch_trace_path", None)
+ trace_path = trace_path_value.strip() if isinstance(trace_path_value, str) else ""
+ self._dispatch_trace = (
+ _DispatchTraceWriter(
+ trace_path,
+ max_events=int(getattr(self._config, "dispatch_trace_max_events", 10_000)),
+ workers={worker_id: list(spec.gpu_ids) for worker_id, spec in self._specs.items()},
+ )
+ if trace_path
+ else None
+ )
async def start(self, *, skip_validation: bool = False) -> None:
"""Spawn and wait for the configured initial replica set."""
@@ -161,6 +259,11 @@ def turboserve_snapshot(self) -> dict[str, object]:
"active_workers": sorted(self._active_workers),
"configured_workers": len(self._specs),
"migration_supported": False,
+ "dispatch_trace": (
+ self._dispatch_trace.snapshot()
+ if getattr(self, "_dispatch_trace", None) is not None
+ else {"enabled": False}
+ ),
}
async def aclose(self) -> None:
@@ -188,6 +291,9 @@ async def aclose(self) -> None:
self._pending_workers.clear()
self._startup.clear()
_close_queue(self._events)
+ trace = getattr(self, "_dispatch_trace", None)
+ if trace is not None:
+ trace.close()
self._started = False
self._closing = False
@@ -278,8 +384,23 @@ async def _event_loop(self) -> None:
return
self._dispatch_event(event)
+ def _record_dispatch_trace(self, event: dict[str, Any]) -> None:
+ """Persist one child-reported model invocation in the parent process."""
+ trace = getattr(self, "_dispatch_trace", None)
+ raw = event.get("trace")
+ if trace is None or not isinstance(raw, dict):
+ return
+ gpu = event.get("gpu")
+ record = dict(raw)
+ record["worker_id"] = str(event.get("worker_id", "unknown"))
+ record["gpu"] = dict(gpu) if isinstance(gpu, dict) else {}
+ trace.append(record)
+
def _dispatch_event(self, event: dict[str, Any]) -> None:
event_type = event.get("type")
+ if event_type == "model_dispatch_trace":
+ self._record_dispatch_trace(event)
+ return
worker_id = event.get("worker_id")
if event_type == "worker_ready":
future = self._startup.get(worker_id)
@@ -465,6 +586,84 @@ def _close_queue(ipc_queue: Any, *, join: bool = True) -> None:
ipc_queue.join_thread()
+def _process_dispatch_trace_gpu_metadata(
+ spec: ProcessWorkerSpec,
+ *,
+ logical_cuda_device: int | None,
+ cuda_visible_devices: str | None = None,
+) -> dict[str, int | str | None]:
+ """Describe the physical CUDA lane without confusing CVD-local indices.
+
+ A process launched as ``CUDA_VISIBLE_DEVICES=1`` sees that card as
+ ``cuda:0``. The worker map therefore correctly contains ``0``, but an
+ experiment timeline must label its lane as physical GPU 1.
+ """
+
+ configured_gpu_id = str(spec.gpu_ids[0]) if spec.gpu_ids else "unknown"
+ logical = logical_cuda_device
+ if logical is None:
+ try:
+ logical = int(configured_gpu_id)
+ except ValueError:
+ logical = None
+ visible = cuda_visible_devices if cuda_visible_devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "")
+ visible_ids = [value.strip() for value in visible.split(",") if value.strip()]
+ physical_gpu_id = configured_gpu_id
+ if logical is not None and 0 <= logical < len(visible_ids):
+ physical_gpu_id = visible_ids[logical]
+ return {
+ "physical_gpu_id": physical_gpu_id,
+ "configured_gpu_id": configured_gpu_id,
+ "logical_cuda_device": logical,
+ }
+
+
+def _current_cuda_device_for_trace(spec: ProcessWorkerSpec) -> int | None:
+ """Return the child CUDA-local device without making tracing mandatory."""
+
+ try:
+ import torch
+
+ if torch.cuda.is_available():
+ return int(torch.cuda.current_device())
+ except Exception: # pragma: no cover - diagnostic metadata must never stop serving
+ pass
+ try:
+ return int(spec.gpu_ids[0]) if spec.gpu_ids else None
+ except ValueError:
+ return None
+
+
+def _install_process_dispatch_trace_callback(
+ *,
+ service: Any,
+ config: LiveKitServeConfig,
+ spec: ProcessWorkerSpec,
+ events: Any,
+ logical_cuda_device: int | None,
+) -> bool:
+ """Forward child service records to the parent's bounded JSONL writer."""
+
+ trace_path_value = config.dispatch_trace_path
+ set_callback = getattr(service, "set_dispatch_trace_callback", None)
+ if not isinstance(trace_path_value, str) or not trace_path_value.strip() or not callable(set_callback):
+ return False
+ gpu = _process_dispatch_trace_gpu_metadata(spec, logical_cuda_device=logical_cuda_device)
+
+ def forward_dispatch_trace(record: dict[str, Any]) -> None:
+ events.put(
+ {
+ "type": "model_dispatch_trace",
+ "worker_id": spec.worker_id,
+ "gpu": gpu,
+ "trace": record,
+ }
+ )
+
+ set_callback(forward_dispatch_trace)
+ return True
+
+
async def _run_process_worker(
spec: ProcessWorkerSpec,
config_values: dict[str, Any],
@@ -498,6 +697,14 @@ async def _run_process_worker(
)
tasks: dict[str, asyncio.Task[None]] = {}
await worker.start(skip_validation=skip_validation)
+ service = getattr(getattr(worker.pipeline_adapter, "stream_service", None), "service", None)
+ _install_process_dispatch_trace_callback(
+ service=service,
+ config=config,
+ spec=spec,
+ events=events,
+ logical_cuda_device=_current_cuda_device_for_trace(spec),
+ )
events.put({"type": "worker_ready", "worker_id": spec.worker_id})
try:
while True:
@@ -662,9 +869,7 @@ def on_session_finished(self, worker_id: str, session_id: str, error: str | None
)
def on_control_received(self, worker_id: str, session_id: str) -> None:
- self.events.put(
- {"type": "control_received", "worker_id": worker_id, "session_id": session_id}
- )
+ self.events.put({"type": "control_received", "worker_id": worker_id, "session_id": session_id})
def on_chunk_published(
self, worker_id: str, session_id: str, frames: int, first_frame_at: float | None = None
diff --git a/telefuser/service/livekit/worker.py b/telefuser/service/livekit/worker.py
index 18d03258..a36c36aa 100644
--- a/telefuser/service/livekit/worker.py
+++ b/telefuser/service/livekit/worker.py
@@ -109,6 +109,8 @@ def __init__(
self.event_sink = event_sink or NullWorkerEventSink()
self.pipeline_adapter = pipeline_adapter or LiveKitPipelineAdapter()
self.room_client = room_client or LiveKitRoomClient()
+ self._publisher_frame_tracking_enabled = False
+ self._publisher_progress_sequence = 0
self.gpu_num = gpu_num
self._active_session_id: str | None = None
self._pipeline_session_id: str | None = None
@@ -158,6 +160,8 @@ async def run_session(self, record: SessionRecord) -> None:
self.event_sink.on_session_status(record.session_id, "starting_pipeline")
if self.pipeline_adapter.stream_mode == STREAM_MODE_BIDIRECTIONAL:
self._pipeline_session_id = self.pipeline_adapter.create_session(record.config)
+ self._publisher_progress_sequence = 0
+ self._publisher_frame_tracking_enabled = self._enable_publisher_frame_tracking()
self.event_sink.on_pipeline_session(record.session_id, self._pipeline_session_id)
chunks = self.pipeline_adapter.pull_chunks(self._pipeline_session_id)
elif self.pipeline_adapter.stream_mode == STREAM_MODE_SERVER_PUSH:
@@ -213,6 +217,58 @@ async def stop_session(self, session_id: str) -> None:
with contextlib.suppress(Exception):
self.pipeline_adapter.push_chunk(self._pipeline_session_id, {"type": "stop"})
+ def _enable_publisher_frame_tracking(self) -> bool:
+ pipeline_session_id = self._pipeline_session_id
+ callback = getattr(self.pipeline_adapter, "enable_publisher_frame_tracking", None)
+ if pipeline_session_id is None or not callable(callback):
+ return False
+ try:
+ return bool(callback(pipeline_session_id))
+ except Exception as exc: # pragma: no cover - feedback must never stop media publication
+ logger.warning(
+ "Could not enable publisher frame tracking: worker=%s session=%s error=%s",
+ self.worker_id,
+ pipeline_session_id,
+ exc,
+ )
+ return False
+
+ def _report_publisher_frame_progress(self, *, event: str, frames_delta: int) -> None:
+ if not self._publisher_frame_tracking_enabled or self._pipeline_session_id is None:
+ return
+ callback = getattr(self.pipeline_adapter, "report_publisher_frame_progress", None)
+ if not callable(callback):
+ self._publisher_frame_tracking_enabled = False
+ return
+ self._publisher_progress_sequence += 1
+ try:
+ reported = callback(
+ self._pipeline_session_id,
+ event=event,
+ frames_delta=frames_delta,
+ sequence=self._publisher_progress_sequence,
+ observed_monotonic_seconds=time.monotonic(),
+ )
+ if not reported:
+ self._publisher_frame_tracking_enabled = False
+ except Exception as exc: # pragma: no cover - feedback must never stop media publication
+ self._publisher_frame_tracking_enabled = False
+ logger.warning(
+ "Could not report publisher frame progress: worker=%s session=%s error=%s",
+ self.worker_id,
+ self._pipeline_session_id,
+ exc,
+ )
+
+ def _abandon_publisher_frames(self, *, tracking: bool, total_frames: int, published_frames: int) -> None:
+ """Release frame credit for a chunk that will not reach LiveKit."""
+ if not tracking or published_frames >= total_frames:
+ return
+ self._report_publisher_frame_progress(
+ event="abandoned",
+ frames_delta=-(total_frames - published_frames),
+ )
+
def _on_data_message(
self,
record: SessionRecord,
@@ -256,10 +312,17 @@ async def _publish_pipeline_chunks(
published_frames = 0
next_frame_at: float | None = None
async for chunk in chunks:
- if self._stop_event.is_set():
- break
+ track_publisher_frames = self._publisher_frame_tracking_enabled and chunk.get("type") == "chunk"
+ chunk_published_frames = 0
frames, audio, metadata = split_chunk_media(chunk)
+ if self._stop_event.is_set():
+ self._abandon_publisher_frames(
+ tracking=track_publisher_frames,
+ total_frames=len(frames),
+ published_frames=chunk_published_frames,
+ )
+ break
model_output_callback = getattr(self.event_sink, "on_model_output", None)
if callable(model_output_callback) and self._pipeline_session_id is not None:
chunk_data_for_metrics = chunk.get("data") if isinstance(chunk.get("data"), dict) else chunk
@@ -290,8 +353,16 @@ async def _publish_pipeline_chunks(
frame_interval = 1.0 / fps
if frames and published_frames == 0 and wait_for_delivery_ack:
height, width = frames[0].shape[:2]
- await self.room_client.publish_video_track("telefuser-output", width, height, fps=fps)
- await asyncio.sleep(_VIDEO_TRACK_SUBSCRIPTION_GRACE_SECONDS)
+ try:
+ await self.room_client.publish_video_track("telefuser-output", width, height, fps=fps)
+ await asyncio.sleep(_VIDEO_TRACK_SUBSCRIPTION_GRACE_SECONDS)
+ except (asyncio.CancelledError, Exception):
+ self._abandon_publisher_frames(
+ tracking=track_publisher_frames,
+ total_frames=len(frames),
+ published_frames=chunk_published_frames,
+ )
+ raise
first_frame_at: float | None = None
for frame in frames:
@@ -301,14 +372,32 @@ async def _publish_pipeline_chunks(
if next_frame_at is None or now - next_frame_at > frame_interval:
next_frame_at = now
delay = next_frame_at - now
- if delay > 0:
- await asyncio.sleep(delay)
- await self.room_client.publish_video_frame(frame, fps=fps)
+ try:
+ if delay > 0:
+ await asyncio.sleep(delay)
+ await self.room_client.publish_video_frame(frame, fps=fps)
+ except (asyncio.CancelledError, Exception):
+ self._abandon_publisher_frames(
+ tracking=track_publisher_frames,
+ total_frames=len(frames),
+ published_frames=chunk_published_frames,
+ )
+ raise
+ chunk_published_frames += 1
+ if track_publisher_frames:
+ self._report_publisher_frame_progress(event="submitted", frames_delta=-1)
if first_frame_at is None:
first_frame_at = time.monotonic()
next_frame_at += frame_interval
published_frames += 1
+ self._abandon_publisher_frames(
+ tracking=track_publisher_frames,
+ total_frames=len(frames),
+ published_frames=chunk_published_frames,
+ )
+ if self._stop_event.is_set():
+ break
if audio is not None:
await self.room_client.publish_audio_frame(
audio.pcm,
@@ -316,8 +405,6 @@ async def _publish_pipeline_chunks(
channels=audio.channels,
)
- if self._stop_event.is_set():
- break
if frames:
published_callback = getattr(self.event_sink, "on_chunk_published", None)
if callable(published_callback):
diff --git a/tests/unit/pipelines/abot_world/test_denoising.py b/tests/unit/pipelines/abot_world/test_denoising.py
index 2224ae34..ec341ce1 100644
--- a/tests/unit/pipelines/abot_world/test_denoising.py
+++ b/tests/unit/pipelines/abot_world/test_denoising.py
@@ -4,10 +4,10 @@
import torch
-from telefuser.core.config import ModelRuntimeConfig
+from telefuser.core.config import AttentionConfig, AttnImplType, ModelRuntimeConfig
from telefuser.core.module_manager import ModuleManager
from telefuser.models.abot_world_dit import ABotWorldDiT
-from telefuser.pipelines.abot_world.denoising import ABotWorldDenoisingStage
+from telefuser.pipelines.abot_world.denoising import ABotWorldDenoisingStage, _ABotSteadyCudaGraph
def _stage_with_recording_dit() -> tuple[ABotWorldDenoisingStage, list[torch.Tensor]]:
@@ -87,3 +87,146 @@ def test_denoising_block_runs_four_model_updates_then_issues_context_cache_updat
for observed, timestep in zip(observed_timesteps[:4], expected, strict=True):
torch.testing.assert_close(observed, torch.full((1, 3), timestep))
assert torch.equal(observed_timesteps[-1], torch.zeros(1, 3))
+
+
+def test_interactive_cuda_graph_wrapper_falls_back_cleanly_on_cpu() -> None:
+ stage, observed_timesteps = _stage_with_recording_dit()
+ stage.configure_cuda_graph(True)
+ self_cache, cross_cache = stage._new_cache(batch_size=1, height=8, width=8)
+ scheduler = stage._scheduler()
+ generator = torch.Generator(device="cpu").manual_seed(43)
+ noise = torch.randn(1, 4, 3, 8, 8, generator=generator)
+
+ output = stage.denoise_interactive_block(
+ session_id="cpu-fallback",
+ latent=noise,
+ prompt_emb=torch.randn(1, 4, 16),
+ action_context=torch.randn(1, 32, 3, 16, 16),
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ current_start=3,
+ generator=generator,
+ scheduler=scheduler,
+ )
+
+ assert output.shape == noise.shape
+ assert len(observed_timesteps) == 5
+ assert stage.last_cuda_graph_metrics() == {
+ "cuda_graph_enabled": 1,
+ "cuda_graph_eligible": 0,
+ "cuda_graph_captured": 0,
+ "cuda_graph_replays": 0,
+ "cuda_graph_fallback": 0,
+ "cuda_graph_batch_size": 0,
+ "cuda_graph_batched": 0,
+ }
+ assert stage.cuda_graph_metrics()["replays"] == 0
+
+
+def test_sage_attention_is_explicitly_eager_only_until_graph_parity_is_available() -> None:
+ stage, _ = _stage_with_recording_dit()
+ stage.dit.set_attention_config(AttentionConfig.dense_attention(AttnImplType.SAGE_ATTN_2_8_8_SM90))
+
+ assert not stage._cuda_graph_backend_is_supported()
+
+
+def test_cuda_graph_capture_replays_slots_before_consuming_static_outputs() -> None:
+ """Capture records work; it must be replayed before sampler state advances."""
+ stage, _ = _stage_with_recording_dit()
+ scheduler = stage._scheduler()
+ generator = torch.Generator(device="cpu").manual_seed(71)
+ latent = torch.randn(1, 4, 3, 8, 8, generator=generator)
+ action_context = torch.randn(1, 32, 3, 16, 16, generator=generator)
+
+ class FakeGraph:
+ def __init__(self) -> None:
+ self.replays = 0
+
+ def replay(self) -> None:
+ self.replays += 1
+
+ entry_graph = FakeGraph()
+ refinement_graph = FakeGraph()
+ state = object.__new__(_ABotSteadyCudaGraph)
+ state.device = torch.device("cpu")
+ state.torch_dtype = stage.torch_dtype
+ state.dit = stage.dit
+ state.frames = latent.shape[2]
+ state.frame_tokens = 16
+ state.static_x = torch.empty_like(latent)
+ state.static_action = torch.empty_like(action_context)
+ state.static_timestep = torch.empty((1, latent.shape[2]), dtype=torch.float32)
+ state.static_context = torch.empty(1, 4, 16)
+ state.current_end = torch.empty(1, dtype=torch.long)
+ state.entry = None
+ state.refinement = None
+
+ def capture_slot(
+ _state: _ABotSteadyCudaGraph,
+ _self_cache: list[dict[str, object]],
+ _cross_cache: list[dict[str, object]],
+ *,
+ update_cache: bool,
+ ) -> object:
+ graph = entry_graph if update_cache else refinement_graph
+ return type("Slot", (), {"graph": graph, "output": torch.zeros_like(latent)})()
+
+ state._capture_slot = MethodType(capture_slot, state)
+ output, replays = state.run(
+ stage,
+ latent,
+ action_context,
+ [{}],
+ [{}],
+ current_start=3,
+ generator=generator,
+ scheduler=scheduler,
+ capture=True,
+ )
+
+ assert output.shape == latent.shape
+ assert replays == 4
+ assert entry_graph.replays == 1
+ assert refinement_graph.replays == 3
+
+
+def test_batched_cuda_graph_arena_binds_rows_and_keeps_independent_cursors() -> None:
+ """The B=2 arena owns K/V while each retained session keeps its cursor."""
+ stage, _ = _stage_with_recording_dit()
+ self_caches = []
+ cross_caches = []
+ for start in (18, 21):
+ self_cache, cross_cache = stage._new_cache(batch_size=1, height=8, width=8)
+ for self_layer, cross_layer in zip(self_cache, cross_cache, strict=True):
+ self_layer["local_end_index"].fill_(stage.dit.local_attn_size * 16)
+ self_layer["global_end_index"].fill_(start * 16)
+ cross_layer["is_init"] = True
+ cross_layer["sequence_length"] = 4
+ self_caches.append(self_cache)
+ cross_caches.append(cross_cache)
+
+ latent = torch.randn(2, 4, 3, 8, 8)
+ prompt_emb = torch.randn(2, 4, 16)
+ action_context = torch.randn(2, 32, 3, 16, 16)
+ state = stage._create_batched_cuda_graph_state(
+ ("a", "b"),
+ latent,
+ prompt_emb,
+ action_context,
+ self_caches,
+ cross_caches,
+ current_starts=(18, 21),
+ )
+
+ stage._bind_batched_cache_arena(state, self_caches, cross_caches)
+ assert state.matches_members(("a", "b"), self_caches, cross_caches)
+ for row, (self_cache, cross_cache) in enumerate(zip(self_caches, cross_caches, strict=True)):
+ assert self_cache[0]["k"].data_ptr() == state.self_cache[0]["k"][row : row + 1].data_ptr()
+ assert cross_cache[0]["v"].data_ptr() == state.cross_cache[0]["v"][row : row + 1].data_ptr()
+
+ stage._advance_batched_cache_cursors(self_caches, current_starts=(18, 21), latent=latent)
+
+ assert int(self_caches[0][0]["global_end_index"].item()) == 21 * 16
+ assert int(self_caches[1][0]["global_end_index"].item()) == 24 * 16
+ assert int(self_caches[0][0]["local_end_index"].item()) == stage.dit.local_attn_size * 16
+ assert int(self_caches[1][0]["local_end_index"].item()) == stage.dit.local_attn_size * 16
diff --git a/tests/unit/pipelines/abot_world/test_livekit_examples.py b/tests/unit/pipelines/abot_world/test_livekit_examples.py
index 6eb0d7de..c4b3b640 100644
--- a/tests/unit/pipelines/abot_world/test_livekit_examples.py
+++ b/tests/unit/pipelines/abot_world/test_livekit_examples.py
@@ -15,6 +15,14 @@ def test_livekit_service_entrypoint_builds_single_gpu_abot_service(monkeypatch:
monkeypatch.delenv("TELEFUSER_ABOT_SCHEDULER_MODE", raising=False)
monkeypatch.delenv("TELEFUSER_ABOT_MAX_BATCH_SIZE", raising=False)
monkeypatch.delenv("TELEFUSER_ABOT_BATCHING_WINDOW_MS", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_ENABLED", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_GUARD_MS", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE", raising=False)
+ monkeypatch.delenv("TELEFUSER_ABOT_BATCH_COMPUTE_SAFETY_FACTOR", raising=False)
def fake_get_pipeline(**kwargs: object) -> object:
captured.update(kwargs)
@@ -31,8 +39,12 @@ def fake_get_pipeline(**kwargs: object) -> object:
assert service.default_session_config["control_latent_frames"] == 3
assert service.scheduler_mode == "batched"
assert service.max_batch_size == 2
+ assert not service.publisher_frame_credit_enabled
assert service.default_session_config["seed"] == 42
assert service.default_session_config["prompt"] == service_example.DEFAULT_PROMPT
+ assert service.batch_compute_profile_name == "none"
+ assert service.batch_compute_safety_factor == pytest.approx(1.10)
+ assert service._batch_compute_priors == {}
assert str(service.default_session_config["image_path"]).endswith("84b90ad568b693d2.png")
@@ -44,6 +56,14 @@ def test_livekit_service_entrypoint_selects_batched_four_session_schedule_from_e
monkeypatch.setenv("TELEFUSER_ABOT_SCHEDULER_MODE", "batched")
monkeypatch.setenv("TELEFUSER_ABOT_MAX_BATCH_SIZE", "4")
monkeypatch.setenv("TELEFUSER_ABOT_BATCHING_WINDOW_MS", "2")
+ monkeypatch.setenv("TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS", "125")
+ monkeypatch.setenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_ENABLED", "true")
+ monkeypatch.setenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS", "1.5")
+ monkeypatch.setenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES", "36")
+ monkeypatch.setenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES", "4")
+ monkeypatch.setenv("TELEFUSER_ABOT_BATCH_COMPUTE_SAFETY_FACTOR", "1.05")
+ monkeypatch.setenv("TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_GUARD_MS", "50")
+ monkeypatch.setenv("TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE", "h100_lf3_eager_full_pipeline_v1")
service = service_example.get_service(gpu_num=1, gpu_ids=["0"])
@@ -51,14 +71,32 @@ def test_livekit_service_entrypoint_selects_batched_four_session_schedule_from_e
assert service.scheduler_mode == "batched"
assert service.max_batch_size == 4
assert service.batching_window_seconds == pytest.approx(0.002)
+ assert service.max_deadline_batch_wait_seconds == pytest.approx(0.125)
+
+ assert service.publisher_frame_credit_enabled
+ assert service.publisher_frame_credit_target_seconds == pytest.approx(1.5)
+ assert service.publisher_frame_credit_target_frames == 36
+ assert service.batch_compute_safety_factor == pytest.approx(1.05)
+ assert service.publisher_frame_credit_reserve_frames == 4
+ assert service.publisher_frame_credit_guard_seconds == pytest.approx(0.05)
+ assert service.batch_compute_profile_name == "h100_lf3_eager_full_pipeline_v1"
+ assert service._batch_compute_priors == {
+ 2: pytest.approx(0.7404982000589371),
+ 3: pytest.approx(1.0691392589360476),
+ 4: pytest.approx(1.407263021916151),
+ }
@pytest.mark.parametrize(
("environment", "expected"),
[
({"TELEFUSER_ABOT_SCHEDULER_MODE": "unknown"}, "SCHEDULER_MODE"),
+ ({"TELEFUSER_ABOT_BATCH_COMPUTE_SAFETY_FACTOR": "0.99"}, "BATCH_COMPUTE_SAFETY_FACTOR"),
({"TELEFUSER_ABOT_MAX_BATCH_SIZE": "0"}, "MAX_BATCH_SIZE"),
({"TELEFUSER_ABOT_BATCHING_WINDOW_MS": "nan"}, "BATCHING_WINDOW_MS"),
+ ({"TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS": "nan"}, "MAX_DEADLINE_BATCH_WAIT_MS"),
+ ({"TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES": "0"}, "TARGET_FRAMES"),
+ ({"TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE": "not-a-profile"}, "BATCH_COMPUTE_PROFILE"),
],
)
def test_livekit_service_entrypoint_rejects_invalid_schedule_environment(
diff --git a/tests/unit/pipelines/abot_world/test_livekit_service.py b/tests/unit/pipelines/abot_world/test_livekit_service.py
index 0e4b9a80..652ae913 100644
--- a/tests/unit/pipelines/abot_world/test_livekit_service.py
+++ b/tests/unit/pipelines/abot_world/test_livekit_service.py
@@ -11,7 +11,11 @@
from PIL import Image
from telefuser.pipelines.abot_world.interactive import ABotWorldSessionLifecycle
-from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService, _ABotWorldLiveKitSession
+from telefuser.pipelines.abot_world.service import (
+ _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS,
+ ABotWorldLiveKitService,
+ _ABotWorldLiveKitSession,
+)
from telefuser.service.core.stream_pipeline_service import BidirectionalService
@@ -35,7 +39,7 @@ def is_resident(self) -> bool:
class _FakePipeline:
- def __init__(self) -> None:
+ def __init__(self, *, use_relative_rope: bool = True) -> None:
self.config = SimpleNamespace(width=8, height=8)
self.device = torch.device("cpu")
self.torch_dtype = torch.float32
@@ -47,6 +51,7 @@ def __init__(self) -> None:
num_layers=2,
local_attn_size=18,
text_len=8,
+ use_relative_rope=use_relative_rope,
)
)
self.generate_calls: list[tuple[str, dict[str, bool]]] = []
@@ -129,8 +134,12 @@ def close(self) -> None:
self.closed = True
-def _service(**kwargs: object) -> tuple[ABotWorldLiveKitService, _FakePipeline]:
- pipeline = _FakePipeline()
+def _service(
+ *,
+ use_relative_rope: bool = True,
+ **kwargs: object,
+) -> tuple[ABotWorldLiveKitService, _FakePipeline]:
+ pipeline = _FakePipeline(use_relative_rope=use_relative_rope)
service = ABotWorldLiveKitService(
pipeline,
default_session_config={"prompt": "test prompt"},
@@ -179,14 +188,41 @@ def test_service_matches_shared_multi_session_bidirectional_contract() -> None:
service.stop()
+def test_offline_batch_compute_prior_seeds_b2_then_keeps_online_high_water() -> None:
+ service, _ = _service(
+ max_batch_size=2,
+ batch_compute_profile_name="h100_lf3_eager_full_pipeline_v1",
+ batch_compute_prior_seconds={2: 0.7404982000589371},
+ batch_compute_safety_factor=1.05,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first, second = (_create(service, value) for value in ("first", "second"))
+ states = [service._session(session_id) for session_id in (first, second)]
+ assert all(state is not None for state in states)
+ try:
+ assert service._estimated_batch_compute_seconds(states) == pytest.approx(0.7404982000589371 * 1.05)
+ service._batch_compute_estimates[2] = 0.80
+ assert service._estimated_batch_compute_seconds(states) == pytest.approx(0.84)
+ assert service.runtime_metrics()["batch_compute_safety_factor"] == pytest.approx(1.05)
+ assert service.runtime_metrics()["batch_compute_profile_name"] == "h100_lf3_eager_full_pipeline_v1"
+ finally:
+ service.stop()
+
+
def test_capacity_profile_accepts_explicit_cuda_device_string(monkeypatch) -> None:
service, pipeline = _service()
pipeline.device = "cuda:3"
monkeypatch.setattr("telefuser.pipelines.abot_world.service.torch.cuda.is_available", lambda: True)
- monkeypatch.setattr(service, "_profile_session_memory", lambda: {
- "profiled_session_bytes": 100,
- "workspace_peak_bytes": 200,
- })
+ monkeypatch.setattr(
+ service,
+ "_profile_session_memory",
+ lambda: {
+ "profiled_session_bytes": 100,
+ "workspace_peak_bytes": 200,
+ },
+ )
observed = {}
def fake_mem_get_info(device):
@@ -201,15 +237,20 @@ def fake_mem_get_info(device):
assert profile["effective_capacity"] == 2
service.stop()
+
def test_batched_capacity_accounts_for_active_batch_workspace(monkeypatch) -> None:
service, pipeline = _service(max_batch_size=8)
pipeline.device = "cuda:0"
monkeypatch.setattr("telefuser.pipelines.abot_world.service.torch.cuda.is_available", lambda: True)
monkeypatch.setattr(service, "_estimate_session_bytes", lambda: 100)
- monkeypatch.setattr(service, "_profile_session_memory", lambda: {
- "profiled_session_bytes": 100,
- "workspace_peak_bytes": 200,
- })
+ monkeypatch.setattr(
+ service,
+ "_profile_session_memory",
+ lambda: {
+ "profiled_session_bytes": 100,
+ "workspace_peak_bytes": 200,
+ },
+ )
monkeypatch.setattr(
"telefuser.pipelines.abot_world.service.torch.cuda.mem_get_info",
lambda device: (1_000, 2_000),
@@ -224,7 +265,6 @@ def test_batched_capacity_accounts_for_active_batch_workspace(monkeypatch) -> No
service.stop()
-
def test_two_ready_sessions_are_generated_in_one_batch_and_keep_order() -> None:
service, pipeline = _service(output_queue_size=4, batching_window_ms=30)
service.configure_session_capacity(2)
@@ -441,6 +481,539 @@ def test_latest_mode_rendezvouses_staggered_continuations_within_deadline_slack(
service.stop()
+def test_deadline_batch_wait_uses_one_persistent_timeout_then_falls_back() -> None:
+ service, _ = _service(
+ output_queue_size=4,
+ batching_window_ms=2,
+ max_deadline_batch_wait_ms=150,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ first_state = service._session(first)
+ assert first_state is not None
+ try:
+ now = 70_000.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.70})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+
+ ready = service._ready_sessions(now)
+ first_wait = service._batch_formation_wait_seconds(ready, now)
+ assert first_wait == pytest.approx(0.15)
+ assert first_state.deadline_batch_wait_until == pytest.approx(now + 0.15)
+
+ # An unrelated condition wake must not restart the full timeout.
+ remaining_wait = service._batch_formation_wait_seconds(ready, now + 0.05)
+ assert remaining_wait == pytest.approx(0.10)
+
+ timeout_wait = service._batch_formation_wait_seconds(ready, now + 0.151)
+ assert timeout_wait == 0.0
+ assert first_state.deadline_batch_wait_until is None
+ assert service.runtime_metrics()["deadline_batch_wait_timeouts"] == 1
+ finally:
+ service.stop()
+
+
+def test_deadline_batch_wait_dispatches_b2_when_peer_arrives_before_timeout() -> None:
+ service, pipeline = _service(
+ output_queue_size=4,
+ batching_window_ms=2,
+ max_deadline_batch_wait_ms=200,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first, second = (_create(service, value) for value in ("first", "second"))
+ first_state = service._session(first)
+ second_state = service._session(second)
+ assert first_state is not None and second_state is not None
+ try:
+ now = 71_000.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.60})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ initial_wait = service._batch_formation_wait_seconds(service._ready_sessions(now), now)
+ assert initial_wait == pytest.approx(0.20)
+
+ peer_ready_at = now + 0.05
+ _prepare_latest_continuation(
+ second_state,
+ now=peer_ready_at,
+ pacing_ready_at=peer_ready_at,
+ next_playout_deadline=peer_ready_at + 1.0,
+ )
+ ready = service._ready_sessions(peer_ready_at)
+ batch = service._select_batch(ready, now=peer_ready_at)
+ assert [state.session_id for state in batch] == [first, second]
+ assert service._batch_formation_wait_seconds(ready, peer_ready_at) == 0.0
+ service._execute_batch(batch, [{"W": True}, {"D": True}])
+ assert pipeline.batch_sizes[-1] == 2
+ finally:
+ service.stop()
+
+
+def test_deadline_batch_wait_promotes_held_b2_to_b3_for_safe_future_peer() -> None:
+ """A persistent A hold may keep A+B together long enough to board safe C."""
+ service, pipeline = _service(
+ output_queue_size=4,
+ batching_window_ms=2,
+ max_batch_size=3,
+ max_deadline_batch_wait_ms=500,
+ batch_compute_safety_factor=1.0,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(3)
+ first, second, third = (_create(service, value) for value in ("first", "second", "third"))
+ states = [service._session(session_id) for session_id in (first, second, third)]
+ assert all(state is not None for state in states)
+ first_state, second_state, third_state = states
+ assert first_state is not None and second_state is not None and third_state is not None
+ try:
+ now = 71_500.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.60, 3: 0.70})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.50,
+ )
+ # A starts one persistent B=2-bounded hold before either peer is ready.
+ assert service._batch_formation_wait_seconds(service._ready_sessions(now), now) == pytest.approx(0.50)
+ assert first_state.deadline_batch_wait_until == pytest.approx(now + 0.50)
+
+ peer_ready_at = now + 0.05
+ _prepare_latest_continuation(
+ second_state,
+ now=peer_ready_at,
+ pacing_ready_at=peer_ready_at,
+ next_playout_deadline=peer_ready_at + 1.50,
+ )
+ # Latest-mode pacing admits a continuation 10 ms ahead of its nominal
+ # pacing point. C therefore becomes eligible at ``third_release_at``.
+ third_release_at = now + 0.15
+ _prepare_latest_continuation(
+ third_state,
+ now=peer_ready_at,
+ pacing_ready_at=third_release_at + 0.01,
+ next_playout_deadline=now + 1.70,
+ )
+
+ ready = service._ready_sessions(peer_ready_at)
+ assert [state.session_id for state in ready] == [first, second]
+ expected_wait = third_release_at - peer_ready_at + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS
+ assert service._batch_formation_wait_seconds(ready, peer_ready_at) == pytest.approx(expected_wait)
+ # The original A cap remains the B=2 fallback, rather than being reset
+ # when B arrives.
+ assert first_state.deadline_batch_wait_until == pytest.approx(now + 0.50)
+
+ selected_at = peer_ready_at + expected_wait
+ ready = service._ready_sessions(selected_at)
+ assert [state.session_id for state in ready] == [first, second, third]
+ batch = service._select_batch(ready, now=selected_at)
+ assert [state.session_id for state in batch] == [first, second, third]
+ assert selected_at <= service._latest_safe_batch_start(batch, now=selected_at)
+ service._execute_batch(batch, [{"W": True}, {"D": True}, {"A": True}])
+ assert pipeline.batch_sizes[-1] == 3
+ finally:
+ service.stop()
+
+
+def test_deadline_batch_wait_dispatches_b2_when_future_third_peer_misses_b3_deadline() -> None:
+ """A C that is safe for B=2 but late for B=3 must not extend the A+B hold."""
+ service, pipeline = _service(
+ output_queue_size=4,
+ batching_window_ms=2,
+ max_batch_size=3,
+ max_deadline_batch_wait_ms=500,
+ batch_compute_safety_factor=1.0,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(3)
+ first, second, third = (_create(service, value) for value in ("first", "second", "third"))
+ states = [service._session(session_id) for session_id in (first, second, third)]
+ assert all(state is not None for state in states)
+ first_state, second_state, third_state = states
+ assert first_state is not None and second_state is not None and third_state is not None
+ try:
+ now = 71_600.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.60, 3: 0.65})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 0.80,
+ )
+ # B=2 is the A fallback and fixes the original hold at +200 ms.
+ assert service._batch_formation_wait_seconds(service._ready_sessions(now), now) == pytest.approx(0.20)
+ assert first_state.deadline_batch_wait_until == pytest.approx(now + 0.20)
+
+ peer_ready_at = now + 0.05
+ _prepare_latest_continuation(
+ second_state,
+ now=peer_ready_at,
+ pacing_ready_at=peer_ready_at,
+ next_playout_deadline=peer_ready_at + 0.80,
+ )
+ third_release_at = now + 0.18
+ _prepare_latest_continuation(
+ third_state,
+ now=peer_ready_at,
+ pacing_ready_at=third_release_at + 0.01,
+ next_playout_deadline=now + 1.00,
+ )
+
+ ready = service._ready_sessions(peer_ready_at)
+ assert [state.session_id for state in ready] == [first, second]
+ b2_latest_safe_start = service._latest_safe_batch_start(ready, now=peer_ready_at)
+ assert b2_latest_safe_start == pytest.approx(now + 0.20)
+ b3_latest_safe_start = service._latest_safe_batch_start([*ready, third_state], now=peer_ready_at)
+ assert b3_latest_safe_start == pytest.approx(now + 0.15)
+ assert third_release_at + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS <= b2_latest_safe_start
+ assert third_release_at + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS > b3_latest_safe_start
+
+ # C could arrive before the retained B=2 fallback, but it would miss
+ # the B=3 deadline. Launch B=2 now rather than extending the hold.
+ assert service._batch_formation_wait_seconds(ready, peer_ready_at) == 0.0
+ batch = service._select_batch(ready, now=peer_ready_at)
+ assert [state.session_id for state in batch] == [first, second]
+ assert peer_ready_at <= service._latest_safe_batch_start(batch, now=peer_ready_at)
+ service._execute_batch(batch, [{"W": True}, {"D": True}])
+ assert pipeline.batch_sizes[-1] == 2
+ finally:
+ service.stop()
+
+
+def test_deadline_batch_wait_is_clamped_by_predicted_b2_deadline() -> None:
+ service, _ = _service(
+ output_queue_size=4,
+ batching_window_ms=2,
+ max_deadline_batch_wait_ms=300,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first = _create(service, "first")
+ first_state = service._session(first)
+ assert first_state is not None
+ try:
+ now = 72_000.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.75})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+
+ wait_seconds = service._batch_formation_wait_seconds(service._ready_sessions(now), now)
+ # B=2 uses a 10% safety factor: 1.0 - 1.1 * 0.75 = 0.175 s.
+ assert wait_seconds == pytest.approx(0.175)
+ finally:
+ service.stop()
+
+
+@pytest.mark.parametrize(
+ ("first_deadline_offset", "urgent_deadline_offset", "expected_session", "expected_fillers"),
+ [
+ (1.0, 0.30, "urgent", 1),
+ (0.70, 0.50, "first", 0),
+ ],
+ ids=("runs-safe-earlier-edf-filler", "protects-held-singleton-fallback"),
+)
+def test_deadline_batch_wait_only_allows_edf_work_that_preserves_held_fallback(
+ first_deadline_offset: float,
+ urgent_deadline_offset: float,
+ expected_session: str,
+ expected_fillers: int,
+) -> None:
+ service, _ = _service(
+ output_queue_size=4,
+ batching_window_ms=2,
+ max_deadline_batch_wait_ms=250,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ first, urgent = (_create(service, value) for value in ("first", "urgent"))
+ first_state = service._session(first)
+ urgent_state = service._session(urgent)
+ assert first_state is not None and urgent_state is not None
+ try:
+ now = 73_000.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.06})
+ if expected_session == first:
+ service._batch_compute_estimates.update({1: 0.40, 2: 0.50})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + first_deadline_offset,
+ )
+ # Start the peer-wait before the incompatible EDF job appears.
+ assert service._batch_formation_wait_seconds(service._ready_sessions(now), now) > 0
+
+ _prepare_latest_continuation(
+ urgent_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + urgent_deadline_offset,
+ )
+ urgent_state.pipeline_session.self_cache[0]["local_end_index"].fill_(1)
+ ready = service._ready_sessions(now)
+ assert [state.session_id for state in ready] == [urgent, first]
+
+ assert service._batch_formation_wait_seconds(ready, now) == 0.0
+ batch = service._select_batch(ready, now=now)
+ assert [state.session_id for state in batch] == [expected_session]
+ assert service.runtime_metrics()["deadline_batch_filler_dispatches"] == expected_fillers
+ finally:
+ service.stop()
+
+
+def test_latest_mode_aligns_three_staggered_lf3_continuations_without_frame_shrinking() -> None:
+ """A 3-way LF3 batch forms through safe wakeups, not unvalidated LF1/LF2 bridges."""
+ service, pipeline = _service(output_queue_size=4, batching_window_ms=2, control_idle_timeout=30)
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(3)
+ first, second, third = (_create(service, value) for value in ("first", "second", "third"))
+ states = [service._session(session_id) for session_id in (first, second, third)]
+ assert all(state is not None for state in states)
+ first_state, second_state, third_state = states
+ assert first_state is not None and second_state is not None and third_state is not None
+ try:
+ now = 40_000.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.06, 3: 0.08})
+ _prepare_latest_continuation(
+ first_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ _prepare_latest_continuation(
+ second_state,
+ now=now,
+ pacing_ready_at=now + 0.06,
+ next_playout_deadline=now + 1.06,
+ )
+ _prepare_latest_continuation(
+ third_state,
+ now=now,
+ pacing_ready_at=now + 0.12,
+ next_playout_deadline=now + 1.12,
+ )
+
+ ready = service._ready_sessions(now)
+ assert [state.session_id for state in ready] == [first]
+ first_wait = service._batch_formation_wait_seconds(ready, now)
+ assert 0 < first_wait < 0.10
+
+ after_first_wait = now + first_wait + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS
+ ready = service._ready_sessions(after_first_wait)
+ assert [state.session_id for state in ready] == [first, second]
+ second_wait = service._batch_formation_wait_seconds(ready, after_first_wait)
+ assert 0 < second_wait < 0.10
+
+ aligned_at = after_first_wait + second_wait + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS
+ ready = service._ready_sessions(aligned_at)
+ batch = service._select_batch(ready, now=aligned_at)
+ assert [state.session_id for state in batch] == [first, second, third]
+ assert aligned_at <= service._latest_safe_batch_start(batch)
+
+ service._execute_batch(batch, [{"W": True}, {"D": True}, {"A": True}])
+ assert pipeline.batch_sizes[-1] == 3
+ assert [state.pipeline_session.next_latent_frame for state in batch] == [9, 9, 9]
+ finally:
+ service.stop()
+
+
+@pytest.mark.parametrize(
+ ("lagging_local_end", "expected_ids", "expected_batch_size"),
+ [
+ (72, ["ahead", "lagging"], 2),
+ (60, ["ahead"], 1),
+ ],
+ ids=("matching-local-window", "different-local-window"),
+)
+def test_relative_rope_rendezvouses_mixed_global_positions_with_compatible_local_windows(
+ lagging_local_end: int,
+ expected_ids: list[str],
+ expected_batch_size: int,
+) -> None:
+ """Only a matching retained KV layout may join a Relative-RoPE micro-batch."""
+ service, pipeline = _service(
+ use_relative_rope=True,
+ output_queue_size=4,
+ batching_window_ms=20,
+ control_idle_timeout=30,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ ahead, lagging = (_create(service, value) for value in ("ahead", "lagging"))
+ ahead_state = service._session(ahead)
+ lagging_state = service._session(lagging)
+ assert ahead_state is not None and lagging_state is not None
+ try:
+ now = 45_000.0
+ service._batch_compute_estimates.update({1: 0.05, 2: 0.06})
+ _prepare_latest_continuation(
+ ahead_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ _prepare_latest_continuation(
+ lagging_state,
+ now=now,
+ pacing_ready_at=now + 0.025,
+ next_playout_deadline=now + 1.025,
+ )
+ ahead_state.pipeline_session.next_latent_frame = 9
+ lagging_state.pipeline_session.next_latent_frame = 12
+ ahead_cache = ahead_state.pipeline_session.self_cache[0]
+ lagging_cache = lagging_state.pipeline_session.self_cache[0]
+ ahead_cache["global_end_index"].fill_(108)
+ lagging_cache["global_end_index"].fill_(144)
+ ahead_cache["local_end_index"].fill_(72)
+ lagging_cache["local_end_index"].fill_(lagging_local_end)
+
+ ready = service._ready_sessions(now)
+ assert [state.session_id for state in ready] == [ahead]
+ wait_seconds = service._batch_formation_wait_seconds(ready, now)
+ if expected_batch_size == 2:
+ assert 0 < wait_seconds < service.batching_window_seconds
+ else:
+ assert wait_seconds == pytest.approx(service.batching_window_seconds)
+
+ selected_at = now + wait_seconds + _PACING_RENDEZVOUS_WAKE_GUARD_SECONDS
+ ready = service._ready_sessions(selected_at)
+ assert [state.session_id for state in ready] == [ahead, lagging]
+ assert (service._batch_key(ahead_state) == service._batch_key(lagging_state)) is (expected_batch_size == 2)
+ batch = service._select_batch(ready, now=selected_at)
+ assert [state.session_id for state in batch] == expected_ids
+ service._execute_batch(batch, [{"W": True} for _ in batch])
+ assert pipeline.batch_sizes[-1] == expected_batch_size
+ finally:
+ service.stop()
+
+
+def test_absolute_rope_sessions_at_different_positions_do_not_share_a_batch() -> None:
+ """A lagging LF3 session must catch up alone before it can phase-lock."""
+ service, _ = _service(use_relative_rope=False, output_queue_size=4, batching_window_ms=2, control_idle_timeout=30)
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ ahead, lagging = (_create(service, value) for value in ("ahead", "lagging"))
+ ahead_state = service._session(ahead)
+ lagging_state = service._session(lagging)
+ assert ahead_state is not None and lagging_state is not None
+ try:
+ now = 50_000.0
+ _prepare_latest_continuation(
+ ahead_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ _prepare_latest_continuation(
+ lagging_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ ahead_state.pipeline_session.next_latent_frame = 9
+ lagging_state.pipeline_session.next_latent_frame = 6
+
+ ready = service._ready_sessions(now)
+
+ assert [state.session_id for state in ready] == [ahead, lagging]
+ assert service._batch_key(ahead_state) != service._batch_key(lagging_state)
+ assert [state.session_id for state in service._select_batch(ready, now=now)] == [ahead]
+ finally:
+ service.stop()
+
+
+def test_lagging_lf3_session_can_catch_up_then_rejoin_a_deadline_safe_batch() -> None:
+ """A conservative phase-lock trace needs no LF shrink or model-interface change."""
+ service, pipeline = _service(
+ use_relative_rope=False, output_queue_size=4, batching_window_ms=2, control_idle_timeout=30
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(2)
+ ahead, lagging = (_create(service, value) for value in ("ahead", "lagging"))
+ ahead_state = service._session(ahead)
+ lagging_state = service._session(lagging)
+ assert ahead_state is not None and lagging_state is not None
+ assert _take_and_notify(service, ahead_state)["type"] == "preview"
+ assert _take_and_notify(service, lagging_state)["type"] == "preview"
+ try:
+ now = 60_000.0
+ _prepare_latest_continuation(
+ ahead_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ _prepare_latest_continuation(
+ lagging_state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 0.5,
+ )
+ ahead_state.pipeline_session.next_latent_frame = 9
+ lagging_state.pipeline_session.next_latent_frame = 6
+
+ catch_up = service._select_batch(service._ready_sessions(now), now=now)
+ assert [state.session_id for state in catch_up] == [lagging]
+ assert now <= service._latest_safe_batch_start(catch_up)
+ service._execute_batch(catch_up, [{"W": True}])
+ assert pipeline.batch_sizes[-1] == 1
+ assert lagging_state.pipeline_session.next_latent_frame == 9
+ assert _take_and_notify(service, lagging_state)["type"] == "chunk"
+
+ # Once the lagging session has completed a full LF3 block, it has the
+ # same Absolute-RoPE position as the buffered-ahead peer. The existing
+ # deadline-safe selector can now form B=2 without any frame shrink.
+ aligned_at = time.monotonic()
+ service._batch_compute_estimates.update({1: 0.04, 2: 0.06})
+ for state in (ahead_state, lagging_state):
+ _prepare_latest_continuation(
+ state,
+ now=aligned_at,
+ pacing_ready_at=aligned_at,
+ next_playout_deadline=aligned_at + 1.0,
+ )
+ state.pipeline_session.next_latent_frame = 9
+ phase_locked = service._select_batch(service._ready_sessions(aligned_at), now=aligned_at)
+ assert [state.session_id for state in phase_locked] == [ahead, lagging]
+ assert aligned_at <= service._latest_safe_batch_start(phase_locked)
+ service._execute_batch(phase_locked, [{"W": True}, {"D": True}])
+ assert pipeline.batch_sizes[-2:] == [1, 2]
+ assert [state.pipeline_session.next_latent_frame for state in phase_locked] == [12, 12]
+ finally:
+ service.stop()
+
+
def test_latest_mode_rendezvous_never_waits_past_a_playout_deadline() -> None:
service, _ = _service(output_queue_size=4, batching_window_ms=2, control_idle_timeout=30)
with service._scheduler_condition:
@@ -474,7 +1047,9 @@ def test_latest_mode_rendezvous_never_waits_past_a_playout_deadline() -> None:
assert wait_seconds == pytest.approx(service.batching_window_seconds)
assert now + wait_seconds < service._latest_safe_batch_start([first_state, second_state])
ready_after_window = service._ready_sessions(now + wait_seconds)
- assert [state.session_id for state in service._select_batch(ready_after_window, now=now + wait_seconds)] == [first]
+ assert [state.session_id for state in service._select_batch(ready_after_window, now=now + wait_seconds)] == [
+ first
+ ]
finally:
service.stop()
@@ -637,3 +1212,151 @@ def test_invalid_livekit_control_payloads_are_rejected(payload: dict) -> None:
service.push_chunk(session_id, payload)
finally:
service.stop()
+
+
+def test_publisher_frame_credit_moves_dequeued_chunk_and_ignores_duplicate_progress() -> None:
+ service, _ = _service(
+ output_queue_size=4,
+ publisher_frame_credit_enabled=True,
+ publisher_frame_credit_target_seconds=1.5,
+ publisher_frame_credit_reserve_frames=4,
+ publisher_frame_credit_guard_ms=50,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(1)
+ session_id = _create(service, "frame-credit", fps=12, control_latent_frames=3)
+ state = service._session(session_id)
+ assert state is not None
+ try:
+ assert state.output_queue.get(timeout=1)["type"] == "preview"
+ assert service.enable_publisher_frame_tracking(session_id)
+ state.output_queue.put({"type": "chunk", "frames": [Image.new("RGB", (8, 8)) for _ in range(12)]})
+
+ async def pull_one() -> dict:
+ iterator = service.pull_chunks(session_id)
+ payload = await anext(iterator)
+ await iterator.aclose()
+ return payload
+
+ assert asyncio.run(pull_one())["type"] == "chunk"
+ metrics = service.runtime_metrics(session_id)
+ assert metrics["queued_video_frames"] == 0
+ assert metrics["publisher_unsubmitted_frames"] == 12
+ assert metrics["frame_credit_frames"] == 12
+ assert service.report_publisher_frame_progress(session_id, event="submitted", frames_delta=-1, sequence=1)
+ assert not service.report_publisher_frame_progress(session_id, event="submitted", frames_delta=-1, sequence=1)
+ metrics = service.runtime_metrics(session_id)
+ assert metrics["publisher_unsubmitted_frames"] == 11
+ assert metrics["publisher_frames_submitted"] == 1
+ finally:
+ service.stop()
+
+
+def test_frame_credit_edf_uses_livekit_buffer_not_queue_payload_count() -> None:
+ service, _ = _service(
+ output_queue_size=4,
+ publisher_frame_credit_enabled=True,
+ publisher_frame_credit_target_seconds=1.5,
+ publisher_frame_credit_reserve_frames=4,
+ publisher_frame_credit_guard_ms=50,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(1)
+ session_id = _create(service, "edf-credit", fps=12, control_latent_frames=3)
+ state = service._session(session_id)
+ assert state is not None
+ try:
+ state.output_queue.get(timeout=1)
+ assert service.enable_publisher_frame_tracking(session_id)
+ now = 91_000.0
+ _prepare_latest_continuation(
+ state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 0.01,
+ )
+ with state.lock:
+ state.publisher_unsubmitted_frames = 19
+ assert service._ready_sessions(now) == []
+ assert service._frame_credit_ready_at(state, now) == pytest.approx(now + 1 / 12)
+ with state.lock:
+ state.publisher_unsubmitted_frames = 18
+ assert service._ready_sessions(now) == [state]
+ assert service._session_deadline(state, now) == pytest.approx(now + (14 / 12) - 0.05)
+ metrics = service.runtime_metrics(session_id)
+ assert metrics["frame_credit_frames"] == 18
+ assert metrics["frame_credit_target_frames"] == 18
+ assert metrics["pacing_buffered_video_payloads"] == 0
+ finally:
+ service.stop()
+
+
+def test_frame_credit_explicit_target_frames_overrides_seconds_but_not_reserve_deadline_floor() -> None:
+ service, _ = _service(
+ output_queue_size=4,
+ publisher_frame_credit_enabled=True,
+ publisher_frame_credit_target_seconds=1.5,
+ publisher_frame_credit_target_frames=36,
+ publisher_frame_credit_reserve_frames=4,
+ publisher_frame_credit_guard_ms=50,
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(1)
+ session_id = _create(service, "edf-credit-frames", fps=12, control_latent_frames=3)
+ state = service._session(session_id)
+ assert state is not None
+ try:
+ state.output_queue.get(timeout=1)
+ assert service.enable_publisher_frame_tracking(session_id)
+ now = 92_000.0
+ _prepare_latest_continuation(
+ state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 0.01,
+ )
+ with state.lock:
+ state.publisher_unsubmitted_frames = 37
+ assert service._ready_sessions(now) == []
+ assert service._frame_credit_ready_at(state, now) == pytest.approx(now + 1 / 12)
+ with state.lock:
+ state.publisher_unsubmitted_frames = 36
+ assert service._ready_sessions(now) == [state]
+ assert service._session_deadline(state, now) == pytest.approx(now + (32 / 12) - 0.05)
+ metrics = service.runtime_metrics(session_id)
+ assert metrics["frame_credit_frames"] == 36
+ assert metrics["frame_credit_target_frames"] == 36
+ finally:
+ service.stop()
+
+
+def test_offline_batch_compute_prior_expands_b2_deadline_wait_before_first_batch() -> None:
+ prior_seconds = 0.7404982000589371
+ service, _ = _service(
+ max_batch_size=2,
+ max_deadline_batch_wait_ms=1000,
+ batch_compute_profile_name="h100_lf3_eager_full_pipeline_v1",
+ batch_compute_prior_seconds={2: prior_seconds},
+ )
+ with service._scheduler_condition:
+ service._scheduler_paused = True
+ service.configure_session_capacity(1)
+ session_id = _create(service, "first")
+ state = service._session(session_id)
+ assert state is not None
+ try:
+ now = 10_000.0
+ _prepare_latest_continuation(
+ state,
+ now=now,
+ pacing_ready_at=now,
+ next_playout_deadline=now + 1.0,
+ )
+ assert service._batch_formation_wait_seconds(service._ready_sessions(now), now) == pytest.approx(
+ 1.0 - prior_seconds * 1.10
+ )
+ finally:
+ service.stop()
diff --git a/tests/unit/pipelines/abot_world/test_model.py b/tests/unit/pipelines/abot_world/test_model.py
index c22f59ea..5fcdb8f6 100644
--- a/tests/unit/pipelines/abot_world/test_model.py
+++ b/tests/unit/pipelines/abot_world/test_model.py
@@ -154,3 +154,134 @@ def test_small_dit_forward_preserves_latent_and_cache_contract() -> None:
assert torch.isfinite(output).all()
assert all(int(cache["global_end_index"].item()) == frame_tokens for cache in self_cache)
assert all(bool(cache["is_init"]) for cache in cross_cache)
+
+
+def test_full_window_steady_state_matches_regular_relative_rope_continuation() -> None:
+ """The graph-safe path must preserve the normal rolling-cache result."""
+ torch.manual_seed(7)
+ model = _tiny_dit().eval()
+ model.set_causal_attention_window(local_attn_size=3, sink_size=1)
+ batch, frames, latent_height, latent_width = 1, 1, 8, 8
+ frame_tokens = (latent_height // 2) * (latent_width // 2)
+ head_dim = model.dim // model.num_heads
+ self_cache = [
+ {
+ "k": torch.zeros(batch, 3 * frame_tokens, model.num_heads, head_dim),
+ "v": torch.zeros(batch, 3 * frame_tokens, model.num_heads, head_dim),
+ "global_end_index": torch.zeros(1, dtype=torch.long),
+ "local_end_index": torch.zeros(1, dtype=torch.long),
+ }
+ for _ in range(model.num_layers)
+ ]
+ cross_cache = [
+ {
+ "k": torch.zeros(batch, model.text_len, model.num_heads, head_dim),
+ "v": torch.zeros(batch, model.text_len, model.num_heads, head_dim),
+ "is_init": False,
+ "sequence_length": 0,
+ }
+ for _ in range(model.num_layers)
+ ]
+ context = torch.randn(batch, model.text_len, 16)
+
+ def model_inputs() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ return (
+ torch.randn(batch, model.in_dim, frames, latent_height, latent_width),
+ torch.randn(batch, 32, frames, latent_height * 2, latent_width * 2),
+ torch.tensor([[0.5]]),
+ )
+
+ # Fill the local [sink, rolling-tail] window through the original path.
+ for frame_index in range(model.local_attn_size):
+ latent, action_context, timestep = model_inputs()
+ model(
+ x=latent,
+ timestep=timestep,
+ context=context,
+ act_context=action_context,
+ kv_cache=self_cache,
+ crossattn_cache=cross_cache,
+ current_start=frame_index * frame_tokens,
+ )
+
+ def clone_cache(cache_list: list[dict[str, object]]) -> list[dict[str, object]]:
+ return [
+ {key: value.clone() if isinstance(value, torch.Tensor) else value for key, value in layer.items()}
+ for layer in cache_list
+ ]
+
+ def assert_cache_equal(actual: list[dict[str, object]], expected: list[dict[str, object]]) -> None:
+ for actual_layer, expected_layer in zip(actual, expected, strict=True):
+ assert actual_layer.keys() == expected_layer.keys()
+ for key, actual_value in actual_layer.items():
+ expected_value = expected_layer[key]
+ if isinstance(actual_value, torch.Tensor):
+ assert isinstance(expected_value, torch.Tensor)
+ torch.testing.assert_close(actual_value, expected_value)
+ else:
+ assert actual_value == expected_value
+
+ reference_self = clone_cache(self_cache)
+ reference_cross = clone_cache(cross_cache)
+ steady_self = clone_cache(self_cache)
+ steady_cross = clone_cache(cross_cache)
+ roll_tokens = (model.local_attn_size - model.sink_size - frames) * frame_tokens
+ roll_scratch_k = torch.empty(batch, roll_tokens, model.num_heads, head_dim)
+ roll_scratch_v = torch.empty_like(roll_scratch_k)
+ current_end = torch.tensor([(model.local_attn_size + frames) * frame_tokens], dtype=torch.long)
+ latent, action_context, timestep = model_inputs()
+
+ expected = model(
+ x=latent,
+ timestep=timestep,
+ context=context,
+ act_context=action_context,
+ kv_cache=reference_self,
+ crossattn_cache=reference_cross,
+ current_start=model.local_attn_size * frame_tokens,
+ )
+ actual = model.forward_steady_state(
+ x=latent,
+ timestep=timestep,
+ context=context,
+ act_context=action_context,
+ kv_cache=steady_self,
+ crossattn_cache=steady_cross,
+ current_end=current_end,
+ roll_scratch_k=roll_scratch_k,
+ roll_scratch_v=roll_scratch_v,
+ update_cache=True,
+ )
+
+ torch.testing.assert_close(actual, expected)
+ assert_cache_equal(steady_self, reference_self)
+ assert_cache_equal(steady_cross, reference_cross)
+
+ # Remaining sampler calls use the same logical chunk and overwrite only
+ # its fixed tail slot; they must not roll the window again.
+ latent, action_context, timestep = model_inputs()
+ expected = model(
+ x=latent,
+ timestep=timestep,
+ context=context,
+ act_context=action_context,
+ kv_cache=reference_self,
+ crossattn_cache=reference_cross,
+ current_start=model.local_attn_size * frame_tokens,
+ )
+ actual = model.forward_steady_state(
+ x=latent,
+ timestep=timestep,
+ context=context,
+ act_context=action_context,
+ kv_cache=steady_self,
+ crossattn_cache=steady_cross,
+ current_end=current_end,
+ roll_scratch_k=roll_scratch_k,
+ roll_scratch_v=roll_scratch_v,
+ update_cache=False,
+ )
+
+ torch.testing.assert_close(actual, expected)
+ assert_cache_equal(steady_self, reference_self)
+ assert_cache_equal(steady_cross, reference_cross)
diff --git a/tests/unit/pipelines/abot_world/test_pipeline.py b/tests/unit/pipelines/abot_world/test_pipeline.py
index 144ae8e6..0131c032 100644
--- a/tests/unit/pipelines/abot_world/test_pipeline.py
+++ b/tests/unit/pipelines/abot_world/test_pipeline.py
@@ -9,6 +9,11 @@
from telefuser.pipelines.abot_world.pipeline import ABotWorldPipelineConfig
+def test_cuda_graph_configuration_is_opt_in() -> None:
+
+ assert ABotWorldPipelineConfig().cuda_graph_enabled is False
+
+
def test_action_context_uses_official_wasd_ijkl_channel_layout() -> None:
action = ABotWorldPipeline.build_action_context(
{"W": True, "D": True, "L": True},
diff --git a/tests/unit/service/livekit/test_dispatch_trace.py b/tests/unit/service/livekit/test_dispatch_trace.py
new file mode 100644
index 00000000..359c5432
--- /dev/null
+++ b/tests/unit/service/livekit/test_dispatch_trace.py
@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import json
+from queue import SimpleQueue
+
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+from telefuser.service.livekit.config import LiveKitServeConfig
+from telefuser.service.livekit.nccl_process_worker_pool import NCCLProcessLiveKitWorkerPool, _DispatchTraceWriter
+from telefuser.service.livekit.process_worker_pool import (
+ ProcessLiveKitWorkerPool,
+ ProcessWorkerSpec,
+ _install_process_dispatch_trace_callback,
+)
+from telefuser.service.livekit.process_worker_pool import (
+ _DispatchTraceWriter as ProcessDispatchTraceWriter,
+)
+
+
+def test_service_dispatch_trace_has_stage_vae_and_session_audit_fields() -> None:
+ service = object.__new__(ABotWorldLiveKitService)
+ service.scheduler_mode = "batched"
+ service._dispatch_trace_sequence = 0
+ records: list[dict] = []
+ service._dispatch_trace_callback = records.append
+
+ service._emit_dispatch_trace(
+ selected_at=100.0,
+ selected_wall_time=1_700_000_000.0,
+ model_started_at=100.01,
+ model_started_wall_time=1_700_000_000.01,
+ completed_at=100.51,
+ completed_wall_time=1_700_000_000.51,
+ control_latent_frames=3,
+ session_traces=[
+ {
+ "session_id": "user-7",
+ "chunk_index": 4,
+ "next_latent_frame_before": 12,
+ "next_latent_frame_after": 15,
+ "emitted_frames_before": 48,
+ "emitted_frames_after": 60,
+ "frames": 12,
+ "controls": ["W"],
+ "queue_wait_seconds": 0.02,
+ }
+ ],
+ stage_metrics={
+ "input_prepare_seconds": 0.01,
+ "cache_collate_seconds": 0.02,
+ "denoise_seconds": 0.34,
+ "cache_scatter_seconds": 0.03,
+ "vae_decode_seconds": 0.07,
+ "postprocess_seconds": 0.04,
+ "total_seconds": 0.51,
+ "taew_decode_mode": 1,
+ "taew_decode_items": 2,
+ "taew_decode_batch_size": 2,
+ "taew_decode_invocations": 1,
+ },
+ outcome="ok",
+ )
+
+ assert len(records) == 1
+ record = records[0]
+ assert record["trace_sequence"] == 1
+ assert record["batch_size"] == 1
+ assert record["control_latent_frames"] == 3
+ assert record["model_started_monotonic_seconds"] == 100.01
+ assert record["model_completed_monotonic_seconds"] == 100.51
+ assert record["stages_seconds"]["denoise"] == 0.34
+ assert record["vae_decode"] == {
+ "mode": 1,
+ "mode_name": "synchronized_batch",
+ "items": 2,
+ "effective_batch_size": 2,
+ "invocations": 1,
+ }
+ assert record["sessions"][0]["session_id"] == "user-7"
+ assert record["sessions"][0]["chunk_index"] == 4
+
+
+def test_parent_dispatch_trace_jsonl_is_bounded_and_enriched(tmp_path) -> None:
+ path = tmp_path / "dispatch-trace.jsonl"
+ writer = _DispatchTraceWriter(str(path), max_events=1, workers={"worker-0": ["0"]})
+ pool = object.__new__(NCCLProcessLiveKitWorkerPool)
+ pool._dispatch_trace = writer
+
+ pool._dispatch_event(
+ {
+ "type": "model_dispatch_trace",
+ "worker_id": "worker-0",
+ "gpu": {"physical_gpu_id": "0", "logical_cuda_device": 0},
+ "trace": {
+ "schema_version": 1,
+ "event_type": "model_dispatch",
+ "batch_size": 2,
+ "sessions": [{"session_id": "user-a"}, {"session_id": "user-b"}],
+ },
+ }
+ )
+ pool._dispatch_event(
+ {
+ "type": "model_dispatch_trace",
+ "worker_id": "worker-0",
+ "gpu": {"physical_gpu_id": "0", "logical_cuda_device": 0},
+ "trace": {"schema_version": 1, "event_type": "model_dispatch", "batch_size": 1},
+ }
+ )
+ snapshot = writer.snapshot()
+ writer.close()
+
+ records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
+ assert [record["event_type"] for record in records] == ["trace_metadata", "model_dispatch"]
+ assert records[1]["parent_sequence"] == 1
+ assert records[1]["worker_id"] == "worker-0"
+ assert records[1]["gpu"]["physical_gpu_id"] == "0"
+ assert records[1]["sessions"] == [{"session_id": "user-a"}, {"session_id": "user-b"}]
+ assert snapshot["written_events"] == 1
+ assert snapshot["dropped_events"] == 1
+
+
+class _TraceCallbackService:
+ def __init__(self) -> None:
+ self.callback = None
+
+ def set_dispatch_trace_callback(self, callback) -> None:
+ self.callback = callback
+
+
+def test_process_mode_trace_forwards_to_parent_and_preserves_physical_gpu(monkeypatch, tmp_path) -> None:
+ path = tmp_path / "dispatch-trace.jsonl"
+ config = LiveKitServeConfig(
+ livekit_url="wss://livekit.example",
+ livekit_api_key="key",
+ livekit_api_secret="secret",
+ worker_mode="process",
+ dispatch_trace_path=str(path),
+ )
+ service = _TraceCallbackService()
+ events = SimpleQueue()
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
+ spec = ProcessWorkerSpec("worker-0", ["0"])
+
+ assert _install_process_dispatch_trace_callback(
+ service=service,
+ config=config,
+ spec=spec,
+ events=events,
+ logical_cuda_device=0,
+ )
+ assert callable(service.callback)
+ service.callback(
+ {
+ "schema_version": 1,
+ "event_type": "model_dispatch",
+ "batch_size": 1,
+ "sessions": [{"session_id": "user-a"}],
+ }
+ )
+ event = events.get()
+ assert event["gpu"] == {
+ "physical_gpu_id": "1",
+ "configured_gpu_id": "0",
+ "logical_cuda_device": 0,
+ }
+
+ writer = ProcessDispatchTraceWriter(str(path), max_events=1, workers={"worker-0": ["0"]})
+ pool = object.__new__(ProcessLiveKitWorkerPool)
+ pool._dispatch_trace = writer
+ ProcessLiveKitWorkerPool._dispatch_event(pool, event)
+ snapshot = writer.snapshot()
+ writer.close()
+
+ records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
+ assert records[1]["worker_id"] == "worker-0"
+ assert records[1]["gpu"] == event["gpu"]
+ assert records[1]["parent_sequence"] == 1
+ assert snapshot["written_events"] == 1
diff --git a/tests/unit/service/livekit/test_nccl_process_worker_pool.py b/tests/unit/service/livekit/test_nccl_process_worker_pool.py
index b24640bc..719026de 100644
--- a/tests/unit/service/livekit/test_nccl_process_worker_pool.py
+++ b/tests/unit/service/livekit/test_nccl_process_worker_pool.py
@@ -6,6 +6,7 @@
from telefuser.service.livekit.config import LiveKitServeConfig
from telefuser.service.livekit.nccl_process_worker_pool import (
_MODEL_OUTPUT_PARENT_QUEUE_SIZE,
+ _NCCL_INIT_PARENT_TIMEOUT_SECONDS,
NCCLProcessLiveKitWorkerPool,
_pump_model_outputs,
)
@@ -75,6 +76,14 @@ def _model_output(session_id: str, payload: dict[str, Any]) -> dict[str, Any]:
}
+def _model_output_eos(session_id: str) -> dict[str, Any]:
+ return {
+ "type": "model_output_eos",
+ "worker_id": "worker-0",
+ "session_id": session_id,
+ }
+
+
async def _wait_for_count(events: _EventCollector, count: int) -> None:
while len(events.items) < count:
events.updated.clear()
@@ -112,6 +121,51 @@ async def run() -> None:
asyncio.run(run())
+def test_child_pump_emits_eos_when_abot_output_iterator_ends() -> None:
+ async def run() -> None:
+ events = _EventCollector()
+ await _pump_model_outputs(
+ _PumpAdapter([]),
+ _PumpService(),
+ worker_id="worker-0",
+ session_id="pipeline-1",
+ credits=asyncio.BoundedSemaphore(_MODEL_OUTPUT_PARENT_QUEUE_SIZE),
+ events=events,
+ )
+
+ assert [(item["type"], item["worker_id"], item["session_id"]) for item in events.items] == [
+ ("model_output_eos", "worker-0", "pipeline-1")
+ ]
+
+ asyncio.run(run())
+
+
+def test_parent_eos_finishes_pull_and_releases_route() -> None:
+ async def run() -> None:
+ pool = _pool()
+ sent: list[tuple[str, dict[str, Any]]] = []
+ pool._send = lambda worker_id, command: sent.append((worker_id, command))
+ pool.create_model_session("worker-0", "pipeline-1", {})
+ sent.clear()
+
+ chunks = pool.pull_model_chunks("pipeline-1")
+ pool._dispatch_event(_model_output_eos("pipeline-1"))
+ try:
+ await asyncio.wait_for(chunks.__anext__(), timeout=0.2)
+ except StopAsyncIteration:
+ pass
+ else:
+ raise AssertionError("parent pull did not finish after child model-output EOF")
+
+ assert "pipeline-1" not in pool._pipeline_routes
+ assert "pipeline-1" not in pool._session_workers
+ assert "pipeline-1" not in pool._model_outputs
+ assert sent == [("worker-0", {"type": "model_close", "session_id": "pipeline-1"})]
+ await chunks.aclose()
+
+ asyncio.run(run())
+
+
def test_parent_queue_preserves_preview_then_replaces_stale_video_and_returns_credit() -> None:
async def run() -> None:
pool = _pool()
@@ -190,3 +244,146 @@ async def fake_init_nccl() -> None:
assert pool._nccl_ranks == {"worker-0": 0, "worker-1": 1}
asyncio.run(run())
+
+
+def test_init_nccl_uses_dedicated_parent_timeout() -> None:
+ async def run() -> None:
+ pool = _pool()
+ pool._active_workers = {"worker-0", "worker-1"}
+ requests: list[tuple[str, str, dict[str, object]]] = []
+
+ async def fake_request(worker_id: str, command_type: str, **kwargs: object) -> dict[str, object]:
+ requests.append((worker_id, command_type, kwargs))
+ return {"result": True}
+
+ pool._request = fake_request
+ await pool._init_nccl()
+
+ assert [(worker_id, command_type) for worker_id, command_type, _ in requests] == [
+ ("worker-0", "nccl_init"),
+ ("worker-1", "nccl_init"),
+ ]
+ assert [kwargs["rank"] for _, _, kwargs in requests] == [0, 1]
+ assert all(kwargs["world_size"] == 2 for _, _, kwargs in requests)
+ assert all(kwargs["timeout"] == _NCCL_INIT_PARENT_TIMEOUT_SECONDS for _, _, kwargs in requests)
+ assert len({kwargs["init_method"] for _, _, kwargs in requests}) == 1
+ assert pool._nccl_ranks == {"worker-0": 0, "worker-1": 1}
+
+ asyncio.run(run())
+
+
+def test_parent_progress_follows_dequeued_payload_owner_after_route_change() -> None:
+ async def run() -> None:
+ pool = _pool()
+ pool._active_workers = {"worker-0", "worker-1"}
+ sent: list[tuple[str, dict[str, Any]]] = []
+ pool._send = lambda worker_id, command: sent.append((worker_id, command))
+ pool.create_model_session("worker-0", "pipeline-1", {})
+ sent.clear()
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "chunk", "frames": [object(), object()]}))
+ chunks = pool.pull_model_chunks("pipeline-1")
+ assert (await chunks.__anext__())["type"] == "chunk"
+ pool._pipeline_routes["pipeline-1"] = "worker-1"
+ assert pool.report_publisher_frame_progress(
+ "pipeline-1", event="submitted", frames_delta=-1, observed_monotonic_seconds=12.0
+ )
+ progress = [
+ (worker_id, command) for worker_id, command in sent if command["type"] == "model_publisher_frame_progress"
+ ]
+ assert progress == [
+ (
+ "worker-0",
+ {
+ "type": "model_publisher_frame_progress",
+ "session_id": "pipeline-1",
+ "event": "submitted",
+ "frames_delta": -1,
+ "sequence": 0,
+ "observed_monotonic_seconds": 12.0,
+ },
+ )
+ ]
+ await chunks.aclose()
+ assert "pipeline-1" not in pool._model_output_inflight_owner
+
+ asyncio.run(run())
+
+
+def test_latest_parent_queue_replacement_rebates_source_publisher_credit() -> None:
+ pool = _pool()
+ sent: list[tuple[str, dict[str, Any]]] = []
+ pool._send = lambda worker_id, command: sent.append((worker_id, command))
+ pool.create_model_session("worker-0", "pipeline-1", {})
+ sent.clear()
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "chunk", "index": 0, "frames": [object()] * 12}))
+ pool._dispatch_event(_model_output("pipeline-1", {"type": "chunk", "index": 1, "frames": [object()] * 12}))
+ progress = [
+ (worker_id, command) for worker_id, command in sent if command["type"] == "model_publisher_frame_progress"
+ ]
+ assert progress == [
+ (
+ "worker-0",
+ {
+ "type": "model_publisher_frame_progress",
+ "session_id": "pipeline-1",
+ "event": "dropped",
+ "frames_delta": -12,
+ "sequence": 0,
+ "observed_monotonic_seconds": progress[0][1]["observed_monotonic_seconds"],
+ },
+ )
+ ]
+
+
+def test_migration_drain_waits_for_child_queue_and_publisher_frames(monkeypatch) -> None:
+ async def run() -> None:
+ pool = _pool()
+ calls: list[tuple[str, object]] = []
+ statuses = iter(
+ [
+ # Child Fq remains: the parent cannot accept this snapshot.
+ {"in_flight": False, "output_queue_empty": False, "publisher_unsubmitted_frames": 0},
+ # Fq has crossed to the publisher, but child Fp remains.
+ {"in_flight": False, "output_queue_empty": True, "publisher_unsubmitted_frames": 12},
+ # First zero snapshot is followed by a parent-drain barrier.
+ {"in_flight": False, "output_queue_empty": True, "publisher_unsubmitted_frames": 0},
+ # Only a second zero status after that barrier is safe.
+ {"in_flight": False, "output_queue_empty": True, "publisher_unsubmitted_frames": 0},
+ ]
+ )
+
+ async def fake_wait_for_model_output_drain(session_id: str, *, timeout: float) -> None:
+ assert session_id == "pipeline-1"
+ assert timeout > 0
+ calls.append(("parent_drain", None))
+
+ async def fake_request(worker_id: str, request_type: str, **kwargs: object) -> dict[str, object]:
+ assert worker_id == "worker-0"
+ assert request_type == "model_output_drain_status"
+ assert kwargs["session_id"] == "pipeline-1"
+ assert float(kwargs["timeout"]) > 0
+ status = next(statuses)
+ calls.append(("child_status", status))
+ return {"result": status}
+
+ monkeypatch.setattr(pool, "_wait_for_model_output_drain", fake_wait_for_model_output_drain)
+ monkeypatch.setattr(pool, "_request", fake_request)
+
+ await pool._drain_model_outputs_for_migration("pipeline-1", source_worker_id="worker-0", timeout=1.0)
+
+ assert [kind for kind, _ in calls] == [
+ "parent_drain",
+ "child_status",
+ "parent_drain",
+ "child_status",
+ "parent_drain",
+ "child_status",
+ "parent_drain",
+ "child_status",
+ ]
+ assert [value for kind, value in calls if kind == "child_status"] == [
+ {"in_flight": False, "output_queue_empty": False, "publisher_unsubmitted_frames": 0},
+ {"in_flight": False, "output_queue_empty": True, "publisher_unsubmitted_frames": 12},
+ {"in_flight": False, "output_queue_empty": True, "publisher_unsubmitted_frames": 0},
+ {"in_flight": False, "output_queue_empty": True, "publisher_unsubmitted_frames": 0},
+ ]
diff --git a/tests/unit/service/livekit/test_serving_metrics.py b/tests/unit/service/livekit/test_serving_metrics.py
index 93a7ed72..2e93f3fe 100644
--- a/tests/unit/service/livekit/test_serving_metrics.py
+++ b/tests/unit/service/livekit/test_serving_metrics.py
@@ -40,6 +40,10 @@ def __init__(self) -> None:
"pipeline-session-1": {
"active": 1,
"emitted_frames": 12,
+ "publisher_frame_tracking_enabled": 1,
+ "queued_video_frames": 12,
+ "publisher_unsubmitted_frames": 6,
+ "frame_credit_frames": 18,
}
},
}
@@ -109,12 +113,20 @@ def test_serving_metrics_render_scheduler_pipeline_slo_and_no_session_id_labels(
assert 'telefuser_serving_slo_chunks_total{result="met"} 1' in rendered
assert "telefuser_serving_action_to_first_frame_seconds_count 1" in rendered
assert 'telefuser_serving_published_fps{scope="aggregate"}' in rendered
+ assert 'telefuser_serving_frame_credit_frames{state="queued"} 12' in rendered
+ assert 'telefuser_serving_frame_credit_frames{state="total"} 18' in rendered
assert created.record.session_id not in rendered
assert "pipeline-session-1" not in rendered
summary = runtime.serving_metrics_snapshot()["summary"]
assert summary["sessions"] == {"retained": 1, "active": 1, "idle": 0, "waiting": 0}
assert summary["scheduler_mode"] == "batched"
+ assert summary["frame_credit"] == {
+ "tracked_sessions": 1,
+ "queued_frames": 12,
+ "publisher_unsubmitted_frames": 6,
+ "total_frames": 18,
+ }
def test_serving_metrics_records_migration_errors() -> None:
@@ -249,7 +261,6 @@ def test_nccl_parent_transport_and_model_event_hooks_preserve_scheduler_mode() -
]
-
def test_serving_metrics_distinguish_native_taew_batch_from_dit_batch() -> None:
runtime = _runtime()
synchronized_scheduler = {
diff --git a/tests/unit/service/livekit/test_worker.py b/tests/unit/service/livekit/test_worker.py
index e42c5f2a..c34064e5 100644
--- a/tests/unit/service/livekit/test_worker.py
+++ b/tests/unit/service/livekit/test_worker.py
@@ -31,6 +31,9 @@ def __init__(self, stream_mode: str = STREAM_MODE_BIDIRECTIONAL) -> None:
self.closed_service = False
self.created = asyncio.Event()
self.output_queue: asyncio.Queue[dict | None] = asyncio.Queue()
+ self.publisher_tracking_enabled = False
+ self.publisher_tracking_sessions: list[str] = []
+ self.publisher_progress: list[dict[str, object]] = []
def start(self, pipeline_file: str, *, skip_validation: bool = False, gpu_num: int = 1) -> None:
self.started.append({"pipeline_file": pipeline_file, "skip_validation": skip_validation, "gpu_num": gpu_num})
@@ -64,6 +67,14 @@ async def stream_task(self, config: dict):
def close_session(self, session_id: str) -> None:
self.closed.append(session_id)
+ def enable_publisher_frame_tracking(self, session_id: str) -> bool:
+ self.publisher_tracking_sessions.append(session_id)
+ return self.publisher_tracking_enabled
+
+ def report_publisher_frame_progress(self, session_id: str, **payload: object) -> bool:
+ self.publisher_progress.append({"session_id": session_id, **payload})
+ return self.publisher_tracking_enabled
+
class FakeRoomClient:
def __init__(self) -> None:
@@ -389,3 +400,88 @@ async def _run() -> None:
assert room.disconnected is False
asyncio.run(_run())
+
+
+def test_livekit_worker_reports_each_successfully_captured_video_frame(monkeypatch) -> None:
+ async def run() -> None:
+ monkeypatch.setattr(worker_module, "_VIDEO_TRACK_SUBSCRIPTION_GRACE_SECONDS", 0)
+ monkeypatch.setattr(worker_module, "_VIDEO_DRAIN_GRACE_SECONDS", 0)
+ adapter = FakePipelineAdapter()
+ adapter.publisher_tracking_enabled = True
+ room = FakeRoomClient()
+ worker = LiveKitWorker(
+ worker_id="worker-0",
+ config=LiveKitServeConfig(
+ livekit_url="wss://livekit.example", livekit_api_key="key", livekit_api_secret="secret"
+ ),
+ pipeline_file="pipeline.py",
+ token_service=FakeTokenService(),
+ event_sink=FakeSink(),
+ pipeline_adapter=adapter,
+ room_client=room,
+ )
+ worker._pipeline_session_id = "pipeline-session-1"
+ worker._publisher_frame_tracking_enabled = worker._enable_publisher_frame_tracking()
+
+ async def chunks():
+ yield {"type": "chunk", "fps": 12, "frames": [Image.new("RGB", (8, 8)) for _ in range(3)]}
+
+ await worker._publish_pipeline_chunks("public-session", chunks(), wait_for_delivery_ack=False)
+ assert adapter.publisher_tracking_sessions == ["pipeline-session-1"]
+ assert [event["event"] for event in adapter.publisher_progress] == ["submitted"] * 3
+ assert [event["frames_delta"] for event in adapter.publisher_progress] == [-1, -1, -1]
+ assert [event["sequence"] for event in adapter.publisher_progress] == [1, 2, 3]
+ assert len(room.video_frames) == 3
+
+ asyncio.run(run())
+
+
+def test_livekit_worker_releases_unpublished_frame_credit_on_publish_failure_or_cancellation(monkeypatch) -> None:
+ class FailingRoomClient(FakeRoomClient):
+ def __init__(self, failure: BaseException) -> None:
+ super().__init__()
+ self.failure = failure
+
+ async def publish_video_frame(self, frame_rgb: np.ndarray, *, fps: float = 16.0) -> None:
+ del frame_rgb, fps
+ raise self.failure
+
+ async def run(failure: BaseException) -> None:
+ monkeypatch.setattr(worker_module, "_VIDEO_TRACK_SUBSCRIPTION_GRACE_SECONDS", 0)
+ monkeypatch.setattr(worker_module, "_VIDEO_DRAIN_GRACE_SECONDS", 0)
+ adapter = FakePipelineAdapter()
+ adapter.publisher_tracking_enabled = True
+ room = FailingRoomClient(failure)
+ worker = LiveKitWorker(
+ worker_id="worker-0",
+ config=LiveKitServeConfig(
+ livekit_url="wss://livekit.example", livekit_api_key="key", livekit_api_secret="secret"
+ ),
+ pipeline_file="pipeline.py",
+ token_service=FakeTokenService(),
+ event_sink=FakeSink(),
+ pipeline_adapter=adapter,
+ room_client=room,
+ )
+ worker._pipeline_session_id = "pipeline-session-1"
+ worker._publisher_frame_tracking_enabled = worker._enable_publisher_frame_tracking()
+
+ async def chunks():
+ yield {"type": "chunk", "fps": 12, "frames": [Image.new("RGB", (8, 8)) for _ in range(3)]}
+
+ try:
+ await worker._publish_pipeline_chunks("public-session", chunks(), wait_for_delivery_ack=False)
+ except BaseException as caught:
+ assert caught is failure
+ else: # pragma: no cover - protects the assertion below from a false positive
+ raise AssertionError("publisher failure must propagate")
+
+ # A frame is accounted as submitted only after LiveKit accepts it.
+ # Failed/cancelled first-frame publication must rebate the entire chunk.
+ assert room.video_frames == []
+ assert [(event["event"], event["frames_delta"], event["sequence"]) for event in adapter.publisher_progress] == [
+ ("abandoned", -3, 1)
+ ]
+
+ asyncio.run(run(RuntimeError("publish failed")))
+ asyncio.run(run(asyncio.CancelledError()))
diff --git a/tests/unit/validation/test_abot_livekit_burst.py b/tests/unit/validation/test_abot_livekit_burst.py
index 01bced1d..bbf6825a 100644
--- a/tests/unit/validation/test_abot_livekit_burst.py
+++ b/tests/unit/validation/test_abot_livekit_burst.py
@@ -2,6 +2,7 @@
import asyncio
import json
+import sys
from pathlib import Path
from typing import Any
@@ -40,6 +41,32 @@ def _scenario_payload(image_path: Path) -> dict[str, Any]:
}
+def _diagnostic_scenario_payload(image_path: Path) -> dict[str, Any]:
+ payload = _scenario_payload(image_path)
+ payload["name"] = "diagnostic-phase-aligned-unit-wave"
+ payload["phases"] = [
+ {
+ "name": "diagnostic_phase_aligned_16_users",
+ "duration_seconds": 30,
+ "target_users": 16,
+ "arrival_window_seconds": 0,
+ "active_input_fraction": 1.0,
+ },
+ {"name": "diagnostic_drain", "duration_seconds": 2, "target_users": 0},
+ ]
+ payload["diagnostic"] = {
+ "initial_control_barrier": {
+ "enabled": True,
+ "kind": "phase_aligned_initial_control",
+ "not_a_real_user_trace": True,
+ "phase": "diagnostic_phase_aligned_16_users",
+ "expected_connected_sessions": 16,
+ "timeout_seconds": 10,
+ }
+ }
+ return payload
+
+
def _load_scenario(tmp_path: Path) -> wave.Scenario:
image = tmp_path / "initial.png"
image.write_bytes(b"test image placeholder")
@@ -82,6 +109,7 @@ def test_load_scenario_validates_lf3_process_nccl_wave(tmp_path: Path) -> None:
assert scenario.session.control_latent_frames == 3
assert scenario.first_generation_grace_seconds == 15
assert scenario.slo_fps_tolerance == 0.25
+ assert scenario.diagnostic_initial_control_barrier is None
assert [phase.target_users for phase in scenario.phases] == [4, 2]
@@ -119,6 +147,7 @@ def test_scale_up_spreads_only_new_arrivals_across_its_window(tmp_path: Path) ->
runner._schedule_transition(wave.Phase("up", 2, target_users=8, arrival_window_seconds=3))
assert [session.index for session in runner._sessions] == list(range(8))
+ assert all(session.initial_control_gate is None for session in runner._sessions)
assert len(scheduled) == 4
for coroutine in scheduled:
coroutine.close()
@@ -226,3 +255,211 @@ def test_intermittent_peak16_trace_models_pauses_and_reengagement() -> None:
assert phases[2]["active_input_fraction"] == 0.5
assert phases[5]["active_input_fraction"] == 0.5
assert phases[6]["active_input_fraction"] == 1.0
+
+
+def test_phase_aligned_16_workload_is_explicitly_diagnostic() -> None:
+ scenario_path = (
+ wave._REPO_ROOT / "tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json"
+ )
+
+ scenario = wave.load_scenario(scenario_path)
+
+ barrier = scenario.diagnostic_initial_control_barrier
+ assert barrier is not None
+ assert barrier.phase_name == "diagnostic_phase_aligned_16_users"
+ assert barrier.expected_connected_sessions == 16
+ assert scenario.phases[0].target_users == 16
+
+
+def test_diagnostic_initial_control_barrier_is_explicit_and_validated(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ payload = _diagnostic_scenario_payload(image)
+ scenario_path = tmp_path / "diagnostic.json"
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+
+ scenario = wave.load_scenario(scenario_path)
+
+ barrier = scenario.diagnostic_initial_control_barrier
+ assert barrier is not None
+ assert barrier.phase_name == "diagnostic_phase_aligned_16_users"
+ assert barrier.expected_connected_sessions == 16
+ assert barrier.timeout_seconds == 10
+
+ payload["diagnostic"]["initial_control_barrier"].pop("not_a_real_user_trace")
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+ with pytest.raises(wave.ScenarioError, match="not_a_real_user_trace"):
+ wave.load_scenario(scenario_path)
+
+
+def test_diagnostic_schedule_assigns_one_shared_gate_to_the_fresh_cohort(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ payload = _diagnostic_scenario_payload(image)
+ payload["diagnostic"]["initial_control_barrier"]["expected_connected_sessions"] = 2
+ payload["phases"][0]["target_users"] = 2
+ scenario_path = tmp_path / "diagnostic.json"
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+ scenario = wave.load_scenario(scenario_path)
+
+ async def run() -> None:
+ runner, scheduled = _runner_for_scheduling(scenario)
+ runner._schedule_transition(scenario.phases[0])
+
+ assert len(runner._sessions) == 2
+ gates = {id(session.initial_control_gate) for session in runner._sessions}
+ assert len(gates) == 1
+ assert all(
+ session.diagnostic_initial_control_barrier_phase == "diagnostic_phase_aligned_16_users"
+ for session in runner._sessions
+ )
+ # Two delayed session starts plus the non-blocking barrier coroutine.
+ assert len(scheduled) == 3
+ for coroutine in scheduled:
+ coroutine.close()
+
+ asyncio.run(run())
+
+
+def test_diagnostic_barrier_opens_only_after_every_session_connects(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ payload = _diagnostic_scenario_payload(image)
+ payload["diagnostic"]["initial_control_barrier"]["expected_connected_sessions"] = 2
+ payload["phases"][0]["target_users"] = 2
+ scenario_path = tmp_path / "diagnostic.json"
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+ scenario = wave.load_scenario(scenario_path)
+ barrier = scenario.diagnostic_initial_control_barrier
+ assert barrier is not None
+
+ async def run() -> None:
+ runner = object.__new__(wave.LiveKitWaveRunner)
+ runner.started_at = 0.0
+ runner._warnings = []
+ runner._diagnostic_initial_control_barrier_results = []
+ events: list[tuple[str, dict[str, Any]]] = []
+ runner.record_event = lambda event, **values: events.append((event, values))
+ gate = asyncio.Event()
+ sessions = [_session(index, scenario) for index in range(2)]
+ for session in sessions:
+ session.initial_control_gate = gate
+ session.diagnostic_initial_control_barrier_phase = barrier.phase_name
+ task = asyncio.create_task(
+ runner._run_diagnostic_initial_control_barrier(scenario.phases[0], barrier, sessions, gate)
+ )
+ await asyncio.sleep(0.01)
+ sessions[0].connected = True
+ await asyncio.sleep(0.01)
+ assert not gate.is_set()
+ sessions[1].connected = True
+ await task
+
+ assert gate.is_set()
+ assert all(session.initial_control_barrier_released_at is not None for session in sessions)
+ assert runner._diagnostic_initial_control_barrier_results[-1]["status"] == "released_aligned"
+ assert events[-1][0] == "diagnostic_initial_control_barrier_released"
+
+ asyncio.run(run())
+
+
+def test_diagnostic_barrier_timeout_releases_connected_sessions_unaligned(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ payload = _diagnostic_scenario_payload(image)
+ payload["diagnostic"]["initial_control_barrier"]["expected_connected_sessions"] = 2
+ payload["diagnostic"]["initial_control_barrier"]["timeout_seconds"] = 0.001
+ payload["phases"][0]["target_users"] = 2
+ scenario_path = tmp_path / "diagnostic.json"
+ scenario_path.write_text(json.dumps(payload), encoding="utf-8")
+ scenario = wave.load_scenario(scenario_path)
+ barrier = scenario.diagnostic_initial_control_barrier
+ assert barrier is not None
+
+ async def run() -> None:
+ runner = object.__new__(wave.LiveKitWaveRunner)
+ runner.started_at = 0.0
+ runner._warnings = []
+ runner._diagnostic_initial_control_barrier_results = []
+ runner.record_event = lambda *args, **kwargs: None
+ gate = asyncio.Event()
+ sessions = [_session(index, scenario) for index in range(2)]
+ sessions[0].connected = True
+ await runner._run_diagnostic_initial_control_barrier(scenario.phases[0], barrier, sessions, gate)
+
+ assert gate.is_set()
+ assert sessions[0].initial_control_barrier_released_at is not None
+ assert sessions[1].initial_control_barrier_released_at is None
+ assert runner._diagnostic_initial_control_barrier_results[-1]["status"] == "released_unaligned_timeout"
+ assert runner._warnings
+
+ asyncio.run(run())
+
+
+def test_session_control_gate_blocks_first_active_control_until_release(tmp_path: Path) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ scenario_path = tmp_path / "scenario.json"
+ scenario_path.write_text(json.dumps(_scenario_payload(image)), encoding="utf-8")
+ scenario = wave.load_scenario(scenario_path)
+
+ class Participant:
+ def __init__(self) -> None:
+ self.messages: list[tuple[bytes, str, bool]] = []
+
+ async def publish_data(self, payload: bytes, *, topic: str, reliable: bool) -> None:
+ self.messages.append((payload, topic, reliable))
+
+ class Room:
+ def __init__(self, participant: Participant) -> None:
+ self.local_participant = participant
+
+ async def run() -> None:
+ events: list[str] = []
+ gate = asyncio.Event()
+ participant = Participant()
+ session = wave.LiveKitWaveSession(
+ index=0,
+ scenario=scenario,
+ http=object(),
+ rtc=object(),
+ record_event=lambda event, **values: events.append(event),
+ started_at=0.0,
+ diagnostic_initial_control_barrier_phase="diagnostic_phase_aligned_16_users",
+ initial_control_gate=gate,
+ )
+ session._room = Room(participant)
+ session.connected = True
+ task = asyncio.create_task(session._send_controls())
+ await asyncio.sleep(0)
+ assert participant.messages == []
+
+ gate.set()
+ for _ in range(10):
+ await asyncio.sleep(0)
+ if participant.messages:
+ break
+ assert participant.messages
+ assert session.first_active_control_at is not None
+ assert "first_active_control" in events
+ session.stop_requested = True
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+
+ asyncio.run(run())
+
+
+def test_dry_run_discloses_diagnostic_initial_control_barrier(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+ image = tmp_path / "initial.png"
+ image.write_bytes(b"test image placeholder")
+ scenario_path = tmp_path / "diagnostic.json"
+ scenario_path.write_text(json.dumps(_diagnostic_scenario_payload(image)), encoding="utf-8")
+ monkeypatch.setattr(sys, "argv", ["benchmark", "--scenario", str(scenario_path), "--dry-run"])
+
+ wave.main()
+
+ output = capsys.readouterr().out
+ assert "DIAGNOSTIC ONLY" in output
+ assert "not a real-user arrival trace" in output
diff --git a/tests/unit/validation/test_abot_scheduler_timeline.py b/tests/unit/validation/test_abot_scheduler_timeline.py
new file mode 100644
index 00000000..c189f477
--- /dev/null
+++ b/tests/unit/validation/test_abot_scheduler_timeline.py
@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from tools.validation import render_abot_dispatch_timeline as dispatch_timeline
+from tools.validation import trace_abot_scheduler_timeline as timeline
+
+
+def test_three_user_timeline_proves_phase_alignment_changes_real_scheduler_batching(tmp_path: Path) -> None:
+ """The CPU backend replaces compute only; ABot service selects the batches."""
+ staggered_config = timeline.TimelineConfig(
+ chunks_per_session=3,
+ fps=12,
+ frames_per_chunk=3,
+ stagger_offsets_ms=(0.0, 260.0, 520.0),
+ output_timeout_seconds=5.0,
+ )
+
+ staggered = timeline.run_scenario("staggered", staggered_config)
+ aligned_config = timeline.TimelineConfig(
+ chunks_per_session=1,
+ fps=12,
+ frames_per_chunk=3,
+ stagger_offsets_ms=(0.0, 260.0, 520.0),
+ output_timeout_seconds=5.0,
+ )
+ aligned = timeline.run_scenario("aligned", aligned_config)
+
+ assert staggered["summary"]["batch_size_histogram"] == {"1": 9}
+ assert staggered["summary"]["classification"] == "time_sliced_singletons"
+ assert staggered["summary"]["serialized_scheduler_thread"] is True
+ assert all(batch["batch_size"] == 1 for batch in staggered["batches"])
+
+ assert aligned["summary"]["batch_size_histogram"] == {"3": 1}
+ assert aligned["summary"]["first_batch_size"] == 3
+ assert aligned["summary"]["classification"] == "coalesced_microbatching"
+ assert all(batch["session_ids"] == ["user-1", "user-2", "user-3"] for batch in aligned["batches"])
+
+ output = tmp_path / "aligned"
+ timeline._write_result(output, aligned)
+ payload = json.loads((output / "timeline.json").read_text())
+ assert payload["backend"]["kind"] == "cpu_fake_pipeline_with_production_abot_service_scheduler"
+ for name in ("timeline.json", "events.csv", "batches.csv", "chunks.csv", "summary.csv", "timeline.png"):
+ assert (output / name).is_file()
+ assert (output / "timeline.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
+
+
+def test_dispatch_timeline_keeps_public_trace_session_generations_distinct() -> None:
+ assert dispatch_timeline._short_user("ts-00079-g01") == "u79g01"
+ assert dispatch_timeline._short_user("ts-00079-g02") == "u79g02"
diff --git a/tests/unit/validation/test_abot_turboserve_trace_adapter.py b/tests/unit/validation/test_abot_turboserve_trace_adapter.py
new file mode 100644
index 00000000..10f715eb
--- /dev/null
+++ b/tests/unit/validation/test_abot_turboserve_trace_adapter.py
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from collections.abc import Mapping
+from pathlib import Path
+
+from tools.validation import benchmark_abot_livekit_burst as wave
+from tools.validation import derive_abot_turboserve_trace as adapter
+from tools.validation import replay_abot_livekit_lifecycle_trace as replay
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_SOURCE = _REPO_ROOT.parent / "TurboServe" / "traces" / "example_8gpu.json"
+_WORKLOADS = _REPO_ROOT / "tools" / "validation" / "workloads"
+
+
+def _peak_retained(events: list[Mapping[str, object]]) -> int:
+ retained: set[str] = set()
+ peak = 0
+ for event in events:
+ trace_session_id = event["trace_session_id"]
+ assert isinstance(trace_session_id, str)
+ if event["event"] == "session_arrival":
+ retained.add(trace_session_id)
+ elif event["event"] == "session_departure":
+ retained.remove(trace_session_id)
+ peak = max(peak, len(retained))
+ assert not retained
+ return peak
+
+
+def test_checked_in_turboserve_public_demo_scenarios_are_deterministic_and_runnable() -> None:
+ expected_1gpu, expected_4gpu = adapter.build_scenarios(_SOURCE)
+ for expected in (expected_1gpu, expected_4gpu):
+ filename = f"{expected['name']}.json"
+ actual = json.loads((_WORKLOADS / filename).read_text(encoding="utf-8"))
+ assert actual == expected
+
+ contract = actual["trace_contract"]
+ assert contract["not_a_turboserve_production_trace"] is True
+ assert contract["not_a_reproduction_of_private_paper_t1_to_t6_traces"] is True
+ assert contract["time_transform"]["source_to_derived_scale"] == 1.0
+ assert contract["execution_contract"].startswith("No diagnostic barrier")
+
+ trace = actual["lifecycle_trace"]
+ assert trace["kind"] == "explicit_session_lifecycle_v1"
+ assert trace["duration_seconds"] == 1800.0
+ events = trace["events"]
+ assert isinstance(events, list)
+ assert _peak_retained(events) == contract["capacity_transform"]["target_peak_retained_sessions"]
+
+ scenario = wave.load_scenario(_WORKLOADS / filename)
+ parsed = replay.load_explicit_lifecycle_trace(scenario)
+ assert parsed.duration_seconds == 1800.0
+ assert parsed.events
+ assert scenario.diagnostic_initial_control_barrier is None
+
+
+def test_public_demo_capacity_normalization_retains_real_pause_resume_events() -> None:
+ one_gpu, four_gpu = adapter.build_scenarios(_SOURCE)
+ for scenario, peak, expected_counts in (
+ (one_gpu, 4, {"session_arrival": 61, "session_departure": 61, "user_active": 66, "user_idle": 69}),
+ (four_gpu, 16, {"session_arrival": 300, "session_departure": 300, "user_active": 282, "user_idle": 323}),
+ ):
+ trace = scenario["lifecycle_trace"]
+ assert isinstance(trace, Mapping)
+ events = trace["events"]
+ assert isinstance(events, list)
+ actual_counts: dict[str, int] = {}
+ for event in events:
+ assert isinstance(event, Mapping)
+ name = event["event"]
+ assert isinstance(name, str)
+ actual_counts[name] = actual_counts.get(name, 0) + 1
+ assert actual_counts == expected_counts
+ assert _peak_retained(events) == peak
+ assert actual_counts["user_active"] > 0
+ assert actual_counts["user_idle"] > 0
+
+
+def test_lifecycle_replay_waits_for_inflight_departure_at_capacity() -> None:
+ """A same-timestamp replacement cannot POST before DELETE completes."""
+
+ async def check() -> None:
+ runner = object.__new__(replay.ExplicitLifecycleRunner)
+ runner.started_at = time.perf_counter()
+ runner._events = []
+ runner._trace_sessions = {"retained-0": object(), "retained-1": object(), "retained-2": object()}
+ runner._lifecycle_admission_capacity = 4
+ departure_completed = asyncio.Event()
+
+ async def finish_departure() -> None:
+ await asyncio.sleep(0)
+ departure_completed.set()
+
+ departure = asyncio.create_task(finish_departure())
+ runner._lifecycle_departure_tasks = {"departing": departure}
+
+ await runner._wait_for_departures_before_arrival()
+
+ assert departure_completed.is_set()
+ assert [event["event"] for event in runner._events] == [
+ "lifecycle_arrival_waiting_for_departure",
+ "lifecycle_arrival_departure_wait_completed",
+ ]
+
+ asyncio.run(check())
diff --git a/tests/unit/validation/test_analyze_abot_serving_trace.py b/tests/unit/validation/test_analyze_abot_serving_trace.py
new file mode 100644
index 00000000..ed8f9df3
--- /dev/null
+++ b/tests/unit/validation/test_analyze_abot_serving_trace.py
@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from tools.validation import analyze_abot_serving_trace as analysis
+
+
+def _prometheus_snapshot(*, items: float, executions: float, b1: float, b2_cumulative: float) -> str:
+ return "\n".join(
+ (
+ f"telefuser_serving_batch_items_total {items}",
+ f"telefuser_serving_batches_total {executions}",
+ f'telefuser_serving_batch_size_bucket{{le="1.0"}} {b1}',
+ f'telefuser_serving_batch_size_bucket{{le="2.0"}} {b2_cumulative}',
+ f'telefuser_serving_batch_size_bucket{{le="3.0"}} {b2_cumulative}',
+ f'telefuser_serving_batch_size_bucket{{le="4.0"}} {b2_cumulative}',
+ "",
+ )
+ )
+
+
+def test_batch_report_distinguishes_item_and_execution_percentages(tmp_path: Path) -> None:
+ metrics_dir = tmp_path / "serving_metrics"
+ prometheus_dir = metrics_dir / "prometheus"
+ prometheus_dir.mkdir(parents=True)
+ (prometheus_dir / "000001.prom").write_text(
+ _prometheus_snapshot(items=0, executions=0, b1=0, b2_cumulative=0), encoding="utf-8"
+ )
+ # Ten B=1 executions plus two B=2 executions produce fourteen items but
+ # only twelve executions. The B=2 item and execution shares differ.
+ (prometheus_dir / "000002.prom").write_text(
+ _prometheus_snapshot(items=14, executions=12, b1=10, b2_cumulative=14), encoding="utf-8"
+ )
+ records = [
+ {"sequence": 1, "offset_seconds": 0.0, "prometheus": {"path": "prometheus/000001.prom"}},
+ {"sequence": 2, "offset_seconds": 10.0, "prometheus": {"path": "prometheus/000002.prom"}},
+ ]
+ (metrics_dir / "serving-metrics.jsonl").write_text(
+ "\n".join(json.dumps(record) for record in records) + "\n", encoding="utf-8"
+ )
+
+ snapshots = analysis.load_snapshots(metrics_dir)
+ summary = analysis.summarize_interval(
+ phase="test",
+ requested_start_seconds=0,
+ requested_end_seconds=10,
+ before=snapshots[0],
+ after=snapshots[1],
+ )
+
+ assert summary.batch_items == 14
+ assert summary.batch_executions == 12
+ assert summary.b1_execution_equivalents == 10
+ assert summary.b2_execution_equivalents == 2
+ assert summary.b2_item_share_percent == pytest.approx(4 / 14 * 100)
+ assert summary.b2_execution_share_percent == pytest.approx(2 / 12 * 100)
+ assert summary.mean_execution_batch_size == pytest.approx(14 / 12)
diff --git a/tools/validation/abot_steady_eager.py b/tools/validation/abot_steady_eager.py
new file mode 100644
index 00000000..ff56fbf0
--- /dev/null
+++ b/tools/validation/abot_steady_eager.py
@@ -0,0 +1,307 @@
+"""Benchmark-only eager executor for ABot's fixed-window continuation path.
+
+This module intentionally does not change production serving behavior. It
+temporarily replaces one pipeline instance's continuation entry points so an
+offline benchmark can distinguish the cost of the fixed-window
+``forward_steady_state`` path from the additional benefit of CUDA Graph
+replay. The hook accepts a compatible native microbatch, not merely B=1.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from types import MethodType
+from typing import Any
+
+import torch
+
+
+def _has_one_generator_per_batch_item(generator: torch.Generator | Sequence[torch.Generator], batch_size: int) -> bool:
+ """Match the eager sampler's one-generator-per-session RNG contract."""
+ if isinstance(generator, torch.Generator):
+ return batch_size == 1
+ return (
+ isinstance(generator, Sequence)
+ and len(generator) == batch_size
+ and all(isinstance(item, torch.Generator) for item in generator)
+ )
+
+
+def _cache_cursor_matches(value: Any, expected: int) -> bool:
+ """Accept singleton or per-item cursor tensors only when all values agree."""
+ return isinstance(value, torch.Tensor) and value.numel() > 0 and bool(torch.all(value == expected).item())
+
+
+def _is_steady_eager_eligible(
+ stage: Any,
+ *,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ current_start: int,
+ generator: torch.Generator | Sequence[torch.Generator],
+) -> bool:
+ """Check fixed-window invariants for one compatible native microbatch.
+
+ This is deliberately stricter than the legacy batch path. A false result
+ is safe: the hook delegates to the original dynamic eager implementation,
+ and the benchmark reports that no static eager observation was made.
+ """
+ dit = stage.dit
+ if latent.ndim != 5 or latent.shape[0] < 1 or latent.shape[2] != 3:
+ return False
+ batch_size = latent.shape[0]
+ if prompt_emb.ndim != 3 or prompt_emb.shape[0] != batch_size:
+ return False
+ if action_context.ndim != 5 or action_context.shape[0] != batch_size:
+ return False
+ if not dit.use_relative_rope or not _has_one_generator_per_batch_item(generator, batch_size):
+ return False
+ if len(self_cache) != dit.num_layers or len(cross_cache) != dit.num_layers:
+ return False
+ if dit.local_attn_size <= latent.shape[2] or dit.sink_size < 0:
+ return False
+ frame_tokens = (latent.shape[-2] // dit.patch_size[1]) * (latent.shape[-1] // dit.patch_size[2])
+ expected_global_end = current_start * frame_tokens
+ capacity = dit.local_attn_size * frame_tokens
+ for self_layer, cross_layer in zip(self_cache, cross_cache, strict=True):
+ key = self_layer.get("k")
+ value = self_layer.get("v")
+ if not isinstance(key, torch.Tensor) or not isinstance(value, torch.Tensor):
+ return False
+ if key.shape[0] != batch_size or key.shape[1] != capacity or value.shape != key.shape:
+ return False
+ if not _cache_cursor_matches(self_layer.get("local_end_index"), capacity):
+ return False
+ if not _cache_cursor_matches(self_layer.get("global_end_index"), expected_global_end):
+ return False
+ cross_key = cross_layer.get("k")
+ cross_value = cross_layer.get("v")
+ if not isinstance(cross_key, torch.Tensor) or not isinstance(cross_value, torch.Tensor):
+ return False
+ if cross_key.shape[0] != batch_size or cross_value.shape != cross_key.shape:
+ return False
+ if not bool(cross_layer["is_init"]) or int(cross_layer["sequence_length"]) != prompt_emb.shape[1]:
+ return False
+
+ return True
+
+
+class _SteadyEagerRunner:
+ """Persistent scratch allocations for one fixed-shape native microbatch."""
+
+ def __init__(self, stage: Any, latent: torch.Tensor, self_cache: list[dict[str, Any]]) -> None:
+ dit = stage.dit
+ self.shape = tuple(latent.shape)
+ self.dtype = latent.dtype
+ self.device = latent.device
+ self.frames = latent.shape[2]
+ self.frame_tokens = (latent.shape[-2] // dit.patch_size[1]) * (latent.shape[-1] // dit.patch_size[2])
+ rolled_tokens = (dit.local_attn_size - dit.sink_size - self.frames) * self.frame_tokens
+ if rolled_tokens < 0:
+ raise ValueError("ABot steady eager block does not fit in the rolling cache tail")
+ scratch_shape = (latent.shape[0], rolled_tokens, dit.num_heads, dit.dim // dit.num_heads)
+ self.roll_scratch_k = torch.empty(scratch_shape, dtype=latent.dtype, device=latent.device)
+ self.roll_scratch_v = torch.empty_like(self.roll_scratch_k)
+ cursor = self_cache[0]["global_end_index"]
+ if not isinstance(cursor, torch.Tensor):
+ raise TypeError("ABot steady eager cache cursor must be a tensor")
+ self.current_end = torch.empty_like(cursor, dtype=torch.long, device=latent.device)
+
+ def matches(self, latent: torch.Tensor, self_cache: list[dict[str, Any]]) -> bool:
+ cursor = self_cache[0].get("global_end_index")
+ return (
+ tuple(latent.shape) == self.shape
+ and latent.dtype == self.dtype
+ and latent.device == self.device
+ and isinstance(cursor, torch.Tensor)
+ and tuple(cursor.shape) == tuple(self.current_end.shape)
+ )
+
+ @staticmethod
+ def _draw_noise(
+ current: torch.Tensor,
+ generator: torch.Generator | Sequence[torch.Generator],
+ ) -> torch.Tensor:
+ if isinstance(generator, torch.Generator):
+ return torch.randn(current.shape, generator=generator, dtype=current.dtype, device=current.device)
+ return torch.cat(
+ [
+ torch.randn(
+ (1, *current.shape[1:]),
+ generator=item_generator,
+ dtype=current.dtype,
+ device=current.device,
+ )
+ for item_generator in generator
+ ],
+ dim=0,
+ )
+
+ def run(
+ self,
+ stage: Any,
+ *,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ current_start: int,
+ generator: torch.Generator | Sequence[torch.Generator],
+ scheduler: Any,
+ ) -> torch.Tensor:
+ """Mirror ``_denoise_block`` using eager steady-state DiT calls."""
+ timesteps = stage._official_denoising_timesteps(scheduler).to(device=self.device)
+ self.current_end.fill_((current_start + self.frames) * self.frame_tokens)
+ current = latent
+ for index, current_timestep in enumerate(timesteps):
+ timestep = torch.full(
+ (latent.shape[0], self.frames),
+ current_timestep,
+ dtype=timesteps.dtype,
+ device=self.device,
+ )
+ with torch.autocast(self.device.type, dtype=stage.torch_dtype, enabled=self.device.type == "cuda"):
+ flow_prediction = stage.dit.forward_steady_state(
+ x=current.to(dtype=stage.torch_dtype),
+ timestep=timestep,
+ context=prompt_emb,
+ act_context=action_context,
+ kv_cache=self_cache,
+ crossattn_cache=cross_cache,
+ current_end=self.current_end,
+ roll_scratch_k=self.roll_scratch_k,
+ roll_scratch_v=self.roll_scratch_v,
+ update_cache=index == 0,
+ )
+ x0 = stage._x0_prediction(flow_prediction, current, timestep, scheduler)
+ if index < len(timesteps) - 1:
+ noise = self._draw_noise(x0, generator)
+ current = scheduler.add_noise(x0, noise, timesteps[index + 1])
+ else:
+ current = x0
+ # Keep the benchmark-only steady path faithful to _denoise_block:
+ # final cache-only context update runs outside sampler autocast.
+ stage.dit(
+ x=current.to(dtype=stage.torch_dtype),
+ timestep=torch.zeros_like(timestep),
+ context=prompt_emb,
+ act_context=action_context,
+ kv_cache=self_cache,
+ crossattn_cache=cross_cache,
+ current_start=current_start * self.frame_tokens,
+ )
+ return current
+
+
+class SteadyEagerHook:
+ """Temporarily route an offline benchmark pipeline to steady-state eager.
+
+ Pre-full-window chunks retain the original dynamic method. The hook only
+ runs for the fixed B=1, LF=3, Relative-RoPE continuation shape; it records
+ every substitution so callers cannot mislabel a legacy fallback as a
+ steady-state eager result.
+ """
+
+ def __init__(self, stage: Any) -> None:
+ self.stage = stage
+ self._original = stage.denoise_interactive_block
+ self._runners: dict[str, _SteadyEagerRunner] = {}
+ self._installed = False
+ self._measurement_active = False
+ self.total_steady_calls = 0
+ self.measured_steady_calls = 0
+ self.total_legacy_calls = 0
+
+ def install(self) -> None:
+ if self._installed:
+ return
+
+ def dispatch(
+ stage: Any,
+ *,
+ session_id: str,
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ current_start: int,
+ generator: torch.Generator,
+ scheduler: Any,
+ ) -> torch.Tensor:
+ if not _is_steady_eager_eligible(
+ stage,
+ latent=latent,
+ prompt_emb=prompt_emb,
+ action_context=action_context,
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ current_start=current_start,
+ generator=generator,
+ ):
+ self.total_legacy_calls += 1
+ return self._original(
+ session_id=session_id,
+ latent=latent,
+ prompt_emb=prompt_emb,
+ action_context=action_context,
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ current_start=current_start,
+ generator=generator,
+ scheduler=scheduler,
+ )
+ runner = self._runners.get(session_id)
+ if runner is None or not runner.matches(latent, self_cache):
+ runner = _SteadyEagerRunner(stage, latent, self_cache)
+ self._runners[session_id] = runner
+ output = runner.run(
+ stage,
+ latent=latent,
+ prompt_emb=prompt_emb,
+ action_context=action_context,
+ self_cache=self_cache,
+ cross_cache=cross_cache,
+ current_start=current_start,
+ generator=generator,
+ scheduler=scheduler,
+ )
+ self.total_steady_calls += 1
+ if self._measurement_active:
+ self.measured_steady_calls += 1
+ set_last_metrics = getattr(stage, "_set_cuda_graph_last_metrics", None)
+ if callable(set_last_metrics):
+ set_last_metrics(eligible=True)
+ return output
+
+ self.stage.denoise_interactive_block = MethodType(dispatch, self.stage)
+ self._installed = True
+
+ def begin_measurement(self) -> None:
+ self.measured_steady_calls = 0
+ self._measurement_active = True
+
+ def runtime_metrics(self) -> dict[str, int]:
+ return {
+ "installed": int(self._installed),
+ "steady_calls_total": self.total_steady_calls,
+ "steady_calls_measured": self.measured_steady_calls,
+ "legacy_calls_total": self.total_legacy_calls,
+ }
+
+ def close(self) -> None:
+ if self._installed:
+ self.stage.denoise_interactive_block = self._original
+ self._installed = False
+ self._runners.clear()
+
+
+def install_steady_eager_hook(pipeline: Any) -> SteadyEagerHook:
+ """Install a scoped benchmark-only steady-state eager route on ``pipeline``."""
+ hook = SteadyEagerHook(pipeline.denoise_stage)
+ hook.install()
+ return hook
diff --git a/tools/validation/analyze_abot_serving_trace.py b/tools/validation/analyze_abot_serving_trace.py
new file mode 100644
index 00000000..c452e242
--- /dev/null
+++ b/tools/validation/analyze_abot_serving_trace.py
@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+"""Summarize and visualize a captured ABot-World serving metrics trace.
+
+The serving collector writes cumulative Prometheus snapshots once per sample.
+This tool turns those snapshots into a batch-execution table and a small
+dependency-light PNG suitable for an experiment notebook or paper appendix.
+
+The distinction between *items* and *executions* matters here: every member of
+a coalesced B=N execution emits one model-output event. Therefore the
+``batch_size`` histogram counts session-chunk items, whereas
+``telefuser_serving_batches_total`` is incremented by ``1/N`` per member and
+is an execution-equivalent counter. The report makes both denominators
+explicit rather than accidentally calling the B=2 item percentage a B=2
+execution percentage.
+
+Example:
+
+ PYTHONPATH=$PWD python tools/validation/analyze_abot_serving_trace.py \
+ --serving-metrics-dir /path/to/serving_metrics \
+ --result /path/to/result.json \
+ --gpu-metrics /path/to/gpu_metrics/gpu-metrics.jsonl \
+ --output-dir /tmp/abot-peak16-analysis
+
+It never contacts a server or GPU. ``Pillow`` is used only for the PNG; JSON
+and CSV summaries are still written when it is unavailable.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+_BATCH_BUCKET_LABELS = ("1.0", "2.0", "3.0", "4.0")
+_BATCH_SIZES = (1, 2, 3, 4)
+
+
+@dataclass(frozen=True)
+class BatchCounters:
+ """Cumulative counters at one Prometheus scrape point."""
+
+ offset_seconds: float
+ sequence: int
+ batch_items: float
+ batch_executions: float
+ bucket_items: tuple[float, float, float, float]
+
+
+@dataclass(frozen=True)
+class PhaseSummary:
+ """A counter-delta summary for a named workload phase."""
+
+ phase: str
+ requested_start_seconds: float
+ requested_end_seconds: float
+ sampled_start_seconds: float
+ sampled_end_seconds: float
+ batch_items: float
+ batch_executions: float
+ b1_items: float
+ b2_items: float
+ b3_items: float
+ b4_items: float
+ b_gt4_items: float
+ b1_execution_equivalents: float
+ b2_execution_equivalents: float
+ b3_execution_equivalents: float
+ b4_execution_equivalents: float
+ b2_execution_share_percent: float
+ b2_item_share_percent: float
+ mean_execution_batch_size: float
+
+
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--serving-metrics-dir",
+ required=True,
+ type=Path,
+ help="Directory containing serving-metrics.jsonl and prometheus/ snapshots.",
+ )
+ parser.add_argument(
+ "--result",
+ type=Path,
+ help="Optional benchmark result.json; its phase boundaries split the report.",
+ )
+ parser.add_argument(
+ "--gpu-metrics",
+ type=Path,
+ help="Optional gpu-metrics.jsonl captured on the same monotonic timeline.",
+ )
+ parser.add_argument(
+ "--output-dir",
+ required=True,
+ type=Path,
+ help="New or empty directory for summary.json, phase-batches.csv, and PNG.",
+ )
+ return parser.parse_args(argv)
+
+
+def _metric_value(metrics: dict[str, float], name: str) -> float:
+ """Read a numeric exposition line, treating not-yet-created metrics as zero."""
+
+ return float(metrics.get(name, 0.0))
+
+
+def _read_prometheus_metrics(path: Path) -> dict[str, float]:
+ metrics: dict[str, float] = {}
+ for line in path.read_text(encoding="utf-8").splitlines():
+ if not line or line.startswith("#") or " " not in line:
+ continue
+ name, value_text = line.rsplit(" ", 1)
+ if not name.startswith("telefuser_serving_"):
+ continue
+ try:
+ value = float(value_text)
+ except ValueError:
+ continue
+ if math.isfinite(value):
+ metrics[name] = value
+ return metrics
+
+
+def load_snapshots(serving_metrics_dir: Path) -> list[BatchCounters]:
+ """Load valid Prometheus snapshots in capture order."""
+
+ root = serving_metrics_dir.expanduser().resolve()
+ jsonl_path = root / "serving-metrics.jsonl"
+ if not jsonl_path.is_file():
+ raise ValueError(f"missing serving metrics JSONL: {jsonl_path}")
+ snapshots: list[BatchCounters] = []
+ for raw in jsonl_path.read_text(encoding="utf-8").splitlines():
+ if not raw.strip():
+ continue
+ record = json.loads(raw)
+ prom = record.get("prometheus")
+ if not isinstance(prom, dict) or not isinstance(prom.get("path"), str):
+ continue
+ prom_path = root / prom["path"]
+ if not prom_path.is_file():
+ continue
+ metrics = _read_prometheus_metrics(prom_path)
+ buckets = tuple(
+ _metric_value(metrics, f'telefuser_serving_batch_size_bucket{{le="{label}"}}')
+ for label in _BATCH_BUCKET_LABELS
+ )
+ snapshots.append(
+ BatchCounters(
+ offset_seconds=float(record["offset_seconds"]),
+ sequence=int(record["sequence"]),
+ batch_items=_metric_value(metrics, "telefuser_serving_batch_items_total"),
+ batch_executions=_metric_value(metrics, "telefuser_serving_batches_total"),
+ bucket_items=buckets,
+ )
+ )
+ if len(snapshots) < 2:
+ raise ValueError(f"need at least two valid Prometheus snapshots under {root}")
+ if any(later.offset_seconds < earlier.offset_seconds for earlier, later in zip(snapshots, snapshots[1:])):
+ raise ValueError("serving metrics snapshots are not ordered by monotonic offset")
+ return snapshots
+
+
+def _nearest_snapshot(snapshots: list[BatchCounters], offset_seconds: float) -> BatchCounters:
+ return min(snapshots, key=lambda item: abs(item.offset_seconds - offset_seconds))
+
+
+def _counter_delta(after: float, before: float, label: str) -> float:
+ delta = after - before
+ if delta < -1e-6:
+ raise ValueError(f"counter reset or reversed phase for {label}: {before} -> {after}")
+ return max(0.0, delta)
+
+
+def _bucket_deltas(after: BatchCounters, before: BatchCounters) -> tuple[float, float, float, float, float]:
+ cumulative = [
+ _counter_delta(current, prior, f"batch histogram <= {size}")
+ for size, current, prior in zip(_BATCH_SIZES, after.bucket_items, before.bucket_items)
+ ]
+ b1 = cumulative[0]
+ b2 = cumulative[1] - cumulative[0]
+ b3 = cumulative[2] - cumulative[1]
+ b4 = cumulative[3] - cumulative[2]
+ items = _counter_delta(after.batch_items, before.batch_items, "batch items")
+ b_gt4 = items - cumulative[3]
+ if min(b2, b3, b4, b_gt4) < -1e-6:
+ raise ValueError("batch histogram is not monotonically cumulative")
+ return b1, max(0.0, b2), max(0.0, b3), max(0.0, b4), max(0.0, b_gt4)
+
+
+def summarize_interval(
+ *,
+ phase: str,
+ requested_start_seconds: float,
+ requested_end_seconds: float,
+ before: BatchCounters,
+ after: BatchCounters,
+) -> PhaseSummary:
+ """Compute observed execution-equivalent batch distribution over an interval."""
+
+ items = _counter_delta(after.batch_items, before.batch_items, "batch items")
+ executions = _counter_delta(after.batch_executions, before.batch_executions, "batch executions")
+ b1, b2, b3, b4, b_gt4 = _bucket_deltas(after, before)
+ b1 + b2 / 2.0 + b3 / 3.0 + b4 / 4.0
+ # The metric contains explicit 1--4 buckets. Items above four do not have
+ # a one-to-one bucket (5/6 are together), so do not invent an exact count.
+ # They are intentionally left out of the named B=1..4 execution bars.
+ mean_batch = items / executions if executions else 0.0
+ return PhaseSummary(
+ phase=phase,
+ requested_start_seconds=requested_start_seconds,
+ requested_end_seconds=requested_end_seconds,
+ sampled_start_seconds=before.offset_seconds,
+ sampled_end_seconds=after.offset_seconds,
+ batch_items=items,
+ batch_executions=executions,
+ b1_items=b1,
+ b2_items=b2,
+ b3_items=b3,
+ b4_items=b4,
+ b_gt4_items=b_gt4,
+ b1_execution_equivalents=b1,
+ b2_execution_equivalents=b2 / 2.0,
+ b3_execution_equivalents=b3 / 3.0,
+ b4_execution_equivalents=b4 / 4.0,
+ b2_execution_share_percent=(100.0 * (b2 / 2.0) / executions if executions else 0.0),
+ b2_item_share_percent=(100.0 * b2 / items if items else 0.0),
+ mean_execution_batch_size=mean_batch,
+ )
+
+
+def _load_phase_bounds(result_path: Path) -> list[tuple[str, float, float]]:
+ payload = json.loads(result_path.expanduser().resolve().read_text(encoding="utf-8"))
+ phase_results = payload.get("phase_results")
+ if not isinstance(phase_results, list):
+ raise ValueError(f"result has no phase_results list: {result_path}")
+ bounds: list[tuple[str, float, float]] = []
+ for entry in phase_results:
+ if not isinstance(entry, dict):
+ continue
+ name = entry.get("phase")
+ start = entry.get("started_offset_seconds")
+ end = entry.get("completed_offset_seconds")
+ if isinstance(name, str) and isinstance(start, int | float) and isinstance(end, int | float):
+ bounds.append((name, float(start), float(end)))
+ if not bounds:
+ raise ValueError(f"result has no usable phase bounds: {result_path}")
+ return bounds
+
+
+def _summaries_from_result(snapshots: list[BatchCounters], result_path: Path | None) -> list[PhaseSummary]:
+ if result_path is None:
+ return [
+ summarize_interval(
+ phase="entire_capture",
+ requested_start_seconds=snapshots[0].offset_seconds,
+ requested_end_seconds=snapshots[-1].offset_seconds,
+ before=snapshots[0],
+ after=snapshots[-1],
+ )
+ ]
+ summaries: list[PhaseSummary] = []
+ for name, start, end in _load_phase_bounds(result_path):
+ summaries.append(
+ summarize_interval(
+ phase=name,
+ requested_start_seconds=start,
+ requested_end_seconds=end,
+ before=_nearest_snapshot(snapshots, start),
+ after=_nearest_snapshot(snapshots, end),
+ )
+ )
+ return summaries
+
+
+def _load_gpu_phase_means(
+ gpu_metrics_path: Path | None,
+ summaries: Iterable[PhaseSummary],
+) -> dict[str, dict[str, float]]:
+ """Return only physical-GPU facts present in the optional NVML artifact."""
+
+ if gpu_metrics_path is None:
+ return {}
+ samples = [
+ json.loads(raw)
+ for raw in gpu_metrics_path.expanduser().resolve().read_text(encoding="utf-8").splitlines()
+ if raw.strip()
+ ]
+ result: dict[str, dict[str, float]] = {}
+ for summary in summaries:
+ selected = [
+ gpu
+ for sample in samples
+ if summary.sampled_start_seconds <= float(sample.get("offset_seconds", -1)) < summary.sampled_end_seconds
+ for gpu in sample.get("gpus", [])
+ if isinstance(gpu, dict)
+ ]
+ if not selected:
+ continue
+ utilization = [float(gpu["gpu_utilization_percent"]) for gpu in selected]
+ memory_gib = [float(gpu["memory_used_bytes"]) / (1024**3) for gpu in selected]
+ result[summary.phase] = {
+ "physical_gpu_samples": float(len(selected)),
+ "mean_gpu_utilization_percent": sum(utilization) / len(utilization),
+ "mean_memory_used_gib": sum(memory_gib) / len(memory_gib),
+ }
+ return result
+
+
+def _write_csv(path: Path, summaries: Iterable[PhaseSummary]) -> None:
+ rows = [asdict(summary) for summary in summaries]
+ if not rows:
+ return
+ with path.open("x", encoding="utf-8", newline="") as output:
+ writer = csv.DictWriter(output, fieldnames=list(rows[0]))
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def _draw_png(path: Path, summaries: list[PhaseSummary]) -> str | None:
+ """Render B=1..4 execution-equivalent proportions using Pillow when present."""
+
+ try:
+ from PIL import Image, ImageDraw, ImageFont
+ except ImportError:
+ return None
+ width = max(960, 190 + 150 * len(summaries))
+ height = 560
+ image = Image.new("RGB", (width, height), "white")
+ draw = ImageDraw.Draw(image)
+ font = ImageFont.load_default()
+ title = "ABot serving: observed model-batch execution distribution"
+ draw.text((28, 20), title, fill="#111827", font=font)
+ draw.text(
+ (28, 40),
+ "Bars use execution-equivalents (B=2 items / 2); B>4 is omitted from colored B=1..4 bars.",
+ fill="#4b5563",
+ font=font,
+ )
+ colors = ("#e67e22", "#3b82f6", "#16a34a", "#8b5cf6")
+ labels = ("B=1", "B=2", "B=3", "B=4")
+ chart_left, chart_top, chart_bottom = 72, 98, 430
+ chart_height = chart_bottom - chart_top
+ draw.line((chart_left, chart_top, chart_left, chart_bottom), fill="#6b7280", width=1)
+ draw.line((chart_left, chart_bottom, width - 30, chart_bottom), fill="#6b7280", width=1)
+ for percent in range(0, 101, 20):
+ y = chart_bottom - chart_height * percent / 100.0
+ draw.line((chart_left, y, width - 30, y), fill="#e5e7eb", width=1)
+ draw.text((20, y - 5), f"{percent}%", fill="#4b5563", font=font)
+ group_width = max(88, (width - chart_left - 40) / max(1, len(summaries)))
+ for index, summary in enumerate(summaries):
+ values = (
+ summary.b1_execution_equivalents,
+ summary.b2_execution_equivalents,
+ summary.b3_execution_equivalents,
+ summary.b4_execution_equivalents,
+ )
+ total = summary.batch_executions
+ x = chart_left + index * group_width + group_width * 0.16
+ usable_width = group_width * 0.68
+ cursor = chart_bottom
+ for value, color in zip(values, colors):
+ height_px = chart_height * value / total if total else 0.0
+ draw.rectangle((x, cursor - height_px, x + usable_width, cursor), fill=color)
+ cursor -= height_px
+ short_name = summary.phase[:18]
+ draw.text((x, chart_bottom + 10), short_name, fill="#111827", font=font)
+ draw.text((x, chart_bottom + 24), f"mean B={summary.mean_execution_batch_size:.3f}", fill="#4b5563", font=font)
+ draw.text(
+ (x, chart_bottom + 38), f"B2 exec={summary.b2_execution_share_percent:.1f}%", fill="#4b5563", font=font
+ )
+ legend_x = 30
+ for label, color in zip(labels, colors):
+ draw.rectangle((legend_x, 485, legend_x + 13, 498), fill=color)
+ draw.text((legend_x + 18, 486), label, fill="#111827", font=font)
+ legend_x += 100
+ draw.text(
+ (30, 520),
+ "A half execution-equivalent can appear at a capture boundary because each B=2 member is forwarded separately.",
+ fill="#4b5563",
+ font=font,
+ )
+ image.save(path)
+ return str(path)
+
+
+def _print_table(summaries: list[PhaseSummary]) -> None:
+ print("phase executions mean-B B1-exec B2-exec B2-exec% B2-item% B>=3-items")
+ for item in summaries:
+ print(
+ f"{item.phase[:27]:27} {item.batch_executions:10.1f} {item.mean_execution_batch_size:7.3f}"
+ f" {item.b1_execution_equivalents:8.1f} {item.b2_execution_equivalents:8.1f}"
+ f" {item.b2_execution_share_percent:9.2f}% {item.b2_item_share_percent:8.2f}%"
+ f" {item.b3_items + item.b4_items + item.b_gt4_items:11.1f}"
+ )
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = _parse_args(argv)
+ output_dir = args.output_dir.expanduser().resolve()
+ if output_dir.exists() and any(output_dir.iterdir()):
+ raise ValueError(f"refusing to reuse non-empty output directory: {output_dir}")
+ output_dir.mkdir(parents=True, exist_ok=True)
+ snapshots = load_snapshots(args.serving_metrics_dir)
+ summaries = _summaries_from_result(snapshots, args.result)
+ whole_capture = summarize_interval(
+ phase="entire_capture",
+ requested_start_seconds=snapshots[0].offset_seconds,
+ requested_end_seconds=snapshots[-1].offset_seconds,
+ before=snapshots[0],
+ after=snapshots[-1],
+ )
+ gpu_phase_means = _load_gpu_phase_means(args.gpu_metrics, summaries)
+ _write_csv(output_dir / "phase-batches.csv", summaries)
+ png_path = _draw_png(output_dir / "batch-distribution.png", summaries)
+ report = {
+ "schema_version": 1,
+ "metric_semantics": {
+ "batch_items": "Session-chunk items; every member of a coalesced execution contributes one.",
+ "batch_executions": "Execution-equivalent total; serving increments 1/B per emitted B-member.",
+ "batch_histogram": "Counts items, not executions. B=2 execution share is B2-items / 2 / executions.",
+ },
+ "capture": {
+ "first_offset_seconds": snapshots[0].offset_seconds,
+ "last_offset_seconds": snapshots[-1].offset_seconds,
+ "valid_prometheus_snapshots": len(snapshots),
+ },
+ "entire_capture": asdict(whole_capture),
+ "phases": [asdict(summary) for summary in summaries],
+ "physical_gpu_phase_means": gpu_phase_means,
+ "png": png_path,
+ }
+ (output_dir / "summary.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ _print_table([whole_capture])
+ if args.result is not None:
+ print()
+ _print_table(summaries)
+ print(f"Wrote analysis: {output_dir}")
+ if png_path is None:
+ print("Pillow unavailable: wrote JSON/CSV but no PNG.", file=sys.stderr)
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except (OSError, ValueError, json.JSONDecodeError) as exc:
+ print(f"ABot serving trace analysis failed: {exc}", file=sys.stderr)
+ raise SystemExit(2) from exc
diff --git a/tools/validation/benchmark_abot_cuda_graph.py b/tools/validation/benchmark_abot_cuda_graph.py
new file mode 100644
index 00000000..92ceafcd
--- /dev/null
+++ b/tools/validation/benchmark_abot_cuda_graph.py
@@ -0,0 +1,840 @@
+"""A/B benchmark for ABot-World's steady-state CUDA Graph continuation path.
+
+The benchmark deliberately bypasses the serving scheduler: it measures a
+single compatible microbatch after the causal KV cache has reached its fixed
+window. This makes a CUDA-Graph on/off comparison reproducible and avoids
+mistaking scheduler queueing for model-runtime speedup.
+
+``eager`` and ``cuda_graph`` accept B=1/2/3 compatible microbatches. For each
+CUDA-Graph B>1 point, this tool requires explicit runtime evidence that one
+batched graph replay occurred; it never treats B independent singleton graphs
+as a native B=2/3 result. ``steady_eager`` remains a benchmark-only static
+DiT control path and reports unsupported rather than silently falling back
+when its requested native batch is unavailable.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import gc
+import importlib.util
+import json
+import math
+import os
+import statistics
+import time
+from collections.abc import Mapping, Sequence
+from numbers import Real
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+_GRAPH_ENV = "TELEFUSER_ABOT_CUDA_GRAPH_ENABLED"
+_MODES = ("eager", "steady_eager", "cuda_graph")
+
+
+def _load_example_loader(mode: str) -> Any:
+ """Load the example loader after selecting the graph environment flag."""
+ loader_path = Path(__file__).resolve().parents[2] / "examples/abot_world/_loader.py"
+ spec = importlib.util.spec_from_file_location(f"abot_cuda_graph_loader_{mode}", loader_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot example loader: {loader_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _load_steady_eager_hook() -> Any:
+ """Load the sibling benchmark-only hook without relying on PYTHONPATH."""
+ hook_path = Path(__file__).with_name("abot_steady_eager.py")
+ spec = importlib.util.spec_from_file_location("abot_steady_eager_benchmark_hook", hook_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load steady-eager benchmark hook: {hook_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.install_steady_eager_hook
+
+
+def _parse_modes(value: str) -> list[str]:
+ modes = [item.strip() for item in value.split(",") if item.strip()]
+ invalid = sorted(set(modes).difference(_MODES))
+ if not modes or invalid:
+ raise argparse.ArgumentTypeError(f"modes must be a comma-separated subset of {','.join(_MODES)}; got {value!r}")
+ return modes
+
+
+def _parse_batch_sizes(value: str) -> list[int]:
+ try:
+ values = [int(item) for item in value.split(",") if item.strip()]
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("batch sizes must be positive integers") from exc
+ if not values or any(item < 1 for item in values):
+ raise argparse.ArgumentTypeError("batch sizes must be positive integers")
+ return values
+
+
+def _percentile(values: Sequence[float], quantile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ return ordered[min(len(ordered) - 1, math.ceil(len(ordered) * quantile) - 1)]
+
+
+def _summary(values: Sequence[float]) -> dict[str, float]:
+ if not values:
+ return {"count": 0.0, "mean": 0.0, "p50": 0.0, "p95": 0.0, "min": 0.0, "max": 0.0}
+ return {
+ "count": float(len(values)),
+ "mean": statistics.fmean(values),
+ "p50": _percentile(values, 0.50),
+ "p95": _percentile(values, 0.95),
+ "min": min(values),
+ "max": max(values),
+ }
+
+
+def _json_safe(value: Any) -> Any:
+ if isinstance(value, Path):
+ return str(value)
+ if isinstance(value, torch.Tensor):
+ if value.numel() == 1:
+ return value.item()
+ return {"tensor_shape": list(value.shape), "tensor_dtype": str(value.dtype)}
+ if isinstance(value, Mapping):
+ return {str(key): _json_safe(item) for key, item in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [_json_safe(item) for item in value]
+ if isinstance(value, (str, int, float, bool)) or value is None:
+ return value
+ return str(value)
+
+
+def _numeric_stage_summary(samples: Sequence[Mapping[str, Any]]) -> dict[str, dict[str, float]]:
+ values_by_key: dict[str, list[float]] = {}
+ for sample in samples:
+ for key, value in sample.items():
+ if isinstance(value, Real) and not isinstance(value, bool):
+ values_by_key.setdefault(str(key), []).append(float(value))
+ return {key: _summary(values) for key, values in sorted(values_by_key.items())}
+
+
+def _graph_metrics_from_runtime(pipeline: Any) -> dict[str, Any]:
+ """Best-effort extraction; core code owns the exact CUDA-Graph counters."""
+ result: dict[str, Any] = {}
+ candidates = [
+ ("pipeline", pipeline),
+ ("denoise_stage", getattr(pipeline, "denoise_stage", None)),
+ ]
+ for owner_name, owner in candidates:
+ if owner is None:
+ continue
+ for attr in ("cuda_graph_metrics", "cuda_graph_runtime_metrics", "graph_runtime_metrics"):
+ callback = getattr(owner, attr, None)
+ if not callable(callback):
+ continue
+ try:
+ metrics = callback()
+ except Exception as exc: # A diagnostic must never hide a benchmark result.
+ result[f"{owner_name}.{attr}_error"] = f"{type(exc).__name__}: {exc}"
+ continue
+ if isinstance(metrics, Mapping):
+ result[f"{owner_name}.{attr}"] = _json_safe(metrics)
+ return result
+
+
+def _graph_replay_observed(stage_samples: Sequence[Mapping[str, Any]], runtime_metrics: Mapping[str, Any]) -> bool:
+ """Return true only for an explicit replay/used counter, never by inference."""
+
+ def walk(value: Any, graph_context: bool = False) -> bool:
+ if isinstance(value, Mapping):
+ for raw_key, item in value.items():
+ key = str(raw_key).lower()
+ item_graph_context = graph_context or "graph" in key
+ if isinstance(item, (Mapping, list, tuple)) and walk(item, item_graph_context):
+ return True
+ if not item_graph_context:
+ continue
+ if isinstance(item, str) and item.lower() in {"true", "yes", "replay", "replayed", "used", "hit"}:
+ return True
+ if not any(token in key for token in ("replay", "used", "hit")):
+ continue
+ if isinstance(item, bool) and item:
+ return True
+ if isinstance(item, Real) and item > 0:
+ return True
+ return False
+ if isinstance(value, (list, tuple)):
+ return any(walk(item, graph_context) for item in value)
+ return False
+
+ return any(walk(sample) for sample in stage_samples) or walk(runtime_metrics)
+
+
+_TAEW_DECODE_MODE_NAMES = {
+ 0: "singleton",
+ 1: "synchronized_native_batch",
+ 2: "serial_fallback",
+}
+
+
+def _as_optional_int(value: Any) -> int | None:
+ """Return a strict integer metric, excluding booleans and opaque values."""
+ if isinstance(value, bool):
+ return None
+ if isinstance(value, Real) and float(value).is_integer():
+ return int(value)
+ return None
+
+
+def _normalise_graph_mode(value: Any) -> str | None:
+ if isinstance(value, str):
+ normalised = value.strip().lower()
+ return normalised or None
+ return None
+
+
+def _graph_batch_verification(
+ stage_samples: Sequence[Mapping[str, Any]],
+ requested_batch_size: int,
+) -> dict[str, Any]:
+ """Prove that measured Graph chunks were one native B-sized replay.
+
+ A positive replay counter alone is insufficient for B>1: it could describe
+ one or more singleton graph slots. B>1 therefore requires a B-sized graph
+ metric and, where exported, its explicit native-batched marker. Older core
+ revisions can instead export cuda_graph_mode=batched.
+ """
+
+ chunks: list[dict[str, Any]] = []
+ for sample in stage_samples:
+ scheduler_batch_size = _as_optional_int(sample.get("batch_size"))
+ graph_batch_size = _as_optional_int(sample.get("cuda_graph_batch_size"))
+ graph_mode = _normalise_graph_mode(sample.get("cuda_graph_mode"))
+ graph_batched = _as_optional_int(sample.get("cuda_graph_batched"))
+ replay_count = _as_optional_int(sample.get("cuda_graph_replays")) or 0
+ fallback_count = _as_optional_int(sample.get("cuda_graph_fallback")) or 0
+ enabled = _as_optional_int(sample.get("cuda_graph_enabled")) == 1
+ eligible = _as_optional_int(sample.get("cuda_graph_eligible")) == 1
+ scheduler_batch_matches = scheduler_batch_size == requested_batch_size
+ if requested_batch_size == 1:
+ graph_batch_matches = True
+ elif graph_batched is not None:
+ graph_batch_matches = graph_batch_size == requested_batch_size and graph_batched == 1
+ else:
+ graph_batch_matches = graph_mode == "batched" or graph_batch_size == requested_batch_size
+ chunks.append(
+ {
+ "scheduler_batch_size": scheduler_batch_size,
+ "cuda_graph_batch_size": graph_batch_size,
+ "cuda_graph_mode": graph_mode,
+ "cuda_graph_batched": graph_batched,
+ "cuda_graph_replays": replay_count,
+ "cuda_graph_fallback": fallback_count,
+ "cuda_graph_enabled": enabled,
+ "cuda_graph_eligible": eligible,
+ "scheduler_batch_matches": scheduler_batch_matches,
+ "graph_batch_matches": graph_batch_matches,
+ "replay_observed": replay_count > 0,
+ "fallback_observed": fallback_count > 0,
+ "verified": scheduler_batch_matches
+ and graph_batch_matches
+ and enabled
+ and eligible
+ and replay_count > 0
+ and fallback_count == 0,
+ }
+ )
+ return {
+ "requested_batch_size": requested_batch_size,
+ "measured_chunks": len(chunks),
+ "all_scheduler_batches_exact": bool(chunks) and all(item["scheduler_batch_matches"] for item in chunks),
+ "all_graph_batches_native": bool(chunks) and all(item["graph_batch_matches"] for item in chunks),
+ "all_chunks_replayed": bool(chunks) and all(item["replay_observed"] for item in chunks),
+ "fallback_observed": any(item["fallback_observed"] for item in chunks),
+ "verified": bool(chunks) and all(item["verified"] for item in chunks),
+ "chunks": chunks,
+ }
+
+
+def _taew_batch_verification(
+ stage_samples: Sequence[Mapping[str, Any]],
+ requested_batch_size: int,
+) -> dict[str, Any]:
+ """Report actual LightVAE/TAeW decode behavior, including serial fallback."""
+ expected_mode = 0 if requested_batch_size == 1 else 1
+ chunks: list[dict[str, Any]] = []
+ for sample in stage_samples:
+ mode = _as_optional_int(sample.get("taew_decode_mode"))
+ effective_batch_size = _as_optional_int(sample.get("taew_decode_batch_size"))
+ invocations = _as_optional_int(sample.get("taew_decode_invocations"))
+ items = _as_optional_int(sample.get("taew_decode_items"))
+ verified = (
+ mode == expected_mode
+ and effective_batch_size == requested_batch_size
+ and invocations == 1
+ and items == requested_batch_size
+ )
+ chunks.append(
+ {
+ "mode": mode,
+ "mode_name": _TAEW_DECODE_MODE_NAMES.get(mode, "unreported"),
+ "effective_batch_size": effective_batch_size,
+ "invocations": invocations,
+ "items": items,
+ "native_batch_verified": verified,
+ }
+ )
+ return {
+ "requested_batch_size": requested_batch_size,
+ "expected_mode": expected_mode,
+ "expected_mode_name": _TAEW_DECODE_MODE_NAMES[expected_mode],
+ "measured_chunks": len(chunks),
+ "reported": bool(chunks) and all(item["mode"] is not None for item in chunks),
+ "verified": bool(chunks) and all(item["native_batch_verified"] for item in chunks),
+ "fallback_observed": any(item["mode"] == 2 for item in chunks),
+ "chunks": chunks,
+ }
+
+
+class _NativeBatchProbe:
+ """Observe Python model entry points in a benchmark-only pipeline.
+
+ This validates eager/static-eager DiT calls. CUDA Graph replays deliberately
+ bypass Python after capture, so graph-native batching is instead proved by
+ _graph_batch_verification and core-reported batch facts.
+ """
+
+ def __init__(self, pipeline: Any) -> None:
+ self._patches: list[tuple[Any, str, Any]] = []
+ self._measurement_active = False
+ self._calls: dict[str, list[int | None]] = {
+ "dit_dynamic": [],
+ "dit_steady_state": [],
+ "taew_decode": [],
+ }
+ self._install_on_pipeline(pipeline)
+
+ @staticmethod
+ def _batch_size(args: tuple[Any, ...], kwargs: Mapping[str, Any]) -> int | None:
+ value = kwargs.get("x")
+ if value is None:
+ value = kwargs.get("latents")
+ if value is None and args:
+ value = args[0]
+ if isinstance(value, torch.Tensor) and value.ndim > 0:
+ return int(value.shape[0])
+ return None
+
+ def _wrap(self, owner: Any, attribute: str, bucket: str) -> None:
+ original = getattr(owner, attribute, None)
+ if not callable(original):
+ return
+
+ def wrapped(*args: Any, **kwargs: Any) -> Any:
+ if self._measurement_active:
+ self._calls[bucket].append(self._batch_size(args, kwargs))
+ return original(*args, **kwargs)
+
+ try:
+ setattr(owner, attribute, wrapped)
+ except (AttributeError, TypeError):
+ return
+ self._patches.append((owner, attribute, original))
+
+ def _install_on_pipeline(self, pipeline: Any) -> None:
+ denoise_stage = getattr(pipeline, "denoise_stage", None)
+ dit = getattr(denoise_stage, "dit", None)
+ if dit is not None:
+ self._wrap(dit, "forward", "dit_dynamic")
+ self._wrap(dit, "forward_steady_state", "dit_steady_state")
+ taew_stage = getattr(pipeline, "taew_decode_stage", None)
+ if taew_stage is not None:
+ self._wrap(taew_stage, "decode_chunks", "taew_decode")
+
+ def begin_measurement(self) -> None:
+ for values in self._calls.values():
+ values.clear()
+ self._measurement_active = True
+
+ def metrics(self) -> dict[str, Any]:
+ return {
+ "installed_wrappers": len(self._patches),
+ "dit_dynamic_batch_sizes_measured": list(self._calls["dit_dynamic"]),
+ "dit_steady_state_batch_sizes_measured": list(self._calls["dit_steady_state"]),
+ "taew_decode_input_batch_sizes_measured": list(self._calls["taew_decode"]),
+ }
+
+ def close(self) -> None:
+ for owner, attribute, original in reversed(self._patches):
+ setattr(owner, attribute, original)
+ self._patches.clear()
+
+
+def _dit_batch_verification(
+ probe_metrics: Mapping[str, Any],
+ requested_batch_size: int,
+ *,
+ cuda_graph: bool,
+ graph_verification: Mapping[str, Any],
+) -> dict[str, Any]:
+ """Separate Python-observed eager batching from CUDA-Graph batch evidence."""
+ dynamic = [item for item in probe_metrics.get("dit_dynamic_batch_sizes_measured", []) if item is not None]
+ steady = [item for item in probe_metrics.get("dit_steady_state_batch_sizes_measured", []) if item is not None]
+ observed = dynamic + steady
+ if cuda_graph:
+ verified = bool(graph_verification.get("verified"))
+ evidence = "core_cuda_graph_batch_metrics"
+ else:
+ verified = bool(observed) and all(item == requested_batch_size for item in observed)
+ evidence = "python_dit_entrypoint_probe"
+ return {
+ "requested_batch_size": requested_batch_size,
+ "cuda_graph": cuda_graph,
+ "evidence": evidence,
+ "dynamic_calls": dynamic,
+ "steady_state_calls": steady,
+ "verified": verified,
+ }
+
+
+def _warmup_chunks_for_steady_state(pipeline: Any, control_latent_frames: int, requested: int) -> tuple[int, int]:
+ """Return (effective warmups, automatic minimum) after the first chunk.
+
+ An initial chunk advances the session by ``control_latent_frames``. To
+ exercise the fixed-window continuation graph once, fill the local window
+ and issue one more continuation chunk. This keeps graph capture outside
+ the measured samples.
+ """
+ dit = getattr(getattr(pipeline, "denoise_stage", None), "dit", None)
+ local_frames = int(getattr(dit, "local_attn_size", 0))
+ fill_chunks = max(0, math.ceil(max(0, local_frames - control_latent_frames) / control_latent_frames))
+ automatic_minimum = fill_chunks + 1
+ return max(requested, automatic_minimum), automatic_minimum
+
+
+def _make_pipeline(args: argparse.Namespace, mode: str) -> Any:
+ os.environ[_GRAPH_ENV] = "1" if mode == "cuda_graph" else "0"
+ loader = _load_example_loader(mode)
+ # Import only after setting the environment flag. The core implementation
+ # reads the flag while constructing a fresh pipeline for every A/B point.
+ from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+
+ return loader.get_pipeline(
+ model_root=args.model_root,
+ device_id=args.device_id,
+ pipeline_class=ABotWorldInteractivePipeline,
+ )
+
+
+def _run_point(args: argparse.Namespace, mode: str, batch_size: int, image: Image.Image) -> dict[str, Any]:
+ graph_requested = mode == "cuda_graph"
+ steady_eager_requested = mode == "steady_eager"
+ if steady_eager_requested and batch_size != 1:
+ return {
+ "mode": mode,
+ "cuda_graph_requested": graph_requested,
+ "steady_eager_requested": steady_eager_requested,
+ "batch": batch_size,
+ "status": "unsupported",
+ "reason": (
+ "steady_eager is a B=1-only static DiT control path; use eager and cuda_graph to benchmark "
+ "native B=2/3 continuation batches."
+ ),
+ }
+
+ batch_probe: _NativeBatchProbe | None = None
+ pipeline = None
+ sessions: list[Any] = []
+ steady_eager_hook: Any | None = None
+ try:
+ pipeline = _make_pipeline(args, mode)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(f"CUDA Graph benchmark requires a CUDA pipeline, got {pipeline.device!r}")
+ pipeline.preload_models()
+ batch_probe = _NativeBatchProbe(pipeline)
+ if steady_eager_requested:
+ steady_eager_hook = _load_steady_eager_hook()(pipeline)
+ for index in range(batch_size):
+ sessions.append(
+ pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=args.seed + index,
+ session_id=f"cuda-graph-{mode}-b{batch_size}-s{index}",
+ )
+ )
+ controls = [{"W": True} for _ in sessions]
+ expected_frames = 4 * args.control_latent_frames
+
+ # First block is deliberately outside steady-state timing.
+ first = pipeline.generate_next_blocks(sessions, controls, control_latent_frames=args.control_latent_frames)
+ if any(len(frames) != expected_frames for frames in first):
+ raise RuntimeError(f"first block emitted unexpected frame counts: {[len(frames) for frames in first]}")
+
+ effective_warmups, automatic_warmup_minimum = _warmup_chunks_for_steady_state(
+ pipeline, args.control_latent_frames, args.warmup_chunks
+ )
+ warmup_stage_samples: list[dict[str, Any]] = []
+ for _ in range(effective_warmups):
+ frames = pipeline.generate_next_blocks(sessions, controls, control_latent_frames=args.control_latent_frames)
+ if any(len(item) != expected_frames for item in frames):
+ raise RuntimeError(f"warmup emitted unexpected frame counts: {[len(item) for item in frames]}")
+ warmup_stage_samples.append(_json_safe(pipeline.last_stage_metrics()))
+
+ if steady_eager_hook is not None:
+ steady_eager_hook.begin_measurement()
+ if batch_probe is not None:
+ batch_probe.begin_measurement()
+
+ torch.cuda.synchronize(device)
+ allocated_before = int(torch.cuda.memory_allocated(device))
+ reserved_before = int(torch.cuda.memory_reserved(device))
+ torch.cuda.reset_peak_memory_stats(device)
+ wall_samples: list[float] = []
+ stage_samples: list[dict[str, Any]] = []
+ for _ in range(args.repeats):
+ torch.cuda.synchronize(device)
+ started_at = time.perf_counter()
+ frames = pipeline.generate_next_blocks(sessions, controls, control_latent_frames=args.control_latent_frames)
+ torch.cuda.synchronize(device)
+ elapsed = time.perf_counter() - started_at
+ if any(len(item) != expected_frames for item in frames):
+ raise RuntimeError(f"sample emitted unexpected frame counts: {[len(item) for item in frames]}")
+ wall_samples.append(elapsed)
+ stage_samples.append(_json_safe(pipeline.last_stage_metrics()))
+
+ runtime_graph_metrics = _graph_metrics_from_runtime(pipeline)
+ graph_verification = _graph_batch_verification(stage_samples, batch_size)
+ graph_replay_observed = graph_requested and bool(graph_verification["verified"])
+ steady_eager_metrics = steady_eager_hook.runtime_metrics() if steady_eager_hook is not None else {}
+ steady_eager_observed = bool(steady_eager_metrics.get("steady_calls_measured", 0))
+ probe_metrics = batch_probe.metrics() if batch_probe is not None else {}
+ dit_batch_verification = _dit_batch_verification(
+ probe_metrics,
+ batch_size,
+ cuda_graph=graph_requested,
+ graph_verification=graph_verification,
+ )
+ taew_batch_verification = _taew_batch_verification(stage_samples, batch_size)
+ native_microbatch_verified = bool(dit_batch_verification["verified"]) and bool(
+ taew_batch_verification["verified"]
+ )
+ wall = _summary(wall_samples)
+ stage_summary = _numeric_stage_summary(stage_samples)
+ chunk_seconds = wall["mean"]
+ result = {
+ "mode": mode,
+ "execution_path": {
+ "eager": "legacy_dynamic",
+ "steady_eager": "steady_state_eager_benchmark_hook",
+ "cuda_graph": "steady_state_cuda_graph",
+ }[mode],
+ "cuda_graph_requested": graph_requested,
+ "steady_eager_requested": steady_eager_requested,
+ "steady_eager_observed": steady_eager_observed,
+ "steady_eager_metrics": steady_eager_metrics,
+ "batch": batch_size,
+ "status": "ok",
+ "device": str(device),
+ "control_latent_frames": args.control_latent_frames,
+ "frames_per_session_per_chunk": expected_frames,
+ "repeats": args.repeats,
+ "warmup_chunks_requested": args.warmup_chunks,
+ "warmup_chunks_effective": effective_warmups,
+ "warmup_chunks_steady_state_minimum": automatic_warmup_minimum,
+ "chunk_wall_seconds": wall,
+ "chunk_time_seconds": chunk_seconds,
+ "aggregate_fps": (expected_frames * batch_size / chunk_seconds) if chunk_seconds else 0.0,
+ "fps_per_session": (expected_frames / chunk_seconds) if chunk_seconds else 0.0,
+ "stage_seconds": stage_summary,
+ "stage_samples": stage_samples,
+ "warmup_stage_samples": warmup_stage_samples,
+ "runtime_graph_metrics": runtime_graph_metrics,
+ "cuda_graph_verification": graph_verification,
+ "cuda_graph_replay_observed": graph_replay_observed,
+ "native_batch_probe": probe_metrics,
+ "dit_batch_verification": dit_batch_verification,
+ "taew_batch_verification": taew_batch_verification,
+ "native_microbatch_verified": native_microbatch_verified,
+ "gpu_memory": {
+ "allocated_before_measured_bytes": allocated_before,
+ "reserved_before_measured_bytes": reserved_before,
+ "peak_allocated_measured_bytes": int(torch.cuda.max_memory_allocated(device)),
+ "allocated_after_measured_bytes": int(torch.cuda.memory_allocated(device)),
+ "reserved_after_measured_bytes": int(torch.cuda.memory_reserved(device)),
+ },
+ }
+ if graph_requested and args.require_graph_replay and not graph_replay_observed:
+ result["status"] = "graph_unverified"
+ result["error"] = (
+ "No verified measured CUDA-Graph replay was observed for the requested batch. "
+ "For B>1 this requires batch_size=B, replay>0, fallback=0, and "
+ "cuda_graph_mode=batched or cuda_graph_batch_size=B on every measured chunk."
+ )
+ if steady_eager_requested and not steady_eager_observed:
+ result["status"] = "steady_eager_unverified"
+ result["error"] = (
+ "No measured full-window forward_steady_state eager invocation was observed. "
+ "This prevents a legacy dynamic fallback from being reported as steady eager."
+ )
+ if args.require_native_batch and result["status"] == "ok" and not native_microbatch_verified:
+ result["status"] = "native_batch_unverified"
+ result["error"] = (
+ "The requested scheduler batch did not prove both native DiT and synchronized LightVAE batching. "
+ "Inspect dit_batch_verification and taew_batch_verification for the exact fallback."
+ )
+ return result
+ except torch.OutOfMemoryError as exc:
+ return {
+ "mode": mode,
+ "cuda_graph_requested": graph_requested,
+ "steady_eager_requested": steady_eager_requested,
+ "batch": batch_size,
+ "status": "oom",
+ "error": str(exc).splitlines()[0],
+ }
+ except Exception as exc:
+ return {
+ "mode": mode,
+ "cuda_graph_requested": graph_requested,
+ "steady_eager_requested": steady_eager_requested,
+ "batch": batch_size,
+ "status": "error",
+ "error": f"{type(exc).__name__}: {exc}",
+ }
+ finally:
+ if steady_eager_hook is not None:
+ try:
+ steady_eager_hook.close()
+ except Exception:
+ pass
+ if batch_probe is not None:
+ try:
+ batch_probe.close()
+ except Exception:
+ pass
+ if pipeline is not None:
+ for session in sessions:
+ try:
+ pipeline.close_interactive_session(session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ del sessions
+ del pipeline
+ gc.collect()
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _display(value: Any, digits: int = 3) -> str:
+ if isinstance(value, Real) and not isinstance(value, bool):
+ return f"{float(value):.{digits}f}"
+ return str(value if value is not None else "")
+
+
+def _observed_fact(verification: Mapping[str, Any], key: str) -> str:
+ """Render distinct per-chunk evidence values compactly for CSV/Markdown."""
+ chunks = verification.get("chunks", [])
+ if not isinstance(chunks, Sequence):
+ return ""
+ values = sorted({str(item.get(key)) for item in chunks if isinstance(item, Mapping) and item.get(key) is not None})
+ return ",".join(values)
+
+
+def _write_outputs(output_dir: Path, results: Sequence[Mapping[str, Any]], args: argparse.Namespace) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ (output_dir / "results.json").write_text(
+ json.dumps(_json_safe({"arguments": vars(args), "results": list(results)}), indent=2, sort_keys=True) + "\n",
+ encoding="utf-8",
+ )
+ fields = [
+ "mode",
+ "execution_path",
+ "batch",
+ "status",
+ "chunk_time_seconds",
+ "aggregate_fps",
+ "fps_per_session",
+ "denoise_seconds_mean",
+ "vae_decode_seconds_mean",
+ "postprocess_seconds_mean",
+ "steady_eager_observed",
+ "cuda_graph_replay_observed",
+ "cuda_graph_batch_verified",
+ "cuda_graph_mode",
+ "cuda_graph_batch_size",
+ "dit_native_batch_verified",
+ "taew_native_batch_verified",
+ "taew_decode_mode",
+ "taew_effective_batch_size",
+ "native_microbatch_verified",
+ "error",
+ ]
+ rows: list[dict[str, Any]] = []
+ for result in results:
+ stage = result.get("stage_seconds", {}) if isinstance(result, Mapping) else {}
+ graph = result.get("cuda_graph_verification", {}) if isinstance(result, Mapping) else {}
+ dit = result.get("dit_batch_verification", {}) if isinstance(result, Mapping) else {}
+ taew = result.get("taew_batch_verification", {}) if isinstance(result, Mapping) else {}
+ rows.append(
+ {
+ "mode": result.get("mode"),
+ "execution_path": result.get("execution_path"),
+ "batch": result.get("batch"),
+ "status": result.get("status"),
+ "chunk_time_seconds": result.get("chunk_time_seconds"),
+ "aggregate_fps": result.get("aggregate_fps"),
+ "fps_per_session": result.get("fps_per_session"),
+ "denoise_seconds_mean": stage.get("denoise_seconds", {}).get("mean"),
+ "vae_decode_seconds_mean": stage.get("vae_decode_seconds", {}).get("mean"),
+ "postprocess_seconds_mean": stage.get("postprocess_seconds", {}).get("mean"),
+ "steady_eager_observed": result.get("steady_eager_observed"),
+ "cuda_graph_replay_observed": result.get("cuda_graph_replay_observed"),
+ "cuda_graph_batch_verified": graph.get("verified") if isinstance(graph, Mapping) else None,
+ "cuda_graph_mode": _observed_fact(graph, "cuda_graph_mode") if isinstance(graph, Mapping) else "",
+ "cuda_graph_batch_size": _observed_fact(graph, "cuda_graph_batch_size")
+ if isinstance(graph, Mapping)
+ else "",
+ "dit_native_batch_verified": dit.get("verified") if isinstance(dit, Mapping) else None,
+ "taew_native_batch_verified": taew.get("verified") if isinstance(taew, Mapping) else None,
+ "taew_decode_mode": _observed_fact(taew, "mode_name") if isinstance(taew, Mapping) else "",
+ "taew_effective_batch_size": _observed_fact(taew, "effective_batch_size")
+ if isinstance(taew, Mapping)
+ else "",
+ "native_microbatch_verified": result.get("native_microbatch_verified"),
+ "error": result.get("error") or result.get("reason"),
+ }
+ )
+ with (output_dir / "results.csv").open("w", newline="", encoding="utf-8") as handle:
+ writer = csv.DictWriter(handle, fieldnames=fields)
+ writer.writeheader()
+ writer.writerows(rows)
+
+ lines = [
+ "# ABot steady-state path comparison",
+ "",
+ (
+ "`steady_eager` is benchmark-only and invokes the same full-window "
+ "`forward_steady_state` path without CUDA Graph capture. Its row is valid only "
+ "when `steady_eager_observed` is `True`; `cuda_graph` is valid only when "
+ "`cuda_graph_replay_observed` is `True`. For B>1, the CSV also records native-DiT, "
+ "native-LightVAE, and graph-batch evidence; a scheduler batch is not treated as native merely "
+ "because it contains more than one session."
+ ),
+ "",
+ (
+ "| Mode | Path | B | Status | Chunk (s) | Aggregate FPS | FPS/session | DiT (s) | "
+ "VAE decode (s) | Postprocess (s) | Steady eager | Graph replay | Graph B native | "
+ "DiT native | LightVAE native |"
+ ),
+ "| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- |",
+ ]
+ for row in rows:
+ lines.append(
+ (
+ "| {mode} | {path} | {batch} | {status} | {chunk} | {aggregate} | {session} | "
+ "{denoise} | {vae} | {post} | {steady} | {replay} | {graph_batch} | {dit_batch} | {taew_batch} |"
+ ).format(
+ mode=_display(row["mode"]),
+ path=_display(row["execution_path"]),
+ batch=_display(row["batch"], 0),
+ status=_display(row["status"]),
+ chunk=_display(row["chunk_time_seconds"]),
+ aggregate=_display(row["aggregate_fps"], 2),
+ session=_display(row["fps_per_session"], 2),
+ denoise=_display(row["denoise_seconds_mean"]),
+ vae=_display(row["vae_decode_seconds_mean"]),
+ post=_display(row["postprocess_seconds_mean"]),
+ steady=_display(row["steady_eager_observed"]),
+ replay=_display(row["cuda_graph_replay_observed"]),
+ graph_batch=_display(row["cuda_graph_batch_verified"]),
+ dit_batch=_display(row["dit_native_batch_verified"]),
+ taew_batch=_display(row["taew_native_batch_verified"]),
+ )
+ )
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--modes", type=_parse_modes, default=["eager", "steady_eager", "cuda_graph"])
+ parser.add_argument("--batch-sizes", type=_parse_batch_sizes, default=[1])
+ parser.add_argument("--control-latent-frames", type=int, choices=(1, 2, 3), default=3)
+ parser.add_argument("--warmup-chunks", type=int, default=6)
+ parser.add_argument("--repeats", type=int, default=12)
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument(
+ "--device-id",
+ type=int,
+ default=0,
+ help="Logical CUDA device after CUDA_VISIBLE_DEVICES remapping (normally 0).",
+ )
+ parser.add_argument(
+ "--require-graph-replay",
+ action="store_true",
+ help="Mark graph results invalid unless core metrics explicitly report a graph replay/use.",
+ )
+ parser.add_argument(
+ "--require-native-batch",
+ action="store_true",
+ help=("Mark a point invalid unless DiT and LightVAE both prove one native requested-size batch."),
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Validate and print the planned A/B sweep without loading a model.",
+ )
+ args = parser.parse_args()
+ if args.warmup_chunks < 0:
+ parser.error("warmup-chunks must be non-negative")
+ if args.repeats < 1:
+ parser.error("repeats must be positive")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ plan = {
+ "modes": args.modes,
+ "batch_sizes": args.batch_sizes,
+ "graph_environment_variable": _GRAPH_ENV,
+ "graph_environment_values": {"eager": "0", "steady_eager": "0", "cuda_graph": "1"},
+ "device_id": args.device_id,
+ "cuda_graph_batch_sizes": [1, 2, 3],
+ "steady_eager_batch_sizes": [1],
+ "B_gt_1_graph_gate": (
+ "batch_size=B, cuda_graph_batch_size=B plus cuda_graph_batched=1 when exported "
+ "(or cuda_graph_mode=batched on older cores), replay>0, fallback=0"
+ ),
+ "B_gt_1_native_batch_gate": "native DiT evidence and TAeW synchronized-native-batch evidence",
+ "steady_eager": "benchmark-only forward_steady_state execution without CUDA Graph capture",
+ }
+ if args.dry_run:
+ print(json.dumps(plan, indent=2, sort_keys=True))
+ return
+
+ image = Image.open(args.image).convert("RGB")
+ results: list[dict[str, Any]] = []
+ for mode in args.modes:
+ for batch_size in args.batch_sizes:
+ print(f"running mode={mode} batch={batch_size}", flush=True)
+ results.append(_run_point(args, mode, batch_size, image))
+ _write_outputs(args.output_dir, results, args)
+ _write_outputs(args.output_dir, results, args)
+ print(json.dumps(_json_safe(results), indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/benchmark_abot_livekit_burst.py b/tools/validation/benchmark_abot_livekit_burst.py
index 847c023c..ce4213e8 100644
--- a/tools/validation/benchmark_abot_livekit_burst.py
+++ b/tools/validation/benchmark_abot_livekit_burst.py
@@ -98,6 +98,37 @@ class AdmissionExpectation:
expected_queue_size: int | None = None
+@dataclass(frozen=True)
+class DiagnosticInitialControlBarrier:
+ """Synthetic synchronized-first-control configuration for a diagnostic trace."""
+
+ phase_name: str
+ expected_connected_sessions: int
+ timeout_seconds: float
+
+
+@dataclass(frozen=True)
+class LifecycleTraceEvent:
+ """One explicit user lifecycle transition in a black-box replay."""
+
+ offset_seconds: float
+ sequence: int
+ event: str
+ trace_session_id: str
+ source_session_id: int | None
+ source_user_id: int | None
+ input_enabled: bool | None
+
+
+@dataclass(frozen=True)
+class LifecycleTrace:
+ """An exact per-session lifecycle trace, unlike aggregate phase fractions."""
+
+ kind: str
+ duration_seconds: float
+ events: tuple[LifecycleTraceEvent, ...]
+
+
@dataclass(frozen=True)
class Scenario:
"""Validated configuration of one complete black-box experiment."""
@@ -116,6 +147,8 @@ class Scenario:
expected_worker_mode: str | None
admission: AdmissionExpectation
expected_num_workers: int | None
+ diagnostic_initial_control_barrier: DiagnosticInitialControlBarrier | None
+ lifecycle_trace: LifecycleTrace | None
raw: dict[str, Any]
@@ -233,6 +266,70 @@ def optional_positive_int(key: str) -> int | None:
)
+def _parse_diagnostic_initial_control_barrier(
+ value: object,
+ phases: Sequence[Phase],
+) -> DiagnosticInitialControlBarrier | None:
+ """Parse an explicitly synthetic synchronized-first-control diagnostic.
+
+ This is deliberately restricted to a fresh first phase. A later phase
+ would contain users that have already sent controls, so it could not be
+ accurately described as an initial-control alignment.
+ """
+ raw = _require_mapping(value, "diagnostic")
+ barrier_value = raw.get("initial_control_barrier")
+ if barrier_value is None:
+ return None
+ barrier = _require_mapping(barrier_value, "diagnostic.initial_control_barrier")
+ enabled = barrier.get("enabled", False)
+ if not isinstance(enabled, bool):
+ raise ScenarioError("diagnostic.initial_control_barrier.enabled must be a boolean")
+ if not enabled:
+ return None
+ if barrier.get("kind") != "phase_aligned_initial_control":
+ raise ScenarioError("diagnostic.initial_control_barrier.kind must be phase_aligned_initial_control")
+ if barrier.get("not_a_real_user_trace") is not True:
+ raise ScenarioError("diagnostic.initial_control_barrier.not_a_real_user_trace must be true")
+ phase_name = barrier.get("phase")
+ if not isinstance(phase_name, str) or not phase_name:
+ raise ScenarioError("diagnostic.initial_control_barrier.phase must be a non-empty string")
+ phase_index = next((index for index, phase in enumerate(phases) if phase.name == phase_name), None)
+ if phase_index is None:
+ raise ScenarioError("diagnostic.initial_control_barrier.phase must name a scenario phase")
+ if phase_index != 0:
+ raise ScenarioError(
+ "diagnostic.initial_control_barrier only supports the first phase; "
+ "later phases are not initial-control traces"
+ )
+ phase = phases[phase_index]
+ expected = _require_non_negative_int(
+ barrier.get("expected_connected_sessions"),
+ "diagnostic.initial_control_barrier.expected_connected_sessions",
+ )
+ if expected < 1:
+ raise ScenarioError("diagnostic.initial_control_barrier.expected_connected_sessions must be positive")
+ if expected != phase.target_users:
+ raise ScenarioError(
+ "diagnostic.initial_control_barrier.expected_connected_sessions must equal the fresh phase target_users"
+ )
+ if phase.active_input_fraction != 1.0:
+ raise ScenarioError(
+ "diagnostic.initial_control_barrier requires active_input_fraction=1.0 "
+ "so every released first control is active"
+ )
+ timeout = _require_positive_float(
+ barrier.get("timeout_seconds"),
+ "diagnostic.initial_control_barrier.timeout_seconds",
+ )
+ if timeout >= phase.duration_seconds:
+ raise ScenarioError("diagnostic.initial_control_barrier.timeout_seconds must be shorter than its phase")
+ return DiagnosticInitialControlBarrier(
+ phase_name=phase.name,
+ expected_connected_sessions=expected,
+ timeout_seconds=timeout,
+ )
+
+
def load_scenario(path: Path, *, server_url_override: str | None = None) -> Scenario:
"""Load and validate a JSON user-wave scenario."""
try:
@@ -313,11 +410,7 @@ def load_scenario(path: Path, *, server_url_override: str | None = None) -> Scen
f"phases[{index}].input_transition_window_seconds",
allow_zero=True,
)
- if (
- arrival_window > duration
- or departure_window > duration
- or input_transition_window > duration
- ):
+ if arrival_window > duration or departure_window > duration or input_transition_window > duration:
raise ScenarioError(f"phases[{index}] transition windows cannot exceed duration")
phases.append(
Phase(
@@ -331,6 +424,11 @@ def load_scenario(path: Path, *, server_url_override: str | None = None) -> Scen
)
)
+ diagnostic_initial_control_barrier = _parse_diagnostic_initial_control_barrier(
+ raw.get("diagnostic", {}),
+ phases,
+ )
+
measurement = _require_mapping(raw.get("measurement", {}), "measurement")
expected_worker_mode = raw.get("expected_worker_mode")
if expected_worker_mode is not None and not isinstance(expected_worker_mode, str):
@@ -377,6 +475,8 @@ def load_scenario(path: Path, *, server_url_override: str | None = None) -> Scen
seed=seed,
expected_worker_mode=expected_worker_mode,
expected_num_workers=expected_num_workers,
+ diagnostic_initial_control_barrier=diagnostic_initial_control_barrier,
+ lifecycle_trace=None,
raw=raw,
)
@@ -424,6 +524,11 @@ class LiveKitWaveSession:
rtc: Any
record_event: Any
started_at: float
+ trace_session_id: str | None = None
+ source_trace_session_id: int | None = None
+ source_trace_user_id: int | None = None
+ diagnostic_initial_control_barrier_phase: str | None = None
+ initial_control_gate: asyncio.Event | None = field(default=None, repr=False)
_room: Any | None = field(default=None, init=False, repr=False)
_video_streams: list[Any] = field(default_factory=list, init=False, repr=False)
_video_tasks: list[asyncio.Task[None]] = field(default_factory=list, init=False, repr=False)
@@ -433,6 +538,8 @@ class LiveKitWaveSession:
create_started_at: float | None = field(default=None, init=False)
created_at: float | None = field(default=None, init=False)
connected_at: float | None = field(default=None, init=False)
+ initial_control_barrier_arrived_at: float | None = field(default=None, init=False)
+ initial_control_barrier_released_at: float | None = field(default=None, init=False)
first_media_frame_at: float | None = field(default=None, init=False)
first_generated_frame_at: float | None = field(default=None, init=False)
last_generated_frame_at: float | None = field(default=None, init=False)
@@ -463,7 +570,7 @@ def __post_init__(self) -> None:
@property
def logical_id(self) -> str:
- return f"wave-{self.index:03d}"
+ return self.trace_session_id or f"wave-{self.index:03d}"
@property
def active_controls(self) -> bool:
@@ -483,9 +590,7 @@ async def set_input_enabled(self, enabled: bool, *, reason: str) -> None:
if enabled:
self.input_resumes += 1
paused_for = (
- max(0.0, now - self.input_pause_started_at)
- if self.input_pause_started_at is not None
- else None
+ max(0.0, now - self.input_pause_started_at) if self.input_pause_started_at is not None else None
)
self.input_pause_started_at = None
self.record_event(
@@ -502,7 +607,11 @@ async def set_input_enabled(self, enabled: bool, *, reason: str) -> None:
controls = ()
# Do not wait for the next heartbeat to clear a stale key state.
- if self.connected and self._room is not None:
+ if (
+ self.connected
+ and self._room is not None
+ and (self.initial_control_gate is None or self.initial_control_gate.is_set())
+ ):
try:
await self._publish_control_state(controls)
except Exception as exc: # noqa: BLE001 - a transition failure is a workload fact
@@ -620,7 +729,20 @@ def on_disconnected(reason: Any) -> None:
connected_seconds=round(self.connected_at - (self.create_started_at or self.connected_at), 6),
)
if not self.stop_requested:
- self._control_task = asyncio.create_task(self._send_controls(), name=f"abot-controls-{self.logical_id}")
+ if self.initial_control_gate is not None:
+ self.initial_control_barrier_arrived_at = self.connected_at
+ self.record_event(
+ "diagnostic_initial_control_barrier_arrived",
+ session=self.logical_id,
+ phase=self.diagnostic_initial_control_barrier_phase,
+ )
+ self._start_control_task()
+
+ def _start_control_task(self) -> None:
+ """Start the heartbeat once; a diagnostic gate may hold its first send."""
+ if self.stop_requested or self._control_task is not None:
+ return
+ self._control_task = asyncio.create_task(self._send_controls(), name=f"abot-controls-{self.logical_id}")
async def _consume_video(self, stream: Any) -> None:
try:
@@ -652,6 +774,11 @@ async def _consume_video(self, stream: Any) -> None:
self.record_event("video_stream_error", session=self.logical_id, error=self.error)
async def _send_controls(self) -> None:
+ gate = self.initial_control_gate
+ if gate is not None:
+ await gate.wait()
+ if self.stop_requested:
+ return
assert self._room is not None
control = self.scenario.session.control
idle_until = 0.0
@@ -683,6 +810,9 @@ async def _publish_control_state(self, controls: tuple[str, ...]) -> None:
"""Publish one reliable control heartbeat and retain its public state."""
if self._room is None:
return
+ gate = self.initial_control_gate
+ if gate is not None and not gate.is_set():
+ return
payload = {"type": "control_state", "controls": list(controls)}
await self._room.local_participant.publish_data(
json.dumps(payload, separators=(",", ":")).encode("utf-8"),
@@ -758,6 +888,8 @@ def snapshot(self, now: float) -> dict[str, Any]:
)
return {
"logical_session_id": self.logical_id,
+ "source_trace_session_id": self.source_trace_session_id,
+ "source_trace_user_id": self.source_trace_user_id,
"server_session_id": self.server_session_id,
"worker_id_at_admission": self.worker_id,
"admission_status": self.admission_status,
@@ -768,6 +900,20 @@ def snapshot(self, now: float) -> dict[str, Any]:
"stop_requested": self.stop_requested,
"departure_scheduled": self.departure_scheduled,
"remote_session_deleted": self.remote_session_deleted,
+ "diagnostic_initial_control_barrier_phase": self.diagnostic_initial_control_barrier_phase,
+ "diagnostic_initial_control_barrier_waiting": (
+ self.initial_control_gate is not None and not self.initial_control_gate.is_set()
+ ),
+ "diagnostic_initial_control_barrier_arrived_offset_seconds": (
+ round(self.initial_control_barrier_arrived_at - self.started_at, 6)
+ if self.initial_control_barrier_arrived_at is not None
+ else None
+ ),
+ "diagnostic_initial_control_barrier_released_offset_seconds": (
+ round(self.initial_control_barrier_released_at - self.started_at, 6)
+ if self.initial_control_barrier_released_at is not None
+ else None
+ ),
"input_enabled": self.input_enabled,
"active_controls": self.active_controls,
"input_pauses": self.input_pauses,
@@ -804,6 +950,12 @@ def __init__(self, scenario: Scenario) -> None:
self.scenario = scenario
self.rtc = _load_livekit_rtc()
self.started_at = 0.0
+ # ``offset_seconds`` is measured with perf_counter(). Keep a sampled
+ # wall-clock/monotonic triplet from the same workload origin so an
+ # external per-dispatch trace can align client events without guessing
+ # from process start time.
+ self._trace_started_monotonic_seconds = 0.0
+ self._trace_started_unix_seconds = 0.0
self._sessions: list[LiveKitWaveSession] = []
self._background_tasks: set[asyncio.Task[None]] = set()
self._monitor_task: asyncio.Task[None] | None = None
@@ -818,6 +970,9 @@ def __init__(self, scenario: Scenario) -> None:
self._previous_sample_at: float | None = None
self._server_metadata: list[dict[str, Any]] = []
self._warnings: list[str] = []
+ # These records are kept separate from user-wave results because a
+ # synchronized first action is a harness diagnostic, not user behavior.
+ self._diagnostic_initial_control_barrier_results: list[dict[str, Any]] = []
def record_event(self, event: str, **values: Any) -> None:
now = time.perf_counter()
@@ -844,6 +999,8 @@ async def run(self) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=timeout, limits=limits, trust_env=False) as http:
self._http = http
self.started_at = time.perf_counter()
+ self._trace_started_monotonic_seconds = time.monotonic()
+ self._trace_started_unix_seconds = time.time()
await self._capture_server_metadata("before_workload")
self._monitoring = True
self._monitor_task = asyncio.create_task(self._monitor(), name="abot-livekit-wave-monitor")
@@ -873,6 +1030,20 @@ async def run(self) -> dict[str, Any]:
"expected_max_sessions_per_worker": self.scenario.admission.expected_max_sessions_per_worker,
"expected_queue_size": self.scenario.admission.expected_queue_size,
},
+ "diagnostic_initial_control_barrier": (
+ {
+ "enabled": True,
+ "kind": "phase_aligned_initial_control",
+ "not_a_real_user_trace": True,
+ "phase": self.scenario.diagnostic_initial_control_barrier.phase_name,
+ "expected_connected_sessions": (
+ self.scenario.diagnostic_initial_control_barrier.expected_connected_sessions
+ ),
+ "timeout_seconds": self.scenario.diagnostic_initial_control_barrier.timeout_seconds,
+ }
+ if self.scenario.diagnostic_initial_control_barrier is not None
+ else {"enabled": False}
+ ),
"phases": [
{
"name": phase.name,
@@ -886,10 +1057,19 @@ async def run(self) -> dict[str, Any]:
for phase in self.scenario.phases
],
},
- "started_at_unix_seconds": time.time() - max(0.0, completed_at - self.started_at),
+ "trace_clock": {
+ "offset_clock": "time.perf_counter",
+ "origin_performance_counter_seconds": round(self.started_at, 9),
+ "origin_monotonic_seconds": round(self._trace_started_monotonic_seconds, 9),
+ "origin_unix_seconds": round(self._trace_started_unix_seconds, 9),
+ "offset_to_unix_seconds": "origin_unix_seconds + offset_seconds",
+ "offset_to_monotonic_seconds": "origin_monotonic_seconds + offset_seconds",
+ },
+ "started_at_unix_seconds": round(self._trace_started_unix_seconds, 9),
"elapsed_seconds": round(completed_at - self.started_at, 6),
"server_metadata": self._server_metadata,
"warnings": self._warnings,
+ "diagnostic_initial_control_barrier_results": self._diagnostic_initial_control_barrier_results,
"phase_results": self._phase_results,
"sessions": [session.snapshot(completed_at) for session in self._sessions],
"samples": self._samples,
@@ -928,7 +1108,14 @@ def _schedule_transition(self, phase: Phase) -> None:
session for session in self._sessions if not session.stop_requested and not session.departure_scheduled
]
difference = phase.target_users - len(present)
+ barrier = self.scenario.diagnostic_initial_control_barrier
+ gate: asyncio.Event | None = None
+ if barrier is not None and barrier.phase_name == phase.name:
+ if present or difference != barrier.expected_connected_sessions:
+ raise RuntimeError("Diagnostic initial-control barrier no longer describes a fresh target population")
+ gate = asyncio.Event()
if difference > 0:
+ barrier_sessions: list[LiveKitWaveSession] = []
for ordinal in range(difference):
session = LiveKitWaveSession(
index=len(self._sessions),
@@ -937,11 +1124,25 @@ def _schedule_transition(self, phase: Phase) -> None:
rtc=self.rtc,
record_event=self.record_event,
started_at=self.started_at,
+ diagnostic_initial_control_barrier_phase=phase.name if gate is not None else None,
+ initial_control_gate=gate,
)
session.scheduled_at = time.perf_counter()
self._sessions.append(session)
+ barrier_sessions.append(session)
offset = self._spread_offset(ordinal + 1, difference, phase.arrival_window_seconds)
self._spawn_background(self._delayed_start(session, offset))
+ if gate is not None:
+ assert barrier is not None
+ self.record_event(
+ "diagnostic_initial_control_barrier_opened",
+ phase=phase.name,
+ expected_connected_sessions=barrier.expected_connected_sessions,
+ not_a_real_user_trace=True,
+ )
+ self._spawn_background(
+ self._run_diagnostic_initial_control_barrier(phase, barrier, barrier_sessions, gate)
+ )
return
if difference < 0:
# Newest users leave first. This also removes still-queued arrivals before
@@ -952,6 +1153,91 @@ def _schedule_transition(self, phase: Phase) -> None:
offset = self._spread_offset(ordinal + 1, -difference, phase.departure_window_seconds)
self._spawn_background(self._delayed_stop(session, offset))
+ async def _run_diagnostic_initial_control_barrier(
+ self,
+ phase: Phase,
+ barrier: DiagnosticInitialControlBarrier,
+ sessions: Sequence[LiveKitWaveSession],
+ gate: asyncio.Event,
+ ) -> None:
+ """Release held control tasks only after the synthetic cohort connects.
+
+ A timeout or failed admission deliberately opens the gate for already
+ connected clients, so the harness never leaves serving sessions held
+ forever. The artifact records that result as unaligned and invalid for
+ a phase-alignment comparison.
+ """
+ opened_at = time.perf_counter()
+ deadline = opened_at + barrier.timeout_seconds
+ status = "released_aligned"
+ warning: str | None = None
+ connected: list[LiveKitWaveSession] = []
+ try:
+ while True:
+ now = time.perf_counter()
+ connected = [session for session in sessions if session.connected and not session.stop_requested]
+ failed = [session for session in sessions if session.stop_requested or session.error is not None]
+ if failed:
+ status = "released_unaligned_failure"
+ warning = (
+ "Diagnostic initial-control barrier saw failed or stopped sessions; "
+ "released connected sessions without phase alignment."
+ )
+ break
+ if len(connected) == barrier.expected_connected_sessions:
+ break
+ if now >= deadline:
+ status = "released_unaligned_timeout"
+ warning = (
+ "Diagnostic initial-control barrier timed out before every session connected; "
+ "released connected sessions without phase alignment."
+ )
+ break
+ await asyncio.sleep(min(0.02, max(0.001, deadline - now)))
+ except asyncio.CancelledError:
+ cancelled_at = time.perf_counter()
+ result = {
+ "kind": "phase_aligned_initial_control",
+ "not_a_real_user_trace": True,
+ "phase": phase.name,
+ "expected_connected_sessions": barrier.expected_connected_sessions,
+ "connected_sessions_at_release": len(connected),
+ "status": "cancelled",
+ "opened_offset_seconds": round(opened_at - self.started_at, 6),
+ "released_offset_seconds": round(cancelled_at - self.started_at, 6),
+ "wait_seconds": round(cancelled_at - opened_at, 6),
+ }
+ self._diagnostic_initial_control_barrier_results.append(result)
+ self.record_event("diagnostic_initial_control_barrier_cancelled", phase=phase.name)
+ raise
+
+ released_at = time.perf_counter()
+ for session in connected:
+ session.initial_control_barrier_released_at = released_at
+ gate.set()
+ result = {
+ "kind": "phase_aligned_initial_control",
+ "not_a_real_user_trace": True,
+ "phase": phase.name,
+ "expected_connected_sessions": barrier.expected_connected_sessions,
+ "connected_sessions_at_release": len(connected),
+ "status": status,
+ "opened_offset_seconds": round(opened_at - self.started_at, 6),
+ "released_offset_seconds": round(released_at - self.started_at, 6),
+ "wait_seconds": round(released_at - opened_at, 6),
+ }
+ self._diagnostic_initial_control_barrier_results.append(result)
+ self.record_event(
+ "diagnostic_initial_control_barrier_released",
+ phase=phase.name,
+ status=status,
+ connected_sessions=len(connected),
+ expected_connected_sessions=barrier.expected_connected_sessions,
+ not_a_real_user_trace=True,
+ )
+ if warning is not None:
+ self._warnings.append(warning)
+
def _schedule_input_activity(self, phase: Phase) -> None:
"""Schedule long input pauses/resumes for sessions present in this phase.
@@ -1353,6 +1639,13 @@ def main() -> None:
scenario = load_scenario(scenario_path, server_url_override=args.server_url)
if args.dry_run:
print(json.dumps(scenario.raw, indent=2, sort_keys=True))
+ barrier = scenario.diagnostic_initial_control_barrier
+ if barrier is not None:
+ print(
+ "\nDIAGNOSTIC ONLY: this scenario synchronizes the first active control after "
+ f"{barrier.expected_connected_sessions} sessions connect in phase {barrier.phase_name!r}. "
+ "It is not a real-user arrival trace."
+ )
print(f"\nValidated scenario: {scenario.name}")
return
if args.output is None:
diff --git a/tools/validation/derive_abot_turboserve_trace.py b/tools/validation/derive_abot_turboserve_trace.py
new file mode 100644
index 00000000..62def926
--- /dev/null
+++ b/tools/validation/derive_abot_turboserve_trace.py
@@ -0,0 +1,592 @@
+#!/usr/bin/env python3
+"""Derive runnable ABot LiveKit traces from TurboServe's public demo trace.
+
+TurboServe's ``traces/example_8gpu.json`` is a simulator lifecycle trace, not
+an ABot action stream. This tool deliberately preserves its wall-clock
+arrival / active / idle / departure sequence while normalizing *retained
+session concurrency* to an ABot serving capacity. The generated scenario is
+replayed by :mod:`benchmark_abot_livekit_burst` exclusively through the public
+HTTP and LiveKit interfaces.
+
+The transformation is intentionally deterministic and recorded verbatim in
+each output's ``trace_contract``. It must be described as a
+``TurboServe-public-demo-trace-derived`` workload, never as TurboServe's
+production trace or a reproduction of the paper's private T1--T6 traces.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections import Counter
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_WORKSPACE_ROOT = _REPO_ROOT.parent
+_DEFAULT_SOURCE = _WORKSPACE_ROOT / "TurboServe" / "traces" / "example_8gpu.json"
+_DEFAULT_OUTPUT_DIR = _REPO_ROOT / "tools" / "validation" / "workloads"
+_DERIVATION_VERSION = "turboserve-public-demo-capacity-normalized-v1"
+_SELECTION_SEED = 20260815
+
+
+class TraceAdapterError(ValueError):
+ """Raised when a source TurboServe lifecycle trace is malformed."""
+
+
+@dataclass(frozen=True)
+class SourceEvent:
+ """The small source-trace subset required for lifecycle replay."""
+
+ time_seconds: float
+ sequence: int
+ event_type: str
+ session_id: int
+ user_id: int | None
+ active_on_arrival: bool | None
+
+
+@dataclass
+class SourceSession:
+ """Mutable source session state during deterministic capacity normalization."""
+
+ session_id: int
+ user_id: int | None
+ input_enabled: bool
+
+
+@dataclass(frozen=True)
+class DerivedTrace:
+ """A complete explicit lifecycle replay plus reproducibility metadata."""
+
+ events: tuple[dict[str, Any], ...]
+ source_duration_seconds: float
+ derived_duration_seconds: float
+ source_peak_retained_sessions: int
+ source_peak_active_sessions: int
+ derived_peak_retained_sessions: int
+ derived_peak_active_sessions: int
+ source_event_counts: dict[str, int]
+ derived_event_counts: dict[str, int]
+ source_sha256: str
+ selected_source_session_count: int
+ derived_connection_count: int
+
+
+def _as_mapping(value: object, label: str) -> Mapping[str, Any]:
+ if not isinstance(value, Mapping):
+ raise TraceAdapterError(f"{label} must be an object")
+ return value
+
+
+def _as_non_negative_float(value: object, label: str) -> float:
+ if not isinstance(value, int | float) or isinstance(value, bool):
+ raise TraceAdapterError(f"{label} must be a number")
+ result = float(value)
+ if result < 0:
+ raise TraceAdapterError(f"{label} must be non-negative")
+ return result
+
+
+def _as_int(value: object, label: str) -> int:
+ if not isinstance(value, int) or isinstance(value, bool):
+ raise TraceAdapterError(f"{label} must be an integer")
+ return int(value)
+
+
+def _load_source_trace(path: Path) -> tuple[list[SourceEvent], dict[str, Any], str]:
+ """Load and minimally validate the public TurboServe JSON trace."""
+ try:
+ raw_bytes = path.read_bytes()
+ except OSError as exc:
+ raise TraceAdapterError(f"Could not read source trace {path}: {exc}") from exc
+ try:
+ document = json.loads(raw_bytes)
+ except json.JSONDecodeError as exc:
+ raise TraceAdapterError(f"Source trace is not valid JSON: {exc}") from exc
+ root = _as_mapping(document, "source trace")
+ raw_events = root.get("events")
+ if not isinstance(raw_events, list) or not raw_events:
+ raise TraceAdapterError("source trace.events must be a non-empty list")
+
+ events: list[SourceEvent] = []
+ for index, raw_event in enumerate(raw_events):
+ event = _as_mapping(raw_event, f"source trace.events[{index}]")
+ event_type = event.get("event_type")
+ if event_type not in {"session_arrival", "user_active", "user_idle", "session_departure"}:
+ raise TraceAdapterError(f"Unsupported source event type at index {index}: {event_type!r}")
+ payload = _as_mapping(event.get("payload", {}), f"source trace.events[{index}].payload")
+ active_on_arrival: bool | None = None
+ if event_type == "session_arrival":
+ candidate = payload.get("active", True)
+ if not isinstance(candidate, bool):
+ raise TraceAdapterError(f"source trace.events[{index}].payload.active must be boolean")
+ active_on_arrival = candidate
+ user_id = event.get("user_id")
+ if user_id is not None:
+ user_id = _as_int(user_id, f"source trace.events[{index}].user_id")
+ events.append(
+ SourceEvent(
+ time_seconds=_as_non_negative_float(event.get("time_s"), f"source trace.events[{index}].time_s"),
+ sequence=_as_int(event.get("sequence"), f"source trace.events[{index}].sequence"),
+ event_type=str(event_type),
+ session_id=_as_int(event.get("session_id"), f"source trace.events[{index}].session_id"),
+ user_id=user_id,
+ active_on_arrival=active_on_arrival,
+ )
+ )
+ events.sort(key=lambda event: (event.time_seconds, event.sequence))
+ config = dict(_as_mapping(root.get("config", {}), "source trace.config"))
+ return events, config, hashlib.sha256(raw_bytes).hexdigest()
+
+
+def _stable_rank(source_session_id: int, *, seed: int) -> int:
+ """Return a stable pseudo-random rank without depending on Python hash salt."""
+ digest = hashlib.sha256(f"{seed}:{source_session_id}".encode("ascii")).digest()
+ return int.from_bytes(digest[:8], byteorder="big", signed=False)
+
+
+def _scaled_target(source_retained: int, *, source_peak: int, target_peak: int) -> int:
+ """Round source concurrency proportionally, guaranteeing the requested peak."""
+ if source_retained <= 0:
+ return 0
+ # Source peak is observed from this exact file. Half-up rounding makes the
+ # rescaling independent of Python's banker-rounding implementation.
+ target = (source_retained * target_peak * 2 + source_peak) // (source_peak * 2)
+ return max(0, min(target_peak, int(target)))
+
+
+def _trace_event(
+ *,
+ offset_seconds: float,
+ event: str,
+ trace_session_id: str,
+ source_session: SourceSession,
+ source_event: SourceEvent,
+ derived_sequence: int,
+ **extra: Any,
+) -> dict[str, Any]:
+ """Build a self-contained replay event with direct provenance."""
+ return {
+ "offset_seconds": round(offset_seconds, 6),
+ "sequence": derived_sequence,
+ "event": event,
+ "trace_session_id": trace_session_id,
+ "source_session_id": source_session.session_id,
+ "source_user_id": source_session.user_id,
+ "source_time_seconds": round(source_event.time_seconds, 6),
+ "source_event_sequence": source_event.sequence,
+ **extra,
+ }
+
+
+def _observed_source_peak(source_events: Iterable[SourceEvent]) -> int:
+ """Return retained-session peak after replaying source lifecycle events."""
+ present: set[int] = set()
+ peak = 0
+ for event in sorted(source_events, key=lambda item: (item.time_seconds, item.sequence)):
+ if event.event_type == "session_arrival":
+ present.add(event.session_id)
+ elif event.event_type == "session_departure":
+ present.discard(event.session_id)
+ peak = max(peak, len(present))
+ return peak
+
+
+def derive_trace(
+ source_events: Iterable[SourceEvent],
+ *,
+ target_peak: int,
+ source_sha256: str,
+ selection_seed: int = _SELECTION_SEED,
+) -> DerivedTrace:
+ """Capacity-normalize source lifecycles while retaining selected identities.
+
+ The selected population is sticky: it is only evicted on a source
+ departure or when the scaled target decreases. When a scaled target grows
+ we fill the free slots using a stable hash order among currently present
+ source sessions. That retains each selected source session's repeated
+ ``user_idle``/``user_active`` sequence whenever capacity permits, rather
+ than resampling a different cohort on every source event.
+ """
+ if target_peak < 1:
+ raise TraceAdapterError("target_peak must be positive")
+ ordered = sorted(source_events, key=lambda event: (event.time_seconds, event.sequence))
+ if not ordered:
+ raise TraceAdapterError("source_events must not be empty")
+
+ source_peak_for_scaling = _observed_source_peak(ordered)
+ if source_peak_for_scaling < 1:
+ raise TraceAdapterError("source trace never retains a session")
+ source_sessions: dict[int, SourceSession] = {}
+ selected: dict[int, str] = {}
+ generations: Counter[int] = Counter()
+ derived_events: list[dict[str, Any]] = []
+ source_event_counts: Counter[str] = Counter()
+ derived_event_counts: Counter[str] = Counter()
+ selected_source_ids: set[int] = set()
+ source_peak_retained = 0
+ source_peak_active = 0
+ derived_peak_retained = 0
+ derived_peak_active = 0
+ derived_sequence = 0
+
+ def emit(
+ event: str,
+ trace_session_id: str,
+ source_session: SourceSession,
+ source_event: SourceEvent,
+ **extra: Any,
+ ) -> None:
+ nonlocal derived_sequence
+ derived_events.append(
+ _trace_event(
+ offset_seconds=source_event.time_seconds,
+ event=event,
+ trace_session_id=trace_session_id,
+ source_session=source_session,
+ source_event=source_event,
+ derived_sequence=derived_sequence,
+ **extra,
+ )
+ )
+ derived_sequence += 1
+ derived_event_counts[event] += 1
+
+ def remove_selection(
+ source_session_id: int,
+ source_event: SourceEvent,
+ *,
+ reason: str,
+ source_session: SourceSession | None = None,
+ ) -> None:
+ trace_session_id = selected.pop(source_session_id, None)
+ if trace_session_id is None:
+ return
+ source_session = source_session or source_sessions.get(source_session_id)
+ if source_session is None:
+ # A departing session is removed from ``source_sessions`` only
+ # after its selection's corresponding departure is emitted.
+ raise TraceAdapterError(f"Selected source session {source_session_id} disappeared before departure")
+ emit(
+ "session_departure",
+ trace_session_id,
+ source_session,
+ source_event,
+ departure_reason=reason,
+ )
+
+ def fill_selection(desired_count: int, source_event: SourceEvent) -> set[int]:
+ """Fill scaled capacity with stable-ranked live source sessions."""
+ newly_selected: set[int] = set()
+ candidates = sorted(
+ (session for session_id, session in source_sessions.items() if session_id not in selected),
+ key=lambda session: (_stable_rank(session.session_id, seed=selection_seed), session.session_id),
+ )
+ for source_session in candidates[: max(0, desired_count - len(selected))]:
+ generations[source_session.session_id] += 1
+ trace_session_id = f"ts-{source_session.session_id:05d}-g{generations[source_session.session_id]:02d}"
+ selected[source_session.session_id] = trace_session_id
+ selected_source_ids.add(source_session.session_id)
+ newly_selected.add(source_session.session_id)
+ emit(
+ "session_arrival",
+ trace_session_id,
+ source_session,
+ source_event,
+ input_enabled=source_session.input_enabled,
+ arrival_reason=(
+ "source_session_arrival"
+ if (
+ source_event.event_type == "session_arrival"
+ and source_event.session_id == source_session.session_id
+ )
+ else "capacity_normalization_scale_up"
+ ),
+ )
+ return newly_selected
+
+ for source_event in ordered:
+ source_event_counts[source_event.event_type] += 1
+ source_session_id = source_event.session_id
+ previous_input_enabled: bool | None = None
+ if source_event.event_type == "session_arrival":
+ if source_session_id in source_sessions:
+ raise TraceAdapterError(f"Source session {source_session_id} arrived while already present")
+ assert source_event.active_on_arrival is not None
+ source_sessions[source_session_id] = SourceSession(
+ session_id=source_session_id,
+ user_id=source_event.user_id,
+ input_enabled=source_event.active_on_arrival,
+ )
+ else:
+ source_session = source_sessions.get(source_session_id)
+ if source_session is None:
+ raise TraceAdapterError(
+ f"Source event {source_event.event_type} references non-present session {source_session_id}"
+ )
+ previous_input_enabled = source_session.input_enabled
+ if source_event.event_type == "user_active":
+ source_session.input_enabled = True
+ elif source_event.event_type == "user_idle":
+ source_session.input_enabled = False
+ elif source_event.event_type == "session_departure":
+ remove_selection(source_session_id, source_event, reason="source_session_departure")
+ del source_sessions[source_session_id]
+
+ source_peak_retained = max(source_peak_retained, len(source_sessions))
+ source_peak_active = max(
+ source_peak_active,
+ sum(session.input_enabled for session in source_sessions.values()),
+ )
+ desired_count = _scaled_target(
+ len(source_sessions), source_peak=source_peak_for_scaling, target_peak=target_peak
+ )
+ # Remove least-preferred selected sessions only when scaled capacity
+ # genuinely shrinks. This avoids churn on ordinary source arrivals.
+ excess = len(selected) - desired_count
+ if excess > 0:
+ evicted_ids = sorted(
+ selected,
+ key=lambda session_id: (_stable_rank(session_id, seed=selection_seed), session_id),
+ reverse=True,
+ )[:excess]
+ for session_id in evicted_ids:
+ remove_selection(session_id, source_event, reason="capacity_normalization_scale_down")
+
+ newly_selected = fill_selection(desired_count, source_event)
+ selected_trace_session_id = selected.get(source_session_id)
+ if (
+ source_event.event_type in {"user_active", "user_idle"}
+ and selected_trace_session_id is not None
+ and source_session_id not in newly_selected
+ and previous_input_enabled != source_sessions[source_session_id].input_enabled
+ ):
+ emit(
+ source_event.event_type,
+ selected_trace_session_id,
+ source_sessions[source_session_id],
+ source_event,
+ input_enabled=source_sessions[source_session_id].input_enabled,
+ )
+
+ derived_peak_retained = max(derived_peak_retained, len(selected))
+ derived_peak_active = max(
+ derived_peak_active,
+ sum(source_sessions[session_id].input_enabled for session_id in selected),
+ )
+
+ if selected:
+ raise TraceAdapterError(
+ "Source trace ended with retained sessions; expected departure events before duration end"
+ )
+ if source_peak_retained != 186:
+ raise TraceAdapterError(
+ "This adapter pins the published example trace's observed retained-session peak at 186; "
+ f"got {source_peak_retained}. Regenerate policy only after reviewing the source trace."
+ )
+ if derived_peak_retained != target_peak:
+ raise TraceAdapterError(
+ f"Capacity normalization did not reach requested peak {target_peak}; got {derived_peak_retained}"
+ )
+ source_duration = max(event.time_seconds for event in ordered)
+ return DerivedTrace(
+ events=tuple(derived_events),
+ source_duration_seconds=source_duration,
+ derived_duration_seconds=source_duration,
+ source_peak_retained_sessions=source_peak_retained,
+ source_peak_active_sessions=source_peak_active,
+ derived_peak_retained_sessions=derived_peak_retained,
+ derived_peak_active_sessions=derived_peak_active,
+ source_event_counts=dict(sorted(source_event_counts.items())),
+ derived_event_counts=dict(sorted(derived_event_counts.items())),
+ source_sha256=source_sha256,
+ selected_source_session_count=len(selected_source_ids),
+ derived_connection_count=derived_event_counts["session_arrival"],
+ )
+
+
+def _scenario_payload(*, name: str, workers: int, target_peak: int, trace: DerivedTrace) -> dict[str, Any]:
+ """Build the existing ABot workload format plus explicit lifecycle events."""
+ expected_worker_mode = "process-nccl" if workers > 1 else "process"
+ source_relative_path = "../../../TurboServe/traces/example_8gpu.json"
+ return {
+ "name": name,
+ "trace_contract": {
+ "kind": "turboserve_public_demo_trace_derived_abot_lifecycle",
+ "derivation_version": _DERIVATION_VERSION,
+ "not_a_turboserve_production_trace": True,
+ "not_a_reproduction_of_private_paper_t1_to_t6_traces": True,
+ "source": {
+ "public_demo_repository_relative_path": source_relative_path,
+ "public_demo_trace_filename": "example_8gpu.json",
+ "sha256": trace.source_sha256,
+ "source_event_counts": trace.source_event_counts,
+ "source_duration_seconds": trace.source_duration_seconds,
+ "source_peak_retained_sessions": trace.source_peak_retained_sessions,
+ "source_peak_active_sessions": trace.source_peak_active_sessions,
+ },
+ "time_transform": {
+ "kind": "identity_wall_clock",
+ "source_to_derived_scale": 1.0,
+ "derived_duration_seconds": trace.derived_duration_seconds,
+ "description": (
+ "No time compression: arrival, active, idle, and departure offsets retain "
+ "the source 30-minute wall-clock scale."
+ ),
+ },
+ "capacity_transform": {
+ "kind": "sticky_capacity_normalized_session_sampling",
+ "selection_seed": _SELECTION_SEED,
+ "source_observed_peak_retained_sessions": trace.source_peak_retained_sessions,
+ "target_peak_retained_sessions": target_peak,
+ "target_workers": workers,
+ "target_sessions_per_worker": 4,
+ "scaling_rule": (
+ "round_half_up(source_retained_sessions * target_peak / source_peak); sticky selected "
+ "sessions are retained until source departure or a scaled capacity decrease; scale-up "
+ "uses stable SHA-256(seed:source_session_id) rank among currently present sessions."
+ ),
+ "derived_peak_retained_sessions": trace.derived_peak_retained_sessions,
+ "derived_peak_active_sessions": trace.derived_peak_active_sessions,
+ "selected_source_session_count": trace.selected_source_session_count,
+ "derived_connection_count": trace.derived_connection_count,
+ },
+ "event_mapping": {
+ "session_arrival": (
+ "create one ABot LiveKit session; arrival input_enabled follows source payload.active/current state"
+ ),
+ "user_active": ("resume that selected ABot client's action heartbeat without dropping its session"),
+ "user_idle": (
+ "pause that selected ABot client's action heartbeat without dropping its session or retained state"
+ ),
+ "session_departure": "stop and delete that selected ABot LiveKit session",
+ },
+ "execution_contract": (
+ "No diagnostic barrier. The black-box runner schedules each lifecycle event at its "
+ "explicit source-derived offset and never assigns a GPU from the client side."
+ ),
+ },
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": expected_worker_mode,
+ "expected_num_workers": workers,
+ "seed": _SELECTION_SEED,
+ "admission": {
+ "require_immediate_assignment": True,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0,
+ },
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.15,
+ "idle_probability": 0.0,
+ "idle_min_seconds": 0.0,
+ "idle_max_seconds": 0.0,
+ "action_states": [["KeyW"], ["KeyW", "KeyA"], ["KeyW", "KeyD"], ["KeyI"]],
+ },
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25,
+ },
+ "phases": [
+ {
+ "name": "turboserve_public_demo_lifecycle_replay",
+ "duration_seconds": trace.derived_duration_seconds,
+ "target_users": target_peak,
+ "active_input_fraction": 1.0,
+ }
+ ],
+ "lifecycle_trace": {
+ "kind": "explicit_session_lifecycle_v1",
+ "duration_seconds": trace.derived_duration_seconds,
+ "events": list(trace.events),
+ },
+ }
+
+
+def build_scenarios(source_path: Path) -> tuple[dict[str, Any], dict[str, Any]]:
+ """Build the canonical single-GPU and four-GPU scenario documents."""
+ source_events, _source_config, source_sha256 = _load_source_trace(source_path)
+ trace_1gpu = derive_trace(source_events, target_peak=4, source_sha256=source_sha256)
+ trace_4gpu = derive_trace(source_events, target_peak=16, source_sha256=source_sha256)
+ return (
+ _scenario_payload(
+ name="abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4",
+ workers=1,
+ target_peak=4,
+ trace=trace_1gpu,
+ ),
+ _scenario_payload(
+ name="abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16",
+ workers=4,
+ target_peak=16,
+ trace=trace_4gpu,
+ ),
+ )
+
+
+def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--source", type=Path, default=_DEFAULT_SOURCE, help="TurboServe public demo JSON trace")
+ parser.add_argument(
+ "--output-dir", type=Path, default=_DEFAULT_OUTPUT_DIR, help="Directory for generated scenarios"
+ )
+ parser.add_argument(
+ "--check",
+ action="store_true",
+ help="Fail if canonical checked-in scenario files differ from deterministic regenerated content.",
+ )
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = _parse_args()
+ source = args.source.expanduser().resolve()
+ output_dir = args.output_dir.expanduser().resolve()
+ scenario_1gpu, scenario_4gpu = build_scenarios(source)
+ outputs = {
+ output_dir / "abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json": scenario_1gpu,
+ output_dir / "abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json": scenario_4gpu,
+ }
+ mismatches: list[Path] = []
+ for path, payload in outputs.items():
+ rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n"
+ if args.check:
+ try:
+ actual = path.read_text(encoding="utf-8")
+ except OSError:
+ mismatches.append(path)
+ continue
+ if actual != rendered:
+ mismatches.append(path)
+ continue
+ _write_json(path, payload)
+ print(f"Wrote {path}")
+ if args.check and mismatches:
+ raise SystemExit("Derived scenario files are stale or missing: " + ", ".join(str(path) for path in mismatches))
+ if args.check:
+ print("Canonical TurboServe-public-demo-derived scenarios are current.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py b/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py
new file mode 100644
index 00000000..7ec2cb11
--- /dev/null
+++ b/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py
@@ -0,0 +1,807 @@
+"""Isolate persistent CUDA-Graph replay errors for ABot B=1, B=2, and B=3.
+
+This correctness diagnostic starts three same-seed retained-session cohorts:
+
+* ordinary public eager;
+* persistent-static eager, using the same fixed static tensors and, for B=2/3,
+ the same persistent KV arena layout as the graph path, but calling
+ forward_steady_state normally rather than replaying a CUDA Graph;
+* ordinary public CUDA-Graph capture then persistent replay.
+
+Every cohort is warmed through the public eager path until the causal KV
+window is full. It then executes exactly two continuation chunks. After
+each chunk, it compares every lane's DiT latent, rendered RGB frames, and the
+complete retained state (KV, cross-cache, RNG, decoder state, and counters).
+
+The first comparison identifies a static-model or capture error. The second
+comparison is the persistent-replay test that a capture-only parity check
+misses. B=2/3 additionally prove that the graph metrics describe one native
+batched graph rather than singleton graphs.
+
+Example:
+
+ CUDA_VISIBLE_DEVICES=3 PYTHONPATH=$PWD \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py \\
+ --batch-size 1 \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir results/validation/abot_cuda_graph_b1_persistent_three_way
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+
+def _load_base_validator() -> Any:
+ path = Path(__file__).with_name("validate_abot_cuda_graph_parity.py")
+ spec = importlib.util.spec_from_file_location("abot_cuda_graph_persistent_base", path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"could not load base CUDA-Graph validator: {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _tree_exactness(left: Any, right: Any) -> dict[str, Any]:
+ """Compare a retained-state tree directly on device without CPU KV copies."""
+ tensor_leaves = 0
+ checked_leaves = 0
+ mismatches: list[str] = []
+
+ def visit(lhs: Any, rhs: Any, path: str) -> bool:
+ nonlocal checked_leaves, tensor_leaves
+ checked_leaves += 1
+ if isinstance(lhs, torch.Tensor) or isinstance(rhs, torch.Tensor):
+ tensor_leaves += 1
+ if not isinstance(lhs, torch.Tensor) or not isinstance(rhs, torch.Tensor):
+ mismatches.append(f"{path}: tensor/non-tensor type mismatch")
+ return False
+ if lhs.shape != rhs.shape or lhs.dtype != rhs.dtype or lhs.device != rhs.device:
+ mismatches.append(
+ f"{path}: tensor metadata differs "
+ f"({tuple(lhs.shape)}, {lhs.dtype}, {lhs.device}) != "
+ f"({tuple(rhs.shape)}, {rhs.dtype}, {rhs.device})"
+ )
+ return False
+ if not bool(torch.equal(lhs, rhs)):
+ mismatches.append(f"{path}: tensor values differ")
+ return False
+ return True
+ if isinstance(lhs, Mapping) or isinstance(rhs, Mapping):
+ if not isinstance(lhs, Mapping) or not isinstance(rhs, Mapping) or set(lhs) != set(rhs):
+ mismatches.append(f"{path}: mapping keys/type differ")
+ return False
+ return all(visit(lhs[key], rhs[key], f"{path}.{key}") for key in sorted(lhs, key=str))
+ if isinstance(lhs, (list, tuple)) or isinstance(rhs, (list, tuple)):
+ if not isinstance(lhs, (list, tuple)) or not isinstance(rhs, (list, tuple)) or len(lhs) != len(rhs):
+ mismatches.append(f"{path}: sequence length/type differs")
+ return False
+ return all(
+ visit(lhs_item, rhs_item, f"{path}[{index}]")
+ for index, (lhs_item, rhs_item) in enumerate(zip(lhs, rhs, strict=True))
+ )
+ if lhs != rhs:
+ mismatches.append(f"{path}: {lhs!r} != {rhs!r}")
+ return False
+ return True
+
+ exact = visit(left, right, "session")
+ return {
+ "exact": exact,
+ "checked_leaves": checked_leaves,
+ "tensor_leaves": tensor_leaves,
+ "mismatch_count_at_least": len(mismatches),
+ "mismatches": mismatches[:20],
+ }
+
+
+def _session_state_tree(session: Any, pipeline: Any) -> dict[str, Any]:
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW decode state")
+ return {
+ "prompt_emb": session.prompt_emb,
+ "first_frame_latent": session.first_frame_latent,
+ "self_cache": session.self_cache,
+ "cross_cache": session.cross_cache,
+ "generator_state": session.generator.get_state(),
+ "wan_decode_state": {
+ "feat_cache": session.vae_decode_state.feat_cache,
+ "feat_idx": session.vae_decode_state.feat_idx,
+ },
+ "taew_decode_state": pipeline.taew_decode_stage.export_decode_state_for_nccl(session.taew_decode_state),
+ "next_latent_frame": session.next_latent_frame,
+ "emitted_frames": session.emitted_frames,
+ }
+
+
+def _compare_tensor(left: torch.Tensor | None, right: torch.Tensor | None) -> dict[str, Any]:
+ if left is None or right is None:
+ return {
+ "comparable": False,
+ "exact": False,
+ "left_captured": left is not None,
+ "right_captured": right is not None,
+ }
+ if left.shape != right.shape or left.dtype != right.dtype or left.device != right.device:
+ return {
+ "comparable": False,
+ "exact": False,
+ "left_shape": list(left.shape),
+ "right_shape": list(right.shape),
+ "left_dtype": str(left.dtype),
+ "right_dtype": str(right.dtype),
+ "left_device": str(left.device),
+ "right_device": str(right.device),
+ }
+ difference = (left.float() - right.float()).abs()
+ return {
+ "comparable": True,
+ "exact": bool(torch.equal(left, right)),
+ "shape": list(left.shape),
+ "dtype": str(left.dtype),
+ "device": str(left.device),
+ "max_abs_difference": float(difference.max().item()),
+ "mean_abs_difference": float(difference.mean().item()),
+ }
+
+
+class _DecodeLatentCapture:
+ """Record the exact denoised tensor consumed by public LightVAE decode."""
+
+ def __init__(self, decode_stage: Any) -> None:
+ self._decode_stage = decode_stage
+ self._original: Any = None
+ self.latents: torch.Tensor | None = None
+
+ def __enter__(self) -> "_DecodeLatentCapture":
+ self._original = self._decode_stage.decode_chunks
+
+ def capture(latents: torch.Tensor, *args: Any, **kwargs: Any) -> Any:
+ self.latents = latents.detach().clone()
+ return self._original(latents, *args, **kwargs)
+
+ self._decode_stage.decode_chunks = capture
+ return self
+
+ def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
+ self._decode_stage.decode_chunks = self._original
+
+
+def _parse_actions(raw: str, batch_size: int, base: Any) -> list[dict[str, bool]]:
+ items = [item.strip() for item in raw.split(";") if item.strip()]
+ if len(items) < batch_size:
+ raise ValueError(
+ f"--session-actions supplies {len(items)} action sets, but batch size {batch_size} needs one per lane"
+ )
+ return [base._parse_action_keys(item) for item in items[:batch_size]]
+
+
+@torch.inference_mode()
+def _run_public(
+ pipeline: Any,
+ sessions: Sequence[Any],
+ actions: Sequence[Mapping[str, bool]],
+ *,
+ control_latent_frames: int,
+ device: torch.device,
+) -> dict[str, Any]:
+ with _DecodeLatentCapture(pipeline.taew_decode_stage) as capture:
+ frames = pipeline.generate_next_blocks(sessions, actions, control_latent_frames=control_latent_frames)
+ torch.cuda.synchronize(device)
+ return {
+ "latents": capture.latents,
+ "frames": frames,
+ "stage_metrics": dict(pipeline.last_stage_metrics()),
+ }
+
+
+@torch.inference_mode()
+def _prepare_inputs(
+ pipeline: Any,
+ sessions: Sequence[Any],
+ actions: Sequence[Mapping[str, bool]],
+ frames: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Mirror public generate_next_blocks input construction byte-for-byte."""
+ noises: list[torch.Tensor] = []
+ action_contexts: list[torch.Tensor] = []
+ for session, session_actions in zip(sessions, actions, strict=True):
+ shape = session.first_frame_latent.shape
+ noises.append(
+ torch.randn(
+ (1, shape[1], frames, shape[3], shape[4]),
+ generator=session.generator,
+ device=pipeline.device,
+ dtype=torch.float32,
+ )
+ )
+ action_contexts.append(
+ pipeline.build_action_context(
+ session_actions,
+ latent_frames=frames,
+ height=pipeline.config.height,
+ width=pipeline.config.width,
+ device=pipeline.device,
+ dtype=pipeline.torch_dtype,
+ )
+ )
+ return (
+ torch.cat(noises, dim=0).to(dtype=pipeline.torch_dtype),
+ torch.cat([session.prompt_emb for session in sessions], dim=0),
+ torch.cat(action_contexts, dim=0),
+ )
+
+
+def _static_state(
+ pipeline: Any,
+ sessions: Sequence[Any],
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+) -> dict[str, Any]:
+ """Allocate a persistent static control with graph-equivalent storage."""
+ stage = pipeline.denoise_stage
+ if len(sessions) == 1:
+ from telefuser.pipelines.abot_world.denoising import _ABotSteadyCudaGraph
+
+ session = sessions[0]
+ graph = _ABotSteadyCudaGraph(
+ stage.dit,
+ latent,
+ prompt_emb,
+ action_context,
+ session.self_cache,
+ session.cross_cache,
+ torch_dtype=pipeline.torch_dtype,
+ )
+ return {
+ "graph": graph,
+ "self_cache": session.self_cache,
+ "cross_cache": session.cross_cache,
+ "arena_state": None,
+ }
+ arena = stage._create_batched_cuda_graph_state(
+ tuple(session.session_id for session in sessions),
+ latent,
+ prompt_emb,
+ action_context,
+ [session.self_cache for session in sessions],
+ [session.cross_cache for session in sessions],
+ current_starts=[session.next_latent_frame for session in sessions],
+ )
+ return {
+ "graph": arena.graph,
+ "self_cache": arena.self_cache,
+ "cross_cache": arena.cross_cache,
+ "arena_state": arena,
+ }
+
+
+@torch.inference_mode()
+def _static_denoise(
+ pipeline: Any,
+ state: Mapping[str, Any],
+ latent: torch.Tensor,
+ action_context: torch.Tensor,
+ *,
+ current_start: int,
+ generators: Sequence[torch.Generator],
+ scheduler: Any,
+) -> torch.Tensor:
+ """Run graph-equivalent persistent buffers through eager forward_steady_state."""
+ stage = pipeline.denoise_stage
+ graph = state["graph"]
+ current_end = (current_start + graph.frames) * graph.frame_tokens
+ timesteps = stage._official_denoising_timesteps(scheduler).to(device=latent.device)
+ generator: torch.Generator | Sequence[torch.Generator]
+ generator = generators[0] if len(generators) == 1 else generators
+ current = latent
+ for index, current_timestep in enumerate(timesteps):
+ graph._set_inputs(current, action_context, current_timestep, current_end=current_end)
+ with torch.autocast(latent.device.type, dtype=pipeline.torch_dtype, enabled=latent.device.type == "cuda"):
+ flow_prediction = stage.dit.forward_steady_state(
+ x=graph.static_x,
+ timestep=graph.static_timestep,
+ context=graph.static_context,
+ act_context=graph.static_action,
+ kv_cache=state["self_cache"],
+ crossattn_cache=state["cross_cache"],
+ current_end=graph.current_end,
+ roll_scratch_k=graph.roll_scratch_k,
+ roll_scratch_v=graph.roll_scratch_v,
+ update_cache=index == 0,
+ )
+ x0 = stage._x0_prediction(flow_prediction, current, graph.static_timestep, scheduler)
+ if index < len(timesteps) - 1:
+ current = scheduler.add_noise(x0, graph._draw_noise(x0, generator), timesteps[index + 1])
+ else:
+ current = x0
+ # Mirror the production graph path: the final dynamic cache-only call
+ # receives independent x0 storage rather than a static graph output view.
+ context_input = current.clone()
+ # Match the public eager cache-only call: it is outside the sampler's
+ # autocast scope.
+ stage.dit(
+ x=context_input.to(dtype=pipeline.torch_dtype),
+ timestep=torch.zeros_like(graph.static_timestep),
+ context=graph.static_context,
+ act_context=action_context,
+ kv_cache=state["self_cache"],
+ crossattn_cache=state["cross_cache"],
+ current_start=current_start * graph.frame_tokens,
+ )
+ return current
+
+
+@torch.inference_mode()
+def _run_static(
+ pipeline: Any,
+ sessions: Sequence[Any],
+ actions: Sequence[Mapping[str, bool]],
+ *,
+ control_latent_frames: int,
+ state: dict[str, Any] | None,
+ device: torch.device,
+) -> tuple[dict[str, Any], dict[str, Any]]:
+ latent, prompt_emb, action_context = _prepare_inputs(pipeline, sessions, actions, control_latent_frames)
+ starts = [session.next_latent_frame for session in sessions]
+ if len(set(starts)) != 1:
+ raise RuntimeError("persistent static control requires aligned continuation positions")
+ if state is None:
+ state = _static_state(pipeline, sessions, latent, prompt_emb, action_context)
+ latents = _static_denoise(
+ pipeline,
+ state,
+ latent,
+ action_context,
+ current_start=starts[0],
+ generators=[session.generator for session in sessions],
+ scheduler=sessions[0].scheduler,
+ )
+ arena = state["arena_state"]
+ if arena is not None:
+ pipeline.denoise_stage._bind_batched_cache_arena(
+ arena,
+ [session.self_cache for session in sessions],
+ [session.cross_cache for session in sessions],
+ )
+ pipeline.denoise_stage._advance_batched_cache_cursors(
+ [session.self_cache for session in sessions],
+ current_starts=starts,
+ latent=latents,
+ )
+ if any(session.taew_decode_state is None for session in sessions):
+ raise RuntimeError("ABot session is missing its TAeW decode state")
+ decoded = pipeline.taew_decode_stage.decode_chunks(latents, [session.taew_decode_state for session in sessions])
+ frames = []
+ for index, session in enumerate(sessions):
+ session_frames = pipeline.tensor2video(decoded[index])
+ session.next_latent_frame += control_latent_frames
+ session.emitted_frames += len(session_frames)
+ frames.append(session_frames)
+ torch.cuda.synchronize(device)
+ return {"latents": latents.detach().clone(), "frames": frames}, state
+
+
+def _graph_evidence(metrics: Mapping[str, Any], batch_size: int, *, capture: bool, base: Any) -> dict[str, Any]:
+ graph = dict(base._graph_verified(metrics))
+ observed_batch_size = metrics.get("batch_size")
+ graph_batch_size = metrics.get("cuda_graph_batch_size")
+ graph_batched = bool(int(metrics.get("cuda_graph_batched", 0)))
+ batch_valid = batch_size == 1 or (
+ observed_batch_size == batch_size and graph_batch_size == batch_size and graph_batched
+ )
+ graph["expected_batch_size"] = batch_size
+ graph["observed_batch_size"] = observed_batch_size
+ graph["observed_cuda_graph_batch_size"] = graph_batch_size
+ graph["cuda_graph_batched"] = graph_batched
+ graph["native_batch_valid"] = batch_valid
+ graph["verified"] = bool(
+ graph["enabled"]
+ and graph["eligible"]
+ and graph["replay_observed"]
+ and not graph["fallback_observed"]
+ and (graph["captured"] if capture else True)
+ and batch_valid
+ )
+ return graph
+
+
+def _round_comparisons(
+ base: Any,
+ regular: Mapping[str, Any],
+ static: Mapping[str, Any],
+ graph: Mapping[str, Any],
+ regular_sessions: Sequence[Any],
+ static_sessions: Sequence[Any],
+ graph_sessions: Sequence[Any],
+ pipeline: Any,
+) -> dict[str, Any]:
+ lanes: list[dict[str, Any]] = []
+ for index in range(len(regular_sessions)):
+ regular_latent = regular["latents"][index : index + 1]
+ static_latent = static["latents"][index : index + 1]
+ graph_latent = graph["latents"][index : index + 1] if graph["latents"] is not None else None
+ lanes.append(
+ {
+ "lane": index,
+ "regular_vs_static": {
+ "latent": _compare_tensor(regular_latent, static_latent),
+ "rgb": base._compare_frames(regular["frames"][index], static["frames"][index]),
+ "state": _tree_exactness(
+ _session_state_tree(regular_sessions[index], pipeline),
+ _session_state_tree(static_sessions[index], pipeline),
+ ),
+ },
+ "static_vs_graph": {
+ "latent": _compare_tensor(static_latent, graph_latent),
+ "rgb": base._compare_frames(static["frames"][index], graph["frames"][index]),
+ "state": _tree_exactness(
+ _session_state_tree(static_sessions[index], pipeline),
+ _session_state_tree(graph_sessions[index], pipeline),
+ ),
+ },
+ "regular_vs_graph": {
+ "latent": _compare_tensor(regular_latent, graph_latent),
+ "rgb": base._compare_frames(regular["frames"][index], graph["frames"][index]),
+ "state": _tree_exactness(
+ _session_state_tree(regular_sessions[index], pipeline),
+ _session_state_tree(graph_sessions[index], pipeline),
+ ),
+ },
+ }
+ )
+ return {"lanes": lanes}
+
+
+def _pair_exact(round_report: Mapping[str, Any], pair: str) -> bool:
+ for lane in round_report.get("lanes", []):
+ comparison = lane.get(pair, {})
+ latent = comparison.get("latent", {})
+ rgb = comparison.get("rgb", {})
+ state = comparison.get("state", {})
+ if not (latent.get("exact") and rgb.get("all_frame_hashes_equal") and state.get("exact")):
+ return False
+ return bool(round_report.get("lanes"))
+
+
+@torch.inference_mode()
+def _run(args: argparse.Namespace, base: Any) -> dict[str, Any]:
+ if not torch.cuda.is_available():
+ raise RuntimeError("persistent CUDA-Graph diagnostic requires CUDA")
+ image = Image.open(args.image).convert("RGB")
+ pipeline = None
+ cohorts: dict[str, list[Any]] = {"regular": [], "static": [], "graph": []}
+ try:
+ pipeline = base._make_pipeline(args)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(f"persistent CUDA-Graph diagnostic requires CUDA, got {pipeline.device!r}")
+ pipeline.preload_models()
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ actions = _parse_actions(args.session_actions, args.batch_size, base)
+ seeds = [args.seed + 9973 * index for index in range(args.batch_size)]
+ for role, sessions in cohorts.items():
+ for index, seed in enumerate(seeds):
+ sessions.append(
+ pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=seed,
+ session_id=f"persistent-{role}-b{args.batch_size}-{index}",
+ )
+ )
+
+ warmup_chunks = base._required_warmup_chunks(
+ int(pipeline.denoise_stage.dit.local_attn_size),
+ args.control_latent_frames,
+ args.extra_warmup_chunks,
+ )
+ warmup_hashes: list[list[bool]] = []
+ for _ in range(warmup_chunks):
+ outputs = {
+ role: pipeline.generate_next_blocks(
+ sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ )
+ for role, sessions in cohorts.items()
+ }
+ warmup_hashes.append(
+ [
+ base._sequence_hash(outputs["regular"][lane])
+ == base._sequence_hash(outputs["static"][lane])
+ == base._sequence_hash(outputs["graph"][lane])
+ for lane in range(args.batch_size)
+ ]
+ )
+ torch.cuda.synchronize(device)
+ warmup_states = {
+ role: [
+ {
+ "cache": base._cache_readiness(session, pipeline),
+ "regular_state_exact": _tree_exactness(
+ _session_state_tree(cohorts["regular"][index], pipeline),
+ _session_state_tree(session, pipeline),
+ ),
+ }
+ for index, session in enumerate(sessions)
+ ]
+ for role, sessions in cohorts.items()
+ }
+ warmup_valid = all(all(chunk) for chunk in warmup_hashes) and all(
+ item["cache"]["ready"] and item["regular_state_exact"]["exact"]
+ for role_items in warmup_states.values()
+ for item in role_items
+ )
+
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ regular_first = _run_public(
+ pipeline,
+ cohorts["regular"],
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ static_first, static_state = _run_static(
+ pipeline,
+ cohorts["static"],
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ state=None,
+ device=device,
+ )
+
+ pipeline.denoise_stage.configure_cuda_graph(True)
+ graph_first = _run_public(
+ pipeline,
+ cohorts["graph"],
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ graph_runtime_after_first = dict(pipeline.denoise_stage.cuda_graph_metrics())
+ # All three cohorts are still at continuation one here. Compare now;
+ # later state is mutable and cannot stand in for this checkpoint.
+ first = _round_comparisons(
+ base,
+ regular_first,
+ static_first,
+ graph_first,
+ cohorts["regular"],
+ cohorts["static"],
+ cohorts["graph"],
+ pipeline,
+ )
+ # Avoid configure_cuda_graph(False): it deliberately clears resident
+ # graphs. This diagnostic-only flag toggle preserves graph cohort state.
+ stage = pipeline.denoise_stage
+ stage._cuda_graph_enabled = False
+ regular_second = _run_public(
+ pipeline,
+ cohorts["regular"],
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ static_second, static_state = _run_static(
+ pipeline,
+ cohorts["static"],
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ state=static_state,
+ device=device,
+ )
+ stage._cuda_graph_enabled = True
+ graph_second = _run_public(
+ pipeline,
+ cohorts["graph"],
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ graph_runtime_after_second = dict(pipeline.denoise_stage.cuda_graph_metrics())
+ first_evidence = _graph_evidence(graph_first["stage_metrics"], args.batch_size, capture=True, base=base)
+ second_evidence = _graph_evidence(graph_second["stage_metrics"], args.batch_size, capture=False, base=base)
+
+ second = _round_comparisons(
+ base,
+ regular_second,
+ static_second,
+ graph_second,
+ cohorts["regular"],
+ cohorts["static"],
+ cohorts["graph"],
+ pipeline,
+ )
+ static_first_exact = _pair_exact(first, "regular_vs_static")
+ graph_first_exact = _pair_exact(first, "static_vs_graph")
+ static_second_exact = _pair_exact(second, "regular_vs_static")
+ graph_second_exact = _pair_exact(second, "static_vs_graph")
+ if not warmup_valid:
+ status = "invalid_warmup"
+ elif not first_evidence["verified"] or not second_evidence["verified"]:
+ status = "graph_unverified"
+ elif not static_first_exact:
+ status = "static_eager_first_continuation_mismatch"
+ elif not graph_first_exact:
+ status = "cuda_graph_capture_mismatch"
+ elif not static_second_exact:
+ status = "persistent_static_eager_mismatch"
+ elif not graph_second_exact:
+ status = "cuda_graph_persistent_replay_mismatch"
+ else:
+ status = "pass"
+ return {
+ "status": status,
+ "diagnostic": "ordinary eager vs persistent-static eager vs CUDA Graph across two continuations",
+ "device": str(device),
+ "batch_size": args.batch_size,
+ "control_latent_frames": args.control_latent_frames,
+ "session_actions": actions,
+ "session_seeds": seeds,
+ "warmup": {
+ "chunks": warmup_chunks,
+ "per_chunk_per_lane_hash_equal": warmup_hashes,
+ "cohort_state": warmup_states,
+ "valid": warmup_valid,
+ },
+ "cuda_graph": {
+ "first_continuation": {
+ "stage_metrics": graph_first["stage_metrics"],
+ "runtime_metrics": graph_runtime_after_first,
+ "verification": first_evidence,
+ },
+ "second_continuation": {
+ "stage_metrics": graph_second["stage_metrics"],
+ "runtime_metrics": graph_runtime_after_second,
+ "verification": second_evidence,
+ },
+ },
+ "rounds": {
+ "first_continuation": first,
+ "second_continuation_persistent_replay": second,
+ },
+ "classification": {
+ "regular_vs_static_first_exact": static_first_exact,
+ "static_vs_graph_first_exact": graph_first_exact,
+ "regular_vs_static_second_exact": static_second_exact,
+ "static_vs_graph_second_exact": graph_second_exact,
+ },
+ }
+ finally:
+ if pipeline is not None:
+ for sessions in cohorts.values():
+ for session in sessions:
+ try:
+ pipeline.close_interactive_session(session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _write_results(output_dir: Path, result: Mapping[str, Any], args: argparse.Namespace, base: Any) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ payload = {"arguments": base._json_safe(vars(args)), "result": base._json_safe(result)}
+ (output_dir / "results.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ classification = result.get("classification", {})
+ graph = result.get("cuda_graph", {})
+ first_graph = graph.get("first_continuation", {}) if isinstance(graph, Mapping) else {}
+ second_graph = graph.get("second_continuation", {}) if isinstance(graph, Mapping) else {}
+ first_verification = first_graph.get("verification", {}) if isinstance(first_graph, Mapping) else {}
+ second_verification = second_graph.get("verification", {}) if isinstance(second_graph, Mapping) else {}
+ lines = [
+ f"# ABot B={args.batch_size} persistent CUDA-Graph three-way diagnostic",
+ "",
+ (
+ "All three same-seed cohorts were eagerly warmed to full KV, then each ran two continuation chunks. "
+ "The static control uses the graph's fixed buffers and, at B=2/3, its persistent KV arena but executes "
+ "forward_steady_state eagerly."
+ ),
+ "",
+ "| Check | First continuation | Second continuation / persistent replay |",
+ "| --- | --- | --- |",
+ (
+ "| ordinary eager == persistent-static eager | "
+ f"{classification.get('regular_vs_static_first_exact', False)} | "
+ f"{classification.get('regular_vs_static_second_exact', False)} |"
+ ),
+ (
+ "| persistent-static eager == CUDA Graph | "
+ f"{classification.get('static_vs_graph_first_exact', False)} | "
+ f"{classification.get('static_vs_graph_second_exact', False)} |"
+ ),
+ (
+ "| CUDA Graph verified | "
+ f"{first_verification.get('verified', False)} | {second_verification.get('verified', False)} |"
+ ),
+ "",
+ f"Status: {result.get('status', 'error')}.",
+ "",
+ (
+ "results.json includes per-lane strict latent, RGB SHA-256/pixel, and complete retained-state "
+ "comparisons after both continuations."
+ ),
+ ]
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args(base: Any) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model-root", type=Path)
+ parser.add_argument("--image", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--batch-size", type=int, choices=(1, 2, 3), default=1)
+ parser.add_argument("--session-actions", default="W;A;S")
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
+ parser.add_argument("--extra-warmup-chunks", type=int, default=0)
+ parser.add_argument("--device-id", type=int, default=0)
+ parser.add_argument("--dry-run", action="store_true")
+ args = parser.parse_args()
+ if args.extra_warmup_chunks < 0:
+ parser.error("--extra-warmup-chunks must be non-negative")
+ try:
+ _parse_actions(args.session_actions, args.batch_size, base)
+ except (ValueError, argparse.ArgumentTypeError) as exc:
+ parser.error(str(exc))
+ if not args.dry_run:
+ if args.model_root is None or args.image is None or args.output_dir is None:
+ parser.error("--model-root, --image, and --output-dir are required unless --dry-run is used")
+ if not args.model_root.is_dir():
+ parser.error(f"model root does not exist: {args.model_root}")
+ if not args.image.is_file():
+ parser.error(f"image does not exist: {args.image}")
+ return args
+
+
+def main() -> None:
+ base = _load_base_validator()
+ args = _parse_args(base)
+ if args.dry_run:
+ print(
+ json.dumps(
+ {
+ "mode": "dry_run",
+ "batch_size": args.batch_size,
+ "cohorts": ["ordinary_eager", "persistent_static_eager", "cuda_graph"],
+ "continuations": ["first_capture", "second_persistent_replay"],
+ "comparisons_after_each": ["per-lane latent", "RGB frame hashes", "full retained state"],
+ "B2_graph_gate": (
+ "batch_size=B, cuda_graph_batch_size=B, cuda_graph_batched=1, replay>0, fallback=0"
+ ),
+ },
+ indent=2,
+ sort_keys=True,
+ )
+ )
+ return
+ assert args.output_dir is not None
+ try:
+ result = _run(args, base)
+ except Exception as exc:
+ result = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
+ _write_results(args.output_dir, result, args, base)
+ print(json.dumps(base._json_safe(result), indent=2, sort_keys=True))
+ if result.get("status") != "pass":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/diagnose_abot_cuda_graph_three_way.py b/tools/validation/diagnose_abot_cuda_graph_three_way.py
new file mode 100644
index 00000000..7a869f5f
--- /dev/null
+++ b/tools/validation/diagnose_abot_cuda_graph_three_way.py
@@ -0,0 +1,625 @@
+"""Separate ABot CUDA-Graph mismatch sources on a real GPU.
+
+This diagnostic starts three identical B=1 retained sessions and warms all of
+them through the ordinary eager interactive path until the 18-latent-frame KV
+window is full. It then compares one continuation through:
+
+1. the regular dynamic eager DiT path;
+2. ``ABotWorldDiT.forward_steady_state`` invoked eagerly (no CUDA capture);
+3. the captured/replayed CUDA-Graph wrapper.
+
+It compares the initial random latent, retained state, final latent, and
+decoded RGB frames. Thus it distinguishes a static-model semantic error from
+a CUDA-Graph wrapper/capture error without modifying serving code.
+
+Example (GPU 3 remapped to CUDA device 0)::
+
+ CUDA_VISIBLE_DEVICES=3 \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/diagnose_abot_cuda_graph_three_way.py \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir results/validation/abot_cuda_graph_three_way_gpu3
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import math
+import time
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+
+def _load_base_validator() -> Any:
+ """Reuse the two-way tool's loader, cache, and pixel-comparison helpers."""
+ path = Path(__file__).with_name("validate_abot_cuda_graph_parity.py")
+ spec = importlib.util.spec_from_file_location("abot_cuda_graph_parity_base", path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load the base parity validator: {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _tree_exactness(left: Any, right: Any) -> dict[str, Any]:
+ """Compare a session-state tree without copying its multi-GB KV cache to CPU."""
+ tensor_leaves = 0
+ checked_leaves = 0
+ mismatches: list[str] = []
+
+ def visit(lhs: Any, rhs: Any, path: str) -> bool:
+ nonlocal tensor_leaves, checked_leaves
+ checked_leaves += 1
+ if isinstance(lhs, torch.Tensor) or isinstance(rhs, torch.Tensor):
+ tensor_leaves += 1
+ if not isinstance(lhs, torch.Tensor) or not isinstance(rhs, torch.Tensor):
+ mismatches.append(f"{path}: tensor/non-tensor type mismatch")
+ return False
+ if lhs.shape != rhs.shape or lhs.dtype != rhs.dtype or lhs.device != rhs.device:
+ mismatches.append(
+ f"{path}: tensor metadata differs "
+ f"({tuple(lhs.shape)}, {lhs.dtype}, {lhs.device}) != "
+ f"({tuple(rhs.shape)}, {rhs.dtype}, {rhs.device})"
+ )
+ return False
+ if not bool(torch.equal(lhs, rhs)):
+ mismatches.append(f"{path}: tensor values differ")
+ return False
+ return True
+ if isinstance(lhs, Mapping) or isinstance(rhs, Mapping):
+ if not isinstance(lhs, Mapping) or not isinstance(rhs, Mapping) or set(lhs) != set(rhs):
+ mismatches.append(f"{path}: mapping keys/type differ")
+ return False
+ return all(visit(lhs[key], rhs[key], f"{path}.{key}") for key in sorted(lhs, key=str))
+ if isinstance(lhs, (list, tuple)) or isinstance(rhs, (list, tuple)):
+ if not isinstance(lhs, (list, tuple)) or not isinstance(rhs, (list, tuple)) or len(lhs) != len(rhs):
+ mismatches.append(f"{path}: sequence length/type differs")
+ return False
+ return all(
+ visit(item_lhs, item_rhs, f"{path}[{index}]")
+ for index, (item_lhs, item_rhs) in enumerate(zip(lhs, rhs, strict=True))
+ )
+ if lhs != rhs:
+ mismatches.append(f"{path}: {lhs!r} != {rhs!r}")
+ return False
+ return True
+
+ exact = visit(left, right, "session")
+ return {
+ "exact": exact,
+ "checked_leaves": checked_leaves,
+ "tensor_leaves": tensor_leaves,
+ "mismatches": mismatches[:20],
+ "mismatch_count_at_least": len(mismatches),
+ }
+
+
+def _session_state_tree(session: Any, pipeline: Any) -> dict[str, Any]:
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW decode state")
+ return {
+ "prompt_emb": session.prompt_emb,
+ "first_frame_latent": session.first_frame_latent,
+ "self_cache": session.self_cache,
+ "cross_cache": session.cross_cache,
+ "generator_state": session.generator.get_state(),
+ "wan_decode_state": {
+ "feat_cache": session.vae_decode_state.feat_cache,
+ "feat_idx": session.vae_decode_state.feat_idx,
+ },
+ "taew_decode_state": pipeline.taew_decode_stage.export_decode_state_for_nccl(session.taew_decode_state),
+ "next_latent_frame": session.next_latent_frame,
+ "emitted_frames": session.emitted_frames,
+ }
+
+
+def _compare_tensor(left: torch.Tensor, right: torch.Tensor) -> dict[str, Any]:
+ if left.shape != right.shape or left.dtype != right.dtype or left.device != right.device:
+ return {
+ "comparable": False,
+ "left_shape": list(left.shape),
+ "right_shape": list(right.shape),
+ "left_dtype": str(left.dtype),
+ "right_dtype": str(right.dtype),
+ "left_device": str(left.device),
+ "right_device": str(right.device),
+ "exact": False,
+ }
+ difference = (left.float() - right.float()).abs()
+ return {
+ "comparable": True,
+ "shape": list(left.shape),
+ "dtype": str(left.dtype),
+ "device": str(left.device),
+ "exact": bool(torch.equal(left, right)),
+ "max_abs_difference": float(difference.max().item()),
+ "mean_abs_difference": float(difference.mean().item()),
+ }
+
+
+def _named_frame_comparison(
+ base: Any, left_name: str, left: Sequence[Image.Image], right_name: str, right: Sequence[Image.Image]
+) -> dict[str, Any]:
+ """Turn the base tool's generic pixel report into an explicitly named pair."""
+ raw = base._compare_frames(left, right)
+ frames = [
+ {
+ "frame_index": item["frame_index"],
+ f"{left_name}_sha256": item["graph_sha256"],
+ f"{right_name}_sha256": item["eager_sha256"],
+ "hash_equal": item["hash_equal"],
+ f"{left_name}_size": item["graph_size"],
+ f"{right_name}_size": item["eager_size"],
+ "max_abs_rgb_difference": item["max_abs_rgb_difference"],
+ "mean_abs_rgb_difference": item["mean_abs_rgb_difference"],
+ "nonzero_rgb_values": item["nonzero_rgb_values"],
+ }
+ for item in raw["frames"]
+ ]
+ return {
+ "left": left_name,
+ "right": right_name,
+ "comparable": raw["comparable"],
+ f"frame_count_{left_name}": raw["frame_count_graph"],
+ f"frame_count_{right_name}": raw["frame_count_eager"],
+ f"sequence_sha256_{left_name}": raw["sequence_sha256_graph"],
+ f"sequence_sha256_{right_name}": raw["sequence_sha256_eager"],
+ "all_frame_hashes_equal": raw["all_frame_hashes_equal"],
+ "max_abs_rgb_difference": raw["max_abs_rgb_difference"],
+ "mean_abs_rgb_difference": raw["mean_abs_rgb_difference"],
+ "nonzero_rgb_values": raw["nonzero_rgb_values"],
+ "total_rgb_values": raw["total_rgb_values"],
+ "frames": frames,
+ }
+
+
+def _within_pixel_tolerance(comparison: Mapping[str, Any], args: argparse.Namespace) -> bool:
+ maximum = comparison.get("max_abs_rgb_difference")
+ mean = comparison.get("mean_abs_rgb_difference")
+ return bool(
+ comparison.get("comparable")
+ and maximum is not None
+ and mean is not None
+ and maximum <= args.max_abs_rgb_difference
+ and mean <= args.mean_abs_rgb_difference
+ )
+
+
+@torch.inference_mode()
+def _prepare_continuation_input(
+ pipeline: Any, session: Any, actions: Mapping[str, bool], frames: int
+) -> tuple[torch.Tensor, torch.Tensor]:
+ shape = session.first_frame_latent.shape
+ noise = torch.randn(
+ (1, shape[1], frames, shape[3], shape[4]),
+ generator=session.generator,
+ device=pipeline.device,
+ dtype=torch.float32,
+ ).to(dtype=pipeline.torch_dtype)
+ action_context = pipeline.build_action_context(
+ actions,
+ latent_frames=frames,
+ height=pipeline.config.height,
+ width=pipeline.config.width,
+ device=pipeline.device,
+ dtype=pipeline.torch_dtype,
+ )
+ return noise, action_context
+
+
+@torch.inference_mode()
+def _steady_state_eager_denoise(
+ pipeline: Any,
+ session: Any,
+ latent: torch.Tensor,
+ action_context: torch.Tensor,
+) -> torch.Tensor:
+ """Execute exactly the graph wrapper's static DiT calls, but without capture."""
+ stage = pipeline.denoise_stage
+ dit = stage.dit
+ frames = latent.shape[2]
+ frame_tokens = (latent.shape[-2] // dit.patch_size[1]) * (latent.shape[-1] // dit.patch_size[2])
+ capacity = session.self_cache[0]["k"].shape[1]
+ sink_tokens = dit.sink_size * frame_tokens
+ rolled_tokens = capacity - sink_tokens - frames * frame_tokens
+ if rolled_tokens < 0:
+ raise ValueError("fixed ABot block does not fit in the rolling cache tail")
+ scratch_shape = (latent.shape[0], rolled_tokens, dit.num_heads, dit.dim // dit.num_heads)
+ roll_scratch_k = torch.empty(scratch_shape, dtype=latent.dtype, device=latent.device)
+ roll_scratch_v = torch.empty_like(roll_scratch_k)
+ current_end = torch.tensor(
+ [(session.next_latent_frame + frames) * frame_tokens],
+ dtype=torch.long,
+ device=latent.device,
+ )
+ timesteps = stage._official_denoising_timesteps(session.scheduler).to(device=latent.device)
+ current = latent
+ for index, timestep_value in enumerate(timesteps):
+ timestep = torch.full((1, frames), timestep_value, dtype=timesteps.dtype, device=latent.device)
+ with torch.autocast(latent.device.type, dtype=pipeline.torch_dtype, enabled=latent.device.type == "cuda"):
+ flow_prediction = dit.forward_steady_state(
+ x=current.to(dtype=pipeline.torch_dtype),
+ timestep=timestep,
+ context=session.prompt_emb,
+ act_context=action_context,
+ kv_cache=session.self_cache,
+ crossattn_cache=session.cross_cache,
+ current_end=current_end,
+ roll_scratch_k=roll_scratch_k,
+ roll_scratch_v=roll_scratch_v,
+ update_cache=index == 0,
+ )
+ x0 = stage._x0_prediction(flow_prediction, current, timestep, session.scheduler)
+ if index < len(timesteps) - 1:
+ noise = torch.randn(x0.shape, generator=session.generator, dtype=x0.dtype, device=latent.device)
+ current = session.scheduler.add_noise(x0, noise, timesteps[index + 1])
+ else:
+ # Mirror _ABotSteadyCudaGraph.run(): its refinement graph's output
+ # buffer is reused by the context-cache replay, so it explicitly
+ # preserves the terminal latent first.
+ current = x0.clone()
+ with torch.autocast(latent.device.type, dtype=pipeline.torch_dtype, enabled=latent.device.type == "cuda"):
+ dit.forward_steady_state(
+ x=current.to(dtype=pipeline.torch_dtype),
+ timestep=torch.zeros_like(timestep),
+ context=session.prompt_emb,
+ act_context=action_context,
+ kv_cache=session.self_cache,
+ crossattn_cache=session.cross_cache,
+ current_end=current_end,
+ roll_scratch_k=roll_scratch_k,
+ roll_scratch_v=roll_scratch_v,
+ update_cache=False,
+ )
+ return current
+
+
+@torch.inference_mode()
+def _decode_and_advance(pipeline: Any, session: Any, latents: torch.Tensor) -> list[Image.Image]:
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW decode state")
+ decoded = pipeline.taew_decode_stage.decode_chunks(latents, [session.taew_decode_state])
+ frames = pipeline.tensor2video(decoded[0])
+ session.next_latent_frame += latents.shape[2]
+ session.emitted_frames += len(frames)
+ return frames
+
+
+def _timed(device: torch.device, callback: Any) -> tuple[Any, float]:
+ torch.cuda.synchronize(device)
+ started_at = time.perf_counter()
+ result = callback()
+ torch.cuda.synchronize(device)
+ return result, time.perf_counter() - started_at
+
+
+@torch.inference_mode()
+def _run(args: argparse.Namespace) -> dict[str, Any]:
+ if not torch.cuda.is_available():
+ raise RuntimeError("three-way CUDA Graph diagnostic requires CUDA")
+ base = _load_base_validator()
+ image = Image.open(args.image).convert("RGB")
+ pipeline = None
+ sessions: dict[str, Any] = {}
+ try:
+ pipeline = base._make_pipeline(args)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(f"three-way CUDA Graph diagnostic requires CUDA, got {pipeline.device!r}")
+ pipeline.preload_models()
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ roles = ("regular_eager", "steady_state_eager", "cuda_graph")
+ for role in roles:
+ sessions[role] = pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=args.seed,
+ session_id=f"cuda-graph-three-way-{role}",
+ )
+
+ local_attn_size = int(pipeline.denoise_stage.dit.local_attn_size)
+ warmup_chunks = math.ceil(local_attn_size / args.control_latent_frames) + args.extra_warmup_chunks
+ warmup_hashes_equal: list[bool] = []
+ for _ in range(warmup_chunks):
+ warm_frames = {
+ role: pipeline.generate_next_block(
+ sessions[role],
+ args.action_keys,
+ control_latent_frames=args.control_latent_frames,
+ )
+ for role in roles
+ }
+ reference = base._sequence_hash(warm_frames["regular_eager"])
+ warmup_hashes_equal.append(all(base._sequence_hash(warm_frames[role]) == reference for role in roles[1:]))
+ torch.cuda.synchronize(device)
+
+ cache_readiness = {role: base._cache_readiness(session, pipeline) for role, session in sessions.items()}
+ regular_tree = _session_state_tree(sessions["regular_eager"], pipeline)
+ state_equivalence = {
+ "regular_vs_steady_state": _tree_exactness(
+ regular_tree,
+ _session_state_tree(sessions["steady_state_eager"], pipeline),
+ ),
+ "regular_vs_cuda_graph": _tree_exactness(
+ regular_tree,
+ _session_state_tree(sessions["cuda_graph"], pipeline),
+ ),
+ }
+
+ inputs = {
+ role: _prepare_continuation_input(
+ pipeline,
+ sessions[role],
+ args.action_keys,
+ args.control_latent_frames,
+ )
+ for role in roles
+ }
+ input_equivalence = {
+ "regular_vs_steady_state": _compare_tensor(inputs["regular_eager"][0], inputs["steady_state_eager"][0]),
+ "regular_vs_cuda_graph": _compare_tensor(inputs["regular_eager"][0], inputs["cuda_graph"][0]),
+ }
+
+ stage = pipeline.denoise_stage
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ regular_latent, regular_denoise_seconds = _timed(
+ device,
+ lambda: stage._denoise_block(
+ inputs["regular_eager"][0],
+ sessions["regular_eager"].prompt_emb,
+ inputs["regular_eager"][1],
+ None,
+ sessions["regular_eager"].self_cache,
+ sessions["regular_eager"].cross_cache,
+ sessions["regular_eager"].next_latent_frame,
+ sessions["regular_eager"].generator,
+ sessions["regular_eager"].scheduler,
+ ),
+ )
+ steady_latent, steady_denoise_seconds = _timed(
+ device,
+ lambda: _steady_state_eager_denoise(
+ pipeline,
+ sessions["steady_state_eager"],
+ inputs["steady_state_eager"][0],
+ inputs["steady_state_eager"][1],
+ ),
+ )
+ pipeline.denoise_stage.configure_cuda_graph(True)
+ graph_latent, graph_denoise_seconds = _timed(
+ device,
+ lambda: stage.denoise_interactive_block(
+ session_id=sessions["cuda_graph"].session_id,
+ latent=inputs["cuda_graph"][0],
+ prompt_emb=sessions["cuda_graph"].prompt_emb,
+ action_context=inputs["cuda_graph"][1],
+ self_cache=sessions["cuda_graph"].self_cache,
+ cross_cache=sessions["cuda_graph"].cross_cache,
+ current_start=sessions["cuda_graph"].next_latent_frame,
+ generator=sessions["cuda_graph"].generator,
+ scheduler=sessions["cuda_graph"].scheduler,
+ ),
+ )
+ graph_last_metrics = dict(stage.last_cuda_graph_metrics())
+ graph_runtime_metrics = dict(stage.cuda_graph_metrics())
+ graph_verification = base._graph_verified(graph_last_metrics)
+ pipeline.denoise_stage.configure_cuda_graph(False)
+
+ latent_comparisons = {
+ "regular_vs_steady_state": _compare_tensor(regular_latent, steady_latent),
+ "steady_state_vs_cuda_graph": _compare_tensor(steady_latent, graph_latent),
+ "regular_vs_cuda_graph": _compare_tensor(regular_latent, graph_latent),
+ }
+ regular_frames, regular_decode_seconds = _timed(
+ device,
+ lambda: _decode_and_advance(pipeline, sessions["regular_eager"], regular_latent),
+ )
+ steady_frames, steady_decode_seconds = _timed(
+ device,
+ lambda: _decode_and_advance(pipeline, sessions["steady_state_eager"], steady_latent),
+ )
+ graph_frames, graph_decode_seconds = _timed(
+ device,
+ lambda: _decode_and_advance(pipeline, sessions["cuda_graph"], graph_latent),
+ )
+ frame_comparisons = {
+ "regular_vs_steady_state": _named_frame_comparison(
+ base, "regular_eager", regular_frames, "steady_state_eager", steady_frames
+ ),
+ "steady_state_vs_cuda_graph": _named_frame_comparison(
+ base, "steady_state_eager", steady_frames, "cuda_graph", graph_frames
+ ),
+ "regular_vs_cuda_graph": _named_frame_comparison(
+ base, "regular_eager", regular_frames, "cuda_graph", graph_frames
+ ),
+ }
+ warmup_valid = all(warmup_hashes_equal) and all(item["ready"] for item in cache_readiness.values())
+ state_valid = all(item["exact"] for item in state_equivalence.values()) and all(
+ item["exact"] for item in input_equivalence.values()
+ )
+ pixels_valid = all(_within_pixel_tolerance(item, args) for item in frame_comparisons.values())
+ latents_valid = all(item["exact"] for item in latent_comparisons.values())
+ if not warmup_valid or not state_valid:
+ status = "invalid_precondition"
+ elif not graph_verification["verified"]:
+ status = "graph_unverified"
+ elif not latent_comparisons["regular_vs_steady_state"]["exact"] or not _within_pixel_tolerance(
+ frame_comparisons["regular_vs_steady_state"], args
+ ):
+ status = "steady_state_model_mismatch"
+ elif not latent_comparisons["steady_state_vs_cuda_graph"]["exact"] or not _within_pixel_tolerance(
+ frame_comparisons["steady_state_vs_cuda_graph"], args
+ ):
+ status = "cuda_graph_wrapper_mismatch"
+ elif not latents_valid or not pixels_valid:
+ status = "cuda_graph_parity_mismatch"
+ else:
+ status = "pass"
+ return {
+ "status": status,
+ "diagnostic": "regular eager vs eager steady-state vs captured CUDA Graph",
+ "device": str(device),
+ "seed": args.seed,
+ "actions": args.action_keys,
+ "control_latent_frames": args.control_latent_frames,
+ "warmup": {
+ "chunks": warmup_chunks,
+ "per_chunk_all_three_hashes_equal": warmup_hashes_equal,
+ "all_three_outputs_equal": all(warmup_hashes_equal),
+ "cache_readiness": cache_readiness,
+ },
+ "state_equivalence_before_continuation": state_equivalence,
+ "initial_noise_equivalence": input_equivalence,
+ "continuation_timings_seconds": {
+ "regular_eager": {"denoise": regular_denoise_seconds, "decode": regular_decode_seconds},
+ "steady_state_eager": {"denoise": steady_denoise_seconds, "decode": steady_decode_seconds},
+ "cuda_graph": {"denoise": graph_denoise_seconds, "decode": graph_decode_seconds},
+ },
+ "cuda_graph": {
+ "last_metrics": graph_last_metrics,
+ "runtime_metrics": graph_runtime_metrics,
+ "verification": graph_verification,
+ },
+ "latent_comparisons": latent_comparisons,
+ "frame_comparisons": frame_comparisons,
+ "pixel_tolerance": {
+ "max_abs_rgb_difference": args.max_abs_rgb_difference,
+ "mean_abs_rgb_difference": args.mean_abs_rgb_difference,
+ },
+ }
+ finally:
+ if pipeline is not None:
+ for session in sessions.values():
+ try:
+ pipeline.close_interactive_session(session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _write_output(output_dir: Path, args: argparse.Namespace, result: Mapping[str, Any], base: Any) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ (output_dir / "results.json").write_text(
+ json.dumps(
+ {"arguments": base._json_safe(vars(args)), "result": base._json_safe(result)}, indent=2, sort_keys=True
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ graph = result.get("cuda_graph", {})
+ verification = graph.get("verification", {}) if isinstance(graph, Mapping) else {}
+ lines = [
+ "# ABot CUDA Graph three-way diagnostic",
+ "",
+ (
+ "Three same-seed B=1 sessions were warmed eagerly to a full KV window, then run through "
+ "regular eager, static eager, and captured CUDA-Graph continuations."
+ ),
+ "",
+ "| Pair / fact | Exact RGB hashes | Max RGB abs diff | Mean RGB abs diff |",
+ "| --- | --- | ---: | ---: |",
+ ]
+ for name, comparison in result.get("frame_comparisons", {}).items():
+ lines.append(
+ f"| {name} | {comparison.get('all_frame_hashes_equal', False)} | "
+ f"{comparison.get('max_abs_rgb_difference', '')} | {comparison.get('mean_abs_rgb_difference', '')} |"
+ )
+ lines.extend(
+ [
+ "",
+ f"Status: `{result.get('status', 'error')}`.",
+ "",
+ f"Graph capture/replay/fallback: `{verification.get('captured', False)}` / "
+ f"`{verification.get('replay_observed', False)}` / `{verification.get('fallback_observed', False)}`.",
+ "",
+ (
+ "`results.json` includes exact pre-continuation session-state checks, initial-noise checks, "
+ "latent comparisons, and per-frame hashes."
+ ),
+ ]
+ )
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args() -> argparse.Namespace:
+ base = _load_base_validator()
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model-root", type=Path)
+ parser.add_argument("--image", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--action-keys", type=base._parse_action_keys, default={"W": True})
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
+ parser.add_argument("--extra-warmup-chunks", type=int, default=0)
+ parser.add_argument("--max-abs-rgb-difference", type=int, default=0)
+ parser.add_argument("--mean-abs-rgb-difference", type=float, default=0.0)
+ parser.add_argument("--device-id", type=int, default=0)
+ parser.add_argument("--dry-run", action="store_true")
+ args = parser.parse_args()
+ if args.extra_warmup_chunks < 0:
+ parser.error("--extra-warmup-chunks must be non-negative")
+ if args.max_abs_rgb_difference < 0 or args.mean_abs_rgb_difference < 0:
+ parser.error("pixel-difference tolerances must be non-negative")
+ if not args.dry_run:
+ if args.model_root is None or args.image is None or args.output_dir is None:
+ parser.error("--model-root, --image, and --output-dir are required unless --dry-run is used")
+ if not args.model_root.is_dir():
+ parser.error(f"model root does not exist: {args.model_root}")
+ if not args.image.is_file():
+ parser.error(f"image does not exist: {args.image}")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ if args.dry_run:
+ print(
+ json.dumps(
+ {
+ "mode": "dry_run",
+ "batch_size": 1,
+ "sessions": ["regular_eager", "steady_state_eager", "cuda_graph"],
+ "warmup": "all three sessions eager until the 18-latent-frame KV window is full",
+ "continuation_paths": [
+ "regular dynamic eager",
+ "forward_steady_state eager",
+ "captured CUDA Graph",
+ ],
+ "comparisons": ["state", "initial noise", "final latent", "decoded RGB pixels/hashes"],
+ "zero_tolerance": {
+ "max_abs_rgb_difference": args.max_abs_rgb_difference,
+ "mean_abs_rgb_difference": args.mean_abs_rgb_difference,
+ },
+ },
+ indent=2,
+ sort_keys=True,
+ )
+ )
+ return
+ assert args.output_dir is not None
+ base = _load_base_validator()
+ try:
+ result = _run(args)
+ except Exception as exc:
+ result = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
+ _write_output(args.output_dir, args, result, base)
+ print(json.dumps(base._json_safe(result), indent=2, sort_keys=True))
+ if result.get("status") != "pass":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/localize_abot_steady_state_cache_mismatch.py b/tools/validation/localize_abot_steady_state_cache_mismatch.py
new file mode 100644
index 00000000..7ae58cc9
--- /dev/null
+++ b/tools/validation/localize_abot_steady_state_cache_mismatch.py
@@ -0,0 +1,570 @@
+"""Locate ABot B=1 dynamic-versus-steady-state KV cache divergence.
+
+This is a tools-only, single-continuation diagnostic. It warms ordinary and
+steady-state twins to the same full causal KV window, then interleaves each
+ordinary DiT denoising call with its persistent-static counterpart. After each
+pair it performs an exact on-device comparison of layer-0 K/V and cache
+cursors. It separately compares the final dynamic context-cache update and
+records the value/layout (shape, stride, memory format) of its x0, prompt, and
+action inputs. Token intervals are only materialized after an exact K/V check
+fails, so the tool does not retain multi-GB cache snapshots.
+
+Example:
+
+ CUDA_VISIBLE_DEVICES=3 PYTHONPATH=$PWD \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/localize_abot_steady_state_cache_mismatch.py \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir results/validation/abot_b1_steady_cache_localization_gpu3
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+
+def _load_module(filename: str, module_name: str) -> Any:
+ path = Path(__file__).with_name(filename)
+ spec = importlib.util.spec_from_file_location(module_name, path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"could not load diagnostic helper: {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _intervals(indices: torch.Tensor) -> list[list[int]]:
+ """Turn a compact CPU token-index tensor into half-open intervals."""
+ values = [int(value) for value in indices.to(device="cpu").tolist()]
+ if not values:
+ return []
+ result: list[list[int]] = []
+ start = previous = values[0]
+ for value in values[1:]:
+ if value != previous + 1:
+ result.append([start, previous + 1])
+ start = value
+ previous = value
+ result.append([start, previous + 1])
+ return result
+
+
+def _changed_token_intervals(left: torch.Tensor, right: torch.Tensor) -> dict[str, Any]:
+ """Strictly identify which KV token rows differ, without copying full KV."""
+ if left.shape != right.shape or left.dtype != right.dtype or left.device != right.device:
+ return {
+ "comparable": False,
+ "equal": False,
+ "left_shape": list(left.shape),
+ "right_shape": list(right.shape),
+ }
+ exact = bool(torch.equal(left, right))
+ if exact:
+ return {
+ "comparable": True,
+ "equal": True,
+ "changed_token_count": 0,
+ "first_changed_token": None,
+ "last_changed_token": None,
+ "changed_token_intervals": [],
+ }
+ changed = torch.any(left != right, dim=(0, 2, 3))
+ indices = torch.nonzero(changed, as_tuple=False).flatten()
+ return {
+ "comparable": True,
+ "equal": False,
+ "changed_token_count": int(indices.numel()),
+ "first_changed_token": int(indices[0].item()) if indices.numel() else None,
+ "last_changed_token": int(indices[-1].item()) if indices.numel() else None,
+ "changed_token_intervals": _intervals(indices),
+ }
+
+
+def _exact_layer_cache_comparison(dynamic: Mapping[str, Any], steady: Mapping[str, Any]) -> dict[str, Any]:
+ """Strict layer-cache equality plus ranges only when an exact check fails."""
+ k = _changed_token_intervals(dynamic["k"], steady["k"])
+ v = _changed_token_intervals(dynamic["v"], steady["v"])
+ dynamic_global = int(dynamic["global_end_index"].item())
+ steady_global = int(steady["global_end_index"].item())
+ dynamic_local = int(dynamic["local_end_index"].item())
+ steady_local = int(steady["local_end_index"].item())
+ return {
+ "exact": bool(k["equal"] and v["equal"] and dynamic_global == steady_global and dynamic_local == steady_local),
+ "k": k,
+ "v": v,
+ "dynamic_global_end_index": dynamic_global,
+ "steady_global_end_index": steady_global,
+ "dynamic_local_end_index": dynamic_local,
+ "steady_local_end_index": steady_local,
+ }
+
+
+def _tensor_layout(value: torch.Tensor) -> dict[str, Any]:
+ """Capture the tensor facts that can change a CUDA kernel choice."""
+ return {
+ "shape": list(value.shape),
+ "stride": list(value.stride()),
+ "dtype": str(value.dtype),
+ "is_contiguous": bool(value.is_contiguous()),
+ "is_channels_last_3d": bool(value.is_contiguous(memory_format=torch.channels_last_3d)),
+ }
+
+
+def _tensor_pair(dynamic: torch.Tensor, steady: torch.Tensor) -> dict[str, Any]:
+ return {
+ "values_exact": bool(torch.equal(dynamic, steady)),
+ "dynamic_layout": _tensor_layout(dynamic),
+ "steady_layout": _tensor_layout(steady),
+ }
+
+
+def _fingerprint(cache: Mapping[str, Any]) -> dict[str, Any]:
+ """Produce collision-resistant, per-token K/V summaries for one layer."""
+
+ def summarise(value: torch.Tensor) -> torch.Tensor:
+ # Reductions operate on the full KV layer but only materialize a tiny
+ # [tokens, 3] result. Avoid float()/square() full-cache temporaries:
+ # two B=1 retained sessions are already substantial GPU residents.
+ detached = value.detach()
+ return torch.stack(
+ (
+ detached.sum(dim=(0, 2, 3), dtype=torch.float32),
+ detached.amin(dim=(0, 2, 3)).float(),
+ detached.amax(dim=(0, 2, 3)).float(),
+ ),
+ dim=1,
+ ).cpu()
+
+ return {
+ "k": summarise(cache["k"]),
+ "v": summarise(cache["v"]),
+ "global_end_index": int(cache["global_end_index"].item()),
+ "local_end_index": int(cache["local_end_index"].item()),
+ }
+
+
+def _fingerprint_difference(left: Mapping[str, Any], right: Mapping[str, Any]) -> dict[str, Any]:
+ def changed(value_name: str) -> dict[str, Any]:
+ lhs = left[value_name]
+ rhs = right[value_name]
+ if lhs.shape != rhs.shape:
+ return {"comparable": False, "changed_token_intervals": []}
+ rows = torch.any(lhs != rhs, dim=1)
+ indices = torch.nonzero(rows, as_tuple=False).flatten()
+ return {
+ "comparable": True,
+ "changed_token_count": int(indices.numel()),
+ "first_changed_token": int(indices[0].item()) if indices.numel() else None,
+ "changed_token_intervals": _intervals(indices),
+ }
+
+ return {
+ "k_fingerprint": changed("k"),
+ "v_fingerprint": changed("v"),
+ "global_end_index": {
+ "dynamic": left["global_end_index"],
+ "steady_state": right["global_end_index"],
+ "equal": left["global_end_index"] == right["global_end_index"],
+ },
+ "local_end_index": {
+ "dynamic": left["local_end_index"],
+ "steady_state": right["local_end_index"],
+ "equal": left["local_end_index"] == right["local_end_index"],
+ },
+ }
+
+
+class _LayerZeroCallProbe:
+ """Fingerprint layer-0 K/V after every dynamic or steady DiT invocation."""
+
+ def __init__(self, dit: Any, layer: int) -> None:
+ self._dit = dit
+ self._layer = layer
+ self._dynamic_original: Any = None
+ self._steady_original: Any = None
+ self.dynamic: list[dict[str, Any]] = []
+ self.steady: list[dict[str, Any]] = []
+
+ def __enter__(self) -> "_LayerZeroCallProbe":
+ self._dynamic_original = self._dit.forward
+ self._steady_original = self._dit.forward_steady_state
+
+ def dynamic(*args: Any, **kwargs: Any) -> Any:
+ output = self._dynamic_original(*args, **kwargs)
+ cache = kwargs["kv_cache"][self._layer]
+ self.dynamic.append(_fingerprint(cache))
+ return output
+
+ def steady(*args: Any, **kwargs: Any) -> Any:
+ output = self._steady_original(*args, **kwargs)
+ cache = kwargs["kv_cache"][self._layer]
+ self.steady.append(_fingerprint(cache))
+ return output
+
+ self._dit.forward = dynamic
+ self._dit.forward_steady_state = steady
+ return self
+
+ def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
+ self._dit.forward = self._dynamic_original
+ self._dit.forward_steady_state = self._steady_original
+
+
+def _call_reports(probe: _LayerZeroCallProbe) -> list[dict[str, Any]]:
+ count = max(len(probe.dynamic), len(probe.steady))
+ reports: list[dict[str, Any]] = []
+ for index in range(count):
+ dynamic = probe.dynamic[index] if index < len(probe.dynamic) else None
+ steady = probe.steady[index] if index < len(probe.steady) else None
+ reports.append(
+ {
+ "call_index": index,
+ "phase": f"denoise_step_{index}" if index < 4 else "context_cache_update",
+ "dynamic_seen": dynamic is not None,
+ "steady_state_seen": steady is not None,
+ "fingerprint_comparison": (
+ _fingerprint_difference(dynamic, steady) if dynamic is not None and steady is not None else None
+ ),
+ }
+ )
+ return reports
+
+
+def _cache_layout(pipeline: Any, session: Any, frames: int) -> dict[str, int]:
+ dit = pipeline.denoise_stage.dit
+ height, width = session.first_frame_latent.shape[-2:]
+ frame_tokens = (height // dit.patch_size[1]) * (width // dit.patch_size[2])
+ capacity = int(session.self_cache[0]["k"].shape[1])
+ sink_tokens = int(dit.sink_size) * frame_tokens
+ return {
+ "frame_tokens": frame_tokens,
+ "continuation_tokens": frames * frame_tokens,
+ "capacity_tokens": capacity,
+ "sink_tokens": sink_tokens,
+ "rolling_tail_start": sink_tokens,
+ "new_block_tail_start": capacity - frames * frame_tokens,
+ }
+
+
+@torch.inference_mode()
+def _run(args: argparse.Namespace, base: Any, control: Any) -> dict[str, Any]:
+ if not torch.cuda.is_available():
+ raise RuntimeError("cache localization requires CUDA")
+ image = Image.open(args.image).convert("RGB")
+ pipeline = None
+ dynamic_session = None
+ static_session = None
+ try:
+ pipeline = base._make_pipeline(args)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(f"cache localization requires CUDA, got {pipeline.device!r}")
+ pipeline.preload_models()
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ actions = base._parse_action_keys(args.action_keys)
+ dynamic_session = pipeline.create_interactive_session(
+ image, args.prompt, seed=args.seed, session_id="cache-localization-dynamic"
+ )
+ static_session = pipeline.create_interactive_session(
+ image, args.prompt, seed=args.seed, session_id="cache-localization-steady"
+ )
+ warmup_chunks = base._required_warmup_chunks(
+ int(pipeline.denoise_stage.dit.local_attn_size),
+ args.control_latent_frames,
+ args.extra_warmup_chunks,
+ )
+ warmup_equal: list[bool] = []
+ for _ in range(warmup_chunks):
+ dynamic_frames = pipeline.generate_next_block(
+ dynamic_session, actions, control_latent_frames=args.control_latent_frames
+ )
+ static_frames = pipeline.generate_next_block(
+ static_session, actions, control_latent_frames=args.control_latent_frames
+ )
+ warmup_equal.append(base._sequence_hash(dynamic_frames) == base._sequence_hash(static_frames))
+ torch.cuda.synchronize(device)
+ warmup_state_exact = control._tree_exactness(
+ control._session_state_tree(dynamic_session, pipeline),
+ control._session_state_tree(static_session, pipeline),
+ )
+ dynamic_latent, dynamic_prompt, dynamic_action = control._prepare_inputs(
+ pipeline, [dynamic_session], [actions], args.control_latent_frames
+ )
+ static_latent, static_prompt, static_action = control._prepare_inputs(
+ pipeline, [static_session], [actions], args.control_latent_frames
+ )
+ input_exact = control._compare_tensor(dynamic_latent, static_latent)
+ stage = pipeline.denoise_stage
+ static_state = control._static_state(pipeline, [static_session], static_latent, static_prompt, static_action)
+ graph = static_state["graph"]
+ current_start = dynamic_session.next_latent_frame
+ if current_start != static_session.next_latent_frame:
+ raise RuntimeError("dynamic and steady sessions must share a continuation position")
+ timesteps = stage._official_denoising_timesteps(dynamic_session.scheduler).to(device=device)
+ dynamic_current = dynamic_latent
+ steady_current = static_latent
+ step_reports: list[dict[str, Any]] = []
+ for index, current_timestep in enumerate(timesteps):
+ dynamic_timestep = torch.full(
+ (1, args.control_latent_frames),
+ current_timestep,
+ dtype=timesteps.dtype,
+ device=device,
+ )
+ with torch.autocast(device.type, dtype=pipeline.torch_dtype, enabled=device.type == "cuda"):
+ dynamic_prediction = stage.dit(
+ x=dynamic_current.to(dtype=pipeline.torch_dtype),
+ timestep=dynamic_timestep,
+ context=dynamic_prompt,
+ act_context=dynamic_action,
+ kv_cache=dynamic_session.self_cache,
+ crossattn_cache=dynamic_session.cross_cache,
+ current_start=current_start * graph.frame_tokens,
+ )
+ graph._set_inputs(
+ steady_current,
+ static_action,
+ current_timestep,
+ current_end=(current_start + args.control_latent_frames) * graph.frame_tokens,
+ )
+ with torch.autocast(device.type, dtype=pipeline.torch_dtype, enabled=device.type == "cuda"):
+ steady_prediction = stage.dit.forward_steady_state(
+ x=graph.static_x,
+ timestep=graph.static_timestep,
+ context=graph.static_context,
+ act_context=graph.static_action,
+ kv_cache=static_state["self_cache"],
+ crossattn_cache=static_state["cross_cache"],
+ current_end=graph.current_end,
+ roll_scratch_k=graph.roll_scratch_k,
+ roll_scratch_v=graph.roll_scratch_v,
+ update_cache=index == 0,
+ )
+ torch.cuda.synchronize(device)
+ layer_cache = _exact_layer_cache_comparison(
+ dynamic_session.self_cache[args.layer], static_session.self_cache[args.layer]
+ )
+ dynamic_x0 = stage._x0_prediction(
+ dynamic_prediction, dynamic_current, dynamic_timestep, dynamic_session.scheduler
+ )
+ steady_x0 = stage._x0_prediction(
+ steady_prediction, steady_current, graph.static_timestep, static_session.scheduler
+ )
+ step_reports.append(
+ {
+ "call_index": index,
+ "phase": f"denoise_step_{index}",
+ "flow_prediction": _tensor_pair(dynamic_prediction, steady_prediction),
+ "x0": _tensor_pair(dynamic_x0, steady_x0),
+ "layer0_cache": layer_cache,
+ }
+ )
+ if index < len(timesteps) - 1:
+ dynamic_noise = torch.randn(
+ dynamic_x0.shape,
+ generator=dynamic_session.generator,
+ dtype=dynamic_x0.dtype,
+ device=device,
+ )
+ steady_noise = torch.randn(
+ steady_x0.shape,
+ generator=static_session.generator,
+ dtype=steady_x0.dtype,
+ device=device,
+ )
+ dynamic_current = dynamic_session.scheduler.add_noise(dynamic_x0, dynamic_noise, timesteps[index + 1])
+ steady_current = static_session.scheduler.add_noise(steady_x0, steady_noise, timesteps[index + 1])
+ else:
+ dynamic_current = dynamic_x0
+ steady_current = steady_x0.clone()
+
+ dynamic_context_timestep = torch.zeros_like(dynamic_timestep)
+ graph.static_timestep.zero_()
+ dynamic_context_input = dynamic_current.to(dtype=pipeline.torch_dtype)
+ steady_context_input = steady_current.to(dtype=pipeline.torch_dtype)
+ final_dynamic_context_inputs = {
+ "x0": _tensor_pair(dynamic_context_input, steady_context_input),
+ "timestep": _tensor_pair(dynamic_context_timestep, graph.static_timestep),
+ "prompt": _tensor_pair(dynamic_prompt, graph.static_context),
+ "action": _tensor_pair(dynamic_action, static_action),
+ }
+ # The public _denoise_block commits final x0 outside its sampler
+ # autocast scope; mirror that precision boundary on both sides.
+ stage.dit(
+ x=dynamic_context_input,
+ timestep=dynamic_context_timestep,
+ context=dynamic_prompt,
+ act_context=dynamic_action,
+ kv_cache=dynamic_session.self_cache,
+ crossattn_cache=dynamic_session.cross_cache,
+ current_start=current_start * graph.frame_tokens,
+ )
+ stage.dit(
+ x=steady_context_input,
+ timestep=graph.static_timestep,
+ context=graph.static_context,
+ act_context=static_action,
+ kv_cache=static_state["self_cache"],
+ crossattn_cache=static_state["cross_cache"],
+ current_start=current_start * graph.frame_tokens,
+ )
+ torch.cuda.synchronize(device)
+ context_cache = _exact_layer_cache_comparison(
+ dynamic_session.self_cache[args.layer], static_session.self_cache[args.layer]
+ )
+ output = _tensor_pair(dynamic_current, steady_current)
+ first_exact_cache_divergence = next(
+ (item["call_index"] for item in step_reports if not item["layer0_cache"]["exact"]),
+ 4 if not context_cache["exact"] else None,
+ )
+ all_step_caches_exact = all(item["layer0_cache"]["exact"] for item in step_reports)
+ status = (
+ "pass"
+ if output["values_exact"] and all_step_caches_exact and context_cache["exact"]
+ else "cache_mismatch_localized"
+ )
+ return {
+ "status": status,
+ "scope": {
+ "public_denoise_block_exercised": False,
+ "dynamic_control": "handwritten per-DiT-call loop",
+ "steady_control": "handwritten forward_steady_state loop",
+ "execution_order": "dynamic_then_steady_per_denoise_step",
+ "cache_coverage": f"layer {args.layer} only",
+ "note": (
+ "This localizer isolates individual DiT/cache writes; it is not a parity proof for "
+ "generate_next_block -> denoise_interactive_block -> _denoise_block. "
+ "Use validate_abot_public_vs_steady_state.py for that public-path check."
+ ),
+ },
+ "device": str(device),
+ "layer": args.layer,
+ "control_latent_frames": args.control_latent_frames,
+ "warmup": {
+ "chunks": warmup_chunks,
+ "per_chunk_frames_equal": warmup_equal,
+ "state_exact": warmup_state_exact,
+ },
+ "continuation_input_exact": input_exact,
+ "continuation_output_exact": output,
+ "cache_layout": _cache_layout(pipeline, dynamic_session, args.control_latent_frames),
+ "per_denoise_step_layer0_exact": step_reports,
+ "all_denoise_step_caches_exact": all_step_caches_exact,
+ "first_exact_cache_divergence_call": first_exact_cache_divergence,
+ "final_dynamic_context_inputs": final_dynamic_context_inputs,
+ "final_layer_cache_exact": context_cache,
+ }
+ finally:
+ if pipeline is not None:
+ for session in (dynamic_session, static_session):
+ if session is not None:
+ try:
+ pipeline.close_interactive_session(session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _write_output(output_dir: Path, result: Mapping[str, Any], args: argparse.Namespace, base: Any) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ (output_dir / "results.json").write_text(
+ json.dumps({"arguments": base._json_safe(vars(args)), "result": base._json_safe(result)}, indent=2) + "\n",
+ encoding="utf-8",
+ )
+ final = result.get("final_layer_cache_exact", {})
+ lines = [
+ "# ABot steady-state cache mismatch localization",
+ "",
+ f"Status: {result.get('status', 'error')}.",
+ "",
+ "Scope: handwritten dynamic and steady-state DiT loops, interleaved per denoising step; "
+ "this is not the public `_denoise_block` parity check.",
+ "",
+ f"First exact layer-0 cache divergence call: {result.get('first_exact_cache_divergence_call')}.",
+ "",
+ f"Final K changed intervals: {final.get('k', {}).get('changed_token_intervals', [])}.",
+ f"Final V changed intervals: {final.get('v', {}).get('changed_token_intervals', [])}.",
+ "",
+ "Call indices 0..3 are denoising calls; index 4 is the context-cache update.",
+ "results.json contains exact per-step K/V comparisons, context-input layouts, and cursor values.",
+ ]
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args(base: Any) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model-root", type=Path)
+ parser.add_argument("--image", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--action-keys", default="W")
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
+ parser.add_argument("--extra-warmup-chunks", type=int, default=0)
+ parser.add_argument("--layer", type=int, default=0)
+ parser.add_argument("--device-id", type=int, default=0)
+ parser.add_argument("--dry-run", action="store_true")
+ args = parser.parse_args()
+ if args.extra_warmup_chunks < 0:
+ parser.error("--extra-warmup-chunks must be non-negative")
+ try:
+ base._parse_action_keys(args.action_keys)
+ except argparse.ArgumentTypeError as exc:
+ parser.error(str(exc))
+ if args.layer < 0:
+ parser.error("--layer must be non-negative")
+ if not args.dry_run:
+ if args.model_root is None or args.image is None or args.output_dir is None:
+ parser.error("--model-root, --image, and --output-dir are required unless --dry-run is used")
+ if not args.model_root.is_dir():
+ parser.error(f"model root does not exist: {args.model_root}")
+ if not args.image.is_file():
+ parser.error(f"image does not exist: {args.image}")
+ return args
+
+
+def main() -> None:
+ base = _load_module("validate_abot_cuda_graph_parity.py", "abot_cache_localization_base")
+ control = _load_module("diagnose_abot_cuda_graph_persistent_three_way.py", "abot_cache_localization_control")
+ args = _parse_args(base)
+ if args.dry_run:
+ print(
+ json.dumps(
+ {
+ "mode": "dry_run",
+ "batch_size": 1,
+ "layer": args.layer,
+ "calls": ["denoise_step_0", "denoise_step_1", "denoise_step_2", "denoise_step_3", "context"],
+ "output": ["first exact cache divergence", "per-step K/V token intervals", "context-input layouts"],
+ },
+ indent=2,
+ )
+ )
+ return
+ assert args.output_dir is not None
+ try:
+ result = _run(args, base, control)
+ except Exception as exc:
+ result = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
+ _write_output(args.output_dir, result, args, base)
+ print(json.dumps(base._json_safe(result), indent=2))
+ if result.get("status") != "pass":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/render_abot_demand_scatter.py b/tools/validation/render_abot_demand_scatter.py
new file mode 100644
index 00000000..59c3324e
--- /dev/null
+++ b/tools/validation/render_abot_demand_scatter.py
@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+"""Render client-side ABot demand onsets and active intervals for one trace.
+
+A demand onset is either a session arriving with active input enabled, or a
+user resuming input after an explicit idle interval. The runner does not write
+every heartbeat to ``result.json``; the horizontal lane segments therefore
+represent the continuous active-demand intervals between those onsets and the
+corresponding pause/departure. Orange spans come from actual parent-owned
+model-dispatch records whose batch size is greater than one.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from PIL import Image, ImageDraw, ImageFont
+
+_NAVY = "#0f172a"
+_SLATE = "#475569"
+_GRID = "#cbd5e1"
+_PANEL = "#f8fafc"
+_WHITE = "#ffffff"
+_ACTIVE = "#188038"
+_RESUME = "#1a73e8"
+_INTERVAL = "#94a3b8"
+_BATCH = "#f9ab00"
+_BATCH_BORDER = "#d93025"
+
+
+def _font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont:
+ face = "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf"
+ return ImageFont.truetype(face, size=size)
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+ value = json.loads(path.read_text(encoding="utf-8"))
+ if not isinstance(value, dict):
+ raise ValueError(f"Expected JSON object: {path}")
+ return value
+
+
+def _events(result: dict[str, Any]) -> list[dict[str, Any]]:
+ raw = result.get("events")
+ if not isinstance(raw, list):
+ raise ValueError("result.events is missing")
+ return [event for event in raw if isinstance(event, dict) and isinstance(event.get("offset_seconds"), int | float)]
+
+
+def _session_for_event(event: dict[str, Any]) -> str | None:
+ for key in ("session", "trace_session_id"):
+ value = event.get(key)
+ if isinstance(value, str) and value:
+ return value
+ return None
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--result", type=Path, required=True)
+ parser.add_argument("--dispatch-trace", type=Path, required=True)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ return parser.parse_args()
+
+
+def _time_to_x(value: float, *, left: int, width: int, last_offset: float) -> int:
+ return left + round(max(0.0, min(value, last_offset)) / last_offset * width)
+
+
+def _circle(draw: ImageDraw.ImageDraw, x: int, y: int, color: str) -> None:
+ radius = 5
+ draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=color, outline=_WHITE, width=1)
+
+
+def _triangle(draw: ImageDraw.ImageDraw, x: int, y: int, color: str) -> None:
+ radius = 6
+ draw.polygon(((x, y - radius), (x - radius, y + radius), (x + radius, y + radius)), fill=color, outline=_WHITE)
+
+
+def _deduplicated(values: list[tuple[float, str, str]]) -> list[tuple[float, str, str]]:
+ seen: set[tuple[float, str, str]] = set()
+ result: list[tuple[float, str, str]] = []
+ for value in values:
+ if value not in seen:
+ seen.add(value)
+ result.append(value)
+ return result
+
+
+def main() -> None:
+ args = _parse_args()
+ result = _load_json(args.result.expanduser().resolve())
+ events = sorted(_events(result), key=lambda event: float(event["offset_seconds"]))
+ clock = result.get("trace_clock")
+ if not isinstance(clock, dict) or not isinstance(clock.get("origin_unix_seconds"), int | float):
+ raise ValueError("result.trace_clock.origin_unix_seconds is required")
+ origin_unix = float(clock["origin_unix_seconds"])
+
+ arrivals: dict[str, float] = {}
+ onsets: list[tuple[float, str, str]] = []
+ intervals: dict[str, list[list[float | None]]] = defaultdict(list)
+ active_open: set[str] = set()
+ for event in events:
+ timestamp = float(event["offset_seconds"])
+ event_name = event.get("event")
+ session = _session_for_event(event)
+ if session is None:
+ continue
+ if event_name == "lifecycle_session_arrival_scheduled":
+ arrivals.setdefault(session, timestamp)
+ if event.get("input_enabled") is True:
+ onsets.append((timestamp, session, "arrival_active"))
+ intervals[session].append([timestamp, None])
+ active_open.add(session)
+ elif event_name == "input_resumed":
+ if session not in active_open:
+ onsets.append((timestamp, session, "resumed"))
+ intervals[session].append([timestamp, None])
+ active_open.add(session)
+ elif event_name == "input_paused" and session in active_open:
+ intervals[session][-1][1] = timestamp
+ active_open.remove(session)
+ elif event_name in {"lifecycle_session_departure_scheduled", "session_stopped"}:
+ if session in active_open:
+ intervals[session][-1][1] = timestamp
+ active_open.remove(session)
+
+ if not arrivals:
+ raise ValueError("result contains no lifecycle session arrivals")
+ last_offset = max(float(event["offset_seconds"]) for event in events)
+ for session_intervals in intervals.values():
+ for interval in session_intervals:
+ if interval[1] is None:
+ interval[1] = last_offset
+ onsets = _deduplicated(onsets)
+ lanes = sorted(arrivals, key=lambda session: (arrivals[session], session))
+ lane_index = {session: index for index, session in enumerate(lanes)}
+
+ trace_lines = args.dispatch_trace.expanduser().resolve().read_text(encoding="utf-8").splitlines()
+ dispatches = [json.loads(line) for line in trace_lines[1:] if line.strip()]
+ batched = [record for record in dispatches if int(record.get("batch_size", 0)) > 1]
+
+ active_delta: list[tuple[float, int]] = []
+ for session_intervals in intervals.values():
+ for start, end in session_intervals:
+ active_delta.append((float(start), 1))
+ active_delta.append((float(end), -1))
+ active_delta.sort(key=lambda item: (item[0], item[1]))
+
+ margin_left = 250
+ margin_right = 55
+ margin_top = 115
+ count_height = 180
+ lane_height = 18
+ lane_top = margin_top + count_height + 70
+ width = 3000
+ plot_width = width - margin_left - margin_right
+ height = lane_top + max(1, len(lanes)) * lane_height + 105
+ image = Image.new("RGB", (width, height), _WHITE)
+ draw = ImageDraw.Draw(image)
+
+ draw.rectangle((0, 0, width, height), fill=_WHITE)
+ draw.text(
+ (margin_left, 28), "ABot client demand onsets and active intervals", font=_font(30, bold=True), fill=_NAVY
+ )
+ subtitle = (
+ "Green circle: arrival with active input; blue triangle: resume; gray: active-demand interval; "
+ "orange: actual B>1 dispatch"
+ )
+ draw.text((margin_left, 67), subtitle, font=_font(15), fill=_SLATE)
+
+ count_top = margin_top
+ count_bottom = count_top + count_height
+ lane_bottom = lane_top + len(lanes) * lane_height
+ draw.rectangle((margin_left, count_top, width - margin_right, count_bottom), fill=_PANEL, outline=_GRID)
+ draw.rectangle((margin_left, lane_top, width - margin_right, lane_bottom), fill=_PANEL, outline=_GRID)
+
+ for seconds in range(0, int(last_offset) + 1, 120):
+ x = _time_to_x(seconds, left=margin_left, width=plot_width, last_offset=last_offset)
+ draw.line((x, count_top, x, lane_bottom), fill="#e2e8f0", width=1)
+ draw.text((x, lane_bottom + 16), f"{seconds}s", font=_font(12), fill=_SLATE, anchor="ma")
+
+ active_count = 0
+ max_seen_active = 0
+ for _, delta in active_delta:
+ active_count += delta
+ max_seen_active = max(max_seen_active, active_count)
+ chart_max = max(1, max_seen_active)
+ for value in range(chart_max + 1):
+ y = count_bottom - round(value / chart_max * (count_height - 25))
+ draw.line((margin_left, y, width - margin_right, y), fill="#e2e8f0", width=1)
+ draw.text((margin_left - 10, y), str(value), font=_font(12), fill=_SLATE, anchor="rm")
+ draw.text(
+ (margin_left - 18, count_top + count_height // 2),
+ "active users",
+ font=_font(14, bold=True),
+ fill=_NAVY,
+ anchor="ms",
+ )
+
+ for record in batched:
+ start = float(record["model_started_unix_seconds"]) - origin_unix
+ end = float(record["model_completed_unix_seconds"]) - origin_unix
+ x0 = _time_to_x(start, left=margin_left, width=plot_width, last_offset=last_offset)
+ x1 = _time_to_x(end, left=margin_left, width=plot_width, last_offset=last_offset)
+ draw.rectangle(
+ (x0, count_top, max(x0 + 1, x1), count_bottom),
+ fill="#fef3c7",
+ outline=_BATCH_BORDER,
+ width=1,
+ )
+ draw.text((x0 + 3, count_top + 4), f"B={record['batch_size']}", font=_font(12, bold=True), fill=_BATCH_BORDER)
+
+ previous_time = 0.0
+ active_count = 0
+ previous_x = _time_to_x(previous_time, left=margin_left, width=plot_width, last_offset=last_offset)
+ previous_y = count_bottom
+ for timestamp, delta in active_delta:
+ x = _time_to_x(timestamp, left=margin_left, width=plot_width, last_offset=last_offset)
+ draw.line((previous_x, previous_y, x, previous_y), fill="#355c7d", width=3)
+ active_count += delta
+ y = count_bottom - round(active_count / chart_max * (count_height - 25))
+ draw.line((x, previous_y, x, y), fill="#355c7d", width=3)
+ previous_x = x
+ previous_y = y
+ draw.line((previous_x, previous_y, width - margin_right, previous_y), fill="#355c7d", width=3)
+
+ for session, lane in lane_index.items():
+ y = lane_top + lane * lane_height + lane_height // 2
+ if lane % 2 == 0:
+ draw.rectangle(
+ (margin_left, y - lane_height // 2, width - margin_right, y + lane_height // 2), fill="#ffffff"
+ )
+ draw.text((margin_left - 10, y), session, font=_font(9), fill=_SLATE, anchor="rm")
+ for start, end in intervals.get(session, []):
+ x0 = _time_to_x(float(start), left=margin_left, width=plot_width, last_offset=last_offset)
+ x1 = _time_to_x(float(end), left=margin_left, width=plot_width, last_offset=last_offset)
+ draw.line((x0, y, x1, y), fill=_INTERVAL, width=3)
+
+ for record in batched:
+ start = float(record["model_started_unix_seconds"]) - origin_unix
+ x0 = _time_to_x(start, left=margin_left, width=plot_width, last_offset=last_offset)
+ draw.line((x0, lane_top, x0, lane_bottom), fill=_BATCH_BORDER, width=2)
+
+ for timestamp, session, kind in onsets:
+ y = lane_top + lane_index[session] * lane_height + lane_height // 2
+ x = _time_to_x(timestamp, left=margin_left, width=plot_width, last_offset=last_offset)
+ if kind == "arrival_active":
+ _circle(draw, x, y, _ACTIVE)
+ else:
+ _triangle(draw, x, y, _RESUME)
+
+ draw.text(
+ (margin_left - 18, (lane_top + lane_bottom) // 2),
+ "logical user generation",
+ font=_font(14, bold=True),
+ fill=_NAVY,
+ anchor="ms",
+ )
+ draw.text(
+ (width // 2, height - 24), "seconds since workload start", font=_font(15, bold=True), fill=_NAVY, anchor="ms"
+ )
+
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ image.save(args.output_dir / "demand-scatter.png")
+ with (args.output_dir / "demand-events.csv").open("w", newline="", encoding="utf-8") as stream:
+ writer = csv.DictWriter(stream, fieldnames=["offset_seconds", "session", "event"])
+ writer.writeheader()
+ for timestamp, session, kind in onsets:
+ writer.writerow({"offset_seconds": f"{timestamp:.6f}", "session": session, "event": kind})
+ summary = {
+ "demand_onsets": len(onsets),
+ "arrival_active_onsets": sum(kind == "arrival_active" for _, _, kind in onsets),
+ "resume_onsets": sum(kind == "resumed" for _, _, kind in onsets),
+ "active_intervals": sum(len(value) for value in intervals.values()),
+ "batched_dispatches": len(batched),
+ }
+ (args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
+ print(json.dumps(summary, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/render_abot_dispatch_timeline.py b/tools/validation/render_abot_dispatch_timeline.py
new file mode 100644
index 00000000..43af30a6
--- /dev/null
+++ b/tools/validation/render_abot_dispatch_timeline.py
@@ -0,0 +1,1001 @@
+#!/usr/bin/env python3
+"""Render an evidence-preserving physical-GPU ABot dispatch timeline.
+
+The input is the parent-owned JSONL produced by
+``TELEFUSER_LIVEKIT_DISPATCH_TRACE_PATH``. Each line corresponds to one real
+``generate_next_block(s)`` invocation in a model worker, rather than a sampled
+Prometheus counter or an inferred batch. The renderer only reads saved
+artifacts; it never contacts the serving system or initializes CUDA.
+
+Example:
+
+ PYTHONPATH=$PWD python tools/validation/render_abot_dispatch_timeline.py \
+ --dispatch-trace results/experiments/run/dispatch-trace.jsonl \
+ --result results/experiments/run/result.json \
+ --output-dir results/experiments/run/dispatch-analysis
+
+Outputs:
+
+* ``dispatch-timeline.png``: physical-GPU timeline with workload-phase bands
+ and a labelled zoom;
+* ``dispatches.csv``: one human-readable row per actual model dispatch;
+* ``stage-projections.csv``: the stage-duration partition used for the inner
+ rectangle strips (explicitly a visual projection, not kernel timestamps);
+* ``phase-summary.csv``: dispatch/batch accounting aligned to workload phases;
+* ``summary.json`` and ``summary.md``: physical-GPU accounting and provenance.
+
+The outer rectangles use the measured host wall-clock interval from model
+dispatch start through completion. The narrow coloured stage strip inside a
+rectangle is a sequential projection of CUDA-measured DiT/LightVAE stage
+durations into that wall-clock interval. It is deliberately not presented as
+an Nsight kernel trace; the raw JSONL remains the source of truth.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+import statistics
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+from PIL import Image, ImageDraw, ImageFont
+
+_NAVY = "#0f172a"
+_SLATE = "#475569"
+_GRID = "#cbd5e1"
+_PANEL = "#f8fafc"
+_WHITE = "#ffffff"
+_BATCH_COLORS = {1: "#2563eb", 2: "#059669", 3: "#d97706", 4: "#dc2626"}
+_PHASE_COLORS = (
+ "#dbeafe",
+ "#dcfce7",
+ "#fef3c7",
+ "#fee2e2",
+ "#f3e8ff",
+ "#cffafe",
+ "#fae8ff",
+ "#ffedd5",
+ "#e0e7ff",
+ "#ecfccb",
+)
+_STAGE_COLORS = {
+ "input_prepare": "#94a3b8",
+ "cache_collate": "#7c3aed",
+ "denoise": "#0ea5e9",
+ "cache_scatter": "#a855f7",
+ "vae_decode": "#f97316",
+ "postprocess": "#64748b",
+}
+_STAGE_LABELS = {
+ "input_prepare": "input",
+ "cache_collate": "KV collect",
+ "denoise": "DiT",
+ "cache_scatter": "KV scatter",
+ "vae_decode": "LightVAE",
+ "postprocess": "post",
+}
+_STAGE_KEYS = tuple(_STAGE_COLORS)
+
+
+@dataclass(frozen=True)
+class Dispatch:
+ """One completed or failed real model invocation."""
+
+ sequence: int
+ worker_id: str
+ configured_gpu_id: str
+ gpu_id: str
+ logical_cuda_device: str
+ selected: float
+ started: float
+ completed: float
+ started_unix: float | None
+ duration: float
+ batch_size: int
+ control_latent_frames: int
+ sessions: tuple[dict[str, Any], ...]
+ stages: dict[str, float]
+ vae_mode: str
+ vae_effective_batch_size: int
+ vae_invocations: int
+ outcome: str
+ error: str | None
+
+
+@dataclass(frozen=True)
+class WorkloadPhase:
+ """A benchmark phase mapped onto the dispatch trace clock."""
+
+ index: int
+ name: str
+ started: float
+ completed: float
+ target_users: int | None
+ active_input_fraction: float | None
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--dispatch-trace", type=Path, required=True, help="Parent-owned dispatch-trace.jsonl.")
+ parser.add_argument(
+ "--result",
+ type=Path,
+ help="Optional black-box result.json; maps opaque session IDs to wave-xxx users.",
+ )
+ parser.add_argument("--output-dir", type=Path, required=True)
+ parser.add_argument(
+ "--zoom-seconds",
+ type=float,
+ default=12.0,
+ help="Duration of auto-selected dense labelled window (default: 12).",
+ )
+ parser.add_argument(
+ "--zoom-start-seconds",
+ type=float,
+ help="Optional time relative to first dispatch, overriding automatic window selection.",
+ )
+ return parser.parse_args()
+
+
+def _number(value: object, default: float = 0.0) -> float:
+ return float(value) if isinstance(value, int | float) and not isinstance(value, bool) else default
+
+
+def _finite_or_none(value: object) -> float | None:
+ candidate = _number(value, math.nan)
+ return candidate if math.isfinite(candidate) else None
+
+
+def _integer(value: object, default: int = 0) -> int:
+ return int(value) if isinstance(value, int) and not isinstance(value, bool) else default
+
+
+def _short_user(value: str) -> str:
+ if value.startswith("wave-"):
+ return "u" + value.removeprefix("wave-")
+ # Public TurboServe-derived lifecycle trace IDs are stable source-session
+ # IDs plus a generation (for example ``ts-00079-g02``). Keeping the
+ # generation avoids visually merging a departed/re-arrived user in the
+ # labelled 12-second zoom of a 30-minute replay.
+ parts = value.split("-")
+ if len(parts) == 3 and parts[0] == "ts" and parts[1].isdigit() and parts[2].startswith("g"):
+ source = str(int(parts[1]))
+ generation = parts[2].removeprefix("g")
+ return f"u{source}g{generation}"
+ return value[:8]
+
+
+def _font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont:
+ return ImageFont.truetype("DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf", size=size)
+
+
+def _text(
+ draw: ImageDraw.ImageDraw,
+ xy: tuple[float, float],
+ text: str,
+ *,
+ size: int = 18,
+ fill: str = _NAVY,
+ bold: bool = False,
+ anchor: str | None = None,
+) -> None:
+ draw.text(xy, text, font=_font(size, bold=bold), fill=fill, anchor=anchor)
+
+
+def _load_dispatches(path: Path) -> list[Dispatch]:
+ source = path.expanduser().resolve()
+ if not source.is_file():
+ raise ValueError(f"missing dispatch trace: {source}")
+ dispatches: list[Dispatch] = []
+ for line_number, raw in enumerate(source.read_text(encoding="utf-8").splitlines(), start=1):
+ if not raw.strip():
+ continue
+ try:
+ event = json.loads(raw)
+ except json.JSONDecodeError as exc:
+ raise ValueError(f"invalid JSON at {source}:{line_number}: {exc}") from exc
+ if not isinstance(event, dict) or event.get("event_type") != "model_dispatch":
+ continue
+ sessions = event.get("sessions", [])
+ if not isinstance(sessions, list) or not all(isinstance(item, dict) for item in sessions):
+ raise ValueError(f"model_dispatch at line {line_number} has invalid sessions")
+ started = _number(event.get("model_started_monotonic_seconds"), math.nan)
+ completed = _number(event.get("model_completed_monotonic_seconds"), math.nan)
+ duration = _number(event.get("model_duration_seconds"), math.nan)
+ if not all(math.isfinite(item) for item in (started, completed, duration)) or completed < started:
+ raise ValueError(f"model_dispatch at line {line_number} has invalid timing")
+ stages_raw = event.get("stages_seconds", {})
+ stages = (
+ {key: max(0.0, _number(stages_raw.get(key))) for key in _STAGE_KEYS}
+ if isinstance(stages_raw, dict)
+ else {key: 0.0 for key in _STAGE_KEYS}
+ )
+ gpu = event.get("gpu", {})
+ configured_gpu_id = "unknown"
+ gpu_id = "unknown"
+ logical_cuda_device = "unknown"
+ if isinstance(gpu, dict):
+ configured = gpu.get("configured_gpu_id")
+ physical = gpu.get("physical_gpu_id")
+ logical = gpu.get("logical_cuda_device", gpu.get("cuda_device_index"))
+ configured_gpu_id = str(configured) if configured is not None else "unknown"
+ logical_cuda_device = str(logical) if logical is not None else "unknown"
+ gpu_id = str(
+ physical if physical is not None else configured if configured is not None else logical_cuda_device
+ )
+ vae = event.get("vae_decode", {})
+ if not isinstance(vae, dict):
+ vae = {}
+ dispatches.append(
+ Dispatch(
+ sequence=_integer(event.get("parent_sequence"), line_number),
+ worker_id=str(event.get("worker_id", "unknown")),
+ configured_gpu_id=configured_gpu_id,
+ gpu_id=gpu_id,
+ logical_cuda_device=logical_cuda_device,
+ selected=_number(event.get("selected_monotonic_seconds"), started),
+ started=started,
+ completed=completed,
+ started_unix=_finite_or_none(event.get("model_started_unix_seconds")),
+ duration=max(0.0, duration),
+ batch_size=max(1, _integer(event.get("batch_size"), len(sessions) or 1)),
+ control_latent_frames=_integer(event.get("control_latent_frames"), 0),
+ sessions=tuple(dict(item) for item in sessions),
+ stages=stages,
+ vae_mode=str(vae.get("mode_name", vae.get("mode", "unknown"))),
+ vae_effective_batch_size=_integer(vae.get("effective_batch_size"), 0),
+ vae_invocations=_integer(vae.get("invocations"), 0),
+ outcome=str(event.get("outcome", "unknown")),
+ error=str(event["error"]) if event.get("error") is not None else None,
+ )
+ )
+ if not dispatches:
+ raise ValueError(f"no model_dispatch events in {source}")
+ return sorted(dispatches, key=lambda item: (item.started, item.sequence))
+
+
+def _session_labels(result_path: Path | None) -> dict[str, str]:
+ if result_path is None:
+ return {}
+ result = json.loads(result_path.expanduser().resolve().read_text(encoding="utf-8"))
+ events = result.get("events", []) if isinstance(result, dict) else []
+ labels: dict[str, str] = {}
+ if not isinstance(events, list):
+ return labels
+ for event in events:
+ if not isinstance(event, dict) or event.get("event") != "session_created":
+ continue
+ session_id = event.get("server_session_id")
+ user = event.get("session")
+ if isinstance(session_id, str) and isinstance(user, str):
+ labels[session_id] = user
+ return labels
+
+
+def _optional_int(value: object) -> int | None:
+ return _integer(value) if isinstance(value, int) and not isinstance(value, bool) else None
+
+
+def _optional_float(value: object) -> float | None:
+ return _number(value, math.nan) if math.isfinite(_number(value, math.nan)) else None
+
+
+def _load_workload_phases(
+ result_path: Path | None,
+ *,
+ dispatch_origin_unix: float | None,
+) -> list[WorkloadPhase]:
+ """Map measured benchmark phase boundaries onto the dispatch trace clock.
+
+ The benchmark records phase offsets from its own Unix-clock origin, while
+ dispatch JSONL records model-start Unix timestamps. This is an exact clock
+ conversion when both are present; no sampled counter alignment is used.
+ """
+
+ if result_path is None or dispatch_origin_unix is None:
+ return []
+ payload = json.loads(result_path.expanduser().resolve().read_text(encoding="utf-8"))
+ if not isinstance(payload, dict):
+ return []
+ trace_clock = payload.get("trace_clock")
+ origin_unix = None
+ if isinstance(trace_clock, dict):
+ origin_unix = _optional_float(trace_clock.get("origin_unix_seconds"))
+ if origin_unix is None:
+ origin_unix = _optional_float(payload.get("started_at_unix_seconds"))
+ phase_results = payload.get("phase_results")
+ if origin_unix is None or not isinstance(phase_results, list):
+ return []
+
+ scenario = payload.get("scenario")
+ declared = scenario.get("phases", []) if isinstance(scenario, dict) else []
+ declared_by_name = {
+ str(item.get("name")): item for item in declared if isinstance(item, dict) and isinstance(item.get("name"), str)
+ }
+ phases: list[WorkloadPhase] = []
+ for index, raw in enumerate(phase_results, start=1):
+ if not isinstance(raw, dict) or not isinstance(raw.get("phase"), str):
+ continue
+ start_offset = _optional_float(raw.get("started_offset_seconds"))
+ completed_offset = _optional_float(raw.get("completed_offset_seconds"))
+ if start_offset is None or completed_offset is None or completed_offset < start_offset:
+ continue
+ name = str(raw["phase"])
+ declared_phase = declared_by_name.get(name, {})
+ phase_summary = raw.get("summary")
+ if not isinstance(phase_summary, dict):
+ phase_summary = {}
+ target_users = _optional_int(phase_summary.get("target_users"))
+ if target_users is None and isinstance(declared_phase, dict):
+ target_users = _optional_int(declared_phase.get("target_users"))
+ active_input_fraction = None
+ if isinstance(declared_phase, dict):
+ active_input_fraction = _optional_float(declared_phase.get("active_input_fraction"))
+ phases.append(
+ WorkloadPhase(
+ index=index,
+ name=name,
+ started=origin_unix + start_offset - dispatch_origin_unix,
+ completed=origin_unix + completed_offset - dispatch_origin_unix,
+ target_users=target_users,
+ active_input_fraction=active_input_fraction,
+ )
+ )
+ return phases
+
+
+def _phase_name_at(phases: Iterable[WorkloadPhase], value: float) -> str:
+ for phase in phases:
+ if phase.started <= value <= phase.completed:
+ return phase.name
+ return ""
+
+
+def _relative(dispatches: Iterable[Dispatch], origin: float) -> list[Dispatch]:
+ return [
+ Dispatch(
+ **{
+ **item.__dict__,
+ "selected": item.selected - origin,
+ "started": item.started - origin,
+ "completed": item.completed - origin,
+ }
+ )
+ for item in dispatches
+ ]
+
+
+def _gpu_sort_key(value: str) -> tuple[int, int, str]:
+ return (0, int(value), value) if value.isdigit() else (1, 0, value)
+
+
+def _worker_order(dispatches: Iterable[Dispatch]) -> list[str]:
+ return sorted({item.gpu_id for item in dispatches}, key=_gpu_sort_key)
+
+
+def _percentile(values: list[float], percentile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * percentile) - 1))
+ return ordered[index]
+
+
+def _auto_zoom_start(dispatches: list[Dispatch], window: float) -> float:
+ if not dispatches:
+ return 0.0
+ candidates = sorted({max(0.0, item.started) for item in dispatches})
+ best_start = candidates[0]
+ best_score = (-1, -1, -1, -1.0)
+ for start in candidates:
+ end = start + window
+ selected = [item for item in dispatches if item.completed >= start and item.started <= end]
+ score = (
+ sum(item.batch_size > 1 for item in selected),
+ sum(max(0, item.batch_size - 1) for item in selected),
+ len(selected),
+ sum(min(item.completed, end) - max(item.started, start) for item in selected),
+ )
+ if score > best_score:
+ best_score = score
+ best_start = start
+ return best_start
+
+
+def _interval_union_seconds(items: Iterable[Dispatch]) -> float:
+ """Return physical-GPU busy time without double-counting overlaps."""
+
+ intervals = sorted((item.started, item.completed) for item in items)
+ if not intervals:
+ return 0.0
+ total = 0.0
+ left, right = intervals[0]
+ for next_left, next_right in intervals[1:]:
+ if next_left <= right:
+ right = max(right, next_right)
+ continue
+ total += right - left
+ left, right = next_left, next_right
+ return total + right - left
+
+
+def _worker_summary(dispatches: list[Dispatch], workers: list[str], span: float) -> list[dict[str, Any]]:
+ """Summarize one physical GPU lane per row.
+
+ The legacy parameter name workers now contains physical GPU IDs.
+ """
+
+ grouped: dict[str, list[Dispatch]] = defaultdict(list)
+ for dispatch in dispatches:
+ grouped[dispatch.gpu_id].append(dispatch)
+ rows: list[dict[str, Any]] = []
+ for gpu_id in workers:
+ items = grouped[gpu_id]
+ batch_counts = Counter(item.batch_size for item in items)
+ raw_busy = sum(item.duration for item in items)
+ busy = _interval_union_seconds(items)
+ rows.append(
+ {
+ "gpu_id": gpu_id,
+ "worker_ids": sorted({item.worker_id for item in items}),
+ "configured_gpu_ids": sorted({item.configured_gpu_id for item in items}),
+ "logical_cuda_devices": sorted({item.logical_cuda_device for item in items}),
+ "dispatches": len(items),
+ "busy_seconds": round(busy, 6),
+ "overlap_seconds": round(max(0.0, raw_busy - busy), 6),
+ "busy_fraction_of_trace": round(busy / span, 6) if span > 0 else 0.0,
+ "mean_duration_seconds": round(statistics.fmean(item.duration for item in items), 6) if items else 0.0,
+ "p95_duration_seconds": round(_percentile([item.duration for item in items], 0.95), 6),
+ "batches_by_size": {str(size): batch_counts[size] for size in sorted(batch_counts)},
+ "batch_items": sum(item.batch_size for item in items),
+ }
+ )
+ return rows
+
+
+def _x(value: float, start: float, end: float, x0: int, x1: int) -> int:
+ if end <= start:
+ return x0
+ return x0 + round((max(start, min(end, value)) - start) / (end - start) * (x1 - x0))
+
+
+def _draw_axis(
+ draw: ImageDraw.ImageDraw,
+ *,
+ x0: int,
+ x1: int,
+ y0: int,
+ y1: int,
+ start: float,
+ end: float,
+ ticks: int,
+) -> None:
+ for index in range(ticks + 1):
+ value = start + (end - start) * index / ticks
+ px = _x(value, start, end, x0, x1)
+ draw.line((px, y0, px, y1), fill=_GRID, width=1)
+ _text(draw, (px, y1 + 8), f"{value:.0f}s", size=14, fill=_SLATE, anchor="ma")
+
+
+def _session_text(dispatch: Dispatch, labels: dict[str, str]) -> str:
+ members: list[str] = []
+ for session in dispatch.sessions:
+ session_id = str(session.get("session_id", "?"))
+ user = _short_user(labels.get(session_id, session_id))
+ chunk_index = _integer(session.get("chunk_index"), -1)
+ members.append(f"{user}@c{chunk_index}")
+ return f"B{dispatch.batch_size}\n" + "+".join(members)
+
+
+def _draw_stage_strip(
+ draw: ImageDraw.ImageDraw,
+ *,
+ dispatch: Dispatch,
+ box: tuple[int, int, int, int],
+) -> None:
+ x0, y0, x1, y1 = box
+ width = max(1, x1 - x0)
+ total = sum(dispatch.stages.values())
+ if total <= 0:
+ draw.rectangle((x0, y0, x1, y1), fill=_SLATE)
+ return
+ cursor = x0
+ for index, key in enumerate(_STAGE_KEYS):
+ seconds = dispatch.stages.get(key, 0.0)
+ fraction = seconds / total
+ right = x1 if index == len(_STAGE_KEYS) - 1 else min(x1, cursor + max(1, round(width * fraction)))
+ if right > cursor:
+ draw.rectangle((cursor, y0, right, y1), fill=_STAGE_COLORS[key])
+ cursor = right
+
+
+def _draw_phase_bands(
+ draw: ImageDraw.ImageDraw,
+ *,
+ phases: Iterable[WorkloadPhase],
+ x0: int,
+ x1: int,
+ y0: int,
+ y1: int,
+ start: float,
+ end: float,
+) -> None:
+ """Draw a narrow workload-phase band on the same clock as dispatches."""
+
+ for phase in phases:
+ if phase.completed < start or phase.started > end:
+ continue
+ left = _x(phase.started, start, end, x0, x1)
+ right = max(left + 1, _x(phase.completed, start, end, x0, x1))
+ color = _PHASE_COLORS[(phase.index - 1) % len(_PHASE_COLORS)]
+ draw.rectangle((left, y0, right, y1), fill=color)
+ draw.line((left, y0, left, y1), fill=_SLATE, width=1)
+ if right - left >= 30:
+ _text(
+ draw,
+ ((left + right) // 2, (y0 + y1) // 2),
+ f"P{phase.index}",
+ size=13,
+ fill=_NAVY,
+ bold=True,
+ anchor="mm",
+ )
+
+
+def _draw_timeline(
+ dispatches: list[Dispatch],
+ labels: dict[str, str],
+ path: Path,
+ phases: list[WorkloadPhase],
+ *,
+ zoom_start: float,
+ zoom_seconds: float,
+) -> None:
+ workers = _worker_order(dispatches)
+ trace_end = max(item.completed for item in dispatches)
+ width, height = 2600, 1720
+ image = Image.new("RGB", (width, height), _WHITE)
+ draw = ImageDraw.Draw(image)
+ _text(draw, (72, 48), "ABot-World real physical-GPU dispatch timeline", size=38, bold=True)
+ _text(
+ draw,
+ (72, 96),
+ "One rectangle = one real generate_next_block(s) invocation; time origin is the first model dispatch.",
+ size=19,
+ fill=_SLATE,
+ )
+ _text(
+ draw,
+ (72, 124),
+ "Outline colour: batch size. Inner strip: input / KV / DiT / LightVAE / postprocess durations "
+ "projected into dispatch wall time.",
+ size=17,
+ fill=_SLATE,
+ )
+
+ overview = (62, 178, width - 62, 668)
+ draw.rounded_rectangle(overview, radius=18, fill=_PANEL, outline=_GRID, width=2)
+ _text(draw, (overview[0] + 24, overview[1] + 18), f"Full trace — 0 to {trace_end:.1f}s", size=26, bold=True)
+ ov_x0, ov_x1 = overview[0] + 178, overview[2] - 28
+ ov_phase_y0, ov_phase_y1 = overview[1] + 62, overview[1] + 82
+ ov_y0, ov_y1 = overview[1] + 98, overview[3] - 58
+ _draw_phase_bands(
+ draw, phases=phases, x0=ov_x0, x1=ov_x1, y0=ov_phase_y0, y1=ov_phase_y1, start=0.0, end=max(1.0, trace_end)
+ )
+ _draw_axis(draw, x0=ov_x0, x1=ov_x1, y0=ov_y0, y1=ov_y1, start=0.0, end=max(1.0, trace_end), ticks=10)
+ lane_height = max(64, (ov_y1 - ov_y0) // max(1, len(workers)))
+ for index, worker in enumerate(workers):
+ lane_top = ov_y0 + index * lane_height + 12
+ lane_bottom = min(ov_y1 - 6, lane_top + lane_height - 24)
+ lane_workers = ",".join(sorted({item.worker_id for item in dispatches if item.gpu_id == worker}))
+ _text(
+ draw,
+ (overview[0] + 22, (lane_top + lane_bottom) // 2),
+ f"GPU {worker}\n{lane_workers}",
+ size=16,
+ fill=_SLATE,
+ anchor="lm",
+ )
+ draw.line((ov_x0, lane_bottom + 8, ov_x1, lane_bottom + 8), fill=_GRID, width=1)
+ for item in dispatches:
+ if item.gpu_id != worker:
+ continue
+ left = _x(item.started, 0.0, max(1.0, trace_end), ov_x0, ov_x1)
+ right = max(left + 2, _x(item.completed, 0.0, max(1.0, trace_end), ov_x0, ov_x1))
+ color = _BATCH_COLORS.get(item.batch_size, "#7c3aed")
+ draw.rounded_rectangle((left, lane_top, right, lane_bottom), radius=4, fill=color)
+
+ zoom_end = min(trace_end, zoom_start + zoom_seconds)
+ if zoom_end <= zoom_start:
+ zoom_start, zoom_end = max(0.0, trace_end - zoom_seconds), trace_end
+ panel = (62, 720, width - 62, height - 66)
+ draw.rounded_rectangle(panel, radius=18, fill=_PANEL, outline=_GRID, width=2)
+ _text(
+ draw,
+ (panel[0] + 24, panel[1] + 18),
+ f"Labelled dense window — {zoom_start:.2f}s to {zoom_end:.2f}s",
+ size=26,
+ bold=True,
+ )
+ _text(
+ draw,
+ (panel[0] + 24, panel[1] + 52),
+ "Each label is B + user/session alias + chunk index; no inferred batches.",
+ size=17,
+ fill=_SLATE,
+ )
+ z_x0, z_x1 = panel[0] + 178, panel[2] - 28
+ z_phase_y0, z_phase_y1 = panel[1] + 78, panel[1] + 98
+ z_y0, z_y1 = panel[1] + 116, panel[3] - 76
+ _draw_phase_bands(
+ draw, phases=phases, x0=z_x0, x1=z_x1, y0=z_phase_y0, y1=z_phase_y1, start=zoom_start, end=zoom_end
+ )
+ _draw_axis(draw, x0=z_x0, x1=z_x1, y0=z_y0, y1=z_y1, start=zoom_start, end=zoom_end, ticks=12)
+ lane_height = max(95, (z_y1 - z_y0) // max(1, len(workers)))
+ for index, worker in enumerate(workers):
+ lane_top = z_y0 + index * lane_height + 16
+ lane_bottom = min(z_y1 - 8, lane_top + lane_height - 30)
+ lane_workers = ",".join(sorted({item.worker_id for item in dispatches if item.gpu_id == worker}))
+ _text(
+ draw,
+ (panel[0] + 22, (lane_top + lane_bottom) // 2),
+ f"GPU {worker}\n{lane_workers}",
+ size=18,
+ fill=_SLATE,
+ anchor="lm",
+ )
+ draw.line((z_x0, lane_bottom + 10, z_x1, lane_bottom + 10), fill=_GRID, width=1)
+ for item in dispatches:
+ if item.gpu_id != worker or item.completed < zoom_start or item.started > zoom_end:
+ continue
+ left = _x(item.started, zoom_start, zoom_end, z_x0, z_x1)
+ right = max(left + 3, _x(item.completed, zoom_start, zoom_end, z_x0, z_x1))
+ color = _BATCH_COLORS.get(item.batch_size, "#7c3aed")
+ draw.rounded_rectangle((left, lane_top, right, lane_bottom), radius=7, fill=color, outline=_NAVY, width=1)
+ strip_top = max(lane_top + 4, lane_bottom - 12)
+ _draw_stage_strip(draw, dispatch=item, box=(left + 1, strip_top, right - 1, lane_bottom - 2))
+ if right - left >= 42:
+ label = _session_text(item, labels)
+ label_size = 13 if right - left < 90 else 15
+ _text(draw, (left + 4, lane_top + 5), label, size=label_size, fill=_WHITE, bold=True)
+
+ legend_x = 86
+ legend_y = height - 38
+ for batch_size in (1, 2, 3, 4):
+ draw.rounded_rectangle(
+ (legend_x, legend_y - 12, legend_x + 22, legend_y + 10), radius=4, fill=_BATCH_COLORS[batch_size]
+ )
+ _text(draw, (legend_x + 29, legend_y - 11), f"B{batch_size}", size=15, fill=_SLATE)
+ legend_x += 84
+ _text(draw, (legend_x + 6, legend_y - 11), "stage strip:", size=15, fill=_SLATE)
+ legend_x += 112
+ for key in _STAGE_KEYS:
+ draw.rectangle((legend_x, legend_y - 12, legend_x + 18, legend_y + 10), fill=_STAGE_COLORS[key])
+ _text(draw, (legend_x + 24, legend_y - 11), _STAGE_LABELS[key], size=14, fill=_SLATE)
+ legend_x += 24 + int(draw.textlength(_STAGE_LABELS[key], font=_font(14))) + 24
+ image.save(path)
+
+
+def _write_csv(path: Path, dispatches: list[Dispatch], labels: dict[str, str], phases: list[WorkloadPhase]) -> None:
+ fields = (
+ "sequence",
+ "worker_id",
+ "gpu_id",
+ "configured_gpu_id",
+ "logical_cuda_device",
+ "workload_phase",
+ "start_seconds",
+ "end_seconds",
+ "duration_seconds",
+ "batch_size",
+ "control_latent_frames",
+ "users",
+ "session_ids",
+ "chunk_indexes",
+ "frame_positions_before",
+ "frame_positions_after",
+ "denoise_seconds",
+ "vae_decode_seconds",
+ "postprocess_seconds",
+ "vae_mode",
+ "vae_effective_batch_size",
+ "vae_invocations",
+ "outcome",
+ "error",
+ )
+ with path.open("w", newline="", encoding="utf-8") as output:
+ writer = csv.DictWriter(output, fieldnames=fields)
+ writer.writeheader()
+ for item in dispatches:
+ session_ids = [str(session.get("session_id", "")) for session in item.sessions]
+ phase_name = _phase_name_at(phases, item.started)
+ writer.writerow(
+ {
+ "sequence": item.sequence,
+ "worker_id": item.worker_id,
+ "gpu_id": item.gpu_id,
+ "configured_gpu_id": item.configured_gpu_id,
+ "logical_cuda_device": item.logical_cuda_device,
+ "workload_phase": phase_name,
+ "start_seconds": f"{item.started:.6f}",
+ "end_seconds": f"{item.completed:.6f}",
+ "duration_seconds": f"{item.duration:.6f}",
+ "batch_size": item.batch_size,
+ "control_latent_frames": item.control_latent_frames,
+ "users": "+".join(labels.get(value, value) for value in session_ids),
+ "session_ids": "+".join(session_ids),
+ "chunk_indexes": "+".join(
+ str(_integer(session.get("chunk_index"), -1)) for session in item.sessions
+ ),
+ "frame_positions_before": "+".join(
+ str(_integer(session.get("next_latent_frame_before"), -1)) for session in item.sessions
+ ),
+ "frame_positions_after": "+".join(
+ str(_integer(session.get("next_latent_frame_after"), -1)) for session in item.sessions
+ ),
+ "denoise_seconds": f"{item.stages.get('denoise', 0.0):.6f}",
+ "vae_decode_seconds": f"{item.stages.get('vae_decode', 0.0):.6f}",
+ "postprocess_seconds": f"{item.stages.get('postprocess', 0.0):.6f}",
+ "vae_mode": item.vae_mode,
+ "vae_effective_batch_size": item.vae_effective_batch_size,
+ "vae_invocations": item.vae_invocations,
+ "outcome": item.outcome,
+ "error": item.error or "",
+ }
+ )
+
+
+def _phase_summary_rows(
+ phases: Iterable[WorkloadPhase],
+ dispatches: Iterable[Dispatch],
+) -> list[dict[str, Any]]:
+ dispatch_list = list(dispatches)
+ rows: list[dict[str, Any]] = []
+ for phase in phases:
+ # Attribute a dispatch to the phase in which its model invocation starts.
+ # This makes phase batch counts mutually exclusive and sum to the global
+ # histogram, unlike interval-overlap accounting at a phase boundary.
+ items = [item for item in dispatch_list if phase.started <= item.started <= phase.completed]
+ histogram = Counter(item.batch_size for item in items)
+ executions = len(items)
+ batch_items = sum(item.batch_size for item in items)
+ rows.append(
+ {
+ "phase_id": f"P{phase.index}",
+ "phase": phase.name,
+ "start_seconds": round(phase.started, 6),
+ "end_seconds": round(phase.completed, 6),
+ "duration_seconds": round(max(0.0, phase.completed - phase.started), 6),
+ "target_users": phase.target_users,
+ "active_input_fraction": phase.active_input_fraction,
+ "dispatches": executions,
+ "batch_items": batch_items,
+ "mean_batch_size": round(batch_items / executions, 6) if executions else 0.0,
+ "batches_b1": histogram[1],
+ "batches_b2": histogram[2],
+ "batches_b3": histogram[3],
+ "batches_b4": histogram[4],
+ }
+ )
+ return rows
+
+
+def _write_phase_csv(path: Path, rows: Iterable[dict[str, Any]]) -> None:
+ fields = (
+ "phase_id",
+ "phase",
+ "start_seconds",
+ "end_seconds",
+ "duration_seconds",
+ "target_users",
+ "active_input_fraction",
+ "dispatches",
+ "batch_items",
+ "mean_batch_size",
+ "batches_b1",
+ "batches_b2",
+ "batches_b3",
+ "batches_b4",
+ )
+ with path.open("w", newline="", encoding="utf-8") as output:
+ writer = csv.DictWriter(output, fieldnames=fields)
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def _write_stage_projection_csv(path: Path, dispatches: Iterable[Dispatch], labels: dict[str, str]) -> None:
+ """Write the exact stage-strip partition used in the PNG.
+
+ CUDA event durations are real measured durations, but the start/end columns
+ are scaled into the enclosing host dispatch span solely for visualization.
+ """
+
+ fields = (
+ "sequence",
+ "physical_gpu_id",
+ "worker_id",
+ "batch_size",
+ "users",
+ "chunk_indexes",
+ "stage",
+ "measured_stage_seconds",
+ "projected_start_seconds",
+ "projected_end_seconds",
+ "projection_note",
+ )
+ with path.open("w", newline="", encoding="utf-8") as output:
+ writer = csv.DictWriter(output, fieldnames=fields)
+ writer.writeheader()
+ for dispatch in dispatches:
+ total = sum(dispatch.stages.values())
+ scale = dispatch.duration / total if total > 0 else 0.0
+ cursor = dispatch.started
+ session_ids = [str(session.get("session_id", "")) for session in dispatch.sessions]
+ users = "+".join(labels.get(value, value) for value in session_ids)
+ chunks = "+".join(str(_integer(session.get("chunk_index"), -1)) for session in dispatch.sessions)
+ for index, stage in enumerate(_STAGE_KEYS):
+ measured = dispatch.stages.get(stage, 0.0)
+ projected_end = dispatch.completed if index == len(_STAGE_KEYS) - 1 else cursor + measured * scale
+ writer.writerow(
+ {
+ "sequence": dispatch.sequence,
+ "physical_gpu_id": dispatch.gpu_id,
+ "worker_id": dispatch.worker_id,
+ "batch_size": dispatch.batch_size,
+ "users": users,
+ "chunk_indexes": chunks,
+ "stage": stage,
+ "measured_stage_seconds": f"{measured:.9f}",
+ "projected_start_seconds": f"{cursor:.9f}",
+ "projected_end_seconds": f"{projected_end:.9f}",
+ "projection_note": "scaled into model dispatch wall interval; not kernel timestamps",
+ }
+ )
+ cursor = projected_end
+
+
+def _write_summary(
+ path: Path,
+ *,
+ source: Path,
+ rows: list[dict[str, Any]],
+ phase_rows: list[dict[str, Any]],
+ dispatches: list[Dispatch],
+ span: float,
+ zoom_start: float,
+ zoom_seconds: float,
+) -> None:
+ batch_histogram = Counter(item.batch_size for item in dispatches)
+ payload = {
+ "schema_version": 1,
+ "source_dispatch_trace": str(source.resolve()),
+ "time_origin": "first model_started_monotonic_seconds in the input trace",
+ "trace_span_seconds": round(span, 6),
+ "dispatches": len(dispatches),
+ "batch_histogram": {str(key): batch_histogram[key] for key in sorted(batch_histogram)},
+ "physical_gpus": rows,
+ "workload_phases": phase_rows,
+ "labelled_zoom": {"start_seconds": round(zoom_start, 6), "duration_seconds": round(zoom_seconds, 6)},
+ "interpretation": {
+ "rectangle": "Measured model dispatch host wall-clock start-to-completion interval.",
+ "stage_strip": "Sequential visual projection of measured stage durations; not an Nsight kernel trace.",
+ },
+ }
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _write_markdown(path: Path, *, summary: dict[str, Any], output_dir: Path) -> None:
+ rows = summary["physical_gpus"]
+ lines = [
+ "# Physical-GPU ABot dispatch timeline",
+ "",
+ "This report is computed solely from the real parent-owned dispatch JSONL.",
+ "Each row is one model invocation, not a sampled counter.",
+ "",
+ f"- Trace span: {summary['trace_span_seconds']:.3f} s",
+ f"- Dispatches: {summary['dispatches']}",
+ f"- Batch histogram: {summary['batch_histogram']}",
+ "",
+ "| Physical GPU | Workers | Dispatches | Busy time | Busy fraction | Batches by size |",
+ "| --- | --- | ---: | ---: | ---: | --- |",
+ ]
+ for row in rows:
+ lines.append(
+ f"| {row['gpu_id']} | {', '.join(row['worker_ids'])} | {row['dispatches']} | "
+ f"{row['busy_seconds']:.3f}s | {100 * row['busy_fraction_of_trace']:.1f}% | "
+ f"{row['batches_by_size']} |"
+ )
+ phase_rows = summary["workload_phases"]
+ if phase_rows:
+ lines.extend(
+ [
+ "",
+ "## Workload phases aligned to the dispatch clock",
+ "",
+ "| ID | Phase | Time | Users | Active input | Dispatches | B1 | B2 |",
+ "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: |",
+ ]
+ )
+ for row in phase_rows:
+ input_fraction = row["active_input_fraction"]
+ input_text = f"{100 * input_fraction:.0f}%" if input_fraction is not None else "—"
+ lines.append(
+ f"| {row['phase_id']} | {row['phase']} | {row['start_seconds']:.1f}–{row['end_seconds']:.1f}s | "
+ f"{row['target_users'] if row['target_users'] is not None else '—'} | {input_text} | "
+ f"{row['dispatches']} | {row['batches_b1']} | {row['batches_b2']} |"
+ )
+ lines.extend(
+ [
+ "",
+ "Artifacts:",
+ "",
+ "- `dispatch-timeline.png`: physical-GPU lanes, phase bands, and labelled zoom.",
+ "- `dispatches.csv`: raw dispatch rows with workload user/phase mapping.",
+ "- `phase-summary.csv`: phase-aligned batch distribution.",
+ "- `stage-projections.csv`: visual stage-strip partition (not kernel timestamps).",
+ "",
+ "The coloured stage strip is a visual projection of measured stage durations within the actual "
+ "wall-clock dispatch interval; it is not a kernel-level Nsight timeline.",
+ ]
+ )
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+ args = _parse_args()
+ if args.zoom_seconds <= 0:
+ raise ValueError("--zoom-seconds must be positive")
+ source = args.dispatch_trace.expanduser().resolve()
+ raw_dispatches = _load_dispatches(source)
+ origin = min(item.started for item in raw_dispatches)
+ dispatches = _relative(raw_dispatches, origin)
+ dispatch_origin_unix = raw_dispatches[0].started_unix
+ phases = _load_workload_phases(args.result, dispatch_origin_unix=dispatch_origin_unix)
+ phase_rows = _phase_summary_rows(phases, dispatches)
+ labels = _session_labels(args.result)
+ span = max(item.completed for item in dispatches)
+ zoom_start = args.zoom_start_seconds
+ if zoom_start is None:
+ zoom_start = _auto_zoom_start(dispatches, args.zoom_seconds)
+ zoom_start = max(0.0, min(float(zoom_start), max(0.0, span - min(args.zoom_seconds, span))))
+ output_dir = args.output_dir.expanduser().resolve()
+ output_dir.mkdir(parents=True, exist_ok=True)
+ workers = _worker_order(dispatches)
+ rows = _worker_summary(dispatches, workers, span)
+ _write_csv(output_dir / "dispatches.csv", dispatches, labels, phases)
+ _write_stage_projection_csv(output_dir / "stage-projections.csv", dispatches, labels)
+ _write_phase_csv(output_dir / "phase-summary.csv", phase_rows)
+ summary_path = output_dir / "summary.json"
+ _write_summary(
+ summary_path,
+ source=source,
+ rows=rows,
+ phase_rows=phase_rows,
+ dispatches=dispatches,
+ span=span,
+ zoom_start=zoom_start,
+ zoom_seconds=min(args.zoom_seconds, span),
+ )
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
+ _write_markdown(output_dir / "summary.md", summary=summary, output_dir=output_dir)
+ _draw_timeline(
+ dispatches,
+ labels,
+ output_dir / "dispatch-timeline.png",
+ phases=phases,
+ zoom_start=zoom_start,
+ zoom_seconds=min(args.zoom_seconds, span),
+ )
+ print(
+ json.dumps({"output_dir": str(output_dir), "dispatches": len(dispatches), "span_seconds": span}, sort_keys=True)
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/validation/render_abot_scheduler_summary.py b/tools/validation/render_abot_scheduler_summary.py
new file mode 100644
index 00000000..dcb5c6e5
--- /dev/null
+++ b/tools/validation/render_abot_scheduler_summary.py
@@ -0,0 +1,455 @@
+#!/usr/bin/env python3
+"""Render a read-only summary figure from saved ABot-World experiment artifacts.
+
+This program deliberately consumes only JSON files produced by the native
+three-session scheduler timeline and the four-GPU LiveKit workload trace. It
+does not import serving code, initialize CUDA, or contact a server. The PNG
+is designed as a compact, reproducible companion to the raw timeline PNGs and
+Prometheus-based batch analysis.
+
+Example:
+
+ PYTHONPATH=$PWD /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \
+ tools/validation/render_abot_scheduler_summary.py \
+ --staggered results/experiments/abot_session_strategy_3user_20260814/native_12fps/staggered/timeline.json \
+ --aligned results/experiments/abot_session_strategy_3user_20260814/native_12fps/aligned/timeline.json \
+ --four-gpu-result results/experiments/abot_4gpu_lf3_12fps_intermittent_proxyfree_20260814/result.json \
+ --four-gpu-analysis results/experiments/abot_4gpu_lf3_12fps_intermittent_proxyfree_20260814/analysis/summary.json \
+ --output-dir results/experiments/abot_4gpu_lf3_12fps_intermittent_proxyfree_20260814/analysis
+
+Outputs ``scheduler-summary.png`` and ``scheduler-summary.md``.
+""" # noqa: E501
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from PIL import Image, ImageDraw, ImageFont
+
+_NAVY = "#0f172a"
+_SLATE = "#475569"
+_GRID = "#cbd5e1"
+_PANEL = "#f8fafc"
+_WHITE = "#ffffff"
+_BLUE = "#2563eb"
+_CYAN = "#0891b2"
+_ORANGE = "#ea580c"
+_RED = "#dc2626"
+_GREEN = "#059669"
+_PURPLE = "#7c3aed"
+_USERS = ("#2563eb", "#ea580c", "#059669")
+
+
+def _args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--staggered", type=Path, required=True)
+ parser.add_argument("--aligned", type=Path, required=True)
+ parser.add_argument("--four-gpu-result", type=Path, required=True)
+ parser.add_argument("--four-gpu-analysis", type=Path, required=True)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ return parser.parse_args()
+
+
+def _load(path: Path) -> dict[str, Any]:
+ return json.loads(path.expanduser().resolve().read_text(encoding="utf-8"))
+
+
+def _font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont:
+ face = "DejaVuSans-Bold.ttf" if bold else "DejaVuSans.ttf"
+ return ImageFont.truetype(face, size=size)
+
+
+def _text(
+ draw: ImageDraw.ImageDraw,
+ xy: tuple[float, float],
+ text: str,
+ *,
+ size: int = 22,
+ fill: str = _NAVY,
+ bold: bool = False,
+ anchor: str | None = None,
+) -> None:
+ draw.text(xy, text, fill=fill, font=_font(size, bold=bold), anchor=anchor)
+
+
+def _rounded(
+ draw: ImageDraw.ImageDraw,
+ box: tuple[float, float, float, float],
+ *,
+ fill: str = _PANEL,
+ outline: str | None = _GRID,
+ radius: int = 18,
+ width: int = 2,
+) -> None:
+ draw.rounded_rectangle(box, radius=radius, fill=fill, outline=outline, width=width)
+
+
+def _safe_float(value: Any, default: float = 0.0) -> float:
+ return float(value) if isinstance(value, int | float) else default
+
+
+def _timeline_stats(payload: dict[str, Any]) -> dict[str, Any]:
+ batches = [item for item in payload.get("batches", []) if isinstance(item, dict)]
+ histogram = Counter(int(item.get("batch_size", 0)) for item in batches)
+ durations = [_safe_float(item.get("duration_ms")) for item in batches]
+ scenario = payload.get("scenario", {}) if isinstance(payload.get("scenario"), dict) else {}
+ return {
+ "batches": batches,
+ "calls": len(batches),
+ "histogram": histogram,
+ "mean_duration_ms": sum(durations) / len(durations) if durations else 0.0,
+ "min_duration_ms": min(durations, default=0.0),
+ "max_duration_ms": max(durations, default=0.0),
+ "offsets_ms": [float(value) for value in scenario.get("session_arrival_offsets_ms", [])],
+ "frames": int(scenario.get("control_latent_frames", 3)) * 4,
+ "fps": _safe_float(scenario.get("fps"), 12.0),
+ }
+
+
+def _phase(result: dict[str, Any], name: str) -> dict[str, Any]:
+ for item in result.get("phase_results", []):
+ if isinstance(item, dict) and item.get("phase") == name:
+ return item
+ raise ValueError(f"missing phase {name!r} in four-GPU result")
+
+
+def _draw_single_card_panel(
+ draw: ImageDraw.ImageDraw,
+ *,
+ box: tuple[int, int, int, int],
+ staggered: dict[str, Any],
+ aligned: dict[str, Any],
+) -> None:
+ x0, y0, x1, y1 = box
+ _rounded(draw, box)
+ _text(draw, (x0 + 26, y0 + 20), "One H100: native 3-session scheduler trace", size=28, bold=True)
+ _text(
+ draw,
+ (x0 + 26, y0 + 57),
+ "ABot-World LF=3, 12 frames/chunk, 12 FPS target; model calls measured at generate_next_blocks().",
+ size=16,
+ fill=_SLATE,
+ )
+ divider = (x0 + x1) // 2
+ draw.line((divider, y0 + 92, divider, y1 - 24), fill=_GRID, width=2)
+
+ columns = (
+ (x0 + 26, divider - 22, staggered, "Staggered controls", _ORANGE),
+ (divider + 22, x1 - 26, aligned, "Barrier-aligned controls", _GREEN),
+ )
+ for col_x0, col_x1, stats, title, accent in columns:
+ hist: Counter[int] = stats["histogram"]
+ call_count = stats["calls"]
+ b3_count = hist[3]
+ b1_count = hist[1]
+ _text(draw, (col_x0, y0 + 108), title, size=22, bold=True)
+ if title.startswith("Staggered"):
+ offsets = "/".join(f"{value:.0f}" for value in stats["offsets_ms"])
+ _text(draw, (col_x0, y0 + 140), f"arrival offsets: {offsets} ms", size=16, fill=_SLATE)
+ result = f"{b1_count}/{call_count} calls are B=1; B=3: {b3_count}/{call_count}"
+ else:
+ _text(draw, (col_x0, y0 + 140), "all three controls activated in one scheduler turn", size=16, fill=_SLATE)
+ result = f"B=3: {b3_count}/{call_count} calls; B=1: {b1_count}/{call_count}"
+ _text(draw, (col_x0, y0 + 169), result, size=17, fill=accent, bold=True)
+
+ bars_y0 = y0 + 217
+ y0 + 357
+ duration_max = max((_safe_float(item.get("end_seconds")) for item in stats["batches"]), default=1.0)
+ duration_max = max(duration_max, 0.1)
+ lane_width = col_x1 - col_x0 - 10
+ for index, batch in enumerate(stats["batches"]):
+ start = _safe_float(batch.get("start_seconds")) / duration_max
+ end = _safe_float(batch.get("end_seconds")) / duration_max
+ bx0 = col_x0 + 3 + int(lane_width * start)
+ bx1 = col_x0 + 3 + int(lane_width * end)
+ by0 = bars_y0 + (index % 3) * 42
+ by1 = by0 + 28
+ batch_size = int(batch.get("batch_size", 0))
+ if batch_size == 1:
+ session_ids = batch.get("session_ids", [])
+ session_id = session_ids[0] if isinstance(session_ids, list) and session_ids else "user-1"
+ user_index = (
+ max(0, min(2, int(str(session_id).split("-")[-1]) - 1))
+ if str(session_id).split("-")[-1].isdigit()
+ else 0
+ )
+ fill = _USERS[user_index]
+ else:
+ fill = _GREEN
+ draw.rounded_rectangle((bx0, by0, max(bx0 + 3, bx1), by1), radius=6, fill=fill)
+ if batch_size == 3:
+ stripe = max(1, (max(bx0 + 3, bx1) - bx0) // 3)
+ for stripe_index, color in enumerate(_USERS):
+ left = bx0 + stripe_index * stripe
+ right = bx0 + (stripe_index + 1) * stripe if stripe_index < 2 else max(bx0 + 3, bx1)
+ draw.rectangle((left, by0, right, by1), fill=color)
+ label = f"B{batch_size} {float(batch.get('duration_ms', 0.0)):.0f}ms"
+ _text(draw, (bx0 + 5, by0 + 4), label, size=12, fill=_WHITE, bold=True)
+ for lane_index, label in enumerate(("u1", "u2", "u3")):
+ _text(draw, (col_x0 - 2, bars_y0 + lane_index * 42 - 15), label, size=13, fill=_SLATE)
+ _text(draw, (col_x0, y0 + 378), f"mean native call: {stats['mean_duration_ms']:.1f} ms", size=16, fill=_SLATE)
+ if title.startswith("Staggered"):
+ _text(
+ draw,
+ (col_x0, y0 + 405),
+ "Interpretation: non-preemptive, chunk-level time division.",
+ size=15,
+ fill=_SLATE,
+ )
+ else:
+ _text(
+ draw,
+ (col_x0, y0 + 405),
+ "Interpretation: scheduler can make a true coalesced B=3 call.",
+ size=15,
+ fill=_SLATE,
+ )
+
+
+def _draw_fps_trace(
+ draw: ImageDraw.ImageDraw,
+ *,
+ box: tuple[int, int, int, int],
+ result: dict[str, Any],
+ peak: dict[str, Any],
+) -> None:
+ x0, y0, x1, y1 = box
+ _rounded(draw, box)
+ _text(draw, (x0 + 24, y0 + 18), "Four H100s: client-visible FPS / active session", size=24, bold=True)
+ summary = peak["summary"]
+ mean = _safe_float(summary["per_active_session_delivery_fps"]["mean"])
+ p50 = _safe_float(summary["per_active_session_delivery_fps"]["p50"])
+ attainment = 100.0 * _safe_float(summary.get("slo_sample_attainment"))
+ _text(
+ draw,
+ (x0 + 24, y0 + 51),
+ f"Peak-16 continuous: mean {mean:.3f} FPS (p50 {p50:.3f}); 12-FPS SLO attainment {attainment:.1f}%.",
+ size=15,
+ fill=_SLATE,
+ )
+ chart = (x0 + 55, y0 + 92, x1 - 28, y1 - 49)
+ cx0, cy0, cx1, cy1 = chart
+ samples = [item for item in result.get("samples", []) if isinstance(item, dict)]
+ phase_results = [item for item in result.get("phase_results", []) if isinstance(item, dict)]
+ max_x = max((_safe_float(item.get("offset_seconds")) for item in samples), default=1.0)
+ max_x = max(max_x, 1.0)
+ y_max = 14.0
+ peak_start = _safe_float(peak.get("started_offset_seconds"))
+ peak_end = _safe_float(peak.get("completed_offset_seconds"))
+ px0 = cx0 + (cx1 - cx0) * peak_start / max_x
+ px1 = cx0 + (cx1 - cx0) * peak_end / max_x
+ draw.rectangle((px0, cy0, px1, cy1), fill="#fef3c7")
+ _text(draw, ((px0 + px1) / 2, cy0 + 8), "peak 16", size=13, fill="#92400e", bold=True, anchor="ma")
+ for fps in (0, 4, 8, 12):
+ y = cy1 - (cy1 - cy0) * fps / y_max
+ draw.line((cx0, y, cx1, y), fill=_GRID, width=1)
+ _text(draw, (cx0 - 10, y), str(fps), size=13, fill=_SLATE, anchor="rm")
+ y_target = cy1 - (cy1 - cy0) * 12.0 / y_max
+ draw.line((cx0, y_target, cx1, y_target), fill=_RED, width=2)
+ _text(draw, (cx1 - 4, y_target - 4), "12 FPS target", size=13, fill=_RED, anchor="rs")
+ points: list[tuple[float, float]] = []
+ for sample in samples:
+ value = sample.get("per_active_session_delivery_fps")
+ if not isinstance(value, int | float):
+ continue
+ x = cx0 + (cx1 - cx0) * _safe_float(sample.get("offset_seconds")) / max_x
+ y = cy1 - (cy1 - cy0) * min(y_max, max(0.0, float(value))) / y_max
+ points.append((x, y))
+ if len(points) > 1:
+ draw.line(points, fill=_BLUE, width=3, joint="curve")
+ for phase in phase_results:
+ end = _safe_float(phase.get("completed_offset_seconds"))
+ x = cx0 + (cx1 - cx0) * end / max_x
+ draw.line((x, cy1, x, cy1 + 5), fill=_SLATE, width=1)
+ _text(draw, (cx0, cy1 + 13), "0 s", size=13, fill=_SLATE)
+ _text(draw, (cx1, cy1 + 13), f"{max_x:.0f} s", size=13, fill=_SLATE, anchor="ra")
+ _text(
+ draw,
+ (cx0, y1 - 25),
+ "Consumer-side rolling delivery rate; not an aggregate/model-throughput metric.",
+ size=13,
+ fill=_SLATE,
+ )
+
+
+def _draw_batch_panel(
+ draw: ImageDraw.ImageDraw,
+ *,
+ box: tuple[int, int, int, int],
+ analysis: dict[str, Any],
+ peak: dict[str, Any],
+) -> None:
+ x0, y0, x1, y1 = box
+ _rounded(draw, box)
+ entire = analysis["entire_capture"]
+ executions = _safe_float(entire.get("batch_executions"))
+ b1 = _safe_float(entire.get("b1_execution_equivalents"))
+ b2 = _safe_float(entire.get("b2_execution_equivalents"))
+ b3 = _safe_float(entire.get("b3_execution_equivalents"))
+ b4 = _safe_float(entire.get("b4_execution_equivalents"))
+ peak_summary = peak["summary"]
+ _text(draw, (x0 + 24, y0 + 18), "Why the realistic trace gets almost no batching", size=24, bold=True)
+ _text(
+ draw,
+ (x0 + 24, y0 + 51),
+ f"Entire 419.2-s capture: {executions:.0f} model executions; B=2/3/4 all zero.",
+ size=15,
+ fill=_SLATE,
+ )
+ values = (("B=1", b1, _BLUE), ("B=2", b2, _ORANGE), ("B=3", b3, _GREEN), ("B=4", b4, _PURPLE))
+ bar_x0, _bar_y0, bar_x1, bar_y1 = x0 + 52, y0 + 109, x1 - 34, y0 + 258
+ max_value = max([value for _, value, _ in values] or [1.0])
+ for index, (label, value, color) in enumerate(values):
+ baseline = bar_y1 - index * 34
+ draw.rounded_rectangle(
+ (bar_x0, baseline - 21, bar_x0 + (bar_x1 - bar_x0) * value / max_value, baseline), radius=5, fill=color
+ )
+ _text(draw, (bar_x0 - 10, baseline - 11), label, size=15, fill=_SLATE, anchor="rm")
+ _text(
+ draw,
+ (bar_x0 + (bar_x1 - bar_x0) * value / max_value + 8, baseline - 11),
+ f"{value:.0f}",
+ size=15,
+ fill=_NAVY,
+ bold=True,
+ )
+ _text(
+ draw,
+ (x0 + 24, y0 + 288),
+ "Peak-16: 16/16 immediately admitted; no queue. Mean execution batch size = 1.000.",
+ size=15,
+ fill=_SLATE,
+ )
+ _text(
+ draw,
+ (x0 + 24, y0 + 319),
+ "Conclusion: the four GPUs supply parallel B=1 service; timing/position mismatch prevents within-GPU coalescing.", # noqa: E501
+ size=14,
+ fill=_SLATE,
+ )
+ _text(
+ draw,
+ (x0 + 24, y0 + 355),
+ f"Peak-16 aggregate delivery mean: {_safe_float(peak_summary['aggregate_delivery_fps']['mean']):.2f} FPS",
+ size=15,
+ fill=_NAVY,
+ bold=True,
+ )
+ _text(
+ draw,
+ (x0 + 24, y0 + 383),
+ "(Aggregate is included only as context; SLO judgment uses per-active-session FPS at left.)",
+ size=13,
+ fill=_SLATE,
+ )
+
+
+def _render(
+ *,
+ staggered: dict[str, Any],
+ aligned: dict[str, Any],
+ four_result: dict[str, Any],
+ four_analysis: dict[str, Any],
+ output_png: Path,
+) -> dict[str, Any]:
+ canvas = Image.new("RGB", (1800, 1320), _WHITE)
+ draw = ImageDraw.Draw(canvas)
+ _text(draw, (50, 28), "ABot-World serving: alignment enables B=3, realistic arrivals do not", size=36, bold=True)
+ _text(
+ draw,
+ (50, 76),
+ "Native model timeline + 4-GPU LiveKit trace. All quantities are derived from the saved artifacts named in scheduler-summary.md.", # noqa: E501
+ size=18,
+ fill=_SLATE,
+ )
+ staggered_stats = _timeline_stats(staggered)
+ aligned_stats = _timeline_stats(aligned)
+ peak = _phase(four_result, "peak_16_continuous")
+ _draw_single_card_panel(draw, box=(45, 122, 1755, 590), staggered=staggered_stats, aligned=aligned_stats)
+ _draw_fps_trace(draw, box=(45, 620, 1125, 1260), result=four_result, peak=peak)
+ _draw_batch_panel(draw, box=(1150, 620, 1755, 1260), analysis=four_analysis, peak=peak)
+ canvas.save(output_png)
+ return {
+ "staggered": staggered_stats,
+ "aligned": aligned_stats,
+ "peak": peak,
+ "entire": four_analysis["entire_capture"],
+ }
+
+
+def _write_markdown(
+ *,
+ path: Path,
+ stats: dict[str, Any],
+ source_paths: dict[str, Path],
+) -> None:
+ staggered = stats["staggered"]
+ aligned = stats["aligned"]
+ peak = stats["peak"]["summary"]
+ entire = stats["entire"]
+ staggered_hist: Counter[int] = staggered["histogram"]
+ aligned_hist: Counter[int] = aligned["histogram"]
+ content = f"""# ABot-World scheduler summary (artifact-derived)
+
+
+
+| Experiment | Native model dispatch evidence | Client-visible result |
+|---|---:|---:|
+| 1 H100, 3 sessions, staggered controls (0/450/900 ms) | {staggered_hist[1]}/{staggered["calls"]} B=1 calls; {staggered_hist[3]}/{staggered["calls"]} B=3 calls; mean call {staggered["mean_duration_ms"]:.1f} ms | Non-preemptive, chunk-level time division (no overlapping GPU calls) |
+| 1 H100, 3 sessions, scheduler barrier aligned | {aligned_hist[3]}/{aligned["calls"]} B=3 calls; {aligned_hist[1]}/{aligned["calls"]} B=1 calls; mean B=3 call {aligned["mean_duration_ms"]:.1f} ms | Actual coalesced B=3 reached the native `generate_next_blocks()` path |
+| 4 H100, intermittent 16-user LiveKit trace | B=1 {entire["b1_execution_equivalents"]:.0f}/{entire["batch_executions"]:.0f}; B=2/3/4 = 0; mean execution batch {entire["mean_execution_batch_size"]:.3f} | `peak_16_continuous`: mean **{peak["per_active_session_delivery_fps"]["mean"]:.3f} FPS/active session**, p50 {peak["per_active_session_delivery_fps"]["p50"]:.3f}, 12-FPS SLO attainment {100.0 * peak["slo_sample_attainment"]:.1f}% |
+
+The 4-GPU result is **not** evidence that native batching lacks value: the controlled one-GPU barrier proves the production scheduler can form B=3. It is evidence that this realistic, staggered/intermittent trace has no compatible sessions ready together on a worker at the same frame/cache boundary, so the system operates as four parallel B=1 workers.
+
+## Reproduce the figure without starting a server or GPU
+
+```bash
+cd /public/fanyk1/lwb/TeleFuser-abot-world
+PYTHONPATH=$PWD /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/render_abot_scheduler_summary.py \\
+ --staggered {source_paths["staggered"]} \\
+ --aligned {source_paths["aligned"]} \\
+ --four-gpu-result {source_paths["four_result"]} \\
+ --four-gpu-analysis {source_paths["four_analysis"]} \\
+ --output-dir {path.parent}
+```
+
+## Source artifacts
+
+- `{source_paths["staggered"]}`
+- `{source_paths["aligned"]}`
+- `{source_paths["four_result"]}`
+- `{source_paths["four_analysis"]}`
+""" # noqa: E501
+ path.write_text(content, encoding="utf-8")
+
+
+def main() -> int:
+ args = _args()
+ paths = {
+ "staggered": args.staggered.expanduser().resolve(),
+ "aligned": args.aligned.expanduser().resolve(),
+ "four_result": args.four_gpu_result.expanduser().resolve(),
+ "four_analysis": args.four_gpu_analysis.expanduser().resolve(),
+ }
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ stats = _render(
+ staggered=_load(paths["staggered"]),
+ aligned=_load(paths["aligned"]),
+ four_result=_load(paths["four_result"]),
+ four_analysis=_load(paths["four_analysis"]),
+ output_png=args.output_dir / "scheduler-summary.png",
+ )
+ _write_markdown(path=args.output_dir / "scheduler-summary.md", stats=stats, source_paths=paths)
+ print(args.output_dir / "scheduler-summary.png")
+ print(args.output_dir / "scheduler-summary.md")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/validation/replay_abot_livekit_lifecycle_trace.py b/tools/validation/replay_abot_livekit_lifecycle_trace.py
new file mode 100644
index 00000000..d9963a88
--- /dev/null
+++ b/tools/validation/replay_abot_livekit_lifecycle_trace.py
@@ -0,0 +1,418 @@
+#!/usr/bin/env python3
+"""Replay an explicit per-session ABot LiveKit lifecycle trace.
+
+Unlike ``benchmark_abot_livekit_burst.py``'s aggregate phase fractions, this
+runner schedules the ``lifecycle_trace.events`` embedded in a scenario at their
+declared offsets. It still uses the same public HTTP and LiveKit client path;
+the workload never selects a GPU or accesses model internals.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+from tools.validation import benchmark_abot_livekit_burst as wave
+
+
+@dataclass(frozen=True)
+class ExplicitLifecycleEvent:
+ """One validated arrival, pause, resume, or departure."""
+
+ offset_seconds: float
+ sequence: int
+ event: str
+ trace_session_id: str
+ source_session_id: int | None
+ source_user_id: int | None
+ input_enabled: bool | None
+
+
+@dataclass(frozen=True)
+class ExplicitLifecycleTrace:
+ """Exact lifecycle schedule for one reporting phase."""
+
+ duration_seconds: float
+ events: tuple[ExplicitLifecycleEvent, ...]
+
+
+class LifecycleTraceError(ValueError):
+ """Raised before the service is contacted for an invalid replay trace."""
+
+
+def _mapping(value: object, label: str) -> Mapping[str, Any]:
+ if not isinstance(value, Mapping):
+ raise LifecycleTraceError(f"{label} must be an object")
+ return value
+
+
+def _number(value: object, label: str, *, allow_zero: bool) -> float:
+ if not isinstance(value, int | float) or isinstance(value, bool):
+ raise LifecycleTraceError(f"{label} must be a number")
+ parsed = float(value)
+ if parsed < 0 or (not allow_zero and parsed == 0):
+ raise LifecycleTraceError(f"{label} must be {'non-negative' if allow_zero else 'positive'}")
+ return parsed
+
+
+def _non_negative_int(value: object, label: str) -> int:
+ if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+ raise LifecycleTraceError(f"{label} must be a non-negative integer")
+ return int(value)
+
+
+def load_explicit_lifecycle_trace(scenario: wave.Scenario) -> ExplicitLifecycleTrace:
+ """Parse the generated trace before opening any HTTP or LiveKit connection."""
+ raw = _mapping(scenario.raw.get("lifecycle_trace"), "lifecycle_trace")
+ if raw.get("kind") != "explicit_session_lifecycle_v1":
+ raise LifecycleTraceError("lifecycle_trace.kind must be explicit_session_lifecycle_v1")
+ if scenario.diagnostic_initial_control_barrier is not None:
+ raise LifecycleTraceError("explicit lifecycle replay cannot use diagnostic.initial_control_barrier")
+ if len(scenario.phases) != 1:
+ raise LifecycleTraceError("explicit lifecycle replay requires exactly one reporting phase")
+ duration = _number(raw.get("duration_seconds"), "lifecycle_trace.duration_seconds", allow_zero=False)
+ if abs(duration - scenario.phases[0].duration_seconds) > 1e-6:
+ raise LifecycleTraceError("lifecycle_trace.duration_seconds must equal its reporting phase duration")
+ raw_events = raw.get("events")
+ if not isinstance(raw_events, list) or not raw_events:
+ raise LifecycleTraceError("lifecycle_trace.events must be a non-empty list")
+
+ retained: dict[str, bool] = {}
+ previous_key = (-1.0, -1)
+ parsed_events: list[ExplicitLifecycleEvent] = []
+ valid_events = {"session_arrival", "user_active", "user_idle", "session_departure"}
+ for index, event_value in enumerate(raw_events):
+ event = _mapping(event_value, f"lifecycle_trace.events[{index}]")
+ offset = _number(
+ event.get("offset_seconds"),
+ f"lifecycle_trace.events[{index}].offset_seconds",
+ allow_zero=True,
+ )
+ sequence = _non_negative_int(event.get("sequence"), f"lifecycle_trace.events[{index}].sequence")
+ if offset > duration:
+ raise LifecycleTraceError(f"lifecycle_trace.events[{index}] occurs after duration")
+ if (offset, sequence) <= previous_key:
+ raise LifecycleTraceError("lifecycle_trace.events must be strictly ordered by (offset_seconds, sequence)")
+ previous_key = (offset, sequence)
+ event_name = event.get("event")
+ if event_name not in valid_events:
+ raise LifecycleTraceError(f"Unsupported lifecycle event {event_name!r} at index {index}")
+ trace_session_id = event.get("trace_session_id")
+ if not isinstance(trace_session_id, str) or not trace_session_id:
+ raise LifecycleTraceError(f"lifecycle_trace.events[{index}].trace_session_id must be a non-empty string")
+ input_enabled = event.get("input_enabled")
+ if event_name != "session_departure" and not isinstance(input_enabled, bool):
+ raise LifecycleTraceError(f"lifecycle_trace.events[{index}].input_enabled must be boolean")
+ if event_name == "session_arrival":
+ if trace_session_id in retained:
+ raise LifecycleTraceError(f"Trace session {trace_session_id!r} arrived while retained")
+ retained[trace_session_id] = bool(input_enabled)
+ elif event_name == "session_departure":
+ if trace_session_id not in retained:
+ raise LifecycleTraceError(f"Trace session {trace_session_id!r} departed before arrival")
+ del retained[trace_session_id]
+ else:
+ current = retained.get(trace_session_id)
+ if current is None:
+ raise LifecycleTraceError(f"Trace session {trace_session_id!r} changed input before arrival")
+ desired = bool(input_enabled)
+ if current == desired:
+ raise LifecycleTraceError(f"Trace session {trace_session_id!r} has a redundant {event_name}")
+ retained[trace_session_id] = desired
+ source_session_id = event.get("source_session_id")
+ if source_session_id is not None:
+ source_session_id = _non_negative_int(
+ source_session_id, f"lifecycle_trace.events[{index}].source_session_id"
+ )
+ source_user_id = event.get("source_user_id")
+ if source_user_id is not None:
+ source_user_id = _non_negative_int(source_user_id, f"lifecycle_trace.events[{index}].source_user_id")
+ parsed_events.append(
+ ExplicitLifecycleEvent(
+ offset_seconds=offset,
+ sequence=sequence,
+ event=str(event_name),
+ trace_session_id=trace_session_id,
+ source_session_id=source_session_id,
+ source_user_id=source_user_id,
+ input_enabled=input_enabled if isinstance(input_enabled, bool) else None,
+ )
+ )
+ if retained:
+ raise LifecycleTraceError("lifecycle_trace must depart every session before its duration ends")
+ return ExplicitLifecycleTrace(duration_seconds=duration, events=tuple(parsed_events))
+
+
+class ExplicitLifecycleRunner(wave.LiveKitWaveRunner):
+ """Reuse the normal black-box runner while replacing phase scheduling."""
+
+ def __init__(self, scenario: wave.Scenario, trace: ExplicitLifecycleTrace) -> None:
+ super().__init__(scenario)
+ self._explicit_trace = trace
+ self._trace_sessions: dict[str, wave.LiveKitWaveSession] = {}
+ # A trace departure means that the logical session is gone at its source
+ # timestamp, but the public DELETE/LiveKit teardown is asynchronous. Keep
+ # that physical teardown in the admission accounting until ``stop()`` has
+ # completed, otherwise an arrival at the same (or a nearby) timestamp can
+ # race the server's fixed session capacity and receive a false 429.
+ self._lifecycle_start_tasks: dict[str, asyncio.Task[None]] = {}
+ self._lifecycle_departure_tasks: dict[str, asyncio.Task[None]] = {}
+ self._lifecycle_admission_capacity = self._derive_admission_capacity(trace)
+
+ def _derive_admission_capacity(self, trace: ExplicitLifecycleTrace) -> int:
+ """Return the public capacity used to guard lifecycle arrivals.
+
+ A scenario with an explicit admission contract should use that contract.
+ Older/free-form traces can still be replayed safely by treating their
+ observed retained-session peak as the capacity limit.
+ """
+ per_worker = self.scenario.admission.expected_max_sessions_per_worker
+ workers = self.scenario.expected_num_workers
+ if per_worker is not None and workers is not None:
+ return per_worker * workers
+
+ retained = 0
+ peak = 0
+ for event in trace.events:
+ if event.event == "session_arrival":
+ retained += 1
+ peak = max(peak, retained)
+ elif event.event == "session_departure":
+ retained -= 1
+ if peak < 1:
+ raise RuntimeError("Explicit lifecycle trace has no retained-session capacity")
+ return peak
+
+ async def _run_phase(self, phase: wave.Phase) -> None:
+ """Run the single reporting phase from its exact event schedule."""
+ trace = self._explicit_trace
+ if abs(trace.duration_seconds - phase.duration_seconds) > 1e-6:
+ raise RuntimeError("Explicit lifecycle trace and reporting phase duration diverged")
+ phase_started = time.perf_counter()
+ sample_start = len(self._samples)
+ self._phase_name = phase.name
+ self._phase_target_users = phase.target_users
+ self._phase_active_input_fraction = phase.active_input_fraction
+ self.record_event(
+ "phase_started",
+ phase=phase.name,
+ target_users=phase.target_users,
+ active_input_fraction=phase.active_input_fraction,
+ lifecycle_trace_kind="explicit_session_lifecycle_v1",
+ lifecycle_trace_event_count=len(trace.events),
+ )
+ await self._capture_server_metadata(f"phase_start:{phase.name}")
+ self.record_event("lifecycle_trace_started", duration_seconds=trace.duration_seconds)
+ for event in trace.events:
+ remaining = event.offset_seconds - (time.perf_counter() - phase_started)
+ if remaining > 0:
+ await asyncio.sleep(remaining)
+ await self._schedule_lifecycle_event(event)
+ remaining = trace.duration_seconds - (time.perf_counter() - phase_started)
+ if remaining > 0:
+ await asyncio.sleep(remaining)
+ await self._wait_for_all_lifecycle_departures()
+ phase_completed = time.perf_counter()
+ await self._capture_server_metadata(f"phase_end:{phase.name}")
+ result = self._summarize_phase(
+ phase,
+ phase_started=phase_started,
+ phase_completed=phase_completed,
+ samples=self._samples[sample_start:],
+ )
+ self._phase_results.append(result)
+ self.record_event("lifecycle_trace_completed", scheduled_event_count=len(trace.events))
+ self.record_event("phase_completed", phase=phase.name, summary=result["summary"])
+
+ def _spawn_lifecycle_task(
+ self,
+ tasks: dict[str, asyncio.Task[None]],
+ trace_session_id: str,
+ coroutine: Any,
+ ) -> asyncio.Task[None]:
+ """Track a lifecycle task both for cleanup and admission ordering."""
+ task = asyncio.create_task(coroutine)
+ tasks[trace_session_id] = task
+ self._background_tasks.add(task)
+
+ def _complete(completed: asyncio.Task[None]) -> None:
+ self._background_tasks.discard(completed)
+ if tasks.get(trace_session_id) is completed:
+ del tasks[trace_session_id]
+
+ task.add_done_callback(_complete)
+ return task
+
+ async def _wait_for_departures_before_arrival(self) -> None:
+ """Wait only when unfinished teardowns still consume all capacity.
+
+ The source trace's retained-session count is capacity-normalized, but
+ public HTTP deletion/LiveKit disconnect are asynchronous. At a source
+ timestamp that replaces a departing session, issuing the new POST while
+ the old DELETE is still in flight produces a harness-only 429. Keep
+ the source schedule concurrent unless that narrow physical-capacity
+ collision exists.
+ """
+ closing = tuple(task for task in self._lifecycle_departure_tasks.values() if not task.done())
+ physical_retained = len(self._trace_sessions) + len(closing)
+ if physical_retained < self._lifecycle_admission_capacity:
+ return
+ self.record_event(
+ "lifecycle_arrival_waiting_for_departure",
+ physical_retained=physical_retained,
+ admission_capacity=self._lifecycle_admission_capacity,
+ closing_sessions=len(closing),
+ )
+ await asyncio.gather(*closing, return_exceptions=True)
+ self.record_event(
+ "lifecycle_arrival_departure_wait_completed",
+ admission_capacity=self._lifecycle_admission_capacity,
+ closing_sessions=len(closing),
+ )
+
+ async def _wait_for_all_lifecycle_departures(self) -> None:
+ """Finish trace departures before recording the phase-end metadata."""
+ closing = tuple(task for task in self._lifecycle_departure_tasks.values() if not task.done())
+ if closing:
+ await asyncio.gather(*closing, return_exceptions=True)
+
+ async def _stop_after_start(
+ self,
+ session: wave.LiveKitWaveSession,
+ start_task: asyncio.Task[None] | None,
+ ) -> None:
+ """Ensure an arrival already in progress is deleted before replacement."""
+ if start_task is not None:
+ await asyncio.gather(start_task, return_exceptions=True)
+ await session.stop()
+
+ async def _schedule_lifecycle_event(self, event: ExplicitLifecycleEvent) -> None:
+ """Schedule one trace event, preserving public-session capacity."""
+ shared = {
+ "trace_session_id": event.trace_session_id,
+ "source_trace_session_id": event.source_session_id,
+ "source_trace_user_id": event.source_user_id,
+ "trace_event_sequence": event.sequence,
+ "trace_scheduled_offset_seconds": event.offset_seconds,
+ }
+ if event.event == "session_arrival":
+ if event.trace_session_id in self._trace_sessions:
+ raise RuntimeError(f"Trace session {event.trace_session_id!r} arrived twice")
+ await self._wait_for_departures_before_arrival()
+ session = wave.LiveKitWaveSession(
+ index=len(self._sessions),
+ scenario=self.scenario,
+ http=self._http,
+ rtc=self.rtc,
+ record_event=self.record_event,
+ started_at=self.started_at,
+ trace_session_id=event.trace_session_id,
+ source_trace_session_id=event.source_session_id,
+ source_trace_user_id=event.source_user_id,
+ )
+ session.input_enabled = bool(event.input_enabled)
+ session.scheduled_at = time.perf_counter()
+ self._sessions.append(session)
+ self._trace_sessions[event.trace_session_id] = session
+ self.record_event("lifecycle_session_arrival_scheduled", input_enabled=session.input_enabled, **shared)
+ self._spawn_lifecycle_task(
+ self._lifecycle_start_tasks,
+ event.trace_session_id,
+ self._delayed_start(session, 0.0),
+ )
+ return
+
+ session = self._trace_sessions.get(event.trace_session_id)
+ if session is None:
+ raise RuntimeError(f"Trace event references unknown session {event.trace_session_id!r}")
+ if event.event == "session_departure":
+ del self._trace_sessions[event.trace_session_id]
+ session.departure_scheduled = True
+ self.record_event("lifecycle_session_departure_scheduled", **shared)
+ self._spawn_lifecycle_task(
+ self._lifecycle_departure_tasks,
+ event.trace_session_id,
+ self._stop_after_start(
+ session,
+ self._lifecycle_start_tasks.get(event.trace_session_id),
+ ),
+ )
+ return
+
+ enabled = bool(event.input_enabled)
+ self.record_event(
+ "lifecycle_input_transition_scheduled",
+ input_enabled=enabled,
+ source_event=event.event,
+ **shared,
+ )
+ self._spawn_background(
+ self._delayed_set_input_enabled(
+ session,
+ enabled,
+ 0.0,
+ reason=f"lifecycle_trace:{event.event}",
+ )
+ )
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--scenario", type=Path, required=True, help="Explicit lifecycle-trace scenario JSON")
+ parser.add_argument("--server-url", help="Override scenario.server_url")
+ parser.add_argument("--output", type=Path, help="Write complete workload artifact")
+ parser.add_argument("--dry-run", action="store_true", help="Validate and summarize without contacting the service")
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = _parse_args()
+ scenario_path = args.scenario.expanduser()
+ if not scenario_path.is_absolute():
+ scenario_path = (wave._REPO_ROOT / scenario_path).resolve()
+ scenario = wave.load_scenario(scenario_path, server_url_override=args.server_url)
+ trace = load_explicit_lifecycle_trace(scenario)
+ if args.dry_run:
+ counts: dict[str, int] = {}
+ for event in trace.events:
+ counts[event.event] = counts.get(event.event, 0) + 1
+ print(
+ json.dumps(
+ {
+ "scenario": scenario.name,
+ "duration_seconds": trace.duration_seconds,
+ "event_count": len(trace.events),
+ "event_counts": counts,
+ "reporting_phase": scenario.phases[0].name,
+ "diagnostic_initial_control_barrier": False,
+ },
+ indent=2,
+ sort_keys=True,
+ )
+ )
+ return
+ if args.output is None:
+ raise SystemExit("--output is required unless --dry-run is used")
+ output = args.output.expanduser()
+ if not output.is_absolute():
+ output = (wave._REPO_ROOT / output).resolve()
+ output.parent.mkdir(parents=True, exist_ok=True)
+ result = asyncio.run(ExplicitLifecycleRunner(scenario, trace).run())
+ output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ wave._print_summary(result)
+ print(f"Wrote complete artifact: {output}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/run_abot_4gpu_30min_trace.sh b/tools/validation/run_abot_4gpu_30min_trace.sh
new file mode 100644
index 00000000..8e3b420a
--- /dev/null
+++ b/tools/validation/run_abot_4gpu_30min_trace.sh
@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# Run the public 30-minute ABot lifecycle trace on four physical GPUs.
+#
+# Default: physical GPUs 4,5,6,7 -> logical worker GPUs 0,1,2,3.
+# Override, for example:
+# GPU_IDS=0,1,2,3 CUDA_GRAPH_ENABLED=0 \
+# tools/validation/run_abot_4gpu_30min_trace.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+cd "${REPO_ROOT}"
+
+PYTHON_BIN="${PYTHON_BIN:-/public/fanyk1/lwb/envs/telefuser_sage291/bin/python}"
+MODEL_ZOO_PATH="${TF_MODEL_ZOO_PATH:-/public/fanyk1/lwb/model_zoo}"
+GPU_IDS="${GPU_IDS:-4,5,6,7}"
+PORT="${PORT:-8088}"
+LIVEKIT_URL="${LIVEKIT_URL:-ws://127.0.0.1:7880}"
+LIVEKIT_API_KEY="${LIVEKIT_API_KEY:-devkey}"
+LIVEKIT_API_SECRET="${LIVEKIT_API_SECRET:-secret}"
+SCENARIO="${SCENARIO:-tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json}"
+TRACE_DURATION_SECONDS="${TRACE_DURATION_SECONDS:-1800}"
+METRICS_DURATION_SECONDS="${METRICS_DURATION_SECONDS:-1860}"
+CUDA_GRAPH_ENABLED="${CUDA_GRAPH_ENABLED:-1}"
+MAX_BATCH_SIZE="${MAX_BATCH_SIZE:-3}"
+MAX_DEADLINE_WAIT_MS="${MAX_DEADLINE_WAIT_MS:-1000}"
+FRAME_CREDIT_TARGET_FRAMES="${FRAME_CREDIT_TARGET_FRAMES:-36}"
+BATCH_SAFETY_FACTOR="${BATCH_SAFETY_FACTOR:-1.05}"
+RUN="${RUN:-results/experiments/abot_4gpu_lf3_12fps_publicdemo_b${MAX_BATCH_SIZE}_f${FRAME_CREDIT_TARGET_FRAMES}_30min_$(date -u +%Y%m%dT%H%M%SZ)}"
+
+IFS=',' read -r -a GPUS <<<"${GPU_IDS}"
+if [[ "${#GPUS[@]}" -ne 4 ]]; then
+ echo "GPU_IDS must contain exactly four comma-separated physical GPU IDs; got: ${GPU_IDS}" >&2
+ exit 2
+fi
+for gpu in "${GPUS[@]}"; do
+ if ! [[ "${gpu}" =~ ^[0-9]+$ ]]; then
+ echo "Invalid GPU ID: ${gpu}" >&2
+ exit 2
+ fi
+ owners="$(nvidia-smi -i "${gpu}" --query-compute-apps=pid,process_name,used_gpu_memory --format=csv,noheader,nounits | sed '/^$/d')"
+ if [[ -n "${owners}" ]]; then
+ echo "Refusing to share physical GPU ${gpu}; active compute process(es):" >&2
+ echo "${owners}" >&2
+ exit 1
+ fi
+done
+
+if ss -ltn "( sport = :${PORT} )" | tail -n +2 | grep -q .; then
+ echo "Refusing to start: TCP port ${PORT} is already listening." >&2
+ exit 1
+fi
+if [[ ! -f "${SCENARIO}" ]]; then
+ echo "Scenario not found: ${SCENARIO}" >&2
+ exit 2
+fi
+if [[ -e "${RUN}" ]]; then
+ echo "Refusing to overwrite existing run directory: ${RUN}" >&2
+ exit 2
+fi
+
+mkdir -p "${RUN}"
+cp "${SCENARIO}" "${RUN}/scenario.json"
+printf '%s\n' \
+ "GPU_IDS=${GPU_IDS}" \
+ "CUDA_GRAPH_ENABLED=${CUDA_GRAPH_ENABLED}" \
+ "MAX_BATCH_SIZE=${MAX_BATCH_SIZE}" \
+ "MAX_DEADLINE_WAIT_MS=${MAX_DEADLINE_WAIT_MS}" \
+ "FRAME_CREDIT_TARGET_FRAMES=${FRAME_CREDIT_TARGET_FRAMES}" \
+ "BATCH_SAFETY_FACTOR=${BATCH_SAFETY_FACTOR}" \
+ "SCENARIO=${SCENARIO}" \
+ >"${RUN}/run-config.env"
+
+SERVER_PID=""
+SERVING_METRICS_PID=""
+GPU_METRICS_PID=""
+cleanup() {
+ local status=$?
+ for pid in "${SERVING_METRICS_PID}" "${GPU_METRICS_PID}" "${SERVER_PID}"; do
+ if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
+ kill -TERM "${pid}" 2>/dev/null || true
+ fi
+ done
+ wait "${SERVING_METRICS_PID}" 2>/dev/null || true
+ wait "${GPU_METRICS_PID}" 2>/dev/null || true
+ wait "${SERVER_PID}" 2>/dev/null || true
+ exit "${status}"
+}
+trap cleanup EXIT INT TERM
+
+unset http_proxy https_proxy all_proxy HTTP_PROXY HTTPS_PROXY ALL_PROXY
+
+echo "Starting 4-GPU service on physical GPUs ${GPU_IDS}; artifacts: ${RUN}"
+CUDA_VISIBLE_DEVICES="${GPU_IDS}" \
+PYTHONPATH="${REPO_ROOT}" \
+TF_MODEL_ZOO_PATH="${MODEL_ZOO_PATH}" \
+TELEFUSER_ABOT_CUDA_GRAPH_ENABLED="${CUDA_GRAPH_ENABLED}" \
+TELEFUSER_ABOT_SCHEDULER_MODE=batched \
+TELEFUSER_ABOT_MAX_BATCH_SIZE="${MAX_BATCH_SIZE}" \
+TELEFUSER_ABOT_BATCHING_WINDOW_MS=2 \
+TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS="${MAX_DEADLINE_WAIT_MS}" \
+TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_ENABLED=1 \
+TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_SECONDS=3.0 \
+TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_TARGET_FRAMES="${FRAME_CREDIT_TARGET_FRAMES}" \
+TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_RESERVE_FRAMES=4 \
+TELEFUSER_ABOT_PUBLISHER_FRAME_CREDIT_GUARD_MS=50 \
+TELEFUSER_ABOT_BATCH_COMPUTE_PROFILE=h100_lf3_eager_full_pipeline_v1 \
+TELEFUSER_ABOT_BATCH_COMPUTE_SAFETY_FACTOR="${BATCH_SAFETY_FACTOR}" \
+TELEFUSER_LIVEKIT_DISPATCH_TRACE_PATH="${REPO_ROOT}/${RUN}/dispatch-trace.jsonl" \
+TELEFUSER_LIVEKIT_DISPATCH_TRACE_MAX_EVENTS=100000 \
+"${PYTHON_BIN}" -m telefuser.entrypoints.cli.main stream-serve \
+ examples/abot_world/abot_world_livekit_service.py \
+ --host 127.0.0.1 \
+ --port "${PORT}" \
+ --livekit-url "${LIVEKIT_URL}" \
+ --livekit-api-key "${LIVEKIT_API_KEY}" \
+ --livekit-api-secret "${LIVEKIT_API_SECRET}" \
+ --num-workers 4 \
+ --worker-gpu-map '0;1;2;3' \
+ --worker-mode process-nccl \
+ --max-sessions-per-worker 4 \
+ --queue-size 0 \
+ --skip-validation \
+ >"${RUN}/server.log" 2>&1 &
+SERVER_PID=$!
+printf '%s\n' "${SERVER_PID}" >"${RUN}/server.pid"
+
+for _ in $(seq 1 240); do
+ if curl --noproxy '*' -fsS "http://127.0.0.1:${PORT}/v1/service/ready" >"${RUN}/ready.json"; then
+ break
+ fi
+ if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
+ echo "Server exited before becoming ready; see ${RUN}/server.log" >&2
+ exit 1
+ fi
+ sleep 1
+done
+test -s "${RUN}/ready.json"
+curl --noproxy '*' -fsS "http://127.0.0.1:${PORT}/v1/service/metadata" >"${RUN}/metadata-before.json"
+
+PYTHONPATH="${REPO_ROOT}" "${PYTHON_BIN}" tools/validation/capture_abot_serving_metrics.py \
+ --server-url "http://127.0.0.1:${PORT}" \
+ --duration "${METRICS_DURATION_SECONDS}" \
+ --interval 1 \
+ --output-dir "${RUN}/serving_metrics" \
+ >"${RUN}/serving-metrics.log" 2>&1 &
+SERVING_METRICS_PID=$!
+
+PYTHONPATH="${REPO_ROOT}" "${PYTHON_BIN}" tools/validation/capture_gpu_nvml_metrics.py \
+ --gpu-indices "${GPU_IDS}" \
+ --duration "${METRICS_DURATION_SECONDS}" \
+ --interval 1 \
+ --output-dir "${RUN}/gpu_metrics" \
+ >"${RUN}/gpu-metrics.log" 2>&1 &
+GPU_METRICS_PID=$!
+
+PYTHONPATH="${REPO_ROOT}" "${PYTHON_BIN}" tools/validation/replay_abot_livekit_lifecycle_trace.py \
+ --scenario "${RUN}/scenario.json" \
+ --output "${RUN}/result.json" \
+ 2>&1 | tee "${RUN}/replay.log"
+
+wait "${SERVING_METRICS_PID}"
+SERVING_METRICS_PID=""
+wait "${GPU_METRICS_PID}"
+GPU_METRICS_PID=""
+
+curl --noproxy '*' -fsS "http://127.0.0.1:${PORT}/v1/service/metadata" >"${RUN}/metadata-after.json"
+
+PYTHONPATH="${REPO_ROOT}" "${PYTHON_BIN}" tools/validation/render_abot_dispatch_timeline.py \
+ --dispatch-trace "${RUN}/dispatch-trace.jsonl" \
+ --result "${RUN}/result.json" \
+ --output-dir "${RUN}/dispatch_analysis"
+
+PYTHONPATH="${REPO_ROOT}" "${PYTHON_BIN}" tools/validation/analyze_abot_serving_trace.py \
+ --serving-metrics-dir "${RUN}/serving_metrics" \
+ --gpu-metrics "${RUN}/gpu_metrics/gpu-metrics.jsonl" \
+ --output-dir "${RUN}/serving_analysis"
+
+echo "Completed: ${RUN}"
diff --git a/tools/validation/trace_abot_scheduler_timeline.py b/tools/validation/trace_abot_scheduler_timeline.py
new file mode 100644
index 00000000..7d37627a
--- /dev/null
+++ b/tools/validation/trace_abot_scheduler_timeline.py
@@ -0,0 +1,736 @@
+#!/usr/bin/env python3
+"""Produce deterministic, inspectable ABot scheduler timelines on CPU.
+
+This is a *scheduler-semantics* experiment, not a model-performance benchmark.
+It drives the production :class:`ABotWorldLiveKitService` with a small CPU fake
+pipeline, so the scheduler thread, readiness predicates, batching window,
+playout pacing, and per-session ordering are the real implementation. The
+fake pipeline only replaces DiT/VAE execution with a controlled sleep and
+records the actual calls the service makes.
+
+The default ``both`` run writes two complementary traces:
+
+* ``staggered``: three independently arriving clients stay phase-shifted and
+ demonstrate the service's singleton/time-sliced dispatches;
+* ``aligned``: the three clients become ready in one scheduler turn and
+ demonstrate a real ``generate_next_blocks(..., B=3)`` dispatch.
+
+Example (no GPU/model checkpoint required)::
+
+ python tools/validation/trace_abot_scheduler_timeline.py \
+ --scenario both --output-dir /tmp/abot-scheduler-timeline
+
+Each scenario directory contains ``timeline.json``, ``events.csv``,
+``batches.csv``, ``chunks.csv``, and ``timeline.png``. The JSON and CSVs are
+intended for paper plots and independent analysis; the PNG is deliberately
+dependency-free (Pillow only) so it can be inspected on a bare serving node.
+"""
+
+# ruff: noqa: I001
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+import queue
+import sys
+import threading
+import time
+from collections import Counter
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, Literal
+
+import torch
+from PIL import Image, ImageDraw, ImageFont
+
+# Validation tools are often invoked by absolute path from a server that also
+# has another TeleFuser checkout installed editable. Prefer this checkout so
+# the trace probes the service implementation adjacent to this file.
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+from telefuser.pipelines.abot_world.interactive import ABotWorldSessionLifecycle
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+
+
+_SCENARIOS = ("staggered", "aligned")
+_BATCH_COLORS = {
+ 1: "#e67e22", # orange: serialized singleton turn
+ 2: "#3b82f6", # blue: B=2 coalesced turn
+ 3: "#16a34a", # green: B=3 coalesced turn
+}
+
+
+@dataclass(frozen=True)
+class TimelineConfig:
+ """Configuration shared by the deterministic scheduler traces."""
+
+ sessions: int = 3
+ chunks_per_session: int = 3
+ fps: int = 12
+ frames_per_chunk: int = 12
+ control_latent_frames: int = 3
+ batching_window_ms: float = 2.0
+ fake_batch_overhead_ms: float = 4.0
+ fake_per_item_ms: float = 8.0
+ stagger_offsets_ms: tuple[float, ...] = (0.0, 260.0, 520.0)
+ output_timeout_seconds: float = 20.0
+
+ @property
+ def chunk_playout_seconds(self) -> float:
+ return self.frames_per_chunk / self.fps
+
+
+class _TimelineRecorder:
+ """Thread-safe wall-clock recorder shared by fake pipeline and clients."""
+
+ def __init__(self) -> None:
+ self.origin = time.monotonic()
+ self._lock = threading.Lock()
+ self.events: list[dict[str, Any]] = []
+ self.batches: list[dict[str, Any]] = []
+ self.chunks: list[dict[str, Any]] = []
+ self._next_batch_id = 0
+
+ def elapsed(self, timestamp: float | None = None) -> float:
+ return (time.monotonic() if timestamp is None else timestamp) - self.origin
+
+ def event(self, kind: str, *, session_id: str | None = None, **fields: Any) -> None:
+ payload: dict[str, Any] = {
+ "time_seconds": round(self.elapsed(), 6),
+ "kind": kind,
+ "session_id": session_id or "",
+ **fields,
+ }
+ with self._lock:
+ self.events.append(payload)
+
+ def begin_batch(self, session_ids: list[str], control_latent_frames: int) -> tuple[int, float]:
+ started = time.monotonic()
+ with self._lock:
+ batch_id = self._next_batch_id
+ self._next_batch_id += 1
+ self.event(
+ "batch_start",
+ batch_id=batch_id,
+ batch_size=len(session_ids),
+ session_ids="|".join(session_ids),
+ control_latent_frames=control_latent_frames,
+ )
+ return batch_id, started
+
+ def end_batch(
+ self,
+ *,
+ batch_id: int,
+ started: float,
+ session_ids: list[str],
+ control_latent_frames: int,
+ ) -> None:
+ ended = time.monotonic()
+ batch = {
+ "batch_id": batch_id,
+ "start_seconds": round(self.elapsed(started), 6),
+ "end_seconds": round(self.elapsed(ended), 6),
+ "duration_ms": round((ended - started) * 1000.0, 3),
+ "batch_size": len(session_ids),
+ "session_ids": list(session_ids),
+ "control_latent_frames": control_latent_frames,
+ }
+ with self._lock:
+ self.batches.append(batch)
+ self.event(
+ "batch_end",
+ batch_id=batch_id,
+ batch_size=len(session_ids),
+ session_ids="|".join(session_ids),
+ duration_ms=batch["duration_ms"],
+ )
+
+ def chunk(self, payload: dict[str, Any]) -> None:
+ with self._lock:
+ self.chunks.append(payload)
+
+
+class _FakePipelineSession:
+ """Small subset of a resident ABot session used by the service scheduler."""
+
+ def __init__(self, session_id: str) -> None:
+ self.session_id = session_id
+ self.next_latent_frame = 0
+ self.first_frame_latent = torch.zeros(1, 1, 1, 1, 1)
+ self.self_cache = [
+ {
+ "local_end_index": torch.zeros(1, dtype=torch.long),
+ "global_end_index": torch.zeros(1, dtype=torch.long),
+ }
+ ]
+ self.lifecycle = ABotWorldSessionLifecycle.READY
+ self.closed = False
+
+ @property
+ def is_resident(self) -> bool:
+ return self.lifecycle != ABotWorldSessionLifecycle.SUSPENDED
+
+
+class _TimelineFakePipeline:
+ """CPU pipeline whose batch calls are recorded after real service selection."""
+
+ def __init__(self, recorder: _TimelineRecorder, config: TimelineConfig) -> None:
+ self.config = SimpleNamespace(width=8, height=8)
+ self.device = torch.device("cpu")
+ self.torch_dtype = torch.float32
+ self.denoise_stage = SimpleNamespace(
+ dit=SimpleNamespace(
+ patch_size=(1, 2, 2),
+ dim=8,
+ num_heads=2,
+ num_layers=2,
+ local_attn_size=18,
+ text_len=8,
+ )
+ )
+ self.recorder = recorder
+ self.timeline_config = config
+ self.closed = False
+
+ def preload_models(self) -> None:
+ return None
+
+ def create_interactive_session(
+ self,
+ image: Image.Image,
+ prompt: str,
+ *,
+ seed: int,
+ session_id: str | None = None,
+ ) -> _FakePipelineSession:
+ del image, prompt, seed
+ if session_id is None:
+ raise ValueError("timeline harness requires a stable session id")
+ return _FakePipelineSession(session_id)
+
+ def _generate(
+ self,
+ sessions: list[_FakePipelineSession],
+ controls: list[dict[str, bool]],
+ *,
+ control_latent_frames: int,
+ ) -> list[list[Image.Image]]:
+ if len(sessions) != len(controls):
+ raise AssertionError("session/control cardinality mismatch")
+ session_ids = [session.session_id for session in sessions]
+ batch_id, started = self.recorder.begin_batch(session_ids, control_latent_frames)
+ # A controllable synthetic kernel model: it makes B=3 visibly shorter
+ # than three B=1 calls, while remaining clearly labelled synthetic.
+ sleep_seconds = (
+ self.timeline_config.fake_batch_overhead_ms + self.timeline_config.fake_per_item_ms * len(sessions)
+ ) / 1000.0
+ time.sleep(sleep_seconds)
+ results: list[list[Image.Image]] = []
+ for index, session in enumerate(sessions):
+ session.next_latent_frame += control_latent_frames
+ session.self_cache[0]["local_end_index"] += control_latent_frames
+ color = (30, (batch_id * 47 + index * 71) % 255, 70)
+ results.append(
+ [Image.new("RGB", (8, 8), color=color) for _ in range(self.timeline_config.frames_per_chunk)]
+ )
+ self.recorder.end_batch(
+ batch_id=batch_id,
+ started=started,
+ session_ids=session_ids,
+ control_latent_frames=control_latent_frames,
+ )
+ return results
+
+ def generate_next_block(
+ self,
+ session: _FakePipelineSession,
+ controls: dict[str, bool],
+ *,
+ control_latent_frames: int,
+ ) -> list[Image.Image]:
+ return self._generate([session], [controls], control_latent_frames=control_latent_frames)[0]
+
+ def generate_next_blocks(
+ self,
+ sessions: list[_FakePipelineSession],
+ controls: list[dict[str, bool]],
+ *,
+ control_latent_frames: int,
+ ) -> list[list[Image.Image]]:
+ return self._generate(sessions, controls, control_latent_frames=control_latent_frames)
+
+ def suspend_interactive_session(self, session: _FakePipelineSession) -> None:
+ session.lifecycle = ABotWorldSessionLifecycle.SUSPENDED
+
+ def restore_interactive_session(self, session: _FakePipelineSession) -> None:
+ session.lifecycle = ABotWorldSessionLifecycle.READY
+
+ def close_interactive_session(self, session: _FakePipelineSession) -> None:
+ session.closed = True
+
+ def close(self) -> None:
+ self.closed = True
+
+
+@dataclass
+class _Consumer:
+ session_id: str
+ state: Any
+ thread: threading.Thread
+ error: BaseException | None = None
+
+
+def _consumer_loop(
+ *,
+ service: ABotWorldLiveKitService,
+ consumer: _Consumer,
+ recorder: _TimelineRecorder,
+ config: TimelineConfig,
+) -> None:
+ """Consume chunks at the requested playback cadence, like a real publisher."""
+ received = 0
+ try:
+ preview = consumer.state.output_queue.get(timeout=config.output_timeout_seconds)
+ if preview.get("type") != "preview":
+ raise RuntimeError(f"{consumer.session_id}: expected preview, got {preview.get('type')!r}")
+ recorder.event("preview_dequeued", session_id=consumer.session_id)
+ with service._scheduler_condition: # noqa: SLF001 - consumer notification mirrors pull_chunks().
+ service._scheduler_condition.notify_all() # noqa: SLF001
+ while received < config.chunks_per_session:
+ payload = consumer.state.output_queue.get(timeout=config.output_timeout_seconds)
+ with service._scheduler_condition: # noqa: SLF001
+ service._scheduler_condition.notify_all() # noqa: SLF001
+ if payload.get("type") == "error":
+ raise RuntimeError(f"{consumer.session_id}: service error: {payload.get('error')}")
+ if payload.get("type") != "chunk":
+ continue
+ dequeued_at = recorder.elapsed()
+ scheduler = dict(payload.get("scheduler", {}))
+ chunk = {
+ "session_id": consumer.session_id,
+ "chunk_index": int(payload.get("index", -1)),
+ "dequeued_seconds": round(dequeued_at, 6),
+ "frames": len(payload.get("frames", [])),
+ "batch_size": int(scheduler.get("batch_size", 0)),
+ "queue_wait_ms": round(float(scheduler.get("queue_wait_seconds", 0.0)) * 1000.0, 3),
+ "compute_ms": round(float(scheduler.get("compute_seconds", 0.0)) * 1000.0, 3),
+ }
+ recorder.chunk(chunk)
+ recorder.event(
+ "chunk_dequeued",
+ session_id=consumer.session_id,
+ chunk_index=chunk["chunk_index"],
+ batch_size=chunk["batch_size"],
+ )
+ received += 1
+ if received == config.chunks_per_session:
+ # Avoid a free-running extra prefetch after the measured tail.
+ service.push_chunk(consumer.session_id, {"type": "control_state", "controls": []})
+ recorder.event("control_released", session_id=consumer.session_id)
+ recorder.event("playback_start", session_id=consumer.session_id, chunk_index=chunk["chunk_index"])
+ time.sleep(config.chunk_playout_seconds)
+ recorder.event("playback_end", session_id=consumer.session_id, chunk_index=chunk["chunk_index"])
+ except BaseException as exc: # pragma: no cover - surfaced deterministically by caller.
+ consumer.error = exc
+
+
+def _make_consumer(
+ service: ABotWorldLiveKitService,
+ session_id: str,
+ recorder: _TimelineRecorder,
+ config: TimelineConfig,
+) -> _Consumer:
+ state = service._session(session_id) # noqa: SLF001 - this is an in-process scheduler probe.
+ if state is None:
+ raise KeyError(session_id)
+ placeholder = _Consumer(session_id=session_id, state=state, thread=threading.Thread())
+ thread = threading.Thread(
+ target=_consumer_loop,
+ kwargs={"service": service, "consumer": placeholder, "recorder": recorder, "config": config},
+ name=f"abot-timeline-consumer-{session_id}",
+ daemon=True,
+ )
+ placeholder.thread = thread
+ thread.start()
+ return placeholder
+
+
+def _arrival_offsets(scenario: Literal["staggered", "aligned"], config: TimelineConfig) -> tuple[float, ...]:
+ if scenario == "aligned":
+ return (0.0,) * config.sessions
+ if len(config.stagger_offsets_ms) != config.sessions:
+ raise ValueError(
+ f"--stagger-offsets-ms must contain exactly {config.sessions} values, got {len(config.stagger_offsets_ms)}"
+ )
+ offsets = tuple(value / 1000.0 for value in config.stagger_offsets_ms)
+ if offsets[0] != 0.0 or any(right < left for left, right in zip(offsets, offsets[1:])):
+ raise ValueError("stagger offsets must start at 0 and be non-decreasing")
+ return offsets
+
+
+def run_scenario(
+ scenario: Literal["staggered", "aligned"],
+ config: TimelineConfig,
+) -> dict[str, Any]:
+ """Run one trace against the production scheduler with the CPU fake backend."""
+ if config.sessions != 3:
+ raise ValueError("this focused harness intentionally requires exactly three sessions")
+ if config.chunks_per_session < 1 or config.fps < 1 or config.frames_per_chunk < 1:
+ raise ValueError("chunks-per-session, fps, and frames-per-chunk must be positive")
+ if config.control_latent_frames not in {1, 2, 3}:
+ raise ValueError("control-latent-frames must be one of 1, 2, 3")
+ if config.batching_window_ms < 0 or config.fake_batch_overhead_ms < 0 or config.fake_per_item_ms < 0:
+ raise ValueError("timing values must be non-negative")
+
+ recorder = _TimelineRecorder()
+ pipeline = _TimelineFakePipeline(recorder, config)
+ service = ABotWorldLiveKitService(
+ pipeline,
+ default_fps=config.fps,
+ default_session_config={"prompt": "ABot scheduler timeline probe"},
+ output_queue_size=4,
+ control_idle_timeout=30.0,
+ idle_suspension_seconds=600.0,
+ max_batch_size=3,
+ batching_window_ms=config.batching_window_ms,
+ scheduler_mode="batched",
+ )
+ session_ids: list[str] = []
+ consumers: list[_Consumer] = []
+ offsets = _arrival_offsets(scenario, config)
+ try:
+ service.configure_session_capacity(config.sessions)
+ service.start()
+ if scenario == "aligned":
+ # The pause is a harness-level barrier: session creation and control
+ # activation complete before the real scheduler examines readiness.
+ # It does not change scheduler selection or service code.
+ with service._scheduler_condition: # noqa: SLF001
+ service._scheduler_paused = True # noqa: SLF001
+ recorder.event("scheduler_barrier_closed")
+
+ for index, offset in enumerate(offsets):
+ target = recorder.origin + offset
+ remaining = target - time.monotonic()
+ if remaining > 0:
+ time.sleep(remaining)
+ session_id = f"user-{index + 1}"
+ service.create_session(
+ {
+ "session_id": session_id,
+ "image": Image.new("RGB", (8, 8), color=(30, index * 50, 90)),
+ "prompt": "ABot scheduler timeline probe",
+ "seed": 100 + index,
+ "fps": config.fps,
+ "control_latent_frames": config.control_latent_frames,
+ "delivery_mode": "latest",
+ }
+ )
+ session_ids.append(session_id)
+ recorder.event("session_created", session_id=session_id, arrival_offset_ms=round(offset * 1000.0, 3))
+ consumers.append(_make_consumer(service, session_id, recorder, config))
+ if scenario == "staggered":
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
+ recorder.event("control_activated", session_id=session_id)
+
+ if scenario == "aligned":
+ for session_id in session_ids:
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
+ recorder.event("control_activated", session_id=session_id)
+ with service._scheduler_condition: # noqa: SLF001
+ service._scheduler_paused = False # noqa: SLF001
+ service._scheduler_condition.notify_all() # noqa: SLF001
+ recorder.event("scheduler_barrier_opened")
+
+ join_timeout = config.output_timeout_seconds + config.chunks_per_session * config.chunk_playout_seconds + 5.0
+ for consumer in consumers:
+ consumer.thread.join(timeout=join_timeout)
+ if consumer.thread.is_alive():
+ raise TimeoutError(f"{consumer.session_id} did not consume requested chunks")
+ if consumer.error is not None:
+ raise consumer.error
+ finally:
+ for session_id in session_ids:
+ service.close_session(session_id, timeout=5.0)
+ service.stop(close_pipeline=True)
+
+ batches = sorted(recorder.batches, key=lambda value: int(value["batch_id"]))
+ chunks = sorted(recorder.chunks, key=lambda value: (str(value["session_id"]), int(value["chunk_index"])))
+ events = sorted(recorder.events, key=lambda value: float(value["time_seconds"]))
+ histogram = Counter(int(batch["batch_size"]) for batch in batches)
+ expected_chunks = config.sessions * config.chunks_per_session
+ if len(chunks) != expected_chunks:
+ raise RuntimeError(f"expected {expected_chunks} measured chunks, received {len(chunks)}")
+ overlaps = sum(
+ 1
+ for earlier, later in zip(batches, batches[1:])
+ if float(later["start_seconds"]) < float(earlier["end_seconds"])
+ )
+ session_chunk_counts = Counter(str(chunk["session_id"]) for chunk in chunks)
+ summary = {
+ "batch_calls": len(batches),
+ "batch_items": sum(int(batch["batch_size"]) for batch in batches),
+ "batch_size_histogram": {str(key): histogram[key] for key in sorted(histogram)},
+ "mean_batch_size": round(
+ sum(int(batch["batch_size"]) for batch in batches) / len(batches) if batches else 0.0,
+ 6,
+ ),
+ "singleton_dispatch_fraction": round(
+ histogram[1] / len(batches) if batches else 0.0,
+ 6,
+ ),
+ "maximum_observed_batch_size": max(histogram, default=0),
+ "overlapping_gpu_batch_calls": overlaps,
+ "serialized_scheduler_thread": overlaps == 0,
+ "per_session_measured_chunks": dict(sorted(session_chunk_counts.items())),
+ "first_batch_size": int(batches[0]["batch_size"]) if batches else 0,
+ "classification": (
+ "time_sliced_singletons"
+ if batches and all(int(batch["batch_size"]) == 1 for batch in batches)
+ else "coalesced_microbatching"
+ ),
+ }
+ return {
+ "schema_version": 1,
+ "backend": {
+ "kind": "cpu_fake_pipeline_with_production_abot_service_scheduler",
+ "claim": (
+ "Batch membership/timing comes from ABotWorldLiveKitService. "
+ "The synthetic sleep is not a DiT/VAE performance measurement."
+ ),
+ "synthetic_kernel_model": {
+ "batch_overhead_ms": config.fake_batch_overhead_ms,
+ "per_item_ms": config.fake_per_item_ms,
+ "batch_duration_ms_formula": "overhead_ms + per_item_ms * batch_size",
+ },
+ },
+ "scenario": {
+ "name": scenario,
+ "scheduler_mode": "batched",
+ "max_batch_size": 3,
+ "delivery_mode": "latest",
+ "session_arrival_offsets_ms": [round(offset * 1000.0, 3) for offset in offsets],
+ "aligned_activation_barrier": scenario == "aligned",
+ "plausible_chunk_playout_ms": round(config.chunk_playout_seconds * 1000.0, 3),
+ **asdict(config),
+ },
+ "summary": summary,
+ "batches": batches,
+ "chunks": chunks,
+ "events": events,
+ "service_runtime_metrics": service.runtime_metrics(),
+ }
+
+
+def _write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+ fields = sorted({key for row in rows for key in row})
+ with path.open("w", newline="", encoding="utf-8") as handle:
+ writer = csv.DictWriter(handle, fieldnames=fields)
+ writer.writeheader()
+ for row in rows:
+ writer.writerow({key: "|".join(value) if isinstance(value, list) else value for key, value in row.items()})
+
+
+def _draw_timeline(result: dict[str, Any], path: Path) -> None:
+ """Render a paper-friendly lane chart without requiring matplotlib."""
+ batches = list(result["batches"])
+ events = list(result["events"])
+ scenario = dict(result["scenario"])
+ sessions = [f"user-{index}" for index in range(1, 4)]
+ maximum = max(
+ [0.001]
+ + [float(batch["end_seconds"]) for batch in batches]
+ + [float(event["time_seconds"]) for event in events]
+ )
+ maximum = max(maximum * 1.08, 0.1)
+ width, left, right = 1600, 170, 60
+ top, lane_height, bottom = 110, 90, 95
+ height = top + lane_height * len(sessions) + bottom
+ image = Image.new("RGB", (width, height), "white")
+ draw = ImageDraw.Draw(image)
+ font = ImageFont.load_default()
+ bold = ImageFont.load_default()
+ chart_width = width - left - right
+
+ def x(value: float) -> int:
+ return left + round(chart_width * value / maximum)
+
+ title = (
+ f"ABot scheduler timeline — {scenario['name']} | "
+ f"hist={result['summary']['batch_size_histogram']} | "
+ f"mean B={result['summary']['mean_batch_size']}"
+ )
+ draw.text((20, 16), title, fill="black", font=bold)
+ subtitle = (
+ "Native ABot model and production service scheduler"
+ if str(result["backend"]["kind"]).startswith("native_")
+ else "CPU fake compute; batch membership is selected by the production ABotWorldLiveKitService scheduler"
+ )
+ draw.text((20, 36), subtitle, fill="#444444", font=font)
+ draw.text(
+ (20, 56),
+ "orange=B1 singleton, blue=B2, green=B3; vertical ticks: A=arrival, C=control",
+ fill="#444444",
+ font=font,
+ )
+
+ for index, session_id in enumerate(sessions):
+ center_y = top + index * lane_height + lane_height // 2
+ draw.line((left, center_y, width - right, center_y), fill="#d1d5db", width=1)
+ draw.text((18, center_y - 6), session_id, fill="black", font=font)
+
+ # Grid and ticks are generated from the observed wall-clock span.
+ tick_count = 8
+ for index in range(tick_count + 1):
+ value = maximum * index / tick_count
+ px = x(value)
+ draw.line((px, top - 12, px, height - bottom + 6), fill="#f0f0f0", width=1)
+ draw.text((px - 14, height - bottom + 18), f"{value * 1000:.0f}", fill="#555555", font=font)
+ draw.text((width // 2 - 42, height - 28), "elapsed milliseconds", fill="#333333", font=font)
+
+ for batch in batches:
+ start = x(float(batch["start_seconds"]))
+ end = max(start + 4, x(float(batch["end_seconds"])))
+ batch_size = int(batch["batch_size"])
+ color = _BATCH_COLORS.get(batch_size, "#8b5cf6")
+ for session_id in batch["session_ids"]:
+ lane = sessions.index(session_id)
+ center_y = top + lane * lane_height + lane_height // 2
+ draw.rounded_rectangle((start, center_y - 18, end, center_y + 18), radius=4, fill=color, outline="#1f2937")
+ if end - start >= 28:
+ draw.text((start + 4, center_y - 5), f"B{batch_size}", fill="white", font=font)
+
+ for event in events:
+ session_id = str(event.get("session_id", ""))
+ if session_id not in sessions:
+ continue
+ if event["kind"] not in {"session_created", "control_activated", "control_released"}:
+ continue
+ lane = sessions.index(session_id)
+ center_y = top + lane * lane_height + lane_height // 2
+ px = x(float(event["time_seconds"]))
+ label = {"session_created": "A", "control_activated": "C", "control_released": "R"}[str(event["kind"])]
+ color = {"A": "#111827", "C": "#7c3aed", "R": "#dc2626"}[label]
+ draw.line((px, center_y - 32, px, center_y + 32), fill=color, width=2)
+ draw.text((px + 3, center_y - 33), label, fill=color, font=font)
+
+ image.save(path)
+
+
+def _write_result(output_dir: Path, result: dict[str, Any]) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ (output_dir / "timeline.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ _write_csv(output_dir / "events.csv", list(result["events"]))
+ _write_csv(output_dir / "batches.csv", list(result["batches"]))
+ _write_csv(output_dir / "chunks.csv", list(result["chunks"]))
+ _write_csv(output_dir / "summary.csv", [dict(result["summary"])])
+ _draw_timeline(result, output_dir / "timeline.png")
+
+
+def _draw_comparison(rows: list[dict[str, Any]], path: Path) -> None:
+ width, height = 900, 340
+ image = Image.new("RGB", (width, height), "white")
+ draw = ImageDraw.Draw(image)
+ font = ImageFont.load_default()
+ draw.text(
+ (20, 18), "ABot service scheduler: phase alignment changes observed microbatching", fill="black", font=font
+ )
+ max_calls = max([1] + [int(row["batch_calls"]) for row in rows])
+ for index, row in enumerate(rows):
+ x0 = 90 + index * 390
+ y_base = 275
+ bar_width = 58
+ histogram = {int(key): int(value) for key, value in dict(row["batch_size_histogram"]).items()}
+ draw.text((x0, 55), str(row["scenario"]), fill="black", font=font)
+ for batch_size in (1, 2, 3):
+ count = histogram.get(batch_size, 0)
+ bar_height = int(175 * count / max_calls)
+ left = x0 + (batch_size - 1) * 90
+ color = _BATCH_COLORS[batch_size]
+ draw.rectangle((left, y_base - bar_height, left + bar_width, y_base), fill=color, outline="#1f2937")
+ draw.text((left + 20, y_base - bar_height - 17), str(count), fill="black", font=font)
+ draw.text((left + 19, y_base + 8), f"B{batch_size}", fill="#333333", font=font)
+ draw.text((x0, 306), f"mean batch={float(row['mean_batch_size']):.2f}", fill="#333333", font=font)
+ draw.text(
+ (20, height - 18),
+ "Counts are actual service pipeline calls; only compute duration is synthetic.",
+ fill="#555555",
+ font=font,
+ )
+ image.save(path)
+
+
+def _parse_offsets(value: str) -> tuple[float, ...]:
+ try:
+ parsed = tuple(float(part.strip()) for part in value.split(",") if part.strip())
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("stagger offsets must be comma-separated milliseconds") from exc
+ if len(parsed) != 3 or any(not math.isfinite(part) or part < 0 for part in parsed):
+ raise argparse.ArgumentTypeError("stagger offsets must contain three non-negative finite milliseconds")
+ return parsed
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--scenario", choices=("staggered", "aligned", "both"), default="both")
+ parser.add_argument("--output-dir", type=Path, required=True)
+ parser.add_argument("--chunks-per-session", type=int, default=3)
+ parser.add_argument("--fps", type=int, default=12)
+ parser.add_argument("--frames-per-chunk", type=int, default=12)
+ parser.add_argument("--control-latent-frames", type=int, choices=(1, 2, 3), default=3)
+ parser.add_argument("--batching-window-ms", type=float, default=2.0)
+ parser.add_argument("--fake-batch-overhead-ms", type=float, default=4.0)
+ parser.add_argument("--fake-per-item-ms", type=float, default=8.0)
+ parser.add_argument("--stagger-offsets-ms", type=_parse_offsets, default=(0.0, 260.0, 520.0))
+ parser.add_argument("--output-timeout-seconds", type=float, default=20.0)
+ args = parser.parse_args()
+ if args.chunks_per_session < 1 or args.fps < 1 or args.frames_per_chunk < 1:
+ parser.error("chunks-per-session, fps, and frames-per-chunk must be positive")
+ if args.batching_window_ms < 0 or args.fake_batch_overhead_ms < 0 or args.fake_per_item_ms < 0:
+ parser.error("timing arguments must be non-negative")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ config = TimelineConfig(
+ chunks_per_session=args.chunks_per_session,
+ fps=args.fps,
+ frames_per_chunk=args.frames_per_chunk,
+ control_latent_frames=args.control_latent_frames,
+ batching_window_ms=args.batching_window_ms,
+ fake_batch_overhead_ms=args.fake_batch_overhead_ms,
+ fake_per_item_ms=args.fake_per_item_ms,
+ stagger_offsets_ms=args.stagger_offsets_ms,
+ output_timeout_seconds=args.output_timeout_seconds,
+ )
+ scenarios: tuple[Literal["staggered", "aligned"], ...] = _SCENARIOS if args.scenario == "both" else (args.scenario,) # type: ignore[assignment]
+ comparison_rows: list[dict[str, Any]] = []
+ for scenario in scenarios:
+ result = run_scenario(scenario, config)
+ _write_result(args.output_dir / scenario, result)
+ comparison_rows.append({"scenario": scenario, **result["summary"]})
+ print(
+ json.dumps(
+ {
+ "scenario": scenario,
+ "summary": result["summary"],
+ "timeline": str((args.output_dir / scenario / "timeline.png").resolve()),
+ },
+ sort_keys=True,
+ ),
+ flush=True,
+ )
+ _write_csv(args.output_dir / "comparison.csv", comparison_rows)
+ if len(comparison_rows) > 1:
+ _draw_comparison(comparison_rows, args.output_dir / "comparison.png")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/trace_abot_scheduler_timeline_native.py b/tools/validation/trace_abot_scheduler_timeline_native.py
new file mode 100755
index 00000000..665a78c8
--- /dev/null
+++ b/tools/validation/trace_abot_scheduler_timeline_native.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""Trace three native ABot sessions through the production service scheduler.
+
+This companion to ``trace_abot_scheduler_timeline.py`` is deliberately small:
+it uses the exact same ``ABotWorldLiveKitService`` and the native
+``ABotWorldInteractivePipeline``. The only instrumentation is an instance
+wrapper around ``generate_next_blocks`` that records the actual session IDs,
+batch size, and wall-clock duration of each model invocation. It does not
+change service or model source code.
+
+Unlike the CPU semantic probe, session initialization is completed before the
+trace clock starts. The experiment therefore isolates the relevant event for
+continuous batching: when independently retained sessions become *control
+ready*. ``aligned`` uses a scheduler barrier to make the three controls ready
+in one scheduler turn; ``staggered`` activates those controls at the supplied
+offsets.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import threading
+import time
+from collections import Counter
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Literal
+
+from PIL import Image
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+from examples.abot_world._loader import DEFAULT_PROMPT, get_pipeline
+from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+from telefuser.pipelines.abot_world.service import ABotWorldLiveKitService
+from tools.validation import trace_abot_scheduler_timeline as common
+
+
+def _record_native_pipeline_calls(pipeline: ABotWorldInteractivePipeline, recorder: common._TimelineRecorder) -> None:
+ """Record actual native B=1/B>1 calls without altering source code."""
+ original = pipeline.generate_next_blocks
+
+ def traced(
+ sessions: list[Any],
+ actions: list[Any],
+ *,
+ control_latent_frames: int = 3,
+ ) -> list[list[Image.Image]]:
+ session_ids = [str(session.session_id) for session in sessions]
+ batch_id, started = recorder.begin_batch(session_ids, control_latent_frames)
+ try:
+ return original(sessions, actions, control_latent_frames=control_latent_frames)
+ except Exception as exc:
+ recorder.event(
+ "batch_error",
+ batch_id=batch_id,
+ batch_size=len(session_ids),
+ session_ids="|".join(session_ids),
+ error=repr(exc),
+ )
+ raise
+ finally:
+ recorder.end_batch(
+ batch_id=batch_id,
+ started=started,
+ session_ids=session_ids,
+ control_latent_frames=control_latent_frames,
+ )
+
+ # Instance assignment means generate_next_block() also reaches this
+ # wrapper, because its implementation calls self.generate_next_blocks().
+ pipeline.generate_next_blocks = traced # type: ignore[method-assign]
+
+
+def _wait_for_previews(recorder: common._TimelineRecorder, expected: int, timeout: float) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ with recorder._lock: # noqa: SLF001 - harness-owned recorder.
+ seen = sum(event["kind"] == "preview_dequeued" for event in recorder.events)
+ if seen >= expected:
+ return
+ time.sleep(0.002)
+ raise TimeoutError(f"only {seen}/{expected} session previews were consumed")
+
+
+def _reset_trace_clock(recorder: common._TimelineRecorder) -> None:
+ """Exclude model preload/session initialization from scheduling evidence."""
+ with recorder._lock: # noqa: SLF001 - harness-owned recorder.
+ recorder.origin = time.monotonic()
+ recorder.events.clear()
+ recorder.batches.clear()
+ recorder.chunks.clear()
+ recorder._next_batch_id = 0 # noqa: SLF001
+
+
+def _run_native_scenario(
+ scenario: Literal["staggered", "aligned"],
+ *,
+ config: common.TimelineConfig,
+ model_root: Path,
+ image_path: Path,
+ device_id: int,
+) -> dict[str, Any]:
+ recorder = common._TimelineRecorder()
+ pipeline = get_pipeline(
+ model_root=model_root,
+ device_id=device_id,
+ pipeline_class=ABotWorldInteractivePipeline,
+ )
+ _record_native_pipeline_calls(pipeline, recorder)
+ service = ABotWorldLiveKitService(
+ pipeline,
+ default_fps=config.fps,
+ default_session_config={"image_path": str(image_path), "prompt": DEFAULT_PROMPT},
+ output_queue_size=4,
+ control_idle_timeout=30.0,
+ idle_suspension_seconds=600.0,
+ max_batch_size=3,
+ batching_window_ms=config.batching_window_ms,
+ scheduler_mode="batched",
+ )
+ session_ids: list[str] = []
+ consumers: list[common._Consumer] = []
+ offsets = common._arrival_offsets(scenario, config)
+ runtime_metrics: dict[str, Any] = {}
+ try:
+ # This experiment has a known three-session B=3 target. Do not add a
+ # fourth hidden capacity-profiling session to a trace intended to show
+ # exactly three retained sessions.
+ service._capacity_profile = {"effective_capacity": config.sessions} # noqa: SLF001
+ service.start()
+ for index in range(config.sessions):
+ session_id = f"user-{index + 1}"
+ service.create_session(
+ {
+ "session_id": session_id,
+ "image_path": str(image_path),
+ "prompt": DEFAULT_PROMPT,
+ "seed": 100 + index,
+ "fps": config.fps,
+ "control_latent_frames": config.control_latent_frames,
+ "delivery_mode": "latest",
+ }
+ )
+ session_ids.append(session_id)
+ consumers.append(common._make_consumer(service, session_id, recorder, config))
+ _wait_for_previews(recorder, config.sessions, config.output_timeout_seconds)
+ _reset_trace_clock(recorder)
+ for session_id in session_ids:
+ recorder.event("session_prepared", session_id=session_id)
+
+ if scenario == "aligned":
+ # This only defers the existing scheduler thread. After resuming,
+ # normal batch-key and deadline checks select the actual B=3/B=1.
+ with service._scheduler_condition: # noqa: SLF001
+ service._scheduler_paused = True # noqa: SLF001
+ recorder.event("scheduler_barrier_closed")
+ for session_id in session_ids:
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
+ recorder.event("control_activated", session_id=session_id, arrival_offset_ms=0.0)
+ with service._scheduler_condition: # noqa: SLF001
+ service._scheduler_paused = False # noqa: SLF001
+ service._scheduler_condition.notify_all() # noqa: SLF001
+ recorder.event("scheduler_barrier_opened")
+ else:
+ for session_id, offset in zip(session_ids, offsets):
+ target = recorder.origin + offset
+ remaining = target - time.monotonic()
+ if remaining > 0:
+ time.sleep(remaining)
+ service.push_chunk(session_id, {"type": "control_state", "controls": ["KeyW"]})
+ recorder.event("control_activated", session_id=session_id, arrival_offset_ms=round(offset * 1000.0, 3))
+
+ join_timeout = config.output_timeout_seconds + config.chunks_per_session * config.chunk_playout_seconds + 30.0
+ for consumer in consumers:
+ consumer.thread.join(timeout=join_timeout)
+ if consumer.thread.is_alive():
+ raise TimeoutError(f"{consumer.session_id} did not consume requested chunks")
+ if consumer.error is not None:
+ raise consumer.error
+ runtime_metrics = service.runtime_metrics()
+ finally:
+ for session_id in session_ids:
+ service.close_session(session_id, timeout=20.0)
+ service.stop(close_pipeline=True)
+
+ batches = sorted(recorder.batches, key=lambda value: int(value["batch_id"]))
+ chunks = sorted(recorder.chunks, key=lambda value: (str(value["session_id"]), int(value["chunk_index"])))
+ events = sorted(recorder.events, key=lambda value: float(value["time_seconds"]))
+ expected_chunks = config.sessions * config.chunks_per_session
+ if len(chunks) != expected_chunks:
+ raise RuntimeError(f"expected {expected_chunks} measured chunks, received {len(chunks)}")
+ histogram = Counter(int(batch["batch_size"]) for batch in batches)
+ overlaps = sum(
+ 1
+ for earlier, later in zip(batches, batches[1:])
+ if float(later["start_seconds"]) < float(earlier["end_seconds"])
+ )
+ per_session = Counter(str(chunk["session_id"]) for chunk in chunks)
+ summary = {
+ "batch_calls": len(batches),
+ "batch_items": sum(int(batch["batch_size"]) for batch in batches),
+ "batch_size_histogram": {str(key): histogram[key] for key in sorted(histogram)},
+ "mean_batch_size": round(
+ sum(int(batch["batch_size"]) for batch in batches) / len(batches) if batches else 0.0,
+ 6,
+ ),
+ "singleton_dispatch_fraction": round(histogram[1] / len(batches) if batches else 0.0, 6),
+ "maximum_observed_batch_size": max(histogram, default=0),
+ "overlapping_gpu_batch_calls": overlaps,
+ "serialized_scheduler_thread": overlaps == 0,
+ "per_session_measured_chunks": dict(sorted(per_session.items())),
+ "first_batch_size": int(batches[0]["batch_size"]) if batches else 0,
+ "classification": (
+ "time_sliced_singletons"
+ if batches and all(int(batch["batch_size"]) == 1 for batch in batches)
+ else "coalesced_microbatching"
+ ),
+ }
+ return {
+ "schema_version": 1,
+ "backend": {
+ "kind": "native_abot_world_model_with_production_abot_service_scheduler",
+ "claim": (
+ "Batch membership and duration are measured around the native "
+ "ABotWorldInteractivePipeline.generate_next_blocks call."
+ ),
+ "model_root": str(model_root),
+ "image_path": str(image_path),
+ "device_id": device_id,
+ },
+ "scenario": {
+ "name": scenario,
+ "scheduler_mode": "batched",
+ "max_batch_size": 3,
+ "delivery_mode": "latest",
+ "session_arrival_offsets_ms": [round(offset * 1000.0, 3) for offset in offsets],
+ "aligned_activation_barrier": scenario == "aligned",
+ "session_preparation": "all three retained sessions are initialized before trace time zero",
+ "nominal_chunk_playout_ms": round(config.chunk_playout_seconds * 1000.0, 3),
+ **asdict(config),
+ },
+ "summary": summary,
+ "batches": batches,
+ "chunks": chunks,
+ "events": events,
+ "service_runtime_metrics": runtime_metrics,
+ }
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--scenario", choices=("staggered", "aligned", "both"), default="both")
+ parser.add_argument("--model-root", type=Path, required=True)
+ parser.add_argument("--image", type=Path, required=True)
+ parser.add_argument("--device-id", type=int, default=0)
+ parser.add_argument("--output-dir", type=Path, required=True)
+ parser.add_argument("--chunks-per-session", type=int, default=3)
+ parser.add_argument("--fps", type=int, default=12)
+ parser.add_argument("--frames-per-chunk", type=int, default=12)
+ parser.add_argument("--control-latent-frames", type=int, choices=(1, 2, 3), default=3)
+ parser.add_argument("--batching-window-ms", type=float, default=2.0)
+ parser.add_argument("--stagger-offsets-ms", type=common._parse_offsets, default=(0.0, 450.0, 900.0))
+ parser.add_argument("--output-timeout-seconds", type=float, default=120.0)
+ args = parser.parse_args()
+ if not args.model_root.is_dir() or not args.image.is_file():
+ parser.error("--model-root must be a directory and --image must be an existing image")
+ if args.chunks_per_session < 1 or args.fps < 1 or args.frames_per_chunk < 1:
+ parser.error("chunks-per-session, fps, and frames-per-chunk must be positive")
+ if args.batching_window_ms < 0:
+ parser.error("batching-window-ms must be non-negative")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ config = common.TimelineConfig(
+ chunks_per_session=args.chunks_per_session,
+ fps=args.fps,
+ frames_per_chunk=args.frames_per_chunk,
+ control_latent_frames=args.control_latent_frames,
+ batching_window_ms=args.batching_window_ms,
+ stagger_offsets_ms=args.stagger_offsets_ms,
+ output_timeout_seconds=args.output_timeout_seconds,
+ )
+ scenarios: tuple[Literal["staggered", "aligned"], ...] = (
+ common._SCENARIOS if args.scenario == "both" else (args.scenario,)
+ ) # type: ignore[assignment]
+ comparison_rows: list[dict[str, Any]] = []
+ for scenario in scenarios:
+ result = _run_native_scenario(
+ scenario,
+ config=config,
+ model_root=args.model_root.resolve(),
+ image_path=args.image.resolve(),
+ device_id=args.device_id,
+ )
+ common._write_result(args.output_dir / scenario, result)
+ comparison_rows.append({"scenario": scenario, **result["summary"]})
+ print(
+ json.dumps(
+ {
+ "scenario": scenario,
+ "summary": result["summary"],
+ "timeline": str((args.output_dir / scenario / "timeline.png").resolve()),
+ },
+ sort_keys=True,
+ ),
+ flush=True,
+ )
+ common._write_csv(args.output_dir / "comparison.csv", comparison_rows)
+ if len(comparison_rows) > 1:
+ common._draw_comparison(comparison_rows, args.output_dir / "comparison.png")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/validate_abot_cuda_graph_batch_parity.py b/tools/validation/validate_abot_cuda_graph_batch_parity.py
new file mode 100644
index 00000000..ab8b9864
--- /dev/null
+++ b/tools/validation/validate_abot_cuda_graph_batch_parity.py
@@ -0,0 +1,663 @@
+"""Strict B=2/B=3 ABot CUDA-Graph batch-continuation parity validation.
+
+This correctness tool drives the public interactive batching API twice. For
+each batch lane it creates an eager twin and a candidate CUDA-Graph twin with
+the same per-session seed, warms both *batched* groups eagerly until their
+causal KV windows are full, and then runs two B=2 or B=3 continuations through
+the ordinary eager and candidate graph-mode paths. The first candidate chunk
+must capture; the second must reuse that graph. It verifies every retained
+session state, the DiT latents handed to LightVAE, and decoded RGB frames.
+
+The result is deliberately rejected unless the candidate path reports a real
+capture and replay with no fallback. On a revision that has not implemented
+batched graphs yet, this tool therefore exits ``graph_unverified`` rather than
+mistaking the regular eager fallback for a graph result.
+
+Example (physical GPU 3 remapped to logical CUDA device 0)::
+
+ CUDA_VISIBLE_DEVICES=3 PYTHONPATH=$PWD \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/validate_abot_cuda_graph_batch_parity.py \\
+ --batch-size 2 \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir results/validation/abot_cuda_graph_batch2_parity_gpu3
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+
+def _load_base_validator() -> Any:
+ """Load the B=1 validator's model-loading and pixel-comparison helpers."""
+ path = Path(__file__).with_name("validate_abot_cuda_graph_parity.py")
+ spec = importlib.util.spec_from_file_location("abot_cuda_graph_batch_parity_base", path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot base parity validator: {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _tree_exactness(left: Any, right: Any) -> dict[str, Any]:
+ """Strictly compare an on-device retained-state tensor tree.
+
+ No cache is copied to CPU: ``torch.equal`` checks each CUDA leaf directly,
+ which keeps this useful for the multi-GB KV state of B=2/B=3 sessions.
+ """
+ tensor_leaves = 0
+ checked_leaves = 0
+ mismatches: list[str] = []
+
+ def visit(lhs: Any, rhs: Any, path: str) -> bool:
+ nonlocal checked_leaves, tensor_leaves
+ checked_leaves += 1
+ if isinstance(lhs, torch.Tensor) or isinstance(rhs, torch.Tensor):
+ tensor_leaves += 1
+ if not isinstance(lhs, torch.Tensor) or not isinstance(rhs, torch.Tensor):
+ mismatches.append(f"{path}: tensor/non-tensor type mismatch")
+ return False
+ if lhs.shape != rhs.shape or lhs.dtype != rhs.dtype or lhs.device != rhs.device:
+ mismatches.append(
+ f"{path}: tensor metadata differs "
+ f"({tuple(lhs.shape)}, {lhs.dtype}, {lhs.device}) != "
+ f"({tuple(rhs.shape)}, {rhs.dtype}, {rhs.device})"
+ )
+ return False
+ if not bool(torch.equal(lhs, rhs)):
+ mismatches.append(f"{path}: tensor values differ")
+ return False
+ return True
+ if isinstance(lhs, Mapping) or isinstance(rhs, Mapping):
+ if not isinstance(lhs, Mapping) or not isinstance(rhs, Mapping) or set(lhs) != set(rhs):
+ mismatches.append(f"{path}: mapping keys/type differ")
+ return False
+ return all(visit(lhs[key], rhs[key], f"{path}.{key}") for key in sorted(lhs, key=str))
+ if isinstance(lhs, (list, tuple)) or isinstance(rhs, (list, tuple)):
+ if not isinstance(lhs, (list, tuple)) or not isinstance(rhs, (list, tuple)) or len(lhs) != len(rhs):
+ mismatches.append(f"{path}: sequence length/type differs")
+ return False
+ return all(
+ visit(item_lhs, item_rhs, f"{path}[{index}]")
+ for index, (item_lhs, item_rhs) in enumerate(zip(lhs, rhs, strict=True))
+ )
+ if lhs != rhs:
+ mismatches.append(f"{path}: {lhs!r} != {rhs!r}")
+ return False
+ return True
+
+ exact = visit(left, right, "session")
+ return {
+ "exact": exact,
+ "checked_leaves": checked_leaves,
+ "tensor_leaves": tensor_leaves,
+ "mismatches": mismatches[:20],
+ "mismatch_count_at_least": len(mismatches),
+ }
+
+
+def _session_state_tree(session: Any, pipeline: Any) -> dict[str, Any]:
+ """Return every stateful object that may affect a later interactive chunk."""
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW decode state")
+ return {
+ "prompt_emb": session.prompt_emb,
+ "first_frame_latent": session.first_frame_latent,
+ "self_cache": session.self_cache,
+ "cross_cache": session.cross_cache,
+ "generator_state": session.generator.get_state(),
+ "wan_decode_state": {
+ "feat_cache": session.vae_decode_state.feat_cache,
+ "feat_idx": session.vae_decode_state.feat_idx,
+ },
+ "taew_decode_state": pipeline.taew_decode_stage.export_decode_state_for_nccl(session.taew_decode_state),
+ "next_latent_frame": session.next_latent_frame,
+ "emitted_frames": session.emitted_frames,
+ }
+
+
+def _compare_tensor(left: torch.Tensor | None, right: torch.Tensor | None) -> dict[str, Any]:
+ """Compare captured DiT output latents exactly without moving them to CPU."""
+ if left is None or right is None:
+ return {
+ "comparable": False,
+ "exact": False,
+ "left_captured": left is not None,
+ "right_captured": right is not None,
+ }
+ if left.shape != right.shape or left.dtype != right.dtype or left.device != right.device:
+ return {
+ "comparable": False,
+ "exact": False,
+ "left_shape": list(left.shape),
+ "right_shape": list(right.shape),
+ "left_dtype": str(left.dtype),
+ "right_dtype": str(right.dtype),
+ "left_device": str(left.device),
+ "right_device": str(right.device),
+ }
+ difference = (left.float() - right.float()).abs()
+ return {
+ "comparable": True,
+ "exact": bool(torch.equal(left, right)),
+ "shape": list(left.shape),
+ "dtype": str(left.dtype),
+ "device": str(left.device),
+ "max_abs_difference": float(difference.max().item()),
+ "mean_abs_difference": float(difference.mean().item()),
+ }
+
+
+class _DecodeLatentCapture:
+ """Record the exact DiT output consumed by the public LightVAE batch call."""
+
+ def __init__(self, decode_stage: Any) -> None:
+ self._decode_stage = decode_stage
+ self._original: Any = None
+ self.latents: torch.Tensor | None = None
+
+ def __enter__(self) -> "_DecodeLatentCapture":
+ self._original = self._decode_stage.decode_chunks
+
+ def capture(latents: torch.Tensor, *args: Any, **kwargs: Any) -> Any:
+ # Enqueue the clone before decode mutates any session-owned stream
+ # state. A later synchronize makes the captured tensor concrete.
+ self.latents = latents.detach().clone()
+ return self._original(latents, *args, **kwargs)
+
+ self._decode_stage.decode_chunks = capture
+ return self
+
+ def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
+ self._decode_stage.decode_chunks = self._original
+
+
+def _parse_actions(raw: str, batch_size: int, base: Any) -> list[dict[str, bool]]:
+ """Parse semicolon-delimited action sets, one deterministic set per lane."""
+ items = [item.strip() for item in raw.split(";") if item.strip()]
+ if len(items) < batch_size:
+ raise ValueError(
+ f"--session-actions supplies {len(items)} action sets, but batch size {batch_size} needs one per session"
+ )
+ return [base._parse_action_keys(item) for item in items[:batch_size]]
+
+
+def _all_exact(items: Sequence[Mapping[str, Any]]) -> bool:
+ return bool(items) and all(bool(item.get("exact", False)) for item in items)
+
+
+def _all_pixel_valid(items: Sequence[Mapping[str, Any]], args: argparse.Namespace) -> bool:
+ return bool(items) and all(
+ bool(item.get("comparable"))
+ and item.get("max_abs_rgb_difference") is not None
+ and item.get("mean_abs_rgb_difference") is not None
+ and item["max_abs_rgb_difference"] <= args.max_abs_rgb_difference
+ and item["mean_abs_rgb_difference"] <= args.mean_abs_rgb_difference
+ for item in items
+ )
+
+
+def _attach_batch_graph_evidence(graph: dict[str, Any], metrics: Mapping[str, Any], batch_size: int) -> dict[str, Any]:
+ """Prove this was one native B=N graph, not N singleton graph calls."""
+ observed_batch_size = metrics.get("batch_size")
+ observed_graph_batch_size = metrics.get("cuda_graph_batch_size")
+ graph["expected_batch_size"] = batch_size
+ graph["observed_batch_size"] = observed_batch_size
+ graph["batch_size_matches"] = observed_batch_size == batch_size
+ graph["observed_cuda_graph_batch_size"] = observed_graph_batch_size
+ graph["cuda_graph_batch_size_matches"] = observed_graph_batch_size == batch_size
+ graph["cuda_graph_batched"] = bool(int(metrics.get("cuda_graph_batched", 0)))
+ graph["actual_batched_graph"] = bool(
+ graph["batch_size_matches"] and graph["cuda_graph_batch_size_matches"] and graph["cuda_graph_batched"]
+ )
+ return graph
+
+
+def _batch_graph_replay_verified(metrics: Mapping[str, Any], batch_size: int, base: Any) -> dict[str, Any]:
+ """Require a reuse call to replay a graph after its prior capture."""
+ graph = _attach_batch_graph_evidence(base._graph_verified(metrics), metrics, batch_size)
+ graph["verified"] = bool(
+ graph["enabled"]
+ and graph["eligible"]
+ and graph["replay_observed"]
+ and not graph["fallback_observed"]
+ and graph["actual_batched_graph"]
+ )
+ return graph
+
+
+@torch.inference_mode()
+def _run_public_batch(
+ pipeline: Any,
+ sessions: Sequence[Any],
+ actions: Sequence[Mapping[str, bool]],
+ *,
+ control_latent_frames: int,
+ device: torch.device,
+) -> dict[str, Any]:
+ """Run one public batch and retain the exact latent passed to LightVAE."""
+ with _DecodeLatentCapture(pipeline.taew_decode_stage) as capture:
+ frames = pipeline.generate_next_blocks(
+ sessions,
+ actions,
+ control_latent_frames=control_latent_frames,
+ )
+ torch.cuda.synchronize(device)
+ return {
+ "frames": frames,
+ "latents": capture.latents,
+ "stage_metrics": dict(pipeline.last_stage_metrics()),
+ }
+
+
+def _batch_graph_verified(metrics: Mapping[str, Any], batch_size: int, base: Any) -> dict[str, Any]:
+ """Require proof that this B>1 request captured one native B=N graph."""
+ graph = _attach_batch_graph_evidence(base._graph_verified(metrics), metrics, batch_size)
+ graph["verified"] = bool(graph["verified"] and graph["actual_batched_graph"])
+ return graph
+
+
+@torch.inference_mode()
+def _run_validation(args: argparse.Namespace, base: Any) -> dict[str, Any]:
+ if not torch.cuda.is_available():
+ raise RuntimeError("batched CUDA Graph parity validation requires CUDA")
+ image = Image.open(args.image).convert("RGB")
+ pipeline = None
+ graph_sessions: list[Any] = []
+ eager_sessions: list[Any] = []
+ try:
+ pipeline = base._make_pipeline(args)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(
+ f"batched CUDA Graph parity validation requires a CUDA pipeline, got {pipeline.device!r}"
+ )
+ pipeline.preload_models()
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ actions = _parse_actions(args.session_actions, args.batch_size, base)
+ seeds = [args.seed + index * 9973 for index in range(args.batch_size)]
+ for index, seed in enumerate(seeds):
+ graph_sessions.append(
+ pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=seed,
+ session_id=f"cuda-graph-batch-{args.batch_size}-candidate-{index}",
+ )
+ )
+ eager_sessions.append(
+ pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=seed,
+ session_id=f"cuda-graph-batch-{args.batch_size}-eager-{index}",
+ )
+ )
+
+ local_attn_size = int(pipeline.denoise_stage.dit.local_attn_size)
+ warmup_chunks = base._required_warmup_chunks(
+ local_attn_size,
+ args.control_latent_frames,
+ args.extra_warmup_chunks,
+ )
+ warmup_per_session_hash_equal: list[list[bool]] = []
+ for _ in range(warmup_chunks):
+ candidate_frames = pipeline.generate_next_blocks(
+ graph_sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ )
+ eager_frames = pipeline.generate_next_blocks(
+ eager_sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ )
+ warmup_per_session_hash_equal.append(
+ [
+ base._sequence_hash(candidate) == base._sequence_hash(eager)
+ for candidate, eager in zip(candidate_frames, eager_frames, strict=True)
+ ]
+ )
+ torch.cuda.synchronize(device)
+
+ pre_continuation: list[dict[str, Any]] = []
+ for index, (candidate, eager) in enumerate(zip(graph_sessions, eager_sessions, strict=True)):
+ pre_continuation.append(
+ {
+ "session_index": index,
+ "candidate_session_id": candidate.session_id,
+ "eager_session_id": eager.session_id,
+ "candidate_cache": base._cache_readiness(candidate, pipeline),
+ "eager_cache": base._cache_readiness(eager, pipeline),
+ "state_exact": _tree_exactness(
+ _session_state_tree(candidate, pipeline),
+ _session_state_tree(eager, pipeline),
+ ),
+ }
+ )
+
+ # First advance the eager twins twice while graph execution is disabled.
+ # Then run the candidate cohort twice without a mode switch in-between:
+ # capture on round one and a real persistent-slot replay on round two.
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ eager_capture_round = _run_public_batch(
+ pipeline,
+ eager_sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ eager_replay_round = _run_public_batch(
+ pipeline,
+ eager_sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+
+ pipeline.denoise_stage.configure_cuda_graph(True)
+ graph_capture_round = _run_public_batch(
+ pipeline,
+ graph_sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ graph_runtime_after_capture = dict(pipeline.denoise_stage.cuda_graph_metrics())
+ graph_replay_round = _run_public_batch(
+ pipeline,
+ graph_sessions,
+ actions,
+ control_latent_frames=args.control_latent_frames,
+ device=device,
+ )
+ graph_runtime_after_replay = dict(pipeline.denoise_stage.cuda_graph_metrics())
+ graph_capture_verification = _batch_graph_verified(graph_capture_round["stage_metrics"], args.batch_size, base)
+ graph_replay_verification = _batch_graph_replay_verified(
+ graph_replay_round["stage_metrics"], args.batch_size, base
+ )
+ pipeline.denoise_stage.configure_cuda_graph(False)
+
+ continuation_rounds = (
+ ("capture_continuation", graph_capture_round, eager_capture_round),
+ ("replay_continuation", graph_replay_round, eager_replay_round),
+ )
+
+ per_session: list[dict[str, Any]] = []
+ for index, (candidate, eager) in enumerate(zip(graph_sessions, eager_sessions, strict=True)):
+ session_rounds: list[dict[str, Any]] = []
+ for round_name, candidate_round, eager_round in continuation_rounds:
+ candidate_latents = candidate_round["latents"]
+ eager_latents = eager_round["latents"]
+ graph_latent = candidate_latents[index : index + 1] if candidate_latents is not None else None
+ eager_latent = eager_latents[index : index + 1] if eager_latents is not None else None
+ session_rounds.append(
+ {
+ "round": round_name,
+ "latent_comparison": _compare_tensor(graph_latent, eager_latent),
+ "rgb_comparison": base._compare_frames(
+ candidate_round["frames"][index],
+ eager_round["frames"][index],
+ ),
+ }
+ )
+ per_session.append(
+ {
+ "session_index": index,
+ "seed": seeds[index],
+ "actions": actions[index],
+ "candidate_session_id": candidate.session_id,
+ "eager_session_id": eager.session_id,
+ "continuations": session_rounds,
+ "post_replay_state_exact": _tree_exactness(
+ _session_state_tree(candidate, pipeline),
+ _session_state_tree(eager, pipeline),
+ ),
+ }
+ )
+
+ precondition_valid = all(
+ bool(item["candidate_cache"]["ready"])
+ and bool(item["eager_cache"]["ready"])
+ and bool(item["state_exact"]["exact"])
+ for item in pre_continuation
+ ) and all(all(item) for item in warmup_per_session_hash_equal)
+ round_comparisons = [round_item for item in per_session for round_item in item["continuations"]]
+ latent_valid = _all_exact([item["latent_comparison"] for item in round_comparisons])
+ state_valid = _all_exact([item["post_replay_state_exact"] for item in per_session])
+ pixels_valid = _all_pixel_valid([item["rgb_comparison"] for item in round_comparisons], args)
+ if not precondition_valid:
+ status = "invalid_warmup"
+ elif not graph_capture_verification["verified"] or not graph_replay_verification["verified"]:
+ status = "graph_unverified"
+ elif not latent_valid:
+ status = "latent_mismatch"
+ elif not state_valid:
+ status = "state_mismatch"
+ elif not pixels_valid:
+ status = "pixel_mismatch"
+ else:
+ status = "pass"
+ return {
+ "status": status,
+ "device": str(device),
+ "batch_size": args.batch_size,
+ "control_latent_frames": args.control_latent_frames,
+ "session_actions": actions,
+ "session_seeds": seeds,
+ "warmup": {
+ "chunks": warmup_chunks,
+ "extra_chunks": args.extra_warmup_chunks,
+ "per_chunk_per_session_output_hash_equal": warmup_per_session_hash_equal,
+ "all_output_hashes_equal": all(all(item) for item in warmup_per_session_hash_equal),
+ "pre_continuation_sessions": pre_continuation,
+ },
+ "candidate_graph_continuations": {
+ "capture_continuation": {
+ "stage_metrics": graph_capture_round["stage_metrics"],
+ "runtime_metrics": graph_runtime_after_capture,
+ "verification": graph_capture_verification,
+ "captured_latent_batch_shape": (
+ list(graph_capture_round["latents"].shape)
+ if graph_capture_round["latents"] is not None
+ else None
+ ),
+ },
+ "replay_continuation": {
+ "stage_metrics": graph_replay_round["stage_metrics"],
+ "runtime_metrics": graph_runtime_after_replay,
+ "verification": graph_replay_verification,
+ "captured_latent_batch_shape": (
+ list(graph_replay_round["latents"].shape) if graph_replay_round["latents"] is not None else None
+ ),
+ },
+ },
+ "ordinary_batched_eager_continuations": {
+ "capture_continuation": {
+ "stage_metrics": eager_capture_round["stage_metrics"],
+ "captured_latent_batch_shape": (
+ list(eager_capture_round["latents"].shape)
+ if eager_capture_round["latents"] is not None
+ else None
+ ),
+ },
+ "replay_continuation": {
+ "stage_metrics": eager_replay_round["stage_metrics"],
+ "captured_latent_batch_shape": (
+ list(eager_replay_round["latents"].shape) if eager_replay_round["latents"] is not None else None
+ ),
+ },
+ },
+ "per_session": per_session,
+ "pixel_tolerance": {
+ "max_abs_rgb_difference": args.max_abs_rgb_difference,
+ "mean_abs_rgb_difference": args.mean_abs_rgb_difference,
+ "within_tolerance": pixels_valid,
+ },
+ }
+ finally:
+ if pipeline is not None:
+ for session in [*graph_sessions, *eager_sessions]:
+ try:
+ pipeline.close_interactive_session(session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _write_results(output_dir: Path, result: Mapping[str, Any], args: argparse.Namespace, base: Any) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ payload = {"arguments": base._json_safe(vars(args)), "result": base._json_safe(result)}
+ (output_dir / "results.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+ graph = result.get("candidate_graph_continuations", {})
+ capture = graph.get("capture_continuation", {}) if isinstance(graph, Mapping) else {}
+ replay = graph.get("replay_continuation", {}) if isinstance(graph, Mapping) else {}
+ capture_verification = capture.get("verification", {}) if isinstance(capture, Mapping) else {}
+ replay_verification = replay.get("verification", {}) if isinstance(replay, Mapping) else {}
+ lines = [
+ f"# ABot CUDA Graph B={args.batch_size} continuation parity",
+ "",
+ (
+ "Two same-seed B=N retained-session groups are warmed through the ordinary public batched path "
+ "until every KV window is full. The eager and candidate groups then run two B=N continuations: "
+ "candidate capture followed by a persistent graph replay."
+ ),
+ "",
+ (
+ "| Session | Round | Candidate / eager RGB sequence SHA-256 | Latent exact | RGB exact | "
+ "Max RGB abs diff | Final state exact |"
+ ),
+ "| ---: | --- | --- | --- | --- | ---: | --- |",
+ ]
+ for item in result.get("per_session", []):
+ state = item.get("post_replay_state_exact", {})
+ for round_item in item.get("continuations", []):
+ latent = round_item.get("latent_comparison", {})
+ rgb = round_item.get("rgb_comparison", {})
+ lines.append(
+ f"| {item.get('session_index', '')} | {round_item.get('round', '')} | "
+ f"{rgb.get('sequence_sha256_graph', '')} / {rgb.get('sequence_sha256_eager', '')} | "
+ f"{latent.get('exact', False)} | {rgb.get('all_frame_hashes_equal', False)} | "
+ f"{rgb.get('max_abs_rgb_difference', '')} | {state.get('exact', False)} |"
+ )
+ lines.extend(
+ [
+ "",
+ f"Status: `{result.get('status', 'error')}`.",
+ "",
+ f"Capture chunk capture/replay/fallback: `{capture_verification.get('captured', False)}` / "
+ f"`{capture_verification.get('replay_observed', False)}` / "
+ f"`{capture_verification.get('fallback_observed', False)}`.",
+ "",
+ f"Reuse chunk replay/fallback: `{replay_verification.get('replay_observed', False)}` / "
+ f"`{replay_verification.get('fallback_observed', False)}`.",
+ "",
+ (
+ "A passing result requires capture on the first B=N request and an actual no-fallback reuse replay "
+ "on the second, plus full-KV paired warmup and exact per-session state/latent/RGB checks. "
+ "`results.json` contains all per-frame and per-session SHA-256 values."
+ ),
+ ]
+ )
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args(base: Any) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model-root", type=Path)
+ parser.add_argument("--image", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--batch-size", type=int, choices=(2, 3), required=False, default=2)
+ parser.add_argument(
+ "--session-actions",
+ default="W;A;S",
+ help="Semicolon-separated action sets for lanes 0..B-1, e.g. 'W;A;S'; NONE means idle.",
+ )
+ parser.add_argument("--seed", type=int, default=42, help="Lane i uses seed + 9973*i in both paired groups.")
+ parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
+ parser.add_argument("--extra-warmup-chunks", type=int, default=0)
+ parser.add_argument("--max-abs-rgb-difference", type=int, default=0)
+ parser.add_argument("--mean-abs-rgb-difference", type=float, default=0.0)
+ parser.add_argument(
+ "--device-id",
+ type=int,
+ default=0,
+ help="Logical CUDA device after CUDA_VISIBLE_DEVICES remapping (normally 0).",
+ )
+ parser.add_argument("--dry-run", action="store_true", help="Print the validation plan without loading a model.")
+ args = parser.parse_args()
+ if args.extra_warmup_chunks < 0:
+ parser.error("--extra-warmup-chunks must be non-negative")
+ if args.max_abs_rgb_difference < 0 or args.mean_abs_rgb_difference < 0:
+ parser.error("pixel-difference tolerances must be non-negative")
+ try:
+ _parse_actions(args.session_actions, args.batch_size, base)
+ except (ValueError, argparse.ArgumentTypeError) as exc:
+ parser.error(str(exc))
+ if not args.dry_run:
+ if args.model_root is None or args.image is None or args.output_dir is None:
+ parser.error("--model-root, --image, and --output-dir are required unless --dry-run is used")
+ if not args.model_root.is_dir():
+ parser.error(f"model root does not exist: {args.model_root}")
+ if not args.image.is_file():
+ parser.error(f"image does not exist: {args.image}")
+ return args
+
+
+def main() -> None:
+ base = _load_base_validator()
+ args = _parse_args(base)
+ if args.dry_run:
+ print(
+ json.dumps(
+ {
+ "mode": "dry_run",
+ "batch_size": args.batch_size,
+ "paired_groups": ["candidate_cuda_graph", "ordinary_batched_eager"],
+ "session_actions": _parse_actions(args.session_actions, args.batch_size, base),
+ "warmup": "both B=N groups batched eagerly until all per-session KV windows are full",
+ "candidate_rounds": ["capture_continuation", "persistent_replay_continuation"],
+ "candidate_gate": (
+ "requires native cuda_graph_batch_size=B, cuda_graph_batched=1, capture/replay, and no fallback"
+ ),
+ "comparisons_per_session": ["pre/post retained state", "DiT latent", "RGB frame SHA-256/pixels"],
+ "control_latent_frames": args.control_latent_frames,
+ "pixel_tolerance": {
+ "max_abs_rgb_difference": args.max_abs_rgb_difference,
+ "mean_abs_rgb_difference": args.mean_abs_rgb_difference,
+ },
+ },
+ indent=2,
+ sort_keys=True,
+ )
+ )
+ return
+ assert args.output_dir is not None
+ try:
+ result = _run_validation(args, base)
+ except Exception as exc:
+ result = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
+ _write_results(args.output_dir, result, args, base)
+ print(json.dumps(base._json_safe(result), indent=2, sort_keys=True))
+ if result.get("status") != "pass":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/validate_abot_cuda_graph_parity.py b/tools/validation/validate_abot_cuda_graph_parity.py
new file mode 100644
index 00000000..e52d382e
--- /dev/null
+++ b/tools/validation/validate_abot_cuda_graph_parity.py
@@ -0,0 +1,512 @@
+"""Validate ABot-World's real CUDA-Graph continuation against eager output.
+
+This is deliberately a correctness tool, not a throughput benchmark. It
+loads one interactive pipeline, creates two independent B=1 retained
+sessions with the same seed, and advances both through the normal eager path
+until their causal KV windows are full. One subsequent continuation uses the
+experimental CUDA-Graph path while its twin stays eager. The resulting
+rendered frames are compared byte-for-byte and by RGB absolute pixel error.
+
+The graph result is accepted only when the serving stage explicitly reports a
+capture and at least one graph replay without a fallback. This avoids
+mistaking the safety fallback for a graph correctness result.
+
+Example (GPU 3 remapped to logical CUDA device 0)::
+
+ CUDA_VISIBLE_DEVICES=3 \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/validate_abot_cuda_graph_parity.py \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir results/validation/abot_cuda_graph_parity_gpu3
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.util
+import json
+import math
+import os
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image, ImageChops
+
+_GRAPH_ENV = "TELEFUSER_ABOT_CUDA_GRAPH_ENABLED"
+_ACTION_KEYS = ("W", "A", "S", "D", "I", "J", "K", "L")
+
+
+def _load_example_loader() -> Any:
+ loader_path = Path(__file__).resolve().parents[2] / "examples/abot_world/_loader.py"
+ spec = importlib.util.spec_from_file_location("abot_cuda_graph_parity_loader", loader_path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load ABot example loader: {loader_path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _parse_action_keys(value: str) -> dict[str, bool]:
+ keys = [item.strip().upper() for item in value.split(",") if item.strip()]
+ if len(keys) == 1 and keys[0] in {"NONE", "IDLE"}:
+ return {}
+ unknown = sorted(set(keys).difference(_ACTION_KEYS))
+ if unknown:
+ raise argparse.ArgumentTypeError(f"unknown ABot action keys: {', '.join(unknown)}")
+ return {key: True for key in keys}
+
+
+def _json_safe(value: Any) -> Any:
+ if isinstance(value, Path):
+ return str(value)
+ if isinstance(value, torch.Tensor):
+ if value.numel() == 1:
+ return value.item()
+ return {"tensor_shape": list(value.shape), "tensor_dtype": str(value.dtype)}
+ if isinstance(value, Mapping):
+ return {str(key): _json_safe(item) for key, item in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [_json_safe(item) for item in value]
+ if isinstance(value, (str, int, float, bool)) or value is None:
+ return value
+ return str(value)
+
+
+def _frame_hash(frame: Image.Image) -> str:
+ """Hash RGB pixels together with dimensions, independent of PIL metadata."""
+ rgb = frame.convert("RGB")
+ digest = hashlib.sha256()
+ digest.update(f"RGB:{rgb.width}x{rgb.height}:".encode("ascii"))
+ digest.update(rgb.tobytes())
+ return digest.hexdigest()
+
+
+def _sequence_hash(frames: Sequence[Image.Image]) -> str:
+ digest = hashlib.sha256()
+ for frame in frames:
+ digest.update(bytes.fromhex(_frame_hash(frame)))
+ return digest.hexdigest()
+
+
+def _compare_frames(graph_frames: Sequence[Image.Image], eager_frames: Sequence[Image.Image]) -> dict[str, Any]:
+ """Compare rendered RGB frames without adding a NumPy dependency.
+
+ ``PIL.ImageChops`` calculates the per-value absolute difference in C; the
+ histogram then gives exact maximum/mean differences over all RGB values.
+ """
+ graph_hashes = [_frame_hash(frame) for frame in graph_frames]
+ eager_hashes = [_frame_hash(frame) for frame in eager_frames]
+ details: list[dict[str, Any]] = []
+ total_absolute_difference = 0
+ total_values = 0
+ nonzero_values = 0
+ maximum_absolute_difference = 0
+ comparable = len(graph_frames) == len(eager_frames)
+
+ for index, (graph_frame, eager_frame) in enumerate(zip(graph_frames, eager_frames, strict=False)):
+ graph_rgb = graph_frame.convert("RGB")
+ eager_rgb = eager_frame.convert("RGB")
+ identical_shape = graph_rgb.size == eager_rgb.size
+ frame_maximum = None
+ frame_mean = None
+ frame_nonzero = None
+ if identical_shape:
+ histogram = ImageChops.difference(graph_rgb, eager_rgb).histogram()
+ channel_total = 0
+ channel_values = 0
+ channel_nonzero = 0
+ channel_maximum = 0
+ for channel_index in range(3):
+ channel_histogram = histogram[channel_index * 256 : (channel_index + 1) * 256]
+ channel_total += sum(value * count for value, count in enumerate(channel_histogram))
+ channel_values += sum(channel_histogram)
+ channel_nonzero += sum(channel_histogram[1:])
+ channel_maximum = max(
+ channel_maximum, max((value for value, count in enumerate(channel_histogram) if count), default=0)
+ )
+ total_absolute_difference += channel_total
+ total_values += channel_values
+ nonzero_values += channel_nonzero
+ maximum_absolute_difference = max(maximum_absolute_difference, channel_maximum)
+ frame_maximum = channel_maximum
+ frame_mean = (channel_total / channel_values) if channel_values else 0.0
+ frame_nonzero = channel_nonzero
+ else:
+ comparable = False
+ details.append(
+ {
+ "frame_index": index,
+ "graph_sha256": graph_hashes[index],
+ "eager_sha256": eager_hashes[index],
+ "hash_equal": graph_hashes[index] == eager_hashes[index],
+ "graph_size": list(graph_rgb.size),
+ "eager_size": list(eager_rgb.size),
+ "max_abs_rgb_difference": frame_maximum,
+ "mean_abs_rgb_difference": frame_mean,
+ "nonzero_rgb_values": frame_nonzero,
+ }
+ )
+
+ return {
+ "comparable": comparable,
+ "frame_count_graph": len(graph_frames),
+ "frame_count_eager": len(eager_frames),
+ "sequence_sha256_graph": _sequence_hash(graph_frames),
+ "sequence_sha256_eager": _sequence_hash(eager_frames),
+ "all_frame_hashes_equal": graph_hashes == eager_hashes,
+ "max_abs_rgb_difference": maximum_absolute_difference if comparable else None,
+ "mean_abs_rgb_difference": (total_absolute_difference / total_values) if total_values else None,
+ "nonzero_rgb_values": nonzero_values if comparable else None,
+ "total_rgb_values": total_values if comparable else None,
+ "frames": details,
+ }
+
+
+def _cache_readiness(session: Any, pipeline: Any) -> dict[str, Any]:
+ """Return only compact metadata proving that a session reached full KV."""
+ dit = pipeline.denoise_stage.dit
+ latent_height, latent_width = session.first_frame_latent.shape[-2:]
+ frame_tokens = (latent_height // dit.patch_size[1]) * (latent_width // dit.patch_size[2])
+ expected_capacity = dit.local_attn_size * frame_tokens
+ expected_global_end = session.next_latent_frame * frame_tokens
+ local_ends = [int(layer["local_end_index"].item()) for layer in session.self_cache]
+ global_ends = [int(layer["global_end_index"].item()) for layer in session.self_cache]
+ cache_capacities = [int(layer["k"].shape[1]) for layer in session.self_cache]
+ cross_initialized = sum(bool(layer["is_init"]) for layer in session.cross_cache)
+ cross_lengths = sorted({int(layer["sequence_length"]) for layer in session.cross_cache})
+ ready = (
+ bool(session.self_cache)
+ and all(value == expected_capacity for value in local_ends)
+ and all(value == expected_global_end for value in global_ends)
+ and all(value == expected_capacity for value in cache_capacities)
+ and cross_initialized == len(session.cross_cache)
+ )
+ return {
+ "ready": ready,
+ "next_latent_frame": int(session.next_latent_frame),
+ "frame_tokens": frame_tokens,
+ "local_attn_size_frames": int(dit.local_attn_size),
+ "expected_capacity_tokens": expected_capacity,
+ "expected_global_end_tokens": expected_global_end,
+ "self_cache_layers": len(session.self_cache),
+ "unique_local_end_tokens": sorted(set(local_ends)),
+ "unique_global_end_tokens": sorted(set(global_ends)),
+ "unique_capacity_tokens": sorted(set(cache_capacities)),
+ "cross_attention_initialized_layers": cross_initialized,
+ "cross_attention_sequence_lengths": cross_lengths,
+ }
+
+
+def _required_warmup_chunks(local_attn_size: int, control_latent_frames: int, extra_chunks: int) -> int:
+ if local_attn_size < 1:
+ raise ValueError("ABot local attention window must be positive")
+ if control_latent_frames < 1:
+ raise ValueError("control_latent_frames must be positive")
+ if extra_chunks < 0:
+ raise ValueError("extra warmup chunks must be non-negative")
+ return math.ceil(local_attn_size / control_latent_frames) + extra_chunks
+
+
+def _graph_verified(metrics: Mapping[str, Any]) -> dict[str, Any]:
+ captured = int(metrics.get("cuda_graph_captured", 0)) > 0
+ replays = int(metrics.get("cuda_graph_replays", 0)) > 0
+ fallback = int(metrics.get("cuda_graph_fallback", 0)) > 0
+ eligible = int(metrics.get("cuda_graph_eligible", 0)) > 0
+ enabled = int(metrics.get("cuda_graph_enabled", 0)) > 0
+ return {
+ "enabled": enabled,
+ "eligible": eligible,
+ "captured": captured,
+ "replay_observed": replays,
+ "fallback_observed": fallback,
+ "verified": enabled and eligible and captured and replays and not fallback,
+ }
+
+
+def _make_pipeline(args: argparse.Namespace) -> Any:
+ # Warmup has to be eager for both sessions. The stage is toggled later
+ # rather than loading two model copies on the same GPU.
+ original = os.environ.get(_GRAPH_ENV)
+ os.environ[_GRAPH_ENV] = "0"
+ try:
+ loader = _load_example_loader()
+ from telefuser.pipelines.abot_world.interactive import ABotWorldInteractivePipeline
+
+ return loader.get_pipeline(
+ model_root=args.model_root,
+ device_id=args.device_id,
+ pipeline_class=ABotWorldInteractivePipeline,
+ )
+ finally:
+ if original is None:
+ os.environ.pop(_GRAPH_ENV, None)
+ else:
+ os.environ[_GRAPH_ENV] = original
+
+
+def _run_validation(args: argparse.Namespace) -> dict[str, Any]:
+ if not torch.cuda.is_available():
+ raise RuntimeError("CUDA Graph parity validation requires CUDA, but torch.cuda.is_available() is false")
+ image = Image.open(args.image).convert("RGB")
+ pipeline = None
+ graph_session = None
+ eager_session = None
+ try:
+ pipeline = _make_pipeline(args)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(f"CUDA Graph parity validation requires a CUDA pipeline, got {pipeline.device!r}")
+ pipeline.preload_models()
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ graph_session = pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=args.seed,
+ session_id="cuda-graph-parity-graph",
+ )
+ eager_session = pipeline.create_interactive_session(
+ image,
+ args.prompt,
+ seed=args.seed,
+ session_id="cuda-graph-parity-eager",
+ )
+
+ local_attn_size = int(pipeline.denoise_stage.dit.local_attn_size)
+ warmup_chunks = _required_warmup_chunks(
+ local_attn_size,
+ args.control_latent_frames,
+ args.extra_warmup_chunks,
+ )
+ warmup_hash_equal: list[bool] = []
+ for _ in range(warmup_chunks):
+ graph_warm_frames = pipeline.generate_next_block(
+ graph_session,
+ args.action_keys,
+ control_latent_frames=args.control_latent_frames,
+ )
+ eager_warm_frames = pipeline.generate_next_block(
+ eager_session,
+ args.action_keys,
+ control_latent_frames=args.control_latent_frames,
+ )
+ warmup_hash_equal.append(_sequence_hash(graph_warm_frames) == _sequence_hash(eager_warm_frames))
+ torch.cuda.synchronize(device)
+ graph_warmup_cache = _cache_readiness(graph_session, pipeline)
+ eager_warmup_cache = _cache_readiness(eager_session, pipeline)
+ warmup_equivalent = bool(warmup_hash_equal) and all(warmup_hash_equal)
+ warmup_ready = graph_warmup_cache["ready"] and eager_warmup_cache["ready"]
+
+ # Capture/replay graph continuation from exactly the same full-window
+ # session state as the eager continuation below.
+ pipeline.denoise_stage.configure_cuda_graph(True)
+ graph_frames = pipeline.generate_next_block(
+ graph_session,
+ args.action_keys,
+ control_latent_frames=args.control_latent_frames,
+ )
+ torch.cuda.synchronize(device)
+ graph_stage_metrics = dict(pipeline.last_stage_metrics())
+ graph_runtime_metrics = dict(pipeline.denoise_stage.cuda_graph_metrics())
+ graph_verification = _graph_verified(graph_stage_metrics)
+
+ # Clear graph ownership before running the twin, so this call cannot
+ # accidentally use an existing graph slot.
+ pipeline.denoise_stage.configure_cuda_graph(False)
+ eager_frames = pipeline.generate_next_block(
+ eager_session,
+ args.action_keys,
+ control_latent_frames=args.control_latent_frames,
+ )
+ torch.cuda.synchronize(device)
+ eager_stage_metrics = dict(pipeline.last_stage_metrics())
+ comparison = _compare_frames(graph_frames, eager_frames)
+ within_pixel_tolerance = (
+ comparison["comparable"]
+ and comparison["max_abs_rgb_difference"] is not None
+ and comparison["mean_abs_rgb_difference"] is not None
+ and comparison["max_abs_rgb_difference"] <= args.max_abs_rgb_difference
+ and comparison["mean_abs_rgb_difference"] <= args.mean_abs_rgb_difference
+ )
+ if not warmup_ready or not warmup_equivalent:
+ status = "invalid_warmup"
+ elif not graph_verification["verified"]:
+ status = "graph_unverified"
+ elif not within_pixel_tolerance:
+ status = "pixel_mismatch"
+ else:
+ status = "pass"
+ return {
+ "status": status,
+ "device": str(device),
+ "control_latent_frames": args.control_latent_frames,
+ "actions": args.action_keys,
+ "seed": args.seed,
+ "warmup": {
+ "chunks": warmup_chunks,
+ "extra_chunks": args.extra_warmup_chunks,
+ "per_chunk_output_hash_equal": warmup_hash_equal,
+ "all_output_hashes_equal": warmup_equivalent,
+ "graph_session_cache": graph_warmup_cache,
+ "eager_session_cache": eager_warmup_cache,
+ },
+ "graph_continuation": {
+ "stage_metrics": graph_stage_metrics,
+ "runtime_metrics": graph_runtime_metrics,
+ "verification": graph_verification,
+ },
+ "eager_continuation": {"stage_metrics": eager_stage_metrics},
+ "comparison": comparison,
+ "pixel_tolerance": {
+ "max_abs_rgb_difference": args.max_abs_rgb_difference,
+ "mean_abs_rgb_difference": args.mean_abs_rgb_difference,
+ "within_tolerance": within_pixel_tolerance,
+ },
+ }
+ finally:
+ if pipeline is not None:
+ if graph_session is not None:
+ try:
+ pipeline.close_interactive_session(graph_session)
+ except Exception:
+ pass
+ if eager_session is not None:
+ try:
+ pipeline.close_interactive_session(eager_session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _write_results(output_dir: Path, result: Mapping[str, Any], args: argparse.Namespace) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ payload = {"arguments": _json_safe(vars(args)), "result": _json_safe(result)}
+ (output_dir / "results.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+ comparison = result.get("comparison", {})
+ graph = result.get("graph_continuation", {})
+ verification = graph.get("verification", {}) if isinstance(graph, Mapping) else {}
+ warmup = result.get("warmup", {})
+ tolerance = result.get("pixel_tolerance", {})
+ graph_cache_ready = warmup.get("graph_session_cache", {}).get("ready", False)
+ eager_cache_ready = warmup.get("eager_session_cache", {}).get("ready", False)
+ graph_frame_count = comparison.get("frame_count_graph", "")
+ eager_frame_count = comparison.get("frame_count_eager", "")
+ maximum_tolerance = tolerance.get("max_abs_rgb_difference", "")
+ mean_tolerance = tolerance.get("mean_abs_rgb_difference", "")
+ lines = [
+ "# ABot CUDA Graph continuation parity",
+ "",
+ (
+ "This validates one B=1 retained-session continuation after two same-seed sessions "
+ "were advanced eagerly to a full KV window."
+ ),
+ "",
+ "| Metric | Value |",
+ "| --- | --- |",
+ f"| Status | {result.get('status', 'error')} |",
+ f"| Device | {result.get('device', '')} |",
+ f"| Eager warmup chunks/session | {warmup.get('chunks', '')} |",
+ f"| Warmup outputs identical | {warmup.get('all_output_hashes_equal', False)} |",
+ f"| Full KV ready (graph / eager) | {graph_cache_ready} / {eager_cache_ready} |",
+ f"| Graph capture observed | {verification.get('captured', False)} |",
+ f"| Graph replay observed | {verification.get('replay_observed', False)} |",
+ f"| Graph fallback observed | {verification.get('fallback_observed', False)} |",
+ f"| Frame count (graph / eager) | {graph_frame_count} / {eager_frame_count} |",
+ f"| All frame SHA-256 equal | {comparison.get('all_frame_hashes_equal', False)} |",
+ f"| Max abs RGB difference | {comparison.get('max_abs_rgb_difference', '')} |",
+ f"| Mean abs RGB difference | {comparison.get('mean_abs_rgb_difference', '')} |",
+ f"| Accepted max / mean tolerance | {maximum_tolerance} / {mean_tolerance} |",
+ "",
+ (
+ "The full per-frame SHA-256 values, pixel differences, and CUDA-Graph stage/runtime metrics "
+ "are in `results.json`."
+ ),
+ ]
+ if result.get("error"):
+ lines.extend(["", "## Error", "", f"`{result['error']}`"])
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model-root", type=Path)
+ parser.add_argument("--image", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--action-keys", type=_parse_action_keys, default={"W": True})
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
+ parser.add_argument("--extra-warmup-chunks", type=int, default=0)
+ parser.add_argument("--max-abs-rgb-difference", type=int, default=0)
+ parser.add_argument("--mean-abs-rgb-difference", type=float, default=0.0)
+ parser.add_argument(
+ "--device-id",
+ type=int,
+ default=0,
+ help="Logical CUDA device after CUDA_VISIBLE_DEVICES remapping (normally 0).",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Print the fixed B=1 validation plan without loading a model or requiring paths.",
+ )
+ args = parser.parse_args()
+ if args.extra_warmup_chunks < 0:
+ parser.error("--extra-warmup-chunks must be non-negative")
+ if args.max_abs_rgb_difference < 0 or args.mean_abs_rgb_difference < 0:
+ parser.error("pixel-difference tolerances must be non-negative")
+ if not args.dry_run:
+ if args.model_root is None or args.image is None or args.output_dir is None:
+ parser.error("--model-root, --image, and --output-dir are required unless --dry-run is used")
+ if not args.model_root.is_dir():
+ parser.error(f"model root does not exist: {args.model_root}")
+ if not args.image.is_file():
+ parser.error(f"image does not exist: {args.image}")
+ return args
+
+
+def main() -> None:
+ args = _parse_args()
+ if args.dry_run:
+ print(
+ json.dumps(
+ {
+ "mode": "dry_run",
+ "batch_size": 1,
+ "sessions": ["graph", "eager"],
+ "warmup": "both sessions eager until the 18-latent-frame KV window is full",
+ "continuation": "one CUDA-Graph-enabled session versus one eager session",
+ "graph_verification": "requires explicit capture and replay metrics with no fallback",
+ "pixel_comparison": "RGB SHA-256 plus maximum and mean absolute per-channel difference",
+ "control_latent_frames": args.control_latent_frames,
+ "extra_warmup_chunks": args.extra_warmup_chunks,
+ "device_id": args.device_id,
+ },
+ indent=2,
+ sort_keys=True,
+ )
+ )
+ return
+
+ assert args.output_dir is not None
+ try:
+ result = _run_validation(args)
+ except Exception as exc:
+ result = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
+ _write_results(args.output_dir, result, args)
+ print(json.dumps(_json_safe(result), indent=2, sort_keys=True))
+ if result.get("status") != "pass":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/validate_abot_public_vs_steady_state.py b/tools/validation/validate_abot_public_vs_steady_state.py
new file mode 100644
index 00000000..13482add
--- /dev/null
+++ b/tools/validation/validate_abot_public_vs_steady_state.py
@@ -0,0 +1,462 @@
+"""Compare ABot's real public B=1 continuation with handwritten steady state.
+
+This is a narrow diagnostic for reconciling two different validation scopes:
+
+* the public path, ``generate_next_block -> denoise_interactive_block ->
+ _denoise_block`` with CUDA Graph explicitly disabled; and
+* the graph-shaped eager control, which invokes ``forward_steady_state`` with
+ the persistent static tensors/scratch buffers but never captures a graph.
+
+Two equal-seed sessions are warmed through public eager generation until their
+KV windows are full. The tool then hooks the *actual* public ``_denoise_block``
+call for one continuation, records its exact input contract, and runs the
+steady-state control for the twin. It checks the public call inputs, sampled
+latent, self/cross cache plus RNG state, rendered frames, and complete retained
+session state.
+
+``--static-state-timing after_public`` mirrors the three-way diagnostic's
+allocator/order. ``before_public`` creates the steady static buffers before
+the public continuation and is useful when contrasting that outcome with the
+per-step-interleaved localizer. Neither mode is a CUDA-Graph test.
+
+Example (physical GPU 3 mapped to logical CUDA device 0)::
+
+ CUDA_VISIBLE_DEVICES=3 PYTHONPATH=$PWD \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/validate_abot_public_vs_steady_state.py \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir /public/fanyk1/lwb/results/validation/abot_public_vs_steady_gpu3
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+
+
+def _load_module(filename: str, module_name: str) -> Any:
+ path = Path(__file__).with_name(filename)
+ spec = importlib.util.spec_from_file_location(module_name, path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"could not load diagnostic helper: {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _tensor_layout(value: torch.Tensor) -> dict[str, Any]:
+ return {
+ "shape": list(value.shape),
+ "stride": list(value.stride()),
+ "dtype": str(value.dtype),
+ "device": str(value.device),
+ "is_contiguous": bool(value.is_contiguous()),
+ "is_channels_last_3d": bool(value.is_contiguous(memory_format=torch.channels_last_3d)),
+ }
+
+
+def _tensor_contract(left: torch.Tensor | None, right: torch.Tensor | None) -> dict[str, Any]:
+ """Compare values and layout, deliberately not allocator-specific pointers."""
+ if left is None or right is None:
+ return {
+ "comparable": False,
+ "values_exact": False,
+ "left_present": left is not None,
+ "right_present": right is not None,
+ }
+ comparable = left.shape == right.shape and left.dtype == right.dtype and left.device == right.device
+ return {
+ "comparable": comparable,
+ "values_exact": bool(torch.equal(left, right)) if comparable else False,
+ "left_layout": _tensor_layout(left),
+ "right_layout": _tensor_layout(right),
+ }
+
+
+def _sampling_state_tree(session: Any) -> dict[str, Any]:
+ """The state that must agree before VAE decoding and counter updates."""
+ return {
+ "prompt_emb": session.prompt_emb,
+ "first_frame_latent": session.first_frame_latent,
+ "self_cache": session.self_cache,
+ "cross_cache": session.cross_cache,
+ "generator_state": session.generator.get_state(),
+ }
+
+
+def _cache_cursor_contract(caches: list[dict[str, Any]]) -> list[dict[str, int]]:
+ return [
+ {
+ "global_end_index": int(layer["global_end_index"].item()),
+ "local_end_index": int(layer["local_end_index"].item()),
+ }
+ for layer in caches
+ ]
+
+
+class _PublicBlockProbe:
+ """Capture the real singleton public ``_denoise_block`` contract once."""
+
+ def __init__(self, stage: Any) -> None:
+ self._stage = stage
+ self._original: Any = None
+ self.call_count = 0
+ self.inputs: dict[str, Any] | None = None
+ self.output: torch.Tensor | None = None
+
+ def __enter__(self) -> "_PublicBlockProbe":
+ self._original = self._stage._denoise_block
+
+ def wrapped(
+ latent: torch.Tensor,
+ prompt_emb: torch.Tensor,
+ action_context: torch.Tensor,
+ first_frame_latent: torch.Tensor | None,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ current_start: int,
+ generator: torch.Generator,
+ scheduler: Any,
+ ) -> torch.Tensor:
+ self.call_count += 1
+ if self.call_count != 1:
+ raise RuntimeError("expected exactly one public _denoise_block call for one continuation")
+ self.inputs = {
+ "latent": latent,
+ "prompt_emb": prompt_emb,
+ "action_context": action_context,
+ "first_frame_is_none": first_frame_latent is None,
+ "current_start": int(current_start),
+ "generator_state_after_input_draw": generator.get_state().clone(),
+ "self_cache_cursors_before": _cache_cursor_contract(self_cache),
+ "cross_cache_initialized_before": [bool(layer["is_init"]) for layer in cross_cache],
+ "scheduler_type": type(scheduler).__name__,
+ }
+ output = self._original(
+ latent,
+ prompt_emb,
+ action_context,
+ first_frame_latent,
+ self_cache,
+ cross_cache,
+ current_start,
+ generator,
+ scheduler,
+ )
+ self.output = output.detach().clone()
+ return output
+
+ self._stage._denoise_block = wrapped
+ return self
+
+ def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
+ self._stage._denoise_block = self._original
+
+
+def _prepare_steady_control(
+ pipeline: Any,
+ control: Any,
+ session: Any,
+ actions: Mapping[str, bool],
+ frames: int,
+) -> dict[str, Any]:
+ latent, prompt_emb, action_context = control._prepare_inputs(pipeline, [session], [actions], frames)
+ return {
+ "latent": latent,
+ "prompt_emb": prompt_emb,
+ "action_context": action_context,
+ "generator_state_after_input_draw": session.generator.get_state().clone(),
+ "current_start": int(session.next_latent_frame),
+ "state": control._static_state(pipeline, [session], latent, prompt_emb, action_context),
+ }
+
+
+def _finish_steady_lifecycle(pipeline: Any, session: Any, latents: torch.Tensor, frames: int) -> list[Image.Image]:
+ if session.taew_decode_state is None:
+ raise RuntimeError("ABot session is missing its TAeW decode state")
+ decoded = pipeline.taew_decode_stage.decode_chunks(latents, [session.taew_decode_state])
+ output = pipeline.tensor2video(decoded[0])
+ session.next_latent_frame += frames
+ session.emitted_frames += len(output)
+ return output
+
+
+def _contracts_all_exact(contracts: Mapping[str, Mapping[str, Any]]) -> bool:
+ return all(
+ bool(contract.get("comparable")) and bool(contract.get("values_exact")) for contract in contracts.values()
+ )
+
+
+@torch.inference_mode()
+def _run(args: argparse.Namespace, base: Any, control: Any) -> dict[str, Any]:
+ if not torch.cuda.is_available():
+ raise RuntimeError("public-vs-steady diagnostic requires CUDA")
+ image = Image.open(args.image).convert("RGB")
+ pipeline = None
+ public_session = None
+ steady_session = None
+ try:
+ pipeline = base._make_pipeline(args)
+ device = torch.device(pipeline.device)
+ if device.type != "cuda":
+ raise RuntimeError(f"public-vs-steady diagnostic requires CUDA, got {pipeline.device!r}")
+ pipeline.preload_models()
+ stage = pipeline.denoise_stage
+ stage.configure_cuda_graph(False)
+ actions = base._parse_action_keys(args.action_keys)
+ public_session = pipeline.create_interactive_session(
+ image, args.prompt, seed=args.seed, session_id="public-vs-steady-public"
+ )
+ steady_session = pipeline.create_interactive_session(
+ image, args.prompt, seed=args.seed, session_id="public-vs-steady-steady"
+ )
+ warmup_chunks = base._required_warmup_chunks(
+ int(stage.dit.local_attn_size), args.control_latent_frames, args.extra_warmup_chunks
+ )
+ warmup_hashes: list[bool] = []
+ for _ in range(warmup_chunks):
+ public_frames = pipeline.generate_next_block(
+ public_session, actions, control_latent_frames=args.control_latent_frames
+ )
+ steady_frames = pipeline.generate_next_block(
+ steady_session, actions, control_latent_frames=args.control_latent_frames
+ )
+ warmup_hashes.append(base._sequence_hash(public_frames) == base._sequence_hash(steady_frames))
+ torch.cuda.synchronize(device)
+ warmup_state = control._tree_exactness(
+ control._session_state_tree(public_session, pipeline),
+ control._session_state_tree(steady_session, pipeline),
+ )
+ warmup_ready = {
+ "public": base._cache_readiness(public_session, pipeline),
+ "steady": base._cache_readiness(steady_session, pipeline),
+ }
+ if public_session.next_latent_frame != steady_session.next_latent_frame:
+ raise RuntimeError("same-seed sessions did not reach the same continuation position")
+
+ steady_control: dict[str, Any] | None = None
+ if args.static_state_timing == "before_public":
+ steady_control = _prepare_steady_control(
+ pipeline, control, steady_session, actions, args.control_latent_frames
+ )
+ with _PublicBlockProbe(stage) as probe:
+ public_frames = pipeline.generate_next_block(
+ public_session, actions, control_latent_frames=args.control_latent_frames
+ )
+ if probe.inputs is None or probe.output is None:
+ raise RuntimeError("public continuation did not enter _denoise_block")
+ if args.static_state_timing == "after_public":
+ steady_control = _prepare_steady_control(
+ pipeline, control, steady_session, actions, args.control_latent_frames
+ )
+ assert steady_control is not None
+
+ public_inputs = probe.inputs
+ input_contracts = {
+ "latent": _tensor_contract(public_inputs["latent"], steady_control["latent"]),
+ "prompt_emb": _tensor_contract(public_inputs["prompt_emb"], steady_control["prompt_emb"]),
+ "action_context": _tensor_contract(public_inputs["action_context"], steady_control["action_context"]),
+ "generator_state_after_input_draw": _tensor_contract(
+ public_inputs["generator_state_after_input_draw"],
+ steady_control["generator_state_after_input_draw"],
+ ),
+ }
+ static_start = int(steady_control["current_start"])
+ if static_start != int(public_inputs["current_start"]):
+ raise RuntimeError("public and steady controls disagree on current_start")
+ steady_latent = control._static_denoise(
+ pipeline,
+ steady_control["state"],
+ steady_control["latent"],
+ steady_control["action_context"],
+ current_start=static_start,
+ generators=[steady_session.generator],
+ scheduler=steady_session.scheduler,
+ )
+ torch.cuda.synchronize(device)
+ sampling_state = control._tree_exactness(
+ _sampling_state_tree(public_session), _sampling_state_tree(steady_session)
+ )
+ latent_contract = _tensor_contract(probe.output, steady_latent)
+ steady_frames = _finish_steady_lifecycle(pipeline, steady_session, steady_latent, args.control_latent_frames)
+ torch.cuda.synchronize(device)
+ rgb = base._compare_frames(public_frames, steady_frames)
+ full_state = control._tree_exactness(
+ control._session_state_tree(public_session, pipeline),
+ control._session_state_tree(steady_session, pipeline),
+ )
+ public_metrics = dict(pipeline.last_stage_metrics())
+ public_path_valid = (
+ probe.call_count == 1
+ and bool(public_inputs["first_frame_is_none"])
+ and int(public_metrics.get("cuda_graph_enabled", 0)) == 0
+ and int(public_metrics.get("cuda_graph_replays", 0)) == 0
+ and int(public_metrics.get("cuda_graph_captured", 0)) == 0
+ )
+ exact = (
+ all(warmup_hashes)
+ and bool(warmup_state["exact"])
+ and bool(warmup_ready["public"]["ready"])
+ and bool(warmup_ready["steady"]["ready"])
+ and public_path_valid
+ and _contracts_all_exact(input_contracts)
+ and bool(latent_contract["values_exact"])
+ and bool(sampling_state["exact"])
+ and bool(rgb["all_frame_hashes_equal"])
+ and bool(full_state["exact"])
+ )
+ return {
+ "status": "pass" if exact else "mismatch",
+ "scope": {
+ "public_path": "generate_next_block -> denoise_interactive_block -> _denoise_block",
+ "steady_control": "handwritten forward_steady_state with graph-shaped static buffers",
+ "continuations": 1,
+ "cuda_graph_enabled": False,
+ "static_state_timing": args.static_state_timing,
+ },
+ "device": str(device),
+ "warmup": {
+ "chunks": warmup_chunks,
+ "per_chunk_frame_hash_equal": warmup_hashes,
+ "public_ready": warmup_ready["public"],
+ "steady_ready": warmup_ready["steady"],
+ "complete_state_exact": warmup_state,
+ },
+ "public_block": {
+ "call_count": probe.call_count,
+ "first_frame_is_none": public_inputs["first_frame_is_none"],
+ "current_start": public_inputs["current_start"],
+ "self_cache_cursors_before": public_inputs["self_cache_cursors_before"],
+ "cross_cache_initialized_before": public_inputs["cross_cache_initialized_before"],
+ "scheduler_type": public_inputs["scheduler_type"],
+ "stage_metrics": public_metrics,
+ "valid_eager_public_path": public_path_valid,
+ },
+ "input_contracts": input_contracts,
+ "post_denoise": {
+ "latent": latent_contract,
+ "sampling_state_exact": sampling_state,
+ },
+ "post_lifecycle": {
+ "rgb": rgb,
+ "complete_state_exact": full_state,
+ },
+ }
+ finally:
+ if pipeline is not None:
+ for session in (public_session, steady_session):
+ if session is not None:
+ try:
+ pipeline.close_interactive_session(session)
+ except Exception:
+ pass
+ try:
+ pipeline.close()
+ except Exception:
+ pass
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+
+def _write_output(output_dir: Path, result: Mapping[str, Any], args: argparse.Namespace, base: Any) -> None:
+ output_dir.mkdir(parents=True, exist_ok=True)
+ payload = {"arguments": base._json_safe(vars(args)), "result": base._json_safe(result)}
+ (output_dir / "results.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ post_denoise = result.get("post_denoise", {})
+ post_lifecycle = result.get("post_lifecycle", {})
+ lines = [
+ "# ABot public `_denoise_block` vs steady-state control",
+ "",
+ f"Status: {result.get('status', 'error')}.",
+ "",
+ "The public side was observed through `generate_next_block -> denoise_interactive_block -> "
+ "_denoise_block` with CUDA Graph disabled.",
+ "",
+ f"Post-denoise latent exact: {post_denoise.get('latent', {}).get('values_exact', False)}.",
+ f"Post-denoise sampling state exact: {post_denoise.get('sampling_state_exact', {}).get('exact', False)}.",
+ f"Rendered RGB exact: {post_lifecycle.get('rgb', {}).get('all_frame_hashes_equal', False)}.",
+ f"Full lifecycle state exact: {post_lifecycle.get('complete_state_exact', {}).get('exact', False)}.",
+ "",
+ "results.json records actual public `_denoise_block` input values/layouts, generator position, "
+ "cache cursors, and strict retained-state comparisons.",
+ ]
+ (output_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _parse_args(base: Any) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--model-root", type=Path)
+ parser.add_argument("--image", type=Path)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--prompt", default="A smooth first-person exploration through a vivid natural landscape.")
+ parser.add_argument("--action-keys", default="W")
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
+ parser.add_argument("--extra-warmup-chunks", type=int, default=0)
+ parser.add_argument("--device-id", type=int, default=0)
+ parser.add_argument("--static-state-timing", choices=("after_public", "before_public"), default="after_public")
+ parser.add_argument("--dry-run", action="store_true")
+ args = parser.parse_args()
+ if args.extra_warmup_chunks < 0:
+ parser.error("--extra-warmup-chunks must be non-negative")
+ try:
+ base._parse_action_keys(args.action_keys)
+ except argparse.ArgumentTypeError as exc:
+ parser.error(str(exc))
+ if not args.dry_run:
+ if args.model_root is None or args.image is None or args.output_dir is None:
+ parser.error("--model-root, --image, and --output-dir are required unless --dry-run is used")
+ if not args.model_root.is_dir():
+ parser.error(f"model root does not exist: {args.model_root}")
+ if not args.image.is_file():
+ parser.error(f"image does not exist: {args.image}")
+ return args
+
+
+def main() -> None:
+ base = _load_module("validate_abot_cuda_graph_parity.py", "abot_public_vs_steady_base")
+ control = _load_module("diagnose_abot_cuda_graph_persistent_three_way.py", "abot_public_vs_steady_control")
+ args = _parse_args(base)
+ if args.dry_run:
+ print(
+ json.dumps(
+ {
+ "mode": "dry_run",
+ "public_path": "generate_next_block -> denoise_interactive_block -> _denoise_block",
+ "steady_control": "forward_steady_state with graph-shaped static buffers",
+ "cuda_graph": "disabled",
+ "continuations": 1,
+ "comparisons": [
+ "warmup full retained state",
+ "actual public input values/layouts and generator position",
+ "post-denoise latent and sampling state",
+ "rendered RGB and complete lifecycle state",
+ ],
+ "static_state_timing": args.static_state_timing,
+ },
+ indent=2,
+ sort_keys=True,
+ )
+ )
+ return
+ assert args.output_dir is not None
+ try:
+ result = _run(args, base, control)
+ except Exception as exc:
+ result = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
+ _write_output(args.output_dir, result, args, base)
+ print(json.dumps(base._json_safe(result), indent=2, sort_keys=True))
+ if result.get("status") != "pass":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validation/workloads/README-turboserve-public-demo-trace.md b/tools/validation/workloads/README-turboserve-public-demo-trace.md
new file mode 100644
index 00000000..8f5f5ad1
--- /dev/null
+++ b/tools/validation/workloads/README-turboserve-public-demo-trace.md
@@ -0,0 +1,50 @@
+# TurboServe public-demo-derived ABot workloads
+
+These two scenarios are deterministic, 30-minute ABot LiveKit workload
+projections of TurboServe's public simulator trace:
+`../../../TurboServe/traces/example_8gpu.json`.
+
+They are **not** TurboServe production traces and are **not** reproductions of
+the private paper T1--T6 traces. The source records session lifecycle events,
+not real ABot keyboard actions. The adapter maps its selected events as:
+
+- `session_arrival` → create an ABot LiveKit session;
+- `user_active` → resume its action heartbeat;
+- `user_idle` → pause its action heartbeat while retaining the session/state;
+- `session_departure` → stop and delete the session.
+
+The source wall clock is retained exactly (1,800 seconds; no time compression).
+Its observed retained-session peak is 186. The capacity transform uses
+half-up proportional scaling to an ABot peak of `workers × 4`, then keeps a
+selected source session sticky until source departure or scaled capacity
+decrease. Scale-up selection uses a stable SHA-256 rank with seed `20260815`.
+The full transform, source SHA-256, and event provenance are embedded in each
+JSON's `trace_contract`.
+
+| Scenario | ABot workers | Peak retained sessions | Duration | Arrivals / departures | Active / idle transitions |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| `abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json` | 1 | 4 | 1,800 s | 61 / 61 | 66 / 69 |
+| `abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json` | 4 | 16 | 1,800 s | 300 / 300 | 282 / 323 |
+
+Regenerate and verify deterministic files:
+
+```bash
+cd /public/fanyk1/lwb/TeleFuser-abot-world
+TF_PY=/public/fanyk1/lwb/envs/telefuser_sage291/bin/python
+$TF_PY tools/validation/derive_abot_turboserve_trace.py --check
+```
+
+Validate, then replay through the normal public serving interfaces:
+
+```bash
+$TF_PY tools/validation/replay_abot_livekit_lifecycle_trace.py \
+ --scenario tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json \
+ --dry-run
+
+$TF_PY tools/validation/replay_abot_livekit_lifecycle_trace.py \
+ --scenario tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json \
+ --output results/experiments/abot_turboserve_public_demo_4gpu/result.json
+```
+
+The replay client never names or selects a GPU; placement, batching, and any
+migration remain black-box serving-system behavior.
diff --git a/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json b/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json
new file mode 100644
index 00000000..16125e68
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json
@@ -0,0 +1,2999 @@
+{
+ "admission": {
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0,
+ "require_immediate_assignment": true
+ },
+ "expected_num_workers": 1,
+ "expected_worker_mode": "process",
+ "lifecycle_trace": {
+ "duration_seconds": 1800.0,
+ "events": [
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 14.309365,
+ "sequence": 0,
+ "source_event_sequence": 163,
+ "source_session_id": 10,
+ "source_time_seconds": 14.309365,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 21.695837,
+ "sequence": 1,
+ "source_event_sequence": 64,
+ "source_session_id": 10,
+ "source_time_seconds": 21.695837,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 35.836162,
+ "sequence": 2,
+ "source_event_sequence": 65,
+ "source_session_id": 10,
+ "source_time_seconds": 35.836162,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 38.042371,
+ "sequence": 3,
+ "source_event_sequence": 66,
+ "source_session_id": 10,
+ "source_time_seconds": 38.042371,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 43.586842,
+ "sequence": 4,
+ "source_event_sequence": 485,
+ "source_session_id": 90,
+ "source_time_seconds": 43.586842,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00090-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 43.804396,
+ "sequence": 5,
+ "source_event_sequence": 239,
+ "source_session_id": 10,
+ "source_time_seconds": 43.804396,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 43.911814,
+ "sequence": 6,
+ "source_event_sequence": 496,
+ "source_session_id": 79,
+ "source_time_seconds": 43.911814,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 45.298333,
+ "sequence": 7,
+ "source_event_sequence": 270,
+ "source_session_id": 79,
+ "source_time_seconds": 45.298333,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 46.441598,
+ "sequence": 8,
+ "source_event_sequence": 511,
+ "source_session_id": 79,
+ "source_time_seconds": 46.441598,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 57.056317,
+ "sequence": 9,
+ "source_event_sequence": 401,
+ "source_session_id": 79,
+ "source_time_seconds": 57.056317,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 57.976271,
+ "sequence": 10,
+ "source_event_sequence": 478,
+ "source_session_id": 90,
+ "source_time_seconds": 57.976271,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00090-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 57.976271,
+ "sequence": 11,
+ "source_event_sequence": 478,
+ "source_session_id": 114,
+ "source_time_seconds": 57.976271,
+ "source_user_id": 28,
+ "trace_session_id": "ts-00114-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 74.513964,
+ "sequence": 12,
+ "source_event_sequence": 614,
+ "source_session_id": 114,
+ "source_time_seconds": 74.513964,
+ "source_user_id": 28,
+ "trace_session_id": "ts-00114-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 80.476437,
+ "sequence": 13,
+ "source_event_sequence": 615,
+ "source_session_id": 114,
+ "source_time_seconds": 80.476437,
+ "source_user_id": 28,
+ "trace_session_id": "ts-00114-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 80.476437,
+ "sequence": 14,
+ "source_event_sequence": 615,
+ "source_session_id": 152,
+ "source_time_seconds": 80.476437,
+ "source_user_id": 359,
+ "trace_session_id": "ts-00152-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 97.978621,
+ "sequence": 15,
+ "source_event_sequence": 402,
+ "source_session_id": 79,
+ "source_time_seconds": 97.978621,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 97.978621,
+ "sequence": 16,
+ "source_event_sequence": 402,
+ "source_session_id": 138,
+ "source_time_seconds": 97.978621,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 102.371272,
+ "sequence": 17,
+ "source_event_sequence": 868,
+ "source_session_id": 152,
+ "source_time_seconds": 102.371272,
+ "source_user_id": 359,
+ "trace_session_id": "ts-00152-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 109.660992,
+ "sequence": 18,
+ "source_event_sequence": 1257,
+ "source_session_id": 202,
+ "source_time_seconds": 109.660992,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 109.916331,
+ "sequence": 19,
+ "source_event_sequence": 786,
+ "source_session_id": 202,
+ "source_time_seconds": 109.916331,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 110.100638,
+ "sequence": 20,
+ "source_event_sequence": 1259,
+ "source_session_id": 202,
+ "source_time_seconds": 110.100638,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 110.236648,
+ "sequence": 21,
+ "source_event_sequence": 1256,
+ "source_session_id": 202,
+ "source_time_seconds": 110.236648,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 110.496001,
+ "sequence": 22,
+ "source_event_sequence": 1264,
+ "source_session_id": 202,
+ "source_time_seconds": 110.496001,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 110.958133,
+ "sequence": 23,
+ "source_event_sequence": 1164,
+ "source_session_id": 202,
+ "source_time_seconds": 110.958133,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 110.958133,
+ "sequence": 24,
+ "source_event_sequence": 1164,
+ "source_session_id": 174,
+ "source_time_seconds": 110.958133,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 113.700009,
+ "sequence": 25,
+ "source_event_sequence": 749,
+ "source_session_id": 138,
+ "source_time_seconds": 113.700009,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 114.220071,
+ "sequence": 26,
+ "source_event_sequence": 869,
+ "source_session_id": 152,
+ "source_time_seconds": 114.220071,
+ "source_user_id": 359,
+ "trace_session_id": "ts-00152-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 114.220071,
+ "sequence": 27,
+ "source_event_sequence": 869,
+ "source_session_id": 196,
+ "source_time_seconds": 114.220071,
+ "source_user_id": 489,
+ "trace_session_id": "ts-00196-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 117.265631,
+ "sequence": 28,
+ "source_event_sequence": 1121,
+ "source_session_id": 196,
+ "source_time_seconds": 117.265631,
+ "source_user_id": 489,
+ "trace_session_id": "ts-00196-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 119.062112,
+ "sequence": 29,
+ "source_event_sequence": 986,
+ "source_session_id": 174,
+ "source_time_seconds": 119.062112,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 133.12222,
+ "sequence": 30,
+ "source_event_sequence": 750,
+ "source_session_id": 138,
+ "source_time_seconds": 133.12222,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 133.291668,
+ "sequence": 31,
+ "source_event_sequence": 751,
+ "source_session_id": 138,
+ "source_time_seconds": 133.291668,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 138.474331,
+ "sequence": 32,
+ "source_event_sequence": 987,
+ "source_session_id": 174,
+ "source_time_seconds": 138.474331,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 143.985667,
+ "sequence": 33,
+ "source_event_sequence": 752,
+ "source_session_id": 138,
+ "source_time_seconds": 143.985667,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 150.74123,
+ "sequence": 34,
+ "source_event_sequence": 1122,
+ "source_session_id": 196,
+ "source_time_seconds": 150.74123,
+ "source_user_id": 489,
+ "trace_session_id": "ts-00196-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 150.74123,
+ "sequence": 35,
+ "source_event_sequence": 1122,
+ "source_session_id": 286,
+ "source_time_seconds": 150.74123,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 157.985186,
+ "sequence": 36,
+ "source_event_sequence": 753,
+ "source_session_id": 138,
+ "source_time_seconds": 157.985186,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 158.327266,
+ "sequence": 37,
+ "source_event_sequence": 988,
+ "source_session_id": 174,
+ "source_time_seconds": 158.327266,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 158.654055,
+ "sequence": 38,
+ "source_event_sequence": 1213,
+ "source_session_id": 174,
+ "source_time_seconds": 158.654055,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 174.885548,
+ "sequence": 39,
+ "source_event_sequence": 754,
+ "source_session_id": 138,
+ "source_time_seconds": 174.885548,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 177.401261,
+ "sequence": 40,
+ "source_event_sequence": 1677,
+ "source_session_id": 286,
+ "source_time_seconds": 177.401261,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 182.107632,
+ "sequence": 41,
+ "source_event_sequence": 755,
+ "source_session_id": 138,
+ "source_time_seconds": 182.107632,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 182.107632,
+ "sequence": 42,
+ "source_event_sequence": 755,
+ "source_session_id": 277,
+ "source_time_seconds": 182.107632,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 188.357514,
+ "sequence": 43,
+ "source_event_sequence": 1618,
+ "source_session_id": 277,
+ "source_time_seconds": 188.357514,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 188.638603,
+ "sequence": 44,
+ "source_event_sequence": 1678,
+ "source_session_id": 286,
+ "source_time_seconds": 188.638603,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 194.784748,
+ "sequence": 45,
+ "source_event_sequence": 1619,
+ "source_session_id": 277,
+ "source_time_seconds": 194.784748,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 201.108461,
+ "sequence": 46,
+ "source_event_sequence": 1679,
+ "source_session_id": 286,
+ "source_time_seconds": 201.108461,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 204.466977,
+ "sequence": 47,
+ "source_event_sequence": 1620,
+ "source_session_id": 277,
+ "source_time_seconds": 204.466977,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 210.144496,
+ "sequence": 48,
+ "source_event_sequence": 1680,
+ "source_session_id": 286,
+ "source_time_seconds": 210.144496,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 220.353476,
+ "sequence": 49,
+ "source_event_sequence": 1621,
+ "source_session_id": 277,
+ "source_time_seconds": 220.353476,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 227.592502,
+ "sequence": 50,
+ "source_event_sequence": 1622,
+ "source_session_id": 277,
+ "source_time_seconds": 227.592502,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 227.879206,
+ "sequence": 51,
+ "source_event_sequence": 127,
+ "source_session_id": 277,
+ "source_time_seconds": 227.879206,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 246.191514,
+ "sequence": 52,
+ "source_event_sequence": 1681,
+ "source_session_id": 286,
+ "source_time_seconds": 246.191514,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 273.589353,
+ "sequence": 53,
+ "source_event_sequence": 1682,
+ "source_session_id": 286,
+ "source_time_seconds": 273.589353,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 276.375874,
+ "sequence": 54,
+ "source_event_sequence": 1683,
+ "source_session_id": 286,
+ "source_time_seconds": 276.375874,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 305.647256,
+ "sequence": 55,
+ "source_event_sequence": 1684,
+ "source_session_id": 286,
+ "source_time_seconds": 305.647256,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 315.256035,
+ "sequence": 56,
+ "source_event_sequence": 1685,
+ "source_session_id": 286,
+ "source_time_seconds": 315.256035,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 372.912756,
+ "sequence": 57,
+ "source_event_sequence": 1686,
+ "source_session_id": 286,
+ "source_time_seconds": 372.912756,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 379.045738,
+ "sequence": 58,
+ "source_event_sequence": 1687,
+ "source_session_id": 286,
+ "source_time_seconds": 379.045738,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 384.316815,
+ "sequence": 59,
+ "source_event_sequence": 1688,
+ "source_session_id": 286,
+ "source_time_seconds": 384.316815,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 404.934145,
+ "sequence": 60,
+ "source_event_sequence": 1689,
+ "source_session_id": 286,
+ "source_time_seconds": 404.934145,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 417.860477,
+ "sequence": 61,
+ "source_event_sequence": 855,
+ "source_session_id": 286,
+ "source_time_seconds": 417.860477,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 419.912345,
+ "sequence": 62,
+ "source_event_sequence": 2335,
+ "source_session_id": 286,
+ "source_time_seconds": 419.912345,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 420.715846,
+ "sequence": 63,
+ "source_event_sequence": 2188,
+ "source_session_id": 286,
+ "source_time_seconds": 420.715846,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 422.041628,
+ "sequence": 64,
+ "source_event_sequence": 2345,
+ "source_session_id": 286,
+ "source_time_seconds": 422.041628,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 422.456713,
+ "sequence": 65,
+ "source_event_sequence": 2346,
+ "source_session_id": 286,
+ "source_time_seconds": 422.456713,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 435.874063,
+ "sequence": 66,
+ "source_event_sequence": 2347,
+ "source_session_id": 286,
+ "source_time_seconds": 435.874063,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 437.256452,
+ "sequence": 67,
+ "source_event_sequence": 1878,
+ "source_session_id": 286,
+ "source_time_seconds": 437.256452,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 459.140079,
+ "sequence": 68,
+ "source_event_sequence": 2377,
+ "source_session_id": 371,
+ "source_time_seconds": 459.140079,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 469.300732,
+ "sequence": 69,
+ "source_event_sequence": 2360,
+ "source_session_id": 371,
+ "source_time_seconds": 469.300732,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 481.875477,
+ "sequence": 70,
+ "source_event_sequence": 2361,
+ "source_session_id": 371,
+ "source_time_seconds": 481.875477,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 496.366056,
+ "sequence": 71,
+ "source_event_sequence": 2776,
+ "source_session_id": 392,
+ "source_time_seconds": 496.366056,
+ "source_user_id": 51,
+ "trace_session_id": "ts-00392-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 512.228726,
+ "sequence": 72,
+ "source_event_sequence": 2358,
+ "source_session_id": 371,
+ "source_time_seconds": 512.228726,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 512.738332,
+ "sequence": 73,
+ "source_event_sequence": 2867,
+ "source_session_id": 441,
+ "source_time_seconds": 512.738332,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 513.045975,
+ "sequence": 74,
+ "source_event_sequence": 2525,
+ "source_session_id": 441,
+ "source_time_seconds": 513.045975,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 513.417242,
+ "sequence": 75,
+ "source_event_sequence": 2871,
+ "source_session_id": 441,
+ "source_time_seconds": 513.417242,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 513.826252,
+ "sequence": 76,
+ "source_event_sequence": 2496,
+ "source_session_id": 392,
+ "source_time_seconds": 513.826252,
+ "source_user_id": 51,
+ "trace_session_id": "ts-00392-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 513.826252,
+ "sequence": 77,
+ "source_event_sequence": 2496,
+ "source_session_id": 432,
+ "source_time_seconds": 513.826252,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 514.16625,
+ "sequence": 78,
+ "source_event_sequence": 2368,
+ "source_session_id": 432,
+ "source_time_seconds": 514.16625,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 514.287995,
+ "sequence": 79,
+ "source_event_sequence": 2877,
+ "source_session_id": 432,
+ "source_time_seconds": 514.287995,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 516.154408,
+ "sequence": 80,
+ "source_event_sequence": 2834,
+ "source_session_id": 432,
+ "source_time_seconds": 516.154408,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 517.3122,
+ "sequence": 81,
+ "source_event_sequence": 2897,
+ "source_session_id": 454,
+ "source_time_seconds": 517.3122,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 521.064599,
+ "sequence": 82,
+ "source_event_sequence": 2820,
+ "source_session_id": 441,
+ "source_time_seconds": 521.064599,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 521.345451,
+ "sequence": 83,
+ "source_event_sequence": 2883,
+ "source_session_id": 454,
+ "source_time_seconds": 521.345451,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 524.4719,
+ "sequence": 84,
+ "source_event_sequence": 2821,
+ "source_session_id": 441,
+ "source_time_seconds": 524.4719,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 534.269848,
+ "sequence": 85,
+ "source_event_sequence": 2822,
+ "source_session_id": 441,
+ "source_time_seconds": 534.269848,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 541.816604,
+ "sequence": 86,
+ "source_event_sequence": 2884,
+ "source_session_id": 454,
+ "source_time_seconds": 541.816604,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 551.661066,
+ "sequence": 87,
+ "source_event_sequence": 2885,
+ "source_session_id": 454,
+ "source_time_seconds": 551.661066,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 561.648913,
+ "sequence": 88,
+ "source_event_sequence": 2823,
+ "source_session_id": 441,
+ "source_time_seconds": 561.648913,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 566.274453,
+ "sequence": 89,
+ "source_event_sequence": 2824,
+ "source_session_id": 441,
+ "source_time_seconds": 566.274453,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 566.274453,
+ "sequence": 90,
+ "source_event_sequence": 2824,
+ "source_session_id": 520,
+ "source_time_seconds": 566.274453,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 571.612985,
+ "sequence": 91,
+ "source_event_sequence": 2886,
+ "source_session_id": 454,
+ "source_time_seconds": 571.612985,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 591.480795,
+ "sequence": 92,
+ "source_event_sequence": 2887,
+ "source_session_id": 454,
+ "source_time_seconds": 591.480795,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 591.542398,
+ "sequence": 93,
+ "source_event_sequence": 2888,
+ "source_session_id": 454,
+ "source_time_seconds": 591.542398,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 603.125462,
+ "sequence": 94,
+ "source_event_sequence": 2889,
+ "source_session_id": 454,
+ "source_time_seconds": 603.125462,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 611.359375,
+ "sequence": 95,
+ "source_event_sequence": 3307,
+ "source_session_id": 520,
+ "source_time_seconds": 611.359375,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 611.518632,
+ "sequence": 96,
+ "source_event_sequence": 3308,
+ "source_session_id": 520,
+ "source_time_seconds": 611.518632,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 634.915132,
+ "sequence": 97,
+ "source_event_sequence": 4045,
+ "source_session_id": 559,
+ "source_time_seconds": 634.915132,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 649.638252,
+ "sequence": 98,
+ "source_event_sequence": 3523,
+ "source_session_id": 559,
+ "source_time_seconds": 649.638252,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 652.478105,
+ "sequence": 99,
+ "source_event_sequence": 3309,
+ "source_session_id": 520,
+ "source_time_seconds": 652.478105,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 653.244159,
+ "sequence": 100,
+ "source_event_sequence": 2890,
+ "source_session_id": 454,
+ "source_time_seconds": 653.244159,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 659.658631,
+ "sequence": 101,
+ "source_event_sequence": 3524,
+ "source_session_id": 559,
+ "source_time_seconds": 659.658631,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 659.827741,
+ "sequence": 102,
+ "source_event_sequence": 2891,
+ "source_session_id": 454,
+ "source_time_seconds": 659.827741,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 659.827741,
+ "sequence": 103,
+ "source_event_sequence": 2891,
+ "source_session_id": 641,
+ "source_time_seconds": 659.827741,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 674.792196,
+ "sequence": 104,
+ "source_event_sequence": 4047,
+ "source_session_id": 641,
+ "source_time_seconds": 674.792196,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 680.026054,
+ "sequence": 105,
+ "source_event_sequence": 3310,
+ "source_session_id": 520,
+ "source_time_seconds": 680.026054,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 686.641081,
+ "sequence": 106,
+ "source_event_sequence": 3525,
+ "source_session_id": 559,
+ "source_time_seconds": 686.641081,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 692.125069,
+ "sequence": 107,
+ "source_event_sequence": 3311,
+ "source_session_id": 520,
+ "source_time_seconds": 692.125069,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 709.867895,
+ "sequence": 108,
+ "source_event_sequence": 3526,
+ "source_session_id": 559,
+ "source_time_seconds": 709.867895,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 715.220506,
+ "sequence": 109,
+ "source_event_sequence": 4048,
+ "source_session_id": 641,
+ "source_time_seconds": 715.220506,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 733.842348,
+ "sequence": 110,
+ "source_event_sequence": 3312,
+ "source_session_id": 520,
+ "source_time_seconds": 733.842348,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 736.742681,
+ "sequence": 111,
+ "source_event_sequence": 3313,
+ "source_session_id": 520,
+ "source_time_seconds": 736.742681,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 737.001167,
+ "sequence": 112,
+ "source_event_sequence": 5317,
+ "source_session_id": 698,
+ "source_time_seconds": 737.001167,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 742.119453,
+ "sequence": 113,
+ "source_event_sequence": 5250,
+ "source_session_id": 520,
+ "source_time_seconds": 742.119453,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 754.601312,
+ "sequence": 114,
+ "source_event_sequence": 5568,
+ "source_session_id": 875,
+ "source_time_seconds": 754.601312,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 760.868031,
+ "sequence": 115,
+ "source_event_sequence": 4243,
+ "source_session_id": 875,
+ "source_time_seconds": 760.868031,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 761.009269,
+ "sequence": 116,
+ "source_event_sequence": 5651,
+ "source_session_id": 875,
+ "source_time_seconds": 761.009269,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 762.263555,
+ "sequence": 117,
+ "source_event_sequence": 5448,
+ "source_session_id": 875,
+ "source_time_seconds": 762.263555,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 762.691475,
+ "sequence": 118,
+ "source_event_sequence": 5679,
+ "source_session_id": 875,
+ "source_time_seconds": 762.691475,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 769.860037,
+ "sequence": 119,
+ "source_event_sequence": 4049,
+ "source_session_id": 641,
+ "source_time_seconds": 769.860037,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 775.127178,
+ "sequence": 120,
+ "source_event_sequence": 5544,
+ "source_session_id": 875,
+ "source_time_seconds": 775.127178,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 777.347912,
+ "sequence": 121,
+ "source_event_sequence": 5545,
+ "source_session_id": 875,
+ "source_time_seconds": 777.347912,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 777.347912,
+ "sequence": 122,
+ "source_event_sequence": 5545,
+ "source_session_id": 804,
+ "source_time_seconds": 777.347912,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 780.316429,
+ "sequence": 123,
+ "source_event_sequence": 4367,
+ "source_session_id": 698,
+ "source_time_seconds": 780.316429,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 780.316429,
+ "sequence": 124,
+ "source_event_sequence": 4367,
+ "source_session_id": 705,
+ "source_time_seconds": 780.316429,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 781.454774,
+ "sequence": 125,
+ "source_event_sequence": 4050,
+ "source_session_id": 641,
+ "source_time_seconds": 781.454774,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 781.454774,
+ "sequence": 126,
+ "source_event_sequence": 4050,
+ "source_session_id": 834,
+ "source_time_seconds": 781.454774,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 783.132594,
+ "sequence": 127,
+ "source_event_sequence": 5241,
+ "source_session_id": 834,
+ "source_time_seconds": 783.132594,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 783.132594,
+ "sequence": 128,
+ "source_event_sequence": 5241,
+ "source_session_id": 945,
+ "source_time_seconds": 783.132594,
+ "source_user_id": 271,
+ "trace_session_id": "ts-00945-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 799.055406,
+ "sequence": 129,
+ "source_event_sequence": 5994,
+ "source_session_id": 945,
+ "source_time_seconds": 799.055406,
+ "source_user_id": 271,
+ "trace_session_id": "ts-00945-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 799.055406,
+ "sequence": 130,
+ "source_event_sequence": 5994,
+ "source_session_id": 944,
+ "source_time_seconds": 799.055406,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 799.231668,
+ "sequence": 131,
+ "source_event_sequence": 5986,
+ "source_session_id": 944,
+ "source_time_seconds": 799.231668,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 799.436833,
+ "sequence": 132,
+ "source_event_sequence": 4402,
+ "source_session_id": 705,
+ "source_time_seconds": 799.436833,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 803.937331,
+ "sequence": 133,
+ "source_event_sequence": 4403,
+ "source_session_id": 705,
+ "source_time_seconds": 803.937331,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 807.921726,
+ "sequence": 134,
+ "source_event_sequence": 5054,
+ "source_session_id": 804,
+ "source_time_seconds": 807.921726,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 809.703847,
+ "sequence": 135,
+ "source_event_sequence": 4404,
+ "source_session_id": 705,
+ "source_time_seconds": 809.703847,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 810.454703,
+ "sequence": 136,
+ "source_event_sequence": 3527,
+ "source_session_id": 559,
+ "source_time_seconds": 810.454703,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 816.203564,
+ "sequence": 137,
+ "source_event_sequence": 4405,
+ "source_session_id": 705,
+ "source_time_seconds": 816.203564,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 823.202618,
+ "sequence": 138,
+ "source_event_sequence": 5987,
+ "source_session_id": 944,
+ "source_time_seconds": 823.202618,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 824.959308,
+ "sequence": 139,
+ "source_event_sequence": 4406,
+ "source_session_id": 705,
+ "source_time_seconds": 824.959308,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 832.87162,
+ "sequence": 140,
+ "source_event_sequence": 5055,
+ "source_session_id": 804,
+ "source_time_seconds": 832.87162,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 841.670216,
+ "sequence": 141,
+ "source_event_sequence": 5056,
+ "source_session_id": 804,
+ "source_time_seconds": 841.670216,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 846.641358,
+ "sequence": 142,
+ "source_event_sequence": 5057,
+ "source_session_id": 804,
+ "source_time_seconds": 846.641358,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 859.117473,
+ "sequence": 143,
+ "source_event_sequence": 5058,
+ "source_session_id": 804,
+ "source_time_seconds": 859.117473,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 862.787985,
+ "sequence": 144,
+ "source_event_sequence": 5988,
+ "source_session_id": 944,
+ "source_time_seconds": 862.787985,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 865.132629,
+ "sequence": 145,
+ "source_event_sequence": 4407,
+ "source_session_id": 705,
+ "source_time_seconds": 865.132629,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 871.458527,
+ "sequence": 146,
+ "source_event_sequence": 6565,
+ "source_session_id": 705,
+ "source_time_seconds": 871.458527,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 883.405857,
+ "sequence": 147,
+ "source_event_sequence": 5059,
+ "source_session_id": 804,
+ "source_time_seconds": 883.405857,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 887.283673,
+ "sequence": 148,
+ "source_event_sequence": 5989,
+ "source_session_id": 944,
+ "source_time_seconds": 887.283673,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 895.880573,
+ "sequence": 149,
+ "source_event_sequence": 5990,
+ "source_session_id": 944,
+ "source_time_seconds": 895.880573,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 897.199014,
+ "sequence": 150,
+ "source_event_sequence": 3528,
+ "source_session_id": 559,
+ "source_time_seconds": 897.199014,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 897.199014,
+ "sequence": 151,
+ "source_event_sequence": 3528,
+ "source_session_id": 1094,
+ "source_time_seconds": 897.199014,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 900.688697,
+ "sequence": 152,
+ "source_event_sequence": 6956,
+ "source_session_id": 1094,
+ "source_time_seconds": 900.688697,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 908.591286,
+ "sequence": 153,
+ "source_event_sequence": 5060,
+ "source_session_id": 804,
+ "source_time_seconds": 908.591286,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 911.502401,
+ "sequence": 154,
+ "source_event_sequence": 5991,
+ "source_session_id": 944,
+ "source_time_seconds": 911.502401,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 913.385895,
+ "sequence": 155,
+ "source_event_sequence": 5992,
+ "source_session_id": 944,
+ "source_time_seconds": 913.385895,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 913.385895,
+ "sequence": 156,
+ "source_event_sequence": 5992,
+ "source_session_id": 1077,
+ "source_time_seconds": 913.385895,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 913.476179,
+ "sequence": 157,
+ "source_event_sequence": 6957,
+ "source_session_id": 1094,
+ "source_time_seconds": 913.476179,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 914.073126,
+ "sequence": 158,
+ "source_event_sequence": 5061,
+ "source_session_id": 804,
+ "source_time_seconds": 914.073126,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 914.073126,
+ "sequence": 159,
+ "source_event_sequence": 5061,
+ "source_session_id": 705,
+ "source_time_seconds": 914.073126,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 917.673593,
+ "sequence": 160,
+ "source_event_sequence": 6317,
+ "source_session_id": 705,
+ "source_time_seconds": 917.673593,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 920.544168,
+ "sequence": 161,
+ "source_event_sequence": 7332,
+ "source_session_id": 705,
+ "source_time_seconds": 920.544168,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 920.960382,
+ "sequence": 162,
+ "source_event_sequence": 5745,
+ "source_session_id": 705,
+ "source_time_seconds": 920.960382,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 921.894621,
+ "sequence": 163,
+ "source_event_sequence": 7351,
+ "source_session_id": 705,
+ "source_time_seconds": 921.894621,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 922.228446,
+ "sequence": 164,
+ "source_event_sequence": 5479,
+ "source_session_id": 705,
+ "source_time_seconds": 922.228446,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 922.309931,
+ "sequence": 165,
+ "source_event_sequence": 7359,
+ "source_session_id": 705,
+ "source_time_seconds": 922.309931,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g05"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 926.647149,
+ "sequence": 166,
+ "source_event_sequence": 7331,
+ "source_session_id": 705,
+ "source_time_seconds": 926.647149,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g05"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 927.163825,
+ "sequence": 167,
+ "source_event_sequence": 7373,
+ "source_session_id": 1165,
+ "source_time_seconds": 927.163825,
+ "source_user_id": 101,
+ "trace_session_id": "ts-01165-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 927.265319,
+ "sequence": 168,
+ "source_event_sequence": 7095,
+ "source_session_id": 1077,
+ "source_time_seconds": 927.265319,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 928.447765,
+ "sequence": 169,
+ "source_event_sequence": 7369,
+ "source_session_id": 1165,
+ "source_time_seconds": 928.447765,
+ "source_user_id": 101,
+ "trace_session_id": "ts-01165-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 928.447765,
+ "sequence": 170,
+ "source_event_sequence": 7369,
+ "source_session_id": 1077,
+ "source_time_seconds": 928.447765,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 937.552942,
+ "sequence": 171,
+ "source_event_sequence": 6864,
+ "source_session_id": 1077,
+ "source_time_seconds": 937.552942,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 943.797195,
+ "sequence": 172,
+ "source_event_sequence": 6865,
+ "source_session_id": 1077,
+ "source_time_seconds": 943.797195,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 944.012298,
+ "sequence": 173,
+ "source_event_sequence": 6958,
+ "source_session_id": 1094,
+ "source_time_seconds": 944.012298,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 944.012298,
+ "sequence": 174,
+ "source_event_sequence": 6958,
+ "source_session_id": 705,
+ "source_time_seconds": 944.012298,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g06"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 949.937713,
+ "sequence": 175,
+ "source_event_sequence": 4410,
+ "source_session_id": 705,
+ "source_time_seconds": 949.937713,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g06"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 960.936668,
+ "sequence": 176,
+ "source_event_sequence": 4411,
+ "source_session_id": 705,
+ "source_time_seconds": 960.936668,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g06"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 962.845237,
+ "sequence": 177,
+ "source_event_sequence": 6866,
+ "source_session_id": 1077,
+ "source_time_seconds": 962.845237,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 965.87947,
+ "sequence": 178,
+ "source_event_sequence": 4412,
+ "source_session_id": 705,
+ "source_time_seconds": 965.87947,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g06"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 965.912076,
+ "sequence": 179,
+ "source_event_sequence": 4413,
+ "source_session_id": 705,
+ "source_time_seconds": 965.912076,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g06"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 965.912076,
+ "sequence": 180,
+ "source_event_sequence": 4413,
+ "source_session_id": 846,
+ "source_time_seconds": 965.912076,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 973.547782,
+ "sequence": 181,
+ "source_event_sequence": 6867,
+ "source_session_id": 1077,
+ "source_time_seconds": 973.547782,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 973.547782,
+ "sequence": 182,
+ "source_event_sequence": 6867,
+ "source_session_id": 783,
+ "source_time_seconds": 973.547782,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 975.490952,
+ "sequence": 183,
+ "source_event_sequence": 5334,
+ "source_session_id": 846,
+ "source_time_seconds": 975.490952,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 975.490952,
+ "sequence": 184,
+ "source_event_sequence": 5334,
+ "source_session_id": 1152,
+ "source_time_seconds": 975.490952,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 996.954911,
+ "sequence": 185,
+ "source_event_sequence": 4910,
+ "source_session_id": 783,
+ "source_time_seconds": 996.954911,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1004.398955,
+ "sequence": 186,
+ "source_event_sequence": 4911,
+ "source_session_id": 783,
+ "source_time_seconds": 1004.398955,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1006.761378,
+ "sequence": 187,
+ "source_event_sequence": 4912,
+ "source_session_id": 783,
+ "source_time_seconds": 1006.761378,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1016.550868,
+ "sequence": 188,
+ "source_event_sequence": 7300,
+ "source_session_id": 1152,
+ "source_time_seconds": 1016.550868,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1021.585148,
+ "sequence": 189,
+ "source_event_sequence": 7301,
+ "source_session_id": 1152,
+ "source_time_seconds": 1021.585148,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1028.702571,
+ "sequence": 190,
+ "source_event_sequence": 4913,
+ "source_session_id": 783,
+ "source_time_seconds": 1028.702571,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1032.910469,
+ "sequence": 191,
+ "source_event_sequence": 7302,
+ "source_session_id": 1152,
+ "source_time_seconds": 1032.910469,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1034.474491,
+ "sequence": 192,
+ "source_event_sequence": 6619,
+ "source_session_id": 1152,
+ "source_time_seconds": 1034.474491,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1096.494941,
+ "sequence": 193,
+ "source_event_sequence": 4914,
+ "source_session_id": 783,
+ "source_time_seconds": 1096.494941,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1105.741921,
+ "sequence": 194,
+ "source_event_sequence": 4915,
+ "source_session_id": 783,
+ "source_time_seconds": 1105.741921,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1127.811051,
+ "sequence": 195,
+ "source_event_sequence": 7642,
+ "source_session_id": 783,
+ "source_time_seconds": 1127.811051,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1417.197789,
+ "sequence": 196,
+ "source_event_sequence": 7961,
+ "source_session_id": 1242,
+ "source_time_seconds": 1417.197789,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1438.218241,
+ "sequence": 197,
+ "source_event_sequence": 7775,
+ "source_session_id": 1242,
+ "source_time_seconds": 1438.218241,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1450.188313,
+ "sequence": 198,
+ "source_event_sequence": 7776,
+ "source_session_id": 1242,
+ "source_time_seconds": 1450.188313,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1450.188313,
+ "sequence": 199,
+ "source_event_sequence": 7776,
+ "source_session_id": 1284,
+ "source_time_seconds": 1450.188313,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1466.099846,
+ "sequence": 200,
+ "source_event_sequence": 8473,
+ "source_session_id": 1301,
+ "source_time_seconds": 1466.099846,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1470.155889,
+ "sequence": 201,
+ "source_event_sequence": 8171,
+ "source_session_id": 1301,
+ "source_time_seconds": 1470.155889,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1486.385084,
+ "sequence": 202,
+ "source_event_sequence": 8053,
+ "source_session_id": 1284,
+ "source_time_seconds": 1486.385084,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1492.507789,
+ "sequence": 203,
+ "source_event_sequence": 8172,
+ "source_session_id": 1301,
+ "source_time_seconds": 1492.507789,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1498.403229,
+ "sequence": 204,
+ "source_event_sequence": 8054,
+ "source_session_id": 1284,
+ "source_time_seconds": 1498.403229,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1503.287146,
+ "sequence": 205,
+ "source_event_sequence": 8173,
+ "source_session_id": 1301,
+ "source_time_seconds": 1503.287146,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1524.299729,
+ "sequence": 206,
+ "source_event_sequence": 8055,
+ "source_session_id": 1284,
+ "source_time_seconds": 1524.299729,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1532.113769,
+ "sequence": 207,
+ "source_event_sequence": 9115,
+ "source_session_id": 1396,
+ "source_time_seconds": 1532.113769,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1532.387868,
+ "sequence": 208,
+ "source_event_sequence": 8199,
+ "source_session_id": 1301,
+ "source_time_seconds": 1532.387868,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1534.62677,
+ "sequence": 209,
+ "source_event_sequence": 8056,
+ "source_session_id": 1284,
+ "source_time_seconds": 1534.62677,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1534.62677,
+ "sequence": 210,
+ "source_event_sequence": 8056,
+ "source_session_id": 1381,
+ "source_time_seconds": 1534.62677,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1537.500305,
+ "sequence": 211,
+ "source_event_sequence": 8780,
+ "source_session_id": 1396,
+ "source_time_seconds": 1537.500305,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1537.607792,
+ "sequence": 212,
+ "source_event_sequence": 8692,
+ "source_session_id": 1381,
+ "source_time_seconds": 1537.607792,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1539.773922,
+ "sequence": 213,
+ "source_event_sequence": 9190,
+ "source_session_id": 1301,
+ "source_time_seconds": 1539.773922,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1539.83924,
+ "sequence": 214,
+ "source_event_sequence": 8909,
+ "source_session_id": 1301,
+ "source_time_seconds": 1539.83924,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1540.029449,
+ "sequence": 215,
+ "source_event_sequence": 9192,
+ "source_session_id": 1301,
+ "source_time_seconds": 1540.029449,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1543.498967,
+ "sequence": 216,
+ "source_event_sequence": 8693,
+ "source_session_id": 1381,
+ "source_time_seconds": 1543.498967,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1551.74539,
+ "sequence": 217,
+ "source_event_sequence": 8174,
+ "source_session_id": 1301,
+ "source_time_seconds": 1551.74539,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1558.051137,
+ "sequence": 218,
+ "source_event_sequence": 8175,
+ "source_session_id": 1301,
+ "source_time_seconds": 1558.051137,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1558.051137,
+ "sequence": 219,
+ "source_event_sequence": 8175,
+ "source_session_id": 1444,
+ "source_time_seconds": 1558.051137,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1564.474429,
+ "sequence": 220,
+ "source_event_sequence": 9055,
+ "source_session_id": 1444,
+ "source_time_seconds": 1564.474429,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1568.10545,
+ "sequence": 221,
+ "source_event_sequence": 8781,
+ "source_session_id": 1396,
+ "source_time_seconds": 1568.10545,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1570.00837,
+ "sequence": 222,
+ "source_event_sequence": 9056,
+ "source_session_id": 1444,
+ "source_time_seconds": 1570.00837,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1570.57061,
+ "sequence": 223,
+ "source_event_sequence": 9057,
+ "source_session_id": 1444,
+ "source_time_seconds": 1570.57061,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1592.75124,
+ "sequence": 224,
+ "source_event_sequence": 8694,
+ "source_session_id": 1381,
+ "source_time_seconds": 1592.75124,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1610.994032,
+ "sequence": 225,
+ "source_event_sequence": 8782,
+ "source_session_id": 1396,
+ "source_time_seconds": 1610.994032,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1613.408104,
+ "sequence": 226,
+ "source_event_sequence": 9058,
+ "source_session_id": 1444,
+ "source_time_seconds": 1613.408104,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1619.092012,
+ "sequence": 227,
+ "source_event_sequence": 8783,
+ "source_session_id": 1396,
+ "source_time_seconds": 1619.092012,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1624.984381,
+ "sequence": 228,
+ "source_event_sequence": 8784,
+ "source_session_id": 1396,
+ "source_time_seconds": 1624.984381,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1635.615655,
+ "sequence": 229,
+ "source_event_sequence": 9059,
+ "source_session_id": 1444,
+ "source_time_seconds": 1635.615655,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1640.240626,
+ "sequence": 230,
+ "source_event_sequence": 8695,
+ "source_session_id": 1381,
+ "source_time_seconds": 1640.240626,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1642.541576,
+ "sequence": 231,
+ "source_event_sequence": 9060,
+ "source_session_id": 1444,
+ "source_time_seconds": 1642.541576,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1642.541576,
+ "sequence": 232,
+ "source_event_sequence": 9060,
+ "source_session_id": 1600,
+ "source_time_seconds": 1642.541576,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1657.422997,
+ "sequence": 233,
+ "source_event_sequence": 8696,
+ "source_session_id": 1381,
+ "source_time_seconds": 1657.422997,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1658.06817,
+ "sequence": 234,
+ "source_event_sequence": 8697,
+ "source_session_id": 1381,
+ "source_time_seconds": 1658.06817,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1658.967534,
+ "sequence": 235,
+ "source_event_sequence": 10074,
+ "source_session_id": 1600,
+ "source_time_seconds": 1658.967534,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1662.950365,
+ "sequence": 236,
+ "source_event_sequence": 8785,
+ "source_session_id": 1396,
+ "source_time_seconds": 1662.950365,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1662.950365,
+ "sequence": 237,
+ "source_event_sequence": 8785,
+ "source_session_id": 1610,
+ "source_time_seconds": 1662.950365,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1676.368851,
+ "sequence": 238,
+ "source_event_sequence": 8698,
+ "source_session_id": 1381,
+ "source_time_seconds": 1676.368851,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1676.368851,
+ "sequence": 239,
+ "source_event_sequence": 8698,
+ "source_session_id": 1662,
+ "source_time_seconds": 1676.368851,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1680.55763,
+ "sequence": 240,
+ "source_event_sequence": 10305,
+ "source_session_id": 1610,
+ "source_time_seconds": 1680.55763,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1685.733335,
+ "sequence": 241,
+ "source_event_sequence": 10075,
+ "source_session_id": 1600,
+ "source_time_seconds": 1685.733335,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1688.012034,
+ "sequence": 242,
+ "source_event_sequence": 10076,
+ "source_session_id": 1600,
+ "source_time_seconds": 1688.012034,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1691.22206,
+ "sequence": 243,
+ "source_event_sequence": 10077,
+ "source_session_id": 1600,
+ "source_time_seconds": 1691.22206,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1691.22206,
+ "sequence": 244,
+ "source_event_sequence": 10077,
+ "source_session_id": 1610,
+ "source_time_seconds": 1691.22206,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1691.677246,
+ "sequence": 245,
+ "source_event_sequence": 10145,
+ "source_session_id": 1610,
+ "source_time_seconds": 1691.677246,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1692.564977,
+ "sequence": 246,
+ "source_event_sequence": 10447,
+ "source_session_id": 1662,
+ "source_time_seconds": 1692.564977,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1697.003339,
+ "sequence": 247,
+ "source_event_sequence": 10146,
+ "source_session_id": 1610,
+ "source_time_seconds": 1697.003339,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1724.209194,
+ "sequence": 248,
+ "source_event_sequence": 10448,
+ "source_session_id": 1662,
+ "source_time_seconds": 1724.209194,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1730.836261,
+ "sequence": 249,
+ "source_event_sequence": 10147,
+ "source_session_id": 1610,
+ "source_time_seconds": 1730.836261,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1736.020456,
+ "sequence": 250,
+ "source_event_sequence": 10449,
+ "source_session_id": 1662,
+ "source_time_seconds": 1736.020456,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1742.602373,
+ "sequence": 251,
+ "source_event_sequence": 10148,
+ "source_session_id": 1610,
+ "source_time_seconds": 1742.602373,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1792.806651,
+ "sequence": 252,
+ "source_event_sequence": 10450,
+ "source_session_id": 1662,
+ "source_time_seconds": 1792.806651,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 253,
+ "source_event_sequence": 8472,
+ "source_session_id": 1610,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 254,
+ "source_event_sequence": 10451,
+ "source_session_id": 1662,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1800.0,
+ "sequence": 255,
+ "source_event_sequence": 10451,
+ "source_session_id": 1717,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 283,
+ "trace_session_id": "ts-01717-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 256,
+ "source_event_sequence": 10612,
+ "source_session_id": 1717,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 283,
+ "trace_session_id": "ts-01717-g01"
+ }
+ ],
+ "kind": "explicit_session_lifecycle_v1"
+ },
+ "measurement": {
+ "connect_timeout_seconds": 90.0,
+ "first_generation_grace_seconds": 15.0,
+ "http_timeout_seconds": 30.0,
+ "sample_interval_seconds": 1.0,
+ "shutdown_timeout_seconds": 20.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "name": "abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4",
+ "phases": [
+ {
+ "active_input_fraction": 1.0,
+ "duration_seconds": 1800.0,
+ "name": "turboserve_public_demo_lifecycle_replay",
+ "target_users": 4
+ }
+ ],
+ "seed": 20260815,
+ "server_url": "http://127.0.0.1:8088",
+ "session": {
+ "control": {
+ "action_states": [
+ [
+ "KeyW"
+ ],
+ [
+ "KeyW",
+ "KeyA"
+ ],
+ [
+ "KeyW",
+ "KeyD"
+ ],
+ [
+ "KeyI"
+ ]
+ ],
+ "idle_max_seconds": 0.0,
+ "idle_min_seconds": 0.0,
+ "idle_probability": 0.0,
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.15
+ },
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "fps": 12,
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "prompt": "A smooth first-person exploration through a vivid natural landscape."
+ },
+ "trace_contract": {
+ "capacity_transform": {
+ "derived_connection_count": 61,
+ "derived_peak_active_sessions": 4,
+ "derived_peak_retained_sessions": 4,
+ "kind": "sticky_capacity_normalized_session_sampling",
+ "scaling_rule": "round_half_up(source_retained_sessions * target_peak / source_peak); sticky selected sessions are retained until source departure or a scaled capacity decrease; scale-up uses stable SHA-256(seed:source_session_id) rank among currently present sessions.",
+ "selected_source_session_count": 42,
+ "selection_seed": 20260815,
+ "source_observed_peak_retained_sessions": 186,
+ "target_peak_retained_sessions": 4,
+ "target_sessions_per_worker": 4,
+ "target_workers": 1
+ },
+ "derivation_version": "turboserve-public-demo-capacity-normalized-v1",
+ "event_mapping": {
+ "session_arrival": "create one ABot LiveKit session; arrival input_enabled follows source payload.active/current state",
+ "session_departure": "stop and delete that selected ABot LiveKit session",
+ "user_active": "resume that selected ABot client's action heartbeat without dropping its session",
+ "user_idle": "pause that selected ABot client's action heartbeat without dropping its session or retained state"
+ },
+ "execution_contract": "No diagnostic barrier. The black-box runner schedules each lifecycle event at its explicit source-derived offset and never assigns a GPU from the client side.",
+ "kind": "turboserve_public_demo_trace_derived_abot_lifecycle",
+ "not_a_reproduction_of_private_paper_t1_to_t6_traces": true,
+ "not_a_turboserve_production_trace": true,
+ "source": {
+ "public_demo_repository_relative_path": "../../../TurboServe/traces/example_8gpu.json",
+ "public_demo_trace_filename": "example_8gpu.json",
+ "sha256": "7dc3bb8934df656a710b76df16c663686ceae8f8db7d1a9b3da98e1ecf2eda31",
+ "source_duration_seconds": 1800.0,
+ "source_event_counts": {
+ "session_arrival": 1719,
+ "session_departure": 1719,
+ "user_active": 3205,
+ "user_idle": 4042
+ },
+ "source_peak_active_sessions": 107,
+ "source_peak_retained_sessions": 186
+ },
+ "time_transform": {
+ "derived_duration_seconds": 1800.0,
+ "description": "No time compression: arrival, active, idle, and departure offsets retain the source 30-minute wall-clock scale.",
+ "kind": "identity_wall_clock",
+ "source_to_derived_scale": 1.0
+ }
+ }
+}
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json
new file mode 100644
index 00000000..54643d33
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json
@@ -0,0 +1,67 @@
+{
+ "name": "abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16",
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "seed": 20260814,
+ "admission": {
+ "require_immediate_assignment": true,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0
+ },
+ "diagnostic": {
+ "initial_control_barrier": {
+ "enabled": true,
+ "kind": "phase_aligned_initial_control",
+ "not_a_real_user_trace": true,
+ "description": "DIAGNOSTIC ONLY: hold first active controls until all 16 newly created sessions have connected. This is a synthetic phase-aligned trace, not a real-user arrival trace.",
+ "phase": "diagnostic_phase_aligned_16_users",
+ "expected_connected_sessions": 16,
+ "timeout_seconds": 120.0
+ }
+ },
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.1,
+ "idle_probability": 0.0,
+ "idle_min_seconds": 0.2,
+ "idle_max_seconds": 1.0,
+ "action_states": [
+ ["KeyW"],
+ ["KeyW", "KeyA"],
+ ["KeyW", "KeyD"],
+ ["KeyI"]
+ ]
+ }
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "phases": [
+ {
+ "name": "diagnostic_phase_aligned_16_users",
+ "duration_seconds": 180.0,
+ "target_users": 16,
+ "arrival_window_seconds": 0.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "diagnostic_drain",
+ "duration_seconds": 30.0,
+ "target_users": 0,
+ "departure_window_seconds": 0.0
+ }
+ ]
+}
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json
new file mode 100644
index 00000000..f65a5300
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json
@@ -0,0 +1,106 @@
+{
+ "name": "abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min",
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "admission": {
+ "require_immediate_assignment": true,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0
+ },
+ "seed": 20260818,
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.1,
+ "idle_probability": 0.03,
+ "idle_min_seconds": 1.5,
+ "idle_max_seconds": 6.0,
+ "action_states": [
+ ["KeyW"],
+ ["KeyW", "KeyA"],
+ ["KeyW", "KeyD"],
+ ["KeyI"]
+ ]
+ }
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "phases": [
+ {
+ "name": "warmup_4_continuous",
+ "duration_seconds": 30.0,
+ "target_users": 4,
+ "arrival_window_seconds": 10.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "ramp_8_continuous",
+ "duration_seconds": 35.0,
+ "target_users": 8,
+ "arrival_window_seconds": 15.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "input_lull_8_half_paused",
+ "duration_seconds": 25.0,
+ "target_users": 8,
+ "active_input_fraction": 0.5,
+ "input_transition_window_seconds": 8.0
+ },
+ {
+ "name": "ramp_16_resume_and_arrive",
+ "duration_seconds": 45.0,
+ "target_users": 16,
+ "arrival_window_seconds": 20.0,
+ "active_input_fraction": 1.0,
+ "input_transition_window_seconds": 8.0
+ },
+ {
+ "name": "peak_16_continuous",
+ "duration_seconds": 40.0,
+ "target_users": 16,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "peak_16_half_paused",
+ "duration_seconds": 35.0,
+ "target_users": 16,
+ "active_input_fraction": 0.5,
+ "input_transition_window_seconds": 15.0
+ },
+ {
+ "name": "peak_16_reengage",
+ "duration_seconds": 35.0,
+ "target_users": 16,
+ "active_input_fraction": 1.0,
+ "input_transition_window_seconds": 10.0
+ },
+ {
+ "name": "recovery_8_departures",
+ "duration_seconds": 30.0,
+ "target_users": 8,
+ "departure_window_seconds": 15.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "recovery_4_departures",
+ "duration_seconds": 25.0,
+ "target_users": 4,
+ "departure_window_seconds": 12.0,
+ "active_input_fraction": 1.0
+ }
+ ]
+}
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json
new file mode 100644
index 00000000..2148f8e1
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json
@@ -0,0 +1,122 @@
+{
+ "name": "abot_livekit_4gpu_lf3_12fps_realistic_async_peak16",
+ "trace_contract": {
+ "kind": "deterministic_asynchronous_synthetic_interactive_user_trace",
+ "not_a_phase_aligned_diagnostic": true,
+ "description": "A repeatable, browser-equivalent user trace. There is deliberately no diagnostic.initial_control_barrier: every user joins, sends controls, pauses, resumes, and leaves on its own schedule.",
+ "arrival_process": "Scale-up arrivals are spread deterministically over each non-zero arrival_window_seconds; they are never released behind a common first-control gate.",
+ "interaction_process": "Connected controllers send a reliable control heartbeat every 0.50 +/- 0.15 seconds. While input is enabled, each heartbeat can begin a 2--8 second stochastic key-release interval; phase transitions additionally create long foreground/background pauses and later re-engagement.",
+ "lifecycle_process": "Scale-down removes the newest retained users over the declared departure window. A paused input is not a session departure, so it preserves that user's retained world-model state.",
+ "admission_contract": "All sixteen retained sessions must be assigned immediately across four workers with capacity four and HTTP queue size zero. Any queued or unassigned user invalidates the workload."
+ },
+ "server_url": "http://127.0.0.1:8088",
+ "expected_worker_mode": "process-nccl",
+ "expected_num_workers": 4,
+ "seed": 20260814,
+ "admission": {
+ "require_immediate_assignment": true,
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0
+ },
+ "session": {
+ "prompt": "A smooth first-person exploration through a vivid natural landscape.",
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "fps": 12,
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "control": {
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.15,
+ "idle_probability": 0.025,
+ "idle_min_seconds": 2.0,
+ "idle_max_seconds": 8.0,
+ "action_states": [
+ ["KeyW"],
+ ["KeyW", "KeyA"],
+ ["KeyW", "KeyD"],
+ ["KeyI"]
+ ]
+ }
+ },
+ "measurement": {
+ "sample_interval_seconds": 1.0,
+ "connect_timeout_seconds": 90.0,
+ "http_timeout_seconds": 30.0,
+ "shutdown_timeout_seconds": 20.0,
+ "first_generation_grace_seconds": 15.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "phases": [
+ {
+ "name": "warmup_4_asynchronous_arrivals",
+ "duration_seconds": 45.0,
+ "target_users": 4,
+ "arrival_window_seconds": 20.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "ramp_to_8_asynchronous_arrivals",
+ "duration_seconds": 55.0,
+ "target_users": 8,
+ "arrival_window_seconds": 25.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "eight_user_foreground_background_mix",
+ "duration_seconds": 35.0,
+ "target_users": 8,
+ "active_input_fraction": 0.625,
+ "input_transition_window_seconds": 18.0
+ },
+ {
+ "name": "surge_to_16_partially_active",
+ "duration_seconds": 65.0,
+ "target_users": 16,
+ "arrival_window_seconds": 35.0,
+ "active_input_fraction": 0.75,
+ "input_transition_window_seconds": 18.0
+ },
+ {
+ "name": "peak_16_all_reengaged",
+ "duration_seconds": 55.0,
+ "target_users": 16,
+ "active_input_fraction": 1.0,
+ "input_transition_window_seconds": 18.0
+ },
+ {
+ "name": "peak_16_mixed_interaction",
+ "duration_seconds": 45.0,
+ "target_users": 16,
+ "active_input_fraction": 0.5,
+ "input_transition_window_seconds": 25.0
+ },
+ {
+ "name": "peak_16_reengagement",
+ "duration_seconds": 45.0,
+ "target_users": 16,
+ "active_input_fraction": 1.0,
+ "input_transition_window_seconds": 18.0
+ },
+ {
+ "name": "post_surge_departure_to_8",
+ "duration_seconds": 35.0,
+ "target_users": 8,
+ "departure_window_seconds": 20.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "post_surge_departure_to_4",
+ "duration_seconds": 30.0,
+ "target_users": 4,
+ "departure_window_seconds": 18.0,
+ "active_input_fraction": 1.0
+ },
+ {
+ "name": "final_departures",
+ "duration_seconds": 20.0,
+ "target_users": 0,
+ "departure_window_seconds": 15.0
+ }
+ ]
+}
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json
new file mode 100644
index 00000000..71d300f7
--- /dev/null
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json
@@ -0,0 +1,13666 @@
+{
+ "admission": {
+ "expected_max_sessions_per_worker": 4,
+ "expected_queue_size": 0,
+ "require_immediate_assignment": true
+ },
+ "expected_num_workers": 4,
+ "expected_worker_mode": "process-nccl",
+ "lifecycle_trace": {
+ "duration_seconds": 1800.0,
+ "events": [
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1.837163,
+ "sequence": 0,
+ "source_event_sequence": 43,
+ "source_session_id": 0,
+ "source_time_seconds": 1.837163,
+ "source_user_id": 77,
+ "trace_session_id": "ts-00000-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 10.248572,
+ "sequence": 1,
+ "source_event_sequence": 109,
+ "source_session_id": 10,
+ "source_time_seconds": 10.248572,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 13.391337,
+ "sequence": 2,
+ "source_event_sequence": 2,
+ "source_session_id": 0,
+ "source_time_seconds": 13.391337,
+ "source_user_id": 77,
+ "trace_session_id": "ts-00000-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 14.694811,
+ "sequence": 3,
+ "source_event_sequence": 3,
+ "source_session_id": 0,
+ "source_time_seconds": 14.694811,
+ "source_user_id": 77,
+ "trace_session_id": "ts-00000-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 16.662313,
+ "sequence": 4,
+ "source_event_sequence": 190,
+ "source_session_id": 19,
+ "source_time_seconds": 16.662313,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 16.786891,
+ "sequence": 5,
+ "source_event_sequence": 4,
+ "source_session_id": 0,
+ "source_time_seconds": 16.786891,
+ "source_user_id": 77,
+ "trace_session_id": "ts-00000-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 17.292557,
+ "sequence": 6,
+ "source_event_sequence": 114,
+ "source_session_id": 19,
+ "source_time_seconds": 17.292557,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 21.695837,
+ "sequence": 7,
+ "source_event_sequence": 64,
+ "source_session_id": 10,
+ "source_time_seconds": 21.695837,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 22.939008,
+ "sequence": 8,
+ "source_event_sequence": 245,
+ "source_session_id": 43,
+ "source_time_seconds": 22.939008,
+ "source_user_id": 419,
+ "trace_session_id": "ts-00043-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 26.73278,
+ "sequence": 9,
+ "source_event_sequence": 5,
+ "source_session_id": 0,
+ "source_time_seconds": 26.73278,
+ "source_user_id": 77,
+ "trace_session_id": "ts-00000-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 29.546965,
+ "sequence": 10,
+ "source_event_sequence": 319,
+ "source_session_id": 58,
+ "source_time_seconds": 29.546965,
+ "source_user_id": 408,
+ "trace_session_id": "ts-00058-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 29.748993,
+ "sequence": 11,
+ "source_event_sequence": 111,
+ "source_session_id": 0,
+ "source_time_seconds": 29.748993,
+ "source_user_id": 77,
+ "trace_session_id": "ts-00000-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 30.152961,
+ "sequence": 12,
+ "source_event_sequence": 115,
+ "source_session_id": 19,
+ "source_time_seconds": 30.152961,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 30.232783,
+ "sequence": 13,
+ "source_event_sequence": 324,
+ "source_session_id": 49,
+ "source_time_seconds": 30.232783,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00049-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 31.697206,
+ "sequence": 14,
+ "source_event_sequence": 241,
+ "source_session_id": 43,
+ "source_time_seconds": 31.697206,
+ "source_user_id": 419,
+ "trace_session_id": "ts-00043-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 34.019193,
+ "sequence": 15,
+ "source_event_sequence": 311,
+ "source_session_id": 58,
+ "source_time_seconds": 34.019193,
+ "source_user_id": 408,
+ "trace_session_id": "ts-00058-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 35.508513,
+ "sequence": 16,
+ "source_event_sequence": 242,
+ "source_session_id": 43,
+ "source_time_seconds": 35.508513,
+ "source_user_id": 419,
+ "trace_session_id": "ts-00043-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 35.836162,
+ "sequence": 17,
+ "source_event_sequence": 65,
+ "source_session_id": 10,
+ "source_time_seconds": 35.836162,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 36.645842,
+ "sequence": 18,
+ "source_event_sequence": 266,
+ "source_session_id": 49,
+ "source_time_seconds": 36.645842,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00049-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 38.042371,
+ "sequence": 19,
+ "source_event_sequence": 66,
+ "source_session_id": 10,
+ "source_time_seconds": 38.042371,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 39.511174,
+ "sequence": 20,
+ "source_event_sequence": 421,
+ "source_session_id": 79,
+ "source_time_seconds": 39.511174,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 39.526075,
+ "sequence": 21,
+ "source_event_sequence": 312,
+ "source_session_id": 58,
+ "source_time_seconds": 39.526075,
+ "source_user_id": 408,
+ "trace_session_id": "ts-00058-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 45.549611,
+ "sequence": 22,
+ "source_event_sequence": 267,
+ "source_session_id": 49,
+ "source_time_seconds": 45.549611,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00049-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 45.774669,
+ "sequence": 23,
+ "source_event_sequence": 67,
+ "source_session_id": 10,
+ "source_time_seconds": 45.774669,
+ "source_user_id": 485,
+ "trace_session_id": "ts-00010-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 45.774669,
+ "sequence": 24,
+ "source_event_sequence": 67,
+ "source_session_id": 90,
+ "source_time_seconds": 45.774669,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00090-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 48.698448,
+ "sequence": 25,
+ "source_event_sequence": 540,
+ "source_session_id": 57,
+ "source_time_seconds": 48.698448,
+ "source_user_id": 41,
+ "trace_session_id": "ts-00057-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 48.860319,
+ "sequence": 26,
+ "source_event_sequence": 512,
+ "source_session_id": 19,
+ "source_time_seconds": 48.860319,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 49.060396,
+ "sequence": 27,
+ "source_event_sequence": 544,
+ "source_session_id": 19,
+ "source_time_seconds": 49.060396,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 50.379424,
+ "sequence": 28,
+ "source_event_sequence": 167,
+ "source_session_id": 19,
+ "source_time_seconds": 50.379424,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 50.484398,
+ "sequence": 29,
+ "source_event_sequence": 566,
+ "source_session_id": 19,
+ "source_time_seconds": 50.484398,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 51.064958,
+ "sequence": 30,
+ "source_event_sequence": 396,
+ "source_session_id": 19,
+ "source_time_seconds": 51.064958,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 51.126556,
+ "sequence": 31,
+ "source_event_sequence": 568,
+ "source_session_id": 19,
+ "source_time_seconds": 51.126556,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 51.629268,
+ "sequence": 32,
+ "source_event_sequence": 556,
+ "source_session_id": 19,
+ "source_time_seconds": 51.629268,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 52.484371,
+ "sequence": 33,
+ "source_event_sequence": 585,
+ "source_session_id": 19,
+ "source_time_seconds": 52.484371,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g05"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 52.808962,
+ "sequence": 34,
+ "source_event_sequence": 254,
+ "source_session_id": 19,
+ "source_time_seconds": 52.808962,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g05"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 54.28964,
+ "sequence": 35,
+ "source_event_sequence": 613,
+ "source_session_id": 114,
+ "source_time_seconds": 54.28964,
+ "source_user_id": 28,
+ "trace_session_id": "ts-00114-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 55.852713,
+ "sequence": 36,
+ "source_event_sequence": 49,
+ "source_session_id": 57,
+ "source_time_seconds": 55.852713,
+ "source_user_id": 41,
+ "trace_session_id": "ts-00057-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 56.616763,
+ "sequence": 37,
+ "source_event_sequence": 631,
+ "source_session_id": 19,
+ "source_time_seconds": 56.616763,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g06"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 57.056317,
+ "sequence": 38,
+ "source_event_sequence": 401,
+ "source_session_id": 79,
+ "source_time_seconds": 57.056317,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 57.976271,
+ "sequence": 39,
+ "source_event_sequence": 478,
+ "source_session_id": 90,
+ "source_time_seconds": 57.976271,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00090-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 57.976271,
+ "sequence": 40,
+ "source_event_sequence": 478,
+ "source_session_id": 11,
+ "source_time_seconds": 57.976271,
+ "source_user_id": 245,
+ "trace_session_id": "ts-00011-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 60.387855,
+ "sequence": 41,
+ "source_event_sequence": 313,
+ "source_session_id": 58,
+ "source_time_seconds": 60.387855,
+ "source_user_id": 408,
+ "trace_session_id": "ts-00058-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 60.387855,
+ "sequence": 42,
+ "source_event_sequence": 313,
+ "source_session_id": 123,
+ "source_time_seconds": 60.387855,
+ "source_user_id": 173,
+ "trace_session_id": "ts-00123-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 66.523629,
+ "sequence": 43,
+ "source_event_sequence": 71,
+ "source_session_id": 11,
+ "source_time_seconds": 66.523629,
+ "source_user_id": 245,
+ "trace_session_id": "ts-00011-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 66.523629,
+ "sequence": 44,
+ "source_event_sequence": 71,
+ "source_session_id": 136,
+ "source_time_seconds": 66.523629,
+ "source_user_id": 0,
+ "trace_session_id": "ts-00136-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 68.127274,
+ "sequence": 45,
+ "source_event_sequence": 675,
+ "source_session_id": 123,
+ "source_time_seconds": 68.127274,
+ "source_user_id": 173,
+ "trace_session_id": "ts-00123-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 68.127274,
+ "sequence": 46,
+ "source_event_sequence": 675,
+ "source_session_id": 138,
+ "source_time_seconds": 68.127274,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 70.380046,
+ "sequence": 47,
+ "source_event_sequence": 741,
+ "source_session_id": 136,
+ "source_time_seconds": 70.380046,
+ "source_user_id": 0,
+ "trace_session_id": "ts-00136-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 70.890596,
+ "sequence": 48,
+ "source_event_sequence": 825,
+ "source_session_id": 145,
+ "source_time_seconds": 70.890596,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 70.920197,
+ "sequence": 49,
+ "source_event_sequence": 268,
+ "source_session_id": 49,
+ "source_time_seconds": 70.920197,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00049-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 72.057986,
+ "sequence": 50,
+ "source_event_sequence": 832,
+ "source_session_id": 147,
+ "source_time_seconds": 72.057986,
+ "source_user_id": 58,
+ "trace_session_id": "ts-00147-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 72.19753,
+ "sequence": 51,
+ "source_event_sequence": 817,
+ "source_session_id": 145,
+ "source_time_seconds": 72.19753,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 73.71529,
+ "sequence": 52,
+ "source_event_sequence": 742,
+ "source_session_id": 136,
+ "source_time_seconds": 73.71529,
+ "source_user_id": 0,
+ "trace_session_id": "ts-00136-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 74.513964,
+ "sequence": 53,
+ "source_event_sequence": 614,
+ "source_session_id": 114,
+ "source_time_seconds": 74.513964,
+ "source_user_id": 28,
+ "trace_session_id": "ts-00114-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 75.517811,
+ "sequence": 54,
+ "source_event_sequence": 116,
+ "source_session_id": 19,
+ "source_time_seconds": 75.517811,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g06"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 78.869808,
+ "sequence": 55,
+ "source_event_sequence": 934,
+ "source_session_id": 152,
+ "source_time_seconds": 78.869808,
+ "source_user_id": 359,
+ "trace_session_id": "ts-00152-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 79.339283,
+ "sequence": 56,
+ "source_event_sequence": 56,
+ "source_session_id": 19,
+ "source_time_seconds": 79.339283,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g06"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 79.435881,
+ "sequence": 57,
+ "source_event_sequence": 937,
+ "source_session_id": 159,
+ "source_time_seconds": 79.435881,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 80.476437,
+ "sequence": 58,
+ "source_event_sequence": 615,
+ "source_session_id": 114,
+ "source_time_seconds": 80.476437,
+ "source_user_id": 28,
+ "trace_session_id": "ts-00114-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 80.476437,
+ "sequence": 59,
+ "source_event_sequence": 615,
+ "source_session_id": 19,
+ "source_time_seconds": 80.476437,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 82.762369,
+ "sequence": 60,
+ "source_event_sequence": 914,
+ "source_session_id": 159,
+ "source_time_seconds": 82.762369,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 83.486272,
+ "sequence": 61,
+ "source_event_sequence": 915,
+ "source_session_id": 159,
+ "source_time_seconds": 83.486272,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 86.125386,
+ "sequence": 62,
+ "source_event_sequence": 916,
+ "source_session_id": 159,
+ "source_time_seconds": 86.125386,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 87.04212,
+ "sequence": 63,
+ "source_event_sequence": 833,
+ "source_session_id": 147,
+ "source_time_seconds": 87.04212,
+ "source_user_id": 58,
+ "trace_session_id": "ts-00147-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 87.04212,
+ "sequence": 64,
+ "source_event_sequence": 833,
+ "source_session_id": 174,
+ "source_time_seconds": 87.04212,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 89.417146,
+ "sequence": 65,
+ "source_event_sequence": 243,
+ "source_session_id": 43,
+ "source_time_seconds": 89.417146,
+ "source_user_id": 419,
+ "trace_session_id": "ts-00043-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 93.811859,
+ "sequence": 66,
+ "source_event_sequence": 917,
+ "source_session_id": 159,
+ "source_time_seconds": 93.811859,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 97.831161,
+ "sequence": 67,
+ "source_event_sequence": 918,
+ "source_session_id": 159,
+ "source_time_seconds": 97.831161,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 97.978621,
+ "sequence": 68,
+ "source_event_sequence": 402,
+ "source_session_id": 79,
+ "source_time_seconds": 97.978621,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00079-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 97.978621,
+ "sequence": 69,
+ "source_event_sequence": 402,
+ "source_session_id": 182,
+ "source_time_seconds": 97.978621,
+ "source_user_id": 460,
+ "trace_session_id": "ts-00182-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 98.951416,
+ "sequence": 70,
+ "source_event_sequence": 919,
+ "source_session_id": 159,
+ "source_time_seconds": 98.951416,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 99.164471,
+ "sequence": 71,
+ "source_event_sequence": 743,
+ "source_session_id": 136,
+ "source_time_seconds": 99.164471,
+ "source_user_id": 0,
+ "trace_session_id": "ts-00136-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 99.164471,
+ "sequence": 72,
+ "source_event_sequence": 743,
+ "source_session_id": 176,
+ "source_time_seconds": 99.164471,
+ "source_user_id": 121,
+ "trace_session_id": "ts-00176-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 102.371272,
+ "sequence": 73,
+ "source_event_sequence": 868,
+ "source_session_id": 152,
+ "source_time_seconds": 102.371272,
+ "source_user_id": 359,
+ "trace_session_id": "ts-00152-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 102.488496,
+ "sequence": 74,
+ "source_event_sequence": 117,
+ "source_session_id": 19,
+ "source_time_seconds": 102.488496,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 104.913324,
+ "sequence": 75,
+ "source_event_sequence": 1197,
+ "source_session_id": 202,
+ "source_time_seconds": 104.913324,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 105.971603,
+ "sequence": 76,
+ "source_event_sequence": 922,
+ "source_session_id": 176,
+ "source_time_seconds": 105.971603,
+ "source_user_id": 121,
+ "trace_session_id": "ts-00176-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 106.489943,
+ "sequence": 77,
+ "source_event_sequence": 1211,
+ "source_session_id": 196,
+ "source_time_seconds": 106.489943,
+ "source_user_id": 489,
+ "trace_session_id": "ts-00196-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 108.99885,
+ "sequence": 78,
+ "source_event_sequence": 1046,
+ "source_session_id": 182,
+ "source_time_seconds": 108.99885,
+ "source_user_id": 460,
+ "trace_session_id": "ts-00182-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 110.958133,
+ "sequence": 79,
+ "source_event_sequence": 1164,
+ "source_session_id": 202,
+ "source_time_seconds": 110.958133,
+ "source_user_id": 317,
+ "trace_session_id": "ts-00202-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 110.958133,
+ "sequence": 80,
+ "source_event_sequence": 1164,
+ "source_session_id": 209,
+ "source_time_seconds": 110.958133,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 111.10924,
+ "sequence": 81,
+ "source_event_sequence": 1047,
+ "source_session_id": 182,
+ "source_time_seconds": 111.10924,
+ "source_user_id": 460,
+ "trace_session_id": "ts-00182-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 112.30381,
+ "sequence": 82,
+ "source_event_sequence": 1323,
+ "source_session_id": 228,
+ "source_time_seconds": 112.30381,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00228-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 112.568459,
+ "sequence": 83,
+ "source_event_sequence": 244,
+ "source_session_id": 43,
+ "source_time_seconds": 112.568459,
+ "source_user_id": 419,
+ "trace_session_id": "ts-00043-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 113.449389,
+ "sequence": 84,
+ "source_event_sequence": 1329,
+ "source_session_id": 176,
+ "source_time_seconds": 113.449389,
+ "source_user_id": 121,
+ "trace_session_id": "ts-00176-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 113.700009,
+ "sequence": 85,
+ "source_event_sequence": 749,
+ "source_session_id": 138,
+ "source_time_seconds": 113.700009,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 114.220071,
+ "sequence": 86,
+ "source_event_sequence": 869,
+ "source_session_id": 152,
+ "source_time_seconds": 114.220071,
+ "source_user_id": 359,
+ "trace_session_id": "ts-00152-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 115.10244,
+ "sequence": 87,
+ "source_event_sequence": 1342,
+ "source_session_id": 163,
+ "source_time_seconds": 115.10244,
+ "source_user_id": 473,
+ "trace_session_id": "ts-00163-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 115.702612,
+ "sequence": 88,
+ "source_event_sequence": 1008,
+ "source_session_id": 176,
+ "source_time_seconds": 115.702612,
+ "source_user_id": 121,
+ "trace_session_id": "ts-00176-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 116.939685,
+ "sequence": 89,
+ "source_event_sequence": 1009,
+ "source_session_id": 176,
+ "source_time_seconds": 116.939685,
+ "source_user_id": 121,
+ "trace_session_id": "ts-00176-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 117.265631,
+ "sequence": 90,
+ "source_event_sequence": 1121,
+ "source_session_id": 196,
+ "source_time_seconds": 117.265631,
+ "source_user_id": 489,
+ "trace_session_id": "ts-00196-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 117.677579,
+ "sequence": 91,
+ "source_event_sequence": 932,
+ "source_session_id": 163,
+ "source_time_seconds": 117.677579,
+ "source_user_id": 473,
+ "trace_session_id": "ts-00163-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 117.97378,
+ "sequence": 92,
+ "source_event_sequence": 933,
+ "source_session_id": 163,
+ "source_time_seconds": 117.97378,
+ "source_user_id": 473,
+ "trace_session_id": "ts-00163-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 117.97378,
+ "sequence": 93,
+ "source_event_sequence": 933,
+ "source_session_id": 239,
+ "source_time_seconds": 117.97378,
+ "source_user_id": 389,
+ "trace_session_id": "ts-00239-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 119.062112,
+ "sequence": 94,
+ "source_event_sequence": 986,
+ "source_session_id": 174,
+ "source_time_seconds": 119.062112,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 123.122502,
+ "sequence": 95,
+ "source_event_sequence": 118,
+ "source_session_id": 19,
+ "source_time_seconds": 123.122502,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 125.36796,
+ "sequence": 96,
+ "source_event_sequence": 1010,
+ "source_session_id": 176,
+ "source_time_seconds": 125.36796,
+ "source_user_id": 121,
+ "trace_session_id": "ts-00176-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 125.36796,
+ "sequence": 97,
+ "source_event_sequence": 1010,
+ "source_session_id": 248,
+ "source_time_seconds": 125.36796,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00248-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 126.74807,
+ "sequence": 98,
+ "source_event_sequence": 1201,
+ "source_session_id": 209,
+ "source_time_seconds": 126.74807,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 128.637466,
+ "sequence": 99,
+ "source_event_sequence": 1202,
+ "source_session_id": 209,
+ "source_time_seconds": 128.637466,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 133.12222,
+ "sequence": 100,
+ "source_event_sequence": 750,
+ "source_session_id": 138,
+ "source_time_seconds": 133.12222,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 133.291668,
+ "sequence": 101,
+ "source_event_sequence": 751,
+ "source_session_id": 138,
+ "source_time_seconds": 133.291668,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 135.564763,
+ "sequence": 102,
+ "source_event_sequence": 1313,
+ "source_session_id": 228,
+ "source_time_seconds": 135.564763,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00228-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 137.053685,
+ "sequence": 103,
+ "source_event_sequence": 1314,
+ "source_session_id": 228,
+ "source_time_seconds": 137.053685,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00228-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 138.474331,
+ "sequence": 104,
+ "source_event_sequence": 987,
+ "source_session_id": 174,
+ "source_time_seconds": 138.474331,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 140.226658,
+ "sequence": 105,
+ "source_event_sequence": 1615,
+ "source_session_id": 277,
+ "source_time_seconds": 140.226658,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 140.848709,
+ "sequence": 106,
+ "source_event_sequence": 1048,
+ "source_session_id": 182,
+ "source_time_seconds": 140.848709,
+ "source_user_id": 460,
+ "trace_session_id": "ts-00182-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 140.848709,
+ "sequence": 107,
+ "source_event_sequence": 1048,
+ "source_session_id": 272,
+ "source_time_seconds": 140.848709,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 142.346647,
+ "sequence": 108,
+ "source_event_sequence": 1016,
+ "source_session_id": 272,
+ "source_time_seconds": 142.346647,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 142.439214,
+ "sequence": 109,
+ "source_event_sequence": 1644,
+ "source_session_id": 272,
+ "source_time_seconds": 142.439214,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 143.985667,
+ "sequence": 110,
+ "source_event_sequence": 752,
+ "source_session_id": 138,
+ "source_time_seconds": 143.985667,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 145.165853,
+ "sequence": 111,
+ "source_event_sequence": 1487,
+ "source_session_id": 272,
+ "source_time_seconds": 145.165853,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 146.147656,
+ "sequence": 112,
+ "source_event_sequence": 1203,
+ "source_session_id": 209,
+ "source_time_seconds": 146.147656,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 149.744402,
+ "sequence": 113,
+ "source_event_sequence": 1315,
+ "source_session_id": 228,
+ "source_time_seconds": 149.744402,
+ "source_user_id": 467,
+ "trace_session_id": "ts-00228-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 149.744402,
+ "sequence": 114,
+ "source_event_sequence": 1315,
+ "source_session_id": 286,
+ "source_time_seconds": 149.744402,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 150.055599,
+ "sequence": 115,
+ "source_event_sequence": 119,
+ "source_session_id": 19,
+ "source_time_seconds": 150.055599,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 150.351982,
+ "sequence": 116,
+ "source_event_sequence": 1442,
+ "source_session_id": 248,
+ "source_time_seconds": 150.351982,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00248-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 150.74123,
+ "sequence": 117,
+ "source_event_sequence": 1122,
+ "source_session_id": 196,
+ "source_time_seconds": 150.74123,
+ "source_user_id": 489,
+ "trace_session_id": "ts-00196-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 151.105585,
+ "sequence": 118,
+ "source_event_sequence": 818,
+ "source_session_id": 145,
+ "source_time_seconds": 151.105585,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 151.474293,
+ "sequence": 119,
+ "source_event_sequence": 1719,
+ "source_session_id": 272,
+ "source_time_seconds": 151.474293,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 153.305467,
+ "sequence": 120,
+ "source_event_sequence": 261,
+ "source_session_id": 272,
+ "source_time_seconds": 153.305467,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 153.625904,
+ "sequence": 121,
+ "source_event_sequence": 120,
+ "source_session_id": 19,
+ "source_time_seconds": 153.625904,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 156.373375,
+ "sequence": 122,
+ "source_event_sequence": 121,
+ "source_session_id": 19,
+ "source_time_seconds": 156.373375,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 157.985186,
+ "sequence": 123,
+ "source_event_sequence": 753,
+ "source_session_id": 138,
+ "source_time_seconds": 157.985186,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 158.11555,
+ "sequence": 124,
+ "source_event_sequence": 819,
+ "source_session_id": 145,
+ "source_time_seconds": 158.11555,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 158.327266,
+ "sequence": 125,
+ "source_event_sequence": 988,
+ "source_session_id": 174,
+ "source_time_seconds": 158.327266,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 159.950001,
+ "sequence": 126,
+ "source_event_sequence": 1362,
+ "source_session_id": 239,
+ "source_time_seconds": 159.950001,
+ "source_user_id": 389,
+ "trace_session_id": "ts-00239-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 159.950001,
+ "sequence": 127,
+ "source_event_sequence": 1362,
+ "source_session_id": 272,
+ "source_time_seconds": 159.950001,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 159.971597,
+ "sequence": 128,
+ "source_event_sequence": 1616,
+ "source_session_id": 277,
+ "source_time_seconds": 159.971597,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 163.144502,
+ "sequence": 129,
+ "source_event_sequence": 989,
+ "source_session_id": 174,
+ "source_time_seconds": 163.144502,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 163.162996,
+ "sequence": 130,
+ "source_event_sequence": 1617,
+ "source_session_id": 277,
+ "source_time_seconds": 163.162996,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 163.976765,
+ "sequence": 131,
+ "source_event_sequence": 820,
+ "source_session_id": 145,
+ "source_time_seconds": 163.976765,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 164.482095,
+ "sequence": 132,
+ "source_event_sequence": 920,
+ "source_session_id": 159,
+ "source_time_seconds": 164.482095,
+ "source_user_id": 347,
+ "trace_session_id": "ts-00159-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 164.482095,
+ "sequence": 133,
+ "source_event_sequence": 920,
+ "source_session_id": 295,
+ "source_time_seconds": 164.482095,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 165.508436,
+ "sequence": 134,
+ "source_event_sequence": 1643,
+ "source_session_id": 272,
+ "source_time_seconds": 165.508436,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 165.718857,
+ "sequence": 135,
+ "source_event_sequence": 1785,
+ "source_session_id": 272,
+ "source_time_seconds": 165.718857,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g05"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 166.188201,
+ "sequence": 136,
+ "source_event_sequence": 821,
+ "source_session_id": 145,
+ "source_time_seconds": 166.188201,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 166.944297,
+ "sequence": 137,
+ "source_event_sequence": 1287,
+ "source_session_id": 272,
+ "source_time_seconds": 166.944297,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g05"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 167.248822,
+ "sequence": 138,
+ "source_event_sequence": 1443,
+ "source_session_id": 248,
+ "source_time_seconds": 167.248822,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00248-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 167.248822,
+ "sequence": 139,
+ "source_event_sequence": 1443,
+ "source_session_id": 272,
+ "source_time_seconds": 167.248822,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g06"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 168.779905,
+ "sequence": 140,
+ "source_event_sequence": 1594,
+ "source_session_id": 272,
+ "source_time_seconds": 168.779905,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g06"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 174.885548,
+ "sequence": 141,
+ "source_event_sequence": 754,
+ "source_session_id": 138,
+ "source_time_seconds": 174.885548,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 175.391505,
+ "sequence": 142,
+ "source_event_sequence": 1204,
+ "source_session_id": 209,
+ "source_time_seconds": 175.391505,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 175.544324,
+ "sequence": 143,
+ "source_event_sequence": 990,
+ "source_session_id": 174,
+ "source_time_seconds": 175.544324,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 176.214854,
+ "sequence": 144,
+ "source_event_sequence": 822,
+ "source_session_id": 145,
+ "source_time_seconds": 176.214854,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 176.951594,
+ "sequence": 145,
+ "source_event_sequence": 1768,
+ "source_session_id": 295,
+ "source_time_seconds": 176.951594,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 177.401261,
+ "sequence": 146,
+ "source_event_sequence": 1677,
+ "source_session_id": 286,
+ "source_time_seconds": 177.401261,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 177.520199,
+ "sequence": 147,
+ "source_event_sequence": 1341,
+ "source_session_id": 272,
+ "source_time_seconds": 177.520199,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g06"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 177.608252,
+ "sequence": 148,
+ "source_event_sequence": 122,
+ "source_session_id": 19,
+ "source_time_seconds": 177.608252,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 178.028771,
+ "sequence": 149,
+ "source_event_sequence": 1205,
+ "source_session_id": 209,
+ "source_time_seconds": 178.028771,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 179.916139,
+ "sequence": 150,
+ "source_event_sequence": 123,
+ "source_session_id": 19,
+ "source_time_seconds": 179.916139,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 181.318517,
+ "sequence": 151,
+ "source_event_sequence": 124,
+ "source_session_id": 19,
+ "source_time_seconds": 181.318517,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 182.107632,
+ "sequence": 152,
+ "source_event_sequence": 755,
+ "source_session_id": 138,
+ "source_time_seconds": 182.107632,
+ "source_user_id": 384,
+ "trace_session_id": "ts-00138-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 182.107632,
+ "sequence": 153,
+ "source_event_sequence": 755,
+ "source_session_id": 272,
+ "source_time_seconds": 182.107632,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 183.476698,
+ "sequence": 154,
+ "source_event_sequence": 823,
+ "source_session_id": 145,
+ "source_time_seconds": 183.476698,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 188.357514,
+ "sequence": 155,
+ "source_event_sequence": 1618,
+ "source_session_id": 277,
+ "source_time_seconds": 188.357514,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 188.638603,
+ "sequence": 156,
+ "source_event_sequence": 1678,
+ "source_session_id": 286,
+ "source_time_seconds": 188.638603,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 191.744029,
+ "sequence": 157,
+ "source_event_sequence": 991,
+ "source_session_id": 174,
+ "source_time_seconds": 191.744029,
+ "source_user_id": 21,
+ "trace_session_id": "ts-00174-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 191.744029,
+ "sequence": 158,
+ "source_event_sequence": 991,
+ "source_session_id": 305,
+ "source_time_seconds": 191.744029,
+ "source_user_id": 266,
+ "trace_session_id": "ts-00305-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 192.552842,
+ "sequence": 159,
+ "source_event_sequence": 1769,
+ "source_session_id": 295,
+ "source_time_seconds": 192.552842,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 193.705516,
+ "sequence": 160,
+ "source_event_sequence": 1519,
+ "source_session_id": 305,
+ "source_time_seconds": 193.705516,
+ "source_user_id": 266,
+ "trace_session_id": "ts-00305-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 194.784748,
+ "sequence": 161,
+ "source_event_sequence": 1619,
+ "source_session_id": 277,
+ "source_time_seconds": 194.784748,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 195.230095,
+ "sequence": 162,
+ "source_event_sequence": 824,
+ "source_session_id": 145,
+ "source_time_seconds": 195.230095,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00145-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 195.230095,
+ "sequence": 163,
+ "source_event_sequence": 824,
+ "source_session_id": 305,
+ "source_time_seconds": 195.230095,
+ "source_user_id": 266,
+ "trace_session_id": "ts-00305-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 196.779654,
+ "sequence": 164,
+ "source_event_sequence": 1770,
+ "source_session_id": 295,
+ "source_time_seconds": 196.779654,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 201.108461,
+ "sequence": 165,
+ "source_event_sequence": 1679,
+ "source_session_id": 286,
+ "source_time_seconds": 201.108461,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 201.959954,
+ "sequence": 166,
+ "source_event_sequence": 125,
+ "source_session_id": 19,
+ "source_time_seconds": 201.959954,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 203.165654,
+ "sequence": 167,
+ "source_event_sequence": 1771,
+ "source_session_id": 295,
+ "source_time_seconds": 203.165654,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 203.551697,
+ "sequence": 168,
+ "source_event_sequence": 1772,
+ "source_session_id": 295,
+ "source_time_seconds": 203.551697,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 204.466977,
+ "sequence": 169,
+ "source_event_sequence": 1620,
+ "source_session_id": 277,
+ "source_time_seconds": 204.466977,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 208.587985,
+ "sequence": 170,
+ "source_event_sequence": 1595,
+ "source_session_id": 272,
+ "source_time_seconds": 208.587985,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g07"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 210.084274,
+ "sequence": 171,
+ "source_event_sequence": 1377,
+ "source_session_id": 305,
+ "source_time_seconds": 210.084274,
+ "source_user_id": 266,
+ "trace_session_id": "ts-00305-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 210.144496,
+ "sequence": 172,
+ "source_event_sequence": 1680,
+ "source_session_id": 286,
+ "source_time_seconds": 210.144496,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 213.696309,
+ "sequence": 173,
+ "source_event_sequence": 1922,
+ "source_session_id": 313,
+ "source_time_seconds": 213.696309,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 214.466292,
+ "sequence": 174,
+ "source_event_sequence": 1726,
+ "source_session_id": 272,
+ "source_time_seconds": 214.466292,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g07"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 215.469258,
+ "sequence": 175,
+ "source_event_sequence": 1926,
+ "source_session_id": 272,
+ "source_time_seconds": 215.469258,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g08"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 216.056124,
+ "sequence": 176,
+ "source_event_sequence": 1596,
+ "source_session_id": 272,
+ "source_time_seconds": 216.056124,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g08"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 218.205423,
+ "sequence": 177,
+ "source_event_sequence": 1383,
+ "source_session_id": 272,
+ "source_time_seconds": 218.205423,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g08"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 218.604477,
+ "sequence": 178,
+ "source_event_sequence": 126,
+ "source_session_id": 19,
+ "source_time_seconds": 218.604477,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 220.353476,
+ "sequence": 179,
+ "source_event_sequence": 1621,
+ "source_session_id": 277,
+ "source_time_seconds": 220.353476,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 224.739454,
+ "sequence": 180,
+ "source_event_sequence": 1206,
+ "source_session_id": 209,
+ "source_time_seconds": 224.739454,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 227.592502,
+ "sequence": 181,
+ "source_event_sequence": 1622,
+ "source_session_id": 277,
+ "source_time_seconds": 227.592502,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 227.879206,
+ "sequence": 182,
+ "source_event_sequence": 127,
+ "source_session_id": 19,
+ "source_time_seconds": 227.879206,
+ "source_user_id": 31,
+ "trace_session_id": "ts-00019-g07"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 227.879206,
+ "sequence": 183,
+ "source_event_sequence": 127,
+ "source_session_id": 272,
+ "source_time_seconds": 227.879206,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g09"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 234.406243,
+ "sequence": 184,
+ "source_event_sequence": 1623,
+ "source_session_id": 277,
+ "source_time_seconds": 234.406243,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 235.933952,
+ "sequence": 185,
+ "source_event_sequence": 1597,
+ "source_session_id": 272,
+ "source_time_seconds": 235.933952,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g09"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 236.56828,
+ "sequence": 186,
+ "source_event_sequence": 1624,
+ "source_session_id": 277,
+ "source_time_seconds": 236.56828,
+ "source_user_id": 414,
+ "trace_session_id": "ts-00277-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 236.56828,
+ "sequence": 187,
+ "source_event_sequence": 1624,
+ "source_session_id": 212,
+ "source_time_seconds": 236.56828,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 245.501072,
+ "sequence": 188,
+ "source_event_sequence": 1946,
+ "source_session_id": 212,
+ "source_time_seconds": 245.501072,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 246.191514,
+ "sequence": 189,
+ "source_event_sequence": 1681,
+ "source_session_id": 286,
+ "source_time_seconds": 246.191514,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 249.480667,
+ "sequence": 190,
+ "source_event_sequence": 1207,
+ "source_session_id": 209,
+ "source_time_seconds": 249.480667,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 249.899341,
+ "sequence": 191,
+ "source_event_sequence": 1598,
+ "source_session_id": 272,
+ "source_time_seconds": 249.899341,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00272-g09"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 249.899341,
+ "sequence": 192,
+ "source_event_sequence": 1598,
+ "source_session_id": 325,
+ "source_time_seconds": 249.899341,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 254.769742,
+ "sequence": 193,
+ "source_event_sequence": 1208,
+ "source_session_id": 209,
+ "source_time_seconds": 254.769742,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 256.252215,
+ "sequence": 194,
+ "source_event_sequence": 1773,
+ "source_session_id": 295,
+ "source_time_seconds": 256.252215,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 259.491272,
+ "sequence": 195,
+ "source_event_sequence": 1209,
+ "source_session_id": 209,
+ "source_time_seconds": 259.491272,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 262.060988,
+ "sequence": 196,
+ "source_event_sequence": 495,
+ "source_session_id": 313,
+ "source_time_seconds": 262.060988,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 262.771428,
+ "sequence": 197,
+ "source_event_sequence": 1210,
+ "source_session_id": 209,
+ "source_time_seconds": 262.771428,
+ "source_user_id": 5,
+ "trace_session_id": "ts-00209-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 262.771428,
+ "sequence": 198,
+ "source_event_sequence": 1210,
+ "source_session_id": 313,
+ "source_time_seconds": 262.771428,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 267.097755,
+ "sequence": 199,
+ "source_event_sequence": 2031,
+ "source_session_id": 212,
+ "source_time_seconds": 267.097755,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 267.359922,
+ "sequence": 200,
+ "source_event_sequence": 1984,
+ "source_session_id": 212,
+ "source_time_seconds": 267.359922,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 267.976881,
+ "sequence": 201,
+ "source_event_sequence": 1970,
+ "source_session_id": 325,
+ "source_time_seconds": 267.976881,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 271.618782,
+ "sequence": 202,
+ "source_event_sequence": 2044,
+ "source_session_id": 212,
+ "source_time_seconds": 271.618782,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 273.589353,
+ "sequence": 203,
+ "source_event_sequence": 1682,
+ "source_session_id": 286,
+ "source_time_seconds": 273.589353,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 276.274046,
+ "sequence": 204,
+ "source_event_sequence": 1294,
+ "source_session_id": 212,
+ "source_time_seconds": 276.274046,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 276.375874,
+ "sequence": 205,
+ "source_event_sequence": 1683,
+ "source_session_id": 286,
+ "source_time_seconds": 276.375874,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 278.547296,
+ "sequence": 206,
+ "source_event_sequence": 1774,
+ "source_session_id": 295,
+ "source_time_seconds": 278.547296,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 278.810898,
+ "sequence": 207,
+ "source_event_sequence": 2081,
+ "source_session_id": 212,
+ "source_time_seconds": 278.810898,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 281.380329,
+ "sequence": 208,
+ "source_event_sequence": 1986,
+ "source_session_id": 212,
+ "source_time_seconds": 281.380329,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 283.842768,
+ "sequence": 209,
+ "source_event_sequence": 1910,
+ "source_session_id": 313,
+ "source_time_seconds": 283.842768,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 285.877938,
+ "sequence": 210,
+ "source_event_sequence": 2100,
+ "source_session_id": 212,
+ "source_time_seconds": 285.877938,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g05"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 288.504087,
+ "sequence": 211,
+ "source_event_sequence": 1971,
+ "source_session_id": 325,
+ "source_time_seconds": 288.504087,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 289.642215,
+ "sequence": 212,
+ "source_event_sequence": 1565,
+ "source_session_id": 212,
+ "source_time_seconds": 289.642215,
+ "source_user_id": 247,
+ "trace_session_id": "ts-00212-g05"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 291.530947,
+ "sequence": 213,
+ "source_event_sequence": 2150,
+ "source_session_id": 347,
+ "source_time_seconds": 291.530947,
+ "source_user_id": 482,
+ "trace_session_id": "ts-00347-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 292.731878,
+ "sequence": 214,
+ "source_event_sequence": 1281,
+ "source_session_id": 313,
+ "source_time_seconds": 292.731878,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 300.104746,
+ "sequence": 215,
+ "source_event_sequence": 1972,
+ "source_session_id": 325,
+ "source_time_seconds": 300.104746,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 305.647256,
+ "sequence": 216,
+ "source_event_sequence": 1684,
+ "source_session_id": 286,
+ "source_time_seconds": 305.647256,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 314.720449,
+ "sequence": 217,
+ "source_event_sequence": 2151,
+ "source_session_id": 347,
+ "source_time_seconds": 314.720449,
+ "source_user_id": 482,
+ "trace_session_id": "ts-00347-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 314.720449,
+ "sequence": 218,
+ "source_event_sequence": 2151,
+ "source_session_id": 313,
+ "source_time_seconds": 314.720449,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 315.256035,
+ "sequence": 219,
+ "source_event_sequence": 1685,
+ "source_session_id": 286,
+ "source_time_seconds": 315.256035,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 317.567673,
+ "sequence": 220,
+ "source_event_sequence": 1973,
+ "source_session_id": 325,
+ "source_time_seconds": 317.567673,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 326.363457,
+ "sequence": 221,
+ "source_event_sequence": 1974,
+ "source_session_id": 325,
+ "source_time_seconds": 326.363457,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 332.026227,
+ "sequence": 222,
+ "source_event_sequence": 1913,
+ "source_session_id": 313,
+ "source_time_seconds": 332.026227,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 333.019696,
+ "sequence": 223,
+ "source_event_sequence": 1975,
+ "source_session_id": 325,
+ "source_time_seconds": 333.019696,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 336.573893,
+ "sequence": 224,
+ "source_event_sequence": 1914,
+ "source_session_id": 313,
+ "source_time_seconds": 336.573893,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 338.047758,
+ "sequence": 225,
+ "source_event_sequence": 1915,
+ "source_session_id": 313,
+ "source_time_seconds": 338.047758,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 338.316481,
+ "sequence": 226,
+ "source_event_sequence": 1916,
+ "source_session_id": 313,
+ "source_time_seconds": 338.316481,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 342.481965,
+ "sequence": 227,
+ "source_event_sequence": 1775,
+ "source_session_id": 295,
+ "source_time_seconds": 342.481965,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 347.883593,
+ "sequence": 228,
+ "source_event_sequence": 1976,
+ "source_session_id": 325,
+ "source_time_seconds": 347.883593,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 348.996705,
+ "sequence": 229,
+ "source_event_sequence": 1977,
+ "source_session_id": 325,
+ "source_time_seconds": 348.996705,
+ "source_user_id": 342,
+ "trace_session_id": "ts-00325-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 348.996705,
+ "sequence": 230,
+ "source_event_sequence": 1977,
+ "source_session_id": 352,
+ "source_time_seconds": 348.996705,
+ "source_user_id": 458,
+ "trace_session_id": "ts-00352-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 349.817659,
+ "sequence": 231,
+ "source_event_sequence": 2041,
+ "source_session_id": 313,
+ "source_time_seconds": 349.817659,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 372.912756,
+ "sequence": 232,
+ "source_event_sequence": 1686,
+ "source_session_id": 286,
+ "source_time_seconds": 372.912756,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 379.045738,
+ "sequence": 233,
+ "source_event_sequence": 1687,
+ "source_session_id": 286,
+ "source_time_seconds": 379.045738,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 380.487764,
+ "sequence": 234,
+ "source_event_sequence": 1776,
+ "source_session_id": 295,
+ "source_time_seconds": 380.487764,
+ "source_user_id": 42,
+ "trace_session_id": "ts-00295-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 380.487764,
+ "sequence": 235,
+ "source_event_sequence": 1776,
+ "source_session_id": 313,
+ "source_time_seconds": 380.487764,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 382.938426,
+ "sequence": 236,
+ "source_event_sequence": 1918,
+ "source_session_id": 313,
+ "source_time_seconds": 382.938426,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g04"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 384.316815,
+ "sequence": 237,
+ "source_event_sequence": 1688,
+ "source_session_id": 286,
+ "source_time_seconds": 384.316815,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 386.242517,
+ "sequence": 238,
+ "source_event_sequence": 2292,
+ "source_session_id": 313,
+ "source_time_seconds": 386.242517,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g04"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 387.467495,
+ "sequence": 239,
+ "source_event_sequence": 2186,
+ "source_session_id": 352,
+ "source_time_seconds": 387.467495,
+ "source_user_id": 458,
+ "trace_session_id": "ts-00352-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 387.467495,
+ "sequence": 240,
+ "source_event_sequence": 2186,
+ "source_session_id": 313,
+ "source_time_seconds": 387.467495,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g05"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 404.934145,
+ "sequence": 241,
+ "source_event_sequence": 1689,
+ "source_session_id": 286,
+ "source_time_seconds": 404.934145,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 417.517785,
+ "sequence": 242,
+ "source_event_sequence": 1919,
+ "source_session_id": 313,
+ "source_time_seconds": 417.517785,
+ "source_user_id": 37,
+ "trace_session_id": "ts-00313-g05"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 417.517785,
+ "sequence": 243,
+ "source_event_sequence": 1919,
+ "source_session_id": 348,
+ "source_time_seconds": 417.517785,
+ "source_user_id": 26,
+ "trace_session_id": "ts-00348-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 452.323407,
+ "sequence": 244,
+ "source_event_sequence": 1690,
+ "source_session_id": 286,
+ "source_time_seconds": 452.323407,
+ "source_user_id": 90,
+ "trace_session_id": "ts-00286-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 452.323407,
+ "sequence": 245,
+ "source_event_sequence": 1690,
+ "source_session_id": 361,
+ "source_time_seconds": 452.323407,
+ "source_user_id": 189,
+ "trace_session_id": "ts-00361-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 456.584735,
+ "sequence": 246,
+ "source_event_sequence": 2160,
+ "source_session_id": 348,
+ "source_time_seconds": 456.584735,
+ "source_user_id": 26,
+ "trace_session_id": "ts-00348-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 462.164509,
+ "sequence": 247,
+ "source_event_sequence": 2397,
+ "source_session_id": 371,
+ "source_time_seconds": 462.164509,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 462.184414,
+ "sequence": 248,
+ "source_event_sequence": 2378,
+ "source_session_id": 361,
+ "source_time_seconds": 462.184414,
+ "source_user_id": 189,
+ "trace_session_id": "ts-00361-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 462.458731,
+ "sequence": 249,
+ "source_event_sequence": 2400,
+ "source_session_id": 361,
+ "source_time_seconds": 462.458731,
+ "source_user_id": 189,
+ "trace_session_id": "ts-00361-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 469.300732,
+ "sequence": 250,
+ "source_event_sequence": 2360,
+ "source_session_id": 371,
+ "source_time_seconds": 469.300732,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 473.617724,
+ "sequence": 251,
+ "source_event_sequence": 2518,
+ "source_session_id": 392,
+ "source_time_seconds": 473.617724,
+ "source_user_id": 51,
+ "trace_session_id": "ts-00392-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 474.248822,
+ "sequence": 252,
+ "source_event_sequence": 2317,
+ "source_session_id": 361,
+ "source_time_seconds": 474.248822,
+ "source_user_id": 189,
+ "trace_session_id": "ts-00361-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 476.662999,
+ "sequence": 253,
+ "source_event_sequence": 2495,
+ "source_session_id": 392,
+ "source_time_seconds": 476.662999,
+ "source_user_id": 51,
+ "trace_session_id": "ts-00392-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 481.001691,
+ "sequence": 254,
+ "source_event_sequence": 2645,
+ "source_session_id": 414,
+ "source_time_seconds": 481.001691,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00414-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 481.875477,
+ "sequence": 255,
+ "source_event_sequence": 2361,
+ "source_session_id": 371,
+ "source_time_seconds": 481.875477,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 483.940791,
+ "sequence": 256,
+ "source_event_sequence": 2318,
+ "source_session_id": 361,
+ "source_time_seconds": 483.940791,
+ "source_user_id": 189,
+ "trace_session_id": "ts-00361-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 483.940791,
+ "sequence": 257,
+ "source_event_sequence": 2318,
+ "source_session_id": 405,
+ "source_time_seconds": 483.940791,
+ "source_user_id": 170,
+ "trace_session_id": "ts-00405-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 487.786249,
+ "sequence": 258,
+ "source_event_sequence": 2646,
+ "source_session_id": 414,
+ "source_time_seconds": 487.786249,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00414-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 490.479846,
+ "sequence": 259,
+ "source_event_sequence": 2161,
+ "source_session_id": 348,
+ "source_time_seconds": 490.479846,
+ "source_user_id": 26,
+ "trace_session_id": "ts-00348-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 493.825089,
+ "sequence": 260,
+ "source_event_sequence": 2745,
+ "source_session_id": 432,
+ "source_time_seconds": 493.825089,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 495.856477,
+ "sequence": 261,
+ "source_event_sequence": 2746,
+ "source_session_id": 432,
+ "source_time_seconds": 495.856477,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 504.497538,
+ "sequence": 262,
+ "source_event_sequence": 2647,
+ "source_session_id": 414,
+ "source_time_seconds": 504.497538,
+ "source_user_id": 455,
+ "trace_session_id": "ts-00414-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 504.497538,
+ "sequence": 263,
+ "source_event_sequence": 2647,
+ "source_session_id": 441,
+ "source_time_seconds": 504.497538,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 505.832122,
+ "sequence": 264,
+ "source_event_sequence": 2587,
+ "source_session_id": 405,
+ "source_time_seconds": 505.832122,
+ "source_user_id": 170,
+ "trace_session_id": "ts-00405-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 513.826252,
+ "sequence": 265,
+ "source_event_sequence": 2496,
+ "source_session_id": 392,
+ "source_time_seconds": 513.826252,
+ "source_user_id": 51,
+ "trace_session_id": "ts-00392-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 513.826252,
+ "sequence": 266,
+ "source_event_sequence": 2496,
+ "source_session_id": 424,
+ "source_time_seconds": 513.826252,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00424-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 520.182539,
+ "sequence": 267,
+ "source_event_sequence": 2588,
+ "source_session_id": 405,
+ "source_time_seconds": 520.182539,
+ "source_user_id": 170,
+ "trace_session_id": "ts-00405-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 520.433343,
+ "sequence": 268,
+ "source_event_sequence": 2362,
+ "source_session_id": 371,
+ "source_time_seconds": 520.433343,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 521.064599,
+ "sequence": 269,
+ "source_event_sequence": 2820,
+ "source_session_id": 441,
+ "source_time_seconds": 521.064599,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 524.4719,
+ "sequence": 270,
+ "source_event_sequence": 2821,
+ "source_session_id": 441,
+ "source_time_seconds": 524.4719,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 525.487102,
+ "sequence": 271,
+ "source_event_sequence": 2971,
+ "source_session_id": 454,
+ "source_time_seconds": 525.487102,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 525.956241,
+ "sequence": 272,
+ "source_event_sequence": 1484,
+ "source_session_id": 348,
+ "source_time_seconds": 525.956241,
+ "source_user_id": 26,
+ "trace_session_id": "ts-00348-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 525.987208,
+ "sequence": 273,
+ "source_event_sequence": 2975,
+ "source_session_id": 461,
+ "source_time_seconds": 525.987208,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00461-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 526.465465,
+ "sequence": 274,
+ "source_event_sequence": 2706,
+ "source_session_id": 424,
+ "source_time_seconds": 526.465465,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00424-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 534.269848,
+ "sequence": 275,
+ "source_event_sequence": 2822,
+ "source_session_id": 441,
+ "source_time_seconds": 534.269848,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 536.428056,
+ "sequence": 276,
+ "source_event_sequence": 2925,
+ "source_session_id": 461,
+ "source_time_seconds": 536.428056,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00461-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 536.428056,
+ "sequence": 277,
+ "source_event_sequence": 2925,
+ "source_session_id": 478,
+ "source_time_seconds": 536.428056,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 540.94168,
+ "sequence": 278,
+ "source_event_sequence": 3018,
+ "source_session_id": 478,
+ "source_time_seconds": 540.94168,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 541.029905,
+ "sequence": 279,
+ "source_event_sequence": 3116,
+ "source_session_id": 480,
+ "source_time_seconds": 541.029905,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 541.816604,
+ "sequence": 280,
+ "source_event_sequence": 2884,
+ "source_session_id": 454,
+ "source_time_seconds": 541.816604,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 543.053109,
+ "sequence": 281,
+ "source_event_sequence": 3028,
+ "source_session_id": 480,
+ "source_time_seconds": 543.053109,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 543.065204,
+ "sequence": 282,
+ "source_event_sequence": 3029,
+ "source_session_id": 480,
+ "source_time_seconds": 543.065204,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 548.460451,
+ "sequence": 283,
+ "source_event_sequence": 3030,
+ "source_session_id": 480,
+ "source_time_seconds": 548.460451,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 551.661066,
+ "sequence": 284,
+ "source_event_sequence": 2885,
+ "source_session_id": 454,
+ "source_time_seconds": 551.661066,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 552.921184,
+ "sequence": 285,
+ "source_event_sequence": 3283,
+ "source_session_id": 475,
+ "source_time_seconds": 552.921184,
+ "source_user_id": 47,
+ "trace_session_id": "ts-00475-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 553.004252,
+ "sequence": 286,
+ "source_event_sequence": 2747,
+ "source_session_id": 432,
+ "source_time_seconds": 553.004252,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 553.484162,
+ "sequence": 287,
+ "source_event_sequence": 3000,
+ "source_session_id": 475,
+ "source_time_seconds": 553.484162,
+ "source_user_id": 47,
+ "trace_session_id": "ts-00475-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 554.291217,
+ "sequence": 288,
+ "source_event_sequence": 3001,
+ "source_session_id": 475,
+ "source_time_seconds": 554.291217,
+ "source_user_id": 47,
+ "trace_session_id": "ts-00475-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 555.522155,
+ "sequence": 289,
+ "source_event_sequence": 2707,
+ "source_session_id": 424,
+ "source_time_seconds": 555.522155,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00424-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 556.447147,
+ "sequence": 290,
+ "source_event_sequence": 2748,
+ "source_session_id": 432,
+ "source_time_seconds": 556.447147,
+ "source_user_id": 325,
+ "trace_session_id": "ts-00432-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 556.447147,
+ "sequence": 291,
+ "source_event_sequence": 2748,
+ "source_session_id": 520,
+ "source_time_seconds": 556.447147,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 556.913935,
+ "sequence": 292,
+ "source_event_sequence": 2363,
+ "source_session_id": 371,
+ "source_time_seconds": 556.913935,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 557.440724,
+ "sequence": 293,
+ "source_event_sequence": 2708,
+ "source_session_id": 424,
+ "source_time_seconds": 557.440724,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00424-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 558.441389,
+ "sequence": 294,
+ "source_event_sequence": 2364,
+ "source_session_id": 371,
+ "source_time_seconds": 558.441389,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 558.529241,
+ "sequence": 295,
+ "source_event_sequence": 2589,
+ "source_session_id": 405,
+ "source_time_seconds": 558.529241,
+ "source_user_id": 170,
+ "trace_session_id": "ts-00405-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 558.529241,
+ "sequence": 296,
+ "source_event_sequence": 2589,
+ "source_session_id": 470,
+ "source_time_seconds": 558.529241,
+ "source_user_id": 10,
+ "trace_session_id": "ts-00470-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 559.014105,
+ "sequence": 297,
+ "source_event_sequence": 3019,
+ "source_session_id": 478,
+ "source_time_seconds": 559.014105,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 561.648913,
+ "sequence": 298,
+ "source_event_sequence": 2823,
+ "source_session_id": 441,
+ "source_time_seconds": 561.648913,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 562.403511,
+ "sequence": 299,
+ "source_event_sequence": 2365,
+ "source_session_id": 371,
+ "source_time_seconds": 562.403511,
+ "source_user_id": 218,
+ "trace_session_id": "ts-00371-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 562.403511,
+ "sequence": 300,
+ "source_event_sequence": 2365,
+ "source_session_id": 504,
+ "source_time_seconds": 562.403511,
+ "source_user_id": 167,
+ "trace_session_id": "ts-00504-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 563.799371,
+ "sequence": 301,
+ "source_event_sequence": 2709,
+ "source_session_id": 424,
+ "source_time_seconds": 563.799371,
+ "source_user_id": 481,
+ "trace_session_id": "ts-00424-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 563.799371,
+ "sequence": 302,
+ "source_event_sequence": 2709,
+ "source_session_id": 516,
+ "source_time_seconds": 563.799371,
+ "source_user_id": 190,
+ "trace_session_id": "ts-00516-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 566.274453,
+ "sequence": 303,
+ "source_event_sequence": 2824,
+ "source_session_id": 441,
+ "source_time_seconds": 566.274453,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00441-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 566.274453,
+ "sequence": 304,
+ "source_event_sequence": 2824,
+ "source_session_id": 540,
+ "source_time_seconds": 566.274453,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 568.979723,
+ "sequence": 305,
+ "source_event_sequence": 3002,
+ "source_session_id": 475,
+ "source_time_seconds": 568.979723,
+ "source_user_id": 47,
+ "trace_session_id": "ts-00475-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 569.605762,
+ "sequence": 306,
+ "source_event_sequence": 3003,
+ "source_session_id": 475,
+ "source_time_seconds": 569.605762,
+ "source_user_id": 47,
+ "trace_session_id": "ts-00475-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 571.612985,
+ "sequence": 307,
+ "source_event_sequence": 2886,
+ "source_session_id": 454,
+ "source_time_seconds": 571.612985,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 571.84728,
+ "sequence": 308,
+ "source_event_sequence": 3004,
+ "source_session_id": 475,
+ "source_time_seconds": 571.84728,
+ "source_user_id": 47,
+ "trace_session_id": "ts-00475-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 571.84728,
+ "sequence": 309,
+ "source_event_sequence": 3004,
+ "source_session_id": 438,
+ "source_time_seconds": 571.84728,
+ "source_user_id": 369,
+ "trace_session_id": "ts-00438-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 572.420691,
+ "sequence": 310,
+ "source_event_sequence": 3429,
+ "source_session_id": 540,
+ "source_time_seconds": 572.420691,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 572.561184,
+ "sequence": 311,
+ "source_event_sequence": 3031,
+ "source_session_id": 480,
+ "source_time_seconds": 572.561184,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 572.767356,
+ "sequence": 312,
+ "source_event_sequence": 2782,
+ "source_session_id": 438,
+ "source_time_seconds": 572.767356,
+ "source_user_id": 369,
+ "trace_session_id": "ts-00438-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 577.128269,
+ "sequence": 313,
+ "source_event_sequence": 3020,
+ "source_session_id": 478,
+ "source_time_seconds": 577.128269,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 580.258063,
+ "sequence": 314,
+ "source_event_sequence": 2783,
+ "source_session_id": 438,
+ "source_time_seconds": 580.258063,
+ "source_user_id": 369,
+ "trace_session_id": "ts-00438-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 580.734334,
+ "sequence": 315,
+ "source_event_sequence": 3175,
+ "source_session_id": 504,
+ "source_time_seconds": 580.734334,
+ "source_user_id": 167,
+ "trace_session_id": "ts-00504-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 583.291368,
+ "sequence": 316,
+ "source_event_sequence": 2982,
+ "source_session_id": 470,
+ "source_time_seconds": 583.291368,
+ "source_user_id": 10,
+ "trace_session_id": "ts-00470-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 583.291368,
+ "sequence": 317,
+ "source_event_sequence": 2982,
+ "source_session_id": 559,
+ "source_time_seconds": 583.291368,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 586.529387,
+ "sequence": 318,
+ "source_event_sequence": 3272,
+ "source_session_id": 516,
+ "source_time_seconds": 586.529387,
+ "source_user_id": 190,
+ "trace_session_id": "ts-00516-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 588.451484,
+ "sequence": 319,
+ "source_event_sequence": 3273,
+ "source_session_id": 516,
+ "source_time_seconds": 588.451484,
+ "source_user_id": 190,
+ "trace_session_id": "ts-00516-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 588.818277,
+ "sequence": 320,
+ "source_event_sequence": 3021,
+ "source_session_id": 478,
+ "source_time_seconds": 588.818277,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 590.37907,
+ "sequence": 321,
+ "source_event_sequence": 2784,
+ "source_session_id": 438,
+ "source_time_seconds": 590.37907,
+ "source_user_id": 369,
+ "trace_session_id": "ts-00438-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 591.261824,
+ "sequence": 322,
+ "source_event_sequence": 3274,
+ "source_session_id": 516,
+ "source_time_seconds": 591.261824,
+ "source_user_id": 190,
+ "trace_session_id": "ts-00516-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 591.261824,
+ "sequence": 323,
+ "source_event_sequence": 3274,
+ "source_session_id": 549,
+ "source_time_seconds": 591.261824,
+ "source_user_id": 402,
+ "trace_session_id": "ts-00549-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 591.480795,
+ "sequence": 324,
+ "source_event_sequence": 2887,
+ "source_session_id": 454,
+ "source_time_seconds": 591.480795,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 591.542398,
+ "sequence": 325,
+ "source_event_sequence": 2888,
+ "source_session_id": 454,
+ "source_time_seconds": 591.542398,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 593.804321,
+ "sequence": 326,
+ "source_event_sequence": 3176,
+ "source_session_id": 504,
+ "source_time_seconds": 593.804321,
+ "source_user_id": 167,
+ "trace_session_id": "ts-00504-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 595.685795,
+ "sequence": 327,
+ "source_event_sequence": 3671,
+ "source_session_id": 557,
+ "source_time_seconds": 595.685795,
+ "source_user_id": 475,
+ "trace_session_id": "ts-00557-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 597.231499,
+ "sequence": 328,
+ "source_event_sequence": 2785,
+ "source_session_id": 438,
+ "source_time_seconds": 597.231499,
+ "source_user_id": 369,
+ "trace_session_id": "ts-00438-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 598.502962,
+ "sequence": 329,
+ "source_event_sequence": 3022,
+ "source_session_id": 478,
+ "source_time_seconds": 598.502962,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 598.565255,
+ "sequence": 330,
+ "source_event_sequence": 3032,
+ "source_session_id": 480,
+ "source_time_seconds": 598.565255,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 599.208342,
+ "sequence": 331,
+ "source_event_sequence": 3441,
+ "source_session_id": 438,
+ "source_time_seconds": 599.208342,
+ "source_user_id": 369,
+ "trace_session_id": "ts-00438-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 599.565922,
+ "sequence": 332,
+ "source_event_sequence": 3710,
+ "source_session_id": 579,
+ "source_time_seconds": 599.565922,
+ "source_user_id": 473,
+ "trace_session_id": "ts-00579-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 599.600384,
+ "sequence": 333,
+ "source_event_sequence": 2638,
+ "source_session_id": 504,
+ "source_time_seconds": 599.600384,
+ "source_user_id": 167,
+ "trace_session_id": "ts-00504-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 602.134498,
+ "sequence": 334,
+ "source_event_sequence": 3511,
+ "source_session_id": 557,
+ "source_time_seconds": 602.134498,
+ "source_user_id": 475,
+ "trace_session_id": "ts-00557-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 603.125462,
+ "sequence": 335,
+ "source_event_sequence": 2889,
+ "source_session_id": 454,
+ "source_time_seconds": 603.125462,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 603.728659,
+ "sequence": 336,
+ "source_event_sequence": 3751,
+ "source_session_id": 593,
+ "source_time_seconds": 603.728659,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 607.090926,
+ "sequence": 337,
+ "source_event_sequence": 3520,
+ "source_session_id": 559,
+ "source_time_seconds": 607.090926,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 608.182293,
+ "sequence": 338,
+ "source_event_sequence": 3512,
+ "source_session_id": 557,
+ "source_time_seconds": 608.182293,
+ "source_user_id": 475,
+ "trace_session_id": "ts-00557-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 609.032946,
+ "sequence": 339,
+ "source_event_sequence": 3485,
+ "source_session_id": 480,
+ "source_time_seconds": 609.032946,
+ "source_user_id": 401,
+ "trace_session_id": "ts-00480-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 611.359375,
+ "sequence": 340,
+ "source_event_sequence": 3307,
+ "source_session_id": 520,
+ "source_time_seconds": 611.359375,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 611.518632,
+ "sequence": 341,
+ "source_event_sequence": 3308,
+ "source_session_id": 520,
+ "source_time_seconds": 611.518632,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 612.530877,
+ "sequence": 342,
+ "source_event_sequence": 3430,
+ "source_session_id": 540,
+ "source_time_seconds": 612.530877,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 612.589427,
+ "sequence": 343,
+ "source_event_sequence": 3521,
+ "source_session_id": 559,
+ "source_time_seconds": 612.589427,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 612.831124,
+ "sequence": 344,
+ "source_event_sequence": 3431,
+ "source_session_id": 540,
+ "source_time_seconds": 612.831124,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 619.978715,
+ "sequence": 345,
+ "source_event_sequence": 3023,
+ "source_session_id": 478,
+ "source_time_seconds": 619.978715,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 621.425318,
+ "sequence": 346,
+ "source_event_sequence": 3893,
+ "source_session_id": 604,
+ "source_time_seconds": 621.425318,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 622.599362,
+ "sequence": 347,
+ "source_event_sequence": 2974,
+ "source_session_id": 478,
+ "source_time_seconds": 622.599362,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 623.153089,
+ "sequence": 348,
+ "source_event_sequence": 3898,
+ "source_session_id": 478,
+ "source_time_seconds": 623.153089,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 624.198148,
+ "sequence": 349,
+ "source_event_sequence": 3513,
+ "source_session_id": 557,
+ "source_time_seconds": 624.198148,
+ "source_user_id": 475,
+ "trace_session_id": "ts-00557-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 625.040216,
+ "sequence": 350,
+ "source_event_sequence": 3818,
+ "source_session_id": 604,
+ "source_time_seconds": 625.040216,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 627.590844,
+ "sequence": 351,
+ "source_event_sequence": 3731,
+ "source_session_id": 478,
+ "source_time_seconds": 627.590844,
+ "source_user_id": 227,
+ "trace_session_id": "ts-00478-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 631.35885,
+ "sequence": 352,
+ "source_event_sequence": 3988,
+ "source_session_id": 627,
+ "source_time_seconds": 631.35885,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 631.45773,
+ "sequence": 353,
+ "source_event_sequence": 2509,
+ "source_session_id": 604,
+ "source_time_seconds": 631.45773,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 631.728348,
+ "sequence": 354,
+ "source_event_sequence": 3998,
+ "source_session_id": 604,
+ "source_time_seconds": 631.728348,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 631.86018,
+ "sequence": 355,
+ "source_event_sequence": 3752,
+ "source_session_id": 593,
+ "source_time_seconds": 631.86018,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 631.870926,
+ "sequence": 356,
+ "source_event_sequence": 3138,
+ "source_session_id": 604,
+ "source_time_seconds": 631.870926,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g02"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 632.823956,
+ "sequence": 357,
+ "source_event_sequence": 4003,
+ "source_session_id": 634,
+ "source_time_seconds": 632.823956,
+ "source_user_id": 64,
+ "trace_session_id": "ts-00634-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 632.953375,
+ "sequence": 358,
+ "source_event_sequence": 3662,
+ "source_session_id": 627,
+ "source_time_seconds": 632.953375,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 633.223047,
+ "sequence": 359,
+ "source_event_sequence": 4008,
+ "source_session_id": 627,
+ "source_time_seconds": 633.223047,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 634.854063,
+ "sequence": 360,
+ "source_event_sequence": 3522,
+ "source_session_id": 559,
+ "source_time_seconds": 634.854063,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 634.993815,
+ "sequence": 361,
+ "source_event_sequence": 3432,
+ "source_session_id": 540,
+ "source_time_seconds": 634.993815,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 637.680861,
+ "sequence": 362,
+ "source_event_sequence": 4076,
+ "source_session_id": 641,
+ "source_time_seconds": 637.680861,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 638.913947,
+ "sequence": 363,
+ "source_event_sequence": 3944,
+ "source_session_id": 627,
+ "source_time_seconds": 638.913947,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 640.792645,
+ "sequence": 364,
+ "source_event_sequence": 3514,
+ "source_session_id": 557,
+ "source_time_seconds": 640.792645,
+ "source_user_id": 475,
+ "trace_session_id": "ts-00557-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 643.36394,
+ "sequence": 365,
+ "source_event_sequence": 3515,
+ "source_session_id": 557,
+ "source_time_seconds": 643.36394,
+ "source_user_id": 475,
+ "trace_session_id": "ts-00557-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 643.36394,
+ "sequence": 366,
+ "source_event_sequence": 3515,
+ "source_session_id": 644,
+ "source_time_seconds": 643.36394,
+ "source_user_id": 135,
+ "trace_session_id": "ts-00644-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 644.232581,
+ "sequence": 367,
+ "source_event_sequence": 3479,
+ "source_session_id": 549,
+ "source_time_seconds": 644.232581,
+ "source_user_id": 402,
+ "trace_session_id": "ts-00549-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 644.232581,
+ "sequence": 368,
+ "source_event_sequence": 3479,
+ "source_session_id": 649,
+ "source_time_seconds": 644.232581,
+ "source_user_id": 101,
+ "trace_session_id": "ts-00649-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 644.260631,
+ "sequence": 369,
+ "source_event_sequence": 3945,
+ "source_session_id": 627,
+ "source_time_seconds": 644.260631,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 645.295527,
+ "sequence": 370,
+ "source_event_sequence": 4004,
+ "source_session_id": 634,
+ "source_time_seconds": 645.295527,
+ "source_user_id": 64,
+ "trace_session_id": "ts-00634-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 645.750987,
+ "sequence": 371,
+ "source_event_sequence": 4077,
+ "source_session_id": 649,
+ "source_time_seconds": 645.750987,
+ "source_user_id": 101,
+ "trace_session_id": "ts-00649-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 646.133616,
+ "sequence": 372,
+ "source_event_sequence": 4046,
+ "source_session_id": 641,
+ "source_time_seconds": 646.133616,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 647.626572,
+ "sequence": 373,
+ "source_event_sequence": 4196,
+ "source_session_id": 604,
+ "source_time_seconds": 647.626572,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 648.788163,
+ "sequence": 374,
+ "source_event_sequence": 3819,
+ "source_session_id": 604,
+ "source_time_seconds": 648.788163,
+ "source_user_id": 394,
+ "trace_session_id": "ts-00604-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 648.788163,
+ "sequence": 375,
+ "source_event_sequence": 3819,
+ "source_session_id": 665,
+ "source_time_seconds": 648.788163,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 649.638252,
+ "sequence": 376,
+ "source_event_sequence": 3523,
+ "source_session_id": 559,
+ "source_time_seconds": 649.638252,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 649.706477,
+ "sequence": 377,
+ "source_event_sequence": 3946,
+ "source_session_id": 627,
+ "source_time_seconds": 649.706477,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 649.808686,
+ "sequence": 378,
+ "source_event_sequence": 3753,
+ "source_session_id": 593,
+ "source_time_seconds": 649.808686,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 652.478105,
+ "sequence": 379,
+ "source_event_sequence": 3309,
+ "source_session_id": 520,
+ "source_time_seconds": 652.478105,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 652.53223,
+ "sequence": 380,
+ "source_event_sequence": 3433,
+ "source_session_id": 540,
+ "source_time_seconds": 652.53223,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 652.982894,
+ "sequence": 381,
+ "source_event_sequence": 3657,
+ "source_session_id": 579,
+ "source_time_seconds": 652.982894,
+ "source_user_id": 473,
+ "trace_session_id": "ts-00579-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 652.982894,
+ "sequence": 382,
+ "source_event_sequence": 3657,
+ "source_session_id": 578,
+ "source_time_seconds": 652.982894,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 653.244159,
+ "sequence": 383,
+ "source_event_sequence": 2890,
+ "source_session_id": 454,
+ "source_time_seconds": 653.244159,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 656.206722,
+ "sequence": 384,
+ "source_event_sequence": 2948,
+ "source_session_id": 578,
+ "source_time_seconds": 656.206722,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 656.358292,
+ "sequence": 385,
+ "source_event_sequence": 4304,
+ "source_session_id": 685,
+ "source_time_seconds": 656.358292,
+ "source_user_id": 36,
+ "trace_session_id": "ts-00685-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 656.468743,
+ "sequence": 386,
+ "source_event_sequence": 3804,
+ "source_session_id": 665,
+ "source_time_seconds": 656.468743,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 656.748036,
+ "sequence": 387,
+ "source_event_sequence": 4307,
+ "source_session_id": 686,
+ "source_time_seconds": 656.748036,
+ "source_user_id": 441,
+ "trace_session_id": "ts-00686-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 657.869617,
+ "sequence": 388,
+ "source_event_sequence": 3144,
+ "source_session_id": 627,
+ "source_time_seconds": 657.869617,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 658.285289,
+ "sequence": 389,
+ "source_event_sequence": 4310,
+ "source_session_id": 627,
+ "source_time_seconds": 658.285289,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 658.347334,
+ "sequence": 390,
+ "source_event_sequence": 4290,
+ "source_session_id": 627,
+ "source_time_seconds": 658.347334,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 658.445023,
+ "sequence": 391,
+ "source_event_sequence": 4316,
+ "source_session_id": 627,
+ "source_time_seconds": 658.445023,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 659.296219,
+ "sequence": 392,
+ "source_event_sequence": 3282,
+ "source_session_id": 627,
+ "source_time_seconds": 659.296219,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 659.46077,
+ "sequence": 393,
+ "source_event_sequence": 4323,
+ "source_session_id": 627,
+ "source_time_seconds": 659.46077,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g05"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 659.658631,
+ "sequence": 394,
+ "source_event_sequence": 3524,
+ "source_session_id": 559,
+ "source_time_seconds": 659.658631,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 659.808573,
+ "sequence": 395,
+ "source_event_sequence": 2517,
+ "source_session_id": 627,
+ "source_time_seconds": 659.808573,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g05"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 659.827741,
+ "sequence": 396,
+ "source_event_sequence": 2891,
+ "source_session_id": 454,
+ "source_time_seconds": 659.827741,
+ "source_user_id": 221,
+ "trace_session_id": "ts-00454-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 659.827741,
+ "sequence": 397,
+ "source_event_sequence": 2891,
+ "source_session_id": 627,
+ "source_time_seconds": 659.827741,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g06"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 659.843822,
+ "sequence": 398,
+ "source_event_sequence": 4303,
+ "source_session_id": 686,
+ "source_time_seconds": 659.843822,
+ "source_user_id": 441,
+ "trace_session_id": "ts-00686-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 659.843822,
+ "sequence": 399,
+ "source_event_sequence": 4303,
+ "source_session_id": 693,
+ "source_time_seconds": 659.843822,
+ "source_user_id": 386,
+ "trace_session_id": "ts-00693-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 661.35515,
+ "sequence": 400,
+ "source_event_sequence": 4359,
+ "source_session_id": 665,
+ "source_time_seconds": 661.35515,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 661.399899,
+ "sequence": 401,
+ "source_event_sequence": 3573,
+ "source_session_id": 665,
+ "source_time_seconds": 661.399899,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 661.490347,
+ "sequence": 402,
+ "source_event_sequence": 4057,
+ "source_session_id": 644,
+ "source_time_seconds": 661.490347,
+ "source_user_id": 135,
+ "trace_session_id": "ts-00644-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 661.490347,
+ "sequence": 403,
+ "source_event_sequence": 4057,
+ "source_session_id": 665,
+ "source_time_seconds": 661.490347,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 661.854382,
+ "sequence": 404,
+ "source_event_sequence": 4078,
+ "source_session_id": 649,
+ "source_time_seconds": 661.854382,
+ "source_user_id": 101,
+ "trace_session_id": "ts-00649-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 663.044393,
+ "sequence": 405,
+ "source_event_sequence": 4368,
+ "source_session_id": 698,
+ "source_time_seconds": 663.044393,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 663.30989,
+ "sequence": 406,
+ "source_event_sequence": 3857,
+ "source_session_id": 665,
+ "source_time_seconds": 663.30989,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 663.460878,
+ "sequence": 407,
+ "source_event_sequence": 4374,
+ "source_session_id": 665,
+ "source_time_seconds": 663.460878,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 663.543947,
+ "sequence": 408,
+ "source_event_sequence": 3826,
+ "source_session_id": 665,
+ "source_time_seconds": 663.543947,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g04"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 663.866008,
+ "sequence": 409,
+ "source_event_sequence": 4364,
+ "source_session_id": 698,
+ "source_time_seconds": 663.866008,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 664.025578,
+ "sequence": 410,
+ "source_event_sequence": 4079,
+ "source_session_id": 649,
+ "source_time_seconds": 664.025578,
+ "source_user_id": 101,
+ "trace_session_id": "ts-00649-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 664.025578,
+ "sequence": 411,
+ "source_event_sequence": 4079,
+ "source_session_id": 665,
+ "source_time_seconds": 664.025578,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g05"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 665.743692,
+ "sequence": 412,
+ "source_event_sequence": 3434,
+ "source_session_id": 540,
+ "source_time_seconds": 665.743692,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 668.440524,
+ "sequence": 413,
+ "source_event_sequence": 4424,
+ "source_session_id": 705,
+ "source_time_seconds": 668.440524,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 670.544154,
+ "sequence": 414,
+ "source_event_sequence": 4393,
+ "source_session_id": 705,
+ "source_time_seconds": 670.544154,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 674.792196,
+ "sequence": 415,
+ "source_event_sequence": 4047,
+ "source_session_id": 641,
+ "source_time_seconds": 674.792196,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 674.811714,
+ "sequence": 416,
+ "source_event_sequence": 4183,
+ "source_session_id": 665,
+ "source_time_seconds": 674.811714,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g05"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 680.026054,
+ "sequence": 417,
+ "source_event_sequence": 3310,
+ "source_session_id": 520,
+ "source_time_seconds": 680.026054,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 681.699297,
+ "sequence": 418,
+ "source_event_sequence": 3435,
+ "source_session_id": 540,
+ "source_time_seconds": 681.699297,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 682.666305,
+ "sequence": 419,
+ "source_event_sequence": 4328,
+ "source_session_id": 693,
+ "source_time_seconds": 682.666305,
+ "source_user_id": 386,
+ "trace_session_id": "ts-00693-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 684.825987,
+ "sequence": 420,
+ "source_event_sequence": 4394,
+ "source_session_id": 705,
+ "source_time_seconds": 684.825987,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 685.474724,
+ "sequence": 421,
+ "source_event_sequence": 3436,
+ "source_session_id": 540,
+ "source_time_seconds": 685.474724,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 686.161993,
+ "sequence": 422,
+ "source_event_sequence": 4005,
+ "source_session_id": 634,
+ "source_time_seconds": 686.161993,
+ "source_user_id": 64,
+ "trace_session_id": "ts-00634-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 686.19572,
+ "sequence": 423,
+ "source_event_sequence": 3754,
+ "source_session_id": 593,
+ "source_time_seconds": 686.19572,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 686.641081,
+ "sequence": 424,
+ "source_event_sequence": 3525,
+ "source_session_id": 559,
+ "source_time_seconds": 686.641081,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 686.828242,
+ "sequence": 425,
+ "source_event_sequence": 4365,
+ "source_session_id": 698,
+ "source_time_seconds": 686.828242,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 687.040162,
+ "sequence": 426,
+ "source_event_sequence": 4637,
+ "source_session_id": 720,
+ "source_time_seconds": 687.040162,
+ "source_user_id": 472,
+ "trace_session_id": "ts-00720-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 687.76928,
+ "sequence": 427,
+ "source_event_sequence": 3755,
+ "source_session_id": 593,
+ "source_time_seconds": 687.76928,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 687.870986,
+ "sequence": 428,
+ "source_event_sequence": 4306,
+ "source_session_id": 665,
+ "source_time_seconds": 687.870986,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00665-g05"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 690.563396,
+ "sequence": 429,
+ "source_event_sequence": 4006,
+ "source_session_id": 634,
+ "source_time_seconds": 690.563396,
+ "source_user_id": 64,
+ "trace_session_id": "ts-00634-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 690.66215,
+ "sequence": 430,
+ "source_event_sequence": 3947,
+ "source_session_id": 627,
+ "source_time_seconds": 690.66215,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g06"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 690.840202,
+ "sequence": 431,
+ "source_event_sequence": 4395,
+ "source_session_id": 705,
+ "source_time_seconds": 690.840202,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 692.125069,
+ "sequence": 432,
+ "source_event_sequence": 3311,
+ "source_session_id": 520,
+ "source_time_seconds": 692.125069,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 696.547721,
+ "sequence": 433,
+ "source_event_sequence": 4492,
+ "source_session_id": 720,
+ "source_time_seconds": 696.547721,
+ "source_user_id": 472,
+ "trace_session_id": "ts-00720-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 696.547721,
+ "sequence": 434,
+ "source_event_sequence": 4492,
+ "source_session_id": 744,
+ "source_time_seconds": 696.547721,
+ "source_user_id": 65,
+ "trace_session_id": "ts-00744-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 698.239078,
+ "sequence": 435,
+ "source_event_sequence": 4298,
+ "source_session_id": 685,
+ "source_time_seconds": 698.239078,
+ "source_user_id": 36,
+ "trace_session_id": "ts-00685-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 699.587918,
+ "sequence": 436,
+ "source_event_sequence": 3756,
+ "source_session_id": 593,
+ "source_time_seconds": 699.587918,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 701.856177,
+ "sequence": 437,
+ "source_event_sequence": 4299,
+ "source_session_id": 685,
+ "source_time_seconds": 701.856177,
+ "source_user_id": 36,
+ "trace_session_id": "ts-00685-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 702.661617,
+ "sequence": 438,
+ "source_event_sequence": 4300,
+ "source_session_id": 685,
+ "source_time_seconds": 702.661617,
+ "source_user_id": 36,
+ "trace_session_id": "ts-00685-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 702.661617,
+ "sequence": 439,
+ "source_event_sequence": 4300,
+ "source_session_id": 767,
+ "source_time_seconds": 702.661617,
+ "source_user_id": 109,
+ "trace_session_id": "ts-00767-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 707.083916,
+ "sequence": 440,
+ "source_event_sequence": 4366,
+ "source_session_id": 698,
+ "source_time_seconds": 707.083916,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 708.155643,
+ "sequence": 441,
+ "source_event_sequence": 4892,
+ "source_session_id": 772,
+ "source_time_seconds": 708.155643,
+ "source_user_id": 461,
+ "trace_session_id": "ts-00772-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 708.756288,
+ "sequence": 442,
+ "source_event_sequence": 4860,
+ "source_session_id": 627,
+ "source_time_seconds": 708.756288,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g06"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 709.210613,
+ "sequence": 443,
+ "source_event_sequence": 4917,
+ "source_session_id": 776,
+ "source_time_seconds": 709.210613,
+ "source_user_id": 465,
+ "trace_session_id": "ts-00776-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 709.651669,
+ "sequence": 444,
+ "source_event_sequence": 4834,
+ "source_session_id": 772,
+ "source_time_seconds": 709.651669,
+ "source_user_id": 461,
+ "trace_session_id": "ts-00772-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 709.651669,
+ "sequence": 445,
+ "source_event_sequence": 4834,
+ "source_session_id": 768,
+ "source_time_seconds": 709.651669,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 709.830516,
+ "sequence": 446,
+ "source_event_sequence": 4396,
+ "source_session_id": 705,
+ "source_time_seconds": 709.830516,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 709.867895,
+ "sequence": 447,
+ "source_event_sequence": 3526,
+ "source_session_id": 559,
+ "source_time_seconds": 709.867895,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 711.082956,
+ "sequence": 448,
+ "source_event_sequence": 4788,
+ "source_session_id": 768,
+ "source_time_seconds": 711.082956,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 712.374027,
+ "sequence": 449,
+ "source_event_sequence": 4639,
+ "source_session_id": 744,
+ "source_time_seconds": 712.374027,
+ "source_user_id": 65,
+ "trace_session_id": "ts-00744-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 713.396319,
+ "sequence": 450,
+ "source_event_sequence": 3437,
+ "source_session_id": 540,
+ "source_time_seconds": 713.396319,
+ "source_user_id": 362,
+ "trace_session_id": "ts-00540-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 713.396319,
+ "sequence": 451,
+ "source_event_sequence": 3437,
+ "source_session_id": 627,
+ "source_time_seconds": 713.396319,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 713.431756,
+ "sequence": 452,
+ "source_event_sequence": 4329,
+ "source_session_id": 693,
+ "source_time_seconds": 713.431756,
+ "source_user_id": 386,
+ "trace_session_id": "ts-00693-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 715.220506,
+ "sequence": 453,
+ "source_event_sequence": 4048,
+ "source_session_id": 641,
+ "source_time_seconds": 715.220506,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 721.298454,
+ "sequence": 454,
+ "source_event_sequence": 4640,
+ "source_session_id": 744,
+ "source_time_seconds": 721.298454,
+ "source_user_id": 65,
+ "trace_session_id": "ts-00744-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 723.27308,
+ "sequence": 455,
+ "source_event_sequence": 4330,
+ "source_session_id": 693,
+ "source_time_seconds": 723.27308,
+ "source_user_id": 386,
+ "trace_session_id": "ts-00693-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 724.347425,
+ "sequence": 456,
+ "source_event_sequence": 3948,
+ "source_session_id": 627,
+ "source_time_seconds": 724.347425,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 724.736559,
+ "sequence": 457,
+ "source_event_sequence": 4784,
+ "source_session_id": 767,
+ "source_time_seconds": 724.736559,
+ "source_user_id": 109,
+ "trace_session_id": "ts-00767-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 725.74643,
+ "sequence": 458,
+ "source_event_sequence": 4785,
+ "source_session_id": 767,
+ "source_time_seconds": 725.74643,
+ "source_user_id": 109,
+ "trace_session_id": "ts-00767-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 725.74643,
+ "sequence": 459,
+ "source_event_sequence": 4785,
+ "source_session_id": 804,
+ "source_time_seconds": 725.74643,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 726.033517,
+ "sequence": 460,
+ "source_event_sequence": 4331,
+ "source_session_id": 693,
+ "source_time_seconds": 726.033517,
+ "source_user_id": 386,
+ "trace_session_id": "ts-00693-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 729.460378,
+ "sequence": 461,
+ "source_event_sequence": 3757,
+ "source_session_id": 593,
+ "source_time_seconds": 729.460378,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 730.579337,
+ "sequence": 462,
+ "source_event_sequence": 4641,
+ "source_session_id": 744,
+ "source_time_seconds": 730.579337,
+ "source_user_id": 65,
+ "trace_session_id": "ts-00744-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 730.770661,
+ "sequence": 463,
+ "source_event_sequence": 4332,
+ "source_session_id": 693,
+ "source_time_seconds": 730.770661,
+ "source_user_id": 386,
+ "trace_session_id": "ts-00693-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 730.770661,
+ "sequence": 464,
+ "source_event_sequence": 4332,
+ "source_session_id": 808,
+ "source_time_seconds": 730.770661,
+ "source_user_id": 270,
+ "trace_session_id": "ts-00808-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 732.621273,
+ "sequence": 465,
+ "source_event_sequence": 5215,
+ "source_session_id": 788,
+ "source_time_seconds": 732.621273,
+ "source_user_id": 366,
+ "trace_session_id": "ts-00788-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 733.368765,
+ "sequence": 466,
+ "source_event_sequence": 4195,
+ "source_session_id": 788,
+ "source_time_seconds": 733.368765,
+ "source_user_id": 366,
+ "trace_session_id": "ts-00788-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 733.476008,
+ "sequence": 467,
+ "source_event_sequence": 3758,
+ "source_session_id": 593,
+ "source_time_seconds": 733.476008,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 733.694732,
+ "sequence": 468,
+ "source_event_sequence": 5237,
+ "source_session_id": 834,
+ "source_time_seconds": 733.694732,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 733.842348,
+ "sequence": 469,
+ "source_event_sequence": 3312,
+ "source_session_id": 520,
+ "source_time_seconds": 733.842348,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 734.796551,
+ "sequence": 470,
+ "source_event_sequence": 4789,
+ "source_session_id": 768,
+ "source_time_seconds": 734.796551,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 735.210969,
+ "sequence": 471,
+ "source_event_sequence": 4790,
+ "source_session_id": 768,
+ "source_time_seconds": 735.210969,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 736.542933,
+ "sequence": 472,
+ "source_event_sequence": 4397,
+ "source_session_id": 705,
+ "source_time_seconds": 736.542933,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 736.742681,
+ "sequence": 473,
+ "source_event_sequence": 3313,
+ "source_session_id": 520,
+ "source_time_seconds": 736.742681,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 739.244242,
+ "sequence": 474,
+ "source_event_sequence": 4398,
+ "source_session_id": 705,
+ "source_time_seconds": 739.244242,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 739.343493,
+ "sequence": 475,
+ "source_event_sequence": 5072,
+ "source_session_id": 808,
+ "source_time_seconds": 739.343493,
+ "source_user_id": 270,
+ "trace_session_id": "ts-00808-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 739.484382,
+ "sequence": 476,
+ "source_event_sequence": 5238,
+ "source_session_id": 834,
+ "source_time_seconds": 739.484382,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 742.514534,
+ "sequence": 477,
+ "source_event_sequence": 4007,
+ "source_session_id": 634,
+ "source_time_seconds": 742.514534,
+ "source_user_id": 64,
+ "trace_session_id": "ts-00634-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 742.514534,
+ "sequence": 478,
+ "source_event_sequence": 4007,
+ "source_session_id": 846,
+ "source_time_seconds": 742.514534,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 744.442945,
+ "sequence": 479,
+ "source_event_sequence": 3759,
+ "source_session_id": 593,
+ "source_time_seconds": 744.442945,
+ "source_user_id": 474,
+ "trace_session_id": "ts-00593-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 744.442945,
+ "sequence": 480,
+ "source_event_sequence": 3759,
+ "source_session_id": 788,
+ "source_time_seconds": 744.442945,
+ "source_user_id": 366,
+ "trace_session_id": "ts-00788-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 745.318289,
+ "sequence": 481,
+ "source_event_sequence": 3314,
+ "source_session_id": 520,
+ "source_time_seconds": 745.318289,
+ "source_user_id": 223,
+ "trace_session_id": "ts-00520-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 745.654695,
+ "sequence": 482,
+ "source_event_sequence": 5073,
+ "source_session_id": 808,
+ "source_time_seconds": 745.654695,
+ "source_user_id": 270,
+ "trace_session_id": "ts-00808-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 746.100555,
+ "sequence": 483,
+ "source_event_sequence": 5414,
+ "source_session_id": 578,
+ "source_time_seconds": 746.100555,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 749.283463,
+ "sequence": 484,
+ "source_event_sequence": 4571,
+ "source_session_id": 578,
+ "source_time_seconds": 749.283463,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g02"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 749.945746,
+ "sequence": 485,
+ "source_event_sequence": 5480,
+ "source_session_id": 866,
+ "source_time_seconds": 749.945746,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 750.644536,
+ "sequence": 486,
+ "source_event_sequence": 3949,
+ "source_session_id": 627,
+ "source_time_seconds": 750.644536,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 751.238389,
+ "sequence": 487,
+ "source_event_sequence": 4399,
+ "source_session_id": 705,
+ "source_time_seconds": 751.238389,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 752.250928,
+ "sequence": 488,
+ "source_event_sequence": 4791,
+ "source_session_id": 768,
+ "source_time_seconds": 752.250928,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 753.175388,
+ "sequence": 489,
+ "source_event_sequence": 4849,
+ "source_session_id": 776,
+ "source_time_seconds": 753.175388,
+ "source_user_id": 465,
+ "trace_session_id": "ts-00776-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 755.495393,
+ "sequence": 490,
+ "source_event_sequence": 5053,
+ "source_session_id": 804,
+ "source_time_seconds": 755.495393,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 755.807178,
+ "sequence": 491,
+ "source_event_sequence": 4400,
+ "source_session_id": 705,
+ "source_time_seconds": 755.807178,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 756.434532,
+ "sequence": 492,
+ "source_event_sequence": 5481,
+ "source_session_id": 866,
+ "source_time_seconds": 756.434532,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 756.499559,
+ "sequence": 493,
+ "source_event_sequence": 5322,
+ "source_session_id": 846,
+ "source_time_seconds": 756.499559,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 759.419224,
+ "sequence": 494,
+ "source_event_sequence": 4938,
+ "source_session_id": 788,
+ "source_time_seconds": 759.419224,
+ "source_user_id": 366,
+ "trace_session_id": "ts-00788-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 759.419224,
+ "sequence": 495,
+ "source_event_sequence": 4938,
+ "source_session_id": 875,
+ "source_time_seconds": 759.419224,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 760.713681,
+ "sequence": 496,
+ "source_event_sequence": 5239,
+ "source_session_id": 834,
+ "source_time_seconds": 760.713681,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 761.968292,
+ "sequence": 497,
+ "source_event_sequence": 5240,
+ "source_session_id": 834,
+ "source_time_seconds": 761.968292,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 763.780609,
+ "sequence": 498,
+ "source_event_sequence": 4792,
+ "source_session_id": 768,
+ "source_time_seconds": 763.780609,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 764.474577,
+ "sequence": 499,
+ "source_event_sequence": 5746,
+ "source_session_id": 891,
+ "source_time_seconds": 764.474577,
+ "source_user_id": 404,
+ "trace_session_id": "ts-00891-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 764.554757,
+ "sequence": 500,
+ "source_event_sequence": 4952,
+ "source_session_id": 627,
+ "source_time_seconds": 764.554757,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g07"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 764.761106,
+ "sequence": 501,
+ "source_event_sequence": 5755,
+ "source_session_id": 896,
+ "source_time_seconds": 764.761106,
+ "source_user_id": 205,
+ "trace_session_id": "ts-00896-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 764.968668,
+ "sequence": 502,
+ "source_event_sequence": 5656,
+ "source_session_id": 891,
+ "source_time_seconds": 764.968668,
+ "source_user_id": 404,
+ "trace_session_id": "ts-00891-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 765.183292,
+ "sequence": 503,
+ "source_event_sequence": 5628,
+ "source_session_id": 808,
+ "source_time_seconds": 765.183292,
+ "source_user_id": 270,
+ "trace_session_id": "ts-00808-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 765.192439,
+ "sequence": 504,
+ "source_event_sequence": 5772,
+ "source_session_id": 877,
+ "source_time_seconds": 765.192439,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 765.209698,
+ "sequence": 505,
+ "source_event_sequence": 5338,
+ "source_session_id": 846,
+ "source_time_seconds": 765.209698,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 766.060128,
+ "sequence": 506,
+ "source_event_sequence": 5793,
+ "source_session_id": 890,
+ "source_time_seconds": 766.060128,
+ "source_user_id": 174,
+ "trace_session_id": "ts-00890-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 768.329573,
+ "sequence": 507,
+ "source_event_sequence": 5482,
+ "source_session_id": 866,
+ "source_time_seconds": 768.329573,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 769.860037,
+ "sequence": 508,
+ "source_event_sequence": 4049,
+ "source_session_id": 641,
+ "source_time_seconds": 769.860037,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 770.662314,
+ "sequence": 509,
+ "source_event_sequence": 5657,
+ "source_session_id": 891,
+ "source_time_seconds": 770.662314,
+ "source_user_id": 404,
+ "trace_session_id": "ts-00891-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 770.662314,
+ "sequence": 510,
+ "source_event_sequence": 5657,
+ "source_session_id": 846,
+ "source_time_seconds": 770.662314,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 772.425763,
+ "sequence": 511,
+ "source_event_sequence": 5323,
+ "source_session_id": 846,
+ "source_time_seconds": 772.425763,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 772.86264,
+ "sequence": 512,
+ "source_event_sequence": 4850,
+ "source_session_id": 776,
+ "source_time_seconds": 772.86264,
+ "source_user_id": 465,
+ "trace_session_id": "ts-00776-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 774.522334,
+ "sequence": 513,
+ "source_event_sequence": 4401,
+ "source_session_id": 705,
+ "source_time_seconds": 774.522334,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 774.700073,
+ "sequence": 514,
+ "source_event_sequence": 5680,
+ "source_session_id": 896,
+ "source_time_seconds": 774.700073,
+ "source_user_id": 205,
+ "trace_session_id": "ts-00896-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 775.127178,
+ "sequence": 515,
+ "source_event_sequence": 5544,
+ "source_session_id": 875,
+ "source_time_seconds": 775.127178,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 777.022547,
+ "sequence": 516,
+ "source_event_sequence": 5483,
+ "source_session_id": 866,
+ "source_time_seconds": 777.022547,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 777.121322,
+ "sequence": 517,
+ "source_event_sequence": 4642,
+ "source_session_id": 744,
+ "source_time_seconds": 777.121322,
+ "source_user_id": 65,
+ "trace_session_id": "ts-00744-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 777.347912,
+ "sequence": 518,
+ "source_event_sequence": 5545,
+ "source_session_id": 875,
+ "source_time_seconds": 777.347912,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00875-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 777.347912,
+ "sequence": 519,
+ "source_event_sequence": 5545,
+ "source_session_id": 927,
+ "source_time_seconds": 777.347912,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 778.774336,
+ "sequence": 520,
+ "source_event_sequence": 5552,
+ "source_session_id": 877,
+ "source_time_seconds": 778.774336,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 780.316429,
+ "sequence": 521,
+ "source_event_sequence": 4367,
+ "source_session_id": 698,
+ "source_time_seconds": 780.316429,
+ "source_user_id": 228,
+ "trace_session_id": "ts-00698-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 780.316429,
+ "sequence": 522,
+ "source_event_sequence": 4367,
+ "source_session_id": 808,
+ "source_time_seconds": 780.316429,
+ "source_user_id": 270,
+ "trace_session_id": "ts-00808-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 781.454774,
+ "sequence": 523,
+ "source_event_sequence": 4050,
+ "source_session_id": 641,
+ "source_time_seconds": 781.454774,
+ "source_user_id": 56,
+ "trace_session_id": "ts-00641-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 781.454774,
+ "sequence": 524,
+ "source_event_sequence": 4050,
+ "source_session_id": 627,
+ "source_time_seconds": 781.454774,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g08"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 782.031975,
+ "sequence": 525,
+ "source_event_sequence": 5877,
+ "source_session_id": 927,
+ "source_time_seconds": 782.031975,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 782.984391,
+ "sequence": 526,
+ "source_event_sequence": 5324,
+ "source_session_id": 846,
+ "source_time_seconds": 782.984391,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 783.132594,
+ "sequence": 527,
+ "source_event_sequence": 5241,
+ "source_session_id": 834,
+ "source_time_seconds": 783.132594,
+ "source_user_id": 358,
+ "trace_session_id": "ts-00834-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 783.132594,
+ "sequence": 528,
+ "source_event_sequence": 5241,
+ "source_session_id": 945,
+ "source_time_seconds": 783.132594,
+ "source_user_id": 271,
+ "trace_session_id": "ts-00945-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 784.563945,
+ "sequence": 529,
+ "source_event_sequence": 5075,
+ "source_session_id": 808,
+ "source_time_seconds": 784.563945,
+ "source_user_id": 270,
+ "trace_session_id": "ts-00808-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 784.563945,
+ "sequence": 530,
+ "source_event_sequence": 5075,
+ "source_session_id": 944,
+ "source_time_seconds": 784.563945,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 786.065892,
+ "sequence": 531,
+ "source_event_sequence": 4793,
+ "source_session_id": 768,
+ "source_time_seconds": 786.065892,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 787.338938,
+ "sequence": 532,
+ "source_event_sequence": 3951,
+ "source_session_id": 627,
+ "source_time_seconds": 787.338938,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g08"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 788.742531,
+ "sequence": 533,
+ "source_event_sequence": 5878,
+ "source_session_id": 927,
+ "source_time_seconds": 788.742531,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 790.587029,
+ "sequence": 534,
+ "source_event_sequence": 5484,
+ "source_session_id": 866,
+ "source_time_seconds": 790.587029,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 790.99492,
+ "sequence": 535,
+ "source_event_sequence": 6179,
+ "source_session_id": 914,
+ "source_time_seconds": 790.99492,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 791.063232,
+ "sequence": 536,
+ "source_event_sequence": 6009,
+ "source_session_id": 914,
+ "source_time_seconds": 791.063232,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 791.164418,
+ "sequence": 537,
+ "source_event_sequence": 4643,
+ "source_session_id": 744,
+ "source_time_seconds": 791.164418,
+ "source_user_id": 65,
+ "trace_session_id": "ts-00744-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 791.164418,
+ "sequence": 538,
+ "source_event_sequence": 4643,
+ "source_session_id": 914,
+ "source_time_seconds": 791.164418,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 791.234077,
+ "sequence": 539,
+ "source_event_sequence": 5485,
+ "source_session_id": 866,
+ "source_time_seconds": 791.234077,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 791.97572,
+ "sequence": 540,
+ "source_event_sequence": 6193,
+ "source_session_id": 578,
+ "source_time_seconds": 791.97572,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 791.983953,
+ "sequence": 541,
+ "source_event_sequence": 3981,
+ "source_session_id": 578,
+ "source_time_seconds": 791.983953,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g03"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 792.406488,
+ "sequence": 542,
+ "source_event_sequence": 6202,
+ "source_session_id": 977,
+ "source_time_seconds": 792.406488,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 792.588332,
+ "sequence": 543,
+ "source_event_sequence": 5985,
+ "source_session_id": 944,
+ "source_time_seconds": 792.588332,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 792.653716,
+ "sequence": 544,
+ "source_event_sequence": 4794,
+ "source_session_id": 768,
+ "source_time_seconds": 792.653716,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 793.560773,
+ "sequence": 545,
+ "source_event_sequence": 5879,
+ "source_session_id": 927,
+ "source_time_seconds": 793.560773,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 793.695911,
+ "sequence": 546,
+ "source_event_sequence": 5797,
+ "source_session_id": 914,
+ "source_time_seconds": 793.695911,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 793.699724,
+ "sequence": 547,
+ "source_event_sequence": 4851,
+ "source_session_id": 776,
+ "source_time_seconds": 793.699724,
+ "source_user_id": 465,
+ "trace_session_id": "ts-00776-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 793.699724,
+ "sequence": 548,
+ "source_event_sequence": 4851,
+ "source_session_id": 914,
+ "source_time_seconds": 793.699724,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 794.117002,
+ "sequence": 549,
+ "source_event_sequence": 6216,
+ "source_session_id": 578,
+ "source_time_seconds": 794.117002,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g04"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 795.956286,
+ "sequence": 550,
+ "source_event_sequence": 5880,
+ "source_session_id": 927,
+ "source_time_seconds": 795.956286,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 796.095067,
+ "sequence": 551,
+ "source_event_sequence": 5553,
+ "source_session_id": 877,
+ "source_time_seconds": 796.095067,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 797.131887,
+ "sequence": 552,
+ "source_event_sequence": 5571,
+ "source_session_id": 578,
+ "source_time_seconds": 797.131887,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 797.738251,
+ "sequence": 553,
+ "source_event_sequence": 6240,
+ "source_session_id": 982,
+ "source_time_seconds": 797.738251,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 798.094366,
+ "sequence": 554,
+ "source_event_sequence": 4351,
+ "source_session_id": 914,
+ "source_time_seconds": 798.094366,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 798.292512,
+ "sequence": 555,
+ "source_event_sequence": 6243,
+ "source_session_id": 914,
+ "source_time_seconds": 798.292512,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g04"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 799.055406,
+ "sequence": 556,
+ "source_event_sequence": 5994,
+ "source_session_id": 945,
+ "source_time_seconds": 799.055406,
+ "source_user_id": 271,
+ "trace_session_id": "ts-00945-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 799.204307,
+ "sequence": 557,
+ "source_event_sequence": 6225,
+ "source_session_id": 982,
+ "source_time_seconds": 799.204307,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 799.231668,
+ "sequence": 558,
+ "source_event_sequence": 5986,
+ "source_session_id": 944,
+ "source_time_seconds": 799.231668,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 799.436833,
+ "sequence": 559,
+ "source_event_sequence": 4402,
+ "source_session_id": 705,
+ "source_time_seconds": 799.436833,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 799.449255,
+ "sequence": 560,
+ "source_event_sequence": 6248,
+ "source_session_id": 578,
+ "source_time_seconds": 799.449255,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g05"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 799.604953,
+ "sequence": 561,
+ "source_event_sequence": 5554,
+ "source_session_id": 877,
+ "source_time_seconds": 799.604953,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 800.204835,
+ "sequence": 562,
+ "source_event_sequence": 6059,
+ "source_session_id": 578,
+ "source_time_seconds": 800.204835,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g05"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 800.260462,
+ "sequence": 563,
+ "source_event_sequence": 6263,
+ "source_session_id": 578,
+ "source_time_seconds": 800.260462,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g06"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 800.440507,
+ "sequence": 564,
+ "source_event_sequence": 5486,
+ "source_session_id": 866,
+ "source_time_seconds": 800.440507,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 800.856084,
+ "sequence": 565,
+ "source_event_sequence": 6242,
+ "source_session_id": 578,
+ "source_time_seconds": 800.856084,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g06"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 800.872062,
+ "sequence": 566,
+ "source_event_sequence": 6266,
+ "source_session_id": 578,
+ "source_time_seconds": 800.872062,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g07"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 801.103228,
+ "sequence": 567,
+ "source_event_sequence": 6065,
+ "source_session_id": 578,
+ "source_time_seconds": 801.103228,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g07"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 801.139444,
+ "sequence": 568,
+ "source_event_sequence": 6273,
+ "source_session_id": 578,
+ "source_time_seconds": 801.139444,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g08"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 801.409049,
+ "sequence": 569,
+ "source_event_sequence": 5487,
+ "source_session_id": 866,
+ "source_time_seconds": 801.409049,
+ "source_user_id": 43,
+ "trace_session_id": "ts-00866-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 801.409049,
+ "sequence": 570,
+ "source_event_sequence": 5487,
+ "source_session_id": 849,
+ "source_time_seconds": 801.409049,
+ "source_user_id": 367,
+ "trace_session_id": "ts-00849-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 801.45226,
+ "sequence": 571,
+ "source_event_sequence": 4475,
+ "source_session_id": 849,
+ "source_time_seconds": 801.45226,
+ "source_user_id": 367,
+ "trace_session_id": "ts-00849-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 801.634049,
+ "sequence": 572,
+ "source_event_sequence": 6291,
+ "source_session_id": 849,
+ "source_time_seconds": 801.634049,
+ "source_user_id": 367,
+ "trace_session_id": "ts-00849-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 801.72996,
+ "sequence": 573,
+ "source_event_sequence": 3654,
+ "source_session_id": 578,
+ "source_time_seconds": 801.72996,
+ "source_user_id": 154,
+ "trace_session_id": "ts-00578-g08"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 802.352641,
+ "sequence": 574,
+ "source_event_sequence": 5881,
+ "source_session_id": 927,
+ "source_time_seconds": 802.352641,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 802.66243,
+ "sequence": 575,
+ "source_event_sequence": 3952,
+ "source_session_id": 627,
+ "source_time_seconds": 802.66243,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g08"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 803.937331,
+ "sequence": 576,
+ "source_event_sequence": 4403,
+ "source_session_id": 705,
+ "source_time_seconds": 803.937331,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 804.371757,
+ "sequence": 577,
+ "source_event_sequence": 4795,
+ "source_session_id": 768,
+ "source_time_seconds": 804.371757,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 805.717145,
+ "sequence": 578,
+ "source_event_sequence": 5652,
+ "source_session_id": 890,
+ "source_time_seconds": 805.717145,
+ "source_user_id": 174,
+ "trace_session_id": "ts-00890-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 807.581866,
+ "sequence": 579,
+ "source_event_sequence": 6203,
+ "source_session_id": 977,
+ "source_time_seconds": 807.581866,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 807.659186,
+ "sequence": 580,
+ "source_event_sequence": 6226,
+ "source_session_id": 982,
+ "source_time_seconds": 807.659186,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 807.921726,
+ "sequence": 581,
+ "source_event_sequence": 5054,
+ "source_session_id": 804,
+ "source_time_seconds": 807.921726,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 808.587597,
+ "sequence": 582,
+ "source_event_sequence": 6227,
+ "source_session_id": 982,
+ "source_time_seconds": 808.587597,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 809.127116,
+ "sequence": 583,
+ "source_event_sequence": 5325,
+ "source_session_id": 846,
+ "source_time_seconds": 809.127116,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 809.635186,
+ "sequence": 584,
+ "source_event_sequence": 5326,
+ "source_session_id": 846,
+ "source_time_seconds": 809.635186,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 809.639792,
+ "sequence": 585,
+ "source_event_sequence": 4796,
+ "source_session_id": 768,
+ "source_time_seconds": 809.639792,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 809.703847,
+ "sequence": 586,
+ "source_event_sequence": 4404,
+ "source_session_id": 705,
+ "source_time_seconds": 809.703847,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 809.925805,
+ "sequence": 587,
+ "source_event_sequence": 4797,
+ "source_session_id": 768,
+ "source_time_seconds": 809.925805,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 810.454703,
+ "sequence": 588,
+ "source_event_sequence": 3527,
+ "source_session_id": 559,
+ "source_time_seconds": 810.454703,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 810.743347,
+ "sequence": 589,
+ "source_event_sequence": 5882,
+ "source_session_id": 927,
+ "source_time_seconds": 810.743347,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 810.953334,
+ "sequence": 590,
+ "source_event_sequence": 5344,
+ "source_session_id": 849,
+ "source_time_seconds": 810.953334,
+ "source_user_id": 367,
+ "trace_session_id": "ts-00849-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 811.95778,
+ "sequence": 591,
+ "source_event_sequence": 6204,
+ "source_session_id": 977,
+ "source_time_seconds": 811.95778,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 812.970567,
+ "sequence": 592,
+ "source_event_sequence": 6205,
+ "source_session_id": 977,
+ "source_time_seconds": 812.970567,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 813.12998,
+ "sequence": 593,
+ "source_event_sequence": 5555,
+ "source_session_id": 877,
+ "source_time_seconds": 813.12998,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 816.203564,
+ "sequence": 594,
+ "source_event_sequence": 4405,
+ "source_session_id": 705,
+ "source_time_seconds": 816.203564,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 816.286225,
+ "sequence": 595,
+ "source_event_sequence": 6228,
+ "source_session_id": 982,
+ "source_time_seconds": 816.286225,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 818.234752,
+ "sequence": 596,
+ "source_event_sequence": 4040,
+ "source_session_id": 849,
+ "source_time_seconds": 818.234752,
+ "source_user_id": 367,
+ "trace_session_id": "ts-00849-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 820.608682,
+ "sequence": 597,
+ "source_event_sequence": 6527,
+ "source_session_id": 1016,
+ "source_time_seconds": 820.608682,
+ "source_user_id": 40,
+ "trace_session_id": "ts-01016-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 821.64449,
+ "sequence": 598,
+ "source_event_sequence": 3377,
+ "source_session_id": 914,
+ "source_time_seconds": 821.64449,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 821.763725,
+ "sequence": 599,
+ "source_event_sequence": 6533,
+ "source_session_id": 1015,
+ "source_time_seconds": 821.763725,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 822.095268,
+ "sequence": 600,
+ "source_event_sequence": 6063,
+ "source_session_id": 627,
+ "source_time_seconds": 822.095268,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g08"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 822.097406,
+ "sequence": 601,
+ "source_event_sequence": 6535,
+ "source_session_id": 627,
+ "source_time_seconds": 822.097406,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g09"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 822.928104,
+ "sequence": 602,
+ "source_event_sequence": 6353,
+ "source_session_id": 627,
+ "source_time_seconds": 822.928104,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g09"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 823.202618,
+ "sequence": 603,
+ "source_event_sequence": 5987,
+ "source_session_id": 944,
+ "source_time_seconds": 823.202618,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 823.430961,
+ "sequence": 604,
+ "source_event_sequence": 6556,
+ "source_session_id": 627,
+ "source_time_seconds": 823.430961,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g10"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 824.7594,
+ "sequence": 605,
+ "source_event_sequence": 6534,
+ "source_session_id": 627,
+ "source_time_seconds": 824.7594,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g10"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 824.840032,
+ "sequence": 606,
+ "source_event_sequence": 6566,
+ "source_session_id": 627,
+ "source_time_seconds": 824.840032,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g11"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 824.959308,
+ "sequence": 607,
+ "source_event_sequence": 4406,
+ "source_session_id": 705,
+ "source_time_seconds": 824.959308,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 825.125207,
+ "sequence": 608,
+ "source_event_sequence": 6512,
+ "source_session_id": 627,
+ "source_time_seconds": 825.125207,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g11"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 825.258719,
+ "sequence": 609,
+ "source_event_sequence": 6206,
+ "source_session_id": 977,
+ "source_time_seconds": 825.258719,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 825.683671,
+ "sequence": 610,
+ "source_event_sequence": 6575,
+ "source_session_id": 627,
+ "source_time_seconds": 825.683671,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g12"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 825.723778,
+ "sequence": 611,
+ "source_event_sequence": 4565,
+ "source_session_id": 627,
+ "source_time_seconds": 825.723778,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g12"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 826.161977,
+ "sequence": 612,
+ "source_event_sequence": 5653,
+ "source_session_id": 890,
+ "source_time_seconds": 826.161977,
+ "source_user_id": 174,
+ "trace_session_id": "ts-00890-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 826.616011,
+ "sequence": 613,
+ "source_event_sequence": 6594,
+ "source_session_id": 1038,
+ "source_time_seconds": 826.616011,
+ "source_user_id": 455,
+ "trace_session_id": "ts-01038-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 830.180488,
+ "sequence": 614,
+ "source_event_sequence": 6443,
+ "source_session_id": 1015,
+ "source_time_seconds": 830.180488,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 831.792183,
+ "sequence": 615,
+ "source_event_sequence": 5681,
+ "source_session_id": 896,
+ "source_time_seconds": 831.792183,
+ "source_user_id": 205,
+ "trace_session_id": "ts-00896-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 831.797566,
+ "sequence": 616,
+ "source_event_sequence": 5197,
+ "source_session_id": 1015,
+ "source_time_seconds": 831.797566,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 831.922018,
+ "sequence": 617,
+ "source_event_sequence": 6671,
+ "source_session_id": 1015,
+ "source_time_seconds": 831.922018,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 831.955076,
+ "sequence": 618,
+ "source_event_sequence": 5542,
+ "source_session_id": 1015,
+ "source_time_seconds": 831.955076,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 832.571365,
+ "sequence": 619,
+ "source_event_sequence": 6207,
+ "source_session_id": 977,
+ "source_time_seconds": 832.571365,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 832.692228,
+ "sequence": 620,
+ "source_event_sequence": 5682,
+ "source_session_id": 896,
+ "source_time_seconds": 832.692228,
+ "source_user_id": 205,
+ "trace_session_id": "ts-00896-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 832.803539,
+ "sequence": 621,
+ "source_event_sequence": 6682,
+ "source_session_id": 1015,
+ "source_time_seconds": 832.803539,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 832.87162,
+ "sequence": 622,
+ "source_event_sequence": 5055,
+ "source_session_id": 804,
+ "source_time_seconds": 832.87162,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 834.587709,
+ "sequence": 623,
+ "source_event_sequence": 5654,
+ "source_session_id": 890,
+ "source_time_seconds": 834.587709,
+ "source_user_id": 174,
+ "trace_session_id": "ts-00890-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 834.587709,
+ "sequence": 624,
+ "source_event_sequence": 5654,
+ "source_session_id": 627,
+ "source_time_seconds": 834.587709,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g13"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 836.79774,
+ "sequence": 625,
+ "source_event_sequence": 5556,
+ "source_session_id": 877,
+ "source_time_seconds": 836.79774,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 839.259312,
+ "sequence": 626,
+ "source_event_sequence": 6595,
+ "source_session_id": 1038,
+ "source_time_seconds": 839.259312,
+ "source_user_id": 455,
+ "trace_session_id": "ts-01038-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 839.850046,
+ "sequence": 627,
+ "source_event_sequence": 3955,
+ "source_session_id": 627,
+ "source_time_seconds": 839.850046,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g13"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 840.488145,
+ "sequence": 628,
+ "source_event_sequence": 6208,
+ "source_session_id": 977,
+ "source_time_seconds": 840.488145,
+ "source_user_id": 207,
+ "trace_session_id": "ts-00977-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 840.488145,
+ "sequence": 629,
+ "source_event_sequence": 6208,
+ "source_session_id": 914,
+ "source_time_seconds": 840.488145,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g05"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 840.506953,
+ "sequence": 630,
+ "source_event_sequence": 6846,
+ "source_session_id": 1075,
+ "source_time_seconds": 840.506953,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01075-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 841.147849,
+ "sequence": 631,
+ "source_event_sequence": 4345,
+ "source_session_id": 914,
+ "source_time_seconds": 841.147849,
+ "source_user_id": 241,
+ "trace_session_id": "ts-00914-g05"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 841.240755,
+ "sequence": 632,
+ "source_event_sequence": 6854,
+ "source_session_id": 1077,
+ "source_time_seconds": 841.240755,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 841.315753,
+ "sequence": 633,
+ "source_event_sequence": 6532,
+ "source_session_id": 627,
+ "source_time_seconds": 841.315753,
+ "source_user_id": 492,
+ "trace_session_id": "ts-00627-g13"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 841.670216,
+ "sequence": 634,
+ "source_event_sequence": 5056,
+ "source_session_id": 804,
+ "source_time_seconds": 841.670216,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 841.900849,
+ "sequence": 635,
+ "source_event_sequence": 5883,
+ "source_session_id": 927,
+ "source_time_seconds": 841.900849,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 842.052848,
+ "sequence": 636,
+ "source_event_sequence": 5884,
+ "source_session_id": 927,
+ "source_time_seconds": 842.052848,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 843.20129,
+ "sequence": 637,
+ "source_event_sequence": 6899,
+ "source_session_id": 1078,
+ "source_time_seconds": 843.20129,
+ "source_user_id": 419,
+ "trace_session_id": "ts-01078-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 845.999839,
+ "sequence": 638,
+ "source_event_sequence": 6869,
+ "source_session_id": 1078,
+ "source_time_seconds": 845.999839,
+ "source_user_id": 419,
+ "trace_session_id": "ts-01078-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 846.470431,
+ "sequence": 639,
+ "source_event_sequence": 6805,
+ "source_session_id": 1015,
+ "source_time_seconds": 846.470431,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 846.641358,
+ "sequence": 640,
+ "source_event_sequence": 5057,
+ "source_session_id": 804,
+ "source_time_seconds": 846.641358,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 847.959863,
+ "sequence": 641,
+ "source_event_sequence": 6937,
+ "source_session_id": 1015,
+ "source_time_seconds": 847.959863,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 851.106484,
+ "sequence": 642,
+ "source_event_sequence": 6229,
+ "source_session_id": 982,
+ "source_time_seconds": 851.106484,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 851.663153,
+ "sequence": 643,
+ "source_event_sequence": 6448,
+ "source_session_id": 1016,
+ "source_time_seconds": 851.663153,
+ "source_user_id": 40,
+ "trace_session_id": "ts-01016-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 852.284158,
+ "sequence": 644,
+ "source_event_sequence": 6561,
+ "source_session_id": 1015,
+ "source_time_seconds": 852.284158,
+ "source_user_id": 134,
+ "trace_session_id": "ts-01015-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 853.556728,
+ "sequence": 645,
+ "source_event_sequence": 6975,
+ "source_session_id": 1094,
+ "source_time_seconds": 853.556728,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 853.745237,
+ "sequence": 646,
+ "source_event_sequence": 6870,
+ "source_session_id": 1078,
+ "source_time_seconds": 853.745237,
+ "source_user_id": 419,
+ "trace_session_id": "ts-01078-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 853.756658,
+ "sequence": 647,
+ "source_event_sequence": 6964,
+ "source_session_id": 846,
+ "source_time_seconds": 853.756658,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 853.871896,
+ "sequence": 648,
+ "source_event_sequence": 4798,
+ "source_session_id": 768,
+ "source_time_seconds": 853.871896,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 854.603445,
+ "sequence": 649,
+ "source_event_sequence": 5885,
+ "source_session_id": 927,
+ "source_time_seconds": 854.603445,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 857.194762,
+ "sequence": 650,
+ "source_event_sequence": 6230,
+ "source_session_id": 982,
+ "source_time_seconds": 857.194762,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 858.269861,
+ "sequence": 651,
+ "source_event_sequence": 6449,
+ "source_session_id": 1016,
+ "source_time_seconds": 858.269861,
+ "source_user_id": 40,
+ "trace_session_id": "ts-01016-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 859.117473,
+ "sequence": 652,
+ "source_event_sequence": 5058,
+ "source_session_id": 804,
+ "source_time_seconds": 859.117473,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 859.978809,
+ "sequence": 653,
+ "source_event_sequence": 6855,
+ "source_session_id": 1077,
+ "source_time_seconds": 859.978809,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 859.980663,
+ "sequence": 654,
+ "source_event_sequence": 7065,
+ "source_session_id": 846,
+ "source_time_seconds": 859.980663,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 860.277704,
+ "sequence": 655,
+ "source_event_sequence": 6295,
+ "source_session_id": 846,
+ "source_time_seconds": 860.277704,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 860.330942,
+ "sequence": 656,
+ "source_event_sequence": 7069,
+ "source_session_id": 846,
+ "source_time_seconds": 860.330942,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 860.413043,
+ "sequence": 657,
+ "source_event_sequence": 6952,
+ "source_session_id": 1094,
+ "source_time_seconds": 860.413043,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 860.481205,
+ "sequence": 658,
+ "source_event_sequence": 6953,
+ "source_session_id": 1094,
+ "source_time_seconds": 860.481205,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 860.611117,
+ "sequence": 659,
+ "source_event_sequence": 6450,
+ "source_session_id": 1016,
+ "source_time_seconds": 860.611117,
+ "source_user_id": 40,
+ "trace_session_id": "ts-01016-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 860.611117,
+ "sequence": 660,
+ "source_event_sequence": 6450,
+ "source_session_id": 1116,
+ "source_time_seconds": 860.611117,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01116-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 861.616314,
+ "sequence": 661,
+ "source_event_sequence": 6510,
+ "source_session_id": 846,
+ "source_time_seconds": 861.616314,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 862.089465,
+ "sequence": 662,
+ "source_event_sequence": 4799,
+ "source_session_id": 768,
+ "source_time_seconds": 862.089465,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 862.739112,
+ "sequence": 663,
+ "source_event_sequence": 6856,
+ "source_session_id": 1077,
+ "source_time_seconds": 862.739112,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 862.787985,
+ "sequence": 664,
+ "source_event_sequence": 5988,
+ "source_session_id": 944,
+ "source_time_seconds": 862.787985,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 863.056306,
+ "sequence": 665,
+ "source_event_sequence": 7106,
+ "source_session_id": 846,
+ "source_time_seconds": 863.056306,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g05"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 863.166372,
+ "sequence": 666,
+ "source_event_sequence": 6559,
+ "source_session_id": 846,
+ "source_time_seconds": 863.166372,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g05"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 863.43045,
+ "sequence": 667,
+ "source_event_sequence": 7115,
+ "source_session_id": 846,
+ "source_time_seconds": 863.43045,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g06"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 863.523594,
+ "sequence": 668,
+ "source_event_sequence": 2440,
+ "source_session_id": 846,
+ "source_time_seconds": 863.523594,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g06"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 863.94134,
+ "sequence": 669,
+ "source_event_sequence": 7129,
+ "source_session_id": 846,
+ "source_time_seconds": 863.94134,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 863.993869,
+ "sequence": 670,
+ "source_event_sequence": 6844,
+ "source_session_id": 1075,
+ "source_time_seconds": 863.993869,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01075-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 864.295673,
+ "sequence": 671,
+ "source_event_sequence": 5700,
+ "source_session_id": 846,
+ "source_time_seconds": 864.295673,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 865.132629,
+ "sequence": 672,
+ "source_event_sequence": 4407,
+ "source_session_id": 705,
+ "source_time_seconds": 865.132629,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 865.336418,
+ "sequence": 673,
+ "source_event_sequence": 5557,
+ "source_session_id": 877,
+ "source_time_seconds": 865.336418,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 865.979565,
+ "sequence": 674,
+ "source_event_sequence": 6845,
+ "source_session_id": 1075,
+ "source_time_seconds": 865.979565,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01075-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 865.979565,
+ "sequence": 675,
+ "source_event_sequence": 6845,
+ "source_session_id": 846,
+ "source_time_seconds": 865.979565,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g08"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 866.697155,
+ "sequence": 676,
+ "source_event_sequence": 6675,
+ "source_session_id": 846,
+ "source_time_seconds": 866.697155,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g08"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 867.884854,
+ "sequence": 677,
+ "source_event_sequence": 7143,
+ "source_session_id": 846,
+ "source_time_seconds": 867.884854,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g09"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 868.130167,
+ "sequence": 678,
+ "source_event_sequence": 5329,
+ "source_session_id": 846,
+ "source_time_seconds": 868.130167,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g09"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 868.197622,
+ "sequence": 679,
+ "source_event_sequence": 5683,
+ "source_session_id": 896,
+ "source_time_seconds": 868.197622,
+ "source_user_id": 205,
+ "trace_session_id": "ts-00896-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 868.197622,
+ "sequence": 680,
+ "source_event_sequence": 5683,
+ "source_session_id": 1099,
+ "source_time_seconds": 868.197622,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 868.211374,
+ "sequence": 681,
+ "source_event_sequence": 6408,
+ "source_session_id": 1099,
+ "source_time_seconds": 868.211374,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 869.767674,
+ "sequence": 682,
+ "source_event_sequence": 5558,
+ "source_session_id": 877,
+ "source_time_seconds": 869.767674,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 869.784957,
+ "sequence": 683,
+ "source_event_sequence": 4800,
+ "source_session_id": 768,
+ "source_time_seconds": 869.784957,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 869.992037,
+ "sequence": 684,
+ "source_event_sequence": 6231,
+ "source_session_id": 982,
+ "source_time_seconds": 869.992037,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 870.190946,
+ "sequence": 685,
+ "source_event_sequence": 5886,
+ "source_session_id": 927,
+ "source_time_seconds": 870.190946,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 872.688615,
+ "sequence": 686,
+ "source_event_sequence": 6857,
+ "source_session_id": 1077,
+ "source_time_seconds": 872.688615,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 873.291249,
+ "sequence": 687,
+ "source_event_sequence": 5887,
+ "source_session_id": 927,
+ "source_time_seconds": 873.291249,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 874.615316,
+ "sequence": 688,
+ "source_event_sequence": 6929,
+ "source_session_id": 846,
+ "source_time_seconds": 874.615316,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g09"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 874.818184,
+ "sequence": 689,
+ "source_event_sequence": 7081,
+ "source_session_id": 1116,
+ "source_time_seconds": 874.818184,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01116-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 875.347779,
+ "sequence": 690,
+ "source_event_sequence": 5559,
+ "source_session_id": 877,
+ "source_time_seconds": 875.347779,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 875.746973,
+ "sequence": 691,
+ "source_event_sequence": 6858,
+ "source_session_id": 1077,
+ "source_time_seconds": 875.746973,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 879.685792,
+ "sequence": 692,
+ "source_event_sequence": 6954,
+ "source_session_id": 1094,
+ "source_time_seconds": 879.685792,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 880.148298,
+ "sequence": 693,
+ "source_event_sequence": 6871,
+ "source_session_id": 1078,
+ "source_time_seconds": 880.148298,
+ "source_user_id": 419,
+ "trace_session_id": "ts-01078-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 880.148298,
+ "sequence": 694,
+ "source_event_sequence": 6871,
+ "source_session_id": 846,
+ "source_time_seconds": 880.148298,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g10"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 881.736992,
+ "sequence": 695,
+ "source_event_sequence": 7182,
+ "source_session_id": 846,
+ "source_time_seconds": 881.736992,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g10"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 882.203351,
+ "sequence": 696,
+ "source_event_sequence": 6859,
+ "source_session_id": 1077,
+ "source_time_seconds": 882.203351,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 882.237036,
+ "sequence": 697,
+ "source_event_sequence": 7194,
+ "source_session_id": 846,
+ "source_time_seconds": 882.237036,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g11"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 882.959544,
+ "sequence": 698,
+ "source_event_sequence": 5445,
+ "source_session_id": 846,
+ "source_time_seconds": 882.959544,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g11"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 883.17189,
+ "sequence": 699,
+ "source_event_sequence": 7201,
+ "source_session_id": 846,
+ "source_time_seconds": 883.17189,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g12"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 883.315205,
+ "sequence": 700,
+ "source_event_sequence": 5560,
+ "source_session_id": 877,
+ "source_time_seconds": 883.315205,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 883.321754,
+ "sequence": 701,
+ "source_event_sequence": 5001,
+ "source_session_id": 846,
+ "source_time_seconds": 883.321754,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g12"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 883.388078,
+ "sequence": 702,
+ "source_event_sequence": 6860,
+ "source_session_id": 1077,
+ "source_time_seconds": 883.388078,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 883.405857,
+ "sequence": 703,
+ "source_event_sequence": 5059,
+ "source_session_id": 804,
+ "source_time_seconds": 883.405857,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 885.897448,
+ "sequence": 704,
+ "source_event_sequence": 7218,
+ "source_session_id": 846,
+ "source_time_seconds": 885.897448,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g13"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 886.125776,
+ "sequence": 705,
+ "source_event_sequence": 6025,
+ "source_session_id": 846,
+ "source_time_seconds": 886.125776,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g13"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 887.283673,
+ "sequence": 706,
+ "source_event_sequence": 5989,
+ "source_session_id": 944,
+ "source_time_seconds": 887.283673,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 888.513809,
+ "sequence": 707,
+ "source_event_sequence": 6955,
+ "source_session_id": 1094,
+ "source_time_seconds": 888.513809,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 893.605748,
+ "sequence": 708,
+ "source_event_sequence": 5561,
+ "source_session_id": 877,
+ "source_time_seconds": 893.605748,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 895.880573,
+ "sequence": 709,
+ "source_event_sequence": 5990,
+ "source_session_id": 944,
+ "source_time_seconds": 895.880573,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 897.199014,
+ "sequence": 710,
+ "source_event_sequence": 3528,
+ "source_session_id": 559,
+ "source_time_seconds": 897.199014,
+ "source_user_id": 22,
+ "trace_session_id": "ts-00559-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 897.199014,
+ "sequence": 711,
+ "source_event_sequence": 3528,
+ "source_session_id": 846,
+ "source_time_seconds": 897.199014,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g14"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 897.348958,
+ "sequence": 712,
+ "source_event_sequence": 6861,
+ "source_session_id": 1077,
+ "source_time_seconds": 897.348958,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 899.300957,
+ "sequence": 713,
+ "source_event_sequence": 5888,
+ "source_session_id": 927,
+ "source_time_seconds": 899.300957,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 899.470337,
+ "sequence": 714,
+ "source_event_sequence": 6862,
+ "source_session_id": 1077,
+ "source_time_seconds": 899.470337,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 900.078121,
+ "sequence": 715,
+ "source_event_sequence": 5562,
+ "source_session_id": 877,
+ "source_time_seconds": 900.078121,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 900.144255,
+ "sequence": 716,
+ "source_event_sequence": 6936,
+ "source_session_id": 846,
+ "source_time_seconds": 900.144255,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g14"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 900.263098,
+ "sequence": 717,
+ "source_event_sequence": 7282,
+ "source_session_id": 846,
+ "source_time_seconds": 900.263098,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g15"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 900.312278,
+ "sequence": 718,
+ "source_event_sequence": 6323,
+ "source_session_id": 846,
+ "source_time_seconds": 900.312278,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g15"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 900.688697,
+ "sequence": 719,
+ "source_event_sequence": 6956,
+ "source_session_id": 1094,
+ "source_time_seconds": 900.688697,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 903.483447,
+ "sequence": 720,
+ "source_event_sequence": 5889,
+ "source_session_id": 927,
+ "source_time_seconds": 903.483447,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 904.497098,
+ "sequence": 721,
+ "source_event_sequence": 6863,
+ "source_session_id": 1077,
+ "source_time_seconds": 904.497098,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 907.192533,
+ "sequence": 722,
+ "source_event_sequence": 6596,
+ "source_session_id": 1038,
+ "source_time_seconds": 907.192533,
+ "source_user_id": 455,
+ "trace_session_id": "ts-01038-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 907.192533,
+ "sequence": 723,
+ "source_event_sequence": 6596,
+ "source_session_id": 1151,
+ "source_time_seconds": 907.192533,
+ "source_user_id": 420,
+ "trace_session_id": "ts-01151-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 908.591286,
+ "sequence": 724,
+ "source_event_sequence": 5060,
+ "source_session_id": 804,
+ "source_time_seconds": 908.591286,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 910.706068,
+ "sequence": 725,
+ "source_event_sequence": 4408,
+ "source_session_id": 705,
+ "source_time_seconds": 910.706068,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 911.38527,
+ "sequence": 726,
+ "source_event_sequence": 7082,
+ "source_session_id": 1116,
+ "source_time_seconds": 911.38527,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01116-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 911.502401,
+ "sequence": 727,
+ "source_event_sequence": 5991,
+ "source_session_id": 944,
+ "source_time_seconds": 911.502401,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 911.860124,
+ "sequence": 728,
+ "source_event_sequence": 6232,
+ "source_session_id": 982,
+ "source_time_seconds": 911.860124,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 912.142668,
+ "sequence": 729,
+ "source_event_sequence": 6233,
+ "source_session_id": 982,
+ "source_time_seconds": 912.142668,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 912.925127,
+ "sequence": 730,
+ "source_event_sequence": 5563,
+ "source_session_id": 877,
+ "source_time_seconds": 912.925127,
+ "source_user_id": 180,
+ "trace_session_id": "ts-00877-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 912.925127,
+ "sequence": 731,
+ "source_event_sequence": 5563,
+ "source_session_id": 846,
+ "source_time_seconds": 912.925127,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g16"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 912.983204,
+ "sequence": 732,
+ "source_event_sequence": 3958,
+ "source_session_id": 846,
+ "source_time_seconds": 912.983204,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g16"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 913.131571,
+ "sequence": 733,
+ "source_event_sequence": 7309,
+ "source_session_id": 846,
+ "source_time_seconds": 913.131571,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g17"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 913.385895,
+ "sequence": 734,
+ "source_event_sequence": 5992,
+ "source_session_id": 944,
+ "source_time_seconds": 913.385895,
+ "source_user_id": 365,
+ "trace_session_id": "ts-00944-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 913.41238,
+ "sequence": 735,
+ "source_event_sequence": 7314,
+ "source_session_id": 1099,
+ "source_time_seconds": 913.41238,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 913.476179,
+ "sequence": 736,
+ "source_event_sequence": 6957,
+ "source_session_id": 1094,
+ "source_time_seconds": 913.476179,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 914.073126,
+ "sequence": 737,
+ "source_event_sequence": 5061,
+ "source_session_id": 804,
+ "source_time_seconds": 914.073126,
+ "source_user_id": 494,
+ "trace_session_id": "ts-00804-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 914.362378,
+ "sequence": 738,
+ "source_event_sequence": 7327,
+ "source_session_id": 783,
+ "source_time_seconds": 914.362378,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 914.564537,
+ "sequence": 739,
+ "source_event_sequence": 6703,
+ "source_session_id": 783,
+ "source_time_seconds": 914.564537,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 916.031419,
+ "sequence": 740,
+ "source_event_sequence": 7293,
+ "source_session_id": 1151,
+ "source_time_seconds": 916.031419,
+ "source_user_id": 420,
+ "trace_session_id": "ts-01151-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 918.853407,
+ "sequence": 741,
+ "source_event_sequence": 4801,
+ "source_session_id": 768,
+ "source_time_seconds": 918.853407,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 920.18921,
+ "sequence": 742,
+ "source_event_sequence": 5331,
+ "source_session_id": 846,
+ "source_time_seconds": 920.18921,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g17"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 924.185426,
+ "sequence": 743,
+ "source_event_sequence": 7083,
+ "source_session_id": 1116,
+ "source_time_seconds": 924.185426,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01116-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 929.021989,
+ "sequence": 744,
+ "source_event_sequence": 4802,
+ "source_session_id": 768,
+ "source_time_seconds": 929.021989,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 933.386125,
+ "sequence": 745,
+ "source_event_sequence": 5890,
+ "source_session_id": 927,
+ "source_time_seconds": 933.386125,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 936.728371,
+ "sequence": 746,
+ "source_event_sequence": 5891,
+ "source_session_id": 927,
+ "source_time_seconds": 936.728371,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 937.552942,
+ "sequence": 747,
+ "source_event_sequence": 6864,
+ "source_session_id": 1077,
+ "source_time_seconds": 937.552942,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 937.56496,
+ "sequence": 748,
+ "source_event_sequence": 4803,
+ "source_session_id": 768,
+ "source_time_seconds": 937.56496,
+ "source_user_id": 54,
+ "trace_session_id": "ts-00768-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 937.56496,
+ "sequence": 749,
+ "source_event_sequence": 4803,
+ "source_session_id": 1159,
+ "source_time_seconds": 937.56496,
+ "source_user_id": 34,
+ "trace_session_id": "ts-01159-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 938.921552,
+ "sequence": 750,
+ "source_event_sequence": 4409,
+ "source_session_id": 705,
+ "source_time_seconds": 938.921552,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 939.107911,
+ "sequence": 751,
+ "source_event_sequence": 7335,
+ "source_session_id": 1159,
+ "source_time_seconds": 939.107911,
+ "source_user_id": 34,
+ "trace_session_id": "ts-01159-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 940.7166,
+ "sequence": 752,
+ "source_event_sequence": 6234,
+ "source_session_id": 982,
+ "source_time_seconds": 940.7166,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 941.065774,
+ "sequence": 753,
+ "source_event_sequence": 4218,
+ "source_session_id": 1099,
+ "source_time_seconds": 941.065774,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 941.069209,
+ "sequence": 754,
+ "source_event_sequence": 7415,
+ "source_session_id": 1099,
+ "source_time_seconds": 941.069209,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 942.208311,
+ "sequence": 755,
+ "source_event_sequence": 7399,
+ "source_session_id": 1099,
+ "source_time_seconds": 942.208311,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 943.797195,
+ "sequence": 756,
+ "source_event_sequence": 6865,
+ "source_session_id": 1077,
+ "source_time_seconds": 943.797195,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 944.012298,
+ "sequence": 757,
+ "source_event_sequence": 6958,
+ "source_session_id": 1094,
+ "source_time_seconds": 944.012298,
+ "source_user_id": 279,
+ "trace_session_id": "ts-01094-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 944.012298,
+ "sequence": 758,
+ "source_event_sequence": 6958,
+ "source_session_id": 1099,
+ "source_time_seconds": 944.012298,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g04"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 944.895249,
+ "sequence": 759,
+ "source_event_sequence": 7336,
+ "source_session_id": 1159,
+ "source_time_seconds": 944.895249,
+ "source_user_id": 34,
+ "trace_session_id": "ts-01159-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 944.895249,
+ "sequence": 760,
+ "source_event_sequence": 7336,
+ "source_session_id": 783,
+ "source_time_seconds": 944.895249,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 949.937713,
+ "sequence": 761,
+ "source_event_sequence": 4410,
+ "source_session_id": 705,
+ "source_time_seconds": 949.937713,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 949.988495,
+ "sequence": 762,
+ "source_event_sequence": 7294,
+ "source_session_id": 1151,
+ "source_time_seconds": 949.988495,
+ "source_user_id": 420,
+ "trace_session_id": "ts-01151-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 951.716175,
+ "sequence": 763,
+ "source_event_sequence": 5892,
+ "source_session_id": 927,
+ "source_time_seconds": 951.716175,
+ "source_user_id": 451,
+ "trace_session_id": "ts-00927-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 951.716175,
+ "sequence": 764,
+ "source_event_sequence": 5892,
+ "source_session_id": 1152,
+ "source_time_seconds": 951.716175,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 954.813196,
+ "sequence": 765,
+ "source_event_sequence": 6235,
+ "source_session_id": 982,
+ "source_time_seconds": 954.813196,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 955.488494,
+ "sequence": 766,
+ "source_event_sequence": 6236,
+ "source_session_id": 982,
+ "source_time_seconds": 955.488494,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 956.067502,
+ "sequence": 767,
+ "source_event_sequence": 5332,
+ "source_session_id": 846,
+ "source_time_seconds": 956.067502,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g17"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 957.283727,
+ "sequence": 768,
+ "source_event_sequence": 7084,
+ "source_session_id": 1116,
+ "source_time_seconds": 957.283727,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01116-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 960.936668,
+ "sequence": 769,
+ "source_event_sequence": 4411,
+ "source_session_id": 705,
+ "source_time_seconds": 960.936668,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 962.309468,
+ "sequence": 770,
+ "source_event_sequence": 7295,
+ "source_session_id": 1151,
+ "source_time_seconds": 962.309468,
+ "source_user_id": 420,
+ "trace_session_id": "ts-01151-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 962.309468,
+ "sequence": 771,
+ "source_event_sequence": 7295,
+ "source_session_id": 1160,
+ "source_time_seconds": 962.309468,
+ "source_user_id": 442,
+ "trace_session_id": "ts-01160-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 962.476984,
+ "sequence": 772,
+ "source_event_sequence": 6970,
+ "source_session_id": 1099,
+ "source_time_seconds": 962.476984,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g04"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 962.845237,
+ "sequence": 773,
+ "source_event_sequence": 6866,
+ "source_session_id": 1077,
+ "source_time_seconds": 962.845237,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 963.405785,
+ "sequence": 774,
+ "source_event_sequence": 5333,
+ "source_session_id": 846,
+ "source_time_seconds": 963.405785,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g17"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 963.541098,
+ "sequence": 775,
+ "source_event_sequence": 4908,
+ "source_session_id": 783,
+ "source_time_seconds": 963.541098,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 964.614705,
+ "sequence": 776,
+ "source_event_sequence": 6237,
+ "source_session_id": 982,
+ "source_time_seconds": 964.614705,
+ "source_user_id": 39,
+ "trace_session_id": "ts-00982-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 964.614705,
+ "sequence": 777,
+ "source_event_sequence": 6237,
+ "source_session_id": 1189,
+ "source_time_seconds": 964.614705,
+ "source_user_id": 53,
+ "trace_session_id": "ts-01189-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 965.365511,
+ "sequence": 778,
+ "source_event_sequence": 7339,
+ "source_session_id": 1160,
+ "source_time_seconds": 965.365511,
+ "source_user_id": 442,
+ "trace_session_id": "ts-01160-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 965.365511,
+ "sequence": 779,
+ "source_event_sequence": 7339,
+ "source_session_id": 1190,
+ "source_time_seconds": 965.365511,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 965.87947,
+ "sequence": 780,
+ "source_event_sequence": 4412,
+ "source_session_id": 705,
+ "source_time_seconds": 965.87947,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 965.912076,
+ "sequence": 781,
+ "source_event_sequence": 4413,
+ "source_session_id": 705,
+ "source_time_seconds": 965.912076,
+ "source_user_id": 273,
+ "trace_session_id": "ts-00705-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 965.912076,
+ "sequence": 782,
+ "source_event_sequence": 4413,
+ "source_session_id": 905,
+ "source_time_seconds": 965.912076,
+ "source_user_id": 177,
+ "trace_session_id": "ts-00905-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 969.74603,
+ "sequence": 783,
+ "source_event_sequence": 6971,
+ "source_session_id": 1099,
+ "source_time_seconds": 969.74603,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01099-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 969.74603,
+ "sequence": 784,
+ "source_event_sequence": 6971,
+ "source_session_id": 1150,
+ "source_time_seconds": 969.74603,
+ "source_user_id": 356,
+ "trace_session_id": "ts-01150-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 969.885314,
+ "sequence": 785,
+ "source_event_sequence": 4909,
+ "source_session_id": 783,
+ "source_time_seconds": 969.885314,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 973.547782,
+ "sequence": 786,
+ "source_event_sequence": 6867,
+ "source_session_id": 1077,
+ "source_time_seconds": 973.547782,
+ "source_user_id": 43,
+ "trace_session_id": "ts-01077-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 973.547782,
+ "sequence": 787,
+ "source_event_sequence": 6867,
+ "source_session_id": 832,
+ "source_time_seconds": 973.547782,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 974.148334,
+ "sequence": 788,
+ "source_event_sequence": 5767,
+ "source_session_id": 905,
+ "source_time_seconds": 974.148334,
+ "source_user_id": 177,
+ "trace_session_id": "ts-00905-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 974.320126,
+ "sequence": 789,
+ "source_event_sequence": 5223,
+ "source_session_id": 832,
+ "source_time_seconds": 974.320126,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 975.095698,
+ "sequence": 790,
+ "source_event_sequence": 7287,
+ "source_session_id": 1150,
+ "source_time_seconds": 975.095698,
+ "source_user_id": 356,
+ "trace_session_id": "ts-01150-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 975.459609,
+ "sequence": 791,
+ "source_event_sequence": 7299,
+ "source_session_id": 1152,
+ "source_time_seconds": 975.459609,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 975.490952,
+ "sequence": 792,
+ "source_event_sequence": 5334,
+ "source_session_id": 846,
+ "source_time_seconds": 975.490952,
+ "source_user_id": 490,
+ "trace_session_id": "ts-00846-g17"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 983.744229,
+ "sequence": 793,
+ "source_event_sequence": 5768,
+ "source_session_id": 905,
+ "source_time_seconds": 983.744229,
+ "source_user_id": 177,
+ "trace_session_id": "ts-00905-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 986.681769,
+ "sequence": 794,
+ "source_event_sequence": 5769,
+ "source_session_id": 905,
+ "source_time_seconds": 986.681769,
+ "source_user_id": 177,
+ "trace_session_id": "ts-00905-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 986.681769,
+ "sequence": 795,
+ "source_event_sequence": 5769,
+ "source_session_id": 1197,
+ "source_time_seconds": 986.681769,
+ "source_user_id": 407,
+ "trace_session_id": "ts-01197-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 988.563342,
+ "sequence": 796,
+ "source_event_sequence": 7490,
+ "source_session_id": 1190,
+ "source_time_seconds": 988.563342,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 990.803221,
+ "sequence": 797,
+ "source_event_sequence": 5224,
+ "source_session_id": 832,
+ "source_time_seconds": 990.803221,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 996.954911,
+ "sequence": 798,
+ "source_event_sequence": 4910,
+ "source_session_id": 783,
+ "source_time_seconds": 996.954911,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 999.008937,
+ "sequence": 799,
+ "source_event_sequence": 7539,
+ "source_session_id": 1197,
+ "source_time_seconds": 999.008937,
+ "source_user_id": 407,
+ "trace_session_id": "ts-01197-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 999.008937,
+ "sequence": 800,
+ "source_event_sequence": 7539,
+ "source_session_id": 1206,
+ "source_time_seconds": 999.008937,
+ "source_user_id": 218,
+ "trace_session_id": "ts-01206-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1000.653173,
+ "sequence": 801,
+ "source_event_sequence": 7578,
+ "source_session_id": 1206,
+ "source_time_seconds": 1000.653173,
+ "source_user_id": 218,
+ "trace_session_id": "ts-01206-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1004.398955,
+ "sequence": 802,
+ "source_event_sequence": 4911,
+ "source_session_id": 783,
+ "source_time_seconds": 1004.398955,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1004.86422,
+ "sequence": 803,
+ "source_event_sequence": 7486,
+ "source_session_id": 1189,
+ "source_time_seconds": 1004.86422,
+ "source_user_id": 53,
+ "trace_session_id": "ts-01189-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1006.185782,
+ "sequence": 804,
+ "source_event_sequence": 7579,
+ "source_session_id": 1206,
+ "source_time_seconds": 1006.185782,
+ "source_user_id": 218,
+ "trace_session_id": "ts-01206-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1006.185782,
+ "sequence": 805,
+ "source_event_sequence": 7579,
+ "source_session_id": 1213,
+ "source_time_seconds": 1006.185782,
+ "source_user_id": 336,
+ "trace_session_id": "ts-01213-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1006.761378,
+ "sequence": 806,
+ "source_event_sequence": 4912,
+ "source_session_id": 783,
+ "source_time_seconds": 1006.761378,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1007.056766,
+ "sequence": 807,
+ "source_event_sequence": 7288,
+ "source_session_id": 1150,
+ "source_time_seconds": 1007.056766,
+ "source_user_id": 356,
+ "trace_session_id": "ts-01150-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1007.439045,
+ "sequence": 808,
+ "source_event_sequence": 7289,
+ "source_session_id": 1150,
+ "source_time_seconds": 1007.439045,
+ "source_user_id": 356,
+ "trace_session_id": "ts-01150-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1008.877015,
+ "sequence": 809,
+ "source_event_sequence": 7290,
+ "source_session_id": 1150,
+ "source_time_seconds": 1008.877015,
+ "source_user_id": 356,
+ "trace_session_id": "ts-01150-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1008.922025,
+ "sequence": 810,
+ "source_event_sequence": 7491,
+ "source_session_id": 1190,
+ "source_time_seconds": 1008.922025,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1016.147697,
+ "sequence": 811,
+ "source_event_sequence": 7487,
+ "source_session_id": 1189,
+ "source_time_seconds": 1016.147697,
+ "source_user_id": 53,
+ "trace_session_id": "ts-01189-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1016.550868,
+ "sequence": 812,
+ "source_event_sequence": 7300,
+ "source_session_id": 1152,
+ "source_time_seconds": 1016.550868,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1020.2982,
+ "sequence": 813,
+ "source_event_sequence": 7607,
+ "source_session_id": 1213,
+ "source_time_seconds": 1020.2982,
+ "source_user_id": 336,
+ "trace_session_id": "ts-01213-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1020.2982,
+ "sequence": 814,
+ "source_event_sequence": 7607,
+ "source_session_id": 1216,
+ "source_time_seconds": 1020.2982,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1021.397596,
+ "sequence": 815,
+ "source_event_sequence": 7571,
+ "source_session_id": 832,
+ "source_time_seconds": 1021.397596,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1021.585148,
+ "sequence": 816,
+ "source_event_sequence": 7301,
+ "source_session_id": 1152,
+ "source_time_seconds": 1021.585148,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1021.743881,
+ "sequence": 817,
+ "source_event_sequence": 7671,
+ "source_session_id": 1215,
+ "source_time_seconds": 1021.743881,
+ "source_user_id": 36,
+ "trace_session_id": "ts-01215-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1022.326889,
+ "sequence": 818,
+ "source_event_sequence": 7156,
+ "source_session_id": 1150,
+ "source_time_seconds": 1022.326889,
+ "source_user_id": 356,
+ "trace_session_id": "ts-01150-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1025.648971,
+ "sequence": 819,
+ "source_event_sequence": 7492,
+ "source_session_id": 1190,
+ "source_time_seconds": 1025.648971,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1027.291489,
+ "sequence": 820,
+ "source_event_sequence": 7488,
+ "source_session_id": 1189,
+ "source_time_seconds": 1027.291489,
+ "source_user_id": 53,
+ "trace_session_id": "ts-01189-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1027.291489,
+ "sequence": 821,
+ "source_event_sequence": 7488,
+ "source_session_id": 832,
+ "source_time_seconds": 1027.291489,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1028.702571,
+ "sequence": 822,
+ "source_event_sequence": 4913,
+ "source_session_id": 783,
+ "source_time_seconds": 1028.702571,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1032.798518,
+ "sequence": 823,
+ "source_event_sequence": 7612,
+ "source_session_id": 1215,
+ "source_time_seconds": 1032.798518,
+ "source_user_id": 36,
+ "trace_session_id": "ts-01215-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1032.910469,
+ "sequence": 824,
+ "source_event_sequence": 7302,
+ "source_session_id": 1152,
+ "source_time_seconds": 1032.910469,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1037.683562,
+ "sequence": 825,
+ "source_event_sequence": 7626,
+ "source_session_id": 1216,
+ "source_time_seconds": 1037.683562,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1037.975872,
+ "sequence": 826,
+ "source_event_sequence": 5225,
+ "source_session_id": 832,
+ "source_time_seconds": 1037.975872,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1038.947811,
+ "sequence": 827,
+ "source_event_sequence": 7303,
+ "source_session_id": 1152,
+ "source_time_seconds": 1038.947811,
+ "source_user_id": 463,
+ "trace_session_id": "ts-01152-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1038.947811,
+ "sequence": 828,
+ "source_event_sequence": 7303,
+ "source_session_id": 1229,
+ "source_time_seconds": 1038.947811,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1042.98811,
+ "sequence": 829,
+ "source_event_sequence": 7214,
+ "source_session_id": 832,
+ "source_time_seconds": 1042.98811,
+ "source_user_id": 403,
+ "trace_session_id": "ts-00832-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1044.073348,
+ "sequence": 830,
+ "source_event_sequence": 7493,
+ "source_session_id": 1190,
+ "source_time_seconds": 1044.073348,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1048.450552,
+ "sequence": 831,
+ "source_event_sequence": 7627,
+ "source_session_id": 1216,
+ "source_time_seconds": 1048.450552,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1057.627463,
+ "sequence": 832,
+ "source_event_sequence": 7698,
+ "source_session_id": 1229,
+ "source_time_seconds": 1057.627463,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1057.964729,
+ "sequence": 833,
+ "source_event_sequence": 7628,
+ "source_session_id": 1216,
+ "source_time_seconds": 1057.964729,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1060.392168,
+ "sequence": 834,
+ "source_event_sequence": 7629,
+ "source_session_id": 1216,
+ "source_time_seconds": 1060.392168,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1060.841346,
+ "sequence": 835,
+ "source_event_sequence": 7494,
+ "source_session_id": 1190,
+ "source_time_seconds": 1060.841346,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1067.935434,
+ "sequence": 836,
+ "source_event_sequence": 7229,
+ "source_session_id": 1190,
+ "source_time_seconds": 1067.935434,
+ "source_user_id": 332,
+ "trace_session_id": "ts-01190-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1071.182227,
+ "sequence": 837,
+ "source_event_sequence": 7699,
+ "source_session_id": 1229,
+ "source_time_seconds": 1071.182227,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1093.236423,
+ "sequence": 838,
+ "source_event_sequence": 7713,
+ "source_session_id": 783,
+ "source_time_seconds": 1093.236423,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1094.068795,
+ "sequence": 839,
+ "source_event_sequence": 7630,
+ "source_session_id": 1216,
+ "source_time_seconds": 1094.068795,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1110.738261,
+ "sequence": 840,
+ "source_event_sequence": 7631,
+ "source_session_id": 1216,
+ "source_time_seconds": 1110.738261,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1116.774232,
+ "sequence": 841,
+ "source_event_sequence": 7700,
+ "source_session_id": 1229,
+ "source_time_seconds": 1116.774232,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1116.8381,
+ "sequence": 842,
+ "source_event_sequence": 7632,
+ "source_session_id": 1216,
+ "source_time_seconds": 1116.8381,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1120.723949,
+ "sequence": 843,
+ "source_event_sequence": 7633,
+ "source_session_id": 1216,
+ "source_time_seconds": 1120.723949,
+ "source_user_id": 450,
+ "trace_session_id": "ts-01216-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1120.723949,
+ "sequence": 844,
+ "source_event_sequence": 7633,
+ "source_session_id": 783,
+ "source_time_seconds": 1120.723949,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1123.045705,
+ "sequence": 845,
+ "source_event_sequence": 7613,
+ "source_session_id": 1215,
+ "source_time_seconds": 1123.045705,
+ "source_user_id": 36,
+ "trace_session_id": "ts-01215-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1123.153908,
+ "sequence": 846,
+ "source_event_sequence": 6479,
+ "source_session_id": 783,
+ "source_time_seconds": 1123.153908,
+ "source_user_id": 469,
+ "trace_session_id": "ts-00783-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1128.809861,
+ "sequence": 847,
+ "source_event_sequence": 7614,
+ "source_session_id": 1215,
+ "source_time_seconds": 1128.809861,
+ "source_user_id": 36,
+ "trace_session_id": "ts-01215-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1134.283221,
+ "sequence": 848,
+ "source_event_sequence": 7701,
+ "source_session_id": 1229,
+ "source_time_seconds": 1134.283221,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1137.093899,
+ "sequence": 849,
+ "source_event_sequence": 7702,
+ "source_session_id": 1229,
+ "source_time_seconds": 1137.093899,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1142.425016,
+ "sequence": 850,
+ "source_event_sequence": 7703,
+ "source_session_id": 1229,
+ "source_time_seconds": 1142.425016,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1143.104058,
+ "sequence": 851,
+ "source_event_sequence": 7704,
+ "source_session_id": 1229,
+ "source_time_seconds": 1143.104058,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1145.131859,
+ "sequence": 852,
+ "source_event_sequence": 7705,
+ "source_session_id": 1229,
+ "source_time_seconds": 1145.131859,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1150.657759,
+ "sequence": 853,
+ "source_event_sequence": 7615,
+ "source_session_id": 1215,
+ "source_time_seconds": 1150.657759,
+ "source_user_id": 36,
+ "trace_session_id": "ts-01215-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1154.886366,
+ "sequence": 854,
+ "source_event_sequence": 6176,
+ "source_session_id": 1215,
+ "source_time_seconds": 1154.886366,
+ "source_user_id": 36,
+ "trace_session_id": "ts-01215-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1212.24085,
+ "sequence": 855,
+ "source_event_sequence": 7706,
+ "source_session_id": 1229,
+ "source_time_seconds": 1212.24085,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1233.051099,
+ "sequence": 856,
+ "source_event_sequence": 5733,
+ "source_session_id": 1229,
+ "source_time_seconds": 1233.051099,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1233.262662,
+ "sequence": 857,
+ "source_event_sequence": 7718,
+ "source_session_id": 1229,
+ "source_time_seconds": 1233.262662,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1268.347463,
+ "sequence": 858,
+ "source_event_sequence": 7707,
+ "source_session_id": 1229,
+ "source_time_seconds": 1268.347463,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1303.049277,
+ "sequence": 859,
+ "source_event_sequence": 7708,
+ "source_session_id": 1229,
+ "source_time_seconds": 1303.049277,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1311.357034,
+ "sequence": 860,
+ "source_event_sequence": 7709,
+ "source_session_id": 1229,
+ "source_time_seconds": 1311.357034,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1312.091021,
+ "sequence": 861,
+ "source_event_sequence": 7710,
+ "source_session_id": 1229,
+ "source_time_seconds": 1312.091021,
+ "source_user_id": 358,
+ "trace_session_id": "ts-01229-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1312.091021,
+ "sequence": 862,
+ "source_event_sequence": 7710,
+ "source_session_id": 1242,
+ "source_time_seconds": 1312.091021,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1316.229453,
+ "sequence": 863,
+ "source_event_sequence": 7771,
+ "source_session_id": 1242,
+ "source_time_seconds": 1316.229453,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1333.581125,
+ "sequence": 864,
+ "source_event_sequence": 7772,
+ "source_session_id": 1242,
+ "source_time_seconds": 1333.581125,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1347.996123,
+ "sequence": 865,
+ "source_event_sequence": 7773,
+ "source_session_id": 1242,
+ "source_time_seconds": 1347.996123,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1410.776719,
+ "sequence": 866,
+ "source_event_sequence": 7911,
+ "source_session_id": 1259,
+ "source_time_seconds": 1410.776719,
+ "source_user_id": 395,
+ "trace_session_id": "ts-01259-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1410.973865,
+ "sequence": 867,
+ "source_event_sequence": 7885,
+ "source_session_id": 1259,
+ "source_time_seconds": 1410.973865,
+ "source_user_id": 395,
+ "trace_session_id": "ts-01259-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1411.81446,
+ "sequence": 868,
+ "source_event_sequence": 7917,
+ "source_session_id": 1240,
+ "source_time_seconds": 1411.81446,
+ "source_user_id": 248,
+ "trace_session_id": "ts-01240-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1415.484308,
+ "sequence": 869,
+ "source_event_sequence": 7774,
+ "source_session_id": 1242,
+ "source_time_seconds": 1415.484308,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1420.07776,
+ "sequence": 870,
+ "source_event_sequence": 7995,
+ "source_session_id": 1263,
+ "source_time_seconds": 1420.07776,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1422.39701,
+ "sequence": 871,
+ "source_event_sequence": 7900,
+ "source_session_id": 1263,
+ "source_time_seconds": 1422.39701,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1423.743248,
+ "sequence": 872,
+ "source_event_sequence": 7763,
+ "source_session_id": 1240,
+ "source_time_seconds": 1423.743248,
+ "source_user_id": 248,
+ "trace_session_id": "ts-01240-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1428.362395,
+ "sequence": 873,
+ "source_event_sequence": 8097,
+ "source_session_id": 1284,
+ "source_time_seconds": 1428.362395,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1429.618692,
+ "sequence": 874,
+ "source_event_sequence": 7897,
+ "source_session_id": 1263,
+ "source_time_seconds": 1429.618692,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1429.923354,
+ "sequence": 875,
+ "source_event_sequence": 8116,
+ "source_session_id": 1287,
+ "source_time_seconds": 1429.923354,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1430.336425,
+ "sequence": 876,
+ "source_event_sequence": 8067,
+ "source_session_id": 1287,
+ "source_time_seconds": 1430.336425,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1431.183754,
+ "sequence": 877,
+ "source_event_sequence": 7764,
+ "source_session_id": 1240,
+ "source_time_seconds": 1431.183754,
+ "source_user_id": 248,
+ "trace_session_id": "ts-01240-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1431.183754,
+ "sequence": 878,
+ "source_event_sequence": 7764,
+ "source_session_id": 1263,
+ "source_time_seconds": 1431.183754,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1434.773246,
+ "sequence": 879,
+ "source_event_sequence": 8052,
+ "source_session_id": 1284,
+ "source_time_seconds": 1434.773246,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1435.550093,
+ "sequence": 880,
+ "source_event_sequence": 8068,
+ "source_session_id": 1287,
+ "source_time_seconds": 1435.550093,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1438.218241,
+ "sequence": 881,
+ "source_event_sequence": 7775,
+ "source_session_id": 1242,
+ "source_time_seconds": 1438.218241,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1444.625075,
+ "sequence": 882,
+ "source_event_sequence": 8251,
+ "source_session_id": 1301,
+ "source_time_seconds": 1444.625075,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1444.813994,
+ "sequence": 883,
+ "source_event_sequence": 8169,
+ "source_session_id": 1301,
+ "source_time_seconds": 1444.813994,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1445.49814,
+ "sequence": 884,
+ "source_event_sequence": 8069,
+ "source_session_id": 1287,
+ "source_time_seconds": 1445.49814,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1450.188313,
+ "sequence": 885,
+ "source_event_sequence": 7776,
+ "source_session_id": 1242,
+ "source_time_seconds": 1450.188313,
+ "source_user_id": 220,
+ "trace_session_id": "ts-01242-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1450.188313,
+ "sequence": 886,
+ "source_event_sequence": 7776,
+ "source_session_id": 1310,
+ "source_time_seconds": 1450.188313,
+ "source_user_id": 199,
+ "trace_session_id": "ts-01310-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1450.557588,
+ "sequence": 887,
+ "source_event_sequence": 7976,
+ "source_session_id": 1263,
+ "source_time_seconds": 1450.557588,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1450.659113,
+ "sequence": 888,
+ "source_event_sequence": 8288,
+ "source_session_id": 1307,
+ "source_time_seconds": 1450.659113,
+ "source_user_id": 269,
+ "trace_session_id": "ts-01307-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1452.682519,
+ "sequence": 889,
+ "source_event_sequence": 8070,
+ "source_session_id": 1287,
+ "source_time_seconds": 1452.682519,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1453.93958,
+ "sequence": 890,
+ "source_event_sequence": 8164,
+ "source_session_id": 1307,
+ "source_time_seconds": 1453.93958,
+ "source_user_id": 269,
+ "trace_session_id": "ts-01307-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1454.052772,
+ "sequence": 891,
+ "source_event_sequence": 8227,
+ "source_session_id": 1310,
+ "source_time_seconds": 1454.052772,
+ "source_user_id": 199,
+ "trace_session_id": "ts-01310-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1457.299211,
+ "sequence": 892,
+ "source_event_sequence": 8338,
+ "source_session_id": 1307,
+ "source_time_seconds": 1457.299211,
+ "source_user_id": 269,
+ "trace_session_id": "ts-01307-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1460.017274,
+ "sequence": 893,
+ "source_event_sequence": 8170,
+ "source_session_id": 1301,
+ "source_time_seconds": 1460.017274,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1461.533907,
+ "sequence": 894,
+ "source_event_sequence": 8071,
+ "source_session_id": 1287,
+ "source_time_seconds": 1461.533907,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1464.061486,
+ "sequence": 895,
+ "source_event_sequence": 8394,
+ "source_session_id": 1304,
+ "source_time_seconds": 1464.061486,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01304-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1465.445531,
+ "sequence": 896,
+ "source_event_sequence": 8072,
+ "source_session_id": 1287,
+ "source_time_seconds": 1465.445531,
+ "source_user_id": 47,
+ "trace_session_id": "ts-01287-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1465.445531,
+ "sequence": 897,
+ "source_event_sequence": 8072,
+ "source_session_id": 1263,
+ "source_time_seconds": 1465.445531,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1468.865122,
+ "sequence": 898,
+ "source_event_sequence": 8189,
+ "source_session_id": 1304,
+ "source_time_seconds": 1468.865122,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01304-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1470.155889,
+ "sequence": 899,
+ "source_event_sequence": 8171,
+ "source_session_id": 1301,
+ "source_time_seconds": 1470.155889,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1470.787507,
+ "sequence": 900,
+ "source_event_sequence": 8531,
+ "source_session_id": 1342,
+ "source_time_seconds": 1470.787507,
+ "source_user_id": 271,
+ "trace_session_id": "ts-01342-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1474.878147,
+ "sequence": 901,
+ "source_event_sequence": 8190,
+ "source_session_id": 1304,
+ "source_time_seconds": 1474.878147,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01304-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1475.873224,
+ "sequence": 902,
+ "source_event_sequence": 8584,
+ "source_session_id": 1313,
+ "source_time_seconds": 1475.873224,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1476.881583,
+ "sequence": 903,
+ "source_event_sequence": 8205,
+ "source_session_id": 1307,
+ "source_time_seconds": 1476.881583,
+ "source_user_id": 269,
+ "trace_session_id": "ts-01307-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1476.881583,
+ "sequence": 904,
+ "source_event_sequence": 8205,
+ "source_session_id": 1361,
+ "source_time_seconds": 1476.881583,
+ "source_user_id": 272,
+ "trace_session_id": "ts-01361-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1479.986342,
+ "sequence": 905,
+ "source_event_sequence": 8228,
+ "source_session_id": 1310,
+ "source_time_seconds": 1479.986342,
+ "source_user_id": 199,
+ "trace_session_id": "ts-01310-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1482.263463,
+ "sequence": 906,
+ "source_event_sequence": 8587,
+ "source_session_id": 1361,
+ "source_time_seconds": 1482.263463,
+ "source_user_id": 272,
+ "trace_session_id": "ts-01361-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1484.846956,
+ "sequence": 907,
+ "source_event_sequence": 8434,
+ "source_session_id": 1342,
+ "source_time_seconds": 1484.846956,
+ "source_user_id": 271,
+ "trace_session_id": "ts-01342-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1485.174433,
+ "sequence": 908,
+ "source_event_sequence": 8661,
+ "source_session_id": 1353,
+ "source_time_seconds": 1485.174433,
+ "source_user_id": 288,
+ "trace_session_id": "ts-01353-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1486.385084,
+ "sequence": 909,
+ "source_event_sequence": 8053,
+ "source_session_id": 1284,
+ "source_time_seconds": 1486.385084,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1486.808299,
+ "sequence": 910,
+ "source_event_sequence": 8246,
+ "source_session_id": 1313,
+ "source_time_seconds": 1486.808299,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1488.368392,
+ "sequence": 911,
+ "source_event_sequence": 8247,
+ "source_session_id": 1313,
+ "source_time_seconds": 1488.368392,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1491.91995,
+ "sequence": 912,
+ "source_event_sequence": 7901,
+ "source_session_id": 1263,
+ "source_time_seconds": 1491.91995,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1492.507789,
+ "sequence": 913,
+ "source_event_sequence": 8172,
+ "source_session_id": 1301,
+ "source_time_seconds": 1492.507789,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1492.619484,
+ "sequence": 914,
+ "source_event_sequence": 8514,
+ "source_session_id": 1353,
+ "source_time_seconds": 1492.619484,
+ "source_user_id": 288,
+ "trace_session_id": "ts-01353-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1498.403229,
+ "sequence": 915,
+ "source_event_sequence": 8054,
+ "source_session_id": 1284,
+ "source_time_seconds": 1498.403229,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1500.095987,
+ "sequence": 916,
+ "source_event_sequence": 8804,
+ "source_session_id": 1396,
+ "source_time_seconds": 1500.095987,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1500.342119,
+ "sequence": 917,
+ "source_event_sequence": 8495,
+ "source_session_id": 1353,
+ "source_time_seconds": 1500.342119,
+ "source_user_id": 288,
+ "trace_session_id": "ts-01353-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1503.287146,
+ "sequence": 918,
+ "source_event_sequence": 8173,
+ "source_session_id": 1301,
+ "source_time_seconds": 1503.287146,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1503.961821,
+ "sequence": 919,
+ "source_event_sequence": 8827,
+ "source_session_id": 1393,
+ "source_time_seconds": 1503.961821,
+ "source_user_id": 320,
+ "trace_session_id": "ts-01393-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1503.97902,
+ "sequence": 920,
+ "source_event_sequence": 8808,
+ "source_session_id": 1313,
+ "source_time_seconds": 1503.97902,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1504.096342,
+ "sequence": 921,
+ "source_event_sequence": 8829,
+ "source_session_id": 1381,
+ "source_time_seconds": 1504.096342,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1504.247922,
+ "sequence": 922,
+ "source_event_sequence": 7902,
+ "source_session_id": 1263,
+ "source_time_seconds": 1504.247922,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1504.394516,
+ "sequence": 923,
+ "source_event_sequence": 7883,
+ "source_session_id": 1342,
+ "source_time_seconds": 1504.394516,
+ "source_user_id": 271,
+ "trace_session_id": "ts-01342-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1507.917716,
+ "sequence": 924,
+ "source_event_sequence": 8873,
+ "source_session_id": 1313,
+ "source_time_seconds": 1507.917716,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1508.06072,
+ "sequence": 925,
+ "source_event_sequence": 8229,
+ "source_session_id": 1310,
+ "source_time_seconds": 1508.06072,
+ "source_user_id": 199,
+ "trace_session_id": "ts-01310-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1508.562459,
+ "sequence": 926,
+ "source_event_sequence": 8767,
+ "source_session_id": 1393,
+ "source_time_seconds": 1508.562459,
+ "source_user_id": 320,
+ "trace_session_id": "ts-01393-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1508.946274,
+ "sequence": 927,
+ "source_event_sequence": 8768,
+ "source_session_id": 1393,
+ "source_time_seconds": 1508.946274,
+ "source_user_id": 320,
+ "trace_session_id": "ts-01393-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1508.959598,
+ "sequence": 928,
+ "source_event_sequence": 8249,
+ "source_session_id": 1313,
+ "source_time_seconds": 1508.959598,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1511.071944,
+ "sequence": 929,
+ "source_event_sequence": 8775,
+ "source_session_id": 1313,
+ "source_time_seconds": 1511.071944,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1511.088849,
+ "sequence": 930,
+ "source_event_sequence": 8896,
+ "source_session_id": 1313,
+ "source_time_seconds": 1511.088849,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1513.806377,
+ "sequence": 931,
+ "source_event_sequence": 8769,
+ "source_session_id": 1393,
+ "source_time_seconds": 1513.806377,
+ "source_user_id": 320,
+ "trace_session_id": "ts-01393-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1513.806377,
+ "sequence": 932,
+ "source_event_sequence": 8769,
+ "source_session_id": 1353,
+ "source_time_seconds": 1513.806377,
+ "source_user_id": 288,
+ "trace_session_id": "ts-01353-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1513.927691,
+ "sequence": 933,
+ "source_event_sequence": 8777,
+ "source_session_id": 1396,
+ "source_time_seconds": 1513.927691,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1514.659287,
+ "sequence": 934,
+ "source_event_sequence": 8250,
+ "source_session_id": 1313,
+ "source_time_seconds": 1514.659287,
+ "source_user_id": 371,
+ "trace_session_id": "ts-01313-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1514.659287,
+ "sequence": 935,
+ "source_event_sequence": 8250,
+ "source_session_id": 1322,
+ "source_time_seconds": 1514.659287,
+ "source_user_id": 479,
+ "trace_session_id": "ts-01322-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1518.282426,
+ "sequence": 936,
+ "source_event_sequence": 7903,
+ "source_session_id": 1263,
+ "source_time_seconds": 1518.282426,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1519.899123,
+ "sequence": 937,
+ "source_event_sequence": 8308,
+ "source_session_id": 1322,
+ "source_time_seconds": 1519.899123,
+ "source_user_id": 479,
+ "trace_session_id": "ts-01322-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1519.899123,
+ "sequence": 938,
+ "source_event_sequence": 8308,
+ "source_session_id": 1345,
+ "source_time_seconds": 1519.899123,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01345-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1520.388299,
+ "sequence": 939,
+ "source_event_sequence": 8517,
+ "source_session_id": 1353,
+ "source_time_seconds": 1520.388299,
+ "source_user_id": 288,
+ "trace_session_id": "ts-01353-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1520.388299,
+ "sequence": 940,
+ "source_event_sequence": 8517,
+ "source_session_id": 1292,
+ "source_time_seconds": 1520.388299,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1522.242095,
+ "sequence": 941,
+ "source_event_sequence": 8230,
+ "source_session_id": 1310,
+ "source_time_seconds": 1522.242095,
+ "source_user_id": 199,
+ "trace_session_id": "ts-01310-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1522.242095,
+ "sequence": 942,
+ "source_event_sequence": 8230,
+ "source_session_id": 1410,
+ "source_time_seconds": 1522.242095,
+ "source_user_id": 303,
+ "trace_session_id": "ts-01410-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1524.299729,
+ "sequence": 943,
+ "source_event_sequence": 8055,
+ "source_session_id": 1284,
+ "source_time_seconds": 1524.299729,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1524.388973,
+ "sequence": 944,
+ "source_event_sequence": 8778,
+ "source_session_id": 1396,
+ "source_time_seconds": 1524.388973,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1528.56616,
+ "sequence": 945,
+ "source_event_sequence": 8779,
+ "source_session_id": 1396,
+ "source_time_seconds": 1528.56616,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1528.926216,
+ "sequence": 946,
+ "source_event_sequence": 7904,
+ "source_session_id": 1263,
+ "source_time_seconds": 1528.926216,
+ "source_user_id": 287,
+ "trace_session_id": "ts-01263-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1528.926216,
+ "sequence": 947,
+ "source_event_sequence": 7904,
+ "source_session_id": 1444,
+ "source_time_seconds": 1528.926216,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1529.291717,
+ "sequence": 948,
+ "source_event_sequence": 9087,
+ "source_session_id": 1363,
+ "source_time_seconds": 1529.291717,
+ "source_user_id": 316,
+ "trace_session_id": "ts-01363-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1529.950432,
+ "sequence": 949,
+ "source_event_sequence": 8862,
+ "source_session_id": 1410,
+ "source_time_seconds": 1529.950432,
+ "source_user_id": 303,
+ "trace_session_id": "ts-01410-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1532.596167,
+ "sequence": 950,
+ "source_event_sequence": 8596,
+ "source_session_id": 1363,
+ "source_time_seconds": 1532.596167,
+ "source_user_id": 316,
+ "trace_session_id": "ts-01363-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1534.282867,
+ "sequence": 951,
+ "source_event_sequence": 9051,
+ "source_session_id": 1444,
+ "source_time_seconds": 1534.282867,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1534.62677,
+ "sequence": 952,
+ "source_event_sequence": 8056,
+ "source_session_id": 1284,
+ "source_time_seconds": 1534.62677,
+ "source_user_id": 149,
+ "trace_session_id": "ts-01284-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1534.62677,
+ "sequence": 953,
+ "source_event_sequence": 8056,
+ "source_session_id": 1354,
+ "source_time_seconds": 1534.62677,
+ "source_user_id": 236,
+ "trace_session_id": "ts-01354-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1535.498643,
+ "sequence": 954,
+ "source_event_sequence": 8588,
+ "source_session_id": 1361,
+ "source_time_seconds": 1535.498643,
+ "source_user_id": 272,
+ "trace_session_id": "ts-01361-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1535.498643,
+ "sequence": 955,
+ "source_event_sequence": 8588,
+ "source_session_id": 1309,
+ "source_time_seconds": 1535.498643,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1536.078013,
+ "sequence": 956,
+ "source_event_sequence": 8087,
+ "source_session_id": 1309,
+ "source_time_seconds": 1536.078013,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1536.128666,
+ "sequence": 957,
+ "source_event_sequence": 9123,
+ "source_session_id": 1309,
+ "source_time_seconds": 1536.128666,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1537.500305,
+ "sequence": 958,
+ "source_event_sequence": 8780,
+ "source_session_id": 1396,
+ "source_time_seconds": 1537.500305,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1537.607792,
+ "sequence": 959,
+ "source_event_sequence": 8692,
+ "source_session_id": 1381,
+ "source_time_seconds": 1537.607792,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1540.178046,
+ "sequence": 960,
+ "source_event_sequence": 9052,
+ "source_session_id": 1444,
+ "source_time_seconds": 1540.178046,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1540.389514,
+ "sequence": 961,
+ "source_event_sequence": 8213,
+ "source_session_id": 1309,
+ "source_time_seconds": 1540.389514,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1543.498967,
+ "sequence": 962,
+ "source_event_sequence": 8693,
+ "source_session_id": 1381,
+ "source_time_seconds": 1543.498967,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1546.24306,
+ "sequence": 963,
+ "source_event_sequence": 8520,
+ "source_session_id": 1354,
+ "source_time_seconds": 1546.24306,
+ "source_user_id": 236,
+ "trace_session_id": "ts-01354-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1546.579098,
+ "sequence": 964,
+ "source_event_sequence": 9266,
+ "source_session_id": 1470,
+ "source_time_seconds": 1546.579098,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1547.524144,
+ "sequence": 965,
+ "source_event_sequence": 8618,
+ "source_session_id": 1309,
+ "source_time_seconds": 1547.524144,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1548.287868,
+ "sequence": 966,
+ "source_event_sequence": 9288,
+ "source_session_id": 1309,
+ "source_time_seconds": 1548.287868,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1550.113419,
+ "sequence": 967,
+ "source_event_sequence": 9243,
+ "source_session_id": 1309,
+ "source_time_seconds": 1550.113419,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1550.131466,
+ "sequence": 968,
+ "source_event_sequence": 9332,
+ "source_session_id": 1309,
+ "source_time_seconds": 1550.131466,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1550.949606,
+ "sequence": 969,
+ "source_event_sequence": 8941,
+ "source_session_id": 1309,
+ "source_time_seconds": 1550.949606,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1551.212669,
+ "sequence": 970,
+ "source_event_sequence": 9345,
+ "source_session_id": 1309,
+ "source_time_seconds": 1551.212669,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g05"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1551.311747,
+ "sequence": 971,
+ "source_event_sequence": 9094,
+ "source_session_id": 1309,
+ "source_time_seconds": 1551.311747,
+ "source_user_id": 472,
+ "trace_session_id": "ts-01309-g05"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1551.414445,
+ "sequence": 972,
+ "source_event_sequence": 9355,
+ "source_session_id": 1488,
+ "source_time_seconds": 1551.414445,
+ "source_user_id": 141,
+ "trace_session_id": "ts-01488-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1551.494201,
+ "sequence": 973,
+ "source_event_sequence": 8181,
+ "source_session_id": 1354,
+ "source_time_seconds": 1551.494201,
+ "source_user_id": 236,
+ "trace_session_id": "ts-01354-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1551.74539,
+ "sequence": 974,
+ "source_event_sequence": 8174,
+ "source_session_id": 1301,
+ "source_time_seconds": 1551.74539,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1551.870315,
+ "sequence": 975,
+ "source_event_sequence": 9359,
+ "source_session_id": 1354,
+ "source_time_seconds": 1551.870315,
+ "source_user_id": 236,
+ "trace_session_id": "ts-01354-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1552.607007,
+ "sequence": 976,
+ "source_event_sequence": 7969,
+ "source_session_id": 1354,
+ "source_time_seconds": 1552.607007,
+ "source_user_id": 236,
+ "trace_session_id": "ts-01354-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1553.129054,
+ "sequence": 977,
+ "source_event_sequence": 9368,
+ "source_session_id": 1490,
+ "source_time_seconds": 1553.129054,
+ "source_user_id": 259,
+ "trace_session_id": "ts-01490-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1554.90521,
+ "sequence": 978,
+ "source_event_sequence": 9226,
+ "source_session_id": 1470,
+ "source_time_seconds": 1554.90521,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1555.739376,
+ "sequence": 979,
+ "source_event_sequence": 8102,
+ "source_session_id": 1292,
+ "source_time_seconds": 1555.739376,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1556.587929,
+ "sequence": 980,
+ "source_event_sequence": 8597,
+ "source_session_id": 1363,
+ "source_time_seconds": 1556.587929,
+ "source_user_id": 316,
+ "trace_session_id": "ts-01363-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1556.642726,
+ "sequence": 981,
+ "source_event_sequence": 9053,
+ "source_session_id": 1444,
+ "source_time_seconds": 1556.642726,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1556.936923,
+ "sequence": 982,
+ "source_event_sequence": 9054,
+ "source_session_id": 1444,
+ "source_time_seconds": 1556.936923,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1558.051137,
+ "sequence": 983,
+ "source_event_sequence": 8175,
+ "source_session_id": 1301,
+ "source_time_seconds": 1558.051137,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01301-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1558.051137,
+ "sequence": 984,
+ "source_event_sequence": 8175,
+ "source_session_id": 1491,
+ "source_time_seconds": 1558.051137,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1558.161137,
+ "sequence": 985,
+ "source_event_sequence": 8863,
+ "source_session_id": 1410,
+ "source_time_seconds": 1558.161137,
+ "source_user_id": 303,
+ "trace_session_id": "ts-01410-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1558.314727,
+ "sequence": 986,
+ "source_event_sequence": 9227,
+ "source_session_id": 1470,
+ "source_time_seconds": 1558.314727,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1560.007547,
+ "sequence": 987,
+ "source_event_sequence": 9191,
+ "source_session_id": 1363,
+ "source_time_seconds": 1560.007547,
+ "source_user_id": 316,
+ "trace_session_id": "ts-01363-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1560.988416,
+ "sequence": 988,
+ "source_event_sequence": 9436,
+ "source_session_id": 1500,
+ "source_time_seconds": 1560.988416,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1561.493885,
+ "sequence": 989,
+ "source_event_sequence": 7960,
+ "source_session_id": 1410,
+ "source_time_seconds": 1561.493885,
+ "source_user_id": 303,
+ "trace_session_id": "ts-01410-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1563.063148,
+ "sequence": 990,
+ "source_event_sequence": 9457,
+ "source_session_id": 1410,
+ "source_time_seconds": 1563.063148,
+ "source_user_id": 303,
+ "trace_session_id": "ts-01410-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1564.474429,
+ "sequence": 991,
+ "source_event_sequence": 9055,
+ "source_session_id": 1444,
+ "source_time_seconds": 1564.474429,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1566.435001,
+ "sequence": 992,
+ "source_event_sequence": 8449,
+ "source_session_id": 1345,
+ "source_time_seconds": 1566.435001,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01345-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1566.893516,
+ "sequence": 993,
+ "source_event_sequence": 9369,
+ "source_session_id": 1491,
+ "source_time_seconds": 1566.893516,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1568.10545,
+ "sequence": 994,
+ "source_event_sequence": 8781,
+ "source_session_id": 1396,
+ "source_time_seconds": 1568.10545,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1570.00837,
+ "sequence": 995,
+ "source_event_sequence": 9056,
+ "source_session_id": 1444,
+ "source_time_seconds": 1570.00837,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1570.078448,
+ "sequence": 996,
+ "source_event_sequence": 8450,
+ "source_session_id": 1345,
+ "source_time_seconds": 1570.078448,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01345-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1570.103545,
+ "sequence": 997,
+ "source_event_sequence": 9596,
+ "source_session_id": 1511,
+ "source_time_seconds": 1570.103545,
+ "source_user_id": 383,
+ "trace_session_id": "ts-01511-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1570.57061,
+ "sequence": 998,
+ "source_event_sequence": 9057,
+ "source_session_id": 1444,
+ "source_time_seconds": 1570.57061,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1571.257671,
+ "sequence": 999,
+ "source_event_sequence": 8969,
+ "source_session_id": 1410,
+ "source_time_seconds": 1571.257671,
+ "source_user_id": 303,
+ "trace_session_id": "ts-01410-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1572.187517,
+ "sequence": 1000,
+ "source_event_sequence": 9370,
+ "source_session_id": 1491,
+ "source_time_seconds": 1572.187517,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1572.728406,
+ "sequence": 1001,
+ "source_event_sequence": 9363,
+ "source_session_id": 1490,
+ "source_time_seconds": 1572.728406,
+ "source_user_id": 259,
+ "trace_session_id": "ts-01490-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1573.430794,
+ "sequence": 1002,
+ "source_event_sequence": 9509,
+ "source_session_id": 1511,
+ "source_time_seconds": 1573.430794,
+ "source_user_id": 383,
+ "trace_session_id": "ts-01511-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1573.430794,
+ "sequence": 1003,
+ "source_event_sequence": 9509,
+ "source_session_id": 1506,
+ "source_time_seconds": 1573.430794,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01506-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1574.304535,
+ "sequence": 1004,
+ "source_event_sequence": 8451,
+ "source_session_id": 1345,
+ "source_time_seconds": 1574.304535,
+ "source_user_id": 67,
+ "trace_session_id": "ts-01345-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1574.304535,
+ "sequence": 1005,
+ "source_event_sequence": 8451,
+ "source_session_id": 1510,
+ "source_time_seconds": 1574.304535,
+ "source_user_id": 406,
+ "trace_session_id": "ts-01510-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1575.45708,
+ "sequence": 1006,
+ "source_event_sequence": 9646,
+ "source_session_id": 1531,
+ "source_time_seconds": 1575.45708,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01531-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1578.299045,
+ "sequence": 1007,
+ "source_event_sequence": 8446,
+ "source_session_id": 1510,
+ "source_time_seconds": 1578.299045,
+ "source_user_id": 406,
+ "trace_session_id": "ts-01510-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1578.420826,
+ "sequence": 1008,
+ "source_event_sequence": 9645,
+ "source_session_id": 1531,
+ "source_time_seconds": 1578.420826,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01531-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1578.420826,
+ "sequence": 1009,
+ "source_event_sequence": 9645,
+ "source_session_id": 1535,
+ "source_time_seconds": 1578.420826,
+ "source_user_id": 121,
+ "trace_session_id": "ts-01535-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1579.013788,
+ "sequence": 1010,
+ "source_event_sequence": 9428,
+ "source_session_id": 1500,
+ "source_time_seconds": 1579.013788,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1579.146018,
+ "sequence": 1011,
+ "source_event_sequence": 9228,
+ "source_session_id": 1470,
+ "source_time_seconds": 1579.146018,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1582.902422,
+ "sequence": 1012,
+ "source_event_sequence": 9669,
+ "source_session_id": 1535,
+ "source_time_seconds": 1582.902422,
+ "source_user_id": 121,
+ "trace_session_id": "ts-01535-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1584.434635,
+ "sequence": 1013,
+ "source_event_sequence": 9229,
+ "source_session_id": 1470,
+ "source_time_seconds": 1584.434635,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1584.482585,
+ "sequence": 1014,
+ "source_event_sequence": 9755,
+ "source_session_id": 1544,
+ "source_time_seconds": 1584.482585,
+ "source_user_id": 471,
+ "trace_session_id": "ts-01544-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1588.662378,
+ "sequence": 1015,
+ "source_event_sequence": 8724,
+ "source_session_id": 1292,
+ "source_time_seconds": 1588.662378,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1588.911222,
+ "sequence": 1016,
+ "source_event_sequence": 9771,
+ "source_session_id": 1292,
+ "source_time_seconds": 1588.911222,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1589.097643,
+ "sequence": 1017,
+ "source_event_sequence": 8872,
+ "source_session_id": 1292,
+ "source_time_seconds": 1589.097643,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1589.276523,
+ "sequence": 1018,
+ "source_event_sequence": 9725,
+ "source_session_id": 1544,
+ "source_time_seconds": 1589.276523,
+ "source_user_id": 471,
+ "trace_session_id": "ts-01544-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1589.699562,
+ "sequence": 1019,
+ "source_event_sequence": 9230,
+ "source_session_id": 1470,
+ "source_time_seconds": 1589.699562,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1590.045714,
+ "sequence": 1020,
+ "source_event_sequence": 9776,
+ "source_session_id": 1292,
+ "source_time_seconds": 1590.045714,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1590.648482,
+ "sequence": 1021,
+ "source_event_sequence": 9534,
+ "source_session_id": 1292,
+ "source_time_seconds": 1590.648482,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1591.628424,
+ "sequence": 1022,
+ "source_event_sequence": 9804,
+ "source_session_id": 1292,
+ "source_time_seconds": 1591.628424,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1591.797866,
+ "sequence": 1023,
+ "source_event_sequence": 9704,
+ "source_session_id": 1292,
+ "source_time_seconds": 1591.797866,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1591.836072,
+ "sequence": 1024,
+ "source_event_sequence": 9809,
+ "source_session_id": 1292,
+ "source_time_seconds": 1591.836072,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g05"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1592.6822,
+ "sequence": 1025,
+ "source_event_sequence": 9371,
+ "source_session_id": 1491,
+ "source_time_seconds": 1592.6822,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1592.75124,
+ "sequence": 1026,
+ "source_event_sequence": 8694,
+ "source_session_id": 1381,
+ "source_time_seconds": 1592.75124,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1592.953036,
+ "sequence": 1027,
+ "source_event_sequence": 9470,
+ "source_session_id": 1506,
+ "source_time_seconds": 1592.953036,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01506-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1593.759671,
+ "sequence": 1028,
+ "source_event_sequence": 9670,
+ "source_session_id": 1535,
+ "source_time_seconds": 1593.759671,
+ "source_user_id": 121,
+ "trace_session_id": "ts-01535-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1594.612924,
+ "sequence": 1029,
+ "source_event_sequence": 9775,
+ "source_session_id": 1292,
+ "source_time_seconds": 1594.612924,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g05"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1595.020383,
+ "sequence": 1030,
+ "source_event_sequence": 9828,
+ "source_session_id": 1292,
+ "source_time_seconds": 1595.020383,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g06"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1595.784844,
+ "sequence": 1031,
+ "source_event_sequence": 9726,
+ "source_session_id": 1544,
+ "source_time_seconds": 1595.784844,
+ "source_user_id": 471,
+ "trace_session_id": "ts-01544-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1596.482454,
+ "sequence": 1032,
+ "source_event_sequence": 9671,
+ "source_session_id": 1535,
+ "source_time_seconds": 1596.482454,
+ "source_user_id": 121,
+ "trace_session_id": "ts-01535-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1596.482454,
+ "sequence": 1033,
+ "source_event_sequence": 9671,
+ "source_session_id": 1565,
+ "source_time_seconds": 1596.482454,
+ "source_user_id": 241,
+ "trace_session_id": "ts-01565-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1597.995556,
+ "sequence": 1034,
+ "source_event_sequence": 9844,
+ "source_session_id": 1565,
+ "source_time_seconds": 1597.995556,
+ "source_user_id": 241,
+ "trace_session_id": "ts-01565-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1599.697914,
+ "sequence": 1035,
+ "source_event_sequence": 9364,
+ "source_session_id": 1490,
+ "source_time_seconds": 1599.697914,
+ "source_user_id": 259,
+ "trace_session_id": "ts-01490-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1599.971604,
+ "sequence": 1036,
+ "source_event_sequence": 9065,
+ "source_session_id": 1292,
+ "source_time_seconds": 1599.971604,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g06"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1601.056309,
+ "sequence": 1037,
+ "source_event_sequence": 9727,
+ "source_session_id": 1544,
+ "source_time_seconds": 1601.056309,
+ "source_user_id": 471,
+ "trace_session_id": "ts-01544-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1601.056309,
+ "sequence": 1038,
+ "source_event_sequence": 9727,
+ "source_session_id": 1292,
+ "source_time_seconds": 1601.056309,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g07"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1604.022725,
+ "sequence": 1039,
+ "source_event_sequence": 9923,
+ "source_session_id": 1575,
+ "source_time_seconds": 1604.022725,
+ "source_user_id": 188,
+ "trace_session_id": "ts-01575-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1604.046856,
+ "sequence": 1040,
+ "source_event_sequence": 9842,
+ "source_session_id": 1292,
+ "source_time_seconds": 1604.046856,
+ "source_user_id": 416,
+ "trace_session_id": "ts-01292-g07"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1604.352175,
+ "sequence": 1041,
+ "source_event_sequence": 9935,
+ "source_session_id": 1580,
+ "source_time_seconds": 1604.352175,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1606.710312,
+ "sequence": 1042,
+ "source_event_sequence": 9904,
+ "source_session_id": 1575,
+ "source_time_seconds": 1606.710312,
+ "source_user_id": 188,
+ "trace_session_id": "ts-01575-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1608.0849,
+ "sequence": 1043,
+ "source_event_sequence": 9372,
+ "source_session_id": 1491,
+ "source_time_seconds": 1608.0849,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1608.187841,
+ "sequence": 1044,
+ "source_event_sequence": 9905,
+ "source_session_id": 1575,
+ "source_time_seconds": 1608.187841,
+ "source_user_id": 188,
+ "trace_session_id": "ts-01575-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1609.477996,
+ "sequence": 1045,
+ "source_event_sequence": 9261,
+ "source_session_id": 1488,
+ "source_time_seconds": 1609.477996,
+ "source_user_id": 141,
+ "trace_session_id": "ts-01488-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1609.802728,
+ "sequence": 1046,
+ "source_event_sequence": 9231,
+ "source_session_id": 1470,
+ "source_time_seconds": 1609.802728,
+ "source_user_id": 126,
+ "trace_session_id": "ts-01470-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1609.802728,
+ "sequence": 1047,
+ "source_event_sequence": 9231,
+ "source_session_id": 1583,
+ "source_time_seconds": 1609.802728,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1610.295591,
+ "sequence": 1048,
+ "source_event_sequence": 9997,
+ "source_session_id": 1488,
+ "source_time_seconds": 1610.295591,
+ "source_user_id": 141,
+ "trace_session_id": "ts-01488-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1610.352437,
+ "sequence": 1049,
+ "source_event_sequence": 9196,
+ "source_session_id": 1488,
+ "source_time_seconds": 1610.352437,
+ "source_user_id": 141,
+ "trace_session_id": "ts-01488-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1610.994032,
+ "sequence": 1050,
+ "source_event_sequence": 8782,
+ "source_session_id": 1396,
+ "source_time_seconds": 1610.994032,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1611.334186,
+ "sequence": 1051,
+ "source_event_sequence": 10019,
+ "source_session_id": 1588,
+ "source_time_seconds": 1611.334186,
+ "source_user_id": 154,
+ "trace_session_id": "ts-01588-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1611.347233,
+ "sequence": 1052,
+ "source_event_sequence": 8680,
+ "source_session_id": 1491,
+ "source_time_seconds": 1611.347233,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1611.404095,
+ "sequence": 1053,
+ "source_event_sequence": 9365,
+ "source_session_id": 1490,
+ "source_time_seconds": 1611.404095,
+ "source_user_id": 259,
+ "trace_session_id": "ts-01490-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1612.090639,
+ "sequence": 1054,
+ "source_event_sequence": 9366,
+ "source_session_id": 1490,
+ "source_time_seconds": 1612.090639,
+ "source_user_id": 259,
+ "trace_session_id": "ts-01490-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1612.646539,
+ "sequence": 1055,
+ "source_event_sequence": 10021,
+ "source_session_id": 1491,
+ "source_time_seconds": 1612.646539,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1612.775359,
+ "sequence": 1056,
+ "source_event_sequence": 9167,
+ "source_session_id": 1491,
+ "source_time_seconds": 1612.775359,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g02"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1612.929208,
+ "sequence": 1057,
+ "source_event_sequence": 10033,
+ "source_session_id": 1491,
+ "source_time_seconds": 1612.929208,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1613.408104,
+ "sequence": 1058,
+ "source_event_sequence": 9058,
+ "source_session_id": 1444,
+ "source_time_seconds": 1613.408104,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1614.079589,
+ "sequence": 1059,
+ "source_event_sequence": 9845,
+ "source_session_id": 1565,
+ "source_time_seconds": 1614.079589,
+ "source_user_id": 241,
+ "trace_session_id": "ts-01565-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1616.159748,
+ "sequence": 1060,
+ "source_event_sequence": 9471,
+ "source_session_id": 1506,
+ "source_time_seconds": 1616.159748,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01506-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1618.144885,
+ "sequence": 1061,
+ "source_event_sequence": 10003,
+ "source_session_id": 1588,
+ "source_time_seconds": 1618.144885,
+ "source_user_id": 154,
+ "trace_session_id": "ts-01588-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1618.144885,
+ "sequence": 1062,
+ "source_event_sequence": 10003,
+ "source_session_id": 1600,
+ "source_time_seconds": 1618.144885,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1618.696633,
+ "sequence": 1063,
+ "source_event_sequence": 9967,
+ "source_session_id": 1583,
+ "source_time_seconds": 1618.696633,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1619.092012,
+ "sequence": 1064,
+ "source_event_sequence": 8783,
+ "source_session_id": 1396,
+ "source_time_seconds": 1619.092012,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "arrival_reason": "source_session_arrival",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1620.223635,
+ "sequence": 1065,
+ "source_event_sequence": 10141,
+ "source_session_id": 1610,
+ "source_time_seconds": 1620.223635,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1620.28203,
+ "sequence": 1066,
+ "source_event_sequence": 9367,
+ "source_session_id": 1490,
+ "source_time_seconds": 1620.28203,
+ "source_user_id": 259,
+ "trace_session_id": "ts-01490-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1624.073105,
+ "sequence": 1067,
+ "source_event_sequence": 9968,
+ "source_session_id": 1583,
+ "source_time_seconds": 1624.073105,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1624.984381,
+ "sequence": 1068,
+ "source_event_sequence": 8784,
+ "source_session_id": 1396,
+ "source_time_seconds": 1624.984381,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1625.495948,
+ "sequence": 1069,
+ "source_event_sequence": 7931,
+ "source_session_id": 1491,
+ "source_time_seconds": 1625.495948,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1625.709013,
+ "sequence": 1070,
+ "source_event_sequence": 10159,
+ "source_session_id": 1604,
+ "source_time_seconds": 1625.709013,
+ "source_user_id": 89,
+ "trace_session_id": "ts-01604-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1625.807065,
+ "sequence": 1071,
+ "source_event_sequence": 8850,
+ "source_session_id": 1604,
+ "source_time_seconds": 1625.807065,
+ "source_user_id": 89,
+ "trace_session_id": "ts-01604-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1626.829014,
+ "sequence": 1072,
+ "source_event_sequence": 10177,
+ "source_session_id": 1615,
+ "source_time_seconds": 1626.829014,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01615-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1627.420978,
+ "sequence": 1073,
+ "source_event_sequence": 10020,
+ "source_session_id": 1580,
+ "source_time_seconds": 1627.420978,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1627.580678,
+ "sequence": 1074,
+ "source_event_sequence": 10184,
+ "source_session_id": 1580,
+ "source_time_seconds": 1627.580678,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1627.632651,
+ "sequence": 1075,
+ "source_event_sequence": 9429,
+ "source_session_id": 1500,
+ "source_time_seconds": 1627.632651,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1628.48675,
+ "sequence": 1076,
+ "source_event_sequence": 9430,
+ "source_session_id": 1500,
+ "source_time_seconds": 1628.48675,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1629.484657,
+ "sequence": 1077,
+ "source_event_sequence": 9906,
+ "source_session_id": 1575,
+ "source_time_seconds": 1629.484657,
+ "source_user_id": 188,
+ "trace_session_id": "ts-01575-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1629.914621,
+ "sequence": 1078,
+ "source_event_sequence": 9846,
+ "source_session_id": 1565,
+ "source_time_seconds": 1629.914621,
+ "source_user_id": 241,
+ "trace_session_id": "ts-01565-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1629.914621,
+ "sequence": 1079,
+ "source_event_sequence": 9846,
+ "source_session_id": 1491,
+ "source_time_seconds": 1629.914621,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g04"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1630.597936,
+ "sequence": 1080,
+ "source_event_sequence": 9936,
+ "source_session_id": 1580,
+ "source_time_seconds": 1630.597936,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1633.940948,
+ "sequence": 1081,
+ "source_event_sequence": 10173,
+ "source_session_id": 1615,
+ "source_time_seconds": 1633.940948,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01615-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1635.615655,
+ "sequence": 1082,
+ "source_event_sequence": 9059,
+ "source_session_id": 1444,
+ "source_time_seconds": 1635.615655,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1636.982476,
+ "sequence": 1083,
+ "source_event_sequence": 10174,
+ "source_session_id": 1615,
+ "source_time_seconds": 1636.982476,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01615-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1637.950746,
+ "sequence": 1084,
+ "source_event_sequence": 9907,
+ "source_session_id": 1575,
+ "source_time_seconds": 1637.950746,
+ "source_user_id": 188,
+ "trace_session_id": "ts-01575-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1637.950746,
+ "sequence": 1085,
+ "source_event_sequence": 9907,
+ "source_session_id": 1629,
+ "source_time_seconds": 1637.950746,
+ "source_user_id": 412,
+ "trace_session_id": "ts-01629-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1640.240626,
+ "sequence": 1086,
+ "source_event_sequence": 8695,
+ "source_session_id": 1381,
+ "source_time_seconds": 1640.240626,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1641.15881,
+ "sequence": 1087,
+ "source_event_sequence": 9431,
+ "source_session_id": 1500,
+ "source_time_seconds": 1641.15881,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1642.541576,
+ "sequence": 1088,
+ "source_event_sequence": 9060,
+ "source_session_id": 1444,
+ "source_time_seconds": 1642.541576,
+ "source_user_id": 403,
+ "trace_session_id": "ts-01444-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1642.541576,
+ "sequence": 1089,
+ "source_event_sequence": 9060,
+ "source_session_id": 1638,
+ "source_time_seconds": 1642.541576,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1642.605618,
+ "sequence": 1090,
+ "source_event_sequence": 10142,
+ "source_session_id": 1610,
+ "source_time_seconds": 1642.605618,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1643.491543,
+ "sequence": 1091,
+ "source_event_sequence": 9969,
+ "source_session_id": 1583,
+ "source_time_seconds": 1643.491543,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1645.531398,
+ "sequence": 1092,
+ "source_event_sequence": 10143,
+ "source_session_id": 1610,
+ "source_time_seconds": 1645.531398,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1645.851172,
+ "sequence": 1093,
+ "source_event_sequence": 10270,
+ "source_session_id": 1629,
+ "source_time_seconds": 1645.851172,
+ "source_user_id": 412,
+ "trace_session_id": "ts-01629-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1646.44364,
+ "sequence": 1094,
+ "source_event_sequence": 10144,
+ "source_session_id": 1610,
+ "source_time_seconds": 1646.44364,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1648.040164,
+ "sequence": 1095,
+ "source_event_sequence": 10315,
+ "source_session_id": 1638,
+ "source_time_seconds": 1648.040164,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1648.97576,
+ "sequence": 1096,
+ "source_event_sequence": 10271,
+ "source_session_id": 1629,
+ "source_time_seconds": 1648.97576,
+ "source_user_id": 412,
+ "trace_session_id": "ts-01629-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1652.000582,
+ "sequence": 1097,
+ "source_event_sequence": 9630,
+ "source_session_id": 1491,
+ "source_time_seconds": 1652.000582,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g04"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1652.912027,
+ "sequence": 1098,
+ "source_event_sequence": 10382,
+ "source_session_id": 1644,
+ "source_time_seconds": 1652.912027,
+ "source_user_id": 87,
+ "trace_session_id": "ts-01644-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1653.525267,
+ "sequence": 1099,
+ "source_event_sequence": 10175,
+ "source_session_id": 1615,
+ "source_time_seconds": 1653.525267,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01615-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1656.145894,
+ "sequence": 1100,
+ "source_event_sequence": 8737,
+ "source_session_id": 1580,
+ "source_time_seconds": 1656.145894,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1656.321979,
+ "sequence": 1101,
+ "source_event_sequence": 10176,
+ "source_session_id": 1615,
+ "source_time_seconds": 1656.321979,
+ "source_user_id": 102,
+ "trace_session_id": "ts-01615-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1656.321979,
+ "sequence": 1102,
+ "source_event_sequence": 10176,
+ "source_session_id": 1580,
+ "source_time_seconds": 1656.321979,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1657.422997,
+ "sequence": 1103,
+ "source_event_sequence": 8696,
+ "source_session_id": 1381,
+ "source_time_seconds": 1657.422997,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1658.06817,
+ "sequence": 1104,
+ "source_event_sequence": 8697,
+ "source_session_id": 1381,
+ "source_time_seconds": 1658.06817,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1658.967534,
+ "sequence": 1105,
+ "source_event_sequence": 10074,
+ "source_session_id": 1600,
+ "source_time_seconds": 1658.967534,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1660.820558,
+ "sequence": 1106,
+ "source_event_sequence": 9472,
+ "source_session_id": 1506,
+ "source_time_seconds": 1660.820558,
+ "source_user_id": 158,
+ "trace_session_id": "ts-01506-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1660.820558,
+ "sequence": 1107,
+ "source_event_sequence": 9472,
+ "source_session_id": 1655,
+ "source_time_seconds": 1660.820558,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1662.950365,
+ "sequence": 1108,
+ "source_event_sequence": 8785,
+ "source_session_id": 1396,
+ "source_time_seconds": 1662.950365,
+ "source_user_id": 25,
+ "trace_session_id": "ts-01396-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1662.950365,
+ "sequence": 1109,
+ "source_event_sequence": 8785,
+ "source_session_id": 1635,
+ "source_time_seconds": 1662.950365,
+ "source_user_id": 197,
+ "trace_session_id": "ts-01635-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1663.527597,
+ "sequence": 1110,
+ "source_event_sequence": 9937,
+ "source_session_id": 1580,
+ "source_time_seconds": 1663.527597,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1663.86805,
+ "sequence": 1111,
+ "source_event_sequence": 9938,
+ "source_session_id": 1580,
+ "source_time_seconds": 1663.86805,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1665.919756,
+ "sequence": 1112,
+ "source_event_sequence": 10400,
+ "source_session_id": 1655,
+ "source_time_seconds": 1665.919756,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1669.088407,
+ "sequence": 1113,
+ "source_event_sequence": 10348,
+ "source_session_id": 1644,
+ "source_time_seconds": 1669.088407,
+ "source_user_id": 87,
+ "trace_session_id": "ts-01644-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1669.251563,
+ "sequence": 1114,
+ "source_event_sequence": 10303,
+ "source_session_id": 1635,
+ "source_time_seconds": 1669.251563,
+ "source_user_id": 197,
+ "trace_session_id": "ts-01635-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1670.593782,
+ "sequence": 1115,
+ "source_event_sequence": 8500,
+ "source_session_id": 1635,
+ "source_time_seconds": 1670.593782,
+ "source_user_id": 197,
+ "trace_session_id": "ts-01635-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1674.681343,
+ "sequence": 1116,
+ "source_event_sequence": 10458,
+ "source_session_id": 1662,
+ "source_time_seconds": 1674.681343,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1675.414685,
+ "sequence": 1117,
+ "source_event_sequence": 9754,
+ "source_session_id": 1655,
+ "source_time_seconds": 1675.414685,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1676.255125,
+ "sequence": 1118,
+ "source_event_sequence": 9432,
+ "source_session_id": 1500,
+ "source_time_seconds": 1676.255125,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1676.368851,
+ "sequence": 1119,
+ "source_event_sequence": 8698,
+ "source_session_id": 1381,
+ "source_time_seconds": 1676.368851,
+ "source_user_id": 105,
+ "trace_session_id": "ts-01381-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1676.368851,
+ "sequence": 1120,
+ "source_event_sequence": 8698,
+ "source_session_id": 1655,
+ "source_time_seconds": 1676.368851,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g02"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1676.823146,
+ "sequence": 1121,
+ "source_event_sequence": 10401,
+ "source_session_id": 1655,
+ "source_time_seconds": 1676.823146,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1677.426153,
+ "sequence": 1122,
+ "source_event_sequence": 10402,
+ "source_session_id": 1655,
+ "source_time_seconds": 1677.426153,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g02"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1677.64091,
+ "sequence": 1123,
+ "source_event_sequence": 9970,
+ "source_session_id": 1583,
+ "source_time_seconds": 1677.64091,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1679.567303,
+ "sequence": 1124,
+ "source_event_sequence": 9433,
+ "source_session_id": 1500,
+ "source_time_seconds": 1679.567303,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1684.419171,
+ "sequence": 1125,
+ "source_event_sequence": 9971,
+ "source_session_id": 1583,
+ "source_time_seconds": 1684.419171,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1685.733335,
+ "sequence": 1126,
+ "source_event_sequence": 10075,
+ "source_session_id": 1600,
+ "source_time_seconds": 1685.733335,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1687.065341,
+ "sequence": 1127,
+ "source_event_sequence": 9434,
+ "source_session_id": 1500,
+ "source_time_seconds": 1687.065341,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1687.867634,
+ "sequence": 1128,
+ "source_event_sequence": 9972,
+ "source_session_id": 1583,
+ "source_time_seconds": 1687.867634,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1688.012034,
+ "sequence": 1129,
+ "source_event_sequence": 10076,
+ "source_session_id": 1600,
+ "source_time_seconds": 1688.012034,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1689.253343,
+ "sequence": 1130,
+ "source_event_sequence": 10349,
+ "source_session_id": 1644,
+ "source_time_seconds": 1689.253343,
+ "source_user_id": 87,
+ "trace_session_id": "ts-01644-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1689.253343,
+ "sequence": 1131,
+ "source_event_sequence": 10349,
+ "source_session_id": 1666,
+ "source_time_seconds": 1689.253343,
+ "source_user_id": 297,
+ "trace_session_id": "ts-01666-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1689.901691,
+ "sequence": 1132,
+ "source_event_sequence": 9250,
+ "source_session_id": 1655,
+ "source_time_seconds": 1689.901691,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1691.22206,
+ "sequence": 1133,
+ "source_event_sequence": 10077,
+ "source_session_id": 1600,
+ "source_time_seconds": 1691.22206,
+ "source_user_id": 461,
+ "trace_session_id": "ts-01600-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1691.22206,
+ "sequence": 1134,
+ "source_event_sequence": 10077,
+ "source_session_id": 1655,
+ "source_time_seconds": 1691.22206,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1691.677246,
+ "sequence": 1135,
+ "source_event_sequence": 10145,
+ "source_session_id": 1610,
+ "source_time_seconds": 1691.677246,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1692.564977,
+ "sequence": 1136,
+ "source_event_sequence": 10447,
+ "source_session_id": 1662,
+ "source_time_seconds": 1692.564977,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1693.131605,
+ "sequence": 1137,
+ "source_event_sequence": 10272,
+ "source_session_id": 1629,
+ "source_time_seconds": 1693.131605,
+ "source_user_id": 412,
+ "trace_session_id": "ts-01629-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1695.16151,
+ "sequence": 1138,
+ "source_event_sequence": 9939,
+ "source_session_id": 1580,
+ "source_time_seconds": 1695.16151,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1695.780838,
+ "sequence": 1139,
+ "source_event_sequence": 9940,
+ "source_session_id": 1580,
+ "source_time_seconds": 1695.780838,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1696.156103,
+ "sequence": 1140,
+ "source_event_sequence": 9941,
+ "source_session_id": 1580,
+ "source_time_seconds": 1696.156103,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1697.003339,
+ "sequence": 1141,
+ "source_event_sequence": 10146,
+ "source_session_id": 1610,
+ "source_time_seconds": 1697.003339,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1699.381938,
+ "sequence": 1142,
+ "source_event_sequence": 9973,
+ "source_session_id": 1583,
+ "source_time_seconds": 1699.381938,
+ "source_user_id": 59,
+ "trace_session_id": "ts-01583-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1699.381938,
+ "sequence": 1143,
+ "source_event_sequence": 9973,
+ "source_session_id": 1663,
+ "source_time_seconds": 1699.381938,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g01"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1700.052139,
+ "sequence": 1144,
+ "source_event_sequence": 9435,
+ "source_session_id": 1500,
+ "source_time_seconds": 1700.052139,
+ "source_user_id": 88,
+ "trace_session_id": "ts-01500-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1700.052139,
+ "sequence": 1145,
+ "source_event_sequence": 9435,
+ "source_session_id": 1491,
+ "source_time_seconds": 1700.052139,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g05"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1702.461597,
+ "sequence": 1146,
+ "source_event_sequence": 10379,
+ "source_session_id": 1491,
+ "source_time_seconds": 1702.461597,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g05"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1702.975123,
+ "sequence": 1147,
+ "source_event_sequence": 10469,
+ "source_session_id": 1666,
+ "source_time_seconds": 1702.975123,
+ "source_user_id": 297,
+ "trace_session_id": "ts-01666-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1703.94258,
+ "sequence": 1148,
+ "source_event_sequence": 10517,
+ "source_session_id": 1491,
+ "source_time_seconds": 1703.94258,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g06"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1704.694776,
+ "sequence": 1149,
+ "source_event_sequence": 10273,
+ "source_session_id": 1629,
+ "source_time_seconds": 1704.694776,
+ "source_user_id": 412,
+ "trace_session_id": "ts-01629-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1709.440855,
+ "sequence": 1150,
+ "source_event_sequence": 10537,
+ "source_session_id": 1677,
+ "source_time_seconds": 1709.440855,
+ "source_user_id": 109,
+ "trace_session_id": "ts-01677-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1710.035321,
+ "sequence": 1151,
+ "source_event_sequence": 10403,
+ "source_session_id": 1655,
+ "source_time_seconds": 1710.035321,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1710.068279,
+ "sequence": 1152,
+ "source_event_sequence": 9956,
+ "source_session_id": 1491,
+ "source_time_seconds": 1710.068279,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g06"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1712.090215,
+ "sequence": 1153,
+ "source_event_sequence": 10404,
+ "source_session_id": 1655,
+ "source_time_seconds": 1712.090215,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1713.266417,
+ "sequence": 1154,
+ "source_event_sequence": 10454,
+ "source_session_id": 1663,
+ "source_time_seconds": 1713.266417,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1718.838636,
+ "sequence": 1155,
+ "source_event_sequence": 10455,
+ "source_session_id": 1663,
+ "source_time_seconds": 1718.838636,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1724.209194,
+ "sequence": 1156,
+ "source_event_sequence": 10448,
+ "source_session_id": 1662,
+ "source_time_seconds": 1724.209194,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1728.582061,
+ "sequence": 1157,
+ "source_event_sequence": 10470,
+ "source_session_id": 1666,
+ "source_time_seconds": 1728.582061,
+ "source_user_id": 297,
+ "trace_session_id": "ts-01666-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1728.914398,
+ "sequence": 1158,
+ "source_event_sequence": 10316,
+ "source_session_id": 1638,
+ "source_time_seconds": 1728.914398,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1730.836261,
+ "sequence": 1159,
+ "source_event_sequence": 10147,
+ "source_session_id": 1610,
+ "source_time_seconds": 1730.836261,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1735.325158,
+ "sequence": 1160,
+ "source_event_sequence": 10471,
+ "source_session_id": 1666,
+ "source_time_seconds": 1735.325158,
+ "source_user_id": 297,
+ "trace_session_id": "ts-01666-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1736.020456,
+ "sequence": 1161,
+ "source_event_sequence": 10449,
+ "source_session_id": 1662,
+ "source_time_seconds": 1736.020456,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1739.023373,
+ "sequence": 1162,
+ "source_event_sequence": 9942,
+ "source_session_id": 1580,
+ "source_time_seconds": 1739.023373,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1741.142353,
+ "sequence": 1163,
+ "source_event_sequence": 9943,
+ "source_session_id": 1580,
+ "source_time_seconds": 1741.142353,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1741.552545,
+ "sequence": 1164,
+ "source_event_sequence": 10472,
+ "source_session_id": 1666,
+ "source_time_seconds": 1741.552545,
+ "source_user_id": 297,
+ "trace_session_id": "ts-01666-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1742.602373,
+ "sequence": 1165,
+ "source_event_sequence": 10148,
+ "source_session_id": 1610,
+ "source_time_seconds": 1742.602373,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1746.193767,
+ "sequence": 1166,
+ "source_event_sequence": 10535,
+ "source_session_id": 1677,
+ "source_time_seconds": 1746.193767,
+ "source_user_id": 109,
+ "trace_session_id": "ts-01677-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1748.716171,
+ "sequence": 1167,
+ "source_event_sequence": 10317,
+ "source_session_id": 1638,
+ "source_time_seconds": 1748.716171,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1749.891705,
+ "sequence": 1168,
+ "source_event_sequence": 10318,
+ "source_session_id": 1638,
+ "source_time_seconds": 1749.891705,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1752.810152,
+ "sequence": 1169,
+ "source_event_sequence": 10584,
+ "source_session_id": 1663,
+ "source_time_seconds": 1752.810152,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1752.820779,
+ "sequence": 1170,
+ "source_event_sequence": 10616,
+ "source_session_id": 1663,
+ "source_time_seconds": 1752.820779,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1755.688639,
+ "sequence": 1171,
+ "source_event_sequence": 10536,
+ "source_session_id": 1677,
+ "source_time_seconds": 1755.688639,
+ "source_user_id": 109,
+ "trace_session_id": "ts-01677-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1757.313982,
+ "sequence": 1172,
+ "source_event_sequence": 10624,
+ "source_session_id": 1491,
+ "source_time_seconds": 1757.313982,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g07"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1758.169091,
+ "sequence": 1173,
+ "source_event_sequence": 9944,
+ "source_session_id": 1580,
+ "source_time_seconds": 1758.169091,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1758.26078,
+ "sequence": 1174,
+ "source_event_sequence": 8895,
+ "source_session_id": 1491,
+ "source_time_seconds": 1758.26078,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g07"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1764.585408,
+ "sequence": 1175,
+ "source_event_sequence": 10405,
+ "source_session_id": 1655,
+ "source_time_seconds": 1764.585408,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1764.740755,
+ "sequence": 1176,
+ "source_event_sequence": 10406,
+ "source_session_id": 1655,
+ "source_time_seconds": 1764.740755,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1765.395057,
+ "sequence": 1177,
+ "source_event_sequence": 10473,
+ "source_session_id": 1666,
+ "source_time_seconds": 1765.395057,
+ "source_user_id": 297,
+ "trace_session_id": "ts-01666-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": false,
+ "offset_seconds": 1765.395057,
+ "sequence": 1178,
+ "source_event_sequence": 10473,
+ "source_session_id": 1491,
+ "source_time_seconds": 1765.395057,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g08"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1768.006324,
+ "sequence": 1179,
+ "source_event_sequence": 10319,
+ "source_session_id": 1638,
+ "source_time_seconds": 1768.006324,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1772.709537,
+ "sequence": 1180,
+ "source_event_sequence": 9376,
+ "source_session_id": 1491,
+ "source_time_seconds": 1772.709537,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g08"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1777.092483,
+ "sequence": 1181,
+ "source_event_sequence": 10320,
+ "source_session_id": 1638,
+ "source_time_seconds": 1777.092483,
+ "source_user_id": 313,
+ "trace_session_id": "ts-01638-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1777.092483,
+ "sequence": 1182,
+ "source_event_sequence": 10320,
+ "source_session_id": 1701,
+ "source_time_seconds": 1777.092483,
+ "source_user_id": 140,
+ "trace_session_id": "ts-01701-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1780.45869,
+ "sequence": 1183,
+ "source_event_sequence": 10407,
+ "source_session_id": 1655,
+ "source_time_seconds": 1780.45869,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1783.633011,
+ "sequence": 1184,
+ "source_event_sequence": 9945,
+ "source_session_id": 1580,
+ "source_time_seconds": 1783.633011,
+ "source_user_id": 204,
+ "trace_session_id": "ts-01580-g03"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1783.633011,
+ "sequence": 1185,
+ "source_event_sequence": 9945,
+ "source_session_id": 1708,
+ "source_time_seconds": 1783.633011,
+ "source_user_id": 395,
+ "trace_session_id": "ts-01708-g01"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1784.969239,
+ "sequence": 1186,
+ "source_event_sequence": 10456,
+ "source_session_id": 1663,
+ "source_time_seconds": 1784.969239,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g02"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1787.413705,
+ "sequence": 1187,
+ "source_event_sequence": 10662,
+ "source_session_id": 1708,
+ "source_time_seconds": 1787.413705,
+ "source_user_id": 395,
+ "trace_session_id": "ts-01708-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1787.413705,
+ "sequence": 1188,
+ "source_event_sequence": 10662,
+ "source_session_id": 1711,
+ "source_time_seconds": 1787.413705,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01711-g01"
+ },
+ {
+ "event": "user_idle",
+ "input_enabled": false,
+ "offset_seconds": 1788.283246,
+ "sequence": 1189,
+ "source_event_sequence": 10408,
+ "source_session_id": 1655,
+ "source_time_seconds": 1788.283246,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1791.702347,
+ "sequence": 1190,
+ "source_event_sequence": 10409,
+ "source_session_id": 1655,
+ "source_time_seconds": 1791.702347,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "event": "user_active",
+ "input_enabled": true,
+ "offset_seconds": 1792.806651,
+ "sequence": 1191,
+ "source_event_sequence": 10450,
+ "source_session_id": 1662,
+ "source_time_seconds": 1792.806651,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1797.311241,
+ "sequence": 1192,
+ "source_event_sequence": 10268,
+ "source_session_id": 1491,
+ "source_time_seconds": 1797.311241,
+ "source_user_id": 211,
+ "trace_session_id": "ts-01491-g08"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1798.189025,
+ "sequence": 1193,
+ "source_event_sequence": 10683,
+ "source_session_id": 1717,
+ "source_time_seconds": 1798.189025,
+ "source_user_id": 283,
+ "trace_session_id": "ts-01717-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1798.200717,
+ "sequence": 1194,
+ "source_event_sequence": 8017,
+ "source_session_id": 1663,
+ "source_time_seconds": 1798.200717,
+ "source_user_id": 397,
+ "trace_session_id": "ts-01663-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1195,
+ "source_event_sequence": 9378,
+ "source_session_id": 1711,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01711-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1196,
+ "source_event_sequence": 10018,
+ "source_session_id": 1655,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g03"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1197,
+ "source_event_sequence": 10149,
+ "source_session_id": 1610,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 133,
+ "trace_session_id": "ts-01610-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1800.0,
+ "sequence": 1198,
+ "source_event_sequence": 10149,
+ "source_session_id": 1655,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g04"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1199,
+ "source_event_sequence": 10251,
+ "source_session_id": 1655,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 456,
+ "trace_session_id": "ts-01655-g04"
+ },
+ {
+ "departure_reason": "source_session_departure",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1200,
+ "source_event_sequence": 10451,
+ "source_session_id": 1662,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 215,
+ "trace_session_id": "ts-01662-g01"
+ },
+ {
+ "arrival_reason": "capacity_normalization_scale_up",
+ "event": "session_arrival",
+ "input_enabled": true,
+ "offset_seconds": 1800.0,
+ "sequence": 1201,
+ "source_event_sequence": 10451,
+ "source_session_id": 1711,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01711-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1202,
+ "source_event_sequence": 10556,
+ "source_session_id": 1711,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 195,
+ "trace_session_id": "ts-01711-g02"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1203,
+ "source_event_sequence": 10634,
+ "source_session_id": 1701,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 140,
+ "trace_session_id": "ts-01701-g01"
+ },
+ {
+ "departure_reason": "capacity_normalization_scale_down",
+ "event": "session_departure",
+ "offset_seconds": 1800.0,
+ "sequence": 1204,
+ "source_event_sequence": 10674,
+ "source_session_id": 1717,
+ "source_time_seconds": 1800.0,
+ "source_user_id": 283,
+ "trace_session_id": "ts-01717-g01"
+ }
+ ],
+ "kind": "explicit_session_lifecycle_v1"
+ },
+ "measurement": {
+ "connect_timeout_seconds": 90.0,
+ "first_generation_grace_seconds": 15.0,
+ "http_timeout_seconds": 30.0,
+ "sample_interval_seconds": 1.0,
+ "shutdown_timeout_seconds": 20.0,
+ "slo_fps_tolerance": 0.25
+ },
+ "name": "abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16",
+ "phases": [
+ {
+ "active_input_fraction": 1.0,
+ "duration_seconds": 1800.0,
+ "name": "turboserve_public_demo_lifecycle_replay",
+ "target_users": 16
+ }
+ ],
+ "seed": 20260815,
+ "server_url": "http://127.0.0.1:8088",
+ "session": {
+ "control": {
+ "action_states": [
+ [
+ "KeyW"
+ ],
+ [
+ "KeyW",
+ "KeyA"
+ ],
+ [
+ "KeyW",
+ "KeyD"
+ ],
+ [
+ "KeyI"
+ ]
+ ],
+ "idle_max_seconds": 0.0,
+ "idle_min_seconds": 0.0,
+ "idle_probability": 0.0,
+ "interval_seconds": 0.5,
+ "jitter_seconds": 0.15
+ },
+ "control_latent_frames": 3,
+ "delivery_mode": "latest",
+ "expected_preview_frames": 1,
+ "fps": 12,
+ "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "prompt": "A smooth first-person exploration through a vivid natural landscape."
+ },
+ "trace_contract": {
+ "capacity_transform": {
+ "derived_connection_count": 300,
+ "derived_peak_active_sessions": 12,
+ "derived_peak_retained_sessions": 16,
+ "kind": "sticky_capacity_normalized_session_sampling",
+ "scaling_rule": "round_half_up(source_retained_sessions * target_peak / source_peak); sticky selected sessions are retained until source departure or a scaled capacity decrease; scale-up uses stable SHA-256(seed:source_session_id) rank among currently present sessions.",
+ "selected_source_session_count": 183,
+ "selection_seed": 20260815,
+ "source_observed_peak_retained_sessions": 186,
+ "target_peak_retained_sessions": 16,
+ "target_sessions_per_worker": 4,
+ "target_workers": 4
+ },
+ "derivation_version": "turboserve-public-demo-capacity-normalized-v1",
+ "event_mapping": {
+ "session_arrival": "create one ABot LiveKit session; arrival input_enabled follows source payload.active/current state",
+ "session_departure": "stop and delete that selected ABot LiveKit session",
+ "user_active": "resume that selected ABot client's action heartbeat without dropping its session",
+ "user_idle": "pause that selected ABot client's action heartbeat without dropping its session or retained state"
+ },
+ "execution_contract": "No diagnostic barrier. The black-box runner schedules each lifecycle event at its explicit source-derived offset and never assigns a GPU from the client side.",
+ "kind": "turboserve_public_demo_trace_derived_abot_lifecycle",
+ "not_a_reproduction_of_private_paper_t1_to_t6_traces": true,
+ "not_a_turboserve_production_trace": true,
+ "source": {
+ "public_demo_repository_relative_path": "../../../TurboServe/traces/example_8gpu.json",
+ "public_demo_trace_filename": "example_8gpu.json",
+ "sha256": "7dc3bb8934df656a710b76df16c663686ceae8f8db7d1a9b3da98e1ecf2eda31",
+ "source_duration_seconds": 1800.0,
+ "source_event_counts": {
+ "session_arrival": 1719,
+ "session_departure": 1719,
+ "user_active": 3205,
+ "user_idle": 4042
+ },
+ "source_peak_active_sessions": 107,
+ "source_peak_retained_sessions": 186
+ },
+ "time_transform": {
+ "derived_duration_seconds": 1800.0,
+ "description": "No time compression: arrival, active, idle, and departure offsets retain the source 30-minute wall-clock scale.",
+ "kind": "identity_wall_clock",
+ "source_to_derived_scale": 1.0
+ }
+ }
+}
From 212a4a72f8c61094ba0597d7997ce049998bd248 Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Wed, 19 Aug 2026 06:04:47 +0000
Subject: [PATCH 5/8] fix(abot): warm CUDA graph capture stream
---
telefuser/pipelines/abot_world/denoising.py | 132 ++++++++-
...se_abot_cuda_graph_persistent_three_way.py | 254 +++++++++++++++++-
.../validate_abot_cuda_graph_parity.py | 12 +
3 files changed, 386 insertions(+), 12 deletions(-)
diff --git a/telefuser/pipelines/abot_world/denoising.py b/telefuser/pipelines/abot_world/denoising.py
index ebb907cf..3bec1542 100644
--- a/telefuser/pipelines/abot_world/denoising.py
+++ b/telefuser/pipelines/abot_world/denoising.py
@@ -15,6 +15,8 @@
from telefuser.schedulers.flow_match import FlowMatchScheduler
from telefuser.utils.logging import logger
+_CUDA_GRAPH_WARMUP_ITERATIONS = 1
+
@dataclass
class _CudaGraphSlot:
@@ -94,6 +96,12 @@ def __init__(
self.static_timestep = torch.empty((latent.shape[0], self.frames), dtype=torch.float32, device=self.device)
self.static_context = prompt_emb.detach().clone()
self.current_end = torch.empty(1, dtype=torch.long, device=self.device)
+ # CUDA Graph capture must use an already-warmed non-default stream.
+ # Keeping the stream on this graph state makes the warmup and capture
+ # execute on the same device/stream pair even in a process-NCCL
+ # worker where the assigned CUDA device is not logical device zero.
+ # CPU construction remains supported by lightweight unit tests.
+ self.capture_stream = torch.cuda.Stream(device=self.device) if self.device.type == "cuda" else None
capacity = self_cache[0]["k"].shape[1]
sink_tokens = dit.sink_size * self.frame_tokens
rolled_tokens = capacity - sink_tokens - latent.shape[2] * self.frame_tokens
@@ -187,9 +195,60 @@ def _capture_slot(
update_cache: bool,
) -> _CudaGraphSlot:
graph = torch.cuda.CUDAGraph()
- with torch.cuda.graph(graph, capture_error_mode="thread_local"):
+ assert self.capture_stream is not None
+ # The caller may have restored cache contents on the current stream
+ # after warmup. Make the explicit capture stream observe that work
+ # before entering capture; a graph may not contain a cross-stream
+ # dependency established while capture is active.
+ with torch.cuda.device(self.device):
+ self.capture_stream.wait_stream(torch.cuda.current_stream(self.device))
+ with torch.cuda.graph(
+ graph,
+ stream=self.capture_stream,
+ capture_error_mode="thread_local",
+ ):
+ with torch.autocast(self.device.type, dtype=self.torch_dtype, enabled=self.device.type == "cuda"):
+ output = self.dit.forward_steady_state(
+ x=self.static_x,
+ timestep=self.static_timestep,
+ context=self.static_context,
+ act_context=self.static_action,
+ kv_cache=self_cache,
+ crossattn_cache=cross_cache,
+ current_end=self.current_end,
+ roll_scratch_k=self.roll_scratch_k,
+ roll_scratch_v=self.roll_scratch_v,
+ update_cache=update_cache,
+ )
+ return _CudaGraphSlot(graph=graph, output=output)
+
+ def warmup(
+ self,
+ stage: "ABotWorldDenoisingStage",
+ latent: torch.Tensor,
+ action_context: torch.Tensor,
+ self_cache: list[dict[str, Any]],
+ cross_cache: list[dict[str, Any]],
+ *,
+ current_start: int,
+ scheduler: FlowMatchScheduler,
+ ) -> None:
+ """Warm both fixed DiT call shapes on the eventual capture stream.
+
+ ``forward_steady_state`` is deliberately distinct from the public
+ eager forward that fills the causal KV window. Its first SDPA/kernel
+ plan creation must therefore happen *outside* graph capture. This
+ method is allowed to mutate the supplied caches; callers use private
+ batched arenas or restore their B=1 cache backup before capture.
+ """
+ timesteps = stage._official_denoising_timesteps(scheduler).to(device=self.device)
+ current_end = (current_start + self.frames) * self.frame_tokens
+ assert self.capture_stream is not None
+
+ def warm_slot(timestep: torch.Tensor, *, update_cache: bool) -> None:
+ self._set_inputs(latent, action_context, timestep, current_end=current_end)
with torch.autocast(self.device.type, dtype=self.torch_dtype, enabled=self.device.type == "cuda"):
- output = self.dit.forward_steady_state(
+ self.dit.forward_steady_state(
x=self.static_x,
timestep=self.static_timestep,
context=self.static_context,
@@ -201,7 +260,14 @@ def _capture_slot(
roll_scratch_v=self.roll_scratch_v,
update_cache=update_cache,
)
- return _CudaGraphSlot(graph=graph, output=output)
+
+ with torch.cuda.device(self.device):
+ self.capture_stream.wait_stream(torch.cuda.current_stream(self.device))
+ with torch.cuda.stream(self.capture_stream):
+ for _ in range(_CUDA_GRAPH_WARMUP_ITERATIONS):
+ warm_slot(timesteps[0], update_cache=True)
+ warm_slot(timesteps[1], update_cache=False)
+ self.capture_stream.synchronize()
@staticmethod
def _draw_noise(
@@ -322,6 +388,7 @@ def __init__(self, name: str, module_manager: ModuleManager, model_runtime_confi
self._cuda_graph_captures = 0
self._cuda_graph_replays = 0
self._cuda_graph_capture_failures = 0
+ self._cuda_graph_capture_disabled = False
self._last_cuda_graph_metrics: dict[str, int] = {
"cuda_graph_enabled": 0,
"cuda_graph_eligible": 0,
@@ -335,6 +402,7 @@ def __init__(self, name: str, module_manager: ModuleManager, model_runtime_confi
def configure_cuda_graph(self, enabled: bool) -> None:
"""Enable the experimental fixed-shape CUDA Graph continuation path."""
self._cuda_graph_enabled = bool(enabled)
+ self._cuda_graph_capture_disabled = False
self._cuda_graph_states.clear()
self._cuda_graph_batch_states.clear()
@@ -357,6 +425,7 @@ def cuda_graph_metrics(self) -> dict[str, int]:
"captures": self._cuda_graph_captures,
"replays": self._cuda_graph_replays,
"capture_failures": self._cuda_graph_capture_failures,
+ "capture_disabled": int(self._cuda_graph_capture_disabled),
}
def last_cuda_graph_metrics(self) -> dict[str, int]:
@@ -367,6 +436,13 @@ def record_cuda_graph_not_used(self) -> None:
"""Mark a non-eligible batch without exposing a stale previous hit."""
self._set_cuda_graph_last_metrics(eligible=False)
+ def _record_cuda_graph_capture_failure(self) -> None:
+ """Disable graph capture for this worker after an unsafe capture."""
+ self._cuda_graph_capture_failures += 1
+ self._cuda_graph_capture_disabled = True
+ self._cuda_graph_states.clear()
+ self._cuda_graph_batch_states.clear()
+
def _cuda_graph_backend_is_supported(self) -> bool:
"""Return whether the active attention backend passed graph parity.
@@ -487,7 +563,11 @@ def _is_cuda_graph_eligible(
generator: torch.Generator,
) -> bool:
"""Check static continuation invariants only before first capture."""
- if not self._cuda_graph_enabled or torch.device(self.device).type != "cuda":
+ if (
+ not self._cuda_graph_enabled
+ or self._cuda_graph_capture_disabled
+ or torch.device(self.device).type != "cuda"
+ ):
return False
if not self._cuda_graph_backend_is_supported():
return False
@@ -526,7 +606,11 @@ def _is_cuda_graph_batched_eligible(
) -> bool:
"""Check B=2/3 fixed-window Relative-RoPE continuation invariants."""
batch_size = latent.shape[0]
- if not self._cuda_graph_enabled or torch.device(self.device).type != "cuda":
+ if (
+ not self._cuda_graph_enabled
+ or self._cuda_graph_capture_disabled
+ or torch.device(self.device).type != "cuda"
+ ):
return False
if not self._cuda_graph_backend_is_supported():
return False
@@ -725,6 +809,29 @@ def denoise_interactive_blocks(
cross_caches,
current_starts=current_starts,
)
+ captured.graph.warmup(
+ self,
+ latent,
+ action_context,
+ captured.self_cache,
+ captured.cross_cache,
+ current_start=current_starts[0],
+ scheduler=scheduler,
+ )
+ for row, (source_self, source_cross) in enumerate(zip(self_caches, cross_caches, strict=True)):
+ for source, arena in zip(source_self, captured.self_cache, strict=True):
+ arena["k"][row : row + 1].copy_(source["k"])
+ arena["v"][row : row + 1].copy_(source["v"])
+ for source, arena in zip(source_cross, captured.cross_cache, strict=True):
+ arena["k"][row : row + 1].copy_(source["k"])
+ arena["v"][row : row + 1].copy_(source["v"])
+ frame_tokens = (latent.shape[-2] // self.dit.patch_size[1]) * (latent.shape[-1] // self.dit.patch_size[2])
+ capacity = self.dit.local_attn_size * frame_tokens
+ for self_layer, cross_layer in zip(captured.self_cache, captured.cross_cache, strict=True):
+ self_layer["global_end_index"].fill_(current_starts[0] * frame_tokens)
+ self_layer["local_end_index"].fill_(capacity)
+ cross_layer["is_init"] = True
+ cross_layer["sequence_length"] = prompt_emb.shape[1]
output, replays = captured.graph.run(
self,
latent,
@@ -739,7 +846,7 @@ def denoise_interactive_blocks(
except (RuntimeError, ValueError) as exc:
for generator, saved_state in zip(generators, generator_states, strict=True):
generator.set_state(saved_state)
- self._cuda_graph_capture_failures += 1
+ self._record_cuda_graph_capture_failure()
self._set_cuda_graph_last_metrics(
eligible=True,
fallback=True,
@@ -844,6 +951,17 @@ def denoise_interactive_block(
cross_cache,
torch_dtype=self.torch_dtype,
)
+ captured.warmup(
+ self,
+ latent,
+ action_context,
+ self_cache,
+ cross_cache,
+ current_start=current_start,
+ scheduler=scheduler,
+ )
+ _ABotSteadyCudaGraph.restore_caches(self_cache, self_backup)
+ _ABotSteadyCudaGraph.restore_caches(cross_cache, cross_backup)
output, replays = captured.run(
self,
latent,
@@ -861,7 +979,7 @@ def denoise_interactive_block(
if cross_backup is not None:
_ABotSteadyCudaGraph.restore_caches(cross_cache, cross_backup)
generator.set_state(generator_state)
- self._cuda_graph_capture_failures += 1
+ self._record_cuda_graph_capture_failure()
self._set_cuda_graph_last_metrics(eligible=True, fallback=True)
logger.warning("ABot CUDA Graph capture for session {} failed; falling back to eager: {}", session_id, exc)
return self._denoise_block(
diff --git a/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py b/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py
index 7ec2cb11..e108d8ca 100644
--- a/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py
+++ b/tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py
@@ -27,6 +27,19 @@
--model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
--image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
--output-dir results/validation/abot_cuda_graph_b1_persistent_three_way
+
+Process-NCCL logical-device smoke (four physical devices visible to one
+worker process, using local cuda:1 / physical GPU 5)::
+
+ CUDA_VISIBLE_DEVICES=4,5,6,7 PYTHONPATH=$PWD \\
+ /public/fanyk1/lwb/envs/telefuser_sage291/bin/python \\
+ tools/validation/diagnose_abot_cuda_graph_persistent_three_way.py \\
+ --batch-size 2 --device-id 1 \\
+ --expected-cuda-visible-devices 4,5,6,7 \\
+ --expected-visible-device-count 4 --nccl-single-rank \\
+ --model-root /public/fanyk1/lwb/model_zoo/ABot-World-0-5B-LF \\
+ --image /public/fanyk1/lwb/ABot-World/web_client/datasets/images/84b90ad568b693d2.png \\
+ --output-dir results/validation/abot_cuda_graph_b2_nccl_visible4567_device1
"""
from __future__ import annotations
@@ -34,11 +47,15 @@
import argparse
import importlib.util
import json
+import os
+import socket
from collections.abc import Mapping, Sequence
+from datetime import timedelta
from pathlib import Path
from typing import Any
import torch
+import torch.distributed as dist
from PIL import Image
@@ -52,6 +69,149 @@ def _load_base_validator() -> Any:
return module
+def _split_cuda_visible_devices(value: str | None) -> list[str] | None:
+ """Return the process-visible device tokens without touching CUDA."""
+ if value is None:
+ return None
+ return [item.strip() for item in value.split(",") if item.strip()]
+
+
+def _expected_visible_devices(value: str | None) -> list[str] | None:
+ """Validate the optional exact CVD mapping requested by the operator."""
+ if value is None:
+ return None
+ devices = _split_cuda_visible_devices(value)
+ if not devices:
+ raise ValueError("--expected-cuda-visible-devices must contain at least one device token")
+ return devices
+
+
+def _device_context_plan(args: argparse.Namespace) -> dict[str, Any]:
+ """Validate CVD/local-index assumptions before loading a CUDA model.
+
+ Process-NCCL workers receive *logical* GPU ids. For example, a process
+ launched with ``CUDA_VISIBLE_DEVICES=4,5,6,7`` must use ``device_id=1``
+ to select physical GPU 5; passing ``5`` would be invalid in that process.
+ """
+ raw_visible = os.environ.get("CUDA_VISIBLE_DEVICES")
+ visible = _split_cuda_visible_devices(raw_visible)
+ expected = _expected_visible_devices(args.expected_cuda_visible_devices)
+ if expected is not None and visible != expected:
+ observed = "" if visible is None else ",".join(visible)
+ raise ValueError(
+ "CUDA_VISIBLE_DEVICES does not match --expected-cuda-visible-devices: "
+ f"expected {','.join(expected)!r}, observed {observed!r}"
+ )
+ if args.expected_visible_device_count is not None and visible is not None:
+ if len(visible) != args.expected_visible_device_count:
+ raise ValueError(
+ "CUDA_VISIBLE_DEVICES count does not match --expected-visible-device-count: "
+ f"expected {args.expected_visible_device_count}, observed {len(visible)}"
+ )
+ if visible is not None and not 0 <= args.device_id < len(visible):
+ raise ValueError(f"--device-id {args.device_id} is not a logical index in CUDA_VISIBLE_DEVICES={raw_visible!r}")
+ return {
+ "cuda_visible_devices": raw_visible,
+ "visible_device_tokens": visible,
+ "expected_cuda_visible_devices": expected,
+ "expected_visible_device_count": args.expected_visible_device_count,
+ "logical_device_requested": args.device_id,
+ "physical_device_token_for_logical_device": (
+ visible[args.device_id] if visible is not None and 0 <= args.device_id < len(visible) else None
+ ),
+ }
+
+
+def _activate_cuda_device_context(args: argparse.Namespace, plan: Mapping[str, Any]) -> dict[str, Any]:
+ """Select the requested local CUDA device before any pipeline/graph work.
+
+ ``torch.cuda.graph`` uses the current CUDA device for its capture stream.
+ This explicit selection mirrors ``_run_nccl_model_worker`` and avoids a
+ standalone validator accidentally capturing on logical cuda:0 while its
+ pipeline tensors reside on logical cuda:1.
+ """
+ if not torch.cuda.is_available():
+ raise RuntimeError("persistent CUDA-Graph diagnostic requires CUDA")
+ runtime_count = int(torch.cuda.device_count())
+ if args.expected_visible_device_count is not None and runtime_count != args.expected_visible_device_count:
+ raise RuntimeError(
+ "torch.cuda.device_count() does not match --expected-visible-device-count: "
+ f"expected {args.expected_visible_device_count}, observed {runtime_count}"
+ )
+ if not 0 <= args.device_id < runtime_count:
+ raise RuntimeError(f"--device-id {args.device_id} is out of range for {runtime_count} visible CUDA device(s)")
+ # Do not call current_device before this: that can initialize CUDA on the
+ # wrong default lane and would not reproduce a process-NCCL worker.
+ torch.cuda.set_device(args.device_id)
+ current = int(torch.cuda.current_device())
+ if current != args.device_id:
+ raise RuntimeError(f"failed to select logical cuda:{args.device_id}; current device is cuda:{current}")
+ properties = torch.cuda.get_device_properties(args.device_id)
+ return {
+ **dict(plan),
+ "runtime_visible_device_count": runtime_count,
+ "logical_device_current_after_set": current,
+ "selected_device": f"cuda:{args.device_id}",
+ "selected_device_name": properties.name,
+ "selected_device_capability": [int(properties.major), int(properties.minor)],
+ }
+
+
+def _free_loopback_port() -> int:
+ """Allocate a short-lived loopback rendezvous port for a rank-0 group."""
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
+def _start_single_rank_nccl(args: argparse.Namespace, device: torch.device) -> tuple[dict[str, Any], bool]:
+ """Optionally reproduce the worker-local initialized-NCCL context.
+
+ It is intentionally world-size one: CUDA Graph parity needs the same
+ process-local current-device and communicator initialization ordering as a
+ process-NCCL model worker, not a second 25-GB replica or a migration test.
+ """
+ report: dict[str, Any] = {
+ "requested": bool(args.nccl_single_rank),
+ "initialized_by_tool": False,
+ "already_initialized": bool(dist.is_available() and dist.is_initialized()),
+ }
+ if not args.nccl_single_rank:
+ return report, False
+ if not dist.is_available() or not dist.is_nccl_available():
+ raise RuntimeError("--nccl-single-rank requires a PyTorch build with NCCL support")
+ if dist.is_initialized():
+ raise RuntimeError("--nccl-single-rank refuses to reuse or destroy an existing process group")
+ port = _free_loopback_port()
+ try:
+ dist.init_process_group(
+ backend="nccl",
+ init_method=f"tcp://127.0.0.1:{port}",
+ rank=0,
+ world_size=1,
+ timeout=timedelta(seconds=args.nccl_init_timeout_seconds),
+ device_id=device,
+ )
+ except Exception:
+ if dist.is_initialized():
+ dist.destroy_process_group()
+ raise
+ if not dist.is_initialized() or str(dist.get_backend()) != "nccl":
+ if dist.is_initialized():
+ dist.destroy_process_group()
+ raise RuntimeError("single-rank NCCL initialization did not produce an NCCL process group")
+ report.update(
+ {
+ "initialized_by_tool": True,
+ "backend": str(dist.get_backend()),
+ "rank": int(dist.get_rank()),
+ "world_size": int(dist.get_world_size()),
+ "current_cuda_device": int(torch.cuda.current_device()),
+ }
+ )
+ return report, True
+
+
def _tree_exactness(left: Any, right: Any) -> dict[str, Any]:
"""Compare a retained-state tree directly on device without CPU KV copies."""
tensor_leaves = 0
@@ -477,17 +637,31 @@ def _pair_exact(round_report: Mapping[str, Any], pair: str) -> bool:
@torch.inference_mode()
def _run(args: argparse.Namespace, base: Any) -> dict[str, Any]:
- if not torch.cuda.is_available():
- raise RuntimeError("persistent CUDA-Graph diagnostic requires CUDA")
+ device_context = _activate_cuda_device_context(args, _device_context_plan(args))
image = Image.open(args.image).convert("RGB")
pipeline = None
+ nccl_context: dict[str, Any] = {
+ "requested": bool(args.nccl_single_rank),
+ "initialized_by_tool": False,
+ }
+ nccl_owned = False
cohorts: dict[str, list[Any]] = {"regular": [], "static": [], "graph": []}
try:
pipeline = base._make_pipeline(args)
device = torch.device(pipeline.device)
- if device.type != "cuda":
- raise RuntimeError(f"persistent CUDA-Graph diagnostic requires CUDA, got {pipeline.device!r}")
+ expected_device = torch.device("cuda", args.device_id)
+ if device != expected_device:
+ raise RuntimeError(
+ "pipeline device does not match the requested logical CUDA device: "
+ f"expected {expected_device}, got {pipeline.device!r}"
+ )
pipeline.preload_models()
+ if int(torch.cuda.current_device()) != args.device_id:
+ raise RuntimeError(
+ "pipeline preload changed the current CUDA device: "
+ f"expected cuda:{args.device_id}, got cuda:{torch.cuda.current_device()}"
+ )
+ nccl_context, nccl_owned = _start_single_rank_nccl(args, device)
pipeline.denoise_stage.configure_cuda_graph(False)
actions = _parse_actions(args.session_actions, args.batch_size, base)
seeds = [args.seed + 9973 * index for index in range(args.batch_size)]
@@ -646,6 +820,8 @@ def _run(args: argparse.Namespace, base: Any) -> dict[str, Any]:
"status": status,
"diagnostic": "ordinary eager vs persistent-static eager vs CUDA Graph across two continuations",
"device": str(device),
+ "execution_context": device_context,
+ "nccl_context": nccl_context,
"batch_size": args.batch_size,
"control_latent_frames": args.control_latent_frames,
"session_actions": actions,
@@ -691,6 +867,10 @@ def _run(args: argparse.Namespace, base: Any) -> dict[str, Any]:
pipeline.close()
except Exception:
pass
+ if nccl_owned and dist.is_initialized():
+ dist.destroy_process_group()
+ elif args.nccl_single_rank and dist.is_initialized():
+ raise RuntimeError("single-rank NCCL diagnostic left an unowned process group initialized")
if torch.cuda.is_available():
torch.cuda.empty_cache()
@@ -705,6 +885,23 @@ def _write_results(output_dir: Path, result: Mapping[str, Any], args: argparse.N
second_graph = graph.get("second_continuation", {}) if isinstance(graph, Mapping) else {}
first_verification = first_graph.get("verification", {}) if isinstance(first_graph, Mapping) else {}
second_verification = second_graph.get("verification", {}) if isinstance(second_graph, Mapping) else {}
+ execution_context = result.get("execution_context", {})
+ if not isinstance(execution_context, Mapping):
+ execution_context = {}
+ nccl_context = result.get("nccl_context", {})
+ if not isinstance(nccl_context, Mapping):
+ nccl_context = {}
+ visible_tokens = execution_context.get("visible_device_tokens")
+ visible_text = ",".join(str(item) for item in visible_tokens) if isinstance(visible_tokens, list) else ""
+ logical = execution_context.get("logical_device_requested", "")
+ physical = execution_context.get("physical_device_token_for_logical_device", "")
+ current = execution_context.get("logical_device_current_after_set", "")
+ nccl_summary = (
+ f"requested={nccl_context.get('requested', False)}, "
+ f"initialized={nccl_context.get('initialized_by_tool', False)}, "
+ f"backend={nccl_context.get('backend', '')}, "
+ f"rank/world={nccl_context.get('rank', '')}/{nccl_context.get('world_size', '')}"
+ )
lines = [
f"# ABot B={args.batch_size} persistent CUDA-Graph three-way diagnostic",
"",
@@ -714,6 +911,14 @@ def _write_results(output_dir: Path, result: Mapping[str, Any], args: argparse.N
"forward_steady_state eagerly."
),
"",
+ "| Process-NCCL compatibility context | Value |",
+ "| --- | --- |",
+ f"| CUDA_VISIBLE_DEVICES | `{visible_text}` |",
+ f"| Selected logical CUDA / mapped physical token | `cuda:{logical}` / `{physical}` |",
+ f"| Current logical CUDA after selection | `cuda:{current}` |",
+ f"| Runtime visible CUDA device count | {execution_context.get('runtime_visible_device_count', '')} |",
+ f"| Single-rank NCCL | {nccl_summary} |",
+ "",
"| Check | First continuation | Second continuation / persistent replay |",
"| --- | --- | --- |",
(
@@ -752,13 +957,47 @@ def _parse_args(base: Any) -> argparse.Namespace:
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--control-latent-frames", type=int, choices=(3,), default=3)
parser.add_argument("--extra-warmup-chunks", type=int, default=0)
- parser.add_argument("--device-id", type=int, default=0)
+ parser.add_argument(
+ "--device-id",
+ type=int,
+ default=0,
+ help="Logical CUDA index after CUDA_VISIBLE_DEVICES remapping (not a physical GPU id).",
+ )
+ parser.add_argument(
+ "--expected-cuda-visible-devices",
+ default=None,
+ help="Require this exact comma-separated CVD mapping, e.g. '4,5,6,7'.",
+ )
+ parser.add_argument(
+ "--expected-visible-device-count",
+ type=int,
+ default=None,
+ help="Require this number of CUDA-visible logical devices at runtime.",
+ )
+ parser.add_argument(
+ "--nccl-single-rank",
+ action="store_true",
+ help="Initialize a world-size-one NCCL group after model preload, mirroring a process-NCCL worker.",
+ )
+ parser.add_argument(
+ "--nccl-init-timeout-seconds",
+ type=float,
+ default=60.0,
+ help="Timeout for the optional single-rank NCCL initialization.",
+ )
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
+ if args.device_id < 0:
+ parser.error("--device-id must be non-negative")
+ if args.expected_visible_device_count is not None and args.expected_visible_device_count < 1:
+ parser.error("--expected-visible-device-count must be positive")
+ if args.nccl_init_timeout_seconds <= 0:
+ parser.error("--nccl-init-timeout-seconds must be positive")
if args.extra_warmup_chunks < 0:
parser.error("--extra-warmup-chunks must be non-negative")
try:
_parse_actions(args.session_actions, args.batch_size, base)
+ _device_context_plan(args)
except (ValueError, argparse.ArgumentTypeError) as exc:
parser.error(str(exc))
if not args.dry_run:
@@ -780,6 +1019,11 @@ def main() -> None:
{
"mode": "dry_run",
"batch_size": args.batch_size,
+ "cuda_device_context_plan": _device_context_plan(args),
+ "nccl_single_rank_requested": args.nccl_single_rank,
+ "nccl_ordering": (
+ "logical device is selected before pipeline load; optional NCCL initializes after preload"
+ ),
"cohorts": ["ordinary_eager", "persistent_static_eager", "cuda_graph"],
"continuations": ["first_capture", "second_persistent_replay"],
"comparisons_after_each": ["per-lane latent", "RGB frame hashes", "full retained state"],
diff --git a/tools/validation/validate_abot_cuda_graph_parity.py b/tools/validation/validate_abot_cuda_graph_parity.py
index e52d382e..cafbc152 100644
--- a/tools/validation/validate_abot_cuda_graph_parity.py
+++ b/tools/validation/validate_abot_cuda_graph_parity.py
@@ -230,6 +230,18 @@ def _graph_verified(metrics: Mapping[str, Any]) -> dict[str, Any]:
def _make_pipeline(args: argparse.Namespace) -> Any:
# Warmup has to be eager for both sessions. The stage is toggled later
# rather than loading two model copies on the same GPU.
+ # The CUDA Graph capture stream follows the *current* CUDA device. Set the
+ # CVD-local device before any model load so this remains correct under
+ # CUDA_VISIBLE_DEVICES=4,5,6,7 with --device-id 1, as in a process-NCCL
+ # worker.
+ if not torch.cuda.is_available():
+ raise RuntimeError("CUDA Graph parity validation requires CUDA, but torch.cuda.is_available() is false")
+ visible_count = int(torch.cuda.device_count())
+ if not 0 <= args.device_id < visible_count:
+ raise RuntimeError(f"--device-id {args.device_id} is out of range for {visible_count} visible CUDA device(s)")
+ torch.cuda.set_device(args.device_id)
+ if int(torch.cuda.current_device()) != args.device_id:
+ raise RuntimeError(f"failed to select requested logical CUDA device {args.device_id}")
original = os.environ.get(_GRAPH_ENV)
os.environ[_GRAPH_ENV] = "0"
try:
From 51c8bc35a60ef9c8aa2066dc45b0c80212ac964e Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Wed, 19 Aug 2026 08:23:18 +0000
Subject: [PATCH 6/8] fix(abot): make optional CUDA Graph hooks safe
Handle minimal CPU pipeline stubs without requiring CUDA Graph methods.
Validate CUDA Graph environment values explicitly and document the attention
and CUDA Graph opt-in controls.
Verification:
- 138 focused ABot serving, scheduler, NCCL, and trace tests passed
- branch Ruff and whitespace checks passed
- four-GPU trace runner syntax check passed
---
docs/en/abot_world.md | 19 ++++++++++
examples/abot_world/_loader.py | 7 +++-
telefuser/pipelines/abot_world/interactive.py | 22 +++++++++---
.../pipelines/abot_world/test_pipeline.py | 35 ++++++++++++++++++-
4 files changed, 77 insertions(+), 6 deletions(-)
diff --git a/docs/en/abot_world.md b/docs/en/abot_world.md
index 167e5e1a..5985d16b 100644
--- a/docs/en/abot_world.md
+++ b/docs/en/abot_world.md
@@ -193,6 +193,25 @@ export TELEFUSER_ABOT_BATCHING_WINDOW_MS=2
export TELEFUSER_ABOT_MAX_DEADLINE_BATCH_WAIT_MS=100
```
+### Experimental execution backends
+
+Both controls below are opt-in and keep their existing defaults when unset:
+
+```bash
+# Default: auto (FlashAttention when available, otherwise PyTorch SDPA).
+# sage_sm90 requires an SM90 H100-class GPU and the matching tf-kernel wheel.
+export TELEFUSER_ABOT_ATTENTION=auto
+
+# Default: false. Enable only after the matching B=1/B=2/B=3 parity check has passed.
+export TELEFUSER_ABOT_CUDA_GRAPH_ENABLED=true
+```
+
+`TELEFUSER_ABOT_CUDA_GRAPH_ENABLED` accepts only `1/true/yes/on` and
+`0/false/no/off`; invalid values fail startup rather than silently disabling the
+optimization. CUDA Graph capture is limited to compatible full-window
+continuations and falls back to eager execution if capture cannot be established.
+It does not change the process-NCCL worker map or the four-worker launch contract.
+
### Offline H100 batch-time priors
Before the first real B=2 dispatch, the generic scheduler would otherwise estimate
diff --git a/examples/abot_world/_loader.py b/examples/abot_world/_loader.py
index 31c61eba..3b9e8ad5 100644
--- a/examples/abot_world/_loader.py
+++ b/examples/abot_world/_loader.py
@@ -39,7 +39,12 @@ def _env_flag(name: str, default: bool = False) -> bool:
value = os.environ.get(name)
if value is None:
return default
- return value.strip().lower() in {"1", "true", "yes", "on"}
+ normalized = value.strip().lower()
+ if normalized in {"1", "true", "yes", "on"}:
+ return True
+ if normalized in {"0", "false", "no", "off"}:
+ return False
+ raise ValueError(f"{name} must be a boolean")
def _attention_backend(device_id: int) -> AttnImplType:
diff --git a/telefuser/pipelines/abot_world/interactive.py b/telefuser/pipelines/abot_world/interactive.py
index f433c852..7f82c294 100644
--- a/telefuser/pipelines/abot_world/interactive.py
+++ b/telefuser/pipelines/abot_world/interactive.py
@@ -250,7 +250,7 @@ def generate_next_blocks(
# First chunks initialize per-session caches and stay eager.
for session in sessions:
self._release_cuda_graph(session.session_id)
- self.denoise_stage.record_cuda_graph_not_used()
+ self._record_cuda_graph_not_used()
cache_collate_started_at = time.monotonic()
original_global_ends = [
[int(layer["global_end_index"].item()) for layer in session.self_cache] for session in sessions
@@ -327,7 +327,7 @@ def generate_next_blocks(
# deliberately not eligible until the full local window exists.
assert self_cache is not None and cross_cache is not None
if direct_session_cache:
- self.denoise_stage.record_cuda_graph_not_used()
+ self._record_cuda_graph_not_used()
latents = self.denoise_stage._denoise_block(
noises[0].to(dtype=self.torch_dtype),
sessions[0].prompt_emb,
@@ -403,7 +403,7 @@ def generate_next_blocks(
"cache_scatter_seconds": cache_scatter_seconds,
"vae_decode_seconds": decode_seconds,
**self.taew_decode_stage.last_decode_metrics(),
- **self.denoise_stage.last_cuda_graph_metrics(),
+ **self._last_cuda_graph_metrics(),
"postprocess_seconds": time.monotonic() - postprocess_started_at,
"total_seconds": time.monotonic() - batch_started_at,
}
@@ -623,10 +623,24 @@ def last_stage_metrics(self) -> dict[str, float | int]:
def _release_cuda_graph(self, session_id: str) -> None:
"""Drop optional graph state without coupling test/minimal stages to it."""
- release = getattr(self.denoise_stage, "release_cuda_graph", None)
+ stage = getattr(self, "denoise_stage", None)
+ release = getattr(stage, "release_cuda_graph", None)
if callable(release):
release(session_id)
+ def _record_cuda_graph_not_used(self) -> None:
+ """Record an eager-only dispatch when the optional stage hook is present."""
+ stage = getattr(self, "denoise_stage", None)
+ record = getattr(stage, "record_cuda_graph_not_used", None)
+ if callable(record):
+ record()
+
+ def _last_cuda_graph_metrics(self) -> dict[str, float | int]:
+ """Return optional CUDA-graph facts without requiring a minimal stage stub."""
+ stage = getattr(self, "denoise_stage", None)
+ metrics = getattr(stage, "last_cuda_graph_metrics", None)
+ return dict(metrics()) if callable(metrics) else {}
+
def suspend_interactive_session(self, session: ABotWorldInteractiveSession) -> None:
"""Move all material session tensors to CPU at a chunk boundary."""
with self._execution_lock, session.lock:
diff --git a/tests/unit/pipelines/abot_world/test_pipeline.py b/tests/unit/pipelines/abot_world/test_pipeline.py
index 0131c032..24af21d9 100644
--- a/tests/unit/pipelines/abot_world/test_pipeline.py
+++ b/tests/unit/pipelines/abot_world/test_pipeline.py
@@ -3,6 +3,7 @@
import pytest
import torch
+from examples.abot_world._loader import _attention_backend, _env_flag
from telefuser.core.config import ModelRuntimeConfig, ParallelConfig
from telefuser.core.module_manager import ModuleManager
from telefuser.pipelines.abot_world import ABotWorldPipeline
@@ -10,10 +11,42 @@
def test_cuda_graph_configuration_is_opt_in() -> None:
-
assert ABotWorldPipelineConfig().cuda_graph_enabled is False
+@pytest.mark.parametrize(
+ ("raw_value", "expected"),
+ [
+ ("1", True),
+ ("true", True),
+ ("yes", True),
+ ("on", True),
+ ("0", False),
+ ("false", False),
+ ],
+)
+def test_cuda_graph_environment_flag_is_explicit(
+ monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: bool
+) -> None:
+ monkeypatch.setenv("TELEFUSER_ABOT_CUDA_GRAPH_ENABLED", raw_value)
+
+ assert _env_flag("TELEFUSER_ABOT_CUDA_GRAPH_ENABLED") is expected
+
+
+def test_cuda_graph_environment_flag_rejects_invalid_value(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("TELEFUSER_ABOT_CUDA_GRAPH_ENABLED", "sometimes")
+
+ with pytest.raises(ValueError, match="TELEFUSER_ABOT_CUDA_GRAPH_ENABLED must be a boolean"):
+ _env_flag("TELEFUSER_ABOT_CUDA_GRAPH_ENABLED")
+
+
+def test_attention_environment_rejects_unknown_backend(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("TELEFUSER_ABOT_ATTENTION", "unknown")
+
+ with pytest.raises(ValueError, match="Unsupported TELEFUSER_ABOT_ATTENTION"):
+ _attention_backend(0)
+
+
def test_action_context_uses_official_wasd_ijkl_channel_layout() -> None:
action = ABotWorldPipeline.build_action_context(
{"W": True, "D": True, "L": True},
From 69588438c334015752b30e205043422689cc08fa Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Wed, 19 Aug 2026 08:23:22 +0000
Subject: [PATCH 7/8] style(abot): format serving and validation changes
Apply Ruff formatting and import ordering to the ABot serving,
NCCL, scheduler, and validation changes.
Verification:
- Ruff check and format check passed
- git diff --check passed
---
.../abot_world/abot_world_interactive_web.py | 4 +-
telefuser/entrypoints/cli/main.py | 4 +-
telefuser/models/taew2_2.py | 179 +++++++++++++-----
telefuser/models/wan22_video_vae.py | 4 +-
telefuser/pipelines/abot_world/taew_vae.py | 41 +---
telefuser/service/livekit/nccl_transfer.py | 12 +-
telefuser/service/livekit/runtime.py | 25 +--
telefuser/service/livekit/scheduler.py | 4 +-
telefuser/service/livekit/turboserve.py | 67 +++++--
telefuser/service/livekit/worker_pool.py | 4 +-
.../pipelines/abot_world/test_migration.py | 5 +-
.../livekit/test_multi_session_capacity.py | 4 +-
.../service/livekit/test_nccl_transfer.py | 4 +-
.../test_capture_abot_serving_metrics.py | 7 +-
tools/validation/benchmark_abot_microbatch.py | 16 +-
.../benchmark_abot_turboserve_concurrent.py | 14 +-
tools/validation/run_abot_batch_scaling.py | 4 +-
17 files changed, 243 insertions(+), 155 deletions(-)
diff --git a/examples/abot_world/abot_world_interactive_web.py b/examples/abot_world/abot_world_interactive_web.py
index 74d63577..563e2eca 100644
--- a/examples/abot_world/abot_world_interactive_web.py
+++ b/examples/abot_world/abot_world_interactive_web.py
@@ -806,7 +806,9 @@ def main() -> None:
parser.add_argument("--height", type=int, default=480)
parser.add_argument("--width", type=int, default=832)
parser.add_argument("--latent-frames", type=int, default=31)
- parser.add_argument("--fps", type=int, default=8, help="Playback and downloaded-video FPS; 8 is the real-time target.")
+ parser.add_argument(
+ "--fps", type=int, default=8, help="Playback and downloaded-video FPS; 8 is the real-time target."
+ )
parser.add_argument(
"--control-latent-frames",
type=int,
diff --git a/telefuser/entrypoints/cli/main.py b/telefuser/entrypoints/cli/main.py
index 0a2f40ee..c0232e1b 100644
--- a/telefuser/entrypoints/cli/main.py
+++ b/telefuser/entrypoints/cli/main.py
@@ -160,9 +160,7 @@ def serve(
)
@click.option("--enable-autoscaling", is_flag=True, help="Dynamically load workers from the configured GPU map")
@click.option("--autoscaling-min-workers", default=1, type=int, help="Initially loaded worker replicas")
-@click.option(
- "--autoscaling-target-utilization", default=0.75, type=float, help="Target retained-session utilization"
-)
+@click.option("--autoscaling-target-utilization", default=0.75, type=float, help="Target retained-session utilization")
@click.option("--autoscaling-hysteresis", default=0.10, type=float, help="Scale decision hysteresis band")
@click.option("--autoscaling-cooldown-seconds", default=30.0, type=float, help="Minimum time between scales")
@click.option("--autoscaling-interval-seconds", default=5.0, type=float, help="Autoscaling control interval")
diff --git a/telefuser/models/taew2_2.py b/telefuser/models/taew2_2.py
index f7bc1165..1cf9054c 100644
--- a/telefuser/models/taew2_2.py
+++ b/telefuser/models/taew2_2.py
@@ -8,49 +8,61 @@
Tiny AutoEncoder for Hunyuan Video
(DNN for encoding / decoding videos to Hunyuan Video's latent space)
"""
+from collections import namedtuple
+
import torch
import torch.nn as nn
import torch.nn.functional as F
from tqdm.auto import tqdm
-from collections import namedtuple
TWorkItem = namedtuple("TWorkItem", ("input_tensor", "block_index"))
+
def conv(n_in, n_out, **kwargs):
return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs)
+
class Clamp(nn.Module):
def forward(self, x):
return torch.tanh(x / 3) * 3
+
class MemBlock(nn.Module):
def __init__(self, n_in, n_out):
super().__init__()
- self.conv = nn.Sequential(conv(n_in * 2, n_out), nn.ReLU(inplace=True), conv(n_out, n_out), nn.ReLU(inplace=True), conv(n_out, n_out))
+ self.conv = nn.Sequential(
+ conv(n_in * 2, n_out), nn.ReLU(inplace=True), conv(n_out, n_out), nn.ReLU(inplace=True), conv(n_out, n_out)
+ )
self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
self.act = nn.ReLU(inplace=True)
+
def forward(self, x, past):
return self.act(self.conv(torch.cat([x, past], 1)) + self.skip(x))
+
class TPool(nn.Module):
def __init__(self, n_f, stride):
super().__init__()
self.stride = stride
- self.conv = nn.Conv2d(n_f*stride,n_f, 1, bias=False)
+ self.conv = nn.Conv2d(n_f * stride, n_f, 1, bias=False)
+
def forward(self, x):
_NT, C, H, W = x.shape
return self.conv(x.reshape(-1, self.stride * C, H, W))
+
class TGrow(nn.Module):
def __init__(self, n_f, stride):
super().__init__()
self.stride = stride
- self.conv = nn.Conv2d(n_f, n_f*stride, 1, bias=False)
+ self.conv = nn.Conv2d(n_f, n_f * stride, 1, bias=False)
+
def forward(self, x):
_NT, C, H, W = x.shape
x = self.conv(x)
return x.reshape(-1, C, H, W)
+
def apply_model_with_memblocks_parallel(model, x, show_progress_bar):
"""
Apply a sequential model with memblocks to the given input,
@@ -65,7 +77,7 @@ def apply_model_with_memblocks_parallel(model, x, show_progress_bar):
"""
assert x.ndim == 5, f"TAEHV operates on NTCHW tensors, but got {x.ndim}-dim tensor"
N, T, C, H, W = x.shape
- x = x.reshape(N*T, C, H, W)
+ x = x.reshape(N * T, C, H, W)
# parallel over input timesteps, iterate over blocks
for b in tqdm(model, disable=not show_progress_bar):
@@ -74,7 +86,7 @@ def apply_model_with_memblocks_parallel(model, x, show_progress_bar):
T = NT // N
_x = x.reshape(N, T, C, H, W)
# pad with zeros along time axis (i.e. empty memory), slice
- block_memory = F.pad(_x, (0,0,0,0,0,0,1,0), value=0)[:,:T].reshape(x.shape)
+ block_memory = F.pad(_x, (0, 0, 0, 0, 0, 0, 1, 0), value=0)[:, :T].reshape(x.shape)
x = b(x, block_memory)
else:
x = b(x)
@@ -82,6 +94,7 @@ def apply_model_with_memblocks_parallel(model, x, show_progress_bar):
T = NT // N
return x.view(N, T, C, H, W)
+
def apply_model_with_memblocks_sequential_single_step(model, memory, work_queue, progress_bar=None):
"""
Process the work queue (a graph traversal over blocks and timesteps)
@@ -104,7 +117,7 @@ def apply_model_with_memblocks_sequential_single_step(model, memory, work_queue,
else:
xt_new = b(xt, memory[i])
memory[i] = xt
- work_queue.insert(0, TWorkItem(xt_new, i+1))
+ work_queue.insert(0, TWorkItem(xt_new, i + 1))
elif isinstance(b, TPool):
# pool blocks accumulate inputs until they have enough to pool
if memory[i] is None:
@@ -114,19 +127,20 @@ def apply_model_with_memblocks_sequential_single_step(model, memory, work_queue,
raise ValueError(f"TPool memory overflow: {len(memory[i])} items for stride {b.stride}")
elif len(memory[i]) == b.stride:
N, C, H, W = xt.shape
- xt = b(torch.cat(memory[i], 1).view(N*b.stride, C, H, W))
+ xt = b(torch.cat(memory[i], 1).view(N * b.stride, C, H, W))
memory[i] = []
- work_queue.insert(0, TWorkItem(xt, i+1))
+ work_queue.insert(0, TWorkItem(xt, i + 1))
elif isinstance(b, TGrow):
xt = b(xt)
NT, C, H, W = xt.shape
- for xt_next in reversed(xt.view(NT//b.stride, b.stride*C, H, W).chunk(b.stride, 1)):
- work_queue.insert(0, TWorkItem(xt_next, i+1))
+ for xt_next in reversed(xt.view(NT // b.stride, b.stride * C, H, W).chunk(b.stride, 1)):
+ work_queue.insert(0, TWorkItem(xt_next, i + 1))
else:
xt = b(xt)
- work_queue.insert(0, TWorkItem(xt, i+1))
+ work_queue.insert(0, TWorkItem(xt, i + 1))
return None
+
def apply_model_with_memblocks_sequential(model, x, show_progress_bar):
"""
Apply a sequential model with memblocks to the given input,
@@ -151,6 +165,7 @@ def apply_model_with_memblocks_sequential(model, x, show_progress_bar):
progress_bar.close()
return torch.cat(out, 1)
+
def apply_model_with_memblocks(model, x, parallel, show_progress_bar):
"""
Apply a sequential model with memblocks to the given input.
@@ -168,15 +183,27 @@ def apply_model_with_memblocks(model, x, parallel, show_progress_bar):
else:
return apply_model_with_memblocks_sequential(model, x, show_progress_bar)
+
class TAEHV(nn.Module):
- def __init__(self, checkpoint_path="taehv.pth", encoder_time_downscale=(True, True, False), decoder_time_upscale=(False, True, True), decoder_space_upscale=(True, True, True), patch_size=1, latent_channels=16):
+ def __init__(
+ self,
+ checkpoint_path="taehv.pth",
+ encoder_time_downscale=(True, True, False),
+ decoder_time_upscale=(False, True, True),
+ decoder_space_upscale=(True, True, True),
+ patch_size=1,
+ latent_channels=16,
+ ):
"""Initialize pretrained TAEHV from the given checkpoint.
Arg:
- checkpoint_path: path to weight file to load. taehv.pth for Hunyuan, taew2_1.pth for Wan 2.1.
+ checkpoint_path: Path to the weight file. Use taehv.pth for
+ Hunyuan or taew2_1.pth for Wan 2.1.
encoder_time_downscale: whether temporal downsampling is enabled for each block.
- decoder_time_upscale: whether temporal upsampling is enabled for each block. upsampling can be disabled for a cheaper preview.
- decoder_space_upscale: whether spatial upsampling is enabled for each block. upsampling can be disabled for a cheaper preview.
+ decoder_time_upscale: Whether temporal upsampling is enabled for
+ each block. It can be disabled for a cheaper preview.
+ decoder_space_upscale: Whether spatial upsampling is enabled for
+ each block. It can be disabled for a cheaper preview.
patch_size: input/output pixelshuffle patch-size for this model.
latent_channels: number of latent channels (z dim) for this model.
"""
@@ -191,30 +218,68 @@ def __init__(self, checkpoint_path="taehv.pth", encoder_time_downscale=(True, Tr
self.patch_size, self.latent_channels = 2, 48
if checkpoint_path is not None and "taehv1_5" in checkpoint_path:
self.patch_size, self.latent_channels = 2, 32
- if checkpoint_path is not None and "taeltx" in checkpoint_path: # same for both 2 and 2.3
- self.patch_size, self.latent_channels, encoder_time_downscale, decoder_time_upscale = 4, 128, (True, True, True), (True, True, True)
+ if checkpoint_path is not None and "taeltx" in checkpoint_path: # same for both 2 and 2.3
+ self.patch_size, self.latent_channels, encoder_time_downscale, decoder_time_upscale = (
+ 4,
+ 128,
+ (True, True, True),
+ (True, True, True),
+ )
self.encoder = nn.Sequential(
- conv(self.image_channels*self.patch_size**2, 64), nn.ReLU(inplace=True),
- TPool(64, 2 if encoder_time_downscale[0] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
- TPool(64, 2 if encoder_time_downscale[1] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
- TPool(64, 2 if encoder_time_downscale[2] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
+ conv(self.image_channels * self.patch_size**2, 64),
+ nn.ReLU(inplace=True),
+ TPool(64, 2 if encoder_time_downscale[0] else 1),
+ conv(64, 64, stride=2, bias=False),
+ MemBlock(64, 64),
+ MemBlock(64, 64),
+ MemBlock(64, 64),
+ TPool(64, 2 if encoder_time_downscale[1] else 1),
+ conv(64, 64, stride=2, bias=False),
+ MemBlock(64, 64),
+ MemBlock(64, 64),
+ MemBlock(64, 64),
+ TPool(64, 2 if encoder_time_downscale[2] else 1),
+ conv(64, 64, stride=2, bias=False),
+ MemBlock(64, 64),
+ MemBlock(64, 64),
+ MemBlock(64, 64),
conv(64, self.latent_channels),
)
n_f = [256, 128, 64, 64]
self.decoder = nn.Sequential(
- Clamp(), conv(self.latent_channels, n_f[0]), nn.ReLU(inplace=True),
- MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), TGrow(n_f[0], 2 if decoder_time_upscale[0] else 1), conv(n_f[0], n_f[1], bias=False),
- MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1), TGrow(n_f[1], 2 if decoder_time_upscale[1] else 1), conv(n_f[1], n_f[2], bias=False),
- MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Upsample(scale_factor=2 if decoder_space_upscale[2] else 1), TGrow(n_f[2], 2 if decoder_time_upscale[2] else 1), conv(n_f[2], n_f[3], bias=False),
- nn.ReLU(inplace=True), conv(n_f[3], self.image_channels*self.patch_size**2),
+ Clamp(),
+ conv(self.latent_channels, n_f[0]),
+ nn.ReLU(inplace=True),
+ MemBlock(n_f[0], n_f[0]),
+ MemBlock(n_f[0], n_f[0]),
+ MemBlock(n_f[0], n_f[0]),
+ nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1),
+ TGrow(n_f[0], 2 if decoder_time_upscale[0] else 1),
+ conv(n_f[0], n_f[1], bias=False),
+ MemBlock(n_f[1], n_f[1]),
+ MemBlock(n_f[1], n_f[1]),
+ MemBlock(n_f[1], n_f[1]),
+ nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1),
+ TGrow(n_f[1], 2 if decoder_time_upscale[1] else 1),
+ conv(n_f[1], n_f[2], bias=False),
+ MemBlock(n_f[2], n_f[2]),
+ MemBlock(n_f[2], n_f[2]),
+ MemBlock(n_f[2], n_f[2]),
+ nn.Upsample(scale_factor=2 if decoder_space_upscale[2] else 1),
+ TGrow(n_f[2], 2 if decoder_time_upscale[2] else 1),
+ conv(n_f[2], n_f[3], bias=False),
+ nn.ReLU(inplace=True),
+ conv(n_f[3], self.image_channels * self.patch_size**2),
)
# computed properties
- self.t_downscale = 2**sum(t.stride == 2 for t in self.encoder if isinstance(t, TPool))
- self.t_upscale = 2**sum(t.stride == 2 for t in self.decoder if isinstance(t, TGrow))
+ self.t_downscale = 2 ** sum(t.stride == 2 for t in self.encoder if isinstance(t, TPool))
+ self.t_upscale = 2 ** sum(t.stride == 2 for t in self.decoder if isinstance(t, TGrow))
self.frames_to_trim = self.t_upscale - 1
if checkpoint_path is not None:
- self.load_state_dict(self.patch_tgrow_layers(torch.load(checkpoint_path, map_location="cpu", weights_only=True)))
+ self.load_state_dict(
+ self.patch_tgrow_layers(torch.load(checkpoint_path, map_location="cpu", weights_only=True))
+ )
def patch_tgrow_layers(self, sd):
"""Patch TGrow layers to use a smaller kernel if needed.
@@ -228,12 +293,13 @@ def patch_tgrow_layers(self, sd):
key = f"decoder.{i}.conv.weight"
if sd[key].shape[0] > new_sd[key].shape[0]:
# take the last-timestep output channels
- sd[key] = sd[key][-new_sd[key].shape[0]:]
+ sd[key] = sd[key][-new_sd[key].shape[0] :]
return sd
def preprocess_input_frames(self, x):
"""Preprocess RGB input frames prior to the main encoder sequence."""
- if self.patch_size > 1: x = F.pixel_unshuffle(x, self.patch_size)
+ if self.patch_size > 1:
+ x = F.pixel_unshuffle(x, self.patch_size)
return x
def encode_video(self, x, parallel=True, show_progress_bar=True):
@@ -256,7 +322,8 @@ def encode_video(self, x, parallel=True, show_progress_bar=True):
def postprocess_output_frames(self, x):
"""Postprocess RGB frames after the main decoder sequence."""
- if self.patch_size > 1: x = F.pixel_shuffle(x, self.patch_size)
+ if self.patch_size > 1:
+ x = F.pixel_shuffle(x, self.patch_size)
return x.clamp_(0, 1)
def decode_video(self, x, parallel=True, show_progress_bar=True):
@@ -277,7 +344,8 @@ def decode_video(self, x, parallel=True, show_progress_bar=True):
# this still doesn't have correct temporal alignment for certain frame counts
# (cogvideox seems to pad at the start?), but for multiple-of-4 it's fine.
return x
- return x[:, self.frames_to_trim:]
+ return x[:, self.frames_to_trim :]
+
class StreamingTAEHV(nn.Module):
def __init__(self, taehv):
@@ -331,7 +399,8 @@ def encode(self, x=None):
self.encoder_work_queue.extend(TWorkItem(xt, 0) for xt in x.unbind(1))
self.n_frames_encoded += x.shape[1]
xt = apply_model_with_memblocks_sequential_single_step(
- self.taehv.encoder, self.encoder_memory, self.encoder_work_queue)
+ self.taehv.encoder, self.encoder_memory, self.encoder_work_queue
+ )
return xt
def decode(self, x=None):
@@ -350,7 +419,9 @@ def decode(self, x=None):
Returns: N1CHW decoded RGB frame tensor, or None if the queue needs more input.
"""
if x is not None:
- assert x.ndim == 5 and x.shape[2] == self.taehv.latent_channels, f"Expected NTCHW latents but got {x.shape=}"
+ assert x.ndim == 5 and x.shape[2] == self.taehv.latent_channels, (
+ f"Expected NTCHW latents but got {x.shape=}"
+ )
self.decoder_work_queue.extend(TWorkItem(xt, 0) for xt in x.unbind(1))
imgs = []
@@ -360,12 +431,13 @@ def decode(self, x=None):
first_chunk = False
while True:
xt = apply_model_with_memblocks_sequential_single_step(
- self.taehv.decoder, self.decoder_memory, self.decoder_work_queue)
+ self.taehv.decoder, self.decoder_memory, self.decoder_work_queue
+ )
if xt is not None:
imgs.append(self.taehv.postprocess_output_frames(xt))
else:
if first_chunk:
- return torch.cat(imgs, 1)[:, self.taehv.frames_to_trim:]
+ return torch.cat(imgs, 1)[:, self.taehv.frames_to_trim :]
else:
return torch.cat(imgs, 1)
self.n_frames_decoded += 1
@@ -411,42 +483,54 @@ def flush(self):
frames.extend(self.flush_decoder())
return frames
+
@torch.no_grad()
def main():
"""Run TAEHV roundtrip reconstruction on the given video paths."""
import os
import sys
- import cv2 # no highly esteemed deed is commemorated here
+
+ import cv2 # no highly esteemed deed is commemorated here
class VideoTensorReader:
def __init__(self, video_file_path):
self.cap = cv2.VideoCapture(video_file_path)
assert self.cap.isOpened(), f"Could not load {video_file_path}"
self.fps = self.cap.get(cv2.CAP_PROP_FPS)
+
def __iter__(self):
return self
+
def __next__(self):
ret, frame = self.cap.read()
if not ret:
self.cap.release()
raise StopIteration # End of video or error
- return torch.from_numpy(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)).permute(2, 0, 1) # BGR HWC -> RGB CHW
+ return torch.from_numpy(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)).permute(2, 0, 1) # BGR HWC -> RGB CHW
class VideoTensorWriter:
def __init__(self, video_file_path, width_height, fps=30):
- self.writer = cv2.VideoWriter(video_file_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, width_height)
+ self.writer = cv2.VideoWriter(video_file_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, width_height)
assert self.writer.isOpened(), f"Could not create writer for {video_file_path}"
+
def write(self, frame_tensor):
assert frame_tensor.ndim == 3 and frame_tensor.shape[0] == 3, f"{frame_tensor.shape}??"
- self.writer.write(cv2.cvtColor(frame_tensor.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR)) # RGB CHW -> BGR HWC
+ self.writer.write(
+ cv2.cvtColor(frame_tensor.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR)
+ ) # RGB CHW -> BGR HWC
+
def __del__(self):
- if hasattr(self, 'writer'): self.writer.release()
+ if hasattr(self, "writer"):
+ self.writer.release()
dev = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu")
dtype = torch.float16
checkpoint_path = os.getenv("TAEHV_CHECKPOINT_PATH", "taehv.pth")
checkpoint_name = os.path.splitext(os.path.basename(checkpoint_path))[0]
- print(f"Using device \033[31m{dev}\033[0m, dtype \033[32m{dtype}\033[0m, checkpoint \033[34m{checkpoint_name}\033[0m ({checkpoint_path})")
+ print(
+ f"Using device \033[31m{dev}\033[0m, dtype \033[32m{dtype}\033[0m, "
+ f"checkpoint \033[34m{checkpoint_name}\033[0m ({checkpoint_path})"
+ )
taehv = TAEHV(checkpoint_path=checkpoint_path).to(dev, dtype)
for video_path in sys.argv[1:]:
print(f"Processing {video_path}...")
@@ -469,10 +553,13 @@ def __del__(self):
vid_dec = taehv.decode_video(vid_enc, parallel=False)
print(f" Decoded {video_path} -> {vid_dec.shape}")
video_out_path = video_path + f".reconstructed_by_{checkpoint_name}.mp4"
- video_out = VideoTensorWriter(video_out_path, (vid_dec.shape[-1], vid_dec.shape[-2]), fps=int(round(video_in.fps)))
+ video_out = VideoTensorWriter(
+ video_out_path, (vid_dec.shape[-1], vid_dec.shape[-2]), fps=int(round(video_in.fps))
+ )
for frame in vid_dec.clamp_(0, 1).mul_(255).round_().byte().cpu()[0]:
video_out.write(frame)
print(f" Saved to {video_out_path}")
+
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/telefuser/models/wan22_video_vae.py b/telefuser/models/wan22_video_vae.py
index bb15d021..8ca32a35 100644
--- a/telefuser/models/wan22_video_vae.py
+++ b/telefuser/models/wan22_video_vae.py
@@ -1547,9 +1547,7 @@ def cached_decode_batch_withflag(
hidden_states = hidden_states.to(device)
scale = self._get_scale_on_device(device, hidden_states.dtype)
- z = hidden_states / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(
- 1, self.z_dim, 1, 1, 1
- )
+ z = hidden_states / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(1, self.z_dim, 1, 1, 1)
x = self.model.conv2(z)
feat_idx = [0]
outputs: list[torch.Tensor] = []
diff --git a/telefuser/pipelines/abot_world/taew_vae.py b/telefuser/pipelines/abot_world/taew_vae.py
index 36fb651b..a393898c 100644
--- a/telefuser/pipelines/abot_world/taew_vae.py
+++ b/telefuser/pipelines/abot_world/taew_vae.py
@@ -99,10 +99,7 @@ def restore_decode_state(
)
if direct_device_tensors:
expected_device = torch.device(self.device)
- if any(
- not self._matches_device(tensor.device, expected_device)
- for tensor in self._iter_tensors(tree)
- ):
+ if any(not self._matches_device(tensor.device, expected_device) for tensor in self._iter_tensors(tree)):
raise ValueError("NCCL TAeW migration tensors must already reside on the target decoder device")
state = self.create_decode_state()
self._apply_decode_state_tensor_tree(state, tree)
@@ -202,10 +199,7 @@ def _decode_serial_batch(
decoder_latents: torch.Tensor,
states: Sequence[ABotWorldTAEWDecodeState],
) -> torch.Tensor | None:
- decoded_parts = [
- state.stream.decode(decoder_latents[index : index + 1])
- for index, state in enumerate(states)
- ]
+ decoded_parts = [state.stream.decode(decoder_latents[index : index + 1]) for index, state in enumerate(states)]
if all(item is None for item in decoded_parts):
return None
if any(item is None for item in decoded_parts):
@@ -253,9 +247,7 @@ def _combine_decode_states(self, states: Sequence[ABotWorldTAEWDecodeState]) ->
)
for index, item in enumerate(reference.decoder_work_queue)
]
- combined.stream.decoder_memory = self._collate_state_values(
- [state.stream.decoder_memory for state in states]
- )
+ combined.stream.decoder_memory = self._collate_state_values([state.stream.decoder_memory for state in states])
combined.stream.n_frames_decoded = int(reference.n_frames_decoded)
return combined
@@ -267,15 +259,9 @@ def _collate_state_values(cls, values: Sequence[Any]) -> Any:
if first is None:
return None
if isinstance(first, list):
- return [
- cls._collate_state_values([value[index] for value in values])
- for index in range(len(first))
- ]
+ return [cls._collate_state_values([value[index] for value in values]) for index in range(len(first))]
if isinstance(first, tuple):
- return tuple(
- cls._collate_state_values([value[index] for value in values])
- for index in range(len(first))
- )
+ return tuple(cls._collate_state_values([value[index] for value in values]) for index in range(len(first)))
if all(value == first for value in values[1:]):
return first
raise ValueError("TAeW decoder states cannot be collated")
@@ -388,9 +374,7 @@ def _normalise_decode_state_tree(
if not isinstance(decoder_memory, (list, tuple)):
raise TypeError("TAeW decoder_memory snapshot must be a sequence")
if len(decoder_memory) != len(self.taew.decoder):
- raise ValueError(
- "TAeW decoder_memory snapshot does not match the loaded decoder architecture"
- )
+ raise ValueError("TAeW decoder_memory snapshot does not match the loaded decoder architecture")
n_frames_decoded = int(snapshot["n_frames_decoded"])
if n_frames_decoded < 0:
raise ValueError("TAeW n_frames_decoded must be non-negative")
@@ -408,8 +392,7 @@ def _normalise_decode_state_tree(
def _apply_decode_state_tensor_tree(state: ABotWorldTAEWDecodeState, tree: Mapping[str, Any]) -> None:
stream = state.stream
stream.decoder_work_queue = [
- TWorkItem(item["input_tensor"], int(item["block_index"]))
- for item in tree["decoder_work_queue"]
+ TWorkItem(item["input_tensor"], int(item["block_index"])) for item in tree["decoder_work_queue"]
]
stream.decoder_memory = list(tree["decoder_memory"])
stream.n_frames_decoded = int(tree["n_frames_decoded"])
@@ -428,15 +411,9 @@ def _copy_tensor_tree(
tensor = tensor.to(device)
return tensor.clone() if clone_tensors else tensor
if isinstance(value, list):
- return [
- cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors)
- for item in value
- ]
+ return [cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors) for item in value]
if isinstance(value, tuple):
- return tuple(
- cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors)
- for item in value
- )
+ return tuple(cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors) for item in value)
if isinstance(value, dict):
return {
key: cls._copy_tensor_tree(item, device=device, clone_tensors=clone_tensors)
diff --git a/telefuser/service/livekit/nccl_transfer.py b/telefuser/service/livekit/nccl_transfer.py
index d12cf912..566c83e3 100644
--- a/telefuser/service/livekit/nccl_transfer.py
+++ b/telefuser/service/livekit/nccl_transfer.py
@@ -13,7 +13,9 @@
import torch.distributed as dist
-def flatten_tensor_tree(value: Any, *, path: tuple[Any, ...] = ()) -> tuple[Any, list[dict[str, Any]], dict[tuple[Any, ...], torch.Tensor]]:
+def flatten_tensor_tree(
+ value: Any, *, path: tuple[Any, ...] = ()
+) -> tuple[Any, list[dict[str, Any]], dict[tuple[Any, ...], torch.Tensor]]:
"""Separate a nested tree into scalar skeleton, tensor manifest, and leaves."""
manifest: list[dict[str, Any]] = []
leaves: dict[tuple[Any, ...], torch.Tensor] = {}
@@ -41,9 +43,13 @@ def visit(item: Any, item_path: tuple[Any, ...]) -> Any:
return visit(value, path), manifest, leaves
-def allocate_tensor_tree_leaves(manifest: list[dict[str, Any]], device: torch.device) -> dict[tuple[Any, ...], torch.Tensor]:
+def allocate_tensor_tree_leaves(
+ manifest: list[dict[str, Any]], device: torch.device
+) -> dict[tuple[Any, ...], torch.Tensor]:
"""Allocate target GPU tensors from a source manifest."""
- dtype_table = {name.removeprefix("torch."): value for name, value in vars(torch).items() if isinstance(value, torch.dtype)}
+ dtype_table = {
+ name.removeprefix("torch."): value for name, value in vars(torch).items() if isinstance(value, torch.dtype)
+ }
leaves: dict[tuple[Any, ...], torch.Tensor] = {}
for entry in manifest:
dtype_name = str(entry["dtype"])
diff --git a/telefuser/service/livekit/runtime.py b/telefuser/service/livekit/runtime.py
index 186bc0f4..4efdeff9 100644
--- a/telefuser/service/livekit/runtime.py
+++ b/telefuser/service/livekit/runtime.py
@@ -138,8 +138,10 @@ async def start(self) -> None:
if self._closed:
raise RuntimeError("LiveKit runtime is already closed")
worker_groups = self.config.worker_gpu_groups()
- if self.config.worker_mode == "process" and self.config.num_workers > 1 and any(
- not group for group in worker_groups
+ if (
+ self.config.worker_mode == "process"
+ and self.config.num_workers > 1
+ and any(not group for group in worker_groups)
):
raise ValueError("worker_gpu_map is required for multiple process workers")
groups = [gpu_id for group in worker_groups for gpu_id in group]
@@ -443,9 +445,7 @@ def _create_worker_pool(self) -> WorkerPool:
for state in self.scheduler.workers()
]
pool_type = (
- NCCLProcessLiveKitWorkerPool
- if self.config.worker_mode == "process-nccl"
- else ProcessLiveKitWorkerPool
+ NCCLProcessLiveKitWorkerPool if self.config.worker_mode == "process-nccl" else ProcessLiveKitWorkerPool
)
return pool_type(
specs,
@@ -456,9 +456,7 @@ def _create_worker_pool(self) -> WorkerPool:
initial_workers=initial_workers,
)
worker_states = self.scheduler.workers()
- backends = {
- state.worker_id: LiveKitPipelineAdapter(security_level=security_level) for state in worker_states
- }
+ backends = {state.worker_id: LiveKitPipelineAdapter(security_level=security_level) for state in worker_states}
router = TurboServePipelineRouter(backends)
workers: dict[str, LiveKitWorker] = {}
for worker_state in worker_states:
@@ -468,9 +466,7 @@ def _create_worker_pool(self) -> WorkerPool:
pipeline_file=self.pipeline_file,
token_service=self.token_service,
event_sink=self,
- pipeline_adapter=router.worker_view(
- worker_state.worker_id, gpu_ids=worker_state.gpu_ids or None
- ),
+ pipeline_adapter=router.worker_view(worker_state.worker_id, gpu_ids=worker_state.gpu_ids or None),
gpu_num=max(1, len(worker_state.gpu_ids)),
gpu_ids=worker_state.gpu_ids or None,
)
@@ -594,11 +590,7 @@ async def _turboserve_control_once(self) -> None:
migrations = 0
for session_id, target_worker_id in decision.placement.items():
record = self.registry.require(session_id)
- if (
- target_worker_id is None
- or target_worker_id == record.worker_id
- or record.pipeline_session_id is None
- ):
+ if target_worker_id is None or target_worker_id == record.worker_id or record.pipeline_session_id is None:
continue
try:
await self.migrate_session(session_id, target_worker_id)
@@ -625,6 +617,7 @@ async def _turboserve_control_once(self) -> None:
self._last_migration_error = None
"""Aggregate live ABot control activity when the pool can expose it."""
+
def _workload_snapshot(self) -> dict[str, float | int]:
snapshot = getattr(self.worker_pool, "turboserve_snapshot", None)
routing = snapshot() if callable(snapshot) else None
diff --git a/telefuser/service/livekit/scheduler.py b/telefuser/service/livekit/scheduler.py
index 6ee5bf4a..5e08fed1 100644
--- a/telefuser/service/livekit/scheduler.py
+++ b/telefuser/service/livekit/scheduler.py
@@ -156,9 +156,7 @@ def drain_queue(self) -> list[SchedulerAdmission]:
worker.room_name = queued.room_name
worker.last_heartbeat_at = utc_timestamp()
admissions.append(
- SchedulerAdmission(
- status="assigned", worker_id=worker.worker_id, session_id=queued.session_id
- )
+ SchedulerAdmission(status="assigned", worker_id=worker.worker_id, session_id=queued.session_id)
)
return admissions
diff --git a/telefuser/service/livekit/turboserve.py b/telefuser/service/livekit/turboserve.py
index 5e45c854..06b57a01 100644
--- a/telefuser/service/livekit/turboserve.py
+++ b/telefuser/service/livekit/turboserve.py
@@ -411,6 +411,7 @@ def release(self, session_id: str) -> None:
self._pending.pop(session_id, None)
self._owners.pop(session_id, None)
+
# The classes below intentionally mirror the closed-loop scheduler in the
# TurboServe reference implementation. The older controllers above are kept
# for API compatibility with the first TeleFuser prototype.
@@ -463,9 +464,7 @@ class TurboServeSchedulingSnapshot:
current_workers: int
worker_order: tuple[str, ...]
capacity_per_worker: int
- runtime_calibration: TurboServeRuntimeCalibration = field(
- default_factory=TurboServeRuntimeCalibration
- )
+ runtime_calibration: TurboServeRuntimeCalibration = field(default_factory=TurboServeRuntimeCalibration)
@dataclass(frozen=True)
@@ -500,14 +499,10 @@ class TurboServeLatencyModel:
}
)
- def migration_cost_ms(
- self, session: TurboServeSessionView, calibration: TurboServeRuntimeCalibration
- ) -> float:
+ def migration_cost_ms(self, session: TurboServeSessionView, calibration: TurboServeRuntimeCalibration) -> float:
if calibration.average_migration_total_ms > 0:
return calibration.average_migration_total_ms
- return self.migration_alpha_ms + session.state_size_mb / max(
- 1e-9, self.migration_bandwidth_mb_per_ms
- )
+ return self.migration_alpha_ms + session.state_size_mb / max(1e-9, self.migration_bandwidth_mb_per_ms)
def session_latency_ms(
self,
@@ -523,7 +518,6 @@ def session_latency_ms(
compute *= self.resolution_factors.get(session.resolution, 1.0)
compute *= max(
self.min_frame_factor,
-
(max(1, session.frame_count) / self.frame_reference_count) ** self.frame_exponent,
)
return (
@@ -537,7 +531,9 @@ def session_latency_ms(
class TurboServeClusterScheduler:
"""Source-aligned closed-loop budget and migration-aware placement."""
- def __init__(self, config: TurboServeSchedulerConfig | None = None, latency_model: TurboServeLatencyModel | None = None) -> None:
+ def __init__(
+ self, config: TurboServeSchedulerConfig | None = None, latency_model: TurboServeLatencyModel | None = None
+ ) -> None:
self.config = config or TurboServeSchedulerConfig()
self.latency_model = latency_model or TurboServeLatencyModel()
self._scale_in_target: int | None = None
@@ -546,7 +542,15 @@ def __init__(self, config: TurboServeSchedulerConfig | None = None, latency_mode
def decide(self, snapshot: TurboServeSchedulingSnapshot) -> TurboServeSchedulingDecision:
budget, action = self._autoscale_budget(snapshot)
placement, metadata = self._place_at_budget(snapshot, budget)
- metadata.update({"scheduler": "turboserve", "autoscale_action": action, "worker_budget": budget, "active_sessions": sum(session.active for session in snapshot.sessions.values()), "target_utilization": self.config.target_utilization})
+ metadata.update(
+ {
+ "scheduler": "turboserve",
+ "autoscale_action": action,
+ "worker_budget": budget,
+ "active_sessions": sum(session.active for session in snapshot.sessions.values()),
+ "target_utilization": self.config.target_utilization,
+ }
+ )
return TurboServeSchedulingDecision(budget, placement, metadata)
def _autoscale_budget(self, snapshot: TurboServeSchedulingSnapshot) -> tuple[int, str]:
@@ -587,7 +591,9 @@ def _target_budget(self, active: int, capacity: int, maximum: int) -> int:
def _clamp(self, value: int, maximum: int) -> int:
return min(maximum, self.config.max_workers, max(self.config.min_workers, int(value)))
- def _place_at_budget(self, snapshot: TurboServeSchedulingSnapshot, budget: int) -> tuple[dict[str, str | None], dict[str, object]]:
+ def _place_at_budget(
+ self, snapshot: TurboServeSchedulingSnapshot, budget: int
+ ) -> tuple[dict[str, str | None], dict[str, object]]:
workers = tuple(snapshot.worker_order[:budget])
capacity = max(1, min(snapshot.capacity_per_worker, self.config.capacity_per_worker))
loads: dict[str, list[str]] = {worker: [] for worker in workers}
@@ -615,16 +621,36 @@ def _place_at_budget(self, snapshot: TurboServeSchedulingSnapshot, budget: int)
if self.config.enable_migration:
moves, evaluations = self._rebalance(snapshot, loads, placement, capacity)
after = self._bottleneck(snapshot, loads, capacity)
- unplaced = sum(session.active and placement.get(session_id) is None for session_id, session in snapshot.sessions.items())
+ unplaced = sum(
+ session.active and placement.get(session_id) is None for session_id, session in snapshot.sessions.items()
+ )
rho_max = max((len(items) / capacity for items in loads.values()), default=0.0)
- return placement, {"algorithm": "least_load_with_optional_rebalance", "capacity_per_worker": capacity, "rebalance_moves": moves, "candidate_evaluations": evaluations, "unplaced_active": unplaced, "bottleneck_before_ms": round(before, 3), "bottleneck_after_ms": round(after, 3), "rho_max": round(rho_max, 4)}
+ return placement, {
+ "algorithm": "least_load_with_optional_rebalance",
+ "capacity_per_worker": capacity,
+ "rebalance_moves": moves,
+ "candidate_evaluations": evaluations,
+ "unplaced_active": unplaced,
+ "bottleneck_before_ms": round(before, 3),
+ "bottleneck_after_ms": round(after, 3),
+ "rho_max": round(rho_max, 4),
+ }
- def _rebalance(self, snapshot: TurboServeSchedulingSnapshot, loads: dict[str, list[str]], placement: dict[str, str | None], capacity: int) -> tuple[int, int]:
+ def _rebalance(
+ self,
+ snapshot: TurboServeSchedulingSnapshot,
+ loads: dict[str, list[str]],
+ placement: dict[str, str | None],
+ capacity: int,
+ ) -> tuple[int, int]:
moves = evaluations = 0
for _ in range(self.config.rebalance_iteration_limit):
if not loads:
break
- source = max(loads, key=lambda worker: (self._worker_worst(snapshot, loads[worker], capacity), len(loads[worker]), worker))
+ source = max(
+ loads,
+ key=lambda worker: (self._worker_worst(snapshot, loads[worker], capacity), len(loads[worker]), worker),
+ )
if not loads[source]:
break
current = self._bottleneck(snapshot, loads, capacity)
@@ -659,4 +685,9 @@ def _bottleneck(self, snapshot: TurboServeSchedulingSnapshot, loads: dict[str, l
def _worker_worst(self, snapshot: TurboServeSchedulingSnapshot, sessions: list[str], capacity: int) -> float:
if not sessions:
return 0.0
- return max(self.latency_model.session_latency_ms(snapshot.sessions[session_id], len(sessions), capacity, snapshot.runtime_calibration) for session_id in sessions)
+ return max(
+ self.latency_model.session_latency_ms(
+ snapshot.sessions[session_id], len(sessions), capacity, snapshot.runtime_calibration
+ )
+ for session_id in sessions
+ )
diff --git a/telefuser/service/livekit/worker_pool.py b/telefuser/service/livekit/worker_pool.py
index a4122847..e13bc049 100644
--- a/telefuser/service/livekit/worker_pool.py
+++ b/telefuser/service/livekit/worker_pool.py
@@ -111,9 +111,7 @@ async def stop_session(self, session_id: str) -> None:
f"{_SESSION_CANCEL_GRACE_SECONDS:g}s: session={session_id}"
)
- async def migrate_session(
- self, pipeline_session_id: str, target_worker_id: str
- ) -> TurboServeOwnership:
+ async def migrate_session(self, pipeline_session_id: str, target_worker_id: str) -> TurboServeOwnership:
"""Move model state while the existing LiveKit runner keeps publishing."""
if self.router is None:
raise RuntimeError("Worker pool was created without TurboServe routing")
diff --git a/tests/unit/pipelines/abot_world/test_migration.py b/tests/unit/pipelines/abot_world/test_migration.py
index ce15705c..3a35f292 100644
--- a/tests/unit/pipelines/abot_world/test_migration.py
+++ b/tests/unit/pipelines/abot_world/test_migration.py
@@ -151,10 +151,7 @@ def test_taew_batched_decode_matches_independent_streams() -> None:
torch.testing.assert_close(actual_first, expected_first)
expected_continuation = torch.cat(
- [
- stage._decode_chunks_impl(latents, [state])
- for latents, state in zip(continuation, serial_states)
- ],
+ [stage._decode_chunks_impl(latents, [state]) for latents, state in zip(continuation, serial_states)],
dim=0,
)
actual_continuation = stage._decode_chunks_impl(torch.cat(continuation), batched_states)
diff --git a/tests/unit/service/livekit/test_multi_session_capacity.py b/tests/unit/service/livekit/test_multi_session_capacity.py
index a11f94ab..e79a4811 100644
--- a/tests/unit/service/livekit/test_multi_session_capacity.py
+++ b/tests/unit/service/livekit/test_multi_session_capacity.py
@@ -128,9 +128,7 @@ def test_runtime_assigns_all_peak16_sessions_without_waiting_when_capacity_is_fo
worker_pool=worker_pool,
)
- admissions = [
- runtime.create_session(SessionCreateRequest(identity=f"controller-{index}")) for index in range(16)
- ]
+ admissions = [runtime.create_session(SessionCreateRequest(identity=f"controller-{index}")) for index in range(16)]
assert all(result.admission.status == "assigned" for result in admissions)
assert len(worker_pool.started) == 16
diff --git a/tests/unit/service/livekit/test_nccl_transfer.py b/tests/unit/service/livekit/test_nccl_transfer.py
index 19ad4aed..f3059911 100644
--- a/tests/unit/service/livekit/test_nccl_transfer.py
+++ b/tests/unit/service/livekit/test_nccl_transfer.py
@@ -1,10 +1,10 @@
from __future__ import annotations
-import torch
import pytest
+import torch
-from telefuser.service.livekit.nccl_transfer import flatten_tensor_tree, rebuild_tensor_tree
from telefuser.service.livekit.config import LiveKitServeConfig
+from telefuser.service.livekit.nccl_transfer import flatten_tensor_tree, rebuild_tensor_tree
def test_tensor_manifest_round_trip_preserves_nested_structure() -> None:
diff --git a/tests/unit/validation/test_capture_abot_serving_metrics.py b/tests/unit/validation/test_capture_abot_serving_metrics.py
index c22490fa..9060be9c 100644
--- a/tests/unit/validation/test_capture_abot_serving_metrics.py
+++ b/tests/unit/validation/test_capture_abot_serving_metrics.py
@@ -14,7 +14,7 @@ class _MetricsHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
type(self).paths.append(self.path)
if self.path == "/metrics":
- body = b"telefuser_serving_sessions{state=\"active\"} 4\n"
+ body = b'telefuser_serving_sessions{state="active"} 4\n'
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
elif self.path == "/v1/service/metrics/json":
@@ -22,7 +22,7 @@ def do_GET(self) -> None: # noqa: N802
{
"serving": {
"summary": {"sessions": {"active": 4}},
- "counters": {"telefuser_serving_chunks_total{result=\"processed\"}": 12},
+ "counters": {'telefuser_serving_chunks_total{result="processed"}': 12},
}
}
).encode("utf-8")
@@ -73,8 +73,7 @@ def test_capture_writes_prometheus_jsonl_and_manifest_without_proxy(
assert _MetricsHandler.paths.count("/v1/service/metrics/json") >= 1
records = [
- json.loads(line)
- for line in (output_dir / "serving-metrics.jsonl").read_text(encoding="utf-8").splitlines()
+ json.loads(line) for line in (output_dir / "serving-metrics.jsonl").read_text(encoding="utf-8").splitlines()
]
assert len(records) == manifest["samples"]["attempted"]
assert records[0]["serving"]["snapshot"]["summary"]["sessions"]["active"] == 4
diff --git a/tools/validation/benchmark_abot_microbatch.py b/tools/validation/benchmark_abot_microbatch.py
index 5c01706f..83c570a8 100644
--- a/tools/validation/benchmark_abot_microbatch.py
+++ b/tools/validation/benchmark_abot_microbatch.py
@@ -86,12 +86,12 @@ def _run_point(
f"{[len(item) for item in initial_frames]}"
)
for _ in range(args.warmup_chunks):
- frames = pipeline.generate_next_blocks(
- sessions, controls, control_latent_frames=args.control_latent_frames
- )
+ frames = pipeline.generate_next_blocks(sessions, controls, control_latent_frames=args.control_latent_frames)
expected_frames = 4 * args.control_latent_frames
if any(len(item) != expected_frames for item in frames):
- raise RuntimeError(f"warmup did not emit {expected_frames} frames per session: {[len(item) for item in frames]}")
+ raise RuntimeError(
+ f"warmup did not emit {expected_frames} frames per session: {[len(item) for item in frames]}"
+ )
torch.cuda.synchronize(device)
torch.cuda.reset_peak_memory_stats(device)
@@ -101,14 +101,14 @@ def _run_point(
for _ in range(args.repeats):
torch.cuda.synchronize(device)
started_at = time.perf_counter()
- frames = pipeline.generate_next_blocks(
- sessions, controls, control_latent_frames=args.control_latent_frames
- )
+ frames = pipeline.generate_next_blocks(sessions, controls, control_latent_frames=args.control_latent_frames)
torch.cuda.synchronize(device)
elapsed = time.perf_counter() - started_at
expected_frames = 4 * args.control_latent_frames
if any(len(item) != expected_frames for item in frames):
- raise RuntimeError(f"sample did not emit {expected_frames} frames per session: {[len(item) for item in frames]}")
+ raise RuntimeError(
+ f"sample did not emit {expected_frames} frames per session: {[len(item) for item in frames]}"
+ )
samples.append(elapsed)
stage_metrics = pipeline.last_stage_metrics()
denoise_samples.append(float(stage_metrics.get("denoise_seconds", 0.0)))
diff --git a/tools/validation/benchmark_abot_turboserve_concurrent.py b/tools/validation/benchmark_abot_turboserve_concurrent.py
index 85489442..e099cbff 100644
--- a/tools/validation/benchmark_abot_turboserve_concurrent.py
+++ b/tools/validation/benchmark_abot_turboserve_concurrent.py
@@ -119,9 +119,7 @@ async def consume() -> None:
"consumer_displayed_frames": displayed_frames,
"consumer_end_to_end_seconds": consumer_completed_at - created_at,
"consumer_end_to_end_fps": (
- displayed_frames / (consumer_completed_at - created_at)
- if consumer_completed_at > created_at
- else 0.0
+ displayed_frames / (consumer_completed_at - created_at) if consumer_completed_at > created_at else 0.0
),
"consumer_first_frame_seconds": (first_frame_at - created_at) if first_frame_at is not None else None,
"scheduler_queue_wait_seconds": scheduler_waits,
@@ -249,7 +247,11 @@ def _parse_args() -> argparse.Namespace:
parser.error("consumer playback FPS must be non-negative")
if args.control_update_min_seconds <= 0 or args.control_update_max_seconds < args.control_update_min_seconds:
parser.error("control update range must be positive and ordered")
- if not 0 <= args.idle_probability <= 1 or args.idle_min_seconds < 0 or args.idle_max_seconds < args.idle_min_seconds:
+ if (
+ not 0 <= args.idle_probability <= 1
+ or args.idle_min_seconds < 0
+ or args.idle_max_seconds < args.idle_min_seconds
+ ):
parser.error("invalid idle burst configuration")
return args
@@ -258,7 +260,9 @@ def main() -> None:
args = _parse_args()
result = asyncio.run(_benchmark(args))
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
- print(json.dumps({key: value for key, value in result.items() if key != "sessions_detail"}, indent=2, sort_keys=True))
+ print(
+ json.dumps({key: value for key, value in result.items() if key != "sessions_detail"}, indent=2, sort_keys=True)
+ )
if __name__ == "__main__":
diff --git a/tools/validation/run_abot_batch_scaling.py b/tools/validation/run_abot_batch_scaling.py
index d1817c1a..9ef6060a 100644
--- a/tools/validation/run_abot_batch_scaling.py
+++ b/tools/validation/run_abot_batch_scaling.py
@@ -142,7 +142,9 @@ def _run_point(
vae_decode = [float(item.get("vae_decode_seconds", 0.0)) for item in scheduler]
total_frames = sum(len(payload.get("frames", [])) for payload in samples)
elapsed = ended_at - measurement_started_at
- per_session_frames = [sum(len(payload.get("frames", [])) for payload in outputs[session_id]) for session_id in session_ids]
+ per_session_frames = [
+ sum(len(payload.get("frames", [])) for payload in outputs[session_id]) for session_id in session_ids
+ ]
return {
"sessions": sessions,
"max_batch_size": max_batch_size,
From 64bea89bb298d6cf62560f9322d07ddbc6a6937b Mon Sep 17 00:00:00 2001
From: youngmagician114514
<97871956+youngmagician114514@users.noreply.github.com>
Date: Wed, 19 Aug 2026 09:44:17 +0000
Subject: [PATCH 8/8] fix(ci): make ABot workload tests self-contained
Use tracked workload assets, keep upstream trace regeneration optional, and register the missing service example.
---
.../service/test_example_service_parity.py | 1 +
.../validation/test_abot_livekit_burst.py | 9 +++
.../test_abot_turboserve_trace_adapter.py | 62 +++++++++++++++----
.../derive_abot_turboserve_trace.py | 2 +-
.../README-turboserve-public-demo-trace.md | 4 ++
...ps_turboserve_public_demo_trace_peak4.json | 2 +-
...4gpu_lf3_12fps_all_active_peak16_wave.json | 2 +-
...lf3_12fps_diagnostic_phase_aligned_16.json | 2 +-
...u_lf3_12fps_intermittent_input_peak16.json | 2 +-
..._12fps_intermittent_input_peak16_5min.json | 2 +-
...4gpu_lf3_12fps_realistic_async_peak16.json | 2 +-
...s_turboserve_public_demo_trace_peak16.json | 2 +-
.../abot_livekit_4gpu_lf3_12fps_wave.json | 2 +-
13 files changed, 74 insertions(+), 20 deletions(-)
diff --git a/tests/unit/service/test_example_service_parity.py b/tests/unit/service/test_example_service_parity.py
index 8de6f598..587d0086 100644
--- a/tests/unit/service/test_example_service_parity.py
+++ b/tests/unit/service/test_example_service_parity.py
@@ -30,6 +30,7 @@
"wan21_i2v_service": (Path("examples/wan_video/wan21_14b_image_to_video_480p_service.py"), "i2v", True),
"minimax_h3_fl2va": (Path("examples/minimax_h3/minimax_h3_fl2va_h100.py"), "t2v", True),
"minimax_h3_ref2va": (Path("examples/minimax_h3/minimax_h3_ref2va_h100.py"), "s2v", True),
+ "minimax_h3_turbo_lora": (Path("examples/minimax_h3/minimax_h3_turbo_lora_h100.py"), "i2v", True),
"wan22_i2v_distill": (Path("examples/wan_video/wan22_14b_image_to_video_distill_h100.py"), "i2v", True),
"lingbot_video_dense": (Path("examples/lingbot_video/lingbot_video_dense_1_3b.py"), "t2i", True),
"lingbot_video_moe": (Path("examples/lingbot_video/lingbot_video_moe_30b.py"), "t2i", True),
diff --git a/tests/unit/validation/test_abot_livekit_burst.py b/tests/unit/validation/test_abot_livekit_burst.py
index bbf6825a..087ac3b2 100644
--- a/tests/unit/validation/test_abot_livekit_burst.py
+++ b/tests/unit/validation/test_abot_livekit_burst.py
@@ -463,3 +463,12 @@ def test_dry_run_discloses_diagnostic_initial_control_barrier(
output = capsys.readouterr().out
assert "DIAGNOSTIC ONLY" in output
assert "not a real-user arrival trace" in output
+
+
+def test_checked_in_workloads_resolve_repo_native_images() -> None:
+ workload_dir = wave._REPO_ROOT / "tools" / "validation" / "workloads"
+ scenario_paths = sorted(workload_dir.glob("abot_livekit_*.json"))
+ assert scenario_paths
+ for scenario_path in scenario_paths:
+ scenario = wave.load_scenario(scenario_path)
+ assert Path(scenario.session.image_path).is_relative_to(wave._REPO_ROOT)
diff --git a/tests/unit/validation/test_abot_turboserve_trace_adapter.py b/tests/unit/validation/test_abot_turboserve_trace_adapter.py
index 10f715eb..caa18304 100644
--- a/tests/unit/validation/test_abot_turboserve_trace_adapter.py
+++ b/tests/unit/validation/test_abot_turboserve_trace_adapter.py
@@ -6,6 +6,8 @@
from collections.abc import Mapping
from pathlib import Path
+import pytest
+
from tools.validation import benchmark_abot_livekit_burst as wave
from tools.validation import derive_abot_turboserve_trace as adapter
from tools.validation import replay_abot_livekit_lifecycle_trace as replay
@@ -30,16 +32,17 @@ def _peak_retained(events: list[Mapping[str, object]]) -> int:
return peak
-def test_checked_in_turboserve_public_demo_scenarios_are_deterministic_and_runnable() -> None:
- expected_1gpu, expected_4gpu = adapter.build_scenarios(_SOURCE)
- for expected in (expected_1gpu, expected_4gpu):
- filename = f"{expected['name']}.json"
+def test_checked_in_turboserve_public_demo_scenarios_are_self_contained_and_runnable() -> None:
+ for filename, peak in (
+ ("abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json", 4),
+ ("abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json", 16),
+ ):
actual = json.loads((_WORKLOADS / filename).read_text(encoding="utf-8"))
- assert actual == expected
contract = actual["trace_contract"]
assert contract["not_a_turboserve_production_trace"] is True
assert contract["not_a_reproduction_of_private_paper_t1_to_t6_traces"] is True
+ assert contract["source"]["sha256"] == "7dc3bb8934df656a710b76df16c663686ceae8f8db7d1a9b3da98e1ecf2eda31"
assert contract["time_transform"]["source_to_derived_scale"] == 1.0
assert contract["execution_contract"].startswith("No diagnostic barrier")
@@ -48,7 +51,7 @@ def test_checked_in_turboserve_public_demo_scenarios_are_deterministic_and_runna
assert trace["duration_seconds"] == 1800.0
events = trace["events"]
assert isinstance(events, list)
- assert _peak_retained(events) == contract["capacity_transform"]["target_peak_retained_sessions"]
+ assert _peak_retained(events) == peak
scenario = wave.load_scenario(_WORKLOADS / filename)
parsed = replay.load_explicit_lifecycle_trace(scenario)
@@ -57,12 +60,20 @@ def test_checked_in_turboserve_public_demo_scenarios_are_deterministic_and_runna
assert scenario.diagnostic_initial_control_barrier is None
-def test_public_demo_capacity_normalization_retains_real_pause_resume_events() -> None:
- one_gpu, four_gpu = adapter.build_scenarios(_SOURCE)
- for scenario, peak, expected_counts in (
- (one_gpu, 4, {"session_arrival": 61, "session_departure": 61, "user_active": 66, "user_idle": 69}),
- (four_gpu, 16, {"session_arrival": 300, "session_departure": 300, "user_active": 282, "user_idle": 323}),
+def test_checked_in_public_demo_scenarios_retain_real_pause_resume_events() -> None:
+ for filename, peak, expected_counts in (
+ (
+ "abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json",
+ 4,
+ {"session_arrival": 61, "session_departure": 61, "user_active": 66, "user_idle": 69},
+ ),
+ (
+ "abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json",
+ 16,
+ {"session_arrival": 300, "session_departure": 300, "user_active": 282, "user_idle": 323},
+ ),
):
+ scenario = json.loads((_WORKLOADS / filename).read_text(encoding="utf-8"))
trace = scenario["lifecycle_trace"]
assert isinstance(trace, Mapping)
events = trace["events"]
@@ -106,3 +117,32 @@ async def finish_departure() -> None:
]
asyncio.run(check())
+
+
+@pytest.mark.skipif(not _SOURCE.is_file(), reason="requires a sibling TurboServe public trace checkout")
+def test_checked_in_turboserve_public_demo_scenarios_match_external_source() -> None:
+ expected_1gpu, expected_4gpu = adapter.build_scenarios(_SOURCE)
+ for expected in (expected_1gpu, expected_4gpu):
+ filename = f"{expected['name']}.json"
+ actual = json.loads((_WORKLOADS / filename).read_text(encoding="utf-8"))
+ assert actual == expected
+
+
+def test_public_demo_capacity_normalization_is_deterministic_for_peak_186_source() -> None:
+ def event(event_type: str, session_id: int, sequence: int) -> adapter.SourceEvent:
+ return adapter.SourceEvent(
+ time_seconds=0.0 if event_type == "session_arrival" else 1.0,
+ sequence=sequence,
+ event_type=event_type,
+ session_id=session_id,
+ user_id=session_id,
+ active_on_arrival=True if event_type == "session_arrival" else None,
+ )
+
+ source_events = [event("session_arrival", session_id, session_id) for session_id in range(186)]
+ source_events.extend(event("session_departure", session_id, 186 + session_id) for session_id in range(186))
+ first = adapter.derive_trace(source_events, target_peak=4, source_sha256="unit-test-source")
+ second = adapter.derive_trace(source_events, target_peak=4, source_sha256="unit-test-source")
+
+ assert first == second
+ assert first.source_peak_retained_sessions == 186
diff --git a/tools/validation/derive_abot_turboserve_trace.py b/tools/validation/derive_abot_turboserve_trace.py
index 62def926..3ecd5306 100644
--- a/tools/validation/derive_abot_turboserve_trace.py
+++ b/tools/validation/derive_abot_turboserve_trace.py
@@ -481,7 +481,7 @@ def _scenario_payload(*, name: str, workers: int, target_peak: int, trace: Deriv
},
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",
diff --git a/tools/validation/workloads/README-turboserve-public-demo-trace.md b/tools/validation/workloads/README-turboserve-public-demo-trace.md
index 8f5f5ad1..5c51ce6c 100644
--- a/tools/validation/workloads/README-turboserve-public-demo-trace.md
+++ b/tools/validation/workloads/README-turboserve-public-demo-trace.md
@@ -4,6 +4,10 @@ These two scenarios are deterministic, 30-minute ABot LiveKit workload
projections of TurboServe's public simulator trace:
`../../../TurboServe/traces/example_8gpu.json`.
+The public source trace is intentionally not vendored into this repository.
+CI validates the committed scenarios' provenance and replay schema; the strict
+source-to-artifact regeneration check runs locally when a sibling TurboServe checkout is available.
+
They are **not** TurboServe production traces and are **not** reproductions of
the private paper T1--T6 traces. The source records session lifecycle events,
not real ABot keyboard actions. The adapter maps its selected events as:
diff --git a/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json b/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json
index 16125e68..3abeeef9 100644
--- a/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json
+++ b/tools/validation/workloads/abot_livekit_1gpu_lf3_12fps_turboserve_public_demo_trace_peak4.json
@@ -2947,7 +2947,7 @@
"delivery_mode": "latest",
"expected_preview_frames": 1,
"fps": 12,
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"prompt": "A smooth first-person exploration through a vivid natural landscape."
},
"trace_contract": {
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json
index b1ff7d91..adca2b68 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_all_active_peak16_wave.json
@@ -11,7 +11,7 @@
"seed": 20260814,
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json
index 54643d33..563009ef 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_diagnostic_phase_aligned_16.json
@@ -22,7 +22,7 @@
},
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json
index 1abb0db1..66c58315 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16.json
@@ -11,7 +11,7 @@
"seed": 20260815,
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json
index f65a5300..6b9292f9 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_intermittent_input_peak16_5min.json
@@ -11,7 +11,7 @@
"seed": 20260818,
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json
index 2148f8e1..714f9999 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_realistic_async_peak16.json
@@ -20,7 +20,7 @@
},
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json
index 71d300f7..4af26c09 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_turboserve_public_demo_trace_peak16.json
@@ -13614,7 +13614,7 @@
"delivery_mode": "latest",
"expected_preview_frames": 1,
"fps": 12,
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"prompt": "A smooth first-person exploration through a vivid natural landscape."
},
"trace_contract": {
diff --git a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
index 17bad1ed..2df1f4bf 100644
--- a/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
+++ b/tools/validation/workloads/abot_livekit_4gpu_lf3_12fps_wave.json
@@ -6,7 +6,7 @@
"seed": 20260813,
"session": {
"prompt": "A smooth first-person exploration through a vivid natural landscape.",
- "image_path": "../ABot-World/web_client/datasets/images/84b90ad568b693d2.png",
+ "image_path": "examples/data/1.png",
"fps": 12,
"control_latent_frames": 3,
"delivery_mode": "latest",