diff --git a/README.md b/README.md index d80d163d..e9d2f699 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,12 @@ telefuser/ | `FlashVSR` | VSR | Streaming video super-resolution via [examples/flashvsr/README.md](examples/flashvsr/README.md) | | `SwiftVR` | Causal video restoration | Single-GPU BF16 offline and direct streaming restoration via [examples/swiftvr/README.md](examples/swiftvr/README.md) | +### Robotics and Action Models + +| Pipeline | Task | Notes | +|----------|------|-------| +| `LingBot-VLA v2` | Vision-language-action inference | RobotWin observations to normalized canonical action chunks via the direct SDK or native structured service; see [examples/lingbot_vla_v2/README.md](examples/lingbot_vla_v2/README.md) | + ### Video Generation | Pipeline | Task | Notes | @@ -277,6 +283,7 @@ See [examples/README.md](examples/README.md) for the example runner and baseline - [docs/en/adding_new_model.md](docs/en/adding_new_model.md): integrating new models - [docs/en/adding_new_example.md](docs/en/adding_new_example.md): authoring examples and pipeline contracts - [docs/en/abot_world.md](docs/en/abot_world.md): ABot-World single-GPU interactive pipeline, controls, and tests +- [examples/lingbot_vla_v2/README.md](examples/lingbot_vla_v2/README.md): LingBot-VLA v2 inference, structured service, parity, and validation boundaries - [examples/swiftvr/README.md](examples/swiftvr/README.md): SwiftVR checkpoint loading, streaming usage, performance, and acceleration options ## Known Limitations diff --git a/benchmarks/telefuser_aiperf/README.md b/benchmarks/telefuser_aiperf/README.md index 93f93258..0826f72c 100644 --- a/benchmarks/telefuser_aiperf/README.md +++ b/benchmarks/telefuser_aiperf/README.md @@ -113,9 +113,52 @@ TELEFUSER_AIPERF_CONCURRENCY=2 \ ``` Each terminal result is required to contain a finite `50x55` action chunk and the frozen structured result fields. -The adapter retains an action hash, bounds, dimensions, verification status, target inference time, and peak memory; -it does not copy full action arrays or Base64 cameras into AIPerf response records. This validates service execution -and normalized action structure, not physical robot control semantics. +The VLA adapter registers `vla_inference_time` and `vla_peak_memory` as AIPerf record metrics, so the normal AIPerf +artifact contains their per-request values and aggregated p50/p95/p99 summaries alongside request latency, throughput, +and success rate. It retains an action hash, bounds, dimensions, and verification status; it does not copy full action +arrays or Base64 cameras into AIPerf response records. This validates service execution and normalized action structure, +not physical robot control semantics. + +For a target-process RSS and per-GPU process-memory artifact, pass the native service PID. This is an external bounded +sampler and does not change the TeleFuser service or any shared metric endpoint: + +```bash +TELEFUSER_AIPERF_SERVICE_PID= \ + bash benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh +``` + +The resource summary is written to +`artifacts/telefuser_aiperf/vla_structured/resource_summary.json` and includes sample count, RSS mean/p50/p95/p99, +and per-GPU process-memory mean/p50/p95/p99. It is intentionally separate from AIPerf server metrics because RSS is +an observer-side process-tree fact, while GPU telemetry remains target-side Prometheus data. + +The checked-in AIPerf configuration defaults to two excluded warmup requests and 20 measured requests. For a longer +distribution, override the request count and concurrency without editing the configuration: + +```bash +TELEFUSER_AIPERF_REQUESTS=100 \ +TELEFUSER_AIPERF_CONCURRENCY=2 \ + bash benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh +``` + +## VLA GPU Parallelism Assessment + +The current VLA policy validates `world_size == 1`, so this assessment is deliberately read-only and does not enable +FSDP, tensor parallelism, or pipeline parallelism. It reports visible GPU capacity, current free memory, checkpoint +size, and how many complete replicas fit using a measured resident-memory estimate: + +```bash +.venv-vla/bin/python tools/validation/inspect_lingbot_vla_v2_gpu_plan.py \ + --model-root /hhb-data/aigc/model_zoo/lingbot/lingbot-vla-v2-6b \ + --replica-memory-mib 13302 \ + --output work_dirs/vla_gpu_plan.json +``` + +On the validated four-H100 host, the report showed four visible 80 GB GPUs and four estimated complete replicas at +the 13,302 MiB measured process-memory baseline. This supports request-level multi-replica service capacity; it is +not evidence that one model replica can be split across GPUs. Any future FSDP or tensor-parallel implementation must +be introduced behind the VLA pipeline boundary and re-run the frozen preprocessing, velocity, action, and HTTP +contract tests before it is considered equivalent. ## LingBot-World v2 Streaming diff --git a/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.py b/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.py new file mode 100644 index 00000000..425a694b --- /dev/null +++ b/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.py @@ -0,0 +1,169 @@ +"""Run the VLA AIPerf workload and sample target process resources.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import threading +import time +from pathlib import Path +from statistics import fmean +from typing import Any + +import psutil + +_MIB = 1024.0 * 1024.0 + + +def _percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def _summary(values: list[float]) -> dict[str, float | int | None]: + return { + "count": len(values), + "mean": fmean(values) if values else None, + "p50": _percentile(values, 0.50), + "p95": _percentile(values, 0.95), + "p99": _percentile(values, 0.99), + "max": max(values) if values else None, + } + + +def _process_tree(root_pid: int) -> tuple[set[int], float]: + try: + root = psutil.Process(root_pid) + processes = [root, *root.children(recursive=True)] + except psutil.Error: + return set(), 0.0 + process_ids: set[int] = set() + rss_bytes = 0 + for process in processes: + try: + process_ids.add(process.pid) + rss_bytes += process.memory_info().rss + except psutil.Error: + continue + return process_ids, rss_bytes / _MIB + + +def _gpu_memory(process_ids: set[int]) -> dict[str, float]: + if not process_ids: + return {} + try: + result = subprocess.run( + [ + "nvidia-smi", + "--query-compute-apps=pid,gpu_uuid,used_gpu_memory", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return {} + memory: dict[str, float] = {} + for line in result.stdout.splitlines(): + fields = [field.strip() for field in line.split(",")] + if len(fields) != 3: + continue + try: + pid = int(fields[0]) + value = float(fields[2]) + except ValueError: + continue + if pid in process_ids and value >= 0: + memory[fields[1]] = memory.get(fields[1], 0.0) + value + return memory + + +def _sample(root_pid: int, started_at: float) -> dict[str, Any]: + process_ids, rss = _process_tree(root_pid) + return { + "offset_seconds": time.perf_counter() - started_at, + "process_ids": sorted(process_ids), + "cpu_rss_mib": rss, + "gpu_memory_mib": _gpu_memory(process_ids), + } + + +def run(args: argparse.Namespace) -> int: + root = Path(__file__).resolve().parents[3] + python = os.environ.get("TELEFUSER_AIPERF_PYTHON", str(root / ".venv-aiperf/bin/python")) + command = [python, "-m", "telefuser_aiperf.cli", "profile", "--config", str(args.config)] + environment = os.environ.copy() + adapter_root = root / "benchmarks/telefuser_aiperf" + environment["PYTHONPATH"] = f"{adapter_root}{os.pathsep}{environment.get('PYTHONPATH', '')}".rstrip(os.pathsep) + + started_at = time.perf_counter() + process = subprocess.Popen(command, cwd=root, env=environment) + samples: list[dict[str, Any]] = [] + stop = threading.Event() + + def sample_loop() -> None: + while not stop.is_set(): + samples.append(_sample(args.service_pid, started_at)) + stop.wait(args.sample_interval) + + sampler = threading.Thread(target=sample_loop, name="vla-aiperf-resource-sampler", daemon=True) + sampler.start() + return_code = process.wait() + stop.set() + sampler.join(timeout=max(args.sample_interval * 2.0, 1.0)) + samples.append(_sample(args.service_pid, started_at)) + + rss_values = [float(sample["cpu_rss_mib"]) for sample in samples] + gpu_values: dict[str, list[float]] = {} + for sample in samples: + for gpu, value in sample["gpu_memory_mib"].items(): + gpu_values.setdefault(gpu, []).append(float(value)) + report = { + "schema_version": 1, + "benchmark": "lingbot_vla_v2_aiperf_structured", + "aiperf_command": command, + "service_pid": args.service_pid, + "return_code": return_code, + "elapsed_seconds": time.perf_counter() - started_at, + "cpu_rss_mib": _summary(rss_values), + "gpu_process_memory_mib": {gpu: _summary(values) for gpu, values in sorted(gpu_values.items())}, + "samples": samples[-args.max_samples :], + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + return return_code + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", + type=Path, + default=Path("benchmarks/telefuser_aiperf/configs/vla_structured_e2e.yaml"), + ) + parser.add_argument("--service-pid", type=int, required=True) + parser.add_argument("--sample-interval", type=float, default=1.0) + parser.add_argument("--max-samples", type=int, default=600) + parser.add_argument( + "--output", + type=Path, + default=Path("artifacts/telefuser_aiperf/vla_structured/resource_summary.json"), + ) + args = parser.parse_args() + if args.service_pid < 1 or args.sample_interval <= 0 or args.max_samples < 2: + parser.error("service PID must be positive, interval must be positive, and max-samples must be at least 2") + raise SystemExit(run(args)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh b/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh index adf86b15..6fa5802f 100755 --- a/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh +++ b/benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh @@ -22,4 +22,15 @@ if command -v curl >/dev/null 2>&1; then fi export PYTHONPATH="${ADAPTER_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" +if [[ -n "${TELEFUSER_AIPERF_SERVICE_PID:-}" ]]; then + RESOURCE_PYTHON="${TELEFUSER_VLA_PYTHON:-${ROOT_DIR}/.venv-vla/bin/python}" + if [[ ! -x "${RESOURCE_PYTHON}" ]]; then + echo "The VLA resource sampler interpreter is unavailable: ${RESOURCE_PYTHON}" >&2 + exit 1 + fi + exec "${RESOURCE_PYTHON}" benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.py \ + --config "${CONFIG_PATH}" \ + --service-pid "${TELEFUSER_AIPERF_SERVICE_PID}" \ + --output "${TELEFUSER_AIPERF_RESOURCE_OUTPUT:-artifacts/telefuser_aiperf/vla_structured/resource_summary.json}" +fi exec "${AIPERF_PYTHON}" -m telefuser_aiperf.cli profile --config "${CONFIG_PATH}" diff --git a/benchmarks/telefuser_aiperf/telefuser_aiperf/vla_structured.py b/benchmarks/telefuser_aiperf/telefuser_aiperf/vla_structured.py index a7a64697..1cdbe26b 100644 --- a/benchmarks/telefuser_aiperf/telefuser_aiperf/vla_structured.py +++ b/benchmarks/telefuser_aiperf/telefuser_aiperf/vla_structured.py @@ -14,17 +14,21 @@ from urllib.parse import quote, urlsplit, urlunsplit import orjson -from aiperf.common.exceptions import NotInitializedError +from aiperf.common.enums import MetricFlags, MetricSizeUnit, MetricTimeUnit +from aiperf.common.exceptions import NoMetricValue, NotInitializedError from aiperf.common.models import ( BaseResponseData, ErrorDetails, InferenceServerResponse, ParsedResponse, + ParsedResponseRecord, RequestInfo, RequestRecord, TextResponse, ) from aiperf.endpoints.base_endpoint import BaseEndpoint +from aiperf.metrics import BaseRecordMetric +from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.plugin.schema.schemas import TransportMetadata from aiperf.transports.aiohttp_transport import AioHttpTransport @@ -60,6 +64,43 @@ class VlaActionResponseData(BaseResponseData): peak_memory_mb: float | None = None +class VlaInferenceTimeMetric(BaseRecordMetric[float]): + """Expose server-measured VLA inference time to AIPerf.""" + + tag = "vla_inference_time" + header = "VLA Inference Time" + short_header = "VLA Inference" + unit = MetricTimeUnit.SECONDS + display_unit = MetricTimeUnit.MILLISECONDS + display_order = 310 + flags = MetricFlags.NONE + + def _parse_record(self, record: ParsedResponseRecord, record_metrics: MetricRecordDict) -> float: + del record_metrics + for response in reversed(record.responses): + if isinstance(response.data, VlaActionResponseData) and response.data.inference_time_s is not None: + return response.data.inference_time_s + raise NoMetricValue("VLA inference time is not available in the structured response.") + + +class VlaPeakMemoryMetric(BaseRecordMetric[float]): + """Expose server-measured peak accelerator memory to AIPerf.""" + + tag = "vla_peak_memory" + header = "VLA Peak Memory" + short_header = "VLA Peak Memory" + unit = MetricSizeUnit.MEGABYTES + display_order = 311 + flags = MetricFlags.NONE + + def _parse_record(self, record: ParsedResponseRecord, record_metrics: MetricRecordDict) -> float: + del record_metrics + for response in reversed(record.responses): + if isinstance(response.data, VlaActionResponseData) and response.data.peak_memory_mb is not None: + return response.data.peak_memory_mb + raise NoMetricValue("VLA peak memory is not available in the structured response.") + + def _finite_number(value: Any, *, name: str, allow_none: bool = False) -> float | None: if value is None and allow_none: return None diff --git a/benchmarks/telefuser_aiperf/tests/test_vla_structured.py b/benchmarks/telefuser_aiperf/tests/test_vla_structured.py index 1ca58d81..39f37f4e 100644 --- a/benchmarks/telefuser_aiperf/tests/test_vla_structured.py +++ b/benchmarks/telefuser_aiperf/tests/test_vla_structured.py @@ -39,6 +39,11 @@ def test_registration_uses_aiperf_endpoint_and_transport_plugins() -> None: assert transport_class is TeleFuserStructuredHttpTransport assert plugins.get_endpoint_metadata("telefuser_vla_structured").requires_polling is True + from aiperf.metrics.metric_registry import MetricRegistry + + assert MetricRegistry.get_class("vla_inference_time").__name__ == "VlaInferenceTimeMetric" + assert MetricRegistry.get_class("vla_peak_memory").__name__ == "VlaPeakMemoryMetric" + def test_build_vla_payload_reuses_inline_image_for_three_cameras() -> None: encoded = base64.b64encode(b"image bytes").decode() diff --git a/benchmarks/telefuser_aiperf/vla_structured_contract.yaml b/benchmarks/telefuser_aiperf/vla_structured_contract.yaml index 81ca0cc8..d1ca51b3 100644 --- a/benchmarks/telefuser_aiperf/vla_structured_contract.yaml +++ b/benchmarks/telefuser_aiperf/vla_structured_contract.yaml @@ -33,7 +33,10 @@ metrics: - request_latency - request_throughput - success_rate + - vla_inference_time + - vla_peak_memory - server_metrics artifacts: config: benchmarks/telefuser_aiperf/configs/vla_structured_e2e.yaml dataset: benchmarks/telefuser_aiperf/data/vla_structured.jsonl + resource_summary: artifacts/telefuser_aiperf/vla_structured/resource_summary.json diff --git a/examples/lingbot_vla_v2/README.md b/examples/lingbot_vla_v2/README.md index 918c0090..ae02d878 100644 --- a/examples/lingbot_vla_v2/README.md +++ b/examples/lingbot_vla_v2/README.md @@ -1,141 +1,140 @@ -# LingBot-VLA v2 Base Model SDK +# LingBot-VLA v2 -This example loads the official LingBot-VLA v2 6B base checkpoint through TeleFuser and returns its normalized -55-dimensional canonical action chunk. The RobotWin profile is used only to prepare the example observation; the -result is not converted to physical RobotWin actions. +This example runs the official LingBot-VLA v2 6B base checkpoint through TeleFuser and returns a normalized +`50 x 55` canonical action chunk. The RobotWin profile prepares the observation only; the base output is not a +physical robot command. -## Inputs +## Model Directory -- Three RGB cameras in the upstream RobotWin order: high, left wrist, right wrist. -- A raw 14-dimensional RobotWin state. -- A non-empty task string. +```text +${TF_MODEL_ZOO_PATH}/ + lingbot/lingbot-vla-v2-6b/ + Qwen3-VL-4B-Instruct/ +``` -The SDK applies the bundled upstream RobotWin `bounds_99_woclip` statistics and maps the observation into -LingBot's 55-dimensional canonical state. +```bash +export TF_MODEL_ZOO_PATH=/hhb-data/aigc/model_zoo +``` -## Output +The VLA directory must contain `model.safetensors.index.json` and every referenced shard. The Qwen3-VL directory +provides the visual-language backbone configuration and processor. -The pipeline returns `LingBotVlaV2CanonicalActionChunk` with: +## Validated H100 Environment -- `canonical_normalized_actions`: `[H, 55]` base-model output. -- `horizon`: action chunk length, normally 50 for the official base config. -- `action_dim`: canonical action dimension, normally 55. -- `checkpoint_variant`: `base`. -- `policy_verified=False` and `verification_status="unverified_official_6b_base"`. +Strict parity, runtime comparison, quantization screening, and structured-service validation used this environment: -## Checkpoints +| Component | Validated value | +| --- | --- | +| GPU | NVIDIA H100 80GB HBM3 (SM90) | +| NVIDIA driver | `590.48.01` | +| Python | `3.10.12` | +| PyTorch | `2.11.0+cu130` | +| TorchVision | `0.26.0+cu130` | +| TorchAudio | `2.11.0+cu130` | +| PyTorch CUDA runtime | `13.0` | +| Transformers | `4.57.3` | +| Triton | `3.6.0` | -The VLA directory must contain `model.safetensors.index.json` and every referenced shard. The Qwen3-VL directory -supplies the visual-language backbone configuration and processor. - -## Example +Create the model-specific environment inside this repository. Install the matching CUDA 13.0 PyTorch wheels first +so dependency resolution retains the validated ABI: ```bash -python examples/lingbot_vla_v2/lingbot_vla_v2_inference.py \ - --model-root /hhb-data/aigc/model_zoo/lingbot/lingbot-vla-v2-6b \ - --qwen3vl-root /hhb-data/aigc/model_zoo/Qwen3-VL-4B-Instruct \ - --camera-high /data/cam_high.png \ - --camera-left-wrist /data/cam_left_wrist.png \ - --camera-right-wrist /data/cam_right_wrist.png \ - --task "pick up the red block" \ - --state-json '[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]' \ - --output canonical_action_chunk.npz +python3.10 -m venv .venv-vla +source .venv-vla/bin/activate +python -m pip install --upgrade pip setuptools wheel +python -m pip install --no-index --find-links /path/to/cu130-wheels \ + "torch==2.11.0+cu130" "torchvision==0.26.0+cu130" "torchaudio==2.11.0+cu130" +python -m pip install -e ".[dev]" ``` -The example saves canonical actions and checkpoint metadata in an `.npz` file. The base output must not be sent to -a robot without an embodiment-specific post-training checkpoint, action mapping, and policy validation. +`.venv-vla` is ignored by Git and does not modify Conda base or the system Python. Use its interpreter explicitly in +the commands below. -## Minimal Single-GPU HTTP Service +## Feature Support -The VLA-specific server loads one policy replica and serializes all inference calls on the selected GPU. It does not -use the shared media service, Ray, multi-GPU execution, dynamic batching, or robot control. Start it from the repository -with the isolated VLA environment: +| Feature | Support | +| --- | --- | +| Official 6B base checkpoint | Supported | +| Public loader and RobotWin preprocessing | Supported | +| Canonical action output | Supported, normally `50 x 55` | +| Strict official-upstream parity | Passed, frozen 38-tensor baseline | +| Matched upstream speed comparison | Recorded below for H100 BF16 | +| Native structured HTTP service | Supported | +| AIPerf structured workload | Supported | +| Request-level replicas | Supported, one policy copy per GPU | +| Online quantization | BF16 default; TorchAO FP8 and BNB NF4 smoke-validated | +| tf-kernel FP8 | Code/unit tested; compatible SM90 wheel not validated on this host | +| Single-policy FSDP, TP, or PP | Not enabled | +| Physical action mapping and safety control | Not included | -```bash -.venv-vla/bin/python examples/lingbot_vla_v2/lingbot_vla_v2_server.py \ - --model-root /hhb-data/aigc/model_zoo/lingbot/lingbot-vla-v2-6b \ - --qwen3vl-root /hhb-data/aigc/model_zoo/Qwen3-VL-4B-Instruct \ - --device cuda:0 \ - --host 127.0.0.1 \ - --port 8000 -``` +## Files -The process reports ready only after both the processor and policy have loaded: +| File | Purpose | +| --- | --- | +| `lingbot_vla_v2_inference.py` | Direct in-process inference | +| `lingbot_vla_v2_native_service.py` | Native TeleFuser structured-service contract | +| `../../telefuser/pipelines/lingbot_vla_v2/` | Pipeline, preprocessing, policy, and service adapter | +| `../../tools/validation/` | Parity, runtime, service, fault, and quantization validators | -```bash -curl http://127.0.0.1:8000/health -``` +Generated captures and benchmark reports belong under the Git-ignored `work_dirs/` directory. -`POST /v1/vla/actions` accepts raw Base64 or a Base64 data URL for each camera. The state must contain exactly 14 -finite values. For example: +## Usage -```bash -.venv-vla/bin/python - <<'PY' -import base64 -from pathlib import Path +### Inputs and Output -import httpx +Inputs are three RGB cameras in upstream RobotWin order (high, left wrist, right wrist), a raw 14-dimensional state, +and a non-empty task instruction. The SDK applies the bundled upstream `bounds_99_woclip` statistics and maps the +observation into LingBot's 55-dimensional canonical state. +The returned `LingBotVlaV2CanonicalActionChunk` contains: -def encode(path: str) -> str: - return base64.b64encode(Path(path).read_bytes()).decode("ascii") +- `canonical_normalized_actions`: `[H, 55]` base-model output. +- `horizon`: normally 50 for the official base configuration. +- `action_dim`: normally 55. +- `checkpoint_variant`: `base`. +- `policy_verified=False` and `verification_status="unverified_official_6b_base"`. +### Direct Inference -response = httpx.post( - "http://127.0.0.1:8000/v1/vla/actions", - json={ - "task": "pick up the red block", - "state": [0.0] * 14, - "camera_high": encode("/data/cam_high.png"), - "camera_left_wrist": encode("/data/cam_left_wrist.png"), - "camera_right_wrist": encode("/data/cam_right_wrist.png"), - "seed": 7, - }, - timeout=300.0, -) -response.raise_for_status() -print(response.json()) -PY +```bash +.venv-vla/bin/python examples/lingbot_vla_v2/lingbot_vla_v2_inference.py \ + --model-root "$TF_MODEL_ZOO_PATH/lingbot/lingbot-vla-v2-6b" \ + --qwen3vl-root "$TF_MODEL_ZOO_PATH/Qwen3-VL-4B-Instruct" \ + --camera-high /data/cam_high.png \ + --camera-left-wrist /data/cam_left_wrist.png \ + --camera-right-wrist /data/cam_right_wrist.png \ + --task "pick up the red block" \ + --state-json '[0,0,0,0,0,0,0,0,0,0,0,0,0,0]' \ + --seed 7 \ + --output canonical_action_chunk.npz ``` -The response contains `canonical_normalized_actions`, `horizon`, `action_dim`, `checkpoint_variant`, -`policy_verified`, and `verification_status`. A successful HTTP response confirms service and model execution only; -the normalized base-model output is not a physical robot command. - -## Native TeleFuser Service - -The native service uses the shared `PIPELINE_CONTRACT`, asynchronous task scheduler, pipeline pool, status API, runtime -metrics, and `TFClient`. It keeps the standalone endpoint above as a small debugging path. - -The example resolves checkpoints under the existing `TF_MODEL_ZOO_PATH` layout: +The `.npz` contains canonical actions and checkpoint metadata. -- `lingbot/lingbot-vla-v2-6b` -- `Qwen3-VL-4B-Instruct` +### Native TeleFuser Service -Start one replica on one visible GPU: +The service uses the shared `PIPELINE_CONTRACT`, asynchronous scheduler, pipeline pool, task-status API, runtime +metrics, and `TFClient`. ```bash TF_MODEL_ZOO_PATH=/hhb-data/aigc/model_zoo \ .venv-vla/bin/telefuser serve \ examples/lingbot_vla_v2/lingbot_vla_v2_native_service.py \ - --task vla_action \ - --parallelism 1 \ - --host 127.0.0.1 \ - --port 18080 + --task vla_action --parallelism 1 --host 127.0.0.1 --port 18080 ``` -Submit `POST /v1/tasks/structured` with `task="vla_action"`, an `instruction`, the 14-dimensional `state`, and -the three Base64 camera fields. The creation response contains a task ID. Poll -`GET /v1/tasks/{task_id}/status`; a completed response contains the action payload under `result` and includes -`inference_time_s` and the optional `peak_memory_mb`. +Submit `POST /v1/tasks/structured` with `task="vla_action"`, `instruction`, the 14-dimensional `state`, the three +Base64 camera fields, and an optional `seed`. Poll `GET /v1/tasks/{task_id}/status`; a completed result includes the +action payload, `inference_time_s`, and optional `peak_memory_mb`. -The unified client handles image encoding, submission, polling, and result extraction: +Each encoded camera is limited to 10 MiB and 16,777,216 decoded pixels before RGB conversion. Both limits are +model-specific settings in `PPL_CONFIG` and apply independently to all three cameras. ```python from telefuser.client import TFClient client = TFClient("http://127.0.0.1:18080") -actions = client.predict_vla_actions( +result = client.predict_vla_actions( instruction="pick up the red block", state=[0.0] * 14, camera_high_path="/data/cam_high.png", @@ -143,118 +142,168 @@ actions = client.predict_vla_actions( camera_right_wrist_path="/data/cam_right_wrist.png", seed=7, ) -print(actions["horizon"], actions["action_dim"]) +print(result["horizon"], result["action_dim"]) ``` -For independent replicas, expose one GPU per replica through the existing pipeline pool: +Use independent request-level replicas when multiple GPUs are available: ```bash CUDA_VISIBLE_DEVICES=0,1 TF_MODEL_ZOO_PATH=/hhb-data/aigc/model_zoo \ .venv-vla/bin/telefuser serve \ examples/lingbot_vla_v2/lingbot_vla_v2_native_service.py \ - --task vla_action \ - --parallelism 2 \ - --num-replicas 2 \ - --port 18080 + --task vla_action --parallelism 2 --num-replicas 2 --port 18080 ``` -This is request-level replication, not tensor parallelism inside one policy replica. The response remains a normalized -base-model canonical action chunk and must not be treated as a physical robot command. +This creates one complete policy per GPU. It does not split one policy with tensor or pipeline parallelism. -## Single-GPU Service Benchmark +## Optional Online Quantization -Use the VLA-specific benchmark to measure checkpoint construction, first-request latency, steady-state latency, -sequential throughput, process RSS, CUDA allocator peaks, and source-image-size overhead. The pipeline always converts -the three source images to the official `256x256` model input, so source size affects boundary and preprocessing cost, -not the model token shape. +BF16 remains the default and the only profile covered by strict upstream parity. Quantization is opt-in, does not +modify checkpoint files, and keeps the fused action MoE weights, state/action projections, AdaNorm projections, and +action head in BF16. The frozen official-base manifest covers 492 Qwen text/vision and action-attention Linear layers. + +| CLI value | Backend | Path | Validation status | +| --- | --- | --- | --- | +| `torchao-fp8` | TorchAO | Dynamic FP8 activation/weight or FP8 weight-only fallback | H100 real forward, action comparison, lifecycle | +| `tf-kernel-fp8` | TeleFuser tf-kernel | Per-token activation and per-output-channel weight FP8 | Code/unit tested; compatible SM90 wheel unavailable | +| `bnb-nf4` | bitsandbytes | NF4 weight-only, BF16 compute | H100 real forward, action comparison, lifecycle | + +TeleFuser's base dependency set currently declares TorchAO and bitsandbytes, while selecting a quantized VLA profile +remains optional. The validated VLA environment uses the exact versions below. If either module is unavailable or a +different version was resolved, repair only `.venv-vla` without changing the system environment: ```bash -CUDA_VISIBLE_DEVICES=0 .venv-vla/bin/python \ - tools/validation/benchmark_lingbot_vla_v2_service.py \ - --model-root /hhb-data/aigc/model_zoo/lingbot/lingbot-vla-v2-6b \ - --qwen3vl-root /hhb-data/aigc/model_zoo/Qwen3-VL-4B-Instruct \ - --image examples/data/lingbot_world_fast/image.jpg \ - --image-sizes 256x256,640x480,1280x720 \ - --warmup 1 \ - --runs 20 \ - --output work_dirs/vla_service_benchmark/report.json +uv pip install --python .venv-vla/bin/python --reinstall --no-deps \ + "torchao==0.17.0" "bitsandbytes==0.48.0" ``` -The native service moves the policy to its target GPU and runs one synthetic fixed-shape warmup before readiness. It -also keeps the allocator cache between requests. The report records construction and startup warmup separately, while -the first accepted request represents a ready replica. The default `service-thread` execution mode matches the native -service runner's fixed worker thread; use `--execution-mode direct` only to measure the in-process pipeline ceiling. -Shutdown still offloads the policy explicitly. +Add `--quantization torchao-fp8` or `--quantization bnb-nf4` to direct-inference and benchmark commands. Accepted +values also include `tf-kernel-fp8`; that path requires an SM90 wheel built for the exact PyTorch/CUDA ABI from +`tf-kernel/`. Do not use a wheel built for another SM family or CUDA ABI. For the native service, set +`PPL_CONFIG["quantization"]` in `lingbot_vla_v2_native_service.py`; it defaults to `None`. -## Native Structured API Validation +The public loader uses the same option: + +```python +from telefuser.pipelines.lingbot_vla_v2.runtime import get_lingbot_vla_v2_pipeline + +pipeline = get_lingbot_vla_v2_pipeline( + "/path/to/lingbot-vla-v2-6b", + "/path/to/Qwen3-VL-4B-Instruct", + device="cuda:0", + quantization="torchao-fp8", +) +``` -Use the VLA-specific HTTP validator after the native service reports ready. This is the structured-output counterpart -to the model-specific direct and AIPerf workloads used by the video and LingBot-World integrations: it exercises the -real TeleFuser HTTP boundary, asynchronous scheduler, task status polling, pipeline pool, and result serialization. -It emits raw request facts and aggregate latency distributions to a JSON artifact; it does not add a VLA-specific -service interface or change shared metric semantics. +The official manifest SHA-256 is +`f9efe28620796060ccc46bd18ac153a580b28d01c7719fa55a8e80631f2ce833`. A changed layer count or name manifest fails +before conversion. Reports record this hash, selected groups, wrapper and weight types, package versions, and backend. -Run a single-replica smoke and latency check: +To compare a quantized backend against unchanged TeleFuser BF16, capture both with identical input, seed, and +`--deterministic-moe`, then run: ```bash -.venv-vla/bin/python tools/validation/validate_lingbot_vla_v2_structured_service.py \ - --base-url http://127.0.0.1:18080 \ - --image examples/data/lingbot_world_fast/image.jpg \ - --warmup 1 \ - --requests 20 \ - --concurrency 1 \ - --output work_dirs/vla_service_validation/smoke_20.json +.venv-vla/bin/python tools/validation/compare_lingbot_vla_v2_quantization.py \ + --reference work_dirs/vla_quantization/bf16_seed7.npz \ + --candidate work_dirs/vla_quantization/torchao_seed7.npz \ + --output work_dirs/vla_quantization/bf16_vs_torchao.json ``` -When the target was started with two independent replicas, validate request-level concurrency with: +Optional gates are `--min-cosine`, `--max-relative-l2`, `--max-abs`, and +`--candidate-replay ... --require-exact-replay`. This is a quantization regression against TeleFuser BF16, not strict +official-upstream parity or RoboTwin task-success evidence. + +## Validation + +### Strict Official-Upstream Parity + +The official reference pins `Robbyant/lingbot-vla-v2` at commit +`be27333c9b5f2663b0ec33f069dd7dfd67fa32b5`. Its isolated checkout, uv environment, cache, and artifacts remain under +`work_dirs/`: ```bash -.venv-vla/bin/python tools/validation/validate_lingbot_vla_v2_structured_service.py \ - --base-url http://127.0.0.1:18080 \ - --image examples/data/lingbot_world_fast/image.jpg \ - --warmup 2 \ - --requests 100 \ - --concurrency 2 \ - --output work_dirs/vla_service_validation/two_replica_100.json +mkdir -p work_dirs/.uv-cache-upstream work_dirs/.uv-tmp-upstream +UV_CACHE_DIR="$PWD/work_dirs/.uv-cache-upstream" TMPDIR="$PWD/work_dirs/.uv-tmp-upstream" \ + uv venv work_dirs/.venv-lingbot-upstream --python .venv-vla/bin/python +UV_CACHE_DIR="$PWD/work_dirs/.uv-cache-upstream" TMPDIR="$PWD/work_dirs/.uv-tmp-upstream" \ + uv pip install --python work_dirs/.venv-lingbot-upstream/bin/python \ + -r tools/validation/requirements-lingbot-vla-v2-upstream.txt +UV_CACHE_DIR="$PWD/work_dirs/.uv-cache-upstream" TMPDIR="$PWD/work_dirs/.uv-tmp-upstream" \ + uv pip install --python work_dirs/.venv-lingbot-upstream/bin/python --no-deps \ + "lerobot @ https://github.com/huggingface/lerobot/archive/refs/tags/v0.4.2.tar.gz" +git clone https://github.com/Robbyant/lingbot-vla-v2 work_dirs/lingbot-vla-v2-upstream +git -C work_dirs/lingbot-vla-v2-upstream checkout be27333c9b5f2663b0ec33f069dd7dfd67fa32b5 ``` -Use duration mode for a bounded soak. Workers use closed-loop scheduling: each worker submits its next request only -after its previous task reaches a terminal state. +Generate the official artifact with `capture_lingbot_vla_v2_upstream.py`, the TeleFuser artifact with +`capture_lingbot_vla_v2_telefuser.py`, and compare them with `run_lingbot_vla_v2_parity.py --profile strict`. Both +captures must use identical checkpoints, cameras, task, state, seed, device, `--deterministic-moe`, eager attention, +and deterministic reference MoE metadata. TeleFuser capture metadata records both the commit and whether tracked +files were dirty; release evidence must use a clean worktree and `--full-checkpoint-hash`. + +| Layer | Compared | Passed | Failed | Global max abs | +| --- | ---: | ---: | ---: | ---: | +| Preprocessing tensors | 6 | 6 | 0 | `0.0` | +| Initial action noise | 1 | 1 | 0 | `0.0` | +| Timesteps | 10 | 10 | 0 | `0.0` | +| Per-step `x_t` | 10 | 10 | 0 | `0.0` | +| Per-step velocity | 10 | 10 | 0 | `0.0` | +| Final normalized action (`50 x 55`) | 1 | 1 | 0 | `0.0` | +| **Total** | **38** | **38** | **0** | **`0.0`** | + +The official constructor hard-codes FlashAttention, so the upstream capture selects eager attention only inside the +validation process. Production inference still reaches the upstream Triton MoE through `telefuser.ops`; strict +capture uses deterministic reference MoE because atomic accumulation is not bitwise repeatable across processes. +The comparator rejects mixed attention or MoE backends. + +### TeleFuser Regression Baseline + +For changes that do not require a fresh official capture, run the TeleFuser capture twice with identical arguments +and compare the artifacts with the same strict comparator: + +```bash +.venv-vla/bin/python tools/validation/capture_lingbot_vla_v2_telefuser.py \ + --model-root "$TF_MODEL_ZOO_PATH/lingbot/lingbot-vla-v2-6b" \ + --qwen3vl-root "$TF_MODEL_ZOO_PATH/Qwen3-VL-4B-Instruct" \ + --camera-high /data/cam_high.png --camera-left-wrist /data/cam_left.png \ + --camera-right-wrist /data/cam_right.png --task "pick up the red block" \ + --state-json '[0,0,0,0,0,0,0,0,0,0,0,0,0,0]' --seed 7 --deterministic-moe \ + --output work_dirs/vla_regression/baseline_seed7.npz + +# Repeat the capture with: +# --output work_dirs/vla_regression/replay_seed7.npz + +.venv-vla/bin/python tools/validation/run_lingbot_vla_v2_parity.py \ + --reference work_dirs/vla_regression/baseline_seed7.npz \ + --candidate work_dirs/vla_regression/replay_seed7.npz --profile strict \ + --output work_dirs/vla_regression/strict_report.json +``` + +Each capture has a JSON sidecar with checkpoint, processor, input, runtime, and tensor-contract metadata. This detects +TeleFuser regressions but does not independently establish equivalence with the official repository. + +### Native Structured API + +Run the validator after the service reports ready: ```bash .venv-vla/bin/python tools/validation/validate_lingbot_vla_v2_structured_service.py \ --base-url http://127.0.0.1:18080 \ - --camera-high /data/cam_high.png \ - --camera-left-wrist /data/cam_left_wrist.png \ - --camera-right-wrist /data/cam_right_wrist.png \ - --duration-seconds 7200 \ - --concurrency 1 \ - --service-pid \ - --gpu-indexes 0 \ - --resource-interval-seconds 1 \ - --output work_dirs/vla_service_validation/soak_2h.json + --image examples/data/lingbot_world_fast/image.jpg \ + --quantization-profile bf16 --warmup 1 --requests 20 --concurrency 1 \ + --output work_dirs/vla_service_validation/smoke_20.json ``` -Resource sampling is opt-in and local-only. `--service-pid` must identify the parent `telefuser serve` process; its -replica descendants are discovered on every sample. RSS is summed across that process tree, while `nvidia-smi` -process memory is grouped by physical GPU index. For a two-replica service on physical GPUs 0 and 1, pass -`--gpu-indexes 0,1`. Omitting `--service-pid` keeps remote-service validation lightweight and does not invoke -`nvidia-smi`. Reports retain bounded raw samples plus distributions and first/last 10% trends for latency, RSS, and -per-GPU process memory. - -The validator freezes the current structured contract. Requests contain exactly `task`, `instruction`, `state`, the -three camera fields, and optional `seed`. Action results contain exactly `canonical_normalized_actions`, `horizon`, -`action_dim`, `checkpoint_variant`, `policy_verified`, and `verification_status`. Safe additive task-status metadata -remains allowed, but status responses must not echo the three Base64 camera fields. +For a two-replica run, use `--warmup 2 --requests 100 --concurrency 2`. For a bounded soak, use +`--duration-seconds 7200 --service-pid --gpu-indexes 0`. Local resource sampling sums RSS over +the service process tree and groups `nvidia-smi` process memory by physical GPU. Reports keep bounded samples, +distributions, and first/last 10% trends without storing full actions or Base64 images. -The command exits nonzero if readiness or contract checks fail, any measured request fails, task IDs are duplicated, -or the queue is not drained at the end. Each successful record validates the expected `50x55` finite action tensor -and retains only statistics and a float64 action fingerprint. Full actions and Base64 camera contents are deliberately -excluded from the artifact. `--max-records` bounds retained per-request samples during long runs while aggregate -latency and success counters still cover the complete run. `--max-resource-samples` independently bounds retained -resource samples. +The validator freezes the VLA request and result fields, checks readiness, unique task IDs, queue drain, and finite +`50 x 55` actions, and exits nonzero on any failure. `--quantization-profile` labels the running configuration; it does +not modify the service or inspect its quantized wrappers. -For fault handling, run the independent validator against a ready service: +Fault checks cover missing cameras, invalid state size, invalid Base64, and cancellation: ```bash .venv-vla/bin/python tools/validation/validate_lingbot_vla_v2_service_faults.py \ @@ -262,87 +311,183 @@ For fault handling, run the independent validator against a ready service: --image examples/data/lingbot_world_fast/image.jpg ``` -It checks missing cameras, invalid state size, invalid Base64, and cancellation. Replica termination is opt-in and -requires a disposable two-replica service: add `--service-pid ` and -`--kill-replica-gpu-index `. The tool only selects a GPU compute process inside that parent process -tree, sends `SIGTERM`, and verifies one-replica capacity degradation plus a subsequent valid `50x55` response. It does -not promise automatic replica restart. +Replica termination is opt-in for a disposable two-replica service through `--service-pid` and +`--kill-replica-gpu-index`. The validator checks one-replica capacity degradation and a subsequent valid response; it +does not promise automatic replica restart. -The same structured API is available through the repository-owned AIPerf workload. Install the pinned isolated -AIPerf environment once, then run the workload while the native service is ready: +The same contract is available as a repository-owned AIPerf workload: ```bash bash scripts/setup_aiperf.sh bash benchmarks/telefuser_aiperf/scripts/run_vla_structured_bench.sh ``` -AIPerf excludes the configured warmup, aggregates request latency, throughput, success, traces, and server metrics, -and writes normal AIPerf artifacts. The adapter strictly validates the action contract but retains only bounded action -facts, not full arrays or Base64 inputs. Passing either validator proves serving and normalized action structure, not -embodiment-specific control semantics. +AIPerf owns warmup exclusion, aggregation, traces, server metrics, and artifacts. Passing these validators proves the +serving path and normalized action structure, not physical control semantics. -## TeleFuser Regression Baseline +### Historical Native HTTP and CI Evidence (`baf3d18`) -The validation capture runs through the public loader and pipeline, then records preprocessing tensors, fixed initial -noise, every flow-matching `x_t` and velocity step, and the final canonical action. Run it twice before changing VLA -model code to establish and verify a strict local baseline: +On 2026-08-12, TeleFuser commit `baf3d18840a71363984edb46222ef86200efb689` was validated with one H100 80GB, +Python 3.10.12, PyTorch 2.11.0+cu130/CUDA 13.0, one replica at `127.0.0.1:18080`, one warmup, and 20 sequential +requests. All three camera fields used the same source image; the request used seed 7 and a zero-valued 14-dimensional +RobotWin state. + +The service used the single-replica command above. The measured workload added process and GPU sampling: ```bash -.venv-vla/bin/python tools/validation/capture_lingbot_vla_v2_telefuser.py \ - --model-root /hhb-data/aigc/model_zoo/lingbot/lingbot-vla-v2-6b \ - --qwen3vl-root /hhb-data/aigc/model_zoo/Qwen3-VL-4B-Instruct \ - --camera-high /data/cam_high.png \ - --camera-left-wrist /data/cam_left_wrist.png \ - --camera-right-wrist /data/cam_right_wrist.png \ - --task "pick up the red block" \ - --state-json '[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]' \ - --seed 7 \ - --deterministic-moe \ - --output work_dirs/vla_regression/baseline_seed7.npz +.venv-vla/bin/python tools/validation/validate_lingbot_vla_v2_structured_service.py \ + --base-url http://127.0.0.1:18080 \ + --image examples/data/lingbot_world_fast/image.jpg \ + --warmup 1 --requests 20 --concurrency 1 \ + --service-pid --gpu-indexes 0 \ + --output work_dirs/vla_service_validation/smoke_20_baf3d18.json +``` -# Repeat the same command with: -# --output work_dirs/vla_regression/replay_seed7.npz +| Check | Result | +| --- | --- | +| Overall validation | Passed | +| Measured requests | 20 | +| Successful / failed | 20 (100%) / 0 | +| Unique task IDs / completed | 20 / 20 | +| Action contract | 20 finite `50 x 55` canonical normalized chunks | +| Policy status | 20 `unverified_official_6b_base` | +| Ready before and after / warmup / queue drain | Passed | +| Resource sampling | Passed, 26 CPU and GPU samples | + +| Latency | Mean | p50 | p95 | p99 | Max | +| --- | ---: | ---: | ---: | ---: | ---: | +| End to end | 1.370 s | 1.372 s | 1.391 s | 1.391 s | 1.391 s | +| Accepted to terminal | 1.337 s | 1.339 s | 1.355 s | 1.356 s | 1.356 s | +| Target inference | 1.267 s | 1.252 s | 1.330 s | 1.330 s | 1.330 s | +| Submission | 0.033 s | 0.032 s | 0.035 s | 0.036 s | 0.036 s | + +Throughput was 0.729 requests/s. GPU process memory remained at 13,302 MiB. Process-tree RSS peaked at 3,617.9 MiB +and changed by -0.6 MiB between the first and last sample windows. The service stopped normally, leaving no process or +GPU allocation. The ignored raw report contains bounded statistics and action fingerprints, not complete action or +Base64 payloads. + +The strict 38-item calculation was not repeated at `baf3d18`: the model, loader, preprocessing, velocity sampling, +and action path had not changed since `2d40ee2`; changes through `baf3d18` affected the service and validation boundary. +This HTTP evidence supplements rather than replaces the frozen strict parity result. + +Full-project CI ran in a separate `.venv`; cross-model dependencies such as PyAV, OpenCV, Diffusers, and ImageIO were +not installed into `.venv-vla`: -.venv-vla/bin/python tools/validation/run_lingbot_vla_v2_parity.py \ - --reference work_dirs/vla_regression/baseline_seed7.npz \ - --candidate work_dirs/vla_regression/replay_seed7.npz \ - --profile strict \ - --output work_dirs/vla_regression/strict_report.json +```bash +PATH=/data/telefuser_vla_test/.venv/bin:$PATH \ + bash scripts/run_ci_tests.sh --skip-install ``` -Each `.npz` has a same-name `.json` sidecar containing the checkpoint, processor, input, runtime, and tensor contract -metadata. The default checkpoint identity is a fast filename-and-size manifest. Add `--full-checkpoint-hash` when a -content hash of every checkpoint shard is required. Keep generated artifacts under `work_dirs`; do not commit them. +| Stage | Result | +| --- | --- | +| Ruff check | Passed | +| Ruff format check | Passed, 510 files checked | +| Ruff import check | Passed | +| CPU-only runtime assertion | Passed | +| Unit tests | 1,236 passed, 8 skipped, 114 deselected, 5 subtests passed | +| Server and OpenAI API tests | 62 passed | +| Overall CI | Passed | + +Expected skips covered CUDA-only operations, optional local tf-kernel, and two LingBot-Video refiner parity checks +whose separate upstream checkout was unavailable. No VLA, structured-service, shared-service, or cross-model test +failed. This run proves that the real 6B checkpoint crossed HTTP, scheduler, pipeline service, serialization, and +status polling while preserving `50 x 55`; it does not change `unverified_official_6b_base` or prove robot semantics. + +## Performance -This is a TeleFuser regression check, not upstream parity. It detects changes to the current implementation but does -not establish equivalence with the official repository. +### Official Upstream vs TeleFuser -## Official Upstream Parity +Establish strict numerical parity before comparing speed. The runtime benchmark consumes the same frozen preprocessed +tensors and initial noise, and requires matching checkpoint, device, software, attention, and MoE identities. It times +the device-resident core and a runtime request boundary without parity hooks or intermediate CPU copies. -The strict upstream baseline pins `Robbyant/lingbot-vla-v2` at commit -`be27333c9b5f2663b0ec33f069dd7dfd67fa32b5`. Keep the checkout, uv environment, cache, and artifacts under -`work_dirs`; Git ignores them. Create the isolated runtime with: +Run both implementations sequentially on the same idle GPU, then compare their reports: ```bash -mkdir -p work_dirs/.uv-cache-upstream work_dirs/.uv-tmp-upstream -UV_CACHE_DIR="$PWD/work_dirs/.uv-cache-upstream" TMPDIR="$PWD/work_dirs/.uv-tmp-upstream" uv venv work_dirs/.venv-lingbot-upstream --python .venv-vla/bin/python -UV_CACHE_DIR="$PWD/work_dirs/.uv-cache-upstream" TMPDIR="$PWD/work_dirs/.uv-tmp-upstream" uv pip install --python work_dirs/.venv-lingbot-upstream/bin/python -r tools/validation/requirements-lingbot-vla-v2-upstream.txt -UV_CACHE_DIR="$PWD/work_dirs/.uv-cache-upstream" TMPDIR="$PWD/work_dirs/.uv-tmp-upstream" uv pip install --python work_dirs/.venv-lingbot-upstream/bin/python --no-deps "lerobot @ https://github.com/huggingface/lerobot/archive/refs/tags/v0.4.2.tar.gz" -git clone https://github.com/Robbyant/lingbot-vla-v2 work_dirs/lingbot-vla-v2-upstream -git -C work_dirs/lingbot-vla-v2-upstream checkout be27333c9b5f2663b0ec33f069dd7dfd67fa32b5 +CUDA_VISIBLE_DEVICES=0 work_dirs/.venv-lingbot-upstream/bin/python \ + tools/validation/benchmark_lingbot_vla_v2_runtime.py \ + --implementation upstream --upstream-root work_dirs/lingbot-vla-v2-upstream \ + --model-root "$TF_MODEL_ZOO_PATH/lingbot/lingbot-vla-v2-6b" \ + --qwen3vl-root "$TF_MODEL_ZOO_PATH/Qwen3-VL-4B-Instruct" \ + --input-artifact work_dirs/vla_upstream_parity/upstream_seed7.npz \ + --seed 7 --device cuda:0 --warmup 3 --runs 20 \ + --output work_dirs/vla_runtime_comparison/upstream_h100_runs20.json + +CUDA_VISIBLE_DEVICES=0 .venv-vla/bin/python \ + tools/validation/benchmark_lingbot_vla_v2_runtime.py \ + --implementation telefuser \ + --model-root "$TF_MODEL_ZOO_PATH/lingbot/lingbot-vla-v2-6b" \ + --qwen3vl-root "$TF_MODEL_ZOO_PATH/Qwen3-VL-4B-Instruct" \ + --input-artifact work_dirs/vla_upstream_parity/upstream_seed7.npz \ + --seed 7 --device cuda:0 --warmup 3 --runs 20 \ + --output work_dirs/vla_runtime_comparison/telefuser_h100_runs20.json + +.venv-vla/bin/python tools/validation/compare_lingbot_vla_v2_runtime_benchmarks.py \ + --upstream work_dirs/vla_runtime_comparison/upstream_h100_runs20.json \ + --telefuser work_dirs/vla_runtime_comparison/telefuser_h100_runs20.json \ + --output-json work_dirs/vla_runtime_comparison/upstream_vs_telefuser_h100_runs20.json \ + --output-markdown work_dirs/vla_runtime_comparison/upstream_vs_telefuser_h100_runs20.md +``` + +The recorded run used upstream commit `be27333c9b5f2663b0ec33f069dd7dfd67fa32b5`, TeleFuser model-source commit +`86278d4a22d35f7cd8606dddd80ae3a4637e396c`, one H100 80GB, the validated software versions above, eager attention, +the upstream Robby Triton MoE, three warmups, and 20 measured requests per scope. + +| Scope | Metric | Upstream | TeleFuser | TeleFuser change | +| --- | ---: | ---: | ---: | ---: | +| Core model | mean | 669.382 ms | 660.100 ms | -1.39% | +| Core model | p50 | 668.373 ms | 657.364 ms | -1.65% | +| Core model | p95 | 677.683 ms | 669.343 ms | -1.23% | +| Core model | p99 | 678.620 ms | 685.735 ms | +1.05% | +| Runtime request | mean | 662.462 ms | 658.935 ms | -0.53% | +| Runtime request | p50 | 661.707 ms | 656.974 ms | -0.72% | +| Runtime request | p95 | 666.779 ms | 678.023 ms | +1.69% | +| Runtime request | p99 | 669.705 ms | 682.456 ms | +1.90% | + +Negative change means TeleFuser was faster. `Core model` measures `sample_actions` with device-resident fixed inputs +and noise. `Runtime request` also includes tensor transfer, seeded-noise construction, output validation, and CPU +action delivery; both exclude image decoding and preprocessing. Peak allocated CUDA memory was 12,454.8 MiB for both. +Mean differences below 1.5% show no material TeleFuser overhead in this matched run. The small 20-sample p99 result is +not a tail-latency conclusion, and loader time is not compared because construction boundaries differ. + +### Single-GPU Service Benchmark + +```bash +CUDA_VISIBLE_DEVICES=0 .venv-vla/bin/python \ + tools/validation/benchmark_lingbot_vla_v2_service.py \ + --model-root "$TF_MODEL_ZOO_PATH/lingbot/lingbot-vla-v2-6b" \ + --qwen3vl-root "$TF_MODEL_ZOO_PATH/Qwen3-VL-4B-Instruct" \ + --image examples/data/lingbot_world_fast/image.jpg \ + --image-sizes 256x256,640x480,1280x720 --warmup 1 --runs 20 \ + --output work_dirs/vla_service_benchmark/report.json ``` -Generate the reference with `capture_lingbot_vla_v2_upstream.py` in the upstream uv environment and the candidate -with `capture_lingbot_vla_v2_telefuser.py` in `.venv-vla`. Pass identical model, processor, camera, task, state, seed, -and device arguments to both commands, add `--deterministic-moe`, and pass `--upstream-root` to the upstream command. -Then compare them with the strict comparator shown above. Generated artifacts belong in `work_dirs/vla_upstream_parity`. +The report includes construction, startup warmup, first/steady request latency, p50/p90/p95/p99, throughput, phase +timing, RSS, CUDA peaks, shutdown, and allocator state. The default `service-thread` mode matches the native runner; +`--execution-mode direct` measures only the in-process ceiling. Source images are always converted to the official +`256 x 256` input, so source size changes boundary/preprocessing cost rather than model token shape. + +### Quantization Screening + +Five fixed-input H100 requests produced this screening result. It verifies the path but is not a production baseline: + +| Profile | Mean request | Throughput | Steady GPU allocated | Action cosine vs BF16 | Relative action L2 | +| --- | ---: | ---: | ---: | ---: | ---: | +| BF16 | 0.668 s | 1.496 req/s | 12,328 MiB | 1.00000 | 0.00% | +| TorchAO FP8 | 1.385 s | 0.722 req/s | 8,293 MiB | 0.99969 | 2.48% | +| BNB NF4 | 0.965 s | 1.037 req/s | 6,327 MiB | 0.99822 | 6.06% | + +Both online formats reduced allocated memory but were slower than BF16 on this H100. Treat them as capacity options +until longer performance and RoboTwin task-success evaluations establish deployment benefit. tf-kernel FP8 was not +run because the available CUDA toolkit was 12.8 while `.venv-vla` used CUDA 13.0. -This is a minimal inference-parity runtime, not a LeRobot training environment. The upstream setup itself combines -LeRobot 0.4.2 metadata constraints with versions outside those constraints, so LeRobot is installed with `--no-deps`; -the capture import and end-to-end run are the runtime checks. +## Notes and Limitations -The official code hard-codes FlashAttention during construction. The upstream capture replaces that selection only -inside its validation process so both sides use eager attention on the Python 3.10.12 / PyTorch 2.11 stack. Production -inference keeps the upstream Triton MoE path through `telefuser.ops`; strict capture uses `--deterministic-moe` because -the upstream kernel uses atomic accumulation and is not bitwise repeatable across separate processes. Artifact metadata -records both `attention_backend` and `moe_backend`, and the comparator rejects mixed-backend artifacts. +- BF16 strict parity proves numerical equivalence at the captured model boundary; it does not certify robot behavior. +- `unverified_official_6b_base` is intentional until an embodiment-specific checkpoint and task-success evaluation + establish control semantics. +- Do not execute canonical actions directly. Real control requires de-normalization, joint/action mapping, control + frequency, limits, safety policy, feedback, and emergency-stop behavior. +- Quantized profiles require separate numerical, performance, and task-success acceptance criteria. +- Request replicas scale independent policy copies; single-policy FSDP, TP, and PP are outside this integration. +- Runtime benchmarks exclude HTTP scheduling unless the structured-service validator or AIPerf workload is used. diff --git a/examples/lingbot_vla_v2/lingbot_vla_v2_inference.py b/examples/lingbot_vla_v2/lingbot_vla_v2_inference.py index 608e9bff..a82e901a 100644 --- a/examples/lingbot_vla_v2/lingbot_vla_v2_inference.py +++ b/examples/lingbot_vla_v2/lingbot_vla_v2_inference.py @@ -19,9 +19,10 @@ def get_pipeline( model_root: str, qwen3vl_root: str, device: str = "cuda", + quantization: str | None = None, ) -> LingBotVlaV2Pipeline: """Load the official 6B checkpoint and Qwen3-VL processor.""" - return get_lingbot_vla_v2_pipeline(model_root, qwen3vl_root, device=device) + return get_lingbot_vla_v2_pipeline(model_root, qwen3vl_root, device=device, quantization=quantization) @click.command() @@ -35,6 +36,7 @@ def get_pipeline( @click.option("--output", default="canonical_action_chunk.npz", type=click.Path(dir_okay=False)) @click.option("--seed", default=None, type=int) @click.option("--device", default="cuda") +@click.option("--quantization", type=click.Choice(("torchao-fp8", "tf-kernel-fp8", "bnb-nf4")), default=None) def main( model_root: str, qwen3vl_root: str, @@ -46,6 +48,7 @@ def main( output: str, seed: int | None, device: str, + quantization: str | None, ) -> None: """Predict and save a normalized canonical action chunk.""" try: @@ -69,6 +72,7 @@ def main( model_root, qwen3vl_root, device=device, + quantization=quantization, ) try: chunk = pipeline(observation, seed=seed) diff --git a/examples/lingbot_vla_v2/lingbot_vla_v2_native_service.py b/examples/lingbot_vla_v2/lingbot_vla_v2_native_service.py index deb6b3e2..3c02ea62 100644 --- a/examples/lingbot_vla_v2/lingbot_vla_v2_native_service.py +++ b/examples/lingbot_vla_v2/lingbot_vla_v2_native_service.py @@ -12,6 +12,7 @@ LingBotVlaV2ActionRequest, predict_lingbot_vla_v2_action, ) +from telefuser.utils.logging import logger TF_MODEL_ZOO_PATH = Path(os.environ.get("TF_MODEL_ZOO_PATH", "model_zoo")).expanduser() @@ -19,7 +20,9 @@ "model_root": str(TF_MODEL_ZOO_PATH / "lingbot" / "lingbot-vla-v2-6b"), "qwen3vl_root": str(TF_MODEL_ZOO_PATH / "Qwen3-VL-4B-Instruct"), "device": "cuda:0", + "quantization": None, "max_image_bytes": 10 * 1024 * 1024, + "max_image_pixels": 16 * 1024 * 1024, } PIPELINE_CONTRACT = { @@ -80,11 +83,13 @@ def get_pipeline(parallelism: int = 1) -> LingBotVlaV2Pipeline: """Load one policy replica for the native TeleFuser service.""" if parallelism != 1: raise ValueError("LingBot-VLA v2 supports parallelism=1 per replica; use --num-replicas for a pipeline pool") + logger.info(f"Loading LingBot-VLA v2 service profile quantization={PPL_CONFIG['quantization'] or 'bf16'}") return get_lingbot_vla_v2_pipeline( PPL_CONFIG["model_root"], PPL_CONFIG["qwen3vl_root"], device=PPL_CONFIG["device"], warmup=True, + quantization=PPL_CONFIG["quantization"], ) @@ -111,5 +116,6 @@ def run_structured( pipeline, request, max_image_bytes=int(PPL_CONFIG["max_image_bytes"]), + max_image_pixels=int(PPL_CONFIG["max_image_pixels"]), ) return response.model_dump(mode="json") diff --git a/examples/lingbot_vla_v2/lingbot_vla_v2_server.py b/examples/lingbot_vla_v2/lingbot_vla_v2_server.py deleted file mode 100644 index 22dfecbc..00000000 --- a/examples/lingbot_vla_v2/lingbot_vla_v2_server.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Start a minimal single-GPU LingBot-VLA v2 HTTP service.""" - -from __future__ import annotations - -import click -import uvicorn - -from telefuser.pipelines.lingbot_vla_v2.service import LingBotVlaV2ServiceConfig, create_lingbot_vla_v2_app - - -@click.command() -@click.option("--model-root", required=True, type=click.Path(exists=True, file_okay=False)) -@click.option("--qwen3vl-root", required=True, type=click.Path(exists=True, file_okay=False)) -@click.option("--device", default="cuda:0", show_default=True) -@click.option("--host", default="127.0.0.1", show_default=True) -@click.option("--port", default=8000, show_default=True, type=click.IntRange(1, 65535)) -@click.option("--max-image-mb", default=10, show_default=True, type=click.IntRange(1, 100)) -def main( - model_root: str, - qwen3vl_root: str, - device: str, - host: str, - port: int, - max_image_mb: int, -) -> None: - """Load one policy replica and serve normalized canonical actions.""" - config = LingBotVlaV2ServiceConfig( - model_root=model_root, - qwen3vl_root=qwen3vl_root, - device=device, - max_image_bytes=max_image_mb * 1024 * 1024, - ) - app = create_lingbot_vla_v2_app(config) - uvicorn.run(app, host=host, port=port, workers=1) - - -if __name__ == "__main__": - main() diff --git a/telefuser/models/lingbot_vla_v2.py b/telefuser/models/lingbot_vla_v2.py index 653b0d95..21497ff6 100644 --- a/telefuser/models/lingbot_vla_v2.py +++ b/telefuser/models/lingbot_vla_v2.py @@ -31,6 +31,7 @@ from transformers.models.qwen3_vl.modeling_qwen3_vl import apply_rotary_pos_emb from transformers.utils import logging +from telefuser.core.config import QuantConfig, QuantKernelBackend, QuantType from telefuser.models.lingbot_vla_v2_loader import ( LingBotVLAWeightLoader, LingBotVlaV2StateDictConverter, @@ -51,6 +52,12 @@ Qwen2FusedExperts, Qwen2TokenMoeBlock, ) +from telefuser.models.lingbot_vla_v2_quantization import ( + LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZATION_MANIFEST_SHA256, + LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT, + build_lingbot_vla_v2_linear_manifest, + finalize_lingbot_vla_v2_quantization_identity, +) from telefuser.models.lingbot_vla_v2_qwen import ( Qwen3VLForConditionalGeneration, Qwen3VLPreTrainedModel, @@ -65,6 +72,22 @@ logger = logging.get_logger(__name__) +# Quantize the standard Qwen text/vision blocks and action-expert attention. +# The fused 3-D MoE weights and action/state heads intentionally remain BF16. +LINGBOT_VLA_V2_DEFAULT_QUANTIZE_MODULES = ( + "qwenvl.model.language_model.layers.", + "qwenvl.model.visual.blocks.", + "self_attn.", +) +LINGBOT_VLA_V2_REQUIRED_SKIP_MODULES = ( + "action_in_proj", + "action_out_proj", + "action_time_mlp", + "state_proj", + "lm_head", +) + + class LingbotVLAConfig(PretrainedConfig): """Configuration class for Lingbot-VLA. This is the configuration class to store the configuration of a [`Lingbot-VLA`]. @@ -1489,5 +1512,131 @@ def __init__(self, config, eval=True): def state_dict_converter(**kwargs): return LingBotVlaV2StateDictConverter(**kwargs) + def enable_quant(self, quant_config: QuantConfig) -> None: + """Apply supported online quantization without changing action or MoE heads.""" + if not isinstance(quant_config, QuantConfig): + raise TypeError("LingBot-VLA v2 online quantization requires QuantConfig") + if not quant_config.enabled: + return + + existing_quant_type = getattr(self, "quant_type", None) + if existing_quant_type == quant_config.quant_type: + return + if existing_quant_type is not None: + raise RuntimeError( + f"LingBot-VLA v2 is already quantized as {existing_quant_type}, cannot apply {quant_config.quant_type}" + ) + + profiles = { + QuantType.TORCHAO_FP8: "torchao-fp8", + QuantType.FP8: "tf-kernel-fp8", + QuantType.BNB_NF4: "bnb-nf4", + } + effective_backends = { + QuantType.TORCHAO_FP8: QuantKernelBackend.TORCHAO, + QuantType.FP8: QuantKernelBackend.TF_KERNEL, + QuantType.BNB_NF4: QuantKernelBackend.BITSANDBYTES, + } + if quant_config.quant_type not in profiles: + raise ValueError(f"LingBot-VLA v2 does not support online quantization type {quant_config.quant_type.name}") + + include_names = quant_config.quantize_modules or LINGBOT_VLA_V2_DEFAULT_QUANTIZE_MODULES + exclude_names = tuple(dict.fromkeys((*quant_config.skip_modules, *LINGBOT_VLA_V2_REQUIRED_SKIP_MODULES))) + manifest = build_lingbot_vla_v2_linear_manifest( + self, + include_names=include_names, + exclude_names=exclude_names, + ) + selected_count = int(manifest["selected_count"]) + if selected_count == 0: + raise RuntimeError("LingBot-VLA v2 online quantization did not select any Linear layers") + frozen_official_profile = ( + getattr(getattr(self, "config", None), "checkpoint_variant", None) == "base" + and quant_config.quantize_modules is None + and quant_config.skip_modules == QuantConfig().skip_modules + ) + if frozen_official_profile: + manifest_sha256 = str(manifest["manifest_sha256"]) + if ( + selected_count != LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT + or manifest_sha256 != LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZATION_MANIFEST_SHA256 + ): + raise RuntimeError( + "LingBot-VLA v2 official 6B quantization manifest changed: " + f"expected count={LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT} " + f"sha256={LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZATION_MANIFEST_SHA256}, " + f"got count={selected_count} sha256={manifest_sha256}" + ) + + if quant_config.quant_type == QuantType.TORCHAO_FP8: + if quant_config.kernel_backend not in (QuantKernelBackend.AUTO, QuantKernelBackend.TORCHAO): + raise ValueError( + f"LingBot-VLA v2 TorchAO FP8 requires the TorchAO backend; got {quant_config.kernel_backend.name}" + ) + from telefuser.ops.torchao_fp8_linear import replace_linear_layers_with_torchao_fp8 + + replaced = replace_linear_layers_with_torchao_fp8( + self, + include_names=include_names, + exclude_names=exclude_names, + ) + self.torchao_fp8_replaced_linear = replaced + elif quant_config.quant_type == QuantType.BNB_NF4: + if quant_config.kernel_backend not in (QuantKernelBackend.AUTO, QuantKernelBackend.BITSANDBYTES): + raise ValueError( + f"LingBot-VLA v2 BNB NF4 requires the bitsandbytes backend; got {quant_config.kernel_backend.name}" + ) + from telefuser.ops.bnb_nf4_linear import replace_linear_layers_with_bnb_nf4 + + replaced = replace_linear_layers_with_bnb_nf4( + self, + compute_dtype=torch.bfloat16, + include_names=include_names, + exclude_names=exclude_names, + ) + self.bnb_nf4_replaced_linear = replaced + elif quant_config.quant_type == QuantType.FP8: + if quant_config.kernel_backend not in (QuantKernelBackend.AUTO, QuantKernelBackend.TF_KERNEL): + raise ValueError( + "LingBot-VLA v2 FP8 online quantization requires the tf-kernel backend; " + f"got {quant_config.kernel_backend.name}" + ) + from telefuser.ops.fp8_gemm import FP8GemmOptions, count_linear_layers, enable_fp8_gemm + + def module_filter(name: str, _module: nn.Module) -> bool: + return any(token in name for token in include_names) and not any( + token and token in name for token in exclude_names + ) + + replaced = count_linear_layers(self, module_filter=module_filter) + enable_fp8_gemm( + self, + options=FP8GemmOptions( + fp16_weight_storage="keep" if quant_config.keep_fp16_weight else "discard", + materialize_fp8_on_wrap=True, + ), + module_filter=module_filter, + ) + self.tf_kernel_fp8_replaced_linear = replaced + if replaced != selected_count: + raise RuntimeError( + f"LingBot-VLA v2 quantization selected {selected_count} Linear layers but converted {replaced}" + ) + self.quant_type = quant_config.quant_type + finalize_lingbot_vla_v2_quantization_identity( + self, + profile=profiles[quant_config.quant_type], + quant_type=quant_config.quant_type.name, + kernel_backend=effective_backends[quant_config.quant_type].name, + manifest=manifest, + ) + logger.info( + "LingBot-VLA v2 %s converted %d selected Linear layers (manifest %s); " + "fused MoE and action heads remain BF16", + quant_config.quant_type.name, + replaced, + manifest["manifest_sha256"], + ) + # __WRAPPER_END__ diff --git a/telefuser/models/lingbot_vla_v2_loader.py b/telefuser/models/lingbot_vla_v2_loader.py index 8f74fbc0..ff9c35e8 100644 --- a/telefuser/models/lingbot_vla_v2_loader.py +++ b/telefuser/models/lingbot_vla_v2_loader.py @@ -678,6 +678,8 @@ def forward(self, llm_feats, queries): from transformers import AutoConfig +from telefuser.core.config import QuantConfig + class LingBotVLAWeightLoader: """Minimal native weight-name mapper retained for model compatibility.""" @@ -918,6 +920,7 @@ def load_lingbot_vla_v2( torch_dtype=torch.bfloat16, device=None, checkpoint_variant: str = "base", + quant_config: QuantConfig | None = None, ): from telefuser.models.lingbot_vla_v2 import LingBotVlaV2Model @@ -936,5 +939,6 @@ def load_lingbot_vla_v2( "checkpoint_variant": checkpoint_variant, "checkpoint_path": str(checkpoint_path), }, + quant_config=quant_config, ) return module_manager.fetch_module("lingbot_vla_v2") diff --git a/telefuser/models/lingbot_vla_v2_moe.py b/telefuser/models/lingbot_vla_v2_moe.py index b8c8e91e..79ad177d 100644 --- a/telefuser/models/lingbot_vla_v2_moe.py +++ b/telefuser/models/lingbot_vla_v2_moe.py @@ -5,6 +5,7 @@ import torch +from telefuser.models.lingbot_vla_v2_quantization import linear_compute_dtype from telefuser.ops.lingbot_vla_v2_moe import robby_moe_forward @@ -413,7 +414,7 @@ def forward( ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: # Ensure input dtypes match weight dtype (needed for gradient checkpointing # recomputation where autocast context is lost) - param_dtype = self.self_attn.q_proj.weight.dtype + param_dtype = linear_compute_dtype(self.self_attn.q_proj, hidden_states.dtype) hidden_states = hidden_states.to(param_dtype) if att_output is not None: att_output = att_output.to(param_dtype) @@ -434,8 +435,9 @@ def forward( return query_state, key_state, value_state elif output_atten: - if att_output.dtype != self.self_attn.o_proj.weight.dtype: - att_output = att_output.to(self.self_attn.o_proj.weight.dtype) + output_dtype = linear_compute_dtype(self.self_attn.o_proj, att_output.dtype) + if att_output.dtype != output_dtype: + att_output = att_output.to(output_dtype) out_emb = self.self_attn.o_proj(att_output[:, start:end]) # first residual diff --git a/telefuser/models/lingbot_vla_v2_quantization.py b/telefuser/models/lingbot_vla_v2_quantization.py new file mode 100644 index 00000000..86b727f3 --- /dev/null +++ b/telefuser/models/lingbot_vla_v2_quantization.py @@ -0,0 +1,171 @@ +"""LingBot-VLA v2 helpers for quantized Linear compatibility and identity.""" + +from __future__ import annotations + +import copy +import hashlib +import json +from collections import Counter +from importlib import metadata +from typing import Iterable + +import torch +from torch import nn + +LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT = 492 +LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZATION_MANIFEST_SHA256 = ( + "f9efe28620796060ccc46bd18ac153a580b28d01c7719fa55a8e80631f2ce833" +) + + +def _matches_tokens(name: str, tokens: Iterable[str]) -> bool: + return any(token and token in name for token in tokens) + + +def _qualified_type(value: object) -> str: + value_type = type(value) + return f"{value_type.__module__}.{value_type.__qualname__}" + + +def _package_version(distribution: str) -> str | None: + try: + return metadata.version(distribution) + except metadata.PackageNotFoundError: + return None + + +def _linear_group(name: str) -> str: + if "qwenvl.model.language_model.layers." in name: + return "qwen_language" + if "qwenvl.model.visual.blocks." in name: + return "qwen_visual" + if "qwen_expert" in name and ".self_attn." in name: + return "action_expert_attention" + return "other" + + +def build_lingbot_vla_v2_linear_manifest( + module: nn.Module, + *, + include_names: Iterable[str], + exclude_names: Iterable[str], +) -> dict[str, object]: + """Describe the exact Linear modules selected before online quantization.""" + include_tokens = tuple(include_names) + exclude_tokens = tuple(exclude_names) + selected_names: list[str] = [] + excluded_names: list[str] = [] + for name, child in module.named_modules(): + if not isinstance(child, nn.Linear): + continue + if _matches_tokens(name, exclude_tokens): + excluded_names.append(name) + elif _matches_tokens(name, include_tokens): + selected_names.append(name) + + selected_names.sort() + excluded_names.sort() + group_counts = Counter(_linear_group(name) for name in selected_names) + canonical = json.dumps( + { + "include_names": include_tokens, + "exclude_names": exclude_tokens, + "selected_names": selected_names, + "excluded_names": excluded_names, + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return { + "selected_count": len(selected_names), + "selected_names": selected_names, + "excluded_count": len(excluded_names), + "excluded_names": excluded_names, + "group_counts": dict(sorted(group_counts.items())), + "manifest_sha256": hashlib.sha256(canonical).hexdigest(), + } + + +def finalize_lingbot_vla_v2_quantization_identity( + module: nn.Module, + *, + profile: str, + quant_type: str, + kernel_backend: str, + manifest: dict[str, object], +) -> dict[str, object]: + """Attach JSON-safe backend and wrapper facts after conversion.""" + modules = dict(module.named_modules()) + selected_names = manifest.get("selected_names", []) + if not isinstance(selected_names, list): + raise TypeError("LingBot-VLA v2 quantization manifest selected_names must be a list") + + module_types: Counter[str] = Counter() + weight_types: Counter[str] = Counter() + weight_dtypes: Counter[str] = Counter() + missing_names: list[str] = [] + for name in selected_names: + selected = modules.get(str(name)) + if selected is None: + missing_names.append(str(name)) + continue + module_types[_qualified_type(selected)] += 1 + weight = getattr(selected, "weight", None) + if weight is None: + weight_types["none"] += 1 + weight_dtypes["none"] += 1 + else: + weight_types[_qualified_type(weight)] += 1 + weight_dtypes[str(getattr(weight, "dtype", "unknown")).removeprefix("torch.")] += 1 + if missing_names: + raise RuntimeError(f"quantized Linear modules disappeared from the model: {missing_names[:5]}") + + identity = { + "enabled": True, + "profile": profile, + "quant_type": quant_type, + "kernel_backend": kernel_backend, + "packages": { + "torchao": _package_version("torchao"), + "bitsandbytes": _package_version("bitsandbytes"), + "tf-kernel": _package_version("tf-kernel"), + }, + "implementation": { + "module_types": dict(sorted(module_types.items())), + "weight_types": dict(sorted(weight_types.items())), + "weight_dtypes": dict(sorted(weight_dtypes.items())), + }, + "manifest": copy.deepcopy(manifest), + } + module._lingbot_vla_v2_quantization_identity = identity + return copy.deepcopy(identity) + + +def lingbot_vla_v2_quantization_identity(module: nn.Module) -> dict[str, object]: + """Return bounded runtime identity for BF16 or an applied quantization profile.""" + identity = getattr(module, "_lingbot_vla_v2_quantization_identity", None) + if isinstance(identity, dict): + return copy.deepcopy(identity) + return { + "enabled": False, + "profile": "bf16", + "quant_type": None, + "kernel_backend": None, + "packages": { + "torchao": _package_version("torchao"), + "bitsandbytes": _package_version("bitsandbytes"), + "tf-kernel": _package_version("tf-kernel"), + }, + "implementation": {}, + "manifest": None, + } + + +def linear_compute_dtype(module: nn.Module, fallback: torch.dtype) -> torch.dtype: + """Return a Linear wrapper's activation dtype rather than its packed weight dtype.""" + compute_dtype = getattr(module, "compute_dtype", None) + if isinstance(compute_dtype, torch.dtype): + return compute_dtype + weight = getattr(module, "weight", None) + weight_dtype = getattr(weight, "dtype", None) + return weight_dtype if isinstance(weight_dtype, torch.dtype) else fallback diff --git a/telefuser/models/lingbot_vla_v2_qwen.py b/telefuser/models/lingbot_vla_v2_qwen.py index 47cc54cb..eb951da8 100644 --- a/telefuser/models/lingbot_vla_v2_qwen.py +++ b/telefuser/models/lingbot_vla_v2_qwen.py @@ -42,6 +42,8 @@ from transformers.processing_utils import Unpack from transformers.utils import logging +from telefuser.models.lingbot_vla_v2_quantization import linear_compute_dtype + logger = logging.get_logger(__name__) @@ -186,7 +188,7 @@ def forward( output_atten: bool = False, **kwargs: Unpack[FlashAttentionKwargs], ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: - param_dtype = self.self_attn.q_proj.weight.dtype + param_dtype = linear_compute_dtype(self.self_attn.q_proj, hidden_states.dtype) hidden_states = hidden_states.to(param_dtype) if att_output is not None: att_output = att_output.to(param_dtype) @@ -200,8 +202,9 @@ def forward( return query_state, key_state, value_state if output_atten: - if att_output.dtype != self.self_attn.o_proj.weight.dtype: - att_output = att_output.to(self.self_attn.o_proj.weight.dtype) + output_dtype = linear_compute_dtype(self.self_attn.o_proj, att_output.dtype) + if att_output.dtype != output_dtype: + att_output = att_output.to(output_dtype) out_emb = self.self_attn.o_proj(att_output[:, start:end]) out_emb += hidden_states after_first_residual = out_emb.clone() diff --git a/telefuser/pipelines/lingbot_vla_v2/runtime.py b/telefuser/pipelines/lingbot_vla_v2/runtime.py index 6e3f93a2..fb699841 100644 --- a/telefuser/pipelines/lingbot_vla_v2/runtime.py +++ b/telefuser/pipelines/lingbot_vla_v2/runtime.py @@ -5,12 +5,45 @@ import torch from transformers import AutoProcessor -from telefuser.core.config import ModelRuntimeConfig +from telefuser.core.config import ModelRuntimeConfig, QuantConfig, QuantKernelBackend, QuantType from telefuser.core.module_manager import ModuleManager from telefuser.models.lingbot_vla_v2_loader import load_lingbot_vla_v2 from .pipeline import LingBotVlaV2Pipeline, LingBotVlaV2PipelineConfig +LINGBOT_VLA_V2_QUANTIZATION_CHOICES = ("torchao-fp8", "tf-kernel-fp8", "bnb-nf4") + + +def lingbot_vla_v2_quant_config(quantization: str | QuantType | None) -> QuantConfig: + """Resolve a public LingBot-VLA v2 online-quantization name.""" + if quantization is None: + return QuantConfig() + if isinstance(quantization, str): + normalized = quantization.strip().lower().replace("_", "-") + names = { + "torchao-fp8": QuantType.TORCHAO_FP8, + "tf-kernel-fp8": QuantType.FP8, + "bnb-nf4": QuantType.BNB_NF4, + } + try: + quant_type = names[normalized] + except KeyError as exc: + choices = ", ".join(repr(name) for name in LINGBOT_VLA_V2_QUANTIZATION_CHOICES) + raise ValueError(f"quantization must be one of {choices}, or None") from exc + elif isinstance(quantization, QuantType): + quant_type = quantization + else: + raise TypeError("quantization must be a string, QuantType, or None") + + backends = { + QuantType.TORCHAO_FP8: QuantKernelBackend.TORCHAO, + QuantType.FP8: QuantKernelBackend.TF_KERNEL, + QuantType.BNB_NF4: QuantKernelBackend.BITSANDBYTES, + } + if quant_type not in backends: + raise ValueError(f"LingBot-VLA v2 does not support online quantization type {quant_type.name}") + return QuantConfig(enabled=True, quant_type=quant_type, kernel_backend=backends[quant_type]) + def get_lingbot_vla_v2_pipeline( model_root: str, @@ -18,6 +51,7 @@ def get_lingbot_vla_v2_pipeline( device: str = "cuda:0", *, warmup: bool = False, + quantization: str | QuantType | None = None, ) -> LingBotVlaV2Pipeline: """Load one official 6B base checkpoint replica for inference.""" target_device = torch.device(device) @@ -31,10 +65,20 @@ def get_lingbot_vla_v2_pipeline( ) target_device = torch.device("cuda", device_index) dtype = torch.bfloat16 if target_device.type == "cuda" else torch.float32 + quant_config = lingbot_vla_v2_quant_config(quantization) + if quant_config.enabled and target_device.type != "cuda": + raise ValueError("LingBot-VLA v2 online quantization requires a CUDA device") processor = AutoProcessor.from_pretrained(qwen3vl_root, local_files_only=True, padding_side="right") manager = ModuleManager(torch_dtype=dtype, device="cpu") manager.add_module(processor, "lingbot_vla_v2_processor", path=qwen3vl_root) - load_lingbot_vla_v2(manager, model_root, qwen3vl_root, torch_dtype=dtype) + load_lingbot_vla_v2( + manager, + model_root, + qwen3vl_root, + torch_dtype=dtype, + device=target_device if quant_config.enabled else None, + quant_config=quant_config if quant_config.enabled else None, + ) pipeline = LingBotVlaV2Pipeline(device=str(target_device), torch_dtype=dtype) pipeline.init( manager, @@ -43,6 +87,7 @@ def get_lingbot_vla_v2_pipeline( device_type=target_device.type, device_id=target_device.index or 0, torch_dtype=dtype, + quant_config=quant_config, ), ), ) diff --git a/telefuser/pipelines/lingbot_vla_v2/service.py b/telefuser/pipelines/lingbot_vla_v2/service.py index f0f78b38..79efb291 100644 --- a/telefuser/pipelines/lingbot_vla_v2/service.py +++ b/telefuser/pipelines/lingbot_vla_v2/service.py @@ -1,4 +1,4 @@ -"""Minimal single-GPU HTTP service for LingBot-VLA v2 action inference.""" +"""Structured request adapter for LingBot-VLA v2 action inference.""" from __future__ import annotations @@ -6,35 +6,16 @@ import binascii import io import math -import threading -from collections.abc import Callable -from contextlib import asynccontextmanager -from dataclasses import dataclass from typing import Protocol from PIL import Image, UnidentifiedImageError -from fastapi import FastAPI, HTTPException -from fastapi.concurrency import run_in_threadpool from pydantic import BaseModel, ConfigDict, Field, field_validator from .data import LingBotVlaV2Observation from .pipeline import LingBotVlaV2CanonicalActionChunk from .robot_profile import ROBOTWIN_CAMERA_KEYS -from .runtime import get_lingbot_vla_v2_pipeline - -@dataclass(frozen=True) -class LingBotVlaV2ServiceConfig: - """Configuration for one process-local LingBot-VLA v2 replica.""" - - model_root: str - qwen3vl_root: str - device: str = "cuda:0" - max_image_bytes: int = 10 * 1024 * 1024 - - def __post_init__(self) -> None: - if self.max_image_bytes <= 0: - raise ValueError("max_image_bytes must be positive") +DEFAULT_MAX_IMAGE_PIXELS = 16 * 1024 * 1024 class LingBotVlaV2ActionRequest(BaseModel): @@ -78,15 +59,6 @@ class LingBotVlaV2ActionResponse(BaseModel): verification_status: str -class LingBotVlaV2HealthResponse(BaseModel): - """Readiness state for the process-local model replica.""" - - status: str - model: str - device: str - policy_verified: bool - - class _Pipeline(Protocol): def __call__( self, @@ -94,17 +66,17 @@ def __call__( seed: int | None = None, ) -> LingBotVlaV2CanonicalActionChunk: ... - def close(self) -> None: ... - -PipelineFactory = Callable[[LingBotVlaV2ServiceConfig], _Pipeline] - - -def _default_pipeline_factory(config: LingBotVlaV2ServiceConfig) -> _Pipeline: - return get_lingbot_vla_v2_pipeline(config.model_root, config.qwen3vl_root, device=config.device, warmup=True) - - -def _decode_image(value: str, *, max_image_bytes: int) -> Image.Image: +def _decode_image( + value: str, + *, + max_image_bytes: int, + max_image_pixels: int, +) -> Image.Image: + if max_image_bytes <= 0: + raise ValueError("max_image_bytes must be positive") + if max_image_pixels <= 0: + raise ValueError("max_image_pixels must be positive") payload = value.strip() if payload.startswith("data:"): header, separator, payload = payload.partition(",") @@ -121,7 +93,12 @@ def _decode_image(value: str, *, max_image_bytes: int) -> Image.Image: raise ValueError(f"decoded image must contain 1 to {max_image_bytes} bytes") try: with Image.open(io.BytesIO(decoded)) as image: + pixel_count = image.width * image.height + if pixel_count > max_image_pixels: + raise ValueError(f"decoded image must not exceed {max_image_pixels} pixels") return image.convert("RGB").copy() + except Image.DecompressionBombError as error: + raise ValueError(f"decoded image must not exceed {max_image_pixels} pixels") from error except (UnidentifiedImageError, OSError) as error: raise ValueError("decoded payload must be a supported image") from error @@ -131,11 +108,16 @@ def predict_lingbot_vla_v2_action( request: LingBotVlaV2ActionRequest, *, max_image_bytes: int, + max_image_pixels: int = DEFAULT_MAX_IMAGE_PIXELS, ) -> LingBotVlaV2ActionResponse: """Decode one request and return the canonical normalized action chunk.""" encoded_images = (request.camera_high, request.camera_left_wrist, request.camera_right_wrist) images = { - key: _decode_image(value, max_image_bytes=max_image_bytes) + key: _decode_image( + value, + max_image_bytes=max_image_bytes, + max_image_pixels=max_image_pixels, + ) for key, value in zip(ROBOTWIN_CAMERA_KEYS, encoded_images, strict=True) } observation = LingBotVlaV2Observation(task=request.task, state=request.state, images=images) @@ -148,59 +130,3 @@ def predict_lingbot_vla_v2_action( policy_verified=chunk.policy_verified, verification_status=chunk.verification_status, ) - - -class LingBotVlaV2Service: - """Serialize requests through one loaded policy replica.""" - - def __init__(self, pipeline: _Pipeline, config: LingBotVlaV2ServiceConfig) -> None: - self.pipeline = pipeline - self.config = config - self._inference_lock = threading.Lock() - - def predict(self, request: LingBotVlaV2ActionRequest) -> LingBotVlaV2ActionResponse: - """Decode one request and run it on the process-local replica.""" - with self._inference_lock: - return predict_lingbot_vla_v2_action(self.pipeline, request, max_image_bytes=self.config.max_image_bytes) - - def close(self) -> None: - """Release model resources during application shutdown.""" - self.pipeline.close() - - -def create_lingbot_vla_v2_app( - config: LingBotVlaV2ServiceConfig, - *, - pipeline_factory: PipelineFactory = _default_pipeline_factory, -) -> FastAPI: - """Create a FastAPI application backed by exactly one policy replica.""" - - @asynccontextmanager - async def lifespan(app: FastAPI): - service = LingBotVlaV2Service(pipeline_factory(config), config) - app.state.lingbot_vla_v2_service = service - try: - yield - finally: - service.close() - - app = FastAPI(title="LingBot VLA v2", version="1", lifespan=lifespan) - - @app.get("/health", response_model=LingBotVlaV2HealthResponse) - async def health() -> LingBotVlaV2HealthResponse: - return LingBotVlaV2HealthResponse( - status="ready", - model="lingbot-vla-v2-6b-base", - device=config.device, - policy_verified=False, - ) - - @app.post("/v1/vla/actions", response_model=LingBotVlaV2ActionResponse) - async def predict(request: LingBotVlaV2ActionRequest) -> LingBotVlaV2ActionResponse: - service: LingBotVlaV2Service = app.state.lingbot_vla_v2_service - try: - return await run_in_threadpool(service.predict, request) - except (TypeError, ValueError) as error: - raise HTTPException(status_code=422, detail=str(error)) from error - - return app diff --git a/tests/integration/test_lingbot_vla_v2_quantization_lifecycle.py b/tests/integration/test_lingbot_vla_v2_quantization_lifecycle.py new file mode 100644 index 00000000..7201961f --- /dev/null +++ b/tests/integration/test_lingbot_vla_v2_quantization_lifecycle.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import gc +from importlib import util + +import pytest +import torch +from torch import nn + +from telefuser.core.config import QuantConfig, QuantKernelBackend, QuantType +from telefuser.models.lingbot_vla_v2 import LingBotVlaV2Model +from telefuser.models.lingbot_vla_v2_quantization import lingbot_vla_v2_quantization_identity + +pytestmark = [pytest.mark.gpu, pytest.mark.quant] + + +def _module_available(module_name: str) -> bool: + try: + return util.find_spec(module_name) is not None + except (ImportError, ValueError): + return False + + +def _quantizable_model(width: int = 64) -> LingBotVlaV2Model: + model = LingBotVlaV2Model.__new__(LingBotVlaV2Model) + nn.Module.__init__(model) + model.qwenvl_with_expert = nn.Module() + model.qwenvl_with_expert.qwenvl = nn.Module() + model.qwenvl_with_expert.qwenvl.model = nn.Module() + model.qwenvl_with_expert.qwenvl.model.language_model = nn.Module() + model.qwenvl_with_expert.qwenvl.model.language_model.layers = nn.ModuleList([nn.Linear(width, width)]) + model.qwenvl_with_expert.qwenvl.model.visual = nn.Module() + model.qwenvl_with_expert.qwenvl.model.visual.blocks = nn.ModuleList([nn.Linear(width, width)]) + model.qwenvl_with_expert.qwen_expert = nn.Module() + model.qwenvl_with_expert.qwen_expert.model = nn.Module() + action_layer = nn.Module() + action_layer.self_attn = nn.Module() + action_layer.self_attn.q_proj = nn.Linear(width, width) + model.qwenvl_with_expert.qwen_expert.model.layers = nn.ModuleList([action_layer]) + model.action_out_proj = nn.Linear(width, width) + return model + + +@pytest.mark.parametrize( + ("distribution", "quant_type", "backend", "profile"), + [ + ("torchao", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO, "torchao-fp8"), + ("bitsandbytes", QuantType.BNB_NF4, QuantKernelBackend.BITSANDBYTES, "bnb-nf4"), + ], +) +def test_online_quantization_repeated_forward_and_release( + distribution: str, + quant_type: QuantType, + backend: QuantKernelBackend, + profile: str, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("CUDA is required") + if not _module_available(distribution): + pytest.skip(f"{distribution} is not importable") + + device = torch.device("cuda:0") + torch.cuda.empty_cache() + + def run_cycle() -> int: + model = _quantizable_model().to(device=device, dtype=torch.bfloat16).eval() + model.enable_quant(QuantConfig(enabled=True, quant_type=quant_type, kernel_backend=backend)) + selected = dict(model.named_modules())["qwenvl_with_expert.qwenvl.model.language_model.layers.0"] + inputs = torch.randn(2, 64, device=device, dtype=torch.bfloat16) + + with torch.inference_mode(): + first = selected(inputs) + second = selected(inputs) + torch.cuda.synchronize(device) + + assert first.shape == (2, 64) + assert torch.isfinite(first).all() + assert torch.equal(first, second) + identity = lingbot_vla_v2_quantization_identity(model) + assert identity["profile"] == profile + assert identity["manifest"]["selected_count"] == 3 + + model.to(device="cpu") + del first, second, selected, inputs, model + gc.collect() + torch.cuda.empty_cache() + return torch.cuda.memory_allocated(device) + + first_cycle_floor = run_cycle() + second_cycle_floor = run_cycle() + + # TorchAO may retain one process-wide dispatch cache after first use. Repeating + # the model lifecycle must not retain another model-sized allocation. + assert second_cycle_floor <= first_cycle_floor + 1024**2 diff --git a/tests/unit/models/test_lingbot_vla_v2_loader.py b/tests/unit/models/test_lingbot_vla_v2_loader.py index b0e01136..5995578e 100644 --- a/tests/unit/models/test_lingbot_vla_v2_loader.py +++ b/tests/unit/models/test_lingbot_vla_v2_loader.py @@ -7,6 +7,7 @@ import pytest import torch +from telefuser.core.config import QuantConfig, QuantType from telefuser.models.lingbot_vla_v2_loader import ( build_official_6b_config, load_lingbot_vla_v2, @@ -117,4 +118,17 @@ def fetch_module(self, name: str): "checkpoint_variant": "base", "checkpoint_path": str(tmp_path), }, + "quant_config": None, } + + quant_config = QuantConfig(enabled=True, quant_type=QuantType.TORCHAO_FP8) + load_lingbot_vla_v2( + manager, + tmp_path, + tmp_path / "qwen3vl", + torch_dtype=torch.bfloat16, + device="cuda:0", + quant_config=quant_config, + ) + assert manager.load_kwargs["device"] == "cuda:0" + assert manager.load_kwargs["quant_config"] is quant_config diff --git a/tests/unit/models/test_lingbot_vla_v2_quantization.py b/tests/unit/models/test_lingbot_vla_v2_quantization.py new file mode 100644 index 00000000..67f80b9b --- /dev/null +++ b/tests/unit/models/test_lingbot_vla_v2_quantization.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +from collections.abc import Callable +from types import SimpleNamespace + +import pytest +import torch +from torch import nn + +from telefuser.core.config import QuantConfig, QuantKernelBackend, QuantType +from telefuser.models.lingbot_vla_v2 import ( + LINGBOT_VLA_V2_DEFAULT_QUANTIZE_MODULES, + LingBotVlaV2Model, +) +from telefuser.models.lingbot_vla_v2_quantization import ( + LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZATION_MANIFEST_SHA256, + LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT, + build_lingbot_vla_v2_linear_manifest, + linear_compute_dtype, + lingbot_vla_v2_quantization_identity, +) + + +def _empty_model() -> LingBotVlaV2Model: + model = LingBotVlaV2Model.__new__(LingBotVlaV2Model) + nn.Module.__init__(model) + return model + + +def _quantizable_model() -> LingBotVlaV2Model: + model = _empty_model() + model.qwenvl_with_expert = nn.Module() + model.qwenvl_with_expert.qwenvl = nn.Module() + model.qwenvl_with_expert.qwenvl.model = nn.Module() + model.qwenvl_with_expert.qwenvl.model.language_model = nn.Module() + model.qwenvl_with_expert.qwenvl.model.language_model.layers = nn.ModuleList([nn.Linear(4, 4)]) + model.qwenvl_with_expert.qwenvl.model.visual = nn.Module() + model.qwenvl_with_expert.qwenvl.model.visual.blocks = nn.ModuleList([nn.Linear(4, 4)]) + model.qwenvl_with_expert.qwen_expert = nn.Module() + model.qwenvl_with_expert.qwen_expert.model = nn.Module() + action_layer = nn.Module() + action_layer.self_attn = nn.Module() + action_layer.self_attn.q_proj = nn.Linear(4, 4) + action_layer.mlp = nn.Module() + action_layer.mlp.shared_expert = nn.Module() + action_layer.mlp.shared_expert.up_proj = nn.Linear(4, 4) + model.qwenvl_with_expert.qwen_expert.model.layers = nn.ModuleList([action_layer]) + model.action_out_proj = nn.Linear(4, 4) + return model + + +@pytest.mark.parametrize( + ("quant_type", "backend", "helper_path", "count_attribute"), + [ + ( + QuantType.TORCHAO_FP8, + QuantKernelBackend.TORCHAO, + "telefuser.ops.torchao_fp8_linear.replace_linear_layers_with_torchao_fp8", + "torchao_fp8_replaced_linear", + ), + ( + QuantType.BNB_NF4, + QuantKernelBackend.BITSANDBYTES, + "telefuser.ops.bnb_nf4_linear.replace_linear_layers_with_bnb_nf4", + "bnb_nf4_replaced_linear", + ), + ], +) +def test_online_quantization_uses_vla_safe_linear_selection( + monkeypatch: pytest.MonkeyPatch, + quant_type: QuantType, + backend: QuantKernelBackend, + helper_path: str, + count_attribute: str, +) -> None: + model = _quantizable_model() + calls: list[dict[str, object]] = [] + + def fake_replace(_module: nn.Module, **kwargs: object) -> int: + calls.append(kwargs) + return 3 + + monkeypatch.setattr(helper_path, fake_replace) + model.enable_quant(QuantConfig(enabled=True, quant_type=quant_type, kernel_backend=backend)) + + assert calls[0]["include_names"] == LINGBOT_VLA_V2_DEFAULT_QUANTIZE_MODULES + exclude_names = calls[0]["exclude_names"] + assert isinstance(exclude_names, tuple) + assert "action_out_proj" in exclude_names + assert "state_proj" in exclude_names + assert getattr(model, count_attribute) == 3 + assert model.quant_type == quant_type + identity = lingbot_vla_v2_quantization_identity(model) + assert identity["profile"] in {"torchao-fp8", "bnb-nf4"} + assert identity["kernel_backend"] == backend.name + assert identity["manifest"]["selected_count"] == 3 + + +def test_tf_kernel_fp8_quantization_filters_action_heads_and_moe(monkeypatch: pytest.MonkeyPatch) -> None: + model = _quantizable_model() + captured_filter: Callable[[str, nn.Module], bool] | None = None + + def fake_count(_module: nn.Module, **kwargs: object) -> int: + nonlocal captured_filter + captured_filter = kwargs["module_filter"] # type: ignore[assignment] + return 3 + + def fake_enable(_module: nn.Module, **_kwargs: object) -> nn.Module: + return _module + + monkeypatch.setattr("telefuser.ops.fp8_gemm.count_linear_layers", fake_count) + monkeypatch.setattr("telefuser.ops.fp8_gemm.enable_fp8_gemm", fake_enable) + + model.enable_quant(QuantConfig(enabled=True, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL)) + + assert captured_filter is not None + linear = nn.Linear(2, 2) + assert captured_filter("model.qwenvl_with_expert.qwenvl.model.language_model.layers.0.mlp.up_proj", linear) + assert captured_filter("model.qwenvl_with_expert.qwenvl.model.visual.blocks.0.mlp.linear_fc1", linear) + assert captured_filter("model.qwenvl_with_expert.qwen_expert.model.layers.0.self_attn.q_proj", linear) + assert not captured_filter("model.qwenvl_with_expert.qwen_expert.model.layers.0.mlp.shared_expert.up_proj", linear) + assert not captured_filter("model.action_out_proj", linear) + assert model.tf_kernel_fp8_replaced_linear == 3 + assert model.quant_type == QuantType.FP8 + + +def test_online_quantization_rejects_unsupported_type() -> None: + model = _empty_model() + with pytest.raises(ValueError, match="does not support"): + model.enable_quant(QuantConfig(enabled=True, quant_type=QuantType.INT8)) + + +def test_quantization_manifest_freezes_selected_layers_and_groups() -> None: + model = _quantizable_model() + + manifest = build_lingbot_vla_v2_linear_manifest( + model, + include_names=LINGBOT_VLA_V2_DEFAULT_QUANTIZE_MODULES, + exclude_names=("action_out_proj", "shared_expert"), + ) + + assert manifest["selected_count"] == 3 + assert manifest["group_counts"] == { + "action_expert_attention": 1, + "qwen_language": 1, + "qwen_visual": 1, + } + assert manifest["excluded_names"] == [ + "action_out_proj", + "qwenvl_with_expert.qwen_expert.model.layers.0.mlp.shared_expert.up_proj", + ] + assert len(manifest["manifest_sha256"]) == 64 + + +def test_official_base_profile_rejects_manifest_drift() -> None: + model = _quantizable_model() + model.config = SimpleNamespace(checkpoint_variant="base") + + with pytest.raises( + RuntimeError, + match=rf"expected count={LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT}", + ): + model.enable_quant( + QuantConfig( + enabled=True, + quant_type=QuantType.TORCHAO_FP8, + kernel_backend=QuantKernelBackend.TORCHAO, + ) + ) + + +def test_official_base_profile_rejects_same_count_with_different_manifest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model = _quantizable_model() + model.config = SimpleNamespace(checkpoint_variant="base") + monkeypatch.setattr( + "telefuser.models.lingbot_vla_v2.build_lingbot_vla_v2_linear_manifest", + lambda *_args, **_kwargs: { + "selected_count": LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZED_LINEAR_COUNT, + "selected_names": [], + "excluded_count": 0, + "excluded_names": [], + "group_counts": {}, + "manifest_sha256": "0" * 64, + }, + ) + + with pytest.raises( + RuntimeError, + match=rf"sha256={LINGBOT_VLA_V2_OFFICIAL_6B_QUANTIZATION_MANIFEST_SHA256}", + ): + model.enable_quant( + QuantConfig( + enabled=True, + quant_type=QuantType.TORCHAO_FP8, + kernel_backend=QuantKernelBackend.TORCHAO, + ) + ) + + +def test_online_quantization_is_idempotent_for_same_profile(monkeypatch: pytest.MonkeyPatch) -> None: + model = _quantizable_model() + calls = 0 + + def fake_replace(_module: nn.Module, **_kwargs: object) -> int: + nonlocal calls + calls += 1 + return 3 + + monkeypatch.setattr( + "telefuser.ops.torchao_fp8_linear.replace_linear_layers_with_torchao_fp8", + fake_replace, + ) + config = QuantConfig( + enabled=True, + quant_type=QuantType.TORCHAO_FP8, + kernel_backend=QuantKernelBackend.TORCHAO, + ) + + model.enable_quant(config) + model.enable_quant(config) + + assert calls == 1 + + +def test_online_quantization_rejects_second_backend(monkeypatch: pytest.MonkeyPatch) -> None: + model = _quantizable_model() + monkeypatch.setattr( + "telefuser.ops.torchao_fp8_linear.replace_linear_layers_with_torchao_fp8", + lambda _module, **_kwargs: 3, + ) + model.enable_quant( + QuantConfig( + enabled=True, + quant_type=QuantType.TORCHAO_FP8, + kernel_backend=QuantKernelBackend.TORCHAO, + ) + ) + + with pytest.raises(RuntimeError, match="already quantized"): + model.enable_quant( + QuantConfig( + enabled=True, + quant_type=QuantType.BNB_NF4, + kernel_backend=QuantKernelBackend.BITSANDBYTES, + ) + ) + + +def test_linear_compute_dtype_prefers_wrapper_compute_dtype() -> None: + wrapper = nn.Module() + wrapper.register_buffer("weight", torch.zeros(2, 2, dtype=torch.uint8)) + wrapper.compute_dtype = torch.float16 + + assert linear_compute_dtype(wrapper, torch.float32) == torch.float16 + + +def test_unquantized_identity_reports_bf16_without_manifest() -> None: + identity = lingbot_vla_v2_quantization_identity(_empty_model()) + + assert identity["enabled"] is False + assert identity["profile"] == "bf16" + assert identity["manifest"] is None + + +def test_linear_compute_dtype_falls_back_for_weightless_wrapper() -> None: + assert linear_compute_dtype(nn.Identity(), torch.bfloat16) == torch.bfloat16 diff --git a/tests/unit/pipelines/lingbot_vla_v2/test_request_adapter.py b/tests/unit/pipelines/lingbot_vla_v2/test_request_adapter.py new file mode 100644 index 00000000..e50045a5 --- /dev/null +++ b/tests/unit/pipelines/lingbot_vla_v2/test_request_adapter.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import base64 +import io + +import pytest +import torch +from PIL import Image +from pydantic import ValidationError + +from telefuser.pipelines.lingbot_vla_v2.pipeline import LingBotVlaV2CanonicalActionChunk +from telefuser.pipelines.lingbot_vla_v2.robot_profile import ROBOTWIN_CAMERA_KEYS +from telefuser.pipelines.lingbot_vla_v2.service import ( + LingBotVlaV2ActionRequest, + predict_lingbot_vla_v2_action, +) + + +def _encoded_image(*, data_url: bool = False) -> str: + buffer = io.BytesIO() + Image.new("RGB", (8, 8), color=(10, 20, 30)).save(buffer, format="PNG") + encoded = base64.b64encode(buffer.getvalue()).decode("ascii") + return f"data:image/png;base64,{encoded}" if data_url else encoded + + +def _payload() -> dict: + image = _encoded_image() + return { + "task": "pick up the red block", + "state": [0.0] * 14, + "camera_high": image, + "camera_left_wrist": image, + "camera_right_wrist": image, + "seed": 7, + } + + +class _Pipeline: + def __init__(self) -> None: + self.observations = [] + self.seeds = [] + + def __call__(self, observation, seed=None) -> LingBotVlaV2CanonicalActionChunk: + self.observations.append(observation) + self.seeds.append(seed) + return LingBotVlaV2CanonicalActionChunk( + canonical_normalized_actions=torch.zeros(2, 55), + horizon=2, + action_dim=55, + ) + + +def test_request_adapter_returns_normalized_action_contract() -> None: + pipeline = _Pipeline() + request = LingBotVlaV2ActionRequest.model_validate(_payload()) + + response = predict_lingbot_vla_v2_action(pipeline, request, max_image_bytes=1024 * 1024) + + assert response.horizon == 2 + assert response.action_dim == 55 + assert response.checkpoint_variant == "base" + assert response.policy_verified is False + assert response.verification_status == "unverified_official_6b_base" + assert len(response.canonical_normalized_actions[0]) == 55 + assert pipeline.seeds == [7] + assert tuple(pipeline.observations[0].images) == ROBOTWIN_CAMERA_KEYS + assert all(image.mode == "RGB" for image in pipeline.observations[0].images.values()) + + +def test_request_adapter_accepts_image_data_urls() -> None: + pipeline = _Pipeline() + payload = _payload() + payload["camera_high"] = _encoded_image(data_url=True) + + predict_lingbot_vla_v2_action( + pipeline, + LingBotVlaV2ActionRequest.model_validate(payload), + max_image_bytes=1024 * 1024, + ) + + assert len(pipeline.observations) == 1 + + +def test_request_adapter_rejects_invalid_image() -> None: + pipeline = _Pipeline() + request = LingBotVlaV2ActionRequest.model_validate({**_payload(), "camera_high": "not-base64"}) + + with pytest.raises(ValueError, match="image must be valid base64"): + predict_lingbot_vla_v2_action(pipeline, request, max_image_bytes=1024 * 1024) + + assert pipeline.observations == [] + + +def test_request_adapter_rejects_non_positive_image_limit() -> None: + pipeline = _Pipeline() + request = LingBotVlaV2ActionRequest.model_validate(_payload()) + + with pytest.raises(ValueError, match="max_image_bytes must be positive"): + predict_lingbot_vla_v2_action(pipeline, request, max_image_bytes=0) + + assert pipeline.observations == [] + + +def test_request_adapter_rejects_image_over_pixel_limit() -> None: + pipeline = _Pipeline() + request = LingBotVlaV2ActionRequest.model_validate(_payload()) + + with pytest.raises(ValueError, match="decoded image must not exceed 63 pixels"): + predict_lingbot_vla_v2_action( + pipeline, + request, + max_image_bytes=1024 * 1024, + max_image_pixels=63, + ) + + assert pipeline.observations == [] + + +def test_request_adapter_rejects_non_positive_pixel_limit() -> None: + pipeline = _Pipeline() + request = LingBotVlaV2ActionRequest.model_validate(_payload()) + + with pytest.raises(ValueError, match="max_image_pixels must be positive"): + predict_lingbot_vla_v2_action( + pipeline, + request, + max_image_bytes=1024 * 1024, + max_image_pixels=0, + ) + + assert pipeline.observations == [] + + +@pytest.mark.parametrize( + "payload", + ( + {**_payload(), "state": [0.0] * 13}, + {**_payload(), "state": [0.0] * 13 + [float("inf")]}, + {**_payload(), "output_path": "/tmp/action"}, + ), +) +def test_request_adapter_rejects_invalid_request(payload: dict) -> None: + with pytest.raises(ValidationError): + LingBotVlaV2ActionRequest.model_validate(payload) diff --git a/tests/unit/pipelines/lingbot_vla_v2/test_runtime.py b/tests/unit/pipelines/lingbot_vla_v2/test_runtime.py new file mode 100644 index 00000000..6ae8943f --- /dev/null +++ b/tests/unit/pipelines/lingbot_vla_v2/test_runtime.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import pytest + +from telefuser.core.config import QuantKernelBackend, QuantType +from telefuser.pipelines.lingbot_vla_v2.runtime import ( + get_lingbot_vla_v2_pipeline, + lingbot_vla_v2_quant_config, +) + + +@pytest.mark.parametrize( + ("name", "quant_type", "backend"), + [ + ("torchao-fp8", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO), + ("torchao_fp8", QuantType.TORCHAO_FP8, QuantKernelBackend.TORCHAO), + ("tf-kernel-fp8", QuantType.FP8, QuantKernelBackend.TF_KERNEL), + ("bnb-nf4", QuantType.BNB_NF4, QuantKernelBackend.BITSANDBYTES), + ], +) +def test_quantization_names_resolve_to_existing_runtime_config( + name: str, + quant_type: QuantType, + backend: QuantKernelBackend, +) -> None: + config = lingbot_vla_v2_quant_config(name) + + assert config.enabled is True + assert config.quant_type == quant_type + assert config.kernel_backend == backend + + +def test_default_runtime_quantization_keeps_bf16_path_disabled() -> None: + assert lingbot_vla_v2_quant_config(None).enabled is False + + +def test_online_quantization_rejects_cpu_before_loading_models() -> None: + with pytest.raises(ValueError, match="requires a CUDA device"): + get_lingbot_vla_v2_pipeline("unused", "unused", device="cpu", quantization="bnb-nf4") + + +def test_quantization_rejects_unknown_name() -> None: + with pytest.raises(ValueError, match="quantization must be"): + lingbot_vla_v2_quant_config("int8") diff --git a/tests/unit/pipelines/lingbot_vla_v2/test_service.py b/tests/unit/pipelines/lingbot_vla_v2/test_service.py deleted file mode 100644 index a79c33d4..00000000 --- a/tests/unit/pipelines/lingbot_vla_v2/test_service.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import annotations - -import base64 -import io -import threading -import time -from concurrent.futures import ThreadPoolExecutor - -import torch -from PIL import Image -from fastapi.testclient import TestClient - -from telefuser.pipelines.lingbot_vla_v2.pipeline import LingBotVlaV2CanonicalActionChunk -from telefuser.pipelines.lingbot_vla_v2.robot_profile import ROBOTWIN_CAMERA_KEYS -from telefuser.pipelines.lingbot_vla_v2.service import ( - LingBotVlaV2ActionRequest, - LingBotVlaV2Service, - LingBotVlaV2ServiceConfig, - create_lingbot_vla_v2_app, -) - - -def _encoded_image(*, data_url: bool = False) -> str: - buffer = io.BytesIO() - Image.new("RGB", (8, 8), color=(10, 20, 30)).save(buffer, format="PNG") - encoded = base64.b64encode(buffer.getvalue()).decode("ascii") - return f"data:image/png;base64,{encoded}" if data_url else encoded - - -def _payload() -> dict: - image = _encoded_image() - return { - "task": "pick up the red block", - "state": [0.0] * 14, - "camera_high": image, - "camera_left_wrist": image, - "camera_right_wrist": image, - "seed": 7, - } - - -class _Pipeline: - def __init__(self, *, delay: float = 0.0) -> None: - self.delay = delay - self.closed = False - self.observations = [] - self.seeds = [] - self.active = 0 - self.max_active = 0 - self._counter_lock = threading.Lock() - - def __call__(self, observation, seed=None) -> LingBotVlaV2CanonicalActionChunk: - with self._counter_lock: - self.active += 1 - self.max_active = max(self.max_active, self.active) - try: - time.sleep(self.delay) - self.observations.append(observation) - self.seeds.append(seed) - return LingBotVlaV2CanonicalActionChunk( - canonical_normalized_actions=torch.zeros(2, 55), - horizon=2, - action_dim=55, - ) - finally: - with self._counter_lock: - self.active -= 1 - - def close(self) -> None: - self.closed = True - - -def _config(**kwargs) -> LingBotVlaV2ServiceConfig: - return LingBotVlaV2ServiceConfig( - model_root="/models/lingbot-vla-v2-6b", - qwen3vl_root="/models/Qwen3-VL-4B-Instruct", - **kwargs, - ) - - -def test_app_serves_health_and_normalized_action_contract() -> None: - pipeline = _Pipeline() - config = _config(device="cuda:3") - app = create_lingbot_vla_v2_app(config, pipeline_factory=lambda received: pipeline) - - with TestClient(app) as client: - health = client.get("/health") - response = client.post("/v1/vla/actions", json=_payload()) - - assert health.status_code == 200 - assert health.json() == { - "status": "ready", - "model": "lingbot-vla-v2-6b-base", - "device": "cuda:3", - "policy_verified": False, - } - assert response.status_code == 200 - body = response.json() - assert body["horizon"] == 2 - assert body["action_dim"] == 55 - assert body["checkpoint_variant"] == "base" - assert body["policy_verified"] is False - assert body["verification_status"] == "unverified_official_6b_base" - assert len(body["canonical_normalized_actions"]) == 2 - assert len(body["canonical_normalized_actions"][0]) == 55 - assert pipeline.seeds == [7] - assert tuple(pipeline.observations[0].images) == ROBOTWIN_CAMERA_KEYS - assert all(image.mode == "RGB" for image in pipeline.observations[0].images.values()) - assert pipeline.closed is True - - -def test_app_accepts_image_data_urls() -> None: - pipeline = _Pipeline() - payload = _payload() - payload["camera_high"] = _encoded_image(data_url=True) - app = create_lingbot_vla_v2_app(_config(), pipeline_factory=lambda received: pipeline) - - with TestClient(app) as client: - response = client.post("/v1/vla/actions", json=payload) - - assert response.status_code == 200 - - -def test_app_rejects_invalid_observations_without_running_policy() -> None: - pipeline = _Pipeline() - app = create_lingbot_vla_v2_app(_config(), pipeline_factory=lambda received: pipeline) - payload = _payload() - payload["camera_high"] = "not-base64" - - with TestClient(app) as client: - invalid_image = client.post("/v1/vla/actions", json=payload) - invalid_state = client.post("/v1/vla/actions", json={**_payload(), "state": [0.0] * 13}) - extra_field = client.post("/v1/vla/actions", json={**_payload(), "output_path": "/tmp/action"}) - - assert invalid_image.status_code == 422 - assert invalid_image.json()["detail"] == "image must be valid base64" - assert invalid_state.status_code == 422 - assert extra_field.status_code == 422 - assert pipeline.observations == [] - - -def test_service_serializes_policy_calls() -> None: - pipeline = _Pipeline(delay=0.02) - service = LingBotVlaV2Service(pipeline, _config()) - request = LingBotVlaV2ActionRequest.model_validate(_payload()) - - with ThreadPoolExecutor(max_workers=2) as executor: - futures = [executor.submit(service.predict, request) for _ in range(2)] - responses = [future.result() for future in futures] - - assert [response.horizon for response in responses] == [2, 2] - assert pipeline.max_active == 1 - - -def test_service_config_rejects_non_positive_image_limit() -> None: - try: - _config(max_image_bytes=0) - except ValueError as error: - assert str(error) == "max_image_bytes must be positive" - else: - raise AssertionError("expected an invalid image size limit to be rejected") diff --git a/tests/unit/service/test_structured_tasks.py b/tests/unit/service/test_structured_tasks.py index 41c7adb4..a2004672 100644 --- a/tests/unit/service/test_structured_tasks.py +++ b/tests/unit/service/test_structured_tasks.py @@ -145,6 +145,23 @@ def __call__(self, observation, seed=None): assert len(result["canonical_normalized_actions"][0]) == 55 +def test_native_vla_pipeline_keeps_bf16_default_and_forwards_quantization(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[dict] = [] + sentinel = object() + + def fake_get_pipeline(*_args, **kwargs): + calls.append(kwargs) + return sentinel + + monkeypatch.setattr(lingbot_vla_v2_native_service, "get_lingbot_vla_v2_pipeline", fake_get_pipeline) + assert lingbot_vla_v2_native_service.get_pipeline() is sentinel + assert calls[-1]["quantization"] is None + + monkeypatch.setitem(lingbot_vla_v2_native_service.PPL_CONFIG, "quantization", "torchao-fp8") + assert lingbot_vla_v2_native_service.get_pipeline() is sentinel + assert calls[-1]["quantization"] == "torchao-fp8" + + def test_unified_client_encodes_vla_inputs_and_returns_result(tmp_path: Path) -> None: image_path = tmp_path / "camera.png" Image.new("RGB", (8, 8)).save(image_path) diff --git a/tests/unit/validation/test_lingbot_vla_v2_artifacts.py b/tests/unit/validation/test_lingbot_vla_v2_artifacts.py index 09865983..58585be4 100644 --- a/tests/unit/validation/test_lingbot_vla_v2_artifacts.py +++ b/tests/unit/validation/test_lingbot_vla_v2_artifacts.py @@ -138,6 +138,22 @@ def test_compare_artifacts_rejects_different_artifact_identity(tmp_path: Path) - compare_artifacts(reference, candidate, rtol=0.0, atol=0.0) +def test_compare_artifacts_rejects_weak_checkpoint_hash_for_strict_profile(tmp_path: Path) -> None: + metadata = _metadata() + metadata["checkpoint_hash_mode"] = "filename_and_size" + reference = _write_artifact(tmp_path, "reference", _arrays(), metadata) + candidate = _write_artifact(tmp_path, "candidate", _arrays(), metadata) + + with pytest.raises(ValueError, match="Strict parity requires full_sha256"): + compare_artifacts( + reference, + candidate, + rtol=0.0, + atol=0.0, + require_full_checkpoint_hash=True, + ) + + def test_compare_artifacts_rejects_different_moe_backends(tmp_path: Path) -> None: candidate_metadata = _metadata() candidate_metadata["moe_backend"] = "upstream_triton" diff --git a/tests/unit/validation/test_lingbot_vla_v2_benchmark.py b/tests/unit/validation/test_lingbot_vla_v2_benchmark.py index 1a42b45f..49484c57 100644 --- a/tests/unit/validation/test_lingbot_vla_v2_benchmark.py +++ b/tests/unit/validation/test_lingbot_vla_v2_benchmark.py @@ -31,6 +31,7 @@ def test_latency_summary_reports_interpolated_percentiles_and_throughput() -> No assert result["count"] == 4 assert result["mean_seconds"] == 2.5 assert result["p95_seconds"] == pytest.approx(3.85) + assert result["p99_seconds"] == pytest.approx(3.97) assert result["throughput_requests_per_second"] == 0.4 diff --git a/tests/unit/validation/test_lingbot_vla_v2_quantization_comparison.py b/tests/unit/validation/test_lingbot_vla_v2_quantization_comparison.py new file mode 100644 index 00000000..79e93afa --- /dev/null +++ b/tests/unit/validation/test_lingbot_vla_v2_quantization_comparison.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +from tools.validation.compare_lingbot_vla_v2_quantization import ( + action_error_metrics, + compare_quantized_actions, +) + + +def _artifact(path: Path, action: np.ndarray, *, profile: str = "bf16", input_sha: str = "input") -> Path: + np.savez(path, canonical_normalized_actions=action) + metadata = { + "checkpoint_manifest_sha256": "checkpoint", + "processor_manifest_sha256": "processor", + "norm_stats_sha256": "norm", + "input_sha256": input_sha, + "seed": 7, + "num_steps": 10, + "attention_backend": "eager", + "moe_backend": "deterministic_torch_reference", + "quantization": {"profile": profile, "enabled": profile != "bf16"}, + } + path.with_suffix(".json").write_text(json.dumps(metadata), encoding="utf-8") + return path + + +def test_action_error_metrics_reports_cosine_and_relative_l2() -> None: + reference = np.asarray([[1.0, 2.0], [3.0, 4.0]]) + candidate = reference + 0.1 + + metrics = action_error_metrics(reference, candidate) + + assert metrics["shape"] == [2, 2] + assert metrics["max_abs"] == pytest.approx(0.1) + assert 0.0 < metrics["relative_l2"] < 0.1 + assert 0.99 < metrics["cosine"] < 1.0 + + +def test_comparison_is_report_only_without_thresholds(tmp_path: Path) -> None: + reference = _artifact(tmp_path / "bf16.npz", np.ones((50, 55), dtype=np.float32)) + candidate = _artifact( + tmp_path / "torchao.npz", + np.full((50, 55), 1.01, dtype=np.float32), + profile="torchao-fp8", + ) + + report = compare_quantized_actions(reference, candidate) + + assert report["passed"] is True + assert report["mode"] == "report_only" + assert report["metadata"]["candidate_quantization"]["profile"] == "torchao-fp8" + + +def test_comparison_enforces_threshold_and_exact_replay(tmp_path: Path) -> None: + reference_action = np.arange(20, dtype=np.float32).reshape(4, 5) + candidate_action = reference_action + 0.01 + reference = _artifact(tmp_path / "bf16.npz", reference_action) + candidate = _artifact(tmp_path / "nf4.npz", candidate_action, profile="bnb-nf4") + replay = _artifact(tmp_path / "nf4_replay.npz", candidate_action.copy(), profile="bnb-nf4") + + report = compare_quantized_actions( + reference, + candidate, + candidate_replay_path=replay, + min_cosine=0.99, + max_relative_l2=0.01, + require_exact_replay=True, + ) + + assert report["passed"] is True + assert report["mode"] == "thresholded" + assert report["checks"]["exact_replay"] is True + + +def test_comparison_rejects_mismatched_input_identity(tmp_path: Path) -> None: + action = np.ones((2, 3), dtype=np.float32) + reference = _artifact(tmp_path / "bf16.npz", action, input_sha="one") + candidate = _artifact(tmp_path / "quant.npz", action, profile="torchao-fp8", input_sha="two") + + with pytest.raises(ValueError, match="identity metadata differs"): + compare_quantized_actions(reference, candidate) + + +def test_comparison_rejects_bf16_candidate(tmp_path: Path) -> None: + action = np.ones((2, 3), dtype=np.float32) + reference = _artifact(tmp_path / "reference.npz", action) + candidate = _artifact(tmp_path / "candidate.npz", action) + + with pytest.raises(ValueError, match="non-BF16"): + compare_quantized_actions(reference, candidate) diff --git a/tests/unit/validation/test_lingbot_vla_v2_runtime_benchmark.py b/tests/unit/validation/test_lingbot_vla_v2_runtime_benchmark.py new file mode 100644 index 00000000..5e5c8e41 --- /dev/null +++ b/tests/unit/validation/test_lingbot_vla_v2_runtime_benchmark.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from tools.validation.benchmark_lingbot_vla_v2_runtime import percentile, summarize +from tools.validation.compare_lingbot_vla_v2_runtime_benchmarks import compare_reports, render_markdown + + +def _report(implementation: str, latency: float) -> dict: + summary = { + "mean_seconds": latency, + "p50_seconds": latency, + "p95_seconds": latency, + "p99_seconds": latency, + } + return { + "benchmark": "lingbot_vla_v2_upstream_telefuser_runtime", + "implementation": implementation, + "implementation_commit": "commit", + "model_root": "/models/vla", + "qwen3vl_root": "/models/qwen", + "input_artifact": "/inputs/parity.npz", + "seed": 7, + "warmup_runs": 3, + "measured_runs": 20, + "device": "cuda:0", + "device_name": "H100", + "environment": { + "python_version": "3.10.12", + "torch_version": "2.11.0+cu130", + "cuda_version": "13.0", + "transformers_version": "4.57.3", + "platform": "Linux-test", + }, + "attention_backend": "eager", + "moe_backend": "robby_triton", + "load_seconds": 7.0, + "gpu_peak_allocated_mib": 12000.0, + "core_model_latency": summary, + "runtime_request_latency": summary, + } + + +def test_summary_includes_p99_and_throughput() -> None: + values = [1.0, 2.0, 3.0, 4.0] + + result = summarize(values) + + assert percentile(values, 0.5) == 2.5 + assert result["p99_seconds"] == pytest.approx(3.97) + assert result["throughput_requests_per_second"] == 0.4 + + +def test_compare_reports_calculates_telefuser_change() -> None: + report = compare_reports(_report("official_upstream", 1.0), _report("telefuser", 0.8)) + + comparison = report["core_model_latency"]["mean_seconds"] + assert comparison["telefuser_change_percent"] == pytest.approx(-20.0) + assert comparison["speedup_upstream_over_telefuser"] == pytest.approx(1.25) + assert "TeleFuser (ms)" in render_markdown(report) + + +def test_compare_reports_rejects_backend_mismatch() -> None: + upstream = _report("official_upstream", 1.0) + telefuser = _report("telefuser", 0.8) + telefuser["moe_backend"] = "fused_fallback" + + with pytest.raises(ValueError, match="moe_backend"): + compare_reports(upstream, telefuser) + + +def test_compare_reports_rejects_environment_mismatch() -> None: + upstream = _report("official_upstream", 1.0) + telefuser = _report("telefuser", 0.8) + telefuser["environment"] = {**telefuser["environment"], "torch_version": "different"} + + with pytest.raises(ValueError, match="environment"): + compare_reports(upstream, telefuser) diff --git a/tools/validation/benchmark_lingbot_vla_v2_runtime.py b/tools/validation/benchmark_lingbot_vla_v2_runtime.py new file mode 100644 index 00000000..f9fb4d3d --- /dev/null +++ b/tools/validation/benchmark_lingbot_vla_v2_runtime.py @@ -0,0 +1,318 @@ +"""Benchmark upstream or TeleFuser LingBot-VLA v2 inference without parity capture hooks.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import platform +import statistics +import subprocess +import sys +import time +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import transformers + + +def percentile(values: Sequence[float], fraction: float) -> float: + """Return a linearly interpolated percentile for a non-empty sample.""" + if not values: + raise ValueError("percentile requires at least one value") + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def summarize(values: Sequence[float]) -> dict[str, float | int]: + """Summarize synchronized wall-clock latency samples in seconds.""" + if not values: + raise ValueError("summary requires at least one value") + total = sum(values) + return { + "count": len(values), + "total_seconds": total, + "mean_seconds": statistics.fmean(values), + "stdev_seconds": statistics.pstdev(values), + "min_seconds": min(values), + "p50_seconds": percentile(values, 0.50), + "p90_seconds": percentile(values, 0.90), + "p95_seconds": percentile(values, 0.95), + "p99_seconds": percentile(values, 0.99), + "max_seconds": max(values), + "throughput_requests_per_second": len(values) / total, + } + + +def _git_commit(repository: Path) -> str: + completed = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + +def _load_cpu_inputs(path: Path) -> tuple[dict[str, torch.Tensor], torch.Tensor]: + with np.load(path) as artifact: + required = { + "images", + "img_masks", + "lang_tokens", + "lang_masks", + "state", + "image_grid_thw", + "initial_noise", + } + missing = sorted(required.difference(artifact.files)) + if missing: + raise ValueError(f"parity input artifact is missing arrays: {missing}") + tensors = { + "images": torch.from_numpy(artifact["images"]).to(dtype=torch.bfloat16), + "img_masks": torch.from_numpy(artifact["img_masks"]).to(dtype=torch.bool), + "lang_tokens": torch.from_numpy(artifact["lang_tokens"]).to(dtype=torch.long), + "lang_masks": torch.from_numpy(artifact["lang_masks"]).to(dtype=torch.bool), + "state": torch.from_numpy(artifact["state"]).to(dtype=torch.bfloat16), + "image_grid_thw": torch.from_numpy(artifact["image_grid_thw"]).to(dtype=torch.long), + } + noise = torch.from_numpy(artifact["initial_noise"]).to(dtype=torch.bfloat16) + return tensors, noise + + +def _to_device(tensors: dict[str, torch.Tensor], device: torch.device) -> dict[str, torch.Tensor]: + return {name: tensor.to(device=device) for name, tensor in tensors.items()} + + +def _run_samples( + operation: Callable[[], torch.Tensor], + *, + device: torch.device, + warmup: int, + runs: int, +) -> tuple[dict[str, float | int], torch.Tensor]: + for _ in range(warmup): + operation() + torch.cuda.synchronize(device) + samples: list[float] = [] + output: torch.Tensor | None = None + for _ in range(runs): + torch.cuda.synchronize(device) + started_at = time.perf_counter() + output = operation() + torch.cuda.synchronize(device) + samples.append(time.perf_counter() - started_at) + assert output is not None + return summarize(samples), output + + +def _output_summary(output: torch.Tensor) -> dict[str, Any]: + snapshot = output.detach().to(device="cpu", dtype=torch.float32).contiguous() + if tuple(snapshot.shape) != (1, 50, 55): + raise RuntimeError(f"unexpected action shape: {tuple(snapshot.shape)}") + if not torch.isfinite(snapshot).all(): + raise RuntimeError("benchmark produced non-finite actions") + array = snapshot.numpy() + return { + "shape": list(array.shape), + "dtype": str(array.dtype), + "minimum": float(array.min()), + "maximum": float(array.max()), + "mean": float(array.mean()), + "sha256_float32_le": hashlib.sha256(array.astype(" tuple[Any, dict[str, Any]]: + import capture_lingbot_vla_v2_upstream as upstream_capture + + upstream_root = args.upstream_root.resolve() + commit = upstream_capture._git_commit(upstream_root) + if commit != upstream_capture.UPSTREAM_COMMIT: + raise RuntimeError(f"expected upstream commit {upstream_capture.UPSTREAM_COMMIT}, got {commit}") + sys.path.insert(0, str(upstream_root)) + config = upstream_capture._build_config(args.qwen3vl_root.resolve()) + model = upstream_capture._load_official_model(args.model_root.resolve(), config, device) + + import lingbotvla.models.vla.lingbot_vla.qwen2_action_expert as upstream_moe + + backend = "robby_triton" if upstream_moe.robby_moe_forward is not None else "fused_fallback" + return model, { + "implementation": "official_upstream", + "implementation_commit": commit, + "attention_backend": "eager", + "moe_backend": backend, + } + + +def _load_telefuser(args: argparse.Namespace, device: torch.device) -> tuple[Any, dict[str, Any]]: + from telefuser.pipelines.lingbot_vla_v2.runtime import get_lingbot_vla_v2_pipeline + + pipeline = get_lingbot_vla_v2_pipeline( + str(args.model_root.resolve()), + str(args.qwen3vl_root.resolve()), + device=str(device), + quantization=args.quantization, + ) + model = pipeline.policy_stage.policy + config = model.config + return (pipeline, model), { + "implementation": "telefuser", + "implementation_commit": _git_commit(Path(__file__).resolve().parents[2]), + "attention_backend": str(config.attention_implementation), + "moe_backend": "robby_triton" if bool(config.use_robby_moe_kernel) else "fused_fallback", + } + + +def run_benchmark(args: argparse.Namespace) -> dict[str, Any]: + """Run synchronized core-model and runtime-boundary benchmarks.""" + if args.warmup < 0 or args.runs < 1: + raise ValueError("--warmup must be non-negative and --runs must be positive") + if args.implementation == "upstream" and args.quantization is not None: + raise ValueError("--quantization is only supported with --implementation telefuser") + device = torch.device(args.device) + if device.type != "cuda" or not torch.cuda.is_available(): + raise RuntimeError("LingBot-VLA v2 runtime benchmarking requires CUDA") + torch.cuda.set_device(device) + torch.empty(1, device=device) + cpu_inputs, cpu_noise = _load_cpu_inputs(args.input_artifact) + + torch.cuda.reset_peak_memory_stats(device) + load_started_at = time.perf_counter() + loaded, identity = ( + _load_upstream(args, device) if args.implementation == "upstream" else _load_telefuser(args, device) + ) + torch.cuda.synchronize(device) + load_seconds = time.perf_counter() - load_started_at + pipeline = loaded[0] if args.implementation == "telefuser" else None + model = loaded[1] if args.implementation == "telefuser" else loaded + if pipeline is not None: + from telefuser.models.lingbot_vla_v2_quantization import lingbot_vla_v2_quantization_identity + + quantization_runtime = lingbot_vla_v2_quantization_identity(model) + else: + quantization_runtime = { + "enabled": False, + "profile": "bf16", + "quant_type": None, + "kernel_backend": None, + "implementation": "official_upstream", + "manifest": None, + } + + device_inputs = _to_device(cpu_inputs, device) + device_noise = cpu_noise.to(device=device) + + @torch.inference_mode() + def core_model() -> torch.Tensor: + return model.sample_actions(**device_inputs, noise=device_noise) + + @torch.inference_mode() + def runtime_request() -> torch.Tensor: + if pipeline is not None: + from telefuser.pipelines.lingbot_vla_v2.data import LingBotVlaV2Inputs + + prepared = LingBotVlaV2Inputs(**cpu_inputs) + return pipeline.predict(prepared, seed=args.seed).canonical_normalized_actions.unsqueeze(0) + request_inputs = _to_device(cpu_inputs, device) + generator = torch.Generator(device=device).manual_seed(args.seed) + noise = torch.randn( + 1, + int(model.config.n_action_steps), + int(model.config.max_action_dim), + device=device, + dtype=torch.bfloat16, + generator=generator, + ) + return model.sample_actions(**request_inputs, noise=noise).detach().to(device="cpu", dtype=torch.float32) + + try: + core_latency, core_output = _run_samples( + core_model, + device=device, + warmup=args.warmup, + runs=args.runs, + ) + runtime_latency, runtime_output = _run_samples( + runtime_request, + device=device, + warmup=args.warmup, + runs=args.runs, + ) + return { + "schema_version": 1, + "benchmark": "lingbot_vla_v2_upstream_telefuser_runtime", + **identity, + "model_root": str(args.model_root.resolve()), + "qwen3vl_root": str(args.qwen3vl_root.resolve()), + "input_artifact": str(args.input_artifact.resolve()), + "seed": args.seed, + "warmup_runs": args.warmup, + "measured_runs": args.runs, + "device": str(device), + "quantization": args.quantization or "bf16", + "quantization_runtime": quantization_runtime, + "device_name": torch.cuda.get_device_name(device), + "environment": { + "python_version": sys.version.split()[0], + "torch_version": torch.__version__, + "cuda_version": torch.version.cuda, + "transformers_version": transformers.__version__, + "platform": platform.platform(), + }, + "load_seconds": load_seconds, + "core_model_latency": core_latency, + "runtime_request_latency": runtime_latency, + "core_model_output": _output_summary(core_output), + "runtime_request_output": _output_summary(runtime_output), + "gpu_peak_allocated_mib": torch.cuda.max_memory_allocated(device) / 1024**2, + "measurement_notes": [ + "No parity capture hooks are installed.", + "core_model reuses device-resident parity inputs and fixed initial noise.", + ( + "runtime_request includes CPU-to-GPU input transfer, seeded noise creation, validation, " + "and CPU output transfer." + ), + ( + "Image decoding and preprocessing are excluded because both sides consume the same frozen " + "parity tensors." + ), + ], + } + finally: + if pipeline is not None: + pipeline.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--implementation", choices=("upstream", "telefuser"), required=True) + parser.add_argument("--upstream-root", type=Path, default=Path("work_dirs/lingbot-vla-v2-upstream")) + parser.add_argument("--model-root", type=Path, required=True) + parser.add_argument("--qwen3vl-root", type=Path, required=True) + parser.add_argument("--input-artifact", type=Path, required=True) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--quantization", choices=("torchao-fp8", "tf-kernel-fp8", "bnb-nf4")) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--runs", type=int, default=20) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = run_benchmark(args) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"implementation": args.implementation, "output": str(args.output), "passed": True})) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/benchmark_lingbot_vla_v2_service.py b/tools/validation/benchmark_lingbot_vla_v2_service.py index 7db06807..fc8ac820 100644 --- a/tools/validation/benchmark_lingbot_vla_v2_service.py +++ b/tools/validation/benchmark_lingbot_vla_v2_service.py @@ -20,6 +20,7 @@ from PIL import Image from telefuser.metrics.runtime import collect_runtime_environment +from telefuser.models.lingbot_vla_v2_quantization import lingbot_vla_v2_quantization_identity from telefuser.pipelines.lingbot_vla_v2.runtime import get_lingbot_vla_v2_pipeline from telefuser.pipelines.lingbot_vla_v2.service import ( LingBotVlaV2ActionRequest, @@ -109,6 +110,7 @@ def summarize(values: Sequence[float]) -> dict[str, float | int]: "p50_seconds": percentile(values, 0.50), "p90_seconds": percentile(values, 0.90), "p95_seconds": percentile(values, 0.95), + "p99_seconds": percentile(values, 0.99), "max_seconds": max(values), "throughput_requests_per_second": len(values) / total, } @@ -202,11 +204,13 @@ def run_benchmark(args: argparse.Namespace) -> dict[str, Any]: str(args.model_root), str(args.qwen3vl_root), device=str(device), + quantization=args.quantization, ), device=device, process=process, - synchronize_cuda=False, + synchronize_cuda=True, ) + quantization_runtime = lingbot_vla_v2_quantization_identity(pipeline.policy_stage.policy) startup_warmup = None if args.startup_warmup: _, startup_warmup = measure( @@ -315,6 +319,8 @@ def invoke_service_thread() -> Any: "model_root": str(args.model_root.resolve()), "qwen3vl_root": str(args.qwen3vl_root.resolve()), "device": str(device), + "quantization": args.quantization or "bf16", + "quantization_runtime": quantization_runtime, "seed": args.seed, "instruction": args.instruction, "internal_model_image_size": [pipeline.input_processor.image_size] * 2, @@ -331,7 +337,21 @@ def invoke_service_thread() -> Any: finally: if executor is not None: executor.shutdown(wait=True) - pipeline.close() + + def close_and_release_allocator_cache() -> None: + pipeline.close() + if device.type == "cuda": + torch.cuda.empty_cache() + + _, shutdown = measure( + close_and_release_allocator_cache, + device=device, + process=process, + synchronize_cuda=True, + ) + if "report" in locals(): + report["shutdown"] = shutdown + report["memory_after_close"] = _memory_snapshot(device, process) return report @@ -344,6 +364,7 @@ def main() -> None: parser.add_argument("--instruction", default="pick up the red block") parser.add_argument("--seed", type=int, default=7) parser.add_argument("--device", default="cuda:0") + parser.add_argument("--quantization", choices=("torchao-fp8", "tf-kernel-fp8", "bnb-nf4")) parser.add_argument("--execution-mode", choices=("service-thread", "direct"), default="service-thread") parser.add_argument("--warmup", type=int, default=1) parser.add_argument("--runs", type=int, default=20) diff --git a/tools/validation/capture_lingbot_vla_v2_telefuser.py b/tools/validation/capture_lingbot_vla_v2_telefuser.py index f440510c..02f0f4f1 100644 --- a/tools/validation/capture_lingbot_vla_v2_telefuser.py +++ b/tools/validation/capture_lingbot_vla_v2_telefuser.py @@ -13,17 +13,15 @@ import numpy as np import torch import transformers -from transformers import AutoProcessor -from telefuser.core.config import ModelRuntimeConfig -from telefuser.core.module_manager import ModuleManager -from telefuser.models.lingbot_vla_v2_loader import load_lingbot_vla_v2, resolve_lingbot_vla_v2_shards +from telefuser.models.lingbot_vla_v2_loader import resolve_lingbot_vla_v2_shards +from telefuser.models.lingbot_vla_v2_quantization import lingbot_vla_v2_quantization_identity from telefuser.pipelines.lingbot_vla_v2 import ( ROBOTWIN_CAMERA_KEYS, LingBotVlaV2Observation, LingBotVlaV2Pipeline, - LingBotVlaV2PipelineConfig, ) +from telefuser.pipelines.lingbot_vla_v2.runtime import get_lingbot_vla_v2_pipeline ARTIFACT_SCHEMA_VERSION = 1 @@ -61,16 +59,23 @@ def _input_sha256(task: str, state: Sequence[float], image_paths: Sequence[Path] return digest.hexdigest() -def _git_commit() -> str: +def _git_identity() -> tuple[str, bool]: repository_root = Path(__file__).resolve().parents[2] - completed = subprocess.run( + revision = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=repository_root, check=True, capture_output=True, text=True, ) - return completed.stdout.strip() + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=no"], + cwd=repository_root, + check=True, + capture_output=True, + text=True, + ) + return revision.stdout.strip(), bool(status.stdout.strip()) class TensorCapture: @@ -138,25 +143,18 @@ def trace_predict_velocity(flow_model: Any, capture: TensorCapture) -> Iterator[ flow_model._use_compile_predict_velocity = compile_enabled -def _build_pipeline(model_root: Path, qwen3vl_root: Path, device: str) -> LingBotVlaV2Pipeline: - target_device = torch.device(device) - dtype = torch.bfloat16 if target_device.type == "cuda" else torch.float32 - processor = AutoProcessor.from_pretrained(str(qwen3vl_root), local_files_only=True, padding_side="right") - manager = ModuleManager(torch_dtype=dtype, device="cpu") - manager.add_module(processor, "lingbot_vla_v2_processor", path=str(qwen3vl_root)) - load_lingbot_vla_v2(manager, model_root, qwen3vl_root, torch_dtype=dtype) - pipeline = LingBotVlaV2Pipeline(device=device, torch_dtype=dtype) - pipeline.init( - manager, - LingBotVlaV2PipelineConfig( - policy_config=ModelRuntimeConfig( - device_type=target_device.type, - device_id=target_device.index or 0, - torch_dtype=dtype, - ) - ), +def _build_pipeline( + model_root: Path, + qwen3vl_root: Path, + device: str, + quantization: str | None, +) -> LingBotVlaV2Pipeline: + return get_lingbot_vla_v2_pipeline( + str(model_root), + str(qwen3vl_root), + device=device, + quantization=quantization, ) - return pipeline def capture_artifact( @@ -171,6 +169,7 @@ def capture_artifact( device: str, full_checkpoint_hash: bool, deterministic_moe: bool, + quantization: str | None = None, ) -> tuple[Path, Path]: if len(image_paths) != len(ROBOTWIN_CAMERA_KEYS): raise ValueError(f"expected {len(ROBOTWIN_CAMERA_KEYS)} camera paths, got {len(image_paths)}") @@ -178,7 +177,7 @@ def capture_artifact( metadata_path = output.with_suffix(".json") output.parent.mkdir(parents=True, exist_ok=True) - pipeline = _build_pipeline(model_root, qwen3vl_root, device) + pipeline = _build_pipeline(model_root, qwen3vl_root, device, quantization) if deterministic_moe: for module in pipeline.policy_stage.policy.modules(): if hasattr(module, "_use_robby_moe_kernel"): @@ -205,10 +204,12 @@ def capture_artifact( target_device = torch.device(device) checkpoint_paths = [Path(path) for path in resolve_lingbot_vla_v2_shards(model_root)] + telefuser_commit, telefuser_worktree_dirty = _git_identity() metadata = { "schema_version": ARTIFACT_SCHEMA_VERSION, "artifact_kind": "telefuser_regression", - "telefuser_commit": _git_commit(), + "telefuser_commit": telefuser_commit, + "telefuser_worktree_dirty": telefuser_worktree_dirty, "checkpoint_manifest_sha256": _manifest_sha256( checkpoint_paths, include_contents=full_checkpoint_hash, @@ -232,6 +233,7 @@ def capture_artifact( "device_name": torch.cuda.get_device_name(target_device) if target_device.type == "cuda" else "cpu", "torch_version": torch.__version__, "transformers_version": transformers.__version__, + "quantization": lingbot_vla_v2_quantization_identity(pipeline.policy_stage.policy), "arrays": capture.array_metadata, } np.savez(output, **capture.arrays) @@ -267,6 +269,7 @@ def main() -> None: parser.add_argument("--seed", required=True, type=int) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--device", default="cuda:0") + parser.add_argument("--quantization", choices=("torchao-fp8", "tf-kernel-fp8", "bnb-nf4")) parser.add_argument( "--full-checkpoint-hash", action="store_true", @@ -301,6 +304,7 @@ def main() -> None: device=args.device, full_checkpoint_hash=args.full_checkpoint_hash, deterministic_moe=args.deterministic_moe, + quantization=args.quantization, ) print(f"Saved LingBot-VLA v2 capture: {artifact}") print(f"Saved LingBot-VLA v2 metadata: {metadata}") diff --git a/tools/validation/compare_lingbot_vla_v2_quantization.py b/tools/validation/compare_lingbot_vla_v2_quantization.py new file mode 100644 index 00000000..4fab4983 --- /dev/null +++ b/tools/validation/compare_lingbot_vla_v2_quantization.py @@ -0,0 +1,230 @@ +"""Compare LingBot-VLA v2 BF16 and online-quantized action artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np + +ACTION_KEY = "canonical_normalized_actions" +IDENTITY_KEYS = ( + "checkpoint_manifest_sha256", + "processor_manifest_sha256", + "norm_stats_sha256", + "input_sha256", + "seed", + "num_steps", + "attention_backend", + "moe_backend", +) + + +def _load_action(path: Path) -> np.ndarray: + if not path.is_file(): + raise FileNotFoundError(path) + with np.load(path, allow_pickle=False) as payload: + if ACTION_KEY not in payload.files: + raise ValueError(f"{path} does not contain {ACTION_KEY!r}") + action = np.asarray(payload[ACTION_KEY], dtype=np.float64) + if action.ndim == 3 and action.shape[0] == 1: + action = action[0] + if action.ndim != 2: + raise ValueError(f"{path} action must have shape [H, D] or [1, H, D], got {action.shape}") + return np.ascontiguousarray(action) + + +def _load_metadata(path: Path) -> dict[str, Any] | None: + metadata_path = path.with_suffix(".json") + if not metadata_path.is_file(): + return None + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"metadata must be a JSON object: {metadata_path}") + return payload + + +def _sha256(array: np.ndarray) -> str: + return hashlib.sha256(array.astype(" float: + reference_flat = reference.reshape(-1) + candidate_flat = candidate.reshape(-1) + denominator = float(np.linalg.norm(reference_flat) * np.linalg.norm(candidate_flat)) + if denominator == 0.0: + return 1.0 if np.array_equal(reference_flat, candidate_flat) else 0.0 + return float(np.dot(reference_flat, candidate_flat) / denominator) + + +def action_error_metrics(reference: np.ndarray, candidate: np.ndarray) -> dict[str, Any]: + """Return shape, finiteness, magnitude, L2, and cosine action facts.""" + if reference.shape != candidate.shape: + raise ValueError(f"action shapes differ: reference={reference.shape}, candidate={candidate.shape}") + reference_finite = bool(np.isfinite(reference).all()) + candidate_finite = bool(np.isfinite(candidate).all()) + if not reference_finite or not candidate_finite: + return { + "shape": list(candidate.shape), + "reference_finite": reference_finite, + "candidate_finite": candidate_finite, + "max_abs": math.inf, + "mean_abs": math.inf, + "relative_l2": math.inf, + "cosine": 0.0, + "min_step_cosine": 0.0, + "mean_step_cosine": 0.0, + "exact": False, + } + + difference = candidate - reference + reference_norm = float(np.linalg.norm(reference.reshape(-1))) + step_cosines = [_cosine(expected, actual) for expected, actual in zip(reference, candidate, strict=True)] + return { + "shape": list(candidate.shape), + "reference_finite": True, + "candidate_finite": True, + "max_abs": float(np.max(np.abs(difference))) if difference.size else 0.0, + "mean_abs": float(np.mean(np.abs(difference))) if difference.size else 0.0, + "relative_l2": float(np.linalg.norm(difference.reshape(-1)) / max(reference_norm, 1e-12)), + "cosine": _cosine(reference, candidate), + "min_step_cosine": min(step_cosines, default=1.0), + "mean_step_cosine": float(np.mean(step_cosines)) if step_cosines else 1.0, + "exact": bool(np.array_equal(reference, candidate)), + } + + +def _validate_metadata_pair(reference: dict[str, Any] | None, candidate: dict[str, Any] | None) -> dict[str, Any]: + if reference is None or candidate is None: + return {"available": False} + mismatches = { + key: {"reference": reference.get(key), "candidate": candidate.get(key)} + for key in IDENTITY_KEYS + if reference.get(key) != candidate.get(key) + } + if mismatches: + raise ValueError(f"artifact identity metadata differs: {mismatches}") + reference_quantization = reference.get("quantization", {"profile": "bf16", "enabled": False}) + candidate_quantization = candidate.get("quantization") + if isinstance(reference_quantization, dict) and reference_quantization.get("profile") != "bf16": + raise ValueError("reference artifact must use the BF16 profile") + if not isinstance(candidate_quantization, dict) or candidate_quantization.get("profile") in (None, "bf16"): + raise ValueError("candidate artifact must record a non-BF16 quantization profile") + return { + "available": True, + "reference_quantization": reference_quantization, + "candidate_quantization": candidate_quantization, + "matched_identity": {key: reference.get(key) for key in IDENTITY_KEYS}, + } + + +def compare_quantized_actions( + reference_path: Path, + candidate_path: Path, + *, + candidate_replay_path: Path | None = None, + min_cosine: float | None = None, + max_relative_l2: float | None = None, + max_abs: float | None = None, + require_exact_replay: bool = False, +) -> dict[str, Any]: + """Compare one quantized action against BF16 and an optional replay.""" + if min_cosine is not None and not -1.0 <= min_cosine <= 1.0: + raise ValueError("min_cosine must be between -1 and 1") + if max_relative_l2 is not None and max_relative_l2 < 0: + raise ValueError("max_relative_l2 must be non-negative") + if max_abs is not None and max_abs < 0: + raise ValueError("max_abs must be non-negative") + if require_exact_replay and candidate_replay_path is None: + raise ValueError("require_exact_replay requires candidate_replay_path") + + reference = _load_action(reference_path) + candidate = _load_action(candidate_path) + metrics = action_error_metrics(reference, candidate) + metadata = _validate_metadata_pair(_load_metadata(reference_path), _load_metadata(candidate_path)) + checks = { + "shape_matches": reference.shape == candidate.shape, + "finite": bool(metrics["reference_finite"] and metrics["candidate_finite"]), + } + if min_cosine is not None: + checks["min_cosine"] = float(metrics["cosine"]) >= min_cosine + if max_relative_l2 is not None: + checks["max_relative_l2"] = float(metrics["relative_l2"]) <= max_relative_l2 + if max_abs is not None: + checks["max_abs"] = float(metrics["max_abs"]) <= max_abs + + replay_report = None + if candidate_replay_path is not None: + replay = _load_action(candidate_replay_path) + replay_metrics = action_error_metrics(candidate, replay) + replay_report = { + "path": str(candidate_replay_path.resolve()), + "sha256_float64_le": _sha256(replay), + "metrics_vs_candidate": replay_metrics, + } + if require_exact_replay: + checks["exact_replay"] = bool(replay_metrics["exact"]) + + thresholds_enabled = any(value is not None for value in (min_cosine, max_relative_l2, max_abs)) + return { + "schema_version": 1, + "comparison": "lingbot_vla_v2_bf16_vs_online_quantization", + "passed": all(checks.values()), + "mode": "thresholded" if thresholds_enabled or require_exact_replay else "report_only", + "checks": checks, + "thresholds": { + "min_cosine": min_cosine, + "max_relative_l2": max_relative_l2, + "max_abs": max_abs, + "require_exact_replay": require_exact_replay, + }, + "reference": { + "path": str(reference_path.resolve()), + "sha256_float64_le": _sha256(reference), + }, + "candidate": { + "path": str(candidate_path.resolve()), + "sha256_float64_le": _sha256(candidate), + }, + "metadata": metadata, + "action_metrics": metrics, + "candidate_replay": replay_report, + "interpretation": ( + "This is a numerical quantization comparison against the BF16 TeleFuser baseline. " + "It is not strict upstream parity and does not establish robot-control success." + ), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reference", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--candidate-replay", type=Path) + parser.add_argument("--min-cosine", type=float) + parser.add_argument("--max-relative-l2", type=float) + parser.add_argument("--max-abs", type=float) + parser.add_argument("--require-exact-replay", action="store_true") + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + report = compare_quantized_actions( + args.reference, + args.candidate, + candidate_replay_path=args.candidate_replay, + min_cosine=args.min_cosine, + max_relative_l2=args.max_relative_l2, + max_abs=args.max_abs, + require_exact_replay=args.require_exact_replay, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"passed": report["passed"], "mode": report["mode"], "output": str(args.output)})) + raise SystemExit(0 if report["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/compare_lingbot_vla_v2_runtime_benchmarks.py b/tools/validation/compare_lingbot_vla_v2_runtime_benchmarks.py new file mode 100644 index 00000000..86476ea1 --- /dev/null +++ b/tools/validation/compare_lingbot_vla_v2_runtime_benchmarks.py @@ -0,0 +1,147 @@ +"""Compare matched upstream and TeleFuser LingBot-VLA v2 runtime benchmarks.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +_LATENCY_KEYS = ("mean_seconds", "p50_seconds", "p95_seconds", "p99_seconds") + + +def _validate_pair(upstream: dict[str, Any], telefuser: dict[str, Any]) -> None: + required_equal = ( + "benchmark", + "model_root", + "qwen3vl_root", + "input_artifact", + "seed", + "warmup_runs", + "measured_runs", + "device", + "device_name", + "environment", + "attention_backend", + "moe_backend", + ) + mismatches = [key for key in required_equal if upstream.get(key) != telefuser.get(key)] + if mismatches: + raise ValueError(f"benchmark conditions differ for: {', '.join(mismatches)}") + if upstream.get("implementation") != "official_upstream" or telefuser.get("implementation") != "telefuser": + raise ValueError("expected official_upstream and telefuser benchmark reports") + + +def _compare_latency(upstream: dict[str, Any], telefuser: dict[str, Any]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key in _LATENCY_KEYS: + upstream_value = float(upstream[key]) + telefuser_value = float(telefuser[key]) + result[key] = { + "upstream_seconds": upstream_value, + "telefuser_seconds": telefuser_value, + "telefuser_minus_upstream_seconds": telefuser_value - upstream_value, + "telefuser_change_percent": (telefuser_value / upstream_value - 1.0) * 100.0, + "speedup_upstream_over_telefuser": upstream_value / telefuser_value, + } + return result + + +def compare_reports(upstream: dict[str, Any], telefuser: dict[str, Any]) -> dict[str, Any]: + """Validate matched conditions and produce bounded comparison facts.""" + _validate_pair(upstream, telefuser) + return { + "schema_version": 1, + "comparison": "lingbot_vla_v2_upstream_vs_telefuser_runtime", + "conditions": { + key: upstream[key] + for key in ( + "model_root", + "qwen3vl_root", + "input_artifact", + "seed", + "warmup_runs", + "measured_runs", + "device", + "device_name", + "environment", + "attention_backend", + "moe_backend", + ) + }, + "commits": { + "upstream": upstream["implementation_commit"], + "telefuser": telefuser["implementation_commit"], + }, + "core_model_latency": _compare_latency(upstream["core_model_latency"], telefuser["core_model_latency"]), + "runtime_request_latency": _compare_latency( + upstream["runtime_request_latency"], telefuser["runtime_request_latency"] + ), + "load_seconds": { + "upstream": upstream["load_seconds"], + "telefuser": telefuser["load_seconds"], + "comparable": False, + "reason": "The two implementations construct processors and framework objects at different boundaries.", + }, + "gpu_peak_allocated_mib": { + "upstream": upstream["gpu_peak_allocated_mib"], + "telefuser": telefuser["gpu_peak_allocated_mib"], + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + """Render a compact human-readable benchmark table.""" + lines = [ + "# LingBot-VLA v2 Upstream vs TeleFuser Runtime", + "", + "| Scope | Metric | Upstream (ms) | TeleFuser (ms) | Change |", + "|---|---:|---:|---:|---:|", + ] + for scope_key, scope_label in ( + ("core_model_latency", "Core model"), + ("runtime_request_latency", "Runtime request"), + ): + for metric in ("mean_seconds", "p50_seconds", "p95_seconds", "p99_seconds"): + values = report[scope_key][metric] + lines.append( + f"| {scope_label} | {metric.removesuffix('_seconds')} | " + f"{values['upstream_seconds'] * 1000:.3f} | " + f"{values['telefuser_seconds'] * 1000:.3f} | " + f"{values['telefuser_change_percent']:+.2f}% |" + ) + conditions = report["conditions"] + lines.extend( + [ + "", + f"GPU: `{conditions['device_name']}`. Warmup: {conditions['warmup_runs']}; measured runs: " + f"{conditions['measured_runs']}; attention: `{conditions['attention_backend']}`; " + f"MoE: `{conditions['moe_backend']}`.", + "", + "Negative change means TeleFuser is faster. Core model excludes transfers; runtime request includes " + "CPU/GPU transfers, seeded noise construction, validation, and CPU action delivery. Image preprocessing " + "is excluded on both sides by using the frozen parity tensors.", + "", + ] + ) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream", type=Path, required=True) + parser.add_argument("--telefuser", type=Path, required=True) + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-markdown", type=Path, required=True) + args = parser.parse_args() + upstream = json.loads(args.upstream.read_text(encoding="utf-8")) + telefuser = json.loads(args.telefuser.read_text(encoding="utf-8")) + report = compare_reports(upstream, telefuser) + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.output_markdown.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps({"passed": True, "output": str(args.output_json)})) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/inspect_lingbot_vla_v2_gpu_plan.py b/tools/validation/inspect_lingbot_vla_v2_gpu_plan.py new file mode 100644 index 00000000..2ec9eb48 --- /dev/null +++ b/tools/validation/inspect_lingbot_vla_v2_gpu_plan.py @@ -0,0 +1,107 @@ +"""Inspect GPU capacity and estimate a safe LingBot-VLA v2 replica plan.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path +from typing import Any + + +def _query_gpus() -> list[dict[str, Any]]: + result = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=index,name,memory.total,memory.used,memory.free", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + gpus = [] + for line in result.stdout.splitlines(): + fields = [field.strip() for field in line.split(",")] + if len(fields) != 5: + continue + index, name, total, used, free = fields + gpus.append( + { + "index": int(index), + "name": name, + "memory_total_mib": float(total), + "memory_used_mib": float(used), + "memory_free_mib": float(free), + } + ) + return gpus + + +def _checkpoint_bytes(model_root: Path) -> int | None: + candidates = sorted(model_root.rglob("*.safetensors.index.json")) + if not candidates: + return None + index_path = candidates[0] + index = json.loads(index_path.read_text(encoding="utf-8")) + weight_map = index.get("weight_map", {}) + if not isinstance(weight_map, dict): + return None + total = 0 + for shard in sorted(set(weight_map.values())): + shard_path = index_path.parent / str(shard) + if not shard_path.is_file(): + return None + total += shard_path.stat().st_size + return total + + +def build_report(model_root: Path, *, replica_memory_mib: float | None = None) -> dict[str, Any]: + gpus = _query_gpus() + checkpoint_bytes = _checkpoint_bytes(model_root) + report: dict[str, Any] = { + "schema_version": 1, + "model": "lingbot-vla-v2-6b", + "model_root": str(model_root.resolve()), + "checkpoint_bytes_on_disk": checkpoint_bytes, + "visible_gpus": gpus, + "visible_gpu_count": len(gpus), + "visible_total_memory_mib": sum(float(gpu["memory_total_mib"]) for gpu in gpus), + "visible_free_memory_mib": sum(float(gpu["memory_free_mib"]) for gpu in gpus), + "parallelism": { + "single_gpu_replica_supported": True, + "model_parallel_supported_by_current_pipeline": False, + "request_level_replica_supported": True, + "recommended_current_mode": "one full replica per GPU", + "fsdp_or_tensor_parallel": ( + "requires a separate parity-preserving implementation and is not enabled by this tool" + ), + }, + } + if replica_memory_mib is not None: + usable = [gpu for gpu in gpus if gpu["memory_free_mib"] >= replica_memory_mib] + report["replica_memory_mib"] = replica_memory_mib + report["estimated_fit_replicas"] = len(usable) + report["replica_fit_gpu_indexes"] = [gpu["index"] for gpu in usable] + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-root", type=Path, required=True) + parser.add_argument("--replica-memory-mib", type=float, help="Measured resident memory of one full replica.") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.replica_memory_mib is not None and args.replica_memory_mib <= 0: + parser.error("--replica-memory-mib must be positive") + report = build_report(args.model_root, replica_memory_mib=args.replica_memory_mib) + payload = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(payload, encoding="utf-8") + print(payload, end="") + + +if __name__ == "__main__": + main() diff --git a/tools/validation/run_lingbot_vla_v2_parity.py b/tools/validation/run_lingbot_vla_v2_parity.py index 901378d9..349ec084 100644 --- a/tools/validation/run_lingbot_vla_v2_parity.py +++ b/tools/validation/run_lingbot_vla_v2_parity.py @@ -216,6 +216,7 @@ def compare_artifacts( atol: float, reference_metadata: Path | None = None, candidate_metadata: Path | None = None, + require_full_checkpoint_hash: bool = False, ) -> dict[str, object]: expected = _load_npz(reference) actual = _load_npz(candidate) @@ -224,6 +225,15 @@ def compare_artifacts( _validate_contract(expected, expected_metadata, side="reference") _validate_contract(actual, actual_metadata, side="candidate") + if require_full_checkpoint_hash: + invalid_hash_modes = { + side: metadata.get("checkpoint_hash_mode") + for side, metadata in (("reference", expected_metadata), ("candidate", actual_metadata)) + if metadata.get("checkpoint_hash_mode") != "full_sha256" + } + if invalid_hash_modes: + raise ValueError(f"Strict parity requires full_sha256 checkpoint manifests: {invalid_hash_modes}") + metadata_mismatches = { key: {"reference": expected_metadata[key], "candidate": actual_metadata[key]} for key in IDENTITY_METADATA_KEYS @@ -359,6 +369,7 @@ def main() -> None: atol=atol, reference_metadata=args.reference_metadata, candidate_metadata=args.candidate_metadata, + require_full_checkpoint_hash=args.profile == "strict", ) payload = json.dumps(report, indent=2, sort_keys=True) if args.output is None: diff --git a/tools/validation/validate_lingbot_vla_v2_structured_service.py b/tools/validation/validate_lingbot_vla_v2_structured_service.py index b35e7e28..ee43e8c6 100644 --- a/tools/validation/validate_lingbot_vla_v2_structured_service.py +++ b/tools/validation/validate_lingbot_vla_v2_structured_service.py @@ -895,6 +895,7 @@ def run_validation(args: argparse.Namespace) -> dict[str, Any]: "target": { "base_url": base_url, "transport": "HTTP native TeleFuser asynchronous structured task API", + "declared_quantization_profile": getattr(args, "quantization_profile", "bf16"), "metadata": before["metadata"], "status_before": before["status"], "status_after": after["status"], @@ -955,6 +956,12 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--task-timeout-seconds", type=float, default=300.0) parser.add_argument("--expected-horizon", type=int, default=50) parser.add_argument("--expected-action-dim", type=int, default=55) + parser.add_argument( + "--quantization-profile", + choices=("bf16", "torchao-fp8", "tf-kernel-fp8", "bnb-nf4"), + default="bf16", + help="Operator-declared profile recorded in the report; it does not change the running service.", + ) parser.add_argument("--max-records", type=int, default=1000) parser.add_argument( "--service-pid",