diff --git a/.github/workflows/localvqe-benchmark.yml b/.github/workflows/localvqe-benchmark.yml new file mode 100644 index 000000000..733bca969 --- /dev/null +++ b/.github/workflows/localvqe-benchmark.yml @@ -0,0 +1,119 @@ +name: LocalVQE Benchmark + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/localvqe-benchmark.yml' + - 'Sources/FluidAudio/Enhancement/**' + - 'Sources/FluidAudio/ASR/Parakeet/**' + - 'Sources/FluidAudio/Shared/Download/**' + - 'Sources/FluidAudio/ModelNames.swift' + - 'Sources/FluidAudio/ModelRegistry.swift' + - 'Sources/FluidAudioCLI/Commands/EnhanceBenchmark*.swift' + - 'Sources/FluidAudioCLI/Utils/WERCalculator.swift' + - 'Sources/FluidAudioCLI/Utils/TextNormalizer.swift' + - 'Tests/FluidAudioTests/CLI/EnhanceBenchmark*' + - 'Tests/FluidAudioTests/CLI/WERCalculatorTests.swift' + - 'Scripts/verify_localvqe_benchmark.py' + - 'Scripts/test_verify_localvqe_benchmark.py' + - 'Scripts/localvqe-dataset.json' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: localvqe-benchmark-${{ github.ref }} + cancel-in-progress: true + +# The 200-example run is ~7 min on an M5 Pro but far slower on the 3-core +# macOS runner (a single job exceeded 60 min), so it is split into contiguous +# shards that a final job merges and verifies as one 200-example report. +env: + SHARD_COUNT: 5 + +jobs: + regressions: + name: Scoring and manifest regressions + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + - name: Swift scorer / dataset / registry tests + run: swift test --filter 'EnhanceBenchmark|WERCalculatorTests|ModelRegistryTests' + - name: Python verifier tests + run: python3 Scripts/test_verify_localvqe_benchmark.py + + shard: + name: Score shard ${{ matrix.shard }} + runs-on: macos-15 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4] + steps: + - uses: actions/checkout@v5 + + - name: Record environment + run: | + mkdir -p localvqe-results + { + git rev-parse HEAD + swift --version + xcodebuild -version + sw_vers + sysctl -n machdep.cpu.brand_string + } > localvqe-results/environment.txt + + - name: Build release benchmark + run: swift build -c release --product fluidaudiocli + + - name: Score shard ${{ matrix.shard }}/${{ env.SHARD_COUNT }} of the fixed 200-example dataset + run: | + set -o pipefail + .build/release/fluidaudiocli enhance-benchmark \ + --variants v1.3,v1.2 --chunk 256ms --compute-units cpu-only \ + --shard ${{ matrix.shard }}/${{ env.SHARD_COUNT }} \ + --output localvqe-results/benchmark.json 2>&1 | tee localvqe-results/benchmark.log + + - name: Upload shard evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: localvqe-shard-${{ matrix.shard }}-${{ github.sha }} + path: localvqe-results/ + retention-days: 30 + + verify: + name: LocalVQE 200-example ASR benchmark + needs: [shard] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v5 + + - name: Download shard reports + uses: actions/download-artifact@v4 + with: + pattern: localvqe-shard-*-${{ github.sha }} + path: shards + + - name: Merge shards and verify coverage, scores and improvement over unprocessed audio + run: | + mkdir -p localvqe-results + ls shards/*/benchmark.json + python3 Scripts/verify_localvqe_benchmark.py shards/*/benchmark.json \ + --merged localvqe-results/benchmark.json \ + --expected-files 200 --require-improvement --markdown "$GITHUB_STEP_SUMMARY" + LOCALVQE_BENCHMARK_REPORT=localvqe-results/benchmark.json \ + python3 Scripts/test_verify_localvqe_benchmark.py + + - name: Upload merged benchmark evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: localvqe-benchmark-${{ github.sha }} + path: localvqe-results/ + retention-days: 30 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 782354762..97e87892a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,8 +24,14 @@ jobs: - name: Build package run: swift build - - name: Run tests - run: swift test --parallel --num-workers $(sysctl -n hw.ncpu) + # Enrollment tests load the same Sortformer cache from separate XCTest + # workers. Serialize them on a cold runner to avoid racing .partial files. + - name: Run speaker enrollment tests serially + run: swift test --filter SpeakerEnrollmentTests + timeout-minutes: 20 + + - name: Run remaining tests + run: swift test --parallel --num-workers $(sysctl -n hw.ncpu) --skip SpeakerEnrollmentTests timeout-minutes: 20 build-without-nemo-text-processing: @@ -96,4 +102,4 @@ jobs: xcodebuild -scheme FluidAudio \ -destination 'generic/platform=iOS' \ -derivedDataPath .build \ - build \ No newline at end of file + build diff --git a/Documentation/CLI.md b/Documentation/CLI.md index 58296de56..7bc6f0312 100644 --- a/Documentation/CLI.md +++ b/Documentation/CLI.md @@ -109,6 +109,32 @@ swift run fluidaudiocli vad-benchmark --all-files --output vad_results.json --de `swift run fluidaudiocli vad-analyze --help` lists every tuning option (padding, negative threshold overrides, max-duration splitting, etc.). +## Speech Enhancement (LocalVQE) + +```bash +# Echo cancellation + noise suppression: mic capture plus what the speaker played +swift run -c release fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav + +# Noise suppression / dereverb only (silent far end) +swift run -c release fluidaudiocli enhance mic.wav --output clean.wav + +# Drive the streaming API in 256-sample buffers with the 16 ms export and report per-call latency +swift run -c release fluidaudiocli enhance mic.wav -r speaker.wav --chunk 16ms --streaming --buffer-samples 256 +``` + +`--variant v1.2` selects the 1.3M-param checkpoint, `--compute-units gpu` +moves the model off the CPU, and `--model-dir DIR` loads local `.mlmodelc` +bundles instead of downloading. + +```bash +# Near-end word recall / WER / far-end leakage on the AEC-Challenge synthetic mini set (auto-downloads) +swift run -c release fluidaudiocli enhance-benchmark +swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json +# Split the run across machines: contiguous shard i of n, then merge + verify the shard reports +swift run -c release fluidaudiocli enhance-benchmark --shard 0/5 --output shard0.json +python3 Scripts/verify_localvqe_benchmark.py shard*.json --merged results.json --expected-files 200 +``` + ## Datasets ```bash diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md new file mode 100644 index 000000000..640144e1b --- /dev/null +++ b/Documentation/Enhancement/LocalVQE.md @@ -0,0 +1,345 @@ +# LocalVQE — Echo Cancellation & Noise Suppression + +`LocalVqeManager` runs [LocalVQE](https://github.com/localai-org/LocalVQE) +(Apache-2.0), a compact neural model for acoustic echo cancellation (AEC), +noise suppression and dereverberation of 16 kHz speech. It is a streaming, +CPU-tuned derivative of DeepVQE (Indenbom et al., Interspeech 2023). Typical +use: cleaning up call audio captured without headphones, where the mic picks +up what the loudspeaker plays. + +**Beta.** Port fidelity validated (numerically equivalent to the upstream +PyTorch and GGML engines, scored identically to GGML on the 800-clip +AEC-Challenge blind set); the published benchmark is only partially +reproduced, and its ERLE protocol is ambiguous in the public artifacts (see +[Quality](#quality-aec-challenge-blind-test-set)). Not yet exercised inside +production call pipelines. + +## Inputs + +The model takes two 16 kHz mono signals of equal length: + +- **mic** — the microphone capture. +- **reference** — the far-end signal: a loopback of what the loudspeaker + played. Without it the model still denoises and dereverberates; pass + silence (`process(mic:)` does this for you). + +Output is 16 kHz mono, same length as the input, sample-aligned. Level +matches the upstream GGML engine (the OBS plugin and HF demo). + +## Quick start + +```swift +import FluidAudio + +let vqe = try await LocalVqeManager() // downloads v1.3 (256 ms chunk) on first use +let clean = try await vqe.process(mic: micSamples, reference: farEndSamples) + +// Files (any format / rate; converted to 16 kHz mono) +let cleanFile = try await vqe.process(micURL: micURL, referenceURL: speakerURL) +``` + +### Streaming + +```swift +let vqe = try await LocalVqeManager(config: LocalVqeConfig(chunk: .realtime16ms)) +let stream = try await vqe.makeStream() + +// Push buffers of any size as they arrive (mic and reference must be equal length). +let out = try await stream.enhance(mic: micBuffer, reference: refBuffer) + +// End of clip: drain the delay line so total output == total input. +let tail = try await stream.flush() +``` + +Streams from one manager share its model and may run concurrently: inference +uses Core ML's async prediction API, which Apple documents as thread-safe. +Within each stream, `enhance`, `flush`, and `reset` execute in the order they +reach the actor, including across inference suspension points. Feed a stream +from one consumer task and await each push before submitting the next: tasks +launched independently from audio callbacks can arrive out of capture order +and accumulate queued audio. Run enhancement outside the audio render callback. + +`enhance` returns samples as whole model calls complete. Output sample `i` +corresponds to input sample `i`, delivered one hop (256 samples, 16 ms) +after the input that produced it plus whatever is still buffered toward the +next call. `flush()` resets the stream; call `reset()` to start a new clip +without flushing. + +Cancelling an operation while it is queued removes it without changing the +current clip. Cancelling a running push/flush, or an inference error, discards +the unfinished clip and clears recurrent state before the next operation. +Treat this as an audio discontinuity. `reset()` waits for earlier operations; +to abandon an active push, cancel its task, await its completion, then resume +with the next clip. A cancelled queued reset does not reset the stream. + +For live capture, supply continuous 16 kHz mono mic/reference buffers covering +the same time intervals. The reference must be the actual far-end playback +signal. Equal buffer lengths alone do not establish correct timing. Reset on +capture/playback discontinuities, and validate reference timing, route changes, +sustained latency and recovery in the application on its target devices before +enabling enhancement by default. The offline benchmark does not exercise that +live integration. + +Real-model streaming regression tests can be enabled locally or in CI with: + +```bash +FLUIDAUDIO_LOCALVQE_MODEL_DIR=/path/to/compiled/models swift test --filter LocalVqe +``` + +The explicit directory must contain both v1.3 chunk variants; a missing bundle +fails the tests. Without that setting, model tests skip in CI or when the local +cache is absent. The suite covers overlapping pushes, flush/reset ordering, +queued and active cancellation, fresh-clip recovery, and independent streams. +See the [streaming validation report](LocalVQEValidation.md) for the completed +real-model checks, the local XCTest environment limitation and remaining +live-integration coverage. + +## Configuration + +```swift +LocalVqeConfig( + variant: .v13, // .v13 (4.8M params, default) or .v12 (1.3M, ~1/4 the cost) + chunk: .batch256ms, // .batch256ms (files) or .realtime16ms (live capture) + computeUnits: .cpuOnly // fp32 models; CPU is fastest for the 16 ms chunk +) +``` + +Both variants are joint AEC + NS + dereverb models. The chunk size only +changes how many 16 ms hops each Core ML call consumes; the audio is +bit-identical either way. + +| Variant | Chunk | Compute | Per-call p50 | RTFx | +|---|---|---|---:|---:| +| v1.3 | 256 ms | CPU | 7.1 ms | 36× | +| v1.3 | 16 ms | CPU | 1.2 ms | 14× | +| v1.2 | 256 ms | CPU | 4.2 ms | 60× | +| v1.2 | 16 ms | CPU | 0.7 ms | 24× | + +Apple M5 Pro, release build, `fluidaudiocli enhance --streaming`. RTFx is +audio-per-call ÷ p50 latency. GPU gives ~15% on the 256 ms chunk at the cost +of a ~110 ms first-call compile; ANE is not used (see below). + +## CLI + +```bash +swift run -c release fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav +swift run -c release fluidaudiocli enhance mic.wav --output clean.wav # NS/dereverb only +swift run -c release fluidaudiocli enhance mic.wav -r speaker.wav --chunk 16ms --streaming +``` + +`--variant v1.2`, `--compute-units gpu`, `--buffer-samples N` (streaming +buffer size) and `--model-dir DIR` (load local `.mlmodelc` bundles) are also +available; `--help` lists everything. + +## Models + +HuggingFace: [FluidInference/localvqe-coreml](https://huggingface.co/FluidInference/localvqe-coreml). +One `.mlmodelc` per (variant, chunk); only the configured one is downloaded +(19 MB for v1.3, 5 MB for v1.2). Cached under +`~/Library/Application Support/FluidAudio/Models/localvqe/`. + +Manual loading: + +```swift +let vqe = try LocalVqeManager(config: config, modelDirectory: URL(fileURLWithPath: "/path/with/mlmodelc")) +``` + +The models are fp32 streaming exports with explicit state: every call takes +`mic`/`ref` plus 33 `in_*` state tensors and returns `enhanced` plus the +matching `out_*` tensors. fp16 was rejected: it drops parity with the +reference from 102 dB to 5 dB (CPU) / 33 dB (ANE) because the power-law +front-end epsilons underflow and the S4D recurrence accumulates error. +Conversion lives in the [mobius](https://github.com/FluidInference/mobius) +repo under `models/enhancement/localvqe/coreml`. + +## Parity + +Upstream double-talk demo clip (10 s), Swift `LocalVqeStream` output: + +| Against | max abs diff | SNR | +|---|---:|---:| +| Upstream PyTorch reference (fp32, ×2 to the GGML level) | 3.8e-5 | 74 dB (16-bit WAV limited) | +| Upstream GGML CLI (`localvqe-v1.3-4.8M-f32.gguf`) | 2.8e-5 | 80 dB | + +Streaming in 100 / 256 / 1000 / 4096-sample buffers and whole-clip +processing produce the same audio to 1e-5. + +## Quality: AEC-Challenge blind test set + +**Summary: port fidelity validated; the published benchmark is only partially +reproduced, with unresolved v1.2 far-end differences and ERLE protocol.** + +The upstream quality table is AECMOS on the ICASSP 2022 AEC-Challenge blind +set (800 real device recordings). The Swift port was rendered over all 800 +clips and scored two ways, kept separate because they answer different +questions. Scripts and per-clip results live in the mobius repo +(`models/enhancement/localvqe/coreml/score_blind.py`). + +**Challenge protocol (reference).** Microsoft's current scenario-aware AECMOS +model with the challenge's segment rules (convergence portions excluded), +DNSMOS OVRL on the same rated segment, and blind ERLE with the LocalVQE +technical-report gating (far-end-dominated frames only). AECMOS is 1–5, +higher is better. + +| Scenario | n | Unprocessed echo | v1.3 echo / deg / ERLE / OVRL | v1.2 echo / deg / ERLE / OVRL | +|---|--:|--:|---|---| +| doubletalk | 115 | 2.17 | 4.35 / 3.93 / 6.3 dB / 2.89 | 4.20 / 3.63 / 6.2 dB / 2.77 | +| doubletalk-with-movement | 185 | 2.21 | 4.35 / 3.86 / 6.1 dB / 2.84 | 4.13 / 3.57 / 6.0 dB / 2.73 | +| farend-singletalk | 107 | 1.95 | 2.49 / 5.00 / 54.2 dB / 1.95 | 3.92 / 5.00 / 53.2 dB / 1.89 | +| farend-singletalk-with-movement | 193 | 2.23 | 3.08 / 5.00 / 55.9 dB / 1.96 | 4.13 / 5.00 / 47.3 dB / 1.80 | +| nearend-singletalk | 200 | 5.00 | 4.99 / 4.14 / 2.3 dB / 3.17 | 4.99 / 4.09 / 2.1 dB / 3.17 | + +**Upstream protocol (HF model-card reproduction).** The published table was +compared using the legacy AECMOS model over the first 20 s of each clip; that +protocol reproduces the card's unprocessed baseline exactly +(2.67 / 2.56 / 1.90 / 2.13 / 5.00). The card defines ERLE as a plain +whole-signal energy ratio, but its numbers resemble a separately reconstructed +gated metric. `gERLE*` below is that reconstruction, not a confirmed +interpretation of the card's protocol. Under it, the Core ML port gives: + +| Scenario | HF card v1.3 | Core ML v1.3 (echo / deg / gERLE* / OVRL) | HF card v1.2 | Core ML v1.2 (echo / deg / gERLE* / OVRL) | +|---|---|---|---|---| +| doubletalk | 4.73 / 2.62 | 4.73 / 2.62 | 4.72 / 2.37 | 4.72 / 2.39 | +| doubletalk-with-movement | 4.67 / 2.43 | 4.66 / 2.44 | 4.65 / 2.30 | 4.64 / 2.31 | +| farend-singletalk | 3.69 / 4.83 | 3.54 / 4.82 | 3.78 / 4.91 | 4.07 / 4.93 | +| farend-singletalk-with-movement | 3.88 / 4.98 | 3.75 / 4.96 | 4.12 / 4.96 | 4.27 / 4.96 | +| nearend-singletalk | 5.00 / 4.18 | 5.00 / 4.18 | 5.00 / 4.16 | 5.00 / 4.17 | + +The ERLE definition conflict is material: current GGML gives plain / gated +far-end ERLE of 43.0 / 50.9 and 40.1 / 49.4 dB for v1.3, versus card values +50.9 / 49.9; v1.2 gives 38.7 / 48.0 and 30.5 / 41.1 dB, versus 45.7 / 40.6. +Selecting the closer gated result does not prove that upstream used this gate. + +Unprocessed baseline: exact. v1.3: double-talk and near-end within 0.01 +echo MOS; far-end 0.15 low from aligned float output and within 0.04 when +the upstream CLI's raw 16-bit, one-hop-late output is scored instead; +gated ERLE within 0.8 dB and OVRL within 0.01 (full columns in the mobius +README). v1.2: double-talk and near-end within 0.02 echo, 0.02 deg, 0.1 dB +gated ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo ++0.29 / +0.15, gated ERLE +1.9 / +0.7 dB, OVRL +0.09 / +0.05, from either +runtime). Those values are above the published ones, which is not evidence +that the port outperforms upstream; +0.29 echo MOS is not rounding noise. +Rendering v1.2 at the pre-v1.2 delay window (dmax 32, which the reference +config left on the day that row was published) brings far-end ERLE +(44.9 / 40.5 vs 45.7 / 40.6 dB) and degradation (4.88 / 4.96 vs 4.91 / 4.96) +close to the card, but does not establish which configuration upstream used. +Its echo MOS moves farther from the card; +softmax temperature 1.0, the ReLU6 reference, the upstream CLI's output +format and every scorer/segment variation were also tested and rejected +(details in the mobius README). None of the tested configurations reproduces +the whole table. Upstream's evaluation configuration, rendered audio or +scoring script would help resolve the remaining echo cells. An earlier revision of +this page said the v1.3 far-end row could not have come from the published +weights; that was a protocol mismatch and is retracted. + +The [follow-up investigation in mobius](https://github.com/FluidInference/mobius/blob/a066485e4c65790fa21b180b7ddf5e22e0f2d044/models/enhancement/localvqe/coreml/REPRODUCTION.md) +also compares the published PT/GGUF tensors and isolates a historical +upstream state-copy defect. Testing the original engine on all 300 far-end +recordings still did not reproduce the card's echo MOS. Its per-recording +results and precision diagnostics are retained separately from the main +800-clip benchmark. + +**Port fidelity.** The upstream GGML engine was run on the same 800 clips +and scored on identical, aligned whole-hop samples: every per-scenario mean +matches the Core ML port to two decimals, the per-clip echo-MOS delta has +mean +0.0002 (95th percentile 0.017), degradation-MOS 95th percentile 0.0004, +and the aligned waveforms agree at a median 84 dB SNR (numerically +equivalent within 16-bit quantisation, not bit-identical). The only +differences found are artefacts of the upstream CLI (256-sample output +delay, zero-filled trailing hop, and a 16-bit writer that wraps samples above +full scale); the Swift CLI writes float32. + +## Benchmark: near-end recall / far-end leakage + +**Exploratory.** `fluidaudiocli enhance-benchmark` scores the enhancer with +the in-repo Parakeet TDT v3 ASR on a 200-example subset (the first 200 of +shard 0) of the Microsoft AEC-Challenge synthetic *training* set (mic + +loopback + clean near-end triples; +[FluidInference/aec-challenge-synthetic-mini](https://huggingface.co/datasets/FluidInference/aec-challenge-synthetic-mini), +revision `1f3714b5a3f98cedef1bbb017f21bbd7ae688596`, archive SHA256 +`45ff5d7acfce499558c25a0eace45eb819cec8aa76420fe733de7ee116ae548d`). +The default download and cached metadata are verified before use; malformed +rows, duplicate IDs and missing audio now fail instead of silently shrinking +the benchmark. The ASR transcript of the clean near-end clip is the +reference and the loopback transcript gives the far-end words, so the +metrics are relative to machine transcripts, not human ones; examples whose +clean-near-end transcript is empty (33 of 200) are excluded and reported. +Use it to compare conditions, not as an absolute quality figure; the +AEC-Challenge blind-set table above is the quality reference. + +- **Recall**: reference words kept by the hypothesis, `1 - (D + S) / N`. +- **WER**: `(S + D + I) / N` against the clean-near-end transcript. Above + 100% on unprocessed audio because the ASR transcribes the echo as well. +- **Leakage**: far-end words that appear in the hypothesis without being + near-end words, over the far-end word count. + +167 scored examples, signal-to-echo ratio (SER) −10…+10 dB, M5 Pro, 256 ms +chunk, CPU: + +| Condition | Recall | WER | Leakage | RTFx | +|---|---:|---:|---:|---:| +| Unprocessed mic | 44.1% | 112.2% | 34.0% | – | +| LocalVQE v1.3 | **77.6%** | 29.6% | **1.1%** | 34× | +| LocalVQE v1.2 | 73.0% | 33.9% | 1.0% | 57× | +| v1.3, silent reference (NS only) | 42.3% | 101.8% | 23.9% | 34× | + +By SER: at SER ≤ 0 dB (echo louder than speech, 93 files) v1.3 lifts recall +36.4% → 74.5% and cuts leakage 41.1% → 1.6%; at SER > 0 dB (74 files) +54.4% → 81.7% and 25.9% → 0.5%. The silent-reference row shows the model +needs the loopback to cancel echo; without it, it only denoises. + +An earlier revision of this table reported 87.5% / 86.3% recall: the shared +WER scorer had its insertion/deletion labels swapped, so an empty hypothesis +scored 100% recall. Fixed in `WERCalculator` (WER itself was unaffected). +When `--output` is used, the JSON includes the dataset revision and hashes, +numeric-file-ID selection order, exact selected/excluded IDs, both machine +reference transcripts and the raw word-count numerators and denominators. + +### Automated benchmark + +The **LocalVQE Benchmark** GitHub Actions workflow runs all 200 examples with +both variants, CPU-only enhancement and Parakeet v3's default `int8` encoder +(`Encoder.mlmodelc`) on CPU. It runs on relevant PR changes and supports manual +dispatch after the workflow is on the default branch. The `localvqe-asr-v2` +report records per-file S/D/I counts, duration and enhancement time, exact +audio/model SHA256 fingerprints, compute configuration, OS and source revision. +This CPU-only protocol is explicit; the earlier exploratory numbers above +should not be treated as its frozen regression baseline. + +The independent verifier recomputes edits, word leakage and micro-averaged +summaries from the saved transcripts. It requires exact selected/scored/excluded +coverage, both enhancement conditions, and waveform hashes matching the pinned +archive. `Scripts/localvqe-dataset.json` contains the 200 IDs and 600 waveform +hashes extracted from that checksum-verified archive. IDs are sparse; a three-file +smoke run selects `0`, `1`, `10`. No metrics are used to choose that subset. + +The full cloud run must improve recall and reduce leakage relative to its own +unprocessed condition for both variants. This is a broad quality sanity check, +not a claim of reproducing the upstream AECMOS/ERLE table or a held-out evaluation. +The JSON, logs and environment are uploaded as a 30-day workflow artifact; +results also appear in the workflow summary. Virtual-machine RTFx does not +represent physical-device performance. Model fingerprints identify the bytes +actually loaded; model downloads still follow their published repository defaults. + +For a small local pipeline check: + +```bash +swift run -c release fluidaudiocli enhance-benchmark --max-files 3 --output smoke.json +python3 Scripts/verify_localvqe_benchmark.py smoke.json --expected-files 3 +``` + +For the full run, omit `--max-files` and verify with `--expected-files 200 +--require-improvement`. Empty/mismatched audio, invalid arguments, invalid output +samples and zero scored references fail the command. An empty reference is +excluded from every condition and listed explicitly. + +```bash +swift run -c release fluidaudiocli enhance-benchmark # both variants, 200 files +swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json +``` + +## Not included + +Upstream's `v1.4-AEC` (echo-only, keeps room and noise) and the low-power +GTCRN line are GGUF-only and depend on a C++ adaptive-filter front-end with +no PyTorch reference; they are not converted. diff --git a/Documentation/Enhancement/LocalVQEValidation.md b/Documentation/Enhancement/LocalVQEValidation.md new file mode 100644 index 000000000..b33fe4331 --- /dev/null +++ b/Documentation/Enhancement/LocalVQEValidation.md @@ -0,0 +1,77 @@ +# LocalVQE streaming validation + +The stream wrapper serializes each complete `enhance`, `flush`, and `reset` +operation, including across asynchronous prediction. Previously another +operation could enter during inference and overwrite shared input buffers, +consume the same pending audio, or reset state before the first call resumed. +Independent streams still run concurrently on a shared manager. + +Queued cancellation leaves the active clip untouched. Active cancellation or +inference failure clears the unfinished clip, recurrent state and sample +counters before reuse. Reset waits for earlier operations. Callers should +submit capture buffers through one consumer task, in capture order, and await +each push to bound queued audio. See [usage and recovery](LocalVQE.md#streaming). + +## Verification + +Validation performed on Apple Silicon with cached real v1.3 Core ML models: + +- Debug library/CLI compilation and `swift build -c release` passed. +- Strict swift-format lint and `git diff --check` passed. +- A standalone harness linked to the compiled FluidAudio module passed 13 + audio comparisons using the repository's real + `01-validation-request-21.4s.wav` fixture. It exercised the same concurrency + and recovery scenarios as the added regression tests; it did not replace + the model with a mock. + +| Check | Result | +|---|---| +| Overlapping pushes followed by flush | Exact match to sequential processing | +| Reset queued between pushes | Next clip exactly matches a fresh stream | +| Cancelled queued push, flush or reset | Three comparisons; active clip unchanged | +| Active push cancellation, then reuse | `CancellationError`; next clip exactly matches a fresh stream | +| Two independent concurrent streams | Each exactly matches its sequential result | +| Streaming buffers of 100, 256, 1000 and 4096 samples | Four exact matches to whole-clip output | +| 16 ms versus 256 ms model chunk | Maximum absolute difference 8.20e-8 | + +Three predetermined real AEC-Challenge mic/reference pairs also passed a +release CLI smoke check, using v1.3 CPU inference, 16 ms model chunks and +256-sample input buffers. Output was finite, retained the microphone input +length, and was bit-identical between streaming and whole-clip processing. + +| Recording stem | Duration | Samples | +|---|---:|---:| +| `t2U2oyODeEuQhnAt3oCksQ_doubletalk` | 36.58 s | 585280 | +| `Du0RI678G0yhsNVU5AGKTw_farend-singletalk-with-movement` | 23.90 s | 382400 | +| `f2HsvN51L0ygRLcFvf3udg_nearend-singletalk` | 16.53 s | 264480 | + +This smoke check did not rerun the full 800-clip quality benchmark or tune +the model/scorer. The existing CLI reported p50 1.10 ms, p99 1.18 ms, and +maximum 1.98–2.22 ms on these recordings. These are offline timing observations: +the CLI collects timing only for pushes emitting samples, excluding the +initial dropped-hop call and flush. They do not cover startup or certify +live-call deadlines under device contention. + +## XCTest and CI + +The local `swift test --filter LocalVqe` attempt could not execute: this Mac +has Command Line Tools 6.2.3 without Xcode's XCTest framework (`no such module +'XCTest'`). The standalone checks above are reported separately from XCTest. + +On an Xcode-equipped machine, run the regression suite with real models: + +```bash +FLUIDAUDIO_LOCALVQE_MODEL_DIR=/path/to/compiled/models swift test --filter LocalVqe +``` + +The directory must contain the v1.3 16 ms and 256 ms bundles. Supplying it +enables model tests in CI and makes a missing bundle a failure. Without an +explicit directory, model tests retain the existing CI/missing-cache skip. + +## Production boundary + +The stream-wrapper concurrency issue is fixed and its recovery behavior is +verified locally. Live microphone/playback clock alignment, route changes, +audio-callback scheduling, and sustained performance under contention or +thermal load still require the application's capture/playback pipeline on +its target devices. LocalVQE remains beta pending that integration validation. diff --git a/Documentation/Models.md b/Documentation/Models.md index 3aea8b7f8..ca4daea2c 100644 --- a/Documentation/Models.md +++ b/Documentation/Models.md @@ -43,6 +43,12 @@ TDT/CTC and the non-autoregressive models above are wrapped by `SlidingWindowAsr |-------|-------------|---------| | **Silero VAD** | Voice activity detection; speech vs silence on 256ms windows. Segments audio before ASR or diarization. | Support model that other pipelines build on. Converted at the time being the best model out there | +## Speech Enhancement Models + +| Model | Description | Context | +|-------|-------------|---------| +| **LocalVQE** | Neural acoustic echo cancellation + noise suppression + dereverberation for 16 kHz speech (DeepVQE derivative, Apache-2.0). Takes mic + far-end reference, returns clean near-end speech with 16 ms algorithmic latency. fp32 streaming exports with explicit state: v1.3 (4.8M, default) and v1.2 (1.3M) in 256 ms and 16 ms chunk sizes; 36× / 14× RTFx on CPU (M5 Pro). Managed by `LocalVqeManager` / `LocalVqeStream`. | Requested in [#49](https://github.com/FluidInference/FluidAudio/issues/49#issuecomment-5719663475) for hands-free calls. fp16 rejected (parity 102 → 5 dB); ANE not used. Upstream's GGUF-only v1.4-AEC / GTCRN line not converted. | + ## Diarization Models | Model | Description | Context | @@ -86,6 +92,7 @@ Models we converted and tested but are not supported: too large for on-device de | Parakeet EOU | [FluidInference/parakeet-realtime-eou-120m-coreml](https://huggingface.co/FluidInference/parakeet-realtime-eou-120m-coreml) (subdirs: `/160ms`, `/320ms`, `/1280ms`) | | Cohere Transcribe (INT8 hybrid, default) | [FluidInference/cohere-transcribe-03-2026-coreml](https://huggingface.co/FluidInference/cohere-transcribe-03-2026-coreml) (variant: `/q8`) | | Silero VAD | [FluidInference/silero-vad-coreml](https://huggingface.co/FluidInference/silero-vad-coreml) | +| LocalVQE | [FluidInference/localvqe-coreml](https://huggingface.co/FluidInference/localvqe-coreml) (`localvqe-v1.3-4.8M-{16ms,256ms}.mlmodelc`, `localvqe-v1.2-1.3M-{16ms,256ms}.mlmodelc`) | | Diarization (Pyannote) | [FluidInference/speaker-diarization-coreml](https://huggingface.co/FluidInference/speaker-diarization-coreml) | | LS-EEND | [FluidInference/ls-eend-coreml](https://huggingface.co/FluidInference/ls-eend-coreml) (per-dataset optimized variants: `/optimized/ami`, `/optimized/ch`, `/optimized/dih2`, `/optimized/dih3`) | | Sortformer | [FluidInference/diar-streaming-sortformer-coreml](https://huggingface.co/FluidInference/diar-streaming-sortformer-coreml) | diff --git a/Examples/LocalVQEDemo/Package.swift b/Examples/LocalVQEDemo/Package.swift new file mode 100644 index 000000000..332548e27 --- /dev/null +++ b/Examples/LocalVQEDemo/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "LocalVQEDemo", + platforms: [.macOS(.v14)], + dependencies: [ + .package(name: "FluidAudio", path: "../..") + ], + targets: [ + .executableTarget( + name: "LocalVQEDemo", + dependencies: [ + .product(name: "FluidAudio", package: "FluidAudio") + ] + ) + ] +) diff --git a/Examples/LocalVQEDemo/README.md b/Examples/LocalVQEDemo/README.md new file mode 100644 index 000000000..fe1e9937c --- /dev/null +++ b/Examples/LocalVQEDemo/README.md @@ -0,0 +1,38 @@ +# LocalVQE Demo (macOS) + +SwiftUI app that exercises `LocalVqeManager` / `LocalVqeStream` from the +parent package: acoustic echo cancellation, noise suppression and +dereverberation on 16 kHz speech. + +```bash +cd Examples/LocalVQEDemo +swift run -c release LocalVQEDemo +``` + +No Xcode project is needed; the executable target builds with SwiftPM and +opens a regular window. The first **Load model** downloads the selected +variant from `FluidInference/localvqe-coreml`. + +## Files (offline) + +Pick a mic recording and, optionally, the far-end signal the loudspeaker was +playing, then **Enhance**. **Play before → after** plays the mic input then the enhanced output; **Transcribe all** runs Parakeet TDT v3 on every row so the echo words, the near-end words and what survives can be compared as text. The rows are also individually playable and +are written automatically as WAVs to `~/Downloads/LocalVQEDemo//` (**Show WAVs in Finder** opens the folder). **Use sample pair** loads a benchmark clip (default fileid 1148, a clear win; the menu also offers fileid 0, a hard case where near-end speech is lost) from the +AEC-Challenge synthetic set if `fluidaudiocli enhance-benchmark` has fetched it. + +## Live (mic + speaker) + +Plays a far-end file through the default output while capturing the default +input, pairs the two by elapsed time in 256 ms steps, and streams them through +`LocalVqeStream`. The meters show mic, far-end and enhanced level per step: +while only the playback is audible, Enhanced should sit well below Mic; when +you speak, it should follow your voice. **Stop** flushes the stream and offers +the three captured clips for listening, written to the same Downloads folder. + +Use the built-in speaker and mic without headphones so the mic actually hears +the playback. Microphone permission is attributed to the terminal that +launched the app when it is run with `swift run`. + +The far-end reference is the signal rendered by the app's own player, aligned +to the mic by elapsed time. The device round-trip latency is left to the +model's built-in delay search; production integrations should measure it. diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift new file mode 100644 index 000000000..9757b2d19 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift @@ -0,0 +1,71 @@ +import AVFoundation +import FluidAudio +import Foundation + +/// 16 kHz mono helpers shared by the file and live modes. +enum AudioFiles { + static let sampleRate = Double(LocalVqeManager.sampleRate) + + /// Any format/rate → 16 kHz mono Float32. + static func read(_ url: URL) throws -> [Float] { + try AudioConverter().resampleAudioFile(url) + } + + /// 16-bit PCM WAV at 16 kHz mono. + static func write(_ samples: [Float], to url: URL) throws { + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: sampleRate, + AVNumberOfChannelsKey: 1, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + ] + let file = try AVAudioFile( + forWriting: url, settings: settings, commonFormat: .pcmFormatFloat32, interleaved: false) + let buffer = try pcmBuffer(samples) + try file.write(from: buffer) + } + + static func pcmBuffer(_ samples: [Float]) throws -> AVAudioPCMBuffer { + guard let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1), + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(max(samples.count, 1))) + else { + throw DemoError.message("Could not allocate a 16 kHz mono buffer") + } + buffer.frameLength = AVAudioFrameCount(samples.count) + if let channel = buffer.floatChannelData?[0] { + samples.withUnsafeBufferPointer { channel.update(from: $0.baseAddress!, count: samples.count) } + } + return buffer + } + + /// Peak envelope (max |x| per bin) for drawing; `bins` columns. + static func peaks(_ samples: [Float], bins: Int) -> [Float] { + guard !samples.isEmpty, bins > 0 else { return [] } + let per = max(1, samples.count / bins) + return stride(from: 0, to: samples.count, by: per).prefix(bins).map { start in + samples[start..) -> Float { + guard !samples.isEmpty else { return 0 } + return (samples.reduce(0) { $0 + $1 * $1 } / Float(samples.count)).squareRoot() + } + + /// Level in dBFS, floored at -80. + static func dbfs(_ rms: Float) -> Float { + rms <= 0 ? -80 : max(-80, 20 * log10(rms)) + } +} + +enum DemoError: Error, LocalizedError { + case message(String) + + var errorDescription: String? { + switch self { + case .message(let text): return text + } + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift new file mode 100644 index 000000000..d24c21e13 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift @@ -0,0 +1,308 @@ +import FluidAudio +import SwiftUI +import UniformTypeIdentifiers + +struct ContentView: View { + @EnvironmentObject private var model: DemoModel + + var body: some View { + VStack(alignment: .leading, spacing: 16) { + ModelBar() + TabView { + FileModeView().tabItem { Text("Files (offline)") } + LiveModeView().tabItem { Text("Live (mic + speaker)") } + } + } + .padding(20) + .alert( + "Error", + isPresented: Binding(get: { model.errorMessage != nil }, set: { if !$0 { model.errorMessage = nil } }) + ) { + Button("OK") { model.errorMessage = nil } + } message: { + Text(model.errorMessage ?? "") + } + } +} + +private struct ModelBar: View { + @EnvironmentObject private var model: DemoModel + + var body: some View { + HStack(spacing: 12) { + Text("LocalVQE").font(.title2).bold() + Picker("Variant", selection: $model.variant) { + Text("v1.3 (4.8M)").tag(LocalVqeVariant.v13) + Text("v1.2 (1.3M)").tag(LocalVqeVariant.v12) + } + .frame(width: 170) + Picker("Chunk", selection: $model.chunk) { + Text("256 ms").tag(LocalVqeChunk.batch256ms) + Text("16 ms").tag(LocalVqeChunk.realtime16ms) + } + .frame(width: 150) + Button(model.manager == nil ? "Load model" : "Reload") { model.loadModel() } + .disabled(model.isLoading || model.isLive) + if model.isLoading { ProgressView().controlSize(.small) } + Text(model.loadStatus).foregroundStyle(.secondary).lineLimit(1) + Spacer() + } + } +} + +// MARK: - File mode + +private struct FileModeView: View { + @EnvironmentObject private var model: DemoModel + @State private var picking: PickTarget? + + private enum PickTarget: Identifiable { + case mic, reference + var id: Self { self } + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack { + FileField(title: "Mic recording", url: model.micURL) { picking = .mic } + FileField(title: "Far end (optional)", url: model.referenceURL) { picking = .reference } + } + HStack { + Menu { + ForEach(DemoModel.samples, id: \.id) { sample in + Button("fileid \(sample.id) — \(sample.note)") { model.useSamplePair(id: sample.id) } + } + } label: { + Text("Use sample pair") + } primaryAction: { + model.useSamplePair() + } + .fixedSize() + .disabled(!model.hasSamples) + Button("Enhance") { model.processFiles() } + .keyboardShortcut(.defaultAction) + .disabled(model.isProcessing || model.manager == nil || model.micURL == nil) + if model.isProcessing { ProgressView().controlSize(.small) } + Spacer() + SaveButton(clips: model.fileClips) + } + Text(model.fileStatus).font(.callout).foregroundStyle(.secondary) + ClipList(clips: model.fileClips) + if !model.hasSamples { + Text( + "No sample pair found. Run `swift run fluidaudiocli enhance-benchmark --max-files 1` once to fetch the AEC-Challenge synthetic set, or pick your own files." + ) + .font(.footnote).foregroundStyle(.tertiary) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .padding(.top, 8) + .fileImporter( + isPresented: Binding(get: { picking != nil }, set: { if !$0 { picking = nil } }), + allowedContentTypes: [.audio] + ) { result in + guard let target = picking, case .success(let url) = result else { return } + switch target { + case .mic: model.micURL = url + case .reference: model.referenceURL = url + } + picking = nil + } + } +} + +// MARK: - Live mode + +private struct LiveModeView: View { + @EnvironmentObject private var model: DemoModel + @State private var pickingFarEnd = false + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack { + FileField(title: "Far-end file to play", url: model.farEndURL) { pickingFarEnd = true } + Button("Use sample") { model.useSampleFarEnd() }.disabled(!model.hasSamples) + } + HStack { + if model.isLive { + Button("Stop") { model.stopLive() }.keyboardShortcut(.cancelAction) + } else { + Button("Start") { model.startLive() } + .keyboardShortcut(.defaultAction) + .disabled(model.manager == nil || model.farEndURL == nil) + } + Spacer() + SaveButton(clips: model.liveClips) + } + Text(model.liveStatus).font(.callout).foregroundStyle(.secondary) + + GroupBox("Live levels (dBFS per 256 ms step)") { + VStack(spacing: 8) { + LevelMeter(title: "Mic", db: model.liveLevels.micDb, tint: .orange) + LevelMeter(title: "Far end", db: model.liveLevels.referenceDb, tint: .blue) + LevelMeter(title: "Enhanced", db: model.liveLevels.enhancedDb, tint: .green) + HStack { + Text(String(format: "%.1f s captured", model.liveLevels.seconds)) + Spacer() + Text(String(format: "%.1f ms per 256 ms step", model.liveLevels.stepMilliseconds)) + } + .font(.caption.monospacedDigit()).foregroundStyle(.secondary) + } + .padding(6) + } + Text( + "Use the built-in speaker and mic without headphones so the mic actually hears the playback. While only the far end is playing, the Enhanced meter should sit far below Mic; when you talk, it should follow your voice." + ) + .font(.footnote).foregroundStyle(.tertiary) + ClipList(clips: model.liveClips) + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .padding(.top, 8) + .fileImporter(isPresented: $pickingFarEnd, allowedContentTypes: [.audio]) { result in + if case .success(let url) = result { model.farEndURL = url } + } + } +} + +// MARK: - Pieces + +private struct FileField: View { + let title: String + let url: URL? + let pick: () -> Void + + var body: some View { + HStack { + Text(title).frame(width: 130, alignment: .trailing) + Text(url?.lastPathComponent ?? "—") + .lineLimit(1).truncationMode(.middle) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 6).padding(.vertical, 3) + .background(RoundedRectangle(cornerRadius: 4).fill(.quaternary)) + Button("Choose…", action: pick) + } + } +} + +private struct SaveButton: View { + @EnvironmentObject private var model: DemoModel + let clips: [Clip] + + var body: some View { + Button("Show WAVs in Finder") { model.revealOutput() } + .disabled(clips.isEmpty || model.lastOutputDirectory == nil) + } +} + +private struct ClipList: View { + @EnvironmentObject private var model: DemoModel + let clips: [Clip] + + var body: some View { + VStack(spacing: 8) { + if !clips.isEmpty { + HStack { + Button { + if model.player.playingLabel != nil { + model.player.stop() + } else { + model.playBeforeAfter(clips) + } + } label: { + Label( + model.player.playingLabel == nil ? "Play before → after" : "Stop", + systemImage: model.player.playingLabel == nil ? "play.fill" : "stop.fill") + } + .buttonStyle(.borderedProminent) + .accessibilityLabel("Play before and after") + if let label = model.player.playingLabel { + Text("Playing: \(label)").font(.callout).foregroundStyle(.secondary) + } + Button { + model.transcribeAll(clips) + } label: { + Label("Transcribe all \(clips.count)", systemImage: "text.quote") + } + .disabled(model.isTranscribing) + .accessibilityLabel("Transcribe all") + if model.isTranscribing { ProgressView().controlSize(.small) } + Text(model.asrStatus).font(.callout).foregroundStyle(.secondary).lineLimit(1) + Spacer() + } + } + ForEach(clips) { clip in + HStack(spacing: 10) { + Button { + if model.player.playingLabel == clip.label { model.player.stop() } else { model.play(clip) } + } label: { + Image(systemName: model.player.playingLabel == clip.label ? "stop.fill" : "play.fill") + .frame(width: 14) + } + .buttonStyle(.bordered) + Text(clip.label).frame(width: 150, alignment: .leading) + WaveformView(samples: clip.samples, tint: tint(for: clip)) + .frame(height: 44) + Text(String(format: "%.1f s", clip.seconds)) + .font(.caption.monospacedDigit()).foregroundStyle(.secondary).frame(width: 44) + } + if let transcript = model.transcripts[clip.id] { + Text(transcript) + .font(.callout) + .foregroundStyle(tint(for: clip)) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.leading, 200) + } + } + } + } + + private func tint(for clip: Clip) -> Color { + if clip.label.hasPrefix("Enhanced") { return .green } + if clip.label.hasPrefix("Far") { return .blue } + return .orange + } +} + +private struct WaveformView: View { + let samples: [Float] + let tint: Color + + var body: some View { + Canvas { context, size in + let peaks = AudioFiles.peaks(samples, bins: Int(size.width)) + guard !peaks.isEmpty else { return } + var path = Path() + let mid = size.height / 2 + for (x, peak) in peaks.enumerated() { + let h = CGFloat(min(1, peak)) * mid + path.move(to: CGPoint(x: CGFloat(x), y: mid - h)) + path.addLine(to: CGPoint(x: CGFloat(x), y: mid + h)) + } + context.stroke(path, with: .color(tint), lineWidth: 1) + } + .background(RoundedRectangle(cornerRadius: 4).fill(.quaternary.opacity(0.5))) + } +} + +private struct LevelMeter: View { + let title: String + let db: Float + let tint: Color + + var body: some View { + HStack { + Text(title).frame(width: 70, alignment: .trailing) + GeometryReader { geometry in + let fraction = CGFloat(max(0, min(1, (db + 80) / 80))) + ZStack(alignment: .leading) { + RoundedRectangle(cornerRadius: 3).fill(.quaternary) + RoundedRectangle(cornerRadius: 3).fill(tint).frame(width: geometry.size.width * fraction) + } + } + .frame(height: 14) + .animation(.linear(duration: 0.1), value: db) + Text(String(format: "%6.1f dB", db)).font(.caption.monospacedDigit()).frame(width: 64) + } + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift new file mode 100644 index 000000000..174ceb607 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift @@ -0,0 +1,301 @@ +import AVFoundation +import FluidAudio +import Foundation +import AppKit +import SwiftUI + +/// A processed clip ready for A/B listening. +struct Clip: Identifiable { + let id = UUID() + let label: String + let samples: [Float] + var seconds: Double { Double(samples.count) / AudioFiles.sampleRate } +} + +@MainActor +final class DemoModel: ObservableObject { + // Model + @Published var variant: LocalVqeVariant = .v13 + @Published var chunk: LocalVqeChunk = .batch256ms + @Published private(set) var manager: LocalVqeManager? + @Published private(set) var loadStatus = "Not loaded" + @Published private(set) var isLoading = false + + // File mode + @Published var micURL: URL? + @Published var referenceURL: URL? + @Published private(set) var fileClips: [Clip] = [] + @Published private(set) var fileStatus = "Pick a mic recording and the far-end (loudspeaker) signal." + @Published private(set) var isProcessing = false + + // Live mode + @Published var farEndURL: URL? + @Published private(set) var isLive = false + @Published private(set) var liveLevels = LiveLevels() + @Published private(set) var liveClips: [Clip] = [] + @Published private(set) var liveStatus = "Plays the far-end file through your speakers while capturing the mic." + + @Published var errorMessage: String? + + // Transcription (Parakeet TDT v3, same ASR as `enhance-benchmark`) + @Published private(set) var transcripts: [UUID: String] = [:] + @Published private(set) var isTranscribing = false + @Published private(set) var asrStatus = "" + private var asr: AsrManager? + /// Folder the most recent Enhance / live Stop wrote its WAVs to. + @Published private(set) var lastOutputDirectory: URL? + + /// Every run writes mic / far-end / enhanced WAVs here, one subfolder per run. + static let outputRoot = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Downloads/LocalVQEDemo", isDirectory: true) + + let player = Player() + private var live: LiveCapture? + + /// The pinned benchmark dataset, if `enhance-benchmark` has downloaded it. + static let sampleDirectory = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/FluidAudio/Datasets/aec-synthetic-mini") + + /// Sample clips from the pinned set, chosen from the 200-file benchmark rows. + static let samples: [(id: String, note: String)] = [ + ("1148", "clear win: 27 near-end words, 0% → 100% recall, 9 → 0 leaked far-end words"), + ("102", "clear win: 13 near-end words, 0% → 100% recall, 13 → 0 leaked"), + ("1089", "clear win: 14 near-end words, 0% → 100% recall, 13 → 0 leaked"), + ("0", "hard case: −7 dB SER, nonlinear echo; near-end speech is lost (0% recall in the benchmark)"), + ] + + var hasSamples: Bool { + FileManager.default.fileExists( + atPath: Self.sampleDirectory.appendingPathComponent("fileid_\(Self.samples[0].id)_mic.wav").path) + } + + // MARK: - Model + + func loadModel() { + guard !isLoading else { return } + isLoading = true + loadStatus = "Loading \(variant.rawValue) / \(chunk.rawValue)…" + let config = LocalVqeConfig(variant: variant, chunk: chunk) + Task { + do { + let started = ContinuousClock.now + let loaded = try await LocalVqeManager(config: config) { [weak self] progress in + Task { @MainActor in + self?.loadStatus = String(format: "Downloading… %.0f%%", progress.fractionCompleted * 100) + } + } + let ms = started.duration(to: .now).components + manager = loaded + loadStatus = String( + format: "%@ / %@ loaded in %.1f s (CPU)", variant.rawValue, chunk.rawValue, + Double(ms.seconds) + Double(ms.attoseconds) / 1e18) + } catch { + loadStatus = "Load failed" + errorMessage = error.localizedDescription + } + isLoading = false + } + } + + // MARK: - File mode + + func useSamplePair(id: String = samples[0].id) { + micURL = Self.sampleDirectory.appendingPathComponent("fileid_\(id)_mic.wav") + referenceURL = Self.sampleDirectory.appendingPathComponent("fileid_\(id)_lpb.wav") + let note = Self.samples.first { $0.id == id }?.note ?? "" + fileStatus = "AEC-Challenge synthetic set, fileid \(id) — \(note)." + fileClips = [] + } + + func processFiles() { + guard let manager else { + errorMessage = "Load a model first." + return + } + guard let micURL else { + errorMessage = "Pick a mic recording." + return + } + isProcessing = true + fileStatus = "Processing…" + player.stop() + let referenceURL = referenceURL + Task { + do { + let mic = try AudioFiles.read(micURL) + let started = ContinuousClock.now + let enhanced = try await manager.process(micURL: micURL, referenceURL: referenceURL) + let elapsed = started.duration(to: .now).components + let seconds = Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18 + var clips = [Clip(label: "Mic (unprocessed)", samples: mic)] + if let referenceURL { + clips.append(Clip(label: "Far end (loudspeaker)", samples: try AudioFiles.read(referenceURL))) + } + clips.append(Clip(label: "Enhanced", samples: enhanced)) + fileClips = clips + let stem = micURL.deletingPathExtension().lastPathComponent + let folder = try save(clips, run: "files-\(stem)") + let audioSeconds = Double(mic.count) / AudioFiles.sampleRate + fileStatus = String( + format: + "%.1f s enhanced in %.2f s (%.0fx real time); level %.1f dB → %.1f dB. WAVs in %@", + audioSeconds, seconds, audioSeconds / max(seconds, 1e-6), + AudioFiles.dbfs(AudioFiles.rms(mic[...])), AudioFiles.dbfs(AudioFiles.rms(enhanced[...])), + folder.path.replacingOccurrences(of: NSHomeDirectory(), with: "~")) + } catch { + fileStatus = "Processing failed" + errorMessage = error.localizedDescription + } + isProcessing = false + } + } + + // MARK: - Live mode + + func useSampleFarEnd() { + farEndURL = Self.sampleDirectory.appendingPathComponent("fileid_\(Self.samples[0].id)_lpb.wav") + } + + func startLive() { + guard let manager else { + errorMessage = "Load a model first." + return + } + guard let farEndURL else { + errorMessage = "Pick a far-end file to play." + return + } + player.stop() + liveClips = [] + liveLevels = LiveLevels() + Task { + guard await LiveCapture.requestMicrophoneAccess() else { + errorMessage = "Microphone access denied. Grant it to the terminal that launched this app." + return + } + do { + let file = try AVAudioFile(forReading: farEndURL) + let stream = try await manager.makeStream() + let capture = LiveCapture() + live = capture + try capture.start(farEnd: file, stream: stream, stepSeconds: 0.256) { [weak self] levels in + self?.liveLevels = levels + } + isLive = true + liveStatus = "Live. Speak over the playback; stop when the far-end clip ends." + } catch { + live = nil + errorMessage = error.localizedDescription + } + } + } + + func stopLive() { + guard let capture = live else { return } + live = nil + isLive = false + liveStatus = "Finishing…" + Task { + do { + let recording = try await capture.stop() + liveClips = [ + Clip(label: "Mic (unprocessed)", samples: recording.mic), + Clip(label: "Far end (played)", samples: recording.reference), + Clip(label: "Enhanced", samples: recording.enhanced), + ] + let folder = try save(liveClips, run: "live") + liveStatus = String( + format: "Captured %.1f s. Listen to Mic vs Enhanced. WAVs in %@", + Double(recording.mic.count) / AudioFiles.sampleRate, + folder.path.replacingOccurrences(of: NSHomeDirectory(), with: "~")) + } catch { + liveStatus = "Capture failed" + errorMessage = error.localizedDescription + } + } + } + + // MARK: - Transcription + + /// Transcribes every clip with Parakeet so the echo, the near-end words and + /// what survives enhancement can be compared as text. + func transcribeAll(_ clips: [Clip]) { + guard !isTranscribing, !clips.isEmpty else { return } + isTranscribing = true + Task { + do { + if asr == nil { + asrStatus = "Loading Parakeet TDT v3…" + let manager = AsrManager() + try await manager.loadModels( + try await AsrModels.downloadAndLoad(version: .v3) { [weak self] progress in + Task { @MainActor in + self?.asrStatus = String( + format: "Downloading Parakeet… %.0f%%", progress.fractionCompleted * 100) + } + }) + asr = manager + } + guard let asr else { return } + let started = ContinuousClock.now + for clip in clips { + asrStatus = "Transcribing \(clip.label)…" + var state = TdtDecoderState.make(decoderLayers: await asr.decoderLayerCount) + let text = try await asr.transcribe(clip.samples, decoderState: &state).text + transcripts[clip.id] = text.isEmpty ? "(no speech recognised)" : text + } + let elapsed = started.duration(to: .now).components + asrStatus = String( + format: "Transcribed %d clips in %.1f s", clips.count, + Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18) + } catch { + asrStatus = "Transcription failed" + errorMessage = error.localizedDescription + } + isTranscribing = false + } + } + + // MARK: - Shared + + func play(_ clip: Clip) { + do { + try player.play(clip.samples, label: clip.label) + } catch { + errorMessage = error.localizedDescription + } + } + + /// Mic input followed by the enhanced output, for a before/after listen. + func playBeforeAfter(_ clips: [Clip]) { + let ordered = clips.filter { $0.label.hasPrefix("Mic") } + clips.filter { $0.label.hasPrefix("Enhanced") } + do { + try player.playSequence(ordered.map { ($0.samples, $0.label) }) + } catch { + errorMessage = error.localizedDescription + } + } + + /// Writes the clips as numbered WAVs into a new timestamped run folder and returns it. + @discardableResult + func save(_ clips: [Clip], run: String) throws -> URL { + let formatter = DateFormatter() + formatter.dateFormat = "yyyyMMdd-HHmmss" + let directory = Self.outputRoot.appendingPathComponent( + "\(run)-\(variant.rawValue)-\(formatter.string(from: Date()))", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + for (index, clip) in clips.enumerated() { + let name = clip.label.lowercased() + .replacingOccurrences(of: #"[^a-z0-9]+"#, with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + try AudioFiles.write(clip.samples, to: directory.appendingPathComponent("\(index + 1)-\(name).wav")) + } + lastOutputDirectory = directory + return directory + } + + func revealOutput() { + guard let lastOutputDirectory else { return } + NSWorkspace.shared.activateFileViewerSelecting([lastOutputDirectory]) + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift new file mode 100644 index 000000000..91d53f443 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift @@ -0,0 +1,192 @@ +import AVFoundation +import FluidAudio +import Foundation +import os + +/// One 256 ms step of paired audio at the taps' native rates. +struct LivePair: Sendable { + let mic: [Float] + let micRate: Double + let reference: [Float] + let referenceRate: Double +} + +/// Lock-protected pairing of the two audio taps. The input-node tap appends +/// mic samples and the main-mixer tap appends the rendered far-end signal; +/// whenever both hold at least one step, a `LivePair` is emitted. Both taps +/// start with the engine, so pairing by elapsed time aligns them to within +/// the device round-trip latency, which the model's own delay search absorbs. +final class PairingBox: Sendable { + private struct State { + var mic: [Float] = [] + var reference: [Float] = [] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + private let micRate: Double + private let referenceRate: Double + private let stepSeconds: Double + private let continuation: AsyncStream.Continuation + + init(micRate: Double, referenceRate: Double, stepSeconds: Double, continuation: AsyncStream.Continuation) + { + self.micRate = micRate + self.referenceRate = referenceRate + self.stepSeconds = stepSeconds + self.continuation = continuation + } + + func appendMic(_ samples: [Float]) { + state.withLock { $0.mic.append(contentsOf: samples) } + emitReadyPairs() + } + + func appendReference(_ samples: [Float]) { + state.withLock { $0.reference.append(contentsOf: samples) } + emitReadyPairs() + } + + func finish() { + continuation.finish() + } + + private func emitReadyPairs() { + let micStep = Int(micRate * stepSeconds) + let referenceStep = Int(referenceRate * stepSeconds) + while true { + let pair: LivePair? = state.withLock { s in + guard s.mic.count >= micStep, s.reference.count >= referenceStep else { return nil } + let pair = LivePair( + mic: Array(s.mic[..? + + static func requestMicrophoneAccess() async -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: return true + case .notDetermined: return await AVCaptureDevice.requestAccess(for: .audio) + default: return false + } + } + + /// Starts playback + capture. `onLevels` is called on the main actor per step. + func start( + farEnd: AVAudioFile, stream: LocalVqeStream, stepSeconds: Double, + onLevels: @escaping @MainActor (LiveLevels) -> Void + ) throws { + engine.attach(farEndPlayer) + engine.connect(farEndPlayer, to: engine.mainMixerNode, format: farEnd.processingFormat) + + let input = engine.inputNode + let micFormat = input.outputFormat(forBus: 0) + let mixFormat = engine.mainMixerNode.outputFormat(forBus: 0) + guard micFormat.sampleRate > 0, mixFormat.sampleRate > 0 else { + throw DemoError.message("No audio input/output device available") + } + + let (pairs, continuation) = AsyncStream.makeStream() + let box = PairingBox( + micRate: micFormat.sampleRate, referenceRate: mixFormat.sampleRate, + stepSeconds: stepSeconds, continuation: continuation) + self.box = box + + input.installTap(onBus: 0, bufferSize: 2048, format: micFormat) { buffer, _ in + box.appendMic(Self.channelZero(buffer)) + } + engine.mainMixerNode.installTap(onBus: 0, bufferSize: 2048, format: mixFormat) { buffer, _ in + box.appendReference(Self.channelZero(buffer)) + } + + feeder = Task.detached(priority: .userInitiated) { + var recording = LiveRecording() + let micConverter = AudioConverter() + let referenceConverter = AudioConverter() + var seconds = 0.0 + for await pair in pairs { + var mic = try micConverter.resample(pair.mic, from: pair.micRate) + var reference = try referenceConverter.resample(pair.reference, from: pair.referenceRate) + let n = min(mic.count, reference.count) + mic.removeLast(mic.count - n) + reference.removeLast(reference.count - n) + + let started = ContinuousClock.now + let enhanced = try await stream.enhance(mic: mic, reference: reference) + let elapsed = started.duration(to: .now) + seconds += Double(n) / AudioFiles.sampleRate + + recording.mic.append(contentsOf: mic) + recording.reference.append(contentsOf: reference) + recording.enhanced.append(contentsOf: enhanced) + + let levels = LiveLevels( + micDb: AudioFiles.dbfs(AudioFiles.rms(mic[...])), + referenceDb: AudioFiles.dbfs(AudioFiles.rms(reference[...])), + enhancedDb: AudioFiles.dbfs(AudioFiles.rms(enhanced[...])), + seconds: seconds, + stepMilliseconds: Double(elapsed.components.attoseconds) / 1e15 + + Double(elapsed.components.seconds) * 1000) + await onLevels(levels) + } + // Drain the delay line so enhanced length == mic length. + recording.enhanced.append(contentsOf: try await stream.flush()) + return recording + } + + engine.prepare() + try engine.start() + farEndPlayer.scheduleFile(farEnd, at: nil) + farEndPlayer.play() + } + + /// Stops the engine and returns the captured 16 kHz clips. + func stop() async throws -> LiveRecording { + farEndPlayer.stop() + engine.inputNode.removeTap(onBus: 0) + engine.mainMixerNode.removeTap(onBus: 0) + engine.stop() + engine.detach(farEndPlayer) + box?.finish() + box = nil + defer { feeder = nil } + guard let feeder else { return LiveRecording() } + return try await feeder.value + } + + private nonisolated static func channelZero(_ buffer: AVAudioPCMBuffer) -> [Float] { + guard let data = buffer.floatChannelData else { return [] } + return Array(UnsafeBufferPointer(start: data[0], count: Int(buffer.frameLength))) + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift new file mode 100644 index 000000000..0cb8028fe --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift @@ -0,0 +1,25 @@ +import AppKit +import SwiftUI + +@main +struct LocalVQEDemoApp: App { + @StateObject private var model = DemoModel() + + init() { + // Line-buffer stdout so `swift run … | tee log` shows progress live. + setvbuf(stdout, nil, _IOLBF, 0) + // Bare SwiftPM executables start as background processes; make this one a + // regular windowed app with a menu bar and Dock presence. + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("LocalVQE Demo") { + ContentView() + .environmentObject(model) + .frame(minWidth: 860, minHeight: 620) + } + .windowResizability(.contentSize) + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift new file mode 100644 index 000000000..0ca5c07c7 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift @@ -0,0 +1,56 @@ +import AVFoundation +import Foundation + +/// Plays a 16 kHz mono clip through the default output; used for A/B listening. +@MainActor +final class Player: ObservableObject { + @Published private(set) var playingLabel: String? + + private let engine = AVAudioEngine() + private let node = AVAudioPlayerNode() + private var generation = 0 + + init() { + engine.attach(node) + let format = AVAudioFormat(standardFormatWithSampleRate: AudioFiles.sampleRate, channels: 1) + engine.connect(node, to: engine.mainMixerNode, format: format) + } + + func play(_ samples: [Float], label: String) throws { + try playSequence([(samples, label)]) + } + + /// Plays clips back to back with a short gap, updating `playingLabel` as each starts. + func playSequence(_ clips: [(samples: [Float], label: String)]) throws { + stop() + let clips = clips.filter { !$0.samples.isEmpty } + guard !clips.isEmpty else { return } + if !engine.isRunning { + try engine.start() + } + generation += 1 + let current = generation + let gap = [Float](repeating: 0, count: Int(AudioFiles.sampleRate * 0.6)) + for (index, clip) in clips.enumerated() { + let buffer = try AudioFiles.pcmBuffer(index == 0 ? clip.samples : gap + clip.samples) + let next: String? = index + 1 < clips.count ? clips[index + 1].label : nil + node.scheduleBuffer(buffer, at: nil, options: [], completionCallbackType: .dataPlayedBack) { + [weak self] _ in + Task { @MainActor in + guard let self, self.generation == current else { return } + self.playingLabel = next + print("Player: \(next.map { "now playing \($0)" } ?? "finished")") + } + } + } + node.play() + playingLabel = clips[0].label + print("Player: playing \(clips.map(\.label).joined(separator: " → "))") + } + + func stop() { + generation += 1 + node.stop() + playingLabel = nil + } +} diff --git a/README.md b/README.md index 6ae446d1d..0cf144d03 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Want to convert your own model? Check [möbius](https://github.com/FluidInferenc - **Speaker Diarization (Online + Offline)**: Speaker separation and identification across audio streams. Streaming pipeline for real-time processing and offline batch pipeline with advanced clustering. - **Speaker Embedding Extraction**: Generate speaker embeddings for voice comparison and clustering, you can use this for speaker identification - **Voice Activity Detection (VAD)**: Voice activity detection with Silero models +- **Speech Enhancement (AEC + Noise Suppression)**: [LocalVQE](Documentation/Enhancement/LocalVQE.md) (4.8M) removes loudspeaker echo, noise and reverb from 16 kHz mic audio given a far-end reference; streaming with 16 ms latency (beta) - **Apple Neural Engine**: Models run efficiently on Apple's ANE for maximum performance with minimal power consumption - **Open-Source Models**: All models are publicly available on HuggingFace — converted and optimized by our team; permissive licenses. See [full model catalog](Documentation/Models.md). @@ -268,6 +269,7 @@ The default is `false` — no behaviour change for existing callers. Combine wit - [Speaker Diarization Guide](Documentation/Diarization/GettingStarted.md) - VAD: [Getting Started](Documentation/VAD/GettingStarted.md) - [Segmentation](Documentation/VAD/Segmentation.md) + - Speech Enhancement: [LocalVQE (AEC + NS)](Documentation/Enhancement/LocalVQE.md) - [Model Conversion Code](https://github.com/FluidInference/mobius) - [Benchmarks](Documentation/Benchmarks.md) - [API Reference](Documentation/API.md) @@ -570,6 +572,33 @@ swift run fluidaudiocli vad-benchmark --num-files 50 --threshold 0.3 negative-threshold overrides, max-speech splitting, padding, and chunk size. Offline mode also reports RTFx using the model's per-chunk processing time. +## Speech Enhancement (Echo Cancellation + Noise Suppression) + +> **⚠️ Beta:** port fidelity validated (scores identically to the upstream GGML engine on the 800-clip AEC-Challenge blind set); the published table is only partially reproduced, with unresolved v1.2 far-end values and an ambiguous ERLE protocol. Not yet exercised in production call pipelines. + +[LocalVQE](https://github.com/localai-org/LocalVQE) (Apache-2.0) is a compact +neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz +speech. Feed it the mic capture and a far-end reference (what the speaker +played) and it returns clean near-end speech, sample-aligned with the input. +Two checkpoints (v1.3 4.8M, v1.2 1.3M) in 256 ms and 16 ms chunk exports; +36× / 14× real-time on CPU for v1.3. On an exploratory AEC-Challenge synthetic +subset v1.3 lifts ASR near-end word recall from 44.1% to 77.6% and cuts +far-end word leakage from 34.0% to 1.1%. See +[Documentation/Enhancement/LocalVQE.md](Documentation/Enhancement/LocalVQE.md). + +```swift +let vqe = try await LocalVqeManager() +let clean = try await vqe.process(mic: micSamples, reference: farEndSamples) + +// Live capture: push buffers of any size, 16 ms algorithmic latency +let stream = try await LocalVqeManager(config: LocalVqeConfig(chunk: .realtime16ms)).makeStream() +let out = try await stream.enhance(mic: micBuffer, reference: refBuffer) +``` + +```bash +swift run -c release fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav +``` + ## Text‑To‑Speech (TTS) > **⚠️ Beta:** TTS currently supports American English only. Additional language support is planned. diff --git a/Scripts/localvqe-dataset.json b/Scripts/localvqe-dataset.json new file mode 100644 index 000000000..168e3885e --- /dev/null +++ b/Scripts/localvqe-dataset.json @@ -0,0 +1,806 @@ +{ + "fileids": [ + "0", + "1", + "10", + "11", + "100", + "101", + "102", + "103", + "104", + "105", + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114", + "115", + "116", + "117", + "1000", + "1001", + "1002", + "1003", + "1004", + "1005", + "1006", + "1007", + "1008", + "1009", + "1010", + "1011", + "1012", + "1013", + "1014", + "1015", + "1016", + "1017", + "1018", + "1019", + "1020", + "1021", + "1022", + "1023", + "1024", + "1025", + "1026", + "1027", + "1028", + "1029", + "1030", + "1031", + "1032", + "1033", + "1034", + "1035", + "1036", + "1037", + "1038", + "1039", + "1040", + "1041", + "1042", + "1043", + "1044", + "1045", + "1046", + "1047", + "1048", + "1049", + "1050", + "1051", + "1052", + "1053", + "1054", + "1055", + "1056", + "1057", + "1058", + "1059", + "1060", + "1061", + "1062", + "1063", + "1064", + "1065", + "1066", + "1067", + "1068", + "1069", + "1070", + "1071", + "1072", + "1073", + "1074", + "1075", + "1076", + "1077", + "1078", + "1079", + "1080", + "1081", + "1082", + "1083", + "1084", + "1085", + "1086", + "1087", + "1088", + "1089", + "1090", + "1091", + "1092", + "1093", + "1094", + "1095", + "1096", + "1097", + "1098", + "1099", + "1100", + "1101", + "1102", + "1103", + "1104", + "1105", + "1106", + "1107", + "1108", + "1109", + "1110", + "1111", + "1112", + "1113", + "1114", + "1115", + "1116", + "1117", + "1118", + "1119", + "1120", + "1121", + "1122", + "1123", + "1124", + "1125", + "1126", + "1127", + "1128", + "1129", + "1130", + "1131", + "1132", + "1133", + "1134", + "1135", + "1136", + "1137", + "1138", + "1139", + "1140", + "1141", + "1142", + "1143", + "1144", + "1145", + "1146", + "1147", + "1148", + "1149", + "1150", + "1151", + "1152", + "1153", + "1154", + "1155", + "1156", + "1157", + "1158", + "1159", + "1160", + "1161", + "1162", + "1163", + "1164", + "1165", + "1166", + "1167", + "1168", + "1169", + "1170", + "1171", + "1172", + "1173", + "1174", + "1175", + "1176", + "1177" + ], + "audio_files_sha256": { + "fileid_0_clean.wav": "6074c5e755202a26a19eff67cc985379f846d55394d21532137cdb8ee0eae933", + "fileid_0_lpb.wav": "3637ee0628d0953f77d5a32327980af542c43230c4127d2a72b4df1ea2ffb0be", + "fileid_0_mic.wav": "cc116af609a66f431f94df6b385ff2aa362f8a2d437c2279f5401e47f9178469", + "fileid_1000_clean.wav": "df149928119f40a6774fef98419704cebc81dc7a7bfbb69726e4721b18803891", + "fileid_1000_lpb.wav": "40051927c38c6ce3f1c094fe9693454240f98f0e8434760a778a5888e396c5b5", + "fileid_1000_mic.wav": "aff4ff5c6607b52ff8049dde9ad5626d99cc24502b8c98d49245786287611951", + "fileid_1001_clean.wav": "ca5aa562e440be26b4f671a5c2f691a0a5e180e06f619de384d420f707487e8f", + "fileid_1001_lpb.wav": "1c40f9e11dd6232caba0687f16dd3124e43bd828ea50c19fc789a9e61af94477", + "fileid_1001_mic.wav": "558e86200e0e5330cda71ea394f9026a09e8197f467c7a8ecd9ebcd2ee2aca67", + "fileid_1002_clean.wav": "18a4913d3ef8e382e13a4aef4ff5aaa7df0e66f194a14906720489dc3164ff49", + "fileid_1002_lpb.wav": "072cc0e736df8b0e2ea2ef9c89088dd72e1d17c0eb7777d6057148922d4760d9", + "fileid_1002_mic.wav": "707555498fd65bc6b742d66782c1a06da593ad841dd7791051dbe66cab1f23e6", + "fileid_1003_clean.wav": "a25a40bff71d5b54f7200c67cef3182ba2e057d69493703d9a26b2320c819b6e", + "fileid_1003_lpb.wav": "c012a34321467c9722ec4924c1794cd91989bba3029d155281a92f6916972071", + "fileid_1003_mic.wav": "97213c1f678789d2be66639e7999ac8c07b4dbad4215b34e93aa0f7db0679614", + "fileid_1004_clean.wav": "86d3b1ec882da82af9dba9d78aedfab4c3e39b5f69c66e6c162a4893b2a797df", + "fileid_1004_lpb.wav": "5d1eb13154d13546285ce24ff39917aee26454bdb395816b1a1d6837a1bea66a", + "fileid_1004_mic.wav": "83b3efc2ae13c518f6a4fe9d690c5574d7f0f0bcec9c7107316b9dc38031e0e3", + "fileid_1005_clean.wav": "dee3a7ee2dbbadb83b9ccf210f738a54f69afb07cc0eaf37e68e577edcdb90e9", + "fileid_1005_lpb.wav": "4849e62b30c515cc01370d1b602ba1c48ca82af09c24469a0fadcad53e6b1815", + "fileid_1005_mic.wav": "284160dd3882863f6398872d6f83baffa0a52259fa420621f6e24e14f5c7734b", + "fileid_1006_clean.wav": "93268c2cac0b6498c9c1212d6ca5be242abb14e029cf48e65ed603f6b54770bc", + "fileid_1006_lpb.wav": "c7ed177f7b3f264155926af941985c4ba6bffcbff8df490904cdde0f018c6d62", + "fileid_1006_mic.wav": "ed89152e491ddb2a5f64ecebde8c58e34c7cfa125b9b61738e97a9902354c6e5", + "fileid_1007_clean.wav": "ab12936965d5121e065b106f8776646305b5dd7ea94c6779930532bbca6dbdb1", + "fileid_1007_lpb.wav": "300a49d6ea76fe186aa794a8e05886d51c304978b7b168908061289db9287082", + "fileid_1007_mic.wav": "2af9997468054227915a0a6268e86e03eec4e4ee338022940cd2a83ce4fda180", + "fileid_1008_clean.wav": "41e19f3045f4ab843017f5c9bdd560d22f3213aa8179a248963cacfa964e405d", + "fileid_1008_lpb.wav": "ab0f6a582306e9b336d15a4f3735edc759a9ca856c972b44b3c3d7f6a9483c65", + "fileid_1008_mic.wav": "9e5298c21cd5d8c99e929d455baae0cd89e220e13f684b91ad2bc5d76e7f6a8d", + "fileid_1009_clean.wav": "fc58521a4407d098f50bb2a24d2c970967a4ff8ff4822d65718342989e71ee0d", + "fileid_1009_lpb.wav": "60749f51eff732cbc6acb2417513192443652ffb6a991325d398f97d0cc811b3", + "fileid_1009_mic.wav": "53794275d2fb2ff1cee858a333f7f81cd8b38b23114c408ff5eda35d39e06fe9", + "fileid_100_clean.wav": "1316a98c3e26f994a73fbe8cb685c4311ae8ab17a118f1158796447ce07d679f", + "fileid_100_lpb.wav": "8114eea757666e6820c3b37c389e9219e3a9a18b1dc7eb3f2f3e48f90c35c8c0", + "fileid_100_mic.wav": "19df3af8013577421459c7f0de38f24b3cdf31ed993618ebfe652e1a399adbb0", + "fileid_1010_clean.wav": "c1f8609cffb82ae3f739afd0d153854a3d0b17b718151a6b281dddb065a31276", + "fileid_1010_lpb.wav": "42f3c82693aef2aa4baa97b98aea498bb6abc536eb9d28f785ff853c7ecf9e8f", + "fileid_1010_mic.wav": "1f905a867013729e5b705b63e4c6d4cf346277eb25b7f9ec6ca047e89cb197eb", + "fileid_1011_clean.wav": "e951e70324b05ed309994262a18444b1620c27be36ce04c551b9adb68ee93c10", + "fileid_1011_lpb.wav": "9918b3ae8837e036d83a0f72d883d815b4f73092aeb0a281861b89527dde6f52", + "fileid_1011_mic.wav": "981a8c1c2914a47123f0c39748b667f2e78e85e837927707f8e08e8a945f4350", + "fileid_1012_clean.wav": "dc2a56c94fe767d1ad98c04134f8c34451577d70a1bafa3300cdbb71d85d3d51", + "fileid_1012_lpb.wav": "4ef8279f73266a04519d5c215d956dbc1b7e3e17956b64812f19a7fb4c735e97", + "fileid_1012_mic.wav": "340a09778c1cc2074103ff0f65de8635e1caff5a71e14c71b84a53d381e8db37", + "fileid_1013_clean.wav": "a0734ee2e91312414aeedadcea63f6bbcec41423422f24e4ac3629287ac50f90", + "fileid_1013_lpb.wav": "4d380418224f23a2d0fb5bb1fe7246502948e25076843d9f92b42208fb346612", + "fileid_1013_mic.wav": "153a69facc5726d5cc400da16c66dedf1a85f5dc8260f5eefaae293201208c28", + "fileid_1014_clean.wav": "ba8dc1e099dc48bf98abb6a14c0526748b29cf3b5aedbb050356453c95b33294", + "fileid_1014_lpb.wav": "fd6d95237e053b8dd33080109c5e6aca3fb2311240dfb24e6039370ec29dda34", + "fileid_1014_mic.wav": "80db366818d971df9a2b2e61421f578a9e71226703afa79ae4f3970ace82a236", + "fileid_1015_clean.wav": "6ca45de80a50b609e47670c74f6deddb0f22321acf81253829f99428b32f8850", + "fileid_1015_lpb.wav": "81df3c722c501a0472e7e1e84356815b1b9482d2f3c93c1e03167af691548064", + "fileid_1015_mic.wav": "e17de9a3ced5fb7db2c44f812d323324a7c0b40ea2dada22ae6766634727f666", + "fileid_1016_clean.wav": "23932a0e95f9b9012e4a690895346c5ec0f56cf30749c7f5633bc301b998b1ff", + "fileid_1016_lpb.wav": "3ff85876e86bd786538b5dc05ab4a7f865ab56b1d2e063a736d2874f666a31de", + "fileid_1016_mic.wav": "c96b30692915f1978edee560dd56ed38649e5fd87fb5a43d8df9d1297563a1f6", + "fileid_1017_clean.wav": "98d5f3484bca66af7f1feeb4fd610c25eb7a1dcbef26c9a00fd7952fb53994da", + "fileid_1017_lpb.wav": "620f923b70a8c5350dc579509e0c5be28a2c6bf4008a2de13f31396f3798ed7a", + "fileid_1017_mic.wav": "c69953607d014326ad2f7faa8fd35d9cf732a7d4f3d4fb7383966068bb167226", + "fileid_1018_clean.wav": "ff64ca9b39c67426006784fe2a07dc5dee78be93a472213481db9ba537d9d749", + "fileid_1018_lpb.wav": "3e08388a8642aa7d82361378bf7cbe7deec77b6dc21696159a9d0833f79f8d73", + "fileid_1018_mic.wav": "969c72a00f911862bbd3b4b01965c3864225e2f9fb8902422335d276e59da495", + "fileid_1019_clean.wav": "d40828adba12b7737c63c650fa47e868e1e8e4d40b5fbc8ec7e1844d5b428eea", + "fileid_1019_lpb.wav": "899856570dc269aed1c1054e1b90e54f27a486e1423987e27013cde4b923e035", + "fileid_1019_mic.wav": "751d76b67f60ada518e00055a9d243f84ac494eaf84f53579b061d4e73ec2f6d", + "fileid_101_clean.wav": "afc6a4da70a2c7e413e77180e27d127fc146b485b84ee1139a3508711b85abe4", + "fileid_101_lpb.wav": "c98e1576fa6a479b5336c1822509429c478d9889939f58f42f808e5d4cc37d7c", + "fileid_101_mic.wav": "c54939bcb8923decc42b26e7dfa8e77a827f3a732e6042a20651a8faed4544b8", + "fileid_1020_clean.wav": "b0423b623807358e52e2672d7f7fb6b07f5cffeb8f7cef76543e4fcdd5e9e836", + "fileid_1020_lpb.wav": "76761600e8b35a9ad0f3bc1bc5aa947f2e05ccb81fcbad37f186209f6d4732cf", + "fileid_1020_mic.wav": "9a566d413a07bf747b7cfd85bea23b43a91c2aa863c749508e3c836db4c5984e", + "fileid_1021_clean.wav": "c992093ddf83341565d99f1cb7909b17b409964c568208fe9c2092034d431ab2", + "fileid_1021_lpb.wav": "5755619d40d6a894f3c08ae8c7c5c9ddfe69d84fdd76dcacfb42f12e200e36f6", + "fileid_1021_mic.wav": "6a50ff9933cac5a1f12c74d91118760b913b48b3163dc07276760e07fb51736b", + "fileid_1022_clean.wav": "c7eeff036753fb054593c3759539d9a40d9b665005c17e596d999a3af3983e24", + "fileid_1022_lpb.wav": "179a4d5e43f2e4f46c9bb83c260a721352efdfca9d45e6143a65d4fc9e6e9ba6", + "fileid_1022_mic.wav": "f21cb20283be7360bedb7f75472e06c96a378b5423fcfe483c2422ad2237924c", + "fileid_1023_clean.wav": "728c2b259aa65c2635c27471db59deac11ecde49260797067d3a806a5824c005", + "fileid_1023_lpb.wav": "1d0c2366a532d7820407351833a56f07866aeac7b0f8716adf5284539e5843d8", + "fileid_1023_mic.wav": "62c6fa6f221c114443bb1cde3fb344f3ab8fe176ec2eb7abd0ea82fd2b8f599c", + "fileid_1024_clean.wav": "82cf78b76666aa2cd9aa6a1c616dbb45a070035933b46600a3cf1dc48139fb65", + "fileid_1024_lpb.wav": "84af32c79d829eddbd7eeba9e0f8d9a4b44478cf347c7cf38e6d0da95153b30e", + "fileid_1024_mic.wav": "cfd924667a9da98e840f97d70b32d52c35eefff2e382e031e5c5ca388b25c905", + "fileid_1025_clean.wav": "bb5226bcdbf11c627f5756bdf4f757851950324db5f99873a7a32d94fddc8b30", + "fileid_1025_lpb.wav": "bf45a5923d8413714e7bef47f4060dda2f9d62bb7a0e04578caaea4e582bad6c", + "fileid_1025_mic.wav": "a01160e6583356c063900f432b11a1a77420513b6c5c383dca0c85b12e38696a", + "fileid_1026_clean.wav": "5ea3a9c67669a295b8989dc5c30cd112ecb19a476f2f2fb49c61ec8f65d628d2", + "fileid_1026_lpb.wav": "9d72df25d68bbe6ea38b045be95facefe6dad16d620678f5460b54805d53b4fe", + "fileid_1026_mic.wav": "8de6780c3f41e0adaa4c52cdcb301d012fc65e1e2a9080a50d5e9381d8a49dbf", + "fileid_1027_clean.wav": "7757fae151fe351542ade2ec70e0721f935fc53b79b4a487109c98f543577f82", + "fileid_1027_lpb.wav": "1a9e86da1931ed306954bf4d38bdb3cb865c8920af802f10892de5d550fb4d48", + "fileid_1027_mic.wav": "742ebacf59122c7a8d443e6d48818efcf6a3294cf50146a453df1cbdf60e7824", + "fileid_1028_clean.wav": "6c941d0701e9ee78a46406ded32f31c161246cd730744fbba3a4f20bd926eab6", + "fileid_1028_lpb.wav": "c4524ac6f006647c474858b3800c6f01765ac46d9a7d0d1d7081c799c00b6685", + "fileid_1028_mic.wav": "bee93df069bc869e5655b2d12fb9a84ee3fe8711e781e667a482b93f49269d3b", + "fileid_1029_clean.wav": "c05e2f07d8f5ee88a36f5d0a2ed8bc1c5277c3da7230ca964d49d1225e838a60", + "fileid_1029_lpb.wav": "c9755f7d48bd4dba0a3dff970c823d6c689da7322fe0e4b2f725cdf01c2197f5", + "fileid_1029_mic.wav": "9002a3defc21b4e16307e7fb00155ad4b300de6b7de630b7bf110ec85931dc9f", + "fileid_102_clean.wav": "8935f518811d26959305aa0d5675d0d0b9df02bf4f02faa3f863a67a520d2c4d", + "fileid_102_lpb.wav": "9cdc9d4695c6042ce3acae3160984fc16334f64117e79731940ae05e523917cc", + "fileid_102_mic.wav": "fe68ff0267dea1cd8fd61c41a45f52371d80bed402fa8adacc7ba6c68d6ac778", + "fileid_1030_clean.wav": "0937948efc4bce60557c8b41b7801fed9fbf72dc8f13a09116506ebd7e3dfd25", + "fileid_1030_lpb.wav": "af4dc069b4c0b9da6284862f7088381df135087b27dcc55bd7621ec2eb927152", + "fileid_1030_mic.wav": "f37fa6a1e98118005d9d62991a6d230d1fcecc6091d72e8fb8543d797cd4340f", + "fileid_1031_clean.wav": "49302317572ee0b92699d91b5950825d896d65834e9544915c30423c9272e97e", + "fileid_1031_lpb.wav": "d204dd98dfb546db10bff547818701138e94e380f333e13832bcdaa121799347", + "fileid_1031_mic.wav": "f86e434ca36a250aa516157781a890a0f91341fcb94978ac981808a5ee9b992e", + "fileid_1032_clean.wav": "3100f429fd3f78818239ec026a69c0a34bd3f16e7bd41a06ce03e354eb0243f0", + "fileid_1032_lpb.wav": "03efa4c1980d24815bab148d9329bead9f5423842de84666978eaf7f4ded4e62", + "fileid_1032_mic.wav": "1fdb273c92848f2c12f285349a8f6bf1fea606d89cd43b124620f007a566e8b4", + "fileid_1033_clean.wav": "6a397741c74ec5212d216e50cc06fcfa2dc425e2517c951994422290d009fd96", + "fileid_1033_lpb.wav": "2a881b89467329854808d7ad3e705d9d1502169d993e1edb681f321adab54fa7", + "fileid_1033_mic.wav": "448b05fc076a083f39c610eef61498807c75426ad87b028b275113263b4f2f08", + "fileid_1034_clean.wav": "976c1730fe59bed5e27c77ef9ab59f89080aafc3ba9865423eebff7a13dc3bcc", + "fileid_1034_lpb.wav": "021aa1b66e3ff3c782c9bc110e5891c10ae9872d0596122ba0cc43159e954c95", + "fileid_1034_mic.wav": "adf2761005c7aed1f7fa9900f448f667ebdc99439fa69f2e2e1385bfd2f97159", + "fileid_1035_clean.wav": "4484f600614a4e47561e56ee3c74310a8eccd461131938d1ffde7bbe1b64a9ab", + "fileid_1035_lpb.wav": "59e1ac1653bb554425408b8ce2d04cd08c0aacb019722a8b26f2f387d10fd3d9", + "fileid_1035_mic.wav": "181094b8041b61ff1dcbabb8b782bcd673349c7953a0c7732f41cdefa09cd865", + "fileid_1036_clean.wav": "5e620570ce11ca0e56e9fbfa4ddcc4cb944665576fb1d4e4024aad86f7b4936d", + "fileid_1036_lpb.wav": "a369a5d0eb0ef8a2dd4b467bcb1431501fbf4fe07491aa49e2a806eb05216bc0", + "fileid_1036_mic.wav": "12eb3a113bd60f7d3de5d8c9b83e9bfee00e876c59d68985cf103a4663d8e27f", + "fileid_1037_clean.wav": "d81c1b55309bbc46462b958f51d7ea4c3b9366d4af51d8572a2c4ad4aa543e97", + "fileid_1037_lpb.wav": "6d2a031c484e94214aa7a607c085b1636482973c65b38043081d6204a4cd8ca2", + "fileid_1037_mic.wav": "41a3cc1436009b795bed45e5a3baa084092d6fe75a332897aa6c3682760cda86", + "fileid_1038_clean.wav": "31095d831580042101b17649bffaa770447f6d7b8054f293b44666351a37dd8a", + "fileid_1038_lpb.wav": "e683ccacd8ad833a67a635624470dcd08ce0a107200d64123c2266788e549808", + "fileid_1038_mic.wav": "6273caad3abf9f9605e515126e61a903511fcea4143c21c6f240a67fca356562", + "fileid_1039_clean.wav": "2fb1b73848659f39864dba20c39b280362520510e81b8c1ccdea970dc3f2fb61", + "fileid_1039_lpb.wav": "c7ec1acace8c62f728f279bdcdfd5e978b7eeb8b049f609c8c170eaf029f175f", + "fileid_1039_mic.wav": "bee6b30b22e044efc782642c1c311a6a36da493f160ec6cfa7f1bca38db56d9a", + "fileid_103_clean.wav": "eb25a31db8b5e29b9f11f81f78262f47a0154ad9440a616cfff596d6a59b8a01", + "fileid_103_lpb.wav": "6ebeb1902da7dfa53dce9732644d701977f15d602dfc6c4fb906d95c00498dca", + "fileid_103_mic.wav": "7682af2bcb83cded339351468959157b9e393eb00f85929718c455357ae27c89", + "fileid_1040_clean.wav": "2cca24d755e6ae67d816cf850344700907d70905596193ecf232a24cd47e8cd8", + "fileid_1040_lpb.wav": "7d092bb61b7d4620e87c96597dcdfc0b9450ab183c05df450917debe38fb7fda", + "fileid_1040_mic.wav": "a7e18488ce92a6dcf3f40842fa384168787b82319de1a447e765937106c9fba6", + "fileid_1041_clean.wav": "d814dff8509cad66da0978f59433bd5f7f2699a66a9e059ff93b4534e634abba", + "fileid_1041_lpb.wav": "810e056967069e3eb10ad53edc1c300ab49123b2e67ffc066d20f413734fe994", + "fileid_1041_mic.wav": "6356116268f9d850c45baaf1331722e6c90400b6bcfaa44599fbd5551b56e7df", + "fileid_1042_clean.wav": "c01ccd3200f8b79aba92878d3c04020f22dd7c847a21b429e92a9925c8fb1449", + "fileid_1042_lpb.wav": "104ef6a94ffff6fefc0b61b8038eed6f42f626becbee9ee4c20af25ea3313439", + "fileid_1042_mic.wav": "43d81c27576e9b37f60cd0d644dc1499bd60255bd5553320d75dfb1a376b706f", + "fileid_1043_clean.wav": "ba5f1118bf7cf23fffb93ea5dd3b57337e27fc73fee2cd3688611eec9ea088fe", + "fileid_1043_lpb.wav": "6304427c170ceae2e6282dd3b9857ef8148e6914314719ce7e1d4527676356ea", + "fileid_1043_mic.wav": "6777111e5248f25bf66def30bac475427d8da84e597a64bda2238a0fc863b126", + "fileid_1044_clean.wav": "f04fcb4b4c987fc8c2744ede6b9db0bf2c80c527fea58f5c9b464ad20e945a9c", + "fileid_1044_lpb.wav": "adbcfc1721867184513e5e6195bcd8c4b71a9590d525e428b2cf40d06c4f2ee6", + "fileid_1044_mic.wav": "ea94fbbffb295f780836f458b126a1ee3fd91fb71943c71b28287f7223cc5ebb", + "fileid_1045_clean.wav": "f00b0482ac69af7b18a503eb1d663ed89fe1161368688b511df1f781725ca1ea", + "fileid_1045_lpb.wav": "5efec4f79cd26280288e4c777551cbde719aba4a2a03fe0e6365d2bc8162951d", + "fileid_1045_mic.wav": "aead85aa5257bf682a8eb487e01b16e663e031ebff939749785524687ea64b5b", + "fileid_1046_clean.wav": "2069d9c9e1d4b29b046dda70b24f9530dec3c0c5f40b1f44fbc898034d5090e8", + "fileid_1046_lpb.wav": "c2e865366d3dd2fdb56ead86c1aadd95ac76cbcb3636649292fa1932c8579dd6", + "fileid_1046_mic.wav": "682e6c48ac4c782b32e6c7947d317176bc7ce0b7d8e52a6aa3bee2671d282fd1", + "fileid_1047_clean.wav": "02581306160963dc6d837f08f83e9ea1868bd91a25da1caa1cd4712734b4be2d", + "fileid_1047_lpb.wav": "6a4d0fe473e9c882c812b9ed25759ecdc045abc02c1ff396051bae72a65e08a2", + "fileid_1047_mic.wav": "5fabac67c03c02550e07d79668f47a22ee236902150ebcfb58327e9e080341be", + "fileid_1048_clean.wav": "15fbe4d81663cde54f3accc04b2e2cc7ed4e44217ee9c40b169a390b220a1555", + "fileid_1048_lpb.wav": "e8bbbbc2cce7bc9a449618f1e9ee8fe78d3f46ba7762a8119fe4241488df54d1", + "fileid_1048_mic.wav": "89f5407d43e1b79641e8eb8de2086e88693d61d5d4cf51e3bb133c218a2b9b30", + "fileid_1049_clean.wav": "ef3b6749031294757429eccb24790720168d58a0f0d12794ea5716d067f19e19", + "fileid_1049_lpb.wav": "b1908dd2ae2c87745b4c0a671a2f3c669f088368fbeee23c5cd3a50de0518213", + "fileid_1049_mic.wav": "9a8071df87941ab90807da2ef55085996bb21937a7ea333f94b0f37cb72ee1a8", + "fileid_104_clean.wav": "198040e5b26592ac3b5d3db9b141396903efb171e49dd556596cb564f512b0c4", + "fileid_104_lpb.wav": "7a55a501548485476c591498324c109a8e86d52539516e6ed8fd05f2bc3c0dfe", + "fileid_104_mic.wav": "67dfd2a88f7402f2c95ac60dee9bf88b66477a32ba89a13dbccee18a90943c2a", + "fileid_1050_clean.wav": "21d30fee3d2b9e9ed1ae2bd18cc0113f78d0a40f75d4153d051ed08c4eb5a878", + "fileid_1050_lpb.wav": "1073c6633b92675f651a978fc722c57164d6a24aea5536b54fb1fa481e952176", + "fileid_1050_mic.wav": "0694d6066a3c0a0300f33a5d1b10eb62a14b44b9f2fabc20406a3abc3895b783", + "fileid_1051_clean.wav": "facb83c97b6efe7f69ee728054df3106f3ef8928ddcb27338ad531775d8fb03f", + "fileid_1051_lpb.wav": "c403a9cbf2df6b77452a359645b4fce935862b0821a2d46013e75a3b95d21be6", + "fileid_1051_mic.wav": "d69771dbd44dd0efa03ee038490685c83b7529f0a9c4fb57c2ef6c6fbb3ada2c", + "fileid_1052_clean.wav": "26475bc1bffb6b81ec798d4befcfccb0a00f2265d40507ec8850cbfe75fa5832", + "fileid_1052_lpb.wav": "1886b1caff1bbefe22bea3e23b4c7577430da776559078cff0478c2c7feaca11", + "fileid_1052_mic.wav": "5d3ebac73a4adafa01dbb6b660eb9dbbf720d268cebc3d8127f86e429d811f7a", + "fileid_1053_clean.wav": "9e6b7a8b6b65f9d4b3634d2a400bad0af09b9a14fbc4e5250958866fabb549d0", + "fileid_1053_lpb.wav": "f3c9041fa69ffb61e970b2e557d32755430be020a1067fbf3c2ab61990ee4f69", + "fileid_1053_mic.wav": "7f51ae8664168c5a8b6ea20ac841ba75812a4063d3e22a452d8c2cc10dc8a9da", + "fileid_1054_clean.wav": "7ca4c8253ea1f673f41c18f3fdcc015aeec71032544a78b074c1e6658a35b78b", + "fileid_1054_lpb.wav": "826eb144bbbbc1497bb676188adee5fd3614a53a4ee34b827d0753aa6bf80eae", + "fileid_1054_mic.wav": "5ea6ab83a083ff52af638d1bae3e5207a5535ea7f14cbf0cc818163d47dab4d9", + "fileid_1055_clean.wav": "afe62f37e89c7e3532df7287b2a5413d35baa7ff27ee0f752afeaf08f9ecb337", + "fileid_1055_lpb.wav": "cd4e3a78a87007336c994d13ba6a50158f142aefa12719ced07633464f4e98d2", + "fileid_1055_mic.wav": "c51132ad94140cb9363811c4f81de0b0536c97fbb67935ccdd044ed496249993", + "fileid_1056_clean.wav": "371d6b4e42c285dae87c9d5c99c9338aa6cfc84a4b20f84ea56841726d99c554", + "fileid_1056_lpb.wav": "a3d9942e1f4b989b8a8b9465a76aa777b414a8c122ffcf6fe7972fe437f7667d", + "fileid_1056_mic.wav": "770effa6bf1911de48f2d56becad80b34f62ab2bb22ede0a6ca2eb7928f085a2", + "fileid_1057_clean.wav": "8f5bb8cba32106b444c8c898a83ecbcae7915e82f973e9bc57c98cee1e6e2a7a", + "fileid_1057_lpb.wav": "d429cefc91ef371dcb0f6cf5bf54a8a6061ea940b0dbe49add9ac64722739fab", + "fileid_1057_mic.wav": "22c9658a6fca36382e4e9e3e347c686053e8cc783de48e91d69a4ef9b4a0a23e", + "fileid_1058_clean.wav": "fdc10c7400e825c52f008ca6e6d6dcb24c787b8806fc8bda035625e4dd55728f", + "fileid_1058_lpb.wav": "ec36fc515356c23227cef01193e363b0a71630e351295a22d716feae2c5a7312", + "fileid_1058_mic.wav": "74f751c7530acd81d6e89ddda380eb0a82983e7445f51e64b2741ff3d322e3d4", + "fileid_1059_clean.wav": "45bc44d59193a18b7716206bf8e032fa4758f1c094a40a8595e98c0af582a16e", + "fileid_1059_lpb.wav": "0d69c8410d39417a0913d9b305035a0dce7b56fb0036634adf9ffeeae5db300f", + "fileid_1059_mic.wav": "a161c88bafbf914ba4409513096120fd9f9496c7c50c531336a73eb0f83fbabc", + "fileid_105_clean.wav": "97752c88b62de2bfadb65c9906fb4d4a1315ef98e63214ec6b91897b1a321f5f", + "fileid_105_lpb.wav": "2f6c829708977e0637346323ddcd7172eeee96941223c50c206b5e536cbc579b", + "fileid_105_mic.wav": "61dd7309156c8b1cfc2110171ab274eda53569f1a5740f0d55a8f9ecec62ea11", + "fileid_1060_clean.wav": "19a1635ba18a96ddd0fc3925b398d2b617c836d89da90e6f1a7c68044c57322d", + "fileid_1060_lpb.wav": "4f64bb155fcdb80a334f6016d3c403d6b90425f2aa980f6f572d2d328414b9fc", + "fileid_1060_mic.wav": "441351bb62ac6c56e7bb370f4ded8d901874ebf095a464fe5c14ced649433c63", + "fileid_1061_clean.wav": "037623587109bd56075cafb0b750459f9e863a7dda3b75fd679e3522ebc9d151", + "fileid_1061_lpb.wav": "e8853ed46726978e13578dc69e29a3b41a498c3aa7bd25c0d34e1438e9a61475", + "fileid_1061_mic.wav": "c1666c8cf6248242cfb7fd4c8d682917ff27100b8b40a6ae01c3024fbbea1043", + "fileid_1062_clean.wav": "080a1298e54bb8f9889dfb340cf1f2164e74b245b91aa919cc9c569cf3181397", + "fileid_1062_lpb.wav": "b9f29b16fe5201cfb4f77fab0267a763a5229fbb5f470b1008244a94aff246ea", + "fileid_1062_mic.wav": "d79aa2f6fed5e4c47f7a9492b4436890727d11656f244709ec0a32d7ebfcf029", + "fileid_1063_clean.wav": "58a045845cb6697d157472b73935241dffd39068d01ac3fea0ce27bcc7494ebe", + "fileid_1063_lpb.wav": "d18c7385b0ab4287ede5751d79898b6b30d2aada6bdc5e779e37b8c64c9e278b", + "fileid_1063_mic.wav": "3588dcc36f355c9a615bd30000aa2d662b9421aa561fe557f70703862b95c6f7", + "fileid_1064_clean.wav": "4ee2f44e9b63a2813a65d8e204d9e72baff5fa876dd1c6b00ca177c733558d20", + "fileid_1064_lpb.wav": "caff14e5caa22b0203b149b6902251ce648deac653f7b372fddf5e8b85700280", + "fileid_1064_mic.wav": "19f51293e767cba79d156a97cde7082c200c81cf1d273ff59b854d0f588a8aec", + "fileid_1065_clean.wav": "924d64fa1898d9e8c6f70b53b4e989b94307c90951f5c92ac4b1a9e137cd474d", + "fileid_1065_lpb.wav": "309cf065306acb0daed450083b8b2036f1856dee8ded23827f2b0356dfe5c23a", + "fileid_1065_mic.wav": "cb4b51d7d61c3eba02a36abd0477b8787bb12b1c00450996d8c1a1ef23ff8f54", + "fileid_1066_clean.wav": "5584c7d70bc769b8d98fcfd6cba03184f7fb21bc54cb635c7f746d4dd0ea09d8", + "fileid_1066_lpb.wav": "ab90704fcf5886d5fcb208beff1a51a69c81acc8947a03bb8fa822c28f24e234", + "fileid_1066_mic.wav": "021933245add0b7188e97f6dd3106f117f919b527169fbbcfe7cb07d9c10fb00", + "fileid_1067_clean.wav": "55a91d1c5f47a4e6fe48eec329b77b1448f948fa4db4630b751d232b641f023d", + "fileid_1067_lpb.wav": "3bd7079100f02b158d4eb740dc320ce9c15f6ff8de6d5d51ce7121b369f62c93", + "fileid_1067_mic.wav": "57fb7feff6b746f4682678a4610883e430504a388e1f7cfefae74d6f8a266a48", + "fileid_1068_clean.wav": "c0c87bb875286e3f707ae4ac847fd5ad21fd555c2c10a0627f72877eb205f097", + "fileid_1068_lpb.wav": "02b280440b263a772bfdc4436d5bcdf0faaefa321651e874a91b9ec0c19d305b", + "fileid_1068_mic.wav": "a8699cdd97d8339e95191600d9ddbcbb55c6cbf912f9c8b92fa00b514be846a1", + "fileid_1069_clean.wav": "a5efe1e237505e7d62fbb7b76d28c949fa34d714a07e428eadfcc75a23d23c49", + "fileid_1069_lpb.wav": "bf46c6ccd07aee4d1eb9d1848b2a2cb09877b9f3dbc33cad847e592d0ee880ab", + "fileid_1069_mic.wav": "c0ab7b786aac63713906baf92a3cd32cd0dd1896c2cff08f0be4fcaab52ef319", + "fileid_106_clean.wav": "f43ae25ce5c05138cebb53e658d797d58c09bed9f1e832187058f95353965119", + "fileid_106_lpb.wav": "0779fff3e7a827dbfb3bd7abd506192a64c2cc3ee8d4b2e77e7346319d736f5d", + "fileid_106_mic.wav": "3414efd19565fd3013ec85167737edaccd5048fc9d0b4d0156fb68bb306ba07f", + "fileid_1070_clean.wav": "1293cab2ecc846b0e6fc3f1fa41ed3c59290ca154428fbb01d2c6bf02ae561c0", + "fileid_1070_lpb.wav": "63995cd92181c2d4968493dc2d1f0e49313e7daadc523ed04e1ff4756800ff24", + "fileid_1070_mic.wav": "8de90a26ea3375239f04c96762f0d60e8a549398124f724b779d610ed6fc3784", + "fileid_1071_clean.wav": "ddae1ddae40bb1247f42d0efa39113d7b119b51fe03dcf855761127c31ba8135", + "fileid_1071_lpb.wav": "505dd232ae0011baf01f3ba5c9eefc84bffff4ecc8afdd285b40ff39563367ed", + "fileid_1071_mic.wav": "f397e1a73a16016aa691caf273dfd064344addf7b2219d0e691bd7398da8ecdc", + "fileid_1072_clean.wav": "a6913f893fad3e2bb06e70aca7c7492abe6ceddf2fa87728eeeec824aa6bb9f9", + "fileid_1072_lpb.wav": "246293cca85d3e1f418a4219fd649a0c495c2f842d26a7ee171e841b78385903", + "fileid_1072_mic.wav": "bf3981698459220dc1afa9ccfc15a8709b8f181df385e88e83a49f1551ff34b7", + "fileid_1073_clean.wav": "9621e9518dba91264bb672ef54c822e531bea7d7e253af868544ec4972a880fc", + "fileid_1073_lpb.wav": "99d2247ef4b1b306d0af0c97e00fb3c7fc9d00e5d9b5c4a999ce24fefc3a3302", + "fileid_1073_mic.wav": "7e30c3d228bd69c28ba10da3280fbb609130e2e22c5fc5e76e3c03c4a104ce4f", + "fileid_1074_clean.wav": "e852b24d91e40f243e4e02b5aadb80f8a38b5a14041903b8befc1b4b82ffdf67", + "fileid_1074_lpb.wav": "cc1ef61a4e49a3dd611560d1ab34e53754ec4cf0f5feb47f47aedf9270ee535d", + "fileid_1074_mic.wav": "c44dca278f25f6569234d5b5535afbc73566e1fca0b1d778c29211e6fd789445", + "fileid_1075_clean.wav": "4a68947cb0b02d28b3aa9550aa5fa1428f250cfd21261a8747d97c6c89ee6362", + "fileid_1075_lpb.wav": "705e2f0d1fe965ab9e4bdd4ed866f7747c57f0a37f021a5bd4f7e6d250a99c2a", + "fileid_1075_mic.wav": "a511dedec96e370efee18fea6933f920c554d714f70551843b29266392e7628b", + "fileid_1076_clean.wav": "67d5c980b6b66e911c14655165f3cb91ce15779ae84103018f9f0dc6b2b6222b", + "fileid_1076_lpb.wav": "90ff48f98175f3c0f0bf3f778d08c903255c437e6c220425d47b392638e62bb4", + "fileid_1076_mic.wav": "19a23bd396e3217b18cf6177a03b435b3d1e68d1f636252d088b488a4cfc564d", + "fileid_1077_clean.wav": "d0f382e41e1daac5ae16a5ee224d7f552d9da73e1a32360ae6819b492a56d387", + "fileid_1077_lpb.wav": "7c170ac12ff6e00194d331ad8025e2cc8e4946efa99e77c3a379539a2cbc90c4", + "fileid_1077_mic.wav": "b450521f43c0c8cc0712c092046763d5abd02b82a75c385f373f821e71f77615", + "fileid_1078_clean.wav": "91992a2c3582e0fe50140bfe9c8674128d297a5088318cb299bac6b7fcee5b9a", + "fileid_1078_lpb.wav": "adb8fce55e3a629b205bc7789773a7bfd85d03b90f42794146926d3a422a93ba", + "fileid_1078_mic.wav": "e4da2976ed7a2dc6f0d498ceaeec4d3c342b419b4a81a1fdffe9d2a6e617e095", + "fileid_1079_clean.wav": "a757129b6911e79d11404ce160983044b057e19c6d915ab05efc4bf43d026744", + "fileid_1079_lpb.wav": "fe3b2c126709489a05f346e15e0951cfcd2016f92d9eedcfe462508460660d35", + "fileid_1079_mic.wav": "0206319ed5133b0b134e1f1fbec5c1feec0c281c3b8161ce2b5d528f6364d97c", + "fileid_107_clean.wav": "b8193f3c281320f15ee764ef3bced562a493d745ea19679a1d14e9c5a0d7f2a9", + "fileid_107_lpb.wav": "0a2c2bfb2baa917097ad8d90e644e3d45068dd761587a7d3bb886343360b2969", + "fileid_107_mic.wav": "1c655c9cbd6e3c572ce60df6dcee54fc6483abd1e3f0c8960a77996410d85c17", + "fileid_1080_clean.wav": "b9645c4903593023d9adfdb54584c4bc33d30942f1ccea57bf2ce9b414bf66f6", + "fileid_1080_lpb.wav": "705bd3229e350bdceefdd10e3cd68fafc472749bd73a4886bf3dd675816ba24b", + "fileid_1080_mic.wav": "77da4d1bc0952081de8d620cb48467b361c0108284258e5b326428adefdfc922", + "fileid_1081_clean.wav": "6c0458d7724461663fea96e3fb2e7c72469390e7431dddc93dc6c2f2422ceb6f", + "fileid_1081_lpb.wav": "6893c0eefd17636670b2d84b98a767816d94b4f754b3a5bd64340e33139a0ed7", + "fileid_1081_mic.wav": "42b993bcb5a86508a6ca27e6ec42609b99acfddcdb286184cf422b8b59789e29", + "fileid_1082_clean.wav": "c488383e67b53cd97222828f742569ec7499ae7a9df297f72cc83b7537a0f8eb", + "fileid_1082_lpb.wav": "2cd8fdd39ccf4a73d2fa064538a5f6e08e09aa215b13525e3d6ebf489bd6f762", + "fileid_1082_mic.wav": "a7962ab355447525057b2975b9415907a82e52c627a69e96354420e7a171571d", + "fileid_1083_clean.wav": "acfef8cc57693d32ed097989b6c518ac69e077ac460febc570a7e48ffd607156", + "fileid_1083_lpb.wav": "83cc5d08e03449ad930577c7c692c5b1375b3f24a19f06c46cf27706946df565", + "fileid_1083_mic.wav": "fc0073905795e27c65a641f5713e3cbc60a0656eface654478061e66fa60092f", + "fileid_1084_clean.wav": "05c4f787954e4c2993c72b274fc6f266b8ca5002d21b24d9ecdb89c8b5d63761", + "fileid_1084_lpb.wav": "f05770c86c4347a77523a1d3dc1e622966a1ab78f27ee55451cbf03d5cadbea7", + "fileid_1084_mic.wav": "1cb79fa3d4eff2bccdf2ffc76342cda03783172acf25718f754cc2113f9e48de", + "fileid_1085_clean.wav": "5fa3bc3d70342b0fa2314cdb4e087ee34fdbbdd50fc6b2098edae851102a0593", + "fileid_1085_lpb.wav": "1885234542deda496e84bb57dcc4963a027f1f104d338f3a1ae7c109f3de6465", + "fileid_1085_mic.wav": "9e734e1ec86ed30ad419b5cb0d11102f8ca81481cb4e834fcfa3b72741845e20", + "fileid_1086_clean.wav": "5350ae2e40b4a5bd5ed5bf6375ee7592bb4a13150cfd8514fdfdc2f4203a5fba", + "fileid_1086_lpb.wav": "d8bb37b0b9d9ba89e939490445a58fec1dc5aeb025ef6850cb71f969a8aa487d", + "fileid_1086_mic.wav": "ac23a3dd698a7d1e939590dde03951dc38c731e83988d3339c0d0d5d4c1f01b2", + "fileid_1087_clean.wav": "3a765ad301b7e979ef348f5790a93e277b9a40912b6b46400002186d04f4f6ca", + "fileid_1087_lpb.wav": "17fadd083fd06c5ae551048ec6d676065a1ecf9ec2d49f854f46b3576e084dc9", + "fileid_1087_mic.wav": "c086617c814ca0a211a43d61ee363172a33e5b120abd7e24e6128dd80b21b587", + "fileid_1088_clean.wav": "b31bcff0c36e50867c98a9bac6b119dda49a6981e953de6b89f21e3f844d8285", + "fileid_1088_lpb.wav": "4f9990c07477447d26cd0d9fc34c81eaa89b860f9832c0f469876f539d13e88f", + "fileid_1088_mic.wav": "9bfc0843624dbe690bc6dd5ba901dd062d3849dfd0af3edf408cb79e605a40fb", + "fileid_1089_clean.wav": "0dea7b2a7cad07e4b1f1e12c376bce17f87a11d74f0b4333f78726dcfddb9d59", + "fileid_1089_lpb.wav": "cc4c8ef210baf01a7280f184802f0ce80c0c124c88e157eb142e0f52af223ed2", + "fileid_1089_mic.wav": "853ae4d0338236527706205931acfaebf6b7d5a048f908a74a686a929502aa54", + "fileid_108_clean.wav": "006ac74c0fb2157ec039efbce94f1af63c6ef05f3454904b6c180c846f49e0de", + "fileid_108_lpb.wav": "c8c0b465d4fbe59e1e9e4db14313f33e9e42cf6ab225079d3c573ee6377b15ca", + "fileid_108_mic.wav": "1cec55ee9473c7c6f393583d5a563cb4318a146e3821398784bcc98828057aa5", + "fileid_1090_clean.wav": "4d878891c8492f55310912c790bf8b5edd9a63e49261a864bfbdd25ef5526555", + "fileid_1090_lpb.wav": "15ef0b86c9943a8ddb2fbd5356ec03a4446c3851b6921e08eebced14e7c30f68", + "fileid_1090_mic.wav": "642befc79fdb709263b00d36ca870772903e909b1ed736a0b5d3302f528e26c7", + "fileid_1091_clean.wav": "b22f769ce9d23714919b1c6a85f4ceeac55b8cd3aa1ae54b1889d024e39e5f1a", + "fileid_1091_lpb.wav": "d92919c490208e5982d78077e1c3f3b636aaaa4e06174ff70713a144f3835f75", + "fileid_1091_mic.wav": "cebac45935b2cc02c0afe9e90d1b7c14e0fbb09137eb52fa2b90904e782ce09c", + "fileid_1092_clean.wav": "f04947511722eeb0aa0c5c6cf8b1a36478b312b39bd070f8e7c47f9dfdff1a28", + "fileid_1092_lpb.wav": "42aabe9940b427be8493fd246ad2c87b537941ad0dcc4e8397eae65349b78633", + "fileid_1092_mic.wav": "c6bd208978244feb259d0321025a94006f12d7016d8218665ada2b64056b1f18", + "fileid_1093_clean.wav": "54a9cf1ba95dc043d955bf97a7ac2a34f7198d0b4567f95e02f49bebb2d6d7b9", + "fileid_1093_lpb.wav": "87c23502e73bc60676a4d20824d715e1b9cc9d377e9260f6ae76979952018113", + "fileid_1093_mic.wav": "d60b1cc9e0cc7838c3c0d38d63e263f4d5a330325636d1ad9e995478d4b45cba", + "fileid_1094_clean.wav": "6f9a7da40c332efae4392f3a8a463b15f09ac410edb4ead3162dcba4b415519d", + "fileid_1094_lpb.wav": "4c28bff9f26844fb78ceb3313eaa70572398be17b8cade627314a280aaa7d62b", + "fileid_1094_mic.wav": "fee568cb81af44615139fdca8d68f7fc92da570b6a1d0d6c3b9fb76ae33e5c6b", + "fileid_1095_clean.wav": "f6e175faf3a5fea3b2117d95a310ad6538a12d29ed0409e8171388247eeeee8d", + "fileid_1095_lpb.wav": "a44054078642d9ab3601e48c0a0abcadd6913b044935738c295b7bac5667f2b0", + "fileid_1095_mic.wav": "70895160e647dedea412a60b0d91a434287f176b6796090460e7cdc16ae96b5e", + "fileid_1096_clean.wav": "a3fc951d9f357db199fe45f91da0c2ad77b45aa246fc7296b672eb32712aa96f", + "fileid_1096_lpb.wav": "1373e2348fad852eb95de4fde352891a2b1f03edda4095d25550afa9fc980d81", + "fileid_1096_mic.wav": "2516265e21181b05b23c0df267264d8e78ef6495079737235c1bf174648df0da", + "fileid_1097_clean.wav": "fb112d6e434a05073eac602f6e8414639709c9ea5b26f9ee0b8deff6a55f8117", + "fileid_1097_lpb.wav": "6635ce70853665b51a20a11e021211738845ca56a329934c4d22a64e4443dacf", + "fileid_1097_mic.wav": "a45e3c098f12241b19a3cf28af3a207242b89e4b2f2e9b575db82ed6eb649091", + "fileid_1098_clean.wav": "db63db050d1021ba29cb4d0b34f5db109ff8496daf196ee5159c7459fe0a4361", + "fileid_1098_lpb.wav": "78ed029cf573a8d3dab9ef9de56355f00831c03431d5d5174f91cdd91038dd07", + "fileid_1098_mic.wav": "29a8492ce4d36e15635e1157f103fccaeac345d0f15ac6f5f162f6665e811b26", + "fileid_1099_clean.wav": "6d9e4e0d2274b520e74f8994b6779a09e58de1fff5ec00a7e1fedc94b6e0bf3d", + "fileid_1099_lpb.wav": "e6049ed80d41705ac00cdcc892df5d1314dfe5c22f93b9051d5abe54ad2434e0", + "fileid_1099_mic.wav": "5b63091442c2c4bd39fe23d9b945ee06a3c7ec6494db81773c9e529c9ca5406b", + "fileid_109_clean.wav": "155853a1b8a4e2398ad85fd651fe54d81fa851eecdf02ce815efa35a55174308", + "fileid_109_lpb.wav": "4286379b6dd4e72b9b3d52c9366b5c090273ac3e7e4fc3c0fa4f43e9499e6fde", + "fileid_109_mic.wav": "7c19a257e3dd90e9231c279c5cb8c81ec4c3edf51ebd4b342d93db8a8b9128a6", + "fileid_10_clean.wav": "5ce8e7d40bf374b62d1a8a83149ddd559de87b41522aaceb1a3c360b94b8574d", + "fileid_10_lpb.wav": "c8c03c6011451dcf3ddefd184fa1dd79461468876b25c8079113511b793626dd", + "fileid_10_mic.wav": "7b32880a339f8c1875bded912bc89222783cdbe713b826bae84e476c03f506dc", + "fileid_1100_clean.wav": "87896acc6228e55a68c71d35d458b0f390bdcf30c0db148d6db7849c7c855ad3", + "fileid_1100_lpb.wav": "c3c12a3ed79618c0d6c0bbbc5f8efb2bdd8033d18dfd80771bafde8429cb1a46", + "fileid_1100_mic.wav": "9379fc438f9dcd6c4e598229ce683a7bc9aa83c4ff30b2c239a120ea74403b56", + "fileid_1101_clean.wav": "dbaedc565a2ce1db1a48c77d81615770a82e4f8dd5601d62856a940fa7440cc6", + "fileid_1101_lpb.wav": "fe36d5ac9906f5e42c3c8f4f32e333d897a0e14ee8bd172e464d3b8e7d1990be", + "fileid_1101_mic.wav": "681fa1cf30e0b0fc144dd47513803c50810b2144f0bc1afab72288f52362c8ba", + "fileid_1102_clean.wav": "5f44e130fe5cf875201e344b21331662fd923ae211e49335f7b0d7dfd5c01fa5", + "fileid_1102_lpb.wav": "5549e0f9e645b652f0ff5bc68333fbb4210886655ba423fffb860fb39b9e33db", + "fileid_1102_mic.wav": "f1244747aebd7add7686b7dd8030f612903159f66383675387cf120cd679c796", + "fileid_1103_clean.wav": "77d0eaaec01ea3db6cc8c9cddc4911b2e8d9b036068f31ca1f0c17b78b84679a", + "fileid_1103_lpb.wav": "221a07ac3e299a0b2ae57a710015690e254d14196b76f434bc23afecbef9623c", + "fileid_1103_mic.wav": "98fbfce6d8c7ccd3b83540cb4de19a3f5642b50b0c92e4967e1c79b38031f968", + "fileid_1104_clean.wav": "5abcd2d4c8a04f5474319c6015ba6c56cde25006845063f2190fb8d1aee112b0", + "fileid_1104_lpb.wav": "33bf1c848715326c2d68877d4f934feaf9999ec2b3840e6eb8396fd5c66c47e0", + "fileid_1104_mic.wav": "e52618a9c704c84f35c60617a49aa68113446a2ecaaaf8f967235aba962ab445", + "fileid_1105_clean.wav": "fb291aed5fe708d8a6950e03d90f414a7ae33207e5ed6620da005cb3e8293857", + "fileid_1105_lpb.wav": "6f91640ae06c97377618511a641b52220d7334b1bced728c698e7e675a6a0eda", + "fileid_1105_mic.wav": "032ff3e61b707645b379d591001ba062e455384d836240a7260a563f74deedbc", + "fileid_1106_clean.wav": "d61023b38da589e4fdd893b7b7107e583e0dfbd12b087c0b23a8c2b2529a6901", + "fileid_1106_lpb.wav": "bd0eb6c5c59581478930a3b9b2cadd97015e08f82a45e9836a92e93e65a06bc8", + "fileid_1106_mic.wav": "8f3e2379cec2d2d2ff1a71a93635d9d05e7f861a39c60a06bbb4fe91b167ee0a", + "fileid_1107_clean.wav": "55d56320b74fbaf77cb1ebb3ec47d24d838567240bcf1f9d4d53aff7834d75ed", + "fileid_1107_lpb.wav": "197ac2a36d4f495989305a191aa2070ad6ad1d8e37ce6fbc696e09fba8f0aee2", + "fileid_1107_mic.wav": "784b1ced5c1e2f6e8182efdbb7f168c68d1effb70d17ff39cb70c87cc5a2d02d", + "fileid_1108_clean.wav": "b6aa12ca0f7eccbb2ba9c9a9129891bbd21176d8967802a75562c57972637d7e", + "fileid_1108_lpb.wav": "2baeaa74c1d39706adc30aee0a7b7b0b909d161dc51851643d62c81642c0c605", + "fileid_1108_mic.wav": "518fef69fc2ea1f7b66deb358451e88985612dbdc306400d0dd7c594368265ed", + "fileid_1109_clean.wav": "c3118f611d3f26f69a1fb9e2558ec94c46a7f5f3d2e975688c5e18a5d20c4457", + "fileid_1109_lpb.wav": "677950efd8c3eccb6bfdccb09b768803f501263abc7777eedc5476f42a97a588", + "fileid_1109_mic.wav": "3d0303ca7ca477beb04e97427339317e812405a465f3cadca76a2ce19a4d451b", + "fileid_110_clean.wav": "3d97e13f4ad79b7a1dd0903921e91dba258a4760f5a6ad2294f317c93ce6bb1c", + "fileid_110_lpb.wav": "5697fea8353e254e8419f36fa96a0f34e86245d22c534ea11c09b9a7c2fc9640", + "fileid_110_mic.wav": "fb6f31c761706e608ad107943a27a8a92eb3819a18dc04b43c3aea6579d3ff9c", + "fileid_1110_clean.wav": "8988e6890fa3e958770433d90c0339b57cabd424c301a48b3f23042f4dd6c4c1", + "fileid_1110_lpb.wav": "5ec1e361a59c4064332df2797440f0d6d63fc83176feaf04fc42b547c5e6770a", + "fileid_1110_mic.wav": "d46772bf070f74098a1e08d401e2906450d5a9d4ee084b979c8078d0152e589f", + "fileid_1111_clean.wav": "fe4ebd955692f1e36839a12bfd9df209f90a0bd103deca8478a8ae951447ca5a", + "fileid_1111_lpb.wav": "2037d1e5f68edf434adca8f96c0ffb8d726348b1e195432d1240f84ba98f3cf7", + "fileid_1111_mic.wav": "0c4d8aea35f3465983168e6fdc5afacc6739b7f6ca21671d6d3027983e90ca7c", + "fileid_1112_clean.wav": "8d4d4c9b7583ee0fb4aa4ef7b7b6cfa347780c5616a63bf23a3069338ee7575b", + "fileid_1112_lpb.wav": "4cec72d53edc3cde87de3161f30de2479506901711a0a07d9a6dd5090febddd1", + "fileid_1112_mic.wav": "0224144a4f769e57d75f2c2b6b381d435940f5a3c98a0ae3ac0c2354d6bbe907", + "fileid_1113_clean.wav": "1101d637adb91229b8bafb2c6fb5f0c6532583bf602eef9462d34e1d29c33107", + "fileid_1113_lpb.wav": "f0f32aa99ba89e18e9fffd258276469fd3b53f49c33622fe5aec6e2f8f54f860", + "fileid_1113_mic.wav": "98e98e1aafaa0864765589aae7ff036e0fba52213afe6498dc9210ea856decc8", + "fileid_1114_clean.wav": "199c1159382d11f12a9fe40b6adc8c56913177f635cef06617d111bab9e1cf0e", + "fileid_1114_lpb.wav": "8211c02b1dff9e584237a730ff53d1d404d2c93c7ae2c222c26a638aba207bf6", + "fileid_1114_mic.wav": "809f51b2f0c65f38087651ef0d65654117c1424685e08e3e24a6a7aa112aa3e0", + "fileid_1115_clean.wav": "552ed6534ee03e271cd160698375fbd796c8cce1643bcb5109d63d791ea392a9", + "fileid_1115_lpb.wav": "db9e54e19ed2b5ab715362c282923bdefbfcdaa3d8f7d5a61678b74b91863464", + "fileid_1115_mic.wav": "7cb339e7690a350bf7322b3fdcd17eb891162b3de84cb627b35149dd6b9029ff", + "fileid_1116_clean.wav": "74ff3209ca3691675ca81c7456a706ff436af4964ec969fa8884cfef2149b51e", + "fileid_1116_lpb.wav": "c6572116d85281133891e3d8496fa51c7defbf6e43108e3093a5064616c58cd6", + "fileid_1116_mic.wav": "c335d0ec054fe9a89225e02a49ef73c718ac9bdc282b52dee721d59409c4f20d", + "fileid_1117_clean.wav": "a7b28407f1278cb89da05aa5565ed1402589680e858789ae62ece03e5156520f", + "fileid_1117_lpb.wav": "03299b16d970075490a54ef448c6b7afc11c7f852f2998d72d7f76cb1ee1a0f8", + "fileid_1117_mic.wav": "a28237bb7836a9fd76f0ce877616925cd27bc31e0a68bf7c43815dcfe48c6cc6", + "fileid_1118_clean.wav": "8149c9ab99888e6b32e90eb6b3b6d602289971824949bd927c51cd0b69b70cd6", + "fileid_1118_lpb.wav": "a33ec46e1358ad494523c44b3e1955a50ff18cb5aa8dc0d370e818574ca96c62", + "fileid_1118_mic.wav": "c5f79924203b82d4b9a4d87974be788c799f1877f9a8f53bd50e1486b49b7c14", + "fileid_1119_clean.wav": "2ba03b7e559beb5df462a73f2a8954073028f4a39edc6dad667f55eed8ca3774", + "fileid_1119_lpb.wav": "2a7faad0b4feecdfbae2f165b18b1101708512759f688f56b20e55ba424cbc05", + "fileid_1119_mic.wav": "71f19eb8704fd96fd321771d5adaa818e1bdcf41b992d2d2721376614b2a56a6", + "fileid_111_clean.wav": "411b117f320cb94e9d1524060e35b3fa925deefd6765c812791b758684bcbafd", + "fileid_111_lpb.wav": "d4dc8e820907f65fe52ebffa81577eebceb4e0ba73a3f810439f15d4854f5e23", + "fileid_111_mic.wav": "e873412f0c196ce651b042eda9dec89f62d51e33abbcc6124500bd6e71107836", + "fileid_1120_clean.wav": "62bc61d8dc741dd1e9b883ec35532992ebb6d7b7791d1d49bbf9eb7b4c9d99a9", + "fileid_1120_lpb.wav": "1e94f520466a9f022a26aab4fdf0f8610bf4b5b5a38bd3e1bb880b8d1c63d85d", + "fileid_1120_mic.wav": "8d458028ae9153df47c0b34f518dc7fb62a7550c5fe48d573975687e6e493de3", + "fileid_1121_clean.wav": "ecf7e58aea8bfa075bf539d5c234bbeeff5943e06c960bb2c7240676f79d6a63", + "fileid_1121_lpb.wav": "e067698e887ee66c97aaab74575ca18af1cf0eeb7af661ea2e9a5b5568d6459d", + "fileid_1121_mic.wav": "f9a2fa2ab3038febf11b980f021004667c8d63b420c2a9ff7e1acdc8ef41033c", + "fileid_1122_clean.wav": "69d9831b0597587ce3284482af02905879265264adec7e80732abedbc1208d3c", + "fileid_1122_lpb.wav": "ced1da5319071689ef28bf646a6e03963242a88b143292ca47f6889a69c24b22", + "fileid_1122_mic.wav": "a4a339cac75c3156258f9dbc70afc99a3933c5e687f0e3942e06c53a0dcd7b45", + "fileid_1123_clean.wav": "32a6d1313b80720df0c7173e12316b809d967850e05ee8c73325b4a83fefe08c", + "fileid_1123_lpb.wav": "60e89cdb5aa5bcfeb8d7598e0e96291504c6e37ea81443052ebb884b0e456c2f", + "fileid_1123_mic.wav": "f4a1b52d565ca261d14d0a06c347a4c60e2af14a54594cdcabb4aef973ba8c2c", + "fileid_1124_clean.wav": "31f7ddb726ceb972dbe542bfe15b16549ad7a1bef1ee2f25244ca98c0b81df54", + "fileid_1124_lpb.wav": "417c2a9131c9614588efc4422889258bba2517844737d49fbbbb0b3976745b8e", + "fileid_1124_mic.wav": "6dde20a9d91dc1d32a47d80ad3a58189e37a5ae91924ea89853e7f5eb4c69d17", + "fileid_1125_clean.wav": "9a3765953d4d4ba45e9488053a242047433e0fc166f2994cf531ddbd780026e5", + "fileid_1125_lpb.wav": "463400e9ab70bf7c6d300d25185c2b0d1f68540de01d275e79ce08b1066cf966", + "fileid_1125_mic.wav": "043c6b183d50f482ede636a3877b58ec75e73f0e847bd4599f2d685c5f23ba1b", + "fileid_1126_clean.wav": "4aa1f91183f73d4d41ba59e6f35b9fc0d04168937859c7fc89936979ddcf7a37", + "fileid_1126_lpb.wav": "bd46fc0571a794374ea7b043683420bff53990b79414b273222e21fe4f455ef8", + "fileid_1126_mic.wav": "874ac36765a97c7e8217bc0dea2a6f3cac52c8b9e8593f7bf6afb3a1c18d6b51", + "fileid_1127_clean.wav": "b90a900fc0345ea7b37dc10f6bed581673db53503e4a9214ae24958f9405626a", + "fileid_1127_lpb.wav": "8009fcf4d9f967ed1160f969444e05530cc20f3561ceed6534c8024cf9dccd36", + "fileid_1127_mic.wav": "a3fbc8b65ebb8d973f6ad1b9afcfbd74179b8fb8986ea06fa83b315683fe6ebd", + "fileid_1128_clean.wav": "485e63e025df2da65300f84ba0a734f1f8dd57a16591e534cf8505b68886ac1d", + "fileid_1128_lpb.wav": "c54727b01f27ecf606274d6dd01ded16f2b96a2e0302a26eff86a49cbe88ca65", + "fileid_1128_mic.wav": "d03b4c408de81e29a12a7ec138ce33b8fb3f2c03a947eb8502419b28492d6998", + "fileid_1129_clean.wav": "3b4c7415f564aca0c41bbf7856470630b2aab0e97eaa8aa775f3fbddbbb4f594", + "fileid_1129_lpb.wav": "54f61f008a2f8d18bc68902a7f0e3c429f0f11a0531a82a88e7fe9229c8f35a4", + "fileid_1129_mic.wav": "cec0c8a792dcd28811260ecfd1f01de0258b8bf0a31bce117f1942c8008d7484", + "fileid_112_clean.wav": "4eb2d70c50deb60c53f158467bb12dddd437e5eb00fe7e3db82af3aac4b32198", + "fileid_112_lpb.wav": "000dbe81ebbe12e300f34aa217bbad56287eb41a49e66b13befbb35d5d43941b", + "fileid_112_mic.wav": "50e7dea2f76b3886c7108300b69093357f3aab8307ffd5ed54c2c55fc751917f", + "fileid_1130_clean.wav": "a77542170eb2ee3b868999a4c8c5cd2cada47b0392aab6f16347f3c9c755cf95", + "fileid_1130_lpb.wav": "5ba65643ec90aa0ea04470e2ff2e3cdeb717dedd92c551e98127dae5d2343b22", + "fileid_1130_mic.wav": "9794abc52e296e08144cc9674c973ddadfc5db8f212459cc797b3a27f657eb45", + "fileid_1131_clean.wav": "48fa2786d4b5356380a651d0303c09ad4d67fe01d12ec40c5435680907789959", + "fileid_1131_lpb.wav": "0fc423ab650a48aa9f3f414de782646973187d253f392ae5a9cb2728193fd073", + "fileid_1131_mic.wav": "ca6fbd5ef5ae6ea8a65a4046fa160cafb3b12688780122bffded0893e116cdb8", + "fileid_1132_clean.wav": "97019401a9dfe5f44a4686a668e3ba51df41e6cbc887ef9986ad6a6482298045", + "fileid_1132_lpb.wav": "bbb68ca6c6c01717d197c7b6fbf28940eec5526fe0c5a5dc8ad8014cd724d233", + "fileid_1132_mic.wav": "46b9d3627dbb43fb49eaafea3dff15a8dd8c835bc5a30915e5a1cad1adda6f19", + "fileid_1133_clean.wav": "09f53bea4960640e0424e9713af5350f572a44a9b6e60d871dd2192ab3bd6dd2", + "fileid_1133_lpb.wav": "ba7ba979949d2139749920031f32a8b41f67b28b76ac13a8cb5a733762a6e410", + "fileid_1133_mic.wav": "ea5406cae373f831997d67e75878fd2885bcb38aff5aaf15a2dc7f31cebe75c8", + "fileid_1134_clean.wav": "a4b3332fc2111297b783f7f028a344f4c7f101bb48631ad4b8cb749e26b4a2b6", + "fileid_1134_lpb.wav": "22c61ed26f4b9ac38e3cd60741854d5d41ff0eaba6ac5008fe3528f9d0961909", + "fileid_1134_mic.wav": "d01113f22c73afa5c9fed415cb9d20e8ec4469b7454ecc41492a463ad29c3113", + "fileid_1135_clean.wav": "a27078db327221c122c7ff6cc0760ec3eed71baf976c28e429efa718d22db17b", + "fileid_1135_lpb.wav": "e50d3f90916b565919f06a33e4c9f5782e11301db7436281a2f7f70396c0bf15", + "fileid_1135_mic.wav": "9dca9b84e2cba17cad3c22b19810c681f7fa2138ea75a464d43068e7ba6a4d0a", + "fileid_1136_clean.wav": "9479db265595b026c89ecb8574af3776081ee625dfbcb0844a11e89aef7a5452", + "fileid_1136_lpb.wav": "687b0eaa44a6a0e2b8b95350f4177a27fa6a460392fdee5b5359836a08b95e1f", + "fileid_1136_mic.wav": "bfd769ffeedfcd63885056e95c068637fcaa4f809a06a93c7e569702dcf87139", + "fileid_1137_clean.wav": "4b9ceb62537b8570908feaad2cb7285b3da8aeb4630f49986ec28acb3c727a90", + "fileid_1137_lpb.wav": "1570e613fa9c78c1f3b3f7ff7f6f261c3913015cf77f3f00ef7a53ee5eb853c0", + "fileid_1137_mic.wav": "89fd747639a3c19a337aa9437aac286e917d3fcdbd4025839de2e30ef37a3fe5", + "fileid_1138_clean.wav": "3e709767fd556e19bd155ee4c85b87a197c50bd13acfb6e6c507879f9ba995b5", + "fileid_1138_lpb.wav": "c135f68b802f67b15415af0cba8672c1709df2d4ea9c81b28fd7c10d7aad798d", + "fileid_1138_mic.wav": "1da224e0965a2db35cb0b00170c35a0a1daf86c45b2ee16ad2ad0c67be5ea087", + "fileid_1139_clean.wav": "da3f46237c11a58eb61cb8c11edd7d43ceac204103fa2c9714c4b4fa27ef3909", + "fileid_1139_lpb.wav": "17ee79fff94d81d1e7c65fdd555c92350f9a99a6f80c449c2bd58c43c84e47d9", + "fileid_1139_mic.wav": "efcb1a001fee66ce179f69c710d080b9e16f41c357dd54c50d6ecb84944cb33a", + "fileid_113_clean.wav": "c2a2cb6cfbbb9153eb41cd8dd5d5b698225bd1c0c3adcaf1f8c125a244abe537", + "fileid_113_lpb.wav": "c68e6a3d58917134ffc5f700224bf772771dcd69b6e89a508c9b7946b046a626", + "fileid_113_mic.wav": "efa29df1ab577cf993ef4f99fba19379b58fb9b8c4cfa9aa61dc34c6d5855a8f", + "fileid_1140_clean.wav": "de72bd4b896811bc4a82382a41d74350043f1f5639e0678bcd4ef05315c7141d", + "fileid_1140_lpb.wav": "d5226512f7916a0b7cc06e26dfe67ea3d4a61aa4ca8c2b0c41c266535937753d", + "fileid_1140_mic.wav": "1560cf21835eb3cffb9a55d5129ed1d8ef95b28eff1aa92245a2f09ca652bcca", + "fileid_1141_clean.wav": "080d2d06d71386208e4d92538b042506ccaae1d4d8b9d9de57cbe2b31ecb8764", + "fileid_1141_lpb.wav": "4e1d46cecf0e79ed9d0e83d62376a34f60d59ca40bfefdd230b8e115baaec2d2", + "fileid_1141_mic.wav": "d7316cabc7269891c3a9d8ccdb776d8241b2dd5ccda20624ea9ab105ed6961dc", + "fileid_1142_clean.wav": "c494d186d673e4aea1e2f8a596057a5ed282a76ac6ac57e29253581ab5675113", + "fileid_1142_lpb.wav": "f26b4a2ffe72803db460b0e986b2972037e90de6cf4e01e8010514d226be98ae", + "fileid_1142_mic.wav": "a866c9687dac8b603d181e62a4752c3490585ceeb91a983a0bbaa9b4c2f1c8cf", + "fileid_1143_clean.wav": "a7c1b54338eae8c90c6969a197679dfaa41d5334b6151d64cd439d649f0a5407", + "fileid_1143_lpb.wav": "299a82461b1ee291453658bb165b93334752d15b4bfa9bd464184f858ec5ec96", + "fileid_1143_mic.wav": "9b0fee1a96ca303f520c8231e166b953805c7723c9ce5d5d67f01de698ebc5c1", + "fileid_1144_clean.wav": "92e6aac8e1ad80af3b91246838579867362f2777c524b440d6a0f595bc9086c6", + "fileid_1144_lpb.wav": "d574e0b3ea45da8b0834bb1be72f585e355a4ceef0354412b26004e5be8c2903", + "fileid_1144_mic.wav": "b6df0f61e925d6663c64bdfe40b2d6ee25cecb6f8eb6951932a7bc202a42aef9", + "fileid_1145_clean.wav": "dc53422a43800b2830395e543e7b327daa8a9afc8b3bae09be80c328a413e526", + "fileid_1145_lpb.wav": "6a2c3d43fc7a2817066867c340aa039013208f9f9fb7a7f9c742964bd87cc586", + "fileid_1145_mic.wav": "104200b6b7135c2449d6720684b198f31cbd94e0a13d8bd2febf6594032d608c", + "fileid_1146_clean.wav": "6eeb7c20a61e0e8abbea39b74f5e3db9a69a4cb2099830927a71e779ecbd6470", + "fileid_1146_lpb.wav": "2b9524d4e300a6285e570062013d2c990b2da43864afc38b8d93b5feca9fd66b", + "fileid_1146_mic.wav": "0a4fc9062fd52f8fcbb6292dbd0c45b243c40f7ac1dbafe3db9b50b3f47b7619", + "fileid_1147_clean.wav": "87da8f2505e362047851cfe7e282623a38d9d2561d6039c85913b22784691a44", + "fileid_1147_lpb.wav": "21abf2d4beed1ebae45019217318ab6ee26172d7f9892b3d074a093ea8bdbce9", + "fileid_1147_mic.wav": "8796002d63f18caafcebb7b8f345c6124aff6bd14b66b004ea807876aafe1ef8", + "fileid_1148_clean.wav": "0d061165fcca9ea216c589f9195afb39a57569562c1a796b36c8c68129b9702c", + "fileid_1148_lpb.wav": "f93144365b00d4011b0aef28dd96f3298fabd3643f3d9be42ffa8601fd383fe6", + "fileid_1148_mic.wav": "3799451a8526aef99d0dc7d3f807fed06a901b721c644816518db8eaa1c4ceb5", + "fileid_1149_clean.wav": "9a61905e4c7dbf0c0220abc4d606be6dec39ab56094638cb9fce087f23d0e1e5", + "fileid_1149_lpb.wav": "8ed2f4b508408485399c08f46fd43305db0022198f831bfcc813c40dd25776f7", + "fileid_1149_mic.wav": "86390395de1e8067f8b6bb68220edf59d0565af64a5373de48dbb1bed8b55326", + "fileid_114_clean.wav": "9bedd557c55fd68d9527d070a34e92091e74d0792797c97245958101a925f715", + "fileid_114_lpb.wav": "7a1c995ed43c8221237e6a68fae574527f586df8d45182fdd1c3998ed667c9fb", + "fileid_114_mic.wav": "35aebfa9ba2bf64ec3d0e3d5d48a4f2f9fcff527b6aa57d9299ed693c2450225", + "fileid_1150_clean.wav": "789c4a8dc099d4065978d37bbb9dd6ab033e6cc34b0948c455faa6fddf8b0246", + "fileid_1150_lpb.wav": "98d5f246afd3e45a6384b207bd1f3bb7de8c20845bd3b7e144243972bcf7e477", + "fileid_1150_mic.wav": "98f81d40344ed801c5cc51656c464b07966588ebb0c580a0bb633e547b4162a9", + "fileid_1151_clean.wav": "2b53ff04d899c13aa232dcdc1e65080daa4cccc6e68352b6e8a852fad737f280", + "fileid_1151_lpb.wav": "51d7a4a56b288ef75c7024c1f3aea59e26e09634ce5f1de30f66821b39c25ba3", + "fileid_1151_mic.wav": "ab9dbcb6c8b20c4216fc8f81e19b97ec7ba45dfe099f724da8665414780f39f8", + "fileid_1152_clean.wav": "7762849d866144a17fc93889ed00d9584cff47e6ad53ea7d6f20728da60456db", + "fileid_1152_lpb.wav": "3371e65cdb685f53771fad4d1fde73a380b4e0d4b8b13bfb5f584a927af3b3ff", + "fileid_1152_mic.wav": "c9d0ba3653814cfa914b60769ce427c90517c66bab700c5169042cde1c618ac5", + "fileid_1153_clean.wav": "a901d3dcb88ff6819140fe2f6dab229e7b056e7a4680a4be2ba5c4b9b33b2e2a", + "fileid_1153_lpb.wav": "53262fd0a25a70217646e37db52331fcd77a2fea804d106a446242e77d7b592d", + "fileid_1153_mic.wav": "fd8510509ecff825d6b614378d6367a5fb8edd55ecea09d6197c2fa0aa2655ac", + "fileid_1154_clean.wav": "fde0a5573c708e2a6407d4a1af1957802cc0e2376285d1900ef0ed72b8ab44de", + "fileid_1154_lpb.wav": "7a2f05884441905fbb19bf1293461842b2887498a86c83a5360528fca16ce65b", + "fileid_1154_mic.wav": "ac5e2b422aa915005d15b848c1d37183bf5d118353eef5e92d15bfb0e73a6c66", + "fileid_1155_clean.wav": "8f4e1f54a14710fe55eeb8b2c0213959a322ce09d3069ee153353ee26980cf78", + "fileid_1155_lpb.wav": "58cedc4b9b8197d185267aae5f7a99f12b41447871792192295464ef5f16f224", + "fileid_1155_mic.wav": "ceff24cb6aea89bbdec57bc8a04ff9c2911a518c14ea6a914022aa394e138adc", + "fileid_1156_clean.wav": "81cea824aca20cfeb19f5635f945199218ecbacb4e1fdf8e584a2621e75dd57d", + "fileid_1156_lpb.wav": "e1d352063cdcd676dbed7a0ba98ba218542599c81ced2166199aed0d0ec81ffb", + "fileid_1156_mic.wav": "057fc27a17c5d8ff628565680359f6ec72300f93a945a390ca5e97b2bb81c2bd", + "fileid_1157_clean.wav": "bb17dafb1553f4f401d42675bca2739246fb7837f2ab7c1fa9bea31ec88f9e53", + "fileid_1157_lpb.wav": "ee3b0b3482049ffbcd7a69dd2d226de89b81da3684730d236cb7ac3830708a34", + "fileid_1157_mic.wav": "a475cc89cef6e7d8e0d6604e8eb873432c2dd0527145aed825fe3af7d909e387", + "fileid_1158_clean.wav": "54455126d0695c90e716e7d4ad57cd913df65391fb81275c32bad8f22bcd0517", + "fileid_1158_lpb.wav": "39ebd077ad156688e3d527d53658b3b7811b8da44d1b849eec4601eca8a9cd60", + "fileid_1158_mic.wav": "9e8655d29cf37edaa170a992573ec4df93a566a5f6579aebaf163769ae3151b1", + "fileid_1159_clean.wav": "ac807d5f86a17f475eb27d0428113e9fbf50f67ab6f46ffd770d3ff1a8402d65", + "fileid_1159_lpb.wav": "973e207e21d1665ea7a320f8d10f50f674c40a5dd8e2869f150911e5dff256a9", + "fileid_1159_mic.wav": "62d86c19d523a6a2790a311c070f48e86271fe9e5e4ad3e5e576b1fe27a9ae8e", + "fileid_115_clean.wav": "d80a661d8b26dc8b03d3c7f68f8bba7b7cee4bfa10b059d7be2aa22f2d782a09", + "fileid_115_lpb.wav": "aa9b78ddf82c472fa76c7ab29ae4e87bd562b741c4b7d68d55341bf7ab07522b", + "fileid_115_mic.wav": "eeb3ddc1d9476776e6dd5a11e47862f079cd1503143e6ec22a187be957a5a8fd", + "fileid_1160_clean.wav": "cbc1f3c0bc846da7a0c4699b674e54326c6548219ee051e13ecb9160867142a6", + "fileid_1160_lpb.wav": "f5dc5eab3527cd15341e94cf99544653bc351741b6bd543753da296ff5e8a43f", + "fileid_1160_mic.wav": "3f5e1318d2a951bb1d35fb6d9b6eb5e5c78fa079f1ddb504d33003b88c93d6ac", + "fileid_1161_clean.wav": "76ee6485d926622dbbca8d4af0ae11ac8e31664220da68808a3207897287b93b", + "fileid_1161_lpb.wav": "99a4dbca675ff752dd1b4da68f359de24e1728b6da5878b60ffe4b347cf48dd1", + "fileid_1161_mic.wav": "ffc3dc224bad20d19f8e56aacc8ae47ea9c72be5b1a245a15df2dd5d56bcd914", + "fileid_1162_clean.wav": "26887989387f2d5bc4932a6b7a35db263cd61b8ab1977bad00635e1b041ca4ec", + "fileid_1162_lpb.wav": "97a71e6ff4c814edc10696deca077290009ee47dc7fcba8206f4cc3864c2dbf1", + "fileid_1162_mic.wav": "10a7d8ec87db34128e6e1b7b630c75611a4ead4b289881ae75cefdf2f843daac", + "fileid_1163_clean.wav": "ee1c493fe71876f8a5bdeedb913e2ad009906892704aed546f8e94861f900394", + "fileid_1163_lpb.wav": "1164421f8a5675d43bc03be7f48498dfa71d604a80d7e72a29324976f35b718d", + "fileid_1163_mic.wav": "2110161cc42d2bff44e3b70ab7376b3779a35e70fd61261fcb375ca2bad5c8f9", + "fileid_1164_clean.wav": "e1e99de07afdbab74b3ab1e7108af73b71ee4b432ab7253bfdfaa6bfc924ea4c", + "fileid_1164_lpb.wav": "b2bebc1a61f9fd702fbabf33ee5dad9997b7601ddbf94f2031596cc6efc20d66", + "fileid_1164_mic.wav": "8d19c7de86700766b95452245584a03b2163c6f13953d8e3e2497076583c92e5", + "fileid_1165_clean.wav": "7a3f1913d1aa70b565f1e40704533cc8d79c47831eca7d142ec9efa9c1e03cd3", + "fileid_1165_lpb.wav": "4c311bcc2694ac055e27f765f2dcbbb6490fbf151a432ba772a04eda8fd6857d", + "fileid_1165_mic.wav": "9f0bacd6d714746af1a72a8c2b30e47a59b5f98b572db7dd5ebdd36fd93db593", + "fileid_1166_clean.wav": "2c8f305e31a4b8031c02f438575bf043037ccc7bef7a8824fd9bcc91b0b77e69", + "fileid_1166_lpb.wav": "52cd27ab3960b741fccee4655d998bc857e5a55df748aa50c8dbba4824b8d046", + "fileid_1166_mic.wav": "fbd9902c5589a5a4ec701f1a31c0f50e7295d0bd739e0e81ca4221f9c4f5c22c", + "fileid_1167_clean.wav": "f4fb8aacb274c762671755633d997f2d208759a5be299df92202c7b34addc826", + "fileid_1167_lpb.wav": "51ef54b683a8801b23b7e26ca6f1e8e476e6f2c14d267fd2a03b99bfb2bdf2d0", + "fileid_1167_mic.wav": "35d8ca030508bf56643915308d1b2ea1f75a32ba814747b92e8d631fc76f8ede", + "fileid_1168_clean.wav": "632c9c44fd77910f7a48a2fdc474ed0aa6766c0047c8a45598a2f6530f32af59", + "fileid_1168_lpb.wav": "852879eaad6a0e98ece322174bfc7fd7bf6a5703d2d2a274e7cbb3cb6848f435", + "fileid_1168_mic.wav": "2280d3fb7ca69ac8eb7d36d0cf7173715644f1c8100abc9a906fc7266cd6c0d2", + "fileid_1169_clean.wav": "572cc4e4c9be0f978ce1baf0e93c220bcce0a592ac9a307fef51862bcab838b9", + "fileid_1169_lpb.wav": "e666afd95aaf7e8fd0adb5de16a2192cd759fa1b9ea1f44598be06a94972dfe3", + "fileid_1169_mic.wav": "7748ceaa67fef81f4a2f97951af75a6f70d1098bb9ecb4ec995ab95eefe8efd6", + "fileid_116_clean.wav": "6e444139a8f4e5116b39996d8852751221a4b37619c00bbf001217dfc6b45bff", + "fileid_116_lpb.wav": "93c305fdd75567e164f52ae9d0dbd077dd8d9746e69a7f0e43da9438a141f543", + "fileid_116_mic.wav": "3972936d7040fbcee5ba21547749afc669adf94a9172e4e6d3a172c2978551e1", + "fileid_1170_clean.wav": "0dcefad8d4743c77037d9d889233809f174f04fc95162428516e2856fc1af18f", + "fileid_1170_lpb.wav": "da4f90a994c47e2ae30cdf2e24ff201f61a4db19933b693e1f3d759132038be0", + "fileid_1170_mic.wav": "5ad71e7cafbc479a3f178aecbbaa4bf13015817e2cdd83bb38ff26d0a03d879d", + "fileid_1171_clean.wav": "6258f04a350ab9d0bc6f15d0c8c0e466c288fff1aec558acd432304f9012f2fb", + "fileid_1171_lpb.wav": "2cef1499dfe0a3a92eddd3d97ad2e1b90229b0ae4930f15e4f999abcf0b55664", + "fileid_1171_mic.wav": "6c3f4346519f9022624d1b70e89881745d51f4b1f2aed46f1f96e7ea26459338", + "fileid_1172_clean.wav": "87497ad175ee4d8f3ab6a00b8ff49563f87440f50899bf7fcf9810b60b6167cd", + "fileid_1172_lpb.wav": "6d97789dda417ebf53ac0e956a794fd8d05ddb569e66c140f277491d3a4b9c54", + "fileid_1172_mic.wav": "7d2c17d05c7c2407d6da0eb5d64c32de7a92ad0ea049c29632c5a59419d5e7c3", + "fileid_1173_clean.wav": "f82399e42c2f64763eec90371e8accd32ec3bda1ac637878c0eb16ac6303e161", + "fileid_1173_lpb.wav": "22cbab46edc21124115bc706a1e2cfc65d589f4d53044e613d7eb6c6afdbaf82", + "fileid_1173_mic.wav": "2de4648c3799c101b1d1f9548ac783bb70e9b64fd2d67940fa480abaa53bd7a3", + "fileid_1174_clean.wav": "e7f7d43f0f4dc1b279f0c6befc9ec9b1494e5a0ff26eac18f656fcdbe01dcd21", + "fileid_1174_lpb.wav": "07a8205f1b937f0caa581441c8adc98b4dd7137de1103e7b7e21c7000afb5e2c", + "fileid_1174_mic.wav": "03a47d1479b29c2268cc58c2cc07add54a2834927dd4f3caf03a460f38193cfc", + "fileid_1175_clean.wav": "4b94971dbe498e05ed93ded39fb02dba9b49d2be0780c6cb332e8b978d1ebc28", + "fileid_1175_lpb.wav": "2622493ec9ff5f0a4b3897397d7cd885a2a09f69c21187eb97d62ec2faa8430b", + "fileid_1175_mic.wav": "e5f9e35c532ee1804c20dde90a0f9482bcd671cedfb49a7e8ed7a3ffd5089dbf", + "fileid_1176_clean.wav": "1507f78c09c750c4c7026ef245dbb22b510bceed0195c7c988fec202b01aa6d5", + "fileid_1176_lpb.wav": "c338c8029a5cbd4918ad9072379ba4e6e430bfecebfddb999a75ab142a61e196", + "fileid_1176_mic.wav": "b6eaa3a11af01f767284220f78a00fcc8032891818773ba4d22f76f1c57ee206", + "fileid_1177_clean.wav": "e83ee02ad8cd2bbcbb7ff6d2272a204b24a917263da073aed5274eba0ed29ad1", + "fileid_1177_lpb.wav": "6beac20575e116ff6f08c61235caa0ab15094a8a36dc4b80082a788ced391576", + "fileid_1177_mic.wav": "54bc0fec07d27a9e6c46f71662e4340416cfc0e90d5c5f99aff3329ab6f1b854", + "fileid_117_clean.wav": "31c9351ae3180200aabdcedc8fa339f5419d13c070bc2c9a77c3e971d84fee01", + "fileid_117_lpb.wav": "bc39bce02c16c2ef2848ae30c2b6fe0b457582ba045824bf7c33a00548c70374", + "fileid_117_mic.wav": "45c149c88465efcb3a0de5c44937eccd0014c5c0d99dda6d96a27dc6a395f75b", + "fileid_11_clean.wav": "62e17274e94f90f96fb23b15f3ecd0fabc4fd85298198cc3856ee9d669bd77dd", + "fileid_11_lpb.wav": "fcd58688c0a8d206b0c647ef3d92d1cbf84676b4cddd5312eb107aecf77cfb83", + "fileid_11_mic.wav": "d2c6ddc7709e0e31bf59e79c4321c9dea79b1d725a4e1092372b2e3f7c33b47b", + "fileid_1_clean.wav": "d632e692c48928f62ebe613376f0875216751c282f5d6a89b97e1cf619283cf0", + "fileid_1_lpb.wav": "d24e74237fedc60f271244a882d83f9c4088ff467c4f056ad01de94b495ed571", + "fileid_1_mic.wav": "39d0a1d1c1faaeca7180bb47ce6cf0cab57404541c530b8b56dea6a57ca1a331" + } +} diff --git a/Scripts/test_verify_localvqe_benchmark.py b/Scripts/test_verify_localvqe_benchmark.py new file mode 100644 index 000000000..03beb211f --- /dev/null +++ b/Scripts/test_verify_localvqe_benchmark.py @@ -0,0 +1,163 @@ +"""Scorer tests and corruption checks against a real benchmark report.""" + +from copy import deepcopy +import json +import os +from pathlib import Path +import unittest + +from verify_localvqe_benchmark import edits, merge, verify + + +class EditTests(unittest.TestCase): + def test_empty_hypothesis_is_deletions(self): + self.assertEqual(edits([], ["one", "two"]), (0, 2, 0)) + + def test_empty_reference_is_insertions(self): + self.assertEqual(edits(["one", "two"], []), (0, 0, 2)) + + def test_mixed_edits(self): + self.assertEqual(edits("a x c d e".split(), "a b c d".split()), (1, 0, 1)) + self.assertEqual(edits("a c".split(), "a b c d".split()), (0, 2, 0)) + + def test_substitution_wins_ties(self): + self.assertEqual(edits(["a", "b"], ["b", "a"]), (2, 0, 0)) + + +def split_into_shards(report, count): + """Re-slice a full report the way enhance-benchmark --shard i/n selects examples.""" + selected = report["dataset"]["selected_fileids"] + size = -(-len(selected) // count) + shards = [] + for index in range(count): + ids = selected[index * size:(index + 1) * size] + shard = deepcopy(report) + shard["shard"] = {"index": index, "count": count} + shard["dataset"]["selected_fileids"] = ids + shard["dataset"]["audio_files_sha256"] = { + name: digest for name, digest in report["dataset"]["audio_files_sha256"].items() + if name.split("_")[1] in ids} + shard["files"] = [row for row in report["files"] if row["fileid"] in ids] + shard["excluded_empty_reference_fileids"] = [i for i in report["excluded_empty_reference_fileids"] if i in ids] + for condition, summary in shard["summary"].items(): + rows = shard["files"] + for field, key in (("files", None), ("reference_words", "ref_words"), ("far_end_words", "far_words"), + ("hits", f"{condition}_hits"), ("errors", f"{condition}_errors"), + ("leaked_words", f"{condition}_leaked"), ("audio_seconds", "audio_seconds"), + ("enhancement_seconds", f"{condition}_enhancement_seconds")): + summary[field] = len(rows) if key is None else sum(row[key] for row in rows) + shards.append(shard) + return shards + + +class MergeTests(unittest.TestCase): + def shard(self, index, count, **overrides): + base = { + "schema_version": 2, "protocol": "localvqe-asr-v2", "configuration": {"asr": "x"}, "chunk": "256ms", + "conditions": ["unprocessed"], "model_files_sha256": {"m": "0" * 64}, + "dataset": {"path": "p", "repository": "r", "revision": "v", "archive_sha256": "a", "metadata_sha256": "b", + "selection_order": "numeric fileid", "selected_fileids": [str(index)], + "audio_files_sha256": {f"fileid_{index}_mic.wav": "0" * 64}}, + "files": [{"fileid": str(index)}], "excluded_empty_reference_fileids": [], + "started_at": f"2026-01-01T00:0{index}:00Z", "completed_at": f"2026-01-01T00:0{index}:30Z", + "environment": {"source_revision": "abc"}, "shard": {"index": index, "count": count}, + "summary": {"unprocessed": {"files": 1, "reference_words": 10, "hits": 5, "errors": 6, "far_end_words": 4, + "leaked_words": 1, "audio_seconds": 8.0, "enhancement_seconds": 0.0}}, + } + base.update(overrides) + return base + + def test_merges_in_index_order_regardless_of_input_order(self): + merged = merge([self.shard(1, 2), self.shard(0, 2)]) + self.assertEqual(merged["dataset"]["selected_fileids"], ["0", "1"]) + self.assertEqual([row["fileid"] for row in merged["files"]], ["0", "1"]) + self.assertEqual(merged["summary"]["unprocessed"]["hits"], 10) + self.assertAlmostEqual(merged["summary"]["unprocessed"]["recall"], 0.5) + self.assertEqual(merged["started_at"], "2026-01-01T00:00:00Z") + self.assertEqual(merged["completed_at"], "2026-01-01T00:01:30Z") + self.assertEqual([s["index"] for s in merged["shards"]], [0, 1]) + self.assertIsNone(merged["shard"]) + + def test_missing_duplicate_or_mismatched_shards_fail(self): + with self.assertRaisesRegex(ValueError, "Expected shards"): + merge([self.shard(0, 3), self.shard(1, 3)]) + with self.assertRaisesRegex(ValueError, "Expected shards"): + merge([self.shard(0, 2), self.shard(0, 2)]) + with self.assertRaisesRegex(ValueError, "shard count"): + merge([self.shard(0, 2), self.shard(1, 3)]) + with self.assertRaisesRegex(ValueError, "model_files_sha256"): + merge([self.shard(0, 2), self.shard(1, 2, model_files_sha256={"m": "1" * 64})]) + with self.assertRaisesRegex(ValueError, "source revisions"): + merge([self.shard(0, 2), self.shard(1, 2, environment={"source_revision": "other"})]) + with self.assertRaisesRegex(ValueError, "not a shard"): + merge([self.shard(0, 1, shard=None)]) + + def test_single_shard_report_is_rejected_by_verify(self): + with self.assertRaisesRegex(ValueError, "Shard 0/2"): + verify(self.shard(0, 2), 1) + + +class RealReportTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = os.environ.get("LOCALVQE_BENCHMARK_REPORT") + if not path: + raise unittest.SkipTest("Set LOCALVQE_BENCHMARK_REPORT to a real enhance-benchmark JSON report") + cls.original = json.loads(Path(path).read_text()) + cls.count = len(cls.original["dataset"]["selected_fileids"]) + + def setUp(self): + self.report = deepcopy(self.original) + + def test_real_report_passes(self): + verify(self.report, self.count) + + def test_missing_row_fails(self): + self.report["files"].pop() + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_wrong_summary_fails(self): + self.report["summary"]["localvqe-v1.3"]["recall"] += 0.1 + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_wrong_edit_count_fails(self): + self.report["files"][0]["localvqe-v1.3_deletions"] += 1 + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_modified_audio_fails(self): + name = next(iter(self.report["dataset"]["audio_files_sha256"])) + self.report["dataset"]["audio_files_sha256"][name] = "0" * 64 + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_excluded_and_scored_overlap_fails(self): + self.report["excluded_empty_reference_fileids"].append(self.report["files"][0]["fileid"]) + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_missing_model_fingerprints_fail(self): + self.report["model_files_sha256"] = {} + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_missing_condition_fails(self): + self.report["conditions"].pop() + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_shards_merge_back_to_the_full_report(self): + merged = merge(split_into_shards(self.report, 5)) + self.assertEqual(merged["dataset"]["selected_fileids"], self.report["dataset"]["selected_fileids"]) + self.assertEqual(merged["files"], self.report["files"]) + self.assertEqual(merged["excluded_empty_reference_fileids"], self.report["excluded_empty_reference_fileids"]) + for condition, summary in self.report["summary"].items(): + for field, value in summary.items(): + self.assertAlmostEqual(merged["summary"][condition][field], value, places=9, msg=f"{condition}.{field}") + verify(merged, self.count) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/verify_localvqe_benchmark.py b/Scripts/verify_localvqe_benchmark.py new file mode 100644 index 000000000..6caf0472e --- /dev/null +++ b/Scripts/verify_localvqe_benchmark.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Independently verify enhance-benchmark's versioned JSON; no inference needed.""" + +import argparse +from collections import Counter +from copy import deepcopy +import json +import math +from pathlib import Path +import re + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def edits(hypothesis, reference): + """Levenshtein S/D/I, with the benchmark's substitution-first tie break.""" + table = [list(range(len(reference) + 1))] + for i, word in enumerate(hypothesis, 1): + row = [i] + for j, target in enumerate(reference, 1): + row.append(table[i - 1][j - 1] if word == target else + 1 + min(table[i - 1][j - 1], table[i - 1][j], row[j - 1])) + table.append(row) + i, j = len(hypothesis), len(reference) + substitutions = deletions = insertions = 0 + while i or j: + if i and j and hypothesis[i - 1] == reference[j - 1]: + i, j = i - 1, j - 1 + elif i and j and table[i][j] == table[i - 1][j - 1] + 1: + substitutions += 1 + i, j = i - 1, j - 1 + elif i and table[i][j] == table[i - 1][j] + 1: + insertions += 1 + i -= 1 + else: + deletions += 1 + j -= 1 + return substitutions, deletions, insertions + + +def close(actual, expected, label): + require(isinstance(actual, (int, float)) and not isinstance(actual, bool) + and math.isfinite(actual) and math.isclose(actual, expected, rel_tol=1e-9, abs_tol=1e-9), + f"{label}: expected {expected}, got {actual}") + + +SHARED_FIELDS = ("schema_version", "protocol", "configuration", "chunk", "conditions", "model_files_sha256") +DATASET_SHARED_FIELDS = ("path", "repository", "revision", "archive_sha256", "metadata_sha256", "selection_order") +COUNT_FIELDS = ("files", "reference_words", "hits", "errors", "far_end_words", "leaked_words", + "audio_seconds", "enhancement_seconds") + + +def merge(reports): + """Combine contiguous shard reports (enhance-benchmark --shard i/n) into one full report. + + Shards are concatenated in index order, so the merged selection, scored rows + and exclusions keep the canonical numeric-fileid order that verify() checks. + """ + shards = [] + for report in reports: + shard = report.get("shard") + require(isinstance(shard, dict) and {"index", "count"} <= set(shard), "Report is not a shard report") + shards.append((shard["index"], shard["count"], report)) + counts = {count for _, count, _ in shards} + require(len(counts) == 1, "Shards disagree on the shard count") + count = counts.pop() + require(sorted(index for index, _, _ in shards) == list(range(count)), + f"Expected shards 0..{count - 1} exactly once, got {sorted(index for index, _, _ in shards)}") + shards.sort(key=lambda item: item[0]) + first = shards[0][2] + for _, _, report in shards[1:]: + for field in SHARED_FIELDS: + require(report[field] == first[field], f"Shards disagree on {field}") + for field in DATASET_SHARED_FIELDS: + require(report["dataset"][field] == first["dataset"][field], f"Shards disagree on dataset.{field}") + require(report["environment"]["source_revision"] == first["environment"]["source_revision"], + "Shards were produced from different source revisions") + + merged = {field: deepcopy(first[field]) for field in SHARED_FIELDS} + merged["dataset"] = {field: first["dataset"][field] for field in DATASET_SHARED_FIELDS} + merged["dataset"]["selected_fileids"] = [] + merged["dataset"]["audio_files_sha256"] = {} + merged["files"] = [] + merged["excluded_empty_reference_fileids"] = [] + merged["shards"] = [] + totals = {condition: Counter() for condition in first["conditions"]} + for index, _, report in shards: + merged["dataset"]["selected_fileids"] += report["dataset"]["selected_fileids"] + hashes = report["dataset"]["audio_files_sha256"] + require(set(hashes).isdisjoint(merged["dataset"]["audio_files_sha256"]), "Shards overlap in audio files") + merged["dataset"]["audio_files_sha256"].update(hashes) + merged["files"] += report["files"] + merged["excluded_empty_reference_fileids"] += report["excluded_empty_reference_fileids"] + merged["shards"].append({"index": index, "count": count, "started_at": report["started_at"], + "completed_at": report["completed_at"], "environment": report["environment"]}) + for condition, summary in report["summary"].items(): + totals[condition].update({field: summary[field] for field in COUNT_FIELDS}) + require(len(merged["dataset"]["selected_fileids"]) == len(set(merged["dataset"]["selected_fileids"])), + "Shards overlap in selected fileids") + merged["summary"] = {} + for condition, total in totals.items(): + words, far, seconds = total["reference_words"], total["far_end_words"], total["enhancement_seconds"] + merged["summary"][condition] = { + **{field: total[field] for field in COUNT_FIELDS}, + "recall": total["hits"] / words if words else 0, "wer": total["errors"] / words if words else 0, + "leakage": total["leaked_words"] / far if far else 0, + "rtfx": total["audio_seconds"] / seconds if seconds else 0, + } + merged["started_at"] = min(shard["started_at"] for shard in merged["shards"]) + merged["completed_at"] = max(shard["completed_at"] for shard in merged["shards"]) + merged["environment"] = deepcopy(first["environment"]) + merged["shard"] = None + return merged + + +def verify(report, expected_files, require_improvement=False): + require(report["schema_version"] == 2 and report["protocol"] == "localvqe-asr-v2", "Unsupported protocol") + shard = report.get("shard") + require(shard is None or shard["count"] == 1, + f"Shard {shard['index']}/{shard['count']} report: pass every shard report so they are merged first" + if shard else "") + config = report["configuration"] + require(config["asr"] == "parakeet-tdt-v3-int8" and config["asr_compute_units"] == "cpu-only", + "Unexpected ASR configuration") + require(config["sample_rate"] == 16000 and config["aggregation"] == "micro" + and config["normalization"] == "TextNormalizer.normalize" + and config["empty_reference_policy"] == "exclude-from-all-conditions", "Unexpected scoring protocol") + require(report["chunk"] == "256ms", "This reference run requires the 256ms export") + require(config["enhancement_compute_units"] == 0, "This reference run requires CPU-only enhancement") + dataset = report["dataset"] + require(dataset["repository"] == "FluidInference/aec-challenge-synthetic-mini", "Unexpected dataset") + require(dataset["revision"] == "1f3714b5a3f98cedef1bbb017f21bbd7ae688596", "Unexpected dataset revision") + require(dataset["archive_sha256"] == "45ff5d7acfce499558c25a0eace45eb819cec8aa76420fe733de7ee116ae548d", + "Unexpected archive hash") + require(dataset["metadata_sha256"] == "865aff8e66eb682c292f42a9d747d931f3a2f71f18e16fec53aea80dbdc2eacc", + "Unexpected metadata hash") + require(dataset["selection_order"] == "numeric fileid", "Unexpected selection order") + selected = dataset["selected_fileids"] + canonical = json.loads(Path(__file__).with_name("localvqe-dataset.json").read_text()) + require(0 < expected_files <= len(canonical["fileids"]), "Invalid expected file count") + require(selected == canonical["fileids"][:expected_files], "Incomplete or reordered input selection") + excluded = report["excluded_empty_reference_fileids"] + require(len(excluded) == len(set(excluded)), "Duplicate excluded IDs") + rows = report["files"] + require(bool(rows), "No scored examples") + scored = [row["fileid"] for row in rows] + require(scored == [i for i in selected if i not in excluded], "Scored rows do not match selected IDs") + require(set(scored).isdisjoint(excluded) and set(scored) | set(excluded) == set(selected), + "Unaccounted or overlapping exclusions") + audio_hashes = dataset["audio_files_sha256"] + require(set(audio_hashes) == {f"fileid_{i}_{kind}.wav" for i in selected for kind in ("mic", "lpb", "clean")}, + "Incomplete audio fingerprints") + for name, digest in audio_hashes.items(): + require(digest == canonical["audio_files_sha256"][name], f"Audio differs from pinned archive: {name}") + model_hashes = report["model_files_sha256"] + model_roots = {"/".join(name.split("/")[:2]) for name in model_hashes} + require(model_roots == { + "parakeet-v3/Preprocessor.mlmodelc", "parakeet-v3/Encoder.mlmodelc", + "parakeet-v3/Decoder.mlmodelc", "parakeet-v3/JointDecisionv3.mlmodelc", + "parakeet-v3/parakeet_vocab.json", "localvqe/localvqe-v1.3-4.8M-256ms.mlmodelc", + "localvqe/localvqe-v1.2-1.3M-256ms.mlmodelc", + }, "Incomplete model fingerprints") + for name, digest in {**audio_hashes, **model_hashes}.items(): + require(isinstance(digest, str) and re.fullmatch(r"[0-9a-f]{64}", digest), f"Invalid SHA256 for {name}") + conditions = ["unprocessed", "localvqe-v1.3", "localvqe-v1.2"] + require(report["conditions"] == conditions and set(report["summary"]) == set(conditions), + "Missing, extra or duplicate conditions") + totals = {condition: Counter() for condition in conditions} + for row in rows: + reference = row["reference"].split() + far = row["far_reference"].split() + require(bool(reference), f"Empty reference in scored file {row['fileid']}") + close(row["ref_words"], len(reference), "reference word count") + close(row["far_words"], len(far), "far-end word count") + duration = row["audio_seconds"] + require(math.isfinite(duration) and duration > 0, "Invalid audio duration") + for condition in conditions: + hypothesis = row[f"{condition}_hyp"].split() + substitutions, deletions, insertions = edits(hypothesis, reference) + hits = len(reference) - deletions - substitutions + errors = substitutions + deletions + insertions + leakage = sum(((Counter(hypothesis) - Counter(reference)) & Counter(far)).values()) + for field, value in (("substitutions", substitutions), ("deletions", deletions), + ("insertions", insertions), ("hits", hits), ("errors", errors), ("leaked", leakage), + ("recall", hits / len(reference)), ("wer", errors / len(reference))): + close(row[f"{condition}_{field}"], value, f"{row['fileid']} {condition}.{field}") + elapsed = row[f"{condition}_enhancement_seconds"] + require(math.isfinite(elapsed) and (elapsed == 0 if condition == "unprocessed" else elapsed > 0), + f"Invalid enhancement timing for {condition}") + totals[condition].update({ + "files": 1, "reference_words": len(reference), "far_end_words": len(far), + "hits": hits, "errors": errors, "leaked_words": leakage, + "audio_seconds": duration, "enhancement_seconds": elapsed, + }) + for condition, total in totals.items(): + total["recall"] = total["hits"] / total["reference_words"] + total["wer"] = total["errors"] / total["reference_words"] + total["leakage"] = total["leaked_words"] / total["far_end_words"] if total["far_end_words"] else 0 + total["rtfx"] = total["audio_seconds"] / total["enhancement_seconds"] if total["enhancement_seconds"] else 0 + for field, value in total.items(): + close(report["summary"][condition][field], value, f"{condition}.{field}") + if require_improvement: + require(expected_files == 200, "Quality comparison requires the full 200-file selection") + baseline = totals["unprocessed"] + require(baseline["far_end_words"] > 0, "Cannot evaluate leakage without far-end reference words") + for condition in conditions[1:]: + require(totals[condition]["recall"] > baseline["recall"], f"{condition} did not improve recall") + require(totals[condition]["leakage"] < baseline["leakage"], f"{condition} did not reduce leakage") + return totals + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path, nargs="+", + help="one full report, or every shard report of one run (merged in shard order)") + parser.add_argument("--expected-files", type=int, default=200) + parser.add_argument("--require-improvement", action="store_true") + parser.add_argument("--markdown", type=Path) + parser.add_argument("--merged", type=Path, help="write the merged full report (shard inputs only)") + args = parser.parse_args() + reports = [json.loads(path.read_text()) for path in args.report] + if len(reports) == 1 and not reports[0].get("shard"): + report = reports[0] + require(args.merged is None, "--merged needs shard reports") + else: + report = merge(reports) + if args.merged: + args.merged.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + totals = verify(report, args.expected_files, args.require_improvement) + lines = ["## LocalVQE ASR benchmark", "", + f"Verified {args.expected_files} selected examples; {len(report['files'])} scored; " + f"{len(report['excluded_empty_reference_fileids'])} empty references excluded" + + (f"; merged from {len(report['shards'])} shards." if report.get("shards") else "."), "", + "| Condition | Recall | WER | Leakage | Enhancement RTFx |", + "|---|---:|---:|---:|---:|"] + for condition, total in totals.items(): + speed = f"{total['rtfx']:.2f}x" if condition != "unprocessed" else "—" + lines.append(f"| {condition} | {total['recall']:.2%} | {total['wer']:.2%} | {total['leakage']:.2%} | {speed} |") + lines += ["", "Protocol: `localvqe-asr-v2`, Parakeet v3 int8, CPU-only, 256ms LocalVQE exports.", + "Exploratory training-shard study with machine transcripts. CI timing is not device performance.", + "Model/audio fingerprints, raw counts and excluded IDs are in the JSON artifact."] + output = "\n".join(lines) + "\n" + print(output) + if args.markdown: + args.markdown.write_text(output) + + +if __name__ == "__main__": + main() diff --git a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift new file mode 100644 index 000000000..d92997423 --- /dev/null +++ b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift @@ -0,0 +1,136 @@ +import AVFoundation +@preconcurrency import CoreML +import Foundation +import OSLog + +/// LocalVQE speech enhancement: neural acoustic echo cancellation, noise +/// suppression and dereverberation for 16 kHz speech. +/// +/// Core ML port of [localai-org/LocalVQE](https://github.com/localai-org/LocalVQE) +/// (Apache-2.0), a streaming CPU-tuned derivative of DeepVQE. The model +/// needs two inputs: the microphone signal and a far-end **reference** +/// (a loopback of what the loudspeaker played). Without a reference it +/// still denoises and dereverberates; pass silence. +/// +/// ```swift +/// let vqe = try await LocalVqeManager() +/// let clean = try await vqe.process(mic: micSamples, reference: farEndSamples) +/// ``` +/// +/// For live audio, open a `LocalVqeStream` and push buffers as they arrive: +/// +/// ```swift +/// let stream = try await vqe.makeStream() +/// let out = try await stream.enhance(mic: micHop, reference: refHop) +/// ``` +/// +/// **Beta**: verified against the upstream PyTorch and GGML engines +/// (72 dB SNR, 16-bit-wav limited) on the upstream double-talk demo clip; +/// not yet exercised in production call pipelines. +public actor LocalVqeManager { + + private let logger = AppLogger(category: "LocalVqeManager") + + public static let sampleRate = 16000 + /// Analysis hop: every call consumes and produces a multiple of this. + public static let hopSize = 256 + /// Algorithmic delay of the enhanced output relative to the input. + public static let outputDelaySamples = hopSize + + public let config: LocalVqeConfig + private let audioConverter = AudioConverter() + private var model: MLModel? + + public var isAvailable: Bool { model != nil } + + /// Download (if needed) and load the configured model from HuggingFace. + public init( + config: LocalVqeConfig = .default, + progressHandler: ProgressHandler? = nil + ) async throws { + self.config = config + let start = Date() + let fileName = ModelNames.LocalVQE.modelFile(variant: config.variant, chunk: config.chunk) + let models = try await ModelHub.loadModels( + .localVqe, + modelNames: [fileName], + directory: Self.defaultBaseDirectory().appendingPathComponent("Models"), + computeUnits: config.computeUnits, + variant: ModelNames.LocalVQE.variantKey(variant: config.variant, chunk: config.chunk), + progressHandler: progressHandler + ) + guard let model = models[fileName] else { + throw LocalVqeError.modelLoadingFailed("\(fileName) missing after download") + } + self.model = model + logger.info( + "LocalVQE \(config.variant.rawValue)/\(config.chunk.rawValue) loaded in \(String(format: "%.2f", Date().timeIntervalSince(start)))s" + ) + } + + /// Load a compiled model bundle from a local directory (no download). + /// `modelDirectory` must contain `-.mlmodelc` for the + /// configured variant/chunk (see `ModelNames.LocalVQE.modelFile`). + public init(config: LocalVqeConfig = .default, modelDirectory: URL) throws { + self.config = config + let fileName = ModelNames.LocalVQE.modelFile(variant: config.variant, chunk: config.chunk) + let url = modelDirectory.appendingPathComponent(fileName) + let mlConfig = MLModelConfiguration() + mlConfig.computeUnits = config.computeUnits + do { + self.model = try MLModel(contentsOf: url, configuration: mlConfig) + } catch { + throw LocalVqeError.modelLoadingFailed("\(url.path): \(error.localizedDescription)") + } + } + + /// Wrap an already-loaded model (must match `config.chunk`). + public init(config: LocalVqeConfig = .default, model: MLModel) { + self.config = config + self.model = model + } + + // MARK: - Whole-clip processing + + /// Enhance a complete clip. `mic` and `reference` are 16 kHz mono and + /// must have equal length; the result has the same length and is + /// sample-aligned with `mic`. + public func process(mic: [Float], reference: [Float]) async throws -> [Float] { + let stream = try makeStream() + var out = try await stream.enhance(mic: mic, reference: reference) + out.append(contentsOf: try await stream.flush()) + return out + } + + /// Enhance a clip whose far-end reference is silent (noise suppression + + /// dereverberation only). + public func process(mic: [Float]) async throws -> [Float] { + try await process(mic: mic, reference: [Float](repeating: 0, count: mic.count)) + } + + /// Enhance a mic recording using a reference recording. Both files are + /// converted to 16 kHz mono; the shorter one is zero-padded. + public func process(micURL: URL, referenceURL: URL?) async throws -> [Float] { + let mic = try audioConverter.resampleAudioFile(micURL) + var reference = try referenceURL.map { try audioConverter.resampleAudioFile($0) } ?? [] + if reference.count < mic.count { + reference.append(contentsOf: [Float](repeating: 0, count: mic.count - reference.count)) + } else if reference.count > mic.count { + reference.removeLast(reference.count - mic.count) + } + return try await process(mic: mic, reference: reference) + } + + // MARK: - Streaming + + /// Open an independent streaming session on the loaded model. + public func makeStream() throws -> LocalVqeStream { + guard let model else { throw LocalVqeError.notInitialized } + return try LocalVqeStream(model: model, samplesPerCall: config.chunk.samplesPerCall) + } + + private static func defaultBaseDirectory() -> URL { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + return appSupport.appendingPathComponent("FluidAudio", isDirectory: true) + } +} diff --git a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift new file mode 100644 index 000000000..ba74275ad --- /dev/null +++ b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift @@ -0,0 +1,320 @@ +@preconcurrency import CoreML +import Foundation +import OSLog + +/// Stateful hop-by-hop LocalVQE session. +/// +/// Feed equal-length mic and far-end reference audio (16 kHz mono Float32) +/// in any buffer size; the stream assembles whole model calls internally and +/// returns enhanced samples as they complete. Output is sample-aligned with +/// input (sample `i` out corresponds to sample `i` in) but is delivered one +/// hop (256 samples, 16 ms) later than the input that produced it, plus +/// whatever remains buffered toward the next call. Call `flush()` at the end +/// of a clip to drain the delay line so the total output length equals the +/// total input length. +/// +/// The Core ML model carries every recurrent state (conv histories, delay +/// windows, S4D bottleneck, overlap-add tail) as explicit `in_*`/`out_*` +/// tensors; the stream just passes each call's outputs back in as the next +/// call's inputs. Create one stream per audio channel pair; a stream is not +/// reusable across unrelated clips without `reset()`. +/// +/// Streams created from one `LocalVqeManager` share its `MLModel`. Inference +/// goes through Core ML's async prediction API, which Apple documents as +/// thread-safe (WWDC23 10049), so independent streams may run concurrently; +/// the synchronous API would require serializing every call on the model. +/// Operations on a single stream are serialized in actor-arrival order. Await +/// each push before submitting the next to preserve capture order and bound +/// queued audio. Cancellation while queued leaves the clip untouched; +/// cancellation or inference failure after a push/flush starts discards the +/// unfinished clip and resets the stream before the next operation. +public actor LocalVqeStream { + + private static let logger = AppLogger(category: "LocalVqeStream") + private static let stateInputPrefix = "in_" + private static let stateOutputPrefix = "out_" + + private let model: MLModel + /// Samples consumed (and produced) per Core ML call. + public let samplesPerCall: Int + + private let micInput: MLMultiArray + private let refInput: MLMultiArray + private let stateNames: [String] + private let stateShapes: [String: [NSNumber]] + private var states: [String: MLMultiArray] = [:] + + private var pendingMic: [Float] = [] + private var pendingRef: [Float] = [] + /// The first emitted hop covers t < 0 of the input and is discarded so + /// that cumulative output stays aligned with cumulative input. + private var leadingSamplesToDrop = LocalVqeManager.hopSize + private var samplesIn = 0 + private var samplesOut = 0 + + private struct OperationWaiter { + let id: UUID + let continuation: CheckedContinuation + } + + private var operationInProgress = false + private var operationWaiters: [OperationWaiter] = [] + + /// Internal queue snapshot for deterministic concurrency regression tests. + var operationCount: Int { (operationInProgress ? 1 : 0) + operationWaiters.count } + + init(model: MLModel, samplesPerCall: Int) throws { + self.model = model + self.samplesPerCall = samplesPerCall + + let desc = model.modelDescription + guard let micDesc = desc.inputDescriptionsByName["mic"]?.multiArrayConstraint, + let refDesc = desc.inputDescriptionsByName["ref"]?.multiArrayConstraint + else { + throw LocalVqeError.modelLoadingFailed("model lacks 'mic'/'ref' inputs") + } + let expected = micDesc.shape.map { $0.intValue }.reduce(1, *) + guard expected == samplesPerCall, refDesc.shape == micDesc.shape else { + throw LocalVqeError.modelLoadingFailed( + "model consumes \(expected) samples per call, expected \(samplesPerCall)") + } + micInput = try MLMultiArray(shape: micDesc.shape, dataType: .float32) + refInput = try MLMultiArray(shape: refDesc.shape, dataType: .float32) + + var names: [String] = [] + var shapes: [String: [NSNumber]] = [:] + for (inputName, inputDesc) in desc.inputDescriptionsByName where inputName.hasPrefix(Self.stateInputPrefix) { + guard let constraint = inputDesc.multiArrayConstraint else { continue } + let name = String(inputName.dropFirst(Self.stateInputPrefix.count)) + guard desc.outputDescriptionsByName[Self.stateOutputPrefix + name] != nil else { + throw LocalVqeError.modelLoadingFailed("state '\(name)' has no matching output") + } + names.append(name) + shapes[name] = constraint.shape + } + guard !names.isEmpty else { + throw LocalVqeError.modelLoadingFailed("model exposes no in_*/out_* state tensors") + } + stateNames = names.sorted() + stateShapes = shapes + states = try Self.zeroStates(names: stateNames, shapes: stateShapes) + } + + /// Number of state tensors the model carries between calls. + public var stateCount: Int { stateNames.count } + + /// Clear all recurrent state and buffered audio after earlier operations + /// finish; the next push starts a new clip. Cancel the active task first + /// when abandoning an in-flight push instead of waiting for it to finish. + public func reset() async throws { + try await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + clearClip() + } + + private func clearClip() { + // Allocate fresh zero states lazily on the next model call. Clearing + // after an inference failure must not itself require an allocation. + states.removeAll(keepingCapacity: true) + pendingMic.removeAll(keepingCapacity: true) + pendingRef.removeAll(keepingCapacity: true) + leadingSamplesToDrop = LocalVqeManager.hopSize + samplesIn = 0 + samplesOut = 0 + } + + private func acquireOperation() async throws { + try Task.checkCancellation() + guard operationInProgress else { + operationInProgress = true + return + } + + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + operationWaiters.append(OperationWaiter(id: id, continuation: continuation)) + } + } onCancel: { + Task { await self.cancelOperation(id: id) } + } + } + + private func cancelOperation(id: UUID) { + guard let index = operationWaiters.firstIndex(where: { $0.id == id }) else { return } + let waiter = operationWaiters.remove(at: index) + waiter.continuation.resume(throwing: CancellationError()) + } + + private func releaseOperation() { + guard !operationWaiters.isEmpty else { + operationInProgress = false + return + } + // Transfer ownership without opening a gap for a newly arriving call. + operationWaiters.removeFirst().continuation.resume() + } + + private static func zeroStates(names: [String], shapes: [String: [NSNumber]]) throws -> [String: MLMultiArray] { + var fresh: [String: MLMultiArray] = [:] + for name in names { + guard let shape = shapes[name] else { continue } + let array = try MLMultiArray(shape: shape, dataType: .float32) + array.withUnsafeMutableBytes { ptr, _ in + ptr.initializeMemory(as: UInt8.self, repeating: 0) + } + fresh[name] = array + } + return fresh + } + + /// Push audio and return every enhanced sample that completed. + /// + /// `mic` and `reference` must have the same length; both may be any + /// length (including zero) — partial calls are buffered until enough + /// samples arrive. + public func enhance(mic: [Float], reference: [Float]) async throws -> [Float] { + guard mic.count == reference.count else { + throw LocalVqeError.lengthMismatch(mic: mic.count, reference: reference.count) + } + try await acquireOperation() + defer { releaseOperation() } + // A cancelled waiter may have acquired the operation just before its + // cancellation handler ran. It must not mutate/reset the current clip. + try Task.checkCancellation() + do { + return try await enhanceExclusive(mic: mic, reference: reference) + } catch { + clearClip() + throw error + } + } + + private func enhanceExclusive(mic: [Float], reference: [Float]) async throws -> [Float] { + pendingMic.append(contentsOf: mic) + pendingRef.append(contentsOf: reference) + samplesIn += mic.count + + var out: [Float] = [] + var offset = 0 + while pendingMic.count - offset >= samplesPerCall { + let hop = try await runCall( + mic: pendingMic[offset.. 0 { + pendingMic.removeFirst(offset) + pendingRef.removeFirst(offset) + } + return emit(out) + } + + /// Drain the delay line by feeding silence, returning the remaining + /// samples so that total output length equals total input length. + /// Ends the current clip: the stream is reset afterwards. + public func flush() async throws -> [Float] { + try await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + do { + return try await flushExclusive() + } catch { + clearClip() + throw error + } + } + + private func flushExclusive() async throws -> [Float] { + let outstanding = samplesIn - samplesOut + guard outstanding > 0 else { + clearClip() + return [] + } + // Zeros needed to complete every outstanding sample, rounded up to whole calls. + let needed = pendingMic.count + LocalVqeManager.hopSize + let padded = ((needed + samplesPerCall - 1) / samplesPerCall) * samplesPerCall + let zeros = [Float](repeating: 0, count: padded - pendingMic.count) + pendingMic.append(contentsOf: zeros) + pendingRef.append(contentsOf: zeros) + + var out: [Float] = [] + var offset = 0 + while pendingMic.count - offset >= samplesPerCall { + let hop = try await runCall( + mic: pendingMic[offset.. [Float] { + var result = samples + if leadingSamplesToDrop > 0 { + let drop = min(leadingSamplesToDrop, result.count) + result.removeFirst(drop) + leadingSamplesToDrop -= drop + } + samplesOut += result.count + return result + } + + private func runCall(mic: ArraySlice, reference: ArraySlice) async throws -> [Float] { + try Task.checkCancellation() + if states.isEmpty { + states = try Self.zeroStates(names: stateNames, shapes: stateShapes) + } + micInput.withUnsafeMutableBufferPointer(ofType: Float.self) { buf, _ in + _ = buf.initialize(from: mic) + } + refInput.withUnsafeMutableBufferPointer(ofType: Float.self) { buf, _ in + _ = buf.initialize(from: reference) + } + var features: [String: Any] = ["mic": micInput, "ref": refInput] + for name in stateNames { + features[Self.stateInputPrefix + name] = states[name] + } + + let output: MLFeatureProvider + do { + let provider = try MLDictionaryFeatureProvider(dictionary: features) + output = try await model.compatPrediction(from: provider, options: MLPredictionOptions()) + } catch { + if Task.isCancelled || error is CancellationError { throw CancellationError() } + throw LocalVqeError.modelProcessingFailed(error.localizedDescription) + } + try Task.checkCancellation() + + for name in stateNames { + guard let next = output.featureValue(for: Self.stateOutputPrefix + name)?.multiArrayValue else { + throw LocalVqeError.modelProcessingFailed("missing state output '\(name)'") + } + states[name] = next + } + guard let enhanced = output.featureValue(for: "enhanced")?.multiArrayValue else { + throw LocalVqeError.modelProcessingFailed("missing 'enhanced' output") + } + return Self.floats(from: enhanced, count: samplesPerCall) + } + + private static func floats(from array: MLMultiArray, count: Int) -> [Float] { + var result = [Float](repeating: 0, count: count) + switch array.dataType { + case .float32: + array.withUnsafeBufferPointer(ofType: Float.self) { buf in + for i in 0.. String { + "\(variant.fileStem)-\(chunk.rawValue).mlmodelc" + } + + public static func variantKey(variant: LocalVqeVariant, chunk: LocalVqeChunk) -> String { + "\(variant.rawValue)-\(chunk.rawValue)" + } + + public static var allModels: Set { + var files: Set = [] + for variant in LocalVqeVariant.allCases { + for chunk in LocalVqeChunk.allCases { + files.insert(modelFile(variant: variant, chunk: chunk)) + } + } + return files + } + + /// Required files for a download variant key; the full set when the + /// key is absent or unrecognised. + public static func requiredModels(variant: String?) -> Set { + for v in LocalVqeVariant.allCases { + for c in LocalVqeChunk.allCases where variantKey(variant: v, chunk: c) == variant { + return [modelFile(variant: v, chunk: c)] + } + } + return allModels + } + } + /// Parakeet EOU streaming model names public enum ParakeetEOU { public static let encoder = "streaming_encoder" @@ -1632,6 +1674,8 @@ public enum ModelNames { ] case .vad: return ModelNames.VAD.requiredModels + case .localVqe: + return ModelNames.LocalVQE.requiredModels(variant: variant) case .parakeetV3: let precision = ParakeetEncoderPrecision(rawValue: variant ?? "") ?? .int8 return ModelNames.ASR.requiredModelsV3(precision: precision) diff --git a/Sources/FluidAudio/ModelRegistry.swift b/Sources/FluidAudio/ModelRegistry.swift index f083da063..8c080bea1 100644 --- a/Sources/FluidAudio/ModelRegistry.swift +++ b/Sources/FluidAudio/ModelRegistry.swift @@ -109,8 +109,10 @@ public enum ModelRegistry { } /// Construct download URL for a dataset file - public static func resolveDataset(_ dataset: String, _ filePath: String) throws -> URL { - let urlString = "\(baseURL)/datasets/\(dataset)/resolve/main/\(filePath)" + public static func resolveDataset( + _ dataset: String, _ filePath: String, revision: String = "main" + ) throws -> URL { + let urlString = "\(baseURL)/datasets/\(dataset)/resolve/\(revision)/\(filePath)" guard let url = URL(string: urlString) else { throw Error.invalidURL(urlString) } @@ -118,8 +120,8 @@ public enum ModelRegistry { } /// Construct base URL for dataset directory (without trailing slash) - public static func resolveDatasetBase(_ dataset: String) -> String { - "\(baseURL)/datasets/\(dataset)/resolve/main" + public static func resolveDatasetBase(_ dataset: String, revision: String = "main") -> String { + "\(baseURL)/datasets/\(dataset)/resolve/\(revision)" } // MARK: - Session Configuration diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift new file mode 100644 index 000000000..1e40b891b --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -0,0 +1,491 @@ +#if os(macOS) +import CoreML +import FluidAudio +import Foundation + +/// `enhance-benchmark`: near-end word recall / far-end leakage of LocalVQE on the +/// Microsoft AEC-Challenge synthetic set (mic + loopback + clean near-end triples). +/// +/// The in-repo Parakeet ASR transcribes the clean near-end clip (reference +/// words) and the loopback clip (far-end words), then the unprocessed mic and +/// each enhanced output. Near-end recall is the fraction of reference words the +/// hypothesis keeps (1 - (deletions + substitutions) / N); far-end leakage is +/// the fraction of far-end words that show up in the hypothesis without being +/// near-end words. WER against the clean-near-end transcript is reported too. +enum EnhanceBenchmarkCommand { + private static let logger = AppLogger(category: "EnhanceBenchmark") + + static let datasetRepo = "FluidInference/aec-challenge-synthetic-mini" + static let datasetArchive = "aec-synthetic-mini.tar.gz" + static let datasetFolder = "aec-synthetic-mini" + static let datasetRevision = "1f3714b5a3f98cedef1bbb017f21bbd7ae688596" + static let datasetArchiveSHA256 = "45ff5d7acfce499558c25a0eace45eb819cec8aa76420fe733de7ee116ae548d" + static let datasetMetadataSHA256 = "865aff8e66eb682c292f42a9d747d931f3a2f71f18e16fec53aea80dbdc2eacc" + static let expectedDatasetExamples = 200 + + private struct Options { + var datasetDir: String? + var maxFiles: Int? + var shard: (index: Int, count: Int)? + var variants: [LocalVqeVariant] = [.v13, .v12] + var chunk: LocalVqeChunk = .batch256ms + var computeUnits: MLComputeUnits = .cpuOnly + var includeNoReference = false + var outputPath: String? + } + + private struct ConditionTotals { + var files = 0 + var refWords = 0 + var hits = 0 + var errors = 0 + var farWords = 0 + var leaked = 0 + var enhanceSeconds = 0.0 + var audioSeconds = 0.0 + + var recall: Double { refWords == 0 ? 0 : Double(hits) / Double(refWords) } + var wer: Double { refWords == 0 ? 0 : Double(errors) / Double(refWords) } + var leakage: Double { farWords == 0 ? 0 : Double(leaked) / Double(farWords) } + var rtfx: Double { enhanceSeconds <= 0 ? 0 : audioSeconds / enhanceSeconds } + } + + static func run(arguments: [String]) async { + var options = Options() + var index = 0 + while index < arguments.count { + let arg = arguments[index] + switch arg { + case "--help", "-h": + printUsage() + exit(0) + case "--dataset-dir": + options.datasetDir = requiredValue(arguments, &index) + case "--max-files": + guard let raw = next(arguments, &index), let maxFiles = Int(raw), maxFiles > 0 else { + logger.error("--max-files must be a positive integer") + exit(1) + } + options.maxFiles = maxFiles + case "--shard": + let parts = (next(arguments, &index) ?? "").split(separator: "/", omittingEmptySubsequences: false) + guard parts.count == 2, let shardIndex = Int(parts[0]), let shardCount = Int(parts[1]), + shardCount > 0, (0../ with 0 <= index < count") + exit(1) + } + options.shard = (shardIndex, shardCount) + case "--variants": + let raw = (next(arguments, &index) ?? "").split(separator: ",", omittingEmptySubsequences: false) + .map(String.init) + let parsed = raw.compactMap(LocalVqeVariant.init(rawValue:)) + guard parsed.count == raw.count, !parsed.isEmpty, + Set(parsed.map(\.rawValue)).count == parsed.count + else { + logger.error("--variants must be a comma list of \(LocalVqeVariant.allCases.map(\.rawValue))") + exit(1) + } + options.variants = parsed + case "--chunk": + guard let raw = next(arguments, &index), let c = LocalVqeChunk(rawValue: raw) else { + logger.error("--chunk must be one of \(LocalVqeChunk.allCases.map(\.rawValue))") + exit(1) + } + options.chunk = c + case "--compute-units": + switch next(arguments, &index)?.lowercased() { + case "cpu-only", "cpu": options.computeUnits = .cpuOnly + case "gpu", "cpu-and-gpu": options.computeUnits = .cpuAndGPU + case "ane", "cpu-and-ne": options.computeUnits = .cpuAndNeuralEngine + case "all": options.computeUnits = .all + default: + logger.error("--compute-units must be cpu-only | gpu | ane | all") + exit(1) + } + case "--no-reference": + options.includeNoReference = true + case "--output": + options.outputPath = requiredValue(arguments, &index) + default: + logger.error("Unknown option: \(arg)") + exit(1) + } + index += 1 + } + + do { + let datasetDir = try await resolveDataset(options.datasetDir) + var examples = try EnhanceBenchmarkDataset.loadExamples(from: datasetDir) + if let maxFiles = options.maxFiles { examples = Array(examples.prefix(maxFiles)) } + if let shard = options.shard { + examples = try EnhanceBenchmarkDataset.shard(examples, index: shard.index, count: shard.count) + } + guard !examples.isEmpty else { + logger.error("No examples selected from \(datasetDir.path)") + exit(1) + } + let shardLabel = options.shard.map { " shard \($0.index)/\($0.count)" } ?? "" + report("Dataset: \(datasetDir.path) (\(examples.count) examples\(shardLabel))") + let startedAt = Date() + let audioHashes = options.outputPath == nil ? [:] : try EnhanceBenchmarkProvenance.audioFiles(examples) + + let asr = AsrManager() + let asrConfiguration = MLModelConfigurationUtils.defaultConfiguration(computeUnits: .cpuOnly) + try await asr.loadModels( + try await AsrModels.downloadAndLoad(configuration: asrConfiguration, version: .v3)) + report("ASR: Parakeet TDT v3 int8 loaded (CPU-only)") + + var conditions: [(name: String, manager: LocalVqeManager?, useReference: Bool)] = [ + ("unprocessed", nil, true) + ] + for variant in options.variants { + let config = LocalVqeConfig(variant: variant, chunk: options.chunk, computeUnits: options.computeUnits) + let manager = try await LocalVqeManager(config: config) + conditions.append(("localvqe-\(variant.rawValue)", manager, true)) + if options.includeNoReference { + conditions.append(("localvqe-\(variant.rawValue)-noref", manager, false)) + } + } + report("Conditions: \(conditions.map(\.name).joined(separator: ", "))") + let modelHashes = + options.outputPath == nil + ? [:] + : try EnhanceBenchmarkProvenance.modelFiles( + variants: options.variants, chunk: options.chunk) + + let converter = AudioConverter() + var totals = [String: ConditionTotals]() + var bySer = [String: [String: ConditionTotals]]() // bucket -> condition -> totals + var rows: [[String: Any]] = [] + var emptyReferenceFileIDs: [String] = [] + + for (i, example) in examples.enumerated() { + let mic = try converter.resampleAudioFile(example.mic) + let lpb = try converter.resampleAudioFile(example.lpb) + let clean = try converter.resampleAudioFile(example.clean) + try validateAudio(mic: mic, reference: lpb, clean: clean, fileID: example.fileID) + + let refWords = words(try await transcribe(asr, clean)) + if refWords.isEmpty { + // No reference words to recall: the ASR produced nothing on the + // clean near-end clip. Excluded from every metric and counted. + emptyReferenceFileIDs.append(example.fileID) + continue + } + let farWords = words(try await transcribe(asr, lpb)) + let bucket = serBucket(example.ser) + var row: [String: Any] = [ + "fileid": example.fileID, "ser": example.ser, "ref_words": refWords.count, + "far_words": farWords.count, "reference": refWords.joined(separator: " "), + "far_reference": farWords.joined(separator: " "), + "is_farend_noisy": example.farendNoisy, + "is_nearend_noisy": example.nearendNoisy, + "audio_seconds": Double(mic.count) / Double(LocalVqeManager.sampleRate), + ] + + for condition in conditions { + var enhanced = mic + var enhanceSeconds = 0.0 + if let manager = condition.manager { + let reference = condition.useReference ? lpb : [Float](repeating: 0, count: mic.count) + let start = ContinuousClock.now + enhanced = try await manager.process(mic: mic, reference: reference) + let elapsed = start.duration(to: .now).components + enhanceSeconds = Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18 + } + guard enhanced.count == mic.count, enhanced.allSatisfy(\.isFinite) else { + throw LocalVqeError.modelProcessingFailed( + "Invalid \(condition.name) output for fileid \(example.fileID)") + } + let hypWords = words(try await transcribe(asr, enhanced)) + let m = score(hypothesis: hypWords, reference: refWords, farEnd: farWords) + + var t = totals[condition.name, default: ConditionTotals()] + t.files += 1 + t.refWords += refWords.count + t.hits += m.hits + t.errors += m.errors + t.farWords += farWords.count + t.leaked += m.leaked + t.enhanceSeconds += enhanceSeconds + t.audioSeconds += Double(mic.count) / Double(LocalVqeManager.sampleRate) + totals[condition.name] = t + var b = bySer[bucket, default: [:]][condition.name, default: ConditionTotals()] + b.files += 1 + b.refWords += refWords.count + b.hits += m.hits + b.errors += m.errors + b.farWords += farWords.count + b.leaked += m.leaked + bySer[bucket, default: [:]][condition.name] = b + + row["\(condition.name)_recall"] = refWords.isEmpty ? 0 : Double(m.hits) / Double(refWords.count) + row["\(condition.name)_wer"] = refWords.isEmpty ? 0 : Double(m.errors) / Double(refWords.count) + row["\(condition.name)_leaked"] = m.leaked + row["\(condition.name)_hyp"] = hypWords.joined(separator: " ") + row["\(condition.name)_hits"] = m.hits + row["\(condition.name)_errors"] = m.errors + row["\(condition.name)_insertions"] = m.insertions + row["\(condition.name)_deletions"] = m.deletions + row["\(condition.name)_substitutions"] = m.substitutions + row["\(condition.name)_enhancement_seconds"] = enhanceSeconds + } + rows.append(row) + + if (i + 1) % 10 == 0 || i + 1 == examples.count { + let parts = conditions.map { c -> String in + let t = totals[c.name] ?? ConditionTotals() + return String(format: "%@ R=%.1f%% L=%.1f%%", c.name, t.recall * 100, t.leakage * 100) + } + report("[\(i + 1)/\(examples.count)] " + parts.joined(separator: " | ")) + } + } + + guard !rows.isEmpty else { + throw LocalVqeError.modelProcessingFailed( + "No examples were scored: all \(examples.count) clean-reference transcripts were empty") + } + report("") + if !emptyReferenceFileIDs.isEmpty { + report( + "Excluded \(emptyReferenceFileIDs.count) examples whose clean near-end transcript was empty: " + + emptyReferenceFileIDs.joined(separator: ", ")) + } + report(row(["condition", "files", "recall", "WER", "leakage", "RTFx"])) + for condition in conditions { + let t = totals[condition.name] ?? ConditionTotals() + report( + row([ + condition.name, "\(t.files)", pct(t.recall), pct(t.wer), pct(t.leakage), + condition.manager == nil ? "-" : String(format: "%.1fx", t.rtfx), + ])) + } + for bucket in ["ser<=0", "ser>0", "ser=?"] { + guard let perCondition = bySer[bucket] else { continue } + report("") + report("SER bucket \(bucket):") + for condition in conditions { + let t = perCondition[condition.name] ?? ConditionTotals() + report(row([" " + condition.name, "\(t.files)", pct(t.recall), pct(t.wer), pct(t.leakage), ""])) + } + } + + if let outputPath = options.outputPath { + var summary: [String: Any] = [:] + for (name, t) in totals { + summary[name] = [ + "files": t.files, "recall": t.recall, "wer": t.wer, "leakage": t.leakage, "rtfx": t.rtfx, + "reference_words": t.refWords, "hits": t.hits, "errors": t.errors, + "far_end_words": t.farWords, "leaked_words": t.leaked, + "audio_seconds": t.audioSeconds, "enhancement_seconds": t.enhanceSeconds, + ] + } + let payload: [String: Any] = [ + "schema_version": 2, + "protocol": "localvqe-asr-v2", + "started_at": ISO8601DateFormatter().string(from: startedAt), + "completed_at": ISO8601DateFormatter().string(from: Date()), + "conditions": conditions.map(\.name), + "configuration": [ + "asr": "parakeet-tdt-v3-int8", "asr_compute_units": "cpu-only", + "enhancement_compute_units": options.computeUnits.rawValue, + "normalization": "TextNormalizer.normalize", "sample_rate": LocalVqeManager.sampleRate, + "aggregation": "micro", "empty_reference_policy": "exclude-from-all-conditions", + ], + "model_files_sha256": modelHashes, + "environment": [ + "os": ProcessInfo.processInfo.operatingSystemVersionString, + "processor_count": ProcessInfo.processInfo.processorCount, + "source_revision": ProcessInfo.processInfo.environment["GITHUB_SHA"] ?? "unrecorded", + ], + "dataset": [ + "path": datasetDir.path, + "repository": options.datasetDir == nil ? datasetRepo : "custom", + "revision": options.datasetDir == nil ? datasetRevision : "custom", + "archive_sha256": options.datasetDir == nil ? datasetArchiveSHA256 : NSNull(), + "metadata_sha256": try EnhanceBenchmarkDataset.sha256( + of: datasetDir.appendingPathComponent("meta.csv")), + "selection_order": "numeric fileid", + "selected_fileids": examples.map(\.fileID), + "audio_files_sha256": audioHashes, + ], + "shard": options.shard.map { ["index": $0.index, "count": $0.count] as [String: Any] } ?? NSNull(), + "chunk": options.chunk.rawValue, "summary": summary, "files": rows, + "excluded_empty_reference_fileids": emptyReferenceFileIDs, + ] + let data = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) + try data.write(to: URL(fileURLWithPath: outputPath), options: .atomic) + report("Wrote \(outputPath)") + } + } catch { + logger.error("enhance-benchmark failed: \(error)") + exit(1) + } + } + + // MARK: - Scoring + + private static func transcribe(_ asr: AsrManager, _ samples: [Float]) async throws -> String { + var state = TdtDecoderState.make(decoderLayers: await asr.decoderLayerCount) + return try await asr.transcribe(samples, decoderState: &state).text + } + + private static func words(_ text: String) -> [String] { + TextNormalizer.normalize(text).split(whereSeparator: { $0.isWhitespace }).map(String.init) + } + + /// hits = reference words kept by the hypothesis (N - deletions - substitutions); + /// errors = S + D + I; leaked = far-end words present in the hypothesis beyond + /// what the near-end reference accounts for (multiset). + static func score( + hypothesis: [String], reference: [String], farEnd: [String] + ) -> (hits: Int, errors: Int, leaked: Int, insertions: Int, deletions: Int, substitutions: Int) { + let m = WERCalculator.calculateWordMetrics(hypothesis: hypothesis, reference: reference) + let hits = max(0, m.totalWords - m.deletions - m.substitutions) + let errors = m.insertions + m.deletions + m.substitutions + + var spare = [String: Int]() + for w in hypothesis { spare[w, default: 0] += 1 } + for w in reference where spare[w, default: 0] > 0 { spare[w, default: 0] -= 1 } + var leaked = 0 + for w in farEnd where spare[w, default: 0] > 0 { + spare[w, default: 0] -= 1 + leaked += 1 + } + return (hits, errors, leaked, m.insertions, m.deletions, m.substitutions) + } + + static func validateAudio(mic: [Float], reference: [Float], clean: [Float], fileID: String) throws { + guard !mic.isEmpty, mic.count == reference.count, mic.count == clean.count else { + throw LocalVqeError.modelProcessingFailed("Empty or unequal audio lengths for fileid \(fileID)") + } + guard mic.allSatisfy(\.isFinite), reference.allSatisfy(\.isFinite), clean.allSatisfy(\.isFinite) else { + throw LocalVqeError.modelProcessingFailed("Non-finite audio samples for fileid \(fileID)") + } + } + + private static func pct(_ value: Double) -> String { + String(format: "%.2f%%", value * 100) + } + + /// Fixed-width table row: first column left-aligned, the rest right-aligned. + private static func row(_ cells: [String]) -> String { + let widths = [24, 6, 9, 9, 9, 8] + return cells.enumerated().map { i, cell in + let w = widths[min(i, widths.count - 1)] + let pad = String(repeating: " ", count: max(0, w - cell.count)) + return i == 0 ? cell + pad : pad + cell + }.joined(separator: " ") + } + + private static func serBucket(_ ser: Int?) -> String { + guard let ser else { return "ser=?" } + return ser <= 0 ? "ser<=0" : "ser>0" + } + + // MARK: - Dataset + + private static func resolveDataset(_ override: String?) async throws -> URL { + if let override { + return URL(fileURLWithPath: override) + } + let base = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/FluidAudio/Datasets", isDirectory: true) + let dir = base.appendingPathComponent(datasetFolder, isDirectory: true) + if FileManager.default.fileExists(atPath: dir.appendingPathComponent("meta.csv").path) { + try validatePinnedDataset(dir) + return dir + } + try FileManager.default.createDirectory(at: base, withIntermediateDirectories: true) + let url = try ModelRegistry.resolveDataset(datasetRepo, datasetArchive, revision: datasetRevision) + report("Downloading \(url.absoluteString)") + let (tmp, response) = try await URLSession.shared.download(from: url) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LocalVqeError.modelProcessingFailed("dataset download failed: \(response)") + } + let actualArchiveHash = try EnhanceBenchmarkDataset.sha256(of: tmp) + guard actualArchiveHash == datasetArchiveSHA256 else { + throw LocalVqeError.modelProcessingFailed( + "dataset archive checksum mismatch: expected \(datasetArchiveSHA256), got \(actualArchiveHash)") + } + let archive = base.appendingPathComponent(datasetArchive) + try? FileManager.default.removeItem(at: archive) + try FileManager.default.moveItem(at: tmp, to: archive) + let tar = Process() + tar.executableURL = URL(fileURLWithPath: "/usr/bin/tar") + tar.arguments = ["-xzf", archive.path, "-C", base.path] + try tar.run() + tar.waitUntilExit() + try? FileManager.default.removeItem(at: archive) + guard tar.terminationStatus == 0, + FileManager.default.fileExists(atPath: dir.appendingPathComponent("meta.csv").path) + else { + throw LocalVqeError.modelProcessingFailed("dataset extraction failed (tar status \(tar.terminationStatus))") + } + try validatePinnedDataset(dir) + return dir + } + + private static func validatePinnedDataset(_ dir: URL) throws { + let metaURL = dir.appendingPathComponent("meta.csv") + let actualMetadataHash = try EnhanceBenchmarkDataset.sha256(of: metaURL) + guard actualMetadataHash == datasetMetadataSHA256 else { + throw LocalVqeError.modelProcessingFailed( + "dataset metadata checksum mismatch: expected \(datasetMetadataSHA256), got \(actualMetadataHash)") + } + let examples = try EnhanceBenchmarkDataset.loadExamples(from: dir) + guard examples.count == expectedDatasetExamples else { + throw LocalVqeError.modelProcessingFailed( + "dataset contains \(examples.count) examples; expected \(expectedDatasetExamples)") + } + } + + private static func next(_ arguments: [String], _ index: inout Int) -> String? { + guard index + 1 < arguments.count, !arguments[index + 1].hasPrefix("--") else { return nil } + index += 1 + return arguments[index] + } + + private static func requiredValue(_ arguments: [String], _ index: inout Int) -> String { + let option = arguments[index] + guard let value = next(arguments, &index), !value.isEmpty else { + logger.error("\(option) requires a value") + exit(1) + } + return value + } + + private static func report(_ line: String) { + print(line) + // stdout is block-buffered when piped (CI `tee`); flush so progress is visible live. + fflush(stdout) + logger.info("\(line)") + } + + private static func printUsage() { + // print, not the logger: usage must show in release builds too. + print( + """ + Usage: fluidaudiocli enhance-benchmark [options] + + Scores LocalVQE on the Microsoft AEC-Challenge synthetic set (mic + loopback + clean near-end): + near-end word recall, WER vs the clean-near-end transcript, and far-end word leakage, all + measured with the in-repo Parakeet TDT v3 ASR. + + Options: + --dataset-dir Directory with fileid_*_{mic,lpb,clean}.wav + meta.csv + (default: auto-download \(datasetRepo)). + --max-files Score only the first n examples (numeric fileid order). + --shard / Score contiguous shard i of n (after --max-files); merge the + shard reports with Scripts/verify_localvqe_benchmark.py. + --variants Comma list of v1.3,v1.2 (default both). + --chunk <256ms|16ms> Chunk export to benchmark (default 256ms). + --compute-units + --no-reference Also score each variant with a silent far end (NS-only mode). + --output Write per-file and summary results. + """ + ) + } +} +#endif diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift new file mode 100644 index 000000000..fc56ad7fa --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift @@ -0,0 +1,178 @@ +#if os(macOS) +import CryptoKit +import Foundation + +/// Loads and validates the fixed LocalVQE ASR benchmark manifest. +enum EnhanceBenchmarkDataset { + struct Example: Equatable { + let fileID: String + let mic: URL + let lpb: URL + let clean: URL + let ser: Int + let farendNoisy: Bool + let nearendNoisy: Bool + } + + enum DatasetError: Error, LocalizedError, Equatable { + case duplicateFileID(String) + case emptyMetadata + case invalidInteger(line: Int, column: String, value: String) + case invalidShard(index: Int, count: Int) + case malformedRow(line: Int, expected: Int, actual: Int) + case missingAudio(fileID: String, path: String) + case missingColumn(String) + case missingFileID(line: Int) + + var errorDescription: String? { + switch self { + case .duplicateFileID(let fileID): + return "duplicate benchmark fileid '\(fileID)'" + case .emptyMetadata: + return "benchmark meta.csv is empty" + case .invalidInteger(let line, let column, let value): + return "benchmark meta.csv line \(line) has invalid \(column) value '\(value)'" + case .invalidShard(let index, let count): + return "shard \(index)/\(count) is out of range; expected 0 <= index < count" + case .malformedRow(let line, let expected, let actual): + return "benchmark meta.csv line \(line) has \(actual) fields; expected \(expected)" + case .missingAudio(let fileID, let path): + return "benchmark fileid \(fileID) is missing \(path)" + case .missingColumn(let column): + return "benchmark meta.csv is missing required column '\(column)'" + case .missingFileID(let line): + return "benchmark meta.csv line \(line) has an empty fileid" + } + } + } + + private struct MetadataRow { + let fileID: String + let ser: Int + let farendNoisy: Bool + let nearendNoisy: Bool + } + + static func loadExamples( + from directory: URL, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) throws -> [Example] { + let metadata = try String(contentsOf: directory.appendingPathComponent("meta.csv"), encoding: .utf8) + return try parseMetadata(metadata).map { row in + let stem = "fileid_\(row.fileID)" + let mic = directory.appendingPathComponent("\(stem)_mic.wav") + let lpb = directory.appendingPathComponent("\(stem)_lpb.wav") + let clean = directory.appendingPathComponent("\(stem)_clean.wav") + for url in [mic, lpb, clean] { + guard fileExists(url.path) else { + throw DatasetError.missingAudio(fileID: row.fileID, path: url.lastPathComponent) + } + } + return Example( + fileID: row.fileID, + mic: mic, + lpb: lpb, + clean: clean, + ser: row.ser, + farendNoisy: row.farendNoisy, + nearendNoisy: row.nearendNoisy + ) + } + } + + /// Contiguous slice `index` of `count` equal-sized shards (the last may be + /// shorter), so concatenating shard results in index order restores the + /// full selection order. + static func shard(_ items: [T], index: Int, count: Int) throws -> [T] { + guard count > 0, (0.. String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = SHA256() + while true { + let data = try handle.read(upToCount: 1024 * 1024) ?? Data() + guard !data.isEmpty else { break } + hasher.update(data: data) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + private static func parseMetadata(_ text: String) throws -> [MetadataRow] { + var lines = text.split(omittingEmptySubsequences: true, whereSeparator: \.isNewline).map(String.init) + guard !lines.isEmpty else { throw DatasetError.emptyMetadata } + + let header = fields(in: lines.removeFirst()) + let requiredColumns = ["fileid", "ser", "is_farend_noisy", "is_nearend_noisy"] + for required in requiredColumns where !header.contains(required) { + throw DatasetError.missingColumn(required) + } + + guard let fileIDIndex = header.firstIndex(of: "fileid"), + let serIndex = header.firstIndex(of: "ser"), + let farendNoisyIndex = header.firstIndex(of: "is_farend_noisy"), + let nearendNoisyIndex = header.firstIndex(of: "is_nearend_noisy") + else { + throw DatasetError.missingColumn("internal required-column lookup") + } + + var seen = Set() + var rows: [MetadataRow] = [] + for (offset, line) in lines.enumerated() { + let lineNumber = offset + 2 + let rowFields = fields(in: line) + guard rowFields.count == header.count else { + throw DatasetError.malformedRow(line: lineNumber, expected: header.count, actual: rowFields.count) + } + let fileID = rowFields[fileIDIndex] + guard !fileID.isEmpty else { throw DatasetError.missingFileID(line: lineNumber) } + guard let numericID = Int(fileID), numericID >= 0, String(numericID) == fileID else { + throw DatasetError.invalidInteger(line: lineNumber, column: "fileid", value: fileID) + } + guard seen.insert(fileID).inserted else { throw DatasetError.duplicateFileID(fileID) } + let ser = try integer("ser", value: rowFields[serIndex], line: lineNumber) + let farendNoisy = try integer( + "is_farend_noisy", value: rowFields[farendNoisyIndex], line: lineNumber) + let nearendNoisy = try integer( + "is_nearend_noisy", value: rowFields[nearendNoisyIndex], line: lineNumber) + for (column, value) in [("is_farend_noisy", farendNoisy), ("is_nearend_noisy", nearendNoisy)] { + guard value == 0 || value == 1 else { + throw DatasetError.invalidInteger(line: lineNumber, column: column, value: String(value)) + } + } + rows.append( + MetadataRow( + fileID: fileID, + ser: ser, + farendNoisy: farendNoisy != 0, + nearendNoisy: nearendNoisy != 0 + )) + } + return rows.sorted { lhs, rhs in + guard let lhsID = Int(lhs.fileID), let rhsID = Int(rhs.fileID) else { + return lhs.fileID < rhs.fileID + } + return lhsID < rhsID + } + } + + private static func fields(in line: String) -> [String] { + line.split(separator: ",", omittingEmptySubsequences: false) + .map { String($0).trimmingCharacters(in: .whitespaces) } + } + + private static func integer(_ column: String, value rawValue: String, line: Int) throws -> Int { + guard let value = Int(rawValue) else { + throw DatasetError.invalidInteger(line: line, column: column, value: rawValue) + } + return value + } +} +#endif diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift new file mode 100644 index 000000000..839517f75 --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift @@ -0,0 +1,51 @@ +#if os(macOS) +import FluidAudio +import Foundation + +/// Fingerprints the exact inputs consumed by an enhancement benchmark run. +enum EnhanceBenchmarkProvenance { + static func modelFiles(variants: [LocalVqeVariant], chunk: LocalVqeChunk) throws -> [String: String] { + let asrDirectory = AsrModels.defaultCacheDirectory(for: .v3) + let enhancementDirectory = MLModelConfigurationUtils.defaultModelsDirectory(for: .localVqe) + var hashes: [String: String] = [:] + for name in ModelNames.ASR.requiredModelsV3().union([ModelNames.ASR.vocabularyFile]).sorted() { + try fingerprint(asrDirectory.appendingPathComponent(name), prefix: "parakeet-v3/\(name)", into: &hashes) + } + for variant in variants { + let name = ModelNames.LocalVQE.modelFile(variant: variant, chunk: chunk) + try fingerprint( + enhancementDirectory.appendingPathComponent(name), prefix: "localvqe/\(name)", into: &hashes) + } + return hashes + } + + private static func fingerprint(_ url: URL, prefix: String, into hashes: inout [String: String]) throws { + let values = try url.resourceValues(forKeys: [.isDirectoryKey, .isRegularFileKey]) + if values.isDirectory == true { + let children = try FileManager.default.contentsOfDirectory( + at: url, includingPropertiesForKeys: [.isDirectoryKey, .isRegularFileKey], options: [.skipsHiddenFiles]) + guard !children.isEmpty else { + throw LocalVqeError.modelProcessingFailed("Empty model directory: \(url.path)") + } + for child in children.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }) { + try fingerprint(child, prefix: "\(prefix)/\(child.lastPathComponent)", into: &hashes) + } + return + } + guard values.isRegularFile == true else { + throw LocalVqeError.modelProcessingFailed("Unsupported model file: \(url.path)") + } + hashes[prefix] = try EnhanceBenchmarkDataset.sha256(of: url) + } + + static func audioFiles(_ examples: [EnhanceBenchmarkDataset.Example]) throws -> [String: String] { + var hashes: [String: String] = [:] + for example in examples { + for url in [example.mic, example.lpb, example.clean] { + hashes[url.lastPathComponent] = try EnhanceBenchmarkDataset.sha256(of: url) + } + } + return hashes + } +} +#endif diff --git a/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift new file mode 100644 index 000000000..c15bf8c1f --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift @@ -0,0 +1,250 @@ +#if os(macOS) +import AVFoundation +import CoreML +import FluidAudio +import Foundation + +/// `enhance`: LocalVQE acoustic echo cancellation + noise suppression on a file. +enum EnhanceCommand { + private static let logger = AppLogger(category: "Enhance") + + private struct Options { + var micPath: String? + var referencePath: String? + var outputPath: String? + var modelDirectory: String? + var variant: LocalVqeVariant = .v13 + var chunk: LocalVqeChunk = .batch256ms + var computeUnits: MLComputeUnits = .cpuOnly + var streaming = false + var bufferSamples = 256 + } + + static func run(arguments: [String]) async { + var options = Options() + var index = 0 + while index < arguments.count { + let arg = arguments[index] + switch arg { + case "--help", "-h": + printUsage() + exit(0) + case "--reference", "-r": + options.referencePath = next(arguments, &index) + case "--output", "-o": + options.outputPath = next(arguments, &index) + case "--model-dir": + options.modelDirectory = next(arguments, &index) + case "--variant": + guard let raw = next(arguments, &index), let v = LocalVqeVariant(rawValue: raw) else { + logger.error("--variant must be one of \(LocalVqeVariant.allCases.map(\.rawValue))") + exit(1) + } + options.variant = v + case "--chunk": + guard let raw = next(arguments, &index), let c = LocalVqeChunk(rawValue: raw) else { + logger.error("--chunk must be one of \(LocalVqeChunk.allCases.map(\.rawValue))") + exit(1) + } + options.chunk = c + case "--compute-units": + switch next(arguments, &index)?.lowercased() { + case "cpu-only", "cpu": options.computeUnits = .cpuOnly + case "gpu", "cpu-and-gpu": options.computeUnits = .cpuAndGPU + case "ane", "cpu-and-ne": options.computeUnits = .cpuAndNeuralEngine + case "all": options.computeUnits = .all + default: + logger.error("--compute-units must be cpu-only | gpu | ane | all") + exit(1) + } + case "--streaming": + options.streaming = true + case "--buffer-samples": + options.bufferSamples = Int(next(arguments, &index) ?? "") ?? options.bufferSamples + default: + if arg.hasPrefix("--") { + logger.warning("Unknown option: \(arg)") + } else if options.micPath == nil { + options.micPath = arg + } else { + logger.warning("Ignoring extra argument: \(arg)") + } + } + index += 1 + } + + guard let micPath = options.micPath else { + logger.error("No mic audio file provided") + printUsage() + exit(1) + } + + do { + let converter = AudioConverter() + let mic = try converter.resampleAudioFile(path: micPath) + var reference: [Float] + if let referencePath = options.referencePath { + reference = try converter.resampleAudioFile(path: referencePath) + } else { + logger.warning("No --reference given: running noise suppression / dereverb only (silent far end)") + reference = [] + } + if reference.count < mic.count { + reference.append(contentsOf: [Float](repeating: 0, count: mic.count - reference.count)) + } else if reference.count > mic.count { + reference.removeLast(reference.count - mic.count) + } + + let config = LocalVqeConfig( + variant: options.variant, chunk: options.chunk, computeUnits: options.computeUnits) + let loadStart = Date() + let manager: LocalVqeManager + if let dir = options.modelDirectory { + manager = try LocalVqeManager(config: config, modelDirectory: URL(fileURLWithPath: dir)) + } else { + manager = try await LocalVqeManager(config: config) + } + let loadTime = Date().timeIntervalSince(loadStart) + report( + "Loaded LocalVQE \(options.variant.rawValue) (\(options.chunk.rawValue) chunk, " + + "\(describe(options.computeUnits))) in \(String(format: "%.2f", loadTime))s") + + let audioSeconds = Double(mic.count) / Double(LocalVqeManager.sampleRate) + let enhanced: [Float] + let start = Date() + if options.streaming { + enhanced = try await runStreaming(manager: manager, mic: mic, reference: reference, options: options) + } else { + enhanced = try await manager.process(mic: mic, reference: reference) + } + let wall = Date().timeIntervalSince(start) + + let inRms = rms(mic) + let outRms = rms(enhanced) + report( + String( + format: "Enhanced %.2fs of audio in %.3fs wall (RTFx %.1fx); RMS in %.4f -> out %.4f (%.1f dB)", + audioSeconds, wall, audioSeconds / max(wall, 1e-9), inRms, outRms, + 20 * log10(max(outRms, 1e-9) / max(inRms, 1e-9)))) + + if let outputPath = options.outputPath { + try writeWav(samples: enhanced, sampleRate: LocalVqeManager.sampleRate, to: outputPath) + report("Wrote \(outputPath)") + } + } catch { + logger.error("Enhance failed: \(error)") + exit(1) + } + } + + /// Push audio through a `LocalVqeStream` in `bufferSamples` pieces and + /// report per-call latency. + private static func runStreaming( + manager: LocalVqeManager, mic: [Float], reference: [Float], options: Options + ) async throws -> [Float] { + let stream = try await manager.makeStream() + let step = max(1, options.bufferSamples) + var out: [Float] = [] + out.reserveCapacity(mic.count) + var latencies: [Double] = [] + var offset = 0 + while offset < mic.count { + let end = min(offset + step, mic.count) + let t0 = DispatchTime.now().uptimeNanoseconds + let hop = try await stream.enhance( + mic: Array(mic[offset.. String? { + guard index + 1 < arguments.count else { return nil } + index += 1 + return arguments[index] + } + + private static func describe(_ units: MLComputeUnits) -> String { + switch units { + case .cpuOnly: return "cpu-only" + case .cpuAndGPU: return "gpu" + case .cpuAndNeuralEngine: return "ane" + case .all: return "all" + @unknown default: return "unknown" + } + } + + private static func rms(_ samples: [Float]) -> Float { + guard !samples.isEmpty else { return 0 } + var acc: Float = 0 + for s in samples { acc += s * s } + return (acc / Float(samples.count)).squareRoot() + } + + private static func writeWav(samples: [Float], sampleRate: Int, to path: String) throws { + guard + let format = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: Double(sampleRate), channels: 1, interleaved: false), + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(max(samples.count, 1))) + else { + throw LocalVqeError.modelProcessingFailed("could not allocate output buffer") + } + buffer.frameLength = AVAudioFrameCount(samples.count) + guard let channel = buffer.floatChannelData?[0] else { + throw LocalVqeError.modelProcessingFailed("could not access output channel") + } + samples.withUnsafeBufferPointer { src in + if let base = src.baseAddress { channel.update(from: base, count: samples.count) } + } + let url = URL(fileURLWithPath: path) + try? FileManager.default.removeItem(at: url) + let file = try AVAudioFile( + forWriting: url, settings: format.settings, commonFormat: .pcmFormatFloat32, interleaved: false) + try file.write(from: buffer) + } + + private static func printUsage() { + // print, not the logger: usage must show in release builds too. + print( + """ + Usage: fluidaudiocli enhance [options] + + LocalVQE speech enhancement: acoustic echo cancellation + noise suppression + dereverberation. + + Options: + --reference, -r Far-end reference (what the speaker played). Omit for NS/dereverb only. + --output, -o Write the enhanced 16 kHz mono WAV here. + --variant Checkpoint (default v1.3, 4.8M params; v1.2 is 1.3M). + --chunk <256ms|16ms> Samples per model call (default 256ms; 16ms for live use). + --compute-units Default cpu-only (fp32 models). + --streaming Drive the LocalVqeStream API buffer-by-buffer and report call latency. + --buffer-samples Buffer size for --streaming (default 256). + --model-dir Load /.mlmodelc instead of downloading from HuggingFace. + """ + ) + } +} +#endif diff --git a/Sources/FluidAudioCLI/FluidAudioCLI.swift b/Sources/FluidAudioCLI/FluidAudioCLI.swift index 057ba1263..7c7bd6aef 100644 --- a/Sources/FluidAudioCLI/FluidAudioCLI.swift +++ b/Sources/FluidAudioCLI/FluidAudioCLI.swift @@ -34,6 +34,10 @@ struct FluidAudioCLI { await VadBenchmark.runVadBenchmark(arguments: Array(arguments.dropFirst(2))) case "vad-analyze": await VadAnalyzeCommand.run(arguments: Array(arguments.dropFirst(2))) + case "enhance": + await EnhanceCommand.run(arguments: Array(arguments.dropFirst(2))) + case "enhance-benchmark": + await EnhanceBenchmarkCommand.run(arguments: Array(arguments.dropFirst(2))) case "asr-benchmark": await ASRBenchmark.runASRBenchmark(arguments: Array(arguments.dropFirst(2))) case "unified-benchmark": @@ -130,6 +134,8 @@ struct FluidAudioCLI { vad-benchmark Run VAD-specific benchmark vad-analyze Inspect VAD segmentation and streaming events fsmn-vad-segment Detect speech segments with FSMN-VAD (beta) + enhance LocalVQE echo cancellation + noise suppression on a mic (+ reference) file (beta) + enhance-benchmark Near-end word recall / far-end leakage of LocalVQE on the AEC-Challenge synthetic set asr-benchmark Run ASR benchmark on LibriSpeech fleurs-benchmark Run multilingual ASR benchmark on FLEURS dataset transcribe Transcribe audio file using streaming ASR diff --git a/Sources/FluidAudioCLI/Utils/WERCalculator.swift b/Sources/FluidAudioCLI/Utils/WERCalculator.swift index a7d045d16..a4be289f1 100644 --- a/Sources/FluidAudioCLI/Utils/WERCalculator.swift +++ b/Sources/FluidAudioCLI/Utils/WERCalculator.swift @@ -15,10 +15,18 @@ enum WERCalculator { let hypWords = hypothesis.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } let refWords = reference.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - let distance = editDistance(hypWords, refWords) - let wer = refWords.isEmpty ? 0.0 : Double(distance.total) / Double(refWords.count) + return calculateWordMetrics(hypothesis: hypWords, reference: refWords) + } - return (wer, distance.insertions, distance.deletions, distance.substitutions, refWords.count) + /// Score already-normalized tokens without applying text normalization again. + static func calculateWordMetrics( + hypothesis: [String], reference: [String] + ) + -> (wer: Double, insertions: Int, deletions: Int, substitutions: Int, totalWords: Int) + { + let distance = editDistance(hypothesis, reference) + let wer = reference.isEmpty ? 0.0 : Double(distance.total) / Double(reference.count) + return (wer, distance.insertions, distance.deletions, distance.substitutions, reference.count) } /// Compute character-level CER alongside WER if needed. @@ -179,11 +187,13 @@ enum WERCalculator { let m = seq1.count let n = seq2.count + // seq1 is the hypothesis, seq2 the reference: an empty hypothesis is n + // deletions (reference words missing), an empty reference m insertions. if m == 0 { - return EditDistanceResult(total: n, insertions: n, deletions: 0, substitutions: 0) + return EditDistanceResult(total: n, insertions: 0, deletions: n, substitutions: 0) } if n == 0 { - return EditDistanceResult(total: m, insertions: 0, deletions: m, substitutions: 0) + return EditDistanceResult(total: m, insertions: m, deletions: 0, substitutions: 0) } var dp = Array(repeating: Array(repeating: 0, count: n + 1), count: m + 1) @@ -220,10 +230,12 @@ enum WERCalculator { i -= 1 j -= 1 } else if i > 0 && dp[i][j] == dp[i - 1][j] + 1 { - deletions += 1 + // hypothesis word with no reference counterpart + insertions += 1 i -= 1 } else if j > 0 && dp[i][j] == dp[i][j - 1] + 1 { - insertions += 1 + // reference word missing from the hypothesis + deletions += 1 j -= 1 } else { break diff --git a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift new file mode 100644 index 000000000..e78c7ad85 --- /dev/null +++ b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift @@ -0,0 +1,111 @@ +#if os(macOS) +import Foundation +import XCTest + +@testable import FluidAudioCLI + +final class EnhanceBenchmarkDatasetTests: XCTestCase { + private let header = "fileid,ser,is_farend_noisy,is_nearend_noisy,nearend_scale" + + func testLoadsCompleteManifestInNumericFileIDOrder() throws { + let metadata = [ + header, + "10,-2,1,0,0.5", + "2,3,0,1,0.8", + ].joined(separator: "\n") + let directory = try metadataDirectory(metadata) + let expectedFiles = Set( + [2, 10].flatMap { id in + ["fileid_\(id)_mic.wav", "fileid_\(id)_lpb.wav", "fileid_\(id)_clean.wav"] + }) + + let examples = try EnhanceBenchmarkDataset.loadExamples(from: directory) { + expectedFiles.contains(URL(fileURLWithPath: $0).lastPathComponent) + } + + XCTAssertEqual(examples.map(\.fileID), ["2", "10"]) + XCTAssertEqual(examples.map(\.ser), [3, -2]) + XCTAssertEqual(examples.map(\.farendNoisy), [false, true]) + XCTAssertEqual(examples.map(\.nearendNoisy), [true, false]) + } + + func testShardsAreContiguousAndCoverEveryItemOnce() throws { + let items = Array(1...11) + let shards = try (0..<3).map { try EnhanceBenchmarkDataset.shard(items, index: $0, count: 3) } + XCTAssertEqual(shards, [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11]]) + XCTAssertEqual(try EnhanceBenchmarkDataset.shard(items, index: 0, count: 1), items) + XCTAssertEqual(try EnhanceBenchmarkDataset.shard(items, index: 11, count: 12), []) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.shard(items, index: 3, count: 3)) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.shard(items, index: -1, count: 3)) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.shard(items, index: 0, count: 0)) + } + + func testRejectsDuplicateFileID() throws { + let metadata = [header, "1,0,0,0,1", "1,1,0,0,1"].joined(separator: "\n") + let directory = try metadataDirectory(metadata) + + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) { + XCTAssertEqual($0 as? EnhanceBenchmarkDataset.DatasetError, .duplicateFileID("1")) + } + } + + func testRejectsMalformedInteger() throws { + let metadata = [header, "1,not-an-int,0,0,1"].joined(separator: "\n") + let directory = try metadataDirectory(metadata) + + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) { + XCTAssertEqual( + $0 as? EnhanceBenchmarkDataset.DatasetError, + .invalidInteger(line: 2, column: "ser", value: "not-an-int")) + } + } + + func testRejectsMissingAudioInsteadOfSilentlySkippingRow() throws { + let directory = try metadataDirectory([header, "7,0,0,0,1"].joined(separator: "\n")) + + XCTAssertThrowsError( + try EnhanceBenchmarkDataset.loadExamples(from: directory) { + !URL(fileURLWithPath: $0).lastPathComponent.hasSuffix("_lpb.wav") + } + ) { + XCTAssertEqual( + $0 as? EnhanceBenchmarkDataset.DatasetError, + .missingAudio(fileID: "7", path: "fileid_7_lpb.wav")) + } + } + + func testStreamingSHA256() throws { + let directory = try metadataDirectory("abc") + let url = directory.appendingPathComponent("meta.csv") + + XCTAssertEqual( + try EnhanceBenchmarkDataset.sha256(of: url), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + } + + func testRejectsNonCanonicalFileIDs() throws { + for id in ["01", "-1", "abc", "../other"] { + let directory = try metadataDirectory([header, "\(id),0,0,0,1"].joined(separator: "\n")) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) + } + } + + func testRejectsInvalidBooleanFlags() throws { + let directory = try metadataDirectory([header, "1,0,2,0,1"].joined(separator: "\n")) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) { + XCTAssertEqual( + $0 as? EnhanceBenchmarkDataset.DatasetError, + .invalidInteger(line: 2, column: "is_farend_noisy", value: "2")) + } + } + + private func metadataDirectory(_ metadata: String) throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try metadata.write(to: directory.appendingPathComponent("meta.csv"), atomically: true, encoding: .utf8) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + return directory + } +} +#endif diff --git a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift new file mode 100644 index 000000000..33f161c03 --- /dev/null +++ b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift @@ -0,0 +1,43 @@ +#if os(macOS) +import XCTest + +@testable import FluidAudioCLI + +final class EnhanceBenchmarkScoringTests: XCTestCase { + func testEmptyHypothesisHasZeroRecall() { + let score = EnhanceBenchmarkCommand.score(hypothesis: [], reference: ["one", "two"], farEnd: ["three"]) + XCTAssertEqual(score.hits, 0) + XCTAssertEqual(score.deletions, 2) + XCTAssertEqual(score.errors, 2) + XCTAssertEqual(score.leaked, 0) + } + + func testLeakageSubtractsNearEndWordsAndRespectsMultiplicity() { + let score = EnhanceBenchmarkCommand.score( + hypothesis: ["hello", "echo", "echo"], reference: ["hello"], farEnd: ["hello", "echo"]) + XCTAssertEqual(score.hits, 1) + XCTAssertEqual(score.insertions, 2) + XCTAssertEqual(score.errors, 2) + XCTAssertEqual(score.leaked, 1) + } + + func testSubstitutionAndDeletionReduceRecallButInsertionDoesNot() { + let score = EnhanceBenchmarkCommand.score( + hypothesis: ["a", "x", "c", "d", "e"], reference: ["a", "b", "c", "d"], farEnd: []) + XCTAssertEqual(score.hits, 3) + XCTAssertEqual(score.substitutions, 1) + XCTAssertEqual(score.insertions, 1) + XCTAssertEqual(score.deletions, 0) + XCTAssertEqual(score.errors, 2) + } + + func testAlreadyNormalizedTokensAreNotNormalizedAgain() { + let metrics = WERCalculator.calculateWordMetrics(hypothesis: ["colour"], reference: ["color"]) + XCTAssertEqual(metrics.substitutions, 1) + } + + func testEmptyAudioFailsBeforeInference() { + XCTAssertThrowsError(try EnhanceBenchmarkCommand.validateAudio(mic: [], reference: [], clean: [], fileID: "0")) + } +} +#endif diff --git a/Tests/FluidAudioTests/CLI/WERCalculatorTests.swift b/Tests/FluidAudioTests/CLI/WERCalculatorTests.swift new file mode 100644 index 000000000..cb6c4683b --- /dev/null +++ b/Tests/FluidAudioTests/CLI/WERCalculatorTests.swift @@ -0,0 +1,54 @@ +import XCTest + +@testable import FluidAudioCLI + +/// Insertion/deletion labelling of the shared WER scorer. +/// +/// The backtrace used to label a reference word missing from the hypothesis +/// as an "insertion" and an extra hypothesis word as a "deletion" (swapped). +/// WER was unaffected (it sums all three) but any consumer that reads the +/// breakdown — the recall metric in `enhance-benchmark`, the S/D/I lines in +/// `canary-transcribe` and `tts-asr-verify` — was wrong; an empty hypothesis +/// scored 100% recall. +final class WERCalculatorTests: XCTestCase { + + func testEmptyHypothesisIsAllDeletions() { + let m = WERCalculator.calculateWERMetrics(hypothesis: "", reference: "one two three") + XCTAssertEqual(m.totalWords, 3) + XCTAssertEqual(m.deletions, 3) + XCTAssertEqual(m.insertions, 0) + XCTAssertEqual(m.substitutions, 0) + XCTAssertEqual(m.wer, 1.0, accuracy: 1e-9) + } + + func testEmptyReferenceIsAllInsertions() { + let m = WERCalculator.calculateWERMetrics(hypothesis: "one two", reference: "") + XCTAssertEqual(m.totalWords, 0) + XCTAssertEqual(m.insertions, 2) + XCTAssertEqual(m.deletions, 0) + } + + func testMixedEdits() { + // ref: a b c d ; hyp: a x c d e -> 1 substitution (b->x), 1 insertion (e) + let m = WERCalculator.calculateWERMetrics(hypothesis: "a x c d e", reference: "a b c d") + XCTAssertEqual(m.substitutions, 1) + XCTAssertEqual(m.insertions, 1) + XCTAssertEqual(m.deletions, 0) + XCTAssertEqual(m.wer, 0.5, accuracy: 1e-9) + + // ref: a b c d ; hyp: a c -> 2 deletions + let d = WERCalculator.calculateWERMetrics(hypothesis: "a c", reference: "a b c d") + XCTAssertEqual(d.deletions, 2) + XCTAssertEqual(d.insertions, 0) + XCTAssertEqual(d.substitutions, 0) + } + + func testRecallFromBreakdown() { + // Recall as enhance-benchmark computes it: (N - D - S) / N. + let m = WERCalculator.calculateWERMetrics(hypothesis: "a c z", reference: "a b c d") + let recall = Double(m.totalWords - m.deletions - m.substitutions) / Double(m.totalWords) + XCTAssertEqual(recall, 0.5, accuracy: 1e-9) + let empty = WERCalculator.calculateWERMetrics(hypothesis: "", reference: "a b c d") + XCTAssertEqual(Double(empty.totalWords - empty.deletions - empty.substitutions), 0) + } +} diff --git a/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift b/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift new file mode 100644 index 000000000..75a2e8223 --- /dev/null +++ b/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift @@ -0,0 +1,319 @@ +import CoreML +import Foundation +import XCTest + +@testable import FluidAudio + +/// Model-free checks of the LocalVQE naming / configuration surface. +final class LocalVqeNamingTests: XCTestCase { + + func testModelFileNames() { + XCTAssertEqual( + ModelNames.LocalVQE.modelFile(variant: .v13, chunk: .batch256ms), + "localvqe-v1.3-4.8M-256ms.mlmodelc") + XCTAssertEqual( + ModelNames.LocalVQE.modelFile(variant: .v12, chunk: .realtime16ms), + "localvqe-v1.2-1.3M-16ms.mlmodelc") + XCTAssertEqual(ModelNames.LocalVQE.allModels.count, 4) + } + + func testVariantKeyNarrowsRequiredSet() { + let key = ModelNames.LocalVQE.variantKey(variant: .v13, chunk: .realtime16ms) + XCTAssertEqual(key, "v1.3-16ms") + XCTAssertEqual( + ModelNames.LocalVQE.requiredModels(variant: key), + ["localvqe-v1.3-4.8M-16ms.mlmodelc"]) + XCTAssertEqual(ModelNames.getRequiredModelNames(for: .localVqe, variant: key).count, 1) + XCTAssertEqual(ModelNames.LocalVQE.requiredModels(variant: nil), ModelNames.LocalVQE.allModels) + XCTAssertEqual(ModelNames.LocalVQE.requiredModels(variant: "bogus"), ModelNames.LocalVQE.allModels) + } + + func testChunkSampleCounts() { + XCTAssertEqual(LocalVqeChunk.realtime16ms.samplesPerCall, 256) + XCTAssertEqual(LocalVqeChunk.batch256ms.samplesPerCall, 4096) + XCTAssertEqual(LocalVqeManager.outputDelaySamples, LocalVqeManager.hopSize) + XCTAssertEqual(Repo.localVqe.remotePath, "FluidInference/localvqe-coreml") + XCTAssertEqual(Repo.localVqe.folderName, "localvqe") + } +} + +/// End-to-end checks against a locally available model bundle. +/// +/// Set `FLUIDAUDIO_LOCALVQE_MODEL_DIR` to a directory holding the compiled +/// `localvqe-*.mlmodelc` bundles (e.g. the mobius conversion `build/` dir); +/// otherwise the default model cache is used, and the tests skip when the +/// model is absent or when running in CI. An explicit directory enables +/// these tests in CI and a missing model in that directory is a failure. +final class LocalVqeStreamTests: XCTestCase { + + private static let fixture = "01-validation-request-21.4s" + + override func setUp() async throws { + if ProcessInfo.processInfo.environment["CI"] != nil, + ProcessInfo.processInfo.environment["FLUIDAUDIO_LOCALVQE_MODEL_DIR"] == nil + { + throw XCTSkip("Skipping LocalVQE model tests in CI") + } + } + + private func loadManager(chunk: LocalVqeChunk) throws -> LocalVqeManager { + let config = LocalVqeConfig(variant: .v13, chunk: chunk, computeUnits: .cpuOnly) + let dir: URL + if let override = ProcessInfo.processInfo.environment["FLUIDAUDIO_LOCALVQE_MODEL_DIR"] { + dir = URL(fileURLWithPath: override) + } else { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + dir = appSupport.appendingPathComponent("FluidAudio/Models/\(Repo.localVqe.folderName)") + } + let file = dir.appendingPathComponent(ModelNames.LocalVQE.modelFile(variant: .v13, chunk: chunk)) + guard FileManager.default.fileExists(atPath: file.path) else { + if ProcessInfo.processInfo.environment["FLUIDAUDIO_LOCALVQE_MODEL_DIR"] != nil { + throw LocalVqeError.modelLoadingFailed("Required test model not available at \(file.path)") + } + throw XCTSkip("LocalVQE model not available at \(file.path)") + } + return try LocalVqeManager(config: config, modelDirectory: dir) + } + + private func loadFixture() throws -> [Float] { + guard + let url = Bundle.module.url(forResource: "Fixtures/\(Self.fixture)", withExtension: "wav") + ?? Bundle.module.url(forResource: Self.fixture, withExtension: "wav") + else { + throw XCTSkip("fixture \(Self.fixture).wav not bundled") + } + return try AudioConverter().resampleAudioFile(url) + } + + func testWholeClipOutputIsSameLengthAndBounded() async throws { + let manager = try loadManager(chunk: .batch256ms) + let mic = try loadFixture() + // Silent far end: the model runs as a noise suppressor / dereverberator. + let out = try await manager.process(mic: mic) + XCTAssertEqual(out.count, mic.count) + XCTAssertFalse(out.contains { !$0.isFinite }) + let inRms = (mic.reduce(0) { $0 + $1 * $1 } / Float(mic.count)).squareRoot() + let outRms = (out.reduce(0) { $0 + $1 * $1 } / Float(out.count)).squareRoot() + XCTAssertGreaterThan(outRms, inRms * 0.1, "enhancer removed almost all speech") + XCTAssertLessThan(outRms, inRms * 4, "enhancer output level far above input") + } + + func testStreamingMatchesWholeClipAcrossBufferSizes() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = Array(try loadFixture().prefix(16000 * 4)) + let reference = [Float](repeating: 0, count: mic.count) + let whole = try await manager.process(mic: mic, reference: reference) + XCTAssertEqual(whole.count, mic.count) + + for bufferSize in [100, 256, 1000, 4096] { + let stream = try await manager.makeStream() + var out: [Float] = [] + var offset = 0 + while offset < mic.count { + let end = min(offset + bufferSize, mic.count) + out.append( + contentsOf: try await stream.enhance( + mic: Array(mic[offset.. [Float] in + switch operation { + case "enhance": + return try await stream.enhance(mic: mic, reference: [Float](repeating: 0, count: mic.count)) + case "flush": + return try await stream.flush() + default: + try await stream.reset() + return [] + } + } + defer { queued.cancel() } + try await waitForOperations(2, on: stream) + queued.cancel() + do { + _ = try await queued.value + XCTFail("Queued \(operation) should throw CancellationError") + } catch is CancellationError { + // Cancellation must remove the queued operation without resetting the active clip. + } + var actual = try await first.value + actual.append(contentsOf: try await stream.flush()) + assertAudioEqual(actual, expected) + } + } + + func testCancelledActivePushResetsBeforeReuse() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = try loadFixture() + let nextClip = Array(mic.suffix(16000 + 137)) + let expected = try await manager.process(mic: nextClip) + let stream = try await manager.makeStream() + let active = Task { + try await stream.enhance(mic: mic, reference: [Float](repeating: 0, count: mic.count)) + } + defer { active.cancel() } + try await waitForOperations(1, on: stream) + active.cancel() + do { + _ = try await active.value + XCTFail("Active push should throw CancellationError") + } catch is CancellationError { + // A partially processed clip must not contaminate the next one. + } + var actual = try await stream.enhance( + mic: nextClip, reference: [Float](repeating: 0, count: nextClip.count)) + actual.append(contentsOf: try await stream.flush()) + assertAudioEqual(actual, expected) + } + + func testIndependentStreamsCanRunConcurrently() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = Array(try loadFixture().prefix(16000 + 137)) + let otherMic = Array(try loadFixture().suffix(16000 + 73)) + let expected = try await manager.process(mic: mic) + let otherExpected = try await manager.process(mic: otherMic) + + async let actual = manager.process(mic: mic) + async let otherActual = manager.process(mic: otherMic) + let results = try await (actual, otherActual) + assertAudioEqual(results.0, expected) + assertAudioEqual(results.1, otherExpected) + } +} diff --git a/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift b/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift index 0f33de574..82e021f59 100644 --- a/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift +++ b/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift @@ -147,6 +147,17 @@ final class ModelRegistryTests: XCTestCase { XCTAssertEqual(url.absoluteString, expectedPath, "Resolve dataset URL should use custom registry") } + func testResolveDatasetURLAtPinnedRevision() throws { + let revision = "1f3714b5a3f98cedef1bbb017f21bbd7ae688596" + let url = try ModelRegistry.resolveDataset( + "FluidInference/aec-challenge-synthetic-mini", "aec-synthetic-mini.tar.gz", revision: revision) + + XCTAssertEqual( + url.absoluteString, + "https://huggingface.co/datasets/FluidInference/aec-challenge-synthetic-mini/resolve/\(revision)/aec-synthetic-mini.tar.gz" + ) + } + // MARK: - Dataset Base URL Tests func testResolveDatasetBaseConstruction() {