From fbc53cbc78f02a028de06b4279b14b96dbab916e Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 11:29:29 -0400 Subject: [PATCH 01/14] feat(enhancement): LocalVQE echo cancellation + noise suppression (beta) Adds LocalVqeManager / LocalVqeStream, a Core ML port of LocalVQE (localai-org/LocalVQE, Apache-2.0): joint acoustic echo cancellation, noise suppression and dereverberation for 16 kHz speech, requested in #49 for hands-free calls where the mic picks up the loudspeaker. The models are fp32 streaming exports with explicit recurrent state (33 in_*/out_* tensors). LocalVqeStream discovers the state tensors from the model description, passes each call's outputs back as the next call's inputs, buffers arbitrary input sizes into whole calls, drops the leading hop (the t<0 region) and flushes one hop of zeros at the end so whole-clip output is sample-aligned and length-preserving. Two chunk exports per checkpoint: 256 ms (files, 36x RTFx for v1.3 on M5 Pro CPU) and 16 ms (live capture, 1.2 ms per call). Verified against the upstream PyTorch reference (74 dB, 16-bit WAV limited) and the upstream GGML CLI (80 dB) on the upstream double-talk demo clip; 100/256/1000/4096-sample streaming buffers and whole-clip processing agree to 1e-5. CPU is the default compute unit: the graph is too small for ANE dispatch to pay off and fp16 was rejected for parity (102 -> 5 dB). Upstream's GGUF-only v1.4-AEC / GTCRN line depends on a C++ adaptive-filter front-end and is not ported. CLI: fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav (--streaming reports per-call latency; --model-dir loads local bundles). Model tests skip in CI and when the bundle is absent locally (FLUIDAUDIO_LOCALVQE_MODEL_DIR overrides the cache location). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/CLI.md | 17 ++ Documentation/Enhancement/LocalVQE.md | 131 +++++++++ Documentation/Models.md | 7 + README.md | 27 ++ .../LocalVQE/LocalVqeManager.swift | 136 ++++++++++ .../Enhancement/LocalVQE/LocalVqeStream.swift | 228 ++++++++++++++++ .../Enhancement/LocalVQE/LocalVqeTypes.swift | 86 ++++++ Sources/FluidAudio/ModelNames.swift | 44 ++++ .../Commands/EnhanceCommand.swift | 249 ++++++++++++++++++ Sources/FluidAudioCLI/FluidAudioCLI.swift | 3 + .../Enhancement/LocalVqeTests.swift | 160 +++++++++++ 11 files changed, 1088 insertions(+) create mode 100644 Documentation/Enhancement/LocalVQE.md create mode 100644 Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift create mode 100644 Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift create mode 100644 Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeTypes.swift create mode 100644 Sources/FluidAudioCLI/Commands/EnhanceCommand.swift create mode 100644 Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift diff --git a/Documentation/CLI.md b/Documentation/CLI.md index 58296de56..5cf1643c2 100644 --- a/Documentation/CLI.md +++ b/Documentation/CLI.md @@ -109,6 +109,23 @@ swift run fluidaudiocli vad-benchmark --all-files --output vad_results.json --de `swift run fluidaudiocli vad-analyze --help` lists every tuning option (padding, negative threshold overrides, max-duration splitting, etc.). +## Speech Enhancement (LocalVQE) + +```bash +# Echo cancellation + noise suppression: mic capture plus what the speaker played +swift run -c release fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav + +# Noise suppression / dereverb only (silent far end) +swift run -c release fluidaudiocli enhance mic.wav --output clean.wav + +# Drive the streaming API in 256-sample buffers with the 16 ms export and report per-call latency +swift run -c release fluidaudiocli enhance mic.wav -r speaker.wav --chunk 16ms --streaming --buffer-samples 256 +``` + +`--variant v1.2` selects the 1.3M-param checkpoint, `--compute-units gpu` +moves the model off the CPU, and `--model-dir DIR` loads local `.mlmodelc` +bundles instead of downloading. + ## Datasets ```bash diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md new file mode 100644 index 000000000..4f566baa8 --- /dev/null +++ b/Documentation/Enhancement/LocalVQE.md @@ -0,0 +1,131 @@ +# LocalVQE — Echo Cancellation & Noise Suppression + +`LocalVqeManager` runs [LocalVQE](https://github.com/localai-org/LocalVQE) +(Apache-2.0), a compact neural model for acoustic echo cancellation (AEC), +noise suppression and dereverberation of 16 kHz speech. It is a streaming, +CPU-tuned derivative of DeepVQE (Indenbom et al., Interspeech 2023). Typical +use: cleaning up call audio captured without headphones, where the mic picks +up what the loudspeaker plays. + +**Beta.** Verified against the upstream PyTorch and GGML engines on the +upstream double-talk demo (see [Parity](#parity)); not yet exercised inside +production call pipelines. + +## Inputs + +The model takes two 16 kHz mono signals of equal length: + +- **mic** — the microphone capture. +- **reference** — the far-end signal: a loopback of what the loudspeaker + played. Without it the model still denoises and dereverberates; pass + silence (`process(mic:)` does this for you). + +Output is 16 kHz mono, same length as the input, sample-aligned. Level +matches the upstream GGML engine (the OBS plugin and HF demo). + +## Quick start + +```swift +import FluidAudio + +let vqe = try await LocalVqeManager() // downloads v1.3 (256 ms chunk) on first use +let clean = try await vqe.process(mic: micSamples, reference: farEndSamples) + +// Files (any format / rate; converted to 16 kHz mono) +let cleanFile = try await vqe.process(micURL: micURL, referenceURL: speakerURL) +``` + +### Streaming + +```swift +let vqe = try await LocalVqeManager(config: LocalVqeConfig(chunk: .realtime16ms)) +let stream = try await vqe.makeStream() + +// Push buffers of any size as they arrive (mic and reference must be equal length). +let out = try await stream.enhance(mic: micBuffer, reference: refBuffer) + +// End of clip: drain the delay line so total output == total input. +let tail = try await stream.flush() +``` + +`enhance` returns samples as whole model calls complete. Output sample `i` +corresponds to input sample `i`, delivered one hop (256 samples, 16 ms) +after the input that produced it plus whatever is still buffered toward the +next call. `flush()` resets the stream; call `reset()` to start a new clip +without flushing. + +## Configuration + +```swift +LocalVqeConfig( + variant: .v13, // .v13 (4.8M params, default) or .v12 (1.3M, ~1/4 the cost) + chunk: .batch256ms, // .batch256ms (files) or .realtime16ms (live capture) + computeUnits: .cpuOnly // fp32 models; CPU is fastest for the 16 ms chunk +) +``` + +Both variants are joint AEC + NS + dereverb models. The chunk size only +changes how many 16 ms hops each Core ML call consumes; the audio is +bit-identical either way. + +| Variant | Chunk | Compute | Per-call p50 | RTFx | +|---|---|---|---:|---:| +| v1.3 | 256 ms | CPU | 7.1 ms | 36× | +| v1.3 | 16 ms | CPU | 1.2 ms | 14× | +| v1.2 | 256 ms | CPU | 4.2 ms | 60× | +| v1.2 | 16 ms | CPU | 0.7 ms | 24× | + +Apple M5 Pro, release build, `fluidaudiocli enhance --streaming`. RTFx is +audio-per-call ÷ p50 latency. GPU gives ~15% on the 256 ms chunk at the cost +of a ~110 ms first-call compile; ANE is not used (see below). + +## CLI + +```bash +swift run -c release fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav +swift run -c release fluidaudiocli enhance mic.wav --output clean.wav # NS/dereverb only +swift run -c release fluidaudiocli enhance mic.wav -r speaker.wav --chunk 16ms --streaming +``` + +`--variant v1.2`, `--compute-units gpu`, `--buffer-samples N` (streaming +buffer size) and `--model-dir DIR` (load local `.mlmodelc` bundles) are also +available; `--help` lists everything. + +## Models + +HuggingFace: [FluidInference/localvqe-coreml](https://huggingface.co/FluidInference/localvqe-coreml). +One `.mlmodelc` per (variant, chunk); only the configured one is downloaded +(19 MB for v1.3, 5 MB for v1.2). Cached under +`~/Library/Application Support/FluidAudio/Models/localvqe/`. + +Manual loading: + +```swift +let vqe = try LocalVqeManager(config: config, modelDirectory: URL(fileURLWithPath: "/path/with/mlmodelc")) +``` + +The models are fp32 streaming exports with explicit state: every call takes +`mic`/`ref` plus 33 `in_*` state tensors and returns `enhanced` plus the +matching `out_*` tensors. fp16 was rejected: it drops parity with the +reference from 102 dB to 5 dB (CPU) / 33 dB (ANE) because the power-law +front-end epsilons underflow and the S4D recurrence accumulates error. +Conversion lives in the [mobius](https://github.com/FluidInference/mobius) +repo under `models/enhancement/localvqe/coreml`. + +## Parity + +Upstream double-talk demo clip (10 s), Swift `LocalVqeStream` output: + +| Against | max abs diff | SNR | +|---|---:|---:| +| Upstream PyTorch reference (fp32, ×2 to the GGML level) | 3.8e-5 | 74 dB (16-bit WAV limited) | +| Upstream GGML CLI (`localvqe-v1.3-4.8M-f32.gguf`) | 2.8e-5 | 80 dB | + +Streaming in 100 / 256 / 1000 / 4096-sample buffers and whole-clip +processing produce the same audio to 1e-5. + +## Not included + +Upstream's `v1.4-AEC` (echo-only, keeps room and noise) and the low-power +GTCRN line are GGUF-only and depend on a C++ adaptive-filter front-end with +no PyTorch reference; they are not converted. diff --git a/Documentation/Models.md b/Documentation/Models.md index 3aea8b7f8..ca4daea2c 100644 --- a/Documentation/Models.md +++ b/Documentation/Models.md @@ -43,6 +43,12 @@ TDT/CTC and the non-autoregressive models above are wrapped by `SlidingWindowAsr |-------|-------------|---------| | **Silero VAD** | Voice activity detection; speech vs silence on 256ms windows. Segments audio before ASR or diarization. | Support model that other pipelines build on. Converted at the time being the best model out there | +## Speech Enhancement Models + +| Model | Description | Context | +|-------|-------------|---------| +| **LocalVQE** | Neural acoustic echo cancellation + noise suppression + dereverberation for 16 kHz speech (DeepVQE derivative, Apache-2.0). Takes mic + far-end reference, returns clean near-end speech with 16 ms algorithmic latency. fp32 streaming exports with explicit state: v1.3 (4.8M, default) and v1.2 (1.3M) in 256 ms and 16 ms chunk sizes; 36× / 14× RTFx on CPU (M5 Pro). Managed by `LocalVqeManager` / `LocalVqeStream`. | Requested in [#49](https://github.com/FluidInference/FluidAudio/issues/49#issuecomment-5719663475) for hands-free calls. fp16 rejected (parity 102 → 5 dB); ANE not used. Upstream's GGUF-only v1.4-AEC / GTCRN line not converted. | + ## Diarization Models | Model | Description | Context | @@ -86,6 +92,7 @@ Models we converted and tested but are not supported: too large for on-device de | Parakeet EOU | [FluidInference/parakeet-realtime-eou-120m-coreml](https://huggingface.co/FluidInference/parakeet-realtime-eou-120m-coreml) (subdirs: `/160ms`, `/320ms`, `/1280ms`) | | Cohere Transcribe (INT8 hybrid, default) | [FluidInference/cohere-transcribe-03-2026-coreml](https://huggingface.co/FluidInference/cohere-transcribe-03-2026-coreml) (variant: `/q8`) | | Silero VAD | [FluidInference/silero-vad-coreml](https://huggingface.co/FluidInference/silero-vad-coreml) | +| LocalVQE | [FluidInference/localvqe-coreml](https://huggingface.co/FluidInference/localvqe-coreml) (`localvqe-v1.3-4.8M-{16ms,256ms}.mlmodelc`, `localvqe-v1.2-1.3M-{16ms,256ms}.mlmodelc`) | | Diarization (Pyannote) | [FluidInference/speaker-diarization-coreml](https://huggingface.co/FluidInference/speaker-diarization-coreml) | | LS-EEND | [FluidInference/ls-eend-coreml](https://huggingface.co/FluidInference/ls-eend-coreml) (per-dataset optimized variants: `/optimized/ami`, `/optimized/ch`, `/optimized/dih2`, `/optimized/dih3`) | | Sortformer | [FluidInference/diar-streaming-sortformer-coreml](https://huggingface.co/FluidInference/diar-streaming-sortformer-coreml) | diff --git a/README.md b/README.md index 6ae446d1d..293410ffa 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Want to convert your own model? Check [möbius](https://github.com/FluidInferenc - **Speaker Diarization (Online + Offline)**: Speaker separation and identification across audio streams. Streaming pipeline for real-time processing and offline batch pipeline with advanced clustering. - **Speaker Embedding Extraction**: Generate speaker embeddings for voice comparison and clustering, you can use this for speaker identification - **Voice Activity Detection (VAD)**: Voice activity detection with Silero models +- **Speech Enhancement (AEC + Noise Suppression)**: [LocalVQE](Documentation/Enhancement/LocalVQE.md) (4.8M) removes loudspeaker echo, noise and reverb from 16 kHz mic audio given a far-end reference; streaming with 16 ms latency (beta) - **Apple Neural Engine**: Models run efficiently on Apple's ANE for maximum performance with minimal power consumption - **Open-Source Models**: All models are publicly available on HuggingFace — converted and optimized by our team; permissive licenses. See [full model catalog](Documentation/Models.md). @@ -268,6 +269,7 @@ The default is `false` — no behaviour change for existing callers. Combine wit - [Speaker Diarization Guide](Documentation/Diarization/GettingStarted.md) - VAD: [Getting Started](Documentation/VAD/GettingStarted.md) - [Segmentation](Documentation/VAD/Segmentation.md) + - Speech Enhancement: [LocalVQE (AEC + NS)](Documentation/Enhancement/LocalVQE.md) - [Model Conversion Code](https://github.com/FluidInference/mobius) - [Benchmarks](Documentation/Benchmarks.md) - [API Reference](Documentation/API.md) @@ -570,6 +572,31 @@ swift run fluidaudiocli vad-benchmark --num-files 50 --threshold 0.3 negative-threshold overrides, max-speech splitting, padding, and chunk size. Offline mode also reports RTFx using the model's per-chunk processing time. +## Speech Enhancement (Echo Cancellation + Noise Suppression) + +> **⚠️ Beta:** verified against the upstream engines on the upstream demo clip; not yet exercised in production call pipelines. + +[LocalVQE](https://github.com/localai-org/LocalVQE) (Apache-2.0) is a compact +neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz +speech. Feed it the mic capture and a far-end reference (what the speaker +played) and it returns clean near-end speech, sample-aligned with the input. +Two checkpoints (v1.3 4.8M, v1.2 1.3M) in 256 ms and 16 ms chunk exports; +36× / 14× real-time on CPU for v1.3. See +[Documentation/Enhancement/LocalVQE.md](Documentation/Enhancement/LocalVQE.md). + +```swift +let vqe = try await LocalVqeManager() +let clean = try await vqe.process(mic: micSamples, reference: farEndSamples) + +// Live capture: push buffers of any size, 16 ms algorithmic latency +let stream = try await LocalVqeManager(config: LocalVqeConfig(chunk: .realtime16ms)).makeStream() +let out = try await stream.enhance(mic: micBuffer, reference: refBuffer) +``` + +```bash +swift run -c release fluidaudiocli enhance mic.wav --reference speaker.wav --output clean.wav +``` + ## Text‑To‑Speech (TTS) > **⚠️ Beta:** TTS currently supports American English only. Additional language support is planned. diff --git a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift new file mode 100644 index 000000000..d92997423 --- /dev/null +++ b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeManager.swift @@ -0,0 +1,136 @@ +import AVFoundation +@preconcurrency import CoreML +import Foundation +import OSLog + +/// LocalVQE speech enhancement: neural acoustic echo cancellation, noise +/// suppression and dereverberation for 16 kHz speech. +/// +/// Core ML port of [localai-org/LocalVQE](https://github.com/localai-org/LocalVQE) +/// (Apache-2.0), a streaming CPU-tuned derivative of DeepVQE. The model +/// needs two inputs: the microphone signal and a far-end **reference** +/// (a loopback of what the loudspeaker played). Without a reference it +/// still denoises and dereverberates; pass silence. +/// +/// ```swift +/// let vqe = try await LocalVqeManager() +/// let clean = try await vqe.process(mic: micSamples, reference: farEndSamples) +/// ``` +/// +/// For live audio, open a `LocalVqeStream` and push buffers as they arrive: +/// +/// ```swift +/// let stream = try await vqe.makeStream() +/// let out = try await stream.enhance(mic: micHop, reference: refHop) +/// ``` +/// +/// **Beta**: verified against the upstream PyTorch and GGML engines +/// (72 dB SNR, 16-bit-wav limited) on the upstream double-talk demo clip; +/// not yet exercised in production call pipelines. +public actor LocalVqeManager { + + private let logger = AppLogger(category: "LocalVqeManager") + + public static let sampleRate = 16000 + /// Analysis hop: every call consumes and produces a multiple of this. + public static let hopSize = 256 + /// Algorithmic delay of the enhanced output relative to the input. + public static let outputDelaySamples = hopSize + + public let config: LocalVqeConfig + private let audioConverter = AudioConverter() + private var model: MLModel? + + public var isAvailable: Bool { model != nil } + + /// Download (if needed) and load the configured model from HuggingFace. + public init( + config: LocalVqeConfig = .default, + progressHandler: ProgressHandler? = nil + ) async throws { + self.config = config + let start = Date() + let fileName = ModelNames.LocalVQE.modelFile(variant: config.variant, chunk: config.chunk) + let models = try await ModelHub.loadModels( + .localVqe, + modelNames: [fileName], + directory: Self.defaultBaseDirectory().appendingPathComponent("Models"), + computeUnits: config.computeUnits, + variant: ModelNames.LocalVQE.variantKey(variant: config.variant, chunk: config.chunk), + progressHandler: progressHandler + ) + guard let model = models[fileName] else { + throw LocalVqeError.modelLoadingFailed("\(fileName) missing after download") + } + self.model = model + logger.info( + "LocalVQE \(config.variant.rawValue)/\(config.chunk.rawValue) loaded in \(String(format: "%.2f", Date().timeIntervalSince(start)))s" + ) + } + + /// Load a compiled model bundle from a local directory (no download). + /// `modelDirectory` must contain `-.mlmodelc` for the + /// configured variant/chunk (see `ModelNames.LocalVQE.modelFile`). + public init(config: LocalVqeConfig = .default, modelDirectory: URL) throws { + self.config = config + let fileName = ModelNames.LocalVQE.modelFile(variant: config.variant, chunk: config.chunk) + let url = modelDirectory.appendingPathComponent(fileName) + let mlConfig = MLModelConfiguration() + mlConfig.computeUnits = config.computeUnits + do { + self.model = try MLModel(contentsOf: url, configuration: mlConfig) + } catch { + throw LocalVqeError.modelLoadingFailed("\(url.path): \(error.localizedDescription)") + } + } + + /// Wrap an already-loaded model (must match `config.chunk`). + public init(config: LocalVqeConfig = .default, model: MLModel) { + self.config = config + self.model = model + } + + // MARK: - Whole-clip processing + + /// Enhance a complete clip. `mic` and `reference` are 16 kHz mono and + /// must have equal length; the result has the same length and is + /// sample-aligned with `mic`. + public func process(mic: [Float], reference: [Float]) async throws -> [Float] { + let stream = try makeStream() + var out = try await stream.enhance(mic: mic, reference: reference) + out.append(contentsOf: try await stream.flush()) + return out + } + + /// Enhance a clip whose far-end reference is silent (noise suppression + + /// dereverberation only). + public func process(mic: [Float]) async throws -> [Float] { + try await process(mic: mic, reference: [Float](repeating: 0, count: mic.count)) + } + + /// Enhance a mic recording using a reference recording. Both files are + /// converted to 16 kHz mono; the shorter one is zero-padded. + public func process(micURL: URL, referenceURL: URL?) async throws -> [Float] { + let mic = try audioConverter.resampleAudioFile(micURL) + var reference = try referenceURL.map { try audioConverter.resampleAudioFile($0) } ?? [] + if reference.count < mic.count { + reference.append(contentsOf: [Float](repeating: 0, count: mic.count - reference.count)) + } else if reference.count > mic.count { + reference.removeLast(reference.count - mic.count) + } + return try await process(mic: mic, reference: reference) + } + + // MARK: - Streaming + + /// Open an independent streaming session on the loaded model. + public func makeStream() throws -> LocalVqeStream { + guard let model else { throw LocalVqeError.notInitialized } + return try LocalVqeStream(model: model, samplesPerCall: config.chunk.samplesPerCall) + } + + private static func defaultBaseDirectory() -> URL { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + return appSupport.appendingPathComponent("FluidAudio", isDirectory: true) + } +} diff --git a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift new file mode 100644 index 000000000..c45b91c01 --- /dev/null +++ b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift @@ -0,0 +1,228 @@ +@preconcurrency import CoreML +import Foundation +import OSLog + +/// Stateful hop-by-hop LocalVQE session. +/// +/// Feed equal-length mic and far-end reference audio (16 kHz mono Float32) +/// in any buffer size; the stream assembles whole model calls internally and +/// returns enhanced samples as they complete. Output is sample-aligned with +/// input (sample `i` out corresponds to sample `i` in) but is delivered one +/// hop (256 samples, 16 ms) later than the input that produced it, plus +/// whatever remains buffered toward the next call. Call `flush()` at the end +/// of a clip to drain the delay line so the total output length equals the +/// total input length. +/// +/// The Core ML model carries every recurrent state (conv histories, delay +/// windows, S4D bottleneck, overlap-add tail) as explicit `in_*`/`out_*` +/// tensors; the stream just passes each call's outputs back in as the next +/// call's inputs. Create one stream per audio channel pair; a stream is not +/// reusable across unrelated clips without `reset()`. +public actor LocalVqeStream { + + private static let logger = AppLogger(category: "LocalVqeStream") + private static let stateInputPrefix = "in_" + private static let stateOutputPrefix = "out_" + + private let model: MLModel + /// Samples consumed (and produced) per Core ML call. + public let samplesPerCall: Int + + private let micInput: MLMultiArray + private let refInput: MLMultiArray + private let stateNames: [String] + private let stateShapes: [String: [NSNumber]] + private var states: [String: MLMultiArray] = [:] + + private var pendingMic: [Float] = [] + private var pendingRef: [Float] = [] + /// The first emitted hop covers t < 0 of the input and is discarded so + /// that cumulative output stays aligned with cumulative input. + private var leadingSamplesToDrop = LocalVqeManager.hopSize + private var samplesIn = 0 + private var samplesOut = 0 + + init(model: MLModel, samplesPerCall: Int) throws { + self.model = model + self.samplesPerCall = samplesPerCall + + let desc = model.modelDescription + guard let micDesc = desc.inputDescriptionsByName["mic"]?.multiArrayConstraint, + let refDesc = desc.inputDescriptionsByName["ref"]?.multiArrayConstraint + else { + throw LocalVqeError.modelLoadingFailed("model lacks 'mic'/'ref' inputs") + } + let expected = micDesc.shape.map { $0.intValue }.reduce(1, *) + guard expected == samplesPerCall, refDesc.shape == micDesc.shape else { + throw LocalVqeError.modelLoadingFailed( + "model consumes \(expected) samples per call, expected \(samplesPerCall)") + } + micInput = try MLMultiArray(shape: micDesc.shape, dataType: .float32) + refInput = try MLMultiArray(shape: refDesc.shape, dataType: .float32) + + var names: [String] = [] + var shapes: [String: [NSNumber]] = [:] + for (inputName, inputDesc) in desc.inputDescriptionsByName where inputName.hasPrefix(Self.stateInputPrefix) { + guard let constraint = inputDesc.multiArrayConstraint else { continue } + let name = String(inputName.dropFirst(Self.stateInputPrefix.count)) + guard desc.outputDescriptionsByName[Self.stateOutputPrefix + name] != nil else { + throw LocalVqeError.modelLoadingFailed("state '\(name)' has no matching output") + } + names.append(name) + shapes[name] = constraint.shape + } + guard !names.isEmpty else { + throw LocalVqeError.modelLoadingFailed("model exposes no in_*/out_* state tensors") + } + stateNames = names.sorted() + stateShapes = shapes + states = try Self.zeroStates(names: stateNames, shapes: stateShapes) + } + + /// Number of state tensors the model carries between calls. + public var stateCount: Int { stateNames.count } + + /// Clear all recurrent state and buffered audio; the next call starts a new clip. + public func reset() throws { + try resetStates() + pendingMic.removeAll(keepingCapacity: true) + pendingRef.removeAll(keepingCapacity: true) + leadingSamplesToDrop = LocalVqeManager.hopSize + samplesIn = 0 + samplesOut = 0 + } + + private func resetStates() throws { + states = try Self.zeroStates(names: stateNames, shapes: stateShapes) + } + + private static func zeroStates(names: [String], shapes: [String: [NSNumber]]) throws -> [String: MLMultiArray] { + var fresh: [String: MLMultiArray] = [:] + for name in names { + guard let shape = shapes[name] else { continue } + let array = try MLMultiArray(shape: shape, dataType: .float32) + array.withUnsafeMutableBytes { ptr, _ in + ptr.initializeMemory(as: UInt8.self, repeating: 0) + } + fresh[name] = array + } + return fresh + } + + /// Push audio and return every enhanced sample that completed. + /// + /// `mic` and `reference` must have the same length; both may be any + /// length (including zero) — partial calls are buffered until enough + /// samples arrive. + public func enhance(mic: [Float], reference: [Float]) throws -> [Float] { + guard mic.count == reference.count else { + throw LocalVqeError.lengthMismatch(mic: mic.count, reference: reference.count) + } + pendingMic.append(contentsOf: mic) + pendingRef.append(contentsOf: reference) + samplesIn += mic.count + + var out: [Float] = [] + var offset = 0 + while pendingMic.count - offset >= samplesPerCall { + let hop = try runCall( + mic: pendingMic[offset.. 0 { + pendingMic.removeFirst(offset) + pendingRef.removeFirst(offset) + } + return emit(out) + } + + /// Drain the delay line by feeding silence, returning the remaining + /// samples so that total output length equals total input length. + /// Ends the current clip: the stream is reset afterwards. + public func flush() throws -> [Float] { + let outstanding = samplesIn - samplesOut + guard outstanding > 0 else { + try reset() + return [] + } + // Zeros needed to complete every outstanding sample, rounded up to whole calls. + let needed = pendingMic.count + LocalVqeManager.hopSize + let padded = ((needed + samplesPerCall - 1) / samplesPerCall) * samplesPerCall + let zeros = [Float](repeating: 0, count: padded - pendingMic.count) + pendingMic.append(contentsOf: zeros) + pendingRef.append(contentsOf: zeros) + + var out: [Float] = [] + var offset = 0 + while pendingMic.count - offset >= samplesPerCall { + let hop = try runCall( + mic: pendingMic[offset.. [Float] { + var result = samples + if leadingSamplesToDrop > 0 { + let drop = min(leadingSamplesToDrop, result.count) + result.removeFirst(drop) + leadingSamplesToDrop -= drop + } + samplesOut += result.count + return result + } + + private func runCall(mic: ArraySlice, reference: ArraySlice) throws -> [Float] { + micInput.withUnsafeMutableBufferPointer(ofType: Float.self) { buf, _ in + _ = buf.initialize(from: mic) + } + refInput.withUnsafeMutableBufferPointer(ofType: Float.self) { buf, _ in + _ = buf.initialize(from: reference) + } + var features: [String: Any] = ["mic": micInput, "ref": refInput] + for name in stateNames { + features[Self.stateInputPrefix + name] = states[name] + } + + let output: MLFeatureProvider + do { + let provider = try MLDictionaryFeatureProvider(dictionary: features) + output = try model.prediction(from: provider) + } catch { + throw LocalVqeError.modelProcessingFailed(error.localizedDescription) + } + + for name in stateNames { + guard let next = output.featureValue(for: Self.stateOutputPrefix + name)?.multiArrayValue else { + throw LocalVqeError.modelProcessingFailed("missing state output '\(name)'") + } + states[name] = next + } + guard let enhanced = output.featureValue(for: "enhanced")?.multiArrayValue else { + throw LocalVqeError.modelProcessingFailed("missing 'enhanced' output") + } + return Self.floats(from: enhanced, count: samplesPerCall) + } + + private static func floats(from array: MLMultiArray, count: Int) -> [Float] { + var result = [Float](repeating: 0, count: count) + switch array.dataType { + case .float32: + array.withUnsafeBufferPointer(ofType: Float.self) { buf in + for i in 0.. String { + "\(variant.fileStem)-\(chunk.rawValue).mlmodelc" + } + + public static func variantKey(variant: LocalVqeVariant, chunk: LocalVqeChunk) -> String { + "\(variant.rawValue)-\(chunk.rawValue)" + } + + public static var allModels: Set { + var files: Set = [] + for variant in LocalVqeVariant.allCases { + for chunk in LocalVqeChunk.allCases { + files.insert(modelFile(variant: variant, chunk: chunk)) + } + } + return files + } + + /// Required files for a download variant key; the full set when the + /// key is absent or unrecognised. + public static func requiredModels(variant: String?) -> Set { + for v in LocalVqeVariant.allCases { + for c in LocalVqeChunk.allCases where variantKey(variant: v, chunk: c) == variant { + return [modelFile(variant: v, chunk: c)] + } + } + return allModels + } + } + /// Parakeet EOU streaming model names public enum ParakeetEOU { public static let encoder = "streaming_encoder" @@ -1632,6 +1674,8 @@ public enum ModelNames { ] case .vad: return ModelNames.VAD.requiredModels + case .localVqe: + return ModelNames.LocalVQE.requiredModels(variant: variant) case .parakeetV3: let precision = ParakeetEncoderPrecision(rawValue: variant ?? "") ?? .int8 return ModelNames.ASR.requiredModelsV3(precision: precision) diff --git a/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift new file mode 100644 index 000000000..b98130970 --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift @@ -0,0 +1,249 @@ +#if os(macOS) +import AVFoundation +import CoreML +import FluidAudio +import Foundation + +/// `enhance`: LocalVQE acoustic echo cancellation + noise suppression on a file. +enum EnhanceCommand { + private static let logger = AppLogger(category: "Enhance") + + private struct Options { + var micPath: String? + var referencePath: String? + var outputPath: String? + var modelDirectory: String? + var variant: LocalVqeVariant = .v13 + var chunk: LocalVqeChunk = .batch256ms + var computeUnits: MLComputeUnits = .cpuOnly + var streaming = false + var bufferSamples = 256 + } + + static func run(arguments: [String]) async { + var options = Options() + var index = 0 + while index < arguments.count { + let arg = arguments[index] + switch arg { + case "--help", "-h": + printUsage() + exit(0) + case "--reference", "-r": + options.referencePath = next(arguments, &index) + case "--output", "-o": + options.outputPath = next(arguments, &index) + case "--model-dir": + options.modelDirectory = next(arguments, &index) + case "--variant": + guard let raw = next(arguments, &index), let v = LocalVqeVariant(rawValue: raw) else { + logger.error("--variant must be one of \(LocalVqeVariant.allCases.map(\.rawValue))") + exit(1) + } + options.variant = v + case "--chunk": + guard let raw = next(arguments, &index), let c = LocalVqeChunk(rawValue: raw) else { + logger.error("--chunk must be one of \(LocalVqeChunk.allCases.map(\.rawValue))") + exit(1) + } + options.chunk = c + case "--compute-units": + switch next(arguments, &index)?.lowercased() { + case "cpu-only", "cpu": options.computeUnits = .cpuOnly + case "gpu", "cpu-and-gpu": options.computeUnits = .cpuAndGPU + case "ane", "cpu-and-ne": options.computeUnits = .cpuAndNeuralEngine + case "all": options.computeUnits = .all + default: + logger.error("--compute-units must be cpu-only | gpu | ane | all") + exit(1) + } + case "--streaming": + options.streaming = true + case "--buffer-samples": + options.bufferSamples = Int(next(arguments, &index) ?? "") ?? options.bufferSamples + default: + if arg.hasPrefix("--") { + logger.warning("Unknown option: \(arg)") + } else if options.micPath == nil { + options.micPath = arg + } else { + logger.warning("Ignoring extra argument: \(arg)") + } + } + index += 1 + } + + guard let micPath = options.micPath else { + logger.error("No mic audio file provided") + printUsage() + exit(1) + } + + do { + let converter = AudioConverter() + let mic = try converter.resampleAudioFile(path: micPath) + var reference: [Float] + if let referencePath = options.referencePath { + reference = try converter.resampleAudioFile(path: referencePath) + } else { + logger.warning("No --reference given: running noise suppression / dereverb only (silent far end)") + reference = [] + } + if reference.count < mic.count { + reference.append(contentsOf: [Float](repeating: 0, count: mic.count - reference.count)) + } else if reference.count > mic.count { + reference.removeLast(reference.count - mic.count) + } + + let config = LocalVqeConfig( + variant: options.variant, chunk: options.chunk, computeUnits: options.computeUnits) + let loadStart = Date() + let manager: LocalVqeManager + if let dir = options.modelDirectory { + manager = try LocalVqeManager(config: config, modelDirectory: URL(fileURLWithPath: dir)) + } else { + manager = try await LocalVqeManager(config: config) + } + let loadTime = Date().timeIntervalSince(loadStart) + report( + "Loaded LocalVQE \(options.variant.rawValue) (\(options.chunk.rawValue) chunk, " + + "\(describe(options.computeUnits))) in \(String(format: "%.2f", loadTime))s") + + let audioSeconds = Double(mic.count) / Double(LocalVqeManager.sampleRate) + let enhanced: [Float] + let start = Date() + if options.streaming { + enhanced = try await runStreaming(manager: manager, mic: mic, reference: reference, options: options) + } else { + enhanced = try await manager.process(mic: mic, reference: reference) + } + let wall = Date().timeIntervalSince(start) + + let inRms = rms(mic) + let outRms = rms(enhanced) + report( + String( + format: "Enhanced %.2fs of audio in %.3fs wall (RTFx %.1fx); RMS in %.4f -> out %.4f (%.1f dB)", + audioSeconds, wall, audioSeconds / max(wall, 1e-9), inRms, outRms, + 20 * log10(max(outRms, 1e-9) / max(inRms, 1e-9)))) + + if let outputPath = options.outputPath { + try writeWav(samples: enhanced, sampleRate: LocalVqeManager.sampleRate, to: outputPath) + report("Wrote \(outputPath)") + } + } catch { + logger.error("Enhance failed: \(error)") + exit(1) + } + } + + /// Push audio through a `LocalVqeStream` in `bufferSamples` pieces and + /// report per-call latency. + private static func runStreaming( + manager: LocalVqeManager, mic: [Float], reference: [Float], options: Options + ) async throws -> [Float] { + let stream = try await manager.makeStream() + let step = max(1, options.bufferSamples) + var out: [Float] = [] + out.reserveCapacity(mic.count) + var latencies: [Double] = [] + var offset = 0 + while offset < mic.count { + let end = min(offset + step, mic.count) + let t0 = DispatchTime.now().uptimeNanoseconds + let hop = try await stream.enhance( + mic: Array(mic[offset.. String? { + guard index + 1 < arguments.count else { return nil } + index += 1 + return arguments[index] + } + + private static func describe(_ units: MLComputeUnits) -> String { + switch units { + case .cpuOnly: return "cpu-only" + case .cpuAndGPU: return "gpu" + case .cpuAndNeuralEngine: return "ane" + case .all: return "all" + @unknown default: return "unknown" + } + } + + private static func rms(_ samples: [Float]) -> Float { + guard !samples.isEmpty else { return 0 } + var acc: Float = 0 + for s in samples { acc += s * s } + return (acc / Float(samples.count)).squareRoot() + } + + private static func writeWav(samples: [Float], sampleRate: Int, to path: String) throws { + guard + let format = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: Double(sampleRate), channels: 1, interleaved: false), + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(max(samples.count, 1))) + else { + throw LocalVqeError.modelProcessingFailed("could not allocate output buffer") + } + buffer.frameLength = AVAudioFrameCount(samples.count) + guard let channel = buffer.floatChannelData?[0] else { + throw LocalVqeError.modelProcessingFailed("could not access output channel") + } + samples.withUnsafeBufferPointer { src in + if let base = src.baseAddress { channel.update(from: base, count: samples.count) } + } + let url = URL(fileURLWithPath: path) + try? FileManager.default.removeItem(at: url) + let file = try AVAudioFile( + forWriting: url, settings: format.settings, commonFormat: .pcmFormatFloat32, interleaved: false) + try file.write(from: buffer) + } + + private static func printUsage() { + logger.info( + """ + Usage: fluidaudiocli enhance [options] + + LocalVQE speech enhancement: acoustic echo cancellation + noise suppression + dereverberation. + + Options: + --reference, -r Far-end reference (what the speaker played). Omit for NS/dereverb only. + --output, -o Write the enhanced 16 kHz mono WAV here. + --variant Checkpoint (default v1.3, 4.8M params; v1.2 is 1.3M). + --chunk <256ms|16ms> Samples per model call (default 256ms; 16ms for live use). + --compute-units Default cpu-only (fp32 models). + --streaming Drive the LocalVqeStream API buffer-by-buffer and report call latency. + --buffer-samples Buffer size for --streaming (default 256). + --model-dir Load /.mlmodelc instead of downloading from HuggingFace. + """ + ) + } +} +#endif diff --git a/Sources/FluidAudioCLI/FluidAudioCLI.swift b/Sources/FluidAudioCLI/FluidAudioCLI.swift index 057ba1263..8188d4121 100644 --- a/Sources/FluidAudioCLI/FluidAudioCLI.swift +++ b/Sources/FluidAudioCLI/FluidAudioCLI.swift @@ -34,6 +34,8 @@ struct FluidAudioCLI { await VadBenchmark.runVadBenchmark(arguments: Array(arguments.dropFirst(2))) case "vad-analyze": await VadAnalyzeCommand.run(arguments: Array(arguments.dropFirst(2))) + case "enhance": + await EnhanceCommand.run(arguments: Array(arguments.dropFirst(2))) case "asr-benchmark": await ASRBenchmark.runASRBenchmark(arguments: Array(arguments.dropFirst(2))) case "unified-benchmark": @@ -130,6 +132,7 @@ struct FluidAudioCLI { vad-benchmark Run VAD-specific benchmark vad-analyze Inspect VAD segmentation and streaming events fsmn-vad-segment Detect speech segments with FSMN-VAD (beta) + enhance LocalVQE echo cancellation + noise suppression on a mic (+ reference) file (beta) asr-benchmark Run ASR benchmark on LibriSpeech fleurs-benchmark Run multilingual ASR benchmark on FLEURS dataset transcribe Transcribe audio file using streaming ASR diff --git a/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift b/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift new file mode 100644 index 000000000..82d57f8e5 --- /dev/null +++ b/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift @@ -0,0 +1,160 @@ +import CoreML +import Foundation +import XCTest + +@testable import FluidAudio + +/// Model-free checks of the LocalVQE naming / configuration surface. +final class LocalVqeNamingTests: XCTestCase { + + func testModelFileNames() { + XCTAssertEqual( + ModelNames.LocalVQE.modelFile(variant: .v13, chunk: .batch256ms), + "localvqe-v1.3-4.8M-256ms.mlmodelc") + XCTAssertEqual( + ModelNames.LocalVQE.modelFile(variant: .v12, chunk: .realtime16ms), + "localvqe-v1.2-1.3M-16ms.mlmodelc") + XCTAssertEqual(ModelNames.LocalVQE.allModels.count, 4) + } + + func testVariantKeyNarrowsRequiredSet() { + let key = ModelNames.LocalVQE.variantKey(variant: .v13, chunk: .realtime16ms) + XCTAssertEqual(key, "v1.3-16ms") + XCTAssertEqual( + ModelNames.LocalVQE.requiredModels(variant: key), + ["localvqe-v1.3-4.8M-16ms.mlmodelc"]) + XCTAssertEqual(ModelNames.getRequiredModelNames(for: .localVqe, variant: key).count, 1) + XCTAssertEqual(ModelNames.LocalVQE.requiredModels(variant: nil), ModelNames.LocalVQE.allModels) + XCTAssertEqual(ModelNames.LocalVQE.requiredModels(variant: "bogus"), ModelNames.LocalVQE.allModels) + } + + func testChunkSampleCounts() { + XCTAssertEqual(LocalVqeChunk.realtime16ms.samplesPerCall, 256) + XCTAssertEqual(LocalVqeChunk.batch256ms.samplesPerCall, 4096) + XCTAssertEqual(LocalVqeManager.outputDelaySamples, LocalVqeManager.hopSize) + XCTAssertEqual(Repo.localVqe.remotePath, "FluidInference/localvqe-coreml") + XCTAssertEqual(Repo.localVqe.folderName, "localvqe") + } +} + +/// End-to-end checks against a locally available model bundle. +/// +/// Set `FLUIDAUDIO_LOCALVQE_MODEL_DIR` to a directory holding the compiled +/// `localvqe-*.mlmodelc` bundles (e.g. the mobius conversion `build/` dir); +/// otherwise the default model cache is used, and the tests skip when the +/// model is absent or when running in CI. +final class LocalVqeStreamTests: XCTestCase { + + private static let fixture = "01-validation-request-21.4s" + + override func setUp() async throws { + if ProcessInfo.processInfo.environment["CI"] != nil { + throw XCTSkip("Skipping LocalVQE model tests in CI") + } + } + + private func loadManager(chunk: LocalVqeChunk) throws -> LocalVqeManager { + let config = LocalVqeConfig(variant: .v13, chunk: chunk, computeUnits: .cpuOnly) + let dir: URL + if let override = ProcessInfo.processInfo.environment["FLUIDAUDIO_LOCALVQE_MODEL_DIR"] { + dir = URL(fileURLWithPath: override) + } else { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + dir = appSupport.appendingPathComponent("FluidAudio/Models/\(Repo.localVqe.folderName)") + } + let file = dir.appendingPathComponent(ModelNames.LocalVQE.modelFile(variant: .v13, chunk: chunk)) + guard FileManager.default.fileExists(atPath: file.path) else { + throw XCTSkip("LocalVQE model not available at \(file.path)") + } + return try LocalVqeManager(config: config, modelDirectory: dir) + } + + private func loadFixture() throws -> [Float] { + guard + let url = Bundle.module.url(forResource: "Fixtures/\(Self.fixture)", withExtension: "wav") + ?? Bundle.module.url(forResource: Self.fixture, withExtension: "wav") + else { + throw XCTSkip("fixture \(Self.fixture).wav not bundled") + } + return try AudioConverter().resampleAudioFile(url) + } + + func testWholeClipOutputIsSameLengthAndBounded() async throws { + let manager = try loadManager(chunk: .batch256ms) + let mic = try loadFixture() + // Silent far end: the model runs as a noise suppressor / dereverberator. + let out = try await manager.process(mic: mic) + XCTAssertEqual(out.count, mic.count) + XCTAssertFalse(out.contains { !$0.isFinite }) + let inRms = (mic.reduce(0) { $0 + $1 * $1 } / Float(mic.count)).squareRoot() + let outRms = (out.reduce(0) { $0 + $1 * $1 } / Float(out.count)).squareRoot() + XCTAssertGreaterThan(outRms, inRms * 0.1, "enhancer removed almost all speech") + XCTAssertLessThan(outRms, inRms * 4, "enhancer output level far above input") + } + + func testStreamingMatchesWholeClipAcrossBufferSizes() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = Array(try loadFixture().prefix(16000 * 4)) + let reference = [Float](repeating: 0, count: mic.count) + let whole = try await manager.process(mic: mic, reference: reference) + XCTAssertEqual(whole.count, mic.count) + + for bufferSize in [100, 256, 1000, 4096] { + let stream = try await manager.makeStream() + var out: [Float] = [] + var offset = 0 + while offset < mic.count { + let end = min(offset + bufferSize, mic.count) + out.append( + contentsOf: try await stream.enhance( + mic: Array(mic[offset.. Date: Fri, 18 Sep 2026 11:51:31 -0400 Subject: [PATCH 02/14] =?UTF-8?q?feat(cli):=20enhance-benchmark=20?= =?UTF-8?q?=E2=80=94=20near-end=20word=20recall=20/=20far-end=20leakage=20?= =?UTF-8?q?for=20LocalVQE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scores an echo canceller with the in-repo Parakeet TDT v3 ASR on the Microsoft AEC-Challenge synthetic set (mic + loopback + clean near-end triples, 200-example subset at FluidInference/aec-challenge-synthetic-mini, auto-downloaded). The clean-near-end transcript is the reference, the loopback transcript gives the far-end words; reports recall (1 - (D+S)/N), WER, and far-end word leakage per condition and per SER bucket, plus enhancement RTFx. --no-reference adds a silent-far-end condition to show what the model does without the loopback. Results (200 files, SER -10..+10 dB, M5 Pro, 256 ms chunk, CPU): unprocessed recall 39.5% / leakage 33.8%; v1.3 87.5% / 1.8% (36x RTFx); v1.2 86.3% / 1.9% (62x); v1.3 with silent reference 45.3% / 24.4%. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/CLI.md | 6 + Documentation/Enhancement/LocalVQE.md | 34 ++ README.md | 4 +- .../Commands/EnhanceBenchmarkCommand.swift | 391 ++++++++++++++++++ Sources/FluidAudioCLI/FluidAudioCLI.swift | 3 + 5 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift diff --git a/Documentation/CLI.md b/Documentation/CLI.md index 5cf1643c2..679d3aecf 100644 --- a/Documentation/CLI.md +++ b/Documentation/CLI.md @@ -126,6 +126,12 @@ swift run -c release fluidaudiocli enhance mic.wav -r speaker.wav --chunk 16ms - moves the model off the CPU, and `--model-dir DIR` loads local `.mlmodelc` bundles instead of downloading. +```bash +# Near-end word recall / WER / far-end leakage on the AEC-Challenge synthetic mini set (auto-downloads) +swift run -c release fluidaudiocli enhance-benchmark +swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json +``` + ## Datasets ```bash diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 4f566baa8..6640975dc 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -124,6 +124,40 @@ Upstream double-talk demo clip (10 s), Swift `LocalVqeStream` output: Streaming in 100 / 256 / 1000 / 4096-sample buffers and whole-clip processing produce the same audio to 1e-5. +## Benchmark: near-end recall / far-end leakage + +`fluidaudiocli enhance-benchmark` scores the enhancer with the in-repo +Parakeet TDT v3 ASR on the Microsoft AEC-Challenge synthetic set (mic + +loopback + clean near-end triples; 200-example subset at +[FluidInference/aec-challenge-synthetic-mini](https://huggingface.co/datasets/FluidInference/aec-challenge-synthetic-mini), +auto-downloaded). The ASR transcript of the clean near-end clip is the +reference; the loopback transcript gives the far-end words. + +- **Recall**: reference words kept by the hypothesis, `1 - (D + S) / N`. +- **WER**: `(S + D + I) / N` against the clean-near-end transcript. Above + 100% on unprocessed audio because the ASR transcribes the echo as well. +- **Leakage**: far-end words that appear in the hypothesis without being + near-end words, over the far-end word count. + +200 examples, signal-to-echo ratio (SER) −10…+10 dB, M5 Pro, 256 ms chunk, CPU: + +| Condition | Recall | WER | Leakage | RTFx | +|---|---:|---:|---:|---:| +| Unprocessed mic | 39.5% | 134.2% | 33.8% | – | +| LocalVQE v1.3 | **87.5%** | 43.4% | **1.8%** | 36× | +| LocalVQE v1.2 | 86.3% | 49.4% | 1.9% | 62× | +| v1.3, silent reference (NS only) | 45.3% | 122.6% | 24.4% | 36× | + +By SER: at SER ≤ 0 dB (echo louder than speech, 110 files) v1.3 lifts recall +32.0% → 87.0% and cuts leakage 41.5% → 2.3%; at SER > 0 dB (90 files) +49.5% → 88.3% and 25.2% → 1.3%. The silent-reference row shows the model +needs the loopback to cancel echo; without it, it only denoises. + +```bash +swift run -c release fluidaudiocli enhance-benchmark # both variants, 200 files +swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json +``` + ## Not included Upstream's `v1.4-AEC` (echo-only, keeps room and noise) and the low-power diff --git a/README.md b/README.md index 293410ffa..087bffb97 100644 --- a/README.md +++ b/README.md @@ -581,7 +581,9 @@ neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz speech. Feed it the mic capture and a far-end reference (what the speaker played) and it returns clean near-end speech, sample-aligned with the input. Two checkpoints (v1.3 4.8M, v1.2 1.3M) in 256 ms and 16 ms chunk exports; -36× / 14× real-time on CPU for v1.3. See +36× / 14× real-time on CPU for v1.3. On the AEC-Challenge synthetic set v1.3 +lifts ASR near-end word recall from 39.5% to 87.5% and cuts far-end word +leakage from 33.8% to 1.8%. See [Documentation/Enhancement/LocalVQE.md](Documentation/Enhancement/LocalVQE.md). ```swift diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift new file mode 100644 index 000000000..635da76a7 --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -0,0 +1,391 @@ +#if os(macOS) +import CoreML +import FluidAudio +import Foundation + +/// `enhance-benchmark`: near-end word recall / far-end leakage of LocalVQE on the +/// Microsoft AEC-Challenge synthetic set (mic + loopback + clean near-end triples). +/// +/// The in-repo Parakeet ASR transcribes the clean near-end clip (reference +/// words) and the loopback clip (far-end words), then the unprocessed mic and +/// each enhanced output. Near-end recall is the fraction of reference words the +/// hypothesis keeps (1 - (deletions + substitutions) / N); far-end leakage is +/// the fraction of far-end words that show up in the hypothesis without being +/// near-end words. WER against the clean-near-end transcript is reported too. +enum EnhanceBenchmarkCommand { + private static let logger = AppLogger(category: "EnhanceBenchmark") + + static let datasetRepo = "FluidInference/aec-challenge-synthetic-mini" + static let datasetArchive = "aec-synthetic-mini.tar.gz" + static let datasetFolder = "aec-synthetic-mini" + + private struct Options { + var datasetDir: String? + var maxFiles: Int? + var variants: [LocalVqeVariant] = [.v13, .v12] + var chunk: LocalVqeChunk = .batch256ms + var computeUnits: MLComputeUnits = .cpuOnly + var includeNoReference = false + var outputPath: String? + } + + private struct Example { + let fileID: String + let mic: URL + let lpb: URL + let clean: URL + let ser: Int? + let nearendNoisy: Bool + } + + private struct ConditionTotals { + var files = 0 + var refWords = 0 + var hits = 0 + var errors = 0 + var farWords = 0 + var leaked = 0 + var enhanceSeconds = 0.0 + var audioSeconds = 0.0 + + var recall: Double { refWords == 0 ? 0 : Double(hits) / Double(refWords) } + var wer: Double { refWords == 0 ? 0 : Double(errors) / Double(refWords) } + var leakage: Double { farWords == 0 ? 0 : Double(leaked) / Double(farWords) } + var rtfx: Double { enhanceSeconds <= 0 ? 0 : audioSeconds / enhanceSeconds } + } + + static func run(arguments: [String]) async { + var options = Options() + var index = 0 + while index < arguments.count { + let arg = arguments[index] + switch arg { + case "--help", "-h": + printUsage() + exit(0) + case "--dataset-dir": + options.datasetDir = next(arguments, &index) + case "--max-files": + options.maxFiles = Int(next(arguments, &index) ?? "") + case "--variants": + let raw = (next(arguments, &index) ?? "").split(separator: ",").map(String.init) + let parsed = raw.compactMap(LocalVqeVariant.init(rawValue:)) + guard parsed.count == raw.count, !parsed.isEmpty else { + logger.error("--variants must be a comma list of \(LocalVqeVariant.allCases.map(\.rawValue))") + exit(1) + } + options.variants = parsed + case "--chunk": + guard let raw = next(arguments, &index), let c = LocalVqeChunk(rawValue: raw) else { + logger.error("--chunk must be one of \(LocalVqeChunk.allCases.map(\.rawValue))") + exit(1) + } + options.chunk = c + case "--compute-units": + switch next(arguments, &index)?.lowercased() { + case "cpu-only", "cpu": options.computeUnits = .cpuOnly + case "gpu", "cpu-and-gpu": options.computeUnits = .cpuAndGPU + case "ane", "cpu-and-ne": options.computeUnits = .cpuAndNeuralEngine + case "all": options.computeUnits = .all + default: + logger.error("--compute-units must be cpu-only | gpu | ane | all") + exit(1) + } + case "--no-reference": + options.includeNoReference = true + case "--output": + options.outputPath = next(arguments, &index) + default: + logger.warning("Unknown option: \(arg)") + } + index += 1 + } + + do { + let datasetDir = try await resolveDataset(options.datasetDir) + var examples = try loadExamples(from: datasetDir) + if let maxFiles = options.maxFiles { examples = Array(examples.prefix(maxFiles)) } + guard !examples.isEmpty else { + logger.error("No examples found in \(datasetDir.path)") + exit(1) + } + report("Dataset: \(datasetDir.path) (\(examples.count) examples)") + + let asr = AsrManager() + try await asr.loadModels(try await AsrModels.downloadAndLoad()) + report("ASR: Parakeet TDT v3 loaded") + + var conditions: [(name: String, manager: LocalVqeManager?, useReference: Bool)] = [ + ("unprocessed", nil, true) + ] + for variant in options.variants { + let config = LocalVqeConfig(variant: variant, chunk: options.chunk, computeUnits: options.computeUnits) + let manager = try await LocalVqeManager(config: config) + conditions.append(("localvqe-\(variant.rawValue)", manager, true)) + if options.includeNoReference { + conditions.append(("localvqe-\(variant.rawValue)-noref", manager, false)) + } + } + report("Conditions: \(conditions.map(\.name).joined(separator: ", "))") + + let converter = AudioConverter() + var totals = [String: ConditionTotals]() + var bySer = [String: [String: ConditionTotals]]() // bucket -> condition -> totals + var rows: [[String: Any]] = [] + + for (i, example) in examples.enumerated() { + let mic = try converter.resampleAudioFile(example.mic) + var lpb = try converter.resampleAudioFile(example.lpb) + let clean = try converter.resampleAudioFile(example.clean) + if lpb.count < mic.count { + lpb.append(contentsOf: [Float](repeating: 0, count: mic.count - lpb.count)) + } else if lpb.count > mic.count { + lpb.removeLast(lpb.count - mic.count) + } + + let refWords = words(try await transcribe(asr, clean)) + let farWords = words(try await transcribe(asr, lpb)) + let bucket = serBucket(example.ser) + var row: [String: Any] = [ + "fileid": example.fileID, "ser": example.ser as Any, "ref_words": refWords.count, + "far_words": farWords.count, "reference": refWords.joined(separator: " "), + ] + + for condition in conditions { + var enhanced = mic + var enhanceSeconds = 0.0 + if let manager = condition.manager { + let reference = condition.useReference ? lpb : [Float](repeating: 0, count: mic.count) + let start = Date() + enhanced = try await manager.process(mic: mic, reference: reference) + enhanceSeconds = Date().timeIntervalSince(start) + } + let hypWords = words(try await transcribe(asr, enhanced)) + let m = score(hypothesis: hypWords, reference: refWords, farEnd: farWords) + + var t = totals[condition.name, default: ConditionTotals()] + t.files += 1 + t.refWords += refWords.count + t.hits += m.hits + t.errors += m.errors + t.farWords += farWords.count + t.leaked += m.leaked + t.enhanceSeconds += enhanceSeconds + t.audioSeconds += Double(mic.count) / Double(LocalVqeManager.sampleRate) + totals[condition.name] = t + var b = bySer[bucket, default: [:]][condition.name, default: ConditionTotals()] + b.files += 1 + b.refWords += refWords.count + b.hits += m.hits + b.errors += m.errors + b.farWords += farWords.count + b.leaked += m.leaked + bySer[bucket, default: [:]][condition.name] = b + + row["\(condition.name)_recall"] = refWords.isEmpty ? 0 : Double(m.hits) / Double(refWords.count) + row["\(condition.name)_wer"] = refWords.isEmpty ? 0 : Double(m.errors) / Double(refWords.count) + row["\(condition.name)_leaked"] = m.leaked + row["\(condition.name)_hyp"] = hypWords.joined(separator: " ") + } + rows.append(row) + + if (i + 1) % 10 == 0 || i + 1 == examples.count { + let parts = conditions.map { c -> String in + let t = totals[c.name] ?? ConditionTotals() + return String(format: "%@ R=%.1f%% L=%.1f%%", c.name, t.recall * 100, t.leakage * 100) + } + report("[\(i + 1)/\(examples.count)] " + parts.joined(separator: " | ")) + } + } + + report("") + report(row(["condition", "files", "recall", "WER", "leakage", "RTFx"])) + for condition in conditions { + let t = totals[condition.name] ?? ConditionTotals() + report( + row([ + condition.name, "\(t.files)", pct(t.recall), pct(t.wer), pct(t.leakage), + condition.manager == nil ? "-" : String(format: "%.1fx", t.rtfx), + ])) + } + for bucket in ["ser<=0", "ser>0", "ser=?"] { + guard let perCondition = bySer[bucket] else { continue } + report("") + report("SER bucket \(bucket):") + for condition in conditions { + let t = perCondition[condition.name] ?? ConditionTotals() + report(row([" " + condition.name, "\(t.files)", pct(t.recall), pct(t.wer), pct(t.leakage), ""])) + } + } + + if let outputPath = options.outputPath { + var summary: [String: Any] = [:] + for (name, t) in totals { + summary[name] = [ + "files": t.files, "recall": t.recall, "wer": t.wer, "leakage": t.leakage, "rtfx": t.rtfx, + ] + } + let payload: [String: Any] = [ + "dataset": datasetDir.path, "chunk": options.chunk.rawValue, "summary": summary, "files": rows, + ] + let data = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) + try data.write(to: URL(fileURLWithPath: outputPath)) + report("Wrote \(outputPath)") + } + } catch { + logger.error("enhance-benchmark failed: \(error)") + exit(1) + } + } + + // MARK: - Scoring + + private static func transcribe(_ asr: AsrManager, _ samples: [Float]) async throws -> String { + var state = TdtDecoderState.make(decoderLayers: await asr.decoderLayerCount) + return try await asr.transcribe(samples, decoderState: &state).text + } + + private static func words(_ text: String) -> [String] { + TextNormalizer.normalize(text).split(whereSeparator: { $0.isWhitespace }).map(String.init) + } + + /// hits = reference words kept by the hypothesis (N - deletions - substitutions); + /// errors = S + D + I; leaked = far-end words present in the hypothesis beyond + /// what the near-end reference accounts for (multiset). + private static func score( + hypothesis: [String], reference: [String], farEnd: [String] + ) -> (hits: Int, errors: Int, leaked: Int) { + let m = WERCalculator.calculateWERMetrics( + hypothesis: hypothesis.joined(separator: " "), reference: reference.joined(separator: " ")) + let hits = max(0, m.totalWords - m.deletions - m.substitutions) + let errors = m.insertions + m.deletions + m.substitutions + + var spare = [String: Int]() + for w in hypothesis { spare[w, default: 0] += 1 } + for w in reference where spare[w, default: 0] > 0 { spare[w]! -= 1 } + var leaked = 0 + for w in farEnd where spare[w, default: 0] > 0 { + spare[w]! -= 1 + leaked += 1 + } + return (hits, errors, leaked) + } + + private static func pct(_ value: Double) -> String { + String(format: "%.2f%%", value * 100) + } + + /// Fixed-width table row: first column left-aligned, the rest right-aligned. + private static func row(_ cells: [String]) -> String { + let widths = [24, 6, 9, 9, 9, 8] + return cells.enumerated().map { i, cell in + let w = widths[min(i, widths.count - 1)] + let pad = String(repeating: " ", count: max(0, w - cell.count)) + return i == 0 ? cell + pad : pad + cell + }.joined(separator: " ") + } + + private static func serBucket(_ ser: Int?) -> String { + guard let ser else { return "ser=?" } + return ser <= 0 ? "ser<=0" : "ser>0" + } + + // MARK: - Dataset + + private static func resolveDataset(_ override: String?) async throws -> URL { + if let override { + return URL(fileURLWithPath: override) + } + let base = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/FluidAudio/Datasets", isDirectory: true) + let dir = base.appendingPathComponent(datasetFolder, isDirectory: true) + if FileManager.default.fileExists(atPath: dir.appendingPathComponent("meta.csv").path) { + return dir + } + try FileManager.default.createDirectory(at: base, withIntermediateDirectories: true) + let url = try ModelRegistry.resolveDataset(datasetRepo, datasetArchive) + report("Downloading \(url.absoluteString)") + let (tmp, response) = try await URLSession.shared.download(from: url) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LocalVqeError.modelProcessingFailed("dataset download failed: \(response)") + } + let archive = base.appendingPathComponent(datasetArchive) + try? FileManager.default.removeItem(at: archive) + try FileManager.default.moveItem(at: tmp, to: archive) + let tar = Process() + tar.executableURL = URL(fileURLWithPath: "/usr/bin/tar") + tar.arguments = ["-xzf", archive.path, "-C", base.path] + try tar.run() + tar.waitUntilExit() + try? FileManager.default.removeItem(at: archive) + guard tar.terminationStatus == 0, + FileManager.default.fileExists(atPath: dir.appendingPathComponent("meta.csv").path) + else { + throw LocalVqeError.modelProcessingFailed("dataset extraction failed (tar status \(tar.terminationStatus))") + } + return dir + } + + private static func loadExamples(from dir: URL) throws -> [Example] { + let metaURL = dir.appendingPathComponent("meta.csv") + let text = try String(contentsOf: metaURL, encoding: .utf8) + var lines = text.split(whereSeparator: { $0 == "\n" || $0 == "\r\n" }).map(String.init) + guard !lines.isEmpty else { return [] } + let header = lines.removeFirst().split(separator: ",").map { String($0).trimmingCharacters(in: .whitespaces) } + func column(_ name: String, _ fields: [String]) -> String? { + guard let i = header.firstIndex(of: name), i < fields.count else { return nil } + let v = fields[i].trimmingCharacters(in: .whitespaces) + return v.isEmpty ? nil : v + } + var examples: [Example] = [] + for line in lines { + let fields = line.split(separator: ",", omittingEmptySubsequences: false).map(String.init) + guard let fileID = column("fileid", fields) else { continue } + let stem = "fileid_\(fileID)" + let mic = dir.appendingPathComponent("\(stem)_mic.wav") + let lpb = dir.appendingPathComponent("\(stem)_lpb.wav") + let clean = dir.appendingPathComponent("\(stem)_clean.wav") + guard [mic, lpb, clean].allSatisfy({ FileManager.default.fileExists(atPath: $0.path) }) else { continue } + examples.append( + Example( + fileID: fileID, mic: mic, lpb: lpb, clean: clean, + ser: column("ser", fields).flatMap(Int.init), + nearendNoisy: column("is_nearend_noisy", fields) == "1")) + } + return examples.sorted { ($0.ser ?? 0, $0.fileID) < ($1.ser ?? 0, $1.fileID) } + } + + private static func next(_ arguments: [String], _ index: inout Int) -> String? { + guard index + 1 < arguments.count else { return nil } + index += 1 + return arguments[index] + } + + private static func report(_ line: String) { + print(line) + logger.info("\(line)") + } + + private static func printUsage() { + logger.info( + """ + Usage: fluidaudiocli enhance-benchmark [options] + + Scores LocalVQE on the Microsoft AEC-Challenge synthetic set (mic + loopback + clean near-end): + near-end word recall, WER vs the clean-near-end transcript, and far-end word leakage, all + measured with the in-repo Parakeet TDT v3 ASR. + + Options: + --dataset-dir Directory with fileid_*_{mic,lpb,clean}.wav + meta.csv + (default: auto-download \(datasetRepo)). + --max-files Score only the first n examples (sorted by SER). + --variants Comma list of v1.3,v1.2 (default both). + --chunk <256ms|16ms> Chunk export to benchmark (default 256ms). + --compute-units + --no-reference Also score each variant with a silent far end (NS-only mode). + --output Write per-file and summary results. + """ + ) + } +} +#endif diff --git a/Sources/FluidAudioCLI/FluidAudioCLI.swift b/Sources/FluidAudioCLI/FluidAudioCLI.swift index 8188d4121..7c7bd6aef 100644 --- a/Sources/FluidAudioCLI/FluidAudioCLI.swift +++ b/Sources/FluidAudioCLI/FluidAudioCLI.swift @@ -36,6 +36,8 @@ struct FluidAudioCLI { await VadAnalyzeCommand.run(arguments: Array(arguments.dropFirst(2))) case "enhance": await EnhanceCommand.run(arguments: Array(arguments.dropFirst(2))) + case "enhance-benchmark": + await EnhanceBenchmarkCommand.run(arguments: Array(arguments.dropFirst(2))) case "asr-benchmark": await ASRBenchmark.runASRBenchmark(arguments: Array(arguments.dropFirst(2))) case "unified-benchmark": @@ -133,6 +135,7 @@ struct FluidAudioCLI { vad-analyze Inspect VAD segmentation and streaming events fsmn-vad-segment Detect speech segments with FSMN-VAD (beta) enhance LocalVQE echo cancellation + noise suppression on a mic (+ reference) file (beta) + enhance-benchmark Near-end word recall / far-end leakage of LocalVQE on the AEC-Challenge synthetic set asr-benchmark Run ASR benchmark on LibriSpeech fleurs-benchmark Run multilingual ASR benchmark on FLEURS dataset transcribe Transcribe audio file using streaming ASR From c5f5796f421be28ab2c59d1f7c94380c089a8abe Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 14:04:56 -0400 Subject: [PATCH 03/14] docs(enhancement): LocalVQE AEC-Challenge blind-set quality table + GGML fidelity result AECMOS / ERLE / DNSMOS over the 800-clip ICASSP 2022 blind set for the Swift port (v1.3, v1.2, unprocessed), and the aligned same-clip control against the upstream GGML engine (identical per-scenario means, per-clip echo delta mean -0.0001). Notes the two places the upstream README table cannot be reproduced from the published weights. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 37 +++++++++++++++++++++++++-- README.md | 2 +- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 6640975dc..e5728549f 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -7,8 +7,9 @@ CPU-tuned derivative of DeepVQE (Indenbom et al., Interspeech 2023). Typical use: cleaning up call audio captured without headphones, where the mic picks up what the loudspeaker plays. -**Beta.** Verified against the upstream PyTorch and GGML engines on the -upstream double-talk demo (see [Parity](#parity)); not yet exercised inside +**Beta.** Bit-matched to the upstream PyTorch and GGML engines and scored +identically to GGML on the 800-clip AEC-Challenge blind set (see +[Quality](#quality-aec-challenge-blind-test-set)); not yet exercised inside production call pipelines. ## Inputs @@ -124,6 +125,38 @@ Upstream double-talk demo clip (10 s), Swift `LocalVqeStream` output: Streaming in 100 / 256 / 1000 / 4096-sample buffers and whole-clip processing produce the same audio to 1e-5. +## Quality: AEC-Challenge blind test set + +The upstream quality table is AECMOS on the ICASSP 2022 AEC-Challenge blind +set (800 real device recordings). The Swift port was rendered over all 800 +clips and scored with Microsoft's local AECMOS model (echo and degradation +MOS, 1–5, higher is better), blind ERLE and DNSMOS OVRL, using the +challenge's segment rules. Scripts live in the mobius repo +(`models/enhancement/localvqe/coreml/score_blind.py`). + +| Scenario | n | Unprocessed echo | v1.3 echo / deg | v1.3 ERLE | v1.2 echo / deg | v1.2 ERLE | +|---|--:|--:|---|--:|---|--:| +| doubletalk | 115 | 2.17 | 4.35 / 3.93 | – | 4.20 / 3.63 | – | +| doubletalk-with-movement | 185 | 2.21 | 4.35 / 3.86 | – | 4.13 / 3.57 | – | +| farend-singletalk | 107 | 1.95 | 2.49 / 5.00 | 54.1 dB | 3.92 / 5.00 | 45.7 dB | +| farend-singletalk-with-movement | 193 | 2.23 | 3.08 / 5.00 | 55.0 dB | 4.13 / 5.00 | 38.2 dB | +| nearend-singletalk | 200 | 5.00 | 4.99 / 4.14 | – | 4.99 / 4.09 | – | + +**Port fidelity.** The upstream GGML engine was run on the same 800 clips +and scored on identical, aligned samples: every per-scenario mean matches +the Core ML port to two decimals and the per-clip echo-MOS delta has mean +−0.0001 (95th percentile 0.02). The only differences found are artefacts of +the upstream CLI (256-sample output delay, tail truncation, and a 16-bit +writer that wraps samples above full scale); the Swift CLI writes float32. + +**Against the published table.** v1.2 matches the upstream single-talk +rows (far-end ERLE 45.7 dB vs 45.7 dB). The double-talk rows are scored on +a segment upstream did not document (our unprocessed baseline is 2.17 vs +their 2.67), and the published v1.3 far-end echo MOS is about 1 point above +what the published v1.3 weights produce under this protocol, at higher +ERLE. Treat the table above, not the upstream README, as the reference for +this port. + ## Benchmark: near-end recall / far-end leakage `fluidaudiocli enhance-benchmark` scores the enhancer with the in-repo diff --git a/README.md b/README.md index 087bffb97..e19eccc9c 100644 --- a/README.md +++ b/README.md @@ -574,7 +574,7 @@ Offline mode also reports RTFx using the model's per-chunk processing time. ## Speech Enhancement (Echo Cancellation + Noise Suppression) -> **⚠️ Beta:** verified against the upstream engines on the upstream demo clip; not yet exercised in production call pipelines. +> **⚠️ Beta:** scores identically to the upstream GGML engine on the 800-clip AEC-Challenge blind set (AECMOS); not yet exercised in production call pipelines. [LocalVQE](https://github.com/localai-org/LocalVQE) (Apache-2.0) is a compact neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz From 234438d8634d927e87c3db5ad650e7b0dd2261b6 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 15:14:23 -0400 Subject: [PATCH 04/14] =?UTF-8?q?fix(enhancement):=20LocalVQE=20review=20f?= =?UTF-8?q?ixes=20=E2=80=94=20async=20prediction,=20release=20--help,=20do?= =?UTF-8?q?c=20wording?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LocalVqeStream: use MLModel's async prediction API. Apple documents the synchronous API as not thread-safe (WWDC23 10049); streams created from one manager share its MLModel and could previously call it concurrently. enhance()/flush() become async. - enhance / enhance-benchmark: print usage via stdout so --help shows in release builds (the logger is silent there). - Docs: numerical equivalence rather than bit-match; single-talk agreement with the 38.2 vs 40.6 dB exception; double-talk mismatch stated as an unresolved upstream protocol discrepancy; note concurrent streams. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 35 ++++++++++++------- .../Enhancement/LocalVQE/LocalVqeStream.swift | 17 +++++---- .../Commands/EnhanceBenchmarkCommand.swift | 3 +- .../Commands/EnhanceCommand.swift | 3 +- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index e5728549f..64d12c638 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -7,8 +7,8 @@ CPU-tuned derivative of DeepVQE (Indenbom et al., Interspeech 2023). Typical use: cleaning up call audio captured without headphones, where the mic picks up what the loudspeaker plays. -**Beta.** Bit-matched to the upstream PyTorch and GGML engines and scored -identically to GGML on the 800-clip AEC-Challenge blind set (see +**Beta.** Numerically equivalent to the upstream PyTorch and GGML engines and +scored identically to GGML on the 800-clip AEC-Challenge blind set (see [Quality](#quality-aec-challenge-blind-test-set)); not yet exercised inside production call pipelines. @@ -49,6 +49,9 @@ let out = try await stream.enhance(mic: micBuffer, reference: refBuffer) let tail = try await stream.flush() ``` +Streams from one manager share its model and may run concurrently: inference +uses Core ML's async prediction API, which Apple documents as thread-safe. + `enhance` returns samples as whole model calls complete. Output sample `i` corresponds to input sample `i`, delivered one hop (256 samples, 16 ms) after the input that produced it plus whatever is still buffered toward the @@ -143,17 +146,23 @@ challenge's segment rules. Scripts live in the mobius repo | nearend-singletalk | 200 | 5.00 | 4.99 / 4.14 | – | 4.99 / 4.09 | – | **Port fidelity.** The upstream GGML engine was run on the same 800 clips -and scored on identical, aligned samples: every per-scenario mean matches -the Core ML port to two decimals and the per-clip echo-MOS delta has mean -−0.0001 (95th percentile 0.02). The only differences found are artefacts of -the upstream CLI (256-sample output delay, tail truncation, and a 16-bit -writer that wraps samples above full scale); the Swift CLI writes float32. - -**Against the published table.** v1.2 matches the upstream single-talk -rows (far-end ERLE 45.7 dB vs 45.7 dB). The double-talk rows are scored on -a segment upstream did not document (our unprocessed baseline is 2.17 vs -their 2.67), and the published v1.3 far-end echo MOS is about 1 point above -what the published v1.3 weights produce under this protocol, at higher +and scored on identical, aligned whole-hop samples: every per-scenario mean +matches the Core ML port to two decimals, the per-clip echo-MOS delta has +mean +0.0002 (95th percentile 0.017), degradation-MOS 95th percentile 0.0004, +and the aligned waveforms agree at a median 84 dB SNR (numerically +equivalent within 16-bit quantisation, not bit-identical). The only +differences found are artefacts of the upstream CLI (256-sample output +delay, zero-filled trailing hop, and a 16-bit writer that wraps samples above +full scale); the Swift CLI writes float32. + +**Against the published table.** v1.2 agrees with the upstream single-talk +rows to within about 0.15 MOS and reproduces the far-end ERLE exactly +(45.7 dB), with one exception (with-movement ERLE 38.2 dB vs 40.6 dB +published). The double-talk rows disagree for every model including the +unprocessed baseline (2.17 vs 2.67), and no segment rule tried reproduces +them: an undocumented evaluation-protocol difference, cause not established. +The published v1.3 far-end echo MOS is about 1 point above what the +published v1.3 weights produce under the documented protocol, at higher ERLE. Treat the table above, not the upstream README, as the reference for this port. diff --git a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift index c45b91c01..8d88a2c95 100644 --- a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift +++ b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift @@ -18,6 +18,11 @@ import OSLog /// tensors; the stream just passes each call's outputs back in as the next /// call's inputs. Create one stream per audio channel pair; a stream is not /// reusable across unrelated clips without `reset()`. +/// +/// Streams created from one `LocalVqeManager` share its `MLModel`. Inference +/// goes through Core ML's async prediction API, which Apple documents as +/// thread-safe (WWDC23 10049), so independent streams may run concurrently; +/// the synchronous API would require serializing every call on the model. public actor LocalVqeStream { private static let logger = AppLogger(category: "LocalVqeStream") @@ -114,7 +119,7 @@ public actor LocalVqeStream { /// `mic` and `reference` must have the same length; both may be any /// length (including zero) — partial calls are buffered until enough /// samples arrive. - public func enhance(mic: [Float], reference: [Float]) throws -> [Float] { + public func enhance(mic: [Float], reference: [Float]) async throws -> [Float] { guard mic.count == reference.count else { throw LocalVqeError.lengthMismatch(mic: mic.count, reference: reference.count) } @@ -125,7 +130,7 @@ public actor LocalVqeStream { var out: [Float] = [] var offset = 0 while pendingMic.count - offset >= samplesPerCall { - let hop = try runCall( + let hop = try await runCall( mic: pendingMic[offset.. [Float] { + public func flush() async throws -> [Float] { let outstanding = samplesIn - samplesOut guard outstanding > 0 else { try reset() @@ -157,7 +162,7 @@ public actor LocalVqeStream { var out: [Float] = [] var offset = 0 while pendingMic.count - offset >= samplesPerCall { - let hop = try runCall( + let hop = try await runCall( mic: pendingMic[offset.., reference: ArraySlice) throws -> [Float] { + private func runCall(mic: ArraySlice, reference: ArraySlice) async throws -> [Float] { micInput.withUnsafeMutableBufferPointer(ofType: Float.self) { buf, _ in _ = buf.initialize(from: mic) } @@ -196,7 +201,7 @@ public actor LocalVqeStream { let output: MLFeatureProvider do { let provider = try MLDictionaryFeatureProvider(dictionary: features) - output = try model.prediction(from: provider) + output = try await model.compatPrediction(from: provider, options: MLPredictionOptions()) } catch { throw LocalVqeError.modelProcessingFailed(error.localizedDescription) } diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift index 635da76a7..69089d160 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -367,7 +367,8 @@ enum EnhanceBenchmarkCommand { } private static func printUsage() { - logger.info( + // print, not the logger: usage must show in release builds too. + print( """ Usage: fluidaudiocli enhance-benchmark [options] diff --git a/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift index b98130970..c15bf8c1f 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceCommand.swift @@ -227,7 +227,8 @@ enum EnhanceCommand { } private static func printUsage() { - logger.info( + // print, not the logger: usage must show in release builds too. + print( """ Usage: fluidaudiocli enhance [options] From 96cfc581c04924f89d68e18bc4e4955bba6ccbc0 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 15:30:55 -0400 Subject: [PATCH 05/14] fix(cli): WERCalculator insertion/deletion labels were swapped; correct enhance-benchmark recall editDistance(hyp, ref) labelled a reference word missing from the hypothesis as an insertion and an extra hypothesis word as a deletion. WER was unaffected (it sums all three) but the breakdown was wrong everywhere it is read: enhance-benchmark's recall (N - D - S)/N awarded 100% to an empty transcript, and canary-transcribe / tts-asr-verify printed S/D/I swapped. Corrected recall on the AEC-Challenge synthetic subset (167 scored, the 33 examples with an empty clean-near-end transcript are now excluded and reported): unprocessed 44.1% -> v1.3 77.6% / v1.2 73.0%; previously published as 39.5% -> 87.5% / 86.3%. Leakage 34.0% -> 1.1%. Docs relabel the subset as exploratory (first 200 of a training shard, machine-transcript references). Adds WERCalculatorTests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 36 ++++++++----- README.md | 6 +-- .../Commands/EnhanceBenchmarkCommand.swift | 11 ++++ .../FluidAudioCLI/Utils/WERCalculator.swift | 12 +++-- .../CLI/WERCalculatorTests.swift | 54 +++++++++++++++++++ 5 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 Tests/FluidAudioTests/CLI/WERCalculatorTests.swift diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 64d12c638..8848b7a1c 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -168,12 +168,17 @@ this port. ## Benchmark: near-end recall / far-end leakage -`fluidaudiocli enhance-benchmark` scores the enhancer with the in-repo -Parakeet TDT v3 ASR on the Microsoft AEC-Challenge synthetic set (mic + -loopback + clean near-end triples; 200-example subset at +**Exploratory.** `fluidaudiocli enhance-benchmark` scores the enhancer with +the in-repo Parakeet TDT v3 ASR on a 200-example subset (the first 200 of +shard 0) of the Microsoft AEC-Challenge synthetic *training* set (mic + +loopback + clean near-end triples; [FluidInference/aec-challenge-synthetic-mini](https://huggingface.co/datasets/FluidInference/aec-challenge-synthetic-mini), auto-downloaded). The ASR transcript of the clean near-end clip is the -reference; the loopback transcript gives the far-end words. +reference and the loopback transcript gives the far-end words, so the +metrics are relative to machine transcripts, not human ones; examples whose +clean-near-end transcript is empty (33 of 200) are excluded and reported. +Use it to compare conditions, not as an absolute quality figure; the +AEC-Challenge blind-set table above is the quality reference. - **Recall**: reference words kept by the hypothesis, `1 - (D + S) / N`. - **WER**: `(S + D + I) / N` against the clean-near-end transcript. Above @@ -181,20 +186,25 @@ reference; the loopback transcript gives the far-end words. - **Leakage**: far-end words that appear in the hypothesis without being near-end words, over the far-end word count. -200 examples, signal-to-echo ratio (SER) −10…+10 dB, M5 Pro, 256 ms chunk, CPU: +167 scored examples, signal-to-echo ratio (SER) −10…+10 dB, M5 Pro, 256 ms +chunk, CPU: | Condition | Recall | WER | Leakage | RTFx | |---|---:|---:|---:|---:| -| Unprocessed mic | 39.5% | 134.2% | 33.8% | – | -| LocalVQE v1.3 | **87.5%** | 43.4% | **1.8%** | 36× | -| LocalVQE v1.2 | 86.3% | 49.4% | 1.9% | 62× | -| v1.3, silent reference (NS only) | 45.3% | 122.6% | 24.4% | 36× | - -By SER: at SER ≤ 0 dB (echo louder than speech, 110 files) v1.3 lifts recall -32.0% → 87.0% and cuts leakage 41.5% → 2.3%; at SER > 0 dB (90 files) -49.5% → 88.3% and 25.2% → 1.3%. The silent-reference row shows the model +| Unprocessed mic | 44.1% | 112.2% | 34.0% | – | +| LocalVQE v1.3 | **77.6%** | 29.6% | **1.1%** | 34× | +| LocalVQE v1.2 | 73.0% | 33.9% | 1.0% | 57× | +| v1.3, silent reference (NS only) | 42.3% | 101.8% | 23.9% | 34× | + +By SER: at SER ≤ 0 dB (echo louder than speech, 93 files) v1.3 lifts recall +36.4% → 74.5% and cuts leakage 41.1% → 1.6%; at SER > 0 dB (74 files) +54.4% → 81.7% and 25.9% → 0.5%. The silent-reference row shows the model needs the loopback to cancel echo; without it, it only denoises. +An earlier revision of this table reported 87.5% / 86.3% recall: the shared +WER scorer had its insertion/deletion labels swapped, so an empty hypothesis +scored 100% recall. Fixed in `WERCalculator` (WER itself was unaffected). + ```bash swift run -c release fluidaudiocli enhance-benchmark # both variants, 200 files swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json diff --git a/README.md b/README.md index e19eccc9c..65f92a107 100644 --- a/README.md +++ b/README.md @@ -581,9 +581,9 @@ neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz speech. Feed it the mic capture and a far-end reference (what the speaker played) and it returns clean near-end speech, sample-aligned with the input. Two checkpoints (v1.3 4.8M, v1.2 1.3M) in 256 ms and 16 ms chunk exports; -36× / 14× real-time on CPU for v1.3. On the AEC-Challenge synthetic set v1.3 -lifts ASR near-end word recall from 39.5% to 87.5% and cuts far-end word -leakage from 33.8% to 1.8%. See +36× / 14× real-time on CPU for v1.3. On an exploratory AEC-Challenge synthetic +subset v1.3 lifts ASR near-end word recall from 44.1% to 77.6% and cuts +far-end word leakage from 34.0% to 1.1%. See [Documentation/Enhancement/LocalVQE.md](Documentation/Enhancement/LocalVQE.md). ```swift diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift index 69089d160..e6dedce54 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -132,6 +132,7 @@ enum EnhanceBenchmarkCommand { var totals = [String: ConditionTotals]() var bySer = [String: [String: ConditionTotals]]() // bucket -> condition -> totals var rows: [[String: Any]] = [] + var emptyReferences = 0 for (i, example) in examples.enumerated() { let mic = try converter.resampleAudioFile(example.mic) @@ -144,6 +145,12 @@ enum EnhanceBenchmarkCommand { } let refWords = words(try await transcribe(asr, clean)) + if refWords.isEmpty { + // No reference words to recall: the ASR produced nothing on the + // clean near-end clip. Excluded from every metric and counted. + emptyReferences += 1 + continue + } let farWords = words(try await transcribe(asr, lpb)) let bucket = serBucket(example.ser) var row: [String: Any] = [ @@ -199,6 +206,9 @@ enum EnhanceBenchmarkCommand { } report("") + if emptyReferences > 0 { + report("Excluded \(emptyReferences) examples whose clean near-end transcript was empty.") + } report(row(["condition", "files", "recall", "WER", "leakage", "RTFx"])) for condition in conditions { let t = totals[condition.name] ?? ConditionTotals() @@ -227,6 +237,7 @@ enum EnhanceBenchmarkCommand { } let payload: [String: Any] = [ "dataset": datasetDir.path, "chunk": options.chunk.rawValue, "summary": summary, "files": rows, + "excluded_empty_reference": emptyReferences, ] let data = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) try data.write(to: URL(fileURLWithPath: outputPath)) diff --git a/Sources/FluidAudioCLI/Utils/WERCalculator.swift b/Sources/FluidAudioCLI/Utils/WERCalculator.swift index a7d045d16..771f001c0 100644 --- a/Sources/FluidAudioCLI/Utils/WERCalculator.swift +++ b/Sources/FluidAudioCLI/Utils/WERCalculator.swift @@ -179,11 +179,13 @@ enum WERCalculator { let m = seq1.count let n = seq2.count + // seq1 is the hypothesis, seq2 the reference: an empty hypothesis is n + // deletions (reference words missing), an empty reference m insertions. if m == 0 { - return EditDistanceResult(total: n, insertions: n, deletions: 0, substitutions: 0) + return EditDistanceResult(total: n, insertions: 0, deletions: n, substitutions: 0) } if n == 0 { - return EditDistanceResult(total: m, insertions: 0, deletions: m, substitutions: 0) + return EditDistanceResult(total: m, insertions: m, deletions: 0, substitutions: 0) } var dp = Array(repeating: Array(repeating: 0, count: n + 1), count: m + 1) @@ -220,10 +222,12 @@ enum WERCalculator { i -= 1 j -= 1 } else if i > 0 && dp[i][j] == dp[i - 1][j] + 1 { - deletions += 1 + // hypothesis word with no reference counterpart + insertions += 1 i -= 1 } else if j > 0 && dp[i][j] == dp[i][j - 1] + 1 { - insertions += 1 + // reference word missing from the hypothesis + deletions += 1 j -= 1 } else { break diff --git a/Tests/FluidAudioTests/CLI/WERCalculatorTests.swift b/Tests/FluidAudioTests/CLI/WERCalculatorTests.swift new file mode 100644 index 000000000..cb6c4683b --- /dev/null +++ b/Tests/FluidAudioTests/CLI/WERCalculatorTests.swift @@ -0,0 +1,54 @@ +import XCTest + +@testable import FluidAudioCLI + +/// Insertion/deletion labelling of the shared WER scorer. +/// +/// The backtrace used to label a reference word missing from the hypothesis +/// as an "insertion" and an extra hypothesis word as a "deletion" (swapped). +/// WER was unaffected (it sums all three) but any consumer that reads the +/// breakdown — the recall metric in `enhance-benchmark`, the S/D/I lines in +/// `canary-transcribe` and `tts-asr-verify` — was wrong; an empty hypothesis +/// scored 100% recall. +final class WERCalculatorTests: XCTestCase { + + func testEmptyHypothesisIsAllDeletions() { + let m = WERCalculator.calculateWERMetrics(hypothesis: "", reference: "one two three") + XCTAssertEqual(m.totalWords, 3) + XCTAssertEqual(m.deletions, 3) + XCTAssertEqual(m.insertions, 0) + XCTAssertEqual(m.substitutions, 0) + XCTAssertEqual(m.wer, 1.0, accuracy: 1e-9) + } + + func testEmptyReferenceIsAllInsertions() { + let m = WERCalculator.calculateWERMetrics(hypothesis: "one two", reference: "") + XCTAssertEqual(m.totalWords, 0) + XCTAssertEqual(m.insertions, 2) + XCTAssertEqual(m.deletions, 0) + } + + func testMixedEdits() { + // ref: a b c d ; hyp: a x c d e -> 1 substitution (b->x), 1 insertion (e) + let m = WERCalculator.calculateWERMetrics(hypothesis: "a x c d e", reference: "a b c d") + XCTAssertEqual(m.substitutions, 1) + XCTAssertEqual(m.insertions, 1) + XCTAssertEqual(m.deletions, 0) + XCTAssertEqual(m.wer, 0.5, accuracy: 1e-9) + + // ref: a b c d ; hyp: a c -> 2 deletions + let d = WERCalculator.calculateWERMetrics(hypothesis: "a c", reference: "a b c d") + XCTAssertEqual(d.deletions, 2) + XCTAssertEqual(d.insertions, 0) + XCTAssertEqual(d.substitutions, 0) + } + + func testRecallFromBreakdown() { + // Recall as enhance-benchmark computes it: (N - D - S) / N. + let m = WERCalculator.calculateWERMetrics(hypothesis: "a c z", reference: "a b c d") + let recall = Double(m.totalWords - m.deletions - m.substitutions) / Double(m.totalWords) + XCTAssertEqual(recall, 0.5, accuracy: 1e-9) + let empty = WERCalculator.calculateWERMetrics(hypothesis: "", reference: "a b c d") + XCTAssertEqual(Double(empty.totalWords - empty.deletions - empty.substitutions), 0) + } +} From 0e029d8ce211322be8307b0a8777badf05299a0e Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 16:07:50 -0400 Subject: [PATCH 06/14] docs(enhancement): LocalVQE quality under both protocols; retract the 'not from published weights' claim The HF model-card protocol (legacy AECMOS model, first 20 s; rated-segment DNSMOS; technical-report gated ERLE) reproduces its unprocessed baseline exactly and every doubletalk / near-end cell within 0.02. Reference table stays on the challenge protocol; v1.2 far-end echo rows remain unexplained and are reported as such. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 59 +++++++++++++++++---------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 8848b7a1c..90574e23c 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -132,18 +132,46 @@ processing produce the same audio to 1e-5. The upstream quality table is AECMOS on the ICASSP 2022 AEC-Challenge blind set (800 real device recordings). The Swift port was rendered over all 800 -clips and scored with Microsoft's local AECMOS model (echo and degradation -MOS, 1–5, higher is better), blind ERLE and DNSMOS OVRL, using the -challenge's segment rules. Scripts live in the mobius repo +clips and scored two ways, kept separate because they answer different +questions. Scripts and per-clip results live in the mobius repo (`models/enhancement/localvqe/coreml/score_blind.py`). -| Scenario | n | Unprocessed echo | v1.3 echo / deg | v1.3 ERLE | v1.2 echo / deg | v1.2 ERLE | -|---|--:|--:|---|--:|---|--:| -| doubletalk | 115 | 2.17 | 4.35 / 3.93 | – | 4.20 / 3.63 | – | -| doubletalk-with-movement | 185 | 2.21 | 4.35 / 3.86 | – | 4.13 / 3.57 | – | -| farend-singletalk | 107 | 1.95 | 2.49 / 5.00 | 54.1 dB | 3.92 / 5.00 | 45.7 dB | -| farend-singletalk-with-movement | 193 | 2.23 | 3.08 / 5.00 | 55.0 dB | 4.13 / 5.00 | 38.2 dB | -| nearend-singletalk | 200 | 5.00 | 4.99 / 4.14 | – | 4.99 / 4.09 | – | +**Challenge protocol (reference).** Microsoft's current scenario-aware AECMOS +model with the challenge's segment rules (convergence portions excluded), +DNSMOS OVRL on the same rated segment, and blind ERLE with the LocalVQE +technical-report gating (far-end-dominated frames only). AECMOS is 1–5, +higher is better. + +| Scenario | n | Unprocessed echo | v1.3 echo / deg / ERLE / OVRL | v1.2 echo / deg / ERLE / OVRL | +|---|--:|--:|---|---| +| doubletalk | 115 | 2.17 | 4.35 / 3.93 / 6.3 dB / 2.89 | 4.20 / 3.63 / 6.2 dB / 2.77 | +| doubletalk-with-movement | 185 | 2.21 | 4.35 / 3.86 / 6.1 dB / 2.84 | 4.13 / 3.57 / 6.0 dB / 2.73 | +| farend-singletalk | 107 | 1.95 | 2.49 / 5.00 / 54.2 dB / 1.95 | 3.92 / 5.00 / 53.2 dB / 1.89 | +| farend-singletalk-with-movement | 193 | 2.23 | 3.08 / 5.00 / 55.9 dB / 1.96 | 4.13 / 5.00 / 47.3 dB / 1.80 | +| nearend-singletalk | 200 | 5.00 | 4.99 / 4.14 / 2.3 dB / 3.17 | 4.99 / 4.09 / 2.1 dB / 3.17 | + +**Upstream protocol (HF model-card reproduction).** The published table was +produced with the legacy AECMOS model over the first 20 s of each clip; that +protocol reproduces the card's unprocessed baseline exactly +(2.67 / 2.56 / 1.90 / 2.13 / 5.00). Under it, the Core ML port gives: + +| Scenario | HF card v1.3 | Core ML v1.3 | HF card v1.2 | Core ML v1.2 | +|---|---|---|---|---| +| doubletalk | 4.73 / 2.62 | 4.73 / 2.62 | 4.72 / 2.37 | 4.72 / 2.39 | +| doubletalk-with-movement | 4.67 / 2.43 | 4.66 / 2.44 | 4.65 / 2.30 | 4.64 / 2.31 | +| farend-singletalk | 3.69 / 4.83 | 3.54 / 4.82 | 3.78 / 4.91 | 4.07 / 4.93 | +| farend-singletalk-with-movement | 3.88 / 4.98 | 3.75 / 4.96 | 4.12 / 4.96 | 4.27 / 4.96 | +| nearend-singletalk | 5.00 / 4.18 | 5.00 / 4.18 | 5.00 / 4.16 | 5.00 / 4.17 | + +Every double-talk and near-end cell reproduces within 0.02; ERLE within +about 1 dB and OVRL within 0.06 (full columns in the mobius README). The +v1.3 far-end cells are 0.15 low from aligned float output and within 0.04 +when the upstream CLI's raw 16-bit, one-hop-late output is scored instead. +The v1.2 far-end echo rows are not reproduced from either runtime (ours +score 0.2–0.4 higher); the private upstream scoring script is not public, so +exact reproduction of every cell is not established. An earlier revision of +this page said the v1.3 far-end row could not have come from the published +weights; that was a protocol mismatch and is retracted. **Port fidelity.** The upstream GGML engine was run on the same 800 clips and scored on identical, aligned whole-hop samples: every per-scenario mean @@ -155,17 +183,6 @@ differences found are artefacts of the upstream CLI (256-sample output delay, zero-filled trailing hop, and a 16-bit writer that wraps samples above full scale); the Swift CLI writes float32. -**Against the published table.** v1.2 agrees with the upstream single-talk -rows to within about 0.15 MOS and reproduces the far-end ERLE exactly -(45.7 dB), with one exception (with-movement ERLE 38.2 dB vs 40.6 dB -published). The double-talk rows disagree for every model including the -unprocessed baseline (2.17 vs 2.67), and no segment rule tried reproduces -them: an undocumented evaluation-protocol difference, cause not established. -The published v1.3 far-end echo MOS is about 1 point above what the -published v1.3 weights produce under the documented protocol, at higher -ERLE. Treat the table above, not the upstream README, as the reference for -this port. - ## Benchmark: near-end recall / far-end leakage **Exploratory.** `fluidaudiocli enhance-benchmark` scores the enhancer with From 07e5f0c9f430499660bd8933fe7a080f61cf2a43 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 16:29:29 -0400 Subject: [PATCH 07/14] docs(enhancement): LocalVQE upstream-table agreement stated per metric v1.2 far-end differs on echo, gated ERLE and OVRL, not only echo MOS; v1.3 numbers stated with their actual bounds. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 90574e23c..c858fc589 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -163,13 +163,15 @@ protocol reproduces the card's unprocessed baseline exactly | farend-singletalk-with-movement | 3.88 / 4.98 | 3.75 / 4.96 | 4.12 / 4.96 | 4.27 / 4.96 | | nearend-singletalk | 5.00 / 4.18 | 5.00 / 4.18 | 5.00 / 4.16 | 5.00 / 4.17 | -Every double-talk and near-end cell reproduces within 0.02; ERLE within -about 1 dB and OVRL within 0.06 (full columns in the mobius README). The -v1.3 far-end cells are 0.15 low from aligned float output and within 0.04 -when the upstream CLI's raw 16-bit, one-hop-late output is scored instead. -The v1.2 far-end echo rows are not reproduced from either runtime (ours -score 0.2–0.4 higher); the private upstream scoring script is not public, so -exact reproduction of every cell is not established. An earlier revision of +Unprocessed baseline: exact. v1.3: double-talk and near-end within 0.01 +echo MOS; far-end 0.15 low from aligned float output and within 0.04 when +the upstream CLI's raw 16-bit, one-hop-late output is scored instead; gated +ERLE within 0.8 dB and OVRL within 0.01 (full columns in the mobius +README). v1.2: double-talk and near-end within 0.02 echo, 0.02 deg, 0.1 dB +ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo ++0.29 / +0.15, gated ERLE +1.9 / +0.7 dB, OVRL +0.09 / +0.05, from either +runtime). The private upstream scoring script is not public, so exact +reproduction of every cell is not established. An earlier revision of this page said the v1.3 far-end row could not have come from the published weights; that was a protocol mismatch and is retracted. From e8a66e15d7b5e552705258be059c2f579453b598 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 17:17:06 -0400 Subject: [PATCH 08/14] docs(enhancement): accepted validation summary wording for LocalVQE Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 17 ++++++++++++----- README.md | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index c858fc589..3971ad331 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -7,9 +7,11 @@ CPU-tuned derivative of DeepVQE (Indenbom et al., Interspeech 2023). Typical use: cleaning up call audio captured without headphones, where the mic picks up what the loudspeaker plays. -**Beta.** Numerically equivalent to the upstream PyTorch and GGML engines and -scored identically to GGML on the 800-clip AEC-Challenge blind set (see -[Quality](#quality-aec-challenge-blind-test-set)); not yet exercised inside +**Beta.** Port fidelity validated (numerically equivalent to the upstream +PyTorch and GGML engines, scored identically to GGML on the 800-clip +AEC-Challenge blind set); the published benchmark is substantially +reproduced, with unresolved v1.2 far-end differences (see +[Quality](#quality-aec-challenge-blind-test-set)). Not yet exercised inside production call pipelines. ## Inputs @@ -130,6 +132,9 @@ processing produce the same audio to 1e-5. ## Quality: AEC-Challenge blind test set +**Summary: port fidelity validated; published benchmark substantially +reproduced, with unresolved v1.2 far-end differences.** + The upstream quality table is AECMOS on the ICASSP 2022 AEC-Challenge blind set (800 real device recordings). The Swift port was rendered over all 800 clips and scored two ways, kept separate because they answer different @@ -170,8 +175,10 @@ ERLE within 0.8 dB and OVRL within 0.01 (full columns in the mobius README). v1.2: double-talk and near-end within 0.02 echo, 0.02 deg, 0.1 dB ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo +0.29 / +0.15, gated ERLE +1.9 / +0.7 dB, OVRL +0.09 / +0.05, from either -runtime). The private upstream scoring script is not public, so exact -reproduction of every cell is not established. An earlier revision of +runtime). Those values are above the published ones, which is not evidence +that the port outperforms upstream; +0.29 echo MOS is not rounding noise. +The private upstream scoring script is not public, so exact reproduction of +every cell is not established. An earlier revision of this page said the v1.3 far-end row could not have come from the published weights; that was a protocol mismatch and is retracted. diff --git a/README.md b/README.md index 65f92a107..bd0204e6d 100644 --- a/README.md +++ b/README.md @@ -574,7 +574,7 @@ Offline mode also reports RTFx using the model's per-chunk processing time. ## Speech Enhancement (Echo Cancellation + Noise Suppression) -> **⚠️ Beta:** scores identically to the upstream GGML engine on the 800-clip AEC-Challenge blind set (AECMOS); not yet exercised in production call pipelines. +> **⚠️ Beta:** port fidelity validated (scores identically to the upstream GGML engine on the 800-clip AEC-Challenge blind set); published benchmark substantially reproduced, with unresolved v1.2 far-end differences. Not yet exercised in production call pipelines. [LocalVQE](https://github.com/localai-org/LocalVQE) (Apache-2.0) is a compact neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz From 1f763f0e973c4ae10092cb38839595a526e180af Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Fri, 18 Sep 2026 18:42:27 -0400 Subject: [PATCH 09/14] docs(enhancement): LocalVQE v1.2 far-end investigation outcome Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015S4u7dmbe4skFjUNMpodud --- Documentation/Enhancement/LocalVQE.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 3971ad331..5f7a9ed2c 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -177,8 +177,13 @@ ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo +0.29 / +0.15, gated ERLE +1.9 / +0.7 dB, OVRL +0.09 / +0.05, from either runtime). Those values are above the published ones, which is not evidence that the port outperforms upstream; +0.29 echo MOS is not rounding noise. -The private upstream scoring script is not public, so exact reproduction of -every cell is not established. An earlier revision of +Rendering v1.2 at the pre-v1.2 delay window (dmax 32, which the reference +config left on the day that row was published) reproduces the card's +far-end ERLE (44.9 / 40.5 dB) and deg (4.88 / 4.96) but not its echo MOS; +softmax temperature 1.0, the ReLU6 reference, the upstream CLI's output +format and every scorer/segment variation were also tested and rejected +(details in the mobius README). The private upstream scoring script is not +public, so those two cells remain unexplained. An earlier revision of this page said the v1.3 far-end row could not have come from the published weights; that was a protocol mismatch and is retracted. From 204414bcfeba5074c0bbf9bafaaf00f5a2b52fcf Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sat, 19 Sep 2026 12:17:44 -0400 Subject: [PATCH 10/14] fix(enhancement): serialize LocalVQE operations and document validation --- Documentation/Enhancement/LocalVQE.md | 54 +++++- .../Enhancement/LocalVQEValidation.md | 77 +++++++++ .../Enhancement/LocalVQE/LocalVqeStream.swift | 101 ++++++++++- .../Enhancement/LocalVqeTests.swift | 163 +++++++++++++++++- 4 files changed, 381 insertions(+), 14 deletions(-) create mode 100644 Documentation/Enhancement/LocalVQEValidation.md diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 5f7a9ed2c..b27460efc 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -53,6 +53,11 @@ let tail = try await stream.flush() Streams from one manager share its model and may run concurrently: inference uses Core ML's async prediction API, which Apple documents as thread-safe. +Within each stream, `enhance`, `flush`, and `reset` execute in the order they +reach the actor, including across inference suspension points. Feed a stream +from one consumer task and await each push before submitting the next: tasks +launched independently from audio callbacks can arrive out of capture order +and accumulate queued audio. Run enhancement outside the audio render callback. `enhance` returns samples as whole model calls complete. Output sample `i` corresponds to input sample `i`, delivered one hop (256 samples, 16 ms) @@ -60,6 +65,35 @@ after the input that produced it plus whatever is still buffered toward the next call. `flush()` resets the stream; call `reset()` to start a new clip without flushing. +Cancelling an operation while it is queued removes it without changing the +current clip. Cancelling a running push/flush, or an inference error, discards +the unfinished clip and clears recurrent state before the next operation. +Treat this as an audio discontinuity. `reset()` waits for earlier operations; +to abandon an active push, cancel its task, await its completion, then resume +with the next clip. A cancelled queued reset does not reset the stream. + +For live capture, supply continuous 16 kHz mono mic/reference buffers covering +the same time intervals. The reference must be the actual far-end playback +signal. Equal buffer lengths alone do not establish correct timing. Reset on +capture/playback discontinuities, and validate reference timing, route changes, +sustained latency and recovery in the application on its target devices before +enabling enhancement by default. The offline benchmark does not exercise that +live integration. + +Real-model streaming regression tests can be enabled locally or in CI with: + +```bash +FLUIDAUDIO_LOCALVQE_MODEL_DIR=/path/to/compiled/models swift test --filter LocalVqe +``` + +The explicit directory must contain both v1.3 chunk variants; a missing bundle +fails the tests. Without that setting, model tests skip in CI or when the local +cache is absent. The suite covers overlapping pushes, flush/reset ordering, +queued and active cancellation, fresh-clip recovery, and independent streams. +See the [streaming validation report](LocalVQEValidation.md) for the completed +real-model checks, the local XCTest environment limitation and remaining +live-integration coverage. + ## Configuration ```swift @@ -156,7 +190,7 @@ higher is better. | nearend-singletalk | 200 | 5.00 | 4.99 / 4.14 / 2.3 dB / 3.17 | 4.99 / 4.09 / 2.1 dB / 3.17 | **Upstream protocol (HF model-card reproduction).** The published table was -produced with the legacy AECMOS model over the first 20 s of each clip; that +compared using the legacy AECMOS model over the first 20 s of each clip; that protocol reproduces the card's unprocessed baseline exactly (2.67 / 2.56 / 1.90 / 2.13 / 5.00). Under it, the Core ML port gives: @@ -178,15 +212,25 @@ ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo runtime). Those values are above the published ones, which is not evidence that the port outperforms upstream; +0.29 echo MOS is not rounding noise. Rendering v1.2 at the pre-v1.2 delay window (dmax 32, which the reference -config left on the day that row was published) reproduces the card's -far-end ERLE (44.9 / 40.5 dB) and deg (4.88 / 4.96) but not its echo MOS; +config left on the day that row was published) brings far-end ERLE +(44.9 / 40.5 vs 45.7 / 40.6 dB) and degradation (4.88 / 4.96 vs 4.91 / 4.96) +close to the card, but does not establish which configuration upstream used. +Its echo MOS moves farther from the card; softmax temperature 1.0, the ReLU6 reference, the upstream CLI's output format and every scorer/segment variation were also tested and rejected -(details in the mobius README). The private upstream scoring script is not -public, so those two cells remain unexplained. An earlier revision of +(details in the mobius README). None of the tested configurations reproduces +the whole table. Upstream's evaluation configuration, rendered audio or +scoring script would help resolve the remaining echo cells. An earlier revision of this page said the v1.3 far-end row could not have come from the published weights; that was a protocol mismatch and is retracted. +The [follow-up investigation in mobius](https://github.com/FluidInference/mobius/blob/a066485e4c65790fa21b180b7ddf5e22e0f2d044/models/enhancement/localvqe/coreml/REPRODUCTION.md) +also compares the published PT/GGUF tensors and isolates a historical +upstream state-copy defect. Testing the original engine on all 300 far-end +recordings still did not reproduce the card's echo MOS. Its per-recording +results and precision diagnostics are retained separately from the main +800-clip benchmark. + **Port fidelity.** The upstream GGML engine was run on the same 800 clips and scored on identical, aligned whole-hop samples: every per-scenario mean matches the Core ML port to two decimals, the per-clip echo-MOS delta has diff --git a/Documentation/Enhancement/LocalVQEValidation.md b/Documentation/Enhancement/LocalVQEValidation.md new file mode 100644 index 000000000..b33fe4331 --- /dev/null +++ b/Documentation/Enhancement/LocalVQEValidation.md @@ -0,0 +1,77 @@ +# LocalVQE streaming validation + +The stream wrapper serializes each complete `enhance`, `flush`, and `reset` +operation, including across asynchronous prediction. Previously another +operation could enter during inference and overwrite shared input buffers, +consume the same pending audio, or reset state before the first call resumed. +Independent streams still run concurrently on a shared manager. + +Queued cancellation leaves the active clip untouched. Active cancellation or +inference failure clears the unfinished clip, recurrent state and sample +counters before reuse. Reset waits for earlier operations. Callers should +submit capture buffers through one consumer task, in capture order, and await +each push to bound queued audio. See [usage and recovery](LocalVQE.md#streaming). + +## Verification + +Validation performed on Apple Silicon with cached real v1.3 Core ML models: + +- Debug library/CLI compilation and `swift build -c release` passed. +- Strict swift-format lint and `git diff --check` passed. +- A standalone harness linked to the compiled FluidAudio module passed 13 + audio comparisons using the repository's real + `01-validation-request-21.4s.wav` fixture. It exercised the same concurrency + and recovery scenarios as the added regression tests; it did not replace + the model with a mock. + +| Check | Result | +|---|---| +| Overlapping pushes followed by flush | Exact match to sequential processing | +| Reset queued between pushes | Next clip exactly matches a fresh stream | +| Cancelled queued push, flush or reset | Three comparisons; active clip unchanged | +| Active push cancellation, then reuse | `CancellationError`; next clip exactly matches a fresh stream | +| Two independent concurrent streams | Each exactly matches its sequential result | +| Streaming buffers of 100, 256, 1000 and 4096 samples | Four exact matches to whole-clip output | +| 16 ms versus 256 ms model chunk | Maximum absolute difference 8.20e-8 | + +Three predetermined real AEC-Challenge mic/reference pairs also passed a +release CLI smoke check, using v1.3 CPU inference, 16 ms model chunks and +256-sample input buffers. Output was finite, retained the microphone input +length, and was bit-identical between streaming and whole-clip processing. + +| Recording stem | Duration | Samples | +|---|---:|---:| +| `t2U2oyODeEuQhnAt3oCksQ_doubletalk` | 36.58 s | 585280 | +| `Du0RI678G0yhsNVU5AGKTw_farend-singletalk-with-movement` | 23.90 s | 382400 | +| `f2HsvN51L0ygRLcFvf3udg_nearend-singletalk` | 16.53 s | 264480 | + +This smoke check did not rerun the full 800-clip quality benchmark or tune +the model/scorer. The existing CLI reported p50 1.10 ms, p99 1.18 ms, and +maximum 1.98–2.22 ms on these recordings. These are offline timing observations: +the CLI collects timing only for pushes emitting samples, excluding the +initial dropped-hop call and flush. They do not cover startup or certify +live-call deadlines under device contention. + +## XCTest and CI + +The local `swift test --filter LocalVqe` attempt could not execute: this Mac +has Command Line Tools 6.2.3 without Xcode's XCTest framework (`no such module +'XCTest'`). The standalone checks above are reported separately from XCTest. + +On an Xcode-equipped machine, run the regression suite with real models: + +```bash +FLUIDAUDIO_LOCALVQE_MODEL_DIR=/path/to/compiled/models swift test --filter LocalVqe +``` + +The directory must contain the v1.3 16 ms and 256 ms bundles. Supplying it +enables model tests in CI and makes a missing bundle a failure. Without an +explicit directory, model tests retain the existing CI/missing-cache skip. + +## Production boundary + +The stream-wrapper concurrency issue is fixed and its recovery behavior is +verified locally. Live microphone/playback clock alignment, route changes, +audio-callback scheduling, and sustained performance under contention or +thermal load still require the application's capture/playback pipeline on +its target devices. LocalVQE remains beta pending that integration validation. diff --git a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift index 8d88a2c95..ba74275ad 100644 --- a/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift +++ b/Sources/FluidAudio/Enhancement/LocalVQE/LocalVqeStream.swift @@ -23,6 +23,11 @@ import OSLog /// goes through Core ML's async prediction API, which Apple documents as /// thread-safe (WWDC23 10049), so independent streams may run concurrently; /// the synchronous API would require serializing every call on the model. +/// Operations on a single stream are serialized in actor-arrival order. Await +/// each push before submitting the next to preserve capture order and bound +/// queued audio. Cancellation while queued leaves the clip untouched; +/// cancellation or inference failure after a push/flush starts discards the +/// unfinished clip and resets the stream before the next operation. public actor LocalVqeStream { private static let logger = AppLogger(category: "LocalVqeStream") @@ -47,6 +52,17 @@ public actor LocalVqeStream { private var samplesIn = 0 private var samplesOut = 0 + private struct OperationWaiter { + let id: UUID + let continuation: CheckedContinuation + } + + private var operationInProgress = false + private var operationWaiters: [OperationWaiter] = [] + + /// Internal queue snapshot for deterministic concurrency regression tests. + var operationCount: Int { (operationInProgress ? 1 : 0) + operationWaiters.count } + init(model: MLModel, samplesPerCall: Int) throws { self.model = model self.samplesPerCall = samplesPerCall @@ -87,9 +103,20 @@ public actor LocalVqeStream { /// Number of state tensors the model carries between calls. public var stateCount: Int { stateNames.count } - /// Clear all recurrent state and buffered audio; the next call starts a new clip. - public func reset() throws { - try resetStates() + /// Clear all recurrent state and buffered audio after earlier operations + /// finish; the next push starts a new clip. Cancel the active task first + /// when abandoning an in-flight push instead of waiting for it to finish. + public func reset() async throws { + try await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + clearClip() + } + + private func clearClip() { + // Allocate fresh zero states lazily on the next model call. Clearing + // after an inference failure must not itself require an allocation. + states.removeAll(keepingCapacity: true) pendingMic.removeAll(keepingCapacity: true) pendingRef.removeAll(keepingCapacity: true) leadingSamplesToDrop = LocalVqeManager.hopSize @@ -97,8 +124,36 @@ public actor LocalVqeStream { samplesOut = 0 } - private func resetStates() throws { - states = try Self.zeroStates(names: stateNames, shapes: stateShapes) + private func acquireOperation() async throws { + try Task.checkCancellation() + guard operationInProgress else { + operationInProgress = true + return + } + + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + operationWaiters.append(OperationWaiter(id: id, continuation: continuation)) + } + } onCancel: { + Task { await self.cancelOperation(id: id) } + } + } + + private func cancelOperation(id: UUID) { + guard let index = operationWaiters.firstIndex(where: { $0.id == id }) else { return } + let waiter = operationWaiters.remove(at: index) + waiter.continuation.resume(throwing: CancellationError()) + } + + private func releaseOperation() { + guard !operationWaiters.isEmpty else { + operationInProgress = false + return + } + // Transfer ownership without opening a gap for a newly arriving call. + operationWaiters.removeFirst().continuation.resume() } private static func zeroStates(names: [String], shapes: [String: [NSNumber]]) throws -> [String: MLMultiArray] { @@ -123,6 +178,20 @@ public actor LocalVqeStream { guard mic.count == reference.count else { throw LocalVqeError.lengthMismatch(mic: mic.count, reference: reference.count) } + try await acquireOperation() + defer { releaseOperation() } + // A cancelled waiter may have acquired the operation just before its + // cancellation handler ran. It must not mutate/reset the current clip. + try Task.checkCancellation() + do { + return try await enhanceExclusive(mic: mic, reference: reference) + } catch { + clearClip() + throw error + } + } + + private func enhanceExclusive(mic: [Float], reference: [Float]) async throws -> [Float] { pendingMic.append(contentsOf: mic) pendingRef.append(contentsOf: reference) samplesIn += mic.count @@ -147,9 +216,21 @@ public actor LocalVqeStream { /// samples so that total output length equals total input length. /// Ends the current clip: the stream is reset afterwards. public func flush() async throws -> [Float] { + try await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + do { + return try await flushExclusive() + } catch { + clearClip() + throw error + } + } + + private func flushExclusive() async throws -> [Float] { let outstanding = samplesIn - samplesOut guard outstanding > 0 else { - try reset() + clearClip() return [] } // Zeros needed to complete every outstanding sample, rounded up to whole calls. @@ -170,7 +251,7 @@ public actor LocalVqeStream { } let emitted = emit(out) let tail = Array(emitted.prefix(outstanding)) - try reset() + clearClip() return tail } @@ -187,6 +268,10 @@ public actor LocalVqeStream { } private func runCall(mic: ArraySlice, reference: ArraySlice) async throws -> [Float] { + try Task.checkCancellation() + if states.isEmpty { + states = try Self.zeroStates(names: stateNames, shapes: stateShapes) + } micInput.withUnsafeMutableBufferPointer(ofType: Float.self) { buf, _ in _ = buf.initialize(from: mic) } @@ -203,8 +288,10 @@ public actor LocalVqeStream { let provider = try MLDictionaryFeatureProvider(dictionary: features) output = try await model.compatPrediction(from: provider, options: MLPredictionOptions()) } catch { + if Task.isCancelled || error is CancellationError { throw CancellationError() } throw LocalVqeError.modelProcessingFailed(error.localizedDescription) } + try Task.checkCancellation() for name in stateNames { guard let next = output.featureValue(for: Self.stateOutputPrefix + name)?.multiArrayValue else { diff --git a/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift b/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift index 82d57f8e5..75a2e8223 100644 --- a/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift +++ b/Tests/FluidAudioTests/Enhancement/LocalVqeTests.swift @@ -42,13 +42,16 @@ final class LocalVqeNamingTests: XCTestCase { /// Set `FLUIDAUDIO_LOCALVQE_MODEL_DIR` to a directory holding the compiled /// `localvqe-*.mlmodelc` bundles (e.g. the mobius conversion `build/` dir); /// otherwise the default model cache is used, and the tests skip when the -/// model is absent or when running in CI. +/// model is absent or when running in CI. An explicit directory enables +/// these tests in CI and a missing model in that directory is a failure. final class LocalVqeStreamTests: XCTestCase { private static let fixture = "01-validation-request-21.4s" override func setUp() async throws { - if ProcessInfo.processInfo.environment["CI"] != nil { + if ProcessInfo.processInfo.environment["CI"] != nil, + ProcessInfo.processInfo.environment["FLUIDAUDIO_LOCALVQE_MODEL_DIR"] == nil + { throw XCTSkip("Skipping LocalVQE model tests in CI") } } @@ -64,6 +67,9 @@ final class LocalVqeStreamTests: XCTestCase { } let file = dir.appendingPathComponent(ModelNames.LocalVQE.modelFile(variant: .v13, chunk: chunk)) guard FileManager.default.fileExists(atPath: file.path) else { + if ProcessInfo.processInfo.environment["FLUIDAUDIO_LOCALVQE_MODEL_DIR"] != nil { + throw LocalVqeError.modelLoadingFailed("Required test model not available at \(file.path)") + } throw XCTSkip("LocalVQE model not available at \(file.path)") } return try LocalVqeManager(config: config, modelDirectory: dir) @@ -157,4 +163,157 @@ final class LocalVqeStreamTests: XCTestCase { let stateCount = await stream.stateCount XCTAssertEqual(stateCount, 33) } + + private func waitForOperations(_ count: Int, on stream: LocalVqeStream) async throws { + let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + while await stream.operationCount < count { + guard ContinuousClock.now < deadline else { + throw NSError( + domain: "LocalVqeStreamTests", code: 1, + userInfo: [NSLocalizedDescriptionKey: "Never observed \(count) overlapping operations"]) + } + try await Task.sleep(for: .milliseconds(1)) + } + } + + private func assertAudioEqual( + _ actual: [Float], _ expected: [Float], file: StaticString = #filePath, line: UInt = #line + ) { + XCTAssertEqual(actual.count, expected.count, file: file, line: line) + XCTAssertTrue(actual.allSatisfy(\.isFinite), file: file, line: line) + let maxDiff = zip(actual, expected).reduce(Float.zero) { max($0, abs($1.0 - $1.1)) } + XCTAssertLessThan(maxDiff, 1e-4, file: file, line: line) + } + + func testOverlappingPushesAndFlushMatchSequentialAudio() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = Array(try loadFixture().prefix(16000 * 5 + 137)) + let split = 16000 * 4 + let firstMic = Array(mic[.. [Float] in + switch operation { + case "enhance": + return try await stream.enhance(mic: mic, reference: [Float](repeating: 0, count: mic.count)) + case "flush": + return try await stream.flush() + default: + try await stream.reset() + return [] + } + } + defer { queued.cancel() } + try await waitForOperations(2, on: stream) + queued.cancel() + do { + _ = try await queued.value + XCTFail("Queued \(operation) should throw CancellationError") + } catch is CancellationError { + // Cancellation must remove the queued operation without resetting the active clip. + } + var actual = try await first.value + actual.append(contentsOf: try await stream.flush()) + assertAudioEqual(actual, expected) + } + } + + func testCancelledActivePushResetsBeforeReuse() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = try loadFixture() + let nextClip = Array(mic.suffix(16000 + 137)) + let expected = try await manager.process(mic: nextClip) + let stream = try await manager.makeStream() + let active = Task { + try await stream.enhance(mic: mic, reference: [Float](repeating: 0, count: mic.count)) + } + defer { active.cancel() } + try await waitForOperations(1, on: stream) + active.cancel() + do { + _ = try await active.value + XCTFail("Active push should throw CancellationError") + } catch is CancellationError { + // A partially processed clip must not contaminate the next one. + } + var actual = try await stream.enhance( + mic: nextClip, reference: [Float](repeating: 0, count: nextClip.count)) + actual.append(contentsOf: try await stream.flush()) + assertAudioEqual(actual, expected) + } + + func testIndependentStreamsCanRunConcurrently() async throws { + let manager = try loadManager(chunk: .realtime16ms) + let mic = Array(try loadFixture().prefix(16000 + 137)) + let otherMic = Array(try loadFixture().suffix(16000 + 73)) + let expected = try await manager.process(mic: mic) + let otherExpected = try await manager.process(mic: otherMic) + + async let actual = manager.process(mic: mic) + async let otherActual = manager.process(mic: otherMic) + let results = try await (actual, otherActual) + assertAudioEqual(results.0, expected) + assertAudioEqual(results.1, otherExpected) + } } From 367024e451f94fc8f7dcefe02acc2cf2bb523c15 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sat, 19 Sep 2026 21:32:59 -0400 Subject: [PATCH 11/14] fix(cli): make LocalVQE benchmarks reproducible --- Documentation/Enhancement/LocalVQE.md | 33 +++- README.md | 2 +- Sources/FluidAudio/ModelRegistry.swift | 10 +- .../Commands/EnhanceBenchmarkCommand.swift | 101 ++++++------ .../Commands/EnhanceBenchmarkDataset.swift | 154 ++++++++++++++++++ .../CLI/EnhanceBenchmarkDatasetTests.swift | 84 ++++++++++ .../Shared/ModelRegistryTests.swift | 11 ++ 7 files changed, 335 insertions(+), 60 deletions(-) create mode 100644 Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift create mode 100644 Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index b27460efc..9f692ad19 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -9,8 +9,8 @@ up what the loudspeaker plays. **Beta.** Port fidelity validated (numerically equivalent to the upstream PyTorch and GGML engines, scored identically to GGML on the 800-clip -AEC-Challenge blind set); the published benchmark is substantially -reproduced, with unresolved v1.2 far-end differences (see +AEC-Challenge blind set); the published benchmark is only partially +reproduced, and its ERLE protocol is ambiguous in the public artifacts (see [Quality](#quality-aec-challenge-blind-test-set)). Not yet exercised inside production call pipelines. @@ -166,8 +166,8 @@ processing produce the same audio to 1e-5. ## Quality: AEC-Challenge blind test set -**Summary: port fidelity validated; published benchmark substantially -reproduced, with unresolved v1.2 far-end differences.** +**Summary: port fidelity validated; the published benchmark is only partially +reproduced, with unresolved v1.2 far-end differences and ERLE protocol.** The upstream quality table is AECMOS on the ICASSP 2022 AEC-Challenge blind set (800 real device recordings). The Swift port was rendered over all 800 @@ -192,9 +192,12 @@ higher is better. **Upstream protocol (HF model-card reproduction).** The published table was compared using the legacy AECMOS model over the first 20 s of each clip; that protocol reproduces the card's unprocessed baseline exactly -(2.67 / 2.56 / 1.90 / 2.13 / 5.00). Under it, the Core ML port gives: +(2.67 / 2.56 / 1.90 / 2.13 / 5.00). The card defines ERLE as a plain +whole-signal energy ratio, but its numbers resemble a separately reconstructed +gated metric. `gERLE*` below is that reconstruction, not a confirmed +interpretation of the card's protocol. Under it, the Core ML port gives: -| Scenario | HF card v1.3 | Core ML v1.3 | HF card v1.2 | Core ML v1.2 | +| Scenario | HF card v1.3 | Core ML v1.3 (echo / deg / gERLE* / OVRL) | HF card v1.2 | Core ML v1.2 (echo / deg / gERLE* / OVRL) | |---|---|---|---|---| | doubletalk | 4.73 / 2.62 | 4.73 / 2.62 | 4.72 / 2.37 | 4.72 / 2.39 | | doubletalk-with-movement | 4.67 / 2.43 | 4.66 / 2.44 | 4.65 / 2.30 | 4.64 / 2.31 | @@ -202,12 +205,17 @@ protocol reproduces the card's unprocessed baseline exactly | farend-singletalk-with-movement | 3.88 / 4.98 | 3.75 / 4.96 | 4.12 / 4.96 | 4.27 / 4.96 | | nearend-singletalk | 5.00 / 4.18 | 5.00 / 4.18 | 5.00 / 4.16 | 5.00 / 4.17 | +The ERLE definition conflict is material: current GGML gives plain / gated +far-end ERLE of 43.0 / 50.9 and 40.1 / 49.4 dB for v1.3, versus card values +50.9 / 49.9; v1.2 gives 38.7 / 48.0 and 30.5 / 41.1 dB, versus 45.7 / 40.6. +Selecting the closer gated result does not prove that upstream used this gate. + Unprocessed baseline: exact. v1.3: double-talk and near-end within 0.01 echo MOS; far-end 0.15 low from aligned float output and within 0.04 when the upstream CLI's raw 16-bit, one-hop-late output is scored instead; gated -ERLE within 0.8 dB and OVRL within 0.01 (full columns in the mobius +gated ERLE within 0.8 dB and OVRL within 0.01 (full columns in the mobius README). v1.2: double-talk and near-end within 0.02 echo, 0.02 deg, 0.1 dB -ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo +gated ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo +0.29 / +0.15, gated ERLE +1.9 / +0.7 dB, OVRL +0.09 / +0.05, from either runtime). Those values are above the published ones, which is not evidence that the port outperforms upstream; +0.29 echo MOS is not rounding noise. @@ -248,7 +256,11 @@ the in-repo Parakeet TDT v3 ASR on a 200-example subset (the first 200 of shard 0) of the Microsoft AEC-Challenge synthetic *training* set (mic + loopback + clean near-end triples; [FluidInference/aec-challenge-synthetic-mini](https://huggingface.co/datasets/FluidInference/aec-challenge-synthetic-mini), -auto-downloaded). The ASR transcript of the clean near-end clip is the +revision `1f3714b5a3f98cedef1bbb017f21bbd7ae688596`, archive SHA256 +`45ff5d7acfce499558c25a0eace45eb819cec8aa76420fe733de7ee116ae548d`). +The default download and cached metadata are verified before use; malformed +rows, duplicate IDs and missing audio now fail instead of silently shrinking +the benchmark. The ASR transcript of the clean near-end clip is the reference and the loopback transcript gives the far-end words, so the metrics are relative to machine transcripts, not human ones; examples whose clean-near-end transcript is empty (33 of 200) are excluded and reported. @@ -279,6 +291,9 @@ needs the loopback to cancel echo; without it, it only denoises. An earlier revision of this table reported 87.5% / 86.3% recall: the shared WER scorer had its insertion/deletion labels swapped, so an empty hypothesis scored 100% recall. Fixed in `WERCalculator` (WER itself was unaffected). +When `--output` is used, the JSON includes the dataset revision and hashes, +numeric-file-ID selection order, exact selected/excluded IDs, both machine +reference transcripts and the raw word-count numerators and denominators. ```bash swift run -c release fluidaudiocli enhance-benchmark # both variants, 200 files diff --git a/README.md b/README.md index bd0204e6d..0cf144d03 100644 --- a/README.md +++ b/README.md @@ -574,7 +574,7 @@ Offline mode also reports RTFx using the model's per-chunk processing time. ## Speech Enhancement (Echo Cancellation + Noise Suppression) -> **⚠️ Beta:** port fidelity validated (scores identically to the upstream GGML engine on the 800-clip AEC-Challenge blind set); published benchmark substantially reproduced, with unresolved v1.2 far-end differences. Not yet exercised in production call pipelines. +> **⚠️ Beta:** port fidelity validated (scores identically to the upstream GGML engine on the 800-clip AEC-Challenge blind set); the published table is only partially reproduced, with unresolved v1.2 far-end values and an ambiguous ERLE protocol. Not yet exercised in production call pipelines. [LocalVQE](https://github.com/localai-org/LocalVQE) (Apache-2.0) is a compact neural acoustic echo canceller + noise suppressor + dereverberator for 16 kHz diff --git a/Sources/FluidAudio/ModelRegistry.swift b/Sources/FluidAudio/ModelRegistry.swift index f083da063..8c080bea1 100644 --- a/Sources/FluidAudio/ModelRegistry.swift +++ b/Sources/FluidAudio/ModelRegistry.swift @@ -109,8 +109,10 @@ public enum ModelRegistry { } /// Construct download URL for a dataset file - public static func resolveDataset(_ dataset: String, _ filePath: String) throws -> URL { - let urlString = "\(baseURL)/datasets/\(dataset)/resolve/main/\(filePath)" + public static func resolveDataset( + _ dataset: String, _ filePath: String, revision: String = "main" + ) throws -> URL { + let urlString = "\(baseURL)/datasets/\(dataset)/resolve/\(revision)/\(filePath)" guard let url = URL(string: urlString) else { throw Error.invalidURL(urlString) } @@ -118,8 +120,8 @@ public enum ModelRegistry { } /// Construct base URL for dataset directory (without trailing slash) - public static func resolveDatasetBase(_ dataset: String) -> String { - "\(baseURL)/datasets/\(dataset)/resolve/main" + public static func resolveDatasetBase(_ dataset: String, revision: String = "main") -> String { + "\(baseURL)/datasets/\(dataset)/resolve/\(revision)" } // MARK: - Session Configuration diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift index e6dedce54..1c0563386 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -18,6 +18,10 @@ enum EnhanceBenchmarkCommand { static let datasetRepo = "FluidInference/aec-challenge-synthetic-mini" static let datasetArchive = "aec-synthetic-mini.tar.gz" static let datasetFolder = "aec-synthetic-mini" + static let datasetRevision = "1f3714b5a3f98cedef1bbb017f21bbd7ae688596" + static let datasetArchiveSHA256 = "45ff5d7acfce499558c25a0eace45eb819cec8aa76420fe733de7ee116ae548d" + static let datasetMetadataSHA256 = "865aff8e66eb682c292f42a9d747d931f3a2f71f18e16fec53aea80dbdc2eacc" + static let expectedDatasetExamples = 200 private struct Options { var datasetDir: String? @@ -29,15 +33,6 @@ enum EnhanceBenchmarkCommand { var outputPath: String? } - private struct Example { - let fileID: String - let mic: URL - let lpb: URL - let clean: URL - let ser: Int? - let nearendNoisy: Bool - } - private struct ConditionTotals { var files = 0 var refWords = 0 @@ -66,7 +61,11 @@ enum EnhanceBenchmarkCommand { case "--dataset-dir": options.datasetDir = next(arguments, &index) case "--max-files": - options.maxFiles = Int(next(arguments, &index) ?? "") + guard let raw = next(arguments, &index), let maxFiles = Int(raw), maxFiles > 0 else { + logger.error("--max-files must be a positive integer") + exit(1) + } + options.maxFiles = maxFiles case "--variants": let raw = (next(arguments, &index) ?? "").split(separator: ",").map(String.init) let parsed = raw.compactMap(LocalVqeVariant.init(rawValue:)) @@ -103,7 +102,7 @@ enum EnhanceBenchmarkCommand { do { let datasetDir = try await resolveDataset(options.datasetDir) - var examples = try loadExamples(from: datasetDir) + var examples = try EnhanceBenchmarkDataset.loadExamples(from: datasetDir) if let maxFiles = options.maxFiles { examples = Array(examples.prefix(maxFiles)) } guard !examples.isEmpty else { logger.error("No examples found in \(datasetDir.path)") @@ -132,7 +131,7 @@ enum EnhanceBenchmarkCommand { var totals = [String: ConditionTotals]() var bySer = [String: [String: ConditionTotals]]() // bucket -> condition -> totals var rows: [[String: Any]] = [] - var emptyReferences = 0 + var emptyReferenceFileIDs: [String] = [] for (i, example) in examples.enumerated() { let mic = try converter.resampleAudioFile(example.mic) @@ -148,14 +147,17 @@ enum EnhanceBenchmarkCommand { if refWords.isEmpty { // No reference words to recall: the ASR produced nothing on the // clean near-end clip. Excluded from every metric and counted. - emptyReferences += 1 + emptyReferenceFileIDs.append(example.fileID) continue } let farWords = words(try await transcribe(asr, lpb)) let bucket = serBucket(example.ser) var row: [String: Any] = [ - "fileid": example.fileID, "ser": example.ser as Any, "ref_words": refWords.count, + "fileid": example.fileID, "ser": example.ser, "ref_words": refWords.count, "far_words": farWords.count, "reference": refWords.joined(separator: " "), + "far_reference": farWords.joined(separator: " "), + "is_farend_noisy": example.farendNoisy, + "is_nearend_noisy": example.nearendNoisy, ] for condition in conditions { @@ -206,8 +208,10 @@ enum EnhanceBenchmarkCommand { } report("") - if emptyReferences > 0 { - report("Excluded \(emptyReferences) examples whose clean near-end transcript was empty.") + if !emptyReferenceFileIDs.isEmpty { + report( + "Excluded \(emptyReferenceFileIDs.count) examples whose clean near-end transcript was empty: " + + emptyReferenceFileIDs.joined(separator: ", ")) } report(row(["condition", "files", "recall", "WER", "leakage", "RTFx"])) for condition in conditions { @@ -233,11 +237,24 @@ enum EnhanceBenchmarkCommand { for (name, t) in totals { summary[name] = [ "files": t.files, "recall": t.recall, "wer": t.wer, "leakage": t.leakage, "rtfx": t.rtfx, + "reference_words": t.refWords, "hits": t.hits, "errors": t.errors, + "far_end_words": t.farWords, "leaked_words": t.leaked, + "audio_seconds": t.audioSeconds, "enhancement_seconds": t.enhanceSeconds, ] } let payload: [String: Any] = [ - "dataset": datasetDir.path, "chunk": options.chunk.rawValue, "summary": summary, "files": rows, - "excluded_empty_reference": emptyReferences, + "dataset": [ + "path": datasetDir.path, + "repository": datasetRepo, + "revision": options.datasetDir == nil ? datasetRevision : "custom", + "archive_sha256": options.datasetDir == nil ? datasetArchiveSHA256 : NSNull(), + "metadata_sha256": try EnhanceBenchmarkDataset.sha256( + of: datasetDir.appendingPathComponent("meta.csv")), + "selection_order": "numeric fileid", + "selected_fileids": examples.map(\.fileID), + ], + "chunk": options.chunk.rawValue, "summary": summary, "files": rows, + "excluded_empty_reference_fileids": emptyReferenceFileIDs, ] let data = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) try data.write(to: URL(fileURLWithPath: outputPath)) @@ -273,10 +290,10 @@ enum EnhanceBenchmarkCommand { var spare = [String: Int]() for w in hypothesis { spare[w, default: 0] += 1 } - for w in reference where spare[w, default: 0] > 0 { spare[w]! -= 1 } + for w in reference where spare[w, default: 0] > 0 { spare[w, default: 0] -= 1 } var leaked = 0 for w in farEnd where spare[w, default: 0] > 0 { - spare[w]! -= 1 + spare[w, default: 0] -= 1 leaked += 1 } return (hits, errors, leaked) @@ -311,15 +328,21 @@ enum EnhanceBenchmarkCommand { .appendingPathComponent("Library/Application Support/FluidAudio/Datasets", isDirectory: true) let dir = base.appendingPathComponent(datasetFolder, isDirectory: true) if FileManager.default.fileExists(atPath: dir.appendingPathComponent("meta.csv").path) { + try validatePinnedDataset(dir) return dir } try FileManager.default.createDirectory(at: base, withIntermediateDirectories: true) - let url = try ModelRegistry.resolveDataset(datasetRepo, datasetArchive) + let url = try ModelRegistry.resolveDataset(datasetRepo, datasetArchive, revision: datasetRevision) report("Downloading \(url.absoluteString)") let (tmp, response) = try await URLSession.shared.download(from: url) guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { throw LocalVqeError.modelProcessingFailed("dataset download failed: \(response)") } + let actualArchiveHash = try EnhanceBenchmarkDataset.sha256(of: tmp) + guard actualArchiveHash == datasetArchiveSHA256 else { + throw LocalVqeError.modelProcessingFailed( + "dataset archive checksum mismatch: expected \(datasetArchiveSHA256), got \(actualArchiveHash)") + } let archive = base.appendingPathComponent(datasetArchive) try? FileManager.default.removeItem(at: archive) try FileManager.default.moveItem(at: tmp, to: archive) @@ -334,36 +357,22 @@ enum EnhanceBenchmarkCommand { else { throw LocalVqeError.modelProcessingFailed("dataset extraction failed (tar status \(tar.terminationStatus))") } + try validatePinnedDataset(dir) return dir } - private static func loadExamples(from dir: URL) throws -> [Example] { + private static func validatePinnedDataset(_ dir: URL) throws { let metaURL = dir.appendingPathComponent("meta.csv") - let text = try String(contentsOf: metaURL, encoding: .utf8) - var lines = text.split(whereSeparator: { $0 == "\n" || $0 == "\r\n" }).map(String.init) - guard !lines.isEmpty else { return [] } - let header = lines.removeFirst().split(separator: ",").map { String($0).trimmingCharacters(in: .whitespaces) } - func column(_ name: String, _ fields: [String]) -> String? { - guard let i = header.firstIndex(of: name), i < fields.count else { return nil } - let v = fields[i].trimmingCharacters(in: .whitespaces) - return v.isEmpty ? nil : v + let actualMetadataHash = try EnhanceBenchmarkDataset.sha256(of: metaURL) + guard actualMetadataHash == datasetMetadataSHA256 else { + throw LocalVqeError.modelProcessingFailed( + "dataset metadata checksum mismatch: expected \(datasetMetadataSHA256), got \(actualMetadataHash)") } - var examples: [Example] = [] - for line in lines { - let fields = line.split(separator: ",", omittingEmptySubsequences: false).map(String.init) - guard let fileID = column("fileid", fields) else { continue } - let stem = "fileid_\(fileID)" - let mic = dir.appendingPathComponent("\(stem)_mic.wav") - let lpb = dir.appendingPathComponent("\(stem)_lpb.wav") - let clean = dir.appendingPathComponent("\(stem)_clean.wav") - guard [mic, lpb, clean].allSatisfy({ FileManager.default.fileExists(atPath: $0.path) }) else { continue } - examples.append( - Example( - fileID: fileID, mic: mic, lpb: lpb, clean: clean, - ser: column("ser", fields).flatMap(Int.init), - nearendNoisy: column("is_nearend_noisy", fields) == "1")) + let examples = try EnhanceBenchmarkDataset.loadExamples(from: dir) + guard examples.count == expectedDatasetExamples else { + throw LocalVqeError.modelProcessingFailed( + "dataset contains \(examples.count) examples; expected \(expectedDatasetExamples)") } - return examples.sorted { ($0.ser ?? 0, $0.fileID) < ($1.ser ?? 0, $1.fileID) } } private static func next(_ arguments: [String], _ index: inout Int) -> String? { @@ -390,7 +399,7 @@ enum EnhanceBenchmarkCommand { Options: --dataset-dir Directory with fileid_*_{mic,lpb,clean}.wav + meta.csv (default: auto-download \(datasetRepo)). - --max-files Score only the first n examples (sorted by SER). + --max-files Score only the first n examples (numeric fileid order). --variants Comma list of v1.3,v1.2 (default both). --chunk <256ms|16ms> Chunk export to benchmark (default 256ms). --compute-units diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift new file mode 100644 index 000000000..97df93e62 --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift @@ -0,0 +1,154 @@ +#if os(macOS) +import CryptoKit +import Foundation + +/// Loads and validates the fixed LocalVQE ASR benchmark manifest. +enum EnhanceBenchmarkDataset { + struct Example: Equatable { + let fileID: String + let mic: URL + let lpb: URL + let clean: URL + let ser: Int + let farendNoisy: Bool + let nearendNoisy: Bool + } + + enum DatasetError: Error, LocalizedError, Equatable { + case duplicateFileID(String) + case emptyMetadata + case invalidInteger(line: Int, column: String, value: String) + case malformedRow(line: Int, expected: Int, actual: Int) + case missingAudio(fileID: String, path: String) + case missingColumn(String) + case missingFileID(line: Int) + + var errorDescription: String? { + switch self { + case .duplicateFileID(let fileID): + return "duplicate benchmark fileid '\(fileID)'" + case .emptyMetadata: + return "benchmark meta.csv is empty" + case .invalidInteger(let line, let column, let value): + return "benchmark meta.csv line \(line) has invalid \(column) value '\(value)'" + case .malformedRow(let line, let expected, let actual): + return "benchmark meta.csv line \(line) has \(actual) fields; expected \(expected)" + case .missingAudio(let fileID, let path): + return "benchmark fileid \(fileID) is missing \(path)" + case .missingColumn(let column): + return "benchmark meta.csv is missing required column '\(column)'" + case .missingFileID(let line): + return "benchmark meta.csv line \(line) has an empty fileid" + } + } + } + + private struct MetadataRow { + let fileID: String + let ser: Int + let farendNoisy: Bool + let nearendNoisy: Bool + } + + static func loadExamples( + from directory: URL, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) throws -> [Example] { + let metadata = try String(contentsOf: directory.appendingPathComponent("meta.csv"), encoding: .utf8) + return try parseMetadata(metadata).map { row in + let stem = "fileid_\(row.fileID)" + let mic = directory.appendingPathComponent("\(stem)_mic.wav") + let lpb = directory.appendingPathComponent("\(stem)_lpb.wav") + let clean = directory.appendingPathComponent("\(stem)_clean.wav") + for url in [mic, lpb, clean] { + guard fileExists(url.path) else { + throw DatasetError.missingAudio(fileID: row.fileID, path: url.lastPathComponent) + } + } + return Example( + fileID: row.fileID, + mic: mic, + lpb: lpb, + clean: clean, + ser: row.ser, + farendNoisy: row.farendNoisy, + nearendNoisy: row.nearendNoisy + ) + } + } + + static func sha256(of url: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = SHA256() + while true { + let data = try handle.read(upToCount: 1024 * 1024) ?? Data() + guard !data.isEmpty else { break } + hasher.update(data: data) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + private static func parseMetadata(_ text: String) throws -> [MetadataRow] { + var lines = text.split(omittingEmptySubsequences: true, whereSeparator: \.isNewline).map(String.init) + guard !lines.isEmpty else { throw DatasetError.emptyMetadata } + + let header = fields(in: lines.removeFirst()) + let requiredColumns = ["fileid", "ser", "is_farend_noisy", "is_nearend_noisy"] + for required in requiredColumns where !header.contains(required) { + throw DatasetError.missingColumn(required) + } + + guard let fileIDIndex = header.firstIndex(of: "fileid"), + let serIndex = header.firstIndex(of: "ser"), + let farendNoisyIndex = header.firstIndex(of: "is_farend_noisy"), + let nearendNoisyIndex = header.firstIndex(of: "is_nearend_noisy") + else { + throw DatasetError.missingColumn("internal required-column lookup") + } + + var seen = Set() + var rows: [MetadataRow] = [] + for (offset, line) in lines.enumerated() { + let lineNumber = offset + 2 + let rowFields = fields(in: line) + guard rowFields.count == header.count else { + throw DatasetError.malformedRow(line: lineNumber, expected: header.count, actual: rowFields.count) + } + let fileID = rowFields[fileIDIndex] + guard !fileID.isEmpty else { throw DatasetError.missingFileID(line: lineNumber) } + guard seen.insert(fileID).inserted else { throw DatasetError.duplicateFileID(fileID) } + let ser = try integer("ser", value: rowFields[serIndex], line: lineNumber) + let farendNoisy = try integer( + "is_farend_noisy", value: rowFields[farendNoisyIndex], line: lineNumber) + let nearendNoisy = try integer( + "is_nearend_noisy", value: rowFields[nearendNoisyIndex], line: lineNumber) + rows.append( + MetadataRow( + fileID: fileID, + ser: ser, + farendNoisy: farendNoisy != 0, + nearendNoisy: nearendNoisy != 0 + )) + } + return rows.sorted { lhs, rhs in + guard let lhsID = Int(lhs.fileID), let rhsID = Int(rhs.fileID) else { + return lhs.fileID < rhs.fileID + } + return lhsID < rhsID + } + } + + private static func fields(in line: String) -> [String] { + line.split(separator: ",", omittingEmptySubsequences: false) + .map { String($0).trimmingCharacters(in: .whitespaces) } + } + + private static func integer(_ column: String, value rawValue: String, line: Int) throws -> Int { + guard let value = Int(rawValue) else { + throw DatasetError.invalidInteger(line: line, column: column, value: rawValue) + } + return value + } +} +#endif diff --git a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift new file mode 100644 index 000000000..116228402 --- /dev/null +++ b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift @@ -0,0 +1,84 @@ +#if os(macOS) +import Foundation +import XCTest + +@testable import FluidAudioCLI + +final class EnhanceBenchmarkDatasetTests: XCTestCase { + private let header = "fileid,ser,is_farend_noisy,is_nearend_noisy,nearend_scale" + + func testLoadsCompleteManifestInNumericFileIDOrder() throws { + let metadata = [ + header, + "10,-2,1,0,0.5", + "2,3,0,1,0.8", + ].joined(separator: "\n") + let directory = try metadataDirectory(metadata) + let expectedFiles = Set( + [2, 10].flatMap { id in + ["fileid_\(id)_mic.wav", "fileid_\(id)_lpb.wav", "fileid_\(id)_clean.wav"] + }) + + let examples = try EnhanceBenchmarkDataset.loadExamples(from: directory) { + expectedFiles.contains(URL(fileURLWithPath: $0).lastPathComponent) + } + + XCTAssertEqual(examples.map(\.fileID), ["2", "10"]) + XCTAssertEqual(examples.map(\.ser), [3, -2]) + XCTAssertEqual(examples.map(\.farendNoisy), [false, true]) + XCTAssertEqual(examples.map(\.nearendNoisy), [true, false]) + } + + func testRejectsDuplicateFileID() throws { + let metadata = [header, "1,0,0,0,1", "1,1,0,0,1"].joined(separator: "\n") + let directory = try metadataDirectory(metadata) + + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) { + XCTAssertEqual($0 as? EnhanceBenchmarkDataset.DatasetError, .duplicateFileID("1")) + } + } + + func testRejectsMalformedInteger() throws { + let metadata = [header, "1,not-an-int,0,0,1"].joined(separator: "\n") + let directory = try metadataDirectory(metadata) + + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) { + XCTAssertEqual( + $0 as? EnhanceBenchmarkDataset.DatasetError, + .invalidInteger(line: 2, column: "ser", value: "not-an-int")) + } + } + + func testRejectsMissingAudioInsteadOfSilentlySkippingRow() throws { + let directory = try metadataDirectory([header, "7,0,0,0,1"].joined(separator: "\n")) + + XCTAssertThrowsError( + try EnhanceBenchmarkDataset.loadExamples(from: directory) { + !URL(fileURLWithPath: $0).lastPathComponent.hasSuffix("_lpb.wav") + } + ) { + XCTAssertEqual( + $0 as? EnhanceBenchmarkDataset.DatasetError, + .missingAudio(fileID: "7", path: "fileid_7_lpb.wav")) + } + } + + func testStreamingSHA256() throws { + let directory = try metadataDirectory("abc") + let url = directory.appendingPathComponent("meta.csv") + + XCTAssertEqual( + try EnhanceBenchmarkDataset.sha256(of: url), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + } + + private func metadataDirectory(_ metadata: String) throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try metadata.write(to: directory.appendingPathComponent("meta.csv"), atomically: true, encoding: .utf8) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + return directory + } +} +#endif diff --git a/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift b/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift index 0f33de574..82e021f59 100644 --- a/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift +++ b/Tests/FluidAudioTests/Shared/ModelRegistryTests.swift @@ -147,6 +147,17 @@ final class ModelRegistryTests: XCTestCase { XCTAssertEqual(url.absoluteString, expectedPath, "Resolve dataset URL should use custom registry") } + func testResolveDatasetURLAtPinnedRevision() throws { + let revision = "1f3714b5a3f98cedef1bbb017f21bbd7ae688596" + let url = try ModelRegistry.resolveDataset( + "FluidInference/aec-challenge-synthetic-mini", "aec-synthetic-mini.tar.gz", revision: revision) + + XCTAssertEqual( + url.absoluteString, + "https://huggingface.co/datasets/FluidInference/aec-challenge-synthetic-mini/resolve/\(revision)/aec-synthetic-mini.tar.gz" + ) + } + // MARK: - Dataset Base URL Tests func testResolveDatasetBaseConstruction() { From de4c0c0326f0779bb5534bc80d001f0607a657f4 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sat, 19 Sep 2026 22:21:25 -0400 Subject: [PATCH 12/14] fix(benchmark): run and verify LocalVQE evaluation in CI --- .github/workflows/localvqe-benchmark.yml | 75 ++ .github/workflows/tests.yml | 12 +- Documentation/Enhancement/LocalVQE.md | 40 +- Scripts/localvqe-dataset.json | 806 ++++++++++++++++++ Scripts/test_verify_localvqe_benchmark.py | 80 ++ Scripts/verify_localvqe_benchmark.py | 168 ++++ .../Commands/EnhanceBenchmarkCommand.swift | 106 ++- .../Commands/EnhanceBenchmarkDataset.swift | 8 + .../Commands/EnhanceBenchmarkProvenance.swift | 51 ++ .../FluidAudioCLI/Utils/WERCalculator.swift | 14 +- .../CLI/EnhanceBenchmarkDatasetTests.swift | 16 + .../CLI/EnhanceBenchmarkScoringTests.swift | 43 + 12 files changed, 1389 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/localvqe-benchmark.yml create mode 100644 Scripts/localvqe-dataset.json create mode 100644 Scripts/test_verify_localvqe_benchmark.py create mode 100644 Scripts/verify_localvqe_benchmark.py create mode 100644 Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift create mode 100644 Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift diff --git a/.github/workflows/localvqe-benchmark.yml b/.github/workflows/localvqe-benchmark.yml new file mode 100644 index 000000000..7db008225 --- /dev/null +++ b/.github/workflows/localvqe-benchmark.yml @@ -0,0 +1,75 @@ +name: LocalVQE Benchmark + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/localvqe-benchmark.yml' + - 'Sources/FluidAudio/Enhancement/**' + - 'Sources/FluidAudio/ASR/Parakeet/**' + - 'Sources/FluidAudio/Shared/Download/**' + - 'Sources/FluidAudio/ModelNames.swift' + - 'Sources/FluidAudio/ModelRegistry.swift' + - 'Sources/FluidAudioCLI/Commands/EnhanceBenchmark*.swift' + - 'Sources/FluidAudioCLI/Utils/WERCalculator.swift' + - 'Sources/FluidAudioCLI/Utils/TextNormalizer.swift' + - 'Tests/FluidAudioTests/CLI/EnhanceBenchmark*' + - 'Tests/FluidAudioTests/CLI/WERCalculatorTests.swift' + - 'Scripts/verify_localvqe_benchmark.py' + - 'Scripts/test_verify_localvqe_benchmark.py' + - 'Scripts/localvqe-dataset.json' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: localvqe-benchmark-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + name: LocalVQE 200-example ASR benchmark + runs-on: macos-15 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + + - name: Record environment + run: | + mkdir -p localvqe-results + { + git rev-parse HEAD + swift --version + xcodebuild -version + sw_vers + sysctl -n machdep.cpu.brand_string + } > localvqe-results/environment.txt + + - name: Run scoring and manifest regressions + run: swift test --filter 'EnhanceBenchmark|WERCalculatorTests|ModelRegistryTests' + + - name: Build release benchmark + run: swift build -c release --product fluidaudiocli + + - name: Score fixed 200-example dataset + run: | + set -o pipefail + .build/release/fluidaudiocli enhance-benchmark \ + --variants v1.3,v1.2 --chunk 256ms --compute-units cpu-only \ + --output localvqe-results/benchmark.json 2>&1 | tee localvqe-results/benchmark.log + + - name: Verify coverage, scores and improvement over unprocessed audio + run: | + LOCALVQE_BENCHMARK_REPORT=localvqe-results/benchmark.json \ + python3 Scripts/test_verify_localvqe_benchmark.py + python3 Scripts/verify_localvqe_benchmark.py localvqe-results/benchmark.json \ + --expected-files 200 --require-improvement --markdown "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: localvqe-benchmark-${{ github.sha }} + path: localvqe-results/ + retention-days: 30 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 782354762..97e87892a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,8 +24,14 @@ jobs: - name: Build package run: swift build - - name: Run tests - run: swift test --parallel --num-workers $(sysctl -n hw.ncpu) + # Enrollment tests load the same Sortformer cache from separate XCTest + # workers. Serialize them on a cold runner to avoid racing .partial files. + - name: Run speaker enrollment tests serially + run: swift test --filter SpeakerEnrollmentTests + timeout-minutes: 20 + + - name: Run remaining tests + run: swift test --parallel --num-workers $(sysctl -n hw.ncpu) --skip SpeakerEnrollmentTests timeout-minutes: 20 build-without-nemo-text-processing: @@ -96,4 +102,4 @@ jobs: xcodebuild -scheme FluidAudio \ -destination 'generic/platform=iOS' \ -derivedDataPath .build \ - build \ No newline at end of file + build diff --git a/Documentation/Enhancement/LocalVQE.md b/Documentation/Enhancement/LocalVQE.md index 9f692ad19..640144e1b 100644 --- a/Documentation/Enhancement/LocalVQE.md +++ b/Documentation/Enhancement/LocalVQE.md @@ -212,7 +212,7 @@ Selecting the closer gated result does not prove that upstream used this gate. Unprocessed baseline: exact. v1.3: double-talk and near-end within 0.01 echo MOS; far-end 0.15 low from aligned float output and within 0.04 when -the upstream CLI's raw 16-bit, one-hop-late output is scored instead; gated +the upstream CLI's raw 16-bit, one-hop-late output is scored instead; gated ERLE within 0.8 dB and OVRL within 0.01 (full columns in the mobius README). v1.2: double-talk and near-end within 0.02 echo, 0.02 deg, 0.1 dB gated ERLE and 0.06 OVRL; the far-end rows are not reproduced on any metric (echo @@ -295,6 +295,44 @@ When `--output` is used, the JSON includes the dataset revision and hashes, numeric-file-ID selection order, exact selected/excluded IDs, both machine reference transcripts and the raw word-count numerators and denominators. +### Automated benchmark + +The **LocalVQE Benchmark** GitHub Actions workflow runs all 200 examples with +both variants, CPU-only enhancement and Parakeet v3's default `int8` encoder +(`Encoder.mlmodelc`) on CPU. It runs on relevant PR changes and supports manual +dispatch after the workflow is on the default branch. The `localvqe-asr-v2` +report records per-file S/D/I counts, duration and enhancement time, exact +audio/model SHA256 fingerprints, compute configuration, OS and source revision. +This CPU-only protocol is explicit; the earlier exploratory numbers above +should not be treated as its frozen regression baseline. + +The independent verifier recomputes edits, word leakage and micro-averaged +summaries from the saved transcripts. It requires exact selected/scored/excluded +coverage, both enhancement conditions, and waveform hashes matching the pinned +archive. `Scripts/localvqe-dataset.json` contains the 200 IDs and 600 waveform +hashes extracted from that checksum-verified archive. IDs are sparse; a three-file +smoke run selects `0`, `1`, `10`. No metrics are used to choose that subset. + +The full cloud run must improve recall and reduce leakage relative to its own +unprocessed condition for both variants. This is a broad quality sanity check, +not a claim of reproducing the upstream AECMOS/ERLE table or a held-out evaluation. +The JSON, logs and environment are uploaded as a 30-day workflow artifact; +results also appear in the workflow summary. Virtual-machine RTFx does not +represent physical-device performance. Model fingerprints identify the bytes +actually loaded; model downloads still follow their published repository defaults. + +For a small local pipeline check: + +```bash +swift run -c release fluidaudiocli enhance-benchmark --max-files 3 --output smoke.json +python3 Scripts/verify_localvqe_benchmark.py smoke.json --expected-files 3 +``` + +For the full run, omit `--max-files` and verify with `--expected-files 200 +--require-improvement`. Empty/mismatched audio, invalid arguments, invalid output +samples and zero scored references fail the command. An empty reference is +excluded from every condition and listed explicitly. + ```bash swift run -c release fluidaudiocli enhance-benchmark # both variants, 200 files swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json diff --git a/Scripts/localvqe-dataset.json b/Scripts/localvqe-dataset.json new file mode 100644 index 000000000..168e3885e --- /dev/null +++ b/Scripts/localvqe-dataset.json @@ -0,0 +1,806 @@ +{ + "fileids": [ + "0", + "1", + "10", + "11", + "100", + "101", + "102", + "103", + "104", + "105", + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114", + "115", + "116", + "117", + "1000", + "1001", + "1002", + "1003", + "1004", + "1005", + "1006", + "1007", + "1008", + "1009", + "1010", + "1011", + "1012", + "1013", + "1014", + "1015", + "1016", + "1017", + "1018", + "1019", + "1020", + "1021", + "1022", + "1023", + "1024", + "1025", + "1026", + "1027", + "1028", + "1029", + "1030", + "1031", + "1032", + "1033", + "1034", + "1035", + "1036", + "1037", + "1038", + "1039", + "1040", + "1041", + "1042", + "1043", + "1044", + "1045", + "1046", + "1047", + "1048", + "1049", + "1050", + "1051", + "1052", + "1053", + "1054", + "1055", + "1056", + "1057", + "1058", + "1059", + "1060", + "1061", + "1062", + "1063", + "1064", + "1065", + "1066", + "1067", + "1068", + "1069", + "1070", + "1071", + "1072", + "1073", + "1074", + "1075", + "1076", + "1077", + "1078", + "1079", + "1080", + "1081", + "1082", + "1083", + "1084", + "1085", + "1086", + "1087", + "1088", + "1089", + "1090", + "1091", + "1092", + "1093", + "1094", + "1095", + "1096", + "1097", + "1098", + "1099", + "1100", + "1101", + "1102", + "1103", + "1104", + "1105", + "1106", + "1107", + "1108", + "1109", + "1110", + "1111", + "1112", + "1113", + "1114", + "1115", + "1116", + "1117", + "1118", + "1119", + "1120", + "1121", + "1122", + "1123", + "1124", + "1125", + "1126", + "1127", + "1128", + "1129", + "1130", + "1131", + "1132", + "1133", + "1134", + "1135", + "1136", + "1137", + "1138", + "1139", + "1140", + "1141", + "1142", + "1143", + "1144", + "1145", + "1146", + "1147", + "1148", + "1149", + "1150", + "1151", + "1152", + "1153", + "1154", + "1155", + "1156", + "1157", + "1158", + "1159", + "1160", + "1161", + "1162", + "1163", + "1164", + "1165", + "1166", + "1167", + "1168", + "1169", + "1170", + "1171", + "1172", + "1173", + "1174", + "1175", + "1176", + "1177" + ], + "audio_files_sha256": { + "fileid_0_clean.wav": "6074c5e755202a26a19eff67cc985379f846d55394d21532137cdb8ee0eae933", + "fileid_0_lpb.wav": "3637ee0628d0953f77d5a32327980af542c43230c4127d2a72b4df1ea2ffb0be", + "fileid_0_mic.wav": "cc116af609a66f431f94df6b385ff2aa362f8a2d437c2279f5401e47f9178469", + "fileid_1000_clean.wav": "df149928119f40a6774fef98419704cebc81dc7a7bfbb69726e4721b18803891", + "fileid_1000_lpb.wav": "40051927c38c6ce3f1c094fe9693454240f98f0e8434760a778a5888e396c5b5", + "fileid_1000_mic.wav": "aff4ff5c6607b52ff8049dde9ad5626d99cc24502b8c98d49245786287611951", + "fileid_1001_clean.wav": "ca5aa562e440be26b4f671a5c2f691a0a5e180e06f619de384d420f707487e8f", + "fileid_1001_lpb.wav": "1c40f9e11dd6232caba0687f16dd3124e43bd828ea50c19fc789a9e61af94477", + "fileid_1001_mic.wav": "558e86200e0e5330cda71ea394f9026a09e8197f467c7a8ecd9ebcd2ee2aca67", + "fileid_1002_clean.wav": "18a4913d3ef8e382e13a4aef4ff5aaa7df0e66f194a14906720489dc3164ff49", + "fileid_1002_lpb.wav": "072cc0e736df8b0e2ea2ef9c89088dd72e1d17c0eb7777d6057148922d4760d9", + "fileid_1002_mic.wav": "707555498fd65bc6b742d66782c1a06da593ad841dd7791051dbe66cab1f23e6", + "fileid_1003_clean.wav": "a25a40bff71d5b54f7200c67cef3182ba2e057d69493703d9a26b2320c819b6e", + "fileid_1003_lpb.wav": "c012a34321467c9722ec4924c1794cd91989bba3029d155281a92f6916972071", + "fileid_1003_mic.wav": "97213c1f678789d2be66639e7999ac8c07b4dbad4215b34e93aa0f7db0679614", + "fileid_1004_clean.wav": "86d3b1ec882da82af9dba9d78aedfab4c3e39b5f69c66e6c162a4893b2a797df", + "fileid_1004_lpb.wav": "5d1eb13154d13546285ce24ff39917aee26454bdb395816b1a1d6837a1bea66a", + "fileid_1004_mic.wav": "83b3efc2ae13c518f6a4fe9d690c5574d7f0f0bcec9c7107316b9dc38031e0e3", + "fileid_1005_clean.wav": "dee3a7ee2dbbadb83b9ccf210f738a54f69afb07cc0eaf37e68e577edcdb90e9", + "fileid_1005_lpb.wav": "4849e62b30c515cc01370d1b602ba1c48ca82af09c24469a0fadcad53e6b1815", + "fileid_1005_mic.wav": "284160dd3882863f6398872d6f83baffa0a52259fa420621f6e24e14f5c7734b", + "fileid_1006_clean.wav": "93268c2cac0b6498c9c1212d6ca5be242abb14e029cf48e65ed603f6b54770bc", + "fileid_1006_lpb.wav": "c7ed177f7b3f264155926af941985c4ba6bffcbff8df490904cdde0f018c6d62", + "fileid_1006_mic.wav": "ed89152e491ddb2a5f64ecebde8c58e34c7cfa125b9b61738e97a9902354c6e5", + "fileid_1007_clean.wav": "ab12936965d5121e065b106f8776646305b5dd7ea94c6779930532bbca6dbdb1", + "fileid_1007_lpb.wav": "300a49d6ea76fe186aa794a8e05886d51c304978b7b168908061289db9287082", + "fileid_1007_mic.wav": "2af9997468054227915a0a6268e86e03eec4e4ee338022940cd2a83ce4fda180", + "fileid_1008_clean.wav": "41e19f3045f4ab843017f5c9bdd560d22f3213aa8179a248963cacfa964e405d", + "fileid_1008_lpb.wav": "ab0f6a582306e9b336d15a4f3735edc759a9ca856c972b44b3c3d7f6a9483c65", + "fileid_1008_mic.wav": "9e5298c21cd5d8c99e929d455baae0cd89e220e13f684b91ad2bc5d76e7f6a8d", + "fileid_1009_clean.wav": "fc58521a4407d098f50bb2a24d2c970967a4ff8ff4822d65718342989e71ee0d", + "fileid_1009_lpb.wav": "60749f51eff732cbc6acb2417513192443652ffb6a991325d398f97d0cc811b3", + "fileid_1009_mic.wav": "53794275d2fb2ff1cee858a333f7f81cd8b38b23114c408ff5eda35d39e06fe9", + "fileid_100_clean.wav": "1316a98c3e26f994a73fbe8cb685c4311ae8ab17a118f1158796447ce07d679f", + "fileid_100_lpb.wav": "8114eea757666e6820c3b37c389e9219e3a9a18b1dc7eb3f2f3e48f90c35c8c0", + "fileid_100_mic.wav": "19df3af8013577421459c7f0de38f24b3cdf31ed993618ebfe652e1a399adbb0", + "fileid_1010_clean.wav": "c1f8609cffb82ae3f739afd0d153854a3d0b17b718151a6b281dddb065a31276", + "fileid_1010_lpb.wav": "42f3c82693aef2aa4baa97b98aea498bb6abc536eb9d28f785ff853c7ecf9e8f", + "fileid_1010_mic.wav": "1f905a867013729e5b705b63e4c6d4cf346277eb25b7f9ec6ca047e89cb197eb", + "fileid_1011_clean.wav": "e951e70324b05ed309994262a18444b1620c27be36ce04c551b9adb68ee93c10", + "fileid_1011_lpb.wav": "9918b3ae8837e036d83a0f72d883d815b4f73092aeb0a281861b89527dde6f52", + "fileid_1011_mic.wav": "981a8c1c2914a47123f0c39748b667f2e78e85e837927707f8e08e8a945f4350", + "fileid_1012_clean.wav": "dc2a56c94fe767d1ad98c04134f8c34451577d70a1bafa3300cdbb71d85d3d51", + "fileid_1012_lpb.wav": "4ef8279f73266a04519d5c215d956dbc1b7e3e17956b64812f19a7fb4c735e97", + "fileid_1012_mic.wav": "340a09778c1cc2074103ff0f65de8635e1caff5a71e14c71b84a53d381e8db37", + "fileid_1013_clean.wav": "a0734ee2e91312414aeedadcea63f6bbcec41423422f24e4ac3629287ac50f90", + "fileid_1013_lpb.wav": "4d380418224f23a2d0fb5bb1fe7246502948e25076843d9f92b42208fb346612", + "fileid_1013_mic.wav": "153a69facc5726d5cc400da16c66dedf1a85f5dc8260f5eefaae293201208c28", + "fileid_1014_clean.wav": "ba8dc1e099dc48bf98abb6a14c0526748b29cf3b5aedbb050356453c95b33294", + "fileid_1014_lpb.wav": "fd6d95237e053b8dd33080109c5e6aca3fb2311240dfb24e6039370ec29dda34", + "fileid_1014_mic.wav": "80db366818d971df9a2b2e61421f578a9e71226703afa79ae4f3970ace82a236", + "fileid_1015_clean.wav": "6ca45de80a50b609e47670c74f6deddb0f22321acf81253829f99428b32f8850", + "fileid_1015_lpb.wav": "81df3c722c501a0472e7e1e84356815b1b9482d2f3c93c1e03167af691548064", + "fileid_1015_mic.wav": "e17de9a3ced5fb7db2c44f812d323324a7c0b40ea2dada22ae6766634727f666", + "fileid_1016_clean.wav": "23932a0e95f9b9012e4a690895346c5ec0f56cf30749c7f5633bc301b998b1ff", + "fileid_1016_lpb.wav": "3ff85876e86bd786538b5dc05ab4a7f865ab56b1d2e063a736d2874f666a31de", + "fileid_1016_mic.wav": "c96b30692915f1978edee560dd56ed38649e5fd87fb5a43d8df9d1297563a1f6", + "fileid_1017_clean.wav": "98d5f3484bca66af7f1feeb4fd610c25eb7a1dcbef26c9a00fd7952fb53994da", + "fileid_1017_lpb.wav": "620f923b70a8c5350dc579509e0c5be28a2c6bf4008a2de13f31396f3798ed7a", + "fileid_1017_mic.wav": "c69953607d014326ad2f7faa8fd35d9cf732a7d4f3d4fb7383966068bb167226", + "fileid_1018_clean.wav": "ff64ca9b39c67426006784fe2a07dc5dee78be93a472213481db9ba537d9d749", + "fileid_1018_lpb.wav": "3e08388a8642aa7d82361378bf7cbe7deec77b6dc21696159a9d0833f79f8d73", + "fileid_1018_mic.wav": "969c72a00f911862bbd3b4b01965c3864225e2f9fb8902422335d276e59da495", + "fileid_1019_clean.wav": "d40828adba12b7737c63c650fa47e868e1e8e4d40b5fbc8ec7e1844d5b428eea", + "fileid_1019_lpb.wav": "899856570dc269aed1c1054e1b90e54f27a486e1423987e27013cde4b923e035", + "fileid_1019_mic.wav": "751d76b67f60ada518e00055a9d243f84ac494eaf84f53579b061d4e73ec2f6d", + "fileid_101_clean.wav": "afc6a4da70a2c7e413e77180e27d127fc146b485b84ee1139a3508711b85abe4", + "fileid_101_lpb.wav": "c98e1576fa6a479b5336c1822509429c478d9889939f58f42f808e5d4cc37d7c", + "fileid_101_mic.wav": "c54939bcb8923decc42b26e7dfa8e77a827f3a732e6042a20651a8faed4544b8", + "fileid_1020_clean.wav": "b0423b623807358e52e2672d7f7fb6b07f5cffeb8f7cef76543e4fcdd5e9e836", + "fileid_1020_lpb.wav": "76761600e8b35a9ad0f3bc1bc5aa947f2e05ccb81fcbad37f186209f6d4732cf", + "fileid_1020_mic.wav": "9a566d413a07bf747b7cfd85bea23b43a91c2aa863c749508e3c836db4c5984e", + "fileid_1021_clean.wav": "c992093ddf83341565d99f1cb7909b17b409964c568208fe9c2092034d431ab2", + "fileid_1021_lpb.wav": "5755619d40d6a894f3c08ae8c7c5c9ddfe69d84fdd76dcacfb42f12e200e36f6", + "fileid_1021_mic.wav": "6a50ff9933cac5a1f12c74d91118760b913b48b3163dc07276760e07fb51736b", + "fileid_1022_clean.wav": "c7eeff036753fb054593c3759539d9a40d9b665005c17e596d999a3af3983e24", + "fileid_1022_lpb.wav": "179a4d5e43f2e4f46c9bb83c260a721352efdfca9d45e6143a65d4fc9e6e9ba6", + "fileid_1022_mic.wav": "f21cb20283be7360bedb7f75472e06c96a378b5423fcfe483c2422ad2237924c", + "fileid_1023_clean.wav": "728c2b259aa65c2635c27471db59deac11ecde49260797067d3a806a5824c005", + "fileid_1023_lpb.wav": "1d0c2366a532d7820407351833a56f07866aeac7b0f8716adf5284539e5843d8", + "fileid_1023_mic.wav": "62c6fa6f221c114443bb1cde3fb344f3ab8fe176ec2eb7abd0ea82fd2b8f599c", + "fileid_1024_clean.wav": "82cf78b76666aa2cd9aa6a1c616dbb45a070035933b46600a3cf1dc48139fb65", + "fileid_1024_lpb.wav": "84af32c79d829eddbd7eeba9e0f8d9a4b44478cf347c7cf38e6d0da95153b30e", + "fileid_1024_mic.wav": "cfd924667a9da98e840f97d70b32d52c35eefff2e382e031e5c5ca388b25c905", + "fileid_1025_clean.wav": "bb5226bcdbf11c627f5756bdf4f757851950324db5f99873a7a32d94fddc8b30", + "fileid_1025_lpb.wav": "bf45a5923d8413714e7bef47f4060dda2f9d62bb7a0e04578caaea4e582bad6c", + "fileid_1025_mic.wav": "a01160e6583356c063900f432b11a1a77420513b6c5c383dca0c85b12e38696a", + "fileid_1026_clean.wav": "5ea3a9c67669a295b8989dc5c30cd112ecb19a476f2f2fb49c61ec8f65d628d2", + "fileid_1026_lpb.wav": "9d72df25d68bbe6ea38b045be95facefe6dad16d620678f5460b54805d53b4fe", + "fileid_1026_mic.wav": "8de6780c3f41e0adaa4c52cdcb301d012fc65e1e2a9080a50d5e9381d8a49dbf", + "fileid_1027_clean.wav": "7757fae151fe351542ade2ec70e0721f935fc53b79b4a487109c98f543577f82", + "fileid_1027_lpb.wav": "1a9e86da1931ed306954bf4d38bdb3cb865c8920af802f10892de5d550fb4d48", + "fileid_1027_mic.wav": "742ebacf59122c7a8d443e6d48818efcf6a3294cf50146a453df1cbdf60e7824", + "fileid_1028_clean.wav": "6c941d0701e9ee78a46406ded32f31c161246cd730744fbba3a4f20bd926eab6", + "fileid_1028_lpb.wav": "c4524ac6f006647c474858b3800c6f01765ac46d9a7d0d1d7081c799c00b6685", + "fileid_1028_mic.wav": "bee93df069bc869e5655b2d12fb9a84ee3fe8711e781e667a482b93f49269d3b", + "fileid_1029_clean.wav": "c05e2f07d8f5ee88a36f5d0a2ed8bc1c5277c3da7230ca964d49d1225e838a60", + "fileid_1029_lpb.wav": "c9755f7d48bd4dba0a3dff970c823d6c689da7322fe0e4b2f725cdf01c2197f5", + "fileid_1029_mic.wav": "9002a3defc21b4e16307e7fb00155ad4b300de6b7de630b7bf110ec85931dc9f", + "fileid_102_clean.wav": "8935f518811d26959305aa0d5675d0d0b9df02bf4f02faa3f863a67a520d2c4d", + "fileid_102_lpb.wav": "9cdc9d4695c6042ce3acae3160984fc16334f64117e79731940ae05e523917cc", + "fileid_102_mic.wav": "fe68ff0267dea1cd8fd61c41a45f52371d80bed402fa8adacc7ba6c68d6ac778", + "fileid_1030_clean.wav": "0937948efc4bce60557c8b41b7801fed9fbf72dc8f13a09116506ebd7e3dfd25", + "fileid_1030_lpb.wav": "af4dc069b4c0b9da6284862f7088381df135087b27dcc55bd7621ec2eb927152", + "fileid_1030_mic.wav": "f37fa6a1e98118005d9d62991a6d230d1fcecc6091d72e8fb8543d797cd4340f", + "fileid_1031_clean.wav": "49302317572ee0b92699d91b5950825d896d65834e9544915c30423c9272e97e", + "fileid_1031_lpb.wav": "d204dd98dfb546db10bff547818701138e94e380f333e13832bcdaa121799347", + "fileid_1031_mic.wav": "f86e434ca36a250aa516157781a890a0f91341fcb94978ac981808a5ee9b992e", + "fileid_1032_clean.wav": "3100f429fd3f78818239ec026a69c0a34bd3f16e7bd41a06ce03e354eb0243f0", + "fileid_1032_lpb.wav": "03efa4c1980d24815bab148d9329bead9f5423842de84666978eaf7f4ded4e62", + "fileid_1032_mic.wav": "1fdb273c92848f2c12f285349a8f6bf1fea606d89cd43b124620f007a566e8b4", + "fileid_1033_clean.wav": "6a397741c74ec5212d216e50cc06fcfa2dc425e2517c951994422290d009fd96", + "fileid_1033_lpb.wav": "2a881b89467329854808d7ad3e705d9d1502169d993e1edb681f321adab54fa7", + "fileid_1033_mic.wav": "448b05fc076a083f39c610eef61498807c75426ad87b028b275113263b4f2f08", + "fileid_1034_clean.wav": "976c1730fe59bed5e27c77ef9ab59f89080aafc3ba9865423eebff7a13dc3bcc", + "fileid_1034_lpb.wav": "021aa1b66e3ff3c782c9bc110e5891c10ae9872d0596122ba0cc43159e954c95", + "fileid_1034_mic.wav": "adf2761005c7aed1f7fa9900f448f667ebdc99439fa69f2e2e1385bfd2f97159", + "fileid_1035_clean.wav": "4484f600614a4e47561e56ee3c74310a8eccd461131938d1ffde7bbe1b64a9ab", + "fileid_1035_lpb.wav": "59e1ac1653bb554425408b8ce2d04cd08c0aacb019722a8b26f2f387d10fd3d9", + "fileid_1035_mic.wav": "181094b8041b61ff1dcbabb8b782bcd673349c7953a0c7732f41cdefa09cd865", + "fileid_1036_clean.wav": "5e620570ce11ca0e56e9fbfa4ddcc4cb944665576fb1d4e4024aad86f7b4936d", + "fileid_1036_lpb.wav": "a369a5d0eb0ef8a2dd4b467bcb1431501fbf4fe07491aa49e2a806eb05216bc0", + "fileid_1036_mic.wav": "12eb3a113bd60f7d3de5d8c9b83e9bfee00e876c59d68985cf103a4663d8e27f", + "fileid_1037_clean.wav": "d81c1b55309bbc46462b958f51d7ea4c3b9366d4af51d8572a2c4ad4aa543e97", + "fileid_1037_lpb.wav": "6d2a031c484e94214aa7a607c085b1636482973c65b38043081d6204a4cd8ca2", + "fileid_1037_mic.wav": "41a3cc1436009b795bed45e5a3baa084092d6fe75a332897aa6c3682760cda86", + "fileid_1038_clean.wav": "31095d831580042101b17649bffaa770447f6d7b8054f293b44666351a37dd8a", + "fileid_1038_lpb.wav": "e683ccacd8ad833a67a635624470dcd08ce0a107200d64123c2266788e549808", + "fileid_1038_mic.wav": "6273caad3abf9f9605e515126e61a903511fcea4143c21c6f240a67fca356562", + "fileid_1039_clean.wav": "2fb1b73848659f39864dba20c39b280362520510e81b8c1ccdea970dc3f2fb61", + "fileid_1039_lpb.wav": "c7ec1acace8c62f728f279bdcdfd5e978b7eeb8b049f609c8c170eaf029f175f", + "fileid_1039_mic.wav": "bee6b30b22e044efc782642c1c311a6a36da493f160ec6cfa7f1bca38db56d9a", + "fileid_103_clean.wav": "eb25a31db8b5e29b9f11f81f78262f47a0154ad9440a616cfff596d6a59b8a01", + "fileid_103_lpb.wav": "6ebeb1902da7dfa53dce9732644d701977f15d602dfc6c4fb906d95c00498dca", + "fileid_103_mic.wav": "7682af2bcb83cded339351468959157b9e393eb00f85929718c455357ae27c89", + "fileid_1040_clean.wav": "2cca24d755e6ae67d816cf850344700907d70905596193ecf232a24cd47e8cd8", + "fileid_1040_lpb.wav": "7d092bb61b7d4620e87c96597dcdfc0b9450ab183c05df450917debe38fb7fda", + "fileid_1040_mic.wav": "a7e18488ce92a6dcf3f40842fa384168787b82319de1a447e765937106c9fba6", + "fileid_1041_clean.wav": "d814dff8509cad66da0978f59433bd5f7f2699a66a9e059ff93b4534e634abba", + "fileid_1041_lpb.wav": "810e056967069e3eb10ad53edc1c300ab49123b2e67ffc066d20f413734fe994", + "fileid_1041_mic.wav": "6356116268f9d850c45baaf1331722e6c90400b6bcfaa44599fbd5551b56e7df", + "fileid_1042_clean.wav": "c01ccd3200f8b79aba92878d3c04020f22dd7c847a21b429e92a9925c8fb1449", + "fileid_1042_lpb.wav": "104ef6a94ffff6fefc0b61b8038eed6f42f626becbee9ee4c20af25ea3313439", + "fileid_1042_mic.wav": "43d81c27576e9b37f60cd0d644dc1499bd60255bd5553320d75dfb1a376b706f", + "fileid_1043_clean.wav": "ba5f1118bf7cf23fffb93ea5dd3b57337e27fc73fee2cd3688611eec9ea088fe", + "fileid_1043_lpb.wav": "6304427c170ceae2e6282dd3b9857ef8148e6914314719ce7e1d4527676356ea", + "fileid_1043_mic.wav": "6777111e5248f25bf66def30bac475427d8da84e597a64bda2238a0fc863b126", + "fileid_1044_clean.wav": "f04fcb4b4c987fc8c2744ede6b9db0bf2c80c527fea58f5c9b464ad20e945a9c", + "fileid_1044_lpb.wav": "adbcfc1721867184513e5e6195bcd8c4b71a9590d525e428b2cf40d06c4f2ee6", + "fileid_1044_mic.wav": "ea94fbbffb295f780836f458b126a1ee3fd91fb71943c71b28287f7223cc5ebb", + "fileid_1045_clean.wav": "f00b0482ac69af7b18a503eb1d663ed89fe1161368688b511df1f781725ca1ea", + "fileid_1045_lpb.wav": "5efec4f79cd26280288e4c777551cbde719aba4a2a03fe0e6365d2bc8162951d", + "fileid_1045_mic.wav": "aead85aa5257bf682a8eb487e01b16e663e031ebff939749785524687ea64b5b", + "fileid_1046_clean.wav": "2069d9c9e1d4b29b046dda70b24f9530dec3c0c5f40b1f44fbc898034d5090e8", + "fileid_1046_lpb.wav": "c2e865366d3dd2fdb56ead86c1aadd95ac76cbcb3636649292fa1932c8579dd6", + "fileid_1046_mic.wav": "682e6c48ac4c782b32e6c7947d317176bc7ce0b7d8e52a6aa3bee2671d282fd1", + "fileid_1047_clean.wav": "02581306160963dc6d837f08f83e9ea1868bd91a25da1caa1cd4712734b4be2d", + "fileid_1047_lpb.wav": "6a4d0fe473e9c882c812b9ed25759ecdc045abc02c1ff396051bae72a65e08a2", + "fileid_1047_mic.wav": "5fabac67c03c02550e07d79668f47a22ee236902150ebcfb58327e9e080341be", + "fileid_1048_clean.wav": "15fbe4d81663cde54f3accc04b2e2cc7ed4e44217ee9c40b169a390b220a1555", + "fileid_1048_lpb.wav": "e8bbbbc2cce7bc9a449618f1e9ee8fe78d3f46ba7762a8119fe4241488df54d1", + "fileid_1048_mic.wav": "89f5407d43e1b79641e8eb8de2086e88693d61d5d4cf51e3bb133c218a2b9b30", + "fileid_1049_clean.wav": "ef3b6749031294757429eccb24790720168d58a0f0d12794ea5716d067f19e19", + "fileid_1049_lpb.wav": "b1908dd2ae2c87745b4c0a671a2f3c669f088368fbeee23c5cd3a50de0518213", + "fileid_1049_mic.wav": "9a8071df87941ab90807da2ef55085996bb21937a7ea333f94b0f37cb72ee1a8", + "fileid_104_clean.wav": "198040e5b26592ac3b5d3db9b141396903efb171e49dd556596cb564f512b0c4", + "fileid_104_lpb.wav": "7a55a501548485476c591498324c109a8e86d52539516e6ed8fd05f2bc3c0dfe", + "fileid_104_mic.wav": "67dfd2a88f7402f2c95ac60dee9bf88b66477a32ba89a13dbccee18a90943c2a", + "fileid_1050_clean.wav": "21d30fee3d2b9e9ed1ae2bd18cc0113f78d0a40f75d4153d051ed08c4eb5a878", + "fileid_1050_lpb.wav": "1073c6633b92675f651a978fc722c57164d6a24aea5536b54fb1fa481e952176", + "fileid_1050_mic.wav": "0694d6066a3c0a0300f33a5d1b10eb62a14b44b9f2fabc20406a3abc3895b783", + "fileid_1051_clean.wav": "facb83c97b6efe7f69ee728054df3106f3ef8928ddcb27338ad531775d8fb03f", + "fileid_1051_lpb.wav": "c403a9cbf2df6b77452a359645b4fce935862b0821a2d46013e75a3b95d21be6", + "fileid_1051_mic.wav": "d69771dbd44dd0efa03ee038490685c83b7529f0a9c4fb57c2ef6c6fbb3ada2c", + "fileid_1052_clean.wav": "26475bc1bffb6b81ec798d4befcfccb0a00f2265d40507ec8850cbfe75fa5832", + "fileid_1052_lpb.wav": "1886b1caff1bbefe22bea3e23b4c7577430da776559078cff0478c2c7feaca11", + "fileid_1052_mic.wav": "5d3ebac73a4adafa01dbb6b660eb9dbbf720d268cebc3d8127f86e429d811f7a", + "fileid_1053_clean.wav": "9e6b7a8b6b65f9d4b3634d2a400bad0af09b9a14fbc4e5250958866fabb549d0", + "fileid_1053_lpb.wav": "f3c9041fa69ffb61e970b2e557d32755430be020a1067fbf3c2ab61990ee4f69", + "fileid_1053_mic.wav": "7f51ae8664168c5a8b6ea20ac841ba75812a4063d3e22a452d8c2cc10dc8a9da", + "fileid_1054_clean.wav": "7ca4c8253ea1f673f41c18f3fdcc015aeec71032544a78b074c1e6658a35b78b", + "fileid_1054_lpb.wav": "826eb144bbbbc1497bb676188adee5fd3614a53a4ee34b827d0753aa6bf80eae", + "fileid_1054_mic.wav": "5ea6ab83a083ff52af638d1bae3e5207a5535ea7f14cbf0cc818163d47dab4d9", + "fileid_1055_clean.wav": "afe62f37e89c7e3532df7287b2a5413d35baa7ff27ee0f752afeaf08f9ecb337", + "fileid_1055_lpb.wav": "cd4e3a78a87007336c994d13ba6a50158f142aefa12719ced07633464f4e98d2", + "fileid_1055_mic.wav": "c51132ad94140cb9363811c4f81de0b0536c97fbb67935ccdd044ed496249993", + "fileid_1056_clean.wav": "371d6b4e42c285dae87c9d5c99c9338aa6cfc84a4b20f84ea56841726d99c554", + "fileid_1056_lpb.wav": "a3d9942e1f4b989b8a8b9465a76aa777b414a8c122ffcf6fe7972fe437f7667d", + "fileid_1056_mic.wav": "770effa6bf1911de48f2d56becad80b34f62ab2bb22ede0a6ca2eb7928f085a2", + "fileid_1057_clean.wav": "8f5bb8cba32106b444c8c898a83ecbcae7915e82f973e9bc57c98cee1e6e2a7a", + "fileid_1057_lpb.wav": "d429cefc91ef371dcb0f6cf5bf54a8a6061ea940b0dbe49add9ac64722739fab", + "fileid_1057_mic.wav": "22c9658a6fca36382e4e9e3e347c686053e8cc783de48e91d69a4ef9b4a0a23e", + "fileid_1058_clean.wav": "fdc10c7400e825c52f008ca6e6d6dcb24c787b8806fc8bda035625e4dd55728f", + "fileid_1058_lpb.wav": "ec36fc515356c23227cef01193e363b0a71630e351295a22d716feae2c5a7312", + "fileid_1058_mic.wav": "74f751c7530acd81d6e89ddda380eb0a82983e7445f51e64b2741ff3d322e3d4", + "fileid_1059_clean.wav": "45bc44d59193a18b7716206bf8e032fa4758f1c094a40a8595e98c0af582a16e", + "fileid_1059_lpb.wav": "0d69c8410d39417a0913d9b305035a0dce7b56fb0036634adf9ffeeae5db300f", + "fileid_1059_mic.wav": "a161c88bafbf914ba4409513096120fd9f9496c7c50c531336a73eb0f83fbabc", + "fileid_105_clean.wav": "97752c88b62de2bfadb65c9906fb4d4a1315ef98e63214ec6b91897b1a321f5f", + "fileid_105_lpb.wav": "2f6c829708977e0637346323ddcd7172eeee96941223c50c206b5e536cbc579b", + "fileid_105_mic.wav": "61dd7309156c8b1cfc2110171ab274eda53569f1a5740f0d55a8f9ecec62ea11", + "fileid_1060_clean.wav": "19a1635ba18a96ddd0fc3925b398d2b617c836d89da90e6f1a7c68044c57322d", + "fileid_1060_lpb.wav": "4f64bb155fcdb80a334f6016d3c403d6b90425f2aa980f6f572d2d328414b9fc", + "fileid_1060_mic.wav": "441351bb62ac6c56e7bb370f4ded8d901874ebf095a464fe5c14ced649433c63", + "fileid_1061_clean.wav": "037623587109bd56075cafb0b750459f9e863a7dda3b75fd679e3522ebc9d151", + "fileid_1061_lpb.wav": "e8853ed46726978e13578dc69e29a3b41a498c3aa7bd25c0d34e1438e9a61475", + "fileid_1061_mic.wav": "c1666c8cf6248242cfb7fd4c8d682917ff27100b8b40a6ae01c3024fbbea1043", + "fileid_1062_clean.wav": "080a1298e54bb8f9889dfb340cf1f2164e74b245b91aa919cc9c569cf3181397", + "fileid_1062_lpb.wav": "b9f29b16fe5201cfb4f77fab0267a763a5229fbb5f470b1008244a94aff246ea", + "fileid_1062_mic.wav": "d79aa2f6fed5e4c47f7a9492b4436890727d11656f244709ec0a32d7ebfcf029", + "fileid_1063_clean.wav": "58a045845cb6697d157472b73935241dffd39068d01ac3fea0ce27bcc7494ebe", + "fileid_1063_lpb.wav": "d18c7385b0ab4287ede5751d79898b6b30d2aada6bdc5e779e37b8c64c9e278b", + "fileid_1063_mic.wav": "3588dcc36f355c9a615bd30000aa2d662b9421aa561fe557f70703862b95c6f7", + "fileid_1064_clean.wav": "4ee2f44e9b63a2813a65d8e204d9e72baff5fa876dd1c6b00ca177c733558d20", + "fileid_1064_lpb.wav": "caff14e5caa22b0203b149b6902251ce648deac653f7b372fddf5e8b85700280", + "fileid_1064_mic.wav": "19f51293e767cba79d156a97cde7082c200c81cf1d273ff59b854d0f588a8aec", + "fileid_1065_clean.wav": "924d64fa1898d9e8c6f70b53b4e989b94307c90951f5c92ac4b1a9e137cd474d", + "fileid_1065_lpb.wav": "309cf065306acb0daed450083b8b2036f1856dee8ded23827f2b0356dfe5c23a", + "fileid_1065_mic.wav": "cb4b51d7d61c3eba02a36abd0477b8787bb12b1c00450996d8c1a1ef23ff8f54", + "fileid_1066_clean.wav": "5584c7d70bc769b8d98fcfd6cba03184f7fb21bc54cb635c7f746d4dd0ea09d8", + "fileid_1066_lpb.wav": "ab90704fcf5886d5fcb208beff1a51a69c81acc8947a03bb8fa822c28f24e234", + "fileid_1066_mic.wav": "021933245add0b7188e97f6dd3106f117f919b527169fbbcfe7cb07d9c10fb00", + "fileid_1067_clean.wav": "55a91d1c5f47a4e6fe48eec329b77b1448f948fa4db4630b751d232b641f023d", + "fileid_1067_lpb.wav": "3bd7079100f02b158d4eb740dc320ce9c15f6ff8de6d5d51ce7121b369f62c93", + "fileid_1067_mic.wav": "57fb7feff6b746f4682678a4610883e430504a388e1f7cfefae74d6f8a266a48", + "fileid_1068_clean.wav": "c0c87bb875286e3f707ae4ac847fd5ad21fd555c2c10a0627f72877eb205f097", + "fileid_1068_lpb.wav": "02b280440b263a772bfdc4436d5bcdf0faaefa321651e874a91b9ec0c19d305b", + "fileid_1068_mic.wav": "a8699cdd97d8339e95191600d9ddbcbb55c6cbf912f9c8b92fa00b514be846a1", + "fileid_1069_clean.wav": "a5efe1e237505e7d62fbb7b76d28c949fa34d714a07e428eadfcc75a23d23c49", + "fileid_1069_lpb.wav": "bf46c6ccd07aee4d1eb9d1848b2a2cb09877b9f3dbc33cad847e592d0ee880ab", + "fileid_1069_mic.wav": "c0ab7b786aac63713906baf92a3cd32cd0dd1896c2cff08f0be4fcaab52ef319", + "fileid_106_clean.wav": "f43ae25ce5c05138cebb53e658d797d58c09bed9f1e832187058f95353965119", + "fileid_106_lpb.wav": "0779fff3e7a827dbfb3bd7abd506192a64c2cc3ee8d4b2e77e7346319d736f5d", + "fileid_106_mic.wav": "3414efd19565fd3013ec85167737edaccd5048fc9d0b4d0156fb68bb306ba07f", + "fileid_1070_clean.wav": "1293cab2ecc846b0e6fc3f1fa41ed3c59290ca154428fbb01d2c6bf02ae561c0", + "fileid_1070_lpb.wav": "63995cd92181c2d4968493dc2d1f0e49313e7daadc523ed04e1ff4756800ff24", + "fileid_1070_mic.wav": "8de90a26ea3375239f04c96762f0d60e8a549398124f724b779d610ed6fc3784", + "fileid_1071_clean.wav": "ddae1ddae40bb1247f42d0efa39113d7b119b51fe03dcf855761127c31ba8135", + "fileid_1071_lpb.wav": "505dd232ae0011baf01f3ba5c9eefc84bffff4ecc8afdd285b40ff39563367ed", + "fileid_1071_mic.wav": "f397e1a73a16016aa691caf273dfd064344addf7b2219d0e691bd7398da8ecdc", + "fileid_1072_clean.wav": "a6913f893fad3e2bb06e70aca7c7492abe6ceddf2fa87728eeeec824aa6bb9f9", + "fileid_1072_lpb.wav": "246293cca85d3e1f418a4219fd649a0c495c2f842d26a7ee171e841b78385903", + "fileid_1072_mic.wav": "bf3981698459220dc1afa9ccfc15a8709b8f181df385e88e83a49f1551ff34b7", + "fileid_1073_clean.wav": "9621e9518dba91264bb672ef54c822e531bea7d7e253af868544ec4972a880fc", + "fileid_1073_lpb.wav": "99d2247ef4b1b306d0af0c97e00fb3c7fc9d00e5d9b5c4a999ce24fefc3a3302", + "fileid_1073_mic.wav": "7e30c3d228bd69c28ba10da3280fbb609130e2e22c5fc5e76e3c03c4a104ce4f", + "fileid_1074_clean.wav": "e852b24d91e40f243e4e02b5aadb80f8a38b5a14041903b8befc1b4b82ffdf67", + "fileid_1074_lpb.wav": "cc1ef61a4e49a3dd611560d1ab34e53754ec4cf0f5feb47f47aedf9270ee535d", + "fileid_1074_mic.wav": "c44dca278f25f6569234d5b5535afbc73566e1fca0b1d778c29211e6fd789445", + "fileid_1075_clean.wav": "4a68947cb0b02d28b3aa9550aa5fa1428f250cfd21261a8747d97c6c89ee6362", + "fileid_1075_lpb.wav": "705e2f0d1fe965ab9e4bdd4ed866f7747c57f0a37f021a5bd4f7e6d250a99c2a", + "fileid_1075_mic.wav": "a511dedec96e370efee18fea6933f920c554d714f70551843b29266392e7628b", + "fileid_1076_clean.wav": "67d5c980b6b66e911c14655165f3cb91ce15779ae84103018f9f0dc6b2b6222b", + "fileid_1076_lpb.wav": "90ff48f98175f3c0f0bf3f778d08c903255c437e6c220425d47b392638e62bb4", + "fileid_1076_mic.wav": "19a23bd396e3217b18cf6177a03b435b3d1e68d1f636252d088b488a4cfc564d", + "fileid_1077_clean.wav": "d0f382e41e1daac5ae16a5ee224d7f552d9da73e1a32360ae6819b492a56d387", + "fileid_1077_lpb.wav": "7c170ac12ff6e00194d331ad8025e2cc8e4946efa99e77c3a379539a2cbc90c4", + "fileid_1077_mic.wav": "b450521f43c0c8cc0712c092046763d5abd02b82a75c385f373f821e71f77615", + "fileid_1078_clean.wav": "91992a2c3582e0fe50140bfe9c8674128d297a5088318cb299bac6b7fcee5b9a", + "fileid_1078_lpb.wav": "adb8fce55e3a629b205bc7789773a7bfd85d03b90f42794146926d3a422a93ba", + "fileid_1078_mic.wav": "e4da2976ed7a2dc6f0d498ceaeec4d3c342b419b4a81a1fdffe9d2a6e617e095", + "fileid_1079_clean.wav": "a757129b6911e79d11404ce160983044b057e19c6d915ab05efc4bf43d026744", + "fileid_1079_lpb.wav": "fe3b2c126709489a05f346e15e0951cfcd2016f92d9eedcfe462508460660d35", + "fileid_1079_mic.wav": "0206319ed5133b0b134e1f1fbec5c1feec0c281c3b8161ce2b5d528f6364d97c", + "fileid_107_clean.wav": "b8193f3c281320f15ee764ef3bced562a493d745ea19679a1d14e9c5a0d7f2a9", + "fileid_107_lpb.wav": "0a2c2bfb2baa917097ad8d90e644e3d45068dd761587a7d3bb886343360b2969", + "fileid_107_mic.wav": "1c655c9cbd6e3c572ce60df6dcee54fc6483abd1e3f0c8960a77996410d85c17", + "fileid_1080_clean.wav": "b9645c4903593023d9adfdb54584c4bc33d30942f1ccea57bf2ce9b414bf66f6", + "fileid_1080_lpb.wav": "705bd3229e350bdceefdd10e3cd68fafc472749bd73a4886bf3dd675816ba24b", + "fileid_1080_mic.wav": "77da4d1bc0952081de8d620cb48467b361c0108284258e5b326428adefdfc922", + "fileid_1081_clean.wav": "6c0458d7724461663fea96e3fb2e7c72469390e7431dddc93dc6c2f2422ceb6f", + "fileid_1081_lpb.wav": "6893c0eefd17636670b2d84b98a767816d94b4f754b3a5bd64340e33139a0ed7", + "fileid_1081_mic.wav": "42b993bcb5a86508a6ca27e6ec42609b99acfddcdb286184cf422b8b59789e29", + "fileid_1082_clean.wav": "c488383e67b53cd97222828f742569ec7499ae7a9df297f72cc83b7537a0f8eb", + "fileid_1082_lpb.wav": "2cd8fdd39ccf4a73d2fa064538a5f6e08e09aa215b13525e3d6ebf489bd6f762", + "fileid_1082_mic.wav": "a7962ab355447525057b2975b9415907a82e52c627a69e96354420e7a171571d", + "fileid_1083_clean.wav": "acfef8cc57693d32ed097989b6c518ac69e077ac460febc570a7e48ffd607156", + "fileid_1083_lpb.wav": "83cc5d08e03449ad930577c7c692c5b1375b3f24a19f06c46cf27706946df565", + "fileid_1083_mic.wav": "fc0073905795e27c65a641f5713e3cbc60a0656eface654478061e66fa60092f", + "fileid_1084_clean.wav": "05c4f787954e4c2993c72b274fc6f266b8ca5002d21b24d9ecdb89c8b5d63761", + "fileid_1084_lpb.wav": "f05770c86c4347a77523a1d3dc1e622966a1ab78f27ee55451cbf03d5cadbea7", + "fileid_1084_mic.wav": "1cb79fa3d4eff2bccdf2ffc76342cda03783172acf25718f754cc2113f9e48de", + "fileid_1085_clean.wav": "5fa3bc3d70342b0fa2314cdb4e087ee34fdbbdd50fc6b2098edae851102a0593", + "fileid_1085_lpb.wav": "1885234542deda496e84bb57dcc4963a027f1f104d338f3a1ae7c109f3de6465", + "fileid_1085_mic.wav": "9e734e1ec86ed30ad419b5cb0d11102f8ca81481cb4e834fcfa3b72741845e20", + "fileid_1086_clean.wav": "5350ae2e40b4a5bd5ed5bf6375ee7592bb4a13150cfd8514fdfdc2f4203a5fba", + "fileid_1086_lpb.wav": "d8bb37b0b9d9ba89e939490445a58fec1dc5aeb025ef6850cb71f969a8aa487d", + "fileid_1086_mic.wav": "ac23a3dd698a7d1e939590dde03951dc38c731e83988d3339c0d0d5d4c1f01b2", + "fileid_1087_clean.wav": "3a765ad301b7e979ef348f5790a93e277b9a40912b6b46400002186d04f4f6ca", + "fileid_1087_lpb.wav": "17fadd083fd06c5ae551048ec6d676065a1ecf9ec2d49f854f46b3576e084dc9", + "fileid_1087_mic.wav": "c086617c814ca0a211a43d61ee363172a33e5b120abd7e24e6128dd80b21b587", + "fileid_1088_clean.wav": "b31bcff0c36e50867c98a9bac6b119dda49a6981e953de6b89f21e3f844d8285", + "fileid_1088_lpb.wav": "4f9990c07477447d26cd0d9fc34c81eaa89b860f9832c0f469876f539d13e88f", + "fileid_1088_mic.wav": "9bfc0843624dbe690bc6dd5ba901dd062d3849dfd0af3edf408cb79e605a40fb", + "fileid_1089_clean.wav": "0dea7b2a7cad07e4b1f1e12c376bce17f87a11d74f0b4333f78726dcfddb9d59", + "fileid_1089_lpb.wav": "cc4c8ef210baf01a7280f184802f0ce80c0c124c88e157eb142e0f52af223ed2", + "fileid_1089_mic.wav": "853ae4d0338236527706205931acfaebf6b7d5a048f908a74a686a929502aa54", + "fileid_108_clean.wav": "006ac74c0fb2157ec039efbce94f1af63c6ef05f3454904b6c180c846f49e0de", + "fileid_108_lpb.wav": "c8c0b465d4fbe59e1e9e4db14313f33e9e42cf6ab225079d3c573ee6377b15ca", + "fileid_108_mic.wav": "1cec55ee9473c7c6f393583d5a563cb4318a146e3821398784bcc98828057aa5", + "fileid_1090_clean.wav": "4d878891c8492f55310912c790bf8b5edd9a63e49261a864bfbdd25ef5526555", + "fileid_1090_lpb.wav": "15ef0b86c9943a8ddb2fbd5356ec03a4446c3851b6921e08eebced14e7c30f68", + "fileid_1090_mic.wav": "642befc79fdb709263b00d36ca870772903e909b1ed736a0b5d3302f528e26c7", + "fileid_1091_clean.wav": "b22f769ce9d23714919b1c6a85f4ceeac55b8cd3aa1ae54b1889d024e39e5f1a", + "fileid_1091_lpb.wav": "d92919c490208e5982d78077e1c3f3b636aaaa4e06174ff70713a144f3835f75", + "fileid_1091_mic.wav": "cebac45935b2cc02c0afe9e90d1b7c14e0fbb09137eb52fa2b90904e782ce09c", + "fileid_1092_clean.wav": "f04947511722eeb0aa0c5c6cf8b1a36478b312b39bd070f8e7c47f9dfdff1a28", + "fileid_1092_lpb.wav": "42aabe9940b427be8493fd246ad2c87b537941ad0dcc4e8397eae65349b78633", + "fileid_1092_mic.wav": "c6bd208978244feb259d0321025a94006f12d7016d8218665ada2b64056b1f18", + "fileid_1093_clean.wav": "54a9cf1ba95dc043d955bf97a7ac2a34f7198d0b4567f95e02f49bebb2d6d7b9", + "fileid_1093_lpb.wav": "87c23502e73bc60676a4d20824d715e1b9cc9d377e9260f6ae76979952018113", + "fileid_1093_mic.wav": "d60b1cc9e0cc7838c3c0d38d63e263f4d5a330325636d1ad9e995478d4b45cba", + "fileid_1094_clean.wav": "6f9a7da40c332efae4392f3a8a463b15f09ac410edb4ead3162dcba4b415519d", + "fileid_1094_lpb.wav": "4c28bff9f26844fb78ceb3313eaa70572398be17b8cade627314a280aaa7d62b", + "fileid_1094_mic.wav": "fee568cb81af44615139fdca8d68f7fc92da570b6a1d0d6c3b9fb76ae33e5c6b", + "fileid_1095_clean.wav": "f6e175faf3a5fea3b2117d95a310ad6538a12d29ed0409e8171388247eeeee8d", + "fileid_1095_lpb.wav": "a44054078642d9ab3601e48c0a0abcadd6913b044935738c295b7bac5667f2b0", + "fileid_1095_mic.wav": "70895160e647dedea412a60b0d91a434287f176b6796090460e7cdc16ae96b5e", + "fileid_1096_clean.wav": "a3fc951d9f357db199fe45f91da0c2ad77b45aa246fc7296b672eb32712aa96f", + "fileid_1096_lpb.wav": "1373e2348fad852eb95de4fde352891a2b1f03edda4095d25550afa9fc980d81", + "fileid_1096_mic.wav": "2516265e21181b05b23c0df267264d8e78ef6495079737235c1bf174648df0da", + "fileid_1097_clean.wav": "fb112d6e434a05073eac602f6e8414639709c9ea5b26f9ee0b8deff6a55f8117", + "fileid_1097_lpb.wav": "6635ce70853665b51a20a11e021211738845ca56a329934c4d22a64e4443dacf", + "fileid_1097_mic.wav": "a45e3c098f12241b19a3cf28af3a207242b89e4b2f2e9b575db82ed6eb649091", + "fileid_1098_clean.wav": "db63db050d1021ba29cb4d0b34f5db109ff8496daf196ee5159c7459fe0a4361", + "fileid_1098_lpb.wav": "78ed029cf573a8d3dab9ef9de56355f00831c03431d5d5174f91cdd91038dd07", + "fileid_1098_mic.wav": "29a8492ce4d36e15635e1157f103fccaeac345d0f15ac6f5f162f6665e811b26", + "fileid_1099_clean.wav": "6d9e4e0d2274b520e74f8994b6779a09e58de1fff5ec00a7e1fedc94b6e0bf3d", + "fileid_1099_lpb.wav": "e6049ed80d41705ac00cdcc892df5d1314dfe5c22f93b9051d5abe54ad2434e0", + "fileid_1099_mic.wav": "5b63091442c2c4bd39fe23d9b945ee06a3c7ec6494db81773c9e529c9ca5406b", + "fileid_109_clean.wav": "155853a1b8a4e2398ad85fd651fe54d81fa851eecdf02ce815efa35a55174308", + "fileid_109_lpb.wav": "4286379b6dd4e72b9b3d52c9366b5c090273ac3e7e4fc3c0fa4f43e9499e6fde", + "fileid_109_mic.wav": "7c19a257e3dd90e9231c279c5cb8c81ec4c3edf51ebd4b342d93db8a8b9128a6", + "fileid_10_clean.wav": "5ce8e7d40bf374b62d1a8a83149ddd559de87b41522aaceb1a3c360b94b8574d", + "fileid_10_lpb.wav": "c8c03c6011451dcf3ddefd184fa1dd79461468876b25c8079113511b793626dd", + "fileid_10_mic.wav": "7b32880a339f8c1875bded912bc89222783cdbe713b826bae84e476c03f506dc", + "fileid_1100_clean.wav": "87896acc6228e55a68c71d35d458b0f390bdcf30c0db148d6db7849c7c855ad3", + "fileid_1100_lpb.wav": "c3c12a3ed79618c0d6c0bbbc5f8efb2bdd8033d18dfd80771bafde8429cb1a46", + "fileid_1100_mic.wav": "9379fc438f9dcd6c4e598229ce683a7bc9aa83c4ff30b2c239a120ea74403b56", + "fileid_1101_clean.wav": "dbaedc565a2ce1db1a48c77d81615770a82e4f8dd5601d62856a940fa7440cc6", + "fileid_1101_lpb.wav": "fe36d5ac9906f5e42c3c8f4f32e333d897a0e14ee8bd172e464d3b8e7d1990be", + "fileid_1101_mic.wav": "681fa1cf30e0b0fc144dd47513803c50810b2144f0bc1afab72288f52362c8ba", + "fileid_1102_clean.wav": "5f44e130fe5cf875201e344b21331662fd923ae211e49335f7b0d7dfd5c01fa5", + "fileid_1102_lpb.wav": "5549e0f9e645b652f0ff5bc68333fbb4210886655ba423fffb860fb39b9e33db", + "fileid_1102_mic.wav": "f1244747aebd7add7686b7dd8030f612903159f66383675387cf120cd679c796", + "fileid_1103_clean.wav": "77d0eaaec01ea3db6cc8c9cddc4911b2e8d9b036068f31ca1f0c17b78b84679a", + "fileid_1103_lpb.wav": "221a07ac3e299a0b2ae57a710015690e254d14196b76f434bc23afecbef9623c", + "fileid_1103_mic.wav": "98fbfce6d8c7ccd3b83540cb4de19a3f5642b50b0c92e4967e1c79b38031f968", + "fileid_1104_clean.wav": "5abcd2d4c8a04f5474319c6015ba6c56cde25006845063f2190fb8d1aee112b0", + "fileid_1104_lpb.wav": "33bf1c848715326c2d68877d4f934feaf9999ec2b3840e6eb8396fd5c66c47e0", + "fileid_1104_mic.wav": "e52618a9c704c84f35c60617a49aa68113446a2ecaaaf8f967235aba962ab445", + "fileid_1105_clean.wav": "fb291aed5fe708d8a6950e03d90f414a7ae33207e5ed6620da005cb3e8293857", + "fileid_1105_lpb.wav": "6f91640ae06c97377618511a641b52220d7334b1bced728c698e7e675a6a0eda", + "fileid_1105_mic.wav": "032ff3e61b707645b379d591001ba062e455384d836240a7260a563f74deedbc", + "fileid_1106_clean.wav": "d61023b38da589e4fdd893b7b7107e583e0dfbd12b087c0b23a8c2b2529a6901", + "fileid_1106_lpb.wav": "bd0eb6c5c59581478930a3b9b2cadd97015e08f82a45e9836a92e93e65a06bc8", + "fileid_1106_mic.wav": "8f3e2379cec2d2d2ff1a71a93635d9d05e7f861a39c60a06bbb4fe91b167ee0a", + "fileid_1107_clean.wav": "55d56320b74fbaf77cb1ebb3ec47d24d838567240bcf1f9d4d53aff7834d75ed", + "fileid_1107_lpb.wav": "197ac2a36d4f495989305a191aa2070ad6ad1d8e37ce6fbc696e09fba8f0aee2", + "fileid_1107_mic.wav": "784b1ced5c1e2f6e8182efdbb7f168c68d1effb70d17ff39cb70c87cc5a2d02d", + "fileid_1108_clean.wav": "b6aa12ca0f7eccbb2ba9c9a9129891bbd21176d8967802a75562c57972637d7e", + "fileid_1108_lpb.wav": "2baeaa74c1d39706adc30aee0a7b7b0b909d161dc51851643d62c81642c0c605", + "fileid_1108_mic.wav": "518fef69fc2ea1f7b66deb358451e88985612dbdc306400d0dd7c594368265ed", + "fileid_1109_clean.wav": "c3118f611d3f26f69a1fb9e2558ec94c46a7f5f3d2e975688c5e18a5d20c4457", + "fileid_1109_lpb.wav": "677950efd8c3eccb6bfdccb09b768803f501263abc7777eedc5476f42a97a588", + "fileid_1109_mic.wav": "3d0303ca7ca477beb04e97427339317e812405a465f3cadca76a2ce19a4d451b", + "fileid_110_clean.wav": "3d97e13f4ad79b7a1dd0903921e91dba258a4760f5a6ad2294f317c93ce6bb1c", + "fileid_110_lpb.wav": "5697fea8353e254e8419f36fa96a0f34e86245d22c534ea11c09b9a7c2fc9640", + "fileid_110_mic.wav": "fb6f31c761706e608ad107943a27a8a92eb3819a18dc04b43c3aea6579d3ff9c", + "fileid_1110_clean.wav": "8988e6890fa3e958770433d90c0339b57cabd424c301a48b3f23042f4dd6c4c1", + "fileid_1110_lpb.wav": "5ec1e361a59c4064332df2797440f0d6d63fc83176feaf04fc42b547c5e6770a", + "fileid_1110_mic.wav": "d46772bf070f74098a1e08d401e2906450d5a9d4ee084b979c8078d0152e589f", + "fileid_1111_clean.wav": "fe4ebd955692f1e36839a12bfd9df209f90a0bd103deca8478a8ae951447ca5a", + "fileid_1111_lpb.wav": "2037d1e5f68edf434adca8f96c0ffb8d726348b1e195432d1240f84ba98f3cf7", + "fileid_1111_mic.wav": "0c4d8aea35f3465983168e6fdc5afacc6739b7f6ca21671d6d3027983e90ca7c", + "fileid_1112_clean.wav": "8d4d4c9b7583ee0fb4aa4ef7b7b6cfa347780c5616a63bf23a3069338ee7575b", + "fileid_1112_lpb.wav": "4cec72d53edc3cde87de3161f30de2479506901711a0a07d9a6dd5090febddd1", + "fileid_1112_mic.wav": "0224144a4f769e57d75f2c2b6b381d435940f5a3c98a0ae3ac0c2354d6bbe907", + "fileid_1113_clean.wav": "1101d637adb91229b8bafb2c6fb5f0c6532583bf602eef9462d34e1d29c33107", + "fileid_1113_lpb.wav": "f0f32aa99ba89e18e9fffd258276469fd3b53f49c33622fe5aec6e2f8f54f860", + "fileid_1113_mic.wav": "98e98e1aafaa0864765589aae7ff036e0fba52213afe6498dc9210ea856decc8", + "fileid_1114_clean.wav": "199c1159382d11f12a9fe40b6adc8c56913177f635cef06617d111bab9e1cf0e", + "fileid_1114_lpb.wav": "8211c02b1dff9e584237a730ff53d1d404d2c93c7ae2c222c26a638aba207bf6", + "fileid_1114_mic.wav": "809f51b2f0c65f38087651ef0d65654117c1424685e08e3e24a6a7aa112aa3e0", + "fileid_1115_clean.wav": "552ed6534ee03e271cd160698375fbd796c8cce1643bcb5109d63d791ea392a9", + "fileid_1115_lpb.wav": "db9e54e19ed2b5ab715362c282923bdefbfcdaa3d8f7d5a61678b74b91863464", + "fileid_1115_mic.wav": "7cb339e7690a350bf7322b3fdcd17eb891162b3de84cb627b35149dd6b9029ff", + "fileid_1116_clean.wav": "74ff3209ca3691675ca81c7456a706ff436af4964ec969fa8884cfef2149b51e", + "fileid_1116_lpb.wav": "c6572116d85281133891e3d8496fa51c7defbf6e43108e3093a5064616c58cd6", + "fileid_1116_mic.wav": "c335d0ec054fe9a89225e02a49ef73c718ac9bdc282b52dee721d59409c4f20d", + "fileid_1117_clean.wav": "a7b28407f1278cb89da05aa5565ed1402589680e858789ae62ece03e5156520f", + "fileid_1117_lpb.wav": "03299b16d970075490a54ef448c6b7afc11c7f852f2998d72d7f76cb1ee1a0f8", + "fileid_1117_mic.wav": "a28237bb7836a9fd76f0ce877616925cd27bc31e0a68bf7c43815dcfe48c6cc6", + "fileid_1118_clean.wav": "8149c9ab99888e6b32e90eb6b3b6d602289971824949bd927c51cd0b69b70cd6", + "fileid_1118_lpb.wav": "a33ec46e1358ad494523c44b3e1955a50ff18cb5aa8dc0d370e818574ca96c62", + "fileid_1118_mic.wav": "c5f79924203b82d4b9a4d87974be788c799f1877f9a8f53bd50e1486b49b7c14", + "fileid_1119_clean.wav": "2ba03b7e559beb5df462a73f2a8954073028f4a39edc6dad667f55eed8ca3774", + "fileid_1119_lpb.wav": "2a7faad0b4feecdfbae2f165b18b1101708512759f688f56b20e55ba424cbc05", + "fileid_1119_mic.wav": "71f19eb8704fd96fd321771d5adaa818e1bdcf41b992d2d2721376614b2a56a6", + "fileid_111_clean.wav": "411b117f320cb94e9d1524060e35b3fa925deefd6765c812791b758684bcbafd", + "fileid_111_lpb.wav": "d4dc8e820907f65fe52ebffa81577eebceb4e0ba73a3f810439f15d4854f5e23", + "fileid_111_mic.wav": "e873412f0c196ce651b042eda9dec89f62d51e33abbcc6124500bd6e71107836", + "fileid_1120_clean.wav": "62bc61d8dc741dd1e9b883ec35532992ebb6d7b7791d1d49bbf9eb7b4c9d99a9", + "fileid_1120_lpb.wav": "1e94f520466a9f022a26aab4fdf0f8610bf4b5b5a38bd3e1bb880b8d1c63d85d", + "fileid_1120_mic.wav": "8d458028ae9153df47c0b34f518dc7fb62a7550c5fe48d573975687e6e493de3", + "fileid_1121_clean.wav": "ecf7e58aea8bfa075bf539d5c234bbeeff5943e06c960bb2c7240676f79d6a63", + "fileid_1121_lpb.wav": "e067698e887ee66c97aaab74575ca18af1cf0eeb7af661ea2e9a5b5568d6459d", + "fileid_1121_mic.wav": "f9a2fa2ab3038febf11b980f021004667c8d63b420c2a9ff7e1acdc8ef41033c", + "fileid_1122_clean.wav": "69d9831b0597587ce3284482af02905879265264adec7e80732abedbc1208d3c", + "fileid_1122_lpb.wav": "ced1da5319071689ef28bf646a6e03963242a88b143292ca47f6889a69c24b22", + "fileid_1122_mic.wav": "a4a339cac75c3156258f9dbc70afc99a3933c5e687f0e3942e06c53a0dcd7b45", + "fileid_1123_clean.wav": "32a6d1313b80720df0c7173e12316b809d967850e05ee8c73325b4a83fefe08c", + "fileid_1123_lpb.wav": "60e89cdb5aa5bcfeb8d7598e0e96291504c6e37ea81443052ebb884b0e456c2f", + "fileid_1123_mic.wav": "f4a1b52d565ca261d14d0a06c347a4c60e2af14a54594cdcabb4aef973ba8c2c", + "fileid_1124_clean.wav": "31f7ddb726ceb972dbe542bfe15b16549ad7a1bef1ee2f25244ca98c0b81df54", + "fileid_1124_lpb.wav": "417c2a9131c9614588efc4422889258bba2517844737d49fbbbb0b3976745b8e", + "fileid_1124_mic.wav": "6dde20a9d91dc1d32a47d80ad3a58189e37a5ae91924ea89853e7f5eb4c69d17", + "fileid_1125_clean.wav": "9a3765953d4d4ba45e9488053a242047433e0fc166f2994cf531ddbd780026e5", + "fileid_1125_lpb.wav": "463400e9ab70bf7c6d300d25185c2b0d1f68540de01d275e79ce08b1066cf966", + "fileid_1125_mic.wav": "043c6b183d50f482ede636a3877b58ec75e73f0e847bd4599f2d685c5f23ba1b", + "fileid_1126_clean.wav": "4aa1f91183f73d4d41ba59e6f35b9fc0d04168937859c7fc89936979ddcf7a37", + "fileid_1126_lpb.wav": "bd46fc0571a794374ea7b043683420bff53990b79414b273222e21fe4f455ef8", + "fileid_1126_mic.wav": "874ac36765a97c7e8217bc0dea2a6f3cac52c8b9e8593f7bf6afb3a1c18d6b51", + "fileid_1127_clean.wav": "b90a900fc0345ea7b37dc10f6bed581673db53503e4a9214ae24958f9405626a", + "fileid_1127_lpb.wav": "8009fcf4d9f967ed1160f969444e05530cc20f3561ceed6534c8024cf9dccd36", + "fileid_1127_mic.wav": "a3fbc8b65ebb8d973f6ad1b9afcfbd74179b8fb8986ea06fa83b315683fe6ebd", + "fileid_1128_clean.wav": "485e63e025df2da65300f84ba0a734f1f8dd57a16591e534cf8505b68886ac1d", + "fileid_1128_lpb.wav": "c54727b01f27ecf606274d6dd01ded16f2b96a2e0302a26eff86a49cbe88ca65", + "fileid_1128_mic.wav": "d03b4c408de81e29a12a7ec138ce33b8fb3f2c03a947eb8502419b28492d6998", + "fileid_1129_clean.wav": "3b4c7415f564aca0c41bbf7856470630b2aab0e97eaa8aa775f3fbddbbb4f594", + "fileid_1129_lpb.wav": "54f61f008a2f8d18bc68902a7f0e3c429f0f11a0531a82a88e7fe9229c8f35a4", + "fileid_1129_mic.wav": "cec0c8a792dcd28811260ecfd1f01de0258b8bf0a31bce117f1942c8008d7484", + "fileid_112_clean.wav": "4eb2d70c50deb60c53f158467bb12dddd437e5eb00fe7e3db82af3aac4b32198", + "fileid_112_lpb.wav": "000dbe81ebbe12e300f34aa217bbad56287eb41a49e66b13befbb35d5d43941b", + "fileid_112_mic.wav": "50e7dea2f76b3886c7108300b69093357f3aab8307ffd5ed54c2c55fc751917f", + "fileid_1130_clean.wav": "a77542170eb2ee3b868999a4c8c5cd2cada47b0392aab6f16347f3c9c755cf95", + "fileid_1130_lpb.wav": "5ba65643ec90aa0ea04470e2ff2e3cdeb717dedd92c551e98127dae5d2343b22", + "fileid_1130_mic.wav": "9794abc52e296e08144cc9674c973ddadfc5db8f212459cc797b3a27f657eb45", + "fileid_1131_clean.wav": "48fa2786d4b5356380a651d0303c09ad4d67fe01d12ec40c5435680907789959", + "fileid_1131_lpb.wav": "0fc423ab650a48aa9f3f414de782646973187d253f392ae5a9cb2728193fd073", + "fileid_1131_mic.wav": "ca6fbd5ef5ae6ea8a65a4046fa160cafb3b12688780122bffded0893e116cdb8", + "fileid_1132_clean.wav": "97019401a9dfe5f44a4686a668e3ba51df41e6cbc887ef9986ad6a6482298045", + "fileid_1132_lpb.wav": "bbb68ca6c6c01717d197c7b6fbf28940eec5526fe0c5a5dc8ad8014cd724d233", + "fileid_1132_mic.wav": "46b9d3627dbb43fb49eaafea3dff15a8dd8c835bc5a30915e5a1cad1adda6f19", + "fileid_1133_clean.wav": "09f53bea4960640e0424e9713af5350f572a44a9b6e60d871dd2192ab3bd6dd2", + "fileid_1133_lpb.wav": "ba7ba979949d2139749920031f32a8b41f67b28b76ac13a8cb5a733762a6e410", + "fileid_1133_mic.wav": "ea5406cae373f831997d67e75878fd2885bcb38aff5aaf15a2dc7f31cebe75c8", + "fileid_1134_clean.wav": "a4b3332fc2111297b783f7f028a344f4c7f101bb48631ad4b8cb749e26b4a2b6", + "fileid_1134_lpb.wav": "22c61ed26f4b9ac38e3cd60741854d5d41ff0eaba6ac5008fe3528f9d0961909", + "fileid_1134_mic.wav": "d01113f22c73afa5c9fed415cb9d20e8ec4469b7454ecc41492a463ad29c3113", + "fileid_1135_clean.wav": "a27078db327221c122c7ff6cc0760ec3eed71baf976c28e429efa718d22db17b", + "fileid_1135_lpb.wav": "e50d3f90916b565919f06a33e4c9f5782e11301db7436281a2f7f70396c0bf15", + "fileid_1135_mic.wav": "9dca9b84e2cba17cad3c22b19810c681f7fa2138ea75a464d43068e7ba6a4d0a", + "fileid_1136_clean.wav": "9479db265595b026c89ecb8574af3776081ee625dfbcb0844a11e89aef7a5452", + "fileid_1136_lpb.wav": "687b0eaa44a6a0e2b8b95350f4177a27fa6a460392fdee5b5359836a08b95e1f", + "fileid_1136_mic.wav": "bfd769ffeedfcd63885056e95c068637fcaa4f809a06a93c7e569702dcf87139", + "fileid_1137_clean.wav": "4b9ceb62537b8570908feaad2cb7285b3da8aeb4630f49986ec28acb3c727a90", + "fileid_1137_lpb.wav": "1570e613fa9c78c1f3b3f7ff7f6f261c3913015cf77f3f00ef7a53ee5eb853c0", + "fileid_1137_mic.wav": "89fd747639a3c19a337aa9437aac286e917d3fcdbd4025839de2e30ef37a3fe5", + "fileid_1138_clean.wav": "3e709767fd556e19bd155ee4c85b87a197c50bd13acfb6e6c507879f9ba995b5", + "fileid_1138_lpb.wav": "c135f68b802f67b15415af0cba8672c1709df2d4ea9c81b28fd7c10d7aad798d", + "fileid_1138_mic.wav": "1da224e0965a2db35cb0b00170c35a0a1daf86c45b2ee16ad2ad0c67be5ea087", + "fileid_1139_clean.wav": "da3f46237c11a58eb61cb8c11edd7d43ceac204103fa2c9714c4b4fa27ef3909", + "fileid_1139_lpb.wav": "17ee79fff94d81d1e7c65fdd555c92350f9a99a6f80c449c2bd58c43c84e47d9", + "fileid_1139_mic.wav": "efcb1a001fee66ce179f69c710d080b9e16f41c357dd54c50d6ecb84944cb33a", + "fileid_113_clean.wav": "c2a2cb6cfbbb9153eb41cd8dd5d5b698225bd1c0c3adcaf1f8c125a244abe537", + "fileid_113_lpb.wav": "c68e6a3d58917134ffc5f700224bf772771dcd69b6e89a508c9b7946b046a626", + "fileid_113_mic.wav": "efa29df1ab577cf993ef4f99fba19379b58fb9b8c4cfa9aa61dc34c6d5855a8f", + "fileid_1140_clean.wav": "de72bd4b896811bc4a82382a41d74350043f1f5639e0678bcd4ef05315c7141d", + "fileid_1140_lpb.wav": "d5226512f7916a0b7cc06e26dfe67ea3d4a61aa4ca8c2b0c41c266535937753d", + "fileid_1140_mic.wav": "1560cf21835eb3cffb9a55d5129ed1d8ef95b28eff1aa92245a2f09ca652bcca", + "fileid_1141_clean.wav": "080d2d06d71386208e4d92538b042506ccaae1d4d8b9d9de57cbe2b31ecb8764", + "fileid_1141_lpb.wav": "4e1d46cecf0e79ed9d0e83d62376a34f60d59ca40bfefdd230b8e115baaec2d2", + "fileid_1141_mic.wav": "d7316cabc7269891c3a9d8ccdb776d8241b2dd5ccda20624ea9ab105ed6961dc", + "fileid_1142_clean.wav": "c494d186d673e4aea1e2f8a596057a5ed282a76ac6ac57e29253581ab5675113", + "fileid_1142_lpb.wav": "f26b4a2ffe72803db460b0e986b2972037e90de6cf4e01e8010514d226be98ae", + "fileid_1142_mic.wav": "a866c9687dac8b603d181e62a4752c3490585ceeb91a983a0bbaa9b4c2f1c8cf", + "fileid_1143_clean.wav": "a7c1b54338eae8c90c6969a197679dfaa41d5334b6151d64cd439d649f0a5407", + "fileid_1143_lpb.wav": "299a82461b1ee291453658bb165b93334752d15b4bfa9bd464184f858ec5ec96", + "fileid_1143_mic.wav": "9b0fee1a96ca303f520c8231e166b953805c7723c9ce5d5d67f01de698ebc5c1", + "fileid_1144_clean.wav": "92e6aac8e1ad80af3b91246838579867362f2777c524b440d6a0f595bc9086c6", + "fileid_1144_lpb.wav": "d574e0b3ea45da8b0834bb1be72f585e355a4ceef0354412b26004e5be8c2903", + "fileid_1144_mic.wav": "b6df0f61e925d6663c64bdfe40b2d6ee25cecb6f8eb6951932a7bc202a42aef9", + "fileid_1145_clean.wav": "dc53422a43800b2830395e543e7b327daa8a9afc8b3bae09be80c328a413e526", + "fileid_1145_lpb.wav": "6a2c3d43fc7a2817066867c340aa039013208f9f9fb7a7f9c742964bd87cc586", + "fileid_1145_mic.wav": "104200b6b7135c2449d6720684b198f31cbd94e0a13d8bd2febf6594032d608c", + "fileid_1146_clean.wav": "6eeb7c20a61e0e8abbea39b74f5e3db9a69a4cb2099830927a71e779ecbd6470", + "fileid_1146_lpb.wav": "2b9524d4e300a6285e570062013d2c990b2da43864afc38b8d93b5feca9fd66b", + "fileid_1146_mic.wav": "0a4fc9062fd52f8fcbb6292dbd0c45b243c40f7ac1dbafe3db9b50b3f47b7619", + "fileid_1147_clean.wav": "87da8f2505e362047851cfe7e282623a38d9d2561d6039c85913b22784691a44", + "fileid_1147_lpb.wav": "21abf2d4beed1ebae45019217318ab6ee26172d7f9892b3d074a093ea8bdbce9", + "fileid_1147_mic.wav": "8796002d63f18caafcebb7b8f345c6124aff6bd14b66b004ea807876aafe1ef8", + "fileid_1148_clean.wav": "0d061165fcca9ea216c589f9195afb39a57569562c1a796b36c8c68129b9702c", + "fileid_1148_lpb.wav": "f93144365b00d4011b0aef28dd96f3298fabd3643f3d9be42ffa8601fd383fe6", + "fileid_1148_mic.wav": "3799451a8526aef99d0dc7d3f807fed06a901b721c644816518db8eaa1c4ceb5", + "fileid_1149_clean.wav": "9a61905e4c7dbf0c0220abc4d606be6dec39ab56094638cb9fce087f23d0e1e5", + "fileid_1149_lpb.wav": "8ed2f4b508408485399c08f46fd43305db0022198f831bfcc813c40dd25776f7", + "fileid_1149_mic.wav": "86390395de1e8067f8b6bb68220edf59d0565af64a5373de48dbb1bed8b55326", + "fileid_114_clean.wav": "9bedd557c55fd68d9527d070a34e92091e74d0792797c97245958101a925f715", + "fileid_114_lpb.wav": "7a1c995ed43c8221237e6a68fae574527f586df8d45182fdd1c3998ed667c9fb", + "fileid_114_mic.wav": "35aebfa9ba2bf64ec3d0e3d5d48a4f2f9fcff527b6aa57d9299ed693c2450225", + "fileid_1150_clean.wav": "789c4a8dc099d4065978d37bbb9dd6ab033e6cc34b0948c455faa6fddf8b0246", + "fileid_1150_lpb.wav": "98d5f246afd3e45a6384b207bd1f3bb7de8c20845bd3b7e144243972bcf7e477", + "fileid_1150_mic.wav": "98f81d40344ed801c5cc51656c464b07966588ebb0c580a0bb633e547b4162a9", + "fileid_1151_clean.wav": "2b53ff04d899c13aa232dcdc1e65080daa4cccc6e68352b6e8a852fad737f280", + "fileid_1151_lpb.wav": "51d7a4a56b288ef75c7024c1f3aea59e26e09634ce5f1de30f66821b39c25ba3", + "fileid_1151_mic.wav": "ab9dbcb6c8b20c4216fc8f81e19b97ec7ba45dfe099f724da8665414780f39f8", + "fileid_1152_clean.wav": "7762849d866144a17fc93889ed00d9584cff47e6ad53ea7d6f20728da60456db", + "fileid_1152_lpb.wav": "3371e65cdb685f53771fad4d1fde73a380b4e0d4b8b13bfb5f584a927af3b3ff", + "fileid_1152_mic.wav": "c9d0ba3653814cfa914b60769ce427c90517c66bab700c5169042cde1c618ac5", + "fileid_1153_clean.wav": "a901d3dcb88ff6819140fe2f6dab229e7b056e7a4680a4be2ba5c4b9b33b2e2a", + "fileid_1153_lpb.wav": "53262fd0a25a70217646e37db52331fcd77a2fea804d106a446242e77d7b592d", + "fileid_1153_mic.wav": "fd8510509ecff825d6b614378d6367a5fb8edd55ecea09d6197c2fa0aa2655ac", + "fileid_1154_clean.wav": "fde0a5573c708e2a6407d4a1af1957802cc0e2376285d1900ef0ed72b8ab44de", + "fileid_1154_lpb.wav": "7a2f05884441905fbb19bf1293461842b2887498a86c83a5360528fca16ce65b", + "fileid_1154_mic.wav": "ac5e2b422aa915005d15b848c1d37183bf5d118353eef5e92d15bfb0e73a6c66", + "fileid_1155_clean.wav": "8f4e1f54a14710fe55eeb8b2c0213959a322ce09d3069ee153353ee26980cf78", + "fileid_1155_lpb.wav": "58cedc4b9b8197d185267aae5f7a99f12b41447871792192295464ef5f16f224", + "fileid_1155_mic.wav": "ceff24cb6aea89bbdec57bc8a04ff9c2911a518c14ea6a914022aa394e138adc", + "fileid_1156_clean.wav": "81cea824aca20cfeb19f5635f945199218ecbacb4e1fdf8e584a2621e75dd57d", + "fileid_1156_lpb.wav": "e1d352063cdcd676dbed7a0ba98ba218542599c81ced2166199aed0d0ec81ffb", + "fileid_1156_mic.wav": "057fc27a17c5d8ff628565680359f6ec72300f93a945a390ca5e97b2bb81c2bd", + "fileid_1157_clean.wav": "bb17dafb1553f4f401d42675bca2739246fb7837f2ab7c1fa9bea31ec88f9e53", + "fileid_1157_lpb.wav": "ee3b0b3482049ffbcd7a69dd2d226de89b81da3684730d236cb7ac3830708a34", + "fileid_1157_mic.wav": "a475cc89cef6e7d8e0d6604e8eb873432c2dd0527145aed825fe3af7d909e387", + "fileid_1158_clean.wav": "54455126d0695c90e716e7d4ad57cd913df65391fb81275c32bad8f22bcd0517", + "fileid_1158_lpb.wav": "39ebd077ad156688e3d527d53658b3b7811b8da44d1b849eec4601eca8a9cd60", + "fileid_1158_mic.wav": "9e8655d29cf37edaa170a992573ec4df93a566a5f6579aebaf163769ae3151b1", + "fileid_1159_clean.wav": "ac807d5f86a17f475eb27d0428113e9fbf50f67ab6f46ffd770d3ff1a8402d65", + "fileid_1159_lpb.wav": "973e207e21d1665ea7a320f8d10f50f674c40a5dd8e2869f150911e5dff256a9", + "fileid_1159_mic.wav": "62d86c19d523a6a2790a311c070f48e86271fe9e5e4ad3e5e576b1fe27a9ae8e", + "fileid_115_clean.wav": "d80a661d8b26dc8b03d3c7f68f8bba7b7cee4bfa10b059d7be2aa22f2d782a09", + "fileid_115_lpb.wav": "aa9b78ddf82c472fa76c7ab29ae4e87bd562b741c4b7d68d55341bf7ab07522b", + "fileid_115_mic.wav": "eeb3ddc1d9476776e6dd5a11e47862f079cd1503143e6ec22a187be957a5a8fd", + "fileid_1160_clean.wav": "cbc1f3c0bc846da7a0c4699b674e54326c6548219ee051e13ecb9160867142a6", + "fileid_1160_lpb.wav": "f5dc5eab3527cd15341e94cf99544653bc351741b6bd543753da296ff5e8a43f", + "fileid_1160_mic.wav": "3f5e1318d2a951bb1d35fb6d9b6eb5e5c78fa079f1ddb504d33003b88c93d6ac", + "fileid_1161_clean.wav": "76ee6485d926622dbbca8d4af0ae11ac8e31664220da68808a3207897287b93b", + "fileid_1161_lpb.wav": "99a4dbca675ff752dd1b4da68f359de24e1728b6da5878b60ffe4b347cf48dd1", + "fileid_1161_mic.wav": "ffc3dc224bad20d19f8e56aacc8ae47ea9c72be5b1a245a15df2dd5d56bcd914", + "fileid_1162_clean.wav": "26887989387f2d5bc4932a6b7a35db263cd61b8ab1977bad00635e1b041ca4ec", + "fileid_1162_lpb.wav": "97a71e6ff4c814edc10696deca077290009ee47dc7fcba8206f4cc3864c2dbf1", + "fileid_1162_mic.wav": "10a7d8ec87db34128e6e1b7b630c75611a4ead4b289881ae75cefdf2f843daac", + "fileid_1163_clean.wav": "ee1c493fe71876f8a5bdeedb913e2ad009906892704aed546f8e94861f900394", + "fileid_1163_lpb.wav": "1164421f8a5675d43bc03be7f48498dfa71d604a80d7e72a29324976f35b718d", + "fileid_1163_mic.wav": "2110161cc42d2bff44e3b70ab7376b3779a35e70fd61261fcb375ca2bad5c8f9", + "fileid_1164_clean.wav": "e1e99de07afdbab74b3ab1e7108af73b71ee4b432ab7253bfdfaa6bfc924ea4c", + "fileid_1164_lpb.wav": "b2bebc1a61f9fd702fbabf33ee5dad9997b7601ddbf94f2031596cc6efc20d66", + "fileid_1164_mic.wav": "8d19c7de86700766b95452245584a03b2163c6f13953d8e3e2497076583c92e5", + "fileid_1165_clean.wav": "7a3f1913d1aa70b565f1e40704533cc8d79c47831eca7d142ec9efa9c1e03cd3", + "fileid_1165_lpb.wav": "4c311bcc2694ac055e27f765f2dcbbb6490fbf151a432ba772a04eda8fd6857d", + "fileid_1165_mic.wav": "9f0bacd6d714746af1a72a8c2b30e47a59b5f98b572db7dd5ebdd36fd93db593", + "fileid_1166_clean.wav": "2c8f305e31a4b8031c02f438575bf043037ccc7bef7a8824fd9bcc91b0b77e69", + "fileid_1166_lpb.wav": "52cd27ab3960b741fccee4655d998bc857e5a55df748aa50c8dbba4824b8d046", + "fileid_1166_mic.wav": "fbd9902c5589a5a4ec701f1a31c0f50e7295d0bd739e0e81ca4221f9c4f5c22c", + "fileid_1167_clean.wav": "f4fb8aacb274c762671755633d997f2d208759a5be299df92202c7b34addc826", + "fileid_1167_lpb.wav": "51ef54b683a8801b23b7e26ca6f1e8e476e6f2c14d267fd2a03b99bfb2bdf2d0", + "fileid_1167_mic.wav": "35d8ca030508bf56643915308d1b2ea1f75a32ba814747b92e8d631fc76f8ede", + "fileid_1168_clean.wav": "632c9c44fd77910f7a48a2fdc474ed0aa6766c0047c8a45598a2f6530f32af59", + "fileid_1168_lpb.wav": "852879eaad6a0e98ece322174bfc7fd7bf6a5703d2d2a274e7cbb3cb6848f435", + "fileid_1168_mic.wav": "2280d3fb7ca69ac8eb7d36d0cf7173715644f1c8100abc9a906fc7266cd6c0d2", + "fileid_1169_clean.wav": "572cc4e4c9be0f978ce1baf0e93c220bcce0a592ac9a307fef51862bcab838b9", + "fileid_1169_lpb.wav": "e666afd95aaf7e8fd0adb5de16a2192cd759fa1b9ea1f44598be06a94972dfe3", + "fileid_1169_mic.wav": "7748ceaa67fef81f4a2f97951af75a6f70d1098bb9ecb4ec995ab95eefe8efd6", + "fileid_116_clean.wav": "6e444139a8f4e5116b39996d8852751221a4b37619c00bbf001217dfc6b45bff", + "fileid_116_lpb.wav": "93c305fdd75567e164f52ae9d0dbd077dd8d9746e69a7f0e43da9438a141f543", + "fileid_116_mic.wav": "3972936d7040fbcee5ba21547749afc669adf94a9172e4e6d3a172c2978551e1", + "fileid_1170_clean.wav": "0dcefad8d4743c77037d9d889233809f174f04fc95162428516e2856fc1af18f", + "fileid_1170_lpb.wav": "da4f90a994c47e2ae30cdf2e24ff201f61a4db19933b693e1f3d759132038be0", + "fileid_1170_mic.wav": "5ad71e7cafbc479a3f178aecbbaa4bf13015817e2cdd83bb38ff26d0a03d879d", + "fileid_1171_clean.wav": "6258f04a350ab9d0bc6f15d0c8c0e466c288fff1aec558acd432304f9012f2fb", + "fileid_1171_lpb.wav": "2cef1499dfe0a3a92eddd3d97ad2e1b90229b0ae4930f15e4f999abcf0b55664", + "fileid_1171_mic.wav": "6c3f4346519f9022624d1b70e89881745d51f4b1f2aed46f1f96e7ea26459338", + "fileid_1172_clean.wav": "87497ad175ee4d8f3ab6a00b8ff49563f87440f50899bf7fcf9810b60b6167cd", + "fileid_1172_lpb.wav": "6d97789dda417ebf53ac0e956a794fd8d05ddb569e66c140f277491d3a4b9c54", + "fileid_1172_mic.wav": "7d2c17d05c7c2407d6da0eb5d64c32de7a92ad0ea049c29632c5a59419d5e7c3", + "fileid_1173_clean.wav": "f82399e42c2f64763eec90371e8accd32ec3bda1ac637878c0eb16ac6303e161", + "fileid_1173_lpb.wav": "22cbab46edc21124115bc706a1e2cfc65d589f4d53044e613d7eb6c6afdbaf82", + "fileid_1173_mic.wav": "2de4648c3799c101b1d1f9548ac783bb70e9b64fd2d67940fa480abaa53bd7a3", + "fileid_1174_clean.wav": "e7f7d43f0f4dc1b279f0c6befc9ec9b1494e5a0ff26eac18f656fcdbe01dcd21", + "fileid_1174_lpb.wav": "07a8205f1b937f0caa581441c8adc98b4dd7137de1103e7b7e21c7000afb5e2c", + "fileid_1174_mic.wav": "03a47d1479b29c2268cc58c2cc07add54a2834927dd4f3caf03a460f38193cfc", + "fileid_1175_clean.wav": "4b94971dbe498e05ed93ded39fb02dba9b49d2be0780c6cb332e8b978d1ebc28", + "fileid_1175_lpb.wav": "2622493ec9ff5f0a4b3897397d7cd885a2a09f69c21187eb97d62ec2faa8430b", + "fileid_1175_mic.wav": "e5f9e35c532ee1804c20dde90a0f9482bcd671cedfb49a7e8ed7a3ffd5089dbf", + "fileid_1176_clean.wav": "1507f78c09c750c4c7026ef245dbb22b510bceed0195c7c988fec202b01aa6d5", + "fileid_1176_lpb.wav": "c338c8029a5cbd4918ad9072379ba4e6e430bfecebfddb999a75ab142a61e196", + "fileid_1176_mic.wav": "b6eaa3a11af01f767284220f78a00fcc8032891818773ba4d22f76f1c57ee206", + "fileid_1177_clean.wav": "e83ee02ad8cd2bbcbb7ff6d2272a204b24a917263da073aed5274eba0ed29ad1", + "fileid_1177_lpb.wav": "6beac20575e116ff6f08c61235caa0ab15094a8a36dc4b80082a788ced391576", + "fileid_1177_mic.wav": "54bc0fec07d27a9e6c46f71662e4340416cfc0e90d5c5f99aff3329ab6f1b854", + "fileid_117_clean.wav": "31c9351ae3180200aabdcedc8fa339f5419d13c070bc2c9a77c3e971d84fee01", + "fileid_117_lpb.wav": "bc39bce02c16c2ef2848ae30c2b6fe0b457582ba045824bf7c33a00548c70374", + "fileid_117_mic.wav": "45c149c88465efcb3a0de5c44937eccd0014c5c0d99dda6d96a27dc6a395f75b", + "fileid_11_clean.wav": "62e17274e94f90f96fb23b15f3ecd0fabc4fd85298198cc3856ee9d669bd77dd", + "fileid_11_lpb.wav": "fcd58688c0a8d206b0c647ef3d92d1cbf84676b4cddd5312eb107aecf77cfb83", + "fileid_11_mic.wav": "d2c6ddc7709e0e31bf59e79c4321c9dea79b1d725a4e1092372b2e3f7c33b47b", + "fileid_1_clean.wav": "d632e692c48928f62ebe613376f0875216751c282f5d6a89b97e1cf619283cf0", + "fileid_1_lpb.wav": "d24e74237fedc60f271244a882d83f9c4088ff467c4f056ad01de94b495ed571", + "fileid_1_mic.wav": "39d0a1d1c1faaeca7180bb47ce6cf0cab57404541c530b8b56dea6a57ca1a331" + } +} diff --git a/Scripts/test_verify_localvqe_benchmark.py b/Scripts/test_verify_localvqe_benchmark.py new file mode 100644 index 000000000..8e9d21617 --- /dev/null +++ b/Scripts/test_verify_localvqe_benchmark.py @@ -0,0 +1,80 @@ +"""Scorer tests and corruption checks against a real benchmark report.""" + +from copy import deepcopy +import json +import os +from pathlib import Path +import unittest + +from verify_localvqe_benchmark import edits, verify + + +class EditTests(unittest.TestCase): + def test_empty_hypothesis_is_deletions(self): + self.assertEqual(edits([], ["one", "two"]), (0, 2, 0)) + + def test_empty_reference_is_insertions(self): + self.assertEqual(edits(["one", "two"], []), (0, 0, 2)) + + def test_mixed_edits(self): + self.assertEqual(edits("a x c d e".split(), "a b c d".split()), (1, 0, 1)) + self.assertEqual(edits("a c".split(), "a b c d".split()), (0, 2, 0)) + + def test_substitution_wins_ties(self): + self.assertEqual(edits(["a", "b"], ["b", "a"]), (2, 0, 0)) + + +class RealReportTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = os.environ.get("LOCALVQE_BENCHMARK_REPORT") + if not path: + raise unittest.SkipTest("Set LOCALVQE_BENCHMARK_REPORT to a real enhance-benchmark JSON report") + cls.original = json.loads(Path(path).read_text()) + cls.count = len(cls.original["dataset"]["selected_fileids"]) + + def setUp(self): + self.report = deepcopy(self.original) + + def test_real_report_passes(self): + verify(self.report, self.count) + + def test_missing_row_fails(self): + self.report["files"].pop() + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_wrong_summary_fails(self): + self.report["summary"]["localvqe-v1.3"]["recall"] += 0.1 + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_wrong_edit_count_fails(self): + self.report["files"][0]["localvqe-v1.3_deletions"] += 1 + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_modified_audio_fails(self): + name = next(iter(self.report["dataset"]["audio_files_sha256"])) + self.report["dataset"]["audio_files_sha256"][name] = "0" * 64 + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_excluded_and_scored_overlap_fails(self): + self.report["excluded_empty_reference_fileids"].append(self.report["files"][0]["fileid"]) + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_missing_model_fingerprints_fail(self): + self.report["model_files_sha256"] = {} + with self.assertRaises(ValueError): + verify(self.report, self.count) + + def test_missing_condition_fails(self): + self.report["conditions"].pop() + with self.assertRaises(ValueError): + verify(self.report, self.count) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/verify_localvqe_benchmark.py b/Scripts/verify_localvqe_benchmark.py new file mode 100644 index 000000000..88e19b612 --- /dev/null +++ b/Scripts/verify_localvqe_benchmark.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Independently verify enhance-benchmark's versioned JSON; no inference needed.""" + +import argparse +from collections import Counter +import json +import math +from pathlib import Path +import re + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def edits(hypothesis, reference): + """Levenshtein S/D/I, with the benchmark's substitution-first tie break.""" + table = [list(range(len(reference) + 1))] + for i, word in enumerate(hypothesis, 1): + row = [i] + for j, target in enumerate(reference, 1): + row.append(table[i - 1][j - 1] if word == target else + 1 + min(table[i - 1][j - 1], table[i - 1][j], row[j - 1])) + table.append(row) + i, j = len(hypothesis), len(reference) + substitutions = deletions = insertions = 0 + while i or j: + if i and j and hypothesis[i - 1] == reference[j - 1]: + i, j = i - 1, j - 1 + elif i and j and table[i][j] == table[i - 1][j - 1] + 1: + substitutions += 1 + i, j = i - 1, j - 1 + elif i and table[i][j] == table[i - 1][j] + 1: + insertions += 1 + i -= 1 + else: + deletions += 1 + j -= 1 + return substitutions, deletions, insertions + + +def close(actual, expected, label): + require(isinstance(actual, (int, float)) and not isinstance(actual, bool) + and math.isfinite(actual) and math.isclose(actual, expected, rel_tol=1e-9, abs_tol=1e-9), + f"{label}: expected {expected}, got {actual}") + + +def verify(report, expected_files, require_improvement=False): + require(report["schema_version"] == 2 and report["protocol"] == "localvqe-asr-v2", "Unsupported protocol") + config = report["configuration"] + require(config["asr"] == "parakeet-tdt-v3-int8" and config["asr_compute_units"] == "cpu-only", + "Unexpected ASR configuration") + require(config["sample_rate"] == 16000 and config["aggregation"] == "micro" + and config["normalization"] == "TextNormalizer.normalize" + and config["empty_reference_policy"] == "exclude-from-all-conditions", "Unexpected scoring protocol") + require(report["chunk"] == "256ms", "This reference run requires the 256ms export") + require(config["enhancement_compute_units"] == 0, "This reference run requires CPU-only enhancement") + dataset = report["dataset"] + require(dataset["repository"] == "FluidInference/aec-challenge-synthetic-mini", "Unexpected dataset") + require(dataset["revision"] == "1f3714b5a3f98cedef1bbb017f21bbd7ae688596", "Unexpected dataset revision") + require(dataset["archive_sha256"] == "45ff5d7acfce499558c25a0eace45eb819cec8aa76420fe733de7ee116ae548d", + "Unexpected archive hash") + require(dataset["metadata_sha256"] == "865aff8e66eb682c292f42a9d747d931f3a2f71f18e16fec53aea80dbdc2eacc", + "Unexpected metadata hash") + require(dataset["selection_order"] == "numeric fileid", "Unexpected selection order") + selected = dataset["selected_fileids"] + canonical = json.loads(Path(__file__).with_name("localvqe-dataset.json").read_text()) + require(0 < expected_files <= len(canonical["fileids"]), "Invalid expected file count") + require(selected == canonical["fileids"][:expected_files], "Incomplete or reordered input selection") + excluded = report["excluded_empty_reference_fileids"] + require(len(excluded) == len(set(excluded)), "Duplicate excluded IDs") + rows = report["files"] + require(bool(rows), "No scored examples") + scored = [row["fileid"] for row in rows] + require(scored == [i for i in selected if i not in excluded], "Scored rows do not match selected IDs") + require(set(scored).isdisjoint(excluded) and set(scored) | set(excluded) == set(selected), + "Unaccounted or overlapping exclusions") + audio_hashes = dataset["audio_files_sha256"] + require(set(audio_hashes) == {f"fileid_{i}_{kind}.wav" for i in selected for kind in ("mic", "lpb", "clean")}, + "Incomplete audio fingerprints") + for name, digest in audio_hashes.items(): + require(digest == canonical["audio_files_sha256"][name], f"Audio differs from pinned archive: {name}") + model_hashes = report["model_files_sha256"] + model_roots = {"/".join(name.split("/")[:2]) for name in model_hashes} + require(model_roots == { + "parakeet-v3/Preprocessor.mlmodelc", "parakeet-v3/Encoder.mlmodelc", + "parakeet-v3/Decoder.mlmodelc", "parakeet-v3/JointDecisionv3.mlmodelc", + "parakeet-v3/parakeet_vocab.json", "localvqe/localvqe-v1.3-4.8M-256ms.mlmodelc", + "localvqe/localvqe-v1.2-1.3M-256ms.mlmodelc", + }, "Incomplete model fingerprints") + for name, digest in {**audio_hashes, **model_hashes}.items(): + require(isinstance(digest, str) and re.fullmatch(r"[0-9a-f]{64}", digest), f"Invalid SHA256 for {name}") + conditions = ["unprocessed", "localvqe-v1.3", "localvqe-v1.2"] + require(report["conditions"] == conditions and set(report["summary"]) == set(conditions), + "Missing, extra or duplicate conditions") + totals = {condition: Counter() for condition in conditions} + for row in rows: + reference = row["reference"].split() + far = row["far_reference"].split() + require(bool(reference), f"Empty reference in scored file {row['fileid']}") + close(row["ref_words"], len(reference), "reference word count") + close(row["far_words"], len(far), "far-end word count") + duration = row["audio_seconds"] + require(math.isfinite(duration) and duration > 0, "Invalid audio duration") + for condition in conditions: + hypothesis = row[f"{condition}_hyp"].split() + substitutions, deletions, insertions = edits(hypothesis, reference) + hits = len(reference) - deletions - substitutions + errors = substitutions + deletions + insertions + leakage = sum(((Counter(hypothesis) - Counter(reference)) & Counter(far)).values()) + for field, value in (("substitutions", substitutions), ("deletions", deletions), + ("insertions", insertions), ("hits", hits), ("errors", errors), ("leaked", leakage), + ("recall", hits / len(reference)), ("wer", errors / len(reference))): + close(row[f"{condition}_{field}"], value, f"{row['fileid']} {condition}.{field}") + elapsed = row[f"{condition}_enhancement_seconds"] + require(math.isfinite(elapsed) and (elapsed == 0 if condition == "unprocessed" else elapsed > 0), + f"Invalid enhancement timing for {condition}") + totals[condition].update({ + "files": 1, "reference_words": len(reference), "far_end_words": len(far), + "hits": hits, "errors": errors, "leaked_words": leakage, + "audio_seconds": duration, "enhancement_seconds": elapsed, + }) + for condition, total in totals.items(): + total["recall"] = total["hits"] / total["reference_words"] + total["wer"] = total["errors"] / total["reference_words"] + total["leakage"] = total["leaked_words"] / total["far_end_words"] if total["far_end_words"] else 0 + total["rtfx"] = total["audio_seconds"] / total["enhancement_seconds"] if total["enhancement_seconds"] else 0 + for field, value in total.items(): + close(report["summary"][condition][field], value, f"{condition}.{field}") + if require_improvement: + require(expected_files == 200, "Quality comparison requires the full 200-file selection") + baseline = totals["unprocessed"] + require(baseline["far_end_words"] > 0, "Cannot evaluate leakage without far-end reference words") + for condition in conditions[1:]: + require(totals[condition]["recall"] > baseline["recall"], f"{condition} did not improve recall") + require(totals[condition]["leakage"] < baseline["leakage"], f"{condition} did not reduce leakage") + return totals + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + parser.add_argument("--expected-files", type=int, default=200) + parser.add_argument("--require-improvement", action="store_true") + parser.add_argument("--markdown", type=Path) + args = parser.parse_args() + report = json.loads(args.report.read_text()) + totals = verify(report, args.expected_files, args.require_improvement) + lines = ["## LocalVQE ASR benchmark", "", + f"Verified {args.expected_files} selected examples; {len(report['files'])} scored; " + f"{len(report['excluded_empty_reference_fileids'])} empty references excluded.", "", + "| Condition | Recall | WER | Leakage | Enhancement RTFx |", + "|---|---:|---:|---:|---:|"] + for condition, total in totals.items(): + speed = f"{total['rtfx']:.2f}x" if condition != "unprocessed" else "—" + lines.append(f"| {condition} | {total['recall']:.2%} | {total['wer']:.2%} | {total['leakage']:.2%} | {speed} |") + lines += ["", "Protocol: `localvqe-asr-v2`, Parakeet v3 int8, CPU-only, 256ms LocalVQE exports.", + "Exploratory training-shard study with machine transcripts. CI timing is not device performance.", + "Model/audio fingerprints, raw counts and excluded IDs are in the JSON artifact."] + output = "\n".join(lines) + "\n" + print(output) + if args.markdown: + args.markdown.write_text(output) + + +if __name__ == "__main__": + main() diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift index 1c0563386..03982d57b 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -59,7 +59,7 @@ enum EnhanceBenchmarkCommand { printUsage() exit(0) case "--dataset-dir": - options.datasetDir = next(arguments, &index) + options.datasetDir = requiredValue(arguments, &index) case "--max-files": guard let raw = next(arguments, &index), let maxFiles = Int(raw), maxFiles > 0 else { logger.error("--max-files must be a positive integer") @@ -67,9 +67,12 @@ enum EnhanceBenchmarkCommand { } options.maxFiles = maxFiles case "--variants": - let raw = (next(arguments, &index) ?? "").split(separator: ",").map(String.init) + let raw = (next(arguments, &index) ?? "").split(separator: ",", omittingEmptySubsequences: false) + .map(String.init) let parsed = raw.compactMap(LocalVqeVariant.init(rawValue:)) - guard parsed.count == raw.count, !parsed.isEmpty else { + guard parsed.count == raw.count, !parsed.isEmpty, + Set(parsed.map(\.rawValue)).count == parsed.count + else { logger.error("--variants must be a comma list of \(LocalVqeVariant.allCases.map(\.rawValue))") exit(1) } @@ -93,9 +96,10 @@ enum EnhanceBenchmarkCommand { case "--no-reference": options.includeNoReference = true case "--output": - options.outputPath = next(arguments, &index) + options.outputPath = requiredValue(arguments, &index) default: - logger.warning("Unknown option: \(arg)") + logger.error("Unknown option: \(arg)") + exit(1) } index += 1 } @@ -109,10 +113,14 @@ enum EnhanceBenchmarkCommand { exit(1) } report("Dataset: \(datasetDir.path) (\(examples.count) examples)") + let startedAt = Date() + let audioHashes = options.outputPath == nil ? [:] : try EnhanceBenchmarkProvenance.audioFiles(examples) let asr = AsrManager() - try await asr.loadModels(try await AsrModels.downloadAndLoad()) - report("ASR: Parakeet TDT v3 loaded") + let asrConfiguration = MLModelConfigurationUtils.defaultConfiguration(computeUnits: .cpuOnly) + try await asr.loadModels( + try await AsrModels.downloadAndLoad(configuration: asrConfiguration, version: .v3)) + report("ASR: Parakeet TDT v3 int8 loaded (CPU-only)") var conditions: [(name: String, manager: LocalVqeManager?, useReference: Bool)] = [ ("unprocessed", nil, true) @@ -126,6 +134,11 @@ enum EnhanceBenchmarkCommand { } } report("Conditions: \(conditions.map(\.name).joined(separator: ", "))") + let modelHashes = + options.outputPath == nil + ? [:] + : try EnhanceBenchmarkProvenance.modelFiles( + variants: options.variants, chunk: options.chunk) let converter = AudioConverter() var totals = [String: ConditionTotals]() @@ -135,13 +148,9 @@ enum EnhanceBenchmarkCommand { for (i, example) in examples.enumerated() { let mic = try converter.resampleAudioFile(example.mic) - var lpb = try converter.resampleAudioFile(example.lpb) + let lpb = try converter.resampleAudioFile(example.lpb) let clean = try converter.resampleAudioFile(example.clean) - if lpb.count < mic.count { - lpb.append(contentsOf: [Float](repeating: 0, count: mic.count - lpb.count)) - } else if lpb.count > mic.count { - lpb.removeLast(lpb.count - mic.count) - } + try validateAudio(mic: mic, reference: lpb, clean: clean, fileID: example.fileID) let refWords = words(try await transcribe(asr, clean)) if refWords.isEmpty { @@ -158,6 +167,7 @@ enum EnhanceBenchmarkCommand { "far_reference": farWords.joined(separator: " "), "is_farend_noisy": example.farendNoisy, "is_nearend_noisy": example.nearendNoisy, + "audio_seconds": Double(mic.count) / Double(LocalVqeManager.sampleRate), ] for condition in conditions { @@ -165,9 +175,14 @@ enum EnhanceBenchmarkCommand { var enhanceSeconds = 0.0 if let manager = condition.manager { let reference = condition.useReference ? lpb : [Float](repeating: 0, count: mic.count) - let start = Date() + let start = ContinuousClock.now enhanced = try await manager.process(mic: mic, reference: reference) - enhanceSeconds = Date().timeIntervalSince(start) + let elapsed = start.duration(to: .now).components + enhanceSeconds = Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18 + } + guard enhanced.count == mic.count, enhanced.allSatisfy(\.isFinite) else { + throw LocalVqeError.modelProcessingFailed( + "Invalid \(condition.name) output for fileid \(example.fileID)") } let hypWords = words(try await transcribe(asr, enhanced)) let m = score(hypothesis: hypWords, reference: refWords, farEnd: farWords) @@ -195,6 +210,12 @@ enum EnhanceBenchmarkCommand { row["\(condition.name)_wer"] = refWords.isEmpty ? 0 : Double(m.errors) / Double(refWords.count) row["\(condition.name)_leaked"] = m.leaked row["\(condition.name)_hyp"] = hypWords.joined(separator: " ") + row["\(condition.name)_hits"] = m.hits + row["\(condition.name)_errors"] = m.errors + row["\(condition.name)_insertions"] = m.insertions + row["\(condition.name)_deletions"] = m.deletions + row["\(condition.name)_substitutions"] = m.substitutions + row["\(condition.name)_enhancement_seconds"] = enhanceSeconds } rows.append(row) @@ -207,6 +228,10 @@ enum EnhanceBenchmarkCommand { } } + guard !rows.isEmpty else { + throw LocalVqeError.modelProcessingFailed( + "No examples were scored: all \(examples.count) clean-reference transcripts were empty") + } report("") if !emptyReferenceFileIDs.isEmpty { report( @@ -243,21 +268,39 @@ enum EnhanceBenchmarkCommand { ] } let payload: [String: Any] = [ + "schema_version": 2, + "protocol": "localvqe-asr-v2", + "started_at": ISO8601DateFormatter().string(from: startedAt), + "completed_at": ISO8601DateFormatter().string(from: Date()), + "conditions": conditions.map(\.name), + "configuration": [ + "asr": "parakeet-tdt-v3-int8", "asr_compute_units": "cpu-only", + "enhancement_compute_units": options.computeUnits.rawValue, + "normalization": "TextNormalizer.normalize", "sample_rate": LocalVqeManager.sampleRate, + "aggregation": "micro", "empty_reference_policy": "exclude-from-all-conditions", + ], + "model_files_sha256": modelHashes, + "environment": [ + "os": ProcessInfo.processInfo.operatingSystemVersionString, + "processor_count": ProcessInfo.processInfo.processorCount, + "source_revision": ProcessInfo.processInfo.environment["GITHUB_SHA"] ?? "unrecorded", + ], "dataset": [ "path": datasetDir.path, - "repository": datasetRepo, + "repository": options.datasetDir == nil ? datasetRepo : "custom", "revision": options.datasetDir == nil ? datasetRevision : "custom", "archive_sha256": options.datasetDir == nil ? datasetArchiveSHA256 : NSNull(), "metadata_sha256": try EnhanceBenchmarkDataset.sha256( of: datasetDir.appendingPathComponent("meta.csv")), "selection_order": "numeric fileid", "selected_fileids": examples.map(\.fileID), + "audio_files_sha256": audioHashes, ], "chunk": options.chunk.rawValue, "summary": summary, "files": rows, "excluded_empty_reference_fileids": emptyReferenceFileIDs, ] let data = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) - try data.write(to: URL(fileURLWithPath: outputPath)) + try data.write(to: URL(fileURLWithPath: outputPath), options: .atomic) report("Wrote \(outputPath)") } } catch { @@ -280,11 +323,10 @@ enum EnhanceBenchmarkCommand { /// hits = reference words kept by the hypothesis (N - deletions - substitutions); /// errors = S + D + I; leaked = far-end words present in the hypothesis beyond /// what the near-end reference accounts for (multiset). - private static func score( + static func score( hypothesis: [String], reference: [String], farEnd: [String] - ) -> (hits: Int, errors: Int, leaked: Int) { - let m = WERCalculator.calculateWERMetrics( - hypothesis: hypothesis.joined(separator: " "), reference: reference.joined(separator: " ")) + ) -> (hits: Int, errors: Int, leaked: Int, insertions: Int, deletions: Int, substitutions: Int) { + let m = WERCalculator.calculateWordMetrics(hypothesis: hypothesis, reference: reference) let hits = max(0, m.totalWords - m.deletions - m.substitutions) let errors = m.insertions + m.deletions + m.substitutions @@ -296,7 +338,16 @@ enum EnhanceBenchmarkCommand { spare[w, default: 0] -= 1 leaked += 1 } - return (hits, errors, leaked) + return (hits, errors, leaked, m.insertions, m.deletions, m.substitutions) + } + + static func validateAudio(mic: [Float], reference: [Float], clean: [Float], fileID: String) throws { + guard !mic.isEmpty, mic.count == reference.count, mic.count == clean.count else { + throw LocalVqeError.modelProcessingFailed("Empty or unequal audio lengths for fileid \(fileID)") + } + guard mic.allSatisfy(\.isFinite), reference.allSatisfy(\.isFinite), clean.allSatisfy(\.isFinite) else { + throw LocalVqeError.modelProcessingFailed("Non-finite audio samples for fileid \(fileID)") + } } private static func pct(_ value: Double) -> String { @@ -376,11 +427,20 @@ enum EnhanceBenchmarkCommand { } private static func next(_ arguments: [String], _ index: inout Int) -> String? { - guard index + 1 < arguments.count else { return nil } + guard index + 1 < arguments.count, !arguments[index + 1].hasPrefix("--") else { return nil } index += 1 return arguments[index] } + private static func requiredValue(_ arguments: [String], _ index: inout Int) -> String { + let option = arguments[index] + guard let value = next(arguments, &index), !value.isEmpty else { + logger.error("\(option) requires a value") + exit(1) + } + return value + } + private static func report(_ line: String) { print(line) logger.info("\(line)") diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift index 97df93e62..f9989cf22 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift @@ -117,12 +117,20 @@ enum EnhanceBenchmarkDataset { } let fileID = rowFields[fileIDIndex] guard !fileID.isEmpty else { throw DatasetError.missingFileID(line: lineNumber) } + guard let numericID = Int(fileID), numericID >= 0, String(numericID) == fileID else { + throw DatasetError.invalidInteger(line: lineNumber, column: "fileid", value: fileID) + } guard seen.insert(fileID).inserted else { throw DatasetError.duplicateFileID(fileID) } let ser = try integer("ser", value: rowFields[serIndex], line: lineNumber) let farendNoisy = try integer( "is_farend_noisy", value: rowFields[farendNoisyIndex], line: lineNumber) let nearendNoisy = try integer( "is_nearend_noisy", value: rowFields[nearendNoisyIndex], line: lineNumber) + for (column, value) in [("is_farend_noisy", farendNoisy), ("is_nearend_noisy", nearendNoisy)] { + guard value == 0 || value == 1 else { + throw DatasetError.invalidInteger(line: lineNumber, column: column, value: String(value)) + } + } rows.append( MetadataRow( fileID: fileID, diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift new file mode 100644 index 000000000..839517f75 --- /dev/null +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkProvenance.swift @@ -0,0 +1,51 @@ +#if os(macOS) +import FluidAudio +import Foundation + +/// Fingerprints the exact inputs consumed by an enhancement benchmark run. +enum EnhanceBenchmarkProvenance { + static func modelFiles(variants: [LocalVqeVariant], chunk: LocalVqeChunk) throws -> [String: String] { + let asrDirectory = AsrModels.defaultCacheDirectory(for: .v3) + let enhancementDirectory = MLModelConfigurationUtils.defaultModelsDirectory(for: .localVqe) + var hashes: [String: String] = [:] + for name in ModelNames.ASR.requiredModelsV3().union([ModelNames.ASR.vocabularyFile]).sorted() { + try fingerprint(asrDirectory.appendingPathComponent(name), prefix: "parakeet-v3/\(name)", into: &hashes) + } + for variant in variants { + let name = ModelNames.LocalVQE.modelFile(variant: variant, chunk: chunk) + try fingerprint( + enhancementDirectory.appendingPathComponent(name), prefix: "localvqe/\(name)", into: &hashes) + } + return hashes + } + + private static func fingerprint(_ url: URL, prefix: String, into hashes: inout [String: String]) throws { + let values = try url.resourceValues(forKeys: [.isDirectoryKey, .isRegularFileKey]) + if values.isDirectory == true { + let children = try FileManager.default.contentsOfDirectory( + at: url, includingPropertiesForKeys: [.isDirectoryKey, .isRegularFileKey], options: [.skipsHiddenFiles]) + guard !children.isEmpty else { + throw LocalVqeError.modelProcessingFailed("Empty model directory: \(url.path)") + } + for child in children.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }) { + try fingerprint(child, prefix: "\(prefix)/\(child.lastPathComponent)", into: &hashes) + } + return + } + guard values.isRegularFile == true else { + throw LocalVqeError.modelProcessingFailed("Unsupported model file: \(url.path)") + } + hashes[prefix] = try EnhanceBenchmarkDataset.sha256(of: url) + } + + static func audioFiles(_ examples: [EnhanceBenchmarkDataset.Example]) throws -> [String: String] { + var hashes: [String: String] = [:] + for example in examples { + for url in [example.mic, example.lpb, example.clean] { + hashes[url.lastPathComponent] = try EnhanceBenchmarkDataset.sha256(of: url) + } + } + return hashes + } +} +#endif diff --git a/Sources/FluidAudioCLI/Utils/WERCalculator.swift b/Sources/FluidAudioCLI/Utils/WERCalculator.swift index 771f001c0..a4be289f1 100644 --- a/Sources/FluidAudioCLI/Utils/WERCalculator.swift +++ b/Sources/FluidAudioCLI/Utils/WERCalculator.swift @@ -15,10 +15,18 @@ enum WERCalculator { let hypWords = hypothesis.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } let refWords = reference.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } - let distance = editDistance(hypWords, refWords) - let wer = refWords.isEmpty ? 0.0 : Double(distance.total) / Double(refWords.count) + return calculateWordMetrics(hypothesis: hypWords, reference: refWords) + } - return (wer, distance.insertions, distance.deletions, distance.substitutions, refWords.count) + /// Score already-normalized tokens without applying text normalization again. + static func calculateWordMetrics( + hypothesis: [String], reference: [String] + ) + -> (wer: Double, insertions: Int, deletions: Int, substitutions: Int, totalWords: Int) + { + let distance = editDistance(hypothesis, reference) + let wer = reference.isEmpty ? 0.0 : Double(distance.total) / Double(reference.count) + return (wer, distance.insertions, distance.deletions, distance.substitutions, reference.count) } /// Compute character-level CER alongside WER if needed. diff --git a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift index 116228402..9ebe2eca6 100644 --- a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift +++ b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift @@ -72,6 +72,22 @@ final class EnhanceBenchmarkDatasetTests: XCTestCase { "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") } + func testRejectsNonCanonicalFileIDs() throws { + for id in ["01", "-1", "abc", "../other"] { + let directory = try metadataDirectory([header, "\(id),0,0,0,1"].joined(separator: "\n")) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) + } + } + + func testRejectsInvalidBooleanFlags() throws { + let directory = try metadataDirectory([header, "1,0,2,0,1"].joined(separator: "\n")) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.loadExamples(from: directory, fileExists: { _ in true })) { + XCTAssertEqual( + $0 as? EnhanceBenchmarkDataset.DatasetError, + .invalidInteger(line: 2, column: "is_farend_noisy", value: "2")) + } + } + private func metadataDirectory(_ metadata: String) throws -> URL { let directory = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) diff --git a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift new file mode 100644 index 000000000..33f161c03 --- /dev/null +++ b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkScoringTests.swift @@ -0,0 +1,43 @@ +#if os(macOS) +import XCTest + +@testable import FluidAudioCLI + +final class EnhanceBenchmarkScoringTests: XCTestCase { + func testEmptyHypothesisHasZeroRecall() { + let score = EnhanceBenchmarkCommand.score(hypothesis: [], reference: ["one", "two"], farEnd: ["three"]) + XCTAssertEqual(score.hits, 0) + XCTAssertEqual(score.deletions, 2) + XCTAssertEqual(score.errors, 2) + XCTAssertEqual(score.leaked, 0) + } + + func testLeakageSubtractsNearEndWordsAndRespectsMultiplicity() { + let score = EnhanceBenchmarkCommand.score( + hypothesis: ["hello", "echo", "echo"], reference: ["hello"], farEnd: ["hello", "echo"]) + XCTAssertEqual(score.hits, 1) + XCTAssertEqual(score.insertions, 2) + XCTAssertEqual(score.errors, 2) + XCTAssertEqual(score.leaked, 1) + } + + func testSubstitutionAndDeletionReduceRecallButInsertionDoesNot() { + let score = EnhanceBenchmarkCommand.score( + hypothesis: ["a", "x", "c", "d", "e"], reference: ["a", "b", "c", "d"], farEnd: []) + XCTAssertEqual(score.hits, 3) + XCTAssertEqual(score.substitutions, 1) + XCTAssertEqual(score.insertions, 1) + XCTAssertEqual(score.deletions, 0) + XCTAssertEqual(score.errors, 2) + } + + func testAlreadyNormalizedTokensAreNotNormalizedAgain() { + let metrics = WERCalculator.calculateWordMetrics(hypothesis: ["colour"], reference: ["color"]) + XCTAssertEqual(metrics.substitutions, 1) + } + + func testEmptyAudioFailsBeforeInference() { + XCTAssertThrowsError(try EnhanceBenchmarkCommand.validateAudio(mic: [], reference: [], clean: [], fileID: "0")) + } +} +#endif From 0b81c51c7ab020d90d9fa72c33ef595849255831 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sun, 20 Sep 2026 00:21:47 -0400 Subject: [PATCH 13/14] fix(ci): shard the LocalVQE 200-example benchmark across runners The single macOS job hit the 60-minute limit on de4c0c03 and showed no progress because stdout was block-buffered through `tee`. The same 200-example run takes 6.6 min on an M5 Pro; the 3-core runner is simply far slower. - enhance-benchmark: `--shard i/n` scores contiguous shard i of the numeric-fileid selection (after --max-files) and records it in the report; progress lines flush stdout. - verify_localvqe_benchmark.py: accepts every shard report of one run, checks index/count coverage and shared configuration, merges them in shard order (`--merged out.json`), then runs the existing full-report verification. Lone shard reports are rejected. - Workflow: 5 parallel shard jobs (60 min each) + ubuntu verify job that merges and applies the 200-file coverage and improvement gates; scorer/dataset regressions run in their own job. - Tests: shard slicing (Swift); merge ordering, mismatch/missing-shard errors, and a real-report split/merge round trip (Python). --- .github/workflows/localvqe-benchmark.yml | 64 ++++++++++--- Documentation/CLI.md | 3 + Scripts/test_verify_localvqe_benchmark.py | 85 +++++++++++++++++- Scripts/verify_localvqe_benchmark.py | 90 ++++++++++++++++++- .../Commands/EnhanceBenchmarkCommand.swift | 23 ++++- .../Commands/EnhanceBenchmarkDataset.swift | 16 ++++ .../CLI/EnhanceBenchmarkDatasetTests.swift | 11 +++ 7 files changed, 276 insertions(+), 16 deletions(-) diff --git a/.github/workflows/localvqe-benchmark.yml b/.github/workflows/localvqe-benchmark.yml index 7db008225..733bca969 100644 --- a/.github/workflows/localvqe-benchmark.yml +++ b/.github/workflows/localvqe-benchmark.yml @@ -27,11 +27,32 @@ concurrency: group: localvqe-benchmark-${{ github.ref }} cancel-in-progress: true +# The 200-example run is ~7 min on an M5 Pro but far slower on the 3-core +# macOS runner (a single job exceeded 60 min), so it is split into contiguous +# shards that a final job merges and verifies as one 200-example report. +env: + SHARD_COUNT: 5 + jobs: - benchmark: - name: LocalVQE 200-example ASR benchmark + regressions: + name: Scoring and manifest regressions + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + - name: Swift scorer / dataset / registry tests + run: swift test --filter 'EnhanceBenchmark|WERCalculatorTests|ModelRegistryTests' + - name: Python verifier tests + run: python3 Scripts/test_verify_localvqe_benchmark.py + + shard: + name: Score shard ${{ matrix.shard }} runs-on: macos-15 timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4] steps: - uses: actions/checkout@v5 @@ -46,27 +67,50 @@ jobs: sysctl -n machdep.cpu.brand_string } > localvqe-results/environment.txt - - name: Run scoring and manifest regressions - run: swift test --filter 'EnhanceBenchmark|WERCalculatorTests|ModelRegistryTests' - - name: Build release benchmark run: swift build -c release --product fluidaudiocli - - name: Score fixed 200-example dataset + - name: Score shard ${{ matrix.shard }}/${{ env.SHARD_COUNT }} of the fixed 200-example dataset run: | set -o pipefail .build/release/fluidaudiocli enhance-benchmark \ --variants v1.3,v1.2 --chunk 256ms --compute-units cpu-only \ + --shard ${{ matrix.shard }}/${{ env.SHARD_COUNT }} \ --output localvqe-results/benchmark.json 2>&1 | tee localvqe-results/benchmark.log - - name: Verify coverage, scores and improvement over unprocessed audio + - name: Upload shard evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: localvqe-shard-${{ matrix.shard }}-${{ github.sha }} + path: localvqe-results/ + retention-days: 30 + + verify: + name: LocalVQE 200-example ASR benchmark + needs: [shard] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v5 + + - name: Download shard reports + uses: actions/download-artifact@v4 + with: + pattern: localvqe-shard-*-${{ github.sha }} + path: shards + + - name: Merge shards and verify coverage, scores and improvement over unprocessed audio run: | + mkdir -p localvqe-results + ls shards/*/benchmark.json + python3 Scripts/verify_localvqe_benchmark.py shards/*/benchmark.json \ + --merged localvqe-results/benchmark.json \ + --expected-files 200 --require-improvement --markdown "$GITHUB_STEP_SUMMARY" LOCALVQE_BENCHMARK_REPORT=localvqe-results/benchmark.json \ python3 Scripts/test_verify_localvqe_benchmark.py - python3 Scripts/verify_localvqe_benchmark.py localvqe-results/benchmark.json \ - --expected-files 200 --require-improvement --markdown "$GITHUB_STEP_SUMMARY" - - name: Upload benchmark evidence + - name: Upload merged benchmark evidence if: always() uses: actions/upload-artifact@v4 with: diff --git a/Documentation/CLI.md b/Documentation/CLI.md index 679d3aecf..7bc6f0312 100644 --- a/Documentation/CLI.md +++ b/Documentation/CLI.md @@ -130,6 +130,9 @@ bundles instead of downloading. # Near-end word recall / WER / far-end leakage on the AEC-Challenge synthetic mini set (auto-downloads) swift run -c release fluidaudiocli enhance-benchmark swift run -c release fluidaudiocli enhance-benchmark --max-files 50 --variants v1.3 --no-reference --output results.json +# Split the run across machines: contiguous shard i of n, then merge + verify the shard reports +swift run -c release fluidaudiocli enhance-benchmark --shard 0/5 --output shard0.json +python3 Scripts/verify_localvqe_benchmark.py shard*.json --merged results.json --expected-files 200 ``` ## Datasets diff --git a/Scripts/test_verify_localvqe_benchmark.py b/Scripts/test_verify_localvqe_benchmark.py index 8e9d21617..03beb211f 100644 --- a/Scripts/test_verify_localvqe_benchmark.py +++ b/Scripts/test_verify_localvqe_benchmark.py @@ -6,7 +6,7 @@ from pathlib import Path import unittest -from verify_localvqe_benchmark import edits, verify +from verify_localvqe_benchmark import edits, merge, verify class EditTests(unittest.TestCase): @@ -24,6 +24,79 @@ def test_substitution_wins_ties(self): self.assertEqual(edits(["a", "b"], ["b", "a"]), (2, 0, 0)) +def split_into_shards(report, count): + """Re-slice a full report the way enhance-benchmark --shard i/n selects examples.""" + selected = report["dataset"]["selected_fileids"] + size = -(-len(selected) // count) + shards = [] + for index in range(count): + ids = selected[index * size:(index + 1) * size] + shard = deepcopy(report) + shard["shard"] = {"index": index, "count": count} + shard["dataset"]["selected_fileids"] = ids + shard["dataset"]["audio_files_sha256"] = { + name: digest for name, digest in report["dataset"]["audio_files_sha256"].items() + if name.split("_")[1] in ids} + shard["files"] = [row for row in report["files"] if row["fileid"] in ids] + shard["excluded_empty_reference_fileids"] = [i for i in report["excluded_empty_reference_fileids"] if i in ids] + for condition, summary in shard["summary"].items(): + rows = shard["files"] + for field, key in (("files", None), ("reference_words", "ref_words"), ("far_end_words", "far_words"), + ("hits", f"{condition}_hits"), ("errors", f"{condition}_errors"), + ("leaked_words", f"{condition}_leaked"), ("audio_seconds", "audio_seconds"), + ("enhancement_seconds", f"{condition}_enhancement_seconds")): + summary[field] = len(rows) if key is None else sum(row[key] for row in rows) + shards.append(shard) + return shards + + +class MergeTests(unittest.TestCase): + def shard(self, index, count, **overrides): + base = { + "schema_version": 2, "protocol": "localvqe-asr-v2", "configuration": {"asr": "x"}, "chunk": "256ms", + "conditions": ["unprocessed"], "model_files_sha256": {"m": "0" * 64}, + "dataset": {"path": "p", "repository": "r", "revision": "v", "archive_sha256": "a", "metadata_sha256": "b", + "selection_order": "numeric fileid", "selected_fileids": [str(index)], + "audio_files_sha256": {f"fileid_{index}_mic.wav": "0" * 64}}, + "files": [{"fileid": str(index)}], "excluded_empty_reference_fileids": [], + "started_at": f"2026-01-01T00:0{index}:00Z", "completed_at": f"2026-01-01T00:0{index}:30Z", + "environment": {"source_revision": "abc"}, "shard": {"index": index, "count": count}, + "summary": {"unprocessed": {"files": 1, "reference_words": 10, "hits": 5, "errors": 6, "far_end_words": 4, + "leaked_words": 1, "audio_seconds": 8.0, "enhancement_seconds": 0.0}}, + } + base.update(overrides) + return base + + def test_merges_in_index_order_regardless_of_input_order(self): + merged = merge([self.shard(1, 2), self.shard(0, 2)]) + self.assertEqual(merged["dataset"]["selected_fileids"], ["0", "1"]) + self.assertEqual([row["fileid"] for row in merged["files"]], ["0", "1"]) + self.assertEqual(merged["summary"]["unprocessed"]["hits"], 10) + self.assertAlmostEqual(merged["summary"]["unprocessed"]["recall"], 0.5) + self.assertEqual(merged["started_at"], "2026-01-01T00:00:00Z") + self.assertEqual(merged["completed_at"], "2026-01-01T00:01:30Z") + self.assertEqual([s["index"] for s in merged["shards"]], [0, 1]) + self.assertIsNone(merged["shard"]) + + def test_missing_duplicate_or_mismatched_shards_fail(self): + with self.assertRaisesRegex(ValueError, "Expected shards"): + merge([self.shard(0, 3), self.shard(1, 3)]) + with self.assertRaisesRegex(ValueError, "Expected shards"): + merge([self.shard(0, 2), self.shard(0, 2)]) + with self.assertRaisesRegex(ValueError, "shard count"): + merge([self.shard(0, 2), self.shard(1, 3)]) + with self.assertRaisesRegex(ValueError, "model_files_sha256"): + merge([self.shard(0, 2), self.shard(1, 2, model_files_sha256={"m": "1" * 64})]) + with self.assertRaisesRegex(ValueError, "source revisions"): + merge([self.shard(0, 2), self.shard(1, 2, environment={"source_revision": "other"})]) + with self.assertRaisesRegex(ValueError, "not a shard"): + merge([self.shard(0, 1, shard=None)]) + + def test_single_shard_report_is_rejected_by_verify(self): + with self.assertRaisesRegex(ValueError, "Shard 0/2"): + verify(self.shard(0, 2), 1) + + class RealReportTests(unittest.TestCase): @classmethod def setUpClass(cls): @@ -75,6 +148,16 @@ def test_missing_condition_fails(self): with self.assertRaises(ValueError): verify(self.report, self.count) + def test_shards_merge_back_to_the_full_report(self): + merged = merge(split_into_shards(self.report, 5)) + self.assertEqual(merged["dataset"]["selected_fileids"], self.report["dataset"]["selected_fileids"]) + self.assertEqual(merged["files"], self.report["files"]) + self.assertEqual(merged["excluded_empty_reference_fileids"], self.report["excluded_empty_reference_fileids"]) + for condition, summary in self.report["summary"].items(): + for field, value in summary.items(): + self.assertAlmostEqual(merged["summary"][condition][field], value, places=9, msg=f"{condition}.{field}") + verify(merged, self.count) + if __name__ == "__main__": unittest.main() diff --git a/Scripts/verify_localvqe_benchmark.py b/Scripts/verify_localvqe_benchmark.py index 88e19b612..6caf0472e 100644 --- a/Scripts/verify_localvqe_benchmark.py +++ b/Scripts/verify_localvqe_benchmark.py @@ -3,6 +3,7 @@ import argparse from collections import Counter +from copy import deepcopy import json import math from pathlib import Path @@ -46,8 +47,81 @@ def close(actual, expected, label): f"{label}: expected {expected}, got {actual}") +SHARED_FIELDS = ("schema_version", "protocol", "configuration", "chunk", "conditions", "model_files_sha256") +DATASET_SHARED_FIELDS = ("path", "repository", "revision", "archive_sha256", "metadata_sha256", "selection_order") +COUNT_FIELDS = ("files", "reference_words", "hits", "errors", "far_end_words", "leaked_words", + "audio_seconds", "enhancement_seconds") + + +def merge(reports): + """Combine contiguous shard reports (enhance-benchmark --shard i/n) into one full report. + + Shards are concatenated in index order, so the merged selection, scored rows + and exclusions keep the canonical numeric-fileid order that verify() checks. + """ + shards = [] + for report in reports: + shard = report.get("shard") + require(isinstance(shard, dict) and {"index", "count"} <= set(shard), "Report is not a shard report") + shards.append((shard["index"], shard["count"], report)) + counts = {count for _, count, _ in shards} + require(len(counts) == 1, "Shards disagree on the shard count") + count = counts.pop() + require(sorted(index for index, _, _ in shards) == list(range(count)), + f"Expected shards 0..{count - 1} exactly once, got {sorted(index for index, _, _ in shards)}") + shards.sort(key=lambda item: item[0]) + first = shards[0][2] + for _, _, report in shards[1:]: + for field in SHARED_FIELDS: + require(report[field] == first[field], f"Shards disagree on {field}") + for field in DATASET_SHARED_FIELDS: + require(report["dataset"][field] == first["dataset"][field], f"Shards disagree on dataset.{field}") + require(report["environment"]["source_revision"] == first["environment"]["source_revision"], + "Shards were produced from different source revisions") + + merged = {field: deepcopy(first[field]) for field in SHARED_FIELDS} + merged["dataset"] = {field: first["dataset"][field] for field in DATASET_SHARED_FIELDS} + merged["dataset"]["selected_fileids"] = [] + merged["dataset"]["audio_files_sha256"] = {} + merged["files"] = [] + merged["excluded_empty_reference_fileids"] = [] + merged["shards"] = [] + totals = {condition: Counter() for condition in first["conditions"]} + for index, _, report in shards: + merged["dataset"]["selected_fileids"] += report["dataset"]["selected_fileids"] + hashes = report["dataset"]["audio_files_sha256"] + require(set(hashes).isdisjoint(merged["dataset"]["audio_files_sha256"]), "Shards overlap in audio files") + merged["dataset"]["audio_files_sha256"].update(hashes) + merged["files"] += report["files"] + merged["excluded_empty_reference_fileids"] += report["excluded_empty_reference_fileids"] + merged["shards"].append({"index": index, "count": count, "started_at": report["started_at"], + "completed_at": report["completed_at"], "environment": report["environment"]}) + for condition, summary in report["summary"].items(): + totals[condition].update({field: summary[field] for field in COUNT_FIELDS}) + require(len(merged["dataset"]["selected_fileids"]) == len(set(merged["dataset"]["selected_fileids"])), + "Shards overlap in selected fileids") + merged["summary"] = {} + for condition, total in totals.items(): + words, far, seconds = total["reference_words"], total["far_end_words"], total["enhancement_seconds"] + merged["summary"][condition] = { + **{field: total[field] for field in COUNT_FIELDS}, + "recall": total["hits"] / words if words else 0, "wer": total["errors"] / words if words else 0, + "leakage": total["leaked_words"] / far if far else 0, + "rtfx": total["audio_seconds"] / seconds if seconds else 0, + } + merged["started_at"] = min(shard["started_at"] for shard in merged["shards"]) + merged["completed_at"] = max(shard["completed_at"] for shard in merged["shards"]) + merged["environment"] = deepcopy(first["environment"]) + merged["shard"] = None + return merged + + def verify(report, expected_files, require_improvement=False): require(report["schema_version"] == 2 and report["protocol"] == "localvqe-asr-v2", "Unsupported protocol") + shard = report.get("shard") + require(shard is None or shard["count"] == 1, + f"Shard {shard['index']}/{shard['count']} report: pass every shard report so they are merged first" + if shard else "") config = report["configuration"] require(config["asr"] == "parakeet-tdt-v3-int8" and config["asr_compute_units"] == "cpu-only", "Unexpected ASR configuration") @@ -140,16 +214,26 @@ def verify(report, expected_files, require_improvement=False): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("report", type=Path) + parser.add_argument("report", type=Path, nargs="+", + help="one full report, or every shard report of one run (merged in shard order)") parser.add_argument("--expected-files", type=int, default=200) parser.add_argument("--require-improvement", action="store_true") parser.add_argument("--markdown", type=Path) + parser.add_argument("--merged", type=Path, help="write the merged full report (shard inputs only)") args = parser.parse_args() - report = json.loads(args.report.read_text()) + reports = [json.loads(path.read_text()) for path in args.report] + if len(reports) == 1 and not reports[0].get("shard"): + report = reports[0] + require(args.merged is None, "--merged needs shard reports") + else: + report = merge(reports) + if args.merged: + args.merged.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") totals = verify(report, args.expected_files, args.require_improvement) lines = ["## LocalVQE ASR benchmark", "", f"Verified {args.expected_files} selected examples; {len(report['files'])} scored; " - f"{len(report['excluded_empty_reference_fileids'])} empty references excluded.", "", + f"{len(report['excluded_empty_reference_fileids'])} empty references excluded" + + (f"; merged from {len(report['shards'])} shards." if report.get("shards") else "."), "", "| Condition | Recall | WER | Leakage | Enhancement RTFx |", "|---|---:|---:|---:|---:|"] for condition, total in totals.items(): diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift index 03982d57b..1e40b891b 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkCommand.swift @@ -26,6 +26,7 @@ enum EnhanceBenchmarkCommand { private struct Options { var datasetDir: String? var maxFiles: Int? + var shard: (index: Int, count: Int)? var variants: [LocalVqeVariant] = [.v13, .v12] var chunk: LocalVqeChunk = .batch256ms var computeUnits: MLComputeUnits = .cpuOnly @@ -66,6 +67,15 @@ enum EnhanceBenchmarkCommand { exit(1) } options.maxFiles = maxFiles + case "--shard": + let parts = (next(arguments, &index) ?? "").split(separator: "/", omittingEmptySubsequences: false) + guard parts.count == 2, let shardIndex = Int(parts[0]), let shardCount = Int(parts[1]), + shardCount > 0, (0../ with 0 <= index < count") + exit(1) + } + options.shard = (shardIndex, shardCount) case "--variants": let raw = (next(arguments, &index) ?? "").split(separator: ",", omittingEmptySubsequences: false) .map(String.init) @@ -108,11 +118,15 @@ enum EnhanceBenchmarkCommand { let datasetDir = try await resolveDataset(options.datasetDir) var examples = try EnhanceBenchmarkDataset.loadExamples(from: datasetDir) if let maxFiles = options.maxFiles { examples = Array(examples.prefix(maxFiles)) } + if let shard = options.shard { + examples = try EnhanceBenchmarkDataset.shard(examples, index: shard.index, count: shard.count) + } guard !examples.isEmpty else { - logger.error("No examples found in \(datasetDir.path)") + logger.error("No examples selected from \(datasetDir.path)") exit(1) } - report("Dataset: \(datasetDir.path) (\(examples.count) examples)") + let shardLabel = options.shard.map { " shard \($0.index)/\($0.count)" } ?? "" + report("Dataset: \(datasetDir.path) (\(examples.count) examples\(shardLabel))") let startedAt = Date() let audioHashes = options.outputPath == nil ? [:] : try EnhanceBenchmarkProvenance.audioFiles(examples) @@ -296,6 +310,7 @@ enum EnhanceBenchmarkCommand { "selected_fileids": examples.map(\.fileID), "audio_files_sha256": audioHashes, ], + "shard": options.shard.map { ["index": $0.index, "count": $0.count] as [String: Any] } ?? NSNull(), "chunk": options.chunk.rawValue, "summary": summary, "files": rows, "excluded_empty_reference_fileids": emptyReferenceFileIDs, ] @@ -443,6 +458,8 @@ enum EnhanceBenchmarkCommand { private static func report(_ line: String) { print(line) + // stdout is block-buffered when piped (CI `tee`); flush so progress is visible live. + fflush(stdout) logger.info("\(line)") } @@ -460,6 +477,8 @@ enum EnhanceBenchmarkCommand { --dataset-dir Directory with fileid_*_{mic,lpb,clean}.wav + meta.csv (default: auto-download \(datasetRepo)). --max-files Score only the first n examples (numeric fileid order). + --shard / Score contiguous shard i of n (after --max-files); merge the + shard reports with Scripts/verify_localvqe_benchmark.py. --variants Comma list of v1.3,v1.2 (default both). --chunk <256ms|16ms> Chunk export to benchmark (default 256ms). --compute-units diff --git a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift index f9989cf22..fc56ad7fa 100644 --- a/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift +++ b/Sources/FluidAudioCLI/Commands/EnhanceBenchmarkDataset.swift @@ -18,6 +18,7 @@ enum EnhanceBenchmarkDataset { case duplicateFileID(String) case emptyMetadata case invalidInteger(line: Int, column: String, value: String) + case invalidShard(index: Int, count: Int) case malformedRow(line: Int, expected: Int, actual: Int) case missingAudio(fileID: String, path: String) case missingColumn(String) @@ -31,6 +32,8 @@ enum EnhanceBenchmarkDataset { return "benchmark meta.csv is empty" case .invalidInteger(let line, let column, let value): return "benchmark meta.csv line \(line) has invalid \(column) value '\(value)'" + case .invalidShard(let index, let count): + return "shard \(index)/\(count) is out of range; expected 0 <= index < count" case .malformedRow(let line, let expected, let actual): return "benchmark meta.csv line \(line) has \(actual) fields; expected \(expected)" case .missingAudio(let fileID, let path): @@ -77,6 +80,19 @@ enum EnhanceBenchmarkDataset { } } + /// Contiguous slice `index` of `count` equal-sized shards (the last may be + /// shorter), so concatenating shard results in index order restores the + /// full selection order. + static func shard(_ items: [T], index: Int, count: Int) throws -> [T] { + guard count > 0, (0.. String { let handle = try FileHandle(forReadingFrom: url) defer { try? handle.close() } diff --git a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift index 9ebe2eca6..e78c7ad85 100644 --- a/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift +++ b/Tests/FluidAudioTests/CLI/EnhanceBenchmarkDatasetTests.swift @@ -29,6 +29,17 @@ final class EnhanceBenchmarkDatasetTests: XCTestCase { XCTAssertEqual(examples.map(\.nearendNoisy), [true, false]) } + func testShardsAreContiguousAndCoverEveryItemOnce() throws { + let items = Array(1...11) + let shards = try (0..<3).map { try EnhanceBenchmarkDataset.shard(items, index: $0, count: 3) } + XCTAssertEqual(shards, [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11]]) + XCTAssertEqual(try EnhanceBenchmarkDataset.shard(items, index: 0, count: 1), items) + XCTAssertEqual(try EnhanceBenchmarkDataset.shard(items, index: 11, count: 12), []) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.shard(items, index: 3, count: 3)) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.shard(items, index: -1, count: 3)) + XCTAssertThrowsError(try EnhanceBenchmarkDataset.shard(items, index: 0, count: 0)) + } + func testRejectsDuplicateFileID() throws { let metadata = [header, "1,0,0,0,1", "1,1,0,0,1"].joined(separator: "\n") let directory = try metadataDirectory(metadata) From 37707fe99c3556accd7e6f61a3eb29c9e89c5977 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sun, 20 Sep 2026 01:09:02 -0400 Subject: [PATCH 14/14] feat(examples): LocalVQE demo app (offline A/B + live mic/speaker streaming) --- Examples/LocalVQEDemo/Package.swift | 18 + Examples/LocalVQEDemo/README.md | 38 +++ .../Sources/LocalVQEDemo/AudioFiles.swift | 71 ++++ .../Sources/LocalVQEDemo/ContentView.swift | 308 ++++++++++++++++++ .../Sources/LocalVQEDemo/DemoModel.swift | 301 +++++++++++++++++ .../Sources/LocalVQEDemo/LiveCapture.swift | 192 +++++++++++ .../LocalVQEDemo/LocalVQEDemoApp.swift | 25 ++ .../Sources/LocalVQEDemo/Player.swift | 56 ++++ 8 files changed, 1009 insertions(+) create mode 100644 Examples/LocalVQEDemo/Package.swift create mode 100644 Examples/LocalVQEDemo/README.md create mode 100644 Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift create mode 100644 Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift create mode 100644 Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift create mode 100644 Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift create mode 100644 Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift create mode 100644 Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift diff --git a/Examples/LocalVQEDemo/Package.swift b/Examples/LocalVQEDemo/Package.swift new file mode 100644 index 000000000..332548e27 --- /dev/null +++ b/Examples/LocalVQEDemo/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "LocalVQEDemo", + platforms: [.macOS(.v14)], + dependencies: [ + .package(name: "FluidAudio", path: "../..") + ], + targets: [ + .executableTarget( + name: "LocalVQEDemo", + dependencies: [ + .product(name: "FluidAudio", package: "FluidAudio") + ] + ) + ] +) diff --git a/Examples/LocalVQEDemo/README.md b/Examples/LocalVQEDemo/README.md new file mode 100644 index 000000000..fe1e9937c --- /dev/null +++ b/Examples/LocalVQEDemo/README.md @@ -0,0 +1,38 @@ +# LocalVQE Demo (macOS) + +SwiftUI app that exercises `LocalVqeManager` / `LocalVqeStream` from the +parent package: acoustic echo cancellation, noise suppression and +dereverberation on 16 kHz speech. + +```bash +cd Examples/LocalVQEDemo +swift run -c release LocalVQEDemo +``` + +No Xcode project is needed; the executable target builds with SwiftPM and +opens a regular window. The first **Load model** downloads the selected +variant from `FluidInference/localvqe-coreml`. + +## Files (offline) + +Pick a mic recording and, optionally, the far-end signal the loudspeaker was +playing, then **Enhance**. **Play before → after** plays the mic input then the enhanced output; **Transcribe all** runs Parakeet TDT v3 on every row so the echo words, the near-end words and what survives can be compared as text. The rows are also individually playable and +are written automatically as WAVs to `~/Downloads/LocalVQEDemo//` (**Show WAVs in Finder** opens the folder). **Use sample pair** loads a benchmark clip (default fileid 1148, a clear win; the menu also offers fileid 0, a hard case where near-end speech is lost) from the +AEC-Challenge synthetic set if `fluidaudiocli enhance-benchmark` has fetched it. + +## Live (mic + speaker) + +Plays a far-end file through the default output while capturing the default +input, pairs the two by elapsed time in 256 ms steps, and streams them through +`LocalVqeStream`. The meters show mic, far-end and enhanced level per step: +while only the playback is audible, Enhanced should sit well below Mic; when +you speak, it should follow your voice. **Stop** flushes the stream and offers +the three captured clips for listening, written to the same Downloads folder. + +Use the built-in speaker and mic without headphones so the mic actually hears +the playback. Microphone permission is attributed to the terminal that +launched the app when it is run with `swift run`. + +The far-end reference is the signal rendered by the app's own player, aligned +to the mic by elapsed time. The device round-trip latency is left to the +model's built-in delay search; production integrations should measure it. diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift new file mode 100644 index 000000000..9757b2d19 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/AudioFiles.swift @@ -0,0 +1,71 @@ +import AVFoundation +import FluidAudio +import Foundation + +/// 16 kHz mono helpers shared by the file and live modes. +enum AudioFiles { + static let sampleRate = Double(LocalVqeManager.sampleRate) + + /// Any format/rate → 16 kHz mono Float32. + static func read(_ url: URL) throws -> [Float] { + try AudioConverter().resampleAudioFile(url) + } + + /// 16-bit PCM WAV at 16 kHz mono. + static func write(_ samples: [Float], to url: URL) throws { + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: sampleRate, + AVNumberOfChannelsKey: 1, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + ] + let file = try AVAudioFile( + forWriting: url, settings: settings, commonFormat: .pcmFormatFloat32, interleaved: false) + let buffer = try pcmBuffer(samples) + try file.write(from: buffer) + } + + static func pcmBuffer(_ samples: [Float]) throws -> AVAudioPCMBuffer { + guard let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1), + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(max(samples.count, 1))) + else { + throw DemoError.message("Could not allocate a 16 kHz mono buffer") + } + buffer.frameLength = AVAudioFrameCount(samples.count) + if let channel = buffer.floatChannelData?[0] { + samples.withUnsafeBufferPointer { channel.update(from: $0.baseAddress!, count: samples.count) } + } + return buffer + } + + /// Peak envelope (max |x| per bin) for drawing; `bins` columns. + static func peaks(_ samples: [Float], bins: Int) -> [Float] { + guard !samples.isEmpty, bins > 0 else { return [] } + let per = max(1, samples.count / bins) + return stride(from: 0, to: samples.count, by: per).prefix(bins).map { start in + samples[start..) -> Float { + guard !samples.isEmpty else { return 0 } + return (samples.reduce(0) { $0 + $1 * $1 } / Float(samples.count)).squareRoot() + } + + /// Level in dBFS, floored at -80. + static func dbfs(_ rms: Float) -> Float { + rms <= 0 ? -80 : max(-80, 20 * log10(rms)) + } +} + +enum DemoError: Error, LocalizedError { + case message(String) + + var errorDescription: String? { + switch self { + case .message(let text): return text + } + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift new file mode 100644 index 000000000..d24c21e13 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/ContentView.swift @@ -0,0 +1,308 @@ +import FluidAudio +import SwiftUI +import UniformTypeIdentifiers + +struct ContentView: View { + @EnvironmentObject private var model: DemoModel + + var body: some View { + VStack(alignment: .leading, spacing: 16) { + ModelBar() + TabView { + FileModeView().tabItem { Text("Files (offline)") } + LiveModeView().tabItem { Text("Live (mic + speaker)") } + } + } + .padding(20) + .alert( + "Error", + isPresented: Binding(get: { model.errorMessage != nil }, set: { if !$0 { model.errorMessage = nil } }) + ) { + Button("OK") { model.errorMessage = nil } + } message: { + Text(model.errorMessage ?? "") + } + } +} + +private struct ModelBar: View { + @EnvironmentObject private var model: DemoModel + + var body: some View { + HStack(spacing: 12) { + Text("LocalVQE").font(.title2).bold() + Picker("Variant", selection: $model.variant) { + Text("v1.3 (4.8M)").tag(LocalVqeVariant.v13) + Text("v1.2 (1.3M)").tag(LocalVqeVariant.v12) + } + .frame(width: 170) + Picker("Chunk", selection: $model.chunk) { + Text("256 ms").tag(LocalVqeChunk.batch256ms) + Text("16 ms").tag(LocalVqeChunk.realtime16ms) + } + .frame(width: 150) + Button(model.manager == nil ? "Load model" : "Reload") { model.loadModel() } + .disabled(model.isLoading || model.isLive) + if model.isLoading { ProgressView().controlSize(.small) } + Text(model.loadStatus).foregroundStyle(.secondary).lineLimit(1) + Spacer() + } + } +} + +// MARK: - File mode + +private struct FileModeView: View { + @EnvironmentObject private var model: DemoModel + @State private var picking: PickTarget? + + private enum PickTarget: Identifiable { + case mic, reference + var id: Self { self } + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack { + FileField(title: "Mic recording", url: model.micURL) { picking = .mic } + FileField(title: "Far end (optional)", url: model.referenceURL) { picking = .reference } + } + HStack { + Menu { + ForEach(DemoModel.samples, id: \.id) { sample in + Button("fileid \(sample.id) — \(sample.note)") { model.useSamplePair(id: sample.id) } + } + } label: { + Text("Use sample pair") + } primaryAction: { + model.useSamplePair() + } + .fixedSize() + .disabled(!model.hasSamples) + Button("Enhance") { model.processFiles() } + .keyboardShortcut(.defaultAction) + .disabled(model.isProcessing || model.manager == nil || model.micURL == nil) + if model.isProcessing { ProgressView().controlSize(.small) } + Spacer() + SaveButton(clips: model.fileClips) + } + Text(model.fileStatus).font(.callout).foregroundStyle(.secondary) + ClipList(clips: model.fileClips) + if !model.hasSamples { + Text( + "No sample pair found. Run `swift run fluidaudiocli enhance-benchmark --max-files 1` once to fetch the AEC-Challenge synthetic set, or pick your own files." + ) + .font(.footnote).foregroundStyle(.tertiary) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .padding(.top, 8) + .fileImporter( + isPresented: Binding(get: { picking != nil }, set: { if !$0 { picking = nil } }), + allowedContentTypes: [.audio] + ) { result in + guard let target = picking, case .success(let url) = result else { return } + switch target { + case .mic: model.micURL = url + case .reference: model.referenceURL = url + } + picking = nil + } + } +} + +// MARK: - Live mode + +private struct LiveModeView: View { + @EnvironmentObject private var model: DemoModel + @State private var pickingFarEnd = false + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack { + FileField(title: "Far-end file to play", url: model.farEndURL) { pickingFarEnd = true } + Button("Use sample") { model.useSampleFarEnd() }.disabled(!model.hasSamples) + } + HStack { + if model.isLive { + Button("Stop") { model.stopLive() }.keyboardShortcut(.cancelAction) + } else { + Button("Start") { model.startLive() } + .keyboardShortcut(.defaultAction) + .disabled(model.manager == nil || model.farEndURL == nil) + } + Spacer() + SaveButton(clips: model.liveClips) + } + Text(model.liveStatus).font(.callout).foregroundStyle(.secondary) + + GroupBox("Live levels (dBFS per 256 ms step)") { + VStack(spacing: 8) { + LevelMeter(title: "Mic", db: model.liveLevels.micDb, tint: .orange) + LevelMeter(title: "Far end", db: model.liveLevels.referenceDb, tint: .blue) + LevelMeter(title: "Enhanced", db: model.liveLevels.enhancedDb, tint: .green) + HStack { + Text(String(format: "%.1f s captured", model.liveLevels.seconds)) + Spacer() + Text(String(format: "%.1f ms per 256 ms step", model.liveLevels.stepMilliseconds)) + } + .font(.caption.monospacedDigit()).foregroundStyle(.secondary) + } + .padding(6) + } + Text( + "Use the built-in speaker and mic without headphones so the mic actually hears the playback. While only the far end is playing, the Enhanced meter should sit far below Mic; when you talk, it should follow your voice." + ) + .font(.footnote).foregroundStyle(.tertiary) + ClipList(clips: model.liveClips) + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .padding(.top, 8) + .fileImporter(isPresented: $pickingFarEnd, allowedContentTypes: [.audio]) { result in + if case .success(let url) = result { model.farEndURL = url } + } + } +} + +// MARK: - Pieces + +private struct FileField: View { + let title: String + let url: URL? + let pick: () -> Void + + var body: some View { + HStack { + Text(title).frame(width: 130, alignment: .trailing) + Text(url?.lastPathComponent ?? "—") + .lineLimit(1).truncationMode(.middle) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 6).padding(.vertical, 3) + .background(RoundedRectangle(cornerRadius: 4).fill(.quaternary)) + Button("Choose…", action: pick) + } + } +} + +private struct SaveButton: View { + @EnvironmentObject private var model: DemoModel + let clips: [Clip] + + var body: some View { + Button("Show WAVs in Finder") { model.revealOutput() } + .disabled(clips.isEmpty || model.lastOutputDirectory == nil) + } +} + +private struct ClipList: View { + @EnvironmentObject private var model: DemoModel + let clips: [Clip] + + var body: some View { + VStack(spacing: 8) { + if !clips.isEmpty { + HStack { + Button { + if model.player.playingLabel != nil { + model.player.stop() + } else { + model.playBeforeAfter(clips) + } + } label: { + Label( + model.player.playingLabel == nil ? "Play before → after" : "Stop", + systemImage: model.player.playingLabel == nil ? "play.fill" : "stop.fill") + } + .buttonStyle(.borderedProminent) + .accessibilityLabel("Play before and after") + if let label = model.player.playingLabel { + Text("Playing: \(label)").font(.callout).foregroundStyle(.secondary) + } + Button { + model.transcribeAll(clips) + } label: { + Label("Transcribe all \(clips.count)", systemImage: "text.quote") + } + .disabled(model.isTranscribing) + .accessibilityLabel("Transcribe all") + if model.isTranscribing { ProgressView().controlSize(.small) } + Text(model.asrStatus).font(.callout).foregroundStyle(.secondary).lineLimit(1) + Spacer() + } + } + ForEach(clips) { clip in + HStack(spacing: 10) { + Button { + if model.player.playingLabel == clip.label { model.player.stop() } else { model.play(clip) } + } label: { + Image(systemName: model.player.playingLabel == clip.label ? "stop.fill" : "play.fill") + .frame(width: 14) + } + .buttonStyle(.bordered) + Text(clip.label).frame(width: 150, alignment: .leading) + WaveformView(samples: clip.samples, tint: tint(for: clip)) + .frame(height: 44) + Text(String(format: "%.1f s", clip.seconds)) + .font(.caption.monospacedDigit()).foregroundStyle(.secondary).frame(width: 44) + } + if let transcript = model.transcripts[clip.id] { + Text(transcript) + .font(.callout) + .foregroundStyle(tint(for: clip)) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.leading, 200) + } + } + } + } + + private func tint(for clip: Clip) -> Color { + if clip.label.hasPrefix("Enhanced") { return .green } + if clip.label.hasPrefix("Far") { return .blue } + return .orange + } +} + +private struct WaveformView: View { + let samples: [Float] + let tint: Color + + var body: some View { + Canvas { context, size in + let peaks = AudioFiles.peaks(samples, bins: Int(size.width)) + guard !peaks.isEmpty else { return } + var path = Path() + let mid = size.height / 2 + for (x, peak) in peaks.enumerated() { + let h = CGFloat(min(1, peak)) * mid + path.move(to: CGPoint(x: CGFloat(x), y: mid - h)) + path.addLine(to: CGPoint(x: CGFloat(x), y: mid + h)) + } + context.stroke(path, with: .color(tint), lineWidth: 1) + } + .background(RoundedRectangle(cornerRadius: 4).fill(.quaternary.opacity(0.5))) + } +} + +private struct LevelMeter: View { + let title: String + let db: Float + let tint: Color + + var body: some View { + HStack { + Text(title).frame(width: 70, alignment: .trailing) + GeometryReader { geometry in + let fraction = CGFloat(max(0, min(1, (db + 80) / 80))) + ZStack(alignment: .leading) { + RoundedRectangle(cornerRadius: 3).fill(.quaternary) + RoundedRectangle(cornerRadius: 3).fill(tint).frame(width: geometry.size.width * fraction) + } + } + .frame(height: 14) + .animation(.linear(duration: 0.1), value: db) + Text(String(format: "%6.1f dB", db)).font(.caption.monospacedDigit()).frame(width: 64) + } + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift new file mode 100644 index 000000000..174ceb607 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/DemoModel.swift @@ -0,0 +1,301 @@ +import AVFoundation +import FluidAudio +import Foundation +import AppKit +import SwiftUI + +/// A processed clip ready for A/B listening. +struct Clip: Identifiable { + let id = UUID() + let label: String + let samples: [Float] + var seconds: Double { Double(samples.count) / AudioFiles.sampleRate } +} + +@MainActor +final class DemoModel: ObservableObject { + // Model + @Published var variant: LocalVqeVariant = .v13 + @Published var chunk: LocalVqeChunk = .batch256ms + @Published private(set) var manager: LocalVqeManager? + @Published private(set) var loadStatus = "Not loaded" + @Published private(set) var isLoading = false + + // File mode + @Published var micURL: URL? + @Published var referenceURL: URL? + @Published private(set) var fileClips: [Clip] = [] + @Published private(set) var fileStatus = "Pick a mic recording and the far-end (loudspeaker) signal." + @Published private(set) var isProcessing = false + + // Live mode + @Published var farEndURL: URL? + @Published private(set) var isLive = false + @Published private(set) var liveLevels = LiveLevels() + @Published private(set) var liveClips: [Clip] = [] + @Published private(set) var liveStatus = "Plays the far-end file through your speakers while capturing the mic." + + @Published var errorMessage: String? + + // Transcription (Parakeet TDT v3, same ASR as `enhance-benchmark`) + @Published private(set) var transcripts: [UUID: String] = [:] + @Published private(set) var isTranscribing = false + @Published private(set) var asrStatus = "" + private var asr: AsrManager? + /// Folder the most recent Enhance / live Stop wrote its WAVs to. + @Published private(set) var lastOutputDirectory: URL? + + /// Every run writes mic / far-end / enhanced WAVs here, one subfolder per run. + static let outputRoot = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Downloads/LocalVQEDemo", isDirectory: true) + + let player = Player() + private var live: LiveCapture? + + /// The pinned benchmark dataset, if `enhance-benchmark` has downloaded it. + static let sampleDirectory = FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/FluidAudio/Datasets/aec-synthetic-mini") + + /// Sample clips from the pinned set, chosen from the 200-file benchmark rows. + static let samples: [(id: String, note: String)] = [ + ("1148", "clear win: 27 near-end words, 0% → 100% recall, 9 → 0 leaked far-end words"), + ("102", "clear win: 13 near-end words, 0% → 100% recall, 13 → 0 leaked"), + ("1089", "clear win: 14 near-end words, 0% → 100% recall, 13 → 0 leaked"), + ("0", "hard case: −7 dB SER, nonlinear echo; near-end speech is lost (0% recall in the benchmark)"), + ] + + var hasSamples: Bool { + FileManager.default.fileExists( + atPath: Self.sampleDirectory.appendingPathComponent("fileid_\(Self.samples[0].id)_mic.wav").path) + } + + // MARK: - Model + + func loadModel() { + guard !isLoading else { return } + isLoading = true + loadStatus = "Loading \(variant.rawValue) / \(chunk.rawValue)…" + let config = LocalVqeConfig(variant: variant, chunk: chunk) + Task { + do { + let started = ContinuousClock.now + let loaded = try await LocalVqeManager(config: config) { [weak self] progress in + Task { @MainActor in + self?.loadStatus = String(format: "Downloading… %.0f%%", progress.fractionCompleted * 100) + } + } + let ms = started.duration(to: .now).components + manager = loaded + loadStatus = String( + format: "%@ / %@ loaded in %.1f s (CPU)", variant.rawValue, chunk.rawValue, + Double(ms.seconds) + Double(ms.attoseconds) / 1e18) + } catch { + loadStatus = "Load failed" + errorMessage = error.localizedDescription + } + isLoading = false + } + } + + // MARK: - File mode + + func useSamplePair(id: String = samples[0].id) { + micURL = Self.sampleDirectory.appendingPathComponent("fileid_\(id)_mic.wav") + referenceURL = Self.sampleDirectory.appendingPathComponent("fileid_\(id)_lpb.wav") + let note = Self.samples.first { $0.id == id }?.note ?? "" + fileStatus = "AEC-Challenge synthetic set, fileid \(id) — \(note)." + fileClips = [] + } + + func processFiles() { + guard let manager else { + errorMessage = "Load a model first." + return + } + guard let micURL else { + errorMessage = "Pick a mic recording." + return + } + isProcessing = true + fileStatus = "Processing…" + player.stop() + let referenceURL = referenceURL + Task { + do { + let mic = try AudioFiles.read(micURL) + let started = ContinuousClock.now + let enhanced = try await manager.process(micURL: micURL, referenceURL: referenceURL) + let elapsed = started.duration(to: .now).components + let seconds = Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18 + var clips = [Clip(label: "Mic (unprocessed)", samples: mic)] + if let referenceURL { + clips.append(Clip(label: "Far end (loudspeaker)", samples: try AudioFiles.read(referenceURL))) + } + clips.append(Clip(label: "Enhanced", samples: enhanced)) + fileClips = clips + let stem = micURL.deletingPathExtension().lastPathComponent + let folder = try save(clips, run: "files-\(stem)") + let audioSeconds = Double(mic.count) / AudioFiles.sampleRate + fileStatus = String( + format: + "%.1f s enhanced in %.2f s (%.0fx real time); level %.1f dB → %.1f dB. WAVs in %@", + audioSeconds, seconds, audioSeconds / max(seconds, 1e-6), + AudioFiles.dbfs(AudioFiles.rms(mic[...])), AudioFiles.dbfs(AudioFiles.rms(enhanced[...])), + folder.path.replacingOccurrences(of: NSHomeDirectory(), with: "~")) + } catch { + fileStatus = "Processing failed" + errorMessage = error.localizedDescription + } + isProcessing = false + } + } + + // MARK: - Live mode + + func useSampleFarEnd() { + farEndURL = Self.sampleDirectory.appendingPathComponent("fileid_\(Self.samples[0].id)_lpb.wav") + } + + func startLive() { + guard let manager else { + errorMessage = "Load a model first." + return + } + guard let farEndURL else { + errorMessage = "Pick a far-end file to play." + return + } + player.stop() + liveClips = [] + liveLevels = LiveLevels() + Task { + guard await LiveCapture.requestMicrophoneAccess() else { + errorMessage = "Microphone access denied. Grant it to the terminal that launched this app." + return + } + do { + let file = try AVAudioFile(forReading: farEndURL) + let stream = try await manager.makeStream() + let capture = LiveCapture() + live = capture + try capture.start(farEnd: file, stream: stream, stepSeconds: 0.256) { [weak self] levels in + self?.liveLevels = levels + } + isLive = true + liveStatus = "Live. Speak over the playback; stop when the far-end clip ends." + } catch { + live = nil + errorMessage = error.localizedDescription + } + } + } + + func stopLive() { + guard let capture = live else { return } + live = nil + isLive = false + liveStatus = "Finishing…" + Task { + do { + let recording = try await capture.stop() + liveClips = [ + Clip(label: "Mic (unprocessed)", samples: recording.mic), + Clip(label: "Far end (played)", samples: recording.reference), + Clip(label: "Enhanced", samples: recording.enhanced), + ] + let folder = try save(liveClips, run: "live") + liveStatus = String( + format: "Captured %.1f s. Listen to Mic vs Enhanced. WAVs in %@", + Double(recording.mic.count) / AudioFiles.sampleRate, + folder.path.replacingOccurrences(of: NSHomeDirectory(), with: "~")) + } catch { + liveStatus = "Capture failed" + errorMessage = error.localizedDescription + } + } + } + + // MARK: - Transcription + + /// Transcribes every clip with Parakeet so the echo, the near-end words and + /// what survives enhancement can be compared as text. + func transcribeAll(_ clips: [Clip]) { + guard !isTranscribing, !clips.isEmpty else { return } + isTranscribing = true + Task { + do { + if asr == nil { + asrStatus = "Loading Parakeet TDT v3…" + let manager = AsrManager() + try await manager.loadModels( + try await AsrModels.downloadAndLoad(version: .v3) { [weak self] progress in + Task { @MainActor in + self?.asrStatus = String( + format: "Downloading Parakeet… %.0f%%", progress.fractionCompleted * 100) + } + }) + asr = manager + } + guard let asr else { return } + let started = ContinuousClock.now + for clip in clips { + asrStatus = "Transcribing \(clip.label)…" + var state = TdtDecoderState.make(decoderLayers: await asr.decoderLayerCount) + let text = try await asr.transcribe(clip.samples, decoderState: &state).text + transcripts[clip.id] = text.isEmpty ? "(no speech recognised)" : text + } + let elapsed = started.duration(to: .now).components + asrStatus = String( + format: "Transcribed %d clips in %.1f s", clips.count, + Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18) + } catch { + asrStatus = "Transcription failed" + errorMessage = error.localizedDescription + } + isTranscribing = false + } + } + + // MARK: - Shared + + func play(_ clip: Clip) { + do { + try player.play(clip.samples, label: clip.label) + } catch { + errorMessage = error.localizedDescription + } + } + + /// Mic input followed by the enhanced output, for a before/after listen. + func playBeforeAfter(_ clips: [Clip]) { + let ordered = clips.filter { $0.label.hasPrefix("Mic") } + clips.filter { $0.label.hasPrefix("Enhanced") } + do { + try player.playSequence(ordered.map { ($0.samples, $0.label) }) + } catch { + errorMessage = error.localizedDescription + } + } + + /// Writes the clips as numbered WAVs into a new timestamped run folder and returns it. + @discardableResult + func save(_ clips: [Clip], run: String) throws -> URL { + let formatter = DateFormatter() + formatter.dateFormat = "yyyyMMdd-HHmmss" + let directory = Self.outputRoot.appendingPathComponent( + "\(run)-\(variant.rawValue)-\(formatter.string(from: Date()))", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + for (index, clip) in clips.enumerated() { + let name = clip.label.lowercased() + .replacingOccurrences(of: #"[^a-z0-9]+"#, with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + try AudioFiles.write(clip.samples, to: directory.appendingPathComponent("\(index + 1)-\(name).wav")) + } + lastOutputDirectory = directory + return directory + } + + func revealOutput() { + guard let lastOutputDirectory else { return } + NSWorkspace.shared.activateFileViewerSelecting([lastOutputDirectory]) + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift new file mode 100644 index 000000000..91d53f443 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LiveCapture.swift @@ -0,0 +1,192 @@ +import AVFoundation +import FluidAudio +import Foundation +import os + +/// One 256 ms step of paired audio at the taps' native rates. +struct LivePair: Sendable { + let mic: [Float] + let micRate: Double + let reference: [Float] + let referenceRate: Double +} + +/// Lock-protected pairing of the two audio taps. The input-node tap appends +/// mic samples and the main-mixer tap appends the rendered far-end signal; +/// whenever both hold at least one step, a `LivePair` is emitted. Both taps +/// start with the engine, so pairing by elapsed time aligns them to within +/// the device round-trip latency, which the model's own delay search absorbs. +final class PairingBox: Sendable { + private struct State { + var mic: [Float] = [] + var reference: [Float] = [] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + private let micRate: Double + private let referenceRate: Double + private let stepSeconds: Double + private let continuation: AsyncStream.Continuation + + init(micRate: Double, referenceRate: Double, stepSeconds: Double, continuation: AsyncStream.Continuation) + { + self.micRate = micRate + self.referenceRate = referenceRate + self.stepSeconds = stepSeconds + self.continuation = continuation + } + + func appendMic(_ samples: [Float]) { + state.withLock { $0.mic.append(contentsOf: samples) } + emitReadyPairs() + } + + func appendReference(_ samples: [Float]) { + state.withLock { $0.reference.append(contentsOf: samples) } + emitReadyPairs() + } + + func finish() { + continuation.finish() + } + + private func emitReadyPairs() { + let micStep = Int(micRate * stepSeconds) + let referenceStep = Int(referenceRate * stepSeconds) + while true { + let pair: LivePair? = state.withLock { s in + guard s.mic.count >= micStep, s.reference.count >= referenceStep else { return nil } + let pair = LivePair( + mic: Array(s.mic[..? + + static func requestMicrophoneAccess() async -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: return true + case .notDetermined: return await AVCaptureDevice.requestAccess(for: .audio) + default: return false + } + } + + /// Starts playback + capture. `onLevels` is called on the main actor per step. + func start( + farEnd: AVAudioFile, stream: LocalVqeStream, stepSeconds: Double, + onLevels: @escaping @MainActor (LiveLevels) -> Void + ) throws { + engine.attach(farEndPlayer) + engine.connect(farEndPlayer, to: engine.mainMixerNode, format: farEnd.processingFormat) + + let input = engine.inputNode + let micFormat = input.outputFormat(forBus: 0) + let mixFormat = engine.mainMixerNode.outputFormat(forBus: 0) + guard micFormat.sampleRate > 0, mixFormat.sampleRate > 0 else { + throw DemoError.message("No audio input/output device available") + } + + let (pairs, continuation) = AsyncStream.makeStream() + let box = PairingBox( + micRate: micFormat.sampleRate, referenceRate: mixFormat.sampleRate, + stepSeconds: stepSeconds, continuation: continuation) + self.box = box + + input.installTap(onBus: 0, bufferSize: 2048, format: micFormat) { buffer, _ in + box.appendMic(Self.channelZero(buffer)) + } + engine.mainMixerNode.installTap(onBus: 0, bufferSize: 2048, format: mixFormat) { buffer, _ in + box.appendReference(Self.channelZero(buffer)) + } + + feeder = Task.detached(priority: .userInitiated) { + var recording = LiveRecording() + let micConverter = AudioConverter() + let referenceConverter = AudioConverter() + var seconds = 0.0 + for await pair in pairs { + var mic = try micConverter.resample(pair.mic, from: pair.micRate) + var reference = try referenceConverter.resample(pair.reference, from: pair.referenceRate) + let n = min(mic.count, reference.count) + mic.removeLast(mic.count - n) + reference.removeLast(reference.count - n) + + let started = ContinuousClock.now + let enhanced = try await stream.enhance(mic: mic, reference: reference) + let elapsed = started.duration(to: .now) + seconds += Double(n) / AudioFiles.sampleRate + + recording.mic.append(contentsOf: mic) + recording.reference.append(contentsOf: reference) + recording.enhanced.append(contentsOf: enhanced) + + let levels = LiveLevels( + micDb: AudioFiles.dbfs(AudioFiles.rms(mic[...])), + referenceDb: AudioFiles.dbfs(AudioFiles.rms(reference[...])), + enhancedDb: AudioFiles.dbfs(AudioFiles.rms(enhanced[...])), + seconds: seconds, + stepMilliseconds: Double(elapsed.components.attoseconds) / 1e15 + + Double(elapsed.components.seconds) * 1000) + await onLevels(levels) + } + // Drain the delay line so enhanced length == mic length. + recording.enhanced.append(contentsOf: try await stream.flush()) + return recording + } + + engine.prepare() + try engine.start() + farEndPlayer.scheduleFile(farEnd, at: nil) + farEndPlayer.play() + } + + /// Stops the engine and returns the captured 16 kHz clips. + func stop() async throws -> LiveRecording { + farEndPlayer.stop() + engine.inputNode.removeTap(onBus: 0) + engine.mainMixerNode.removeTap(onBus: 0) + engine.stop() + engine.detach(farEndPlayer) + box?.finish() + box = nil + defer { feeder = nil } + guard let feeder else { return LiveRecording() } + return try await feeder.value + } + + private nonisolated static func channelZero(_ buffer: AVAudioPCMBuffer) -> [Float] { + guard let data = buffer.floatChannelData else { return [] } + return Array(UnsafeBufferPointer(start: data[0], count: Int(buffer.frameLength))) + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift new file mode 100644 index 000000000..0cb8028fe --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/LocalVQEDemoApp.swift @@ -0,0 +1,25 @@ +import AppKit +import SwiftUI + +@main +struct LocalVQEDemoApp: App { + @StateObject private var model = DemoModel() + + init() { + // Line-buffer stdout so `swift run … | tee log` shows progress live. + setvbuf(stdout, nil, _IOLBF, 0) + // Bare SwiftPM executables start as background processes; make this one a + // regular windowed app with a menu bar and Dock presence. + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("LocalVQE Demo") { + ContentView() + .environmentObject(model) + .frame(minWidth: 860, minHeight: 620) + } + .windowResizability(.contentSize) + } +} diff --git a/Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift new file mode 100644 index 000000000..0ca5c07c7 --- /dev/null +++ b/Examples/LocalVQEDemo/Sources/LocalVQEDemo/Player.swift @@ -0,0 +1,56 @@ +import AVFoundation +import Foundation + +/// Plays a 16 kHz mono clip through the default output; used for A/B listening. +@MainActor +final class Player: ObservableObject { + @Published private(set) var playingLabel: String? + + private let engine = AVAudioEngine() + private let node = AVAudioPlayerNode() + private var generation = 0 + + init() { + engine.attach(node) + let format = AVAudioFormat(standardFormatWithSampleRate: AudioFiles.sampleRate, channels: 1) + engine.connect(node, to: engine.mainMixerNode, format: format) + } + + func play(_ samples: [Float], label: String) throws { + try playSequence([(samples, label)]) + } + + /// Plays clips back to back with a short gap, updating `playingLabel` as each starts. + func playSequence(_ clips: [(samples: [Float], label: String)]) throws { + stop() + let clips = clips.filter { !$0.samples.isEmpty } + guard !clips.isEmpty else { return } + if !engine.isRunning { + try engine.start() + } + generation += 1 + let current = generation + let gap = [Float](repeating: 0, count: Int(AudioFiles.sampleRate * 0.6)) + for (index, clip) in clips.enumerated() { + let buffer = try AudioFiles.pcmBuffer(index == 0 ? clip.samples : gap + clip.samples) + let next: String? = index + 1 < clips.count ? clips[index + 1].label : nil + node.scheduleBuffer(buffer, at: nil, options: [], completionCallbackType: .dataPlayedBack) { + [weak self] _ in + Task { @MainActor in + guard let self, self.generation == current else { return } + self.playingLabel = next + print("Player: \(next.map { "now playing \($0)" } ?? "finished")") + } + } + } + node.play() + playingLabel = clips[0].label + print("Player: playing \(clips.map(\.label).joined(separator: " → "))") + } + + func stop() { + generation += 1 + node.stop() + playingLabel = nil + } +}