From a6251df86b267a6514042effe5dbf8825c60cad1 Mon Sep 17 00:00:00 2001 From: sumerc Date: Thu, 6 Aug 2026 20:12:52 +0300 Subject: [PATCH 1/8] perf(whisper): reuse the language-detect encoder pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit whisper_full encoded the same audio twice in auto mode: once inside whisper_lang_auto_detect_with_state, then again for the first decode window. Tag the encoder output with the (mel_offset, n_audio_ctx) it was computed from and skip the identical re-encode; assign exp_n_audio_ctx before detection so one call encodes at one window size. Auto-detect now costs the same as a forced language: 1.94x on dictation-length clips (530 -> 274 ms, M5 Pro, turbo-q5), with transcripts unchanged — the skipped work was bit-identical. The forced-language path never had the second encode and is untouched. Upstream as ggml-org/whisper.cpp#3954, unmerged, so it lives in patches/whisper.cpp and make whisper-lib applies it. Guarded twice, because an unpatched build is still correct and merely 2x slower on auto, which no test can distinguish from a fast one: WHISPER_BASE stops the build on a submodule bump, and TestWhisperPatchesApplied matches the checkout against the patch byte-for-byte (also catching hand-edits that ignore = dirty hides). Co-Authored-By: Claude Opus 5 --- .gitmodules | 1 + Makefile | 28 +++++ internal/whisper/patch_test.go | 75 +++++++++++++ internal/whisper/whisper.go | 24 +++-- .../0001-reuse-detect-encoder-output.patch | 102 ++++++++++++++++++ 5 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 internal/whisper/patch_test.go create mode 100644 patches/whisper.cpp/0001-reuse-detect-encoder-output.patch diff --git a/.gitmodules b/.gitmodules index d4915e1..ae01354 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,4 @@ [submodule "third_party/whisper.cpp"] path = third_party/whisper.cpp url = https://github.com/ggml-org/whisper.cpp + ignore = dirty diff --git a/Makefile b/Makefile index 6f26c48..65e8e75 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,16 @@ PARAKEET_LIB := $(PARAKEET_DIR)/build-release/libparakeet.a GGML_PREFIX := $(CURDIR)/$(PARAKEET_DIR)/build-release/ggml-prefix WHISPER_DIR := third_party/whisper.cpp WHISPER_LIB := $(WHISPER_DIR)/build-release/src/libwhisper.a +# In-tree patches applied to the pinned whisper.cpp checkout before it builds. +# whisper-lib applies any that are missing and forces a reconfigure when it does, +# so a `git submodule update` that resets the checkout cannot silently drop them. +WHISPER_PATCHES := patches/whisper.cpp +# The upstream commit those patches were written and benchmarked against. +# `git apply` only matches context lines, so a patch can still apply cleanly onto +# a restructured encode path and be quietly wrong — a correct-but-slow build that +# no test can distinguish from a fast one. Bumping the submodule therefore has to +# stop the build until a human re-validates and moves this pin. +WHISPER_BASE := f049fff95a089aa9969deb009cdd4892b3e74916 HOST := $(shell go env GOOS)/$(shell go env GOARCH) ifeq ($(HOST),darwin/arm64) CGO_ENV := MACOSX_DEPLOYMENT_TARGET=$(MACOS_MIN) CGO_CFLAGS=-mmacosx-version-min=$(MACOS_MIN) CGO_LDFLAGS=-mmacosx-version-min=$(MACOS_MIN) @@ -77,6 +87,24 @@ whisper-lib: parakeet-lib echo "==> initializing whisper.cpp submodule (first checkout)"; \ git submodule update --init --recursive $(WHISPER_DIR); \ fi; \ + head=$$(git -C $(WHISPER_DIR) rev-parse HEAD 2>/dev/null); \ + if [ "$$head" != "$(WHISPER_BASE)" ]; then \ + echo "ERROR: whisper.cpp is at $$head"; \ + echo " but $(WHISPER_PATCHES)/*.patch were validated against $(WHISPER_BASE)."; \ + echo ""; \ + echo " git apply checks context lines, not meaning: these patches may still apply"; \ + echo " cleanly onto a moved encode path and silently stop working. Re-validate:"; \ + echo " ZEE_AC_DEBUG=1 go test ./internal/whisper -run FaultMatrix -v # H must pass"; \ + echo " make bench-local # auto ~= forced"; \ + echo " then regenerate the patch and set WHISPER_BASE to $$head."; \ + exit 1; \ + fi; \ + for p in $(CURDIR)/$(WHISPER_PATCHES)/*.patch; do \ + if git -C $(WHISPER_DIR) apply --reverse --check $$p 2>/dev/null; then continue; fi; \ + echo "==> applying $$(basename $$p)"; \ + git -C $(WHISPER_DIR) apply $$p || exit 1; \ + rm -rf $(WHISPER_DIR)/build-release; \ + done; \ if [ ! -d $(WHISPER_DIR)/build-release ]; then \ echo "==> configuring whisper.cpp (one-time)"; \ cmake -S $(WHISPER_DIR) -B $(WHISPER_DIR)/build-release \ diff --git a/internal/whisper/patch_test.go b/internal/whisper/patch_test.go new file mode 100644 index 0000000..b7b54bb --- /dev/null +++ b/internal/whisper/patch_test.go @@ -0,0 +1,75 @@ +//go:build darwin && arm64 + +package whisper_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// zee builds whisper.cpp from a pinned submodule plus the in-tree patches in +// patches/whisper.cpp, applied by `make whisper-lib`. Two ways that goes wrong +// silently, neither of which any other test can see: +// +// - a `git submodule update` resets the checkout and drops the patches. The +// build still succeeds and transcripts are still correct — auto-detect just +// quietly costs twice what it should again. +// - someone hand-edits the submodule source. `ignore = dirty` in .gitmodules +// (needed because the patches make the checkout permanently dirty) means +// `git status` will not show it. +// +// Comparing the submodule's diff against the patch files byte-for-byte catches +// both, and also fires when the submodule is bumped: hunk offsets move, so the +// diff stops matching and a human has to re-validate rather than trust a clean +// `git apply`. The Makefile's WHISPER_BASE pin is the build-time half of this. +func TestWhisperPatchesApplied(t *testing.T) { + root := filepath.Join("..", "..") + patchDir := filepath.Join(root, "patches", "whisper.cpp") + + entries, err := os.ReadDir(patchDir) + if err != nil { + t.Fatalf("read %s: %v", patchDir, err) + } + var want strings.Builder + for _, e := range entries { + if filepath.Ext(e.Name()) != ".patch" { + continue + } + b, err := os.ReadFile(filepath.Join(patchDir, e.Name())) + if err != nil { + t.Fatalf("read patch: %v", err) + } + want.Write(b) + } + if want.Len() == 0 { + t.Fatalf("no patches found in %s", patchDir) + } + + // Same flags the patches are generated with, so the comparison is not at the + // mercy of the developer's diff config. + cmd := exec.Command("git", "-c", "core.pager=cat", "-C", + filepath.Join(root, "third_party", "whisper.cpp"), + "diff", "--no-color", "--no-ext-diff") + out, err := cmd.Output() + if err != nil { + t.Skipf("whisper.cpp submodule not checked out: %v", err) + } + + if string(out) != want.String() { + t.Fatalf("third_party/whisper.cpp does not match patches/whisper.cpp/*.patch\n\n"+ + "got %d bytes of diff, want %d.\n\n"+ + "Either the patches were dropped (run `make whisper-lib` to reapply), the\n"+ + "submodule source was hand-edited, or the submodule was bumped and the\n"+ + "patches now apply at different offsets. In the last case re-validate the\n"+ + "optimisation before regenerating — a clean `git apply` does not prove the\n"+ + "patch still does anything:\n"+ + " ZEE_AC_DEBUG=1 go test ./internal/whisper -run FaultMatrix -v\n"+ + " make bench-local # auto must still cost ~the same as forced\n"+ + " git -C third_party/whisper.cpp -c core.pager=cat diff --no-color \\\n"+ + " --no-ext-diff > patches/whisper.cpp/0001-reuse-detect-encoder-output.patch", + len(out), want.Len()) + } +} diff --git a/internal/whisper/whisper.go b/internal/whisper/whisper.go index a729acc..67b1954 100644 --- a/internal/whisper/whisper.go +++ b/internal/whisper/whisper.go @@ -57,7 +57,7 @@ static char *zee_wsp_transcribe(struct whisper_context *ctx, const float *pcm, // a fixed 30 s per decode and skips the retry-on-failed-decode path, so // whatever the model does not emit in a window is lost for good. p.translate = false; // transcribe in-language, never translate to English - p.language = lang; // "auto" => detect (costs one extra encoder pass) + p.language = lang; // "auto" => detect (one extra decode step; see audioCtxFor) p.audio_ctx = audio_ctx; // 0 = full window; see audioCtxFor // Vocabulary hints ride in as the initial prompt — the same string the @@ -144,10 +144,15 @@ const sampleRate = 16000 // encode reads whatever the PREVIOUS call left in exp_n_audio_ctx, so it only // shrinks from a cold state (0, which every read site expands to 1500). Primed // once at a small size, later auto calls at a LARGER size are grows and are -// correct (matrix cases M/N). Returning 0 is now a cost decision, not a -// correctness one — measured ~1.9x at a floor of 800, with an over-tight window -// actively slower. See design-notes "audio_ctx sizing" for the numbers and the -// open questions. +// correct (matrix cases M/N). +// +// Superseded 2026-08-06 for the cold case: patches/whisper.cpp now assigns +// exp_n_audio_ctx before the detect encode, so one call encodes at one size and +// case H (cold, auto, sized) passes. Sizing on a REUSED state still garbles +// (D/F/G/I/J/L unchanged), so the lever needs a fresh whisper_state per +// utterance — measured ~10 ms, i.e. affordable. Returning 0 stays a deliberate +// choice: sizing is worth a further ~1.7x but does not preserve the transcript +// word-for-word. See design-notes "audio_ctx sizing". func audioCtxFor(int) int { return 0 } // Available reports whether local Whisper transcription is compiled in. @@ -167,7 +172,7 @@ var hushOnce sync.Once // also warms up: one throwaway transcribe so the backend's first-use init // (Metal pipeline compilation, buffer/kernel setup) happens now rather than // stalling the first real dictation. The warm-up runs in auto-detect mode -// because that is the default path, so both encoder passes get warmed. +// because that is the default path. func New(path string) (*Ctx, error) { c, err := newNoWarm(path) if err != nil { @@ -194,9 +199,10 @@ func newNoWarm(path string) (*Ctx, error) { } // Transcribe runs the model over mono 16 kHz float32 PCM and returns the -// transcript. lang is an ISO-639-1 code; "" means auto-detect, which costs one -// extra encoder pass but is the only mode that survives code-switching — a -// wrong forced language garbles the output rather than merely mislabelling it. +// transcript. lang is an ISO-639-1 code; "" means auto-detect, which is the +// only mode that survives code-switching — a wrong forced language garbles the +// output rather than merely mislabelling it. Detection is close to free: it +// shares its encoder pass with the first decode window (patches/whisper.cpp). // // hints is optional vocabulary biasing (the same comma-separated string the // cloud providers take as `prompt`); "" disables it. diff --git a/patches/whisper.cpp/0001-reuse-detect-encoder-output.patch b/patches/whisper.cpp/0001-reuse-detect-encoder-output.patch new file mode 100644 index 0000000..791a08d --- /dev/null +++ b/patches/whisper.cpp/0001-reuse-detect-encoder-output.patch @@ -0,0 +1,102 @@ +diff --git a/src/whisper.cpp b/src/whisper.cpp +index 5ffc70af..9cc528f5 100644 +--- a/src/whisper.cpp ++++ b/src/whisper.cpp +@@ -920,6 +920,13 @@ struct whisper_state { + // [EXPERIMENTAL] speed-up techniques + int32_t exp_n_audio_ctx = 0; // 0 - use default + ++ // Identifies the encoder pass whose output (embd_enc + kv_cross) the state ++ // currently holds, so an identical re-encode can be skipped. Auto-detect ++ // encodes exactly the window the first decode window needs. Invalidated ++ // whenever the mel changes, and while an encode is in flight. ++ int enc_mel_offset = -1; ++ int enc_n_ctx = -1; ++ + whisper_vad_context * vad_context = nullptr; + + struct vad_segment_info { +@@ -2364,6 +2371,20 @@ static bool whisper_encode_internal( + void * abort_callback_data) { + const int64_t t_start_us = ggml_time_us(); + ++ const int n_ctx_enc = wstate.exp_n_audio_ctx > 0 ? wstate.exp_n_audio_ctx : wctx.model.hparams.n_audio_ctx; ++ ++ // The encoder output already in the state was computed from this same mel ++ // window at this same size, so recomputing it would be bit-identical work. ++ // Only the decoder's self-KV changes between such calls, and that is reset ++ // per window by the caller. ++ if (wstate.enc_mel_offset == mel_offset && wstate.enc_n_ctx == n_ctx_enc) { ++ return !(abort_callback && abort_callback(abort_callback_data)); ++ } ++ ++ // A partially written embd_enc/kv_cross must never look reusable. ++ wstate.enc_mel_offset = -1; ++ wstate.enc_n_ctx = -1; ++ + // conv + { + auto & sched = wstate.sched_conv.sched; +@@ -2449,6 +2470,9 @@ static bool whisper_encode_internal( + } + } + ++ wstate.enc_mel_offset = mel_offset; ++ wstate.enc_n_ctx = n_ctx_enc; ++ + wstate.t_encode_us += ggml_time_us() - t_start_us; + wstate.n_encode++; + +@@ -3892,6 +3916,9 @@ int whisper_pcm_to_mel_with_state(struct whisper_context * ctx, struct whisper_s + return -1; + } + ++ state->enc_mel_offset = -1; ++ state->enc_n_ctx = -1; ++ + return 0; + } + +@@ -3917,6 +3944,9 @@ int whisper_set_mel_with_state( + state->mel.data.resize(n_len*n_mel); + memcpy(state->mel.data.data(), data, n_len*n_mel*sizeof(float)); + ++ state->enc_mel_offset = -1; ++ state->enc_n_ctx = -1; ++ + return 0; + } + +@@ -6829,6 +6859,18 @@ int whisper_full_with_state( + } + } + ++ // overwrite audio_ctx, max allowed is hparams.n_audio_ctx ++ // ++ // This has to happen before language auto-detection: detection runs the ++ // encoder too, and an encoder pass at a different window size than the ++ // decode that follows produces a cross-attention layout the decoder then ++ // misreads (the stride is derived from the same n_audio_ctx). ++ if (params.audio_ctx > whisper_n_audio_ctx(ctx)) { ++ WHISPER_LOG_ERROR("%s: audio_ctx is larger than the maximum allowed (%d > %d)\n", __func__, params.audio_ctx, whisper_n_audio_ctx(ctx)); ++ return -5; ++ } ++ state->exp_n_audio_ctx = params.audio_ctx; ++ + // auto-detect language if not specified + if (params.language == nullptr || strlen(params.language) == 0 || strcmp(params.language, "auto") == 0 || params.detect_language) { + std::vector probs(whisper_lang_max_id() + 1, 0.0f); +@@ -6964,13 +7006,6 @@ int whisper_full_with_state( + } + } + +- // overwrite audio_ctx, max allowed is hparams.n_audio_ctx +- if (params.audio_ctx > whisper_n_audio_ctx(ctx)) { +- WHISPER_LOG_ERROR("%s: audio_ctx is larger than the maximum allowed (%d > %d)\n", __func__, params.audio_ctx, whisper_n_audio_ctx(ctx)); +- return -5; +- } +- state->exp_n_audio_ctx = params.audio_ctx; +- + // these tokens determine the task that will be performed + std::vector prompt_init = { whisper_token_sot(ctx), }; + From c1bdb7e0a5a88ef42da7d85392590c568996abb8 Mon Sep 17 00:00:00 2001 From: sumerc Date: Thu, 6 Aug 2026 20:12:52 +0300 Subject: [PATCH 2/8] docs: record the auto-detect patch, time budget and engine survey What was measured while landing the encoder-reuse patch, so none of it gets re-derived: the before/after numbers, whisper's own time budget (the encoder is 94% of a dictation transcribe and flat in clip length), and the ggml 0.13 -> 0.18 bump measuring neutral on M5 Pro. Also corrects two entries the patch invalidates. The audio_ctx rejection is partly superseded: fault-matrix case H no longer garbles and whisper_init_state is ~10 ms, not the Metal setup it was assumed to be, so sizing is now a quality call rather than a correctness one. The silence-trimming entry gains a conditional, since "not a speed lever" holds only while the encoder window is fixed. Adds the alternative-engine survey: the search space collapses because the 30 s padding is a Whisper architecture property, not a whisper.cpp one. Notes that Parakeet's coverage excludes Turkish and every non-European language, so for those users Whisper is the only local engine and audio_ctx sizing is the only remaining lever; that upstream whisper.cpp now ships Parakeet but with an incompatible model format, so migrating costs a model release and buys build simplicity only; and that MLX is the one untested alternative that survives the coverage filter. Co-Authored-By: Claude Opus 5 --- docs/design-notes.md | 191 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/docs/design-notes.md b/docs/design-notes.md index ab2a47e..9b236ff 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -430,6 +430,118 @@ are now known to conflict: the ladder is what repairs the decodes a tight window breaks. Working notes and the full tables are in `whisper-optimize.md` item 1. +**Superseded in part 2026-08-06** by the encoder-reuse patch below: case H (cold +state, auto, sized) no longer garbles, so sizing no longer needs a forced +language *or* a primed state — it needs a fresh state, and `whisper_init_state` +is now measured at **~10 ms** (M5 Pro, best of 10, warm process), not the Metal +setup this paragraph feared. What still stands: the reused-state shrink fault +(D/F/G/I/J/L), the ~800 floor, and the ladder conflict. + +## Auto-detect costs one encoder pass, not two (patched 2026-08-06) + +`whisper_full_with_state` in auto mode encoded the same audio **twice**: +`whisper_lang_auto_detect_with_state` runs the encoder at `seek = 0`, and the +main decode loop then re-encodes that identical first window. On dictation-length +clips the encoder is the whole cost, so auto-detect simply doubled it. + +`patches/whisper.cpp/0001-reuse-detect-encoder-output.patch` (applied by +`make whisper-lib`) does two things: + +- tags the encoder output in `whisper_state` with the `(mel_offset, n_audio_ctx)` + it was computed from, and skips a re-encode that would reproduce it. The detect + decode only writes self-KV, so `embd_enc`/`kv_cross` are still valid; the mel + setters invalidate the tag. +- assigns `exp_n_audio_ctx` **before** the detect block instead of after + (v1.9.1 :6862 vs :6997). One call now encodes at exactly one window size. + +Measured on the standard corpus (M5 Pro, `internal/localbench`, best of 5, +whisper-turbo-q5), before → after: + +``` +clip auto before auto after speedup forced lang +short (1.4 s) 530 ms 274 ms 1.94x unchanged +en (1.6 s) 538 ms 278 ms 1.94x unchanged +en-5.2s 538 ms 278 ms 1.94x unchanged +tr-9.8s 588 ms 329 ms 1.79x unchanged +en-70s 1383 ms 1137 ms 1.22x unchanged +en-183s 3343 ms 3075 ms 1.09x unchanged +``` + +Transcripts are unchanged — the skipped work was bit-identical, and auto output +now equals forced-language output on every corpus clip. The forced-language path +never had the second encode and is untouched, within noise. Long clips gain less +because one saved encode amortises over many windows. + +The second half of the patch also fixes fault-matrix case H, which is what +reopens `audio_ctx` sizing (above). Sizing on top of this is worth a further +~1.7× on clips under ~16 s (274 → ~160 ms at ac=800, fresh state per utterance) +but is **not** transcript-preserving: a different window changes punctuation and +wording, so it is a quality call, not a free win. Not taken; `audioCtxFor` +still returns 0. + +This is upstream issue **#3954** (opened 2026-07-25 from this project, still +unanswered), now implemented. The issue proposed restricting the reuse to +`offset_ms == 0` with a default `audio_ctx`; keying it on the +`(mel_offset, n_audio_ctx)` the output was computed from instead is both simpler +and general — it holds for every window, not just the first. + +**Why the patch is guarded twice.** Every failure mode here is silent: an +unpatched build is still *correct*, just 2× slower on auto, and no test can tell +a correct-but-slow build from a fast one. Worse, `git apply` only matches +context lines, so after an upstream bump the patch can apply cleanly onto a +restructured encode path and quietly stop doing anything. So: + +- `make whisper-lib` refuses to build when the submodule HEAD is not the + `WHISPER_BASE` commit the patches were benchmarked against. A bump is a hard + stop until someone re-runs the fault matrix and the benchmark and moves the + pin — deliberate friction, on the rare operation that earns it. +- `TestWhisperPatchesApplied` compares the submodule's diff to + `patches/whisper.cpp/*.patch` byte-for-byte. That catches a dropped patch, a + bump that shifted the hunks, and hand-edits to the submodule source — the last + of which `git status` cannot show, because `ignore = dirty` (needed since the + patches leave the checkout permanently dirty) suppresses it. + +A fork of whisper.cpp carrying the patch as a commit was considered instead. +Rejected while it is a single upstream-bound patch: pinning the submodule to an +*official* commit plus a readable in-tree diff is easier to audit than a +personal fork, does not make the build depend on a personal repo staying alive, +and unwinds to nothing (delete one file, bump the pin) the day #3954 merges. +Revisit if the patch set grows past two or three, or if upstream declines it and +the divergence becomes long-lived — at that point commit history beats a stack +of `.patch` files. + +**Where the remaining time goes (M5 Pro, turbo-q5, whisper's own counters):** + +``` +clip mel sample encode decode prompt total +en (1.6 s) 0.8 1.6 258.2 12.8 0.0 276.1 +en-5.2s 1.6 1.7 253.3 12.1 0.0 271.0 +trim-15s 3.8 8.4 258.9 58.6 0.0 332.1 +en-70s 16.8 38.4 772.7 259.6 13.9 1109.0 +``` + +Auto and forced-language columns are within noise of each other, which is the +patch working: one encode either way. The encoder is **94% of a dictation-length +transcribe and flat in clip length** — 258 ms whether the audio is 1.6 s or 15 s +— because it always processes the padded 30 s window. Everything else is +single-digit milliseconds. So there is exactly one whisper-side lever left, and +it is `audio_ctx` sizing; decode, mel, sampling and parameter tweaks have +nothing left to give. (Checked while looking: `flash_attn` and `use_gpu` are +already on by `whisper_context_default_params`, and greedy `best_of = 5` costs +nothing at temperature 0 — `n_decoders_cur` is 1 until the fallback ladder +fires, which is also what makes that ladder so expensive when a too-tight +window triggers it.) + +**Newer ggml is not a lever (measured 2026-08-06).** whisper.cpp v1.9.2 is +essentially "sync ggml" over v1.9.1 — every other change in it is VAD or +bindings work zee does not use — so it isolates the ggml 0.13.0 → 0.18.1 jump. +Built standalone and run over the same corpus with zee's own turbo-q5 model and +zee's params (greedy, timestamps on): encode ~255 ms and auto ~515 ms on both, +i.e. **no meaningful win on M5 Pro**; the 70 s/183 s clips moved 3–8%, at the +edge of noise. So the shared-ggml bump — which would mean re-validating +parakeet's in-tree ggml patches against a new base — buys nothing on its own +here. Untested on M1/M2, where older Metal kernels might benefit more. + Ruled out while chasing it (each tested, not assumed): sampling strategy (beam search — whisper-cli's actual default at `beam_size=5` — garbles identically), `no_timestamps` (not a cause *here* — but a serious bug in its own right, see @@ -442,6 +554,13 @@ aborted on exit in the C driver; in zee both engines load and tear down cleanly Raw measurement detail: `zee-whisper-poc/FINDINGS.md`. +**Conditional on `audio_ctx = 0` (noted 2026-08-06).** The whole argument below +rests on the encoder window being fixed at 1500 frames. If `audioCtxFor` ever +returns a sized window, trimming silence shortens the clip, which shrinks the +window, which cuts encode time — the two levers multiply instead of being +independent. Re-measure this entry before reusing it in a world where sizing +ships. + **Silence trimming (VAD before inference): not a whisper speed lever.** Handy runs Silero VAD during capture and drops non-speech before the model ever sees it — tempting to copy, but the speedup doesn't transfer. whisper's @@ -896,6 +1015,78 @@ to turbo-auto on low confidence?), memory cost of a second loaded model irrelevant), and whether the detect pass can run on a short prefix. +## The alternative-engine search space collapses (surveyed 2026-08-06) + +Prompted by "did we try every other library?". The budget above is what settles +it: after the detect-encode patch, a dictation-length whisper transcribe is 94% +one encoder pass over a **padded 30 s window**, and that padding is a property +of the *Whisper architecture*, not of whisper.cpp. So a runtime swap can only +make the same fixed encode faster by a constant; it cannot make the cost scale +with audio length. That splits every candidate into two piles. + +**Engines that avoid the fixed window are already in the repo — but only for +languages Parakeet covers.** Moonshine (useful-sensors) is the headline example +— no zero-padding, compute proportional to audio, ~5× Whisper — and SenseVoice +via sherpa-onnx is the same idea. Inside Parakeet's language set they are +dominated by what zee already ships: Parakeet has no fixed window either, runs +18 ms (110m-en) / 33 ms (v3-multi) on a short clip, and beats Moonshine +tiny/base on accuracy (12.7% / 10.1% WER). sherpa-onnx would additionally mean a +second inference runtime (onnxruntime) next to ggml to run a Parakeet zee +already runs. Rejected on architecture, without benchmarking: no capability zee +lacks. + +**That scoping matters more than it looks.** Parakeet v3's "25 languages" are +the EU official set plus Russian and Ukrainian — **Turkish is not among them**, +and neither are Arabic, Hebrew, Hindi, Japanese, Korean, Mandarin or any other +non-European language. For those users Whisper is not the slow fallback, it is +the *only* local engine, so the fast-path escape hatch above does not exist for +them and every millisecond of the padded 30 s window is their daily latency. +Ranking the remaining levers, weight `audio_ctx` sizing accordingly: it is the +only whisper-side lever left, and for a Turkish or Japanese dictation it is the +only lever there is. + +**So the only open question is coverage, not speed.** Whisper earns its place +purely as the ~99-language fallback; anything replacing it must match that +coverage, which Moonshine (en + 4), SenseVoice (5) and Parakeet v3 (25) do not. +Only the Whisper family survives that filter, which is what makes MLX — the same +weights on a different runtime — the one alternative still worth timing. + +**MLX: genuinely open, but the widely-cited number does not transfer.** A +January 2026 benchmark reports `mlx_whisper` 2.03 ± 0.06× faster than +whisper.cpp on large-v3-turbo. Reading the method: one long file, CLI defaults +on both sides, model load included, hardware unstated. whisper.cpp's CLI default +is beam search at `beam_size = 5` where zee decodes greedy, so an unknown part +of that ratio is a sampling-strategy mismatch rather than runtime speed, and a +long-file throughput ratio says little about a 1.6 s clip whose cost is one +fixed encode. Untested here. The blocker is embedding, not plausibility: +`mlx-whisper` is a Python package, there is no mainstream C/C++ Whisper on MLX, +and MLX would be a *second* GPU runtime beside ggml in a Go binary. Worth +timing before ever being worth integrating. + +**Lead worth following (build simplification, not latency): upstream +whisper.cpp now ships Parakeet itself.** v1.9.2 builds `parakeet-cli` and +`parakeet-quantize`, converts its own GGUFs (`ggml-org/parakeet-GGUF`), and runs +`parakeet-tdt-0.6b-v3` — the same model as `parakeet-v3-multi`. Its +implementation is TDT-only (`parakeet-arch.h` has the TDT duration hparams, no +CTC path), and both of zee's Parakeet models use `Decoder: 2` (TDT), so both are +candidates on paper. If it holds, the parakeet.cpp submodule, its *patched* +ggml, the `WHISPER_USE_SYSTEM_GGML` prefix dance and `TestGGMLPinUnchanged` all +collapse into one upstream submodule on an unpatched ggml — which would also +make future ggml bumps free. + +**Priced 2026-08-06, and the price is a model release.** The formats are *not* +compatible: upstream's Parakeet loads whisper.cpp-style `.bin` files, and +feeding it `models-v3`'s `tdt-0.6b-v3-q4_k.gguf` fails outright with +`parakeet_model_load: invalid model data (bad magic)`. Upstream publishes its +own conversions at `ggml-org/parakeet-GGUF` (f32/f16/q8_0/q4_0/**q4_k** — the +same quantization zee ships), so migrating means re-releasing the model set as +`models-v4`, re-validating accuracy on both models, and every user +re-downloading ~900 MB. It buys build simplicity, not latency or quality: same +weights, same quantization. Park it until something else forces a model release, +and fold it in then. Speed parity with mudler's build was never measured — the +comparison was started and dropped as not worth the download, since the outcome +cannot change the migration's value. + ## Open/untested: Voxtral as a local engine (recorded 2026-08-04) **Never measured.** Voxtral exists in zee only as a cloud provider From 8e11dae71005eeb83675de161c76dc2d00b87d4e Mon Sep 17 00:00:00 2001 From: sumerc Date: Fri, 7 Aug 2026 13:03:06 +0300 Subject: [PATCH 3/8] docs: note Metal resource leak under -race in design notes --- docs/design-notes.md | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/design-notes.md b/docs/design-notes.md index 9b236ff..ec537b6 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -552,8 +552,56 @@ search — whisper-cli's actual default at `beam_size=5` — garbles identically aborted on exit in the C driver; in zee both engines load and tear down cleanly (exit 0, repeatedly). That was the driver's teardown order, not ggml sharing. +> **Qualified 2026-08-07 — "tears down cleanly" is true only of Go's exit path.** +> parakeet.cpp does *not* release all its Metal resources on close; you cannot +> see it because Go exits via `exit_group`, which never runs the ObjC/C++ static +> destructors that would notice. See "Known: parakeet.cpp aborts at exit under +> `-race`" below before spending any time on it. + Raw measurement detail: `zee-whisper-poc/FINDINGS.md`. +## Known: parakeet.cpp aborts at exit under `-race` (do not re-investigate) + +`go test -race` on a package that loads a Parakeet model aborts *after* the +tests pass: + +``` +third_party/parakeet.cpp/third_party/ggml/src/ggml-metal/ggml-metal-device.m:618: +GGML_ASSERT([rsets->data count] == 0) failed +``` + +ggml's own comment on that line: "if you hit this assert, most likely you +haven't deallocated all Metal resources before exiting." That is exactly what +happens — in parakeet.cpp, not in zee. + +**It needs two conditions, which is why it looks like a regression when it +appears.** `-race`, *and* the models actually being on disk. Under `-race` tsan +finalises through libc `exit()`, which runs `__cxa_finalize` and therefore the +ObjC destructor; Go's normal exit is an `exit_group` syscall that skips it +entirely. And with an empty models directory every load fails, so nothing is +ever allocated to leak. CI has neither (it sets no `ZEE_MODELS_DIR` and `make +test` does not depend on `download-models`), so CI is green. It only shows up on +a developer machine pointing tests at real models. + +**Cause is upstream, established by elimination (2026-08-07):** + +- `openParakeet(m)` followed immediately by `eng.Close()` — no provider, no + goroutines, no sessions, no concurrency — still aborts. That is the whole + reproduction. +- `internal/whisper` under `-race` with the same models **passes**, so ggml's + Metal teardown is fine when a caller does release everything. The gap is + parakeet.cpp's. +- No upstream issue exists for it (checked mudler/parakeet.cpp, ggml, llama.cpp). + +**Two zee-side theories were tested and both were wrong**, so do not retry them: +the missing parentheses in `parakeet_async_test.go`'s `_ = s.Close` (a real +typo, still there, but not this), and `localProvider.Close()` not being terminal +against a `load()` queued behind it (fixing it changed nothing). + +**Impact is nil**: production exits via Go, so the destructor never runs and the +leak dies with the process. Not worth carrying a patch for. If it ever needs +fixing, it belongs upstream in parakeet.cpp's context teardown. + **Conditional on `audio_ctx = 0` (noted 2026-08-06).** The whole argument below rests on the encoder window being fixed at 1500 frames. If `audioCtxFor` ever returns a sized window, trimming silence shortens the clip, which shrinks the From 9d9bf73e10d38e277fc74bc55d1103a0b2f049df Mon Sep 17 00:00:00 2001 From: sumerc Date: Fri, 7 Aug 2026 16:55:53 +0300 Subject: [PATCH 4/8] feat: default whisper to English and add auto-detect logging --- docs/design-notes.md | 160 +++++++++++++++++++++++++++++++ internal/whisper/whisper.go | 61 +++++++++++- internal/whisper/whisper_stub.go | 2 + main.go | 9 +- transcriber/local_session.go | 23 +++++ transcriber/whisper.go | 26 +++-- 6 files changed, 270 insertions(+), 11 deletions(-) diff --git a/docs/design-notes.md b/docs/design-notes.md index ec537b6..8f4e635 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -472,6 +472,25 @@ now equals forced-language output on every corpus clip. The forced-language path never had the second encode and is untouched, within noise. Long clips gain less because one saved encode amortises over many windows. +> **Superseded 2026-08-07 — "transcripts are unchanged" is false.** It holds for +> the clean corpus clips it was checked against, and not in general. Measured by +> running the patched binary against a pre-patch build of the same commit +> (`../zee`, unpatched submodule) on identical audio, each binary bit-repeatable +> across runs: +> +> - clean/loud audio, single window: identical output, as claimed. +> - marginal audio (low SNR, accented): **37 of 48** transcripts differ. +> - **> 30 s (multi-window): differs even on clean audio.** A 55.8 s synthetic +> clip decodes 5 phrases unpatched vs 10 + a hallucinated `"Thank you."` +> patched. +> +> Reusing the detect pass's encoder output is not numerically identical to +> recomputing it, and marginal decodes flip on it. Across 48 marginal clips the +> patch was not systematically worse (18 wrong-language unpatched vs 16 +> patched) — it reshuffles rather than degrades. The speedup is unaffected and +> stands. What is retracted is only the correctness claim. No test covers +> either case: the fixtures are ~1.6 s and clean. + The second half of the patch also fixes fault-matrix case H, which is what reopens `audio_ctx` sizing (above). Sizing on top of this is worth a further ~1.7× on clips under ~16 s (274 → ~160 ms at ac=800, fresh state per utterance) @@ -560,6 +579,147 @@ aborted on exit in the C driver; in zee both engines load and tear down cleanly Raw measurement detail: `zee-whisper-poc/FINDINGS.md`. +## Why English is the default language for every model, auto-detect included (2026-08-07) + +Auto-detect was the whisper default because "a wrong forced language garbles the +output, and auto is the only mode that survives code-switching". **The second +half of that is backwards**, per the upstream maintainers: whisper is +*"intended for monolingual audio inputs"* and *"doesn't support code-switching +inputs very well"*. Detection reads only the first 30 s and commits that +language to the whole recording, so auto is the mode that *breaks* on mixed +audio. Specifying the language is what preserves it +([openai/whisper #2009](https://github.com/openai/whisper/discussions/2009), +[#49](https://github.com/openai/whisper/discussions/49)). + +The first half is wrong too, and in a way that matters more. A mismatched +forced language does not garble — it **translates**, fluently. Measured on real +dictation (turbo-q5, M5 Pro): + +| audio | `-lang auto` | `-lang tr` | `-lang en` | +|---|---|---|---| +| Turkish, 5.3 s | Turkish ✅ | Turkish ✅ | `Is this working fine right now?` | +| English, 15.1 s | English ✅ | `Yani ben de doğruyuyorum…` | English ✅ | + +The language token conditions the *output* language; `p.translate = false` only +selects the task token and does not prevent this. So a wrong detection produces +a fluent, on-topic transcript in the wrong language — the hardest error class to +notice, and exactly what a mislabel would not do. + +**Why auto is not merely wrong-in-principle here.** Detection accuracy across +whisper's 102 languages is ~65% for large-v2, near-100% only for the top few +languages; specifying the language is reported as 5–10% more accurate +([#1456](https://github.com/openai/whisper/discussions/1456)). On real saved +samples (Turkish-accented English, speech −32 to −36 dBFS, SNR 4–13 dB — quiet, +which is the realistic dictation case, not a contrived one), 4 of 6 clips +detected wrong. The probability vector, dumped via +`whisper_lang_auto_detect`: + +| clip | detected | p(top) | p(en) | correct? | +|---|---|---|---|---| +| 14-30-40 | en | 0.5082 | — | ✅ | +| 14-39-00 | tr | 0.9125 | 0.0567 | ✅ (really Turkish) | +| 14-47-18 | tr | 0.6829 | 0.2642 | ❌ | +| 15-03-11 | ar | 0.6467 | 0.2432 | ❌ | +| 15-07-10 | tr | 0.7008 | 0.2640 | ❌ | +| 15-08-59 | tr | 0.6690 | 0.3000 | ❌ | + +Two things to take from that table. **A confidence threshold on the winner does +not work** — the one *correct* English call is the least confident row (0.51) +while the failures sit at 0.65–0.70. The discriminating signal is the +runner-up (0.057 on genuinely-Turkish audio vs 0.24–0.30 on every failure), and +reading it needs a further whisper.cpp patch. Fitted on six clips with one +negative case, so it is a hypothesis, not a threshold. + +**Not a zee bug, and not the encoder-reuse patch.** Ruled out by measurement: +detection probabilities are byte-identical between the patched and unpatched +libwhisper; a synthetic sweep of 48 marginal clips flips at ~35% on *both* +builds; and Groq's hosted `whisper-large-v3-turbo` — separate implementation, +unquantized, no ggml, no patch — makes the same errors on the same audio, plus +two the local model gets right (it returns French for 15-03-11 and Turkish for +14-30-40). It is the whisper model family's language ID on quiet accented +speech. Peak-normalising +15–22 dB fixes only 1 of 4. + +**Decision: default every model to `en`, including the multilingual ones.** The +failure modes are asymmetric, which is what settles it: + +| | English speech | short Turkish (< ~25 s) | long Turkish | +|---|---|---|---| +| auto | ~35% → fluent Turkish translation | ✅ | ✅ | +| forced `en` | ✅ | English translation (readable) | Turkish (readable) | + +Forced `en` never produces the unusable case. Long Turkish stays Turkish because +the language token is a soft prior the acoustic evidence can override past one +window — the same clip translates at 10 s and 25 s but not at 50 s. + +Auto remains available in the menu; it is the right choice when the language is +genuinely unknown, which is what upstream built it for. It is no longer the +cheaper option either: since the encoder-reuse patch, auto and forced cost the +same (301 vs 314 ms on a 15 s clip, M5 Pro), so the old "auto costs one extra +encoder pass" argument for *avoiding* it is also gone. + +`lang_detect lang= p=` is now logged per auto transcription +(scraped from whisper's own `WHISPER_LOG_INFO`, which `zee_wsp_hush` used to +discard — zero added cost, no decode change). Forced-language calls log nothing. + +**What comparable apps default to** (read from source, not marketing): + +| App | Default | Source | +|---|---|---| +| VoiceInk | **`en`** | `LanguageSelectionView.swift:11` — `@AppStorage("SelectedLanguage") = "en"`; `WhisperPrompt.swift:88` falls back to `"en"` | +| Handy | `auto` | `src-tauri/src/settings.rs:511` — `default_selected_language() -> "auto"` | +| Voquill | not determined | no persisted default located in `apps/desktop/src` | + +The field is split, so this is not an appeal to consensus — VoiceInk, the +closest comparable (macOS, whisper.cpp, same model), ships `en`. + +## Known: bare-list hints flip the transcription language (measured 2026-08-07, no fix shipped) + +Hints reach the whisper-family engines as free-text prompt — local +`initial_prompt` (pinned to every window via `carry_initial_prompt`), Groq and +OpenAI `prompt`. That prompt conditions **language**, not just vocabulary, and +it outranks the `language` parameter. Found via a saved sample that transcribed +as fluent Turkish although the speech was English (verified with Parakeet +110m-en, which has no Turkish and recovered the real words) *and* the language +was forced to `en`. Isolated to hints alone — same clip, `-lang en`: + +| prompt sent | output | +|---|---| +| none | English ✅ | +| `Opus` (a single bare word) | Turkish | +| the full hints.txt list | Turkish | +| same terms inside an English sentence | English ✅ | + +A bare comma list carries no grammatical language signal, so it neutralises the +language token and the acoustics decide — Turkish-accented English tips over. +One word is enough. Groq reproduces it identically (same decoder behind the +API). Deepgram/ElevenLabs/Mistral are immune: their hints go as structured +keyword fields, never through a decoder. + +Auto-detect is unaffected in both directions: `whisper_lang_auto_detect` runs +before the decode and never sees the prompt (probabilities byte-identical with +and without hints). Two independent failure modes, one visible symptom. + +**Tried and reverted: wrapping hints in an English carrier sentence** +("The following terms may appear: …"). It fixes the bare-list case and even +made forced-`en` hold on 50 s Turkish clips where the bare token lost to the +acoustics. Reverted because it does not survive adversarial hint content and +breaks the other direction: + +| case | result | +|---|---| +| English audio, `-lang en`, hints = 8 Turkish words | Turkish — carrier outvoted | +| Turkish audio, `-lang tr`, English carrier | English — carrier overrode the selection | + +There is no neutral prompt form: one text, its dominant language wins. Any +carrier is an arms race against the hint content. The real constraint is on +`hints.txt` itself — **hints must be written in the dictation language** — and +no wrapper removes it. Current state: hints pass through unmodified (the +pre-existing behaviour), the hazard is documented at the pass-through site, and +the practical mitigations if it bites again are: keep hints.txt to +English-shaped technical terms, or clear it when dictating other languages. +Per-language hint files (`hints.en.txt`, …) would be the correct fix if this +ever matters enough. + ## Known: parakeet.cpp aborts at exit under `-race` (do not re-investigate) `go test -race` on a package that loads a Parakeet model aborts *after* the diff --git a/internal/whisper/whisper.go b/internal/whisper/whisper.go index 67b1954..53eeda0 100644 --- a/internal/whisper/whisper.go +++ b/internal/whisper/whisper.go @@ -23,6 +23,7 @@ package whisper #cgo LDFLAGS: ${SRCDIR}/../../third_party/parakeet.cpp/build-release/third_party/ggml/src/ggml-metal/libggml-metal.a #cgo LDFLAGS: ${SRCDIR}/../../third_party/parakeet.cpp/build-release/third_party/ggml/src/libggml-base.a #cgo LDFLAGS: -lc++ -lm -framework Accelerate -framework Metal -framework MetalKit -framework Foundation +#include #include #include #include "whisper.h" @@ -34,11 +35,54 @@ package whisper static void zee_wsp_silent(enum ggml_log_level l, const char *t, void *u) { (void)l; (void)t; (void)u; } + +// Auto-detect result of the last whisper_full call, scraped from whisper's own +// log line. whisper_full picks the argmax language internally and keeps the +// probability vector on its stack, so short of patching whisper.cpp further +// this is the only place a caller can see what it decided: +// +// whisper_full_with_state: auto-detected language: tr (p = 0.700800) +// +// Worth capturing because a wrong detection does not merely mislabel the +// transcript — whisper hard-forces the start-of-transcript token, so the audio +// decodes AS that language and comes back fluent and wrong. The probability is +// what separates a confident call from a coin toss. +// +// Written on whisper's own thread inside whisper_full and read after it +// returns, both under the Ctx mutex, so only one call is ever in flight. +static char zee_wsp_det_lang[16]; +static float zee_wsp_det_prob; + +static void zee_wsp_log(enum ggml_log_level l, const char *t, void *u) { + (void)l; (void)u; + static const char marker[] = "auto-detected language: "; + const char *m = strstr(t, marker); + if (m == NULL) { + return; // every other line stays silenced + } + char lang[16]; + float p; + if (sscanf(m + sizeof(marker) - 1, "%15s (p = %f)", lang, &p) == 2) { + snprintf(zee_wsp_det_lang, sizeof zee_wsp_det_lang, "%s", lang); + zee_wsp_det_prob = p; + } +} + static void zee_wsp_hush(void) { ggml_log_set(zee_wsp_silent, 0); - whisper_log_set(zee_wsp_silent, 0); + whisper_log_set(zee_wsp_log, 0); } +// zee_wsp_det_clear resets the capture, so a forced-language call (which +// detects nothing) cannot report the previous auto call's result. +static void zee_wsp_det_clear(void) { + zee_wsp_det_lang[0] = '\0'; + zee_wsp_det_prob = 0.0f; +} + +static const char *zee_wsp_det_lang_get(void) { return zee_wsp_det_lang; } +static float zee_wsp_det_prob_get(void) { return zee_wsp_det_prob; } + // zee_wsp_transcribe runs one whisper_full pass and returns the concatenated // segment text as a malloc'd C string (caller frees), or NULL on failure. Doing // the param setup and segment join here keeps the Go side to a two-argument @@ -231,6 +275,7 @@ func (c *Ctx) transcribeAt(pcm []float32, lang, hints string, audioCtx int) (str cHints := C.CString(hints) defer C.free(unsafe.Pointer(cHints)) + C.zee_wsp_det_clear() out := C.zee_wsp_transcribe(c.ptr, (*C.float)(unsafe.Pointer(&pcm[0])), C.int(len(pcm)), cLang, cHints, C.int(audioCtx)) @@ -241,6 +286,20 @@ func (c *Ctx) transcribeAt(pcm []float32, lang, hints string, audioCtx int) (str return C.GoString(out), nil } +// LastDetection reports the language auto-detect picked on the most recent +// Transcribe call and the probability it gave that language. lang is "" when +// the call forced a language, so nothing was detected. +// +// Diagnostic only — nothing acts on it. It exists because a mis-detection is +// invisible in the transcript: the output is fluent, confident and in the wrong +// language, and without the probability there is no way to tell a solid call +// (p≈0.95) from a coin toss (p≈0.65 with the runner-up at 0.30). +func (c *Ctx) LastDetection() (lang string, p float64) { + c.mu.Lock() + defer c.mu.Unlock() + return C.GoString(C.zee_wsp_det_lang_get()), float64(C.zee_wsp_det_prob_get()) +} + // Close frees the model. Safe to call more than once. func (c *Ctx) Close() { c.mu.Lock() diff --git a/internal/whisper/whisper_stub.go b/internal/whisper/whisper_stub.go index dd00ed8..2e30573 100644 --- a/internal/whisper/whisper_stub.go +++ b/internal/whisper/whisper_stub.go @@ -19,4 +19,6 @@ func New(string) (*Ctx, error) { return nil, errUnavailable } func (c *Ctx) Transcribe([]float32, string, string) (string, error) { return "", errUnavailable } +func (c *Ctx) LastDetection() (string, float64) { return "", 0 } + func (c *Ctx) Close() {} diff --git a/main.go b/main.go index ad27be3..497ae16 100644 --- a/main.go +++ b/main.go @@ -365,9 +365,12 @@ func run() { } } streamEnabled = modelSupportsStream(activeTranscriber) - if *langFlag != "" { - activeTranscriber.SetLanguage(*langFlag) - } + // Applied even when empty, for the same reason the flag merge above keeps an + // empty value: "" is Auto-detect, a real choice. Skipping it would leave the + // provider's own default in place — "en" for whisper — so an explicit Auto + // (saved setting, or -lang "") would silently transcribe as English on any + // path the tray does not reach, -transcribe included. + activeTranscriber.SetLanguage(*langFlag) log.SetTranscribeEnabled(*debugTranscribeFlag) if err := log.Init(); err != nil { diff --git a/transcriber/local_session.go b/transcriber/local_session.go index cc6a048..dcefb36 100644 --- a/transcriber/local_session.go +++ b/transcriber/local_session.go @@ -8,8 +8,30 @@ import ( "zee/audio" "zee/encoder" + "zee/log" ) +// logDetectedLanguage records what auto-detect chose, for engines that detect +// at all (whisper; parakeet's models are fixed-language and report nothing). +// +// It is a bare diagnostic, deliberately: a mis-detection produces a fluent +// transcript in the wrong language, which looks like a transcription bug rather +// than a detection one, and the log is the only place the difference shows. +// Measured on real dictation, a correct call and a wrong one are NOT separated +// by the winner's probability — see docs/design-notes.md — so the runner-up +// matters too and is worth having on the record when it becomes available. +func logDetectedLanguage(e localEngine) { + d, ok := e.(interface{ LastDetection() (string, float64) }) + if !ok { + return + } + lang, p := d.LastDetection() + if lang == "" { + return // language was forced; nothing was detected + } + log.Info(fmt.Sprintf("lang_detect lang=%s p=%.4f", lang, p)) +} + // localSession buffers raw S16LE PCM during recording, then runs one batch // inference on Close. Same Session interface as the cloud batch path, so the // live hotkey and -transcribe share it — no encoder, no network. @@ -53,6 +75,7 @@ func (s *localSession) Close() (SessionResult, error) { return SessionResult{AudioData: audioData, AudioFormat: "wav"}, err } inferenceMs := float64(time.Since(start).Microseconds()) / 1000 + logDetectedLanguage(s.engine) text = strings.TrimSpace(text) noSpeech := text == "" diff --git a/transcriber/whisper.go b/transcriber/whisper.go index 9d6559b..f3f1790 100644 --- a/transcriber/whisper.go +++ b/transcriber/whisper.go @@ -8,20 +8,32 @@ import ( // Whisper is the on-device multilingual provider. Unlike Parakeet it takes a // language per transcription, so the tray's language menu is live for it. // -// It defaults to auto-detect ("") rather than a fixed language. That is a -// measured choice, not a preference: whisper's language setting hard-forces the -// start-of-transcript token, so dictating Turkish while the model is pinned to -// English does not merely mislabel the output — it garbles it. Auto-detect -// costs one extra encoder pass (~250 ms on M5) and is the only mode that -// survives code-switching mid-sentence. +// It defaults to "en", not auto-detect, even though the model is multilingual. +// Auto-detect reads only the first 30 s and commits that language to the whole +// recording, and when it guesses wrong the output is not mislabelled — the +// language token conditions the decoder, so the audio comes back *translated*: +// fluent, on-topic, wrong language. Measured on real dictation, detection +// misfires often enough on quiet or accented speech to matter, and the failure +// modes are asymmetric — a forced "en" still yields readable text for Turkish +// speech, while auto can hand back Turkish for English and force a redo. +// Auto stays selectable in the menu for genuinely unknown audio. +// See docs/design-notes.md, "Why English is the default language for every +// model". // whisperEngine adapts a loaded ggml model to localEngine. type whisperEngine struct{ ctx *whisper.Ctx } +// hints become initial_prompt, whose language can override lang — a hazard, +// deliberately unfixed. See design-notes, "Known: bare-list hints flip the +// transcription language". func (e whisperEngine) Transcribe(pcm []float32, lang, hints string) (string, error) { return e.ctx.Transcribe(pcm, lang, hints) } +// LastDetection satisfies the optional interface localSession probes to log +// what auto-detect chose. Whisper is the only engine that detects a language. +func (e whisperEngine) LastDetection() (string, float64) { return e.ctx.LastDetection() } + func (e whisperEngine) Close() { e.ctx.Close() } func openWhisper(m localmodel.Model) (localEngine, error) { @@ -39,7 +51,7 @@ func whisperLanguages(localmodel.Model) []Language { return AllLanguages() } func whisperProvider() ProviderInfo { return localProviderInfo( localmodel.EngineWhisper, "Local (Whisper)", - localmodel.IDWhisperQ5, "", // "" = auto-detect + localmodel.IDWhisperQ5, "en", // multilingual, but English by default — see above whisper.Available(), true, // hints: fed in as whisper's initial prompt openWhisper, whisperLanguages, ) From 61f8ca404d3b2b581d217962753b8ac9a67bf05a Mon Sep 17 00:00:00 2001 From: sumerc Date: Fri, 7 Aug 2026 18:02:09 +0300 Subject: [PATCH 5/8] feat(transcriber): default language to multi and split multi-word hints --- docs/design-notes.md | 64 +++++++++++++++++++++++++++++++++++++++++ transcriber/deepgram.go | 23 +++++++++------ transcriber/mistral.go | 8 +++++- 3 files changed, 86 insertions(+), 9 deletions(-) diff --git a/docs/design-notes.md b/docs/design-notes.md index 8f4e635..aa54733 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -720,6 +720,70 @@ English-shaped technical terms, or clear it when dictating other languages. Per-language hint files (`hints.en.txt`, …) would be the correct fix if this ever matters enough. +**How comparable apps handle the same hazard** (read from source 2026-08-07, +same checkouts as the STT-landscape survey). Both competitors keep user +vocabulary **out of the decoder prompt entirely** — zee is the outlier in +feeding raw user keywords to `initial_prompt`: + +- **VoiceInk**: the whisper prompt is a hardcoded *carrier sentence in the + selected language* — a 25-language table in `WhisperPrompt.swift` ("Hello, + how are you doing? …" / "Merhaba, nasılsın? …"), swapped whenever the + language changes, so the prompt always votes *with* the language parameter, + never against it. User vocabulary never enters that prompt: it is applied + afterwards as case-insensitive regex replacement over the finished transcript + (`WordReplacementService.swift`, called at `TranscriptionPipeline.swift:142`). + A user *can* overwrite the carrier per language (`setCustomPrompt(for:)`), + which reopens the hazard for power users — but per language, so a Turkish + prompt can only ever ride with Turkish selected. The replacements are + **manual**: the user authors explicit wrong→right pairs ("super whisper" → + "Superwhisper"), so the wrong form must be known in advance. +- **Handy**: sends **no prompt at all**, and its replacement is **implicit**: + the user lists only the *correct* words, and `apply_custom_words` + (`audio_toolkit/text.rs`) finds near-misses on its own — length-guarded + Levenshtein (≤25% length difference), a Soundex phonetic boost (score ×0.3 + on phonetic match; ASCII-only, guarded, so non-English terms get plain edit + distance), and n-gram merging for multi-word splits ("Charge B" → + "ChargeBee"). The prompt-steering failure class is structurally impossible + there; the trade is that replacement can only repair words the model nearly + got, it cannot bias recognition itself. Notably, its input format is exactly + zee's `hints.txt` — a bare list of correct terms — so it is the drop-in + semantics if hints ever move out of the prompt. + +Neither app is immune on the *detection* side — VoiceInk shipped and closed +"Spoken English gets transcribed to written German", Handy closed a +Canary-model always-translates-on-auto bug — reinforcing that wrong-language +output is endemic to multilingual STT and only the prompt-steering half is +designable-away. If hints biasing is ever revisited here, these are the two +proven shapes: language-matched carrier only (VoiceInk), or post-processing +replacement with no prompt (Handy). + +The closed-source apps (docs, 2026-08-07): **superwhisper** injects vocabulary +into the prompt exactly like zee — and its +[docs](https://superwhisper.com/docs/get-started/interface-vocabulary) carry +the hazard as user-facing caveats: "adding too many words can confuse the AI +transcription model", foreign-language vocabulary "may degrade accuracy", and +vocabulary "affects not just spelling but also punctuation, **language +detection**, and formatting". Their recommended posture is vocabulary +minimally + post-hoc replacements for anything that must be reliable. +**Wispr Flow** claims "word boosting" during transcription plus replacement +rules after; mechanics unverifiable (own model stack). Also confirmed: the +bare-list flip reproduces on Groq's hosted `whisper-large-v3-turbo` verbatim +(same clip, `language=en`: no prompt → English 2/2, hints as prompt → Turkish +2/2), so the hazard is the model family's, not our build's. + +The field at a glance: + +| app | vocab reaches the model? | mechanism | language-flip risk | +|---|---|---|---| +| zee | yes | raw list → `initial_prompt` | live, documented here | +| superwhisper | yes | vocab → prompt | live, documented in their docs | +| Wispr Flow | claimed | "word boosting" + replacements after | unknown (closed stack) | +| VoiceInk | no | language-locked carrier prompt; regex replace after | designed out | +| Handy | no | no prompt; fuzzy replace after | impossible | + +Nobody has both acoustic biasing and safety: the prompt-injectors carry the +hazard, the post-processors gave up biasing to be rid of it. + ## Known: parakeet.cpp aborts at exit under `-race` (do not re-investigate) `go test -race` on a package that loads a Parakeet model aborts *after* the diff --git a/transcriber/deepgram.go b/transcriber/deepgram.go index 1ac0f38..8decead 100644 --- a/transcriber/deepgram.go +++ b/transcriber/deepgram.go @@ -90,19 +90,26 @@ func (d *Deepgram) Transcribe(audioData []byte, format, lang, hints string) (*Re contentType = "audio/mpeg" } - apiURL := d.apiURL + u, err := url.Parse(d.apiURL) + if err != nil { + return nil, err + } + q := u.Query() + if lang != "" { + q.Set("language", lang) + } else { + // Mirrors the streaming session: "" can still arrive from stale configs + // or headless flags, and omitting the param silently means English-only, + // so send nova-3's multilingual mode as the least-wrong interpretation. + q.Set("language", "multi") + } if hints != "" { - u, err := url.Parse(apiURL) - if err != nil { - return nil, err - } - q := u.Query() for _, term := range strings.Split(hints, ",") { q.Add("keyterm", strings.TrimSpace(term)) } - u.RawQuery = q.Encode() - apiURL = u.String() } + u.RawQuery = q.Encode() + apiURL := u.String() req, err := http.NewRequest("POST", apiURL, bytes.NewReader(audioData)) if err != nil { diff --git a/transcriber/mistral.go b/transcriber/mistral.go index e7372db..9cbb5bf 100644 --- a/transcriber/mistral.go +++ b/transcriber/mistral.go @@ -66,8 +66,14 @@ func (m *Mistral) Transcribe(audioData []byte, format, lang, hints string) (*Res writer.WriteField("language", lang) } if hints != "" { + // Mistral 400s the WHOLE request if any context_bias item contains + // whitespace, and multi-word hints ("App Router") are legal in + // hints.txt — the shipped default even contains one. Split such terms + // into single words rather than failing the transcription. for _, word := range strings.Split(hints, ",") { - writer.WriteField("context_bias[]", strings.TrimSpace(word)) + for _, w := range strings.Fields(word) { + writer.WriteField("context_bias[]", w) + } } } writer.Close() From 3144a1cf152966869705673dcc5e50fac6a1299e Mon Sep 17 00:00:00 2001 From: sumerc Date: Mon, 10 Aug 2026 13:11:38 +0300 Subject: [PATCH 6/8] =?UTF-8?q?feat:=20add=20optional=20model=20warm?= =?UTF-8?q?=E2=80=91up=20and=20disable=E2=80=91hints=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 1 + docs/design-notes.md | 133 +++++++++++++++++++++++++++------------ main.go | 15 ++++- transcriber/local.go | 44 +++++++++++++ transcriber/warm_test.go | 49 +++++++++++++++ 5 files changed, 201 insertions(+), 41 deletions(-) create mode 100644 transcriber/warm_test.go diff --git a/CLAUDE.md b/CLAUDE.md index bee021a..dca10e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,6 +79,7 @@ overwriting. `BENCH_FILE=` overrides the destination. - `-runs N` - benchmark iterations (default: 3) - `-logpath ` - log directory (default: `$ZEE_LOG_PATH` or OS-specific, use `./` for current directory) - `-hints ` - comma-separated vocabulary hints (overrides `hints.txt`) +- `-no-hints` - disable vocabulary hints entirely (ignore `hints.txt`) - `-transcribe ` - transcribe an audio file (mp3/flac/wav) and exit The tray's "Save Last Recording" persists the last clip (audio + `info.json`) to `/samples/`; a failed transcription auto-saves there too, with the error recorded in `info.json`. diff --git a/docs/design-notes.md b/docs/design-notes.md index aa54733..23e4b25 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -597,9 +597,14 @@ dictation (turbo-q5, M5 Pro): | audio | `-lang auto` | `-lang tr` | `-lang en` | |---|---|---|---| -| Turkish, 5.3 s | Turkish ✅ | Turkish ✅ | `Is this working fine right now?` | +| English, 5.3 s (14-39-00) | Turkish ❌ | Turkish (translation) | English ✅ | | English, 15.1 s | English ✅ | `Yani ben de doğruyuyorum…` | English ✅ | +(The first row was originally recorded as "Turkish audio" on the strength of +auto's own p=0.91 detection; Parakeet ground truth later proved the speech +English. The mechanism conclusion is unchanged — the `-lang tr` column is a +fluent Turkish *translation* of English speech either way.) + The language token conditions the *output* language; `p.translate = false` only selects the task token and does not prevent this. So a wrong detection produces a fluent, on-topic transcript in the wrong language — the hardest error class to @@ -617,18 +622,31 @@ detected wrong. The probability vector, dumped via | clip | detected | p(top) | p(en) | correct? | |---|---|---|---|---| | 14-30-40 | en | 0.5082 | — | ✅ | -| 14-39-00 | tr | 0.9125 | 0.0567 | ✅ (really Turkish) | +| 14-39-00 | tr | 0.9125 | 0.0567 | ❌ (later ground-truthed: English) | | 14-47-18 | tr | 0.6829 | 0.2642 | ❌ | | 15-03-11 | ar | 0.6467 | 0.2432 | ❌ | | 15-07-10 | tr | 0.7008 | 0.2640 | ❌ | | 15-08-59 | tr | 0.6690 | 0.3000 | ❌ | +| 21-07-10 (live) | tr | 0.8041 | — | ❌ (Parakeet-verified English) | + +The last row is the first production capture from the `lang_detect` log line. +Adjacent unsaved clips in the same session logged tr at 0.9981 and 0.9975; the +one correct call logged en at 0.5473. On this speaker, wrong calls are +consistently *more* confident than right ones. Two things to take from that table. **A confidence threshold on the winner does -not work** — the one *correct* English call is the least confident row (0.51) -while the failures sit at 0.65–0.70. The discriminating signal is the -runner-up (0.057 on genuinely-Turkish audio vs 0.24–0.30 on every failure), and -reading it needs a further whisper.cpp patch. Fitted on six clips with one -negative case, so it is a hypothesis, not a threshold. +not work** — the one correct call is the least confident row (0.51), and +14-39-00 is a *wrong* call at **0.91**: ground-truthing it later the same day +(Parakeet 110m-en, an English-only model with no translation ability, produced +clean idiomatic English — "Is this working fine right now? I don't, I'm not +sure") proved the speech was English. Detection on these samples is therefore +1/7 correct, with its most confident answers wrong. **The runner-up rule died +with that correction**: it read 14-39-00's p(en)=0.057 as the genuine-Turkish +signature separating real Turkish from misdetections — but 14-39-00 was a +misdetection too, so a wrong call can carry a runner-up of 0.057 and the +claimed 4× separation was an artifact of one mislabeled clip. No probability +read-out from a single detect pass — winner or runner-up — separates right +from wrong on this data. **Not a zee bug, and not the encoder-reuse patch.** Ruled out by measurement: detection probabilities are byte-identical between the patched and unpatched @@ -636,20 +654,33 @@ libwhisper; a synthetic sweep of 48 marginal clips flips at ~35% on *both* builds; and Groq's hosted `whisper-large-v3-turbo` — separate implementation, unquantized, no ggml, no patch — makes the same errors on the same audio, plus two the local model gets right (it returns French for 15-03-11 and Turkish for -14-30-40). It is the whisper model family's language ID on quiet accented -speech. Peak-normalising +15–22 dB fixes only 1 of 4. +14-30-40). It is the whisper model family's language ID on accented speech. +Loudness is secondary, not causal: peak-normalising the failing clips by ++15–22 dB fixed only 1 of 4 and made one *more* confidently wrong +(15-07-10: tr 0.70 → 0.80). Low SNR widens the blast radius — the synthetic +sweep flips ~35% at SNR 7–12 vs 0 at SNR 30 on the same accented voice — but +gain alone does not rescue detection on the real recordings. **Decision: default every model to `en`, including the multilingual ones.** The failure modes are asymmetric, which is what settles it: -| | English speech | short Turkish (< ~25 s) | long Turkish | -|---|---|---|---| -| auto | ~35% → fluent Turkish translation | ✅ | ✅ | -| forced `en` | ✅ | English translation (readable) | Turkish (readable) | - -Forced `en` never produces the unusable case. Long Turkish stays Turkish because -the language token is a soft prior the acoustic evidence can override past one -window — the same clip translates at 10 s and 25 s but not at 50 s. +| | English speech | Turkish speech | +|---|---|---| +| auto | wrong-language coin toss on quiet/accented audio | untested — no real Turkish sample exists | +| forced `en` | ✅ correct (7/7 saved samples + 1 live, hints off) | short: rough English translation · long (58 s): stays correct Turkish | + +Forced `en` never produced an unusable output on any real sample. Caveats on +the record: every "Turkish speech" behaviour rests on synthetic Yelda-TTS +clips — Parakeet ground-truthing showed *all* real saved samples were English +speech, so no genuine recording has been through this matrix. The original +length-crossover evidence (translates at 10/25 s, stays Turkish at 50 s) was +invalid twice over — measured on a clip later proven to be English speech, and +contaminated by the auto-created default `hints.txt` — but the behaviour +itself was then re-confirmed clean on genuine synthetic Turkish (hints off): +9.6 s → lossy English translation ("yürüyüş yaptım" became "going to sleep"), +58 s → correct Turkish transcript. The language token holds for about one +window, then the acoustics win. Note the translation is *rough*, not faithful — +"translates it for me" is not a feature to rely on, merely a readable failure. Auto remains available in the menu; it is the right choice when the language is genuinely unknown, which is what upstream built it for. It is no longer the @@ -691,34 +722,56 @@ was forced to `en`. Isolated to hints alone — same clip, `-lang en`: A bare comma list carries no grammatical language signal, so it neutralises the language token and the acoustics decide — Turkish-accented English tips over. -One word is enough. Groq reproduces it identically (same decoder behind the -API). Deepgram/ElevenLabs/Mistral are immune: their hints go as structured -keyword fields, never through a decoder. +One word is enough. Groq reproduces it verbatim (same decoder behind the API; +same clip, `language=en`: no prompt → English 2/2, hints as prompt → Turkish +2/2, deterministic). Deepgram/ElevenLabs/Mistral are immune, verified live: +their hints go as structured keyword fields (`keyterm`, `keyterms[]`, +`context_bias[]`), never through a decoder — even 8 Turkish keyterms on +`-lang en` left the output English on all three. Auto-detect is unaffected in both directions: `whisper_lang_auto_detect` runs before the decode and never sees the prompt (probabilities byte-identical with and without hints). Two independent failure modes, one visible symptom. **Tried and reverted: wrapping hints in an English carrier sentence** -("The following terms may appear: …"). It fixes the bare-list case and even -made forced-`en` hold on 50 s Turkish clips where the bare token lost to the -acoustics. Reverted because it does not survive adversarial hint content and -breaks the other direction: +("The following terms may appear: …"). It fixed the bare-list case, but it +does not survive adversarial hint content, and it breaks the other direction: | case | result | |---|---| -| English audio, `-lang en`, hints = 8 Turkish words | Turkish — carrier outvoted | +| English audio, `-lang en`, short carrier + 8 Turkish hint words | Turkish — carrier outvoted | | Turkish audio, `-lang tr`, English carrier | English — carrier overrode the selection | -There is no neutral prompt form: one text, its dominant language wins. Any -carrier is an arms race against the hint content. The real constraint is on -`hints.txt` itself — **hints must be written in the dictation language** — and -no wrapper removes it. Current state: hints pass through unmodified (the -pre-existing behaviour), the hazard is documented at the pass-through site, and -the practical mitigations if it bites again are: keep hints.txt to -English-shaped technical terms, or clear it when dictating other languages. -Per-language hint files (`hints.en.txt`, …) would be the correct fix if this -ever matters enough. +Follow-up measurements sharpened *why*, and killed the obvious repairs: + +- **Sizing works, per-language authoring required.** A long grammatical + carrier (~3× the hint tokens) beats the 8-Turkish-word attack, and a long + *Turkish* carrier holds `-lang tr` around 12 English tech terms. The app + controls both sides, so the ratio is controllable — up to whisper's + 224-token prompt budget, and only with a hand-written sentence per language. +- **Count-matching fails.** Padding the list with common English words at 1:1 + and 2:1 against the Turkish terms changed nothing; order did not matter + either. The language signal is grammatical coherence, not token count: a + word bag reads as no language, gets discounted wholesale, and the acoustics + decide. There is no neutral prompt form and no cancellation trick — the + prompt channel has exactly two safe states, coherent prose in the + transcription language or empty. + +The real constraint is on `hints.txt` itself — **hints must be written in the +dictation language** — and no wrapper removes it. Current state: hints pass +through unmodified (the pre-existing behaviour), the hazard is documented at +the pass-through site, and `-no-hints` disables the mechanism entirely. If it +bites again: keep hints.txt to English-shaped technical terms, or run with +`-no-hints`. Per-language hint files (`hints.en.txt`, …) would be the correct +fix if this ever matters enough. + +Two adjacent facts caught in the same investigation: `config.GetHints` +**auto-creates** `hints.txt` with the default template on first touch of a +config dir, so "no hints" configurations silently carry the default list (this +contaminated several controls before it was caught — beware in future A/Bs). +And that default contains `App Router`, which Mistral rejects with a 400 for +whitespace — until the sanitization in `mistral.go`, every Mistral +transcription on a default config failed outright. **How comparable apps handle the same hazard** (read from source 2026-08-07, same checkouts as the STT-landscape survey). Both competitors keep user @@ -766,10 +819,7 @@ vocabulary "affects not just spelling but also punctuation, **language detection**, and formatting". Their recommended posture is vocabulary minimally + post-hoc replacements for anything that must be reliable. **Wispr Flow** claims "word boosting" during transcription plus replacement -rules after; mechanics unverifiable (own model stack). Also confirmed: the -bare-list flip reproduces on Groq's hosted `whisper-large-v3-turbo` verbatim -(same clip, `language=en`: no prompt → English 2/2, hints as prompt → Turkish -2/2), so the hazard is the model family's, not our build's. +rules after; mechanics unverifiable (own model stack). The field at a glance: @@ -1146,8 +1196,11 @@ of capitalised terms pushes mid-sentence capitals the other way. Punctuation follows the same rule. Write `hints.txt` the way the output should look. Ordinary prompt-conditioning side effects come with it (a repeated word at the -end of one clip, a dropped comma). Biasing is a trade, not a free win — which is -why it stays opt-in per user rather than being seeded with defaults. +end of one clip, a dropped comma). Biasing is a trade, not a free win. +(Superseded detail: `GetHints` does seed a default `hints.txt` template on +first run, so in practice hints are on by default. And the trade turned out +far worse than style bleed — a bare hint list can flip the entire output +language; see "Known: bare-list hints flip the transcription language".) ## Why the login item is written but never bootstrapped (2026-07-28) diff --git a/main.go b/main.go index 497ae16..30c168d 100644 --- a/main.go +++ b/main.go @@ -245,6 +245,7 @@ func run() { logPathFlag := flag.String("logpath", "", "log directory path (default: OS-specific location, use ./ for current dir)") testFlag := flag.Bool("test", false, "Test mode (headless, stdin-driven)") hintsFlag := flag.String("hints", "", "Vocabulary hints for transcription (comma-separated)") + noHintsFlag := flag.Bool("no-hints", false, "Disable vocabulary hints entirely (ignore hints.txt)") transcribeFlag := flag.String("transcribe", "", "Transcribe audio file(s) and exit; extra files may follow as positional args (one transcript printed per line)") providerFlag := flag.String("provider", "", "Transcription provider (e.g. parakeet, groq); overrides saved config") modelFlag := flag.String("model", "", "Model ID for the selected provider; overrides saved config") @@ -321,7 +322,9 @@ func run() { switch *formatFlag { case "mp3@16", "mp3@64", "flac": activeFormat = *formatFlag - if *hintsFlag != "" { + if *noHintsFlag { + config.SetHints("") // pins hints empty; hints.txt is never read + } else if *hintsFlag != "" { config.SetHints(*hintsFlag) } default: @@ -966,6 +969,16 @@ func tryStartSession(sessions chan<- recSession) *atomic.Bool { denyBusy("Already recording or transcribing.") return nil } + // After a long idle, macOS has paged the local model out and the first + // inference pays seconds of page-in. Re-touch it now, in parallel with the + // recording, so the cost is gone by release. The provider itself decides + // whether a warm is due (idle threshold) — a no-op for cloud providers. + configMu.Lock() + tr := activeTranscriber + configMu.Unlock() + if w, ok := tr.(interface{ Warm() }); ok { + go w.Warm() + } sc := &atomic.Bool{} audio.PlayStart() // reflexive: sound the press now, not after the record loop spins up (playOne is non-blocking) sessions <- recSession{Stop: resetStop(), SilenceClose: sc, PressedAt: time.Now()} diff --git a/transcriber/local.go b/transcriber/local.go index 5ad8c02..c305f3f 100644 --- a/transcriber/local.go +++ b/transcriber/local.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "sync" + "time" "zee/audio" + "zee/encoder" "zee/localmodel" + "zee/log" ) // localEngine is the entire engine-specific surface of an on-device provider: @@ -37,6 +40,7 @@ type localProvider struct { engine localEngine loadErr error lang string + lastUsed time.Time // last time the engine's memory was touched (load, session, warm) name string // provider name, e.g. "parakeet" defaultID string // this engine's model, used when modelID belongs to another @@ -183,6 +187,43 @@ func (p *localProvider) load() { p.mu.Lock() p.engine, p.loadErr, p.loadedID = eng, err, want + p.lastUsed = time.Now() // a fresh load is resident by definition + p.mu.Unlock() +} + +// warmIdleThreshold is the idle gap after which the next keydown pre-touches +// the model. macOS compresses/evicts the loaded weights after long idle +// (observed as RSS ~780 → 160 MB overnight), and the first inference then +// pays the whole page-in serially — measured seconds, not milliseconds, see +// design-notes "felt-latency" notes. Below the threshold the weights are +// still resident and a warm pass would be pure battery cost. +const warmIdleThreshold = 10 * time.Minute + +// Warm re-touches the loaded model if it has been idle long enough to have +// been paged out, so the page-in overlaps the recording instead of delaying +// the transcription. Runs one short silent inference — the only way through +// the engine API to fault every weight page and GPU buffer back in. Blocking; +// callers run it in a goroutine at recording start. The engine mutex +// serializes it against the real inference, so a fast release never races it +// — worst case the inference queues behind the tail of the page-in, which is +// still never slower than paying it cold. +func (p *localProvider) Warm() { + p.mu.Lock() + eng := p.engine + ready := eng != nil && p.loadErr == nil && p.loadedID == p.modelID + idle := time.Since(p.lastUsed) + p.mu.Unlock() + if !ready || idle < warmIdleThreshold { + return // loading, broken, or still resident + } + start := time.Now() + // Fixed language, not auto: a warm pass must not exercise detection (it + // would log a bogus lang_detect line for silence). Result discarded. + eng.Transcribe(make([]float32, encoder.SampleRate), "en", "") + log.Info(fmt.Sprintf("model_warm engine=%s idle_s=%.0f warm_ms=%.0f", + p.name, idle.Seconds(), float64(time.Since(start).Microseconds())/1000)) + p.mu.Lock() + p.lastUsed = time.Now() p.mu.Unlock() } @@ -293,6 +334,9 @@ func (p *localProvider) NewSession(_ context.Context, cfg SessionConfig) (Sessio if cfg.Language != "" { lang = cfg.Language } + p.mu.Lock() + p.lastUsed = time.Now() // inference follows within this record cycle + p.mu.Unlock() return &localSession{engine: eng, lang: lang, hints: cfg.Hints, updates: make(chan string)}, nil } diff --git a/transcriber/warm_test.go b/transcriber/warm_test.go new file mode 100644 index 0000000..256bd3d --- /dev/null +++ b/transcriber/warm_test.go @@ -0,0 +1,49 @@ +package transcriber + +import ( + "sync/atomic" + "testing" + "time" +) + +type warmStubEngine struct{ calls atomic.Int32 } + +func (e *warmStubEngine) Transcribe([]float32, string, string) (string, error) { + e.calls.Add(1) + return "", nil +} +func (e *warmStubEngine) Close() {} + +// Warm must run a touch inference only when the engine is loaded AND idle past +// the threshold. The recently-used and not-ready cases must be no-ops: warming +// on every keydown would burn GPU for nothing, and warming a half-loaded +// provider would race the loader. +func TestWarmIdleGate(t *testing.T) { + eng := &warmStubEngine{} + p := &localProvider{name: "stub", modelID: "m", loadedID: "m", engine: eng} + + p.lastUsed = time.Now() + p.Warm() + if n := eng.calls.Load(); n != 0 { + t.Fatalf("recently used: want no warm inference, got %d", n) + } + + p.lastUsed = time.Now().Add(-warmIdleThreshold - time.Minute) + p.Warm() + if n := eng.calls.Load(); n != 1 { + t.Fatalf("idle past threshold: want 1 warm inference, got %d", n) + } + if time.Since(p.lastUsed) > time.Second { + t.Fatal("Warm did not refresh lastUsed — the next keydown would warm again") + } + + // Mid-switch (loadedID != modelID) and unloaded engines must be skipped. + p.lastUsed = time.Now().Add(-warmIdleThreshold - time.Minute) + p.modelID = "other" + p.Warm() + p.modelID, p.engine = "m", nil + p.Warm() + if n := eng.calls.Load(); n != 1 { + t.Fatalf("not-ready provider: want no extra inference, got %d", n) + } +} From 3a7ee786cb2cf9f426f5cc9603281996897f6fcf Mon Sep 17 00:00:00 2001 From: sumerc Date: Wed, 12 Aug 2026 15:28:30 +0300 Subject: [PATCH 7/8] test: add verification that Warm hook fires when starting a session --- main_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/main_test.go b/main_test.go index 574fba0..31a4d7c 100644 --- a/main_test.go +++ b/main_test.go @@ -291,3 +291,45 @@ func TestDeviceChangeAction(t *testing.T) { } } } + +// warmSpy is a Transcriber that records whether the optional Warm() hook +// fired. It guards the structural assertion in tryStartSession: Warm is wired +// through an anonymous interface, so a rename or receiver change would not +// fail the build — only this test notices the trigger going dead. +type warmSpy struct { + *transcriber.FakeTranscriber + warmed chan struct{} +} + +func (w *warmSpy) Warm() { + select { + case w.warmed <- struct{}{}: + default: + } +} + +func TestTryStartSessionTriggersWarm(t *testing.T) { + isRecording.Store(false) + defer isRecording.Store(false) + + spy := &warmSpy{FakeTranscriber: transcriber.NewFake("", nil), warmed: make(chan struct{}, 1)} + configMu.Lock() + old := activeTranscriber + activeTranscriber = spy + configMu.Unlock() + defer func() { + configMu.Lock() + activeTranscriber = old + configMu.Unlock() + }() + + sessions := make(chan recSession, 1) + if tryStartSession(sessions) == nil { + t.Fatal("expected the session to start") + } + select { + case <-spy.warmed: + case <-time.After(2 * time.Second): + t.Fatal("Warm() never fired on session start — tryStartSession's interface assertion no longer matches") + } +} From 23011814bcc54d7be0f9b8f812bbcc419b090509 Mon Sep 17 00:00:00 2001 From: sumerc Date: Thu, 13 Aug 2026 12:10:23 +0300 Subject: [PATCH 8/8] feat: add listen mode for meeting transcription --- config/config.go | 15 +-- listen.go | 280 ++++++++++++++++++++++++++++++++++++++++++++ listen_test.go | 189 ++++++++++++++++++++++++++++++ main.go | 45 +++++-- tray/tray.go | 17 +++ tray/tray_darwin.go | 21 ++++ tray/tray_other.go | 1 + 7 files changed, 554 insertions(+), 14 deletions(-) create mode 100644 listen.go create mode 100644 listen_test.go diff --git a/config/config.go b/config/config.go index b9c3ddd..f0f2438 100644 --- a/config/config.go +++ b/config/config.go @@ -17,13 +17,14 @@ import ( // keycode; an empty Hotkey (no Mods) means "use the built-in default" // (hotkey.Combo.OrDefault resolves that). type Settings struct { - Language string `json:"language"` - Device string `json:"device"` - Provider string `json:"provider"` - Model string `json:"model"` - Hotkey hotkey.Combo `json:"hotkey"` - AutoPaste bool `json:"auto_paste"` - AutoStart bool `json:"auto_start"` + Language string `json:"language"` + Device string `json:"device"` + Provider string `json:"provider"` + Model string `json:"model"` + Hotkey hotkey.Combo `json:"hotkey"` + AutoPaste bool `json:"auto_paste"` + ListenMode bool `json:"listen_mode"` // transcribe to transcript.txt instead of pasting + AutoStart bool `json:"auto_start"` // TailWaitMs keeps the mic open this many ms after the hotkey is released so // a fast keyup doesn't clip the last word. 0 disables the wait. TailWaitMs int `json:"tail_wait_ms"` diff --git a/listen.go b/listen.go new file mode 100644 index 0000000..b5c75de --- /dev/null +++ b/listen.go @@ -0,0 +1,280 @@ +package main + +// Listen mode (POC): meeting capture. It is a destination, not a second +// capture path — the mic, VAD, overlay, feedback sounds, device selection and +// TCC permission prompt all come from the ordinary recording flow. The single +// difference is where the text goes: appended to transcript.txt in chunks as +// the meeting runs, instead of one transcript to the clipboard at the end. +// +// It plugs in as a transcriber.Session, so handleRecording drives it exactly +// as it drives a normal session — Feed on every capture callback, Close at the +// end. A first attempt owned its own audio.CaptureDevice and bypassed all of +// that; it recorded silence for four minutes because nothing had prompted for +// microphone permission, on the wrong device, with no overlay to show it. +// +// Known POC limits: chunks decode independently (whisper runs with +// no_context), so a sentence split across a cut loses its thread even at a +// clean silence boundary — overlap-and-merge is the real fix. No speaker +// attribution. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "zee/config" + "zee/encoder" + "zee/log" + "zee/transcriber" + "zee/tray" +) + +// Chunking bounds. A cut is preferred at a silence, but only inside a window: +// below listenMinChunk silence is ignored (cutting at every clause boundary +// fragments the audio, and whisper decodes a two-word fragment far worse than +// a sentence — it is trained on 30 s windows), and at listenMaxChunk the cut +// happens regardless so no chunk approaches that window. +// +// listenMinSilence is 500 ms: a clause boundary, above the 200-300 ms gaps +// inside a sentence and safely above stop consonants. At 20 ms VAD frames that +// is 25 consecutive non-speech frames, so one misclassified frame cannot cut. +// The field converges here — faster-whisper setups use 300-500 ms, Silero +// defaults to 2000 ms; nobody cuts as short as 200 ms. +const ( + listenMinSilence = 500 * time.Millisecond + listenMinChunk = 8 * time.Second + listenMaxChunk = 25 * time.Second + listenCutPoll = 100 * time.Millisecond +) + +// listenTranscriptFile sits in the config dir, beside config.json and the +// samples — where every other artefact the app produces lives. +const listenTranscriptFile = "transcript.txt" + +// listenMode is the live setting, mirrored from config so the record path can +// read it without touching the file. Guarded by configMu, like autoPaste. +var listenMode bool + +type listenChunkJob struct { + pcm []byte + reason string // why the cut happened: "silence", "maxlen" or "stop" + cutAt time.Time +} + +// listenSink is the transcriber.Session that writes a meeting to disk. +type listenSink struct { + tr transcriber.Transcriber + lang string + hints string + path string + vp *vadProcessor + + mu sync.Mutex + pcm []byte // raw S16LE from Feed, drained by the cutter + + // queue is deliberately generous rather than bounded: dropping a meeting is + // worse than holding memory. If it ever filled, the cutter would block and + // the audio would accumulate in pcm instead — the same backlog, one buffer + // earlier. listen_chunk logs depth and lag so a real one shows up first. + queue chan listenChunkJob + + updates chan string + stop chan struct{} + stopOnce sync.Once + cutterEnd chan struct{} + workerEnd chan struct{} + + written int // chunks appended, for the closing log line +} + +// newListenSink starts the cutter and the transcription worker. The caller +// feeds it audio; nothing here touches the microphone. +func newListenSink(tr transcriber.Transcriber, lang, hints string) (*listenSink, error) { + dir := config.Dir() + if dir == "" { + return nil, fmt.Errorf("no config directory for the transcript") + } + vp, err := newVADProcessor() + if err != nil { + return nil, fmt.Errorf("VAD init: %w", err) + } + s := &listenSink{ + tr: tr, + lang: lang, + hints: hints, + path: filepath.Join(dir, listenTranscriptFile), + vp: vp, + queue: make(chan listenChunkJob, 256), // ~1 h of backlog at these chunk sizes + updates: make(chan string), + stop: make(chan struct{}), + cutterEnd: make(chan struct{}), + workerEnd: make(chan struct{}), + } + // One header per session, so a single file can hold several meetings and + // stay readable. The file is append-only by design and never truncated: a + // lost meeting is worse than a long file. + s.append(fmt.Sprintf("\n=== listen session %s (%v-%v chunks) ===\n", + time.Now().Format("2006-01-02 15:04:05"), listenMinChunk, listenMaxChunk)) + go s.worker() + go s.cutter() + log.Info(fmt.Sprintf("listen_start min_s=%.0f max_s=%.0f silence_ms=%d path=%s", + listenMinChunk.Seconds(), listenMaxChunk.Seconds(), + listenMinSilence.Milliseconds(), s.path)) + return s, nil +} + +// Feed takes raw PCM from the recording session's capture callback. It copies: +// the caller reuses its buffer after Feed returns. +func (s *listenSink) Feed(pcm []byte) { + buf := make([]byte, len(pcm)) + copy(buf, pcm) + s.mu.Lock() + s.pcm = append(s.pcm, buf...) + s.mu.Unlock() + s.vp.Process(buf) // drives the silence detection the cutter reads +} + +// Updates satisfies the interface; listen mode emits no streaming partials, so +// the channel only ever closes. +func (s *listenSink) Updates() <-chan string { return s.updates } + +// buffered reports how much audio is waiting to be cut. +func (s *listenSink) buffered() time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return time.Duration(len(s.pcm)) * time.Second / (encoder.SampleRate * 2) +} + +// take removes and returns everything buffered so far. +func (s *listenSink) take() []byte { + s.mu.Lock() + defer s.mu.Unlock() + buf := s.pcm + s.pcm = nil + return buf +} + +// cutter decides chunk boundaries. It never transcribes, so a slow inference +// cannot stall cut decisions or the capture callback behind them. +func (s *listenSink) cutter() { + defer close(s.cutterEnd) + ticker := time.NewTicker(listenCutPoll) + defer ticker.Stop() + for { + select { + case <-ticker.C: + switch dur := s.buffered(); { + case dur >= listenMaxChunk: + s.cut("maxlen") + case dur >= listenMinChunk && !s.vp.SpeakingNow(listenMinSilence): + s.cut("silence") + } + case <-s.stop: + s.cut("stop") // tail: whatever arrived since the last cut + return + } + } +} + +// cut moves buffered audio to the queue. Chunks with no speech are dropped +// rather than queued: whisper invents text on pure silence (phantom "Thank +// you." and subtitle credits), which would fill a quiet meeting's transcript +// with words nobody said. +func (s *listenSink) cut(reason string) { + raw := s.take() + if len(raw) == 0 { + return + } + if _, speech := s.vp.StatsDelta(); speech == 0 { + log.Info(fmt.Sprintf("listen_skip reason=%s audio_s=%.1f cause=no_speech", + reason, float64(len(raw))/2/float64(encoder.SampleRate))) + return + } + s.queue <- listenChunkJob{pcm: raw, reason: reason, cutAt: time.Now()} +} + +// worker drains the queue serially: the engine serializes inference anyway, +// and the transcript must stay in order. +func (s *listenSink) worker() { + defer close(s.workerEnd) + for job := range s.queue { + s.transcribe(job) + } +} + +// transcribe decodes one chunk and appends it. Errors are logged and the +// worker continues — one failed chunk must not end a meeting. +func (s *listenSink) transcribe(job listenChunkJob) { + start := time.Now() + sess, err := s.tr.NewSession(context.Background(), transcriber.SessionConfig{ + Format: "wav", + Language: s.lang, + Hints: s.hints, + }) + if err != nil { + log.Errorf("listen: session: %v", err) + return + } + sess.Feed(job.pcm) + res, err := sess.Close() + if err != nil { + log.Errorf("listen: transcribe: %v", err) + return + } + // queue is the backlog depth, lag_ms how long the chunk waited before its + // inference began. Both climbing together means transcription is losing to + // real time — the signal to widen the bounds or pick a faster model. + log.Info(fmt.Sprintf("listen_chunk audio_s=%.1f cut=%s inference_ms=%.0f lag_ms=%.0f queue=%d chars=%d", + float64(len(job.pcm))/2/float64(encoder.SampleRate), job.reason, + float64(time.Since(start).Microseconds())/1000, + float64(time.Since(job.cutAt).Microseconds())/1000, len(s.queue), len(res.Text))) + if res.Text == "" { + return + } + s.written++ + s.append(fmt.Sprintf("[%s] %s\n", time.Now().Format("15:04:05"), res.Text)) +} + +// append adds one line, reopening the file each time so an interrupted session +// (crash, force quit) still leaves everything written so far on disk. +func (s *listenSink) append(line string) { + f, err := os.OpenFile(s.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Errorf("listen: cannot write %s: %v", s.path, err) + return + } + defer f.Close() + if _, err := f.WriteString(line); err != nil { + log.Errorf("listen: write: %v", err) + } +} + +// Close cuts the tail and waits for the queue to drain, so the transcript is +// complete when it returns. A long backlog therefore makes Close slow — right, +// since the alternative is discarding speech already recorded. +func (s *listenSink) Close() (transcriber.SessionResult, error) { + close(s.updates) + s.stopOnce.Do(func() { close(s.stop) }) + <-s.cutterEnd // tail queued + close(s.queue) // no more jobs + <-s.workerEnd // every queued chunk transcribed + s.append(fmt.Sprintf("=== end %s ===\n", time.Now().Format("15:04:05"))) + log.Info(fmt.Sprintf("listen_stop chunks=%d path=%s", s.written, s.path)) + // No text is returned on purpose: listen mode's output is the file, and a + // transcript here would be copied to the clipboard and pasted. + return transcriber.SessionResult{NoSpeech: true}, nil +} + +// setListenMode is the tray handler: a persisted setting, like auto-paste. +// Nothing starts or stops here — the next recording picks it up. +func setListenMode(on bool) { + configMu.Lock() + listenMode = on + configMu.Unlock() + config.Update(func(c *config.Settings) { c.ListenMode = on }) + tray.SetListen(on) + log.Info(fmt.Sprintf("listen_mode on=%v", on)) +} diff --git a/listen_test.go b/listen_test.go new file mode 100644 index 0000000..02cbdae --- /dev/null +++ b/listen_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "encoding/binary" + "math" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "zee/config" + "zee/encoder" + "zee/transcriber" +) + +// newTestSink builds a sink writing into a temp config dir. It never touches +// the microphone: listen mode is fed by the ordinary recording session. +func newTestSink(t *testing.T, text string) *listenSink { + t.Helper() + config.SetDir(t.TempDir()) + vp, err := newVADProcessor() + if err != nil { + t.Fatalf("VAD init: %v", err) + } + return &listenSink{ + tr: transcriber.NewFake(text, nil), + path: filepath.Join(t.TempDir(), "transcript.txt"), + vp: vp, + queue: make(chan listenChunkJob, 8), + updates: make(chan string), + stop: make(chan struct{}), + cutterEnd: make(chan struct{}), + workerEnd: make(chan struct{}), + } +} + +// speechPCM is a loud 220 Hz tone. WebRTC VAD calls it speech, which is what +// the cutter gates on; digital silence would be skipped instead. +func speechPCM(seconds int) []byte { + n := encoder.SampleRate * seconds + b := make([]byte, n*2) + for i := 0; i < n; i++ { + v := int16(9000 * math.Sin(2*math.Pi*220*float64(i)/float64(encoder.SampleRate))) + binary.LittleEndian.PutUint16(b[i*2:], uint16(v)) + } + return b +} + +// listenSink must satisfy transcriber.Session — that is the whole design: the +// ordinary record path drives it, so listen mode inherits the mic, VAD, +// overlay, device selection and permission prompt for free. +func TestListenSinkIsASession(t *testing.T) { + var _ transcriber.Session = (*listenSink)(nil) +} + +// A chunk must be transcribed and appended as one timestamped line, and the +// buffer drained so the next cut cannot re-transcribe the same audio. +func TestListenTranscribeWritesAndDrains(t *testing.T) { + s := newTestSink(t, "hello meeting") + s.Feed(speechPCM(1)) + s.transcribe(listenChunkJob{pcm: s.take(), reason: "test", cutAt: time.Now()}) + + body, err := os.ReadFile(s.path) + if err != nil { + t.Fatalf("transcript not written: %v", err) + } + if !strings.Contains(string(body), "hello meeting") { + t.Fatalf("transcript missing the text, got %q", body) + } + if len(s.pcm) != 0 { + t.Fatalf("buffer not drained: %d bytes would be transcribed twice", len(s.pcm)) + } +} + +// Whisper invents text on pure silence, so a chunk with no speech must be +// dropped rather than queued — otherwise a quiet meeting fills the transcript +// with words nobody said. +func TestListenCutSkipsSilentChunk(t *testing.T) { + s := newTestSink(t, "phantom text") + s.Feed(make([]byte, encoder.SampleRate*2)) // 1 s of digital silence + s.cut("test") + if len(s.queue) != 0 { + t.Fatal("silent chunk was queued — whisper would hallucinate on it") + } +} + +// The cutter must ignore silence below listenMinChunk (cutting every clause +// fragments the audio) and cut once past it. +func TestListenCutterHonoursBounds(t *testing.T) { + s := newTestSink(t, "x") + go s.cutter() + defer func() { s.stopOnce.Do(func() { close(s.stop) }); <-s.cutterEnd }() + + s.Feed(speechPCM(2)) + time.Sleep(3 * listenCutPoll) + if len(s.queue) != 0 { + t.Fatalf("cut below listenMinChunk (%v) — fragments the audio", listenMinChunk) + } + + s.Feed(speechPCM(int(listenMinChunk.Seconds()))) + deadline := time.After(3 * time.Second) + for { + select { + case job := <-s.queue: + if job.reason != "silence" { + t.Fatalf("cut reason = %q, want silence", job.reason) + } + return + case <-deadline: + t.Fatal("no cut after exceeding listenMinChunk with a silent VAD") + case <-time.After(listenCutPoll): + } + } +} + +// Close must cut the tail and drain the queue, so nothing recorded is lost — +// and must return no text, or the record path would paste the meeting into +// whatever window has focus. +func TestListenCloseFlushesTailAndReturnsNoText(t *testing.T) { + s := newTestSink(t, "tail words") + go s.worker() + go s.cutter() + s.Feed(speechPCM(1)) + + res, err := s.Close() + if err != nil { + t.Fatalf("Close: %v", err) + } + if res.Text != "" || res.HasText { + t.Fatalf("Close returned text %q — it would be pasted", res.Text) + } + body, _ := os.ReadFile(s.path) + if !strings.Contains(string(body), "tail words") { + t.Fatalf("tail audio was dropped, transcript=%q", body) + } +} + +// Feed runs on the audio thread while the cutter drains; -race proves the +// buffer is safe. Feed must also copy, since the caller reuses its slice. +func TestListenFeedCopiesAndIsRaceFree(t *testing.T) { + s := newTestSink(t, "x") + shared := []byte{1, 2, 3, 4} + s.Feed(shared) + shared[0] = 99 + if s.pcm[0] == 99 { + t.Fatal("Feed kept the caller's buffer — it is reused after Feed returns") + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + s.Feed([]byte{1, 2, 3, 4}) + } + }() + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + s.take() + } + }() + wg.Wait() +} + +// A persisted listen_mode must reach the tray before the menu is built. +// Without the seed the checkbox renders unchecked while the mode is on, so the +// first click appears to do nothing (it re-sets the value it already had) and +// the toggle feels stuck one step behind. +func TestListenModeSeededFromConfig(t *testing.T) { + config.SetDir(t.TempDir()) + config.Update(func(c *config.Settings) { c.ListenMode = true }) + + if err := config.Load(); err != nil { + t.Fatalf("Load: %v", err) + } + if !config.Get().ListenMode { + t.Fatal("ListenMode did not persist to config.json") + } + + // What main.go does at startup, and the assertion that it happens at all. + listenMode = config.Get().ListenMode + if !listenMode { + t.Fatal("listenMode not seeded from config — the tray would show it off while it is on") + } + t.Cleanup(func() { listenMode = false }) +} diff --git a/main.go b/main.go index 30c168d..ca27cd4 100644 --- a/main.go +++ b/main.go @@ -106,6 +106,7 @@ type recordingConfig struct { lang string hints string autoPaste bool + listen bool // transcribe to transcript.txt instead of the clipboard tailWait time.Duration // mic kept open after release so a fast keyup doesn't clip the last word pressToRecordMs float64 // press→mic-live, filled at record start; logged with the transcription metrics releasedAt time.Time // recording end, filled once it happens; start of the felt-latency metric @@ -315,6 +316,7 @@ func run() { } if !flagSet["autopaste"] { autoPaste = cfg.AutoPaste + listenMode = cfg.ListenMode } else { autoPaste = *autoPasteFlag } @@ -476,6 +478,10 @@ func run() { }) } tray.SetAutoPaste(autoPaste) + // Seeded before Init, like auto-paste: the menu is built from these, so + // without it a persisted listen_mode=true renders as an unchecked box — + // the mode silently on while the UI says off. + tray.SetListen(listenMode) var trayModels []tray.Model modelIndex := map[string]transcriber.ModelInfo{} @@ -649,6 +655,8 @@ func run() { }) tray.SetHotkeyLabel(cfg.Hotkey.OrDefault().Display()) + tray.OnListen(setListenMode) + trayQuit := tray.Init() tray.OnAutoPaste(func(on bool) { configMu.Lock() @@ -834,6 +842,7 @@ func run() { configMu.Lock() apChanged := autoPaste != s.AutoPaste autoPaste = s.AutoPaste + listenMode = s.ListenMode configMu.Unlock() if apChanged { tray.SetAutoPaste(s.AutoPaste) @@ -1157,20 +1166,35 @@ func handleRecording(capture audio.CaptureDevice, sess recSession) (<-chan struc lang: activeTranscriber.GetLanguage(), hints: config.GetHints(), autoPaste: autoPaste, + listen: listenMode, tailWait: time.Duration(config.Get().TailWaitMs) * time.Millisecond, } configMu.Unlock() + // Listen mode writes a meeting to a file; pasting each chunk into whatever + // window has focus would be actively harmful, and streaming partials have + // nowhere to go. + if cfg.listen { + cfg.autoPaste, cfg.stream = false, false + } if cfg.autoPaste && !permissions.HasAccessibility() { cfg.autoPaste = false tray.SetError("Auto-paste is waiting for Accessibility permission") } - tSess, err := cfg.tr.NewSession(context.Background(), transcriber.SessionConfig{ - Stream: cfg.stream, - Format: cfg.format, - Language: cfg.lang, - Hints: cfg.hints, - }) + var tSess transcriber.Session + var err error + if cfg.listen { + // Same capture, VAD, overlay and feedback as a normal recording — only + // the destination differs. See listen.go. + tSess, err = newListenSink(cfg.tr, cfg.lang, cfg.hints) + } else { + tSess, err = cfg.tr.NewSession(context.Background(), transcriber.SessionConfig{ + Stream: cfg.stream, + Format: cfg.format, + Language: cfg.lang, + Hints: cfg.hints, + }) + } if err != nil { return nil, err } @@ -1204,7 +1228,14 @@ func handleRecording(capture audio.CaptureDevice, sess recSession) (<-chan struc } }() - rec, err := newRecordingSession(capture, sess.Stop, tSess, sess.SilenceClose, cfg.tailWait) + // A meeting has long quiet stretches, and toggle mode arms the silence + // auto-close — which would end the session after the first 30 s pause. + // Hand listen mode a handle that is never armed instead. + silenceClose := sess.SilenceClose + if cfg.listen { + silenceClose = &atomic.Bool{} + } + rec, err := newRecordingSession(capture, sess.Stop, tSess, silenceClose, cfg.tailWait) if err != nil { tSess.Close() return nil, err diff --git a/tray/tray.go b/tray/tray.go index 1e7ce56..4ef023d 100644 --- a/tray/tray.go +++ b/tray/tray.go @@ -51,6 +51,9 @@ var ( autoPasteOn bool autoPasteCb func(bool) + listenOn bool + listenCb func(bool) + loginOn bool loginAvailable = true loginCb func(bool) error @@ -84,6 +87,20 @@ func OnRecord(start, stop func()) { recordFn = start; stopFn = stop } func OnAutoPaste(fn func(bool)) { autoPasteCb = fn } func OnLogin(fn func(bool) error) { loginCb = fn } +// OnListen registers the Listen Mode handler. Listen mode is meeting capture: +// the mic stays open and each fixed-length chunk is transcribed to a file, +// independent of the push-to-talk cycle. +func OnListen(fn func(bool)) { listenCb = fn } + +// SetListen re-renders the Listen Mode checkbox. Needed because the app can +// turn the mode off by itself (capture failure), not only by user click. +func SetListen(on bool) { + trayMu.Lock() + listenOn = on + trayMu.Unlock() + updateListenItem(on) +} + // SetAutoPaste / SetLogin set the checkbox state; before Init they seed the // menu build, after Init (config-file reload) they re-render the item. func SetAutoPaste(on bool) { diff --git a/tray/tray_darwin.go b/tray/tray_darwin.go index 716e7d7..0baf186 100644 --- a/tray/tray_darwin.go +++ b/tray/tray_darwin.go @@ -20,6 +20,7 @@ var ( mSettings *systray.MenuItem mAutoPaste *systray.MenuItem + mListen *systray.MenuItem mLogin *systray.MenuItem mHotkey *systray.MenuItem mEditHints *systray.MenuItem @@ -82,6 +83,17 @@ func updateAutoPasteItem(on bool) { } } +func updateListenItem(on bool) { + if mListen == nil { + return + } + if on { + mListen.Check() + } else { + mListen.Uncheck() + } +} + func updateLoginItem(on bool) { if mLogin == nil { return @@ -250,6 +262,15 @@ func onReady() { } }) + mListen = mSettings.AddSubMenuItemCheckbox("Listen Mode", + "Keep the mic open and append each chunk to transcript.txt", listenOn) + mListen.Click(func() { + want := !mListen.Checked() + if listenCb != nil { + listenCb(want) // the app confirms via SetListen; a failed start turns it back off + } + }) + // Greyed out, same title: a suffix like "(installed app only)" would widen // the whole submenu to fit it. The tooltip carries the why. loginTip := "Launch zee when you log in" diff --git a/tray/tray_other.go b/tray/tray_other.go index 0cb203f..7355444 100644 --- a/tray/tray_other.go +++ b/tray/tray_other.go @@ -17,5 +17,6 @@ func updateStatusItem(string) {} func updateModelItem(int) {} func setHintsEnabled(bool) {} func updateAutoPasteItem(bool) {} +func updateListenItem(bool) {} func updateLoginItem(bool) {} func updateHotkeyDisplay() {}