Listen mode poc

.gitmodules

@@ -5,3 +5,4 @@
 [submodule "third_party/whisper.cpp"]
 	path = third_party/whisper.cpp
 	url = https://github.com/ggml-org/whisper.cpp
+	ignore = dirty

CLAUDE.md

@@ -79,6 +79,7 @@ overwriting. `BENCH_FILE=` overrides the destination.
 - `-runs N` - benchmark iterations (default: 3)
 - `-logpath <path>` - log directory (default: `$ZEE_LOG_PATH` or OS-specific, use `./` for current directory)
 - `-hints <words>` - comma-separated vocabulary hints (overrides `hints.txt`)
+- `-no-hints` - disable vocabulary hints entirely (ignore `hints.txt`)
 - `-transcribe <file>` - transcribe an audio file (mp3/flac/wav) and exit
 
 The tray's "Save Last Recording" persists the last clip (audio + `info.json`) to `<config>/samples/`; a failed transcription auto-saves there too, with the error recorded in `info.json`.

Makefile

@@ -16,6 +16,16 @@ PARAKEET_LIB  := $(PARAKEET_DIR)/build-release/libparakeet.a
 GGML_PREFIX   := $(CURDIR)/$(PARAKEET_DIR)/build-release/ggml-prefix
 WHISPER_DIR   := third_party/whisper.cpp
 WHISPER_LIB   := $(WHISPER_DIR)/build-release/src/libwhisper.a
+# In-tree patches applied to the pinned whisper.cpp checkout before it builds.
+# whisper-lib applies any that are missing and forces a reconfigure when it does,
+# so a `git submodule update` that resets the checkout cannot silently drop them.
+WHISPER_PATCHES := patches/whisper.cpp
+# The upstream commit those patches were written and benchmarked against.
+# `git apply` only matches context lines, so a patch can still apply cleanly onto
+# a restructured encode path and be quietly wrong — a correct-but-slow build that
+# no test can distinguish from a fast one. Bumping the submodule therefore has to
+# stop the build until a human re-validates and moves this pin.
+WHISPER_BASE  := f049fff95a089aa9969deb009cdd4892b3e74916
 HOST          := $(shell go env GOOS)/$(shell go env GOARCH)
 ifeq ($(HOST),darwin/arm64)
 CGO_ENV := MACOSX_DEPLOYMENT_TARGET=$(MACOS_MIN) CGO_CFLAGS=-mmacosx-version-min=$(MACOS_MIN) CGO_LDFLAGS=-mmacosx-version-min=$(MACOS_MIN)
@@ -77,6 +87,24 @@ whisper-lib: parakeet-lib
 	  echo "==> initializing whisper.cpp submodule (first checkout)"; \
 	  git submodule update --init --recursive $(WHISPER_DIR); \
 	fi; \
+	head=$$(git -C $(WHISPER_DIR) rev-parse HEAD 2>/dev/null); \
+	if [ "$$head" != "$(WHISPER_BASE)" ]; then \
+	  echo "ERROR: whisper.cpp is at $$head"; \
+	  echo "       but $(WHISPER_PATCHES)/*.patch were validated against $(WHISPER_BASE)."; \
+	  echo ""; \
+	  echo "  git apply checks context lines, not meaning: these patches may still apply"; \
+	  echo "  cleanly onto a moved encode path and silently stop working. Re-validate:"; \
+	  echo "    ZEE_AC_DEBUG=1 go test ./internal/whisper -run FaultMatrix -v   # H must pass"; \
+	  echo "    make bench-local                                                # auto ~= forced"; \
+	  echo "  then regenerate the patch and set WHISPER_BASE to $$head."; \
+	  exit 1; \
+	fi; \
+	for p in $(CURDIR)/$(WHISPER_PATCHES)/*.patch; do \
+	  if git -C $(WHISPER_DIR) apply --reverse --check $$p 2>/dev/null; then continue; fi; \
+	  echo "==> applying $$(basename $$p)"; \
+	  git -C $(WHISPER_DIR) apply $$p || exit 1; \
+	  rm -rf $(WHISPER_DIR)/build-release; \
+	done; \
 	if [ ! -d $(WHISPER_DIR)/build-release ]; then \
 	  echo "==> configuring whisper.cpp (one-time)"; \
 	  cmake -S $(WHISPER_DIR) -B $(WHISPER_DIR)/build-release \

config/config.go

@@ -17,13 +17,14 @@ import (
 // keycode; an empty Hotkey (no Mods) means "use the built-in default"
 // (hotkey.Combo.OrDefault resolves that).
 type Settings struct {
-	Language  string       `json:"language"`
-	Device    string       `json:"device"`
-	Provider  string       `json:"provider"`
-	Model     string       `json:"model"`
-	Hotkey    hotkey.Combo `json:"hotkey"`
-	AutoPaste bool         `json:"auto_paste"`
-	AutoStart bool         `json:"auto_start"`
+	Language   string       `json:"language"`
+	Device     string       `json:"device"`
+	Provider   string       `json:"provider"`
+	Model      string       `json:"model"`
+	Hotkey     hotkey.Combo `json:"hotkey"`
+	AutoPaste  bool         `json:"auto_paste"`
+	ListenMode bool         `json:"listen_mode"` // transcribe to transcript.txt instead of pasting
+	AutoStart  bool         `json:"auto_start"`
 	// TailWaitMs keeps the mic open this many ms after the hotkey is released so
 	// a fast keyup doesn't clip the last word. 0 disables the wait.
 	TailWaitMs int `json:"tail_wait_ms"`

internal/whisper/patch_test.go

@@ -0,0 +1,75 @@
+//go:build darwin && arm64
+
+package whisper_test
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// zee builds whisper.cpp from a pinned submodule plus the in-tree patches in
+// patches/whisper.cpp, applied by `make whisper-lib`. Two ways that goes wrong
+// silently, neither of which any other test can see:
+//
+//   - a `git submodule update` resets the checkout and drops the patches. The
+//     build still succeeds and transcripts are still correct — auto-detect just
+//     quietly costs twice what it should again.
+//   - someone hand-edits the submodule source. `ignore = dirty` in .gitmodules
+//     (needed because the patches make the checkout permanently dirty) means
+//     `git status` will not show it.
+//
+// Comparing the submodule's diff against the patch files byte-for-byte catches
+// both, and also fires when the submodule is bumped: hunk offsets move, so the
+// diff stops matching and a human has to re-validate rather than trust a clean
+// `git apply`. The Makefile's WHISPER_BASE pin is the build-time half of this.
+func TestWhisperPatchesApplied(t *testing.T) {
+	root := filepath.Join("..", "..")
+	patchDir := filepath.Join(root, "patches", "whisper.cpp")
+
+	entries, err := os.ReadDir(patchDir)
+	if err != nil {
+		t.Fatalf("read %s: %v", patchDir, err)
+	}
+	var want strings.Builder
+	for _, e := range entries {
+		if filepath.Ext(e.Name()) != ".patch" {
+			continue
+		}
+		b, err := os.ReadFile(filepath.Join(patchDir, e.Name()))
+		if err != nil {
+			t.Fatalf("read patch: %v", err)
+		}
+		want.Write(b)
+	}
+	if want.Len() == 0 {
+		t.Fatalf("no patches found in %s", patchDir)
+	}
+
+	// Same flags the patches are generated with, so the comparison is not at the
+	// mercy of the developer's diff config.
+	cmd := exec.Command("git", "-c", "core.pager=cat", "-C",
+		filepath.Join(root, "third_party", "whisper.cpp"),
+		"diff", "--no-color", "--no-ext-diff")
+	out, err := cmd.Output()
+	if err != nil {
+		t.Skipf("whisper.cpp submodule not checked out: %v", err)
+	}
+
+	if string(out) != want.String() {
+		t.Fatalf("third_party/whisper.cpp does not match patches/whisper.cpp/*.patch\n\n"+
+			"got %d bytes of diff, want %d.\n\n"+
+			"Either the patches were dropped (run `make whisper-lib` to reapply), the\n"+
+			"submodule source was hand-edited, or the submodule was bumped and the\n"+
+			"patches now apply at different offsets. In the last case re-validate the\n"+
+			"optimisation before regenerating — a clean `git apply` does not prove the\n"+
+			"patch still does anything:\n"+
+			"    ZEE_AC_DEBUG=1 go test ./internal/whisper -run FaultMatrix -v\n"+
+			"    make bench-local     # auto must still cost ~the same as forced\n"+
+			"    git -C third_party/whisper.cpp -c core.pager=cat diff --no-color \\\n"+
+			"        --no-ext-diff > patches/whisper.cpp/0001-reuse-detect-encoder-output.patch",
+			len(out), want.Len())
+	}
+}

internal/whisper/whisper.go

@@ -23,6 +23,7 @@ package whisper
 #cgo LDFLAGS: ${SRCDIR}/../../third_party/parakeet.cpp/build-release/third_party/ggml/src/ggml-metal/libggml-metal.a
 #cgo LDFLAGS: ${SRCDIR}/../../third_party/parakeet.cpp/build-release/third_party/ggml/src/libggml-base.a
 #cgo LDFLAGS: -lc++ -lm -framework Accelerate -framework Metal -framework MetalKit -framework Foundation
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "whisper.h"
@@ -34,11 +35,54 @@ package whisper
 static void zee_wsp_silent(enum ggml_log_level l, const char *t, void *u) {
     (void)l; (void)t; (void)u;
 }
+
+// Auto-detect result of the last whisper_full call, scraped from whisper's own
+// log line. whisper_full picks the argmax language internally and keeps the
+// probability vector on its stack, so short of patching whisper.cpp further
+// this is the only place a caller can see what it decided:
+//
+//   whisper_full_with_state: auto-detected language: tr (p = 0.700800)
+//
+// Worth capturing because a wrong detection does not merely mislabel the
+// transcript — whisper hard-forces the start-of-transcript token, so the audio
+// decodes AS that language and comes back fluent and wrong. The probability is
+// what separates a confident call from a coin toss.
+//
+// Written on whisper's own thread inside whisper_full and read after it
+// returns, both under the Ctx mutex, so only one call is ever in flight.
+static char  zee_wsp_det_lang[16];
+static float zee_wsp_det_prob;
+
+static void zee_wsp_log(enum ggml_log_level l, const char *t, void *u) {
+    (void)l; (void)u;
+    static const char marker[] = "auto-detected language: ";
+    const char *m = strstr(t, marker);
+    if (m == NULL) {
+        return;   // every other line stays silenced
+    }
+    char lang[16];
+    float p;
+    if (sscanf(m + sizeof(marker) - 1, "%15s (p = %f)", lang, &p) == 2) {
+        snprintf(zee_wsp_det_lang, sizeof zee_wsp_det_lang, "%s", lang);
+        zee_wsp_det_prob = p;
+    }
+}
+
 static void zee_wsp_hush(void) {
     ggml_log_set(zee_wsp_silent, 0);
-    whisper_log_set(zee_wsp_silent, 0);
+    whisper_log_set(zee_wsp_log, 0);
 }
 
+// zee_wsp_det_clear resets the capture, so a forced-language call (which
+// detects nothing) cannot report the previous auto call's result.
+static void zee_wsp_det_clear(void) {
+    zee_wsp_det_lang[0] = '\0';
+    zee_wsp_det_prob = 0.0f;
+}
+
+static const char *zee_wsp_det_lang_get(void) { return zee_wsp_det_lang; }
+static float       zee_wsp_det_prob_get(void) { return zee_wsp_det_prob; }
+
 // zee_wsp_transcribe runs one whisper_full pass and returns the concatenated
 // segment text as a malloc'd C string (caller frees), or NULL on failure. Doing
 // the param setup and segment join here keeps the Go side to a two-argument
@@ -57,7 +101,7 @@ static char *zee_wsp_transcribe(struct whisper_context *ctx, const float *pcm,
     // a fixed 30 s per decode and skips the retry-on-failed-decode path, so
     // whatever the model does not emit in a window is lost for good.
     p.translate        = false;   // transcribe in-language, never translate to English
-    p.language         = lang;    // "auto" => detect (costs one extra encoder pass)
+    p.language         = lang;    // "auto" => detect (one extra decode step; see audioCtxFor)
     p.audio_ctx        = audio_ctx;  // 0 = full window; see audioCtxFor
 
     // Vocabulary hints ride in as the initial prompt — the same string the
@@ -144,10 +188,15 @@ const sampleRate = 16000
 // encode reads whatever the PREVIOUS call left in exp_n_audio_ctx, so it only
 // shrinks from a cold state (0, which every read site expands to 1500). Primed
 // once at a small size, later auto calls at a LARGER size are grows and are
-// correct (matrix cases M/N). Returning 0 is now a cost decision, not a
-// correctness one — measured ~1.9x at a floor of 800, with an over-tight window
-// actively slower. See design-notes "audio_ctx sizing" for the numbers and the
-// open questions.
+// correct (matrix cases M/N).
+//
+// Superseded 2026-08-06 for the cold case: patches/whisper.cpp now assigns
+// exp_n_audio_ctx before the detect encode, so one call encodes at one size and
+// case H (cold, auto, sized) passes. Sizing on a REUSED state still garbles
+// (D/F/G/I/J/L unchanged), so the lever needs a fresh whisper_state per
+// utterance — measured ~10 ms, i.e. affordable. Returning 0 stays a deliberate
+// choice: sizing is worth a further ~1.7x but does not preserve the transcript
+// word-for-word. See design-notes "audio_ctx sizing".
 func audioCtxFor(int) int { return 0 }
 
 // Available reports whether local Whisper transcription is compiled in.
@@ -167,7 +216,7 @@ var hushOnce sync.Once
 // also warms up: one throwaway transcribe so the backend's first-use init
 // (Metal pipeline compilation, buffer/kernel setup) happens now rather than
 // stalling the first real dictation. The warm-up runs in auto-detect mode
-// because that is the default path, so both encoder passes get warmed.
+// because that is the default path.
 func New(path string) (*Ctx, error) {
 	c, err := newNoWarm(path)
 	if err != nil {
@@ -194,9 +243,10 @@ func newNoWarm(path string) (*Ctx, error) {
 }
 
 // Transcribe runs the model over mono 16 kHz float32 PCM and returns the
-// transcript. lang is an ISO-639-1 code; "" means auto-detect, which costs one
-// extra encoder pass but is the only mode that survives code-switching — a
-// wrong forced language garbles the output rather than merely mislabelling it.
+// transcript. lang is an ISO-639-1 code; "" means auto-detect, which is the
+// only mode that survives code-switching — a wrong forced language garbles the
+// output rather than merely mislabelling it. Detection is close to free: it
+// shares its encoder pass with the first decode window (patches/whisper.cpp).
 //
 // hints is optional vocabulary biasing (the same comma-separated string the
 // cloud providers take as `prompt`); "" disables it.
@@ -225,6 +275,7 @@ func (c *Ctx) transcribeAt(pcm []float32, lang, hints string, audioCtx int) (str
 	cHints := C.CString(hints)
 	defer C.free(unsafe.Pointer(cHints))
 
+	C.zee_wsp_det_clear()
 	out := C.zee_wsp_transcribe(c.ptr,
 		(*C.float)(unsafe.Pointer(&pcm[0])), C.int(len(pcm)),
 		cLang, cHints, C.int(audioCtx))
@@ -235,6 +286,20 @@ func (c *Ctx) transcribeAt(pcm []float32, lang, hints string, audioCtx int) (str
 	return C.GoString(out), nil
 }
 
+// LastDetection reports the language auto-detect picked on the most recent
+// Transcribe call and the probability it gave that language. lang is "" when
+// the call forced a language, so nothing was detected.
+//
+// Diagnostic only — nothing acts on it. It exists because a mis-detection is
+// invisible in the transcript: the output is fluent, confident and in the wrong
+// language, and without the probability there is no way to tell a solid call
+// (p≈0.95) from a coin toss (p≈0.65 with the runner-up at 0.30).
+func (c *Ctx) LastDetection() (lang string, p float64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return C.GoString(C.zee_wsp_det_lang_get()), float64(C.zee_wsp_det_prob_get())
+}
+
 // Close frees the model. Safe to call more than once.
 func (c *Ctx) Close() {
 	c.mu.Lock()

internal/whisper/whisper_stub.go

@@ -19,4 +19,6 @@ func New(string) (*Ctx, error) { return nil, errUnavailable }
 
 func (c *Ctx) Transcribe([]float32, string, string) (string, error) { return "", errUnavailable }
 
+func (c *Ctx) LastDetection() (string, float64) { return "", 0 }
+
 func (c *Ctx) Close() {}