Exported (it is the whole point — an adb-reachable surface, unlike the app's non-exported - * services), mirroring {@link org.appdevforall.k2go.delivery.debug.DebugDeliveryReceiver}. A huge - * floor makes any real free-space reading CRITICAL, tripping the guard for real. Example: + * services), mirroring {@link org.appdevforall.k2go.delivery.debug.DebugDeliveryReceiver}. + * + *
Two modes. The low-disk path: a huge floor makes any real free-space reading CRITICAL, tripping + * the guard for real: * *
* adb shell am broadcast \ @@ -23,10 +25,20 @@ * --el floor_bytes 999999999999 ** - * Watch it act in logcat: {@code adb logcat -s K2Go-DiskGuard}. The debug hook runs the FORCED path, - * which always CONTAINs: it reaps and reclaims, then leaves the server desired=UP and asks the - * reconciler to relaunch a fresh box. It never advances the real escalation count, so repeated - * triggers cannot stop the box. + * The firehose path (K2GO-386 L3a): pass {@code --ez firehose true} to exercise the second trigger. It + * skips the dash-node signal fetch but STILL runs the real growth re-probe, so it reaps only if a + * {@code .log} is actually growing now -- stage a fast-growing log first: + * + *
+ * adb shell am broadcast \ + * -a org.appdevforall.k2go.DEBUG_DISK_GUARD \ + * -n org.appdevforall.k2go/org.appdevforall.k2go.diskguard.debug.DebugDiskGuardReceiver \ + * --ez firehose true + *+ * + * Watch it act in logcat: {@code adb logcat -s K2Go-DiskGuard}. Both modes always CONTAIN: reap and + * reclaim, then leave the server desired=UP and ask the reconciler to relaunch a fresh box. Neither + * advances the real escalation count, so repeated triggers cannot stop the box. */ public final class DebugDiskGuardReceiver extends BroadcastReceiver { @@ -35,11 +47,17 @@ public final class DebugDiskGuardReceiver extends BroadcastReceiver { @Override public void onReceive(Context context, Intent intent) { final Context app = context.getApplicationContext(); + final boolean firehose = intent.getBooleanExtra("firehose", false); final long floor = intent.getLongExtra("floor_bytes", Long.MAX_VALUE); - Log.w(TAG, "K2GO-386: debug disk-guard test hook fired (floor_bytes=" + floor + ")"); + Log.w(TAG, "K2GO-386: debug disk-guard test hook fired (firehose=" + firehose + + ", floor_bytes=" + floor + ")"); new Thread(() -> { try { - DiskGuard.checkWithFloor(app, floor); + if (firehose) { + DiskGuard.checkFirehoseForced(app); + } else { + DiskGuard.checkWithFloor(app, floor); + } } catch (Throwable t) { Log.w(TAG, "K2GO-386: debug disk-guard test hook failed", t); } diff --git a/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java b/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java index f320a8dd1..ed44a3ce3 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java @@ -55,6 +55,11 @@ public class WatchdogService extends Service { // protected session, stopped on destroy. private ScheduledExecutorService diskGuardPoller; private static final long DISK_GUARD_INTERVAL_S = 25; + // The low-disk check runs every tick (a local StatFs read). The firehose signal is an HTTP GET to + // dash-node, and dash-node only advances it on its 10-min guard tick, so read it every Nth tick + // (~150 s) instead of every 25 s. Touched only by the single poller thread. + private static final int FIREHOSE_POLL_EVERY_N_TICKS = 6; + private int diskGuardTick = 0; @Override public void onCreate() { @@ -130,14 +135,20 @@ private void releaseHardwareLocks() { } // K2GO-386 (Layer 3): the free-space guard. One background poller ticks every DISK_GUARD_INTERVAL_S. - // On a CRITICAL reading DiskGuard confirms, reaps the box, reclaims the runaway log, and by default - // lets it restart; the in-box layers cannot stop an off-proot orphan. Started once per session. + // Two triggers. (1) check() EVERY tick (a local read): on a CRITICAL free-space reading, confirm, + // reap, reclaim, and by default let the box restart. (2) checkFirehoseSignal() every Nth tick (an + // HTTP read): on a fresh recurring firehose that is still growing, reap the off-proot orphan the box + // cannot stop -- even before the disk goes low (ADR-386 §6). The in-box layers cannot stop an + // off-proot orphan. Started once per session. private void startDiskGuard() { if (diskGuardPoller != null) return; diskGuardPoller = Executors.newSingleThreadScheduledExecutor(); diskGuardPoller.scheduleWithFixedDelay(() -> { try { org.appdevforall.k2go.diskguard.DiskGuard.check(getApplicationContext()); + if (diskGuardTick++ % FIREHOSE_POLL_EVERY_N_TICKS == 0) { + org.appdevforall.k2go.diskguard.DiskGuard.checkFirehoseSignal(getApplicationContext()); + } } catch (Throwable t) { Log.w(TAG, "K2GO-386: disk-guard tick failed", t); } diff --git a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java index 8a928c401..a276ca84c 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java @@ -40,19 +40,24 @@ import androidx.core.app.NotificationManagerCompat; import org.appdevforall.k2go.R; -import org.appdevforall.k2go.delivery.DeliveryManager; +import org.appdevforall.k2go.delivery.data.CrashReportConsent; +import org.appdevforall.k2go.diskguard.data.FirehoseSignalSource; import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation; import org.appdevforall.k2go.diskguard.domain.DiskGuardPolicy; +import org.appdevforall.k2go.diskguard.domain.FirehoseSignal; import org.appdevforall.k2go.env.EnvironmentLock; import org.appdevforall.k2go.env.EnvironmentProcess; import org.appdevforall.k2go.env.ServerLifecycleReconciler; import org.appdevforall.k2go.storage.StorageProbe; import org.appdevforall.k2go.system.domain.Operation; -import org.json.JSONObject; +import io.sentry.Sentry; +import io.sentry.SentryLevel; import java.io.File; import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.List; public final class DiskGuard { @@ -71,6 +76,17 @@ public final class DiskGuard { private static final int ESCALATE_AFTER_TRIPS = 3; private static final long TRIP_WINDOW_MS = 30L * 60L * 1000L; + // The firehose trigger (ADR-386 §6): a recurring-firehose signal older than this (in the server's + // own clock) is stale and ignored -- the firehose likely resolved. A live one is re-confirmed by + // growth anyway. About 2.5 guard ticks (the guard runs every 10 min). + private static final long FIREHOSE_FRESH_WINDOW_MS = 25L * 60L * 1000L; + // Growth re-probe: read the .log total, wait, read again. Act only on a delta that is clearly a + // firehose. The observed firehose runs ~600 MB/min to 1.3 GB/min (php-fpm busy-loop), so even the + // low end adds ~30 MB in 3 s. 16 MiB (~327 MB/min) stays below that low end with margin, and far + // above any normal log (KB-MB/min), so a real firehose is caught and a normal log never trips it. + private static final long GROWTH_PROBE_MS = 3000L; + private static final long GROWTH_MIN_BYTES = 16L * 1024 * 1024; // 16 MiB within GROWTH_PROBE_MS + private static final String CHANNEL_ID = "disk_guard_channel"; private static final int NOTIF_ID = 7386; @@ -98,6 +114,30 @@ public static boolean checkWithFloor(Context ctx, long floorBytes) { return run(ctx, floorBytes, true); } + /** + * The SECOND reap trigger (ADR-386 §6). The low-disk path above catches a disk that already went + * low. This path catches a firehose the in-box guard keeps truncating -- so the disk may never go + * low -- but that the box cannot stop because the writer is an off-proot orphan. It reads the live + * dash-node signal, and if the signal is a fresh recurring firehose it CONFIRMS by re-probing live + * log growth before it reaps. Safe to call every poller tick; a no-op unless a firehose is live now. + */ + public static boolean checkFirehoseSignal(Context ctx) { + if (ctx == null) return false; + FirehoseSignal sig = FirehoseSignalSource.read(); + if (sig == null || !sig.isFresh(FIREHOSE_FRESH_WINDOW_MS)) return false; // no live alert + return actOnFirehose(ctx, sig.maxStreak); + } + + /** + * The debug device-verify hook for the firehose path. It skips the signal fetch and freshness gate, + * but STILL runs the real growth re-probe -- so it only reaps when a log is actually growing now. + * Stage a fast-growing .log, then fire it, to verify confirm-before-act plus the reap on device. + */ + public static boolean checkFirehoseForced(Context ctx) { + if (ctx == null) return false; + return actOnFirehose(ctx, -1); + } + private static boolean run(Context ctx, long floorBytes, boolean forced) { if (ctx == null) return false; boolean critical = confirmCritical(ctx, floorBytes); @@ -177,22 +217,109 @@ private static boolean deepOpActive(Context ctx) { return EnvironmentLock.currentHolder(ctx).executionClass == Operation.ExecutionClass.STOPPED; } - /** Report the event to developers through the delivery backbone (unattended; not user-facing). */ + /** + * Act on a firehose that a fresh signal (or the debug hook) flagged. Confirm-before-acting: the + * signal is only an ALERT; reap solely if a log is actually growing fast RIGHT NOW. Then reap and + * restart (restart-to-keep-alive), the same as the low-disk path. The app-side reap DOES reach the + * off-proot orphan (unlike an in-box kill), so a fresh box does not refill. The low-disk path stays + * the sole escalation authority, so a firehose reap never counts toward stop-and-stay-down. + */ + private static boolean actOnFirehose(Context ctx, int streak) { + if (!confirmFirehoseGrowing(ctx)) return false; + if (deepOpActive(ctx)) { + Log.w(TAG, "K2GO-386: firehose confirmed but a deep op holds the box; not reaping this tick"); + return false; + } + boolean reaped = EnvironmentProcess.reapBox(ctx); + long reclaimed = reclaimRunawayLog(ctx); + ServerLifecycleReconciler.get().requestReconcileNow(); + report(ctx, "contained_firehose", 0L, reaped, reclaimed, streak); + Log.w(TAG, "K2GO-386: contained recurring firehose (streak " + streak + "): reaped=" + reaped + + ", reclaimed=" + reclaimed + " B, box restarting"); + return true; + } + + /** + * True when the box's {@code *.log} files are growing fast enough to be a firehose: read the total + * {@code .log} bytes under {@code /var/log}, wait {@link #GROWTH_PROBE_MS}, read again, and require a + * delta of at least {@link #GROWTH_MIN_BYTES}. Summing all logs (not one file) is robust to WHICH log + * the orphan writes. It is not fooled by a normal log, which never grows this fast. A rare race -- the + * in-box guard truncating the firehose during the probe -- reads as no growth this tick, not a false + * reap; the next tick catches it (the guard runs every 10 min, so the overlap is unlikely). + */ + private static boolean confirmFirehoseGrowing(Context ctx) { + File varLog = new File(ctx.getFilesDir(), "rootfs/installed-rootfs/iiab/var/log"); + long before = totalLogBytes(varLog); + try { + Thread.sleep(GROWTH_PROBE_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + long delta = totalLogBytes(varLog) - before; + boolean growing = delta >= GROWTH_MIN_BYTES; + if (!growing) { + Log.i(TAG, "K2GO-386: firehose signal but logs are not growing now (delta " + delta + " B); not acting"); + } + return growing; + } + + /** Every {@code *.log} regular file in the tree rooted at {@code dir}, added to {@code out}. The one + * recursive walker; {@link #totalLogBytes} and {@link #biggestLog} reduce over it. Best-effort + * (unreadable dirs are skipped). Bounded to the small {@code /var/log} tree. */ + private static void collectLogs(File dir, List
Blocking by design; the guard poller already runs off the main thread. The response is a few + * fields, so the byte cap is tiny. + */ +public final class FirehoseSignalSource { + + private static final String TAG = "K2Go-DiskGuard"; + private static final String URL_PATH = BoxEndpoints.API + "/system/disk-guard/firehose"; + private static final int TIMEOUT_MS = 4000; + private static final int MAX_BYTES = 8 * 1024; // a handful of fields; refuse the absurd + + private FirehoseSignalSource() {} + + /** @return the parsed signal, or {@code null} when it could not be read (box off is the usual cause). */ + public static FirehoseSignal read() { + try { + String body = httpGet(URL_PATH); + if (body.isEmpty()) return null; + JSONObject o = new JSONObject(body); + return new FirehoseSignal( + o.optBoolean("recurring", false), + o.optInt("maxStreak", 0), + o.optLong("lastTruncatedAtMs", 0L), + o.optLong("now", 0L)); + } catch (Exception e) { + Log.i(TAG, "K2GO-386: firehose signal read failed: " + e.getMessage()); + return null; + } + } + + private static String httpGet(String urlStr) throws Exception { + HttpURLConnection c = (HttpURLConnection) new URL(urlStr).openConnection(); + try { + c.setUseCaches(false); + c.setConnectTimeout(TIMEOUT_MS); + c.setReadTimeout(TIMEOUT_MS); + c.setRequestMethod("GET"); + c.setRequestProperty("Accept", "application/json"); + int code = c.getResponseCode(); + InputStream is = code >= 200 && code < 400 ? c.getInputStream() : c.getErrorStream(); + String text = readAll(is); + if (code < 200 || code >= 400) throw new Exception("HTTP " + code); + return text; + } finally { + c.disconnect(); + } + } + + private static String readAll(InputStream is) throws Exception { + if (is == null) return ""; + try (InputStream in = is) { + ByteArrayOutputStream buf = new ByteArrayOutputStream(); + byte[] chunk = new byte[2048]; + int n; + int total = 0; + while ((n = in.read(chunk)) != -1) { + total += n; + if (total > MAX_BYTES) throw new Exception("response over " + (MAX_BYTES / 1024) + " KB"); + buf.write(chunk, 0, n); + } + return buf.toString(StandardCharsets.UTF_8.name()); + } + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/FirehoseSignal.java b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/FirehoseSignal.java new file mode 100644 index 000000000..b75f8f190 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/FirehoseSignal.java @@ -0,0 +1,46 @@ +/* + * ============================================================================ + * Name : FirehoseSignal.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-386 (Layer 3). The app's parsed view of the dash-node live + * firehose signal (GET /system/disk-guard/firehose). Pure value + * object, no android.*, so the freshness rule is unit-tested on a + * plain JVM. See ADR-386 §6. + * ============================================================================ + */ +package org.appdevforall.k2go.diskguard.domain; + +/** + * A recurring firehose means the in-box guard truncated a runaway log on several consecutive ticks: + * an off-proot orphan the box cannot stop. This signal is the app's ALERT to look; it is NOT a command + * to reap. The app re-probes live log growth before it acts (ADR-386 §6, confirm before acting). + * + *
{@code nowMs} and {@code lastTruncatedAtMs} are both the dash-node wall-clock, so freshness is
+ * judged in the server's own time frame -- no app-vs-server clock skew.
+ */
+public final class FirehoseSignal {
+
+ public final boolean recurring;
+ public final int maxStreak;
+ public final long lastTruncatedAtMs; // server wall-clock of the last truncation, or 0 if never
+ public final long nowMs; // server wall-clock when it answered
+
+ public FirehoseSignal(boolean recurring, int maxStreak, long lastTruncatedAtMs, long nowMs) {
+ this.recurring = recurring;
+ this.maxStreak = maxStreak;
+ this.lastTruncatedAtMs = lastTruncatedAtMs;
+ this.nowMs = nowMs;
+ }
+
+ /**
+ * True when the signal is worth acting on: it reports a recurring firehose AND the last truncation
+ * is recent (within {@code freshWindowMs} of the server's now). A stale signal -- the guard has not
+ * truncated anything lately -- is ignored, so a firehose that already resolved never triggers a reap.
+ */
+ public boolean isFresh(long freshWindowMs) {
+ if (!recurring || lastTruncatedAtMs <= 0L) return false;
+ long age = nowMs - lastTruncatedAtMs;
+ return age >= 0L && age <= freshWindowMs;
+ }
+}
diff --git a/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/FirehoseSignalTest.java b/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/FirehoseSignalTest.java
new file mode 100644
index 000000000..152f53b88
--- /dev/null
+++ b/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/FirehoseSignalTest.java
@@ -0,0 +1,44 @@
+package org.appdevforall.k2go.diskguard.domain;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class FirehoseSignalTest {
+
+ private static final long WINDOW = 25L * 60L * 1000L;
+
+ @Test
+ public void recurringAndRecent_isFresh() {
+ // last truncation 5 min before the server's now.
+ FirehoseSignal s = new FirehoseSignal(true, 3, 1_000_000L, 1_000_000L + 5L * 60L * 1000L);
+ assertTrue(s.isFresh(WINDOW));
+ }
+
+ @Test
+ public void notRecurring_isNotFresh() {
+ FirehoseSignal s = new FirehoseSignal(false, 1, 1_000_000L, 1_000_000L + 60L * 1000L);
+ assertFalse(s.isFresh(WINDOW));
+ }
+
+ @Test
+ public void recurringButStale_isNotFresh() {
+ // last truncation 40 min before now: beyond the window, the firehose likely resolved.
+ FirehoseSignal s = new FirehoseSignal(true, 4, 1_000_000L, 1_000_000L + 40L * 60L * 1000L);
+ assertFalse(s.isFresh(WINDOW));
+ }
+
+ @Test
+ public void neverTruncated_isNotFresh() {
+ FirehoseSignal s = new FirehoseSignal(true, 2, 0L, 5_000_000L);
+ assertFalse(s.isFresh(WINDOW));
+ }
+
+ @Test
+ public void negativeAge_isNotFresh() {
+ // lastTruncatedAtMs after now (clock went backwards / bad read): reject rather than trust it.
+ FirehoseSignal s = new FirehoseSignal(true, 2, 2_000_000L, 1_000_000L);
+ assertFalse(s.isFresh(WINDOW));
+ }
+}
diff --git a/controller/docs/ADR-386-unattended-disk-containment.md b/controller/docs/ADR-386-unattended-disk-containment.md
index 51c830c59..2114262df 100644
--- a/controller/docs/ADR-386-unattended-disk-containment.md
+++ b/controller/docs/ADR-386-unattended-disk-containment.md
@@ -123,9 +123,9 @@ app can stop an off-proot orphan.
and it emits the recurring-firehose signal.
- **App-side (Android) — STOP.** The only actor that can reap an off-proot orphan. It is driven by the
in-box recurring-firehose signal (a log that keeps refilling after truncation = an orphan) or by disk
- pressure, and it reaps + reports. This is the half that is still to be built.
+ pressure, and it reaps + reports. **Built** -- see "Mechanism (as built)" below.
-**Decisions (direction; detailed mechanism is the next design step, possibly `ADR-386a`):**
+**Decisions (direction):**
- **Targeted, not blind.** Act on the offending vector (a specific runaway log/service), keeping the rest
of the system up. NOT the rejected "disk full → stop everything → stay down → user fixes it."
@@ -152,20 +152,48 @@ app can stop an off-proot orphan.
- **The signal carries a timestamp; a stale one is ignored.** Freshness is part of the contract, not an
assumption.
-**Open (design next):** the exact leading-indicator (growth-rate vs absolute), the targeted-vs-full
-decision, and how relaunch coordinates with the reconciler. The existing `feat/K2GO-386-disk-guard`
-slice (device-verified) is the scaffold to reorient from "stop + stay-down" to this.
-
-## 7. Reporting to developers (unattended, cadence-based)
-
-When any layer contains or recovers a situation, it should tell us — so field robustness is measured, not
-guessed — **without bothering the user**:
-
-- **Optional and cadence/rate-based:** a rare anomaly → a low-cadence digest (daily/weekly/monthly); a
- recurring one (e.g. hourly) → escalate ("we're noticing X; send a report?"). A frequency/rate rule sets
- the cadence.
-- **Reuse the delivery backbone** (`DeliveryManager` / the debug-delivery path) rather than a new channel.
-- The user is *informed*, not *tasked*: a report goes to developers; recovery already happened.
+**Mechanism (as built).** Two triggers, one poller. `WatchdogService` ticks the guard every 25 s and
+calls two entry points in order:
+
+1. **Low-disk trigger** -- `DiskGuard.check`. Free space (StatFs) vs a floor; on CRITICAL it confirms
+ with a second read, then reaps + reclaims + relaunches. This path OWNS escalation: after
+ `ESCALATE_AFTER_TRIPS` consecutive trips (the restart is not fixing it) it stops and stays down as a
+ last resort (`DiskGuardEscalation`, a pure unit-tested rule). Device-verified (L3b).
+2. **Firehose trigger** -- `DiskGuard.checkFirehoseSignal`. It reads dash-node's LIVE signal
+ (`GET /k2go-api/system/disk-guard/firehose`, which returns the in-memory streak, never a log line).
+ `FirehoseSignal.isFresh` gates on `recurring` AND a recent `lastTruncatedAtMs` -- judged in the
+ server's own clock (the endpoint returns `now` too), so there is no app-vs-server skew. The signal is
+ only an ALERT: before reaping, the app **re-probes live growth** -- it sums the `.log` bytes under
+ `/var/log`, waits a few seconds, sums again, and acts only on a firehose-sized delta. This catches an
+ off-proot orphan BEFORE the disk goes low (the in-box guard keeps truncating, so the disk may never go
+ low). It reaps + reclaims + relaunches (restart-to-keep-alive); the app-side reap DOES reach the
+ off-proot orphan, so a fresh box does not refill. It does NOT touch the low-disk escalation counter --
+ the low-disk path stays the single escalation authority.
+
+Both paths skip the reap while a deep op (clone/backup/restore/install) holds the box
+(`EnvironmentLock.currentHolder`), report to developers through the delivery backbone, and relaunch
+through the ADR-5343 reconciler (`requestReconcileNow`, desired stays UP).
+
+## 7. Reporting (two channels, not analytics)
+
+When a layer contains or recovers a situation it tells us -- so field robustness is measured, not guessed.
+A disk-guard event is an OPERATIONAL diagnostic, not behavioural analytics, so it does NOT use the
+analytics backbone (`AnalyticsConsent`, opt-in, default OFF -- which would silently drop it). There are
+two channels, on purpose:
+
+- **Automatic (unattended) -> GlitchTip via Sentry.** `DiskGuard.report(...)` captures a Sentry message
+ (tags: `event=disk_guard`, `action`, `reaped`; extras: `floor_bytes`, `reclaimed_bytes`, `trip`), gated
+ by `CrashReportConsent` (default ON, its own policy, no PII -- see `IIABApplication`, ADFA-4533). It is a
+ no-op when crash reporting is off or Sentry has no DSN. The user is informed, not tasked; recovery
+ already happened. DONE (this ticket).
+- **Active (user-sent) -> feedback email.** The app tells the user it contained an unusual behavior and
+ offers to send a report the user actively sends, through the existing feedback flow (`FeedbackFab` /
+ `EmailFeedbackSender`, mailto), pre-filled with what happened -- the same pattern the install-failed
+ report uses (ADFA-5119). Because the guard runs in the background, the bridge is the notification: it
+ gains a tap action that opens an Activity which launches the pre-filled feedback email. This does NOT
+ wait for GlitchTip to surface the issue; the operator can send it on the spot.
+
+The active channel is the CLOSING piece of the K2GO-386 effort and lands in its own ticket (see section 12).
## 8. Lifecycle (who sets it, who clears it, what if a process dies)
@@ -205,7 +233,8 @@ guessed — **without bothering the user**:
| L1 | rootfs built with the merged patch | php-fpm installed, **not enabled**, not running on a default build |
| L2 | `logrotate -d` after install; a log grown past `size`, then a trigger | parses clean; truncated in place; writer keeps writing; no orphaned deleted-but-open file |
| L2 | short (<10 min) session | no boot rotation (by design); bounded next long session |
-| L3 | fast fill + Vector B (synthetic) | contained/recovered without denying service; system back up; a report enqueued |
+| L3 low-disk | force CRITICAL (debug huge floor) with a staged runaway log | reaps, reclaims, box relaunches (desired=UP), report enqueued; system stays up. Device-verified (HD1901) |
+| L3 firehose | a fresh recurring signal + a `.log` growing fast now | re-probes growth, reaps the orphan, box relaunches; a non-growing signal is ignored (confirm before acting). Full real chain device-verified (HD1901): the guard truncated a >1 GiB log on two ticks -> endpoint `recurring:true` -> the app polled, confirmed growth, and reaped (`trip:2`, the real streak). The forced hook (`--ez firehose true`) covers the same path without the wait. |
## 11. Consequences
@@ -213,5 +242,29 @@ guessed — **without bothering the user**:
- We carry a small owned config set instead of inherited RPi drift; new services get one block, one place.
- One liveness dependency remains in Phase 1 (dash-node as L2 trigger); named, bounded by L3, removed in
Phase 2.
-- Field occurrences are reported to developers, so we tune the parameters from real data rather than
- guesses.
+- Field occurrences are reported to developers (GlitchTip, §7), so we tune the parameters from real data
+ rather than guesses.
+
+## 12. Closing piece -- active user report (its own ticket)
+
+The automatic report (§7) is unattended: it waits for GlitchTip to surface the issue. The closing piece of
+this effort adds the ACTIVE channel -- the operator can send a report on the spot -- and is the broche de
+cierre for K2GO-386. It ships in its own ticket (APK-affecting), and this section is its authority.
+
+Scope:
+
+- **Notify with intent.** When the guard contains a situation (a firehose reap, or the last-resort stop),
+ post a user notification that says what was contained -- calm, non-technical -- and offers "send a
+ report". Today `notifyUser` only fires on the last-resort stop and carries no action.
+- **Bridge background -> Activity.** The guard runs in `WatchdogService` (no Activity), but the feedback
+ email needs one. The notification's tap action opens an Activity (deep-link) that launches the report.
+- **Reuse the feedback flow, pre-filled.** Call `FeedbackFab.sendFeedback(activity, "disk-guard",
+ FeedbackType.BUG, prefilled)` -- the same typed, pre-filled pattern the install-failed report uses
+ (ADFA-5119). The user sends it via email; the app fills in what happened, since the user did not cause it.
+- **What the report carries.** The diagnostic already gathered for §7 (action, reaped, reclaimed bytes,
+ trip/streak, free bytes, the runaway log path) plus the standard feedback envelope (app version, build,
+ Android release, device, ABI, binaries tag), so a single email is enough to triage.
+- **Cadence.** Do not nag: offer the active report on a meaningful containment (an escalation, or a
+ recurring firehose), not on every routine reap. The automatic channel still records every event.
+
+Not in scope here: the GlitchTip DSN and server-side routing (a Worker/ops concern, not the APK).
diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md
index 961e55e60..809244ca2 100644
--- a/static/dashboard/CHANGELOG.md
+++ b/static/dashboard/CHANGELOG.md
@@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i
(the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this
file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`.
+- **1.3.1** - Live firehose signal for the app-side backstop (K2GO-386, ADR-386 §6). New read-only `GET /system/disk-guard/firehose` returns `{ recurring, maxStreak, paths, lastTruncatedAtMs, now }`. The in-box guard (1.3.0) truncates a runaway `.log` every tick, so the disk may never go low -- but a recurring firehose means an off-proot orphan the box CANNOT stop; only an app-side reap can. This endpoint exposes the guard's LIVE in-memory streak state (never a parsed log line, so a restart-resolved firehose reports clean) as the app's SECOND reap trigger. `recurring` is `maxStreak >= 2` (a single `.log` refilled past the cap on at least two consecutive ticks); `lastTruncatedAtMs` (wall-clock, 0 if never) lets the app judge freshness. It is an ALERT only: the app re-probes live log growth before it reaps (confirm before acting). Localhost-only. (K2GO-386)
- **1.3.0** - Proot log rotation, dash-node-triggered (K2GO-386, ADR-386). proot has no systemd/cron, so `/etc/cron.daily/logrotate` never runs — logrotate was installed but never triggered, and a service log (php-fpm, dash-node) could grow until the device hit ENOSPC. dash-node now runs, every 10 min (no work at boot; `timer.unref`), a firehose guard THEN `logrotate /etc/logrotate.conf`: the guard truncates any log past ~1 GiB in place first (a runaway ~GB/min that logrotate would otherwise copy — doubling disk + pegging CPU on a weak phone), so L2 never meets a firehose; a recurring firehose is flagged for the future app-side reap (ADR-386 §6). The K2Go-owned config `/etc/logrotate.d/k2go` (copytruncate + `size 100M`, proot-correct — no reopen signal — overriding the RPi-oriented nginx/php-fpm snippets and adding calibre-web + dash-node; kiwix has no log, kolibri self-rotates) is installed at deploy by `tools/setup-proot-logging.sh` (rootfs build + rebuild/dev-push), not at boot. Not a REST-surface change; the version bump is the delivery mechanism for the new dash-node behavior (no ansible role yet). (K2GO-386)
- **1.2.12** - Dashboard-update card back end (ADFA-5339, Phase 1 server half). New read-only `GET /system/dashboard/rebuild/log`: the last ~200 lines of `/var/log/dash-rebuild.log`, for the card's expandable "Details" (no file yet = empty log, not an error). `POST /system/dashboard/rebuild` now accepts `{ site: true }`: it refreshes the served landing page in the SAME run via `site-updater.sh`, from the same clone the rebuild's git fetch+reset refreshes, in finalize AFTER the core swap verifies live — so the site matches the new source. The site is a separate, versionless artifact: it never touches the reported version, and a site failure is logged, never a rollback of the (already-verified) core update. Both localhost-only. (ADFA-5339)
- **1.2.11** - `/auth/:service/session` mints the session **for the agent that asks** (ADFA-5361). Calibre-Web (Flask-Login) binds a session to a fingerprint of the User-Agent, so a session minted under dash-node's own agent was rejected on the WebView's first request: the identity was dropped, the `remember_token` deleted, and the card opened as the anonymous Guest — the "logged in as Admin" flash comes from the injected session and renders even then, which is why the auto-login looked like it worked. The route now forwards the caller's `User-Agent` through the whole login handshake (every request, not just the POST — the fingerprint is established on the first one), for Calibre-Web and Kolibri alike. The callers that consume the session themselves (downloads runner, `removeBook`) are unchanged. No User-Agent on the request degrades to the previous behaviour, logged. Same ticket: the books runner's private copy of the Calibre-Web login is gone — it never got the ADFA-5043 `remember_me` and was the drift this whole bug rode in on — so `getCalibreSession` is the one source. (ADFA-5361)
diff --git a/static/dashboard/package.json b/static/dashboard/package.json
index 427dff373..82d9088cc 100644
--- a/static/dashboard/package.json
+++ b/static/dashboard/package.json
@@ -1,6 +1,6 @@
{
"name": "dashboard-console",
- "version": "1.3.0",
+ "version": "1.3.1",
"description": "",
"main": "index.js",
"scripts": {
diff --git a/static/dashboard/routes.ts b/static/dashboard/routes.ts
index 123e75bb4..fdb1d86f0 100644
--- a/static/dashboard/routes.ts
+++ b/static/dashboard/routes.ts
@@ -21,6 +21,7 @@ import {
describeCredential, setCredential, clearCredential, isServiceName,
} from './sockets/credentials';
import { isRestartableService, restartService } from './sockets/services';
+import { getFirehoseState } from './sockets/log-rotate';
// ADFA-4879: FQR helpers reached from the app (in-app region download/delete instead of the
// copy-paste-into-a-terminal flow). tile-extract.py is installed on the box by the upstream maps
@@ -271,6 +272,27 @@ apiRouter.get('/system/version', (_req: Request, res: Response): void => {
}
});
+// K2GO-386 / ADR-386 §6: the LIVE firehose signal, the app's SECOND reap trigger (the first is the
+// app's own low-disk read). The in-box guard truncates a runaway .log every tick so the disk may never
+// go low; but a recurring firehose means an off-proot orphan the box CANNOT stop -- only an app-side
+// reap can. maxStreak is the longest run of consecutive ticks a single .log kept refilling past the cap;
+// recurring means it has done so at least twice. lastTruncatedAtMs (wall-clock, 0 if never) lets the app
+// judge freshness. The state is LIVE in-memory (resets on restart), never a parsed log line -- so a
+// restart-resolved firehose reports clean. This is an ALERT only: the app re-probes live growth before
+// it reaps (confirm before acting). Localhost-only, like all of /k2go-api.
+const FIREHOSE_RECUR_THRESHOLD = 2;
+apiRouter.get('/system/disk-guard/firehose', (_req: Request, res: Response): void => {
+ res.set('Cache-Control', 'no-store');
+ const s = getFirehoseState();
+ res.json({
+ recurring: s.maxStreak >= FIREHOSE_RECUR_THRESHOLD,
+ maxStreak: s.maxStreak,
+ paths: s.paths,
+ lastTruncatedAtMs: s.lastTruncatedAtMs,
+ now: Date.now(),
+ });
+});
+
// Current rebuild state: idle | running | done | error (read from the status file the script writes).
apiRouter.get('/system/dashboard/rebuild/status', (_req: Request, res: Response): void => {
let state = 'idle';
diff --git a/static/dashboard/sockets/log-rotate.test.ts b/static/dashboard/sockets/log-rotate.test.ts
index 3d5b3b980..22c741fc4 100644
--- a/static/dashboard/sockets/log-rotate.test.ts
+++ b/static/dashboard/sockets/log-rotate.test.ts
@@ -1,7 +1,7 @@
///