From 7a2d145cf7c6463dcb89776b4b701e32dd4ceeb6 Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Sat, 5 Sep 2026 11:42:25 -0600 Subject: [PATCH 1/3] K2GO-386 feat(app-backstop): app-side disk-fill backstop that keeps the system alive (L3b) The app-side net for a disk-fill the in-box layers cannot stop (a service orphaned off proot; device-proven that only an app-side reap works). A background poller reads free space. On a CRITICAL reading DiskGuard does this: - Confirm with a fresh re-read after a short delay. It never acts on one reading. - Reap the box (EnvironmentProcess.reapBox) and reclaim the runaway log. - By DEFAULT keep the system alive. It does not force the server down. The ADFA-5343 reconciler is left desired=UP and asked to reconcile now, so it relaunches a fresh box (a fresh service under a fresh proot does not busy-loop). - Only when the fill returns 3 times inside 30 min, stop and stay down as a last resort and tell the user. - Report every trip to developers through the delivery backbone (enqueueAnalytics). This reorients the earlier stop-and-stay-down slice to restart-to-keep-alive, so a non-technical user is never left without a system for a transient fill. See ADR-386. Reuses the device-verified reap, the pure floor rule (DiskGuardPolicy, unit-tested), the StorageProbe read, the 25s poller, and the debug device-verify receiver. --- controller/app/src/debug/AndroidManifest.xml | 12 +- .../debug/DebugDiskGuardReceiver.java | 46 ++++ .../appdevforall/k2go/WatchdogService.java | 38 +++ .../k2go/diskguard/DiskGuard.java | 234 ++++++++++++++++++ .../diskguard/domain/DiskGuardPolicy.java | 49 ++++ .../k2go/env/EnvironmentProcess.java | 71 ++++++ .../main/res/values/strings_untranslated.xml | 17 ++ .../diskguard/domain/DiskGuardPolicyTest.java | 44 ++++ 8 files changed, 509 insertions(+), 2 deletions(-) create mode 100644 controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java create mode 100644 controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java create mode 100644 controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicy.java create mode 100644 controller/app/src/main/res/values/strings_untranslated.xml create mode 100644 controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicyTest.java diff --git a/controller/app/src/debug/AndroidManifest.xml b/controller/app/src/debug/AndroidManifest.xml index d6db5520c..05c061470 100644 --- a/controller/app/src/debug/AndroidManifest.xml +++ b/controller/app/src/debug/AndroidManifest.xml @@ -1,7 +1,7 @@ @@ -12,5 +12,13 @@ + + + + + + diff --git a/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java b/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java new file mode 100644 index 000000000..54ae97ec6 --- /dev/null +++ b/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java @@ -0,0 +1,46 @@ +package org.appdevforall.k2go.diskguard.debug; + +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.util.Log; + +import org.appdevforall.k2go.diskguard.DiskGuard; + +/** + * DEBUG-ONLY. K2GO-386 device-verify hook. Forces one disk-guard check with an injected floor so the + * full protective path (set desired=DOWN → reap the box → reclaim the runaway log → notify) can be + * verified on device WITHOUT first filling ~58 GB. Lives in src/debug, so it never ships in release. + * + *

Exported (it is the whole point — an adb-reachable surface, unlike the app's non-exported + * services), mirroring {@link org.appdevforall.k2go.delivery.debug.DebugDeliveryReceiver}. A huge + * floor makes any real free-space reading CRITICAL, tripping the guard for real. Example: + * + * 

+ * adb shell am broadcast \
+ *   -a org.appdevforall.k2go.DEBUG_DISK_GUARD \
+ *   -n org.appdevforall.k2go/org.appdevforall.k2go.diskguard.debug.DebugDiskGuardReceiver \
+ *   --el floor_bytes 999999999999
+ *
+ * + * Watch it act in logcat: {@code adb logcat -s K2Go-DiskGuard}. Because it sets desired=DOWN, the box + * stays down after the reap; re-enable the server from the app to bring it back. + */ +public final class DebugDiskGuardReceiver extends BroadcastReceiver { + + private static final String TAG = "K2Go-DiskGuard"; + + @Override + public void onReceive(Context context, Intent intent) { + final Context app = context.getApplicationContext(); + final long floor = intent.getLongExtra("floor_bytes", Long.MAX_VALUE); + Log.w(TAG, "K2GO-386: debug disk-guard test hook fired (floor_bytes=" + floor + ")"); + new Thread(() -> { + try { + DiskGuard.checkWithFloor(app, floor); + } catch (Throwable t) { + Log.w(TAG, "K2GO-386: debug disk-guard test hook failed", t); + } + }, "debug-disk-guard").start(); + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java b/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java index 13498edc5..f320a8dd1 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/WatchdogService.java @@ -26,6 +26,10 @@ import androidx.core.app.NotificationCompat; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + public class WatchdogService extends Service { private static final String TAG = "IIAB-Watchdog"; private static final String CHANNEL_ID = "watchdog_channel"; @@ -46,6 +50,12 @@ public class WatchdogService extends Service { private PowerManager.WakeLock wakeLock; private WifiManager.WifiLock wifiLock; + // K2GO-386 (Layer 3): a single background poller checks free space while the box is up. On a critical + // reading DiskGuard reaps and reclaims, then by default keeps the system alive. Started once per + // protected session, stopped on destroy. + private ScheduledExecutorService diskGuardPoller; + private static final long DISK_GUARD_INTERVAL_S = 25; + @Override public void onCreate() { super.onCreate(); @@ -80,6 +90,9 @@ private void startWatchdog() { // 2. Acquire CPU WakeLock to prevent sleep during heavy operations (e.g., Tar extraction, Rsync) acquireHardwareLocks(); + // K2GO-386 (barrier 2): guard free space for the life of this protected session. + startDiskGuard(); + // 3. Notify the UI (MainActivity) that the engine is protected and running IIABWatchdog.logSessionStart(this); Intent startIntent = new Intent(ACTION_STATE_STARTED); @@ -116,6 +129,28 @@ private void releaseHardwareLocks() { } } + // K2GO-386 (Layer 3): the free-space guard. One background poller ticks every DISK_GUARD_INTERVAL_S. + // On a CRITICAL reading DiskGuard confirms, reaps the box, reclaims the runaway log, and by default + // lets it restart; the in-box layers cannot stop an off-proot orphan. Started once per session. + private void startDiskGuard() { + if (diskGuardPoller != null) return; + diskGuardPoller = Executors.newSingleThreadScheduledExecutor(); + diskGuardPoller.scheduleWithFixedDelay(() -> { + try { + org.appdevforall.k2go.diskguard.DiskGuard.check(getApplicationContext()); + } catch (Throwable t) { + Log.w(TAG, "K2GO-386: disk-guard tick failed", t); + } + }, DISK_GUARD_INTERVAL_S, DISK_GUARD_INTERVAL_S, TimeUnit.SECONDS); + } + + private void stopDiskGuard() { + if (diskGuardPoller != null) { + diskGuardPoller.shutdownNow(); + diskGuardPoller = null; + } + } + @Override public void onDestroy() { RUNNING = false; // ADFA-5343 (Phase 4b): protection is ending — clear the promoter's state signal @@ -124,6 +159,9 @@ public void onDestroy() { stopIntent.setPackage(getPackageName()); sendBroadcast(stopIntent); + // K2GO-386 (barrier 2): stop the free-space guard — this protected session (box up) is ending. + stopDiskGuard(); + // 2. Release Hardware Locks so the phone can sleep again releaseHardwareLocks(); diff --git a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java new file mode 100644 index 000000000..c9b137cf5 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java @@ -0,0 +1,234 @@ +/* + * ============================================================================ + * Name : DiskGuard.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-386 (Layer 3, app-side backstop). The outside-the-rootfs + * net for a disk-fill the in-box layers cannot stop. + * + * One tick reads free space (StorageProbe) and asks the pure rule + * (DiskGuardPolicy). On CRITICAL it CONFIRMS with a fresh re-read + * (it never acts on one reading), then reaps the box and reclaims + * the runaway log. The default action KEEPS THE SYSTEM ALIVE: it + * does not force the server down. A fresh service under a fresh + * proot does not busy-loop, so the ADFA-5343 reconciler relaunches + * a clean box. Only when the fill KEEPS RETURNING after several + * restarts in a window does the guard stop and stay down as a last + * resort, and tell the user. Every trip is reported to developers + * through the delivery backbone. + * + * Why Android-side: the fill happens when the box proot dies and a + * service is orphaned off proot. An in-box kill does not reach the + * orphan (device-proven 2026-09-04, HD1901); only an app-side reap + * works. See controller/docs/ADR-386. + * ============================================================================ + */ +package org.appdevforall.k2go.diskguard; + +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.content.Context; +import android.os.Build; +import android.os.SystemClock; +import android.util.Log; + +import androidx.core.app.NotificationCompat; +import androidx.core.app.NotificationManagerCompat; + +import org.appdevforall.k2go.R; +import org.appdevforall.k2go.delivery.DeliveryManager; +import org.appdevforall.k2go.diskguard.domain.DiskGuardPolicy; +import org.appdevforall.k2go.env.EnvironmentProcess; +import org.appdevforall.k2go.env.ServerLifecycleReconciler; +import org.appdevforall.k2go.storage.StorageProbe; + +import org.json.JSONObject; + +import java.io.File; +import java.io.FileOutputStream; + +public final class DiskGuard { + + private static final String TAG = "K2Go-DiskGuard"; + + // Only truncate a log that is clearly a runaway, not a normal log. A runaway at a critical-low-space + // moment is many GB, so 1 GiB stays well clear of any legitimate log. + private static final long RUNAWAY_LOG_MIN_BYTES = 1024L * 1024 * 1024; + + // Confirm-before-act: after a CRITICAL reading, wait this long and read again. A real fill persists; + // a momentary spike does not (ADR-386, "confirm before acting"). + private static final long CONFIRM_DELAY_MS = 1000L; + + // Restart-to-keep-alive is the default. If the fill returns this many times inside TRIP_WINDOW_MS, + // the restart is not fixing it, so the guard escalates to stop-and-stay-down as a last resort. + private static final int ESCALATE_AFTER_TRIPS = 3; + private static final long TRIP_WINDOW_MS = 30L * 60L * 1000L; + + private static final String CHANNEL_ID = "disk_guard_channel"; + private static final int NOTIF_ID = 7386; + + // Recent-trip state, in memory on purpose: it resets when the app process restarts, so a stale count + // never carries across a restart. Guarded by the class monitor. + private static long lastTripElapsedMs = -1L; + private static int tripCount = 0; + + private DiskGuard() {} + + /** + * One guard tick with the default critical floor. Returns true when it acted. Safe to call + * repeatedly from a poller. A null or UNKNOWN read is a no-op. + */ + public static boolean check(Context ctx) { + return checkWithFloor(ctx, DiskGuardPolicy.CRITICAL_FLOOR_BYTES); + } + + /** + * As {@link #check} but with an explicit floor. The debug device-verify hook passes a huge floor so + * any real free-space read is CRITICAL. This forces the action without filling the disk for real. + */ + public static boolean checkWithFloor(Context ctx, long floorBytes) { + if (ctx == null) return false; + if (!confirmCritical(ctx, floorBytes)) return false; + + int trip = recordTrip(); + boolean reaped = EnvironmentProcess.reapBox(ctx); + long reclaimed = reclaimRunawayLog(ctx); + + if (trip < ESCALATE_AFTER_TRIPS) { + // Default: keep the system alive. Do NOT force desired=DOWN. The reconciler (ADFA-5343) owns + // "should the box be up"; it is left UP, so it relaunches a fresh box. Ask it to act now + // instead of on its next tick. + ServerLifecycleReconciler.get().requestReconcileNow(); + report(ctx, "contained", floorBytes, reaped, reclaimed, trip); + Log.w(TAG, "K2GO-386: contained disk pressure (trip " + trip + "): reaped=" + reaped + + ", reclaimed=" + reclaimed + " B, box restarting"); + } else { + // Last resort: the fill keeps returning after restarts. Stop and stay down through the one + // persisted lever, and tell the user. The user re-enables the server after freeing space. + ServerLifecycleReconciler.get().setUserWantsOn(ctx, false); + notifyUser(ctx); + report(ctx, "escalated_stopped", floorBytes, reaped, reclaimed, trip); + Log.w(TAG, "K2GO-386: recurring disk pressure (trip " + trip + "): stopped and staying down"); + } + return true; + } + + /** + * True only if free space is CRITICAL on two reads separated by {@link #CONFIRM_DELAY_MS}. Both reads + * are live (StatFs), so this debounces a momentary spike; it never acts on a single reading. + */ + private static boolean confirmCritical(Context ctx, long floorBytes) { + Long free = StorageProbe.freeBytes(ctx); + if (DiskGuardPolicy.evaluate(free, floorBytes) != DiskGuardPolicy.Level.CRITICAL) return false; + try { + Thread.sleep(CONFIRM_DELAY_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + Long free2 = StorageProbe.freeBytes(ctx); + boolean stillCritical = DiskGuardPolicy.evaluate(free2, floorBytes) == DiskGuardPolicy.Level.CRITICAL; + if (!stillCritical) { + Log.i(TAG, "K2GO-386: free space recovered on re-read (" + free2 + " B); not acting"); + } + return stillCritical; + } + + /** Count trips inside a sliding window. Returns the trip number in the current window (1-based). */ + private static synchronized int recordTrip() { + long now = SystemClock.elapsedRealtime(); + if (lastTripElapsedMs < 0L || now - lastTripElapsedMs > TRIP_WINDOW_MS) { + tripCount = 0; + } + lastTripElapsedMs = now; + return ++tripCount; + } + + /** Report the event to developers through the delivery backbone (unattended; not user-facing). */ + private static void report(Context ctx, String action, long floorBytes, boolean reaped, + long reclaimed, int trip) { + try { + String json = new JSONObject() + .put("event", "disk_guard") + .put("action", action) + .put("floor_bytes", floorBytes) + .put("reaped", reaped) + .put("reclaimed_bytes", reclaimed) + .put("trip", trip) + .put("ts", System.currentTimeMillis()) + .toString(); + DeliveryManager.with(ctx).enqueueAnalytics(json); + } catch (Exception e) { + Log.w(TAG, "K2GO-386: could not enqueue disk-guard report", e); + } + } + + /** + * Truncate the biggest {@code *.log} file anywhere under the box's {@code /var/log} to reclaim the + * space the runaway consumed (a real file that persists after its writer dies). Recurses + * subdirectories (for example {@code /var/log/nginx/}) and only considers {@code .log} files over + * {@link #RUNAWAY_LOG_MIN_BYTES}, so a normal or non-log file is never touched. Best-effort. Returns + * the bytes reclaimed, or 0. + */ + private static long reclaimRunawayLog(Context ctx) { + File varLog = new File(ctx.getFilesDir(), "rootfs/installed-rootfs/iiab/var/log"); + File biggest = biggestLogUnder(varLog, null); + if (biggest == null || biggest.length() < RUNAWAY_LOG_MIN_BYTES) return 0L; + long size = biggest.length(); + try (FileOutputStream truncate = new FileOutputStream(biggest)) { + // opening for write with no append truncates to zero + Log.w(TAG, "K2GO-386: truncated runaway log " + biggest.getName() + " (" + size + " B)"); + return size; + } catch (Exception e) { + Log.w(TAG, "K2GO-386: could not truncate " + biggest.getName(), e); + return 0L; + } + } + + /** + * The biggest {@code *.log} regular file in the tree rooted at {@code dir}, or {@code best} if none is + * bigger. Name-filtered to {@code .log} so a non-log large file is never a candidate. Bounded to the + * small {@code /var/log} tree. Best-effort (unreadable dirs are skipped). + */ + private static File biggestLogUnder(File dir, File best) { + File[] entries = dir.listFiles(); + if (entries == null) return best; + for (File f : entries) { + if (f.isDirectory()) { + best = biggestLogUnder(f, best); + } else if (f.isFile() && f.getName().endsWith(".log") + && (best == null || f.length() > best.length())) { + best = f; + } + } + return best; + } + + /** + * Warn the user that the box was stopped to protect the device. Best-effort: a no-op if the + * POST_NOTIFICATIONS permission is not granted (API 33+). The teardown still happened. + */ + private static void notifyUser(Context ctx) { + try { + NotificationManager nm = (NotificationManager) ctx.getSystemService(Context.NOTIFICATION_SERVICE); + if (nm != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + nm.createNotificationChannel(new NotificationChannel( + CHANNEL_ID, ctx.getString(R.string.disk_guard_notif_title), + NotificationManager.IMPORTANCE_HIGH)); + } + Notification n = new NotificationCompat.Builder(ctx, CHANNEL_ID) + .setContentTitle(ctx.getString(R.string.disk_guard_notif_title)) + .setContentText(ctx.getString(R.string.disk_guard_notif_body)) + .setStyle(new NotificationCompat.BigTextStyle() + .bigText(ctx.getString(R.string.disk_guard_notif_body))) + .setSmallIcon(android.R.drawable.stat_sys_warning) + .setPriority(NotificationCompat.PRIORITY_HIGH) + .setAutoCancel(true) + .build(); + NotificationManagerCompat.from(ctx).notify(NOTIF_ID, n); + } catch (Exception e) { + Log.w(TAG, "K2GO-386: could not post the disk-guard notification", e); + } + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicy.java b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicy.java new file mode 100644 index 000000000..070f78890 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicy.java @@ -0,0 +1,49 @@ +/* + * ============================================================================ + * Name : DiskGuardPolicy.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-386. The pure runtime rule for "is free space critically + * low while the server is up?" — the general disk-fill safety net. + * + * Barrier 2 of K2GO-386: a runaway box process (proven: php-fpm + * orphaned off proot, busy-looping into /var/log at ~600 MB/min) + * fills the device to ENOSPC, and the in-box healer cannot help — + * it dies with the box. Only an Android-side guard, independent of + * the box, catches it. And it catches ANY runaway, not just php: + * every disk-fill travels through one common surface, free space. + * + * The critical floor is set BELOW StorageGuard's 2 GiB op-floor: + * every app-driven op reserves >= 2 GiB headroom (StorageGuard), + * so free space only crosses below ~2 GiB when something fills the + * disk WITHOUT that headroom check — i.e. a runaway. 1.5 GiB still + * leaves runway to act before ENOSPC (~2.5 min at ~600 MB/min). + * + * Pure JVM (no android.*) so it is unit-testable; the StatFs read + * is the thin caller (StorageProbe), kept out on purpose. + * ============================================================================ + */ +package org.appdevforall.k2go.diskguard.domain; + +public final class DiskGuardPolicy { + + /** Act when free space drops below this while the server is up. Below StorageGuard's 2 GiB + * op-floor on purpose: legit ops keep >= 2 GiB free, so only a runaway crosses this line. */ + public static final long CRITICAL_FLOOR_BYTES = 1536L * 1024 * 1024; // 1.5 GiB + + public enum Level { OK, CRITICAL, UNKNOWN } + + private DiskGuardPolicy() {} + + /** Evaluate free space against the default critical floor. */ + public static Level evaluate(Long freeBytes) { + return evaluate(freeBytes, CRITICAL_FLOOR_BYTES); + } + + /** As above with an explicit floor. A null/negative read is UNKNOWN — the guard must NOT tear + * down the box on a failed read (fail-safe: the read is the uncertain half). */ + public static Level evaluate(Long freeBytes, long floorBytes) { + if (freeBytes == null || freeBytes < 0L) return Level.UNKNOWN; + return freeBytes < floorBytes ? Level.CRITICAL : Level.OK; + } +} diff --git a/controller/app/src/main/java/org/appdevforall/k2go/env/EnvironmentProcess.java b/controller/app/src/main/java/org/appdevforall/k2go/env/EnvironmentProcess.java index e00336e76..cea87c8a3 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/env/EnvironmentProcess.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/env/EnvironmentProcess.java @@ -184,6 +184,77 @@ public static boolean reapEnvironmentHttpFront() { return reaped; } + /** + * K2GO-386 (barrier 2): a FULL box teardown — kill the proot and reap the box's daemonised services. + * This is the disk-guard's recovery action when free space is critical. Device-proven (2026-09-04): a + * targeted restart or an in-proot kill does NOT stop a runaway orphaned off proot; only a full teardown + * (what {@code am force-stop} does) does. Each service runs in the app's own uid + SELinux domain, so + * {@code killProcess} reaches it — the same technique as {@link #reapEnvironmentHttpFront}. Best-effort + * and idempotent. + * + * @return true when at least one process was signalled. + */ + public static boolean reapBox(Context ctx) { + boolean any = false; + if (ctx != null) { + int proot = findPid(ctx); + if (proot > 0) { + try { + android.os.Process.killProcess(proot); + Log.i(TAG, "K2GO-386: killed the box proot, pid " + proot); + any = true; + } catch (Exception e) { + Log.w(TAG, "K2GO-386: could not kill the box proot pid " + proot, e); + } + } + } + // The box's services daemonise (setsid, reparent to init) and survive the proot; reap them by name. + return reapByNames(BOX_SERVICE_TOKENS) || any; + } + + /** Cmdline tokens of the box's daemonised services (dash-node = "node"). Scoped to us: these run only + * in the app's uid, so a match is always a box process. */ + private static final String[] BOX_SERVICE_TOKENS = + {"php-fpm", "nginx", "mariadb", "mysqld", "kolibri", "kiwix", "calibre", "node"}; + + /** Kill every non-self process whose cmdline contains any of {@code tokens} (same uid + SELinux domain + * as us, so killable). Best-effort; entries vanishing mid-scan are ignored. */ + private static boolean reapByNames(String[] tokens) { + File[] entries = new File("/proc").listFiles(); + if (entries == null) { + return false; + } + int myPid = android.os.Process.myPid(); + boolean reaped = false; + for (File dir : entries) { + int pid; + try { + pid = Integer.parseInt(dir.getName()); + } catch (NumberFormatException notAPid) { + continue; + } + if (pid == myPid) { + continue; + } + String cmd = readCmdline(new File(dir, "cmdline")); + if (cmd == null) { + continue; + } + for (String token : tokens) { + if (cmd.contains(token)) { + try { + android.os.Process.killProcess(pid); + reaped = true; + } catch (Exception ignored) { + // vanished mid-scan, or not ours to signal + } + break; + } + } + } + return reaped; + } + /** {@code /proc//cmdline} as a space-joined string, or null if it cannot be read. */ private static String readCmdline(File cmdline) { try (FileInputStream in = new FileInputStream(cmdline)) { diff --git a/controller/app/src/main/res/values/strings_untranslated.xml b/controller/app/src/main/res/values/strings_untranslated.xml new file mode 100644 index 000000000..c5d784c1f --- /dev/null +++ b/controller/app/src/main/res/values/strings_untranslated.xml @@ -0,0 +1,17 @@ + + + + + + Storage critically low + The server was stopped to protect your device from running out of space. + + diff --git a/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicyTest.java b/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicyTest.java new file mode 100644 index 000000000..d097b275c --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardPolicyTest.java @@ -0,0 +1,44 @@ +/* + * ============================================================================ + * Name : DiskGuardPolicyTest.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-386. JVM unit tests for the pure disk-guard rule. + * ============================================================================ + */ +package org.appdevforall.k2go.diskguard.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class DiskGuardPolicyTest { + + @Test + public void nullOrNegativeFreeSpaceIsUnknown_soTheGuardDoesNotActOnABadRead() { + assertEquals(DiskGuardPolicy.Level.UNKNOWN, DiskGuardPolicy.evaluate(null)); + assertEquals(DiskGuardPolicy.Level.UNKNOWN, DiskGuardPolicy.evaluate(-1L)); + } + + @Test + public void belowTheFloorIsCritical() { + assertEquals(DiskGuardPolicy.Level.CRITICAL, + DiskGuardPolicy.evaluate(DiskGuardPolicy.CRITICAL_FLOOR_BYTES - 1)); + assertEquals(DiskGuardPolicy.Level.CRITICAL, DiskGuardPolicy.evaluate(0L)); + } + + @Test + public void atOrAboveTheFloorIsOk() { + assertEquals(DiskGuardPolicy.Level.OK, + DiskGuardPolicy.evaluate(DiskGuardPolicy.CRITICAL_FLOOR_BYTES)); + assertEquals(DiskGuardPolicy.Level.OK, DiskGuardPolicy.evaluate(10L * 1024 * 1024 * 1024)); + } + + @Test + public void criticalFloorIsBelowStorageGuardOpFloor_soLegitOpsNeverTripIt() { + // Legit ops reserve >= 2 GiB (StorageGuard.DEFAULT_FLOOR_BYTES). Keeping the runtime critical + // floor strictly below that means only a headroom-less runaway can cross it. + assertTrue(DiskGuardPolicy.CRITICAL_FLOOR_BYTES < 2L * 1024 * 1024 * 1024); + } +} From 82b30392e8cb91ce5de4f4950cb6c5ae0ddb275c Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Sat, 5 Sep 2026 12:00:40 -0600 Subject: [PATCH 2/3] K2GO-386 refactor(app-backstop): extract pure escalation rule; guard reap; fix debug/report Second code-review pass on L3b. Five fixes. 1. Deep-op guard. DiskGuard.deepOpActive(ctx) reads EnvironmentLock.currentHolder. If a STOPPED-class op (clone, backup, restore, install) holds the box, the tick logs and returns without reaping. A reap mid-extract would corrupt the rootfs. 2. Debug path no longer escalates. checkWithFloor runs in FORCED mode. FORCED always CONTAINs. It never advances the trip state. Before, three debug triggers could stop the box. 3. Consecutive-trip semantics. A non-critical tick ends the spell and resets the count. Escalation now means the disk stayed critical across consecutive restarts, so the restart is not fixing it. Before, the count reset only on a 30-min gap or app restart. 4. Testable escalation. The decision moves to the pure DiskGuardEscalation.next(...). No android.* imports. DiskGuardEscalationTest covers five cases on a plain JVM. 5. Report cadence. A CONTAIN reports only on the first trip of a spell. An ESCALATE always reports. A thrash no longer enqueues one analytics event per trip. See controller/docs/ADR-386-unattended-disk-containment.md. --- .../k2go/diskguard/DiskGuard.java | 102 ++++++++++++------ .../diskguard/domain/DiskGuardEscalation.java | 64 +++++++++++ .../domain/DiskGuardEscalationTest.java | 55 ++++++++++ 3 files changed, 186 insertions(+), 35 deletions(-) create mode 100644 controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalation.java create mode 100644 controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalationTest.java diff --git a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java index c9b137cf5..8a928c401 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java @@ -8,14 +8,17 @@ * * One tick reads free space (StorageProbe) and asks the pure rule * (DiskGuardPolicy). On CRITICAL it CONFIRMS with a fresh re-read - * (it never acts on one reading), then reaps the box and reclaims - * the runaway log. The default action KEEPS THE SYSTEM ALIVE: it - * does not force the server down. A fresh service under a fresh - * proot does not busy-loop, so the ADFA-5343 reconciler relaunches - * a clean box. Only when the fill KEEPS RETURNING after several - * restarts in a window does the guard stop and stay down as a last - * resort, and tell the user. Every trip is reported to developers - * through the delivery backbone. + * (it never acts on one reading). It does NOT reap while a deep op + * (clone/backup/restore/install) holds the box, because a reap + * mid-operation would corrupt it. Otherwise it reaps the box and + * reclaims the runaway log. + * + * The default action KEEPS THE SYSTEM ALIVE: it does not force the + * server down. A fresh service under a fresh proot does not + * busy-loop, so the ADFA-5343 reconciler relaunches a clean box. + * Only when the disk stays critical for several trips in a row + * (DiskGuardEscalation) does the guard stop and stay down as a last + * resort and tell the user. Trips are reported to developers. * * Why Android-side: the fill happens when the box proot dies and a * service is orphaned off proot. An in-box kill does not reach the @@ -38,10 +41,13 @@ import org.appdevforall.k2go.R; import org.appdevforall.k2go.delivery.DeliveryManager; +import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation; import org.appdevforall.k2go.diskguard.domain.DiskGuardPolicy; +import org.appdevforall.k2go.env.EnvironmentLock; import org.appdevforall.k2go.env.EnvironmentProcess; import org.appdevforall.k2go.env.ServerLifecycleReconciler; import org.appdevforall.k2go.storage.StorageProbe; +import org.appdevforall.k2go.system.domain.Operation; import org.json.JSONObject; @@ -60,8 +66,8 @@ public final class DiskGuard { // a momentary spike does not (ADR-386, "confirm before acting"). private static final long CONFIRM_DELAY_MS = 1000L; - // Restart-to-keep-alive is the default. If the fill returns this many times inside TRIP_WINDOW_MS, - // the restart is not fixing it, so the guard escalates to stop-and-stay-down as a last resort. + // Restart-to-keep-alive is the default. If the disk stays critical this many trips in a row, the + // restart is not fixing it, so the guard escalates to stop-and-stay-down as a last resort. private static final int ESCALATE_AFTER_TRIPS = 3; private static final long TRIP_WINDOW_MS = 30L * 60L * 1000L; @@ -69,7 +75,7 @@ public final class DiskGuard { private static final int NOTIF_ID = 7386; // Recent-trip state, in memory on purpose: it resets when the app process restarts, so a stale count - // never carries across a restart. Guarded by the class monitor. + // never carries across a restart. Read and written only under advanceTripState (class monitor). private static long lastTripElapsedMs = -1L; private static int tripCount = 0; @@ -80,36 +86,57 @@ private DiskGuard() {} * repeatedly from a poller. A null or UNKNOWN read is a no-op. */ public static boolean check(Context ctx) { - return checkWithFloor(ctx, DiskGuardPolicy.CRITICAL_FLOOR_BYTES); + return run(ctx, DiskGuardPolicy.CRITICAL_FLOOR_BYTES, false); } /** - * As {@link #check} but with an explicit floor. The debug device-verify hook passes a huge floor so - * any real free-space read is CRITICAL. This forces the action without filling the disk for real. + * The debug device-verify hook. It passes a huge floor so any real free-space read is CRITICAL, and + * runs in FORCED mode: it exercises the reap/reclaim/restart path once but does NOT advance the real + * escalation count, so triggering it repeatedly cannot stop the box. */ public static boolean checkWithFloor(Context ctx, long floorBytes) { + return run(ctx, floorBytes, true); + } + + private static boolean run(Context ctx, long floorBytes, boolean forced) { if (ctx == null) return false; - if (!confirmCritical(ctx, floorBytes)) return false; + boolean critical = confirmCritical(ctx, floorBytes); + + DiskGuardEscalation.Verdict v; + if (forced) { + // Forced (debug): CONTAIN if critical, and never touch the shared trip state or escalate. + v = new DiskGuardEscalation.Verdict( + critical ? DiskGuardEscalation.Action.CONTAIN : DiskGuardEscalation.Action.NONE, + 0, 0L, true); + } else { + v = advanceTripState(critical); + } + if (v.action == DiskGuardEscalation.Action.NONE) return false; + + // Never reap while a deep op owns the box (clone/backup/restore/install). A reap mid-operation + // would corrupt it. EnvironmentLock is the one owner of "is a stop-class op running". + if (deepOpActive(ctx)) { + Log.w(TAG, "K2GO-386: disk critical but a deep op holds the box; not reaping this tick"); + return false; + } - int trip = recordTrip(); boolean reaped = EnvironmentProcess.reapBox(ctx); long reclaimed = reclaimRunawayLog(ctx); - if (trip < ESCALATE_AFTER_TRIPS) { - // Default: keep the system alive. Do NOT force desired=DOWN. The reconciler (ADFA-5343) owns - // "should the box be up"; it is left UP, so it relaunches a fresh box. Ask it to act now - // instead of on its next tick. - ServerLifecycleReconciler.get().requestReconcileNow(); - report(ctx, "contained", floorBytes, reaped, reclaimed, trip); - Log.w(TAG, "K2GO-386: contained disk pressure (trip " + trip + "): reaped=" + reaped - + ", reclaimed=" + reclaimed + " B, box restarting"); - } else { + if (v.action == DiskGuardEscalation.Action.ESCALATE) { // Last resort: the fill keeps returning after restarts. Stop and stay down through the one // persisted lever, and tell the user. The user re-enables the server after freeing space. ServerLifecycleReconciler.get().setUserWantsOn(ctx, false); notifyUser(ctx); - report(ctx, "escalated_stopped", floorBytes, reaped, reclaimed, trip); - Log.w(TAG, "K2GO-386: recurring disk pressure (trip " + trip + "): stopped and staying down"); + report(ctx, "escalated_stopped", floorBytes, reaped, reclaimed, v.tripCount); + Log.w(TAG, "K2GO-386: recurring disk pressure (trip " + v.tripCount + "): stopped and staying down"); + } else { + // Default: keep the system alive. Leave desired=UP and ask the reconciler to relaunch a fresh + // box now. Report only the first trip of a spell so a thrash does not spam telemetry. + ServerLifecycleReconciler.get().requestReconcileNow(); + if (v.firstOfSpell) report(ctx, "contained", floorBytes, reaped, reclaimed, v.tripCount); + Log.w(TAG, "K2GO-386: contained disk pressure (trip " + v.tripCount + "): reaped=" + reaped + + ", reclaimed=" + reclaimed + " B, box restarting"); } return true; } @@ -135,14 +162,19 @@ private static boolean confirmCritical(Context ctx, long floorBytes) { return stillCritical; } - /** Count trips inside a sliding window. Returns the trip number in the current window (1-based). */ - private static synchronized int recordTrip() { - long now = SystemClock.elapsedRealtime(); - if (lastTripElapsedMs < 0L || now - lastTripElapsedMs > TRIP_WINDOW_MS) { - tripCount = 0; - } - lastTripElapsedMs = now; - return ++tripCount; + /** Advance the shared trip state with the pure rule and return the verdict. */ + private static synchronized DiskGuardEscalation.Verdict advanceTripState(boolean critical) { + DiskGuardEscalation.Verdict v = DiskGuardEscalation.next( + critical, SystemClock.elapsedRealtime(), lastTripElapsedMs, tripCount, + TRIP_WINDOW_MS, ESCALATE_AFTER_TRIPS); + tripCount = v.tripCount; + lastTripElapsedMs = v.lastElapsedMs; + return v; + } + + /** True when a stop-class operation (clone/backup/restore/install) currently holds the box. */ + private static boolean deepOpActive(Context ctx) { + return EnvironmentLock.currentHolder(ctx).executionClass == Operation.ExecutionClass.STOPPED; } /** Report the event to developers through the delivery backbone (unattended; not user-facing). */ diff --git a/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalation.java b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalation.java new file mode 100644 index 000000000..5c672c1ae --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalation.java @@ -0,0 +1,64 @@ +/* + * ============================================================================ + * Name : DiskGuardEscalation.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-386 (Layer 3). The pure escalation rule for the app-side + * disk-fill backstop. It decides, on one guard tick, what to do + * from whether the disk is critical now plus the recent-trip state. + * No android.*, so it is unit-tested on a plain JVM. See ADR-386. + * ============================================================================ + */ +package org.appdevforall.k2go.diskguard.domain; + +/** + * A trip is a confirmed-critical tick. Trips count only while they stay CONSECUTIVE: a non-critical tick + * ends the spell and resets the count, and a long gap (over the window) resets it too. The default action + * is CONTAIN (reap, then let the box restart). If the disk stays critical for {@code escalateAfter} trips + * in a row, the restart is not fixing it, so the action becomes ESCALATE (stop and stay down). + */ +public final class DiskGuardEscalation { + + public enum Action { NONE, CONTAIN, ESCALATE } + + /** The new trip state plus the action to take. Immutable. */ + public static final class Verdict { + public final Action action; + public final int tripCount; // new consecutive-trip count; 0 when not critical + public final long lastElapsedMs; // new last-trip time + public final boolean firstOfSpell; // tripCount == 1; used to report once per spell + + public Verdict(Action action, int tripCount, long lastElapsedMs, boolean firstOfSpell) { + this.action = action; + this.tripCount = tripCount; + this.lastElapsedMs = lastElapsedMs; + this.firstOfSpell = firstOfSpell; + } + } + + private DiskGuardEscalation() {} + + /** + * @param critical whether the disk is critical on this tick (already confirmed). + * @param nowMs a monotonic clock reading (SystemClock.elapsedRealtime). + * @param lastElapsedMs the previous trip's clock reading, or negative if none. + * @param prevCount the previous consecutive-trip count. + * @param windowMs a gap longer than this starts a new spell. + * @param escalateAfter the trip number at which to escalate. + */ + public static Verdict next(boolean critical, long nowMs, long lastElapsedMs, int prevCount, + long windowMs, int escalateAfter) { + if (!critical) { + // The disk recovered. End the spell so a later, separate fill starts fresh. + return new Verdict(Action.NONE, 0, lastElapsedMs, false); + } + int count; + if (prevCount <= 0 || lastElapsedMs < 0L || nowMs - lastElapsedMs > windowMs) { + count = 1; // first trip of a new spell + } else { + count = prevCount + 1; + } + Action action = count >= escalateAfter ? Action.ESCALATE : Action.CONTAIN; + return new Verdict(action, count, nowMs, count == 1); + } +} diff --git a/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalationTest.java b/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalationTest.java new file mode 100644 index 000000000..f53093916 --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/diskguard/domain/DiskGuardEscalationTest.java @@ -0,0 +1,55 @@ +package org.appdevforall.k2go.diskguard.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation.Action; +import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation.Verdict; + +import org.junit.Test; + +public class DiskGuardEscalationTest { + + private static final long WINDOW = 30L * 60L * 1000L; + private static final int ESCALATE = 3; + + @Test + public void notCritical_isNoneAndResetsCount() { + Verdict v = DiskGuardEscalation.next(false, 1000L, 500L, 2, WINDOW, ESCALATE); + assertEquals(Action.NONE, v.action); + assertEquals(0, v.tripCount); + } + + @Test + public void firstCritical_isContainTripOne() { + Verdict v = DiskGuardEscalation.next(true, 1000L, -1L, 0, WINDOW, ESCALATE); + assertEquals(Action.CONTAIN, v.action); + assertEquals(1, v.tripCount); + assertTrue(v.firstOfSpell); + } + + @Test + public void consecutiveCritical_incrementsWithinWindow() { + Verdict v = DiskGuardEscalation.next(true, 2000L, 1000L, 1, WINDOW, ESCALATE); + assertEquals(Action.CONTAIN, v.action); + assertEquals(2, v.tripCount); + assertFalse(v.firstOfSpell); + } + + @Test + public void escalatesAtThreshold() { + Verdict v = DiskGuardEscalation.next(true, 3000L, 2000L, 2, WINDOW, ESCALATE); + assertEquals(Action.ESCALATE, v.action); + assertEquals(3, v.tripCount); + } + + @Test + public void gapBeyondWindow_startsNewSpell() { + // prevCount is 2, but the gap exceeds the window, so the spell resets to trip 1. + Verdict v = DiskGuardEscalation.next(true, 100_000_000L, 1000L, 2, WINDOW, ESCALATE); + assertEquals(Action.CONTAIN, v.action); + assertEquals(1, v.tripCount); + assertTrue(v.firstOfSpell); + } +} From 10e72d0f22c825764a601074dc889fa0953c70ec Mon Sep 17 00:00:00 2001 From: Luis Guzman Date: Sat, 5 Sep 2026 12:08:10 -0600 Subject: [PATCH 3/3] K2GO-386 docs(app-backstop): fix stale debug-receiver Javadoc after the reorient The debug hook doc still described the old stop-and-stay-down path (set desired=DOWN, box stays down). After the L3b reorient the debug hook runs the FORCED path: it always CONTAINs, reaps and reclaims, then leaves desired=UP and lets the reconciler relaunch the box. It never advances the escalation count. Correct the class Javadoc to match. Device-verified on HD1901. --- .../k2go/diskguard/debug/DebugDiskGuardReceiver.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java b/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java index 54ae97ec6..4946cac6d 100644 --- a/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java +++ b/controller/app/src/debug/java/org/appdevforall/k2go/diskguard/debug/DebugDiskGuardReceiver.java @@ -9,8 +9,8 @@ /** * DEBUG-ONLY. K2GO-386 device-verify hook. Forces one disk-guard check with an injected floor so the - * full protective path (set desired=DOWN → reap the box → reclaim the runaway log → notify) can be - * verified on device WITHOUT first filling ~58 GB. Lives in src/debug, so it never ships in release. + * protective path (reap the box -> reclaim the runaway log -> restart) can be verified on device WITHOUT + * first filling ~58 GB. Lives in src/debug, so it never ships in release. * *

Exported (it is the whole point — an adb-reachable surface, unlike the app's non-exported * services), mirroring {@link org.appdevforall.k2go.delivery.debug.DebugDeliveryReceiver}. A huge @@ -23,8 +23,10 @@ * --el floor_bytes 999999999999 * * - * Watch it act in logcat: {@code adb logcat -s K2Go-DiskGuard}. Because it sets desired=DOWN, the box - * stays down after the reap; re-enable the server from the app to bring it back. + * Watch it act in logcat: {@code adb logcat -s K2Go-DiskGuard}. The debug hook runs the FORCED path, + * which always CONTAINs: it reaps and reclaims, then leaves the server desired=UP and asks the + * reconciler to relaunch a fresh box. It never advances the real escalation count, so repeated + * triggers cannot stop the box. */ public final class DebugDiskGuardReceiver extends BroadcastReceiver {