From 17b7015983395fc9088c9bc7f6d04b8fe938e114 Mon Sep 17 00:00:00 2001 From: Guilherme Costa Date: Thu, 16 Jul 2026 09:52:57 +0100 Subject: [PATCH 1/6] fix(updater): skip a component whose branch was deleted upstream so one dead branch can't brick updates --- tests/updater/test_service_unit.py | 20 ++++++++++++++++++++ updater/service.py | 12 ++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/updater/test_service_unit.py b/tests/updater/test_service_unit.py index 980cdf4c..dee839d5 100644 --- a/tests/updater/test_service_unit.py +++ b/tests/updater/test_service_unit.py @@ -665,6 +665,26 @@ async def test_empty_batch_skips_git(self, tmp_path): await svc.update_all() mock_batch.assert_not_called() + @pytest.mark.asyncio + async def test_dead_branch_dropped_not_aborting_batch(self, tmp_path): + # A deleted-upstream branch must be dropped so BlocksScreen still updates. + dead = self._git(tmp_path, "klipper", 2) + dead.branch = "wip/deleted" + good = self._git(tmp_path, "BlocksScreen", 99) + good.branch = "dev" + with ( + patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._run_git_batch") as mock_batch, + patch( + "updater.service.git_ref_hash", + side_effect=lambda path, ref: "" if "deleted" in ref else "a" * 40, + ), + ): + svc = UpdateService() + svc._components = [dead, good] + await svc.update_all() + assert [c.name for c in mock_batch.call_args[0][0]] == ["BlocksScreen"] + class TestAtomicBatch: """_run_git_batch: stage all → deps → hooks → restart each service once; all-or-nothing.""" diff --git a/updater/service.py b/updater/service.py index 77a1671f..d824e1c3 100644 --- a/updater/service.py +++ b/updater/service.py @@ -326,6 +326,18 @@ async def update_all(self, names: set[str] | None = None) -> None: continue # path is now absent: the provision pass below clones fresh self._log.error("%s: %s is not a git repository - skipping", c.name, c.path) self._cb_error_done(c.name, "not a git repository - reinstall required") + # A dead configured branch must not abort the batch and brick BlocksScreen. + for c in list(batch): + if c.branch and not await git_ref_hash(c.path, f"origin/{c.branch}"): + batch.remove(c) + self._log.error( + "%s: branch origin/%s does not resolve (deleted upstream?) - skipping", + c.name, + c.branch, + ) + self._cb_error_done( + c.name, f"branch origin/{c.branch} not found - fix components.yaml" + ) provision = [ c for c in sorted_components From a34b965fd0234e14ac8000d173d6225856d0db6f Mon Sep 17 00:00:00 2001 From: Guilherme Costa Date: Thu, 16 Jul 2026 16:14:30 +0100 Subject: [PATCH 2/6] fix(updater): reclone rung for local==origin corruption; klipper->master; disown klipper from moonraker (fresh installs); actionable dead-branch status --- scripts/bs-common.sh | 2 +- tests/scripts/test_moonraker_ownership.py | 5 + tests/updater/test_executor_unit.py | 17 +++ tests/updater/test_self_heal_unit.py | 1 + tests/updater/test_service_unit.py | 164 ++++++++++++++++++++ updater/__main__.py | 3 +- updater/components.py | 5 +- updater/components.yaml | 2 +- updater/dbus_service.py | 16 +- updater/executor.py | 143 ++++++++---------- updater/locking.py | 21 ++- updater/service.py | 176 ++++++++++++++-------- 12 files changed, 380 insertions(+), 175 deletions(-) diff --git a/scripts/bs-common.sh b/scripts/bs-common.sh index 34ea4f08..26d93be0 100644 --- a/scripts/bs-common.sh +++ b/scripts/bs-common.sh @@ -48,7 +48,7 @@ bs_disable_overlapping_update_managers() { local conf="$1" tag="${2:-bs-common}" [ -f "$conf" ] || return 1 grep -q "blocksscreen-single-owner" "$conf" && return 1 - local owned="RF50-Klipper happy-hare Klippain-ShakeTune mainsail-config crowsnest" + local owned="klipper RF50-Klipper happy-hare Klippain-ShakeTune mainsail-config crowsnest" local tmp # Same-dir temp + atomic rename: a power cut can never truncate moonraker.conf. tmp="$(mktemp -p "$(dirname "$conf")" .moonraker.conf.XXXXXX)" || return 1 diff --git a/tests/scripts/test_moonraker_ownership.py b/tests/scripts/test_moonraker_ownership.py index a62d1952..eab5878a 100644 --- a/tests/scripts/test_moonraker_ownership.py +++ b/tests/scripts/test_moonraker_ownership.py @@ -14,6 +14,10 @@ [update_manager mainsail] type: web +[update_manager klipper] +type: git_repo +path: ~/klipper + [update_manager RF50-Klipper] type: git_repo path: ~/RF50-Klipper @@ -44,6 +48,7 @@ def test_owned_blocks_commented_others_untouched(tmp_path: Path) -> None: conf = tmp_path / "moonraker.conf" conf.write_text(_CONF) out = _run(conf) + assert "#[update_manager klipper]" in out # updater owns klipper, not moonraker assert "#[update_manager RF50-Klipper]" in out assert "#[update_manager crowsnest]" in out assert "[update_manager mainsail]" in out # web client kept diff --git a/tests/updater/test_executor_unit.py b/tests/updater/test_executor_unit.py index af1d3fc7..0121712c 100644 --- a/tests/updater/test_executor_unit.py +++ b/tests/updater/test_executor_unit.py @@ -841,6 +841,23 @@ async def test_branch_mismatch_flagged_when_on_wrong_branch(self, tmp_path): assert result.commits_behind == 0 assert result.branch_mismatch is True + @pytest.mark.asyncio + async def test_dead_configured_branch_reports_actionable_error(self, tmp_path): + # origin/ deleted upstream: report "fix components.yaml", not the + # generic git_commits_behind failure. + procs = [ + _make_proc(0, b"abc1234\n", b""), # git_get_hash + _make_proc(0, b"wip/dead\n", b""), # git_get_current_branch + _make_proc(128, b"", b"fatal: unknown revision\n"), # git_commits_behind -1 + _make_proc(128, b"", b"fatal: unknown revision\n"), # git_ref_hash -> "" + ] + exec_mock = AsyncMock(side_effect=procs) + with patch("asyncio.create_subprocess_exec", exec_mock): + result = await check_git_status( + "klipper", tmp_path, branch="wip/dead", skip_fetch=True + ) + assert result.error == "branch origin/wip/dead not found - fix components.yaml" + class TestCheckAptStatus: @pytest.mark.asyncio diff --git a/tests/updater/test_self_heal_unit.py b/tests/updater/test_self_heal_unit.py index 358d8094..888c631e 100644 --- a/tests/updater/test_self_heal_unit.py +++ b/tests/updater/test_self_heal_unit.py @@ -871,6 +871,7 @@ async def run_test(): patch( "updater.service.git_repair", return_value=(False, "repair fail") ), + patch.object(UpdateService, "_reclone_component", return_value=False), ): await svc._reconcile_locked() # must not raise assert "BlocksScreen" not in svc._read_state() # entry sanitized away diff --git a/tests/updater/test_service_unit.py b/tests/updater/test_service_unit.py index dee839d5..48416eba 100644 --- a/tests/updater/test_service_unit.py +++ b/tests/updater/test_service_unit.py @@ -1590,6 +1590,170 @@ async def test_provision_enable_failure_is_best_effort(self, tmp_path): cb.on_component_done.assert_called_once_with("newcomp", True) +class TestProvisionMissing: + """provision_missing clones absent opted-in components outside a user Update.""" + + def _comp(self, tmp_path: Path, *, name: str = "Spoolman") -> ComponentConfig: + return ComponentConfig( + name=name, + kind="git", + path=tmp_path / name, + branch="master", + url=f"https://github.com/test/{name}", + install_if_missing=True, + ) + + @pytest.mark.asyncio + async def test_provisions_absent_opted_in_component(self, tmp_path): + comp = self._comp(tmp_path) # path does not exist + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch.object( + UpdateService, "_provision_component", return_value=True + ) as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is True + mock_prov.assert_awaited_once_with(comp) + + @pytest.mark.asyncio + async def test_present_component_is_never_provisioned(self, tmp_path): + comp = self._comp(tmp_path) + comp.path.mkdir() # already on disk + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch.object(UpdateService, "_provision_component") as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is False + mock_prov.assert_not_called() + + @pytest.mark.asyncio + async def test_not_opted_in_is_skipped(self, tmp_path): + comp = self._comp(tmp_path) + comp.install_if_missing = False # absent but not declared installable + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch.object(UpdateService, "_provision_component") as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is False + mock_prov.assert_not_called() + + @pytest.mark.asyncio + async def test_defers_when_process_lock_held(self, tmp_path): + # A concurrent update owns the lock: provisioning defers, never races it. + comp = self._comp(tmp_path) + with ( + patch("updater.service.process_lock", lambda: nullcontext(False)), + patch.object(UpdateService, "_provision_component") as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is False + mock_prov.assert_not_called() + + +class TestReclone: + """_reclone_component: temp-clone + atomic swap as the deepest corruption rung.""" + + def _comp(self, tmp_path: Path) -> ComponentConfig: + path = tmp_path / "klipper" + path.mkdir() + (path / "corrupt").write_text("x") # marker proving the tree got replaced + return ComponentConfig( + name="klipper", + kind="git", + path=path, + url="https://github.com/test/klipper", + branch="master", + ) + + @staticmethod + def _fake_clone(url, dest, branch): + Path(dest).mkdir() + (Path(dest) / ".git").mkdir() + (Path(dest) / "fresh").write_text("y") + return (True, "") + + @pytest.mark.asyncio + async def test_swaps_in_fresh_clone(self, tmp_path): + comp = self._comp(tmp_path) + with patch("updater.service.git_clone", side_effect=self._fake_clone): + svc = UpdateService() + svc._components = [comp] + ok = await svc._reclone_component(comp) + assert ok is True + assert (comp.path / "fresh").exists() # fresh clone swapped in + assert not (comp.path / "corrupt").exists() # old tree gone + assert not (comp.path.parent / ".klipper.reclone-old").exists() + assert not (comp.path.parent / ".klipper.reclone-tmp").exists() + + @pytest.mark.asyncio + async def test_clone_failure_leaves_original_intact(self, tmp_path): + comp = self._comp(tmp_path) + with patch("updater.service.git_clone", return_value=(False, "boom")): + svc = UpdateService() + svc._components = [comp] + ok = await svc._reclone_component(comp) + assert ok is False + assert (comp.path / "corrupt").exists() # original untouched on clone failure + assert not (comp.path.parent / ".klipper.reclone-tmp").exists() + + @pytest.mark.asyncio + async def test_no_url_returns_false(self, tmp_path): + comp = self._comp(tmp_path) + comp.url = None + with patch("updater.service.git_clone") as mock_clone: + svc = UpdateService() + svc._components = [comp] + ok = await svc._reclone_component(comp) + assert ok is False + mock_clone.assert_not_called() + + @pytest.mark.asyncio + async def test_stage_reclones_when_repair_fails(self, tmp_path): + comp = self._comp(tmp_path) + with ( + patch("updater.service.git_fetch", return_value=(False, "corrupt")), + patch("updater.service.git_has_corruption", return_value=True), + patch("updater.service.git_repair", return_value=(False, "unrepairable")), + patch.object( + UpdateService, "_reclone_component", return_value=True + ) as mock_reclone, + patch("updater.service.git_checkout", return_value=(True, "")), + patch("updater.service.git_reset_to_hash", return_value=(True, "")), + ): + svc = UpdateService() + svc._components = [comp] + ok, _reason = await svc._stage_component(comp) + assert ok is True + mock_reclone.assert_awaited_once() + + @pytest.mark.asyncio + async def test_reconcile_reclones_when_repair_fails(self, tmp_path): + comp = self._comp(tmp_path) + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch("updater.service.git_get_hash", side_effect=["", "abc123"]), + patch("updater.service.git_repair", return_value=(False, "unrepairable")), + patch.object( + UpdateService, "_reclone_component", return_value=True + ) as mock_reclone, + ): + svc = UpdateService() + svc._components = [comp] + await svc.reconcile() + mock_reclone.assert_awaited_once() + + class TestReconcile: """Boot-time repo healing for every component (Blocker 1).""" diff --git a/updater/__main__.py b/updater/__main__.py index 39e36619..3fc6a019 100644 --- a/updater/__main__.py +++ b/updater/__main__.py @@ -85,8 +85,7 @@ async def _run_daemon() -> None: def _watchdog_ping_interval() -> float: """Half of systemd's WatchdogSec (per sd_notify(3)), or 15s if unset/invalid. - Reading WATCHDOG_USEC from the environment keeps the heartbeat correct if the - unit's WatchdogSec is ever retuned, instead of hardcoding half of 30s. + Reads WATCHDOG_USEC so the heartbeat stays correct if WatchdogSec is retuned. """ try: watchdog_usec = int(os.environ.get("WATCHDOG_USEC", "0")) diff --git a/updater/components.py b/updater/components.py index e4938088..f80d5d47 100644 --- a/updater/components.py +++ b/updater/components.py @@ -166,10 +166,9 @@ def _merge(base: list[dict], override: list[dict]) -> list[dict]: def load_components() -> tuple[list[ComponentConfig], float]: - """Load and validate component definitions from bundled YAML, merged with user override. + """Load/validate components from bundled YAML merged with the user override. - Returns: - A tuple of (list of ComponentConfig, poll_interval in seconds). + Returns (list of ComponentConfig, poll_interval in seconds). """ try: import yaml # noqa: PLC0415 diff --git a/updater/components.yaml b/updater/components.yaml index 38dd5253..334a2afc 100644 --- a/updater/components.yaml +++ b/updater/components.yaml @@ -14,7 +14,7 @@ components: url: https://github.com/Klipper3d/klipper service: klipper.service reset_mode: hard - branch: wip/Happy-Hare-Integration + branch: master order: 2 restart_ui: true diff --git a/updater/dbus_service.py b/updater/dbus_service.py index f7284bf5..d7b3d605 100644 --- a/updater/dbus_service.py +++ b/updater/dbus_service.py @@ -19,11 +19,8 @@ class DbusProgressCallback: - """Adapter that forwards ProgressCallback events to D-Bus signals. - - Holds a reference to the server interface so each on_* call invokes - the corresponding signal's .emit() - decoupling UpdateService from D-Bus. - """ + """Forward ProgressCallback events to D-Bus signals (decouples UpdateService + from D-Bus): each on_* call invokes the matching signal's .emit().""" def __init__(self, iface: UpdaterInterface) -> None: self._iface = iface @@ -55,11 +52,8 @@ class UpdaterInterface( sdbus.DbusInterfaceCommonAsync, interface_name="com.blockscreen.Updater", ): - """D-Bus interface contract shared by server and client proxy. - - Signals are declared once here, sdbus replaces each descriptor with - emit machinery on the server side and an async-iterable on the client. - """ + """D-Bus contract shared by server and client proxy: signals declared once, + sdbus swaps each for server-side emit machinery / client-side async-iterable.""" @sdbus.dbus_signal_async("sii") def step_complete(self) -> tuple[str, int, int]: @@ -186,6 +180,8 @@ async def _periodic_status_check(self) -> None: while True: try: await self._emit_status() + if await self._svc.provision_missing(): + await self._emit_status() # reflect freshly-installed components except Exception as exc: # noqa: BLE001 _log.error("periodic_check failed: %s", exc) await asyncio.sleep(self._svc.poll_interval) diff --git a/updater/executor.py b/updater/executor.py index 437da324..46481a91 100644 --- a/updater/executor.py +++ b/updater/executor.py @@ -45,14 +45,10 @@ def _clear_stale_git_index_lock(path: Path) -> bool: - """Remove .git/index.lock only if it is stale (older than threshold). - - A stale lock is left by a SIGKILL'd or interrupted git operation. Only - remove if mtime indicates lock is NOT held by a live git process (age >= - threshold). At runtime, a fresh lock may indicate a concurrent legitimate - git op; yanking it would corrupt that op. Returns True if lock was - removed or did not exist; False if lock exists but is too fresh to safely - remove. + """Remove .git/index.lock only if stale (mtime age >= threshold). + + A fresh lock may belong to a live/concurrent git op; yanking it would corrupt + that op. Returns True if removed or absent, False if too fresh to remove. """ lock_path = path / ".git" / "index.lock" if not lock_path.exists(): @@ -161,10 +157,9 @@ def _validate_git_ref(ref: str) -> bool: def _resolve_component_pip(path: Path | None) -> str: - """Find the pip binary for a component's venv; falls back to system pip if none found. + """Find the component venv's pip, else system pip. - SEC: Validate venv paths are under component or parent; reject symlinks to prevent - arbitrary venv hijacking. + SEC: only accept venv paths under the component/parent; reject symlink escapes. """ if path is None: logger.warning("Path not found, using system pip") @@ -198,10 +193,9 @@ def _resolve_component_pip(path: Path | None) -> str: async def _list_upgradable_packages() -> tuple[bool, list[str]]: - """Return (success, list_of_package_names) from apt list --upgradable. + """Return (success, package_names) from apt list --upgradable. - SEC: Validate package names against Debian naming rules to prevent - injection attacks even if apt output is malformed. + SEC: validate names against Debian rules to resist injection via malformed output. """ ok, output = await _run([APT, "list", "--upgradable"], timeout=30.0) if not ok: @@ -277,6 +271,14 @@ async def check_git_status( else: commits_behind = await git_commits_behind(path, remote_ref) if commits_behind == -1: + # a configured branch whose origin ref is gone is a config error, not transient + if branch and not await git_ref_hash(path, remote_ref): + return ComponentStatus( + name=name, + current_hash=current_hash, + current_branch=current_branch, + error=f"branch {remote_ref} not found - fix components.yaml", + ) logger.error("Failed at git_commits_behind operation") return ComponentStatus( name=name, error="Failed at git_commits_behind operation" @@ -375,11 +377,8 @@ async def git_fetch( ) -> tuple[bool, str]: """Fetch all remotes for the repo at path. - prune_remotes=False skips the extra-remote cleanup; pass False during status - checks where pruning is unnecessary and adds ~2 subprocess spawns per repo. - - A corrupt local ref (null SHA from an interrupted write) makes --prune abort - the whole fetch and silently disables every update; on that failure the + prune_remotes=False skips extra-remote cleanup (unneeded in status checks). A + corrupt local ref makes --prune abort the whole fetch; on that failure the named ref is deleted and the fetch retried once. """ if path is None: @@ -410,8 +409,7 @@ async def git_clone( ) -> tuple[bool, str]: """Clone url into path (https-only), optionally at branch. Returns (ok, message). - Large repos (klipper, moonraker) can take a while, so the timeout is generous. - git creates the leaf directory; the parent (home) must already exist. + Timeout is generous for large repos; git makes the leaf dir, parent must exist. """ if not path: return (False, "path error") @@ -427,12 +425,11 @@ async def git_clone( async def git_reset_to_hash(path: Path | None, prev_hash: str = "") -> tuple[bool, str]: - """Hard-reset repo at path directly to prev_hash without fetching. + """Hard-reset repo at path directly to prev_hash (no fetch). - This is the rollback/heal primitive (abort, boot revert, recover), so the - timeout is generous: a reset across a large delta on a slow SD card must - not be SIGTERM'd mid-checkout in exactly the path meant to fix things. - If reset fails with an index.lock error, clears it and retries once. + The rollback/heal primitive (abort, boot revert, recover); generous timeout so + a large reset on a slow SD card isn't SIGTERM'd mid-checkout. Clears an + index.lock error and retries once. """ if not path: return (False, "path error") @@ -471,10 +468,9 @@ async def git_reset_to_hash(path: Path | None, prev_hash: str = "") -> tuple[boo async def git_has_corruption(path: Path | None, hint: str = "") -> bool: """True if `hint` or git fsck shows object corruption. - `hint` (e.g. a failed fetch's stderr) is checked first: `git fsck - --connectivity-only` validates reachability, not object content, so it can - miss a corrupt blob that already broke the fetch. The fetch error is the - reliable signal in that case. + `hint` (e.g. a failed fetch's stderr) is checked first: fsck + --connectivity-only validates reachability not content, so it can miss a + corrupt blob the fetch already tripped on. """ if path is None: return False @@ -507,12 +503,9 @@ def _prune_empty_loose_objects(objects: Path) -> int: async def _quarantine_corrupt_objects(path: Path) -> int: """Move loose objects that `git fsck --full` flags as corrupt out of the way. - `git_has_corruption` uses `fsck --connectivity-only`, which never reads blob - content and so cannot see a non-empty object with a bad zlib header - exactly - what a power cut mid-write produces. A plain re-fetch will NOT replace such an - object because git still finds a file at that name. `fsck --full` reads object - content and names the bad ones; we move (not delete) them so a re-fetch - re-downloads them, and a misdiagnosis stays recoverable. Returns count moved. + connectivity-only fsck misses a bad-zlib object (power cut mid-write) and a + re-fetch won't replace it (file still there); fsck --full names them and we + move (not delete) them so a re-fetch restores them. Returns count moved. """ ok, out = await _run( [GIT, "fsck", "--full", "--no-dangling", "--no-progress"], @@ -550,14 +543,11 @@ async def _quarantine_corrupt_objects(path: Path) -> int: async def git_repair(path: Path, branch: str = "main") -> tuple[bool, str]: - """Prune 0-byte loose objects, re-fetch, and re-verify. Mirrors start.sh recovery. - - If empty-object pruning plus a fetch does not clear the corruption, escalate to - quarantining non-empty corrupt loose objects (`fsck --full`) and re-fetch once - more. Working tree is untouched (delete/move + fetch only), so tracked-but- - modified files survive. If fetch fails with index.lock error, clears lock and - retries. Branch parameter is used to repair corrupt HEAD (fallback to "main"). - Returns (ok, message). + """Prune 0-byte loose objects, re-fetch, re-verify. Mirrors start.sh recovery. + + If that doesn't clear it, quarantine non-empty corrupt objects (`fsck --full`) + and re-fetch once more. Working tree untouched (delete/move + fetch only); + `branch` repairs a corrupt HEAD (fallback "main"). Returns (ok, message). """ _clear_stale_git_index_lock(path) objects = path / ".git" / "objects" @@ -706,9 +696,9 @@ async def git_checkout( async def check_apt_status( cache_ttl_seconds: int = 86_400, exclude: tuple[str, ...] = () ) -> ComponentStatus: - """Return apt ComponentStatus from cache, refreshing via apt-get if older than cache_ttl_seconds. + """Return cached apt ComponentStatus, refreshing via apt-get past the TTL. - SEC: Validate cache file ownership and permissions to prevent privilege escalation. + SEC: validate cache ownership/permissions to prevent privilege escalation. """ path = Path.home() / ".cache" / "blockscreen" / "apt_status_cache.json" exclude_key = "|".join(sorted(exclude)) @@ -786,19 +776,16 @@ def _apt_env() -> dict[str, str]: def _apt_cmd(verb: str, pkgs: Sequence[str] = ()) -> list[str]: """Build the sudo apt argv via the root-owned wrapper. - The wrapper is the only apt command sudoers grants. bs-bootstrap reinstalls it - when missing (an old-installer box can have the service but not the wrapper); until - that self-heal runs a missing wrapper surfaces as a plain 'command not found'. + The wrapper is the only apt command sudoers grants; bs-bootstrap reinstalls it + when missing (an old box may have the service but not the wrapper). """ return [SUDO, str(APT_HELPER), verb, *pkgs] async def _apt_snapshot_packages() -> tuple[bool, Path | None]: - """Snapshot current package state with dpkg --get-selections to temp file. + """Snapshot package state via dpkg --get-selections to a temp file (no sudo). - Returns (success, snapshot_path). - Path is stored at ~/.cache/blockscreen/apt_pre_upgrade_snapshot.txt. - No sudo required for dpkg --get-selections. + Returns (success, snapshot_path) under ~/.cache/blockscreen/. """ snapshot_path = ( Path.home() / ".cache" / "blockscreen" / "apt_pre_upgrade_snapshot.txt" @@ -827,21 +814,19 @@ async def _apt_snapshot_packages() -> tuple[bool, Path | None]: async def _apt_get_fix_broken() -> tuple[bool, str]: - """Run apt-get -f install -y to fix broken package state. + """Run apt-get -f install -y to unbreak package state. - Best-effort rollback after failed upgrade. May not fully restore state, - but aims to unbreak the system. Called only on apt_upgrade failure. + Best-effort post-upgrade rollback; called only on apt_upgrade failure. """ return await _run(_apt_cmd("fix-broken"), timeout=120.0, env=_apt_env()) async def _apt_restore_packages(snapshot_path: Path) -> tuple[bool, str]: - """Restore dpkg package *selections* from the pre-upgrade snapshot, then dselect-upgrade. + """Restore dpkg selections from the pre-upgrade snapshot, then dselect-upgrade. - This re-asserts which packages should be installed; it does NOT reliably - downgrade versions, since the prior .debs may no longer be in the apt cache. - Treat it as an unbreak step (keep the set consistent), not a true rollback. - Kernel/firmware are excluded from upgrades, bounding the blast radius. + Re-asserts which packages should be installed; does NOT reliably downgrade + versions (old .debs may be gone). An unbreak step, not a true rollback; + kernel/firmware excluded bounds the blast radius. """ try: stdin_data = snapshot_path.read_bytes() @@ -903,10 +888,9 @@ async def apt_update() -> tuple[bool, str]: async def apt_upgrade(exclude: tuple[str, ...] = ()) -> tuple[bool, str]: - """Run apt-get upgrade via explicit package list to limit blast radius. + """Run apt-get upgrade via an explicit package list to limit blast radius. - Always lists upgradable packages first; applies exclude regexes if any. - Uses 'install --only-upgrade' so only already-installed packages are touched. + Lists upgradables, applies excludes, uses `install --only-upgrade`. """ ok, pkgs = await _list_upgradable_packages() if not ok: @@ -934,9 +918,8 @@ async def run_hook( ) -> tuple[bool, str]: """Run the per-component update hook if it exists. - The 60s default suits tests and trivial hooks; the update and provisioning - flows pass HOOK_TIMEOUT since a hook may sync a full dependency set (e.g. - Spoolman's `uv sync` after its deps changed). + The 60s default suits tests/trivial hooks; update/provision flows pass + HOOK_TIMEOUT since a hook may sync a full dep set (e.g. Spoolman's `uv sync`). """ hook = (_HOOKS_DIR / f"{name}.sh").resolve() # SEC: resolve symlinks try: @@ -967,11 +950,10 @@ async def enable_service(name: str | None) -> tuple[bool, str]: async def wait_for_service_active(name: str, timeout: float = 90.0) -> bool: - """Poll systemctl is-active until the service is active or timeout expires. + """Poll systemctl is-active until active or timeout. - `systemctl is-active --wait` does not exist (--wait applies only to - start/restart/is-system-running/kill and is silently ignored here), so an - explicit poll loop is required to ride out 'activating' after a restart. + `is-active --wait` doesn't exist (--wait is ignored here), so an explicit poll + is needed to ride out 'activating' after a restart. """ if not _SERVICE_RE.match(name): logger.warning("wait_for_service_active: invalid service name %r", name) @@ -998,9 +980,8 @@ async def wait_for_service_active(name: str, timeout: float = 90.0) -> bool: async def restart_service_noblock(name: str | None) -> tuple[bool, str]: """Queue a service restart without waiting (systemctl --no-block). - For the UI service that hosts the updater's D-Bus client: a blocking restart - tears down that client, and a slow restart could time out and abort the - in-flight update. Fire-and-forget and let it reload on its own. + For the UI service hosting the updater's D-Bus client: a blocking restart + tears it down and could time out the in-flight update. Fire-and-forget. """ if name is None: return (False, "service name is None") @@ -1012,10 +993,9 @@ async def restart_service_noblock(name: str | None) -> tuple[bool, str]: async def verify_updater_importable(component_path: Path | None) -> bool: """Self-test the new on-disk updater code before restarting into it. - Imports the updater package in a fresh interpreter from component_path. A - new updater that fails to import must never restart the daemon onto itself: - that would take down the only field update path. On failure the caller keeps - the running (old) daemon and lets the next reboot adopt the new code. + Imports the updater package in a fresh interpreter from component_path: a new + updater that fails to import must never restart the daemon onto itself (that + kills the only field update path). On failure keep the old daemon. """ if component_path is None or not component_path.exists(): return False @@ -1032,10 +1012,9 @@ async def verify_updater_importable(component_path: Path | None) -> bool: async def restart_service(name: str | None) -> tuple[bool, str]: """Restart a systemd service, recovering from a start-limit hit. - If `restart` fails (commonly "start request repeated too quickly"), clear the - rate-limit/failed state with `reset-failed` and retry once. `systemctl kill` - is NOT used as a fallback: it does not clear a start-limit and is not covered - by the scoped NOPASSWD sudoers rules, so it would prompt for a password. + If `restart` fails (e.g. "start request repeated too quickly"), `reset-failed` + and retry once. `systemctl kill` is NOT a fallback: it neither clears a + start-limit nor is covered by the scoped NOPASSWD rules (would prompt). """ if name is None: return (False, "service name is None") diff --git a/updater/locking.py b/updater/locking.py index c7bd5abb..e58fcb9d 100644 --- a/updater/locking.py +++ b/updater/locking.py @@ -1,9 +1,8 @@ """Cross-process advisory lock shared by the updater daemon and the CLI. -A single flock-backed lockfile is the only thing serializing mutating git/apt -work between the long-running daemon and a `python -m updater update` CLI run: -the daemon's asyncio locks are in-process only. Both sides acquire the same path -non-blocking, so whoever is second fails fast instead of corrupting a repo. +A single flock-backed lockfile serializes mutating git/apt work between the +daemon and a `python -m updater update` CLI run (asyncio locks are in-process +only); both acquire it non-blocking so the second caller fails fast. """ from __future__ import annotations @@ -36,12 +35,11 @@ def lock_path() -> Path: def restart_sentinel_path() -> Path: - """Return the sentinel a self-update hook writes instead of restarting. + """Return the sentinel a self-update hook writes instead of restarting mid-batch. - Hooks running under the updater append `install`/`code` here rather than - restarting the daemon mid-batch; the daemon reads it once the batch is done. - Preferring /run (tmpfs) means it clears on reboot, so a crash can never - trigger a spurious restart later; the safe floor is 'adopt on next reboot'. + Hooks append `install`/`code` here; the daemon reads it once the batch ends. + Prefer /run (tmpfs) so it clears on reboot: a crash can't trigger a spurious + restart later (safe floor: adopt on next reboot). """ return _runtime_dir() / "updater-restart-needed" @@ -50,9 +48,8 @@ def restart_sentinel_path() -> Path: def process_lock() -> Iterator[bool]: """Acquire the shared updater lock non-blocking. - Yields True if acquired, False if another updater process (daemon or CLI) - holds it. The fd stays open for the whole `with` block - closing it on exit - is what releases the lock. + Yields True if acquired, else False (another daemon/CLI holds it). The fd + stays open for the whole `with` block; closing it on exit releases the lock. """ try: f = open(lock_path(), "w") # noqa: SIM115, PTH123 diff --git a/updater/service.py b/updater/service.py index d824e1c3..5df70144 100644 --- a/updater/service.py +++ b/updater/service.py @@ -353,16 +353,38 @@ async def update_all(self, names: set[str] | None = None) -> None: for c in provision: await self._provision_component(c) + async def provision_missing(self) -> bool: + """Clone absent install_if_missing components without waiting for a manual + Update (e.g. one added by a self-update); existing repos untouched. Returns + True if anything was provisioned.""" + missing = [ + c + for c in self._components + if c.install_if_missing + and c.url + and (c.path is None or not c.path.exists()) + ] + if not missing: + return False + provisioned = False + with process_lock() as acquired: + if not acquired: + self._log.info("provision_missing: update in progress, deferring") + return False + for c in missing: + if c.path is None or not c.path.exists(): # recheck under lock + await self._provision_component(c) + provisioned = True + return provisioned + async def _preflight_fetch( self, sorted_components: list[ComponentConfig], all_components: list ) -> bool: """Fetch every existing git component up-front (network phase). - Decoupling the download from the apply means a mid-sequence connection - drop aborts before anything is checked out, instead of leaving an - applied/unapplied skew across components. A pre-fetched component skips - its own fetch in the apply phase (recorded fetch time). Repos that need - cloning or are corrupt are left to self-heal in their own flow. + Decoupling download from apply means a mid-sequence drop aborts before + anything is checked out (no applied/unapplied skew). A pre-fetched + component skips its apply-phase fetch; missing/corrupt repos self-heal. """ # Non-repo dirs excluded: their fetch failure is not "offline" (see update_all). targets = [ @@ -396,11 +418,10 @@ async def _preflight_fetch( async def _run_git_batch(self, batch: list[ComponentConfig]) -> None: """Apply existing git components all-or-nothing. - Stage every repo, then deps, then hooks, then restart each unique service - once. While staging/deps/hooks run nothing has been restarted, so the live - services still execute the old code - an abort then just reverts git and - restarts nothing. A restart-phase failure reverts git AND re-restarts the - services already bounced so they come back on the old code. + Stage every repo, then deps, hooks, then restart each unique service once. + Nothing is restarted during stage/deps/hooks, so an abort just reverts git + and restarts nothing. A restart-phase failure reverts git AND re-restarts + the already-bounced services back onto the old code. """ prev: dict[str, str] = {} for c in batch: @@ -601,17 +622,14 @@ async def _abort_batch( self._cb("on_component_done", c.name, False) async def _apply_deferred_restart(self) -> None: - """Apply a daemon restart/reinstall a hook deferred during this batch. - - Hooks running under the updater record `install`/`code` to the tmpfs - sentinel instead of restarting the daemon mid-batch. Called once, after - success is recorded and the UI restart is queued: - - `install`: touch the deploy flag so BlocksScreen-deploy.path runs - install-updater out-of-band (its own cgroup, post-batch). - - `code`: self-test the new on-disk updater; only if it imports - cleanly, `--no-block` restart the daemon onto it. Otherwise keep the - running daemon and let the next reboot adopt it. Never raises: a - completed update must stand regardless of what happens here. + """Apply a daemon restart/reinstall hook deferred during this batch. + + Hooks record `install`/`code` to the tmpfs sentinel instead of restarting + mid-batch. Called once after success + UI restart: `install` touches the + deploy flag (BlocksScreen-deploy.path runs install-updater out-of-band); + `code` self-tests the new updater and `--no-block` restarts onto it only if + it imports, else the next reboot adopts it. Never raises: a completed update + must stand regardless. """ try: sentinel = restart_sentinel_path() @@ -707,10 +725,9 @@ async def recover(self, name: str, hard: bool = False) -> bool: return ok async def bless_healthy(self, name: str, hash_val: str) -> bool: - """Mark a component as healthy (known-good) after a debounce; snapshots baseline. + """Mark a component healthy (known-good) after a debounce; snapshots baseline. - An empty hash_val means "bless the component's current HEAD": the UI does - not need to compute a hash, it just says it is healthy. + Empty hash_val blesses the current HEAD (UI need not compute a hash). """ component = next((c for c in self._components if c.name == name), None) if component is None: @@ -747,8 +764,7 @@ def _apply(state: dict) -> None: def _check_crash_loop(self, name: str, nrestarts: int) -> bool: """Return True if NRestarts rose by >= 5 within the trailing 180s window. - Compares against the oldest in-window sample (largest rise, since NRestarts - is monotonic), then records the current sample and prunes aged-out ones. + Compared against the oldest in-window sample; records+prunes samples. """ if name not in self._nrestarts_samples: self._nrestarts_samples[name] = [] # fresh device: start tracking now @@ -881,9 +897,9 @@ def _rebaseline_samples(self, name: str, nrestarts: int) -> None: async def supervise_ui(self) -> None: """Poll the UI service NRestarts and drive the recovery ladder on a crash-loop. - The single-attempt-in-flight guard is the loop itself: at most one rung runs - per crash-loop, then detection is re-baselined and a settle window elapses so - a new build gets a fair chance to bless before the next rung is counted. + The loop itself is the single-attempt-in-flight guard: at most one rung per + crash-loop, then detection re-baselines and a settle window elapses so a new + build can bless before the next rung is counted. """ while True: await asyncio.sleep(_NRESTARTS_POLL_INTERVAL_S) @@ -918,8 +934,7 @@ async def _handle_crash_loop(self, nrestarts: int) -> None: async def forward_heal_ui(self) -> None: """Slowly re-check origin/main and re-attempt an upgrade once a fix ships. - Jittered and connectivity-gated: desynchronized polls avoid a fleet stampede - and a failed fetch (offline) is skipped without churn. + Jittered + connectivity-gated: avoids a fleet stampede and offline churn. """ while True: delay = _FORWARD_HEAL_BASE_S + secrets.randbelow( @@ -992,16 +1007,12 @@ def _clear_fault_marker(self) -> None: async def reconcile(self) -> None: """Heal repos left damaged by a power loss mid-update, for every component. - start.sh only repairs the BlocksScreen repo; this covers klipper/moonraker/ - config repos too. The gate is the cheap `rev-parse HEAD` (mirrors start.sh's - `cat-file -e HEAD`) so a healthy boot pays almost nothing; deeper corruption - is caught on demand by the normal update flow. Offline-safe: git_repair - fetches when it can, else we reset to the last recorded good hash. - - Held under the shared process lock so the boot-time heal cannot interleave - its git resets/repairs with a near-simultaneous user update or CLI run on - the same repo. If another updater holds the lock, skip: that run is already - mutating the repos, and any real damage is re-detected on demand. + start.sh only repairs BlocksScreen; this covers klipper/moonraker/config too. + The gate is a cheap `rev-parse HEAD` so a healthy boot pays almost nothing; + deeper corruption is caught on demand. Offline-safe: git_repair fetches when + it can, else resets to the last recorded good hash. Held under the process + lock so the boot heal can't interleave with a user update/CLI run; if another + updater holds it, skip (that run is already mutating; damage re-detects later). """ with process_lock() as acquired: if not acquired: @@ -1014,10 +1025,9 @@ async def reconcile(self) -> None: async def _revert_inflight(self) -> None: """Revert any update cut off mid-flight by a power loss before it committed. - The in-flight marker maps each batch component to the commit it was on - before staging. Its mere presence at boot means a batch did not reach a - terminal outcome, so every listed repo is hard-reset back to that commit - and the marker is cleared. A repo already on its prev_hash is left alone. + The in-flight marker maps each batch component to its pre-staging commit; its + presence at boot means a batch didn't finish, so each listed repo is + hard-reset back and the marker cleared. A repo already there is left alone. """ inflight = await asyncio.to_thread(self._read_inflight) if not inflight: @@ -1062,6 +1072,9 @@ async def _reconcile_locked(self) -> None: if ok and await git_get_hash(c.path) != "": self._history("boot_repair", c.name, detail=msg[:80]) continue + if await self._reclone_component(c) and await git_get_hash(c.path): + self._history("boot_reclone", c.name) + continue comp_state = (await asyncio.to_thread(self._read_state)).get(c.name, {}) prev = ( comp_state.get("prev_hash") @@ -1179,9 +1192,8 @@ async def _ping_while( ): """Await coro, re-emitting on_step every `interval` seconds. - A HOOK_TIMEOUT-bounded hook can run silent longer than the client's - 360s busy watchdog; the pings keep it fed. Cancellation propagates to - the inner task so its subprocess is still killed. + A HOOK_TIMEOUT-bounded hook can run silent past the client's 360s busy + watchdog; the pings keep it fed. Cancellation propagates to the inner task. """ task = asyncio.ensure_future(coro) try: @@ -1221,8 +1233,7 @@ def _cb_error_done(self, name: str, reason: str) -> bool: def _history(self, event: str, name: str, **fields: object) -> None: """Append one event to the persistent update-history log (OBS-1). - journald is volatile on this image, so this on-SD JSONL log is the - durable record of update outcomes for field diagnosis. + journald is volatile here, so this on-SD JSONL is the durable field record. """ entry = { "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), @@ -1295,10 +1306,9 @@ async def _fail_provision(self, component: ComponentConfig, reason: str) -> bool async def _provision_component(self, component: ComponentConfig) -> bool: """Clone and set up a missing opted-in component. - A fresh component has no prev_hash to roll back to, so any failure deletes - the partial clone (clean slate to retry next Update) rather than rolling - back. Service install is delegated to the component's hook, mirroring the - existing update flow. + A fresh component has no prev_hash, so any failure deletes the partial clone + (clean slate to retry) instead of rolling back. Service install is delegated + to the component's hook, as in the update flow. """ if not component.url: return self._cb_error_done(component.name, "no clone url") @@ -1380,12 +1390,49 @@ async def _provision_component(self, component: ComponentConfig) -> bool: ) return await self._fail_provision(component, "unexpected_error") + async def _reclone_component(self, component: ComponentConfig) -> bool: + """Reclone an unrepairable repo: fresh clone to a temp dir, then atomic swap.""" + if not component.url or component.path is None: + return False + path = component.path + tmp = path.parent / f".{path.name}.reclone-tmp" + old = path.parent / f".{path.name}.reclone-old" + for stale in (tmp, old): # clear orphans from a crashed prior reclone + await asyncio.to_thread(shutil.rmtree, stale, ignore_errors=True) + self._log.warning("%s: recloning from %s", component.name, component.url) + ok, err = await git_clone(component.url, tmp, component.branch) + if not ok: + await asyncio.to_thread(shutil.rmtree, tmp, ignore_errors=True) + self._log.error("%s: reclone clone failed: %s", component.name, err) + return False + if component.version: + vok, verr = await git_reset_to_hash(tmp, component.version) + if not vok: + await asyncio.to_thread(shutil.rmtree, tmp, ignore_errors=True) + self._log.error("%s: reclone pin failed: %s", component.name, verr) + return False + try: + # move old aside first: renaming onto a nonempty dir is ENOTEMPTY + if path.exists(): + await asyncio.to_thread(os.rename, path, old) + await asyncio.to_thread(os.rename, tmp, path) + except OSError as exc: + if not path.exists() and old.exists(): # restore after a half-done swap + with contextlib.suppress(OSError): + await asyncio.to_thread(os.rename, old, path) + await asyncio.to_thread(shutil.rmtree, tmp, ignore_errors=True) + self._log.error("%s: reclone swap failed: %s", component.name, exc) + return False + await asyncio.to_thread(shutil.rmtree, old, ignore_errors=True) + self._history("reclone", component.name, url=component.url) + self._log.warning("%s: recloned successfully", component.name) + return True + async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str]: """Bring the working tree to its target ref: fetch-gate + reset/checkout/pull. - No deps/hook/restart and no rollback - the caller decides what to do on - failure. Returns (ok, reason) where reason is one of - network/corrupt/reset/version/branch/conflict. + No deps/hook/restart and no rollback (caller decides on failure). Returns + (ok, reason), reason one of network/corrupt/reset/version/branch/conflict. """ if component.path is None: return (False, "path not found") @@ -1403,9 +1450,12 @@ async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str] return (False, "network") self._log.warning("%s: corrupt repo, repairing", component.name) rok, rmsg = await git_repair(component.path) - if not rok: + if rok: + self._history("repair", component.name, detail=rmsg[:80]) + elif await self._reclone_component(component): # deepest rung + self._history("repair", component.name, detail="recloned") + else: return (False, "corrupt") - self._history("repair", component.name, detail=rmsg[:80]) # Switch to the target branch FIRST so reset/pull act on the right branch. if component.branch: @@ -1459,9 +1509,8 @@ async def _restart_one(self, service: str) -> bool: async def _run_git_update(self, component: ComponentConfig) -> bool: """Update a single existing git component, rolling back on any failure. - Shares its git mechanics (_stage_component) with the multi-component batch; - keeps its own per-component rollback since a single update has no peers to - coordinate. + Shares its git mechanics (_stage_component) with the batch; keeps its own + per-component rollback since a single update has no peers to coordinate. """ if component.path is None: return self._cb_error_done(component.name, "path not found") @@ -1780,8 +1829,7 @@ def _fsync_dir(path: Path) -> None: async def background_apt_upgrade(self) -> None: """Run apt update + upgrade + autoremove silently after every Update click. - Uses apt-get upgrade (never dist-upgrade) so the Debian release never changes. - Never reports to the UI - failures are logged only. + apt-get upgrade (never dist-upgrade) so the release can't change; UI-silent. """ async with self._apt_lock: await self._background_apt_upgrade_locked() From 06ed4a2deafa54640c7ce76824a353e2d13baa8c Mon Sep 17 00:00:00 2001 From: Guilherme Costa Date: Fri, 17 Jul 2026 14:31:51 +0100 Subject: [PATCH 3/6] fix(updater): per-component update isolation, main release channel, and pre-main review hardening --- scripts/BlocksScreen-deploy.service | 4 +- scripts/BlocksScreen-start.sh | 13 + scripts/bs-bootstrap.sh | 4 +- scripts/bs-common.sh | 27 +- scripts/bs-splash-holder.py | 9 +- scripts/install-updater.sh | 41 ++- scripts/install.sh | 4 +- scripts/post-merge | 11 +- tests/scripts/test_moonraker_ownership.py | 63 ++++ tests/updater/test_components_unit.py | 37 +++ tests/updater/test_executor_unit.py | 30 ++ tests/updater/test_service_unit.py | 355 +++++++++++++++++---- updater/__main__.py | 7 +- updater/components.py | 21 +- updater/components.yaml | 5 +- updater/dbus_service.py | 11 +- updater/executor.py | 36 ++- updater/models.py | 4 - updater/service.py | 372 ++++++++++++++++------ 19 files changed, 846 insertions(+), 208 deletions(-) diff --git a/scripts/BlocksScreen-deploy.service b/scripts/BlocksScreen-deploy.service index 735c3f44..358d1111 100644 --- a/scripts/BlocksScreen-deploy.service +++ b/scripts/BlocksScreen-deploy.service @@ -6,6 +6,8 @@ DefaultDependencies=no Type=oneshot User=root ExecStart=/bin/bash BS_DIR/scripts/install-updater.sh -ExecStartPost=/bin/rm -f BS_PRIMARY_HOME/.config/blockscreen/.run-install-updater +# ExecStopPost also runs when ExecStart fails: a persistent failure must not +# leave the flag set and storm the path unit into its start limit. +ExecStopPost=-/bin/rm -f BS_PRIMARY_HOME/.config/blockscreen/.run-install-updater StandardOutput=journal StandardError=journal diff --git a/scripts/BlocksScreen-start.sh b/scripts/BlocksScreen-start.sh index 1c22bc67..eef1d15e 100755 --- a/scripts/BlocksScreen-start.sh +++ b/scripts/BlocksScreen-start.sh @@ -92,6 +92,16 @@ git -C "$BS_PATH" config core.hooksPath scripts 2>/dev/null || true bs_migrate_moonraker_conf "$_BSENV_HOME/printer_data/config/moonraker.conf" BlocksScreen-start +# Serialize the boot git surgery below with the updater daemon/CLI (their flock): +# the daemon's boot reconcile can heal this same repo concurrently. Proceed after +# 20s rather than hold up the UI; the fd is closed before exec'ing the UI. +_BS_LOCK_DIR="/run/blockscreen" +mkdir -p "$_BS_LOCK_DIR" 2>/dev/null || _BS_LOCK_DIR="$_BSENV_HOME/.cache/blockscreen" +if exec 9>"$_BS_LOCK_DIR/updater.lock" 2>/dev/null; then + flock -w 20 9 2>/dev/null \ + || echo "[BlocksScreen-start] updater lock busy - proceeding without it" +fi + # Remove stale git index lock left by an interrupted update (e.g. power loss during git reset) rm -f "$BS_PATH/.git/index.lock" @@ -148,6 +158,9 @@ if ! "$BSENV/bin/python3.11" -m compileall -q \ git -C "$BS_PATH" reset --hard HEAD 2>/dev/null || true fi +# Release the updater lock now: the exec'd UI must never inherit (and hold) it. +exec 9>&- 2>/dev/null || true + # -- Deferred bootstrap (background) ------------------------------------------ # Heavy / network-dependent setup (apt packages, pip requirements, updater # install, splash-holder unit, splash precompute) runs detached in a transient diff --git a/scripts/bs-bootstrap.sh b/scripts/bs-bootstrap.sh index cb41b8da..5c0ff018 100755 --- a/scripts/bs-bootstrap.sh +++ b/scripts/bs-bootstrap.sh @@ -25,8 +25,8 @@ command -v feh >/dev/null 2>&1 || _missing="$_missing feh" command -v xsetroot >/dev/null 2>&1 || _missing="$_missing x11-xserver-utils" if [ -n "$_missing" ]; then # shellcheck disable=SC2086 - apt-get install -y --quiet $_missing 2>/dev/null \ - || echo "[bs-bootstrap] apt install deferred (offline?):$_missing" + apt-get -o DPkg::Lock::Timeout=60 install -y --quiet $_missing 2>/dev/null \ + || echo "[bs-bootstrap] apt install deferred (offline or dpkg busy):$_missing" fi # 2. Python deps. PyYAML is required by the updater daemon's component loader. diff --git a/scripts/bs-common.sh b/scripts/bs-common.sh index 26d93be0..1c3ee74a 100644 --- a/scripts/bs-common.sh +++ b/scripts/bs-common.sh @@ -6,18 +6,31 @@ bs_migrate_moonraker_conf() { local conf="$1" tag="${2:-bs-common}" patched=false [ -f "$conf" ] || return 0 - cp "$conf" "${conf}.bak" 2>/dev/null || true + # Back up only a sane-looking conf: never overwrite a good .bak with a corrupt one. + if grep -q '^\[server\]' "$conf" 2>/dev/null; then + cp "$conf" "${conf}.bak" 2>/dev/null || true + fi # Gate on the section existing: else the sed no-ops but patched=true restarts moonraker every boot. if ! grep -q "enable_system_updates" "$conf" && grep -q '^\[update_manager\]$' "$conf"; then sed -i '/^\[update_manager\]$/a enable_system_updates: False' "$conf" patched=true echo "[$tag] moonraker.conf: disabled system apt upgrades" fi - if grep -q "managed_services: klipper moonraker" "$conf"; then + # Scope checks to the BlocksScreen section so an unrelated match can't re-patch every boot. + local bs_um_section + bs_um_section=$(sed -n '/^\[update_manager BlocksScreen\]/,/^\[/p' "$conf" 2>/dev/null) + if printf '%s\n' "$bs_um_section" | grep -q "managed_services: klipper moonraker"; then sed -i '/^\[update_manager BlocksScreen\]/,/^\[/ s/managed_services: klipper moonraker/managed_services: BlocksScreen/' "$conf" patched=true echo "[$tag] moonraker.conf: fixed BlocksScreen managed_services" fi + # Old installers wrote primary_branch: master, which does not exist upstream; + # moonraker refuses to manage a repo whose primary branch is gone. + if printf '%s\n' "$bs_um_section" | grep -q "^primary_branch: master$"; then + sed -i '/^\[update_manager BlocksScreen\]/,/^\[/ s/^primary_branch: master$/primary_branch: main/' "$conf" + patched=true + echo "[$tag] moonraker.conf: fixed BlocksScreen primary_branch (master -> main)" + fi if ! grep -q "blocksscreen-single-owner" "$conf"; then bs_disable_overlapping_update_managers "$conf" "$tag" && patched=true fi @@ -37,7 +50,15 @@ bs_ensure_spoolman_moonraker() { local spoolman_dir="${conf%/printer_data/config/moonraker.conf}/Spoolman" [ -d "$spoolman_dir/.venv" ] || return 1 grep -q '^\[spoolman\]' "$conf" && return 1 - printf '\n[spoolman]\nserver: localhost:7912\n' >>"$conf" + # Same-dir temp + atomic rename: a power cut mid-append could truncate the conf. + local tmp + tmp="$(mktemp -p "$(dirname "$conf")" .moonraker.conf.XXXXXX)" || return 1 + if ! { cat "$conf" && printf '\n[spoolman]\nserver: localhost:7912\n'; } >"$tmp"; then + rm -f "$tmp" + return 1 + fi + chmod --reference="$conf" "$tmp" 2>/dev/null || true + mv -f "$tmp" "$conf" echo "[$tag] moonraker.conf: added [spoolman] (server: localhost:7912)" return 0 } diff --git a/scripts/bs-splash-holder.py b/scripts/bs-splash-holder.py index 378a4dee..560f0f0b 100644 --- a/scripts/bs-splash-holder.py +++ b/scripts/bs-splash-holder.py @@ -93,8 +93,13 @@ def _on_sigterm(_sig: int, _frame: object) -> None: signal.signal(signal.SIGTERM, _on_sigterm) -# Switch to tty8 immediately - VT switch does not require fb0. -_activate_tty8() +# Switch to tty8 immediately - VT switch does not require fb0. Skip when X is +# already up (first-install start lands mid-session): stealing the VT from the +# live GUI on tty7 would look like a bricked screen until a power cycle. +if os.path.exists("/tmp/.X11-unix/X0"): # nosec B108 - canonical X11 socket path + _log("X session present - not activating tty8") +else: + _activate_tty8() # Clear tty8 and hide the cursor (no bare text): the screen stays black until the # splash image is written to fb0 below, so the user only ever sees the logo splash. diff --git a/scripts/install-updater.sh b/scripts/install-updater.sh index 929134f5..2bdc7d73 100755 --- a/scripts/install-updater.sh +++ b/scripts/install-updater.sh @@ -20,13 +20,22 @@ _BSENV_USER="blocks" _BSENV_HOME=$(getent passwd "$_BSENV_USER" | cut -d: -f6) BSENV="${BLOCKSSCREEN_VENV:-${_BSENV_HOME}/.BlocksScreen-env}" +# Venv-mutating commands run as blocks: root-owned dists break later pip-as-blocks runs. +_as_blocks() { if [ "$(id -u)" = "0" ]; then runuser -u "$_BSENV_USER" -- "$@"; else "$@"; fi; } +# Atomic root install: a power cut must never leave a truncated-but-present file +# (the [ -f ] self-heal guards would then never rewrite it). +_install_atomic() { + local mode="$1" src="$2" dst="$3" + sudo install -m "$mode" "$src" "${dst}.new" && sudo mv -Tf "${dst}.new" "$dst" +} + echo_info "Installing D-Bus policy ..." -sudo cp "$SCRIPT_PATH/com.blockscreen.Updater.conf" /etc/dbus-1/system.d/ +_install_atomic 0644 "$SCRIPT_PATH/com.blockscreen.Updater.conf" /etc/dbus-1/system.d/com.blockscreen.Updater.conf sudo systemctl reload dbus || true echo_ok "D-Bus policy installed" echo_info "Installing D-Bus activation file ..." -sudo cp "$SCRIPT_PATH/com.blockscreen.Updater.service" /usr/share/dbus-1/system-services/ +_install_atomic 0644 "$SCRIPT_PATH/com.blockscreen.Updater.service" /usr/share/dbus-1/system-services/com.blockscreen.Updater.service echo_ok "D-Bus activation file installed" echo_info "Installing bs-apt-helper (root-owned apt wrapper) ..." @@ -40,7 +49,8 @@ SERVICE=$(cat "$SCRIPT_PATH/BlocksScreen-updater.service") SERVICE=${SERVICE//BS_DIR/$BS_PATH} SERVICE=${SERVICE//BSENV/$BSENV} SERVICE=${SERVICE//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$SERVICE" | sudo tee /etc/systemd/system/BlocksScreen-updater.service >/dev/null +echo "$SERVICE" | sudo tee /etc/systemd/system/.BlocksScreen-updater.service.new >/dev/null +sudo mv -Tf /etc/systemd/system/.BlocksScreen-updater.service.new /etc/systemd/system/BlocksScreen-updater.service sudo systemctl daemon-reload sudo systemctl enable BlocksScreen-updater.service sudo systemctl restart BlocksScreen-updater.service || true @@ -49,7 +59,8 @@ echo_ok "BlocksScreen-updater service installed and started" echo_info "Installing Spoolman service unit (inactive until the updater provisions it) ..." SPOOLMAN_SVC=$(cat "$SCRIPT_PATH/Spoolman.service") SPOOLMAN_SVC=${SPOOLMAN_SVC//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$SPOOLMAN_SVC" | sudo tee /etc/systemd/system/Spoolman.service >/dev/null +echo "$SPOOLMAN_SVC" | sudo tee /etc/systemd/system/.Spoolman.service.new >/dev/null +sudo mv -Tf /etc/systemd/system/.Spoolman.service.new /etc/systemd/system/Spoolman.service sudo systemctl daemon-reload echo_ok "Spoolman service unit installed" @@ -81,8 +92,10 @@ else fi # Daemon self-restart target; never in components.yaml. _emit_svc_rules BlocksScreen-updater.service -# Spoolman is provisioned on demand; enable rule needed for its first clean start. +# Spoolman is provisioned on demand; enable rules needed for its first clean start +# (hooks/Spoolman.sh uses `enable --now`; sudoers args must match exactly). printf 'blocks ALL=(ALL) NOPASSWD: /usr/bin/systemctl enable Spoolman.service\n' >>"$SUDOERS_TMP" +printf 'blocks ALL=(ALL) NOPASSWD: /usr/bin/systemctl enable --now Spoolman.service\n' >>"$SUDOERS_TMP" if sudo visudo -cf "$SUDOERS_TMP" >/dev/null 2>&1; then sudo install -m 0440 "$SUDOERS_TMP" "$SUDOERS_FILE" echo_ok "Sudoers rules installed" @@ -179,10 +192,12 @@ echo_info "Installing BlocksScreen-deploy path unit (sudo-free hook trigger) ... DEPLOY_SVC=$(cat "$SCRIPT_PATH/BlocksScreen-deploy.service") DEPLOY_SVC=${DEPLOY_SVC//BS_DIR/$BS_PATH} DEPLOY_SVC=${DEPLOY_SVC//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$DEPLOY_SVC" | sudo tee /etc/systemd/system/BlocksScreen-deploy.service >/dev/null +echo "$DEPLOY_SVC" | sudo tee /etc/systemd/system/.BlocksScreen-deploy.service.new >/dev/null +sudo mv -Tf /etc/systemd/system/.BlocksScreen-deploy.service.new /etc/systemd/system/BlocksScreen-deploy.service DEPLOY_PATH=$(cat "$SCRIPT_PATH/BlocksScreen-deploy.path") DEPLOY_PATH=${DEPLOY_PATH//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$DEPLOY_PATH" | sudo tee /etc/systemd/system/BlocksScreen-deploy.path >/dev/null +echo "$DEPLOY_PATH" | sudo tee /etc/systemd/system/.BlocksScreen-deploy.path.new >/dev/null +sudo mv -Tf /etc/systemd/system/.BlocksScreen-deploy.path.new /etc/systemd/system/BlocksScreen-deploy.path sudo systemctl daemon-reload sudo systemctl enable --now BlocksScreen-deploy.path echo_ok "BlocksScreen-deploy path unit installed" @@ -207,23 +222,23 @@ echo_ok "post-merge hook installed" echo_info "Installing Python requirements ..." # Best-effort pip self-update; must not block the requirements install below. -"$BSENV/bin/pip" install --quiet --upgrade pip 2>/dev/null || true -apt-get install -y --quiet libsystemd-dev python3-dev 2>/dev/null || true +_as_blocks "$BSENV/bin/pip" install --quiet --upgrade pip 2>/dev/null || true +sudo apt-get -o DPkg::Lock::Timeout=60 install -y --quiet libsystemd-dev python3-dev 2>/dev/null || true # xsetroot is used as belt-and-suspenders cursor hiding alongside the Xorg -nocursor server flag. -sudo apt-get install -y --quiet x11-xserver-utils 2>/dev/null || true +sudo apt-get -o DPkg::Lock::Timeout=60 install -y --quiet x11-xserver-utils 2>/dev/null || true # sdbus pinned; --no-binary needs libsystemd-dev; only-if-needed skips satisfied. -"$BSENV/bin/pip" install --quiet --only-binary :all: --no-binary sdbus,sdbus-networkmanager \ +_as_blocks "$BSENV/bin/pip" install --quiet --only-binary :all: --no-binary sdbus,sdbus-networkmanager \ --upgrade-strategy=only-if-needed \ -r "$BS_PATH/scripts/requirements.txt" || true echo_ok "Python requirements installed" # uv: Spoolman's dependency installer (run-from-source); provide it for the hook. echo_info "Ensuring uv is available for Spoolman provisioning ..." -"$BSENV/bin/pip" install --quiet --upgrade uv 2>/dev/null || true +_as_blocks "$BSENV/bin/pip" install --quiet --upgrade uv 2>/dev/null || true echo_ok "uv ready" echo_info "Pre-generating framebuffer splash cache ..." -"$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true +_as_blocks "$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true echo_ok "Splash cache ready" echo_ok "BlocksScreen updater setup complete" diff --git a/scripts/install.sh b/scripts/install.sh index 148c9a9e..53b3c46b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -413,11 +413,11 @@ function add_moonraker_update() { [update_manager BlocksScreen] type: git_repo -primary_branch: master +primary_branch: main path: $BS_PATH origin: https://github.com/BlocksTechnology/BlocksScreen.git is_system_service: True -managed_services: klipper moonraker +managed_services: BlocksScreen virtualenv: $BSENV requirements: scripts/requirements.txt system_dependencies: scripts/system-dependencies.json diff --git a/scripts/post-merge b/scripts/post-merge index b10c99f7..8eb5390e 100755 --- a/scripts/post-merge +++ b/scripts/post-merge @@ -8,6 +8,10 @@ _BSENV_USER="blocks" _BSENV_HOME=$(getent passwd "$_BSENV_USER" | cut -d: -f6) BSENV="${BLOCKSSCREEN_VENV:-${_BSENV_HOME}/.BlocksScreen-env}" +# A root pull must not leave root-owned dists in the blocks venv (breaks later pip runs). +_as_blocks() { if [ "$(id -u)" = "0" ]; then runuser -u "$_BSENV_USER" -- "$@"; else "$@"; fi; } +_own() { [ "$(id -u)" = "0" ] && chown "$_BSENV_USER" "$1" 2>/dev/null || true; } + # shellcheck source=/dev/null if [ -f "$SCRIPT_PATH/bs-common.sh" ]; then . "$SCRIPT_PATH/bs-common.sh"; fi @@ -23,12 +27,13 @@ SENTINEL="$BSENV/.blockscreen-reqs-hash" ATTEMPT="$BSENV/.blockscreen-reqs-attempt" if [ ! -f "$SENTINEL" ] || [ "$(cat "$SENTINEL")" != "$REQS_HASH" ]; then echo "[post-merge] requirements changed - pre-installing deps (this may take a few minutes) ..." - if "$BSENV/bin/pip" install --quiet \ + if _as_blocks "$BSENV/bin/pip" install --quiet \ --only-binary :all: \ --no-binary sdbus,sdbus-networkmanager \ --upgrade-strategy=only-if-needed \ -r "$BS_PATH/scripts/requirements.txt"; then echo "$REQS_HASH" >"$SENTINEL" + _own "$SENTINEL" rm -f "$ATTEMPT" 2>/dev/null || true echo "[post-merge] deps installed" else @@ -38,9 +43,11 @@ if [ ! -f "$SENTINEL" ] || [ "$(cat "$SENTINEL")" != "$REQS_HASH" ]; then case "$_cnt" in '' | *[!0-9]*) _cnt=0 ;; esac if [ "$_prev" = "$REQS_HASH" ]; then _cnt=$((_cnt + 1)); else _cnt=1; fi echo "$REQS_HASH $_cnt" >"$ATTEMPT" + _own "$ATTEMPT" if [ "$_cnt" -ge 3 ]; then echo "[post-merge] deps failed ${_cnt}x; giving up to stop thrash (venv left as-is)" echo "$REQS_HASH" >"$SENTINEL" + _own "$SENTINEL" fi fi fi @@ -55,7 +62,7 @@ if [ ! -f /etc/systemd/system/BlocksScreen-updater.service ]; then fi # 4. Precompute splash cache so the first Qt restart shows the right logo. -"$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true +_as_blocks "$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true # Under the daemon's own pull a restart here aborts the batch: defer via sentinel. diff --git a/tests/scripts/test_moonraker_ownership.py b/tests/scripts/test_moonraker_ownership.py index eab5878a..7550eb7a 100644 --- a/tests/scripts/test_moonraker_ownership.py +++ b/tests/scripts/test_moonraker_ownership.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import subprocess from pathlib import Path @@ -62,3 +63,65 @@ def test_idempotent(tmp_path: Path) -> None: first = _run(conf) second = _run(conf) assert first == second # no double-comment on re-run + + +_MIGRATE_CONF = """\ +[server] +host: 0.0.0.0 + +[update_manager] +enable_system_updates: False + +[update_manager BlocksScreen] +type: git_repo +primary_branch: master +managed_services: klipper moonraker + +[update_manager other] +primary_branch: master +""" + + +def _migrate(conf: Path) -> str: + # Shim sudo: the function ends with `sudo systemctl restart moonraker`; real + # sudo would prompt on a dev TTY (test hang) or bounce services on the printer. + shim = conf.parent / "bin" + shim.mkdir(exist_ok=True) + fake_sudo = shim / "sudo" + fake_sudo.write_text("#!/bin/sh\nexit 0\n") + fake_sudo.chmod(0o755) + env = {**os.environ, "PATH": f"{shim}:{os.environ.get('PATH', '')}"} + subprocess.run( + [ + "bash", + "-c", + f'. "{_FN}"; bs_migrate_moonraker_conf "{conf}" test', + ], + check=False, + capture_output=True, + text=True, + env=env, + timeout=20, + ) + return conf.read_text() + + +def test_migrate_fixes_blocksscreen_branch_and_services(tmp_path: Path) -> None: + conf = tmp_path / "moonraker.conf" + conf.write_text(_MIGRATE_CONF) + out = _migrate(conf) + bs = out.split("[update_manager BlocksScreen]")[1].split("[update_manager other]")[ + 0 + ] + assert "primary_branch: main" in bs # origin/master does not exist upstream + assert "managed_services: BlocksScreen" in bs + # Patches are scoped to the BlocksScreen section: others keep their values. + assert "primary_branch: master" in out.split("[update_manager other]")[1] + + +def test_migrate_is_idempotent(tmp_path: Path) -> None: + conf = tmp_path / "moonraker.conf" + conf.write_text(_MIGRATE_CONF) + first = _migrate(conf) + second = _migrate(conf) + assert first == second diff --git a/tests/updater/test_components_unit.py b/tests/updater/test_components_unit.py index b119fd1a..983a7aed 100644 --- a/tests/updater/test_components_unit.py +++ b/tests/updater/test_components_unit.py @@ -438,3 +438,40 @@ def test_defaults_when_absent(self): assert cfg is not None assert cfg.url is None assert cfg.install_if_missing is False + + +def test_invalid_reset_mode_coerced_to_hard(caplog): + bad_yaml = """ +components: + - name: klipper + type: git + path: ~/klipper + reset_mode: Hard + order: 10 +""" + with ( + patch("builtins.open", _mock_load(bad_yaml)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + configs, _ = load_components() + assert configs[-1].reset_mode == "hard" # typo must not become the soft path + assert any("invalid reset_mode" in r.message for r in caplog.records) + + +def test_non_dict_component_entry_skipped(caplog): + bad_yaml = """ +components: + - just-a-string + - name: klipper + type: git + path: ~/klipper + order: 10 +""" + with ( + patch("builtins.open", _mock_load(bad_yaml)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + configs, _ = load_components() + assert [c.name for c in configs if c.kind == "git"] == ["klipper"] diff --git a/tests/updater/test_executor_unit.py b/tests/updater/test_executor_unit.py index 0121712c..e4223006 100644 --- a/tests/updater/test_executor_unit.py +++ b/tests/updater/test_executor_unit.py @@ -16,6 +16,7 @@ _make_clean_env, _repair_corrupt_head, _remove_broken_loose_ref, + git_default_branch, _run, apt_update, apt_upgrade, @@ -1298,3 +1299,32 @@ async def test_has_corruption_detects_inflate_error(self): "asyncio.create_subprocess_exec", new_callable=AsyncMock, return_value=proc ): assert await git_has_corruption(Path("/x")) is True + + +class TestGitDefaultBranch: + """git_default_branch: HEAD symref, else origin/HEAD target, else master.""" + + @pytest.mark.asyncio + async def test_prefers_head_symref(self, tmp_path): + with patch( + "updater.executor._run", return_value=(True, "master\n") + ) as mock_run: + assert await git_default_branch(tmp_path) == "master" + assert mock_run.call_count == 1 + + @pytest.mark.asyncio + async def test_falls_back_to_origin_head(self, tmp_path): + with patch( + "updater.executor._run", + side_effect=[(False, ""), (True, "origin/master\n")], + ): + assert await git_default_branch(tmp_path) == "master" + + @pytest.mark.asyncio + async def test_last_resort_is_master(self, tmp_path): + with patch("updater.executor._run", side_effect=[(False, ""), (False, "")]): + assert await git_default_branch(tmp_path) == "master" + + @pytest.mark.asyncio + async def test_none_path_returns_master(self): + assert await git_default_branch(None) == "master" diff --git a/tests/updater/test_service_unit.py b/tests/updater/test_service_unit.py index 48416eba..ce906500 100644 --- a/tests/updater/test_service_unit.py +++ b/tests/updater/test_service_unit.py @@ -205,12 +205,14 @@ def mock_wait_active(self): yield @pytest.mark.asyncio - async def test_rollback_on_fetch_failure(self): + async def test_fetch_failure_reports_network_without_rollback(self): + # Nothing is checked out before the fetch: no revert, no service restart. cb = MagicMock() with ( patch("pathlib.Path.exists", return_value=True), patch("updater.service.git_get_hash", return_value="abc123"), patch("updater.service.git_fetch", return_value=(False, "timeout")), + patch("updater.service.git_has_corruption", return_value=False), patch( "updater.service.git_reset_to_hash", return_value=(True, "") ) as mock_reset, @@ -219,8 +221,9 @@ async def test_rollback_on_fetch_failure(self): svc = UpdateService(callback=cb) result = await svc.update_component("klipper") assert result is False - mock_reset.assert_called_once() - cb.on_rollback.assert_called_once_with("klipper", True) + mock_reset.assert_not_called() + cb.on_rollback.assert_not_called() + cb.on_error.assert_called_once_with("klipper", "network") cb.on_component_done.assert_called_once_with("klipper", False) @pytest.mark.asyncio @@ -424,6 +427,7 @@ async def test_branch_hard_checks_out_then_resets_target(self): patch("pathlib.Path.exists", return_value=True), patch("updater.service.git_get_hash", return_value="oldhash"), patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch( "updater.service.git_checkout", side_effect=lambda *a, **k: call_order.append("checkout") or (True, ""), @@ -616,7 +620,7 @@ async def test_existing_git_go_to_one_atomic_batch_in_order(self, tmp_path): ] seen: list[list[str]] = [] with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_apt_update") as mock_apt, patch( "updater.service.UpdateService._run_git_batch", @@ -641,7 +645,7 @@ async def test_missing_opted_in_goes_to_provision_not_batch(self, tmp_path): install_if_missing=True, ) with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_git_batch") as mock_batch, patch( "updater.service.UpdateService._provision_component" @@ -656,7 +660,7 @@ async def test_missing_opted_in_goes_to_provision_not_batch(self, tmp_path): @pytest.mark.asyncio async def test_empty_batch_skips_git(self, tmp_path): with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_git_batch") as mock_batch, ): svc = UpdateService() @@ -673,7 +677,7 @@ async def test_dead_branch_dropped_not_aborting_batch(self, tmp_path): good = self._git(tmp_path, "BlocksScreen", 99) good.branch = "dev" with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_git_batch") as mock_batch, patch( "updater.service.git_ref_hash", @@ -687,7 +691,7 @@ async def test_dead_branch_dropped_not_aborting_batch(self, tmp_path): class TestAtomicBatch: - """_run_git_batch: stage all → deps → hooks → restart each service once; all-or-nothing.""" + """_run_git_batch: phased stage/deps/hooks/restart; failures drop per component.""" @pytest.fixture(autouse=True) def _mocks(self): @@ -766,7 +770,8 @@ async def test_inflight_marker_written_then_cleared_on_success(self, tmp_path): assert not svc._inflight_path.exists() # cleared on commit @pytest.mark.asyncio - async def test_inflight_marker_cleared_on_abort(self, tmp_path): + async def test_inflight_marker_cleared_on_partial_failure(self, tmp_path): + # One component fails and reverts cleanly; the survivor commits: no marker left. cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) svc._inflight_path = tmp_path / "inflight.json" @@ -780,9 +785,13 @@ async def test_inflight_marker_cleared_on_abort(self, tmp_path): return_value=(True, ""), ), patch("updater.service.git_reset_to_hash", return_value=(True, "")), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): await svc._run_git_batch(comps) - assert not svc._inflight_path.exists() # cleared by _abort_batch + assert not svc._inflight_path.exists() # settled: nothing pending @pytest.mark.asyncio async def test_restart_ui_component_also_restarts_ui(self, tmp_path): @@ -1005,7 +1014,8 @@ async def test_cancel_after_commit_does_not_revert(self, tmp_path): assert all(c.args[1] is True for c in cb.on_component_done.call_args_list) @pytest.mark.asyncio - async def test_stage_failure_reverts_touched_and_does_not_restart(self, tmp_path): + async def test_stage_failure_drops_only_failed_component(self, tmp_path): + # c1 fails staging: it is reverted and dropped; c0 continues and commits. cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) with ( @@ -1016,19 +1026,27 @@ async def test_stage_failure_reverts_touched_and_does_not_restart(self, tmp_path patch( "updater.service.git_reset_to_hash", return_value=(True, "") ) as mock_reset, - patch("updater.service.restart_service") as mock_rs, + patch("updater.service.run_hook", return_value=(True, "")), + patch( + "updater.service.restart_service", return_value=(True, "") + ) as mock_rs, + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): - await svc._run_git_batch(comps) - assert mock_reset.call_count == 2 # both touched repos reverted - mock_rs.assert_not_called() - assert cb.on_rollback.call_count == 2 - assert all(c.args[1] is True for c in cb.on_rollback.call_args_list) + result = await svc._run_git_batch(comps) + assert result is False # a drop is still a reported failure + mock_reset.assert_called_once() # only the failed repo is reverted + mock_rs.assert_called_once() # the survivor's service still restarts + cb.on_rollback.assert_called_once_with("c1", True) + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"c0": True, "c1": False} @pytest.mark.asyncio - async def test_abort_reports_rollback_false_when_revert_fails(self, tmp_path): - """A failed git revert during abort must report rollback as unsuccessful.""" + async def test_failed_revert_keeps_component_in_inflight_marker(self, tmp_path): + """A failed revert reports rollback False and stays marked for boot retry.""" cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) + svc._inflight_path = tmp_path / "inflight.json" with ( patch( "updater.service.UpdateService._stage_component", @@ -1038,14 +1056,18 @@ async def test_abort_reports_rollback_false_when_revert_fails(self, tmp_path): "updater.service.git_reset_to_hash", return_value=(False, "disk full"), ), - patch("updater.service.restart_service"), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): await svc._run_git_batch(comps) - assert cb.on_rollback.call_count == 2 - assert all(c.args[1] is False for c in cb.on_rollback.call_args_list) + cb.on_rollback.assert_called_once_with("c1", False) + marker = json.loads(svc._inflight_path.read_text()) + assert set(marker) == {"c1"} # survivor c0 must not be boot-reverted @pytest.mark.asyncio - async def test_deps_failure_reverts_all_no_restart(self, tmp_path): + async def test_deps_failure_drops_only_failed_component(self, tmp_path): cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) with ( @@ -1058,17 +1080,27 @@ async def test_deps_failure_reverts_all_no_restart(self, tmp_path): "updater.service.UpdateService._install_dependencies", side_effect=[(True, ""), (False, "pip boom")], ), - patch("updater.service.restart_service") as mock_rs, + patch("updater.service.run_hook", return_value=(True, "")), + patch( + "updater.service.restart_service", return_value=(True, "") + ) as mock_rs, + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): - await svc._run_git_batch(comps) - assert mr.call_count == 2 - mock_rs.assert_not_called() - assert cb.on_error.call_args[0][1] == "deps" + result = await svc._run_git_batch(comps) + assert result is False + mr.assert_called_once() # only c1 reverted + mock_rs.assert_called_once() # c0 still restarted + assert cb.on_error.call_args[0] == ("c1", "deps") + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"c0": True, "c1": False} @pytest.mark.asyncio - async def test_restart_failure_reverts_and_rerestarts_bounced(self, tmp_path): - # Two distinct services: first restarts OK, second fails → revert git and - # re-restart the first so it returns on the old code. + async def test_restart_failure_drops_service_members_and_recovers_it( + self, tmp_path + ): + # Two distinct services: svc-a restarts OK and commits; svc-b fails, its + # component reverts and svc-b is re-restarted onto the old code. cb = MagicMock() c0 = self._git(tmp_path, "c0", 2, "svc-a.service") c1 = self._git(tmp_path, "c1", 3, "svc-b.service") @@ -1094,18 +1126,20 @@ async def fake_restart(name): patch("updater.service.run_hook", return_value=(True, "")), patch("updater.service.restart_service", side_effect=fake_restart), patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): - await svc._run_git_batch([c0, c1]) - assert mr.call_count == 2 # both repos reverted - # Initial: svc-a ok, svc-b fails. Abort re-restarts BOTH bounced services - # (incl. the failed svc-b) onto the reverted code. + result = await svc._run_git_batch([c0, c1]) + assert result is False + mr.assert_called_once() # only c1 reverted; c0's update stands + # svc-a ok; svc-b fails then is re-restarted once onto the old code. assert restart_calls == [ "svc-a.service", "svc-b.service", - "svc-a.service", "svc-b.service", ] - assert cb.on_error.call_args[0][1] == "restart" + assert cb.on_error.call_args[0] == ("c1", "restart") + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"c0": True, "c1": False} @pytest.mark.asyncio async def test_cancel_mid_batch_reverts_staged(self, tmp_path): @@ -1725,6 +1759,7 @@ async def test_stage_reclones_when_repair_fails(self, tmp_path): patch("updater.service.git_fetch", return_value=(False, "corrupt")), patch("updater.service.git_has_corruption", return_value=True), patch("updater.service.git_repair", return_value=(False, "unrepairable")), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch.object( UpdateService, "_reclone_component", return_value=True ) as mock_reclone, @@ -1742,8 +1777,13 @@ async def test_reconcile_reclones_when_repair_fails(self, tmp_path): comp = self._comp(tmp_path) with ( patch("updater.service.process_lock", lambda: nullcontext(True)), - patch("updater.service.git_get_hash", side_effect=["", "abc123"]), + # gate "", post-reclone check, post-reclone hook hash + patch( + "updater.service.git_get_hash", + side_effect=["", "abc123", "abc123"], + ), patch("updater.service.git_repair", return_value=(False, "unrepairable")), + patch("updater.service.run_hook", return_value=(True, "")) as mock_hook, patch.object( UpdateService, "_reclone_component", return_value=True ) as mock_reclone, @@ -1752,6 +1792,8 @@ async def test_reconcile_reclones_when_repair_fails(self, tmp_path): svc._components = [comp] await svc.reconcile() mock_reclone.assert_awaited_once() + # A boot reclone drops in-repo artifacts: the hook must rebuild them. + mock_hook.assert_awaited_once() class TestReconcile: @@ -1788,7 +1830,8 @@ async def test_unreadable_head_triggers_repair(self, tmp_path): svc = UpdateService() svc._components = [comp] await svc.reconcile() - mock_repair.assert_called_once_with(comp.path, "main") + # No configured branch: git_repair derives the repo's own default. + mock_repair.assert_called_once_with(comp.path, None) @pytest.mark.asyncio async def test_repair_failure_falls_back_to_prev_hash_reset(self, tmp_path): @@ -1875,7 +1918,7 @@ async def test_rollback_false_when_service_not_active(self): class TestPreflightFetch: - """update_all fetches every component up-front; a network drop aborts cleanly.""" + """update_all fetches up-front; an offline repo is dropped, the rest continue.""" def _git(self, tmp_path: Path, name: str) -> ComponentConfig: path = tmp_path / name @@ -1888,23 +1931,21 @@ async def test_all_ok_records_fetch_times(self, tmp_path): with patch("updater.service.git_fetch", return_value=(True, "")): svc = UpdateService() svc._components = comps - ok = await svc._preflight_fetch(comps, comps) - assert ok is True + offline = await svc._preflight_fetch(comps) + assert offline == set() assert set(svc._fetch_times) == {"klipper", "moonraker"} @pytest.mark.asyncio - async def test_offline_aborts_and_reports_all(self, tmp_path): + async def test_offline_components_are_returned(self, tmp_path): comps = [self._git(tmp_path, "klipper"), self._git(tmp_path, "moonraker")] - cb = MagicMock() with ( patch("updater.service.git_fetch", return_value=(False, "timeout")), patch("updater.service.git_has_corruption", return_value=False), ): - svc = UpdateService(callback=cb) + svc = UpdateService() svc._components = comps - ok = await svc._preflight_fetch(comps, comps) - assert ok is False - assert cb.on_error.call_count == 2 + offline = await svc._preflight_fetch(comps) + assert offline == {"klipper", "moonraker"} @pytest.mark.asyncio async def test_corrupt_repo_is_not_treated_as_offline(self, tmp_path): @@ -1915,8 +1956,8 @@ async def test_corrupt_repo_is_not_treated_as_offline(self, tmp_path): ): svc = UpdateService() svc._components = comps - ok = await svc._preflight_fetch(comps, comps) - assert ok is True + offline = await svc._preflight_fetch(comps) + assert offline == set() assert "klipper" not in svc._fetch_times # left for the per-component flow @pytest.mark.asyncio @@ -1926,21 +1967,30 @@ async def test_recently_fetched_component_is_not_refetched(self, tmp_path): svc = UpdateService() svc._components = [comp] svc._fetch_times["klipper"] = time.monotonic() - ok = await svc._preflight_fetch([comp], [comp]) - assert ok is True + offline = await svc._preflight_fetch([comp]) + assert offline == set() mock_fetch.assert_not_called() @pytest.mark.asyncio - async def test_update_all_aborts_apply_when_preflight_fails(self, tmp_path): - comps = [self._git(tmp_path, "klipper")] + async def test_update_all_drops_only_offline_components(self, tmp_path): + offline_comp = self._git(tmp_path, "klipper") + online_comp = self._git(tmp_path, "moonraker") + cb = MagicMock() with ( - patch.object(UpdateService, "_preflight_fetch", return_value=False), - patch.object(UpdateService, "update_component") as mock_update, + patch.object(UpdateService, "_preflight_fetch", return_value={"klipper"}), + patch.object( + UpdateService, "_run_git_batch", return_value=True + ) as mock_batch, ): - svc = UpdateService() - svc._components = comps - await svc.update_all({"klipper"}) - mock_update.assert_not_called() + svc = UpdateService(callback=cb) + svc._components = [offline_comp, online_comp] + ok = await svc.update_all({"klipper", "moonraker"}) + assert ok is False # the offline drop is a reported failure + cb.on_error.assert_called_once_with( + "klipper", "network error during pre-flight fetch" + ) + batched = [c.name for c in mock_batch.call_args.args[0]] + assert batched == ["moonraker"] class TestDeferredRestart: @@ -2254,15 +2304,25 @@ async def test_single_insecure_remote_clears_marker(self, tmp_path): cb.on_error.assert_called_once_with("klipper", "insecure remote") @pytest.mark.asyncio - async def test_single_reset_failure_clears_marker(self, tmp_path): + async def test_single_reset_failure_rolls_back_and_clears_marker(self, tmp_path): + # The checkout before the failing reset may have moved the tree: the + # component must roll back to prev_hash, then the marker clears. cb = MagicMock() + + async def fake_reset(path, ref=""): + # Target reset (origin/) fails; the prev_hash rollback works. + return (False, "boom") if str(ref).startswith("origin/") else (True, "") + with ( patch("pathlib.Path.exists", return_value=True), patch("updater.service.git_get_hash", return_value="a" * 40), patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch("updater.service.git_get_current_branch", return_value="master"), patch("updater.service.git_checkout", return_value=(True, "")), - patch("updater.service.git_reset_to_hash", return_value=(False, "boom")), + patch("updater.service.git_reset_to_hash", side_effect=fake_reset), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), patch( "updater.service._assert_https_remote", return_value=(True, "https://github.com/test/repo"), @@ -2272,6 +2332,7 @@ async def test_single_reset_failure_clears_marker(self, tmp_path): assert await svc.update_component("klipper") is False assert not (tmp_path / "inflight.json").exists() cb.on_error.assert_called_once_with("klipper", "reset") + cb.on_rollback.assert_called_once_with("klipper", True) @pytest.mark.asyncio async def test_batch_insecure_remote_clears_marker(self, tmp_path): @@ -2333,7 +2394,7 @@ async def fake_fetch(path, **kw): with patch("updater.service.git_fetch", side_effect=fake_fetch): svc = UpdateService(callback=MagicMock()) - assert await svc._preflight_fetch([good, bad], [good, bad]) is True + assert await svc._preflight_fetch([good, bad]) == set() assert fetched == ["klipper"] @pytest.mark.asyncio @@ -2431,3 +2492,171 @@ async def test_honors_configured_apt_excludes(self): ): await svc.background_apt_upgrade() mock_up.assert_awaited_once_with(exclude=("^linux-image", "^firmware-")) + + +class TestReviewHardeningFixes: + """Regressions for the 2026-07 pre-main review fixes.""" + + def _git( + self, tmp_path: Path, name: str, service: str = "klipper.service" + ) -> ComponentConfig: + path = tmp_path / name + (path / ".git").mkdir(parents=True) + return ComponentConfig(name=name, kind="git", path=path, service=service) + + def _seeded_svc(self, tmp_path: Path, comp: ComponentConfig) -> UpdateService: + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + svc._write_state({comp.name: {"last_good": "b" * 40, "golden": "c" * 40}}) + return svc + + def _happy_patches(self, stack: ExitStack) -> None: + for target, kw in ( + ("updater.service.git_get_hash", {"return_value": "a" * 40}), + ( + "updater.service._assert_https_remote", + {"return_value": (True, "https://x")}, + ), + ( + "updater.service.UpdateService._stage_component", + {"return_value": (True, "")}, + ), + ( + "updater.service.UpdateService._install_dependencies", + {"return_value": (True, "")}, + ), + ("updater.service.run_hook", {"return_value": (True, "")}), + ("updater.service.restart_service_noblock", {"new": AsyncMock()}), + ): + stack.enter_context(patch(target, **kw)) + stack.enter_context( + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()) + ) + + @pytest.mark.asyncio + async def test_batch_update_preserves_self_heal_anchors(self, tmp_path): + comp = self._git(tmp_path, "BlocksScreen", "BlocksScreen.service") + svc = self._seeded_svc(tmp_path, comp) + with ExitStack() as stack: + self._happy_patches(stack) + assert await svc._run_git_batch([comp]) is True + state = svc._read_state()["BlocksScreen"] + assert state["last_good"] == "b" * 40 # anchors survive the update + assert state["golden"] == "c" * 40 + assert state["prev_hash"] == "a" * 40 + + @pytest.mark.asyncio + async def test_single_update_preserves_self_heal_anchors(self, tmp_path): + comp = self._git(tmp_path, "BlocksScreen", "BlocksScreen.service") + svc = self._seeded_svc(tmp_path, comp) + with ExitStack() as stack: + self._happy_patches(stack) + assert await svc._run_git_update(comp) is True + state = svc._read_state()["BlocksScreen"] + assert state["last_good"] == "b" * 40 + assert state["golden"] == "c" * 40 + + @pytest.mark.asyncio + async def test_single_update_dead_branch_errors_without_stranding(self, tmp_path): + comp = self._git(tmp_path, "klipper") + comp.branch = "wip/deleted" + cb = MagicMock() + svc = UpdateService(callback=cb) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + with ( + patch("updater.service.git_get_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://x"), + ), + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value=""), + patch("updater.service.git_checkout") as mock_checkout, + ): + assert await svc._run_git_update(comp) is False + mock_checkout.assert_not_called() # never switched onto the dead branch + assert not (tmp_path / "inflight.json").exists() + assert "not found" in cb.on_error.call_args[0][1] + + @pytest.mark.asyncio + async def test_supervise_ui_survives_handler_exception(self): + svc = UpdateService() + seen: list[int] = [] + + async def fake_nrestarts(_svc=None): + seen.append(1) + if len(seen) >= 2: + raise asyncio.CancelledError # end the loop after the second pass + return 99 + + with ( + patch("updater.service._NRESTARTS_POLL_INTERVAL_S", 0.001), + patch( + "updater.service.get_service_nrestarts", side_effect=fake_nrestarts + ), + patch.object(UpdateService, "_check_crash_loop", return_value=True), + patch.object( + UpdateService, "_handle_crash_loop", side_effect=RuntimeError("boom") + ), + ): + with pytest.raises(asyncio.CancelledError): + await svc.supervise_ui() + assert len(seen) == 2 # the loop survived the first pass exception + + @pytest.mark.asyncio + async def test_recover_tolerates_corrupt_state_shapes(self, tmp_path): + comp = self._git(tmp_path, "klipper") + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._state_path.write_text("[1, 2, 3]") # valid JSON, wrong shape + assert await svc.recover("klipper") is False + svc._state_path.write_text("{\"klipper\": \"not-a-dict\"}") + assert await svc.recover("klipper") is False + + @pytest.mark.asyncio + async def test_revert_inflight_tolerates_corrupt_values(self, tmp_path): + comp = self._git(tmp_path, "klipper") + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + svc._inflight_path = tmp_path / "inflight.json" + svc._inflight_path.write_text("{\"klipper\": 42, \"ghost\": null}") + with patch("updater.service.git_reset_to_hash") as mock_reset: + await svc._revert_inflight() # must not raise + mock_reset.assert_not_called() + + @pytest.mark.asyncio + async def test_reclone_invalidates_cached_pip_path(self, tmp_path): + comp = self._git(tmp_path, "Spoolman", "Spoolman.service") + comp.url = "https://github.com/test/spoolman" + svc = UpdateService() + svc._component_pip_cache[str(comp.path)] = "/stale/.venv/bin/pip" + + async def fake_clone(url, dest, branch): + Path(dest).mkdir() + (Path(dest) / ".git").mkdir() + return (True, "") + + with patch("updater.service.git_clone", side_effect=fake_clone): + assert await svc._reclone_component(comp) is True + assert str(comp.path) not in svc._component_pip_cache + + @pytest.mark.asyncio + async def test_update_all_returns_true_only_when_all_succeed(self, tmp_path): + good = self._git(tmp_path, "klipper") + for batch_result, expected in ((True, True), (False, False)): + with ( + patch.object( + UpdateService, "_preflight_fetch", return_value=set() + ), + patch.object( + UpdateService, "_run_git_batch", return_value=batch_result + ), + ): + svc = UpdateService() + svc._components = [good] + assert await svc.update_all() is expected diff --git a/updater/__main__.py b/updater/__main__.py index 3fc6a019..1b21bb1a 100644 --- a/updater/__main__.py +++ b/updater/__main__.py @@ -121,9 +121,12 @@ async def main() -> None: case "update": with _cli_lock(): if args.name is None: - await svc.update_all() + ok = await svc.update_all() else: - await svc.update_component(args.name) + ok = await svc.update_component(args.name) + if not ok: + # Scripts/harnesses rely on the exit code, not just the log. + raise SystemExit(1) case "status": result = await svc.check_status() for s in sorted(result.values(), key=lambda c: c.name): diff --git a/updater/components.py b/updater/components.py index f80d5d47..9f7ea77b 100644 --- a/updater/components.py +++ b/updater/components.py @@ -105,6 +105,16 @@ def _validate_component(data: dict) -> ComponentConfig | None: ) url = None + reset_mode = data.get("reset_mode", "hard") + if reset_mode not in ("hard", "soft"): + # An unknown value must not silently take the soft path (fleet default is hard). + logger.warning( + "Component %r has invalid reset_mode %r - using 'hard'", + name, + reset_mode, + ) + reset_mode = "hard" + install_if_missing = bool(data.get("install_if_missing", False)) if install_if_missing and not url: logger.warning( @@ -119,7 +129,7 @@ def _validate_component(data: dict) -> ComponentConfig | None: kind=comp_type, path=resolved, service=str(service) if service else None, - reset_mode=data.get("reset_mode", "hard"), + reset_mode=reset_mode, order=order, branch=str(branch) if branch else None, version=str(version) if version else None, @@ -151,8 +161,12 @@ def _validate_component(data: dict) -> ComponentConfig | None: def _merge(base: list[dict], override: list[dict]) -> list[dict]: """Merge override entries into base by name; unknown override names are appended.""" - base_by_name = {c["name"]: dict(c) for c in base if "name" in c} + base_by_name = { + c["name"]: dict(c) for c in base if isinstance(c, dict) and "name" in c + } for entry in override: + if not isinstance(entry, dict): + continue entry_name = entry.get("name") if not entry_name: continue @@ -225,6 +239,9 @@ def load_components() -> tuple[list[ComponentConfig], float]: logger.exception("Failed to load override YAML %s", OVERRIDE_PATH) configs: list[ComponentConfig] = [] for entry in raw_components: + if not isinstance(entry, dict): + logger.warning("Component entry is not a mapping - skipped: %r", entry) + continue cfg = _validate_component(entry) if cfg is not None: configs.append(cfg) diff --git a/updater/components.yaml b/updater/components.yaml index 334a2afc..7d998063 100644 --- a/updater/components.yaml +++ b/updater/components.yaml @@ -63,8 +63,7 @@ components: reset_mode: hard order: 20 - # Provisioned by hooks/Spoolman.sh (uv sync from source + stub client/dist); - # backend-only, used via the moonraker API. + # Provisioned by hooks/Spoolman.sh - name: Spoolman type: git path: ~/Spoolman @@ -78,7 +77,7 @@ components: - name: BlocksScreen type: git path: ~/BlocksScreen - branch: dev + branch: main url: https://github.com/BlocksTechnology/BlocksScreen service: BlocksScreen.service reset_mode: hard diff --git a/updater/dbus_service.py b/updater/dbus_service.py index d7b3d605..58d9de06 100644 --- a/updater/dbus_service.py +++ b/updater/dbus_service.py @@ -108,9 +108,18 @@ def _spawn(self, coro, *, name: str | None = None) -> asyncio.Task: """Create a task and hold a strong reference so GC cannot cancel it.""" task = asyncio.get_running_loop().create_task(coro, name=name) self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + task.add_done_callback(self._task_done) return task + def _task_done(self, task: asyncio.Task) -> None: + """Drop the task ref and log its exception now, not at some later GC.""" + self._background_tasks.discard(task) + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + _log.error("task %r failed", task.get_name(), exc_info=exc) + def _set_busy(self, busy: bool) -> None: # noqa: FBT001 """Emit busy_changed only on state transitions to avoid redundant signals.""" if busy != self._busy: diff --git a/updater/executor.py b/updater/executor.py index 46481a91..50718171 100644 --- a/updater/executor.py +++ b/updater/executor.py @@ -542,12 +542,14 @@ async def _quarantine_corrupt_objects(path: Path) -> int: return moved -async def git_repair(path: Path, branch: str = "main") -> tuple[bool, str]: +async def git_repair(path: Path, branch: str | None = None) -> tuple[bool, str]: """Prune 0-byte loose objects, re-fetch, re-verify. Mirrors start.sh recovery. If that doesn't clear it, quarantine non-empty corrupt objects (`fsck --full`) and re-fetch once more. Working tree untouched (delete/move + fetch only); - `branch` repairs a corrupt HEAD (fallback "main"). Returns (ok, message). + `branch` repairs a corrupt HEAD (None = derive the repo's own default, so a + branchless master-default component is not misrepaired to main). Returns + (ok, message). """ _clear_stale_git_index_lock(path) objects = path / ".git" / "objects" @@ -595,13 +597,14 @@ async def _is_head_readable(path: Path) -> bool: return ok -async def _repair_corrupt_head(path: Path, branch: str = "main") -> bool: +async def _repair_corrupt_head(path: Path, branch: str | None = None) -> bool: """Try to repair a corrupt HEAD by rewriting it to a valid symbolic ref. Returns True if HEAD was repaired or is already readable. """ if await _is_head_readable(path): return True + branch = branch or await git_default_branch(path) ok, _msg = await _run( [GIT, "symbolic-ref", "HEAD", f"refs/heads/{branch}"], cwd=path, @@ -664,6 +667,27 @@ async def git_get_current_branch(path: Path) -> str: return output.strip() if ok else "" +async def git_default_branch(path: Path | None) -> str: + """Best-effort default branch: HEAD's symref, else origin/HEAD's target, else master. + + HEAD's symref survives a missing commit object (plain-text ref file), so a + corrupt repo still reports the branch it was on. + """ + if path is None: + return "master" + ok, out = await _run( + [GIT, "symbolic-ref", "--short", "HEAD"], cwd=path, timeout=10.0 + ) + if ok and out.strip(): + return out.strip() + ok, out = await _run( + [GIT, "rev-parse", "--abbrev-ref", "origin/HEAD"], cwd=path, timeout=10.0 + ) + if ok and out.strip().startswith("origin/"): + return out.strip().removeprefix("origin/") + return "master" + + async def git_describe(path: Path, ref: str | None = None) -> str: cmd = [GIT, "describe", "--tags", "--abbrev=0"] if ref: @@ -1020,12 +1044,14 @@ async def restart_service(name: str | None) -> tuple[bool, str]: return (False, "service name is None") if not _SERVICE_RE.match(name): return (False, f"service name {name!r} is invalid") - ok, err = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=30.0) + # 120s: a blocking restart of a Type=notify unit waits for READY (up to its + # TimeoutStartSec=90s default); 30s used to under-wait a slow cold UI start. + ok, err = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=120.0) if ok: return (True, "") logger.warning("systemctl restart %s failed (%s); reset-failed + retry", name, err) await _run([SUDO, SYSTEMCTL, "reset-failed", name], timeout=10.0) - ok, err2 = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=30.0) + ok, err2 = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=120.0) if ok: return (True, f"recovered via reset-failed (first error: {err})") return (False, err2) diff --git a/updater/models.py b/updater/models.py index cb1af71b..502c4018 100644 --- a/updater/models.py +++ b/updater/models.py @@ -17,9 +17,7 @@ class ComponentConfig: apt_exclude: tuple[str, ...] = () url: str | None = None install_if_missing: bool = False - # Restart BlocksScreen on update even when the component's service differs. restart_ui: bool = False - # Restart klipper on update even when the component's own service differs. restart_klipper: bool = False @@ -36,7 +34,5 @@ class ComponentStatus: error: str | None = None has_local_changes: bool = False needs_install: bool = False - # Checked-out branch differs from configured branch (switch needed). branch_mismatch: bool = False - # Actual checked-out branch, surfaced for debugging branch switches. current_branch: str = "" diff --git a/updater/service.py b/updater/service.py index 5df70144..c4f9fc3d 100644 --- a/updater/service.py +++ b/updater/service.py @@ -291,11 +291,14 @@ async def update_component(self, name: str) -> bool: return await self._run_apt_update(component) return await self._run_git_update(component) - async def update_all(self, names: set[str] | None = None) -> None: - """Update components: apt independently, existing git as one atomic batch. - - apt packages and missing-component provisioning are independent of the git - code set, so they run outside the all-or-nothing batch. + async def update_all(self, names: set[str] | None = None) -> bool: + """Update components: apt independently, existing git as one phased batch. + + apt packages and missing-component provisioning are independent of the + git code set, so they run outside the batch. One failing component never + stops the others: it is reported, reverted and dropped while the rest + continue. Returns True only if every attempted component succeeded (CLI + exit-code contract). """ components = ( self._components @@ -308,24 +311,30 @@ async def update_all(self, names: set[str] | None = None) -> None: self._log.error("update_all: %s", msg) for c in components: self._cb("on_error", c.name, msg) - return + return False sorted_components = sorted(components, key=lambda c: c.order) - if not await self._preflight_fetch(sorted_components, components): - return + offline = await self._preflight_fetch(sorted_components) + ok = True apt = [c for c in sorted_components if c.kind == "apt"] batch = [ c for c in sorted_components if c.kind == "git" and c.path is not None and c.path.exists() ] + # An unreachable remote drops only that component, never the whole update. + for c in [c for c in batch if c.name in offline]: + batch.remove(c) + ok = self._cb_error_done(c.name, "network error during pre-flight fetch") # A dir without .git (tarball install) errors individually, not the batch. for c in [c for c in batch if not is_git_repo(c.path)]: batch.remove(c) if c.install_if_missing and c.url and await self._quarantine_nonrepo(c): continue # path is now absent: the provision pass below clones fresh self._log.error("%s: %s is not a git repository - skipping", c.name, c.path) - self._cb_error_done(c.name, "not a git repository - reinstall required") + ok = self._cb_error_done( + c.name, "not a git repository - reinstall required" + ) # A dead configured branch must not abort the batch and brick BlocksScreen. for c in list(batch): if c.branch and not await git_ref_hash(c.path, f"origin/{c.branch}"): @@ -335,7 +344,7 @@ async def update_all(self, names: set[str] | None = None) -> None: c.name, c.branch, ) - self._cb_error_done( + ok = self._cb_error_done( c.name, f"branch origin/{c.branch} not found - fix components.yaml" ) provision = [ @@ -347,11 +356,12 @@ async def update_all(self, names: set[str] | None = None) -> None: and c.url ] for c in apt: - await self._run_apt_update(c) + ok = await self._run_apt_update(c) and ok if batch: - await self._run_git_batch(batch) + ok = await self._run_git_batch(batch) and ok for c in provision: - await self._provision_component(c) + ok = await self._provision_component(c) and ok + return ok async def provision_missing(self) -> bool: """Clone absent install_if_missing components without waiting for a manual @@ -378,13 +388,14 @@ async def provision_missing(self) -> bool: return provisioned async def _preflight_fetch( - self, sorted_components: list[ComponentConfig], all_components: list - ) -> bool: + self, sorted_components: list[ComponentConfig] + ) -> set[str]: """Fetch every existing git component up-front (network phase). - Decoupling download from apply means a mid-sequence drop aborts before - anything is checked out (no applied/unapplied skew). A pre-fetched - component skips its apply-phase fetch; missing/corrupt repos self-heal. + Decoupling download from apply catches an unreachable remote before + anything is checked out. Returns the names whose fetch failed (offline); + the caller drops just those and continues. A pre-fetched component skips + its apply-phase fetch; missing/corrupt repos self-heal. """ # Non-repo dirs excluded: their fetch failure is not "offline" (see update_all). targets = [ @@ -392,7 +403,7 @@ async def _preflight_fetch( for c in sorted_components if c.kind == "git" and c.path is not None and is_git_repo(c.path) ] - offline: list[str] = [] + offline: set[str] = set() # Lock guards only _fetch_times; git_fetch runs outside it (as check_status). for c in targets: now = time.monotonic() @@ -406,81 +417,108 @@ async def _preflight_fetch( async with self._git_lock: self._fetch_times[c.name] = now elif not await git_has_corruption(c.path, hint=err): - offline.append(c.name) + offline.add(c.name) if offline: - msg = "network error during pre-flight fetch - no components changed" - self._log.error("update_all: %s (failed: %s)", msg, offline) - for c in all_components: - self._cb("on_error", c.name, msg) - return False - return True + self._log.error( + "update_all: pre-flight fetch failed for %s - continuing without them", + sorted(offline), + ) + return offline - async def _run_git_batch(self, batch: list[ComponentConfig]) -> None: - """Apply existing git components all-or-nothing. + async def _run_git_batch(self, batch: list[ComponentConfig]) -> bool: + """Apply existing git components with per-component failure isolation. - Stage every repo, then deps, hooks, then restart each unique service once. - Nothing is restarted during stage/deps/hooks, so an abort just reverts git - and restarts nothing. A restart-phase failure reverts git AND re-restarts - the already-bounced services back onto the old code. + Phased: stage all, deps all, hooks all, then restart each unique service + once. A component failing a pre-restart phase is reverted and dropped + while the rest continue; a restart failure reverts every component behind + that service and re-restarts it onto the old code. Nothing restarts + before the restart phase, so a power cut mid-way still reverts at boot. + Returns True only if every batched component succeeded. """ prev: dict[str, str] = {} + alive: list[ComponentConfig] = [] for c in batch: ph = await git_get_hash(c.path) if ph == "": - self._log.error("%s: prev_hash empty - aborting batch", c.name) - for b in batch: - self._cb_error_done(b.name, "prev_hash empty") - return + self._log.error("%s: prev_hash empty - skipping", c.name) + self._cb_error_done(c.name, "prev_hash empty") + continue prev[c.name] = ph + alive.append(c) + failed = len(alive) != len(batch) + if not alive: + return False async with self._state_lock: state = await asyncio.to_thread(self._read_state) for name, ph in prev.items(): - state[name] = {"prev_hash": ph} + # Merge: replacing the entry would wipe the self-heal anchors (last_good/golden). + _ensure_comp(state, name)["prev_hash"] = ph if not await asyncio.to_thread(self._write_state, state): - for b in batch: - self._cb_error_done(b.name, "failed to persist rollback point") - return + for c in alive: + self._cb_error_done(c.name, "failed to persist rollback point") + return False # Mark in-flight so a pre-commit power cut is reverted on next boot. await asyncio.to_thread(self._write_inflight, dict(prev)) - for c in batch: - ok, reason = await _assert_https_remote(c.path) - if not ok: + checked: list[ComponentConfig] = [] + for c in alive: + sec_ok, reason = await _assert_https_remote(c.path) + if not sec_ok: self._log.error("%s: SEC-4 remote check failed: %s", c.name, reason) - # Nothing staged: drop the marker (avoids a spurious boot revert). - await asyncio.to_thread(self._clear_inflight) - for b in batch: - self._cb_error_done(b.name, "insecure remote") - return + self._cb_error_done(c.name, "insecure remote") + failed = True + continue self._history("update_start", c.name, prev_hash=prev[c.name][:12]) + checked.append(c) + alive = checked + if not alive: + # Nothing staged: drop the marker (avoids a spurious boot revert). + await asyncio.to_thread(self._clear_inflight) + return False self._log.info( "git batch start: %d component(s): %s", - len(batch), - ", ".join(f"{c.name}->{c.service or '-'}" for c in batch), + len(alive), + ", ".join(f"{c.name}->{c.service or '-'}" for c in alive), ) touched: list[ComponentConfig] = [] restarted: list[str] = [] + # Repos whose failure-revert itself failed: kept in the in-flight marker + # so the boot reconcile retries them (never the committed survivors). + pending_revert: dict[str, str] = {} committed = False try: - self._log.info("git batch: stage phase (%d repo(s))", len(batch)) - for c in batch: + self._log.info("git batch: stage phase (%d repo(s))", len(alive)) + survivors: list[ComponentConfig] = [] + for c in alive: self._cb("on_step", c.name, 1, 4) touched.append(c) # mark before staging so a partial stage is reverted ok, reason = await self._stage_component(c) - if not ok: - await self._abort_batch(batch, prev, touched, restarted, reason) - return + if ok: + survivors.append(c) + continue + failed = True + if not await self._drop_component(c, prev[c.name], reason): + pending_revert[c.name] = prev[c.name] + touched.remove(c) + alive = survivors self._log.info("git batch: deps phase") - for c in batch: + survivors = [] + for c in alive: self._cb("on_step", c.name, 2, 4) deps_ok, deps_err = await self._install_dependencies(c) - if not deps_ok: - self._log.warning("%s: deps failed: %s", c.name, deps_err) - await self._abort_batch(batch, prev, touched, restarted, "deps") - return + if deps_ok: + survivors.append(c) + continue + self._log.warning("%s: deps failed: %s", c.name, deps_err) + failed = True + if not await self._drop_component(c, prev[c.name], "deps"): + pending_revert[c.name] = prev[c.name] + touched.remove(c) + alive = survivors self._log.info("git batch: hook phase") new_hashes: dict[str, str] = {} - for c in batch: + survivors = [] + for c in alive: self._cb("on_step", c.name, 3, 4) new_hashes[c.name] = await git_get_hash(c.path) self._log.info( @@ -501,16 +539,22 @@ async def _run_git_batch(self, batch: list[ComponentConfig]) -> None: 3, 4, ) - if not hook_ok: - self._log.error("%s: hook failed: %s", c.name, hook_err) - await self._abort_batch(batch, prev, touched, restarted, "hook") - return - # Restart each unique svc once; bounced marked BEFORE so abort re-restarts. + if hook_ok: + survivors.append(c) + continue + self._log.error("%s: hook failed: %s", c.name, hook_err) + failed = True + if not await self._drop_component(c, prev[c.name], "hook"): + pending_revert[c.name] = prev[c.name] + touched.remove(c) + alive = survivors + # Restart each unique svc once; a failure reverts every component + # behind that service and re-restarts it onto the old code. self._log.info("git batch: restart phase") seen: set[str] = set() ui_components: list[ComponentConfig] = [] - for c in batch: - if not c.service or c.service in seen: + for c in list(alive): + if c not in alive or not c.service or c.service in seen: continue seen.add(c.service) if c.service in _FIRE_AND_FORGET_SERVICES: @@ -523,41 +567,76 @@ async def _run_git_batch(self, batch: list[ComponentConfig]) -> None: self._cb("on_step", c.name, 4, 4) restarted.append(c.service) self._log.info("git batch: restarting %s (verified)", c.service) + if await self._restart_one(c.service): + continue + failed = True + for m in [m for m in alive if m.service == c.service]: + if not await self._drop_component(m, prev[m.name], "restart"): + pending_revert[m.name] = prev[m.name] + alive.remove(m) + touched.remove(m) + # Members reverted: bring the service back up on the old code. if not await self._restart_one(c.service): - await self._abort_batch(batch, prev, touched, restarted, "restart") - return + self._log.error("%s did not recover after revert", c.service) + restarted.remove(c.service) # Bounce klipper once for restart_klipper components that aren't klipper. klipper_svc = self._klipper_service() - if any(c.restart_klipper for c in batch) and klipper_svc not in seen: + requesters = [ + c for c in alive if c.restart_klipper and c.service != klipper_svc + ] + if requesters and klipper_svc not in seen: seen.add(klipper_svc) restarted.append(klipper_svc) self._log.info( "git batch: restarting %s (restart_klipper)", klipper_svc ) if not await self._restart_one(klipper_svc): - await self._abort_batch(batch, prev, touched, restarted, "restart") - return + failed = True + for m in requesters: + if not await self._drop_component(m, prev[m.name], "restart"): + pending_revert[m.name] = prev[m.name] + alive.remove(m) + touched.remove(m) + # Its own service already runs the new code: put it back + # on the old, unless a surviving component shares it. + shared = any(o.service == m.service for o in alive) + if m.service and m.service in restarted and not shared: + if not await self._restart_one(m.service): + self._log.error( + "%s did not recover after revert", m.service + ) + restarted.remove(m.service) + restarted.remove(klipper_svc) + # Requesters reverted: try to bring klipper back up. + if not await self._restart_one(klipper_svc): + self._log.error("%s did not recover after revert", klipper_svc) + if not alive: + await self._settle_inflight(pending_revert) + return False # Persist success BEFORE UI restart: a committed update is never reverted. - for c in batch: + for c in alive: self._history( "update_success", c.name, new_hash=new_hashes[c.name][:12] ) self._cb("on_component_done", c.name, True) # Durable now; the self-restart's CancelledError must never revert. committed = True - # Clear marker before any restart: the update stands from here on. - await asyncio.to_thread(self._clear_inflight) + # Shrink the marker before any restart: survivors stand from here on. + await self._settle_inflight(pending_revert) self._log.info( - "git batch complete: %d component(s) updated; queueing UI restart(s)", + "git batch complete: %d/%d component(s) updated; queueing UI restart(s)", + len(alive), len(batch), ) ui_services: set[str] = set() for c in ui_components: + if c not in alive: + continue self._cb("on_step", c.name, 4, 4) if c.service: ui_services.add(c.service) # klipper/RF50 hold config the UI reads at startup: refresh it too. - if any(c.restart_ui for c in batch): + if any(c.restart_ui for c in alive): ui_services.add(_UI_SERVICE) for svc in ui_services: self._log.info( @@ -565,21 +644,27 @@ async def _run_git_batch(self, batch: list[ComponentConfig]) -> None: ) await restart_service_noblock(svc) await self._apply_deferred_restart() + return not failed except asyncio.CancelledError: if committed: raise self._log.warning("git batch cancelled - aborting") await self._shielded( - self._abort_batch(batch, prev, touched, restarted, "cancelled"), + self._abort_batch( + alive, prev, touched, restarted, "cancelled", pending_revert + ), "git batch cancel-abort", ) raise except Exception: # noqa: BLE001 if committed: self._log.error("post-commit error (update stands)", exc_info=True) - return + return True self._log.error("git batch unexpected error", exc_info=True) - await self._abort_batch(batch, prev, touched, restarted, "unexpected_error") + await self._abort_batch( + alive, prev, touched, restarted, "unexpected_error", pending_revert + ) + return False async def _abort_batch( self, @@ -588,21 +673,23 @@ async def _abort_batch( touched: list[ComponentConfig], restarted: list[str], reason: str, + pending_revert: dict[str, str] | None = None, ) -> None: """Revert touched repos, re-restart bounced services, report all failed.""" self._log.warning( "aborting batch (reason=%s): reverting %d repo(s)", reason, len(touched) ) - revert_ok = True + pending = dict(pending_revert or {}) for c in touched: ok, _ = await git_reset_to_hash(c.path, prev[c.name]) if not ok: - revert_ok = False + pending[c.name] = prev[c.name] self._log.error( "abort: %s reset to %s failed", c.name, prev[c.name][:12] ) - # Repos are back at prev_hash: the batch is resolved, drop the marker. - await asyncio.to_thread(self._clear_inflight) + # Only unresolved repos stay in the marker for a boot-time retry. + await self._settle_inflight(pending) + revert_ok = not pending restart_ok = True for service in restarted: if not await self._restart_one(service): @@ -621,6 +708,38 @@ async def _abort_batch( self._cb("on_rollback", c.name, rollback_ok) self._cb("on_component_done", c.name, False) + async def _drop_component( + self, component: ComponentConfig, prev_hash: str, reason: str + ) -> bool: + """Revert one failed component and report it; the rest of the batch continues. + + Returns True when the revert succeeded; False means the caller must keep + the repo in the in-flight marker for a boot-time retry. + """ + self._log.warning( + "%s: dropped from batch (reason=%s), reverting to %s", + component.name, + reason, + prev_hash[:8], + ) + ok, _ = await git_reset_to_hash(component.path, prev_hash) + if not ok: + self._log.error("%s: revert to %s failed", component.name, prev_hash[:12]) + self._history( + "rollback", component.name, reason=reason, reverted_to=prev_hash[:12], ok=ok + ) + self._cb("on_error", component.name, reason) + self._cb("on_rollback", component.name, ok) + self._cb("on_component_done", component.name, False) + return ok + + async def _settle_inflight(self, pending_revert: dict[str, str]) -> None: + """Shrink the in-flight marker to repos whose revert still needs a boot retry.""" + if pending_revert: + await asyncio.to_thread(self._write_inflight, dict(pending_revert)) + else: + await asyncio.to_thread(self._clear_inflight) + async def _apply_deferred_restart(self) -> None: """Apply a daemon restart/reinstall hook deferred during this batch. @@ -694,13 +813,15 @@ async def recover(self, name: str, hard: bool = False) -> bool: self._log.error("component %s not found", name) self._cb("on_recover", name, False) return False - state = self._read_state() - prev_hash = state.get(name, {}).get("prev_hash") + comp_state = self._read_state().get(name) + prev_hash = ( + comp_state.get("prev_hash") if isinstance(comp_state, dict) else None + ) if prev_hash is None: self._cb("on_recover", name, False) self._log.error("failed to get prev_hash for %s", name) return False - if not _GIT_SHA_RE.match(prev_hash): + if not _is_sha(prev_hash): self._log.error("recover: invalid prev_hash format for %s", name) self._cb("on_recover", name, False) return False @@ -905,11 +1026,11 @@ async def supervise_ui(self) -> None: await asyncio.sleep(_NRESTARTS_POLL_INTERVAL_S) try: nrestarts = await get_service_nrestarts(_UI_SERVICE) - except Exception as exc: # noqa: BLE001 - self._log.debug("supervise_ui: NRestarts read failed: %s", exc) - continue - if self._check_crash_loop(_UI_COMPONENT, nrestarts): - await self._handle_crash_loop(nrestarts) + if self._check_crash_loop(_UI_COMPONENT, nrestarts): + await self._handle_crash_loop(nrestarts) + except Exception: # noqa: BLE001 + # One bad pass must not kill crash-loop supervision for good. + self._log.error("supervise_ui pass failed", exc_info=True) async def _handle_crash_loop(self, nrestarts: int) -> None: """Run one recovery rung for the UI, or saturate if the ladder is exhausted.""" @@ -1060,6 +1181,7 @@ async def _revert_inflight(self) -> None: async def _reconcile_locked(self) -> None: await self._revert_inflight() self._log.info("reconcile: boot-checking git components for damage") + recloned: list[ComponentConfig] = [] for c in self._components: if c.kind != "git" or c.path is None or not c.path.exists(): continue @@ -1067,13 +1189,14 @@ async def _reconcile_locked(self) -> None: if await git_get_hash(c.path) != "": continue self._log.warning("reconcile: %s HEAD unreadable - repairing", c.name) - _branch = c.branch or "main" - ok, msg = await git_repair(c.path, _branch) + # No configured branch: git_repair derives the repo's own default. + ok, msg = await git_repair(c.path, c.branch) if ok and await git_get_hash(c.path) != "": self._history("boot_repair", c.name, detail=msg[:80]) continue if await self._reclone_component(c) and await git_get_hash(c.path): self._history("boot_reclone", c.name) + recloned.append(c) continue comp_state = (await asyncio.to_thread(self._read_state)).get(c.name, {}) prev = ( @@ -1091,6 +1214,21 @@ async def _reconcile_locked(self) -> None: self._log.error( "reconcile: %s unrepairable, no recorded prev_hash", c.name ) + # Outside _git_lock: a reclone drops in-repo artifacts (e.g. Spoolman's + # .venv); the hook rebuilds them, best-effort, without blocking status. + for c in recloned: + try: + new_hash = await git_get_hash(c.path) + hook_ok, hook_err = await run_hook( + c.name, c.path, new_hash, _GIT_EMPTY_TREE, timeout=HOOK_TIMEOUT + ) + if not hook_ok: + self._log.warning( + "%s: post-reclone hook failed: %s", c.name, hook_err + ) + except Exception: # noqa: BLE001 + # Best-effort: a hook crash must not kill the rest of boot heal. + self._log.warning("%s: post-reclone hook raised", c.name, exc_info=True) await self._reconcile_self_heal_state() async def _reconcile_self_heal_state(self) -> None: @@ -1144,7 +1282,11 @@ async def _rollback( "%s: rolling back to %s (reason=%s)", component.name, prev_hash[:8], reason ) ok, _ = await git_reset_to_hash(component.path, prev_hash) - await asyncio.to_thread(self._clear_inflight) + if ok: + await asyncio.to_thread(self._clear_inflight) + else: + # A failed revert keeps the marker so boot _revert_inflight retries it. + self._log.warning("rollback: revert failed - keeping in-flight marker") if component.service and not await self._restart_one(component.service): ok = False self._history( @@ -1183,8 +1325,9 @@ async def _install_dependencies( # Keep pip current (best-effort: a failed upgrade must not block reqs). await _run([pip_path, "install", "--upgrade", "pip", "--quiet"], timeout=120.0) + # Generous: one aarch64 source build (no wheel) easily exceeds 120s on a Pi. return await _run( - [pip_path, "install", "-r", str(req), "--quiet"], timeout=120.0 + [pip_path, "install", "-r", str(req), "--quiet"], timeout=600.0 ) async def _ping_while( @@ -1268,6 +1411,8 @@ def _quarantine_sync(self, path: Path) -> Path | None: except OSError as exc: self._log.error("quarantine of %s failed: %s", path, exc) return None + # The venv moved with the dir: a cached pip path would now dangle. + self._component_pip_cache.pop(str(path), None) return dest async def _quarantine_nonrepo(self, component: ComponentConfig) -> bool: @@ -1424,6 +1569,8 @@ async def _reclone_component(self, component: ComponentConfig) -> bool: self._log.error("%s: reclone swap failed: %s", component.name, exc) return False await asyncio.to_thread(shutil.rmtree, old, ignore_errors=True) + # A fresh clone has no in-repo venv: drop any cached pip path for it. + self._component_pip_cache.pop(str(path), None) self._history("reclone", component.name, url=component.url) self._log.warning("%s: recloned successfully", component.name) return True @@ -1457,6 +1604,15 @@ async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str] else: return (False, "corrupt") + # A deleted upstream branch must fail here, not strand the repo mid-switch. + if component.branch and not await git_ref_hash( + component.path, f"origin/{component.branch}" + ): + return ( + False, + f"branch origin/{component.branch} not found - fix components.yaml", + ) + # Switch to the target branch FIRST so reset/pull act on the right branch. if component.branch: # hard mode forces past untracked collisions (build artifacts). @@ -1539,7 +1695,8 @@ async def _run_git_update(self, component: ComponentConfig) -> bool: return self._cb_error_done(component.name, "prev_hash empty") async with self._state_lock: state = await asyncio.to_thread(self._read_state) - state[component.name] = {"prev_hash": prev_hash} + # Merge: replacing the entry would wipe the self-heal anchors (last_good/golden). + _ensure_comp(state, component.name)["prev_hash"] = prev_hash if not await asyncio.to_thread(self._write_state, state): return self._cb_error_done( component.name, "failed to persist rollback point" @@ -1558,10 +1715,14 @@ async def _run_git_update(self, component: ComponentConfig) -> bool: self._cb("on_step", component.name, 1, 4) stage_ok, stage_reason = await self._stage_component(component) if not stage_ok: - # Failed pre-update reset changed nothing, so nothing to roll back. - if stage_reason == "reset": + # Pre-checkout failures leave the tree untouched: no revert, no restart. + if ( + stage_reason in ("network", "corrupt") + or "not found" in stage_reason + ): await asyncio.to_thread(self._clear_inflight) - return self._cb_error_done(component.name, "reset") + return self._cb_error_done(component.name, stage_reason) + # checkout/reset/pin/pull may have moved the tree: full rollback. await self._rollback(component, prev_hash, stage_reason) return False @@ -1744,16 +1905,21 @@ async def _run_apt_update_locked( def _read_state(self) -> dict: try: - return json.loads(self._state_path.read_text()) + data = json.loads(self._state_path.read_text()) except (FileNotFoundError, json.JSONDecodeError): return {} + # Valid JSON of the wrong shape (torn/corrupt write) must read as empty. + return data if isinstance(data, dict) else {} def _read_inflight(self) -> dict[str, str]: try: data = json.loads(self._inflight_path.read_text()) except (FileNotFoundError, json.JSONDecodeError): return {} - return data if isinstance(data, dict) else {} + if not isinstance(data, dict): + return {} + # Drop corrupt entries so a torn write can't crash the boot revert. + return {k: v for k, v in data.items() if isinstance(k, str) and _is_sha(v)} def _write_inflight(self, mapping: dict[str, str]) -> bool: """Atomically record the in-flight batch's name->prev_hash map (fsync'd).""" From 900411118a101b5d6901917ad78f29cc20e095e3 Mon Sep 17 00:00:00 2001 From: Guilherme Costa Date: Fri, 17 Jul 2026 14:55:40 +0100 Subject: [PATCH 4/6] fix: change to release branch --- updater/components.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/updater/components.yaml b/updater/components.yaml index 7d998063..7efe3e3b 100644 --- a/updater/components.yaml +++ b/updater/components.yaml @@ -77,7 +77,7 @@ components: - name: BlocksScreen type: git path: ~/BlocksScreen - branch: main + branch: origin/Release-2.0 url: https://github.com/BlocksTechnology/BlocksScreen service: BlocksScreen.service reset_mode: hard From 63ca2bc7d0381a60582d8388db2666187cc87d44 Mon Sep 17 00:00:00 2001 From: Guilherme Costa Date: Mon, 20 Jul 2026 15:07:12 +0100 Subject: [PATCH 5/6] fix(updater): guard against host downgrade to a pre-updater commit; one-line all comments and docstrings --- tests/updater/test_components_unit.py | 20 +++ tests/updater/test_self_heal_unit.py | 4 + tests/updater/test_service_unit.py | 114 +++++++++++++ updater/__main__.py | 5 +- updater/components.py | 27 +-- updater/dbus_service.py | 22 ++- updater/executor.py | 162 +++++------------- updater/hooks/Spoolman.sh | 7 +- updater/locking.py | 20 +-- updater/service.py | 232 ++++++++++++-------------- 10 files changed, 323 insertions(+), 290 deletions(-) diff --git a/tests/updater/test_components_unit.py b/tests/updater/test_components_unit.py index 983a7aed..8db31e20 100644 --- a/tests/updater/test_components_unit.py +++ b/tests/updater/test_components_unit.py @@ -475,3 +475,23 @@ def test_non_dict_component_entry_skipped(caplog): ): configs, _ = load_components() assert [c.name for c in configs if c.kind == "git"] == ["klipper"] + + +def test_origin_prefixed_branch_is_stripped(caplog): + bad_yaml = """ +components: + - name: BlocksScreen + type: git + path: ~/BlocksScreen + branch: origin/Release-2.0 + order: 99 +""" + with ( + patch("builtins.open", _mock_load(bad_yaml)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + configs, _ = load_components() + bs = next(c for c in configs if c.name == "BlocksScreen") + assert bs.branch == "Release-2.0" # stray origin/ prefix stripped + assert any("origin/" in r.message for r in caplog.records) diff --git a/tests/updater/test_self_heal_unit.py b/tests/updater/test_self_heal_unit.py index 888c631e..e52f558e 100644 --- a/tests/updater/test_self_heal_unit.py +++ b/tests/updater/test_self_heal_unit.py @@ -464,6 +464,7 @@ async def run_test(): patch("updater.service.git_fetch") as m_fetch, patch("updater.service.git_ref_hash") as m_ref, patch("updater.service.git_reset_to_hash") as m_reset, + patch("updater.service.git_tree_has_path", return_value=True), patch.object(svc, "_restart_ui_service") as m_restart, ): m_fetch.return_value = (True, "") @@ -492,6 +493,7 @@ async def run_test(): patch("updater.service.git_fetch") as m_fetch, patch("updater.service.git_ref_hash") as m_ref, patch("updater.service.git_reset_to_hash") as m_reset, + patch("updater.service.git_tree_has_path", return_value=True), patch.object(svc, "_restart_ui_service") as m_restart, ): m_fetch.return_value = (True, "") @@ -615,6 +617,7 @@ async def run_test(): patch("updater.service.git_fetch") as m_fetch, patch("updater.service.git_ref_hash") as m_ref, patch("updater.service.git_reset_to_hash") as m_reset, + patch("updater.service.git_tree_has_path", return_value=True), patch("updater.service.get_service_nrestarts") as m_nr, patch.object(svc, "_restart_ui_service") as m_restart, ): @@ -750,6 +753,7 @@ async def slow_reset(_path, _ref): with ( patch("updater.service.git_fetch", return_value=(True, "")), patch("updater.service.git_ref_hash", return_value="c" * 40), + patch("updater.service.git_tree_has_path", return_value=True), patch("updater.service.git_reset_to_hash", side_effect=slow_reset), patch("updater.service.get_service_nrestarts", return_value=0), patch.object(svc, "_restart_ui_service", return_value=True), diff --git a/tests/updater/test_service_unit.py b/tests/updater/test_service_unit.py index ce906500..d85f089b 100644 --- a/tests/updater/test_service_unit.py +++ b/tests/updater/test_service_unit.py @@ -2660,3 +2660,117 @@ async def test_update_all_returns_true_only_when_all_succeed(self, tmp_path): svc = UpdateService() svc._components = [good] assert await svc.update_all() is expected + + +class TestUpdaterDowngradeGuard: + """Refuse switching the host onto a tree without the updater (field brick).""" + + def _bs(self, tmp_path: Path) -> ComponentConfig: + path = tmp_path / "BlocksScreen" + (path / ".git").mkdir(parents=True) + return ComponentConfig( + name="BlocksScreen", + kind="git", + path=path, + service="BlocksScreen.service", + branch="main", + ) + + @pytest.mark.asyncio + async def test_stage_refuses_target_without_updater(self, tmp_path): + comp = self._bs(tmp_path) + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + with ( + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + # target lacks updater/ + patch("updater.service.git_tree_has_path", return_value=False), + patch("updater.service.git_checkout") as mock_checkout, + ): + ok, reason = await svc._stage_component(comp) + assert ok is False + assert "refusing" in reason + mock_checkout.assert_not_called() # never switched onto the brick + + @pytest.mark.asyncio + async def test_stage_allows_target_with_updater(self, tmp_path): + comp = self._bs(tmp_path) + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + with ( + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch("updater.service.git_tree_has_path", return_value=True), + patch("updater.service.git_checkout", return_value=(True, "")) as mc, + patch("updater.service.git_reset_to_hash", return_value=(True, "")), + ): + ok, _ = await svc._stage_component(comp) + assert ok is True + mc.assert_called_once() + + @pytest.mark.asyncio + async def test_single_update_refusal_no_rollback_clears_marker(self, tmp_path): + comp = self._bs(tmp_path) + cb = MagicMock() + svc = UpdateService(callback=cb) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + with ( + patch("updater.service.git_get_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://x"), + ), + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch("updater.service.git_tree_has_path", return_value=False), + patch("updater.service.git_reset_to_hash") as mock_reset, + ): + assert await svc._run_git_update(comp) is False + mock_reset.assert_not_called() # nothing moved: no rollback + assert not (tmp_path / "inflight.json").exists() + assert "refusing" in cb.on_error.call_args[0][1] + + @pytest.mark.asyncio + async def test_batch_drops_host_keeps_others(self, tmp_path): + bs = self._bs(tmp_path) + klip_path = tmp_path / "klipper" + (klip_path / ".git").mkdir(parents=True) + klip = ComponentConfig( + name="klipper", kind="git", path=klip_path, service="klipper.service" + ) + cb = MagicMock() + svc = UpdateService(callback=cb) + svc._components = [klip, bs] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + + async def fake_tree(path, ref, repo_path): + return "klipper" in str(path) # only klipper target has updater/ + + with ( + patch("updater.service.git_get_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://x"), + ), + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch("updater.service.git_tree_has_path", side_effect=fake_tree), + patch("updater.service.git_checkout", return_value=(True, "")), + patch("updater.service.git_reset_to_hash", return_value=(True, "")), + patch( + "updater.service.UpdateService._install_dependencies", + return_value=(True, ""), + ), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), + ): + result = await svc._run_git_batch([klip, bs]) + assert result is False # BlocksScreen dropped + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"klipper": True, "BlocksScreen": False} diff --git a/updater/__main__.py b/updater/__main__.py index 1b21bb1a..76f47c8b 100644 --- a/updater/__main__.py +++ b/updater/__main__.py @@ -83,10 +83,7 @@ async def _run_daemon() -> None: def _watchdog_ping_interval() -> float: - """Half of systemd's WatchdogSec (per sd_notify(3)), or 15s if unset/invalid. - - Reads WATCHDOG_USEC so the heartbeat stays correct if WatchdogSec is retuned. - """ + """Half of systemd's WatchdogSec (per sd_notify(3)), or 15s if unset/invalid.""" try: watchdog_usec = int(os.environ.get("WATCHDOG_USEC", "0")) except ValueError: diff --git a/updater/components.py b/updater/components.py index 9f7ea77b..135c533f 100644 --- a/updater/components.py +++ b/updater/components.py @@ -74,13 +74,21 @@ def _validate_component(data: dict) -> ComponentConfig | None: return None branch = data.get("branch") - if branch is not None and not _GIT_BRANCH_RE.match(str(branch)): - logger.warning( - "Component %r has invalid branch name %r, skipped", - name, - branch, - ) - return None + if branch is not None: + branch = str(branch) + # Strip a stray remote prefix: `origin/x` would fetch `origin/origin/x`. + if branch.startswith("origin/"): + logger.warning( + "Component %r branch %r has an 'origin/' prefix - stripping it", + name, + branch, + ) + branch = branch[len("origin/") :] + if not _GIT_BRANCH_RE.match(branch): + logger.warning( + "Component %r has invalid branch name %r, skipped", name, branch + ) + return None version = data.get("version") if version is not None and not _GIT_VERSION_RE.match(str(version)): @@ -180,10 +188,7 @@ def _merge(base: list[dict], override: list[dict]) -> list[dict]: def load_components() -> tuple[list[ComponentConfig], float]: - """Load/validate components from bundled YAML merged with the user override. - - Returns (list of ComponentConfig, poll_interval in seconds). - """ + """Load/validate components from bundled YAML merged with the user override.""" try: import yaml # noqa: PLC0415 except ImportError: diff --git a/updater/dbus_service.py b/updater/dbus_service.py index 58d9de06..fd0ce058 100644 --- a/updater/dbus_service.py +++ b/updater/dbus_service.py @@ -19,13 +19,14 @@ class DbusProgressCallback: - """Forward ProgressCallback events to D-Bus signals (decouples UpdateService - from D-Bus): each on_* call invokes the matching signal's .emit().""" + """Forward ProgressCallback events to D-Bus signals (decouples UpdateService""" def __init__(self, iface: UpdaterInterface) -> None: + """Bind the callback to the D-Bus interface whose signals it emits.""" self._iface = iface def on_step(self, name: str, step: int, total: int) -> None: + """Emit step_complete and write the current step to the status file.""" self._iface.step_complete.emit((name, step, total)) try: _STATUS_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -36,15 +37,19 @@ def on_step(self, name: str, step: int, total: int) -> None: _log.debug("status write failed: %s", e) def on_component_done(self, name: str, success: bool) -> None: # noqa: FBT001 + """Emit the component_done signal.""" self._iface.component_done.emit((name, success)) def on_error(self, name: str, reason: str) -> None: + """Emit the error signal.""" self._iface.error.emit((name, reason)) def on_rollback(self, name: str, success: bool) -> None: # noqa: FBT001 + """Emit the rollback signal.""" self._iface.rollback.emit((name, success)) def on_recover(self, name: str, success: bool) -> None: # noqa: FBT001 + """Emit the recover_done signal.""" self._iface.recover_done.emit((name, success)) @@ -52,8 +57,7 @@ class UpdaterInterface( sdbus.DbusInterfaceCommonAsync, interface_name="com.blockscreen.Updater", ): - """D-Bus contract shared by server and client proxy: signals declared once, - sdbus swaps each for server-side emit machinery / client-side async-iterable.""" + """D-Bus contract shared by server and client proxy: signals declared once,""" @sdbus.dbus_signal_async("sii") def step_complete(self) -> tuple[str, int, int]: @@ -91,6 +95,7 @@ def busy_changed(self) -> tuple[bool]: raise NotImplementedError def __init__(self) -> None: + """Wire the service and busy state, then spawn the boot, poll, and self-heal tasks.""" super().__init__() self._svc = UpdateService(callback=DbusProgressCallback(self)) self._busy: bool = False @@ -128,10 +133,7 @@ def _set_busy(self, busy: bool) -> None: # noqa: FBT001 self.busy_changed.emit((busy,)) def _validate_component_name(self, name: str) -> bool: - """SEC: verify component exists to prevent abuse on unknown names. - - Also rate-limits invalid requests to detect/prevent fuzzing attacks. - """ + """SEC: verify component exists to prevent abuse on unknown names.""" is_valid = self._svc.has_component(name) if not is_valid: self._invalid_requests += 1 @@ -237,6 +239,7 @@ async def bless_healthy(self, name: str, hash_val: str) -> bool: return await self._svc.bless_healthy(name, hash_val) async def _run_update_all(self) -> None: + """Run update_all under the process lock, then a background apt pass; always clears busy.""" ran = False try: with process_lock() as acquired: @@ -258,6 +261,7 @@ async def _run_update_all(self) -> None: ) async def _update_all_locked(self) -> None: + """Update only the components whose status is dirty.""" statuses = await self._svc.check_status() dirty = { name @@ -276,6 +280,7 @@ async def _update_all_locked(self) -> None: _log.info("update_all: no dirty components found") async def _run_update_component(self, name: str) -> None: + """Run a single-component update under the process lock; always clears busy.""" try: with process_lock() as acquired: if not acquired: @@ -289,6 +294,7 @@ async def _run_update_component(self, name: str) -> None: self._set_busy(busy=False) async def _run_recover(self, name: str, hard: bool) -> None: # noqa: FBT001 + """Run a recover under the process lock; always clears busy.""" try: with process_lock() as acquired: if not acquired: diff --git a/updater/executor.py b/updater/executor.py index 50718171..f6269b0a 100644 --- a/updater/executor.py +++ b/updater/executor.py @@ -45,11 +45,7 @@ def _clear_stale_git_index_lock(path: Path) -> bool: - """Remove .git/index.lock only if stale (mtime age >= threshold). - - A fresh lock may belong to a live/concurrent git op; yanking it would corrupt - that op. Returns True if removed or absent, False if too fresh to remove. - """ + """Remove .git/index.lock only if stale (mtime age >= threshold).""" lock_path = path / ".git" / "index.lock" if not lock_path.exists(): return True @@ -80,6 +76,7 @@ def _kill_proc_group(proc, sig): def _make_clean_env() -> dict[str, str]: + """Build a minimal sanitized environment for updater subprocesses.""" env: dict[str, str] = {} for key in ( "PATH", @@ -113,11 +110,7 @@ async def _run( cwd: Path | None = None, env: dict[str, str] | None = None, ) -> tuple[bool, str]: - """Run a subprocess, return (success, stdout_or_stderr). - - On timeout: SIGTERM → 5s grace → SIGKILL. - On CancelledError: SIGKILL → 2s grace, then re-raise. - """ + """Run a subprocess, return (success, stdout_or_stderr).""" proc = await asyncio.create_subprocess_exec( *[str(c) for c in cmd], cwd=cwd, @@ -157,10 +150,7 @@ def _validate_git_ref(ref: str) -> bool: def _resolve_component_pip(path: Path | None) -> str: - """Find the component venv's pip, else system pip. - - SEC: only accept venv paths under the component/parent; reject symlink escapes. - """ + """Find the component venv's pip, else system pip.""" if path is None: logger.warning("Path not found, using system pip") return PIP @@ -193,10 +183,7 @@ def _resolve_component_pip(path: Path | None) -> str: async def _list_upgradable_packages() -> tuple[bool, list[str]]: - """Return (success, package_names) from apt list --upgradable. - - SEC: validate names against Debian rules to resist injection via malformed output. - """ + """Return (success, package_names) from apt list --upgradable.""" ok, output = await _run([APT, "list", "--upgradable"], timeout=30.0) if not ok: return False, [] @@ -226,6 +213,7 @@ def _compile_exclude_patterns(exclude: tuple[str, ...]) -> list[re.Pattern]: def _apply_exclude_patterns(pkgs: list[str], exclude: tuple[str, ...]) -> list[str]: + """Drop packages matching any apt exclude pattern.""" if not exclude: return pkgs compiled = _compile_exclude_patterns(exclude) @@ -233,6 +221,7 @@ def _apply_exclude_patterns(pkgs: list[str], exclude: tuple[str, ...]) -> list[s async def _count_apt_upgradable(exclude: tuple[str, ...] = ()) -> int: + """Return the count of upgradable packages after excludes, or -1 on failure.""" ok, pkgs = await _list_upgradable_packages() if not ok: return -1 @@ -324,6 +313,7 @@ def is_git_repo(path: Path | None) -> bool: async def git_is_dirty(path: Path) -> bool: + """Return True if the repo has uncommitted tracked-file changes.""" ok, output = await _run( [GIT, "status", "--porcelain", "--untracked-files=no"], cwd=path, @@ -333,6 +323,7 @@ async def git_is_dirty(path: Path) -> bool: async def git_prune_extra_remotes(path: Path) -> None: + """Remove every git remote except origin.""" ok, output = await _run([GIT, "remote"], cwd=path, timeout=10.0) if not ok: return @@ -375,12 +366,7 @@ async def _prune_broken_refs(path: Path, output: str) -> bool: async def git_fetch( path: Path | None, *, prune_remotes: bool = True ) -> tuple[bool, str]: - """Fetch all remotes for the repo at path. - - prune_remotes=False skips extra-remote cleanup (unneeded in status checks). A - corrupt local ref makes --prune abort the whole fetch; on that failure the - named ref is deleted and the fetch retried once. - """ + """Fetch all remotes for the repo at path.""" if path is None: return (False, "path does not exist") if prune_remotes: @@ -407,10 +393,7 @@ async def git_pull(path: Path | None) -> tuple[bool, str]: async def git_clone( url: str, path: Path | None, branch: str | None = None ) -> tuple[bool, str]: - """Clone url into path (https-only), optionally at branch. Returns (ok, message). - - Timeout is generous for large repos; git makes the leaf dir, parent must exist. - """ + """Clone url into path (https-only), optionally at branch. Returns (ok, message).""" if not path: return (False, "path error") if not _GIT_URL_RE.match(url): @@ -425,12 +408,7 @@ async def git_clone( async def git_reset_to_hash(path: Path | None, prev_hash: str = "") -> tuple[bool, str]: - """Hard-reset repo at path directly to prev_hash (no fetch). - - The rollback/heal primitive (abort, boot revert, recover); generous timeout so - a large reset on a slow SD card isn't SIGTERM'd mid-checkout. Clears an - index.lock error and retries once. - """ + """Hard-reset repo at path directly to prev_hash (no fetch).""" if not path: return (False, "path error") if prev_hash == "": @@ -466,12 +444,7 @@ async def git_reset_to_hash(path: Path | None, prev_hash: str = "") -> tuple[boo async def git_has_corruption(path: Path | None, hint: str = "") -> bool: - """True if `hint` or git fsck shows object corruption. - - `hint` (e.g. a failed fetch's stderr) is checked first: fsck - --connectivity-only validates reachability not content, so it can miss a - corrupt blob the fetch already tripped on. - """ + """True if `hint` or git fsck shows object corruption.""" if path is None: return False if any(k in hint for k in _GIT_CORRUPT_SIGNATURES): @@ -501,12 +474,7 @@ def _prune_empty_loose_objects(objects: Path) -> int: async def _quarantine_corrupt_objects(path: Path) -> int: - """Move loose objects that `git fsck --full` flags as corrupt out of the way. - - connectivity-only fsck misses a bad-zlib object (power cut mid-write) and a - re-fetch won't replace it (file still there); fsck --full names them and we - move (not delete) them so a re-fetch restores them. Returns count moved. - """ + """Move loose objects that `git fsck --full` flags as corrupt out of the way.""" ok, out = await _run( [GIT, "fsck", "--full", "--no-dangling", "--no-progress"], cwd=path, @@ -543,14 +511,7 @@ async def _quarantine_corrupt_objects(path: Path) -> int: async def git_repair(path: Path, branch: str | None = None) -> tuple[bool, str]: - """Prune 0-byte loose objects, re-fetch, re-verify. Mirrors start.sh recovery. - - If that doesn't clear it, quarantine non-empty corrupt objects (`fsck --full`) - and re-fetch once more. Working tree untouched (delete/move + fetch only); - `branch` repairs a corrupt HEAD (None = derive the repo's own default, so a - branchless master-default component is not misrepaired to main). Returns - (ok, message). - """ + """Prune 0-byte loose objects, re-fetch, re-verify. Mirrors start.sh recovery.""" _clear_stale_git_index_lock(path) objects = path / ".git" / "objects" if not objects.is_dir(): @@ -598,10 +559,7 @@ async def _is_head_readable(path: Path) -> bool: async def _repair_corrupt_head(path: Path, branch: str | None = None) -> bool: - """Try to repair a corrupt HEAD by rewriting it to a valid symbolic ref. - - Returns True if HEAD was repaired or is already readable. - """ + """Try to repair a corrupt HEAD by rewriting it to a valid symbolic ref.""" if await _is_head_readable(path): return True branch = branch or await git_default_branch(path) @@ -625,6 +583,16 @@ async def git_ref_hash(path: Path | None, ref: str) -> str: return output.strip() if ok else "" +async def git_tree_has_path(path: Path | None, ref: str, repo_path: str) -> bool: + """True if repo_path exists in the tree at ref (git cat-file -e ref:path).""" + if path is None: + return False + ok, _ = await _run( + [GIT, "cat-file", "-e", f"{ref}:{repo_path}"], cwd=path, timeout=10.0 + ) + return ok + + async def git_commits_behind(path: Path, remote_ref: str = "origin/HEAD") -> int: """Return how many commits the local branch is behind remote_ref.""" ok, output = await _run( @@ -661,6 +629,7 @@ async def _assert_https_remote(path: Path | None) -> tuple[bool, str]: async def git_get_current_branch(path: Path) -> str: + """Return the repo's current branch name, or empty string on failure.""" ok, output = await _run( [GIT, "rev-parse", "--abbrev-ref", "HEAD"], cwd=path, timeout=10.0 ) @@ -668,11 +637,7 @@ async def git_get_current_branch(path: Path) -> str: async def git_default_branch(path: Path | None) -> str: - """Best-effort default branch: HEAD's symref, else origin/HEAD's target, else master. - - HEAD's symref survives a missing commit object (plain-text ref file), so a - corrupt repo still reports the branch it was on. - """ + """Best-effort default branch: HEAD's symref, else origin/HEAD's target, else master.""" if path is None: return "master" ok, out = await _run( @@ -689,6 +654,7 @@ async def git_default_branch(path: Path | None) -> str: async def git_describe(path: Path, ref: str | None = None) -> str: + """Return the nearest tag for ref (or HEAD), or empty string.""" cmd = [GIT, "describe", "--tags", "--abbrev=0"] if ref: cmd.append(ref) @@ -699,6 +665,7 @@ async def git_describe(path: Path, ref: str | None = None) -> str: async def git_checkout( path: Path | None, branch: str, *, force: bool = False ) -> tuple[bool, str]: + """Check out branch (force overwrites untracked collisions); validates the name.""" if path is None: return (False, "path not found") @@ -720,10 +687,7 @@ async def git_checkout( async def check_apt_status( cache_ttl_seconds: int = 86_400, exclude: tuple[str, ...] = () ) -> ComponentStatus: - """Return cached apt ComponentStatus, refreshing via apt-get past the TTL. - - SEC: validate cache ownership/permissions to prevent privilege escalation. - """ + """Return cached apt ComponentStatus, refreshing via apt-get past the TTL.""" path = Path.home() / ".cache" / "blockscreen" / "apt_status_cache.json" exclude_key = "|".join(sorted(exclude)) packages_upgradable = -1 @@ -790,6 +754,7 @@ async def check_apt_status( def _apt_env() -> dict[str, str]: + """Return the apt subprocess env: noninteractive frontend, needrestart disabled.""" env = _make_clean_env() env["DEBIAN_FRONTEND"] = "noninteractive" # needrestart can otherwise open an interactive prompt mid-upgrade and hang. @@ -798,19 +763,12 @@ def _apt_env() -> dict[str, str]: def _apt_cmd(verb: str, pkgs: Sequence[str] = ()) -> list[str]: - """Build the sudo apt argv via the root-owned wrapper. - - The wrapper is the only apt command sudoers grants; bs-bootstrap reinstalls it - when missing (an old box may have the service but not the wrapper). - """ + """Build the sudo apt argv via the root-owned wrapper.""" return [SUDO, str(APT_HELPER), verb, *pkgs] async def _apt_snapshot_packages() -> tuple[bool, Path | None]: - """Snapshot package state via dpkg --get-selections to a temp file (no sudo). - - Returns (success, snapshot_path) under ~/.cache/blockscreen/. - """ + """Snapshot package state via dpkg --get-selections to a temp file (no sudo).""" snapshot_path = ( Path.home() / ".cache" / "blockscreen" / "apt_pre_upgrade_snapshot.txt" ) @@ -838,20 +796,12 @@ async def _apt_snapshot_packages() -> tuple[bool, Path | None]: async def _apt_get_fix_broken() -> tuple[bool, str]: - """Run apt-get -f install -y to unbreak package state. - - Best-effort post-upgrade rollback; called only on apt_upgrade failure. - """ + """Run apt-get -f install -y to unbreak package state.""" return await _run(_apt_cmd("fix-broken"), timeout=120.0, env=_apt_env()) async def _apt_restore_packages(snapshot_path: Path) -> tuple[bool, str]: - """Restore dpkg selections from the pre-upgrade snapshot, then dselect-upgrade. - - Re-asserts which packages should be installed; does NOT reliably downgrade - versions (old .debs may be gone). An unbreak step, not a true rollback; - kernel/firmware excluded bounds the blast radius. - """ + """Restore dpkg selections from the pre-upgrade snapshot, then dselect-upgrade.""" try: stdin_data = snapshot_path.read_bytes() except OSError as e: @@ -912,10 +862,7 @@ async def apt_update() -> tuple[bool, str]: async def apt_upgrade(exclude: tuple[str, ...] = ()) -> tuple[bool, str]: - """Run apt-get upgrade via an explicit package list to limit blast radius. - - Lists upgradables, applies excludes, uses `install --only-upgrade`. - """ + """Run apt-get upgrade via an explicit package list to limit blast radius.""" ok, pkgs = await _list_upgradable_packages() if not ok: return False, "failed to list upgradable packages" @@ -940,11 +887,7 @@ async def run_hook( prev_hash: str, timeout: float = 60.0, ) -> tuple[bool, str]: - """Run the per-component update hook if it exists. - - The 60s default suits tests/trivial hooks; update/provision flows pass - HOOK_TIMEOUT since a hook may sync a full dep set (e.g. Spoolman's `uv sync`). - """ + """Run the per-component update hook if it exists.""" hook = (_HOOKS_DIR / f"{name}.sh").resolve() # SEC: resolve symlinks try: hook.relative_to(_HOOKS_DIR.resolve()) # SEC: prevent path traversal @@ -974,11 +917,7 @@ async def enable_service(name: str | None) -> tuple[bool, str]: async def wait_for_service_active(name: str, timeout: float = 90.0) -> bool: - """Poll systemctl is-active until active or timeout. - - `is-active --wait` doesn't exist (--wait is ignored here), so an explicit poll - is needed to ride out 'activating' after a restart. - """ + """Poll systemctl is-active until active or timeout.""" if not _SERVICE_RE.match(name): logger.warning("wait_for_service_active: invalid service name %r", name) return False @@ -1002,11 +941,7 @@ async def wait_for_service_active(name: str, timeout: float = 90.0) -> bool: async def restart_service_noblock(name: str | None) -> tuple[bool, str]: - """Queue a service restart without waiting (systemctl --no-block). - - For the UI service hosting the updater's D-Bus client: a blocking restart - tears it down and could time out the in-flight update. Fire-and-forget. - """ + """Queue a service restart without waiting (systemctl --no-block).""" if name is None: return (False, "service name is None") if not _SERVICE_RE.match(name): @@ -1015,12 +950,7 @@ async def restart_service_noblock(name: str | None) -> tuple[bool, str]: async def verify_updater_importable(component_path: Path | None) -> bool: - """Self-test the new on-disk updater code before restarting into it. - - Imports the updater package in a fresh interpreter from component_path: a new - updater that fails to import must never restart the daemon onto itself (that - kills the only field update path). On failure keep the old daemon. - """ + """Self-test the new on-disk updater code before restarting into it.""" if component_path is None or not component_path.exists(): return False ok, out = await _run( @@ -1034,18 +964,12 @@ async def verify_updater_importable(component_path: Path | None) -> bool: async def restart_service(name: str | None) -> tuple[bool, str]: - """Restart a systemd service, recovering from a start-limit hit. - - If `restart` fails (e.g. "start request repeated too quickly"), `reset-failed` - and retry once. `systemctl kill` is NOT a fallback: it neither clears a - start-limit nor is covered by the scoped NOPASSWD rules (would prompt). - """ + """Restart a systemd service, recovering from a start-limit hit.""" if name is None: return (False, "service name is None") if not _SERVICE_RE.match(name): return (False, f"service name {name!r} is invalid") - # 120s: a blocking restart of a Type=notify unit waits for READY (up to its - # TimeoutStartSec=90s default); 30s used to under-wait a slow cold UI start. + # 120s timeout: Type=notify unit READY wait (up to 90s default TimeoutStartSec) plus margin for slow cold UI start. ok, err = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=120.0) if ok: return (True, "") diff --git a/updater/hooks/Spoolman.sh b/updater/hooks/Spoolman.sh index 6dd7b552..8309f676 100755 --- a/updater/hooks/Spoolman.sh +++ b/updater/hooks/Spoolman.sh @@ -1,7 +1,5 @@ #!/bin/bash -# Provision/update Spoolman: run-from-source (not pip-installable), so deps go in -# via uv (provided in the BlocksScreen venv by install-updater.sh). SQLite default -# means no sudo here; the unit is laid down + enabled by install-updater/updater. +# Provision Spoolman via uv (run-from-source, not pip-installable); SQLite default = no sudo; unit installed by install-updater. set -euo pipefail if [ -z "${COMPONENT_PATH:-}" ]; then @@ -21,8 +19,7 @@ fi cd "$COMPONENT_PATH" "$_uv" sync --no-dev -# Spoolman won't start without client/dist (a clone has no prebuilt UI); the API -# is all we need, so a stub satisfies the mount without an npm build. +# Stub client/dist (a clone has no prebuilt UI); the API is all we need, no npm build. mkdir -p client/dist if [ ! -f client/dist/index.html ]; then printf 'Spoolman\n' >client/dist/index.html diff --git a/updater/locking.py b/updater/locking.py index e58fcb9d..d0f4ac10 100644 --- a/updater/locking.py +++ b/updater/locking.py @@ -1,9 +1,4 @@ -"""Cross-process advisory lock shared by the updater daemon and the CLI. - -A single flock-backed lockfile serializes mutating git/apt work between the -daemon and a `python -m updater update` CLI run (asyncio locks are in-process -only); both acquire it non-blocking so the second caller fails fast. -""" +"""Cross-process advisory lock shared by the updater daemon and the CLI.""" from __future__ import annotations @@ -35,22 +30,13 @@ def lock_path() -> Path: def restart_sentinel_path() -> Path: - """Return the sentinel a self-update hook writes instead of restarting mid-batch. - - Hooks append `install`/`code` here; the daemon reads it once the batch ends. - Prefer /run (tmpfs) so it clears on reboot: a crash can't trigger a spurious - restart later (safe floor: adopt on next reboot). - """ + """Return the sentinel a self-update hook writes instead of restarting mid-batch.""" return _runtime_dir() / "updater-restart-needed" @contextlib.contextmanager def process_lock() -> Iterator[bool]: - """Acquire the shared updater lock non-blocking. - - Yields True if acquired, else False (another daemon/CLI holds it). The fd - stays open for the whole `with` block; closing it on exit releases the lock. - """ + """Acquire the shared updater lock non-blocking.""" try: f = open(lock_path(), "w") # noqa: SIM115, PTH123 except OSError: diff --git a/updater/service.py b/updater/service.py index c4f9fc3d..272cb5f0 100644 --- a/updater/service.py +++ b/updater/service.py @@ -42,6 +42,7 @@ git_has_corruption, git_pull, git_ref_hash, + git_tree_has_path, git_repair, git_reset_to_hash, is_git_repo, @@ -143,6 +144,8 @@ def reset(self) -> None: # Self-heal: the UI component name (components.yaml) that the supervisor watches. _UI_COMPONENT = "BlocksScreen" +# Marker file proving updater exists: absence at target ref aborts update (lack bricks Type=notify host with no self-heal). +_UPDATER_MARKER = "updater/dbus_service.py" # Forward-heal always targets the curated-stable channel, not the configured branch. _HEAL_REMOTE_REF = "origin/main" # Settle window after a rung (debounce plus margin) so a new build can bless first. @@ -156,33 +159,53 @@ def reset(self) -> None: class ProgressCallback(Protocol): - def on_step(self, name: str, step: int, total: int) -> None: ... - def on_component_done(self, name: str, success: bool) -> None: ... - def on_error(self, name: str, reason: str) -> None: ... - def on_rollback(self, name: str, success: bool) -> None: ... - def on_recover(self, name: str, success: bool) -> None: ... + def on_step(self, name: str, step: int, total: int) -> None: + """Report a numbered progress step for a component.""" + ... + + def on_component_done(self, name: str, success: bool) -> None: + """Report a component's final success or failure.""" + ... + + def on_error(self, name: str, reason: str) -> None: + """Report a non-recoverable error for a component.""" + ... + + def on_rollback(self, name: str, success: bool) -> None: + """Report a rollback attempt's outcome.""" + ... + + def on_recover(self, name: str, success: bool) -> None: + """Report a recover attempt's outcome.""" + ... class LoggingCallback: def __init__(self) -> None: + """Create the logging progress callback.""" self._log = logging.getLogger("updater") def on_step(self, name: str, step: int, total: int) -> None: + """Log a progress step.""" self._log.info("%s step %d/%d", name, step, total) def on_component_done(self, name: str, success: bool) -> None: + """Log a component's success or failure.""" if success: self._log.info("%s done(ok=True)", name) else: self._log.error("%s done(ok=False)", name) def on_error(self, name: str, reason: str) -> None: + """Log a component error.""" self._log.error("%s error: %s", name, reason) def on_rollback(self, name: str, success: bool) -> None: + """Log a rollback outcome.""" self._log.warning("%s rollback (ok=%s)", name, success) def on_recover(self, name: str, success: bool) -> None: + """Log a recover outcome.""" self._log.info("%s recover (ok=%s)", name, success) @@ -192,6 +215,7 @@ class UpdateService: _FETCH_TTL: float = 300.0 def __init__(self, callback: ProgressCallback | None = None) -> None: + """Load components and initialize locks, caches, and self-heal tracking.""" self._components, self.poll_interval = load_components() self._git_lock = Lock() self._state_lock = Lock() @@ -226,14 +250,11 @@ def component_stubs(self) -> list[tuple[str, str]]: return [(c.name, c.kind) for c in self._components] async def check_status(self, force: bool = False) -> dict[str, ComponentStatus]: - """Concurrently check status of all components. - - force=True bypasses the fetch TTL so git fetch always runs. - Use for explicit user-triggered refreshes. - """ + """Concurrently check status of all components.""" results: dict[str, ComponentStatus] = {} async def _check_one(c: ComponentConfig) -> None: + """Fetch and record one component's status into the results dict.""" if c.kind == "apt": # force bypasses the apt cache, mirroring the git fetch TTL bypass. status = await check_apt_status( @@ -292,14 +313,7 @@ async def update_component(self, name: str) -> bool: return await self._run_git_update(component) async def update_all(self, names: set[str] | None = None) -> bool: - """Update components: apt independently, existing git as one phased batch. - - apt packages and missing-component provisioning are independent of the - git code set, so they run outside the batch. One failing component never - stops the others: it is reported, reverted and dropped while the rest - continue. Returns True only if every attempted component succeeded (CLI - exit-code contract). - """ + """Update components: apt independently, existing git as one phased batch.""" components = ( self._components if names is None @@ -364,9 +378,7 @@ async def update_all(self, names: set[str] | None = None) -> bool: return ok async def provision_missing(self) -> bool: - """Clone absent install_if_missing components without waiting for a manual - Update (e.g. one added by a self-update); existing repos untouched. Returns - True if anything was provisioned.""" + """Clone absent install_if_missing components without waiting for a manual""" missing = [ c for c in self._components @@ -390,13 +402,7 @@ async def provision_missing(self) -> bool: async def _preflight_fetch( self, sorted_components: list[ComponentConfig] ) -> set[str]: - """Fetch every existing git component up-front (network phase). - - Decoupling download from apply catches an unreachable remote before - anything is checked out. Returns the names whose fetch failed (offline); - the caller drops just those and continues. A pre-fetched component skips - its apply-phase fetch; missing/corrupt repos self-heal. - """ + """Fetch every existing git component up-front (network phase).""" # Non-repo dirs excluded: their fetch failure is not "offline" (see update_all). targets = [ c @@ -426,15 +432,7 @@ async def _preflight_fetch( return offline async def _run_git_batch(self, batch: list[ComponentConfig]) -> bool: - """Apply existing git components with per-component failure isolation. - - Phased: stage all, deps all, hooks all, then restart each unique service - once. A component failing a pre-restart phase is reverted and dropped - while the rest continue; a restart failure reverts every component behind - that service and re-restarts it onto the old code. Nothing restarts - before the restart phase, so a power cut mid-way still reverts at boot. - Returns True only if every batched component succeeded. - """ + """Apply existing git components with per-component failure isolation.""" prev: dict[str, str] = {} alive: list[ComponentConfig] = [] for c in batch: @@ -482,8 +480,7 @@ async def _run_git_batch(self, batch: list[ComponentConfig]) -> bool: ) touched: list[ComponentConfig] = [] restarted: list[str] = [] - # Repos whose failure-revert itself failed: kept in the in-flight marker - # so the boot reconcile retries them (never the committed survivors). + # Repos whose revert failed: kept in in-flight marker so boot reconcile retries them, not committed survivors. pending_revert: dict[str, str] = {} committed = False try: @@ -548,8 +545,7 @@ async def _run_git_batch(self, batch: list[ComponentConfig]) -> bool: pending_revert[c.name] = prev[c.name] touched.remove(c) alive = survivors - # Restart each unique svc once; a failure reverts every component - # behind that service and re-restarts it onto the old code. + # Restart each unique service once; failure reverts all components behind it and re-restarts onto old code. self._log.info("git batch: restart phase") seen: set[str] = set() ui_components: list[ComponentConfig] = [] @@ -597,8 +593,7 @@ async def _run_git_batch(self, batch: list[ComponentConfig]) -> bool: pending_revert[m.name] = prev[m.name] alive.remove(m) touched.remove(m) - # Its own service already runs the new code: put it back - # on the old, unless a surviving component shares it. + # Service runs new code: revert it to old code unless a surviving component shares the service. shared = any(o.service == m.service for o in alive) if m.service and m.service in restarted and not shared: if not await self._restart_one(m.service): @@ -711,11 +706,7 @@ async def _abort_batch( async def _drop_component( self, component: ComponentConfig, prev_hash: str, reason: str ) -> bool: - """Revert one failed component and report it; the rest of the batch continues. - - Returns True when the revert succeeded; False means the caller must keep - the repo in the in-flight marker for a boot-time retry. - """ + """Revert one failed component and report it; the rest of the batch continues.""" self._log.warning( "%s: dropped from batch (reason=%s), reverting to %s", component.name, @@ -741,15 +732,7 @@ async def _settle_inflight(self, pending_revert: dict[str, str]) -> None: await asyncio.to_thread(self._clear_inflight) async def _apply_deferred_restart(self) -> None: - """Apply a daemon restart/reinstall hook deferred during this batch. - - Hooks record `install`/`code` to the tmpfs sentinel instead of restarting - mid-batch. Called once after success + UI restart: `install` touches the - deploy flag (BlocksScreen-deploy.path runs install-updater out-of-band); - `code` self-tests the new updater and `--no-block` restarts onto it only if - it imports, else the next reboot adopts it. Never raises: a completed update - must stand regardless. - """ + """Apply a daemon restart/reinstall hook deferred during this batch.""" try: sentinel = restart_sentinel_path() reason = await asyncio.to_thread(self._read_clear_sentinel, sentinel) @@ -846,10 +829,7 @@ async def recover(self, name: str, hard: bool = False) -> bool: return ok async def bless_healthy(self, name: str, hash_val: str) -> bool: - """Mark a component healthy (known-good) after a debounce; snapshots baseline. - - Empty hash_val blesses the current HEAD (UI need not compute a hash). - """ + """Mark a component healthy (known-good) after a debounce; snapshots baseline.""" component = next((c for c in self._components if c.name == name), None) if component is None: self._log.error("bless_healthy: component %s not found", name) @@ -862,6 +842,7 @@ async def bless_healthy(self, name: str, hash_val: str) -> bool: nrestarts_baseline = await get_service_nrestarts(_UI_SERVICE) def _apply(state: dict) -> None: + """Set last_good, seed golden once, reset the attempt counter and baseline.""" comp = _ensure_comp(state, name) comp["last_good"] = hash_val if not _is_sha(comp.get("golden")): @@ -883,10 +864,7 @@ def _apply(state: dict) -> None: return ok def _check_crash_loop(self, name: str, nrestarts: int) -> bool: - """Return True if NRestarts rose by >= 5 within the trailing 180s window. - - Compared against the oldest in-window sample; records+prunes samples. - """ + """Return True if NRestarts rose by >= 5 within the trailing 180s window.""" if name not in self._nrestarts_samples: self._nrestarts_samples[name] = [] # fresh device: start tracking now samples = self._nrestarts_samples[name] @@ -971,6 +949,15 @@ async def run_recovery_rung(self, name: str, attempt: int) -> bool: "recovery rung 2: %s unresolved", _HEAL_REMOTE_REF ) return False + # Never heal the host onto a pre-updater tip (would re-brick, not fix). + if name == _UI_COMPONENT and not await git_tree_has_path( + component.path, _HEAL_REMOTE_REF, _UPDATER_MARKER + ): + self._log.error( + "recovery rung 2: %s lacks the updater package - skipping", + _HEAL_REMOTE_REF, + ) + return False ok_reset, _ = await git_reset_to_hash(component.path, _HEAL_REMOTE_REF) if not ok_reset: self._log.error("reset to %s failed", _HEAL_REMOTE_REF) @@ -1016,12 +1003,7 @@ def _rebaseline_samples(self, name: str, nrestarts: int) -> None: self._nrestarts_samples[name] = [(time.monotonic(), nrestarts)] async def supervise_ui(self) -> None: - """Poll the UI service NRestarts and drive the recovery ladder on a crash-loop. - - The loop itself is the single-attempt-in-flight guard: at most one rung per - crash-loop, then detection re-baselines and a settle window elapses so a new - build can bless before the next rung is counted. - """ + """Poll the UI service NRestarts and drive the recovery ladder on a crash-loop.""" while True: await asyncio.sleep(_NRESTARTS_POLL_INTERVAL_S) try: @@ -1053,10 +1035,7 @@ async def _handle_crash_loop(self, nrestarts: int) -> None: await asyncio.sleep(_RECOVERY_SETTLE_S) async def forward_heal_ui(self) -> None: - """Slowly re-check origin/main and re-attempt an upgrade once a fix ships. - - Jittered + connectivity-gated: avoids a fleet stampede and offline churn. - """ + """Slowly re-check origin/main and re-attempt an upgrade once a fix ships.""" while True: delay = _FORWARD_HEAL_BASE_S + secrets.randbelow( int(_FORWARD_HEAL_JITTER_S) + 1 @@ -1086,6 +1065,15 @@ async def _forward_heal_once(self) -> bool: tip = await git_ref_hash(component.path, _HEAL_REMOTE_REF) if not tip or tip == comp_state.get("last_failed_remote"): return False # no new stable tip since the last failure + # Never forward-heal onto a pre-updater tip (would brick, not upgrade). + if not await git_tree_has_path( + component.path, _HEAL_REMOTE_REF, _UPDATER_MARKER + ): + self._log.warning( + "forward-heal: %s lacks the updater package - skipping", + _HEAL_REMOTE_REF, + ) + return False self._log.info( "forward-heal: new %s tip %s, upgrading", _HEAL_REMOTE_REF, tip[:8] ) @@ -1095,6 +1083,7 @@ async def _forward_heal_once(self) -> bool: return False def _apply(s: dict) -> None: + """Reset the attempt counter and record the healed tip.""" comp = _ensure_comp(s, _UI_COMPONENT) comp["fast_attempt"] = 0 comp["last_failed_remote"] = tip @@ -1126,15 +1115,7 @@ def _clear_fault_marker(self) -> None: self._log.warning("failed to clear self-heal fault marker") async def reconcile(self) -> None: - """Heal repos left damaged by a power loss mid-update, for every component. - - start.sh only repairs BlocksScreen; this covers klipper/moonraker/config too. - The gate is a cheap `rev-parse HEAD` so a healthy boot pays almost nothing; - deeper corruption is caught on demand. Offline-safe: git_repair fetches when - it can, else resets to the last recorded good hash. Held under the process - lock so the boot heal can't interleave with a user update/CLI run; if another - updater holds it, skip (that run is already mutating; damage re-detects later). - """ + """Heal repos left damaged by a power loss mid-update, for every component.""" with process_lock() as acquired: if not acquired: self._log.info( @@ -1144,12 +1125,7 @@ async def reconcile(self) -> None: await self._reconcile_locked() async def _revert_inflight(self) -> None: - """Revert any update cut off mid-flight by a power loss before it committed. - - The in-flight marker maps each batch component to its pre-staging commit; its - presence at boot means a batch didn't finish, so each listed repo is - hard-reset back and the marker cleared. A repo already there is left alone. - """ + """Revert any update cut off mid-flight by a power loss before it committed.""" inflight = await asyncio.to_thread(self._read_inflight) if not inflight: return @@ -1179,6 +1155,7 @@ async def _revert_inflight(self) -> None: await asyncio.to_thread(self._clear_inflight) async def _reconcile_locked(self) -> None: + """Boot heal under the lock: revert in-flight, repair repos, run reclone hooks, sanitize state.""" await self._revert_inflight() self._log.info("reconcile: boot-checking git components for damage") recloned: list[ComponentConfig] = [] @@ -1214,8 +1191,7 @@ async def _reconcile_locked(self) -> None: self._log.error( "reconcile: %s unrepairable, no recorded prev_hash", c.name ) - # Outside _git_lock: a reclone drops in-repo artifacts (e.g. Spoolman's - # .venv); the hook rebuilds them, best-effort, without blocking status. + # Reclone drops in-repo artifacts (e.g. Spoolman's .venv); hook rebuilds them best-effort outside _git_lock. for c in recloned: try: new_hash = await git_get_hash(c.path) @@ -1278,6 +1254,7 @@ async def _reconcile_self_heal_state(self) -> None: async def _rollback( self, component: ComponentConfig, prev_hash: str, reason: str ) -> None: + """Reset one component to prev_hash, restart its service, and report failure.""" self._log.warning( "%s: rolling back to %s (reason=%s)", component.name, prev_hash[:8], reason ) @@ -1333,11 +1310,7 @@ async def _install_dependencies( async def _ping_while( self, coro, name: str, step: int, total: int, interval: float = 60.0 ): - """Await coro, re-emitting on_step every `interval` seconds. - - A HOOK_TIMEOUT-bounded hook can run silent past the client's 360s busy - watchdog; the pings keep it fed. Cancellation propagates to the inner task. - """ + """Await coro, re-emitting on_step every `interval` seconds.""" task = asyncio.ensure_future(coro) try: while True: @@ -1369,15 +1342,13 @@ async def _shielded(self, coro, label: str) -> None: return def _cb_error_done(self, name: str, reason: str) -> bool: + """Emit on_error and on_component_done(False); return False.""" self._cb("on_error", name, reason) self._cb("on_component_done", name, False) return False def _history(self, event: str, name: str, **fields: object) -> None: - """Append one event to the persistent update-history log (OBS-1). - - journald is volatile here, so this on-SD JSONL is the durable field record. - """ + """Append one event to the persistent update-history log (OBS-1).""" entry = { "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "event": event, @@ -1436,11 +1407,7 @@ async def _remove_clone(self, component: ComponentConfig) -> None: await asyncio.to_thread(shutil.rmtree, component.path, ignore_errors=True) async def _fail_provision(self, component: ComponentConfig, reason: str) -> bool: - """Remove the partial clone, log, and report failure. - - No prev_hash to roll back to, so the clean slate is deleting the clone. - The unit is enabled only after a clean start, so nothing is left enabled. - """ + """Remove the partial clone, log, and report failure.""" await self._remove_clone(component) self._history("install_failed", component.name, reason=reason) self._log.warning( @@ -1449,12 +1416,7 @@ async def _fail_provision(self, component: ComponentConfig, reason: str) -> bool return self._cb_error_done(component.name, reason) async def _provision_component(self, component: ComponentConfig) -> bool: - """Clone and set up a missing opted-in component. - - A fresh component has no prev_hash, so any failure deletes the partial clone - (clean slate to retry) instead of rolling back. Service install is delegated - to the component's hook, as in the update flow. - """ + """Clone and set up a missing opted-in component.""" if not component.url: return self._cb_error_done(component.name, "no clone url") self._log.info("%s: provisioning via clone %s", component.name, component.url) @@ -1575,12 +1537,17 @@ async def _reclone_component(self, component: ComponentConfig) -> bool: self._log.warning("%s: recloned successfully", component.name) return True - async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str]: - """Bring the working tree to its target ref: fetch-gate + reset/checkout/pull. + @staticmethod + def _self_update_target(component: ComponentConfig) -> str: + """The ref _stage_component would move the host to (version/branch/HEAD).""" + if component.version: + return component.version + if component.branch: + return f"origin/{component.branch}" + return "origin/HEAD" - No deps/hook/restart and no rollback (caller decides on failure). Returns - (ok, reason), reason one of network/corrupt/reset/version/branch/conflict. - """ + async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str]: + """Bring the working tree to its target ref: fetch-gate + reset/checkout/pull.""" if component.path is None: return (False, "path not found") async with self._git_lock: @@ -1613,6 +1580,19 @@ async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str] f"branch origin/{component.branch} not found - fix components.yaml", ) + # Updater host must never checkout code lacking updater: Type=notify unit lacks sd_notify READY causes crash loop with no self-heal. + if component.name == _UI_COMPONENT: + target = self._self_update_target(component) + if target and not await git_tree_has_path( + component.path, target, _UPDATER_MARKER + ): + self._log.error( + "%s: target %s lacks the updater package - refusing brick downgrade", + component.name, + target, + ) + return (False, "refusing downgrade past updater") + # Switch to the target branch FIRST so reset/pull act on the right branch. if component.branch: # hard mode forces past untracked collisions (build artifacts). @@ -1663,11 +1643,7 @@ async def _restart_one(self, service: str) -> bool: return True async def _run_git_update(self, component: ComponentConfig) -> bool: - """Update a single existing git component, rolling back on any failure. - - Shares its git mechanics (_stage_component) with the batch; keeps its own - per-component rollback since a single update has no peers to coordinate. - """ + """Update a single existing git component, rolling back on any failure.""" if component.path is None: return self._cb_error_done(component.name, "path not found") if not component.path.exists(): @@ -1719,6 +1695,7 @@ async def _run_git_update(self, component: ComponentConfig) -> bool: if ( stage_reason in ("network", "corrupt") or "not found" in stage_reason + or "refusing" in stage_reason ): await asyncio.to_thread(self._clear_inflight) return self._cb_error_done(component.name, stage_reason) @@ -1840,6 +1817,7 @@ async def _run_apt_update(self, component: ComponentConfig) -> bool: async def _run_apt_update_locked( self, component: ComponentConfig, _apt_cache: Path ) -> bool: + """apt refresh, upgrade, and autoremove under the apt lock with snapshot rollback.""" try: if self._apt_backoff.cooling_down(): self._log.debug("apt cooling down; skipping update") @@ -1904,6 +1882,7 @@ async def _run_apt_update_locked( raise def _read_state(self) -> dict: + """Return the self-heal state dict, empty on missing/corrupt/wrong-shape.""" try: data = json.loads(self._state_path.read_text()) except (FileNotFoundError, json.JSONDecodeError): @@ -1912,6 +1891,7 @@ def _read_state(self) -> dict: return data if isinstance(data, dict) else {} def _read_inflight(self) -> dict[str, str]: + """Return the in-flight batch marker as a name->prev_hash dict.""" try: data = json.loads(self._inflight_path.read_text()) except (FileNotFoundError, json.JSONDecodeError): @@ -1952,6 +1932,7 @@ def _clear_inflight(self) -> None: self._log.warning("Failed to clear in-flight marker") def _write_state(self, data: dict) -> bool: + """Atomically write the self-heal state file (temp, fsync, replace).""" try: self._state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) # SEC: atomic write via temp file prevents symlink attacks and partial writes @@ -1983,6 +1964,7 @@ async def _mutate_state(self, mutate: Callable[[dict], None]) -> bool: @staticmethod def _fsync_dir(path: Path) -> None: + """fsync a directory so a contained rename survives power loss.""" try: fd = os.open(path, os.O_RDONLY) try: @@ -1993,14 +1975,12 @@ def _fsync_dir(path: Path) -> None: pass async def background_apt_upgrade(self) -> None: - """Run apt update + upgrade + autoremove silently after every Update click. - - apt-get upgrade (never dist-upgrade) so the release can't change; UI-silent. - """ + """Run apt update + upgrade + autoremove silently after every Update click.""" async with self._apt_lock: await self._background_apt_upgrade_locked() async def _background_apt_upgrade_locked(self) -> None: + """Silent apt update, upgrade, and autoremove honoring the apt excludes.""" if self._apt_backoff.cooling_down(): self._log.debug("apt cooling down; skipping background upgrade") return From acb4eb29d3ac07783ddedbd07948d6423ef41b64 Mon Sep 17 00:00:00 2001 From: Guilherme Costa Date: Tue, 21 Jul 2026 10:45:55 +0100 Subject: [PATCH 6/6] test(updater): mock git_ref_hash and _assert_https_remote in update-flow unit tests --- tests/updater/test_service_unit.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/updater/test_service_unit.py b/tests/updater/test_service_unit.py index d85f089b..ca67500c 100644 --- a/tests/updater/test_service_unit.py +++ b/tests/updater/test_service_unit.py @@ -266,6 +266,11 @@ async def test_emits_step_progress_in_order(self, tmp_path): patch("updater.service.git_checkout", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_pull", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://github.com/test/repo"), + ), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -321,6 +326,11 @@ async def test_rollback_on_service_restart_failure(self): patch("updater.service.git_checkout", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_pull", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://github.com/test/repo"), + ), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -346,6 +356,11 @@ async def test_fetch_skipped_when_snapshot_fresh(self): patch("updater.service.git_checkout", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_pull", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://github.com/test/repo"), + ), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -2230,6 +2245,7 @@ async def test_single_update_passes_hook_timeout(self, tmp_path): patch("updater.service.git_pull", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_get_current_branch", return_value="master"), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""),