diff --git a/scripts/BlocksScreen-deploy.service b/scripts/BlocksScreen-deploy.service index 735c3f44..358d1111 100644 --- a/scripts/BlocksScreen-deploy.service +++ b/scripts/BlocksScreen-deploy.service @@ -6,6 +6,8 @@ DefaultDependencies=no Type=oneshot User=root ExecStart=/bin/bash BS_DIR/scripts/install-updater.sh -ExecStartPost=/bin/rm -f BS_PRIMARY_HOME/.config/blockscreen/.run-install-updater +# ExecStopPost also runs when ExecStart fails: a persistent failure must not +# leave the flag set and storm the path unit into its start limit. +ExecStopPost=-/bin/rm -f BS_PRIMARY_HOME/.config/blockscreen/.run-install-updater StandardOutput=journal StandardError=journal diff --git a/scripts/BlocksScreen-start.sh b/scripts/BlocksScreen-start.sh index 1c22bc67..eef1d15e 100755 --- a/scripts/BlocksScreen-start.sh +++ b/scripts/BlocksScreen-start.sh @@ -92,6 +92,16 @@ git -C "$BS_PATH" config core.hooksPath scripts 2>/dev/null || true bs_migrate_moonraker_conf "$_BSENV_HOME/printer_data/config/moonraker.conf" BlocksScreen-start +# Serialize the boot git surgery below with the updater daemon/CLI (their flock): +# the daemon's boot reconcile can heal this same repo concurrently. Proceed after +# 20s rather than hold up the UI; the fd is closed before exec'ing the UI. +_BS_LOCK_DIR="/run/blockscreen" +mkdir -p "$_BS_LOCK_DIR" 2>/dev/null || _BS_LOCK_DIR="$_BSENV_HOME/.cache/blockscreen" +if exec 9>"$_BS_LOCK_DIR/updater.lock" 2>/dev/null; then + flock -w 20 9 2>/dev/null \ + || echo "[BlocksScreen-start] updater lock busy - proceeding without it" +fi + # Remove stale git index lock left by an interrupted update (e.g. power loss during git reset) rm -f "$BS_PATH/.git/index.lock" @@ -148,6 +158,9 @@ if ! "$BSENV/bin/python3.11" -m compileall -q \ git -C "$BS_PATH" reset --hard HEAD 2>/dev/null || true fi +# Release the updater lock now: the exec'd UI must never inherit (and hold) it. +exec 9>&- 2>/dev/null || true + # -- Deferred bootstrap (background) ------------------------------------------ # Heavy / network-dependent setup (apt packages, pip requirements, updater # install, splash-holder unit, splash precompute) runs detached in a transient diff --git a/scripts/bs-bootstrap.sh b/scripts/bs-bootstrap.sh index cb41b8da..5c0ff018 100755 --- a/scripts/bs-bootstrap.sh +++ b/scripts/bs-bootstrap.sh @@ -25,8 +25,8 @@ command -v feh >/dev/null 2>&1 || _missing="$_missing feh" command -v xsetroot >/dev/null 2>&1 || _missing="$_missing x11-xserver-utils" if [ -n "$_missing" ]; then # shellcheck disable=SC2086 - apt-get install -y --quiet $_missing 2>/dev/null \ - || echo "[bs-bootstrap] apt install deferred (offline?):$_missing" + apt-get -o DPkg::Lock::Timeout=60 install -y --quiet $_missing 2>/dev/null \ + || echo "[bs-bootstrap] apt install deferred (offline or dpkg busy):$_missing" fi # 2. Python deps. PyYAML is required by the updater daemon's component loader. diff --git a/scripts/bs-common.sh b/scripts/bs-common.sh index 34ea4f08..1c3ee74a 100644 --- a/scripts/bs-common.sh +++ b/scripts/bs-common.sh @@ -6,18 +6,31 @@ bs_migrate_moonraker_conf() { local conf="$1" tag="${2:-bs-common}" patched=false [ -f "$conf" ] || return 0 - cp "$conf" "${conf}.bak" 2>/dev/null || true + # Back up only a sane-looking conf: never overwrite a good .bak with a corrupt one. + if grep -q '^\[server\]' "$conf" 2>/dev/null; then + cp "$conf" "${conf}.bak" 2>/dev/null || true + fi # Gate on the section existing: else the sed no-ops but patched=true restarts moonraker every boot. if ! grep -q "enable_system_updates" "$conf" && grep -q '^\[update_manager\]$' "$conf"; then sed -i '/^\[update_manager\]$/a enable_system_updates: False' "$conf" patched=true echo "[$tag] moonraker.conf: disabled system apt upgrades" fi - if grep -q "managed_services: klipper moonraker" "$conf"; then + # Scope checks to the BlocksScreen section so an unrelated match can't re-patch every boot. + local bs_um_section + bs_um_section=$(sed -n '/^\[update_manager BlocksScreen\]/,/^\[/p' "$conf" 2>/dev/null) + if printf '%s\n' "$bs_um_section" | grep -q "managed_services: klipper moonraker"; then sed -i '/^\[update_manager BlocksScreen\]/,/^\[/ s/managed_services: klipper moonraker/managed_services: BlocksScreen/' "$conf" patched=true echo "[$tag] moonraker.conf: fixed BlocksScreen managed_services" fi + # Old installers wrote primary_branch: master, which does not exist upstream; + # moonraker refuses to manage a repo whose primary branch is gone. + if printf '%s\n' "$bs_um_section" | grep -q "^primary_branch: master$"; then + sed -i '/^\[update_manager BlocksScreen\]/,/^\[/ s/^primary_branch: master$/primary_branch: main/' "$conf" + patched=true + echo "[$tag] moonraker.conf: fixed BlocksScreen primary_branch (master -> main)" + fi if ! grep -q "blocksscreen-single-owner" "$conf"; then bs_disable_overlapping_update_managers "$conf" "$tag" && patched=true fi @@ -37,7 +50,15 @@ bs_ensure_spoolman_moonraker() { local spoolman_dir="${conf%/printer_data/config/moonraker.conf}/Spoolman" [ -d "$spoolman_dir/.venv" ] || return 1 grep -q '^\[spoolman\]' "$conf" && return 1 - printf '\n[spoolman]\nserver: localhost:7912\n' >>"$conf" + # Same-dir temp + atomic rename: a power cut mid-append could truncate the conf. + local tmp + tmp="$(mktemp -p "$(dirname "$conf")" .moonraker.conf.XXXXXX)" || return 1 + if ! { cat "$conf" && printf '\n[spoolman]\nserver: localhost:7912\n'; } >"$tmp"; then + rm -f "$tmp" + return 1 + fi + chmod --reference="$conf" "$tmp" 2>/dev/null || true + mv -f "$tmp" "$conf" echo "[$tag] moonraker.conf: added [spoolman] (server: localhost:7912)" return 0 } @@ -48,7 +69,7 @@ bs_disable_overlapping_update_managers() { local conf="$1" tag="${2:-bs-common}" [ -f "$conf" ] || return 1 grep -q "blocksscreen-single-owner" "$conf" && return 1 - local owned="RF50-Klipper happy-hare Klippain-ShakeTune mainsail-config crowsnest" + local owned="klipper RF50-Klipper happy-hare Klippain-ShakeTune mainsail-config crowsnest" local tmp # Same-dir temp + atomic rename: a power cut can never truncate moonraker.conf. tmp="$(mktemp -p "$(dirname "$conf")" .moonraker.conf.XXXXXX)" || return 1 diff --git a/scripts/bs-splash-holder.py b/scripts/bs-splash-holder.py index 378a4dee..560f0f0b 100644 --- a/scripts/bs-splash-holder.py +++ b/scripts/bs-splash-holder.py @@ -93,8 +93,13 @@ def _on_sigterm(_sig: int, _frame: object) -> None: signal.signal(signal.SIGTERM, _on_sigterm) -# Switch to tty8 immediately - VT switch does not require fb0. -_activate_tty8() +# Switch to tty8 immediately - VT switch does not require fb0. Skip when X is +# already up (first-install start lands mid-session): stealing the VT from the +# live GUI on tty7 would look like a bricked screen until a power cycle. +if os.path.exists("/tmp/.X11-unix/X0"): # nosec B108 - canonical X11 socket path + _log("X session present - not activating tty8") +else: + _activate_tty8() # Clear tty8 and hide the cursor (no bare text): the screen stays black until the # splash image is written to fb0 below, so the user only ever sees the logo splash. diff --git a/scripts/install-updater.sh b/scripts/install-updater.sh index 929134f5..2bdc7d73 100755 --- a/scripts/install-updater.sh +++ b/scripts/install-updater.sh @@ -20,13 +20,22 @@ _BSENV_USER="blocks" _BSENV_HOME=$(getent passwd "$_BSENV_USER" | cut -d: -f6) BSENV="${BLOCKSSCREEN_VENV:-${_BSENV_HOME}/.BlocksScreen-env}" +# Venv-mutating commands run as blocks: root-owned dists break later pip-as-blocks runs. +_as_blocks() { if [ "$(id -u)" = "0" ]; then runuser -u "$_BSENV_USER" -- "$@"; else "$@"; fi; } +# Atomic root install: a power cut must never leave a truncated-but-present file +# (the [ -f ] self-heal guards would then never rewrite it). +_install_atomic() { + local mode="$1" src="$2" dst="$3" + sudo install -m "$mode" "$src" "${dst}.new" && sudo mv -Tf "${dst}.new" "$dst" +} + echo_info "Installing D-Bus policy ..." -sudo cp "$SCRIPT_PATH/com.blockscreen.Updater.conf" /etc/dbus-1/system.d/ +_install_atomic 0644 "$SCRIPT_PATH/com.blockscreen.Updater.conf" /etc/dbus-1/system.d/com.blockscreen.Updater.conf sudo systemctl reload dbus || true echo_ok "D-Bus policy installed" echo_info "Installing D-Bus activation file ..." -sudo cp "$SCRIPT_PATH/com.blockscreen.Updater.service" /usr/share/dbus-1/system-services/ +_install_atomic 0644 "$SCRIPT_PATH/com.blockscreen.Updater.service" /usr/share/dbus-1/system-services/com.blockscreen.Updater.service echo_ok "D-Bus activation file installed" echo_info "Installing bs-apt-helper (root-owned apt wrapper) ..." @@ -40,7 +49,8 @@ SERVICE=$(cat "$SCRIPT_PATH/BlocksScreen-updater.service") SERVICE=${SERVICE//BS_DIR/$BS_PATH} SERVICE=${SERVICE//BSENV/$BSENV} SERVICE=${SERVICE//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$SERVICE" | sudo tee /etc/systemd/system/BlocksScreen-updater.service >/dev/null +echo "$SERVICE" | sudo tee /etc/systemd/system/.BlocksScreen-updater.service.new >/dev/null +sudo mv -Tf /etc/systemd/system/.BlocksScreen-updater.service.new /etc/systemd/system/BlocksScreen-updater.service sudo systemctl daemon-reload sudo systemctl enable BlocksScreen-updater.service sudo systemctl restart BlocksScreen-updater.service || true @@ -49,7 +59,8 @@ echo_ok "BlocksScreen-updater service installed and started" echo_info "Installing Spoolman service unit (inactive until the updater provisions it) ..." SPOOLMAN_SVC=$(cat "$SCRIPT_PATH/Spoolman.service") SPOOLMAN_SVC=${SPOOLMAN_SVC//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$SPOOLMAN_SVC" | sudo tee /etc/systemd/system/Spoolman.service >/dev/null +echo "$SPOOLMAN_SVC" | sudo tee /etc/systemd/system/.Spoolman.service.new >/dev/null +sudo mv -Tf /etc/systemd/system/.Spoolman.service.new /etc/systemd/system/Spoolman.service sudo systemctl daemon-reload echo_ok "Spoolman service unit installed" @@ -81,8 +92,10 @@ else fi # Daemon self-restart target; never in components.yaml. _emit_svc_rules BlocksScreen-updater.service -# Spoolman is provisioned on demand; enable rule needed for its first clean start. +# Spoolman is provisioned on demand; enable rules needed for its first clean start +# (hooks/Spoolman.sh uses `enable --now`; sudoers args must match exactly). printf 'blocks ALL=(ALL) NOPASSWD: /usr/bin/systemctl enable Spoolman.service\n' >>"$SUDOERS_TMP" +printf 'blocks ALL=(ALL) NOPASSWD: /usr/bin/systemctl enable --now Spoolman.service\n' >>"$SUDOERS_TMP" if sudo visudo -cf "$SUDOERS_TMP" >/dev/null 2>&1; then sudo install -m 0440 "$SUDOERS_TMP" "$SUDOERS_FILE" echo_ok "Sudoers rules installed" @@ -179,10 +192,12 @@ echo_info "Installing BlocksScreen-deploy path unit (sudo-free hook trigger) ... DEPLOY_SVC=$(cat "$SCRIPT_PATH/BlocksScreen-deploy.service") DEPLOY_SVC=${DEPLOY_SVC//BS_DIR/$BS_PATH} DEPLOY_SVC=${DEPLOY_SVC//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$DEPLOY_SVC" | sudo tee /etc/systemd/system/BlocksScreen-deploy.service >/dev/null +echo "$DEPLOY_SVC" | sudo tee /etc/systemd/system/.BlocksScreen-deploy.service.new >/dev/null +sudo mv -Tf /etc/systemd/system/.BlocksScreen-deploy.service.new /etc/systemd/system/BlocksScreen-deploy.service DEPLOY_PATH=$(cat "$SCRIPT_PATH/BlocksScreen-deploy.path") DEPLOY_PATH=${DEPLOY_PATH//BS_PRIMARY_HOME/$_BSENV_HOME} -echo "$DEPLOY_PATH" | sudo tee /etc/systemd/system/BlocksScreen-deploy.path >/dev/null +echo "$DEPLOY_PATH" | sudo tee /etc/systemd/system/.BlocksScreen-deploy.path.new >/dev/null +sudo mv -Tf /etc/systemd/system/.BlocksScreen-deploy.path.new /etc/systemd/system/BlocksScreen-deploy.path sudo systemctl daemon-reload sudo systemctl enable --now BlocksScreen-deploy.path echo_ok "BlocksScreen-deploy path unit installed" @@ -207,23 +222,23 @@ echo_ok "post-merge hook installed" echo_info "Installing Python requirements ..." # Best-effort pip self-update; must not block the requirements install below. -"$BSENV/bin/pip" install --quiet --upgrade pip 2>/dev/null || true -apt-get install -y --quiet libsystemd-dev python3-dev 2>/dev/null || true +_as_blocks "$BSENV/bin/pip" install --quiet --upgrade pip 2>/dev/null || true +sudo apt-get -o DPkg::Lock::Timeout=60 install -y --quiet libsystemd-dev python3-dev 2>/dev/null || true # xsetroot is used as belt-and-suspenders cursor hiding alongside the Xorg -nocursor server flag. -sudo apt-get install -y --quiet x11-xserver-utils 2>/dev/null || true +sudo apt-get -o DPkg::Lock::Timeout=60 install -y --quiet x11-xserver-utils 2>/dev/null || true # sdbus pinned; --no-binary needs libsystemd-dev; only-if-needed skips satisfied. -"$BSENV/bin/pip" install --quiet --only-binary :all: --no-binary sdbus,sdbus-networkmanager \ +_as_blocks "$BSENV/bin/pip" install --quiet --only-binary :all: --no-binary sdbus,sdbus-networkmanager \ --upgrade-strategy=only-if-needed \ -r "$BS_PATH/scripts/requirements.txt" || true echo_ok "Python requirements installed" # uv: Spoolman's dependency installer (run-from-source); provide it for the hook. echo_info "Ensuring uv is available for Spoolman provisioning ..." -"$BSENV/bin/pip" install --quiet --upgrade uv 2>/dev/null || true +_as_blocks "$BSENV/bin/pip" install --quiet --upgrade uv 2>/dev/null || true echo_ok "uv ready" echo_info "Pre-generating framebuffer splash cache ..." -"$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true +_as_blocks "$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true echo_ok "Splash cache ready" echo_ok "BlocksScreen updater setup complete" diff --git a/scripts/install.sh b/scripts/install.sh index 148c9a9e..53b3c46b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -413,11 +413,11 @@ function add_moonraker_update() { [update_manager BlocksScreen] type: git_repo -primary_branch: master +primary_branch: main path: $BS_PATH origin: https://github.com/BlocksTechnology/BlocksScreen.git is_system_service: True -managed_services: klipper moonraker +managed_services: BlocksScreen virtualenv: $BSENV requirements: scripts/requirements.txt system_dependencies: scripts/system-dependencies.json diff --git a/scripts/post-merge b/scripts/post-merge index b10c99f7..8eb5390e 100755 --- a/scripts/post-merge +++ b/scripts/post-merge @@ -8,6 +8,10 @@ _BSENV_USER="blocks" _BSENV_HOME=$(getent passwd "$_BSENV_USER" | cut -d: -f6) BSENV="${BLOCKSSCREEN_VENV:-${_BSENV_HOME}/.BlocksScreen-env}" +# A root pull must not leave root-owned dists in the blocks venv (breaks later pip runs). +_as_blocks() { if [ "$(id -u)" = "0" ]; then runuser -u "$_BSENV_USER" -- "$@"; else "$@"; fi; } +_own() { [ "$(id -u)" = "0" ] && chown "$_BSENV_USER" "$1" 2>/dev/null || true; } + # shellcheck source=/dev/null if [ -f "$SCRIPT_PATH/bs-common.sh" ]; then . "$SCRIPT_PATH/bs-common.sh"; fi @@ -23,12 +27,13 @@ SENTINEL="$BSENV/.blockscreen-reqs-hash" ATTEMPT="$BSENV/.blockscreen-reqs-attempt" if [ ! -f "$SENTINEL" ] || [ "$(cat "$SENTINEL")" != "$REQS_HASH" ]; then echo "[post-merge] requirements changed - pre-installing deps (this may take a few minutes) ..." - if "$BSENV/bin/pip" install --quiet \ + if _as_blocks "$BSENV/bin/pip" install --quiet \ --only-binary :all: \ --no-binary sdbus,sdbus-networkmanager \ --upgrade-strategy=only-if-needed \ -r "$BS_PATH/scripts/requirements.txt"; then echo "$REQS_HASH" >"$SENTINEL" + _own "$SENTINEL" rm -f "$ATTEMPT" 2>/dev/null || true echo "[post-merge] deps installed" else @@ -38,9 +43,11 @@ if [ ! -f "$SENTINEL" ] || [ "$(cat "$SENTINEL")" != "$REQS_HASH" ]; then case "$_cnt" in '' | *[!0-9]*) _cnt=0 ;; esac if [ "$_prev" = "$REQS_HASH" ]; then _cnt=$((_cnt + 1)); else _cnt=1; fi echo "$REQS_HASH $_cnt" >"$ATTEMPT" + _own "$ATTEMPT" if [ "$_cnt" -ge 3 ]; then echo "[post-merge] deps failed ${_cnt}x; giving up to stop thrash (venv left as-is)" echo "$REQS_HASH" >"$SENTINEL" + _own "$SENTINEL" fi fi fi @@ -55,7 +62,7 @@ if [ ! -f /etc/systemd/system/BlocksScreen-updater.service ]; then fi # 4. Precompute splash cache so the first Qt restart shows the right logo. -"$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true +_as_blocks "$BSENV/bin/python3.11" "$SCRIPT_PATH/bs-splash.py" --precompute 2>/dev/null || true # Under the daemon's own pull a restart here aborts the batch: defer via sentinel. diff --git a/tests/scripts/test_moonraker_ownership.py b/tests/scripts/test_moonraker_ownership.py index a62d1952..7550eb7a 100644 --- a/tests/scripts/test_moonraker_ownership.py +++ b/tests/scripts/test_moonraker_ownership.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import subprocess from pathlib import Path @@ -14,6 +15,10 @@ [update_manager mainsail] type: web +[update_manager klipper] +type: git_repo +path: ~/klipper + [update_manager RF50-Klipper] type: git_repo path: ~/RF50-Klipper @@ -44,6 +49,7 @@ def test_owned_blocks_commented_others_untouched(tmp_path: Path) -> None: conf = tmp_path / "moonraker.conf" conf.write_text(_CONF) out = _run(conf) + assert "#[update_manager klipper]" in out # updater owns klipper, not moonraker assert "#[update_manager RF50-Klipper]" in out assert "#[update_manager crowsnest]" in out assert "[update_manager mainsail]" in out # web client kept @@ -57,3 +63,65 @@ def test_idempotent(tmp_path: Path) -> None: first = _run(conf) second = _run(conf) assert first == second # no double-comment on re-run + + +_MIGRATE_CONF = """\ +[server] +host: 0.0.0.0 + +[update_manager] +enable_system_updates: False + +[update_manager BlocksScreen] +type: git_repo +primary_branch: master +managed_services: klipper moonraker + +[update_manager other] +primary_branch: master +""" + + +def _migrate(conf: Path) -> str: + # Shim sudo: the function ends with `sudo systemctl restart moonraker`; real + # sudo would prompt on a dev TTY (test hang) or bounce services on the printer. + shim = conf.parent / "bin" + shim.mkdir(exist_ok=True) + fake_sudo = shim / "sudo" + fake_sudo.write_text("#!/bin/sh\nexit 0\n") + fake_sudo.chmod(0o755) + env = {**os.environ, "PATH": f"{shim}:{os.environ.get('PATH', '')}"} + subprocess.run( + [ + "bash", + "-c", + f'. "{_FN}"; bs_migrate_moonraker_conf "{conf}" test', + ], + check=False, + capture_output=True, + text=True, + env=env, + timeout=20, + ) + return conf.read_text() + + +def test_migrate_fixes_blocksscreen_branch_and_services(tmp_path: Path) -> None: + conf = tmp_path / "moonraker.conf" + conf.write_text(_MIGRATE_CONF) + out = _migrate(conf) + bs = out.split("[update_manager BlocksScreen]")[1].split("[update_manager other]")[ + 0 + ] + assert "primary_branch: main" in bs # origin/master does not exist upstream + assert "managed_services: BlocksScreen" in bs + # Patches are scoped to the BlocksScreen section: others keep their values. + assert "primary_branch: master" in out.split("[update_manager other]")[1] + + +def test_migrate_is_idempotent(tmp_path: Path) -> None: + conf = tmp_path / "moonraker.conf" + conf.write_text(_MIGRATE_CONF) + first = _migrate(conf) + second = _migrate(conf) + assert first == second diff --git a/tests/updater/test_components_unit.py b/tests/updater/test_components_unit.py index b119fd1a..49eaa534 100644 --- a/tests/updater/test_components_unit.py +++ b/tests/updater/test_components_unit.py @@ -286,6 +286,44 @@ def test_restart_klipper_flag_parsed(self): cfg = next(c for c in components if c.name == "cfg") assert cfg.restart_klipper is True + def test_loopback_health_url_parsed(self): + yaml_text = """ +components: + - name: cfg + type: git + path: ~/cfg + branch: master + order: 10 + health_url: http://127.0.0.1:7912/api/v1/health +""" + with ( + patch("builtins.open", _mock_load(yaml_text)), + patch("pathlib.Path.exists", return_value=False), + ): + components, _ = load_components() + cfg = next(c for c in components if c.name == "cfg") + assert cfg.health_url == "http://127.0.0.1:7912/api/v1/health" + + def test_non_loopback_health_url_dropped(self, caplog): + yaml_text = """ +components: + - name: cfg + type: git + path: ~/cfg + branch: master + order: 10 + health_url: http://evil.example.com/api/v1/health +""" + with ( + patch("builtins.open", _mock_load(yaml_text)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + components, _ = load_components() + cfg = next(c for c in components if c.name == "cfg") + assert cfg.health_url is None + assert any("health_url" in r.message for r in caplog.records) + def test_invalid_branch_name_skipped(self, caplog): bad_yaml = """ components: @@ -438,3 +476,60 @@ def test_defaults_when_absent(self): assert cfg is not None assert cfg.url is None assert cfg.install_if_missing is False + + +def test_invalid_reset_mode_coerced_to_hard(caplog): + bad_yaml = """ +components: + - name: klipper + type: git + path: ~/klipper + reset_mode: Hard + order: 10 +""" + with ( + patch("builtins.open", _mock_load(bad_yaml)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + configs, _ = load_components() + assert configs[-1].reset_mode == "hard" # typo must not become the soft path + assert any("invalid reset_mode" in r.message for r in caplog.records) + + +def test_non_dict_component_entry_skipped(caplog): + bad_yaml = """ +components: + - just-a-string + - name: klipper + type: git + path: ~/klipper + order: 10 +""" + with ( + patch("builtins.open", _mock_load(bad_yaml)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + configs, _ = load_components() + assert [c.name for c in configs if c.kind == "git"] == ["klipper"] + + +def test_origin_prefixed_branch_is_stripped(caplog): + bad_yaml = """ +components: + - name: BlocksScreen + type: git + path: ~/BlocksScreen + branch: origin/Release-2.0 + order: 99 +""" + with ( + patch("builtins.open", _mock_load(bad_yaml)), + patch("pathlib.Path.exists", return_value=False), + caplog.at_level(logging.WARNING, logger="updater.components"), + ): + configs, _ = load_components() + bs = next(c for c in configs if c.name == "BlocksScreen") + assert bs.branch == "Release-2.0" # stray origin/ prefix stripped + assert any("origin/" in r.message for r in caplog.records) diff --git a/tests/updater/test_executor_unit.py b/tests/updater/test_executor_unit.py index af1d3fc7..83676809 100644 --- a/tests/updater/test_executor_unit.py +++ b/tests/updater/test_executor_unit.py @@ -16,6 +16,7 @@ _make_clean_env, _repair_corrupt_head, _remove_broken_loose_ref, + git_default_branch, _run, apt_update, apt_upgrade, @@ -40,6 +41,7 @@ restart_service, restart_service_noblock, run_hook, + wait_for_http_ready, ) @@ -841,6 +843,23 @@ async def test_branch_mismatch_flagged_when_on_wrong_branch(self, tmp_path): assert result.commits_behind == 0 assert result.branch_mismatch is True + @pytest.mark.asyncio + async def test_dead_configured_branch_reports_actionable_error(self, tmp_path): + # origin/ deleted upstream: report "fix components.yaml", not the + # generic git_commits_behind failure. + procs = [ + _make_proc(0, b"abc1234\n", b""), # git_get_hash + _make_proc(0, b"wip/dead\n", b""), # git_get_current_branch + _make_proc(128, b"", b"fatal: unknown revision\n"), # git_commits_behind -1 + _make_proc(128, b"", b"fatal: unknown revision\n"), # git_ref_hash -> "" + ] + exec_mock = AsyncMock(side_effect=procs) + with patch("asyncio.create_subprocess_exec", exec_mock): + result = await check_git_status( + "klipper", tmp_path, branch="wip/dead", skip_fetch=True + ) + assert result.error == "branch origin/wip/dead not found - fix components.yaml" + class TestCheckAptStatus: @pytest.mark.asyncio @@ -1281,3 +1300,53 @@ async def test_has_corruption_detects_inflate_error(self): "asyncio.create_subprocess_exec", new_callable=AsyncMock, return_value=proc ): assert await git_has_corruption(Path("/x")) is True + + +class TestGitDefaultBranch: + """git_default_branch: HEAD symref, else origin/HEAD target, else master.""" + + @pytest.mark.asyncio + async def test_prefers_head_symref(self, tmp_path): + with patch( + "updater.executor._run", return_value=(True, "master\n") + ) as mock_run: + assert await git_default_branch(tmp_path) == "master" + assert mock_run.call_count == 1 + + @pytest.mark.asyncio + async def test_falls_back_to_origin_head(self, tmp_path): + with patch( + "updater.executor._run", + side_effect=[(False, ""), (True, "origin/master\n")], + ): + assert await git_default_branch(tmp_path) == "master" + + @pytest.mark.asyncio + async def test_last_resort_is_master(self, tmp_path): + with patch("updater.executor._run", side_effect=[(False, ""), (False, "")]): + assert await git_default_branch(tmp_path) == "master" + + @pytest.mark.asyncio + async def test_none_path_returns_master(self): + assert await git_default_branch(None) == "master" + + +class TestWaitForHttpReady: + @pytest.mark.asyncio + async def test_returns_true_on_2xx(self): + with patch("updater.executor._http_probe", return_value=True): + assert await wait_for_http_ready("http://127.0.0.1:7912/x") is True + + @pytest.mark.asyncio + async def test_times_out_when_never_ready(self): + with patch("updater.executor._http_probe", return_value=False): + assert await wait_for_http_ready("http://127.0.0.1:7912/x", timeout=0) is False + + @pytest.mark.asyncio + async def test_polls_until_ready(self): + with ( + patch("updater.executor._http_probe", side_effect=[False, True]) as probe, + patch("updater.executor.asyncio.sleep", new=AsyncMock()), + ): + assert await wait_for_http_ready("http://127.0.0.1:7912/x") is True + assert probe.call_count == 2 diff --git a/tests/updater/test_self_heal_unit.py b/tests/updater/test_self_heal_unit.py index 358d8094..e52f558e 100644 --- a/tests/updater/test_self_heal_unit.py +++ b/tests/updater/test_self_heal_unit.py @@ -464,6 +464,7 @@ async def run_test(): patch("updater.service.git_fetch") as m_fetch, patch("updater.service.git_ref_hash") as m_ref, patch("updater.service.git_reset_to_hash") as m_reset, + patch("updater.service.git_tree_has_path", return_value=True), patch.object(svc, "_restart_ui_service") as m_restart, ): m_fetch.return_value = (True, "") @@ -492,6 +493,7 @@ async def run_test(): patch("updater.service.git_fetch") as m_fetch, patch("updater.service.git_ref_hash") as m_ref, patch("updater.service.git_reset_to_hash") as m_reset, + patch("updater.service.git_tree_has_path", return_value=True), patch.object(svc, "_restart_ui_service") as m_restart, ): m_fetch.return_value = (True, "") @@ -615,6 +617,7 @@ async def run_test(): patch("updater.service.git_fetch") as m_fetch, patch("updater.service.git_ref_hash") as m_ref, patch("updater.service.git_reset_to_hash") as m_reset, + patch("updater.service.git_tree_has_path", return_value=True), patch("updater.service.get_service_nrestarts") as m_nr, patch.object(svc, "_restart_ui_service") as m_restart, ): @@ -750,6 +753,7 @@ async def slow_reset(_path, _ref): with ( patch("updater.service.git_fetch", return_value=(True, "")), patch("updater.service.git_ref_hash", return_value="c" * 40), + patch("updater.service.git_tree_has_path", return_value=True), patch("updater.service.git_reset_to_hash", side_effect=slow_reset), patch("updater.service.get_service_nrestarts", return_value=0), patch.object(svc, "_restart_ui_service", return_value=True), @@ -871,6 +875,7 @@ async def run_test(): patch( "updater.service.git_repair", return_value=(False, "repair fail") ), + patch.object(UpdateService, "_reclone_component", return_value=False), ): await svc._reconcile_locked() # must not raise assert "BlocksScreen" not in svc._read_state() # entry sanitized away diff --git a/tests/updater/test_service_unit.py b/tests/updater/test_service_unit.py index 980cdf4c..bc638d00 100644 --- a/tests/updater/test_service_unit.py +++ b/tests/updater/test_service_unit.py @@ -205,12 +205,14 @@ def mock_wait_active(self): yield @pytest.mark.asyncio - async def test_rollback_on_fetch_failure(self): + async def test_fetch_failure_reports_network_without_rollback(self): + # Nothing is checked out before the fetch: no revert, no service restart. cb = MagicMock() with ( patch("pathlib.Path.exists", return_value=True), patch("updater.service.git_get_hash", return_value="abc123"), patch("updater.service.git_fetch", return_value=(False, "timeout")), + patch("updater.service.git_has_corruption", return_value=False), patch( "updater.service.git_reset_to_hash", return_value=(True, "") ) as mock_reset, @@ -219,8 +221,9 @@ async def test_rollback_on_fetch_failure(self): svc = UpdateService(callback=cb) result = await svc.update_component("klipper") assert result is False - mock_reset.assert_called_once() - cb.on_rollback.assert_called_once_with("klipper", True) + mock_reset.assert_not_called() + cb.on_rollback.assert_not_called() + cb.on_error.assert_called_once_with("klipper", "network") cb.on_component_done.assert_called_once_with("klipper", False) @pytest.mark.asyncio @@ -263,6 +266,11 @@ async def test_emits_step_progress_in_order(self, tmp_path): patch("updater.service.git_checkout", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_pull", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://github.com/test/repo"), + ), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -318,6 +326,11 @@ async def test_rollback_on_service_restart_failure(self): patch("updater.service.git_checkout", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_pull", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://github.com/test/repo"), + ), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -343,6 +356,11 @@ async def test_fetch_skipped_when_snapshot_fresh(self): patch("updater.service.git_checkout", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_pull", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://github.com/test/repo"), + ), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -424,6 +442,7 @@ async def test_branch_hard_checks_out_then_resets_target(self): patch("pathlib.Path.exists", return_value=True), patch("updater.service.git_get_hash", return_value="oldhash"), patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch( "updater.service.git_checkout", side_effect=lambda *a, **k: call_order.append("checkout") or (True, ""), @@ -616,7 +635,7 @@ async def test_existing_git_go_to_one_atomic_batch_in_order(self, tmp_path): ] seen: list[list[str]] = [] with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_apt_update") as mock_apt, patch( "updater.service.UpdateService._run_git_batch", @@ -641,7 +660,7 @@ async def test_missing_opted_in_goes_to_provision_not_batch(self, tmp_path): install_if_missing=True, ) with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_git_batch") as mock_batch, patch( "updater.service.UpdateService._provision_component" @@ -656,7 +675,7 @@ async def test_missing_opted_in_goes_to_provision_not_batch(self, tmp_path): @pytest.mark.asyncio async def test_empty_batch_skips_git(self, tmp_path): with ( - patch("updater.service.UpdateService._preflight_fetch", return_value=True), + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), patch("updater.service.UpdateService._run_git_batch") as mock_batch, ): svc = UpdateService() @@ -665,9 +684,49 @@ async def test_empty_batch_skips_git(self, tmp_path): await svc.update_all() mock_batch.assert_not_called() + @pytest.mark.asyncio + async def test_dead_branch_dropped_not_aborting_batch(self, tmp_path): + # A deleted-upstream branch must be dropped so BlocksScreen still updates. + dead = self._git(tmp_path, "klipper", 2) + dead.branch = "wip/deleted" + good = self._git(tmp_path, "BlocksScreen", 99) + good.branch = "dev" + with ( + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), + patch("updater.service.UpdateService._run_git_batch") as mock_batch, + patch( + "updater.service.git_ref_hash", + side_effect=lambda path, ref: "" if "deleted" in ref else "a" * 40, + ), + ): + svc = UpdateService() + svc._components = [dead, good] + await svc.update_all() + assert [c.name for c in mock_batch.call_args[0][0]] == ["BlocksScreen"] + + @pytest.mark.asyncio + async def test_dead_current_branch_dropped_when_unconfigured(self, tmp_path): + # No configured branch + deleted current branch must also be dropped. + dead = self._git(tmp_path, "klipper", 2) # branch=None -> uses current + good = self._git(tmp_path, "BlocksScreen", 99) + good.branch = "dev" + with ( + patch("updater.service.UpdateService._preflight_fetch", return_value=set()), + patch("updater.service.UpdateService._run_git_batch") as mock_batch, + patch("updater.service.git_get_current_branch", return_value="wip/gone"), + patch( + "updater.service.git_ref_hash", + side_effect=lambda path, ref: "" if "gone" in ref else "a" * 40, + ), + ): + svc = UpdateService() + svc._components = [dead, good] + await svc.update_all() + assert [c.name for c in mock_batch.call_args[0][0]] == ["BlocksScreen"] + class TestAtomicBatch: - """_run_git_batch: stage all → deps → hooks → restart each service once; all-or-nothing.""" + """_run_git_batch: phased stage/deps/hooks/restart; failures drop per component.""" @pytest.fixture(autouse=True) def _mocks(self): @@ -746,7 +805,8 @@ async def test_inflight_marker_written_then_cleared_on_success(self, tmp_path): assert not svc._inflight_path.exists() # cleared on commit @pytest.mark.asyncio - async def test_inflight_marker_cleared_on_abort(self, tmp_path): + async def test_inflight_marker_cleared_on_partial_failure(self, tmp_path): + # One component fails and reverts cleanly; the survivor commits: no marker left. cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) svc._inflight_path = tmp_path / "inflight.json" @@ -760,9 +820,13 @@ async def test_inflight_marker_cleared_on_abort(self, tmp_path): return_value=(True, ""), ), patch("updater.service.git_reset_to_hash", return_value=(True, "")), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): await svc._run_git_batch(comps) - assert not svc._inflight_path.exists() # cleared by _abort_batch + assert not svc._inflight_path.exists() # settled: nothing pending @pytest.mark.asyncio async def test_restart_ui_component_also_restarts_ui(self, tmp_path): @@ -985,7 +1049,8 @@ async def test_cancel_after_commit_does_not_revert(self, tmp_path): assert all(c.args[1] is True for c in cb.on_component_done.call_args_list) @pytest.mark.asyncio - async def test_stage_failure_reverts_touched_and_does_not_restart(self, tmp_path): + async def test_stage_failure_drops_only_failed_component(self, tmp_path): + # c1 fails staging: it is reverted and dropped; c0 continues and commits. cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) with ( @@ -996,19 +1061,27 @@ async def test_stage_failure_reverts_touched_and_does_not_restart(self, tmp_path patch( "updater.service.git_reset_to_hash", return_value=(True, "") ) as mock_reset, - patch("updater.service.restart_service") as mock_rs, + patch("updater.service.run_hook", return_value=(True, "")), + patch( + "updater.service.restart_service", return_value=(True, "") + ) as mock_rs, + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): - await svc._run_git_batch(comps) - assert mock_reset.call_count == 2 # both touched repos reverted - mock_rs.assert_not_called() - assert cb.on_rollback.call_count == 2 - assert all(c.args[1] is True for c in cb.on_rollback.call_args_list) + result = await svc._run_git_batch(comps) + assert result is False # a drop is still a reported failure + mock_reset.assert_called_once() # only the failed repo is reverted + mock_rs.assert_called_once() # the survivor's service still restarts + cb.on_rollback.assert_called_once_with("c1", True) + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"c0": True, "c1": False} @pytest.mark.asyncio - async def test_abort_reports_rollback_false_when_revert_fails(self, tmp_path): - """A failed git revert during abort must report rollback as unsuccessful.""" + async def test_failed_revert_keeps_component_in_inflight_marker(self, tmp_path): + """A failed revert reports rollback False and stays marked for boot retry.""" cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) + svc._inflight_path = tmp_path / "inflight.json" with ( patch( "updater.service.UpdateService._stage_component", @@ -1018,14 +1091,18 @@ async def test_abort_reports_rollback_false_when_revert_fails(self, tmp_path): "updater.service.git_reset_to_hash", return_value=(False, "disk full"), ), - patch("updater.service.restart_service"), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): await svc._run_git_batch(comps) - assert cb.on_rollback.call_count == 2 - assert all(c.args[1] is False for c in cb.on_rollback.call_args_list) + cb.on_rollback.assert_called_once_with("c1", False) + marker = json.loads(svc._inflight_path.read_text()) + assert set(marker) == {"c1"} # survivor c0 must not be boot-reverted @pytest.mark.asyncio - async def test_deps_failure_reverts_all_no_restart(self, tmp_path): + async def test_deps_failure_drops_only_failed_component(self, tmp_path): cb = MagicMock() svc, comps = self._svc(tmp_path, cb, n=2) with ( @@ -1038,17 +1115,27 @@ async def test_deps_failure_reverts_all_no_restart(self, tmp_path): "updater.service.UpdateService._install_dependencies", side_effect=[(True, ""), (False, "pip boom")], ), - patch("updater.service.restart_service") as mock_rs, + patch("updater.service.run_hook", return_value=(True, "")), + patch( + "updater.service.restart_service", return_value=(True, "") + ) as mock_rs, + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): - await svc._run_git_batch(comps) - assert mr.call_count == 2 - mock_rs.assert_not_called() - assert cb.on_error.call_args[0][1] == "deps" + result = await svc._run_git_batch(comps) + assert result is False + mr.assert_called_once() # only c1 reverted + mock_rs.assert_called_once() # c0 still restarted + assert cb.on_error.call_args[0] == ("c1", "deps") + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"c0": True, "c1": False} @pytest.mark.asyncio - async def test_restart_failure_reverts_and_rerestarts_bounced(self, tmp_path): - # Two distinct services: first restarts OK, second fails → revert git and - # re-restart the first so it returns on the old code. + async def test_restart_failure_drops_service_members_and_recovers_it( + self, tmp_path + ): + # Two distinct services: svc-a restarts OK and commits; svc-b fails, its + # component reverts and svc-b is re-restarted onto the old code. cb = MagicMock() c0 = self._git(tmp_path, "c0", 2, "svc-a.service") c1 = self._git(tmp_path, "c1", 3, "svc-b.service") @@ -1074,18 +1161,20 @@ async def fake_restart(name): patch("updater.service.run_hook", return_value=(True, "")), patch("updater.service.restart_service", side_effect=fake_restart), patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), ): - await svc._run_git_batch([c0, c1]) - assert mr.call_count == 2 # both repos reverted - # Initial: svc-a ok, svc-b fails. Abort re-restarts BOTH bounced services - # (incl. the failed svc-b) onto the reverted code. + result = await svc._run_git_batch([c0, c1]) + assert result is False + mr.assert_called_once() # only c1 reverted; c0's update stands + # svc-a ok; svc-b fails then is re-restarted once onto the old code. assert restart_calls == [ "svc-a.service", "svc-b.service", - "svc-a.service", "svc-b.service", ] - assert cb.on_error.call_args[0][1] == "restart" + assert cb.on_error.call_args[0] == ("c1", "restart") + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"c0": True, "c1": False} @pytest.mark.asyncio async def test_cancel_mid_batch_reverts_staged(self, tmp_path): @@ -1387,7 +1476,13 @@ async def test_rollback_when_repair_fails(self): class TestProvisionMissingComponent: """install_if_missing: clone + deps + hook + restart, rm -rf on failure.""" - def _comp(self, tmp_path: Path, *, service: str | None = None) -> ComponentConfig: + def _comp( + self, + tmp_path: Path, + *, + service: str | None = None, + health_url: str | None = None, + ) -> ComponentConfig: return ComponentConfig( name="newcomp", kind="git", @@ -1396,6 +1491,7 @@ def _comp(self, tmp_path: Path, *, service: str | None = None) -> ComponentConfi service=service, url="https://github.com/test/newcomp", install_if_missing=True, + health_url=health_url, ) @pytest.mark.asyncio @@ -1497,7 +1593,70 @@ async def test_provision_waits_for_service_active(self, tmp_path): assert ok is False mock_wait.assert_called_once() mock_rmtree.assert_called_once() - assert cb.on_error.call_args[0][1] == "restart_timeout" + assert cb.on_error.call_args[0][1] == "restart" + + @pytest.mark.asyncio + async def test_provision_fails_when_health_check_fails(self, tmp_path): + comp = self._comp( + tmp_path, + service="newcomp.service", + health_url="http://127.0.0.1:7912/health", + ) + cb = MagicMock() + with ( + patch("updater.service.git_clone", return_value=(True, "")), + patch("updater.service.git_get_hash", return_value="newhash"), + patch( + "updater.service.UpdateService._install_dependencies", + return_value=(True, ""), + ), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch( + "updater.service.wait_for_http_ready", return_value=False + ) as mock_health, + patch("updater.service.shutil.rmtree") as mock_rmtree, + ): + svc = UpdateService(callback=cb) + svc._components = [comp] + ok = await svc.update_component("newcomp") + assert ok is False + mock_health.assert_called_once() + mock_rmtree.assert_called_once() + assert cb.on_error.call_args[0][1] == "restart" + + @pytest.mark.asyncio + async def test_provision_succeeds_when_health_ready(self, tmp_path): + comp = self._comp( + tmp_path, + service="newcomp.service", + health_url="http://127.0.0.1:7912/health", + ) + cb = MagicMock() + with ( + patch("updater.service.git_clone", return_value=(True, "")), + patch("updater.service.git_get_hash", return_value="newhash"), + patch( + "updater.service.UpdateService._install_dependencies", + return_value=(True, ""), + ), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch( + "updater.service.wait_for_http_ready", return_value=True + ) as mock_health, + patch("updater.service.enable_service", return_value=(True, "")), + patch("updater.service.shutil.rmtree") as mock_rmtree, + ): + svc = UpdateService(callback=cb) + svc._components = [comp] + ok = await svc.update_component("newcomp") + assert ok is True + mock_health.assert_called_once_with("http://127.0.0.1:7912/health") + mock_rmtree.assert_not_called() + cb.on_component_done.assert_called_with("newcomp", True) @pytest.mark.asyncio async def test_check_status_reports_needs_install(self, tmp_path): @@ -1570,6 +1729,178 @@ async def test_provision_enable_failure_is_best_effort(self, tmp_path): cb.on_component_done.assert_called_once_with("newcomp", True) +class TestProvisionMissing: + """provision_missing clones absent opted-in components outside a user Update.""" + + def _comp(self, tmp_path: Path, *, name: str = "Spoolman") -> ComponentConfig: + return ComponentConfig( + name=name, + kind="git", + path=tmp_path / name, + branch="master", + url=f"https://github.com/test/{name}", + install_if_missing=True, + ) + + @pytest.mark.asyncio + async def test_provisions_absent_opted_in_component(self, tmp_path): + comp = self._comp(tmp_path) # path does not exist + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch.object( + UpdateService, "_provision_component", return_value=True + ) as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is True + mock_prov.assert_awaited_once_with(comp) + + @pytest.mark.asyncio + async def test_present_component_is_never_provisioned(self, tmp_path): + comp = self._comp(tmp_path) + comp.path.mkdir() # already on disk + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch.object(UpdateService, "_provision_component") as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is False + mock_prov.assert_not_called() + + @pytest.mark.asyncio + async def test_not_opted_in_is_skipped(self, tmp_path): + comp = self._comp(tmp_path) + comp.install_if_missing = False # absent but not declared installable + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + patch.object(UpdateService, "_provision_component") as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is False + mock_prov.assert_not_called() + + @pytest.mark.asyncio + async def test_defers_when_process_lock_held(self, tmp_path): + # A concurrent update owns the lock: provisioning defers, never races it. + comp = self._comp(tmp_path) + with ( + patch("updater.service.process_lock", lambda: nullcontext(False)), + patch.object(UpdateService, "_provision_component") as mock_prov, + ): + svc = UpdateService() + svc._components = [comp] + did = await svc.provision_missing() + assert did is False + mock_prov.assert_not_called() + + +class TestReclone: + """_reclone_component: temp-clone + atomic swap as the deepest corruption rung.""" + + def _comp(self, tmp_path: Path) -> ComponentConfig: + path = tmp_path / "klipper" + path.mkdir() + (path / "corrupt").write_text("x") # marker proving the tree got replaced + return ComponentConfig( + name="klipper", + kind="git", + path=path, + url="https://github.com/test/klipper", + branch="master", + ) + + @staticmethod + def _fake_clone(url, dest, branch): + Path(dest).mkdir() + (Path(dest) / ".git").mkdir() + (Path(dest) / "fresh").write_text("y") + return (True, "") + + @pytest.mark.asyncio + async def test_swaps_in_fresh_clone(self, tmp_path): + comp = self._comp(tmp_path) + with patch("updater.service.git_clone", side_effect=self._fake_clone): + svc = UpdateService() + svc._components = [comp] + ok = await svc._reclone_component(comp) + assert ok is True + assert (comp.path / "fresh").exists() # fresh clone swapped in + assert not (comp.path / "corrupt").exists() # old tree gone + assert not (comp.path.parent / ".klipper.reclone-old").exists() + assert not (comp.path.parent / ".klipper.reclone-tmp").exists() + + @pytest.mark.asyncio + async def test_clone_failure_leaves_original_intact(self, tmp_path): + comp = self._comp(tmp_path) + with patch("updater.service.git_clone", return_value=(False, "boom")): + svc = UpdateService() + svc._components = [comp] + ok = await svc._reclone_component(comp) + assert ok is False + assert (comp.path / "corrupt").exists() # original untouched on clone failure + assert not (comp.path.parent / ".klipper.reclone-tmp").exists() + + @pytest.mark.asyncio + async def test_no_url_returns_false(self, tmp_path): + comp = self._comp(tmp_path) + comp.url = None + with patch("updater.service.git_clone") as mock_clone: + svc = UpdateService() + svc._components = [comp] + ok = await svc._reclone_component(comp) + assert ok is False + mock_clone.assert_not_called() + + @pytest.mark.asyncio + async def test_stage_reclones_when_repair_fails(self, tmp_path): + comp = self._comp(tmp_path) + with ( + patch("updater.service.git_fetch", return_value=(False, "corrupt")), + patch("updater.service.git_has_corruption", return_value=True), + patch("updater.service.git_repair", return_value=(False, "unrepairable")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch.object( + UpdateService, "_reclone_component", return_value=True + ) as mock_reclone, + patch("updater.service.git_checkout", return_value=(True, "")), + patch("updater.service.git_reset_to_hash", return_value=(True, "")), + ): + svc = UpdateService() + svc._components = [comp] + ok, _reason = await svc._stage_component(comp) + assert ok is True + mock_reclone.assert_awaited_once() + + @pytest.mark.asyncio + async def test_reconcile_reclones_when_repair_fails(self, tmp_path): + comp = self._comp(tmp_path) + with ( + patch("updater.service.process_lock", lambda: nullcontext(True)), + # gate "", post-reclone check, post-reclone hook hash + patch( + "updater.service.git_get_hash", + side_effect=["", "abc123", "abc123"], + ), + patch("updater.service.git_repair", return_value=(False, "unrepairable")), + patch("updater.service.run_hook", return_value=(True, "")) as mock_hook, + patch.object( + UpdateService, "_reclone_component", return_value=True + ) as mock_reclone, + ): + svc = UpdateService() + svc._components = [comp] + await svc.reconcile() + mock_reclone.assert_awaited_once() + # A boot reclone drops in-repo artifacts: the hook must rebuild them. + mock_hook.assert_awaited_once() + + class TestReconcile: """Boot-time repo healing for every component (Blocker 1).""" @@ -1604,7 +1935,8 @@ async def test_unreadable_head_triggers_repair(self, tmp_path): svc = UpdateService() svc._components = [comp] await svc.reconcile() - mock_repair.assert_called_once_with(comp.path, "main") + # No configured branch: git_repair derives the repo's own default. + mock_repair.assert_called_once_with(comp.path, None) @pytest.mark.asyncio async def test_repair_failure_falls_back_to_prev_hash_reset(self, tmp_path): @@ -1691,7 +2023,7 @@ async def test_rollback_false_when_service_not_active(self): class TestPreflightFetch: - """update_all fetches every component up-front; a network drop aborts cleanly.""" + """update_all fetches up-front; an offline repo is dropped, the rest continue.""" def _git(self, tmp_path: Path, name: str) -> ComponentConfig: path = tmp_path / name @@ -1704,23 +2036,21 @@ async def test_all_ok_records_fetch_times(self, tmp_path): with patch("updater.service.git_fetch", return_value=(True, "")): svc = UpdateService() svc._components = comps - ok = await svc._preflight_fetch(comps, comps) - assert ok is True + offline = await svc._preflight_fetch(comps) + assert offline == set() assert set(svc._fetch_times) == {"klipper", "moonraker"} @pytest.mark.asyncio - async def test_offline_aborts_and_reports_all(self, tmp_path): + async def test_offline_components_are_returned(self, tmp_path): comps = [self._git(tmp_path, "klipper"), self._git(tmp_path, "moonraker")] - cb = MagicMock() with ( patch("updater.service.git_fetch", return_value=(False, "timeout")), patch("updater.service.git_has_corruption", return_value=False), ): - svc = UpdateService(callback=cb) + svc = UpdateService() svc._components = comps - ok = await svc._preflight_fetch(comps, comps) - assert ok is False - assert cb.on_error.call_count == 2 + offline = await svc._preflight_fetch(comps) + assert offline == {"klipper", "moonraker"} @pytest.mark.asyncio async def test_corrupt_repo_is_not_treated_as_offline(self, tmp_path): @@ -1731,8 +2061,8 @@ async def test_corrupt_repo_is_not_treated_as_offline(self, tmp_path): ): svc = UpdateService() svc._components = comps - ok = await svc._preflight_fetch(comps, comps) - assert ok is True + offline = await svc._preflight_fetch(comps) + assert offline == set() assert "klipper" not in svc._fetch_times # left for the per-component flow @pytest.mark.asyncio @@ -1742,21 +2072,30 @@ async def test_recently_fetched_component_is_not_refetched(self, tmp_path): svc = UpdateService() svc._components = [comp] svc._fetch_times["klipper"] = time.monotonic() - ok = await svc._preflight_fetch([comp], [comp]) - assert ok is True + offline = await svc._preflight_fetch([comp]) + assert offline == set() mock_fetch.assert_not_called() @pytest.mark.asyncio - async def test_update_all_aborts_apply_when_preflight_fails(self, tmp_path): - comps = [self._git(tmp_path, "klipper")] + async def test_update_all_drops_only_offline_components(self, tmp_path): + offline_comp = self._git(tmp_path, "klipper") + online_comp = self._git(tmp_path, "moonraker") + cb = MagicMock() with ( - patch.object(UpdateService, "_preflight_fetch", return_value=False), - patch.object(UpdateService, "update_component") as mock_update, + patch.object(UpdateService, "_preflight_fetch", return_value={"klipper"}), + patch.object( + UpdateService, "_run_git_batch", return_value=True + ) as mock_batch, ): - svc = UpdateService() - svc._components = comps - await svc.update_all({"klipper"}) - mock_update.assert_not_called() + svc = UpdateService(callback=cb) + svc._components = [offline_comp, online_comp] + ok = await svc.update_all({"klipper", "moonraker"}) + assert ok is False # the offline drop is a reported failure + cb.on_error.assert_called_once_with( + "klipper", "network error during pre-flight fetch" + ) + batched = [c.name for c in mock_batch.call_args.args[0]] + assert batched == ["moonraker"] class TestDeferredRestart: @@ -1939,6 +2278,23 @@ async def test_revert_rejects_invalid_prev_hash(self, tmp_path): mock_reset.assert_not_called() assert svc._read_inflight() == {} # marker still cleared + @pytest.mark.asyncio + async def test_revert_failed_reset_keeps_marker_for_retry(self, tmp_path): + """A failed boot revert keeps the entry marked so the next boot retries it.""" + comp = self._git(tmp_path) + svc = UpdateService() + svc._components = [comp] + svc._inflight_path = tmp_path / "inflight.json" + svc._write_inflight({"klipper": "a" * 40}) + with ( + patch("updater.service.git_get_hash", return_value="b" * 40), + patch( + "updater.service.git_reset_to_hash", return_value=(False, "disk full") + ), + ): + await svc._revert_inflight() + assert svc._read_inflight() == {"klipper": "a" * 40} # kept for retry + class TestPingWhile: @pytest.mark.asyncio @@ -1996,6 +2352,7 @@ async def test_single_update_passes_hook_timeout(self, tmp_path): patch("updater.service.git_pull", return_value=(True, "")), patch("updater.service.git_reset_to_hash", return_value=(True, "")), patch("updater.service.git_get_current_branch", return_value="master"), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch( "updater.service.UpdateService._install_dependencies", return_value=(True, ""), @@ -2070,15 +2427,25 @@ async def test_single_insecure_remote_clears_marker(self, tmp_path): cb.on_error.assert_called_once_with("klipper", "insecure remote") @pytest.mark.asyncio - async def test_single_reset_failure_clears_marker(self, tmp_path): + async def test_single_reset_failure_rolls_back_and_clears_marker(self, tmp_path): + # The checkout before the failing reset may have moved the tree: the + # component must roll back to prev_hash, then the marker clears. cb = MagicMock() + + async def fake_reset(path, ref=""): + # Target reset (origin/) fails; the prev_hash rollback works. + return (False, "boom") if str(ref).startswith("origin/") else (True, "") + with ( patch("pathlib.Path.exists", return_value=True), patch("updater.service.git_get_hash", return_value="a" * 40), patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), patch("updater.service.git_get_current_branch", return_value="master"), patch("updater.service.git_checkout", return_value=(True, "")), - patch("updater.service.git_reset_to_hash", return_value=(False, "boom")), + patch("updater.service.git_reset_to_hash", side_effect=fake_reset), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), patch( "updater.service._assert_https_remote", return_value=(True, "https://github.com/test/repo"), @@ -2088,6 +2455,7 @@ async def test_single_reset_failure_clears_marker(self, tmp_path): assert await svc.update_component("klipper") is False assert not (tmp_path / "inflight.json").exists() cb.on_error.assert_called_once_with("klipper", "reset") + cb.on_rollback.assert_called_once_with("klipper", True) @pytest.mark.asyncio async def test_batch_insecure_remote_clears_marker(self, tmp_path): @@ -2149,7 +2517,7 @@ async def fake_fetch(path, **kw): with patch("updater.service.git_fetch", side_effect=fake_fetch): svc = UpdateService(callback=MagicMock()) - assert await svc._preflight_fetch([good, bad], [good, bad]) is True + assert await svc._preflight_fetch([good, bad]) == set() assert fetched == ["klipper"] @pytest.mark.asyncio @@ -2247,3 +2615,281 @@ async def test_honors_configured_apt_excludes(self): ): await svc.background_apt_upgrade() mock_up.assert_awaited_once_with(exclude=("^linux-image", "^firmware-")) + + +class TestReviewHardeningFixes: + """Regressions for the 2026-07 pre-main review fixes.""" + + def _git( + self, tmp_path: Path, name: str, service: str = "klipper.service" + ) -> ComponentConfig: + path = tmp_path / name + (path / ".git").mkdir(parents=True) + return ComponentConfig(name=name, kind="git", path=path, service=service) + + def _seeded_svc(self, tmp_path: Path, comp: ComponentConfig) -> UpdateService: + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + svc._write_state({comp.name: {"last_good": "b" * 40, "golden": "c" * 40}}) + return svc + + def _happy_patches(self, stack: ExitStack) -> None: + for target, kw in ( + ("updater.service.git_get_hash", {"return_value": "a" * 40}), + ( + "updater.service._assert_https_remote", + {"return_value": (True, "https://x")}, + ), + ( + "updater.service.UpdateService._stage_component", + {"return_value": (True, "")}, + ), + ( + "updater.service.UpdateService._install_dependencies", + {"return_value": (True, "")}, + ), + ("updater.service.run_hook", {"return_value": (True, "")}), + ("updater.service.restart_service_noblock", {"new": AsyncMock()}), + ): + stack.enter_context(patch(target, **kw)) + stack.enter_context( + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()) + ) + + @pytest.mark.asyncio + async def test_batch_update_preserves_self_heal_anchors(self, tmp_path): + comp = self._git(tmp_path, "BlocksScreen", "BlocksScreen.service") + svc = self._seeded_svc(tmp_path, comp) + with ExitStack() as stack: + self._happy_patches(stack) + assert await svc._run_git_batch([comp]) is True + state = svc._read_state()["BlocksScreen"] + assert state["last_good"] == "b" * 40 # anchors survive the update + assert state["golden"] == "c" * 40 + assert state["prev_hash"] == "a" * 40 + + @pytest.mark.asyncio + async def test_single_update_preserves_self_heal_anchors(self, tmp_path): + comp = self._git(tmp_path, "BlocksScreen", "BlocksScreen.service") + svc = self._seeded_svc(tmp_path, comp) + with ExitStack() as stack: + self._happy_patches(stack) + assert await svc._run_git_update(comp) is True + state = svc._read_state()["BlocksScreen"] + assert state["last_good"] == "b" * 40 + assert state["golden"] == "c" * 40 + + @pytest.mark.asyncio + async def test_single_update_dead_branch_errors_without_stranding(self, tmp_path): + comp = self._git(tmp_path, "klipper") + comp.branch = "wip/deleted" + cb = MagicMock() + svc = UpdateService(callback=cb) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + with ( + patch("updater.service.git_get_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://x"), + ), + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value=""), + patch("updater.service.git_checkout") as mock_checkout, + ): + assert await svc._run_git_update(comp) is False + mock_checkout.assert_not_called() # never switched onto the dead branch + assert not (tmp_path / "inflight.json").exists() + assert "not found" in cb.on_error.call_args[0][1] + + @pytest.mark.asyncio + async def test_supervise_ui_survives_handler_exception(self): + svc = UpdateService() + seen: list[int] = [] + + async def fake_nrestarts(_svc=None): + seen.append(1) + if len(seen) >= 2: + raise asyncio.CancelledError # end the loop after the second pass + return 99 + + with ( + patch("updater.service._NRESTARTS_POLL_INTERVAL_S", 0.001), + patch("updater.service.get_service_nrestarts", side_effect=fake_nrestarts), + patch.object(UpdateService, "_check_crash_loop", return_value=True), + patch.object( + UpdateService, "_handle_crash_loop", side_effect=RuntimeError("boom") + ), + ): + with pytest.raises(asyncio.CancelledError): + await svc.supervise_ui() + assert len(seen) == 2 # the loop survived the first pass exception + + @pytest.mark.asyncio + async def test_recover_tolerates_corrupt_state_shapes(self, tmp_path): + comp = self._git(tmp_path, "klipper") + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._state_path.write_text("[1, 2, 3]") # valid JSON, wrong shape + assert await svc.recover("klipper") is False + svc._state_path.write_text('{"klipper": "not-a-dict"}') + assert await svc.recover("klipper") is False + + @pytest.mark.asyncio + async def test_revert_inflight_tolerates_corrupt_values(self, tmp_path): + comp = self._git(tmp_path, "klipper") + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + svc._inflight_path = tmp_path / "inflight.json" + svc._inflight_path.write_text('{"klipper": 42, "ghost": null}') + with patch("updater.service.git_reset_to_hash") as mock_reset: + await svc._revert_inflight() # must not raise + mock_reset.assert_not_called() + + @pytest.mark.asyncio + async def test_reclone_invalidates_cached_pip_path(self, tmp_path): + comp = self._git(tmp_path, "Spoolman", "Spoolman.service") + comp.url = "https://github.com/test/spoolman" + svc = UpdateService() + svc._component_pip_cache[str(comp.path)] = "/stale/.venv/bin/pip" + + async def fake_clone(url, dest, branch): + Path(dest).mkdir() + (Path(dest) / ".git").mkdir() + return (True, "") + + with patch("updater.service.git_clone", side_effect=fake_clone): + assert await svc._reclone_component(comp) is True + assert str(comp.path) not in svc._component_pip_cache + + @pytest.mark.asyncio + async def test_update_all_returns_true_only_when_all_succeed(self, tmp_path): + good = self._git(tmp_path, "klipper") + for batch_result, expected in ((True, True), (False, False)): + with ( + patch.object(UpdateService, "_preflight_fetch", return_value=set()), + patch.object( + UpdateService, "_run_git_batch", return_value=batch_result + ), + ): + svc = UpdateService() + svc._components = [good] + assert await svc.update_all() is expected + + +class TestUpdaterDowngradeGuard: + """Refuse switching the host onto a tree without the updater (field brick).""" + + def _bs(self, tmp_path: Path) -> ComponentConfig: + path = tmp_path / "BlocksScreen" + (path / ".git").mkdir(parents=True) + return ComponentConfig( + name="BlocksScreen", + kind="git", + path=path, + service="BlocksScreen.service", + branch="main", + ) + + @pytest.mark.asyncio + async def test_stage_refuses_target_without_updater(self, tmp_path): + comp = self._bs(tmp_path) + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + with ( + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + # target lacks updater/ + patch("updater.service.git_tree_has_path", return_value=False), + patch("updater.service.git_checkout") as mock_checkout, + ): + ok, reason = await svc._stage_component(comp) + assert ok is False + assert "refusing" in reason + mock_checkout.assert_not_called() # never switched onto the brick + + @pytest.mark.asyncio + async def test_stage_allows_target_with_updater(self, tmp_path): + comp = self._bs(tmp_path) + svc = UpdateService(callback=MagicMock()) + svc._components = [comp] + with ( + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch("updater.service.git_tree_has_path", return_value=True), + patch("updater.service.git_checkout", return_value=(True, "")) as mc, + patch("updater.service.git_reset_to_hash", return_value=(True, "")), + ): + ok, _ = await svc._stage_component(comp) + assert ok is True + mc.assert_called_once() + + @pytest.mark.asyncio + async def test_single_update_refusal_no_rollback_clears_marker(self, tmp_path): + comp = self._bs(tmp_path) + cb = MagicMock() + svc = UpdateService(callback=cb) + svc._components = [comp] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + with ( + patch("updater.service.git_get_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://x"), + ), + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch("updater.service.git_tree_has_path", return_value=False), + patch("updater.service.git_reset_to_hash") as mock_reset, + ): + assert await svc._run_git_update(comp) is False + mock_reset.assert_not_called() # nothing moved: no rollback + assert not (tmp_path / "inflight.json").exists() + assert "refusing" in cb.on_error.call_args[0][1] + + @pytest.mark.asyncio + async def test_batch_drops_host_keeps_others(self, tmp_path): + bs = self._bs(tmp_path) + klip_path = tmp_path / "klipper" + (klip_path / ".git").mkdir(parents=True) + klip = ComponentConfig( + name="klipper", kind="git", path=klip_path, service="klipper.service" + ) + cb = MagicMock() + svc = UpdateService(callback=cb) + svc._components = [klip, bs] + svc._state_path = tmp_path / "state.json" + svc._inflight_path = tmp_path / "inflight.json" + + async def fake_tree(path, ref, repo_path): + return "klipper" in str(path) # only klipper target has updater/ + + with ( + patch("updater.service.git_get_hash", return_value="a" * 40), + patch( + "updater.service._assert_https_remote", + return_value=(True, "https://x"), + ), + patch("updater.service.git_fetch", return_value=(True, "")), + patch("updater.service.git_ref_hash", return_value="a" * 40), + patch("updater.service.git_tree_has_path", side_effect=fake_tree), + patch("updater.service.git_checkout", return_value=(True, "")), + patch("updater.service.git_reset_to_hash", return_value=(True, "")), + patch( + "updater.service.UpdateService._install_dependencies", + return_value=(True, ""), + ), + patch("updater.service.run_hook", return_value=(True, "")), + patch("updater.service.restart_service", return_value=(True, "")), + patch("updater.service.wait_for_service_active", return_value=True), + patch.object(UpdateService, "_apply_deferred_restart", new=AsyncMock()), + ): + result = await svc._run_git_batch([klip, bs]) + assert result is False # BlocksScreen dropped + done = {c.args[0]: c.args[1] for c in cb.on_component_done.call_args_list} + assert done == {"klipper": True, "BlocksScreen": False} diff --git a/updater/__main__.py b/updater/__main__.py index 39e36619..68614d0b 100644 --- a/updater/__main__.py +++ b/updater/__main__.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from updater.locking import process_lock +from updater.models import ComponentStatus from updater.service import LoggingCallback, UpdateService # NOTE: sdbus imports are lazy (in _run_daemon) so the CLI works without sdbus. @@ -83,11 +84,7 @@ async def _run_daemon() -> None: def _watchdog_ping_interval() -> float: - """Half of systemd's WatchdogSec (per sd_notify(3)), or 15s if unset/invalid. - - Reading WATCHDOG_USEC from the environment keeps the heartbeat correct if the - unit's WatchdogSec is ever retuned, instead of hardcoding half of 30s. - """ + """Half of systemd's WatchdogSec (per sd_notify(3)), or 15s if unset/invalid.""" try: watchdog_usec = int(os.environ.get("WATCHDOG_USEC", "0")) except ValueError: @@ -107,6 +104,31 @@ def _cli_lock() -> Iterator[None]: yield +def _print_component_status(s: ComponentStatus, verbose: bool) -> None: + """Print a one-line status summary for a component (+ verbose detail line).""" + if s.error: + print(f"{s.name}: ERROR: {s.error}") + elif s.branch_mismatch: + print(f"{s.name}: branch switch needed") + elif s.needs_install: + print(f"{s.name}: install required") + elif s.packages_upgradable > 0: + print(f"{s.name}: {s.packages_upgradable} packages upgradable") + elif s.commits_behind > 0: + print(f"{s.name}: {s.commits_behind} commits behind") + elif s.has_local_changes: + print(f"{s.name}: local changes") + else: + print(f"{s.name}: up to date") + if verbose: + print( + f" branch={s.current_branch or '?'} mismatch={s.branch_mismatch} " + f"behind={s.commits_behind} needs_install={s.needs_install} " + f"local_changes={s.has_local_changes} pkgs={s.packages_upgradable} " + f"hash={s.current_hash[:8]}" + ) + + async def main() -> None: """Parse CLI args and dispatch to the appropriate UpdateService method.""" args = build_parser().parse_args() @@ -122,36 +144,20 @@ async def main() -> None: case "update": with _cli_lock(): if args.name is None: - await svc.update_all() + ok = await svc.update_all() else: - await svc.update_component(args.name) + ok = await svc.update_component(args.name) + if not ok: + # Scripts/harnesses rely on the exit code, not just the log. + raise SystemExit(1) case "status": result = await svc.check_status() for s in sorted(result.values(), key=lambda c: c.name): - if s.error: - print(f"{s.name}: ERROR: {s.error}") - elif s.branch_mismatch: - print(f"{s.name}: branch switch needed") - elif s.needs_install: - print(f"{s.name}: install required") - elif s.packages_upgradable > 0: - print(f"{s.name}: {s.packages_upgradable} packages upgradable") - elif s.commits_behind > 0: - print(f"{s.name}: {s.commits_behind} commits behind") - elif s.has_local_changes: - print(f"{s.name}: local changes") - else: - print(f"{s.name}: up to date") - if args.verbose: - print( - f" branch={s.current_branch or '?'} mismatch={s.branch_mismatch} " - f"behind={s.commits_behind} needs_install={s.needs_install} " - f"local_changes={s.has_local_changes} pkgs={s.packages_upgradable} " - f"hash={s.current_hash[:8]}" - ) + _print_component_status(s, args.verbose) case "recover": with _cli_lock(): - await svc.recover(args.name, hard=args.hard) + if not await svc.recover(args.name, hard=args.hard): + raise SystemExit(1) case "bless": with _cli_lock(): ok = await svc.bless_healthy(args.name, args.hash) diff --git a/updater/components.py b/updater/components.py index e4938088..6073b3e5 100644 --- a/updater/components.py +++ b/updater/components.py @@ -15,6 +15,9 @@ _GIT_VERSION_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._/+-]*$") _COMPONENT_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]*$") _GIT_URL_RE = re.compile(r"^https://[a-zA-Z0-9._~:/?#\[\]@!$&'()*+,;=%-]+$") +_HEALTH_URL_RE = re.compile( + r"^http://(127\.0\.0\.1|localhost)(:\d{1,5})?/[a-zA-Z0-9._~:/?#\[\]@!$&'()*+,;=%-]*$" +) _SERVICE_BANNED = set("/\\;&|$`") | {" ", "\t"} OVERRIDE_PATH = Path("~/printer_data/config/blockscreen_updater.yaml").expanduser() @@ -36,103 +39,132 @@ def _validate_service(name: str) -> bool: return bool(_SERVICE_RE.match(name)) -def _validate_component(data: dict) -> ComponentConfig | None: - """Parse and validate a raw component dict; returns None and logs a warning on any error.""" # noqa: E501 - name = data.get("name", "") - if not isinstance(name, str) or not name: - logger.warning("Component missing name, skipped") +def _parse_order(name: str, data: dict) -> int: + """Parse the integer 'order' field, defaulting to 50 on any bad value.""" + try: + return int(data.get("order", 50)) + except (TypeError, ValueError): + logger.warning("Component %r has invalid order value", name) + return 50 + + +def _parse_git_branch(name: str, raw_branch: object) -> tuple[str | None, bool]: + """Normalize/validate a git branch (strip 'origin/'); (branch, ok=False) on invalid.""" + if raw_branch is None: + return None, True + branch = str(raw_branch) + # Strip a stray remote prefix: `origin/x` would fetch `origin/origin/x`. + if branch.startswith("origin/"): + logger.warning( + "Component %r branch %r has an 'origin/' prefix - stripping it", + name, + branch, + ) + branch = branch[len("origin/") :] + if not _GIT_BRANCH_RE.match(branch): + logger.warning("Component %r has invalid branch name %r, skipped", name, branch) + return None, False + return branch, True + + +def _parse_git_url(name: str, url: object) -> object: + """Validate the https url; drop it (None) on any non-https value.""" + if url is not None and not _GIT_URL_RE.match(str(url)): + logger.warning( + "Component %r has invalid (non-https) url %r - dropping url", name, url + ) return None - if len(name) > 255: - logger.warning("Component name too long (%d chars) - skipped", len(name)) + return url + + +def _parse_health_url(name: str, url: object) -> str | None: + """Accept only a loopback http readiness URL; drop anything else.""" + if url is None: return None - if not _COMPONENT_NAME_RE.match(name): + if not _HEALTH_URL_RE.match(str(url)): logger.warning( - "Component %r has invalid name (must match [a-zA-Z0-9_-]+) - skipped", name + "Component %r has non-loopback health_url %r - dropping", name, url ) return None + return str(url) - comp_type = data.get("type", "") - if comp_type not in ("git", "apt"): - logger.warning("Component %r has invalid type %r - skipped", name, comp_type) + +def _parse_reset_mode(name: str, reset_mode: object) -> str: + """Return 'hard'/'soft'; unknown values fall back to 'hard' (fleet default).""" + if reset_mode not in ("hard", "soft"): + # An unknown value must not silently take the soft path (fleet default is hard). + logger.warning( + "Component %r has invalid reset_mode %r - using 'hard'", + name, + reset_mode, + ) + return "hard" + return str(reset_mode) + + +def _validate_git_component(name: str, data: dict) -> ComponentConfig | None: + """Parse/validate a git component dict; None on any invalid field.""" + raw_path = data.get("path") + if not raw_path: + logger.warning("Component %r missing path - skipped", name) + return None + resolved = Path(str(raw_path)).expanduser().resolve() + if not resolved.is_relative_to(Path.home()): + logger.warning("Component %r path escapes home dir - skipped", name) return None - if comp_type == "git": - raw_path = data.get("path") - if not raw_path: - logger.warning("Component %r missing path - skipped", name) - return None - resolved = Path(str(raw_path)).expanduser().resolve() - if not resolved.is_relative_to(Path.home()): - logger.warning("Component %r path escapes home dir - skipped", name) - return None - - service = data.get("service") - if service is not None and not _validate_service(str(service)): - logger.warning( - "Component %r has invalid service %r, skipped", name, service - ) - return None - - branch = data.get("branch") - if branch is not None and not _GIT_BRANCH_RE.match(str(branch)): - logger.warning( - "Component %r has invalid branch name %r, skipped", - name, - branch, - ) - return None - - version = data.get("version") - if version is not None and not _GIT_VERSION_RE.match(str(version)): - logger.warning( - "Component %r has invalid version[%s] for the branch %s", - name, - version, - branch, - ) - return None - - order = 50 - try: - order = int(data.get("order", 50)) - except (TypeError, ValueError): - logger.warning("Component %r has invalid order value", name) - - url = data.get("url") - if url is not None and not _GIT_URL_RE.match(str(url)): - logger.warning( - "Component %r has invalid (non-https) url %r - dropping url", name, url - ) - url = None - - install_if_missing = bool(data.get("install_if_missing", False)) - if install_if_missing and not url: - logger.warning( - "Component %r sets install_if_missing but has no valid url - " - "cannot provision, disabling", - name, - ) - install_if_missing = False - - return ComponentConfig( - name=name, - kind=comp_type, - path=resolved, - service=str(service) if service else None, - reset_mode=data.get("reset_mode", "hard"), - order=order, - branch=str(branch) if branch else None, - version=str(version) if version else None, - url=str(url) if url else None, - install_if_missing=install_if_missing, - restart_ui=bool(data.get("restart_ui", False)), - restart_klipper=bool(data.get("restart_klipper", False)), + service = data.get("service") + if service is not None and not _validate_service(str(service)): + logger.warning("Component %r has invalid service %r, skipped", name, service) + return None + + branch, branch_ok = _parse_git_branch(name, data.get("branch")) + if not branch_ok: + return None + + version = data.get("version") + if version is not None and not _GIT_VERSION_RE.match(str(version)): + logger.warning( + "Component %r has invalid version[%s] for the branch %s", + name, + version, + branch, ) - apt_order = 50 - try: - apt_order = int(data.get("order", 50)) - except (TypeError, ValueError): - logger.warning("Component %r has invalid order value", name) + return None + + order = _parse_order(name, data) + url = _parse_git_url(name, data.get("url")) + reset_mode = _parse_reset_mode(name, data.get("reset_mode", "hard")) + + install_if_missing = bool(data.get("install_if_missing", False)) + if install_if_missing and not url: + logger.warning( + "Component %r sets install_if_missing but has no valid url - " + "cannot provision, disabling", + name, + ) + install_if_missing = False + + return ComponentConfig( + name=name, + kind="git", + path=resolved, + service=str(service) if service else None, + reset_mode=reset_mode, + order=order, + branch=str(branch) if branch else None, + version=str(version) if version else None, + url=str(url) if url else None, + install_if_missing=install_if_missing, + restart_ui=bool(data.get("restart_ui", False)), + restart_klipper=bool(data.get("restart_klipper", False)), + health_url=_parse_health_url(name, data.get("health_url")), + ) + + +def _validate_apt_component(name: str, data: dict) -> ComponentConfig: + """Build an apt ComponentConfig, always unioning the kernel/firmware excludes.""" + apt_order = _parse_order(name, data) raw_exclude = data.get("apt_exclude", []) apt_exclude: tuple[str, ...] = () if isinstance(raw_exclude, list): @@ -149,10 +181,39 @@ def _validate_component(data: dict) -> ComponentConfig | None: ) +def _validate_component(data: dict) -> ComponentConfig | None: + """Parse and validate a raw component dict; returns None and logs a warning on any error.""" # noqa: E501 + name = data.get("name", "") + if not isinstance(name, str) or not name: + logger.warning("Component missing name, skipped") + return None + if len(name) > 255: + logger.warning("Component name too long (%d chars) - skipped", len(name)) + return None + if not _COMPONENT_NAME_RE.match(name): + logger.warning( + "Component %r has invalid name (must match [a-zA-Z0-9_-]+) - skipped", name + ) + return None + + comp_type = data.get("type", "") + if comp_type not in ("git", "apt"): + logger.warning("Component %r has invalid type %r - skipped", name, comp_type) + return None + + if comp_type == "git": + return _validate_git_component(name, data) + return _validate_apt_component(name, data) + + def _merge(base: list[dict], override: list[dict]) -> list[dict]: """Merge override entries into base by name; unknown override names are appended.""" - base_by_name = {c["name"]: dict(c) for c in base if "name" in c} + base_by_name = { + c["name"]: c.copy() for c in base if isinstance(c, dict) and "name" in c + } for entry in override: + if not isinstance(entry, dict): + continue entry_name = entry.get("name") if not entry_name: continue @@ -161,38 +222,30 @@ def _merge(base: list[dict], override: list[dict]) -> list[dict]: if v is not None: base_by_name[entry_name][k] = v else: - base_by_name[entry_name] = dict(entry) + base_by_name[entry_name] = entry.copy() return list(base_by_name.values()) -def load_components() -> tuple[list[ComponentConfig], float]: - """Load and validate component definitions from bundled YAML, merged with user override. - - Returns: - A tuple of (list of ComponentConfig, poll_interval in seconds). - """ - try: - import yaml # noqa: PLC0415 - except ImportError: - logger.error("PyYAML not installed; updater unavailable until next restart") # noqa: TRY400 - return [], 1440 * 60.0 - +def _load_bundled_yaml(yaml) -> dict | None: + """Read + parse the bundled components.yaml; None on any read/parse error.""" bundled_path = Path(__file__).parent / "components.yaml" try: with open(bundled_path) as f: # noqa: PTH123 bundled_data = yaml.safe_load(f) except OSError: logger.exception("bundled_data not found") - return [], 1440 * 60.0 + return None except yaml.YAMLError: logger.exception("Malformed YAML file") - return [], 1440 * 60.0 - + return None if not isinstance(bundled_data, dict): logger.error("Data is empty") - return [], 1440 * 60.0 - raw_components: list[dict] = bundled_data.get("components", []) + return None + return bundled_data + +def _load_override(raw_components: list[dict], yaml) -> list[dict]: + """Merge the user override YAML into raw_components (skips unsafe/malformed overrides).""" try: override_exists = OVERRIDE_PATH.exists() except OSError: @@ -224,8 +277,16 @@ def load_components() -> tuple[list[ComponentConfig], float]: ) except Exception: logger.exception("Failed to load override YAML %s", OVERRIDE_PATH) + return raw_components + + +def _build_configs(raw_components: list[dict]) -> list[ComponentConfig]: + """Validate each raw entry, then auto-inject a system apt component if none exists.""" configs: list[ComponentConfig] = [] for entry in raw_components: + if not isinstance(entry, dict): + logger.warning("Component entry is not a mapping - skipped: %r", entry) + continue cfg = _validate_component(entry) if cfg is not None: configs.append(cfg) @@ -240,6 +301,23 @@ def load_components() -> tuple[list[ComponentConfig], float]: apt_exclude=_KERNEL_FIRMWARE_EXCLUDES, ), ) + return configs + + +def load_components() -> tuple[list[ComponentConfig], float]: + """Load/validate components from bundled YAML merged with the user override.""" + try: + import yaml # noqa: PLC0415 + except ImportError: + logger.error("PyYAML not installed; updater unavailable until next restart") # noqa: TRY400 + return [], 1440 * 60.0 + + bundled_data = _load_bundled_yaml(yaml) + if bundled_data is None: + return [], 1440 * 60.0 + raw_components: list[dict] = bundled_data.get("components", []) + raw_components = _load_override(raw_components, yaml) + configs = _build_configs(raw_components) try: poll_seconds = float(bundled_data.get("poll_interval_minutes", 1440)) * 60.0 except (TypeError, ValueError): diff --git a/updater/components.yaml b/updater/components.yaml index 38dd5253..bfe6696f 100644 --- a/updater/components.yaml +++ b/updater/components.yaml @@ -14,7 +14,7 @@ components: url: https://github.com/Klipper3d/klipper service: klipper.service reset_mode: hard - branch: wip/Happy-Hare-Integration + branch: master order: 2 restart_ui: true @@ -63,8 +63,7 @@ components: reset_mode: hard order: 20 - # Provisioned by hooks/Spoolman.sh (uv sync from source + stub client/dist); - # backend-only, used via the moonraker API. + # Provisioned by hooks/Spoolman.sh - name: Spoolman type: git path: ~/Spoolman @@ -74,11 +73,12 @@ components: reset_mode: hard order: 21 install_if_missing: true + health_url: http://127.0.0.1:7912/api/v1/health - name: BlocksScreen type: git path: ~/BlocksScreen - branch: dev + branch: origin/Release-2.0 url: https://github.com/BlocksTechnology/BlocksScreen service: BlocksScreen.service reset_mode: hard diff --git a/updater/dbus_service.py b/updater/dbus_service.py index f7284bf5..467af38c 100644 --- a/updater/dbus_service.py +++ b/updater/dbus_service.py @@ -19,16 +19,14 @@ class DbusProgressCallback: - """Adapter that forwards ProgressCallback events to D-Bus signals. - - Holds a reference to the server interface so each on_* call invokes - the corresponding signal's .emit() - decoupling UpdateService from D-Bus. - """ + """Forward ProgressCallback events to D-Bus signals (decouples UpdateService""" def __init__(self, iface: UpdaterInterface) -> None: + """Bind the callback to the D-Bus interface whose signals it emits.""" self._iface = iface def on_step(self, name: str, step: int, total: int) -> None: + """Emit step_complete and write the current step to the status file.""" self._iface.step_complete.emit((name, step, total)) try: _STATUS_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -39,15 +37,19 @@ def on_step(self, name: str, step: int, total: int) -> None: _log.debug("status write failed: %s", e) def on_component_done(self, name: str, success: bool) -> None: # noqa: FBT001 + """Emit the component_done signal.""" self._iface.component_done.emit((name, success)) def on_error(self, name: str, reason: str) -> None: + """Emit the error signal.""" self._iface.error.emit((name, reason)) def on_rollback(self, name: str, success: bool) -> None: # noqa: FBT001 + """Emit the rollback signal.""" self._iface.rollback.emit((name, success)) def on_recover(self, name: str, success: bool) -> None: # noqa: FBT001 + """Emit the recover_done signal.""" self._iface.recover_done.emit((name, success)) @@ -55,11 +57,7 @@ class UpdaterInterface( sdbus.DbusInterfaceCommonAsync, interface_name="com.blockscreen.Updater", ): - """D-Bus interface contract shared by server and client proxy. - - Signals are declared once here, sdbus replaces each descriptor with - emit machinery on the server side and an async-iterable on the client. - """ + """D-Bus contract shared by server and client proxy: signals declared once,""" @sdbus.dbus_signal_async("sii") def step_complete(self) -> tuple[str, int, int]: @@ -97,6 +95,7 @@ def busy_changed(self) -> tuple[bool]: raise NotImplementedError def __init__(self) -> None: + """Wire the service and busy state, then spawn the boot, poll, and self-heal tasks.""" super().__init__() self._svc = UpdateService(callback=DbusProgressCallback(self)) self._busy: bool = False @@ -114,9 +113,18 @@ def _spawn(self, coro, *, name: str | None = None) -> asyncio.Task: """Create a task and hold a strong reference so GC cannot cancel it.""" task = asyncio.get_running_loop().create_task(coro, name=name) self._background_tasks.add(task) - task.add_done_callback(self._background_tasks.discard) + task.add_done_callback(self._task_done) return task + def _task_done(self, task: asyncio.Task) -> None: + """Drop the task ref and log its exception now, not at some later GC.""" + self._background_tasks.discard(task) + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + _log.error("task %r failed", task.get_name(), exc_info=exc) + def _set_busy(self, busy: bool) -> None: # noqa: FBT001 """Emit busy_changed only on state transitions to avoid redundant signals.""" if busy != self._busy: @@ -125,10 +133,7 @@ def _set_busy(self, busy: bool) -> None: # noqa: FBT001 self.busy_changed.emit((busy,)) def _validate_component_name(self, name: str) -> bool: - """SEC: verify component exists to prevent abuse on unknown names. - - Also rate-limits invalid requests to detect/prevent fuzzing attacks. - """ + """SEC: verify component exists to prevent abuse on unknown names.""" is_valid = self._svc.has_component(name) if not is_valid: self._invalid_requests += 1 @@ -186,6 +191,8 @@ async def _periodic_status_check(self) -> None: while True: try: await self._emit_status() + if await self._svc.provision_missing(): + await self._emit_status() # reflect freshly-installed components except Exception as exc: # noqa: BLE001 _log.error("periodic_check failed: %s", exc) await asyncio.sleep(self._svc.poll_interval) @@ -232,6 +239,7 @@ async def bless_healthy(self, name: str, hash_val: str) -> bool: return await self._svc.bless_healthy(name, hash_val) async def _run_update_all(self) -> None: + """Run update_all under the process lock, then a background apt pass; always clears busy.""" ran = False try: with process_lock() as acquired: @@ -253,6 +261,7 @@ async def _run_update_all(self) -> None: ) async def _update_all_locked(self) -> None: + """Update only the components whose status is dirty.""" statuses = await self._svc.check_status() dirty = { name @@ -271,6 +280,7 @@ async def _update_all_locked(self) -> None: _log.info("update_all: no dirty components found") async def _run_update_component(self, name: str) -> None: + """Run a single-component update under the process lock; always clears busy.""" try: with process_lock() as acquired: if not acquired: @@ -284,6 +294,7 @@ async def _run_update_component(self, name: str) -> None: self._set_busy(busy=False) async def _run_recover(self, name: str, hard: bool) -> None: # noqa: FBT001 + """Run a recover under the process lock; always clears busy.""" try: with process_lock() as acquired: if not acquired: @@ -312,7 +323,7 @@ async def cancel(self) -> None: """D-Bus method: cancel the running update or recover task and wait for cleanup.""" cancelled_tasks: list[asyncio.Task] = [] for task in list(self._background_tasks): - name = task.get_name() or "" + name = task.get_name() if name.startswith(("update_", "recover_")): task.cancel() cancelled_tasks.append(task) diff --git a/updater/executor.py b/updater/executor.py index 437da324..1605ab35 100644 --- a/updater/executor.py +++ b/updater/executor.py @@ -2,6 +2,7 @@ import asyncio import contextlib +import http.client import json import logging import os @@ -12,6 +13,7 @@ import time from collections.abc import Sequence from pathlib import Path +from urllib.parse import urlsplit from updater.locking import restart_sentinel_path from updater.models import ComponentStatus @@ -45,15 +47,7 @@ def _clear_stale_git_index_lock(path: Path) -> bool: - """Remove .git/index.lock only if it is stale (older than threshold). - - A stale lock is left by a SIGKILL'd or interrupted git operation. Only - remove if mtime indicates lock is NOT held by a live git process (age >= - threshold). At runtime, a fresh lock may indicate a concurrent legitimate - git op; yanking it would corrupt that op. Returns True if lock was - removed or did not exist; False if lock exists but is too fresh to safely - remove. - """ + """Remove .git/index.lock only if stale (mtime age >= threshold).""" lock_path = path / ".git" / "index.lock" if not lock_path.exists(): return True @@ -77,13 +71,12 @@ def _clear_stale_git_index_lock(path: Path) -> bool: def _kill_proc_group(proc, sig): """Kill a process group; suppress errors if already gone.""" - try: + with contextlib.suppress(ProcessLookupError, PermissionError): os.killpg(os.getpgid(proc.pid), sig) - except (ProcessLookupError, PermissionError): - pass # process already gone def _make_clean_env() -> dict[str, str]: + """Build a minimal sanitized environment for updater subprocesses.""" env: dict[str, str] = {} for key in ( "PATH", @@ -117,11 +110,7 @@ async def _run( cwd: Path | None = None, env: dict[str, str] | None = None, ) -> tuple[bool, str]: - """Run a subprocess, return (success, stdout_or_stderr). - - On timeout: SIGTERM → 5s grace → SIGKILL. - On CancelledError: SIGKILL → 2s grace, then re-raise. - """ + """Run a subprocess, return (success, stdout_or_stderr).""" proc = await asyncio.create_subprocess_exec( *[str(c) for c in cmd], cwd=cwd, @@ -161,11 +150,7 @@ def _validate_git_ref(ref: str) -> bool: def _resolve_component_pip(path: Path | None) -> str: - """Find the pip binary for a component's venv; falls back to system pip if none found. - - SEC: Validate venv paths are under component or parent; reject symlinks to prevent - arbitrary venv hijacking. - """ + """Find the component venv's pip, else system pip.""" if path is None: logger.warning("Path not found, using system pip") return PIP @@ -198,11 +183,7 @@ def _resolve_component_pip(path: Path | None) -> str: async def _list_upgradable_packages() -> tuple[bool, list[str]]: - """Return (success, list_of_package_names) from apt list --upgradable. - - SEC: Validate package names against Debian naming rules to prevent - injection attacks even if apt output is malformed. - """ + """Return (success, package_names) from apt list --upgradable.""" ok, output = await _run([APT, "list", "--upgradable"], timeout=30.0) if not ok: return False, [] @@ -232,6 +213,7 @@ def _compile_exclude_patterns(exclude: tuple[str, ...]) -> list[re.Pattern]: def _apply_exclude_patterns(pkgs: list[str], exclude: tuple[str, ...]) -> list[str]: + """Drop packages matching any apt exclude pattern.""" if not exclude: return pkgs compiled = _compile_exclude_patterns(exclude) @@ -239,6 +221,7 @@ def _apply_exclude_patterns(pkgs: list[str], exclude: tuple[str, ...]) -> list[s async def _count_apt_upgradable(exclude: tuple[str, ...] = ()) -> int: + """Return the count of upgradable packages after excludes, or -1 on failure.""" ok, pkgs = await _list_upgradable_packages() if not ok: return -1 @@ -246,6 +229,51 @@ async def _count_apt_upgradable(exclude: tuple[str, ...] = ()) -> int: return len(pkgs) +async def _git_status_preflight( + name: str, path: Path, skip_fetch: bool +) -> tuple[str, ComponentStatus | None]: + """Fetch (unless skipped) + read current hash; return (hash, None) or ('', error).""" + if not skip_fetch: + result, output = await git_fetch(path, prune_remotes=False) + if not result: + logger.error("Failed at git_fetch operation, %s", output) + return "", ComponentStatus(name=name, error=output) + current_hash = await git_get_hash(path) + if current_hash == "": + logger.error("Failed at git_hash operation") + return "", ComponentStatus(name=name, error="Failed at git_hash operation") + return current_hash, None + + +async def _commits_behind_or_error( + path: Path, + name: str, + branch: str | None, + version: str | None, + remote_ref: str, + current_hash: str, + current_branch: str, +) -> tuple[int, ComponentStatus | None]: + """Resolve commits-behind; return (n, None) or (-1, error ComponentStatus).""" + if version: + return 0, None + commits_behind = await git_commits_behind(path, remote_ref) + if commits_behind != -1: + return commits_behind, None + # a configured branch whose origin ref is gone is a config error, not transient + if branch and not await git_ref_hash(path, remote_ref): + return -1, ComponentStatus( + name=name, + current_hash=current_hash, + current_branch=current_branch, + error=f"branch {remote_ref} not found - fix components.yaml", + ) + logger.error("Failed at git_commits_behind operation") + return -1, ComponentStatus( + name=name, error="Failed at git_commits_behind operation" + ) + + async def check_git_status( name: str, path: Path | None, @@ -256,15 +284,9 @@ async def check_git_status( """Return a ComponentStatus snapshot for a git-based component.""" if path is None or not path.exists(): return ComponentStatus(name=name, error=f"path {path} does not exist") - if not skip_fetch: - result, output = await git_fetch(path, prune_remotes=False) - if not result: - logger.error("Failed at git_fetch operation, %s", output) - return ComponentStatus(name=name, error=output) - current_hash = await git_get_hash(path) - if current_hash == "": - logger.error("Failed at git_hash operation") - return ComponentStatus(name=name, error="Failed at git_hash operation") + current_hash, err = await _git_status_preflight(name, path, skip_fetch) + if err is not None: + return err current_branch = await git_get_current_branch(path) if branch: remote_ref = f"origin/{branch}" @@ -272,15 +294,11 @@ async def check_git_status( remote_ref = f"origin/{current_branch}" if current_branch else "origin/HEAD" # Configured branch != checked-out branch: needs an update to switch. branch_mismatch = bool(branch) and current_branch != branch - if version: - commits_behind = 0 - else: - commits_behind = await git_commits_behind(path, remote_ref) - if commits_behind == -1: - logger.error("Failed at git_commits_behind operation") - return ComponentStatus( - name=name, error="Failed at git_commits_behind operation" - ) + commits_behind, err = await _commits_behind_or_error( + path, name, branch, version, remote_ref, current_hash, current_branch + ) + if err is not None: + return err ( remote_url, has_local_changes, @@ -322,6 +340,7 @@ def is_git_repo(path: Path | None) -> bool: async def git_is_dirty(path: Path) -> bool: + """Return True if the repo has uncommitted tracked-file changes.""" ok, output = await _run( [GIT, "status", "--porcelain", "--untracked-files=no"], cwd=path, @@ -331,6 +350,7 @@ async def git_is_dirty(path: Path) -> bool: async def git_prune_extra_remotes(path: Path) -> None: + """Remove every git remote except origin.""" ok, output = await _run([GIT, "remote"], cwd=path, timeout=10.0) if not ok: return @@ -360,7 +380,7 @@ async def _prune_broken_refs(path: Path, output: str) -> bool: """Delete corrupt refs named in a failed fetch. Returns True if any were.""" pruned = False for ref in dict.fromkeys(_BROKEN_REF_RE.findall(output)): - if not ref.startswith("refs/") or any(c.isspace() for c in ref): + if not ref.startswith("refs/") or ".." in ref or any(c.isspace() for c in ref): continue ok, _ = await _run([GIT, "update-ref", "-d", ref], cwd=path, timeout=10.0) if not ok: @@ -373,15 +393,7 @@ async def _prune_broken_refs(path: Path, output: str) -> bool: async def git_fetch( path: Path | None, *, prune_remotes: bool = True ) -> tuple[bool, str]: - """Fetch all remotes for the repo at path. - - prune_remotes=False skips the extra-remote cleanup; pass False during status - checks where pruning is unnecessary and adds ~2 subprocess spawns per repo. - - A corrupt local ref (null SHA from an interrupted write) makes --prune abort - the whole fetch and silently disables every update; on that failure the - named ref is deleted and the fetch retried once. - """ + """Fetch all remotes for the repo at path.""" if path is None: return (False, "path does not exist") if prune_remotes: @@ -408,11 +420,7 @@ async def git_pull(path: Path | None) -> tuple[bool, str]: async def git_clone( url: str, path: Path | None, branch: str | None = None ) -> tuple[bool, str]: - """Clone url into path (https-only), optionally at branch. Returns (ok, message). - - Large repos (klipper, moonraker) can take a while, so the timeout is generous. - git creates the leaf directory; the parent (home) must already exist. - """ + """Clone url into path (https-only), optionally at branch. Returns (ok, message).""" if not path: return (False, "path error") if not _GIT_URL_RE.match(url): @@ -427,13 +435,7 @@ async def git_clone( async def git_reset_to_hash(path: Path | None, prev_hash: str = "") -> tuple[bool, str]: - """Hard-reset repo at path directly to prev_hash without fetching. - - This is the rollback/heal primitive (abort, boot revert, recover), so the - timeout is generous: a reset across a large delta on a slow SD card must - not be SIGTERM'd mid-checkout in exactly the path meant to fix things. - If reset fails with an index.lock error, clears it and retries once. - """ + """Hard-reset repo at path directly to prev_hash (no fetch).""" if not path: return (False, "path error") if prev_hash == "": @@ -469,13 +471,7 @@ async def git_reset_to_hash(path: Path | None, prev_hash: str = "") -> tuple[boo async def git_has_corruption(path: Path | None, hint: str = "") -> bool: - """True if `hint` or git fsck shows object corruption. - - `hint` (e.g. a failed fetch's stderr) is checked first: `git fsck - --connectivity-only` validates reachability, not object content, so it can - miss a corrupt blob that already broke the fetch. The fetch error is the - reliable signal in that case. - """ + """True if `hint` or git fsck shows object corruption.""" if path is None: return False if any(k in hint for k in _GIT_CORRUPT_SIGNATURES): @@ -505,15 +501,7 @@ def _prune_empty_loose_objects(objects: Path) -> int: async def _quarantine_corrupt_objects(path: Path) -> int: - """Move loose objects that `git fsck --full` flags as corrupt out of the way. - - `git_has_corruption` uses `fsck --connectivity-only`, which never reads blob - content and so cannot see a non-empty object with a bad zlib header - exactly - what a power cut mid-write produces. A plain re-fetch will NOT replace such an - object because git still finds a file at that name. `fsck --full` reads object - content and names the bad ones; we move (not delete) them so a re-fetch - re-downloads them, and a misdiagnosis stays recoverable. Returns count moved. - """ + """Move loose objects that `git fsck --full` flags as corrupt out of the way.""" ok, out = await _run( [GIT, "fsck", "--full", "--no-dangling", "--no-progress"], cwd=path, @@ -549,20 +537,12 @@ async def _quarantine_corrupt_objects(path: Path) -> int: return moved -async def git_repair(path: Path, branch: str = "main") -> tuple[bool, str]: - """Prune 0-byte loose objects, re-fetch, and re-verify. Mirrors start.sh recovery. - - If empty-object pruning plus a fetch does not clear the corruption, escalate to - quarantining non-empty corrupt loose objects (`fsck --full`) and re-fetch once - more. Working tree is untouched (delete/move + fetch only), so tracked-but- - modified files survive. If fetch fails with index.lock error, clears lock and - retries. Branch parameter is used to repair corrupt HEAD (fallback to "main"). - Returns (ok, message). - """ +async def _git_repair_cleanup_fetch(path: Path) -> tuple[bool, str, int]: + """Clear locks, prune empty loose objects, fetch (retrying past a stale index.lock).""" _clear_stale_git_index_lock(path) objects = path / ".git" / "objects" if not objects.is_dir(): - return (False, "no .git/objects directory") + return (False, "no .git/objects directory", 0) removed = _prune_empty_loose_objects(objects) logger.warning("git_repair: removed %d empty object(s) from %s", removed, path) ok, out = await git_fetch(path, prune_remotes=False) @@ -570,11 +550,14 @@ async def git_repair(path: Path, branch: str = "main") -> tuple[bool, str]: _clear_stale_git_index_lock(path) ok, out = await git_fetch(path, prune_remotes=False) if not ok: - return (False, f"fetch after cleanup failed: {out}") - if not await git_has_corruption(path): - if not await _repair_corrupt_head(path, branch): - return (False, "repaired objects but HEAD still unreadable") - return (True, f"repaired ({removed} empty objects removed)") + return (False, f"fetch after cleanup failed: {out}", removed) + return (True, "", removed) + + +async def _git_repair_after_quarantine( + path: Path, branch: str | None, removed: int +) -> tuple[bool, str]: + """Quarantine corrupt objects, re-fetch, re-verify HEAD when object cleanup wasn't enough.""" quarantined = await _quarantine_corrupt_objects(path) if quarantined == 0: return (False, "still corrupt after fetch (no quarantinable objects found)") @@ -591,6 +574,18 @@ async def git_repair(path: Path, branch: str = "main") -> tuple[bool, str]: ) +async def git_repair(path: Path, branch: str | None = None) -> tuple[bool, str]: + """Prune 0-byte loose objects, re-fetch, re-verify. Mirrors start.sh recovery.""" + ok, msg, removed = await _git_repair_cleanup_fetch(path) + if not ok: + return (False, msg) + if not await git_has_corruption(path): + if not await _repair_corrupt_head(path, branch): + return (False, "repaired objects but HEAD still unreadable") + return (True, f"repaired ({removed} empty objects removed)") + return await _git_repair_after_quarantine(path, branch, removed) + + async def git_get_hash(path: Path | None) -> str: """Return the current HEAD commit hash, or empty string on failure.""" if path is None: @@ -605,13 +600,11 @@ async def _is_head_readable(path: Path) -> bool: return ok -async def _repair_corrupt_head(path: Path, branch: str = "main") -> bool: - """Try to repair a corrupt HEAD by rewriting it to a valid symbolic ref. - - Returns True if HEAD was repaired or is already readable. - """ +async def _repair_corrupt_head(path: Path, branch: str | None = None) -> bool: + """Try to repair a corrupt HEAD by rewriting it to a valid symbolic ref.""" if await _is_head_readable(path): return True + branch = branch or await git_default_branch(path) ok, _msg = await _run( [GIT, "symbolic-ref", "HEAD", f"refs/heads/{branch}"], cwd=path, @@ -632,6 +625,16 @@ async def git_ref_hash(path: Path | None, ref: str) -> str: return output.strip() if ok else "" +async def git_tree_has_path(path: Path | None, ref: str, repo_path: str) -> bool: + """True if repo_path exists in the tree at ref (git cat-file -e ref:path).""" + if path is None: + return False + ok, _ = await _run( + [GIT, "cat-file", "-e", f"{ref}:{repo_path}"], cwd=path, timeout=10.0 + ) + return ok + + async def git_commits_behind(path: Path, remote_ref: str = "origin/HEAD") -> int: """Return how many commits the local branch is behind remote_ref.""" ok, output = await _run( @@ -668,13 +671,32 @@ async def _assert_https_remote(path: Path | None) -> tuple[bool, str]: async def git_get_current_branch(path: Path) -> str: + """Return the repo's current branch name, or empty string on failure.""" ok, output = await _run( [GIT, "rev-parse", "--abbrev-ref", "HEAD"], cwd=path, timeout=10.0 ) return output.strip() if ok else "" +async def git_default_branch(path: Path | None) -> str: + """Best-effort default branch: HEAD's symref, else origin/HEAD's target, else master.""" + if path is None: + return "master" + ok, out = await _run( + [GIT, "symbolic-ref", "--short", "HEAD"], cwd=path, timeout=10.0 + ) + if ok and out.strip(): + return out.strip() + ok, out = await _run( + [GIT, "rev-parse", "--abbrev-ref", "origin/HEAD"], cwd=path, timeout=10.0 + ) + if ok and out.strip().startswith("origin/"): + return out.strip().removeprefix("origin/") + return "master" + + async def git_describe(path: Path, ref: str | None = None) -> str: + """Return the nearest tag for ref (or HEAD), or empty string.""" cmd = [GIT, "describe", "--tags", "--abbrev=0"] if ref: cmd.append(ref) @@ -685,6 +707,7 @@ async def git_describe(path: Path, ref: str | None = None) -> str: async def git_checkout( path: Path | None, branch: str, *, force: bool = False ) -> tuple[bool, str]: + """Check out branch (force overwrites untracked collisions); validates the name.""" if path is None: return (False, "path not found") @@ -703,16 +726,10 @@ async def git_checkout( return await _run(cmd, cwd=path, timeout=60.0) -async def check_apt_status( - cache_ttl_seconds: int = 86_400, exclude: tuple[str, ...] = () -) -> ComponentStatus: - """Return apt ComponentStatus from cache, refreshing via apt-get if older than cache_ttl_seconds. - - SEC: Validate cache file ownership and permissions to prevent privilege escalation. - """ - path = Path.home() / ".cache" / "blockscreen" / "apt_status_cache.json" - exclude_key = "|".join(sorted(exclude)) - packages_upgradable = -1 +def _read_apt_cache( + path: Path, cache_ttl_seconds: int, exclude_key: str +) -> ComponentStatus | None: + """Return a cached apt ComponentStatus if fresh + owner-safe, else None.""" try: if path.exists(): stat = path.stat() @@ -734,34 +751,51 @@ async def check_apt_status( ) except (json.JSONDecodeError, IOError, ValueError) as e: logger.warning("apt cache miss %s", e) + return None + + +def _write_apt_cache(path: Path, packages_upgradable: int, exclude_key: str) -> None: + """Atomically persist the apt upgradable count + exclude key to the cache file.""" + tmp_path = None + try: + path.parent.mkdir(parents=True, mode=0o0700, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", + dir=path.parent, + delete=False, + prefix=".apt_cache_", + suffix=".tmp", + ) as f: + json.dump( + { + "packages_upgradable": packages_upgradable, + "exclude_key": exclude_key, + "cached_ts": time.time(), + }, + f, + ) + tmp_path = Path(f.name) + tmp_path.chmod(0o600) + tmp_path.replace(path) + except (json.JSONDecodeError, IOError, ValueError) as e: + if tmp_path is not None and tmp_path.exists(): + tmp_path.unlink(missing_ok=True) + logger.error("writing cache apt data: %s", e) + + +async def check_apt_status( + cache_ttl_seconds: int = 86_400, exclude: tuple[str, ...] = () +) -> ComponentStatus: + """Return cached apt ComponentStatus, refreshing via apt-get past the TTL.""" + path = Path.home() / ".cache" / "blockscreen" / "apt_status_cache.json" + exclude_key = "|".join(sorted(exclude)) + cached = _read_apt_cache(path, cache_ttl_seconds, exclude_key) + if cached is not None: + return cached packages_upgradable = await _count_apt_upgradable(exclude=exclude) if packages_upgradable >= 0: - tmp_path = None - try: - path.parent.mkdir(parents=True, mode=0o0700, exist_ok=True) - with tempfile.NamedTemporaryFile( - mode="w", - dir=path.parent, - delete=False, - prefix=".apt_cache_", - suffix=".tmp", - ) as f: - json.dump( - { - "packages_upgradable": packages_upgradable, - "exclude_key": exclude_key, - "cached_ts": time.time(), - }, - f, - ) - tmp_path = Path(f.name) - tmp_path.chmod(0o600) - tmp_path.replace(path) - except (json.JSONDecodeError, IOError, ValueError) as e: - if tmp_path is not None and tmp_path.exists(): - tmp_path.unlink(missing_ok=True) - logger.error("writing cache apt data: %s", e) + _write_apt_cache(path, packages_upgradable, exclude_key) if packages_upgradable < 0: return ComponentStatus( @@ -776,6 +810,7 @@ async def check_apt_status( def _apt_env() -> dict[str, str]: + """Return the apt subprocess env: noninteractive frontend, needrestart disabled.""" env = _make_clean_env() env["DEBIAN_FRONTEND"] = "noninteractive" # needrestart can otherwise open an interactive prompt mid-upgrade and hang. @@ -784,25 +819,16 @@ def _apt_env() -> dict[str, str]: def _apt_cmd(verb: str, pkgs: Sequence[str] = ()) -> list[str]: - """Build the sudo apt argv via the root-owned wrapper. - - The wrapper is the only apt command sudoers grants. bs-bootstrap reinstalls it - when missing (an old-installer box can have the service but not the wrapper); until - that self-heal runs a missing wrapper surfaces as a plain 'command not found'. - """ + """Build the sudo apt argv via the root-owned wrapper.""" return [SUDO, str(APT_HELPER), verb, *pkgs] async def _apt_snapshot_packages() -> tuple[bool, Path | None]: - """Snapshot current package state with dpkg --get-selections to temp file. - - Returns (success, snapshot_path). - Path is stored at ~/.cache/blockscreen/apt_pre_upgrade_snapshot.txt. - No sudo required for dpkg --get-selections. - """ + """Snapshot package state via dpkg --get-selections to a temp file (no sudo).""" snapshot_path = ( Path.home() / ".cache" / "blockscreen" / "apt_pre_upgrade_snapshot.txt" ) + tmp: Path | None = None try: snapshot_path.parent.mkdir(parents=True, mode=0o0700, exist_ok=True) ok, output = await _run([DPKG, "--get-selections"], timeout=15.0) @@ -823,26 +849,19 @@ async def _apt_snapshot_packages() -> tuple[bool, Path | None]: return True, snapshot_path except (IOError, ValueError) as e: logger.error("apt snapshot error: %s", e) + if tmp is not None: + with contextlib.suppress(OSError): + tmp.unlink() return False, None async def _apt_get_fix_broken() -> tuple[bool, str]: - """Run apt-get -f install -y to fix broken package state. - - Best-effort rollback after failed upgrade. May not fully restore state, - but aims to unbreak the system. Called only on apt_upgrade failure. - """ + """Run apt-get -f install -y to unbreak package state.""" return await _run(_apt_cmd("fix-broken"), timeout=120.0, env=_apt_env()) async def _apt_restore_packages(snapshot_path: Path) -> tuple[bool, str]: - """Restore dpkg package *selections* from the pre-upgrade snapshot, then dselect-upgrade. - - This re-asserts which packages should be installed; it does NOT reliably - downgrade versions, since the prior .debs may no longer be in the apt cache. - Treat it as an unbreak step (keep the set consistent), not a true rollback. - Kernel/firmware are excluded from upgrades, bounding the blast radius. - """ + """Restore dpkg selections from the pre-upgrade snapshot, then dselect-upgrade.""" try: stdin_data = snapshot_path.read_bytes() except OSError as e: @@ -903,11 +922,7 @@ async def apt_update() -> tuple[bool, str]: async def apt_upgrade(exclude: tuple[str, ...] = ()) -> tuple[bool, str]: - """Run apt-get upgrade via explicit package list to limit blast radius. - - Always lists upgradable packages first; applies exclude regexes if any. - Uses 'install --only-upgrade' so only already-installed packages are touched. - """ + """Run apt-get upgrade via an explicit package list to limit blast radius.""" ok, pkgs = await _list_upgradable_packages() if not ok: return False, "failed to list upgradable packages" @@ -932,12 +947,7 @@ async def run_hook( prev_hash: str, timeout: float = 60.0, ) -> tuple[bool, str]: - """Run the per-component update hook if it exists. - - The 60s default suits tests and trivial hooks; the update and provisioning - flows pass HOOK_TIMEOUT since a hook may sync a full dependency set (e.g. - Spoolman's `uv sync` after its deps changed). - """ + """Run the per-component update hook if it exists.""" hook = (_HOOKS_DIR / f"{name}.sh").resolve() # SEC: resolve symlinks try: hook.relative_to(_HOOKS_DIR.resolve()) # SEC: prevent path traversal @@ -967,12 +977,7 @@ async def enable_service(name: str | None) -> tuple[bool, str]: async def wait_for_service_active(name: str, timeout: float = 90.0) -> bool: - """Poll systemctl is-active until the service is active or timeout expires. - - `systemctl is-active --wait` does not exist (--wait applies only to - start/restart/is-system-running/kill and is silently ignored here), so an - explicit poll loop is required to ride out 'activating' after a restart. - """ + """Poll systemctl is-active until active or timeout.""" if not _SERVICE_RE.match(name): logger.warning("wait_for_service_active: invalid service name %r", name) return False @@ -995,13 +1000,36 @@ async def wait_for_service_active(name: str, timeout: float = 90.0) -> bool: return False -async def restart_service_noblock(name: str | None) -> tuple[bool, str]: - """Queue a service restart without waiting (systemctl --no-block). +def _http_probe(url: str) -> bool: + """One blocking GET; True only on a 2xx from the loopback health URL.""" + parts = urlsplit(url) + conn = http.client.HTTPConnection( + parts.hostname or "127.0.0.1", parts.port or 80, timeout=5.0 + ) + try: + conn.request("GET", parts.path or "/") + return 200 <= conn.getresponse().status < 300 + except (OSError, http.client.HTTPException): + return False + finally: + conn.close() + + +async def wait_for_http_ready(url: str, timeout: float = 120.0) -> bool: + """Poll a component's loopback health URL until it returns 2xx or timeout.""" + deadline = asyncio.get_running_loop().time() + timeout + while True: + if await asyncio.to_thread(_http_probe, url): + logger.info("health check ok: %s", url) + return True + if asyncio.get_running_loop().time() >= deadline: + logger.warning("health check timed out after %.0fs: %s", timeout, url) + return False + await asyncio.sleep(2.0) + - For the UI service that hosts the updater's D-Bus client: a blocking restart - tears down that client, and a slow restart could time out and abort the - in-flight update. Fire-and-forget and let it reload on its own. - """ +async def restart_service_noblock(name: str | None) -> tuple[bool, str]: + """Queue a service restart without waiting (systemctl --no-block).""" if name is None: return (False, "service name is None") if not _SERVICE_RE.match(name): @@ -1010,13 +1038,7 @@ async def restart_service_noblock(name: str | None) -> tuple[bool, str]: async def verify_updater_importable(component_path: Path | None) -> bool: - """Self-test the new on-disk updater code before restarting into it. - - Imports the updater package in a fresh interpreter from component_path. A - new updater that fails to import must never restart the daemon onto itself: - that would take down the only field update path. On failure the caller keeps - the running (old) daemon and lets the next reboot adopt the new code. - """ + """Self-test the new on-disk updater code before restarting into it.""" if component_path is None or not component_path.exists(): return False ok, out = await _run( @@ -1030,23 +1052,18 @@ async def verify_updater_importable(component_path: Path | None) -> bool: async def restart_service(name: str | None) -> tuple[bool, str]: - """Restart a systemd service, recovering from a start-limit hit. - - If `restart` fails (commonly "start request repeated too quickly"), clear the - rate-limit/failed state with `reset-failed` and retry once. `systemctl kill` - is NOT used as a fallback: it does not clear a start-limit and is not covered - by the scoped NOPASSWD sudoers rules, so it would prompt for a password. - """ + """Restart a systemd service, recovering from a start-limit hit.""" if name is None: return (False, "service name is None") if not _SERVICE_RE.match(name): return (False, f"service name {name!r} is invalid") - ok, err = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=30.0) + # 120s timeout: Type=notify unit READY wait (up to 90s default TimeoutStartSec) plus margin for slow cold UI start. + ok, err = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=120.0) if ok: return (True, "") logger.warning("systemctl restart %s failed (%s); reset-failed + retry", name, err) await _run([SUDO, SYSTEMCTL, "reset-failed", name], timeout=10.0) - ok, err2 = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=30.0) + ok, err2 = await _run([SUDO, SYSTEMCTL, "restart", name], timeout=120.0) if ok: return (True, f"recovered via reset-failed (first error: {err})") return (False, err2) diff --git a/updater/hooks/Spoolman.sh b/updater/hooks/Spoolman.sh index 6dd7b552..8309f676 100755 --- a/updater/hooks/Spoolman.sh +++ b/updater/hooks/Spoolman.sh @@ -1,7 +1,5 @@ #!/bin/bash -# Provision/update Spoolman: run-from-source (not pip-installable), so deps go in -# via uv (provided in the BlocksScreen venv by install-updater.sh). SQLite default -# means no sudo here; the unit is laid down + enabled by install-updater/updater. +# Provision Spoolman via uv (run-from-source, not pip-installable); SQLite default = no sudo; unit installed by install-updater. set -euo pipefail if [ -z "${COMPONENT_PATH:-}" ]; then @@ -21,8 +19,7 @@ fi cd "$COMPONENT_PATH" "$_uv" sync --no-dev -# Spoolman won't start without client/dist (a clone has no prebuilt UI); the API -# is all we need, so a stub satisfies the mount without an npm build. +# Stub client/dist (a clone has no prebuilt UI); the API is all we need, no npm build. mkdir -p client/dist if [ ! -f client/dist/index.html ]; then printf 'Spoolman\n' >client/dist/index.html diff --git a/updater/locking.py b/updater/locking.py index c7bd5abb..d0f4ac10 100644 --- a/updater/locking.py +++ b/updater/locking.py @@ -1,10 +1,4 @@ -"""Cross-process advisory lock shared by the updater daemon and the CLI. - -A single flock-backed lockfile is the only thing serializing mutating git/apt -work between the long-running daemon and a `python -m updater update` CLI run: -the daemon's asyncio locks are in-process only. Both sides acquire the same path -non-blocking, so whoever is second fails fast instead of corrupting a repo. -""" +"""Cross-process advisory lock shared by the updater daemon and the CLI.""" from __future__ import annotations @@ -36,24 +30,13 @@ def lock_path() -> Path: def restart_sentinel_path() -> Path: - """Return the sentinel a self-update hook writes instead of restarting. - - Hooks running under the updater append `install`/`code` here rather than - restarting the daemon mid-batch; the daemon reads it once the batch is done. - Preferring /run (tmpfs) means it clears on reboot, so a crash can never - trigger a spurious restart later; the safe floor is 'adopt on next reboot'. - """ + """Return the sentinel a self-update hook writes instead of restarting mid-batch.""" return _runtime_dir() / "updater-restart-needed" @contextlib.contextmanager def process_lock() -> Iterator[bool]: - """Acquire the shared updater lock non-blocking. - - Yields True if acquired, False if another updater process (daemon or CLI) - holds it. The fd stays open for the whole `with` block - closing it on exit - is what releases the lock. - """ + """Acquire the shared updater lock non-blocking.""" try: f = open(lock_path(), "w") # noqa: SIM115, PTH123 except OSError: diff --git a/updater/models.py b/updater/models.py index cb1af71b..234af7a4 100644 --- a/updater/models.py +++ b/updater/models.py @@ -17,10 +17,9 @@ class ComponentConfig: apt_exclude: tuple[str, ...] = () url: str | None = None install_if_missing: bool = False - # Restart BlocksScreen on update even when the component's service differs. restart_ui: bool = False - # Restart klipper on update even when the component's own service differs. restart_klipper: bool = False + health_url: str | None = None @dataclass(frozen=True) @@ -36,7 +35,5 @@ class ComponentStatus: error: str | None = None has_local_changes: bool = False needs_install: bool = False - # Checked-out branch differs from configured branch (switch needed). branch_mismatch: bool = False - # Actual checked-out branch, surfaced for debugging branch switches. current_branch: str = "" diff --git a/updater/service.py b/updater/service.py index 77a1671f..70f29a01 100644 --- a/updater/service.py +++ b/updater/service.py @@ -42,6 +42,7 @@ git_has_corruption, git_pull, git_ref_hash, + git_tree_has_path, git_repair, git_reset_to_hash, is_git_repo, @@ -49,6 +50,7 @@ restart_service_noblock, run_hook, verify_updater_importable, + wait_for_http_ready, wait_for_service_active, ) from updater.locking import process_lock, restart_sentinel_path @@ -143,6 +145,8 @@ def reset(self) -> None: # Self-heal: the UI component name (components.yaml) that the supervisor watches. _UI_COMPONENT = "BlocksScreen" +# Marker file proving updater exists: absence at target ref aborts update (lack bricks Type=notify host with no self-heal). +_UPDATER_MARKER = "updater/dbus_service.py" # Forward-heal always targets the curated-stable channel, not the configured branch. _HEAL_REMOTE_REF = "origin/main" # Settle window after a rung (debounce plus margin) so a new build can bless first. @@ -156,33 +160,53 @@ def reset(self) -> None: class ProgressCallback(Protocol): - def on_step(self, name: str, step: int, total: int) -> None: ... - def on_component_done(self, name: str, success: bool) -> None: ... - def on_error(self, name: str, reason: str) -> None: ... - def on_rollback(self, name: str, success: bool) -> None: ... - def on_recover(self, name: str, success: bool) -> None: ... + def on_step(self, name: str, step: int, total: int) -> None: + """Report a numbered progress step for a component.""" + ... + + def on_component_done(self, name: str, success: bool) -> None: + """Report a component's final success or failure.""" + ... + + def on_error(self, name: str, reason: str) -> None: + """Report a non-recoverable error for a component.""" + ... + + def on_rollback(self, name: str, success: bool) -> None: + """Report a rollback attempt's outcome.""" + ... + + def on_recover(self, name: str, success: bool) -> None: + """Report a recover attempt's outcome.""" + ... class LoggingCallback: def __init__(self) -> None: + """Create the logging progress callback.""" self._log = logging.getLogger("updater") def on_step(self, name: str, step: int, total: int) -> None: + """Log a progress step.""" self._log.info("%s step %d/%d", name, step, total) def on_component_done(self, name: str, success: bool) -> None: + """Log a component's success or failure.""" if success: self._log.info("%s done(ok=True)", name) else: self._log.error("%s done(ok=False)", name) def on_error(self, name: str, reason: str) -> None: + """Log a component error.""" self._log.error("%s error: %s", name, reason) def on_rollback(self, name: str, success: bool) -> None: + """Log a rollback outcome.""" self._log.warning("%s rollback (ok=%s)", name, success) def on_recover(self, name: str, success: bool) -> None: + """Log a recover outcome.""" self._log.info("%s recover (ok=%s)", name, success) @@ -192,6 +216,7 @@ class UpdateService: _FETCH_TTL: float = 300.0 def __init__(self, callback: ProgressCallback | None = None) -> None: + """Load components and initialize locks, caches, and self-heal tracking.""" self._components, self.poll_interval = load_components() self._git_lock = Lock() self._state_lock = Lock() @@ -226,14 +251,11 @@ def component_stubs(self) -> list[tuple[str, str]]: return [(c.name, c.kind) for c in self._components] async def check_status(self, force: bool = False) -> dict[str, ComponentStatus]: - """Concurrently check status of all components. - - force=True bypasses the fetch TTL so git fetch always runs. - Use for explicit user-triggered refreshes. - """ + """Concurrently check status of all components.""" results: dict[str, ComponentStatus] = {} async def _check_one(c: ComponentConfig) -> None: + """Fetch and record one component's status into the results dict.""" if c.kind == "apt": # force bypasses the apt cache, mirroring the git fetch TTL bypass. status = await check_apt_status( @@ -291,42 +313,42 @@ async def update_component(self, name: str) -> bool: return await self._run_apt_update(component) return await self._run_git_update(component) - async def update_all(self, names: set[str] | None = None) -> None: - """Update components: apt independently, existing git as one atomic batch. - - apt packages and missing-component provisioning are independent of the git - code set, so they run outside the all-or-nothing batch. - """ + async def update_all(self, names: set[str] | None = None) -> bool: + """Update components: apt independently, existing git as one phased batch.""" components = ( self._components if names is None else [c for c in self._components if c.name in names] ) - free_mb = shutil.disk_usage(Path.home()).free / 1_048_576 - if free_mb < 200: - msg = f"insufficient disk space ({free_mb:.0f} MB free, need 200 MB)" - self._log.error("update_all: %s", msg) - for c in components: - self._cb("on_error", c.name, msg) - return + if not self._check_disk_space(components): + return False sorted_components = sorted(components, key=lambda c: c.order) - if not await self._preflight_fetch(sorted_components, components): - return - + offline = await self._preflight_fetch(sorted_components) + + apt, batch = self._build_apt_and_batch_lists(sorted_components) + ok = await self._filter_offline_batch(batch, offline) + ok = await self._filter_nonrepo_batch(batch) and ok + ok = await self._filter_dead_branch_batch(batch) and ok + provision = self._build_provision_list(sorted_components) + return await self._run_update_phases(apt, batch, provision) and ok + + def _build_apt_and_batch_lists( + self, sorted_components: list[ComponentConfig] + ) -> tuple[list[ComponentConfig], list[ComponentConfig]]: + """Partition sorted components into (apt, existing-git-repo batch).""" apt = [c for c in sorted_components if c.kind == "apt"] batch = [ c for c in sorted_components if c.kind == "git" and c.path is not None and c.path.exists() ] - # A dir without .git (tarball install) errors individually, not the batch. - for c in [c for c in batch if not is_git_repo(c.path)]: - batch.remove(c) - if c.install_if_missing and c.url and await self._quarantine_nonrepo(c): - continue # path is now absent: the provision pass below clones fresh - self._log.error("%s: %s is not a git repository - skipping", c.name, c.path) - self._cb_error_done(c.name, "not a git repository - reinstall required") - provision = [ + return apt, batch + + def _build_provision_list( + self, sorted_components: list[ComponentConfig] + ) -> list[ComponentConfig]: + """Git components with no local path (incl. just-quarantined) safe to auto-clone.""" + return [ c for c in sorted_components if c.kind == "git" @@ -334,31 +356,112 @@ async def update_all(self, names: set[str] | None = None) -> None: and c.install_if_missing and c.url ] + + async def _run_update_phases( + self, + apt: list[ComponentConfig], + batch: list[ComponentConfig], + provision: list[ComponentConfig], + ) -> bool: + """Run apt updates, then the git batch, then provisioning; AND all results.""" + ok = True for c in apt: - await self._run_apt_update(c) + ok = await self._run_apt_update(c) and ok if batch: - await self._run_git_batch(batch) + ok = await self._run_git_batch(batch) and ok for c in provision: - await self._provision_component(c) + ok = await self._provision_component(c) and ok + return ok - async def _preflight_fetch( - self, sorted_components: list[ComponentConfig], all_components: list + def _check_disk_space(self, components: list[ComponentConfig]) -> bool: + """Return False (and report on_error for each component) if free space < 200 MB.""" + free_mb = shutil.disk_usage(Path.home()).free / 1_048_576 + if free_mb >= 200: + return True + msg = f"insufficient disk space ({free_mb:.0f} MB free, need 200 MB)" + self._log.error("update_all: %s", msg) + for c in components: + self._cb_error_done(c.name, msg) + return False + + async def _filter_offline_batch( + self, batch: list[ComponentConfig], offline: set[str] ) -> bool: - """Fetch every existing git component up-front (network phase). - - Decoupling the download from the apply means a mid-sequence connection - drop aborts before anything is checked out, instead of leaving an - applied/unapplied skew across components. A pre-fetched component skips - its own fetch in the apply phase (recorded fetch time). Repos that need - cloning or are corrupt are left to self-heal in their own flow. - """ + """Drop offline components from the batch, erroring each individually.""" + # An unreachable remote drops only that component, never the whole update. + ok = True + for c in [c for c in batch if c.name in offline]: + batch.remove(c) + ok = self._cb_error_done(c.name, "network error during pre-flight fetch") + return ok + + async def _filter_nonrepo_batch(self, batch: list[ComponentConfig]) -> bool: + """Drop non-git-repo dirs, quarantining installable ones for a fresh clone.""" + # A dir without .git (tarball install) errors individually, not the batch. + ok = True + for c in [c for c in batch if not is_git_repo(c.path)]: + batch.remove(c) + if c.install_if_missing and c.url and await self._quarantine_nonrepo(c): + continue # path is now absent: the provision pass below clones fresh + self._log.error("%s: %s is not a git repository - skipping", c.name, c.path) + ok = self._cb_error_done( + c.name, "not a git repository - reinstall required" + ) + return ok + + async def _filter_dead_branch_batch(self, batch: list[ComponentConfig]) -> bool: + """Drop components whose effective upstream ref (configured or current) is gone.""" + # A dead ref (configured or deleted current branch) must not abort the batch. + ok = True + for c in batch.copy(): + branch = c.branch + if not branch and c.path is not None: + branch = await git_get_current_branch(c.path) + if branch and not await git_ref_hash(c.path, f"origin/{branch}"): + batch.remove(c) + self._log.error( + "%s: origin/%s does not resolve (deleted upstream?) - skipping", + c.name, + branch, + ) + ok = self._cb_error_done( + c.name, f"branch origin/{branch} not found - fix components.yaml" + ) + return ok + + async def provision_missing(self) -> bool: + """Clone absent install_if_missing components without waiting for a manual""" + missing = [ + c + for c in self._components + if c.install_if_missing + and c.url + and (c.path is None or not c.path.exists()) + ] + if not missing: + return False + provisioned = False + with process_lock() as acquired: + if not acquired: + self._log.info("provision_missing: update in progress, deferring") + return False + for c in missing: + if c.path is None or not c.path.exists(): # recheck under lock + await self._provision_component(c) + provisioned = True + return provisioned + + async def _preflight_fetch( + self, sorted_components: list[ComponentConfig] + ) -> set[str]: + """Fetch every existing git component up-front (network phase).""" # Non-repo dirs excluded: their fetch failure is not "offline" (see update_all). targets = [ c for c in sorted_components if c.kind == "git" and c.path is not None and is_git_repo(c.path) ] - offline: list[str] = [] + offline: set[str] = set() # Lock guards only _fetch_times; git_fetch runs outside it (as check_status). for c in targets: now = time.monotonic() @@ -372,181 +475,384 @@ async def _preflight_fetch( async with self._git_lock: self._fetch_times[c.name] = now elif not await git_has_corruption(c.path, hint=err): - offline.append(c.name) + offline.add(c.name) if offline: - msg = "network error during pre-flight fetch - no components changed" - self._log.error("update_all: %s (failed: %s)", msg, offline) - for c in all_components: - self._cb("on_error", c.name, msg) - return False - return True - - async def _run_git_batch(self, batch: list[ComponentConfig]) -> None: - """Apply existing git components all-or-nothing. + self._log.error( + "update_all: pre-flight fetch failed for %s - continuing without them", + sorted(offline), + ) + return offline - Stage every repo, then deps, then hooks, then restart each unique service - once. While staging/deps/hooks run nothing has been restarted, so the live - services still execute the old code - an abort then just reverts git and - restarts nothing. A restart-phase failure reverts git AND re-restarts the - services already bounced so they come back on the old code. - """ + async def _prepare_git_batch_hashes( + self, batch: list[ComponentConfig] + ) -> tuple[list[ComponentConfig], dict[str, str], bool]: + """Snapshot each component's prev_hash; drop (error) those with an empty hash.""" prev: dict[str, str] = {} + alive: list[ComponentConfig] = [] for c in batch: ph = await git_get_hash(c.path) if ph == "": - self._log.error("%s: prev_hash empty - aborting batch", c.name) - for b in batch: - self._cb_error_done(b.name, "prev_hash empty") - return + self._log.error("%s: prev_hash empty - skipping", c.name) + self._cb_error_done(c.name, "prev_hash empty") + continue prev[c.name] = ph + alive.append(c) + failed = len(alive) != len(batch) + return alive, prev, failed + + async def _persist_batch_rollback(self, prev: dict[str, str]) -> bool: + """Persist per-component prev_hash + the inflight marker under the state lock.""" async with self._state_lock: state = await asyncio.to_thread(self._read_state) for name, ph in prev.items(): - state[name] = {"prev_hash": ph} + # Merge: replacing the entry would wipe the self-heal anchors (last_good/golden). + _ensure_comp(state, name)["prev_hash"] = ph if not await asyncio.to_thread(self._write_state, state): - for b in batch: - self._cb_error_done(b.name, "failed to persist rollback point") - return + return False # Mark in-flight so a pre-commit power cut is reverted on next boot. - await asyncio.to_thread(self._write_inflight, dict(prev)) - for c in batch: - ok, reason = await _assert_https_remote(c.path) - if not ok: + if not await asyncio.to_thread(self._write_inflight, prev.copy()): + return False + return True + + async def _security_check_batch( + self, alive: list[ComponentConfig], prev: dict[str, str] + ) -> tuple[list[ComponentConfig], bool]: + """Assert an https remote per component, dropping any that fail SEC-4.""" + checked: list[ComponentConfig] = [] + failed = False + for c in alive: + sec_ok, reason = await _assert_https_remote(c.path) + if not sec_ok: self._log.error("%s: SEC-4 remote check failed: %s", c.name, reason) - # Nothing staged: drop the marker (avoids a spurious boot revert). - await asyncio.to_thread(self._clear_inflight) - for b in batch: - self._cb_error_done(b.name, "insecure remote") - return + self._cb_error_done(c.name, "insecure remote") + failed = True + continue self._history("update_start", c.name, prev_hash=prev[c.name][:12]) + checked.append(c) + return checked, failed + + async def _batch_stage_phase( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + pending_revert: dict[str, str], + ) -> tuple[list[ComponentConfig], bool]: + """Stage each repo to its target ref, reverting the ones that fail.""" + self._log.info("git batch: stage phase (%d repo(s))", len(alive)) + failed = False + survivors: list[ComponentConfig] = [] + for c in alive: + self._cb("on_step", c.name, 1, 4) + touched.append(c) # mark before staging so a partial stage is reverted + ok, reason = await self._stage_component(c) + if ok: + survivors.append(c) + continue + failed = True + if not await self._drop_component(c, prev[c.name], reason): + pending_revert[c.name] = prev[c.name] + touched.remove(c) + return survivors, failed + async def _batch_deps_phase( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + pending_revert: dict[str, str], + ) -> tuple[list[ComponentConfig], bool]: + """Install each survivor's deps, reverting the ones whose install fails.""" + self._log.info("git batch: deps phase") + failed = False + survivors: list[ComponentConfig] = [] + for c in alive: + self._cb("on_step", c.name, 2, 4) + deps_ok, deps_err = await self._install_dependencies(c) + if deps_ok: + survivors.append(c) + continue + self._log.warning("%s: deps failed: %s", c.name, deps_err) + failed = True + if not await self._drop_component(c, prev[c.name], "deps"): + pending_revert[c.name] = prev[c.name] + touched.remove(c) + return survivors, failed + + async def _batch_hook_phase( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + pending_revert: dict[str, str], + ) -> tuple[list[ComponentConfig], dict[str, str], bool]: + """Run each survivor's post-update hook, reverting the ones whose hook fails.""" + self._log.info("git batch: hook phase") + new_hashes: dict[str, str] = {} + failed = False + survivors: list[ComponentConfig] = [] + for c in alive: + self._cb("on_step", c.name, 3, 4) + new_hashes[c.name] = await git_get_hash(c.path) + self._log.info( + "git batch hook: %s %s -> %s", + c.name, + prev[c.name][:8], + new_hashes[c.name][:8], + ) + hook_ok, hook_err = await self._ping_while( + run_hook( + c.name, + c.path, + new_hashes[c.name], + prev[c.name], + timeout=HOOK_TIMEOUT, + ), + c.name, + 3, + 4, + ) + if hook_ok: + survivors.append(c) + continue + self._log.error("%s: hook failed: %s", c.name, hook_err) + failed = True + if not await self._drop_component(c, prev[c.name], "hook"): + pending_revert[c.name] = prev[c.name] + touched.remove(c) + return survivors, new_hashes, failed + + async def _batch_restart_services( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + restarted: list[str], + pending_revert: dict[str, str], + seen: set[str], + ) -> tuple[bool, list[ComponentConfig]]: + """Restart each component's own service once; revert its members on failure.""" + failed = False + ui_components: list[ComponentConfig] = [] + for c in alive.copy(): + if c not in alive or not c.service or c.service in seen: + continue + seen.add(c.service) + if c.service in _FIRE_AND_FORGET_SERVICES: + self._log.info( + "git batch: deferring self/UI service %s to fire-and-forget", + c.service, + ) + ui_components.append(c) + continue + self._cb("on_step", c.name, 4, 4) + restarted.append(c.service) + self._log.info("git batch: restarting %s (verified)", c.service) + if await self._restart_one(c.service, c.health_url): + continue + failed = True + for m in [m for m in alive if m.service == c.service]: + if not await self._drop_component(m, prev[m.name], "restart"): + pending_revert[m.name] = prev[m.name] + alive.remove(m) + touched.remove(m) + # Members reverted: bring the service back up on the old code. + if not await self._restart_one(c.service): + self._log.error("%s did not recover after revert", c.service) + restarted.remove(c.service) + return failed, ui_components + + async def _batch_restart_klipper_bounce( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + restarted: list[str], + pending_revert: dict[str, str], + seen: set[str], + ) -> bool: + """Bounce klipper once for restart_klipper components, reverting them on failure.""" + # Bounce klipper once for restart_klipper components that aren't klipper. + klipper_svc = self._klipper_service() + requesters = [ + c for c in alive if c.restart_klipper and c.service != klipper_svc + ] + if not requesters or klipper_svc in seen: + return False + seen.add(klipper_svc) + restarted.append(klipper_svc) + self._log.info("git batch: restarting %s (restart_klipper)", klipper_svc) + if await self._restart_one(klipper_svc): + return False + failed = True + for m in requesters: + if not await self._drop_component(m, prev[m.name], "restart"): + pending_revert[m.name] = prev[m.name] + alive.remove(m) + touched.remove(m) + # Service runs new code: revert it to old code unless a surviving component shares the service. + shared = any(o.service == m.service for o in alive) + if m.service and m.service in restarted and not shared: + if not await self._restart_one(m.service): + self._log.error("%s did not recover after revert", m.service) + restarted.remove(m.service) + restarted.remove(klipper_svc) + # Requesters reverted: try to bring klipper back up. + if not await self._restart_one(klipper_svc): + self._log.error("%s did not recover after revert", klipper_svc) + return failed + + async def _batch_restart_phase( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + restarted: list[str], + pending_revert: dict[str, str], + ) -> tuple[bool, list[ComponentConfig]]: + """Restart each unique service once, reverting all components behind a failed one.""" + # Restart each unique service once; failure reverts all components behind it and re-restarts onto old code. + self._log.info("git batch: restart phase") + seen: set[str] = set() + svc_failed, ui_components = await self._batch_restart_services( + alive, prev, touched, restarted, pending_revert, seen + ) + klipper_failed = await self._batch_restart_klipper_bounce( + alive, prev, touched, restarted, pending_revert, seen + ) + return svc_failed or klipper_failed, ui_components + + async def _finalize_git_batch( + self, + alive: list[ComponentConfig], + batch: list[ComponentConfig], + pending_revert: dict[str, str], + ui_components: list[ComponentConfig], + ) -> None: + """Settle the inflight marker and fire-and-forget the UI restarts for survivors.""" + # Shrink the marker before any restart: survivors stand from here on. + await self._settle_inflight(pending_revert) self._log.info( - "git batch start: %d component(s): %s", + "git batch complete: %d/%d component(s) updated; queueing UI restart(s)", + len(alive), len(batch), - ", ".join(f"{c.name}->{c.service or '-'}" for c in batch), + ) + ui_services: set[str] = set() + for c in ui_components: + if c not in alive: + continue + self._cb("on_step", c.name, 4, 4) + if c.service: + ui_services.add(c.service) + # klipper/RF50 hold config the UI reads at startup: refresh it too. + if any(c.restart_ui for c in alive): + ui_services.add(_UI_SERVICE) + for svc in ui_services: + self._log.info("git batch: fire-and-forget restart of %s (no wait)", svc) + await restart_service_noblock(svc) + await self._apply_deferred_restart() + + async def _git_batch_preflight( + self, batch: list[ComponentConfig] + ) -> tuple[list[ComponentConfig], dict[str, str], bool] | None: + """Snapshot hashes, persist the rollback point, SEC-check remotes. None = abort.""" + alive, prev, failed = await self._prepare_git_batch_hashes(batch) + if not alive: + return None + if not await self._persist_batch_rollback(prev): + for c in alive: + self._cb_error_done(c.name, "failed to persist rollback point") + return None + alive, sec_failed = await self._security_check_batch(alive, prev) + failed = failed or sec_failed + if not alive: + # Nothing staged: drop the marker (avoids a spurious boot revert). + await asyncio.to_thread(self._clear_inflight) + return None + return alive, prev, failed + + async def _run_batch_phases( + self, + alive: list[ComponentConfig], + prev: dict[str, str], + touched: list[ComponentConfig], + restarted: list[str], + pending_revert: dict[str, str], + failed: bool, + ) -> tuple[list[ComponentConfig], dict[str, str], list[ComponentConfig], bool]: + """Run stage->deps->hook->restart; return (survivors, new_hashes, ui_components, failed).""" + alive, stage_failed = await self._batch_stage_phase( + alive, prev, touched, pending_revert + ) + failed = failed or stage_failed + alive, deps_failed = await self._batch_deps_phase( + alive, prev, touched, pending_revert + ) + failed = failed or deps_failed + alive, new_hashes, hook_failed = await self._batch_hook_phase( + alive, prev, touched, pending_revert + ) + failed = failed or hook_failed + restart_failed, ui_components = await self._batch_restart_phase( + alive, prev, touched, restarted, pending_revert + ) + failed = failed or restart_failed + return alive, new_hashes, ui_components, failed + + async def _run_git_batch(self, batch: list[ComponentConfig]) -> bool: + """Apply existing git components with per-component failure isolation.""" + pre = await self._git_batch_preflight(batch) + if pre is None: + return False + alive, prev, failed = pre + + self._log.info( + "git batch start: %d component(s): %s", + len(alive), + ", ".join(f"{c.name}->{c.service or '-'}" for c in alive), ) touched: list[ComponentConfig] = [] restarted: list[str] = [] + # Repos whose revert failed: kept in in-flight marker so boot reconcile retries them, not committed survivors. + pending_revert: dict[str, str] = {} committed = False try: - self._log.info("git batch: stage phase (%d repo(s))", len(batch)) - for c in batch: - self._cb("on_step", c.name, 1, 4) - touched.append(c) # mark before staging so a partial stage is reverted - ok, reason = await self._stage_component(c) - if not ok: - await self._abort_batch(batch, prev, touched, restarted, reason) - return - self._log.info("git batch: deps phase") - for c in batch: - self._cb("on_step", c.name, 2, 4) - deps_ok, deps_err = await self._install_dependencies(c) - if not deps_ok: - self._log.warning("%s: deps failed: %s", c.name, deps_err) - await self._abort_batch(batch, prev, touched, restarted, "deps") - return - self._log.info("git batch: hook phase") - new_hashes: dict[str, str] = {} - for c in batch: - self._cb("on_step", c.name, 3, 4) - new_hashes[c.name] = await git_get_hash(c.path) - self._log.info( - "git batch hook: %s %s -> %s", - c.name, - prev[c.name][:8], - new_hashes[c.name][:8], - ) - hook_ok, hook_err = await self._ping_while( - run_hook( - c.name, - c.path, - new_hashes[c.name], - prev[c.name], - timeout=HOOK_TIMEOUT, - ), - c.name, - 3, - 4, - ) - if not hook_ok: - self._log.error("%s: hook failed: %s", c.name, hook_err) - await self._abort_batch(batch, prev, touched, restarted, "hook") - return - # Restart each unique svc once; bounced marked BEFORE so abort re-restarts. - self._log.info("git batch: restart phase") - seen: set[str] = set() - ui_components: list[ComponentConfig] = [] - for c in batch: - if not c.service or c.service in seen: - continue - seen.add(c.service) - if c.service in _FIRE_AND_FORGET_SERVICES: - self._log.info( - "git batch: deferring self/UI service %s to fire-and-forget", - c.service, - ) - ui_components.append(c) - continue - self._cb("on_step", c.name, 4, 4) - restarted.append(c.service) - self._log.info("git batch: restarting %s (verified)", c.service) - if not await self._restart_one(c.service): - await self._abort_batch(batch, prev, touched, restarted, "restart") - return - # Bounce klipper once for restart_klipper components that aren't klipper. - klipper_svc = self._klipper_service() - if any(c.restart_klipper for c in batch) and klipper_svc not in seen: - seen.add(klipper_svc) - restarted.append(klipper_svc) - self._log.info( - "git batch: restarting %s (restart_klipper)", klipper_svc - ) - if not await self._restart_one(klipper_svc): - await self._abort_batch(batch, prev, touched, restarted, "restart") - return + alive, new_hashes, ui_components, failed = await self._run_batch_phases( + alive, prev, touched, restarted, pending_revert, failed + ) + if not alive: + await self._settle_inflight(pending_revert) + return False # Persist success BEFORE UI restart: a committed update is never reverted. - for c in batch: + for c in alive: self._history( "update_success", c.name, new_hash=new_hashes[c.name][:12] ) self._cb("on_component_done", c.name, True) # Durable now; the self-restart's CancelledError must never revert. committed = True - # Clear marker before any restart: the update stands from here on. - await asyncio.to_thread(self._clear_inflight) - self._log.info( - "git batch complete: %d component(s) updated; queueing UI restart(s)", - len(batch), - ) - ui_services: set[str] = set() - for c in ui_components: - self._cb("on_step", c.name, 4, 4) - if c.service: - ui_services.add(c.service) - # klipper/RF50 hold config the UI reads at startup: refresh it too. - if any(c.restart_ui for c in batch): - ui_services.add(_UI_SERVICE) - for svc in ui_services: - self._log.info( - "git batch: fire-and-forget restart of %s (no wait)", svc - ) - await restart_service_noblock(svc) - await self._apply_deferred_restart() + await self._finalize_git_batch(alive, batch, pending_revert, ui_components) + return not failed except asyncio.CancelledError: if committed: raise self._log.warning("git batch cancelled - aborting") await self._shielded( - self._abort_batch(batch, prev, touched, restarted, "cancelled"), + self._abort_batch( + alive, prev, touched, restarted, "cancelled", pending_revert + ), "git batch cancel-abort", ) raise except Exception: # noqa: BLE001 if committed: self._log.error("post-commit error (update stands)", exc_info=True) - return + return True self._log.error("git batch unexpected error", exc_info=True) - await self._abort_batch(batch, prev, touched, restarted, "unexpected_error") + await self._abort_batch( + alive, prev, touched, restarted, "unexpected_error", pending_revert + ) + return False async def _abort_batch( self, @@ -555,21 +861,23 @@ async def _abort_batch( touched: list[ComponentConfig], restarted: list[str], reason: str, + pending_revert: dict[str, str] | None = None, ) -> None: """Revert touched repos, re-restart bounced services, report all failed.""" self._log.warning( "aborting batch (reason=%s): reverting %d repo(s)", reason, len(touched) ) - revert_ok = True + pending = dict(pending_revert or {}) for c in touched: ok, _ = await git_reset_to_hash(c.path, prev[c.name]) if not ok: - revert_ok = False + pending[c.name] = prev[c.name] self._log.error( "abort: %s reset to %s failed", c.name, prev[c.name][:12] ) - # Repos are back at prev_hash: the batch is resolved, drop the marker. - await asyncio.to_thread(self._clear_inflight) + # Only unresolved repos stay in the marker for a boot-time retry. + await self._settle_inflight(pending) + revert_ok = not pending restart_ok = True for service in restarted: if not await self._restart_one(service): @@ -588,19 +896,36 @@ async def _abort_batch( self._cb("on_rollback", c.name, rollback_ok) self._cb("on_component_done", c.name, False) + async def _drop_component( + self, component: ComponentConfig, prev_hash: str, reason: str + ) -> bool: + """Revert one failed component and report it; the rest of the batch continues.""" + self._log.warning( + "%s: dropped from batch (reason=%s), reverting to %s", + component.name, + reason, + prev_hash[:8], + ) + ok, _ = await git_reset_to_hash(component.path, prev_hash) + if not ok: + self._log.error("%s: revert to %s failed", component.name, prev_hash[:12]) + self._history( + "rollback", component.name, reason=reason, reverted_to=prev_hash[:12], ok=ok + ) + self._cb("on_error", component.name, reason) + self._cb("on_rollback", component.name, ok) + self._cb("on_component_done", component.name, False) + return ok + + async def _settle_inflight(self, pending_revert: dict[str, str]) -> None: + """Shrink the in-flight marker to repos whose revert still needs a boot retry.""" + if pending_revert: + await asyncio.to_thread(self._write_inflight, pending_revert.copy()) + else: + await asyncio.to_thread(self._clear_inflight) + async def _apply_deferred_restart(self) -> None: - """Apply a daemon restart/reinstall a hook deferred during this batch. - - Hooks running under the updater record `install`/`code` to the tmpfs - sentinel instead of restarting the daemon mid-batch. Called once, after - success is recorded and the UI restart is queued: - - `install`: touch the deploy flag so BlocksScreen-deploy.path runs - install-updater out-of-band (its own cgroup, post-batch). - - `code`: self-test the new on-disk updater; only if it imports - cleanly, `--no-block` restart the daemon onto it. Otherwise keep the - running daemon and let the next reboot adopt it. Never raises: a - completed update must stand regardless of what happens here. - """ + """Apply a daemon restart/reinstall hook deferred during this batch.""" try: sentinel = restart_sentinel_path() reason = await asyncio.to_thread(self._read_clear_sentinel, sentinel) @@ -664,13 +989,15 @@ async def recover(self, name: str, hard: bool = False) -> bool: self._log.error("component %s not found", name) self._cb("on_recover", name, False) return False - state = self._read_state() - prev_hash = state.get(name, {}).get("prev_hash") + comp_state = self._read_state().get(name) + prev_hash = ( + comp_state.get("prev_hash") if isinstance(comp_state, dict) else None + ) if prev_hash is None: self._cb("on_recover", name, False) self._log.error("failed to get prev_hash for %s", name) return False - if not _GIT_SHA_RE.match(prev_hash): + if not _is_sha(prev_hash): self._log.error("recover: invalid prev_hash format for %s", name) self._cb("on_recover", name, False) return False @@ -695,11 +1022,7 @@ async def recover(self, name: str, hard: bool = False) -> bool: return ok async def bless_healthy(self, name: str, hash_val: str) -> bool: - """Mark a component as healthy (known-good) after a debounce; snapshots baseline. - - An empty hash_val means "bless the component's current HEAD": the UI does - not need to compute a hash, it just says it is healthy. - """ + """Mark a component healthy (known-good) after a debounce; snapshots baseline.""" component = next((c for c in self._components if c.name == name), None) if component is None: self._log.error("bless_healthy: component %s not found", name) @@ -712,6 +1035,7 @@ async def bless_healthy(self, name: str, hash_val: str) -> bool: nrestarts_baseline = await get_service_nrestarts(_UI_SERVICE) def _apply(state: dict) -> None: + """Set last_good, seed golden once, reset the attempt counter and baseline.""" comp = _ensure_comp(state, name) comp["last_good"] = hash_val if not _is_sha(comp.get("golden")): @@ -733,11 +1057,7 @@ def _apply(state: dict) -> None: return ok def _check_crash_loop(self, name: str, nrestarts: int) -> bool: - """Return True if NRestarts rose by >= 5 within the trailing 180s window. - - Compares against the oldest in-window sample (largest rise, since NRestarts - is monotonic), then records the current sample and prunes aged-out ones. - """ + """Return True if NRestarts rose by >= 5 within the trailing 180s window.""" if name not in self._nrestarts_samples: self._nrestarts_samples[name] = [] # fresh device: start tracking now samples = self._nrestarts_samples[name] @@ -790,68 +1110,88 @@ async def run_recovery_rung(self, name: str, attempt: int) -> bool: if not isinstance(comp_state, dict): comp_state = {} if attempt == 1: - last_good = comp_state.get("last_good") - if _is_sha(last_good): - self._log.info("recovery rung 1: reset to last_good %s", last_good[:8]) - if component.path: - async with self._git_lock: - ok, _ = await git_reset_to_hash(component.path, last_good) - if not ok: - self._log.error("reset to last_good failed") - return False - ok = await self._restart_ui_service() - if ok: - self._history( - "recovery_rung1", name, ok=True, reverted_to=last_good[:12] - ) - return ok - self._log.warning("recovery rung 1: no valid last_good, skipping") + return await self._recovery_rung1(component, name, comp_state) + if attempt == 2: + return await self._recovery_rung2(component, name) + return await self._recovery_rung3(component, name, comp_state) + + async def _recovery_rung1( + self, component: ComponentConfig, name: str, comp_state: dict + ) -> bool: + """Recovery rung 1: reset to the stored last_good sha and restart the UI.""" + last_good = comp_state.get("last_good") + if _is_sha(last_good): + self._log.info("recovery rung 1: reset to last_good %s", last_good[:8]) + if component.path: + async with self._git_lock: + ok, _ = await git_reset_to_hash(component.path, last_good) + if not ok: + self._log.error("reset to last_good failed") + return False + ok = await self._restart_ui_service() + if ok: + self._history( + "recovery_rung1", name, ok=True, reverted_to=last_good[:12] + ) + return ok + self._log.warning("recovery rung 1: no valid last_good, skipping") + return False + + async def _recovery_rung2(self, component: ComponentConfig, name: str) -> bool: + """Recovery rung 2: fetch + reset to the heal remote ref (guarding brick tips).""" + self._log.info("recovery rung 2: fetch %s and reset", _HEAL_REMOTE_REF) + if not component.path: return False - elif attempt == 2: - self._log.info("recovery rung 2: fetch %s and reset", _HEAL_REMOTE_REF) - if not component.path: + async with self._git_lock: + ok_fetch, _ = await git_fetch(component.path) + if not ok_fetch: + self._log.warning("recovery rung 2: fetch failed (offline?)") return False - async with self._git_lock: - ok_fetch, _ = await git_fetch(component.path) - if not ok_fetch: - self._log.warning("recovery rung 2: fetch failed (offline?)") - return False - tip = await git_ref_hash(component.path, _HEAL_REMOTE_REF) - if not tip: - self._log.warning( - "recovery rung 2: %s unresolved", _HEAL_REMOTE_REF - ) - return False - ok_reset, _ = await git_reset_to_hash(component.path, _HEAL_REMOTE_REF) - if not ok_reset: - self._log.error("reset to %s failed", _HEAL_REMOTE_REF) + tip = await git_ref_hash(component.path, _HEAL_REMOTE_REF) + if not tip: + self._log.warning("recovery rung 2: %s unresolved", _HEAL_REMOTE_REF) return False - # Persist the tip before restart so an unbootable tip is never retried. - await self._mutate_state( - lambda s: _ensure_comp(s, name).update(last_failed_remote=tip) - ) + # Never heal the host onto a pre-updater tip (would re-brick, not fix). + if name == _UI_COMPONENT and not await git_tree_has_path( + component.path, _HEAL_REMOTE_REF, _UPDATER_MARKER + ): + self._log.error( + "recovery rung 2: %s lacks the updater package - skipping", + _HEAL_REMOTE_REF, + ) + return False + ok_reset, _ = await git_reset_to_hash(component.path, _HEAL_REMOTE_REF) + if not ok_reset: + self._log.error("reset to %s failed", _HEAL_REMOTE_REF) + return False + # Persist the tip before restart so an unbootable tip is never retried. + await self._mutate_state( + lambda s: _ensure_comp(s, name).update(last_failed_remote=tip) + ) + ok = await self._restart_ui_service() + if ok: + self._history("recovery_rung2", name, ok=True, reverted_to=tip[:12]) + return ok + + async def _recovery_rung3( + self, component: ComponentConfig, name: str, comp_state: dict + ) -> bool: + """Recovery rung 3: reset to the stored golden sha and restart the UI.""" + golden = comp_state.get("golden") + if _is_sha(golden): + self._log.info("recovery rung 3: reset to golden %s", golden[:8]) + if component.path: + async with self._git_lock: + ok, _ = await git_reset_to_hash(component.path, golden) + if not ok: + self._log.error("reset to golden failed") + return False ok = await self._restart_ui_service() if ok: - self._history("recovery_rung2", name, ok=True, reverted_to=tip[:12]) + self._history("recovery_rung3", name, ok=True, reverted_to=golden[:12]) return ok - else: - golden = comp_state.get("golden") - if _is_sha(golden): - self._log.info("recovery rung 3: reset to golden %s", golden[:8]) - if component.path: - async with self._git_lock: - ok, _ = await git_reset_to_hash(component.path, golden) - if not ok: - self._log.error("reset to golden failed") - return False - ok = await self._restart_ui_service() - if ok: - self._history( - "recovery_rung3", name, ok=True, reverted_to=golden[:12] - ) - return ok - self._log.warning("recovery rung 3: no valid golden, skipping") - return False + self._log.warning("recovery rung 3: no valid golden, skipping") + return False async def _restart_ui_service(self) -> bool: """Restart the BlocksScreen UI service; return True if successful.""" @@ -867,21 +1207,16 @@ def _rebaseline_samples(self, name: str, nrestarts: int) -> None: self._nrestarts_samples[name] = [(time.monotonic(), nrestarts)] async def supervise_ui(self) -> None: - """Poll the UI service NRestarts and drive the recovery ladder on a crash-loop. - - The single-attempt-in-flight guard is the loop itself: at most one rung runs - per crash-loop, then detection is re-baselined and a settle window elapses so - a new build gets a fair chance to bless before the next rung is counted. - """ + """Poll the UI service NRestarts and drive the recovery ladder on a crash-loop.""" while True: await asyncio.sleep(_NRESTARTS_POLL_INTERVAL_S) try: nrestarts = await get_service_nrestarts(_UI_SERVICE) - except Exception as exc: # noqa: BLE001 - self._log.debug("supervise_ui: NRestarts read failed: %s", exc) - continue - if self._check_crash_loop(_UI_COMPONENT, nrestarts): - await self._handle_crash_loop(nrestarts) + if self._check_crash_loop(_UI_COMPONENT, nrestarts): + await self._handle_crash_loop(nrestarts) + except Exception: # noqa: BLE001 + # One bad pass must not kill crash-loop supervision for good. + self._log.error("supervise_ui pass failed", exc_info=True) async def _handle_crash_loop(self, nrestarts: int) -> None: """Run one recovery rung for the UI, or saturate if the ladder is exhausted.""" @@ -904,11 +1239,7 @@ async def _handle_crash_loop(self, nrestarts: int) -> None: await asyncio.sleep(_RECOVERY_SETTLE_S) async def forward_heal_ui(self) -> None: - """Slowly re-check origin/main and re-attempt an upgrade once a fix ships. - - Jittered and connectivity-gated: desynchronized polls avoid a fleet stampede - and a failed fetch (offline) is skipped without churn. - """ + """Slowly re-check origin/main and re-attempt an upgrade once a fix ships.""" while True: delay = _FORWARD_HEAL_BASE_S + secrets.randbelow( int(_FORWARD_HEAL_JITTER_S) + 1 @@ -919,6 +1250,31 @@ async def forward_heal_ui(self) -> None: except Exception as exc: # noqa: BLE001 self._log.debug("forward_heal_ui: %s", exc) + async def _forward_heal_target( + self, comp_state: dict + ) -> tuple[ComponentConfig, str] | None: + """Resolve the new heal tip to upgrade onto, or None if none is safe/available.""" + component = next((c for c in self._components if c.name == _UI_COMPONENT), None) + if component is None or not component.path: + return None + async with self._git_lock: + ok_fetch, _ = await git_fetch(component.path) + if not ok_fetch: + return None # offline: connectivity gate + tip = await git_ref_hash(component.path, _HEAL_REMOTE_REF) + if not tip or tip == comp_state.get("last_failed_remote"): + return None # no new stable tip since the last failure + # Never forward-heal onto a pre-updater tip (would brick, not upgrade). + if not await git_tree_has_path( + component.path, _HEAL_REMOTE_REF, _UPDATER_MARKER + ): + self._log.warning( + "forward-heal: %s lacks the updater package - skipping", + _HEAL_REMOTE_REF, + ) + return None + return component, tip + async def _forward_heal_once(self) -> bool: """One forward-heal pass: attempt the new origin/main tip if we are in fallback.""" state = await asyncio.to_thread(self._read_state) @@ -928,16 +1284,10 @@ async def _forward_heal_once(self) -> bool: raw = comp_state.get("fast_attempt", 0) if not (isinstance(raw, int) and not isinstance(raw, bool)) or raw < 2: return False # healthy, not yet in deep fallback, or corrupt counter - component = next((c for c in self._components if c.name == _UI_COMPONENT), None) - if component is None or not component.path: + target = await self._forward_heal_target(comp_state) + if target is None: return False - async with self._git_lock: - ok_fetch, _ = await git_fetch(component.path) - if not ok_fetch: - return False # offline: connectivity gate - tip = await git_ref_hash(component.path, _HEAL_REMOTE_REF) - if not tip or tip == comp_state.get("last_failed_remote"): - return False # no new stable tip since the last failure + component, tip = target self._log.info( "forward-heal: new %s tip %s, upgrading", _HEAL_REMOTE_REF, tip[:8] ) @@ -947,8 +1297,10 @@ async def _forward_heal_once(self) -> bool: return False def _apply(s: dict) -> None: + """Reset the attempt counter and record the healed tip.""" comp = _ensure_comp(s, _UI_COMPONENT) comp["fast_attempt"] = 0 + # Reused as "last remote tip attempted": blocks re-healing to the same tip. comp["last_failed_remote"] = tip await self._mutate_state(_apply) @@ -978,19 +1330,7 @@ def _clear_fault_marker(self) -> None: self._log.warning("failed to clear self-heal fault marker") async def reconcile(self) -> None: - """Heal repos left damaged by a power loss mid-update, for every component. - - start.sh only repairs the BlocksScreen repo; this covers klipper/moonraker/ - config repos too. The gate is the cheap `rev-parse HEAD` (mirrors start.sh's - `cat-file -e HEAD`) so a healthy boot pays almost nothing; deeper corruption - is caught on demand by the normal update flow. Offline-safe: git_repair - fetches when it can, else we reset to the last recorded good hash. - - Held under the shared process lock so the boot-time heal cannot interleave - its git resets/repairs with a near-simultaneous user update or CLI run on - the same repo. If another updater holds the lock, skip: that run is already - mutating the repos, and any real damage is re-detected on demand. - """ + """Heal repos left damaged by a power loss mid-update, for every component.""" with process_lock() as acquired: if not acquired: self._log.info( @@ -1000,13 +1340,7 @@ async def reconcile(self) -> None: await self._reconcile_locked() async def _revert_inflight(self) -> None: - """Revert any update cut off mid-flight by a power loss before it committed. - - The in-flight marker maps each batch component to the commit it was on - before staging. Its mere presence at boot means a batch did not reach a - terminal outcome, so every listed repo is hard-reset back to that commit - and the marker is cleared. A repo already on its prev_hash is left alone. - """ + """Revert any update cut off mid-flight by a power loss before it committed.""" inflight = await asyncio.to_thread(self._read_inflight) if not inflight: return @@ -1015,8 +1349,10 @@ async def _revert_inflight(self) -> None: len(inflight), ) by_name = {c.name: c for c in self._components} + unresolved: dict[str, str] = {} # reverts that failed: kept for next-boot retry for name, prev_hash in inflight.items(): comp = by_name.get(name) + # Gone component/path or invalid hash can never revert: drop (bounds retries). if comp is None or comp.path is None or not comp.path.exists(): continue if not _GIT_SHA_RE.match(prev_hash): @@ -1033,39 +1369,72 @@ async def _revert_inflight(self) -> None: prev_hash[:8], rok, ) - await asyncio.to_thread(self._clear_inflight) + if not rok: + unresolved[name] = prev_hash # keep marker so next boot retries + # Clear on full success; else persist only the still-failing entries (retry). + if unresolved: + await asyncio.to_thread(self._write_inflight, unresolved) + else: + await asyncio.to_thread(self._clear_inflight) + + async def _boot_repair_component(self, c: ComponentConfig) -> bool: + """Repair one git component with an unreadable HEAD at boot; True if recloned.""" + if c.path is None: + return False + async with self._git_lock: + if await git_get_hash(c.path) != "": + return False + self._log.warning("reconcile: %s HEAD unreadable - repairing", c.name) + # No configured branch: git_repair derives the repo's own default. + ok, msg = await git_repair(c.path, c.branch) + if ok and await git_get_hash(c.path) != "": + self._history("boot_repair", c.name, detail=msg[:80]) + return False + if await self._reclone_component(c) and await git_get_hash(c.path): + self._history("boot_reclone", c.name) + return True + comp_state = (await asyncio.to_thread(self._read_state)).get(c.name, {}) + prev = comp_state.get("prev_hash") if isinstance(comp_state, dict) else None + if prev and _GIT_SHA_RE.match(prev): + rok, _ = await git_reset_to_hash(c.path, prev) + self._history("boot_reset", c.name, ok=rok, reverted_to=prev[:12]) + self._log.warning( + "reconcile: %s reset to %s (ok=%s)", c.name, prev[:8], rok + ) + else: + self._log.error( + "reconcile: %s unrepairable, no recorded prev_hash", c.name + ) + return False + + async def _boot_reclone_hook(self, c: ComponentConfig) -> None: + """Best-effort post-reclone hook to rebuild in-repo artifacts (e.g. Spoolman .venv).""" + if c.path is None: + return + try: + new_hash = await git_get_hash(c.path) + hook_ok, hook_err = await run_hook( + c.name, c.path, new_hash, _GIT_EMPTY_TREE, timeout=HOOK_TIMEOUT + ) + if not hook_ok: + self._log.warning("%s: post-reclone hook failed: %s", c.name, hook_err) + except Exception: # noqa: BLE001 + # Best-effort: a hook crash must not kill the rest of boot heal. + self._log.warning("%s: post-reclone hook raised", c.name, exc_info=True) async def _reconcile_locked(self) -> None: + """Boot heal under the lock: revert in-flight, repair repos, run reclone hooks, sanitize state.""" await self._revert_inflight() self._log.info("reconcile: boot-checking git components for damage") + recloned: list[ComponentConfig] = [] for c in self._components: if c.kind != "git" or c.path is None or not c.path.exists(): continue - async with self._git_lock: - if await git_get_hash(c.path) != "": - continue - self._log.warning("reconcile: %s HEAD unreadable - repairing", c.name) - _branch = c.branch or "main" - ok, msg = await git_repair(c.path, _branch) - if ok and await git_get_hash(c.path) != "": - self._history("boot_repair", c.name, detail=msg[:80]) - continue - comp_state = (await asyncio.to_thread(self._read_state)).get(c.name, {}) - prev = ( - comp_state.get("prev_hash") - if isinstance(comp_state, dict) - else None - ) - if prev and _GIT_SHA_RE.match(prev): - rok, _ = await git_reset_to_hash(c.path, prev) - self._history("boot_reset", c.name, ok=rok, reverted_to=prev[:12]) - self._log.warning( - "reconcile: %s reset to %s (ok=%s)", c.name, prev[:8], rok - ) - else: - self._log.error( - "reconcile: %s unrepairable, no recorded prev_hash", c.name - ) + if await self._boot_repair_component(c): + recloned.append(c) + # Reclone drops in-repo artifacts; rebuild them best-effort outside _git_lock. + for c in recloned: + await self._boot_reclone_hook(c) await self._reconcile_self_heal_state() async def _reconcile_self_heal_state(self) -> None: @@ -1115,12 +1484,19 @@ async def _reconcile_self_heal_state(self) -> None: async def _rollback( self, component: ComponentConfig, prev_hash: str, reason: str ) -> None: + """Reset one component to prev_hash, restart its service, and report failure.""" self._log.warning( "%s: rolling back to %s (reason=%s)", component.name, prev_hash[:8], reason ) ok, _ = await git_reset_to_hash(component.path, prev_hash) - await asyncio.to_thread(self._clear_inflight) - if component.service and not await self._restart_one(component.service): + if ok: + await asyncio.to_thread(self._clear_inflight) + else: + # A failed revert keeps the marker so boot _revert_inflight retries it. + self._log.warning("rollback: revert failed - keeping in-flight marker") + if component.service and not await self._restart_one( + component.service, component.health_url + ): ok = False self._history( "rollback", component.name, reason=reason, reverted_to=prev_hash[:12], ok=ok @@ -1158,19 +1534,15 @@ async def _install_dependencies( # Keep pip current (best-effort: a failed upgrade must not block reqs). await _run([pip_path, "install", "--upgrade", "pip", "--quiet"], timeout=120.0) + # Generous: one aarch64 source build (no wheel) easily exceeds 120s on a Pi. return await _run( - [pip_path, "install", "-r", str(req), "--quiet"], timeout=120.0 + [pip_path, "install", "-r", str(req), "--quiet"], timeout=600.0 ) async def _ping_while( self, coro, name: str, step: int, total: int, interval: float = 60.0 ): - """Await coro, re-emitting on_step every `interval` seconds. - - A HOOK_TIMEOUT-bounded hook can run silent longer than the client's - 360s busy watchdog; the pings keep it fed. Cancellation propagates to - the inner task so its subprocess is still killed. - """ + """Await coro, re-emitting on_step every `interval` seconds.""" task = asyncio.ensure_future(coro) try: while True: @@ -1202,22 +1574,18 @@ async def _shielded(self, coro, label: str) -> None: return def _cb_error_done(self, name: str, reason: str) -> bool: + """Emit on_error and on_component_done(False); return False.""" self._cb("on_error", name, reason) self._cb("on_component_done", name, False) return False def _history(self, event: str, name: str, **fields: object) -> None: - """Append one event to the persistent update-history log (OBS-1). - - journald is volatile on this image, so this on-SD JSONL log is the - durable record of update outcomes for field diagnosis. - """ + """Append one event to the persistent update-history log (OBS-1).""" entry = { "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "event": event, "component": name, - **fields, - } + } | fields try: self._history_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) with self._history_path.open("a", encoding="utf-8") as f: @@ -1245,6 +1613,8 @@ def _quarantine_sync(self, path: Path) -> Path | None: except OSError as exc: self._log.error("quarantine of %s failed: %s", path, exc) return None + # The venv moved with the dir: a cached pip path would now dangle. + self._component_pip_cache.pop(str(path), None) return dest async def _quarantine_nonrepo(self, component: ComponentConfig) -> bool: @@ -1268,11 +1638,7 @@ async def _remove_clone(self, component: ComponentConfig) -> None: await asyncio.to_thread(shutil.rmtree, component.path, ignore_errors=True) async def _fail_provision(self, component: ComponentConfig, reason: str) -> bool: - """Remove the partial clone, log, and report failure. - - No prev_hash to roll back to, so the clean slate is deleting the clone. - The unit is enabled only after a clean start, so nothing is left enabled. - """ + """Remove the partial clone, log, and report failure.""" await self._remove_clone(component) self._history("install_failed", component.name, reason=reason) self._log.warning( @@ -1280,14 +1646,26 @@ async def _fail_provision(self, component: ComponentConfig, reason: str) -> bool ) return self._cb_error_done(component.name, reason) - async def _provision_component(self, component: ComponentConfig) -> bool: - """Clone and set up a missing opted-in component. + async def _provision_restart_service( + self, component: ComponentConfig + ) -> str | None: + """Restart+health-check+enable the provisioned service; fail reason or None.""" + if not component.service: + return None + if not await self._restart_one(component.service, component.health_url): + return "restart" + # Enable only after a clean start (no boot-looping failed unit). + en_ok, en_err = await enable_service(component.service) + if not en_ok: + self._log.warning( + "%s: enable failed (runs now, may not persist): %s", + component.name, + en_err, + ) + return None - A fresh component has no prev_hash to roll back to, so any failure deletes - the partial clone (clean slate to retry next Update) rather than rolling - back. Service install is delegated to the component's hook, mirroring the - existing update flow. - """ + async def _provision_component(self, component: ComponentConfig) -> bool: + """Clone and set up a missing opted-in component.""" if not component.url: return self._cb_error_done(component.name, "no clone url") self._log.info("%s: provisioning via clone %s", component.name, component.url) @@ -1330,26 +1708,9 @@ async def _provision_component(self, component: ComponentConfig) -> bool: return await self._fail_provision(component, "hook") self._cb("on_step", component.name, 4, 4) - if component.service: - svc_ok, svc_err = await restart_service(component.service) - if not svc_ok: - self._log.error( - "%s: provision restart: %s", component.name, svc_err - ) - return await self._fail_provision(component, "restart") - if not await wait_for_service_active(component.service, timeout=90.0): - self._log.error( - "%s: provisioned service not active", component.name - ) - return await self._fail_provision(component, "restart_timeout") - # Enable only after a clean start (no boot-looping failed unit). - en_ok, en_err = await enable_service(component.service) - if not en_ok: - self._log.warning( - "%s: enable failed (runs now, may not persist): %s", - component.name, - en_err, - ) + reason = await self._provision_restart_service(component) + if reason: + return await self._fail_provision(component, reason) self._history("install_success", component.name, new_hash=new_hash[:12]) self._cb("on_component_done", component.name, True) @@ -1368,13 +1729,75 @@ async def _provision_component(self, component: ComponentConfig) -> bool: ) return await self._fail_provision(component, "unexpected_error") - async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str]: - """Bring the working tree to its target ref: fetch-gate + reset/checkout/pull. + async def _reclone_into_tmp(self, component: ComponentConfig, tmp: Path) -> bool: + """Clone (+ optional version pin) into a temp dir; rmtree + False on failure.""" + if not component.url: + return False + self._log.warning("%s: recloning from %s", component.name, component.url) + ok, err = await git_clone(component.url, tmp, component.branch) + if not ok: + await asyncio.to_thread(shutil.rmtree, tmp, ignore_errors=True) + self._log.error("%s: reclone clone failed: %s", component.name, err) + return False + if component.version: + vok, verr = await git_reset_to_hash(tmp, component.version) + if not vok: + await asyncio.to_thread(shutil.rmtree, tmp, ignore_errors=True) + self._log.error("%s: reclone pin failed: %s", component.name, verr) + return False + return True - No deps/hook/restart and no rollback - the caller decides what to do on - failure. Returns (ok, reason) where reason is one of - network/corrupt/reset/version/branch/conflict. - """ + async def _reclone_swap( + self, component: ComponentConfig, path: Path, tmp: Path, old: Path + ) -> bool: + """Atomically swap the fresh tmp clone into place; restore + False on failure.""" + try: + # move old aside first: renaming onto a nonempty dir is ENOTEMPTY + if path.exists(): + await asyncio.to_thread(os.rename, path, old) + await asyncio.to_thread(os.rename, tmp, path) + except OSError as exc: + if not path.exists() and old.exists(): # restore after a half-done swap + with contextlib.suppress(OSError): + await asyncio.to_thread(os.rename, old, path) + await asyncio.to_thread(shutil.rmtree, tmp, ignore_errors=True) + self._log.error("%s: reclone swap failed: %s", component.name, exc) + return False + await asyncio.to_thread(shutil.rmtree, old, ignore_errors=True) + return True + + async def _reclone_component(self, component: ComponentConfig) -> bool: + """Reclone an unrepairable repo: fresh clone to a temp dir, then atomic swap.""" + if not component.url or component.path is None: + return False + path = component.path + tmp = path.parent / f".{path.name}.reclone-tmp" + old = path.parent / f".{path.name}.reclone-old" + for stale in (tmp, old): # clear orphans from a crashed prior reclone + await asyncio.to_thread(shutil.rmtree, stale, ignore_errors=True) + if not await self._reclone_into_tmp(component, tmp): + return False + if not await self._reclone_swap(component, path, tmp, old): + return False + # A fresh clone has no in-repo venv: drop any cached pip path for it. + self._component_pip_cache.pop(str(path), None) + self._history("reclone", component.name, url=component.url) + self._log.warning("%s: recloned successfully", component.name) + return True + + @staticmethod + def _self_update_target(component: ComponentConfig) -> str: + """The ref _stage_component would move the host to (version/branch/HEAD).""" + if component.version: + return component.version + if component.branch: + return f"origin/{component.branch}" + return "origin/HEAD" + + async def _stage_fetch_gate( + self, component: ComponentConfig + ) -> tuple[bool, str] | None: + """Fetch-gate (30s) + fetch, fsck-repair on corruption. None=proceed, tuple=abort.""" if component.path is None: return (False, "path not found") async with self._git_lock: @@ -1391,10 +1814,45 @@ async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str] return (False, "network") self._log.warning("%s: corrupt repo, repairing", component.name) rok, rmsg = await git_repair(component.path) - if not rok: + if rok: + self._history("repair", component.name, detail=rmsg[:80]) + elif await self._reclone_component(component): # deepest rung + self._history("repair", component.name, detail="recloned") + else: return (False, "corrupt") - self._history("repair", component.name, detail=rmsg[:80]) + return None + async def _stage_guard_target( + self, component: ComponentConfig + ) -> tuple[bool, str] | None: + """Pre-checkout guards: dead upstream branch + updater-marker brick guard.""" + # A deleted upstream branch must fail here, not strand the repo mid-switch. + if component.branch and not await git_ref_hash( + component.path, f"origin/{component.branch}" + ): + return ( + False, + f"branch origin/{component.branch} not found - fix components.yaml", + ) + + # Updater host must never checkout code lacking updater: Type=notify unit lacks sd_notify READY causes crash loop with no self-heal. + if component.name == _UI_COMPONENT: + target = self._self_update_target(component) + if target and not await git_tree_has_path( + component.path, target, _UPDATER_MARKER + ): + self._log.error( + "%s: target %s lacks the updater package - refusing brick downgrade", + component.name, + target, + ) + return (False, "refusing downgrade past updater") + return None + + async def _stage_apply_ref(self, component: ComponentConfig) -> tuple[bool, str]: + """Checkout target branch, then hard-reset / version-pin / soft-pull to the tip.""" + if component.path is None: + return (False, "path not found") # Switch to the target branch FIRST so reset/pull act on the right branch. if component.branch: # hard mode forces past untracked collisions (build artifacts). @@ -1431,7 +1889,17 @@ async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str] return (False, "conflict") return (True, "") - async def _restart_one(self, service: str) -> bool: + async def _stage_component(self, component: ComponentConfig) -> tuple[bool, str]: + """Bring the working tree to its target ref: fetch-gate + reset/checkout/pull.""" + gate = await self._stage_fetch_gate(component) + if gate is not None: + return gate + guard = await self._stage_guard_target(component) + if guard is not None: + return guard + return await self._stage_apply_ref(component) + + async def _restart_one(self, service: str, health_url: str | None = None) -> bool: """Restart a service and verify it came active (kill-fallback aware).""" self._log.info("restarting %s and waiting for active", service) ok, err = await restart_service(service) @@ -1441,16 +1909,14 @@ async def _restart_one(self, service: str) -> bool: if not await wait_for_service_active(service, timeout=90.0): self._log.error("%s did not become active after restart", service) return False + if health_url and not await wait_for_http_ready(health_url): + self._log.error("%s active but health check failed", service) + return False self._log.info("%s active after restart", service) return True - async def _run_git_update(self, component: ComponentConfig) -> bool: - """Update a single existing git component, rolling back on any failure. - - Shares its git mechanics (_stage_component) with the multi-component batch; - keeps its own per-component rollback since a single update has no peers to - coordinate. - """ + async def _preflight_git_update(self, component: ComponentConfig) -> bool | None: + """Validate path/repo; return None to proceed or a bool to short-circuit.""" if component.path is None: return self._cb_error_done(component.name, "path not found") if not component.path.exists(): @@ -1471,101 +1937,146 @@ async def _run_git_update(self, component: ComponentConfig) -> bool: return self._cb_error_done( component.name, "not a git repository - reinstall required" ) + return None + async def _prepare_rollback_point( + self, component: ComponentConfig + ) -> tuple[bool, str]: + """Persist prev_hash rollback point + inflight marker, then verify https remote.""" prev_hash = await git_get_hash(component.path) if prev_hash == "": self._log.error("prev_hash is empty") - return self._cb_error_done(component.name, "prev_hash empty") + return self._cb_error_done(component.name, "prev_hash empty"), "" async with self._state_lock: state = await asyncio.to_thread(self._read_state) - state[component.name] = {"prev_hash": prev_hash} + # Merge: replacing the entry would wipe the self-heal anchors (last_good/golden). + _ensure_comp(state, component.name)["prev_hash"] = prev_hash if not await asyncio.to_thread(self._write_state, state): - return self._cb_error_done( - component.name, "failed to persist rollback point" + return ( + self._cb_error_done( + component.name, "failed to persist rollback point" + ), + "", + ) + if not await asyncio.to_thread( + self._write_inflight, {component.name: prev_hash} + ): + return ( + self._cb_error_done( + component.name, "failed to write in-flight marker" + ), + "", ) - await asyncio.to_thread(self._write_inflight, {component.name: prev_hash}) ok, reason = await _assert_https_remote(component.path) if not ok: self._log.error("SEC-4 remote check failed: %s", reason) # Nothing staged: drop the marker (avoids a spurious boot revert). await asyncio.to_thread(self._clear_inflight) - return self._cb_error_done(component.name, "insecure remote") + return self._cb_error_done(component.name, "insecure remote"), "" self._history("update_start", component.name, prev_hash=prev_hash[:12]) + return True, prev_hash - committed = False - try: - self._cb("on_step", component.name, 1, 4) - stage_ok, stage_reason = await self._stage_component(component) - if not stage_ok: - # Failed pre-update reset changed nothing, so nothing to roll back. - if stage_reason == "reset": - await asyncio.to_thread(self._clear_inflight) - return self._cb_error_done(component.name, "reset") - await self._rollback(component, prev_hash, stage_reason) - return False - - self._cb("on_step", component.name, 2, 4) - deps_ok, deps_err = await self._install_dependencies(component) - if not deps_ok: - self._log.warning("dependencies error: %s", deps_err) - await self._rollback(component, prev_hash, "deps") - return False - - self._cb("on_step", component.name, 3, 4) - new_hash = await git_get_hash(component.path) - hook_ok, hook_err = await self._ping_while( - run_hook( - component.name, - component.path, - new_hash, - prev_hash, - timeout=HOOK_TIMEOUT, - ), - component.name, - 3, - 4, - ) - if not hook_ok: - self._log.error("hook failed for %s: %s", component.name, hook_err) - await self._rollback(component, prev_hash, "hook") - return False - - self._cb("on_step", component.name, 4, 4) - fire_and_forget = component.service in _FIRE_AND_FORGET_SERVICES + async def _run_git_phases( + self, component: ComponentConfig, prev_hash: str + ) -> tuple[bool, str]: + """Run stage->deps->hook->restart; rollback + return (False, '') on any failure.""" + self._cb("on_step", component.name, 1, 4) + stage_ok, stage_reason = await self._stage_component(component) + if not stage_ok: + # Pre-checkout failures leave the tree untouched: no revert, no restart. if ( - component.service - and not fire_and_forget - and not await self._restart_one(component.service) + stage_reason in ("network", "corrupt") + or "not found" in stage_reason + or "refusing" in stage_reason ): + await asyncio.to_thread(self._clear_inflight) + return self._cb_error_done(component.name, stage_reason), "" + # checkout/reset/pin/pull may have moved the tree: full rollback. + await self._rollback(component, prev_hash, stage_reason) + return False, "" + + self._cb("on_step", component.name, 2, 4) + deps_ok, deps_err = await self._install_dependencies(component) + if not deps_ok: + self._log.warning("dependencies error: %s", deps_err) + await self._rollback(component, prev_hash, "deps") + return False, "" + + self._cb("on_step", component.name, 3, 4) + new_hash = await git_get_hash(component.path) + hook_ok, hook_err = await self._ping_while( + run_hook( + component.name, + component.path, + new_hash, + prev_hash, + timeout=HOOK_TIMEOUT, + ), + component.name, + 3, + 4, + ) + if not hook_ok: + self._log.error("hook failed for %s: %s", component.name, hook_err) + await self._rollback(component, prev_hash, "hook") + return False, "" + + self._cb("on_step", component.name, 4, 4) + fire_and_forget = component.service in _FIRE_AND_FORGET_SERVICES + if ( + component.service + and not fire_and_forget + and not await self._restart_one(component.service, component.health_url) + ): + await self._rollback(component, prev_hash, "restart") + return False, "" + # Bounce klipper too when requested and it isn't the component's own service. + klipper_svc = self._klipper_service() + if component.restart_klipper and component.service != klipper_svc: + if not await self._restart_one(klipper_svc): await self._rollback(component, prev_hash, "restart") - return False - # Bounce klipper too when requested and it isn't the component's own service. - klipper_svc = self._klipper_service() - if component.restart_klipper and component.service != klipper_svc: - if not await self._restart_one(klipper_svc): - await self._rollback(component, prev_hash, "restart") - return False + return False, "" + return True, new_hash + + async def _fire_and_forget_restart(self, component: ComponentConfig) -> None: + """Post-commit: kick self/UI/klipper-config restart without waiting.""" + fire_and_forget = component.service in _FIRE_AND_FORGET_SERVICES + # Self/UI service: queue the restart only after success is recorded. + if fire_and_forget and component.service: + self._log.info( + "%s updated; fire-and-forget restart of %s (no wait)", + component.name, + component.service, + ) + await restart_service_noblock(component.service) + # klipper/RF50 hold config the UI reads at startup: refresh it too. + elif component.restart_ui: + self._log.info( + "%s updated (restart_ui); fire-and-forget restart of %s", + component.name, + _UI_SERVICE, + ) + await restart_service_noblock(_UI_SERVICE) + async def _run_git_update(self, component: ComponentConfig) -> bool: + """Update a single existing git component, rolling back on any failure.""" + short = await self._preflight_git_update(component) + if short is not None: + return short + ok, prev_hash = await self._prepare_rollback_point(component) + if not ok: + return False + + committed = False + try: + phases_ok, new_hash = await self._run_git_phases(component, prev_hash) + if not phases_ok: + return False self._history("update_success", component.name, new_hash=new_hash[:12]) self._cb("on_component_done", component.name, True) committed = True await asyncio.to_thread(self._clear_inflight) - # Self/UI service: queue the restart only after success is recorded. - if fire_and_forget and component.service: - self._log.info( - "%s updated; fire-and-forget restart of %s (no wait)", - component.name, - component.service, - ) - await restart_service_noblock(component.service) - # klipper/RF50 hold config the UI reads at startup: refresh it too. - elif component.restart_ui: - self._log.info( - "%s updated (restart_ui); fire-and-forget restart of %s", - component.name, - _UI_SERVICE, - ) - await restart_service_noblock(_UI_SERVICE) + await self._fire_and_forget_restart(component) await self._apply_deferred_restart() return True except asyncio.CancelledError: @@ -1618,6 +2129,7 @@ async def _run_apt_update(self, component: ComponentConfig) -> bool: async def _run_apt_update_locked( self, component: ComponentConfig, _apt_cache: Path ) -> bool: + """apt refresh, upgrade, and autoremove under the apt lock with snapshot rollback.""" try: if self._apt_backoff.cooling_down(): self._log.debug("apt cooling down; skipping update") @@ -1682,17 +2194,24 @@ async def _run_apt_update_locked( raise def _read_state(self) -> dict: + """Return the self-heal state dict, empty on missing/corrupt/wrong-shape.""" try: - return json.loads(self._state_path.read_text()) - except (FileNotFoundError, json.JSONDecodeError): + data = json.loads(self._state_path.read_text()) + except (OSError, ValueError): return {} + # Valid JSON of the wrong shape (torn/corrupt write) must read as empty. + return data if isinstance(data, dict) else {} def _read_inflight(self) -> dict[str, str]: + """Return the in-flight batch marker as a name->prev_hash dict.""" try: data = json.loads(self._inflight_path.read_text()) - except (FileNotFoundError, json.JSONDecodeError): + except (OSError, ValueError): return {} - return data if isinstance(data, dict) else {} + if not isinstance(data, dict): + return {} + # Drop corrupt entries so a torn write can't crash the boot revert. + return {k: v for k, v in data.items() if isinstance(k, str) and _is_sha(v)} def _write_inflight(self, mapping: dict[str, str]) -> bool: """Atomically record the in-flight batch's name->prev_hash map (fsync'd).""" @@ -1725,6 +2244,7 @@ def _clear_inflight(self) -> None: self._log.warning("Failed to clear in-flight marker") def _write_state(self, data: dict) -> bool: + """Atomically write the self-heal state file (temp, fsync, replace).""" try: self._state_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) # SEC: atomic write via temp file prevents symlink attacks and partial writes @@ -1756,25 +2276,21 @@ async def _mutate_state(self, mutate: Callable[[dict], None]) -> bool: @staticmethod def _fsync_dir(path: Path) -> None: - try: + """fsync a directory so a contained rename survives power loss.""" + with contextlib.suppress(OSError): fd = os.open(path, os.O_RDONLY) try: os.fsync(fd) finally: os.close(fd) - except OSError: - pass async def background_apt_upgrade(self) -> None: - """Run apt update + upgrade + autoremove silently after every Update click. - - Uses apt-get upgrade (never dist-upgrade) so the Debian release never changes. - Never reports to the UI - failures are logged only. - """ + """Run apt update + upgrade + autoremove silently after every Update click.""" async with self._apt_lock: await self._background_apt_upgrade_locked() async def _background_apt_upgrade_locked(self) -> None: + """Silent apt update, upgrade, and autoremove honoring the apt excludes.""" if self._apt_backoff.cooling_down(): self._log.debug("apt cooling down; skipping background upgrade") return