diff --git a/.changeset/atomic-control-plane-pair-v2.md b/.changeset/atomic-control-plane-pair-v2.md new file mode 100644 index 00000000..e52f6275 --- /dev/null +++ b/.changeset/atomic-control-plane-pair-v2.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Update Core and updater as one digest-locked release pair, recover stale transactions, and restore both prior images when readiness fails. diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index aedf4a84..0c94d732 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -94,8 +94,30 @@ jobs: username: ${{ vars.SOURCEFUL_GHCR_USER || github.actor }} password: ${{ secrets.SOURCEFUL_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + - name: Refuse to move an existing immutable tag + id: immutable + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}:${{ needs.tag.outputs.version }} + TAG: ${{ needs.tag.outputs.version }} + run: | + set -euo pipefail + if ! scripts/inspect-image-digest.sh "${IMAGE}" >/dev/null 2>&1; then + echo "skip=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + # A finished prior run is safe to repeat. A partial tag has no pair + # manifest, so a maintainer must review it instead of moving the tag. + mkdir -p existing + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir existing + scripts/verify-control-plane-manifest.sh \ + existing/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + echo "skip=true" >> "${GITHUB_OUTPUT}" + - name: Build and publish beta image id: build + if: steps.immutable.outputs.skip != 'true' uses: docker/build-push-action@v6 with: context: . @@ -118,6 +140,7 @@ jobs: cache-to: type=gha,mode=max,scope=${{ matrix.target }} - name: Login to compatibility GHCR namespace + if: steps.immutable.outputs.skip != 'true' uses: docker/login-action@v3 with: registry: ghcr.io @@ -125,6 +148,7 @@ jobs: password: ${{ secrets.LEGACY_GHCR_TOKEN || secrets.GITHUB_TOKEN }} - name: Mirror exact beta manifests to compatibility namespace + if: steps.immutable.outputs.skip != 'true' env: IMAGE_SUFFIX: ${{ matrix.image_suffix }} SOURCE_DIGEST: ${{ steps.build.outputs.digest }} @@ -140,11 +164,44 @@ jobs: test "${SOURCE_DIGEST}" = "${legacy_digest}" done + control-plane: + name: verify immutable Core/updater pair + runs-on: ubuntu-latest + needs: [tag, docker] + steps: + - name: Checkout beta tag + uses: actions/checkout@v5 + with: + ref: ${{ needs.tag.outputs.version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create and verify pair manifest + env: + TAG: ${{ needs.tag.outputs.version }} + run: | + set -euo pipefail + scripts/create-control-plane-manifest.sh "${TAG}" "${GITHUB_SHA}" ftw-control-plane.json + scripts/verify-control-plane-manifest.sh ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + + - name: Save pair manifest for release job + uses: actions/upload-artifact@v4 + with: + name: beta-control-plane-${{ needs.tag.outputs.version }} + path: ftw-control-plane.json + if-no-files-found: error + release: name: publish GitHub prerelease runs-on: ubuntu-latest - needs: [tag, docker] + needs: [tag, control-plane] steps: + - name: Download verified pair manifest + uses: actions/download-artifact@v4 + with: + name: beta-control-plane-${{ needs.tag.outputs.version }} + - name: Publish prerelease after images are available env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -152,11 +209,14 @@ jobs: run: | set -euo pipefail if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then - echo "GitHub prerelease ${TAG} already exists" + gh release upload "${TAG}" ftw-control-plane.json \ + --repo "${GITHUB_REPOSITORY}" --clobber + echo "Verified GitHub prerelease ${TAG} already exists" exit 0 fi gh release create "${TAG}" \ --repo "${GITHUB_REPOSITORY}" \ --title "${TAG}" \ --prerelease \ - --generate-notes + --generate-notes \ + ftw-control-plane.json diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 1c4a0b7f..1240f4f3 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -54,6 +54,11 @@ jobs: tag: ${{ steps.meta.outputs.tag }} version: ${{ steps.meta.outputs.version }} steps: + - name: Checkout stable tag + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Derive tag/version from ref id: meta env: @@ -81,6 +86,30 @@ jobs: echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" echo "Building assets for ${TAG} (version ${VERSION})" + - name: Set up Docker Buildx for promotion check + uses: docker/setup-buildx-action@v3 + + - name: Repeat beta-to-stable pair gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.meta.outputs.tag }} + run: | + set -euo pipefail + TAG_COMMIT="$(git rev-list -n 1 "${TAG}")" + test "${TAG_COMMIT}" = "${GITHUB_SHA}" + BETA_TAG="$(git tag --list "${TAG}-beta.*" --sort=-v:refname | head -n 1)" + test -n "${BETA_TAG}" + BETA_COMMIT="$(git rev-list -n 1 "${BETA_TAG}")" + test "${BETA_COMMIT}" = "${GITHUB_SHA}" + RELEASE_JSON="$(gh release view "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft,isPrerelease)" + test "$(jq -r .isDraft <<<"${RELEASE_JSON}")" = false + test "$(jq -r .isPrerelease <<<"${RELEASE_JSON}")" = true + mkdir beta-control-plane + gh release download "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir beta-control-plane + scripts/verify-control-plane-manifest.sh \ + beta-control-plane/ftw-control-plane.json "${BETA_TAG}" "${GITHUB_SHA}" + binaries: name: build + upload release binaries runs-on: ubuntu-latest @@ -249,8 +278,28 @@ jobs: username: ${{ vars.SOURCEFUL_GHCR_USER || github.actor }} password: ${{ secrets.SOURCEFUL_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + - name: Refuse to move an existing immutable tag + id: immutable + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}:${{ needs.meta.outputs.tag }} + TAG: ${{ needs.meta.outputs.tag }} + run: | + set -euo pipefail + if ! scripts/inspect-image-digest.sh "${IMAGE}" >/dev/null 2>&1; then + echo "skip=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + mkdir -p existing + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir existing + scripts/verify-control-plane-manifest.sh \ + existing/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + echo "skip=true" >> "${GITHUB_OUTPUT}" + - name: Compute image metadata id: meta + if: steps.immutable.outputs.skip != 'true' uses: docker/metadata-action@v5 with: images: ghcr.io/srcfl/ftw${{ matrix.image_suffix }} @@ -263,11 +312,13 @@ jobs: org.opencontainers.image.title=${{ matrix.title }} org.opencontainers.image.description=${{ matrix.description }} org.opencontainers.image.source=https://github.com/srcfl/ftw + org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.version=${{ needs.meta.outputs.version }} org.opencontainers.image.licenses=Apache-2.0 - name: Build and push id: build + if: steps.immutable.outputs.skip != 'true' uses: docker/build-push-action@v6 with: context: . @@ -289,6 +340,7 @@ jobs: # canonical namespace, then copy that exact multi-platform manifest to # the temporary compatibility namespace under its own credentials. - name: Login to compatibility GHCR namespace + if: steps.immutable.outputs.skip != 'true' uses: docker/login-action@v3 with: registry: ghcr.io @@ -296,6 +348,7 @@ jobs: password: ${{ secrets.LEGACY_GHCR_TOKEN || secrets.GITHUB_TOKEN }} - name: Mirror exact manifests to compatibility namespace + if: steps.immutable.outputs.skip != 'true' env: CANONICAL_TAGS: ${{ steps.meta.outputs.tags }} SOURCE_DIGEST: ${{ steps.build.outputs.digest }} @@ -309,13 +362,54 @@ jobs: test "${SOURCE_DIGEST}" = "${legacy_digest}" done + control-plane: + name: verify pair and publish release + runs-on: ubuntu-latest + needs: [meta, binaries, docker, imager-metadata] + permissions: + contents: write + steps: + - name: Checkout stable tag + uses: actions/checkout@v5 + with: + ref: ${{ needs.meta.outputs.tag }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create and verify immutable pair manifest + env: + TAG: ${{ needs.meta.outputs.tag }} + run: | + set -euo pipefail + scripts/create-control-plane-manifest.sh "${TAG}" "${GITHUB_SHA}" ftw-control-plane.json + scripts/verify-control-plane-manifest.sh ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + + - name: Attach pair manifest and publish draft + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.meta.outputs.tag }} + run: | + set -euo pipefail + gh release upload "${TAG}" ftw-control-plane.json \ + --repo "${GITHUB_REPOSITORY}" --clobber + mkdir published + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir published + scripts/verify-control-plane-manifest.sh \ + published/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + IS_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)" + if [ "${IS_DRAFT}" = true ]; then + gh release edit "${TAG}" --repo "${GITHUB_REPOSITORY}" --draft=false + fi + discord: name: announce release on Discord runs-on: ubuntu-latest # Wait for binaries + docker so the message links to assets that # actually exist. The independent rpi-installer channel is intentionally # outside the application release critical path. - needs: [meta, binaries, docker] + needs: [meta, control-plane] permissions: contents: read # gh release view needs read access steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68732c5d..5a8364ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: release # Changesets-driven release. Pattern lifted from umara/u-front. # -# This workflow ONLY handles the changesets flow + tag + GitHub Release -# creation. The cross-platform binaries, docker images, and +# This workflow ONLY handles the changesets flow + tag + draft GitHub Release +# creation. The cross-platform binaries, docker images, pair verification and # Discord announcement live in `release-assets.yml` and are triggered # explicitly via `gh workflow run release-assets.yml --ref vX.Y.Z` at # the end of the publish step. Keeping the heavy build matrix out of @@ -123,6 +123,10 @@ jobs: # needs broader scope, and a narrow token bounds the blast radius. GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} + - name: Set up Docker Buildx for promotion check + if: github.event_name == 'workflow_dispatch' && steps.changesets.outputs.hasChangesets == 'false' + uses: docker/setup-buildx-action@v3 + - name: Tag and create GitHub Release id: publish # Stable is an explicit promotion after real beta validation. Pushes @@ -141,14 +145,16 @@ jobs: TAG_EXISTS=true fi RELEASE_EXISTS=false + RELEASE_DRAFT=false if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then RELEASE_EXISTS=true + RELEASE_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)" fi # Already-done case: a previous run completed cleanly. No # new release was cut on THIS run, so the release-assets # dispatch below should skip — emit `published=false`. - if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ]; then + if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ] && [ "${RELEASE_DRAFT}" = "false" ]; then echo "Tag ${TAG} and GitHub Release already exist — nothing to publish." { echo "published=false" @@ -157,6 +163,15 @@ jobs: exit 0 fi + if [ "${TAG_EXISTS}" = "true" ]; then + git fetch --force origin "refs/tags/${TAG}:refs/tags/${TAG}" + TAG_COMMIT="$(git rev-list -n 1 "${TAG}")" + if [ "${TAG_COMMIT}" != "${GITHUB_SHA}" ]; then + echo "Existing ${TAG} points to ${TAG_COMMIT}, not ${GITHUB_SHA}; refusing to move it." >&2 + exit 1 + fi + fi + # Stable is a promotion, never the first public channel for a commit. # beta.yml also checks that the prerelease base matches package.json. git fetch --force --tags origin @@ -171,6 +186,18 @@ jobs: echo "Publish a beta from this exact commit before promotion." >&2 exit 1 fi + BETA_RELEASE_JSON="$(gh release view "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft,isPrerelease)" + if [ "$(jq -r .isDraft <<<"${BETA_RELEASE_JSON}")" != "false" ] || \ + [ "$(jq -r .isPrerelease <<<"${BETA_RELEASE_JSON}")" != "true" ]; then + echo "${BETA_TAG} is not a published beta prerelease." >&2 + exit 1 + fi + rm -rf beta-control-plane + mkdir beta-control-plane + gh release download "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir beta-control-plane + scripts/verify-control-plane-manifest.sh \ + beta-control-plane/ftw-control-plane.json "${BETA_TAG}" "${GITHUB_SHA}" echo "Promoting tested ${BETA_TAG} to ${TAG}." # Build codename-annotated release notes. The script extracts @@ -196,14 +223,18 @@ jobs: gh release create "${TAG}" \ --repo "${GITHUB_REPOSITORY}" \ --title "${TAG}" \ - --notes-file release-notes.md + --notes-file release-notes.md \ + --draft + elif [ "${RELEASE_DRAFT}" != "true" ]; then + echo "Existing release ${TAG} is public but was not recognized as complete." >&2 + exit 1 fi { echo "published=true" echo "version=${VERSION}" } >> "${GITHUB_OUTPUT}" - echo "Published ${TAG}" + echo "Prepared draft ${TAG}; release-assets will publish it after pair verification" - name: Dispatch release-assets workflow # GitHub does NOT fire downstream `push: tags` workflows from diff --git a/Dockerfile b/Dockerfile index 8d5164e3..2c49703a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,8 +90,10 @@ EXPOSE 8080 # Without this the binary fails fast with "open state … unable to # open database file" because SQLite can't create state.db inside # a directory it doesn't own. -HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=12 \ - CMD wget -q -T 4 -O /dev/null http://127.0.0.1:8080/api/health || exit 1 +# Readiness must stay false until Core has opened and migrated state. The long +# start period lets a valid large migration finish without marking it unhealthy. +HEALTHCHECK --interval=10s --timeout=5s --start-period=30m --retries=12 \ + CMD wget -q -T 4 -O /dev/null http://127.0.0.1:8080/api/status || exit 1 ENTRYPOINT ["/app/ftw"] CMD ["-config", "/app/data/config.yaml", "-web", "/app/web", "-drivers", "/app/drivers", "-user-drivers", "/app/data/drivers"] diff --git a/Makefile b/Makefile index 6fcdaad7..0e196c5f 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ # make dev — start sims + main app (hot-reload workflow) # make clean — remove all build artifacts -.PHONY: help test optimizer-install optimizer-test compose-migration-test container-boundary-test build build-arm64 build-amd64 build-windows-amd64 release \ +.PHONY: help test optimizer-install optimizer-test compose-migration-test container-boundary-test release-contract-test build build-arm64 build-amd64 build-windows-amd64 release \ run-sim dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions @@ -71,6 +71,9 @@ compose-migration-test: container-boundary-test: bash scripts/test-container-boundaries.sh +release-contract-test: + bash scripts/test-control-plane-release.sh + optimizer/.venv/.installed: optimizer/pyproject.toml $(MAKE) optimizer-install @@ -102,7 +105,7 @@ ci-hw-pi: # verify-all adds cross-compile checks for all release targets, catching # platform-specific syscall/import mistakes before push. -verify: test compose-migration-test container-boundary-test +verify: test compose-migration-test container-boundary-test release-contract-test cd go && go vet ./... cd go && go build ./... @echo "verify: vet + test + build clean" diff --git a/docs/self-update.md b/docs/self-update.md index ac5da20d..3019bd7b 100644 --- a/docs/self-update.md +++ b/docs/self-update.md @@ -20,19 +20,26 @@ After that PR merges: 1. run `beta.yml` with `vX.Y.Z-beta.N`; 2. validate that immutable build on real sites; 3. manually dispatch `release.yml` from that same commit; -4. stable promotion verifies that a matching beta tag resolves to the exact +4. stable promotion checks that the beta tag and pair manifest match the exact stable candidate commit; -5. release assets publish `vX.Y.Z` and move the stable aliases. +5. release assets keep the stable release as a draft while they build Core and + updater; +6. the workflow checks both exact tag digests, adds `ftw-control-plane.json`, + checks the uploaded file, then publishes the draft and moves stable aliases. Stable therefore cannot be the first public channel for new code. Beta and stable may have different tags but identify the same source commit. ## Immutable update targets -The checker uses GitHub Releases to select a released version and GHCR to prove -that its exact image tag exists. The updater installs the immutable tag, never -the moving `:latest` or `:beta` alias. This avoids the race where a release -exists before a moving image alias has advanced. +The checker uses GitHub Releases to select a release. It accepts a Core release +only when `ftw-control-plane.json` names that release and pins both +`ghcr.io/srcfl/ftw` and `ghcr.io/srcfl/ftw-updater` by digest. It resolves both +exact tags again and rejects a missing or changed digest. The updater installs +the two approved digest refs, never `:latest` or `:beta`. The tag remains the +release identity, but the pull, Compose override and final running-image check +all use the digests from the verified record. A moved tag cannot change the +bytes installed after the check. Release notes are best-effort UI data. Failure to fetch notes does not weaken tag resolution or image verification. @@ -60,9 +67,27 @@ These bounded points remain on the same disk and are deliberately labelled **Local rollback points**, not full backups. Older incomplete snapshots are visible but cannot be restored. +Core and updater form one release pair. Updater `/status` must report protocol +2 or later, capability `control-plane-pair-v1`, and the exact Core release. +Core checks this before it opens or migrates state. A v1.3.1 updater lacks this +contract, so a new Core stays unready and does not open state with it. + +The updater saves both prior image IDs before it pulls the pair. A detached +helper starts the matching updater first, checks its handshake, then starts +Core. Core must pass `/api/status` after state and migrations are ready. The +helper marks the update done only after both containers report the exact target +digest refs. On failure it restores updater first and then Core. The final state +remains failed even when rollback succeeds. The shared state file carries both +old image IDs, both approved digests and a helper heartbeat. A new updater waits +while that heartbeat is fresh. After five minutes without a heartbeat it starts +a recovery helper from the saved old updater image. A durable transaction ID +ties each heartbeat and helper name to one update. Recovery stops that exact +stale helper before it restores updater first and Core second. It never changes +the final result to `done`. + The updater also requires a running, healthy `ftw-optimizer` service before it updates Core. If the merged Compose files lack that service, or its health -check fails, the update stops before pulling or replacing Core. The updater +check fails, the update stops before pulling or replacing the pair. The updater does not edit operator override files; use the [legacy upgrade guide](upgrade-from-legacy.md) to add the sidecar safely. @@ -73,8 +98,13 @@ retains the pre-restore directory and automatically reactivates it when the restored Core fails health. See [backup-and-restore.md](backup-and-restore.md). Optimizer-only updates use `optimizer-vX.Y.Z[-beta.N]`, recreate and -health-check only `ftw-optimizer`, and never replace Core. Failure restores the -previous Optimizer image while Core continues on its Go fallback. +health-check only `ftw-optimizer`, and never replace Core. Optimizer has its +own SemVer line, which starts at 1.3.2. Core checks `name=ftw-optimizer`, exact +protocol 1, plan schema 1, and required features. It does not compare optimizer +and Core versions. The old optimizer releases 1.3.1-beta.1 and 1.3.1 remain +compatible through protocol 1. A request for recourse or multistage also needs +that feature; auto transport falls back when it is missing. Failure restores +the prior Optimizer image while Core continues on its Go fallback. A Driver update downloads one signed artifact, verifies hash, metadata and host API compatibility, then atomically activates exactly that version. Core puts @@ -95,14 +125,36 @@ The version badge selects `stable` or `beta`, checks availability and starts an update. Changing channel does not deploy anything. A skipped version remains hidden only until a newer version appears. -For manual Core + updater operation: +The first pair-aware release needs one operator bridge from +`v1.10.0-beta.1`. That updater can replace Core only and the old Core cannot +send the verified digests required by the new updater. The old Update Center +therefore must not be used for this one step. + +Use the migration command in [Upgrade an older installation](upgrade-from-legacy.md) +with the exact Core and updater digests from the reviewed +`ftw-control-plane.json`. The script: + +1. saves both running image IDs; +2. pulls both approved digest refs and tags only those local images; +3. replaces updater and confirms old Core remains healthy; +4. replaces Core and waits for `/api/status`; +5. restores updater first and Core second on any failure. + +After this one-time bridge, later pair-aware releases update through Update +Center. A tag-only manual pull is not an approved pair update. + +For inspection after the bridge: ```bash cd ~/ftw -docker compose pull ftw ftw-updater -docker compose up -d --no-deps ftw ftw-updater +docker compose ps +docker inspect "$(docker compose ps -q ftw-updater)" --format '{{.Image}} {{.Config.Image}}' +docker inspect "$(docker compose ps -q ftw)" --format '{{.Image}} {{.Config.Image}}' +curl -fsS http://127.0.0.1:8080/api/status ``` +Do not start Core first and do not mix release records. + Manage Optimizer and Drivers independently in Update Center. A blanket `docker compose pull` is intentionally not the documented upgrade procedure. @@ -129,3 +181,18 @@ tested locally. Main publishes `drivers-beta`; `drivers-stable` promotes the exact signed beta commit and retains per-driver version history. See [device-repository.md](device-repository.md). + +## Compatibility and rollout gate + +| Pair | Result | +|---|---| +| New Core + same-release updater, protocol 2+, pair capability | Allowed | +| New Core + v1.3.1 updater or updater without the pair capability | Core stays unready before state opens | +| New Core + different updater release | Core stays unready before state opens | +| Core/updater release with one missing or changed image digest | Not shown as an update and not published | +| Core + optimizer 1.3.1-beta.1, 1.3.1, or 1.3.2+ with protocol 1 | Allowed; optimizer version need not match Core | + +Do not merge or roll out a Core/updater pair until the release manifest checks +pass, the cross-version updater test passes, migration restores both old image +IDs after a readiness failure, `make verify` passes, and the active beta pilot +reports PASS. diff --git a/docs/upgrade-from-legacy.md b/docs/upgrade-from-legacy.md index 3e97e76d..28219c39 100644 --- a/docs/upgrade-from-legacy.md +++ b/docs/upgrade-from-legacy.md @@ -21,7 +21,13 @@ Kör från installationskatalogen: ```bash curl -fsSL https://raw.githubusercontent.com/srcfl/ftw/master/scripts/migrate-legacy-compose.sh \ -o /tmp/ftw-migrate.sh +RELEASE=vX.Y.Z # byt till den godkända beta- eller stable-releasen +CORE_DIGEST=sha256:... # exakt värde från granskad ftw-control-plane.json +UPDATER_DIGEST=sha256:... # exakt värde från samma fil bash /tmp/ftw-migrate.sh \ + --version "$RELEASE" \ + --core-digest "$CORE_DIGEST" \ + --updater-digest "$UPDATER_DIGEST" \ --dir "$PWD" \ --backup-dir /media/$USER/FTW-BACKUP ``` @@ -30,14 +36,19 @@ Om du saknar extern disk kan du utelämna `--backup-dir`, men kopiera den utskrivna `.ftwbak`-filen från `ftw-backups/` till en annan dator direkt efter migreringen. Skriptet letar annars i aktuell katalog, `~/ftw` och `~/forty-two-watts`; om flera installationer hittas måste `--dir` anges. +Detta är även den enda godkända första uppdateringen från +`v1.10.0-beta.1`: den gamla produktens Update Center uppdaterar inte paret och +skickar inte de verifierade digestvärdena. ### Fyra oberoende faser 1. **Full backup.** Den nya backuphjälparen öppnar den äldre databasen skrivskyddat, gör ingen schemamigrering, bygger en komplett `.ftwbak`, verifierar filhashar och SQLite och stoppar vid minsta fel. -2. **Core + updater.** Det parade kontrollplanet hämtas och återskapas med - samma data-bind. Core måste både vara frisk på `/api/health` och helt +2. **Core + updater.** Samma verifierade releasepost och dess två exakta + digests används. Updatern startas före Core medan den äldre Core fortsätter + vara frisk. Därefter startas Core med samma data-bind. Core måste vara frisk på + `/api/health` och helt startklar på `/api/status`; annars återställs Compose, tidigare oföränderliga image-ID:n och containrar automatiskt. 3. **Optimizer.** Optimizern hämtas och hälsokontrolleras separat. Om den @@ -87,7 +98,13 @@ Run from the installation directory: ```bash curl -fsSL https://raw.githubusercontent.com/srcfl/ftw/master/scripts/migrate-legacy-compose.sh \ -o /tmp/ftw-migrate.sh +RELEASE=vX.Y.Z # replace with the approved beta or stable release +CORE_DIGEST=sha256:... # exact value from the reviewed ftw-control-plane.json +UPDATER_DIGEST=sha256:... # exact value from the same file bash /tmp/ftw-migrate.sh \ + --version "$RELEASE" \ + --core-digest "$CORE_DIGEST" \ + --updater-digest "$UPDATER_DIGEST" \ --dir "$PWD" \ --backup-dir /media/$USER/FTW-BACKUP ``` @@ -96,14 +113,18 @@ If no external disk is available, omit `--backup-dir` and copy the printed archive from `ftw-backups/` to another computer immediately afterwards. The script can also discover the current directory, `~/ftw`, or `~/forty-two-watts`; ambiguous installations require `--dir`. +This is also the only approved first update from `v1.10.0-beta.1`: the old +product Update Center does not update the pair or send the verified digests. ### Four independent phases 1. **Full backup.** The new helper opens the legacy database read-only, performs no schema migration, creates a complete archive, and verifies file hashes plus SQLite before any deployment change. -2. **Core + updater.** The paired control plane is recreated on the same data - bind and must pass both `/api/health` and full readiness on `/api/status`. +2. **Core + updater.** Both use the same verified release record and its two + exact digests. Updater starts first while old Core remains healthy. Core + then starts on the same data bind and must pass `/api/health` and full + readiness on `/api/status`. Failure restores Compose, the prior immutable image IDs, and the previous containers automatically. 3. **Optimizer.** Optimizer is pulled and health-checked separately. Failure diff --git a/go/cmd/ftw-updater/control_plane.go b/go/cmd/ftw-updater/control_plane.go new file mode 100644 index 00000000..34a143dc --- /dev/null +++ b/go/cmd/ftw-updater/control_plane.go @@ -0,0 +1,775 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + "sync" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + controlPlaneHelperUpdater = "updater" + controlPlaneHelperNormal = "normal" + controlPlaneHelperRecovery = "recovery" +) + +type controlPlaneRelease struct { + Target string + Revision string + CoreDigest string + UpdaterDigest string +} + +func (r controlPlaneRelease) validate() error { + if !isImmutableImageTag(r.Target) { + return errors.New("target is not an immutable release tag") + } + if !validHexValue(r.Revision, 40, "") { + return errors.New("source revision must contain 40 lowercase hex characters") + } + if !validHexValue(r.CoreDigest, 64, "sha256:") || !validHexValue(r.UpdaterDigest, 64, "sha256:") { + return errors.New("Core and updater digests must be sha256 values") + } + return nil +} + +func validHexValue(value string, digits int, prefix string) bool { + if !strings.HasPrefix(value, prefix) || len(value) != len(prefix)+digits { + return false + } + for _, r := range value[len(prefix):] { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} + +func (r controlPlaneRelease) coreRef() string { + return canonicalMainImage + "@" + r.CoreDigest +} + +func (r controlPlaneRelease) updaterRef() string { + return canonicalUpdaterImage + "@" + r.UpdaterDigest +} + +func controlPlaneEnv(target string) []string { + return []string{"FTW_IMAGE_TAG=" + target, "FTW_UPDATER_IMAGE_TAG=" + target} +} + +func (s *server) startControlPlaneHeartbeat(target, transactionID, helperKind string, states ...string) func() { + allowed := make(map[string]bool, len(states)) + for _, state := range states { + allowed[state] = true + } + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-ticker.C: + s.refreshControlPlaneHeartbeat(target, transactionID, helperKind, allowed) + case <-stop: + return + } + } + }() + var once sync.Once + return func() { + once.Do(func() { close(stop) }) + <-done + } +} + +func (s *server) refreshControlPlaneHeartbeat(target, transactionID, helperKind string, allowed map[string]bool) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + st := s.readStateLocked() + if s.heartbeatAfterRead != nil { + s.heartbeatAfterRead() + } + if st.Action == "update" && st.Component == "core" && st.Target == target && + st.TransactionID == transactionID && st.HelperKind == helperKind && allowed[st.State] { + st.UpdatedAt = s.nowTime() + s.writeStateLocked(st) + } +} + +// runControlPlaneUpdate captures both rollback images and pulls the matched +// pair. A detached helper then owns replacement and rollback while the updater +// service replaces itself. +func (s *server) runControlPlaneUpdate(release controlPlaneRelease, startedAt time.Time) { + target := release.Target + transactionID := newControlPlaneTransactionID(startedAt) + stopHeartbeat := s.startControlPlaneHeartbeat(target, transactionID, controlPlaneHelperUpdater, "pulling") + defer stopHeartbeat() + base := State{ + State: "pulling", Action: "update", Component: "core", Target: target, + StartedAt: startedAt, UpdatedAt: s.nowTime(), ReleaseRevision: release.Revision, + CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, TransactionID: transactionID, + HelperKind: controlPlaneHelperUpdater, + } + s.writeState(base) + if err := release.validate(); err != nil { + s.failControlPlanePreflight(base, err) + return + } + + cleanup, err := s.prepareControlPlaneDigestPins(release) + if err != nil { + s.failControlPlanePreflight(base, err) + return + } + cleanupActive := true + defer func() { + if cleanupActive { + cleanup() + } + }() + if err := s.validateControlPlaneDigestPins(release); err != nil { + s.failControlPlanePreflight(base, err) + return + } + specs, err := s.controlPlaneSpecs() + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", err.Error(), s.nowTime() + s.writeState(base) + return + } + + previous := make(map[string]string, len(specs)) + for _, spec := range specs { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + imageID, captureErr := s.imageID(ctx, spec.service) + cancel() + if captureErr != nil || imageID == "" { + if captureErr == nil { + captureErr = errors.New("empty image ID") + } + base.State = "failed" + base.Message = "cannot capture " + spec.name + " image for rollback: " + captureErr.Error() + base.UpdatedAt = s.nowTime() + s.writeState(base) + return + } + previous[spec.name] = imageID + } + base.PreviousImages = previous + base.PreviousImageID = previous["core"] + s.writeState(base) + + if !s.skipPull { + pullArgs := s.composeArgs("pull", s.mainServiceName, updaterServiceName) + var pullErr error + for attempt := 1; ; attempt++ { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + pullErr = s.runner(ctx, controlPlaneEnv(target), pullArgs...) + cancel() + if pullErr == nil { + break + } + if s.maxPullAttempts > 0 && attempt >= s.maxPullAttempts { + break + } + time.Sleep(s.pullRetryDelay) + } + if pullErr != nil { + base.State, base.Message, base.UpdatedAt = "failed", "digest-pinned pair pull failed: "+pullErr.Error(), s.nowTime() + s.writeState(base) + return + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + updaterContainer, err := s.containerID(ctx, updaterServiceName) + cancel() + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "cannot locate updater container: "+err.Error(), s.nowTime() + s.writeState(base) + return + } + // The pull is complete. Drop this process's private override before the + // detached helper creates its own digest-pinned override. + cleanup() + cleanupActive = false + // The detached helper becomes the sole state writer at handoff. Wait for + // this updater's heartbeat before publishing helper ownership. + stopHeartbeat() + base.State = "transacting" + base.HelperKind = controlPlaneHelperNormal + base.Message = "paired images pulled; detached helper owns the transaction" + base.UpdatedAt = s.nowTime() + s.writeState(base) + if s.launchTransaction != nil { + if err := s.launchTransaction(release, previous, startedAt, transactionID); err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "cannot start transaction helper: "+err.Error(), s.nowTime() + s.writeState(base) + } + return + } + err = s.launchControlPlaneHelper(updaterContainer, previous["updater"], false, release, previous, startedAt, transactionID) + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "cannot start transaction helper: "+err.Error(), s.nowTime() + s.writeState(base) + } +} + +func (s *server) nowTime() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +func newControlPlaneTransactionID(startedAt time.Time) string { + return fmt.Sprintf("%016x", uint64(startedAt.UnixNano())) +} + +func validControlPlaneTransactionID(transactionID string) bool { + return validHexValue(transactionID, len(transactionID), "") && len(transactionID) >= 16 && len(transactionID) <= 64 +} + +func controlPlaneHelperName(recovery bool, transactionID string) string { + prefix := "ftw-control-plane-" + if recovery { + prefix += "recovery-" + } + return prefix + transactionID +} + +func removeControlPlaneTransactionHelper(transactionID string, recovery bool) error { + if !validControlPlaneTransactionID(transactionID) { + return errors.New("invalid paired transaction ID") + } + name := controlPlaneHelperName(recovery, transactionID) + inspectCtx, cancelInspect := context.WithTimeout(context.Background(), 30*time.Second) + _, err := dockerOutput(inspectCtx, "container", "inspect", "--format", "{{.Id}}", name) + cancelInspect() + if err != nil { + if strings.Contains(err.Error(), "No such object") || strings.Contains(err.Error(), "No such container") { + return nil + } + return fmt.Errorf("inspect transaction helper %s: %w", name, err) + } + stopCtx, cancelStop := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelStop() + if err := dockerCompose(stopCtx, nil, "rm", "-f", name); err != nil { + return fmt.Errorf("stop transaction helper %s: %w", name, err) + } + return nil +} + +func (s *server) launchControlPlaneHelper(updaterContainer, helperImage string, recovery bool, release controlPlaneRelease, previous map[string]string, startedAt time.Time, transactionID string) error { + if !validControlPlaneTransactionID(transactionID) { + return errors.New("invalid paired transaction ID") + } + args := []string{ + "run", "--rm", "--detach", "--name", controlPlaneHelperName(recovery, transactionID), + "--network", "none", "--volumes-from", updaterContainer, + } + for _, pair := range []string{ + "FTW_UPDATER_COMPOSE=" + s.composeFile, + "FTW_UPDATER_MAIN_SERVICE=" + s.mainServiceName, + "FTW_UPDATER_SOCKET=" + s.socketPath, + "FTW_UPDATER_STATUS=" + s.statusPath, + "COMPOSE_PROJECT_NAME=" + os.Getenv("COMPOSE_PROJECT_NAME"), + } { + args = append(args, "--env", pair) + } + args = append(args, helperImage) + if recovery { + args = append(args, "-control-plane-recovery") + } + args = append(args, + "-control-plane-transaction", release.Target, + "-previous-core-image", previous["core"], + "-previous-updater-image", previous["updater"], + "-transaction-started-at", startedAt.Format(time.RFC3339Nano), + "-transaction-id", transactionID, + "-release-revision", release.Revision, + "-core-digest", release.CoreDigest, + "-updater-digest", release.UpdaterDigest, + ) + launchCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + return s.runner(launchCtx, nil, args...) +} + +func (s *server) monitorControlPlaneTransaction() { + poll := s.recoveryPoll + if poll <= 0 { + poll = controlPlaneRecoveryPoll + } + ticker := time.NewTicker(poll) + defer ticker.Stop() + for range ticker.C { + if s.recoverStaleControlPlaneTransaction() { + return + } + if st := s.readState(); !controlPlaneRecoveryInFlight(st) { + return + } + } +} + +// recoverStaleControlPlaneTransaction claims a transaction only after its +// durable heartbeat lease expires. It reads the lease again just before the +// claim so a current heartbeat can stop recovery. +func (s *server) recoverStaleControlPlaneTransaction() bool { + st := s.readState() + if !controlPlaneRecoveryInFlight(st) { + return true + } + ttl := s.transactionTTL + if ttl <= 0 { + ttl = controlPlaneStaleAfter + } + if st.UpdatedAt.IsZero() || s.nowTime().Sub(st.UpdatedAt) <= ttl { + return false + } + if !s.runMu.TryLock() { + return false + } + defer s.runMu.Unlock() + + st = s.readState() + if !controlPlaneRecoveryInFlight(st) { + return true + } + if st.UpdatedAt.IsZero() || s.nowTime().Sub(st.UpdatedAt) <= ttl { + return false + } + if st.PreviousImages["core"] == "" || st.PreviousImages["updater"] == "" { + st.State = "failed" + st.Message = "stale control-plane transaction has no complete rollback image pair; manual recovery required" + st.UpdatedAt = s.nowTime() + s.writeState(st) + return true + } + if !validControlPlaneTransactionID(st.TransactionID) { + st.State = "failed" + st.Message = "stale control-plane transaction has no valid transaction ID; manual recovery required" + st.UpdatedAt = s.nowTime() + s.writeState(st) + return true + } + + var recoveryHelper bool + switch st.HelperKind { + case controlPlaneHelperNormal: + recoveryHelper = false + case controlPlaneHelperRecovery: + recoveryHelper = true + default: + st.State = "failed" + st.Message = "stale control-plane transaction has no valid helper owner; manual recovery required" + st.UpdatedAt = s.nowTime() + s.writeState(st) + return true + } + if s.stopTransactionHelper == nil { + current := s.readState() + if current.TransactionID != st.TransactionID || current.HelperKind != st.HelperKind || !controlPlaneRecoveryInFlight(current) { + return true + } + current.State = "failed" + current.Message = "detached helper heartbeat expired but helper fencing is unavailable; manual recovery required" + current.UpdatedAt = s.nowTime() + s.writeState(current) + return true + } + stopErr := s.stopTransactionHelper(st.TransactionID, recoveryHelper) + // The helper may have written a terminal result while Docker stopped it. + // Keep that result. Otherwise create the recovery claim after the helper + // can no longer overwrite it. + current := s.readState() + if current.TransactionID != st.TransactionID || current.HelperKind != st.HelperKind || !controlPlaneRecoveryInFlight(current) { + return true + } + if stopErr != nil { + current.State = "failed" + current.Message = "detached helper heartbeat expired but it could not be stopped; manual recovery required: " + stopErr.Error() + current.UpdatedAt = s.nowTime() + s.writeState(current) + return true + } + current.State = "restoring" + current.HelperKind = controlPlaneHelperRecovery + current.Message = "detached helper stopped; launching updater-first pair recovery" + current.UpdatedAt = s.nowTime() + s.writeState(current) + var err error + if s.launchRecovery != nil { + err = s.launchRecovery(current) + } else { + err = s.launchControlPlaneRecoveryHelper(current) + } + if err != nil { + current.State = "failed" + current.Message = "detached helper heartbeat expired and recovery helper could not start; pair may be mixed: " + err.Error() + current.UpdatedAt = s.nowTime() + s.writeState(current) + } + return true +} + +func (s *server) launchControlPlaneRecoveryHelper(st State) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + updaterContainer, err := s.containerID(ctx, updaterServiceName) + cancel() + if err != nil { + return fmt.Errorf("locate updater for recovery: %w", err) + } + release := controlPlaneRelease{ + Target: st.Target, Revision: st.ReleaseRevision, + CoreDigest: st.CoreDigest, UpdaterDigest: st.UpdaterDigest, + } + return s.launchControlPlaneHelper(updaterContainer, st.PreviousImages["updater"], true, release, st.PreviousImages, st.StartedAt, st.TransactionID) +} + +func (s *server) runControlPlaneRecovery(st State) { + st.HelperKind = controlPlaneHelperRecovery + stopHeartbeat := s.startControlPlaneHeartbeat(st.Target, st.TransactionID, controlPlaneHelperRecovery, "restoring") + defer stopHeartbeat() + s.rollbackControlPlane(st, errors.New("detached transaction helper stopped heartbeating"), stopHeartbeat) +} + +func controlPlaneRecoveryInFlight(st State) bool { + return st.Action == "update" && st.Component == "core" && + (st.State == "transacting" || st.State == "restoring") +} + +func (s *server) failControlPlanePreflight(base State, err error) { + base.State = "failed" + base.Message = "compose preflight failed: " + err.Error() + base.UpdatedAt = s.nowTime() + s.writeState(base) +} + +func (s *server) runControlPlaneTransaction(release controlPlaneRelease, previous map[string]string, startedAt time.Time, transactionID string) { + stopHeartbeat := s.startControlPlaneHeartbeat(release.Target, transactionID, controlPlaneHelperNormal, "transacting", "restoring") + defer stopHeartbeat() + base := State{ + State: "transacting", Action: "update", Component: "core", Target: release.Target, + StartedAt: startedAt, UpdatedAt: s.nowTime(), PreviousImageID: previous["core"], PreviousImages: previous, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: transactionID, HelperKind: controlPlaneHelperNormal, + } + cleanup := func() {} + cleanupActive := false + fail := func(err error) { + if cleanupActive { + cleanup() + cleanupActive = false + } + s.rollbackControlPlane(base, err, stopHeartbeat) + } + if err := release.validate(); err != nil || previous["core"] == "" || previous["updater"] == "" || !validControlPlaneTransactionID(transactionID) { + if err == nil { + switch { + case previous["core"] == "" || previous["updater"] == "": + err = errors.New("previous Core and updater image IDs are required") + default: + err = errors.New("valid transaction ID is required") + } + } + fail(fmt.Errorf("invalid paired transaction parameters: %w", err)) + return + } + var err error + cleanup, err = s.prepareControlPlaneDigestPins(release) + if err != nil { + fail(fmt.Errorf("compose preflight: %w", err)) + return + } + cleanupActive = true + defer func() { + if cleanupActive { + cleanup() + } + }() + if err := s.validateControlPlaneDigestPins(release); err != nil { + fail(fmt.Errorf("compose preflight: %w", err)) + return + } + env := controlPlaneEnv(release.Target) + + base.Message = "replacing updater with matching release" + base.UpdatedAt = s.nowTime() + s.writeState(base) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + err = s.runner(ctx, env, s.composeArgs("up", "-d", updaterServiceName)...) + cancel() + if err != nil { + fail(fmt.Errorf("replace updater: %w", err)) + return + } + compatCtx, cancelCompat := context.WithTimeout(context.Background(), 3*time.Minute) + if s.updaterReady != nil { + err = s.updaterReady(compatCtx, release.Target) + } else { + err = s.waitForUpdaterRelease(compatCtx, release.Target) + } + cancelCompat() + if err != nil { + fail(err) + return + } + if err := s.requireServiceImageRef(updaterServiceName, release.updaterRef()); err != nil { + fail(err) + return + } + + base.Message = "matching updater ready; replacing Core" + base.UpdatedAt = s.nowTime() + s.writeState(base) + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) + err = s.runner(ctx, env, s.composeArgs("up", "-d", s.mainServiceName)...) + cancel() + if err != nil { + fail(fmt.Errorf("replace Core: %w", err)) + return + } + if s.healthCheck != nil { + healthCtx, cancelHealth := context.WithTimeout(context.Background(), componentHealthTimeout("core")) + err = s.healthCheck(healthCtx, s.mainServiceName) + cancelHealth() + if err != nil { + fail(fmt.Errorf("new Core readiness: %w", err)) + return + } + } + if err := s.requireServiceImageRef(s.mainServiceName, release.coreRef()); err != nil { + fail(err) + return + } + if err := s.requireServiceImageRef(updaterServiceName, release.updaterRef()); err != nil { + fail(err) + return + } + base.State = "done" + base.Message = "Core and updater committed as one digest-locked release pair" + base.UpdatedAt = s.nowTime() + stopHeartbeat() + s.writeState(base) +} + +func (s *server) waitForUpdaterRelease(ctx context.Context, target string) error { + for { + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", s.socketPath) + }}, + } + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/status", nil) + resp, err := client.Do(req) + if err == nil { + var status struct { + ProtocolVersion int `json:"protocol_version"` + UpdaterVersion string `json:"updater_version"` + Capabilities []string `json:"capabilities"` + } + decodeErr := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&status) + resp.Body.Close() + if resp.StatusCode == http.StatusOK && decodeErr == nil && + status.ProtocolVersion >= updaterProtocolVersion && status.UpdaterVersion == target { + for _, capability := range status.Capabilities { + if capability == controlPlaneCapability { + return nil + } + } + } + } + select { + case <-ctx.Done(): + return fmt.Errorf("matching updater %s did not become ready: %w", target, ctx.Err()) + case <-time.After(time.Second): + } + } +} + +func (s *server) requireServiceImageRef(service, want string) error { + if s.imageRef == nil { + return errors.New("image reference inspection is unavailable") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + got, err := s.imageRef(ctx, service) + cancel() + if err != nil { + return fmt.Errorf("inspect %s image reference: %w", service, err) + } + if got != want { + return fmt.Errorf("service %s runs %s, want %s", service, got, want) + } + return nil +} + +func (s *server) rollbackControlPlane(base State, cause error, stopHeartbeat func()) { + base.State = "restoring" + base.Message = "paired update failed; restoring previous updater and Core: " + cause.Error() + base.UpdatedAt = s.nowTime() + s.writeState(base) + cleanup, preflightErr := s.prepareControlPlaneTagPins() + if preflightErr == nil { + defer cleanup() + } + var failures []string + if preflightErr != nil { + failures = append(failures, "rollback preflight: "+preflightErr.Error()) + } else { + updaterSpec, _ := s.componentSpec("updater") + coreSpec, _ := s.componentSpec("core") + if err := s.restorePreviousComponentImage(base.PreviousImages["updater"], updaterSpec); err != nil { + failures = append(failures, "updater: "+err.Error()) + } + if err := s.restorePreviousComponentImage(base.PreviousImages["core"], coreSpec); err != nil { + failures = append(failures, "Core: "+err.Error()) + } + } + base.State = "failed" + base.UpdatedAt = s.nowTime() + if len(failures) == 0 { + base.Message = "paired update failed; previous Core/updater pair restored: " + cause.Error() + } else { + base.Message = "paired update failed and pair rollback was incomplete: " + cause.Error() + "; " + strings.Join(failures, "; ") + } + stopHeartbeat() + s.writeState(base) +} + +func (s *server) controlPlaneSpecs() ([]componentSpec, error) { + core, err := s.componentSpec("core") + if err != nil { + return nil, err + } + updater, err := s.componentSpec("updater") + if err != nil { + return nil, err + } + return []componentSpec{core, updater}, nil +} + +func (s *server) prepareControlPlaneDigestPins(release controlPlaneRelease) (func(), error) { + if err := release.validate(); err != nil { + return func() {}, err + } + return s.prepareControlPlaneOverride(map[string]string{ + s.mainServiceName: release.coreRef(), + updaterServiceName: release.updaterRef(), + }) +} + +func (s *server) validateControlPlaneDigestPins(release controlPlaneRelease) error { + return s.validateControlPlaneRefs(map[string]string{ + s.mainServiceName: release.coreRef(), + updaterServiceName: release.updaterRef(), + }) +} + +func (s *server) prepareControlPlaneTagPins() (func(), error) { + specs, err := s.controlPlaneSpecs() + if err != nil { + return func() {}, err + } + images := make(map[string]string, len(specs)) + for _, spec := range specs { + images[spec.service] = spec.image + ":${" + spec.tagVariable + ":-latest}" + } + return s.prepareControlPlaneOverride(images) +} + +func (s *server) prepareControlPlaneOverride(images map[string]string) (func(), error) { + if s.updateOverrideFile != "" { + return func() {}, errors.New("a control-plane image override is already active") + } + type imageService struct { + Image string `yaml:"image"` + } + doc := struct { + Services map[string]imageService `yaml:"services"` + }{Services: make(map[string]imageService, len(images))} + for service, image := range images { + if _, ok, err := serviceImageFromComposeFiles(s.composeFiles(), service); err != nil { + return func() {}, err + } else if !ok { + return func() {}, fmt.Errorf("service %q has no image", service) + } + doc.Services[service] = imageService{Image: image} + } + data, err := yaml.Marshal(doc) + if err != nil { + return func() {}, err + } + f, err := os.CreateTemp("", "ftw-control-plane-update-*.yml") + if err != nil { + return func() {}, err + } + path := f.Name() + remove := func() { + _ = f.Close() + _ = os.Remove(path) + } + if _, err := f.Write(data); err != nil { + remove() + return func() {}, err + } + if err := f.Sync(); err != nil { + remove() + return func() {}, err + } + if err := f.Close(); err != nil { + _ = os.Remove(path) + return func() {}, err + } + s.updateOverrideFile = path + return func() { + s.updateOverrideFile = "" + _ = os.Remove(path) + }, nil +} + +func (s *server) validateControlPlaneRefs(images map[string]string) error { + for service, want := range images { + got, ok, err := serviceImageFromComposeFiles(s.composeFiles(), service) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("service %q has no image", service) + } + if got != want { + return fmt.Errorf("service %q image %q does not match verified ref %q", service, got, want) + } + } + return nil +} + +func (s *server) currentServiceImageRef(ctx context.Context, service string) (string, error) { + containerID, err := s.serviceContainerID(ctx, service) + if err != nil { + return "", err + } + out, err := dockerOutput(ctx, "inspect", "--format", "{{.Config.Image}}", containerID) + if err != nil { + return "", err + } + ref := strings.TrimSpace(out) + if ref == "" { + return "", errors.New("running container has no image reference") + } + return ref, nil +} diff --git a/go/cmd/ftw-updater/control_plane_test.go b/go/cmd/ftw-updater/control_plane_test.go new file mode 100644 index 00000000..7ee43e6d --- /dev/null +++ b/go/cmd/ftw-updater/control_plane_test.go @@ -0,0 +1,383 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestStatusReportsControlPlaneContract(t *testing.T) { + oldVersion := Version + Version = "v1.10.0-beta.1" + t.Cleanup(func() { Version = oldVersion }) + s, _ := newTestServer(t) + + rr := httptest.NewRecorder() + s.handleStatus(rr, httptest.NewRequest(http.MethodGet, "/status", nil)) + var got struct { + ProtocolVersion int `json:"protocol_version"` + UpdaterVersion string `json:"updater_version"` + Capabilities []string `json:"capabilities"` + } + if err := json.NewDecoder(rr.Body).Decode(&got); err != nil { + t.Fatal(err) + } + if got.ProtocolVersion != updaterProtocolVersion || got.UpdaterVersion != Version || + len(got.Capabilities) != 1 || got.Capabilities[0] != controlPlaneCapability { + t.Fatalf("status contract = %+v", got) + } +} + +func TestTransactingStateRejectsAnotherUpdate(t *testing.T) { + s, runner := newTestServer(t) + s.writeState(State{State: "transacting", Action: "update", Component: "core", Target: "v1.2.3"}) + rr := httptest.NewRecorder() + s.handleUpdate(rr, httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.4")))) + if rr.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409", rr.Code) + } + if calls := runner.snapshot(); len(calls) != 0 { + t.Fatalf("second update ran Docker: %v", calls) + } +} + +func TestControlPlaneDoesNotCommitMixedImageRefs(t *testing.T) { + s, _ := newTestServer(t) + s.skipPull = true + s.imageID = func(_ context.Context, service string) (string, error) { + return "sha256:previous-" + service, nil + } + s.launchTransaction = func(release controlPlaneRelease, previous map[string]string, startedAt time.Time, transactionID string) error { + s.imageRef = func(_ context.Context, service string) (string, error) { + if service == updaterServiceName { + return release.updaterRef(), nil + } + return canonicalMainImage + ":v1.2.2", nil + } + s.runControlPlaneTransaction(release, previous, startedAt, transactionID) + return nil + } + + rr := httptest.NewRecorder() + s.handleUpdate(rr, httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3")))) + state := waitForState(t, s, "failed") + if !strings.Contains(state.Message, "previous Core/updater pair restored") || !strings.Contains(state.Message, "runs ghcr.io/srcfl/ftw:v1.2.2") { + t.Fatalf("mixed pair result = %+v", state) + } +} + +func TestCoreUpdateRequiresVerifiedDigests(t *testing.T) { + s, runner := newTestServer(t) + rr := httptest.NewRecorder() + s.handleUpdate(rr, httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`))) + if rr.Code != http.StatusBadRequest || !strings.Contains(rr.Body.String(), "invalid control-plane release") { + t.Fatalf("missing digests = %d %q, want 400", rr.Code, rr.Body.String()) + } + if calls := runner.snapshot(); len(calls) != 0 { + t.Fatalf("invalid request ran Docker: %v", calls) + } +} + +func TestControlPlaneOverridePinsVerifiedDigests(t *testing.T) { + s, _ := newTestServer(t) + release := testControlPlaneRelease("v1.2.3") + cleanup, err := s.prepareControlPlaneDigestPins(release) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := s.validateControlPlaneDigestPins(release); err != nil { + t.Fatal(err) + } + for service, want := range map[string]string{ + s.mainServiceName: release.coreRef(), updaterServiceName: release.updaterRef(), + } { + got, ok, err := serviceImageFromComposeFiles(s.composeFiles(), service) + if err != nil || !ok || got != want { + t.Fatalf("%s image = %q, %v, %v; want %q", service, got, ok, err, want) + } + } +} + +func TestControlPlaneHelperCarriesTransactionFenceAndDigests(t *testing.T) { + s, runner := newTestServer(t) + release := testControlPlaneRelease("v1.2.3") + previous := map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"} + startedAt := time.Date(2026, 7, 22, 8, 0, 0, 0, time.UTC) + if err := s.launchControlPlaneHelper("updater-container", previous["updater"], false, release, previous, startedAt, testControlPlaneTransactionID); err != nil { + t.Fatal(err) + } + calls := runner.snapshot() + if len(calls) != 1 { + t.Fatalf("helper calls = %v", calls) + } + call := strings.Join(calls[0], " ") + for _, want := range []string{ + "--name " + controlPlaneHelperName(false, testControlPlaneTransactionID), + previous["updater"], + "-transaction-id " + testControlPlaneTransactionID, + "-release-revision " + release.Revision, + "-core-digest " + release.CoreDigest, + "-updater-digest " + release.UpdaterDigest, + } { + if !strings.Contains(call, want) { + t.Fatalf("helper call %q does not contain %q", call, want) + } + } +} + +func TestControlPlaneRejectsMovedTagDigestAtExecution(t *testing.T) { + s, _ := newTestServer(t) + s.skipPull = true + s.imageID = func(_ context.Context, service string) (string, error) { + return "sha256:previous-" + service, nil + } + s.launchTransaction = func(release controlPlaneRelease, previous map[string]string, startedAt time.Time, transactionID string) error { + s.imageRef = func(_ context.Context, service string) (string, error) { + if service == updaterServiceName { + return release.updaterRef(), nil + } + return canonicalMainImage + "@sha256:" + strings.Repeat("c", 64), nil + } + s.runControlPlaneTransaction(release, previous, startedAt, transactionID) + return nil + } + + rr := httptest.NewRecorder() + s.handleUpdate(rr, httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3")))) + state := waitForState(t, s, "failed") + if !strings.Contains(state.Message, "previous Core/updater pair restored") || + !strings.Contains(state.Message, testControlPlaneRelease("v1.2.3").CoreDigest) { + t.Fatalf("moved-tag result = %+v", state) + } +} + +func TestFreshDetachedHelperLeaseIsNotRecovered(t *testing.T) { + s, runner := newTestServer(t) + now := time.Date(2026, 7, 22, 8, 0, 0, 0, time.UTC) + s.now = func() time.Time { return now } + release := testControlPlaneRelease("v1.2.3") + s.writeState(State{ + State: "transacting", Action: "update", Component: "core", Target: release.Target, + StartedAt: now.Add(-time.Minute), UpdatedAt: now, + PreviousImages: map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"}, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + launches := 0 + s.launchRecovery = func(State) error { launches++; return nil } + if monitor := s.recoverCrashedState(); !monitor { + t.Fatal("fresh detached-helper lease should start a monitor") + } + if launches != 0 || s.readState().State != "transacting" || len(runner.snapshot()) != 0 { + t.Fatalf("fresh helper was raced: launches=%d state=%+v calls=%v", launches, s.readState(), runner.snapshot()) + } +} + +func TestStoppedHelperHeartbeatRestoresUpdaterThenCore(t *testing.T) { + s, runner := newTestServer(t) + now := time.Date(2026, 7, 22, 8, 0, 0, 0, time.UTC) + s.now = func() time.Time { return now } + release := testControlPlaneRelease("v1.2.3") + previous := map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"} + s.writeState(State{ + State: "transacting", Action: "update", Component: "core", Target: release.Target, + StartedAt: now.Add(-time.Minute), UpdatedAt: now, PreviousImages: previous, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + s.launchRecovery = func(st State) error { + s.runControlPlaneRecovery(st) + return nil + } + if monitor := s.recoverCrashedState(); !monitor { + t.Fatal("fresh helper should be monitored before its lease expires") + } + now = now.Add(controlPlaneStaleAfter + time.Second) + if recovered := s.recoverStaleControlPlaneTransaction(); !recovered { + t.Fatal("stale helper was not recovered") + } + state := s.readState() + if state.State != "failed" || strings.Contains(state.State, "done") || !strings.Contains(state.Message, "previous Core/updater pair restored") { + t.Fatalf("recovery final state = %+v", state) + } + if state.PreviousImages["core"] != previous["core"] || state.PreviousImages["updater"] != previous["updater"] { + t.Fatalf("recovery lost rollback IDs: %v", state.PreviousImages) + } + calls := runner.snapshot() + if len(calls) != 5 || strings.Join(calls[0], " ") != "rm -f "+controlPlaneHelperName(false, testControlPlaneTransactionID) || + !strings.Contains(strings.Join(calls[1], " "), previous["updater"]) || + calls[2][len(calls[2])-1] != updaterServiceName || + !strings.Contains(strings.Join(calls[3], " "), previous["core"]) || + calls[4][len(calls[4])-1] != s.mainServiceName { + t.Fatalf("recovery was not updater-first: %v", calls) + } +} + +func TestHostRestartAfterUpdaterReplacementLaunchesRecoveryHelper(t *testing.T) { + s, _ := newTestServer(t) + now := time.Date(2026, 7, 22, 8, 10, 0, 0, time.UTC) + s.now = func() time.Time { return now } + release := testControlPlaneRelease("v1.2.3") + previous := map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"} + s.writeState(State{ + State: "transacting", Action: "update", Component: "core", Target: release.Target, + Message: "matching updater ready; replacing Core", StartedAt: now.Add(-10 * time.Minute), + UpdatedAt: now.Add(-controlPlaneStaleAfter - time.Second), PreviousImages: previous, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + var recovered State + s.launchRecovery = func(st State) error { recovered = st; return nil } + if monitor := s.recoverCrashedState(); monitor { + t.Fatal("stale host-restart state should recover before serving") + } + if recovered.State != "restoring" || recovered.PreviousImages["core"] != previous["core"] || recovered.PreviousImages["updater"] != previous["updater"] { + t.Fatalf("recovery helper input = %+v", recovered) + } + if state := s.readState(); state.State != "restoring" || strings.Contains(state.State, "done") { + t.Fatalf("host-restart recovery state = %+v", state) + } +} + +func TestStaleRestoringNormalHelperIsFencedBeforeRecovery(t *testing.T) { + s, runner := newTestServer(t) + now := time.Date(2026, 7, 22, 8, 20, 0, 0, time.UTC) + s.now = func() time.Time { return now } + release := testControlPlaneRelease("v1.2.3") + s.writeState(State{ + State: "restoring", Action: "update", Component: "core", Target: release.Target, + StartedAt: now.Add(-20 * time.Minute), UpdatedAt: now.Add(-controlPlaneStaleAfter - time.Second), + PreviousImages: map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"}, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + launches := 0 + s.launchRecovery = func(State) error { launches++; return nil } + if recovered := s.recoverStaleControlPlaneTransaction(); !recovered { + t.Fatal("stale normal rollback helper was not reclaimed") + } + calls := runner.snapshot() + if launches != 1 || len(calls) != 1 || strings.Join(calls[0], " ") != "rm -f "+controlPlaneHelperName(false, testControlPlaneTransactionID) { + t.Fatalf("normal rollback helper fence: launches=%d calls=%v", launches, calls) + } +} + +func TestStaleRestoringRecoveryHelperIsFencedBeforeRelaunch(t *testing.T) { + s, runner := newTestServer(t) + now := time.Date(2026, 7, 22, 8, 30, 0, 0, time.UTC) + s.now = func() time.Time { return now } + release := testControlPlaneRelease("v1.2.3") + s.writeState(State{ + State: "restoring", Action: "update", Component: "core", Target: release.Target, + StartedAt: now.Add(-20 * time.Minute), UpdatedAt: now.Add(-controlPlaneStaleAfter - time.Second), + PreviousImages: map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"}, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperRecovery, + }) + launches := 0 + s.launchRecovery = func(State) error { launches++; return nil } + if recovered := s.recoverStaleControlPlaneTransaction(); !recovered { + t.Fatal("stale recovery helper was not reclaimed") + } + calls := runner.snapshot() + if launches != 1 || len(calls) != 1 || strings.Join(calls[0], " ") != "rm -f "+controlPlaneHelperName(true, testControlPlaneTransactionID) { + t.Fatalf("recovery helper fence: launches=%d calls=%v", launches, calls) + } +} + +func TestRecoveryPreservesTerminalStateWrittenWhileHelperStops(t *testing.T) { + tests := []struct { + name string + terminalState string + stopErr error + }{ + {name: "done after successful stop", terminalState: "done"}, + {name: "failed before stop error", terminalState: "failed", stopErr: errors.New("helper already exited")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, _ := newTestServer(t) + now := time.Date(2026, 7, 22, 8, 40, 0, 0, time.UTC) + s.now = func() time.Time { return now } + release := testControlPlaneRelease("v1.2.3") + s.writeState(State{ + State: "restoring", Action: "update", Component: "core", Target: release.Target, + StartedAt: now.Add(-20 * time.Minute), UpdatedAt: now.Add(-controlPlaneStaleAfter - time.Second), + PreviousImages: map[string]string{"core": "sha256:core-old", "updater": "sha256:updater-old"}, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + s.stopTransactionHelper = func(string, bool) error { + terminal := s.readState() + terminal.State = tc.terminalState + terminal.Message = "helper terminal result" + terminal.UpdatedAt = now + s.writeState(terminal) + return tc.stopErr + } + launches := 0 + s.launchRecovery = func(State) error { launches++; return nil } + if recovered := s.recoverStaleControlPlaneTransaction(); !recovered { + t.Fatal("stale transaction was not handled") + } + state := s.readState() + if state.State != tc.terminalState || state.Message != "helper terminal result" || launches != 0 { + t.Fatalf("terminal result was overwritten: state=%+v launches=%d", state, launches) + } + }) + } +} + +func TestHeartbeatCannotOverwriteTerminalState(t *testing.T) { + for _, terminalState := range []string{"done", "failed"} { + t.Run(terminalState, func(t *testing.T) { + s, _ := newTestServer(t) + target := "v1.2.3" + s.writeState(State{ + State: "transacting", Action: "update", Component: "core", Target: target, + UpdatedAt: time.Now(), TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + afterRead := make(chan struct{}) + allowHeartbeatWrite := make(chan struct{}) + s.heartbeatAfterRead = func() { + close(afterRead) + <-allowHeartbeatWrite + } + heartbeatDone := make(chan struct{}) + go func() { + s.refreshControlPlaneHeartbeat(target, testControlPlaneTransactionID, controlPlaneHelperNormal, map[string]bool{"transacting": true}) + close(heartbeatDone) + }() + <-afterRead + + terminalStarted := make(chan struct{}) + terminalDone := make(chan struct{}) + go func() { + close(terminalStarted) + s.writeState(State{ + State: terminalState, Action: "update", Component: "core", Target: target, + UpdatedAt: time.Now(), TransactionID: testControlPlaneTransactionID, HelperKind: controlPlaneHelperNormal, + }) + close(terminalDone) + }() + <-terminalStarted + select { + case <-terminalDone: + t.Fatal("terminal write passed heartbeat while its read-modify-write lock was held") + default: + } + close(allowHeartbeatWrite) + <-heartbeatDone + <-terminalDone + if state := s.readState(); state.State != terminalState { + t.Fatalf("heartbeat restored stale in-flight state: %+v", state) + } + }) + } +} diff --git a/go/cmd/ftw-updater/main.go b/go/cmd/ftw-updater/main.go index 5c33d74c..c8e498de 100644 --- a/go/cmd/ftw-updater/main.go +++ b/go/cmd/ftw-updater/main.go @@ -40,14 +40,22 @@ const ( canonicalMainServiceName = "ftw" legacyMainServiceName = "forty-two-watts" canonicalMainImage = "ghcr.io/srcfl/ftw" + updaterServiceName = "ftw-updater" + canonicalUpdaterImage = "ghcr.io/srcfl/ftw-updater" optimizerServiceName = "ftw-optimizer" canonicalOptimizerImage = "ghcr.io/srcfl/ftw-optimizer" + updaterProtocolVersion = 2 + controlPlaneCapability = "control-plane-pair-v1" + controlPlaneStaleAfter = 5 * time.Minute + controlPlaneRecoveryPoll = 15 * time.Second ) +var Version = "dev" + // State mirrors selfupdate.UpdateStatus (we keep a local copy to avoid // importing the main module's internal package from this separate cmd). type State struct { - State string `json:"state"` // idle, pulling, restarting, restoring, done, failed + State string `json:"state"` // idle, pulling, transacting, restarting, restoring, done, failed Action string `json:"action,omitempty"` // update, restart, rollback (#152) Component string `json:"component,omitempty"` Target string `json:"target,omitempty"` @@ -60,6 +68,11 @@ type State struct { Message string `json:"message,omitempty"` PreviousImageID string `json:"previous_image_id,omitempty"` PreviousImages map[string]string `json:"previous_images,omitempty"` + ReleaseRevision string `json:"release_revision,omitempty"` + CoreDigest string `json:"core_digest,omitempty"` + UpdaterDigest string `json:"updater_digest,omitempty"` + TransactionID string `json:"transaction_id,omitempty"` + HelperKind string `json:"helper_kind,omitempty"` } type server struct { @@ -76,6 +89,7 @@ type server struct { // main image. It lets immutable updates migrate safely without modifying // the read-only host Compose file or changing its service/data layout. updateOverrideFile string + socketPath string statusPath string stateMu sync.Mutex // skipPull is a dev-only escape hatch: when true, the "pulling" phase @@ -92,6 +106,9 @@ type server struct { // after an arbitrary N. Tests that exercise the "always-fail" path set // this to a small value to avoid looping forever. maxPullAttempts int + now func() time.Time + transactionTTL time.Duration + recoveryPoll time.Duration // runMu ensures only one pull+up runs at a time. HTTP handlers that // arrive while a job is in flight return 409. @@ -104,12 +121,18 @@ type server struct { // imageID captures the image backing the running service before an update. // healthCheck waits for the recreated service to become healthy. Both are // injectable so the rollback path is testable without Docker. - imageID func(ctx context.Context, service string) (string, error) - containerID func(ctx context.Context, service string) (string, error) - healthCheck func(ctx context.Context, service string) error - chownFile func(string, int, int) error - checkSnapshotFile func(context.Context, string, string, string) error - stageSnapshotFile func(context.Context, string, string, string, string) error + imageID func(ctx context.Context, service string) (string, error) + imageRef func(ctx context.Context, service string) (string, error) + containerID func(ctx context.Context, service string) (string, error) + healthCheck func(ctx context.Context, service string) error + updaterReady func(ctx context.Context, target string) error + launchTransaction func(release controlPlaneRelease, previous map[string]string, startedAt time.Time, transactionID string) error + launchRecovery func(State) error + stopTransactionHelper func(string, bool) error + heartbeatAfterRead func() + chownFile func(string, int, int) error + checkSnapshotFile func(context.Context, string, string, string) error + stageSnapshotFile func(context.Context, string, string, string, string) error } // composeArgs returns the common prefix of every `docker compose` invocation @@ -217,6 +240,15 @@ func main() { compose := flag.String("compose", envOr("FTW_UPDATER_COMPOSE", "/compose/docker-compose.yml"), "Path to docker-compose.yml") mainService := flag.String("main-service", envOr("FTW_UPDATER_MAIN_SERVICE", ""), "Compose service for FTW (auto-detected when empty)") skipPull := flag.Bool("skip-pull", envOr("FTW_UPDATER_SKIP_PULL", "") != "", "Dev: skip docker compose pull (keeps local image)") + transactionTarget := flag.String("control-plane-transaction", "", "Internal: apply a paired Core/updater release") + transactionRecovery := flag.Bool("control-plane-recovery", false, "Internal: restore a stale paired transaction") + transactionCoreImage := flag.String("previous-core-image", "", "Internal: previous Core image ID") + transactionUpdaterImage := flag.String("previous-updater-image", "", "Internal: previous updater image ID") + transactionStartedAt := flag.String("transaction-started-at", "", "Internal: RFC3339 transaction start") + transactionID := flag.String("transaction-id", "", "Internal: durable paired transaction ID") + transactionRevision := flag.String("release-revision", "", "Internal: verified control-plane source revision") + transactionCoreDigest := flag.String("core-digest", "", "Internal: verified Core image digest") + transactionUpdaterDigest := flag.String("updater-digest", "", "Internal: verified updater image digest") flag.Parse() slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))) @@ -237,10 +269,14 @@ func main() { srv := &server{ composeFile: *compose, + socketPath: *socket, statusPath: *statusPath, skipPull: *skipPull, pullRetryDelay: 60 * time.Second, runner: dockerCompose, + now: time.Now, + transactionTTL: controlPlaneStaleAfter, + recoveryPoll: controlPlaneRecoveryPoll, } // Auto-discover override files alongside the base, the same way the // compose CLI does when invoked without -f. Without this the sidecar @@ -258,7 +294,9 @@ func main() { } srv.mainServiceName = selectedService srv.imageID = srv.currentServiceImageID + srv.imageRef = srv.currentServiceImageRef srv.containerID = srv.serviceContainerID + srv.stopTransactionHelper = removeControlPlaneTransactionHelper srv.healthCheck = srv.waitForServiceHealth srv.chownFile = os.Chown srv.checkSnapshotFile = func(ctx context.Context, containerID, snapshotID, file string) error { @@ -271,7 +309,33 @@ func main() { if *skipPull { slog.Warn("ftw-updater: skip-pull enabled — production deploys should leave this off") } - srv.recoverCrashedState() + if *transactionTarget != "" { + startedAt, err := time.Parse(time.RFC3339Nano, *transactionStartedAt) + if err != nil { + startedAt = time.Now() + } + release := controlPlaneRelease{ + Target: *transactionTarget, Revision: *transactionRevision, + CoreDigest: *transactionCoreDigest, UpdaterDigest: *transactionUpdaterDigest, + } + previous := map[string]string{ + "core": *transactionCoreImage, "updater": *transactionUpdaterImage, + } + if *transactionRecovery { + srv.runControlPlaneRecovery(State{ + State: "restoring", Action: "update", Component: "core", Target: release.Target, + StartedAt: startedAt, UpdatedAt: srv.nowTime(), PreviousImageID: previous["core"], PreviousImages: previous, + ReleaseRevision: release.Revision, CoreDigest: release.CoreDigest, UpdaterDigest: release.UpdaterDigest, + TransactionID: *transactionID, HelperKind: controlPlaneHelperRecovery, + }) + } else { + srv.runControlPlaneTransaction(release, previous, startedAt, *transactionID) + } + return + } + if srv.recoverCrashedState() { + go srv.monitorControlPlaneTransaction() + } mux := http.NewServeMux() mux.HandleFunc("POST /update", srv.handleUpdate) @@ -307,14 +371,17 @@ func main() { func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { var body struct { - Action string `json:"action"` - Component string `json:"component,omitempty"` - Target string `json:"target,omitempty"` - Snapshot string `json:"snapshot,omitempty"` // rollback-only (#152) - Files []string `json:"files,omitempty"` // rollback: basenames to restore - SafetySnapshot string `json:"safety_snapshot,omitempty"` - SafetyFiles []string `json:"safety_files,omitempty"` - StartedAt time.Time `json:"started_at,omitempty"` + Action string `json:"action"` + Component string `json:"component,omitempty"` + Target string `json:"target,omitempty"` + Snapshot string `json:"snapshot,omitempty"` // rollback-only (#152) + Files []string `json:"files,omitempty"` // rollback: basenames to restore + SafetySnapshot string `json:"safety_snapshot,omitempty"` + SafetyFiles []string `json:"safety_files,omitempty"` + StartedAt time.Time `json:"started_at,omitempty"` + ReleaseRevision string `json:"release_revision,omitempty"` + CoreDigest string `json:"core_digest,omitempty"` + UpdaterDigest string `json:"updater_digest,omitempty"` } if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<12)).Decode(&body); err != nil { http.Error(w, "bad json: "+err.Error(), 400) @@ -345,6 +412,16 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { http.Error(w, "target must be stable vX.Y.Z or beta vX.Y.Z-beta.N", 400) return } + if body.Component == "core" { + release := controlPlaneRelease{ + Target: body.Target, Revision: body.ReleaseRevision, + CoreDigest: body.CoreDigest, UpdaterDigest: body.UpdaterDigest, + } + if err := release.validate(); err != nil { + http.Error(w, "invalid control-plane release: "+err.Error(), http.StatusBadRequest) + return + } + } case "restart": // target optional — when empty, compose's `${FTW_IMAGE_TAG:-latest}` // substitution falls through to :latest. That's the dev path for @@ -391,6 +468,10 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { http.Error(w, "action must be update, restart, rollback, or component_rollback", 400) return } + if st := s.readState(); updaterStateInFlight(st.State) { + http.Error(w, "control-plane transaction already in progress", http.StatusConflict) + return + } if !s.runMu.TryLock() { http.Error(w, "update already in progress", 409) return @@ -405,7 +486,10 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { } else if body.Action == "component_rollback" { s.runComponentRollback(body.Component, body.StartedAt) } else { - s.runComponentJob(body.Action, body.Target, body.Component, body.StartedAt) + s.runComponentJob(body.Action, body.Target, body.Component, body.StartedAt, controlPlaneRelease{ + Target: body.Target, Revision: body.ReleaseRevision, + CoreDigest: body.CoreDigest, UpdaterDigest: body.UpdaterDigest, + }) } }() @@ -422,7 +506,15 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { func (s *server) handleStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") st := s.readState() - _ = json.NewEncoder(w).Encode(st) + _ = json.NewEncoder(w).Encode(struct { + State + ProtocolVersion int `json:"protocol_version"` + UpdaterVersion string `json:"updater_version"` + Capabilities []string `json:"capabilities"` + }{ + State: st, ProtocolVersion: updaterProtocolVersion, UpdaterVersion: Version, + Capabilities: []string{controlPlaneCapability}, + }) } // runJob executes a pull+up (or pull+up --force-recreate) sequence, @@ -438,7 +530,7 @@ func (s *server) runJob(action, target string) { s.runComponentJob(action, target, "core", time.Time{}) } -func (s *server) runComponentJob(action, target, component string, startedAt time.Time) { +func (s *server) runComponentJob(action, target, component string, startedAt time.Time, releases ...controlPlaneRelease) { now := startedAt if now.IsZero() { now = time.Now() @@ -455,6 +547,12 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim slog.Error("core update blocked", "err", err) return } + var release controlPlaneRelease + if len(releases) > 0 { + release = releases[0] + } + s.runControlPlaneUpdate(release, now) + return } s.writeState(State{State: "pulling", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: now}) @@ -627,6 +725,13 @@ func (s *server) componentSpec(component string) (componentSpec, error) { return componentSpec{}, fmt.Errorf("compose service %q is unavailable", optimizerServiceName) } return componentSpec{name: "optimizer", service: optimizerServiceName, image: canonicalOptimizerImage, tagEnv: "FTW_OPTIMIZER_IMAGE_TAG", tagVariable: "FTW_OPTIMIZER_IMAGE_TAG"}, nil + case "updater": + if _, ok, err := serviceImageFromComposeFiles(s.composeFiles(), updaterServiceName); err != nil { + return componentSpec{}, err + } else if !ok { + return componentSpec{}, fmt.Errorf("compose service %q is unavailable", updaterServiceName) + } + return componentSpec{name: "updater", service: updaterServiceName, image: canonicalUpdaterImage, tagEnv: "FTW_UPDATER_IMAGE_TAG", tagVariable: "FTW_UPDATER_IMAGE_TAG"}, nil default: return componentSpec{}, fmt.Errorf("unsupported component %q", component) } @@ -1063,6 +1168,11 @@ func (s *server) recoverRollbackSafety(ctx context.Context, base State, safetySn func (s *server) writeState(st State) { s.stateMu.Lock() defer s.stateMu.Unlock() + s.writeStateLocked(st) +} + +// writeStateLocked writes state while the caller holds stateMu. +func (s *server) writeStateLocked(st State) { if st.UpdatedAt.IsZero() { st.UpdatedAt = time.Now() } @@ -1134,6 +1244,11 @@ func (s *server) writeState(st State) { func (s *server) readState() State { s.stateMu.Lock() defer s.stateMu.Unlock() + return s.readStateLocked() +} + +// readStateLocked reads state while the caller holds stateMu. +func (s *server) readStateLocked() State { data, err := os.ReadFile(s.statusPath) if err != nil { return State{State: "idle"} @@ -1158,14 +1273,17 @@ func (s *server) previousImageID(component string) string { return "" } -// recoverCrashedState runs once at boot. Any in-flight state belongs to the -// updater process that just died; waiting for a stale timeout would leave the -// UI stuck forever. Rollback carries its safety backup in state.json, so it -// can restore the pre-rollback data before reporting failure. -func (s *server) recoverCrashedState() { +// recoverCrashedState runs once at boot. A fresh paired transaction belongs to +// the detached helper that intentionally replaced this updater, so the caller +// starts a lease monitor instead of racing it. Other interrupted jobs can fail +// immediately because no separate process owns them. +func (s *server) recoverCrashedState() bool { st := s.readState() + if controlPlaneRecoveryInFlight(st) { + return !s.recoverStaleControlPlaneTransaction() + } if st.State != "pulling" && st.State != "restarting" && st.State != "restoring" { - return + return false } prev := st.State if st.Action == "rollback" && validSnapshotID(st.SafetySnapshot) && hasRollbackState(st.SafetyFiles) && rollbackFilesValid(st.SafetyFiles) && s.containerID != nil && s.imageID != nil { @@ -1176,7 +1294,7 @@ func (s *server) recoverCrashedState() { if containerErr == nil && imageErr == nil { s.recoverRollbackSafety(ctx, st, st.SafetySnapshot, st.SafetyFiles, containerID, imageID, "updater restarted during rollback") slog.Warn("recovered interrupted rollback from safety backup", "prev_state", prev) - return + return false } st.Message = fmt.Sprintf("updater restarted during rollback; safety recovery unavailable: container=%v image=%v", containerErr, imageErr) } else if st.Message == "" { @@ -1186,6 +1304,16 @@ func (s *server) recoverCrashedState() { st.UpdatedAt = time.Now() s.writeState(st) slog.Warn("recovered in-flight state as failed", "prev_state", prev) + return false +} + +func updaterStateInFlight(state string) bool { + switch state { + case "pulling", "transacting", "restarting", "restoring": + return true + default: + return false + } } func rollbackFilesValid(files []string) bool { diff --git a/go/cmd/ftw-updater/main_test.go b/go/cmd/ftw-updater/main_test.go index bb6d7cfc..283fe358 100644 --- a/go/cmd/ftw-updater/main_test.go +++ b/go/cmd/ftw-updater/main_test.go @@ -28,6 +28,26 @@ type fakeRunner struct { failOn string } +const testControlPlaneTransactionID = "0123456789abcdef" + +func testControlPlaneRelease(target string) controlPlaneRelease { + return controlPlaneRelease{ + Target: target, Revision: "0123456789abcdef0123456789abcdef01234567", + CoreDigest: "sha256:" + strings.Repeat("a", 64), + UpdaterDigest: "sha256:" + strings.Repeat("b", 64), + } +} + +func controlPlaneUpdateJSON(target string) string { + release := testControlPlaneRelease(target) + body, _ := json.Marshal(map[string]string{ + "action": "update", "target": release.Target, + "release_revision": release.Revision, "core_digest": release.CoreDigest, + "updater_digest": release.UpdaterDigest, + }) + return string(body) +} + func (f *fakeRunner) run(ctx context.Context, env []string, args ...string) error { f.mu.Lock() defer f.mu.Unlock() @@ -69,13 +89,45 @@ func newTestServer(t *testing.T) (*server, *fakeRunner) { s := &server{ composeFile: filepath.Join(dir, "docker-compose.yml"), mainServiceName: canonicalMainServiceName, + socketPath: filepath.Join(dir, "updater.sock"), statusPath: filepath.Join(dir, "state.json"), pullRetryDelay: time.Millisecond, + now: time.Now, + transactionTTL: controlPlaneStaleAfter, + recoveryPoll: time.Hour, runner: runner.run, healthCheck: func(context.Context, string) error { return nil }, imageID: func(context.Context, string) (string, error) { return "sha256:current", nil }, - containerID: func(context.Context, string) (string, error) { return "ftw-container", nil }, - chownFile: func(string, int, int) error { return nil }, + imageRef: func(_ context.Context, service string) (string, error) { + switch service { + case canonicalMainServiceName, legacyMainServiceName: + return canonicalMainImage + ":v1.2.3", nil + case updaterServiceName: + return canonicalUpdaterImage + ":v1.2.3", nil + default: + return "", fmt.Errorf("unknown service %q", service) + } + }, + containerID: func(context.Context, string) (string, error) { return "ftw-container", nil }, + updaterReady: func(context.Context, string) error { return nil }, + chownFile: func(string, int, int) error { return nil }, + } + s.stopTransactionHelper = func(transactionID string, recovery bool) error { + return s.runner(context.Background(), nil, "rm", "-f", controlPlaneHelperName(recovery, transactionID)) + } + s.launchTransaction = func(release controlPlaneRelease, previous map[string]string, startedAt time.Time, transactionID string) error { + s.imageRef = func(_ context.Context, service string) (string, error) { + switch service { + case canonicalMainServiceName, legacyMainServiceName: + return release.coreRef(), nil + case updaterServiceName: + return release.updaterRef(), nil + default: + return "", fmt.Errorf("unknown service %q", service) + } + } + s.runControlPlaneTransaction(release, previous, startedAt, transactionID) + return nil } s.checkSnapshotFile = func(_ context.Context, _ string, snapshotID, file string) error { _, err := os.Stat(filepath.Join(dir, "data", "snapshots", snapshotID, file)) @@ -95,6 +147,8 @@ func newTestServer(t *testing.T) (*server, *fakeRunner) { - ./data:/app/data ftw-optimizer: image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} + ftw-updater: + image: ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest} `) return s, runner } @@ -110,16 +164,17 @@ func TestSkipPull_BypassesPullStep(t *testing.T) { s, runner := newTestServer(t) s.skipPull = true - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) waitForState(t, s, "done") calls := runner.snapshot() - if len(calls) != 1 { - t.Fatalf("skip-pull should yield 1 call (up only), got %d: %v", len(calls), calls) + if len(calls) != 2 { + t.Fatalf("skip-pull should replace updater then Core, got %d calls: %v", len(calls), calls) } - if !strings.Contains(strings.Join(calls[0], " "), "up -d") { - t.Errorf("single call should be `up -d`: %v", calls[0]) + if !strings.Contains(strings.Join(calls[0], " "), "up -d "+updaterServiceName) || + !strings.Contains(strings.Join(calls[1], " "), "up -d "+canonicalMainServiceName) { + t.Errorf("pair replacement order is wrong: %v", calls) } } @@ -193,7 +248,7 @@ func waitForState(t *testing.T, s *server, want string) State { func TestHandleUpdate_HappyPath(t *testing.T) { s, runner := newTestServer(t) - body := bytes.NewBufferString(`{"action":"update","target":"v1.2.3"}`) + body := bytes.NewBufferString(controlPlaneUpdateJSON("v1.2.3")) req := httptest.NewRequest(http.MethodPost, "/update", body) rr := httptest.NewRecorder() s.handleUpdate(rr, req) @@ -205,15 +260,17 @@ func TestHandleUpdate_HappyPath(t *testing.T) { t.Errorf("unexpected final state: %+v", st) } calls := runner.snapshot() - if len(calls) != 2 { - t.Fatalf("expected 2 docker calls, got %d: %v", len(calls), calls) + if len(calls) != 3 { + t.Fatalf("expected pull plus paired replacement, got %d calls: %v", len(calls), calls) } if calls[0][0] != "compose" || !strings.Contains(strings.Join(calls[0], " "), "pull") { t.Errorf("first call should be pull: %v", calls[0]) } - up := strings.Join(calls[1], " ") - if !strings.Contains(up, "up -d") || strings.Contains(up, "--force-recreate") { - t.Errorf("update path should NOT force-recreate: %v", calls[1]) + if up := strings.Join(calls[1], " "); !strings.Contains(up, "up -d "+updaterServiceName) || strings.Contains(up, "--force-recreate") { + t.Errorf("updater replacement is wrong: %v", calls[1]) + } + if up := strings.Join(calls[2], " "); !strings.Contains(up, "up -d "+canonicalMainServiceName) || strings.Contains(up, "--force-recreate") { + t.Errorf("Core replacement is wrong: %v", calls[2]) } } @@ -226,7 +283,7 @@ func TestHandleUpdate_BlocksCoreUpdateWithoutOptimizer(t *testing.T) { - ./data:/app/data `) - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) if rr.Code != http.StatusAccepted { @@ -252,7 +309,7 @@ func TestHandleUpdate_BlocksCoreUpdateWhenOptimizerIsUnhealthy(t *testing.T) { return nil } - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) state := waitForState(t, s, "failed") @@ -280,7 +337,7 @@ func TestHandleUpdate_MissingOptimizerLeavesUserOverrideUntouched(t *testing.T) } s.overrideFiles = []string{override} - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) waitForState(t, s, "failed") @@ -313,7 +370,7 @@ func TestHandleUpdate_PullFailure(t *testing.T) { runner.fail = true s.maxPullAttempts = 3 // cap retries so the always-fail runner doesn't loop forever - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) st := waitForState(t, s, "failed") @@ -326,7 +383,7 @@ func TestHandleUpdate_UpFailure(t *testing.T) { s, runner := newTestServer(t) runner.failOn = "up" - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) st := waitForState(t, s, "failed") @@ -388,7 +445,7 @@ func TestImmutableImageTagChannels(t *testing.T) { // the exact image and is immune to the :latest-retag race. func TestHandleUpdate_PinsImageTagViaEnv(t *testing.T) { s, runner := newTestServer(t) - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v0.44.0"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v0.44.0"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) if rr.Code != 202 { @@ -396,19 +453,21 @@ func TestHandleUpdate_PinsImageTagViaEnv(t *testing.T) { } waitForState(t, s, "done") envs := runner.envSnapshot() - if len(envs) != 2 { - t.Fatalf("expected 2 docker calls (pull, up); got %d", len(envs)) + if len(envs) != 3 { + t.Fatalf("expected pull plus paired replacement; got %d calls", len(envs)) } for i, env := range envs { - found := false + foundCore, foundUpdater := false, false for _, e := range env { if e == "FTW_IMAGE_TAG=v0.44.0" { - found = true - break + foundCore = true + } + if e == "FTW_UPDATER_IMAGE_TAG=v0.44.0" { + foundUpdater = true } } - if !found { - t.Errorf("call %d missing FTW_IMAGE_TAG=v0.44.0; env=%v", i, env) + if !foundCore || !foundUpdater { + t.Errorf("call %d missing paired tag pins; env=%v", i, env) } } } @@ -422,10 +481,12 @@ func TestHandleUpdate_MigratesHardcodedImageWithTransientOverride(t *testing.T) - ./data:/app/data ftw-optimizer: image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} + ftw-updater: + image: ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest} `) s.mainServiceName = legacyMainServiceName - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v0.44.0"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v0.44.0"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) if rr.Code != 202 { @@ -433,17 +494,17 @@ func TestHandleUpdate_MigratesHardcodedImageWithTransientOverride(t *testing.T) } waitForState(t, s, "done") calls := runner.snapshot() - if len(calls) != 2 { - t.Fatalf("want pull + up, got %v", calls) + if len(calls) != 3 { + t.Fatalf("want pair pull and updater-first replacement, got %v", calls) } for _, call := range calls { joined := strings.Join(call, " ") - if !strings.Contains(joined, "-f "+s.composeFile+" -f ") || !strings.Contains(joined, "ftw-compose-update-") { + if !strings.Contains(joined, "-f "+s.composeFile+" -f ") || !strings.Contains(joined, "ftw-control-plane-update-") { t.Fatalf("legacy update must append compatibility override after base file: %v", call) } - if call[len(call)-1] != legacyMainServiceName { - t.Fatalf("legacy service identity must be preserved: %v", call) - } + } + if calls[1][len(calls[1])-1] != updaterServiceName || calls[2][len(calls[2])-1] != legacyMainServiceName { + t.Fatalf("pair replacement order or legacy service changed: %v", calls) } } @@ -456,6 +517,8 @@ func TestHandleUpdate_RestartMigratesHardcodedImageWithTransientOverride(t *test - ./data:/app/data ftw-optimizer: image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} + ftw-updater: + image: ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest} `) s.mainServiceName = legacyMainServiceName @@ -750,7 +813,7 @@ func TestHandleUpdate_ConcurrentRejected(t *testing.T) { return nil } - req1 := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req1 := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr1 := httptest.NewRecorder() s.handleUpdate(rr1, req1) if rr1.Code != 202 { @@ -840,9 +903,16 @@ func TestUpdateHealthFailureRestoresPreviousImage(t *testing.T) { - ./data:/app/data ftw-optimizer: image: ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest} + ftw-updater: + image: ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest} `) s.mainServiceName = legacyMainServiceName - s.imageID = func(context.Context, string) (string, error) { return "sha256:previous", nil } + s.imageID = func(_ context.Context, service string) (string, error) { + if service == updaterServiceName { + return "sha256:updater-previous", nil + } + return "sha256:core-previous", nil + } checks := 0 s.healthCheck = func(_ context.Context, service string) error { if service == optimizerServiceName { @@ -855,25 +925,31 @@ func TestUpdateHealthFailureRestoresPreviousImage(t *testing.T) { return nil } - req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(controlPlaneUpdateJSON("v1.2.3"))) rr := httptest.NewRecorder() s.handleUpdate(rr, req) st := waitForState(t, s, "failed") - if !strings.Contains(st.Message, "previous image restored") { - t.Fatalf("state should report automatic rollback, got %+v", st) + if !strings.Contains(st.Message, "previous Core/updater pair restored") { + t.Fatalf("state should report pair rollback, got %+v", st) } calls := runner.snapshot() - if len(calls) != 4 { - t.Fatalf("want pull, new up, image tag, rollback up; got %v", calls) + if len(calls) != 7 { + t.Fatalf("want pair pull, two replacements, then updater-first pair rollback; got %v", calls) + } + if got := strings.Join(calls[3], " "); !strings.Contains(got, "image tag sha256:updater-previous") { + t.Fatalf("rollback must tag updater first, got %q", got) + } + if calls[4][len(calls[4])-1] != updaterServiceName { + t.Fatalf("rollback must recreate updater first, got %v", calls[4]) } - if got := strings.Join(calls[2], " "); !strings.Contains(got, "image tag sha256:previous") { - t.Fatalf("third call should tag previous image, got %q", got) + if got := strings.Join(calls[5], " "); !strings.Contains(got, "image tag sha256:core-previous") || !strings.Contains(got, canonicalMainImage+":ftw-rollback-") { + t.Fatalf("rollback must tag the previous Core image, got %q", got) } - if got := strings.Join(calls[2], " "); !strings.Contains(got, canonicalMainImage+":ftw-rollback-") { - t.Fatalf("previous legacy image should be retagged into canonical repository, got %q", got) + if calls[6][len(calls[6])-1] != legacyMainServiceName { + t.Fatalf("rollback must preserve the legacy Core service: %v", calls[6]) } - if got := strings.Join(calls[3], " "); !strings.Contains(got, "ftw-compose-update-") || calls[3][len(calls[3])-1] != legacyMainServiceName { - t.Fatalf("rollback must reuse transient pin and legacy service identity, got %q", got) + if st.PreviousImages["core"] != "sha256:core-previous" || st.PreviousImages["updater"] != "sha256:updater-previous" { + t.Fatalf("rollback history = %v", st.PreviousImages) } } diff --git a/go/cmd/ftw/boothealth_test.go b/go/cmd/ftw/boothealth_test.go new file mode 100644 index 00000000..3d865836 --- /dev/null +++ b/go/cmd/ftw/boothealth_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestBootPhaseDoesNotReportReadiness(t *testing.T) { + handler := bootPhaseHandler() + + health := httptest.NewRecorder() + handler.ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/api/health", nil)) + if health.Code != http.StatusOK || health.Body.String() != `{"status":"starting","phase":"initializing state"}` { + t.Fatalf("boot liveness = %d %s", health.Code, health.Body.String()) + } + + status := httptest.NewRecorder() + handler.ServeHTTP(status, httptest.NewRequest(http.MethodGet, "/api/status", nil)) + if status.Code != http.StatusServiceUnavailable { + t.Fatalf("boot readiness = %d, want 503", status.Code) + } + if status.Header().Get("Retry-After") == "" { + t.Fatal("boot readiness lacks Retry-After") + } +} diff --git a/go/cmd/ftw/bootstrap.go b/go/cmd/ftw/bootstrap.go index bae59789..f9704b61 100644 --- a/go/cmd/ftw/bootstrap.go +++ b/go/cmd/ftw/bootstrap.go @@ -34,9 +34,12 @@ func runBootstrap(configPath, webDir, driverDir string) { "env", "FTW_SELFUPDATE_CURRENT_VERSION") } selfUpdater = selfupdate.New(selfupdate.Config{ - CurrentVersion: current, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), + CurrentVersion: current, + RequiredUpdaterVersion: Version, + PairedImage: "srcfl/ftw-updater", + PairManifestAsset: "ftw-control-plane.json", + SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), + StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), }, nil) selfUpdater.Start(context.Background()) slog.Info("selfupdate enabled in bootstrap", "socket", envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock")) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 34ba779d..022510e2 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -181,6 +181,26 @@ func main() { } }() + // Core and updater share one release. Check the updater before state.Open + // so a legacy updater cannot start a new Core alone and let it migrate data. + if envBool("FTW_SELFUPDATE_ENABLED") && Version != "dev" { + socketPath := envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock") + var lastLog time.Time + for { + probeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := selfupdate.RequireUpdaterRelease(probeCtx, socketPath, Version) + cancel() + if err == nil { + break + } + if time.Since(lastLog) >= 30*time.Second { + slog.Warn("waiting for matching updater", "core_version", Version, "err", err) + lastLog = time.Now() + } + time.Sleep(2 * time.Second) + } + } + st, err := state.Open(statePath) if err != nil { slog.Error("open state", "err", err) @@ -1811,9 +1831,12 @@ func main() { "env", "FTW_SELFUPDATE_CURRENT_VERSION") } selfUpdater = selfupdate.New(selfupdate.Config{ - CurrentVersion: current, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), + CurrentVersion: current, + RequiredUpdaterVersion: Version, + PairedImage: "srcfl/ftw-updater", + PairManifestAsset: "ftw-control-plane.json", + SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), + StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), // Publish events.UpdateAvailable when a new release lands so // the notifications service (or any other subscriber) can act // without polling the checker directly. @@ -1835,9 +1858,10 @@ func main() { optimizerUpdater = selfupdate.New(selfupdate.Config{ Repo: "srcfl/ftw", Image: "srcfl/ftw-optimizer", ReleaseTagPrefix: "optimizer-", StoragePrefix: "optimizer.", - CurrentVersion: optimizerCurrent, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), + CurrentVersion: optimizerCurrent, + RequiredUpdaterVersion: Version, + SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), + StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), }, st) optimizerUpdater.Start(ctx) slog.Info("selfupdate enabled", diff --git a/go/internal/api/api_selfupdate_test.go b/go/internal/api/api_selfupdate_test.go index 7a2e6c46..cd9f7a13 100644 --- a/go/internal/api/api_selfupdate_test.go +++ b/go/internal/api/api_selfupdate_test.go @@ -102,6 +102,13 @@ func startFakeSidecar(t *testing.T, statusCode int) string { t.Fatalf("listen fake sidecar: %v", err) } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/status" { + _ = json.NewEncoder(w).Encode(map[string]any{ + "state": "idle", "protocol_version": 2, "updater_version": "v1.9.1", + "capabilities": []string{"control-plane-pair-v1"}, + }) + return + } w.WriteHeader(statusCode) _, _ = w.Write([]byte(`{"status":"stub"}`)) })} diff --git a/go/internal/selfupdate/control_plane_release.go b/go/internal/selfupdate/control_plane_release.go new file mode 100644 index 00000000..1efffbf2 --- /dev/null +++ b/go/internal/selfupdate/control_plane_release.go @@ -0,0 +1,140 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strings" +) + +const controlPlaneManifestSchema = 1 + +var ( + sha256DigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + gitRevisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) +) + +type controlPlaneReleaseManifest struct { + SchemaVersion int `json:"schema_version"` + Release string `json:"release"` + Revision string `json:"revision"` + Components struct { + Core controlPlaneReleaseComponent `json:"core"` + Updater controlPlaneReleaseComponent `json:"updater"` + } `json:"components"` +} + +type controlPlaneReleaseComponent struct { + Image string `json:"image"` + Digest string `json:"digest"` +} + +type verifiedControlPlaneRelease struct { + Target string + Revision string + CoreDigest string + UpdaterDigest string +} + +func (c *Checker) verifyControlPlaneRelease(ctx context.Context, rel ghRelease, targetTag string) (verifiedControlPlaneRelease, bool, error) { + assetURL := "" + for _, asset := range rel.Assets { + if asset.Name == c.cfg.PairManifestAsset { + assetURL = asset.BrowserDownloadURL + break + } + } + if assetURL == "" { + return verifiedControlPlaneRelease{}, false, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, assetURL, nil) + if err != nil { + return verifiedControlPlaneRelease{}, false, err + } + req.Header.Set("Accept", "application/octet-stream") + req.Header.Set("User-Agent", "FTW-selfupdate") + resp, err := c.cfg.HTTPClient.Do(req) + if err != nil { + return verifiedControlPlaneRelease{}, false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return verifiedControlPlaneRelease{}, false, nil + } + if resp.StatusCode >= 400 { + return verifiedControlPlaneRelease{}, false, fmt.Errorf("pair manifest HTTP %d", resp.StatusCode) + } + var manifest controlPlaneReleaseManifest + decoder := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil { + return verifiedControlPlaneRelease{}, false, fmt.Errorf("decode pair manifest: %w", err) + } + if err := c.validateControlPlaneManifest(manifest, targetTag); err != nil { + return verifiedControlPlaneRelease{}, false, err + } + + coreProbe := ®istryProbe{httpClient: c.cfg.HTTPClient, base: c.cfg.RegistryBaseURL, repo: c.cfg.Image, service: c.cfg.RegistryService} + updaterProbe := ®istryProbe{httpClient: c.cfg.HTTPClient, base: c.cfg.RegistryBaseURL, repo: c.cfg.PairedImage, service: c.cfg.RegistryService} + coreDigest, err := coreProbe.manifestDigest(ctx, targetTag) + if err != nil { + return verifiedControlPlaneRelease{}, false, fmt.Errorf("Core digest: %w", err) + } + updaterDigest, err := updaterProbe.manifestDigest(ctx, targetTag) + if err != nil { + return verifiedControlPlaneRelease{}, false, fmt.Errorf("updater digest: %w", err) + } + if coreDigest == "" || updaterDigest == "" { + return verifiedControlPlaneRelease{}, false, nil + } + if coreDigest != manifest.Components.Core.Digest { + return verifiedControlPlaneRelease{}, false, fmt.Errorf("Core tag digest %s does not match release manifest %s", coreDigest, manifest.Components.Core.Digest) + } + if updaterDigest != manifest.Components.Updater.Digest { + return verifiedControlPlaneRelease{}, false, fmt.Errorf("updater tag digest %s does not match release manifest %s", updaterDigest, manifest.Components.Updater.Digest) + } + return verifiedControlPlaneRelease{ + Target: targetTag, Revision: manifest.Revision, + CoreDigest: coreDigest, UpdaterDigest: updaterDigest, + }, true, nil +} + +func (c *Checker) validateControlPlaneManifest(manifest controlPlaneReleaseManifest, targetTag string) error { + if manifest.SchemaVersion != controlPlaneManifestSchema { + return fmt.Errorf("unsupported pair manifest schema %d", manifest.SchemaVersion) + } + if manifest.Release != targetTag { + return fmt.Errorf("pair manifest release %q does not match %q", manifest.Release, targetTag) + } + if !gitRevisionPattern.MatchString(manifest.Revision) { + return errors.New("pair manifest has invalid source revision") + } + wantCore := c.cfg.Image + ":" + targetTag + wantUpdater := c.cfg.PairedImage + ":" + targetTag + if trimRegistryHost(manifest.Components.Core.Image) != wantCore { + return fmt.Errorf("pair manifest Core image %q does not match %q", manifest.Components.Core.Image, wantCore) + } + if trimRegistryHost(manifest.Components.Updater.Image) != wantUpdater { + return fmt.Errorf("pair manifest updater image %q does not match %q", manifest.Components.Updater.Image, wantUpdater) + } + if !sha256DigestPattern.MatchString(manifest.Components.Core.Digest) || !sha256DigestPattern.MatchString(manifest.Components.Updater.Digest) { + return errors.New("pair manifest contains an invalid image digest") + } + return nil +} + +func trimRegistryHost(image string) string { + image = strings.TrimPrefix(image, "https://") + image = strings.TrimPrefix(image, "http://") + if slash := strings.IndexByte(image, '/'); slash >= 0 { + first := image[:slash] + if strings.Contains(first, ".") || strings.Contains(first, ":") || first == "localhost" { + return image[slash+1:] + } + } + return image +} diff --git a/go/internal/selfupdate/control_plane_release_test.go b/go/internal/selfupdate/control_plane_release_test.go new file mode 100644 index 00000000..00914474 --- /dev/null +++ b/go/internal/selfupdate/control_plane_release_test.go @@ -0,0 +1,148 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const ( + testCoreDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testUpdaterDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + testRevision = "0123456789abcdef0123456789abcdef01234567" +) + +type pairReleaseFixture struct { + manifest controlPlaneReleaseManifest + coreDigest string + updaterDigest string + extraOptimizer bool +} + +func newPairReleaseFixture(tag string) pairReleaseFixture { + var manifest controlPlaneReleaseManifest + manifest.SchemaVersion = controlPlaneManifestSchema + manifest.Release = tag + manifest.Revision = testRevision + manifest.Components.Core = controlPlaneReleaseComponent{Image: "ghcr.io/srcfl/ftw:" + tag, Digest: testCoreDigest} + manifest.Components.Updater = controlPlaneReleaseComponent{Image: "ghcr.io/srcfl/ftw-updater:" + tag, Digest: testUpdaterDigest} + return pairReleaseFixture{manifest: manifest, coreDigest: testCoreDigest, updaterDigest: testUpdaterDigest} +} + +func pairReleaseServer(t *testing.T, tag string, fixture *pairReleaseFixture) *httptest.Server { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/release": + _ = json.NewEncoder(w).Encode(map[string]any{ + "tag_name": tag, "draft": false, "prerelease": false, + "assets": []map[string]string{{"name": "ftw-control-plane.json", "browser_download_url": srv.URL + "/asset"}}, + }) + case r.URL.Path == "/asset": + if fixture.extraOptimizer { + data, _ := json.Marshal(fixture.manifest) + var document map[string]any + _ = json.Unmarshal(data, &document) + document["components"].(map[string]any)["optimizer"] = map[string]string{ + "image": "ghcr.io/srcfl/ftw-optimizer:v1.3.2", "digest": testUpdaterDigest, + } + _ = json.NewEncoder(w).Encode(document) + } else { + _ = json.NewEncoder(w).Encode(fixture.manifest) + } + case r.URL.Path == "/token": + _ = json.NewEncoder(w).Encode(map[string]string{"token": "test"}) + case r.URL.Path == "/v2/srcfl/ftw/manifests/"+tag: + if fixture.coreDigest == "" { + http.NotFound(w, r) + return + } + w.Header().Set("Docker-Content-Digest", fixture.coreDigest) + _, _ = w.Write([]byte(`{"schemaVersion":2}`)) + case r.URL.Path == "/v2/srcfl/ftw-updater/manifests/"+tag: + if fixture.updaterDigest == "" { + http.NotFound(w, r) + return + } + w.Header().Set("Docker-Content-Digest", fixture.updaterDigest) + _, _ = w.Write([]byte(`{"schemaVersion":2}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func pairChecker(srv *httptest.Server) *Checker { + return New(Config{ + Repo: "srcfl/ftw", Image: "srcfl/ftw", PairedImage: "srcfl/ftw-updater", + PairManifestAsset: "ftw-control-plane.json", CurrentVersion: "v1.3.1", + RegistryBaseURL: srv.URL, LatestReleaseURL: srv.URL + "/release", + HTTPClient: srv.Client(), + }, newMemStore()) +} + +func TestControlPlaneReleaseRequiresBothPinnedDigests(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + srv := pairReleaseServer(t, tag, &fixture) + info, err := pairChecker(srv).Check(context.Background(), true) + if err != nil { + t.Fatal(err) + } + if !info.UpdateAvailable || info.Latest != tag { + t.Fatalf("pair release = %+v", info) + } +} + +func TestControlPlaneReleaseStaysHiddenUntilUpdaterDigestExists(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.updaterDigest = "" + srv := pairReleaseServer(t, tag, &fixture) + info, err := pairChecker(srv).Check(context.Background(), true) + if err != nil { + t.Fatal(err) + } + if info.UpdateAvailable || info.Latest != "" { + t.Fatalf("partial pair must not be announced: %+v", info) + } +} + +func TestControlPlaneReleaseRejectsMovedTagDigest(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.coreDigest = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + srv := pairReleaseServer(t, tag, &fixture) + _, err := pairChecker(srv).Check(context.Background(), true) + if err == nil || !strings.Contains(err.Error(), "does not match release manifest") { + t.Fatalf("Check error = %v", err) + } +} + +func TestControlPlaneReleaseRejectsWrongReleaseIdentity(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.manifest.Release = "v1.4.1" + srv := pairReleaseServer(t, tag, &fixture) + _, err := pairChecker(srv).Check(context.Background(), true) + if err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("Check error = %v", err) + } +} + +func TestControlPlaneReleaseRejectsOptimizerComponent(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.extraOptimizer = true + srv := pairReleaseServer(t, tag, &fixture) + _, err := pairChecker(srv).Check(context.Background(), true) + if err == nil || !strings.Contains(err.Error(), `unknown field "optimizer"`) { + t.Fatalf("Check error = %v", err) + } +} diff --git a/go/internal/selfupdate/registry.go b/go/internal/selfupdate/registry.go index 7aee0094..d4100249 100644 --- a/go/internal/selfupdate/registry.go +++ b/go/internal/selfupdate/registry.go @@ -2,6 +2,7 @@ package selfupdate import ( "context" + "crypto/sha256" "encoding/json" "fmt" "io" @@ -43,6 +44,47 @@ func (rp *registryProbe) hasTag(ctx context.Context, tag string) (bool, error) { return false, nil } +// manifestDigest resolves one tag to its OCI content digest. A missing +// manifest is a normal publish race and returns an empty digest. +func (rp *registryProbe) manifestDigest(ctx context.Context, tag string) (string, error) { + tok, err := rp.token(ctx) + if err != nil { + return "", err + } + u := rp.base + "/v2/" + rp.repo + "/manifests/" + url.PathEscape(tag) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + req.Header.Set("Accept", strings.Join([]string{ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + }, ", ")) + resp, err := rp.httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + if resp.StatusCode >= 400 { + return "", fmt.Errorf("manifest %s: HTTP %d", tag, resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return "", err + } + digest := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest")) + if digest == "" { + sum := sha256.Sum256(body) + digest = fmt.Sprintf("sha256:%x", sum[:]) + } + return digest, nil +} + func (rp *registryProbe) token(ctx context.Context) (string, error) { u := rp.base + "/token?service=" + url.QueryEscape(rp.service) + "&scope=" + url.QueryEscape("repository:"+rp.repo+":pull") diff --git a/go/internal/selfupdate/selfupdate.go b/go/internal/selfupdate/selfupdate.go index 2b24f82e..09e15b4b 100644 --- a/go/internal/selfupdate/selfupdate.go +++ b/go/internal/selfupdate/selfupdate.go @@ -21,10 +21,9 @@ // Stable and beta require both signals: GitHub tells us *what is released*, // and GHCR tells us whether it is deployable yet. // -// Dispatch passes the resolved version tag (not :latest) to the -// sidecar, which sets FTW_IMAGE_TAG= on the docker exec so -// `docker compose pull` resolves a specific, immutable tag. No race -// possible. +// Dispatch passes the resolved version plus both verified manifest digests to +// the sidecar. The paired transaction pulls and runs those digest refs, so a +// tag move after Check cannot change the installed bytes. // // The check is probe-only — nothing mutates the host until the user // explicitly POSTs /api/version/update or /api/version/restart and the @@ -39,7 +38,6 @@ import ( "fmt" "io" "log/slog" - "net" "net/http" "os" "strconv" @@ -97,6 +95,10 @@ type Config struct { // Image overrides the registry path if it differs from Repo (rare — // normally the GH repo and the GHCR image share a name). Defaults to Repo. Image string + // PairedImage and PairManifestAsset make release checks require both + // control-plane images and their pinned digests. + PairedImage string + PairManifestAsset string // ReleaseTagPrefix selects component-specific GitHub Releases while the // registry continues to use plain vX.Y.Z tags. Example: "optimizer-" // resolves GitHub tag optimizer-v1.2.0 to GHCR tag v1.2.0. @@ -112,6 +114,8 @@ type Config struct { // CurrentVersion is the running binary's version (from main.Version). CurrentVersion string + // RequiredUpdaterVersion is the Core release that the updater must report. + RequiredUpdaterVersion string // CheckInterval is the probe cadence. 0 = 1 h. CheckInterval time.Duration // SocketPath is where the sidecar listens. Empty disables Trigger. @@ -185,6 +189,11 @@ type UpdateStatus struct { Message string `json:"message,omitempty"` PreviousImageID string `json:"previous_image_id,omitempty"` PreviousImages map[string]string `json:"previous_images,omitempty"` + ReleaseRevision string `json:"release_revision,omitempty"` + CoreDigest string `json:"core_digest,omitempty"` + UpdaterDigest string `json:"updater_digest,omitempty"` + TransactionID string `json:"transaction_id,omitempty"` + HelperKind string `json:"helper_kind,omitempty"` } // Checker is the background version-check service. @@ -197,6 +206,7 @@ type Checker struct { lastAnnouncedTag string // dedupe: last tag we emitted UpdateAvailable for skippedKey string channelKey string + verifiedPair verifiedControlPlaneRelease } // New constructs a Checker but does not start the background loop. @@ -310,12 +320,24 @@ func (c *Checker) Check(ctx context.Context, force bool) (Info, error) { return c.recordErr(err) } targetTag := c.releaseTargetTag(rel.TagName) + var verifiedPair verifiedControlPlaneRelease if targetTag != "" { - ok, err := rp.hasTag(ctx, targetTag) - if err != nil { - return c.recordErr(fmt.Errorf("registry probe: %w", err)) + if c.cfg.PairedImage != "" && c.cfg.PairManifestAsset != "" { + record, ok, err := c.verifyControlPlaneRelease(ctx, rel, targetTag) + if err != nil { + return c.recordErr(fmt.Errorf("control-plane release: %w", err)) + } + deployable = ok + if ok { + verifiedPair = record + } + } else { + ok, err := rp.hasTag(ctx, targetTag) + if err != nil { + return c.recordErr(fmt.Errorf("registry probe: %w", err)) + } + deployable = ok } - deployable = ok } c.mu.Lock() @@ -332,11 +354,13 @@ func (c *Checker) Check(ctx context.Context, force bool) (Info, error) { c.info.ReleaseNotesURL = rel.HtmlURL c.info.ReleaseBody = truncateBody(rel.Body) c.info.UpdateAvailable = channelUpdateAvailable(targetTag, c.info.Current) + c.verifiedPair = verifiedPair } else { // Either GH has no published release yet, or the build workflow // hasn't pushed the image for this release yet. Keep the prior // Latest visible (so the UI doesn't flicker) but don't dispatch. c.info.UpdateAvailable = false + c.verifiedPair = verifiedControlPlaneRelease{} } c.info.CheckedAt = c.cfg.Now() c.info.Err = "" @@ -395,6 +419,10 @@ type ghRelease struct { PublishedAt time.Time `json:"published_at"` Draft bool `json:"draft"` Prerelease bool `json:"prerelease"` + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + } `json:"assets"` } // fetchLatestRelease asks GitHub for the most-recently-published @@ -507,10 +535,10 @@ func (c *Checker) releaseTargetTag(releaseTag string) string { // periodic Check. func (c *Checker) Info() Info { c.mu.Lock() - defer c.mu.Unlock() c.reloadSkipLocked() info := c.info - info.SidecarReady = c.sidecarReadyLocked() + c.mu.Unlock() + info.SidecarReady = c.sidecarReady() return info } @@ -553,23 +581,15 @@ func (c *Checker) SetChannel(channel Channel) error { c.info.Err = "" c.info.Skipped = false c.lastAnnouncedTag = "" + c.verifiedPair = verifiedControlPlaneRelease{} c.mu.Unlock() return nil } -// sidecarReadyLocked reports whether the updater socket is present as an -// actual Unix socket. An empty SocketPath means the feature was never -// configured for this deploy — docker-compose sets it, native installs -// typically don't. -func (c *Checker) sidecarReadyLocked() bool { - if c.cfg.SocketPath == "" { - return false - } - fi, err := os.Stat(c.cfg.SocketPath) - if err != nil { - return false - } - return fi.Mode()&os.ModeSocket != 0 +func (c *Checker) sidecarReady() bool { + ctx, cancel := context.WithTimeout(context.Background(), updaterCompatibilityHTTPTimeout) + defer cancel() + return RequireUpdaterRelease(ctx, c.cfg.SocketPath, c.cfg.RequiredUpdaterVersion) == nil } func (c *Checker) reloadSkipLocked() { @@ -648,9 +668,21 @@ func (c *Checker) TriggerComponentAt(ctx context.Context, action, target, compon if action == "component_rollback" && component != "optimizer" { return errors.New("selfupdate: component rollback is only available for optimizer") } - body, _ := json.Marshal(map[string]any{ + request := map[string]any{ "action": action, "target": target, "component": component, "started_at": startedAt, - }) + } + if action == "update" && component == "core" && c.cfg.PairedImage != "" && c.cfg.PairManifestAsset != "" { + c.mu.RLock() + record := c.verifiedPair + c.mu.RUnlock() + if record.Target != target || record.Revision == "" || record.CoreDigest == "" || record.UpdaterDigest == "" { + return fmt.Errorf("selfupdate: no verified control-plane release record for %q", target) + } + request["release_revision"] = record.Revision + request["core_digest"] = record.CoreDigest + request["updater_digest"] = record.UpdaterDigest + } + body, _ := json.Marshal(request) return c.postSidecar(ctx, body) } @@ -684,15 +716,10 @@ func (c *Checker) TriggerRollback(ctx context.Context, snapshotID string, files // endpoint. Shared by Trigger and TriggerRollback so the HTTP client // config (socket dialer + timeout) only lives in one place. func (c *Checker) postSidecar(ctx context.Context, body []byte) error { - cli := &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(ctx, "unix", c.cfg.SocketPath) - }, - }, + if err := RequireUpdaterRelease(ctx, c.cfg.SocketPath, c.cfg.RequiredUpdaterVersion); err != nil { + return err } + cli := updaterHTTPClient(c.cfg.SocketPath) req, _ := http.NewRequestWithContext(ctx, "POST", "http://unix/update", strings.NewReader(string(body))) req.Header.Set("Content-Type", "application/json") resp, err := cli.Do(req) @@ -785,7 +812,7 @@ func (c *Checker) WriteStatus(st UpdateStatus) error { func isInFlightState(state string) bool { switch state { - case "starting", "snapshotting", "pulling", "restarting", "restoring": + case "starting", "snapshotting", "pulling", "transacting", "restarting", "restoring": return true default: return false diff --git a/go/internal/selfupdate/updater_contract.go b/go/internal/selfupdate/updater_contract.go new file mode 100644 index 00000000..4d217364 --- /dev/null +++ b/go/internal/selfupdate/updater_contract.go @@ -0,0 +1,84 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "slices" + "strings" + "time" +) + +const ( + UpdaterProtocolVersion = 2 + ControlPlanePairCapability = "control-plane-pair-v1" + updaterCompatibilityHTTPTimeout = 5 * time.Second +) + +type UpdaterRuntimeInfo struct { + ProtocolVersion int `json:"protocol_version"` + Version string `json:"updater_version"` + Capabilities []string `json:"capabilities"` +} + +func updaterHTTPClient(socketPath string) *http.Client { + return &http.Client{ + Timeout: updaterCompatibilityHTTPTimeout, + Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socketPath) + }, + }, + } +} + +func ProbeUpdater(ctx context.Context, socketPath string) (UpdaterRuntimeInfo, error) { + if socketPath == "" { + return UpdaterRuntimeInfo{}, errors.New("selfupdate: sidecar socket not configured") + } + fi, err := os.Stat(socketPath) + if err != nil { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater socket: %w", err) + } + if fi.Mode()&os.ModeSocket == 0 { + return UpdaterRuntimeInfo{}, errors.New("selfupdate: updater path is not a Unix socket") + } + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/status", nil) + resp, err := updaterHTTPClient(socketPath).Do(req) + if err != nil { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater handshake: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater handshake HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + var info UpdaterRuntimeInfo + if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&info); err != nil { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: decode updater handshake: %w", err) + } + if info.ProtocolVersion < UpdaterProtocolVersion || !slices.Contains(info.Capabilities, ControlPlanePairCapability) { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater lacks %s protocol %d", ControlPlanePairCapability, UpdaterProtocolVersion) + } + if strings.TrimSpace(info.Version) == "" { + return UpdaterRuntimeInfo{}, errors.New("selfupdate: updater did not report its release version") + } + return info, nil +} + +func RequireUpdaterRelease(ctx context.Context, socketPath, release string) error { + info, err := ProbeUpdater(ctx, socketPath) + if err != nil { + return err + } + if release != "" && info.Version != release { + return fmt.Errorf("selfupdate: updater release %s does not match Core release %s", info.Version, release) + } + return nil +} diff --git a/go/internal/selfupdate/updater_contract_test.go b/go/internal/selfupdate/updater_contract_test.go new file mode 100644 index 00000000..efca9ebe --- /dev/null +++ b/go/internal/selfupdate/updater_contract_test.go @@ -0,0 +1,142 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "net" + "net/http" + "os" + "strings" + "sync/atomic" + "testing" +) + +func serveUnixUpdater(t *testing.T, handler http.Handler) string { + t.Helper() + f, err := os.CreateTemp("/tmp", "ftw-updater-*.sock") + if err != nil { + t.Fatal(err) + } + path := f.Name() + _ = f.Close() + _ = os.Remove(path) + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + srv := &http.Server{Handler: handler} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { + _ = srv.Close() + _ = os.Remove(path) + }) + return path +} + +// v1.3.1 exposed GET /status and accepted the same update JSON, but ignored +// unknown fields and recreated Core only. A new Core must reject that contract +// before it sends an update request or opens state with a mixed release pair. +func TestV131UpdaterIsRejectedBeforeCoreUpdate(t *testing.T) { + var posts atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"state":"idle"}`)) + }) + mux.HandleFunc("POST /update", func(w http.ResponseWriter, _ *http.Request) { + posts.Add(1) + w.WriteHeader(http.StatusAccepted) + }) + socket := serveUnixUpdater(t, mux) + + c := New(Config{SocketPath: socket, RequiredUpdaterVersion: "v1.4.0"}, newMemStore()) + if c.Info().SidecarReady { + t.Fatal("v1.3.1 updater must not be ready for a new Core") + } + err := c.Trigger(context.Background(), "update", "v1.4.1") + if err == nil || !strings.Contains(err.Error(), ControlPlanePairCapability) { + t.Fatalf("Trigger error = %v, want missing pair capability", err) + } + if got := posts.Load(); got != 0 { + t.Fatalf("legacy updater received %d POST(s); want zero", got) + } +} + +func TestMatchingUpdaterReleaseIsReadyAndAcceptsUpdate(t *testing.T) { + var posts atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(UpdaterRuntimeInfo{ + ProtocolVersion: UpdaterProtocolVersion, + Version: "v1.4.0", + Capabilities: []string{ControlPlanePairCapability}, + }) + }) + mux.HandleFunc("POST /update", func(w http.ResponseWriter, _ *http.Request) { + posts.Add(1) + w.WriteHeader(http.StatusAccepted) + }) + socket := serveUnixUpdater(t, mux) + + c := New(Config{SocketPath: socket, RequiredUpdaterVersion: "v1.4.0"}, newMemStore()) + if !c.Info().SidecarReady { + t.Fatal("matching updater should be ready") + } + if err := c.Trigger(context.Background(), "update", "v1.4.1"); err != nil { + t.Fatal(err) + } + if got := posts.Load(); got != 1 { + t.Fatalf("POST count = %d, want one", got) + } +} + +func TestUpdaterReleaseMismatchFailsClosed(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(UpdaterRuntimeInfo{ + ProtocolVersion: UpdaterProtocolVersion, + Version: "v1.4.1", + Capabilities: []string{ControlPlanePairCapability}, + }) + }) + socket := serveUnixUpdater(t, mux) + if err := RequireUpdaterRelease(context.Background(), socket, "v1.4.0"); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("RequireUpdaterRelease error = %v", err) + } +} + +func TestTriggerCarriesVerifiedControlPlaneDigests(t *testing.T) { + const target = "v1.4.0" + fixture := newPairReleaseFixture(target) + releaseServer := pairReleaseServer(t, target, &fixture) + requestBody := make(chan map[string]any, 1) + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(UpdaterRuntimeInfo{ + ProtocolVersion: UpdaterProtocolVersion, + Version: "v1.3.1", + Capabilities: []string{ControlPlanePairCapability}, + }) + }) + mux.HandleFunc("POST /update", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + requestBody <- body + w.WriteHeader(http.StatusAccepted) + }) + socket := serveUnixUpdater(t, mux) + + c := pairChecker(releaseServer) + c.cfg.SocketPath = socket + c.cfg.RequiredUpdaterVersion = "v1.3.1" + info, err := c.Check(context.Background(), true) + if err != nil || !info.UpdateAvailable { + t.Fatalf("verified release = %+v, %v", info, err) + } + if err := c.Trigger(context.Background(), "update", target); err != nil { + t.Fatal(err) + } + body := <-requestBody + if body["release_revision"] != testRevision || body["core_digest"] != testCoreDigest || body["updater_digest"] != testUpdaterDigest { + t.Fatalf("update request lost verified record: %+v", body) + } +} diff --git a/scripts/create-control-plane-manifest.sh b/scripts/create-control-plane-manifest.sh new file mode 100755 index 00000000..f102d2ae --- /dev/null +++ b/scripts/create-control-plane-manifest.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +tag="${1:?usage: create-control-plane-manifest.sh TAG REVISION OUTPUT}" +revision="${2:?usage: create-control-plane-manifest.sh TAG REVISION OUTPUT}" +output="${3:?usage: create-control-plane-manifest.sh TAG REVISION OUTPUT}" + +[[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]] || { + echo "invalid control-plane release tag: ${tag}" >&2 + exit 1 +} +[[ "${revision}" =~ ^[0-9a-f]{40}$ ]] || { + echo "invalid control-plane source revision: ${revision}" >&2 + exit 1 +} + +core_image="ghcr.io/srcfl/ftw:${tag}" +updater_image="ghcr.io/srcfl/ftw-updater:${tag}" +core_digest="$(scripts/inspect-image-digest.sh "${core_image}")" +updater_digest="$(scripts/inspect-image-digest.sh "${updater_image}")" + +for digest in "${core_digest}" "${updater_digest}"; do + [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "invalid immutable image digest: ${digest}" >&2 + exit 1 + } +done + +jq -n \ + --arg release "${tag}" \ + --arg revision "${revision}" \ + --arg core_image "${core_image}" \ + --arg core_digest "${core_digest}" \ + --arg updater_image "${updater_image}" \ + --arg updater_digest "${updater_digest}" \ + '{ + schema_version: 1, + release: $release, + revision: $revision, + components: { + core: {image: $core_image, digest: $core_digest}, + updater: {image: $updater_image, digest: $updater_digest} + } + }' >"${output}" diff --git a/scripts/install.sh b/scripts/install.sh index 33179091..ec6697c7 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -75,9 +75,12 @@ if [ -f "$INSTALL_DIR/docker-compose.yml" ]; then An existing FTW Docker Compose installation was found at: $INSTALL_DIR -The fresh installer will not overwrite it. Run the rollback-safe migration: +The fresh installer will not overwrite it. Download the rollback-safe +migration, then run it with the approved release and both digests from +ftw-control-plane.json: - curl -fsSL $MIGRATION_URL -o /tmp/ftw-migrate.sh && bash /tmp/ftw-migrate.sh --dir "$INSTALL_DIR" + curl -fsSL $MIGRATION_URL -o /tmp/ftw-migrate.sh + bash /tmp/ftw-migrate.sh --version vX.Y.Z --core-digest sha256:... --updater-digest sha256:... --dir "$INSTALL_DIR" Guide: https://github.com/srcfl/ftw/blob/$BRANCH/docs/upgrade-from-legacy.md EOF diff --git a/scripts/migrate-legacy-compose.sh b/scripts/migrate-legacy-compose.sh index 1fb76a5c..10c5cccd 100755 --- a/scripts/migrate-legacy-compose.sh +++ b/scripts/migrate-legacy-compose.sh @@ -20,7 +20,9 @@ die() { usage() { cat <<'EOF' -Usage: migrate-legacy-compose.sh [--dir PATH] [--backup-dir PATH] +Usage: migrate-legacy-compose.sh --version vX.Y.Z[-beta.N] \ + --core-digest sha256:... --updater-digest sha256:... \ + [--dir PATH] [--backup-dir PATH] Without --dir, the script uses the current directory when it contains docker-compose.yml, then tries ~/ftw and ~/forty-two-watts. @@ -33,6 +35,9 @@ EOF requested_dir="${FTW_DIR:-}" requested_full_backup_dir="${FTW_BACKUP_DIR:-}" +control_plane_version="${FTW_CONTROL_PLANE_VERSION:-}" +core_digest="${FTW_CONTROL_PLANE_CORE_DIGEST:-}" +updater_digest="${FTW_CONTROL_PLANE_UPDATER_DIGEST:-}" while [ "$#" -gt 0 ]; do case "$1" in --dir) @@ -45,6 +50,21 @@ while [ "$#" -gt 0 ]; do requested_full_backup_dir="$2" shift 2 ;; + --version) + [ "$#" -ge 2 ] || die "--version requires an immutable release tag" + control_plane_version="$2" + shift 2 + ;; + --core-digest) + [ "$#" -ge 2 ] || die "--core-digest requires a sha256 digest" + core_digest="$2" + shift 2 + ;; + --updater-digest) + [ "$#" -ge 2 ] || die "--updater-digest requires a sha256 digest" + updater_digest="$2" + shift 2 + ;; -h|--help) usage exit 0 @@ -55,6 +75,16 @@ while [ "$#" -gt 0 ]; do esac done +if [[ ! "$control_plane_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]]; then + die "--version must be an immutable vX.Y.Z or vX.Y.Z-beta.N release" +fi +if [[ ! "$core_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + die "--core-digest must match the verified ftw-control-plane.json Core digest" +fi +if [[ ! "$updater_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + die "--updater-digest must match the verified ftw-control-plane.json updater digest" +fi + [ "$(uname -s)" = Linux ] || die "automatic migration currently supports Linux Docker hosts only" command -v docker >/dev/null 2>&1 || die "docker is not installed" command -v curl >/dev/null 2>&1 || die "curl is not installed" @@ -130,7 +160,7 @@ if [ -n "$compose_project" ]; then fi compose() { if [ "$canonical_tags" = true ]; then - FTW_IMAGE_TAG=latest FTW_UPDATER_IMAGE_TAG=latest FTW_OPTIMIZER_IMAGE_TAG=latest \ + FTW_IMAGE_TAG="$control_plane_version" FTW_UPDATER_IMAGE_TAG="$control_plane_version" FTW_OPTIMIZER_IMAGE_TAG=latest \ "${compose_command[@]}" "$@" else "${compose_command[@]}" "$@" @@ -164,6 +194,8 @@ previous_updater_image_id="" previous_updater_image_ref="" previous_optimizer_image_id="" previous_optimizer_image_ref="" +pinned_main_image_id="" +pinned_updater_image_id="" restore_image_reference() { local image_id="$1" @@ -214,16 +246,24 @@ restore_after_failure() { # A pull can move a mutable tag such as :latest before any container is # recreated. Put every previous immutable image back under its original # reference before Compose restores the old service definitions. - if ! restore_image_reference "$previous_main_image_id" "$previous_main_image_ref"; then + if ! restore_image_reference "$previous_updater_image_id" "$previous_updater_image_ref"; then restore_ok=false fi - if ! restore_image_reference "$previous_updater_image_id" "$previous_updater_image_ref"; then + if ! restore_image_reference "$previous_main_image_id" "$previous_main_image_ref"; then restore_ok=false fi if ! restore_image_reference "$previous_optimizer_image_id" "$previous_optimizer_image_ref"; then restore_ok=false fi + # Restore updater first. A paired Core waits for its matching updater before + # it opens state, so restoring Core first can leave a good rollback unready. + if [ "$containers_changed" = true ]; then + if ! compose up -d --no-deps --force-recreate ftw-updater >/dev/null 2>&1; then + restore_ok=false + fi + fi + if [ -n "$renamed_container" ]; then # Remove any replacement by Compose service identity as well as by exact # ID/name. A Compose file without container_name uses a generated name. @@ -252,11 +292,6 @@ restore_after_failure() { restore_ok=false fi fi - if [ "$containers_changed" = true ]; then - if ! compose up -d --no-deps --force-recreate ftw-updater >/dev/null 2>&1; then - restore_ok=false - fi - fi if [ "$optimizer_container_changed" = true ] && [ "$optimizer_service_added" = false ]; then if ! compose up -d --no-deps --force-recreate ftw-optimizer >/dev/null 2>&1; then restore_ok=false @@ -563,33 +598,37 @@ rewrite_service_image() { for file in "${compose_files[@]}"; do rewrite_service_image "$file" "$main_service" 'ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest}' - rewrite_service_image "$file" 'ftw-updater' 'ghcr.io/srcfl/ftw-updater:latest' + rewrite_service_image "$file" 'ftw-updater' 'ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest}' rewrite_service_image "$file" 'ftw-optimizer' 'ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest}' done -# A caller's shell or old .env file may contain a development tag. Use the -# canonical stable tag for this one-time migration; future in-app updates pass -# their own immutable FTW_IMAGE_TAG to Compose. +# Pin both control-plane services to the chosen immutable release. Ignore a +# development tag from the caller's shell or an old .env file. canonical_tags=true compose config >/dev/null effective_main_image="$(compose config --images "$main_service")" effective_updater_image="$(compose config --images ftw-updater)" effective_optimizer_image="$(compose config --images ftw-optimizer)" -case "$effective_main_image" in - ghcr.io/srcfl/ftw:*) ;; - *) die "the effective $main_service image is not ghcr.io/srcfl/ftw: $effective_main_image" ;; -esac -case "$effective_updater_image" in - ghcr.io/srcfl/ftw-updater:*) ;; - *) die "the effective ftw-updater image is not ghcr.io/srcfl/ftw-updater: $effective_updater_image" ;; -esac +[ "$effective_main_image" = "ghcr.io/srcfl/ftw:$control_plane_version" ] || \ + die "the effective $main_service image is not the selected release: $effective_main_image" +[ "$effective_updater_image" = "ghcr.io/srcfl/ftw-updater:$control_plane_version" ] || \ + die "the effective ftw-updater image is not the selected release: $effective_updater_image" case "$effective_optimizer_image" in ghcr.io/srcfl/ftw-optimizer:*) ;; *) die "the effective ftw-optimizer image is not ghcr.io/srcfl/ftw-optimizer: $effective_optimizer_image" ;; esac -log "phase 2/4: pulling the paired Core + updater control plane" -compose pull "$main_service" ftw-updater +core_digest_ref="ghcr.io/srcfl/ftw@$core_digest" +updater_digest_ref="ghcr.io/srcfl/ftw-updater@$updater_digest" +log "phase 2/4: pulling the digest-locked Core + updater control plane" +docker pull "$core_digest_ref" >/dev/null +docker pull "$updater_digest_ref" >/dev/null +pinned_main_image_id="$(docker image inspect "$core_digest_ref" --format '{{.Id}}')" +pinned_updater_image_id="$(docker image inspect "$updater_digest_ref" --format '{{.Id}}')" +[ -n "$pinned_main_image_id" ] || die "could not inspect the verified Core digest" +[ -n "$pinned_updater_image_id" ] || die "could not inspect the verified updater digest" +docker image tag "$core_digest_ref" "ghcr.io/srcfl/ftw:$control_plane_version" +docker image tag "$updater_digest_ref" "ghcr.io/srcfl/ftw-updater:$control_plane_version" # Some developer installations replaced the Compose-managed main container # with a manually created container of the same name. Preserve it as a stopped @@ -620,27 +659,45 @@ if docker container inspect "$main_service" >/dev/null 2>&1; then fi fi -log "starting $main_service from ghcr.io/srcfl/ftw" +log "starting ftw-updater from ghcr.io/srcfl/ftw-updater:$control_plane_version" containers_changed=true -compose up -d --no-deps --force-recreate "$main_service" -log "starting ftw-updater from ghcr.io/srcfl/ftw-updater" -compose up -d --no-deps --force-recreate ftw-updater +compose up -d --no-deps --pull never --force-recreate ftw-updater +updater_id="$(compose ps -q --status running ftw-updater | tail -n 1)" +[ -n "$updater_id" ] || die "ftw-updater did not reach running state" +running_updater_image_id="$(docker inspect "$updater_id" --format '{{.Image}}')" +[ "$running_updater_image_id" = "$pinned_updater_image_id" ] || \ + die "running updater does not match the verified digest: $running_updater_image_id" + +# v1.10.0-beta.1 cannot drive the new paired transaction. Its one-time bridge +# starts only the verified updater while old Core remains live, then replaces +# Core. Any loss of old Core health here triggers the existing image-ID restore. +health_url="${FTW_HEALTH_URL:-http://127.0.0.1:8080/api/health}" +ready_url="${FTW_READY_URL:-${health_url%/api/health}/api/status}" +curl -fsS --max-time 3 "$health_url" >/dev/null 2>&1 || \ + die "old Core lost health after the updater-first bootstrap" +log "old Core remained healthy after the updater-first bootstrap" + +log "starting $main_service from ghcr.io/srcfl/ftw:$control_plane_version" +compose up -d --no-deps --pull never --force-recreate "$main_service" new_main_id="$(compose ps -q --status running "$main_service" | tail -n 1)" [ -n "$new_main_id" ] || die "$main_service did not reach running state" updater_id="$(compose ps -q --status running ftw-updater | tail -n 1)" [ -n "$updater_id" ] || die "ftw-updater did not reach running state" +running_main_image_id="$(docker inspect "$new_main_id" --format '{{.Image}}')" +running_updater_image_id="$(docker inspect "$updater_id" --format '{{.Image}}')" +[ "$running_main_image_id" = "$pinned_main_image_id" ] || \ + die "running Core does not match the verified digest: $running_main_image_id" +[ "$running_updater_image_id" = "$pinned_updater_image_id" ] || \ + die "running updater does not match the verified digest: $running_updater_image_id" + main_image="$(docker inspect "$new_main_id" --format '{{.Config.Image}}')" updater_image="$(docker inspect "$updater_id" --format '{{.Config.Image}}')" -case "$main_image" in - ghcr.io/srcfl/ftw:*) ;; - *) die "running main container uses unexpected image: $main_image" ;; -esac -case "$updater_image" in - ghcr.io/srcfl/ftw-updater:*) ;; - *) die "running updater uses unexpected image: $updater_image" ;; -esac +[ "$main_image" = "ghcr.io/srcfl/ftw:$control_plane_version" ] || \ + die "running main container is not the selected release: $main_image" +[ "$updater_image" = "ghcr.io/srcfl/ftw-updater:$control_plane_version" ] || \ + die "running updater is not the selected release: $updater_image" running_data_source="$(docker inspect "$new_main_id" --format '{{range .Mounts}}{{if eq .Destination "/app/data"}}{{.Source}}{{end}}{{end}}')" if [ -d "$running_data_source" ]; then running_data_source="$(cd "$running_data_source" && pwd -P)" @@ -648,10 +705,9 @@ fi [ "$running_data_source" = "$expected_data_source" ] || \ die "running main container uses unexpected /app/data source: $running_data_source" -health_url="${FTW_HEALTH_URL:-http://127.0.0.1:8080/api/health}" -ready_url="${FTW_READY_URL:-${health_url%/api/health}/api/status}" healthy=false -health_deadline=$((SECONDS + 1800)) +health_timeout_s="${FTW_MIGRATION_HEALTH_TIMEOUT_S:-1800}" +health_deadline=$((SECONDS + health_timeout_s)) while [ "$SECONDS" -lt "$health_deadline" ]; do if curl -fsS --max-time 3 "$health_url" >/dev/null 2>&1 && \ curl -fsS --max-time 3 "$ready_url" >/dev/null 2>&1; then @@ -660,7 +716,7 @@ while [ "$SECONDS" -lt "$health_deadline" ]; do fi sleep 2 done -[ "$healthy" = true ] || die "FTW did not finish initialization at $ready_url within 30 minutes" +[ "$healthy" = true ] || die "FTW did not finish initialization at $ready_url within ${health_timeout_s}s" # Phase 3 deliberately starts only after Core + updater have passed their # health gate. Optimizer has its own release and compatibility handshake; a diff --git a/scripts/release.sh b/scripts/release.sh index 195758d0..658834a1 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,12 +1,11 @@ #!/bin/bash -# Build release archives and create (or attach to) a GitHub release. +# Build release archives and attach them to a gated draft release. # Usage: ./scripts/release.sh v0.2.0 # # Produces linux arm64/amd64 tarballs + windows amd64 zip via the # Makefile's `release` target, then uploads them to the GitHub release -# for the given tag. If the release doesn't exist yet it's created with -# auto-generated notes; if it does exist the assets are added with -# --clobber so a re-run safely replaces a broken archive. +# for the given tag. This script cannot create or publish a release. Only the +# release workflow may publish after it checks the Core/updater pair. set -euo pipefail @@ -14,6 +13,20 @@ VERSION=${1:?Usage: $0 } REPO="srcfl/ftw" echo "Building FTW ${VERSION} (linux arm64/amd64, windows amd64)…" + +RELEASE_JSON="$(gh release view "${VERSION}" --repo "${REPO}" --json isDraft,assets)" || { + echo "Release ${VERSION} does not exist. Create it through release.yml." >&2 + exit 1 +} +if [ "$(jq -r .isDraft <<<"${RELEASE_JSON}")" != "true" ]; then + echo "Release ${VERSION} is not a draft; refusing local changes." >&2 + exit 1 +fi +if ! jq -e '.assets | any(.name == "ftw-control-plane.json")' <<<"${RELEASE_JSON}" >/dev/null; then + echo "Draft ${VERSION} lacks ftw-control-plane.json." >&2 + exit 1 +fi + make release ASSETS=( @@ -35,14 +48,7 @@ for f in "${ASSETS[@]}"; do [ -f "$f" ] || { echo "missing: $f"; exit 1; } done -if gh release view "${VERSION}" --repo "${REPO}" >/dev/null 2>&1; then - echo "Release ${VERSION} exists — uploading assets (clobbering duplicates)…" - gh release upload "${VERSION}" --repo "${REPO}" --clobber "${ASSETS[@]}" -else - echo "Creating GitHub release ${VERSION}…" - gh release create "${VERSION}" --repo "${REPO}" \ - --title "${VERSION}" --generate-notes \ - "${ASSETS[@]}" -fi +echo "Uploading assets to verified draft ${VERSION}…" +gh release upload "${VERSION}" --repo "${REPO}" --clobber "${ASSETS[@]}" -echo "Done! Release: https://github.com/${REPO}/releases/tag/${VERSION}" +echo "Assets attached. Only release-assets.yml may publish the draft." diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index c11aa393..cacaabc9 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -13,6 +13,12 @@ grep -q '^FROM alpine:' Dockerfile grep -q '^COPY optimizer/' Dockerfile.optimizer grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile +grep -q 'HEALTHCHECK.*start-period=30m' Dockerfile +grep -q 'http://127.0.0.1:8080/api/status' Dockerfile +if grep -A1 '^HEALTHCHECK' Dockerfile | grep -q '/api/health'; then + echo "Core container health must use readiness, not boot liveness" >&2 + exit 1 +fi grep -q '^ ftw-optimizer:' docker-compose.yml grep -q 'FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' docker-compose.yml diff --git a/scripts/test-control-plane-release.sh b/scripts/test-control-plane-release.sh new file mode 100755 index 00000000..c9c083ad --- /dev/null +++ b/scripts/test-control-plane-release.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +mkdir -p "${TMP}/bin" +cat >"${TMP}/bin/docker" <<'FAKE_DOCKER' +#!/usr/bin/env bash +set -euo pipefail +ref="${*: -1}" +case "${ref}" in + ghcr.io/srcfl/ftw:v1.4.0) + echo 'Name: test' + echo 'Digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + ;; + ghcr.io/srcfl/ftw-updater:v1.4.0) + echo 'Name: test' + echo "Digest: ${FAKE_UPDATER_DIGEST:-sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb}" + ;; + *) exit 1 ;; +esac +FAKE_DOCKER +chmod +x "${TMP}/bin/docker" + +cd "${ROOT}" +PATH="${TMP}/bin:${PATH}" scripts/create-control-plane-manifest.sh \ + v1.4.0 0123456789abcdef0123456789abcdef01234567 "${TMP}/manifest.json" +PATH="${TMP}/bin:${PATH}" scripts/verify-control-plane-manifest.sh \ + "${TMP}/manifest.json" v1.4.0 0123456789abcdef0123456789abcdef01234567 + +if PATH="${TMP}/bin:${PATH}" FAKE_UPDATER_DIGEST=sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc \ + scripts/verify-control-plane-manifest.sh "${TMP}/manifest.json" v1.4.0 \ + 0123456789abcdef0123456789abcdef01234567 >/dev/null 2>&1; then + echo "moved updater tag should fail pair verification" >&2 + exit 1 +fi + +if PATH="${TMP}/bin:${PATH}" scripts/verify-control-plane-manifest.sh \ + "${TMP}/manifest.json" v1.4.1 0123456789abcdef0123456789abcdef01234567 >/dev/null 2>&1; then + echo "wrong release identity should fail pair verification" >&2 + exit 1 +fi + +jq '.components.optimizer = { + image: "ghcr.io/srcfl/ftw-optimizer:v1.3.2", + digest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +}' "${TMP}/manifest.json" >"${TMP}/manifest-with-optimizer.json" +if PATH="${TMP}/bin:${PATH}" scripts/verify-control-plane-manifest.sh \ + "${TMP}/manifest-with-optimizer.json" v1.4.0 \ + 0123456789abcdef0123456789abcdef01234567 >/dev/null 2>&1; then + echo "optimizer must stay outside the control-plane manifest" >&2 + exit 1 +fi + +echo "control-plane release contract verified" diff --git a/scripts/test-modular-compose.sh b/scripts/test-modular-compose.sh index d841666c..7eebb68b 100755 --- a/scripts/test-modular-compose.sh +++ b/scripts/test-modular-compose.sh @@ -88,11 +88,11 @@ mkdir -p "$TMP/migrate/bin" "$TMP/migrate/data" "$TMP/migrate/state" cat >"$TMP/migrate/docker-compose.yml" <<'YAML' services: ftw: - image: example.invalid/old-core:latest + image: ghcr.io/srcfl/ftw:v1.10.0-beta.1 volumes: - ./data:/app/data ftw-updater: - image: example.invalid/old-updater:latest + image: ghcr.io/srcfl/ftw-updater:v1.10.0-beta.1 YAML cat >"$TMP/migrate/bin/uname" <<'FAKE_UNAME' @@ -102,6 +102,10 @@ FAKE_UNAME cat >"$TMP/migrate/bin/curl" <<'FAKE_CURL' #!/usr/bin/env bash +printf 'curl %s\n' "$*" >>"${FAKE_STATE_DIR:?}/events" +if [ "${FAKE_FAIL_READY:-}" = 1 ] && [[ "$*" == *"/api/status"* ]]; then + exit 1 +fi exit 0 FAKE_CURL @@ -190,8 +194,12 @@ case "$command" in case "$format" in *'{{.Image}}'*) case "$subject" in - ftw-id) echo 'sha256:old-core' ;; - ftw-updater-id) echo 'sha256:old-updater' ;; + ftw-id) + if [ -f "$state/canonical-ftw" ]; then echo 'sha256:pinned-core'; else echo 'sha256:old-core'; fi + ;; + ftw-updater-id) + if [ -f "$state/canonical-ftw-updater" ]; then echo 'sha256:pinned-updater'; else echo 'sha256:old-updater'; fi + ;; ftw-optimizer-id) echo 'sha256:old-optimizer' ;; *) exit 1 ;; esac @@ -205,10 +213,10 @@ case "$command" in *Config.Image*) case "$subject" in ftw-id) - if [ -f "$state/canonical-ftw" ]; then echo 'ghcr.io/srcfl/ftw:latest'; else echo 'example.invalid/old-core:latest'; fi + if [ -f "$state/canonical-ftw" ]; then echo "ghcr.io/srcfl/ftw:$(cat "$state/canonical-ftw")"; else echo 'example.invalid/old-core:latest'; fi ;; ftw-updater-id) - if [ -f "$state/canonical-ftw-updater" ]; then echo 'ghcr.io/srcfl/ftw-updater:latest'; else echo 'example.invalid/old-updater:latest'; fi + if [ -f "$state/canonical-ftw-updater" ]; then echo "ghcr.io/srcfl/ftw-updater:$(cat "$state/canonical-ftw-updater")"; else echo 'example.invalid/old-updater:latest'; fi ;; ftw-optimizer-id) if [ -f "$state/canonical-ftw-optimizer" ]; then echo 'ghcr.io/srcfl/ftw-optimizer:latest'; else echo 'example.invalid/old-optimizer:latest'; fi @@ -256,8 +264,8 @@ case "$command" in ;; --images) case "${2:-}" in - ftw) echo 'ghcr.io/srcfl/ftw:latest' ;; - ftw-updater) echo 'ghcr.io/srcfl/ftw-updater:latest' ;; + ftw) echo "ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest}" ;; + ftw-updater) echo "ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest}" ;; ftw-optimizer) echo 'ghcr.io/srcfl/ftw-optimizer:latest' ;; *) exit 1 ;; esac @@ -278,7 +286,13 @@ case "$command" in touch "$state/failure-consumed" exit 1 fi - touch "$state/$service" "$state/$service-id" "$state/canonical-$service" + touch "$state/$service" "$state/$service-id" + printf 'up %s\n' "$service" >>"$state/events" + case "$service" in + ftw) printf '%s\n' "${FTW_IMAGE_TAG:-latest}" >"$state/canonical-$service" ;; + ftw-updater) printf '%s\n' "${FTW_UPDATER_IMAGE_TAG:-latest}" >"$state/canonical-$service" ;; + ftw-optimizer) printf '%s\n' "${FTW_OPTIMIZER_IMAGE_TAG:-latest}" >"$state/canonical-$service" ;; + esac ;; ps) service="${*: -1}" @@ -296,8 +310,21 @@ case "$command" in esac ;; image) - [ "${1:-}" = tag ] || exit 1 - printf '%s %s\n' "${2:-}" "${3:-}" >>"$state/image-tags" + subcommand="${1:-}" + shift || true + case "$subcommand" in + inspect) + case "${1:-}" in + ghcr.io/srcfl/ftw@*) echo 'sha256:pinned-core' ;; + ghcr.io/srcfl/ftw-updater@*) echo 'sha256:pinned-updater' ;; + *) exit 1 ;; + esac + ;; + tag) + printf '%s %s\n' "${1:-}" "${2:-}" >>"$state/image-tags" + ;; + *) exit 1 ;; + esac ;; *) exit 1 @@ -310,17 +337,33 @@ chmod +x \ "$TMP/migrate/bin/chmod" \ "$TMP/migrate/bin/docker" touch "$TMP/migrate/data/state.db" +CORE_DIGEST="sha256:$(printf 'a%.0s' {1..64})" +UPDATER_DIGEST="sha256:$(printf 'b%.0s' {1..64})" +PAIR_VERSION=v1.10.1-beta.1 +PAIR_ARGS=( + --version "$PAIR_VERSION" + --core-digest "$CORE_DIGEST" + --updater-digest "$UPDATER_DIGEST" +) PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/migrate/state" \ FAKE_DATA_DIR="$TMP/migrate/data" \ -bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/migrate" +bash "$ROOT/scripts/migrate-legacy-compose.sh" "${PAIR_ARGS[@]}" --dir "$TMP/migrate" grep -q '^ ftw-optimizer:' "$TMP/migrate/docker-compose.override.yml" test -f "$TMP/migrate/state/ftw" test -f "$TMP/migrate/state/ftw-updater" test -f "$TMP/migrate/state/ftw-optimizer" test -f "$TMP/migrate"/.ftw-migration-backup-*/previous-images.tsv +grep -q "^ghcr.io/srcfl/ftw@$CORE_DIGEST ghcr.io/srcfl/ftw:$PAIR_VERSION$" "$TMP/migrate/state/image-tags" +grep -q "^ghcr.io/srcfl/ftw-updater@$UPDATER_DIGEST ghcr.io/srcfl/ftw-updater:$PAIR_VERSION$" "$TMP/migrate/state/image-tags" +awk ' + $0 == "up ftw-updater" && !updater { updater = NR } + $1 == "curl" && $0 ~ /api\/health/ && !health { health = NR } + $0 == "up ftw" && !core { core = NR } + END { exit !(updater && health && core && updater < health && health < core) } +' "$TMP/migrate/state/events" # Generated container names still carry Compose labels. The migration must # reuse their explicit project name instead of creating a parallel default. @@ -332,7 +375,7 @@ FAKE_STATE_DIR="$TMP/project/state" \ FAKE_DATA_DIR="$TMP/project/data" \ FAKE_PROJECT_NAME=custom-energy \ FAKE_INSTALL_DIR="$TMP/project" \ -bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/project" +bash "$ROOT/scripts/migrate-legacy-compose.sh" "${PAIR_ARGS[@]}" --dir "$TMP/project" test -e "$TMP/project/state/project-custom-energy" if [ -n "$REAL_DOCKER" ]; then @@ -358,7 +401,7 @@ if PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/rollback/state" \ FAKE_DATA_DIR="$TMP/rollback/data" \ FAKE_FAIL_SERVICE=ftw \ - bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/rollback" \ + bash "$ROOT/scripts/migrate-legacy-compose.sh" "${PAIR_ARGS[@]}" --dir "$TMP/rollback" \ >/dev/null 2>&1; then echo "expected a failed core recreate to roll the modular migration back" >&2 exit 1 @@ -392,7 +435,7 @@ if PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/rollback-existing/state" \ FAKE_DATA_DIR="$TMP/rollback-existing/data" \ FAKE_FAIL_SERVICE=ftw \ - bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/rollback-existing" \ + bash "$ROOT/scripts/migrate-legacy-compose.sh" "${PAIR_ARGS[@]}" --dir "$TMP/rollback-existing" \ >/dev/null 2>&1; then echo "expected an existing modular migration failure to roll back" >&2 exit 1 @@ -403,6 +446,35 @@ grep -q '^sha256:old-core example.invalid/old-core:latest$' "$TMP/rollback-exist grep -q '^sha256:old-updater example.invalid/old-updater:latest$' "$TMP/rollback-existing/state/image-tags" grep -q '^sha256:old-optimizer example.invalid/old-optimizer:latest$' "$TMP/rollback-existing/state/image-tags" +# A readiness failure after both replacements must restore both old image IDs +# and return an error. +mkdir -p "$TMP/readiness-rollback/data" "$TMP/readiness-rollback/state" +touch "$TMP/readiness-rollback/data/state.db" +touch "$TMP/readiness-rollback/state/ftw" "$TMP/readiness-rollback/state/ftw-updater" +cat >"$TMP/readiness-rollback/docker-compose.yml" <<'YAML' +services: + ftw: + image: example.invalid/old-core:latest + volumes: + - ./data:/app/data + ftw-updater: + image: example.invalid/old-updater:latest +YAML +if PATH="$TMP/migrate/bin:$PATH" \ + FAKE_STATE_DIR="$TMP/readiness-rollback/state" \ + FAKE_DATA_DIR="$TMP/readiness-rollback/data" \ + FAKE_FAIL_READY=1 \ + FTW_MIGRATION_HEALTH_TIMEOUT_S=1 \ + bash "$ROOT/scripts/migrate-legacy-compose.sh" "${PAIR_ARGS[@]}" --dir "$TMP/readiness-rollback" \ + >/dev/null 2>&1; then + echo "expected readiness failure to roll back the paired migration" >&2 + exit 1 +fi +grep -q 'example.invalid/old-core:latest' "$TMP/readiness-rollback/docker-compose.yml" +grep -q 'example.invalid/old-updater:latest' "$TMP/readiness-rollback/docker-compose.yml" +grep -q '^sha256:old-core example.invalid/old-core:latest$' "$TMP/readiness-rollback/state/image-tags" +grep -q '^sha256:old-updater example.invalid/old-updater:latest$' "$TMP/readiness-rollback/state/image-tags" + # Optimizer is an independent, optional phase. A failed optimizer candidate # must leave the newly healthy Core + updater online and must not fail the # migration or activate a driver. @@ -421,7 +493,7 @@ PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/optimizer-failure/state" \ FAKE_DATA_DIR="$TMP/optimizer-failure/data" \ FAKE_FAIL_SERVICE=ftw-optimizer \ -bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/optimizer-failure" \ +bash "$ROOT/scripts/migrate-legacy-compose.sh" "${PAIR_ARGS[@]}" --dir "$TMP/optimizer-failure" \ >/dev/null test -e "$TMP/optimizer-failure/state/ftw" test -e "$TMP/optimizer-failure/state/ftw-updater" diff --git a/scripts/verify-control-plane-manifest.sh b/scripts/verify-control-plane-manifest.sh new file mode 100755 index 00000000..9c681fa1 --- /dev/null +++ b/scripts/verify-control-plane-manifest.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +manifest="${1:?usage: verify-control-plane-manifest.sh MANIFEST TAG REVISION}" +tag="${2:?usage: verify-control-plane-manifest.sh MANIFEST TAG REVISION}" +revision="${3:?usage: verify-control-plane-manifest.sh MANIFEST TAG REVISION}" + +jq -e \ + --arg release "${tag}" \ + --arg revision "${revision}" \ + --arg core "ghcr.io/srcfl/ftw:${tag}" \ + --arg updater "ghcr.io/srcfl/ftw-updater:${tag}" ' + .schema_version == 1 + and .release == $release + and .revision == $revision + and ((.components | keys | sort) == ["core", "updater"]) + and .components.core.image == $core + and .components.updater.image == $updater + and (.components.core.digest | test("^sha256:[0-9a-f]{64}$")) + and (.components.updater.digest | test("^sha256:[0-9a-f]{64}$")) + ' "${manifest}" >/dev/null + +core_ref="$(jq -r '.components.core.image' "${manifest}")" +updater_ref="$(jq -r '.components.updater.image' "${manifest}")" +expected_core="$(jq -r '.components.core.digest' "${manifest}")" +expected_updater="$(jq -r '.components.updater.digest' "${manifest}")" +actual_core="$(scripts/inspect-image-digest.sh "${core_ref}")" +actual_updater="$(scripts/inspect-image-digest.sh "${updater_ref}")" + +[ "${actual_core}" = "${expected_core}" ] || { + echo "Core tag moved: ${actual_core} != ${expected_core}" >&2 + exit 1 +} +[ "${actual_updater}" = "${expected_updater}" ] || { + echo "updater tag moved: ${actual_updater} != ${expected_updater}" >&2 + exit 1 +} diff --git a/web/release-metadata.test.mjs b/web/release-metadata.test.mjs index 1179637c..0a1658b5 100644 --- a/web/release-metadata.test.mjs +++ b/web/release-metadata.test.mjs @@ -12,6 +12,18 @@ const releaseWorkflow = readFileSync( join(repoRoot, ".github", "workflows", "release.yml"), "utf8", ); +const betaWorkflow = readFileSync( + join(repoRoot, ".github", "workflows", "beta.yml"), + "utf8", +); +const releaseAssetsWorkflow = readFileSync( + join(repoRoot, ".github", "workflows", "release-assets.yml"), + "utf8", +); +const localReleaseScript = readFileSync( + join(repoRoot, "scripts", "release.sh"), + "utf8", +); const changesetCheckWorkflow = readFileSync( join(repoRoot, ".github", "workflows", "changeset-check.yml"), "utf8", @@ -40,4 +52,28 @@ describe("release metadata", () => { /changeset status --since=["']origin\/\$\{\{ github\.base_ref \}\}["']/, ); }); + + it("keeps stable draft-only until the control-plane pair passes", () => { + assert.match(releaseWorkflow, /gh release create "\$\{TAG\}"[\s\S]+--draft/); + assert.match(releaseWorkflow, /verify-control-plane-manifest\.sh[\s\S]+"\$\{BETA_TAG\}"/); + assert.match(releaseAssetsWorkflow, /name: verify pair and publish release/); + assert.match(releaseAssetsWorkflow, /verify-control-plane-manifest\.sh[\s\S]+gh release edit "\$\{TAG\}"[\s\S]+--draft=false/); + assert.match(releaseAssetsWorkflow, /needs: \[meta, control-plane\]/); + }); + + it("checks the same-release Core and updater digests", () => { + for (const workflow of [betaWorkflow, releaseAssetsWorkflow]) { + assert.match(workflow, /create-control-plane-manifest\.sh/); + assert.match(workflow, /verify-control-plane-manifest\.sh/); + assert.match(workflow, /ftw-control-plane\.json/); + assert.match(workflow, /Refuse to move an existing immutable tag/); + } + }); + + it("does not let the local helper create or publish releases", () => { + assert.doesNotMatch(localReleaseScript, /gh release create/); + assert.doesNotMatch(localReleaseScript, /gh release edit/); + assert.match(localReleaseScript, /isDraft/); + assert.match(localReleaseScript, /ftw-control-plane\.json/); + }); });