srcfl · frahlg · Jul 18, 2026
diff --git a/.changeset/atomic-control-plane-pair.md b/.changeset/atomic-control-plane-pair.md
@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Make Core and updater an atomic versioned control-plane pair. Release checks now require a shared manifest with both immutable image digests, Core stays unready until the matching updater is present, and paired updates retain and restore both previous image IDs on failure.
diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml
@@ -94,8 +94,31 @@ jobs:
           username: ${{ vars.SOURCEFUL_GHCR_USER || github.actor }}
           password: ${{ secrets.SOURCEFUL_GHCR_TOKEN || secrets.GITHUB_TOKEN }}
 
+      - name: Refuse to move an existing immutable tag
+        id: immutable
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IMAGE: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}:${{ needs.tag.outputs.version }}
+          TAG: ${{ needs.tag.outputs.version }}
+        run: |
+          set -euo pipefail
+          if ! scripts/inspect-image-digest.sh "${IMAGE}" >/dev/null 2>&1; then
+            echo "skip=false" >> "${GITHUB_OUTPUT}"
+            exit 0
+          fi
+          # A complete prior run is idempotent. A partial exact tag has no
+          # authoritative pair manifest and must be removed/reviewed manually;
+          # rebuilding must never move it silently.
+          mkdir -p existing
+          gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \
+            --pattern ftw-control-plane.json --dir existing
+          scripts/verify-control-plane-manifest.sh \
+            existing/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}"
+          echo "skip=true" >> "${GITHUB_OUTPUT}"
+
       - name: Build and publish beta image
         id: build
+        if: steps.immutable.outputs.skip != 'true'
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -118,13 +141,15 @@ jobs:
           cache-to: type=gha,mode=max,scope=${{ matrix.target }}
 
       - name: Login to compatibility GHCR namespace
+        if: steps.immutable.outputs.skip != 'true'
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: frahlg
           password: ${{ secrets.LEGACY_GHCR_TOKEN || secrets.GITHUB_TOKEN }}
 
       - name: Mirror exact beta manifests to compatibility namespace
+        if: steps.immutable.outputs.skip != 'true'
         env:
           IMAGE_SUFFIX: ${{ matrix.image_suffix }}
           SOURCE_DIGEST: ${{ steps.build.outputs.digest }}
@@ -140,23 +165,61 @@ jobs:
             test "${SOURCE_DIGEST}" = "${legacy_digest}"
           done
 
+  control-plane:
+    name: verify immutable Core/updater pair
+    runs-on: ubuntu-latest
+    needs: [tag, docker]
+    steps:
+      - name: Checkout beta tag
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ needs.tag.outputs.version }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build paired digest manifest
+        env:
+          TAG: ${{ needs.tag.outputs.version }}
+        run: scripts/create-control-plane-manifest.sh "${TAG}" "${GITHUB_SHA}" ftw-control-plane.json
+
+      - name: Re-verify both immutable tags
+        env:
+          TAG: ${{ needs.tag.outputs.version }}
+        run: scripts/verify-control-plane-manifest.sh ftw-control-plane.json "${TAG}" "${GITHUB_SHA}"
+
+      - name: Upload pair manifest for release job
+        uses: actions/upload-artifact@v4
+        with:
+          name: beta-control-plane-${{ needs.tag.outputs.version }}
+          path: ftw-control-plane.json
+          if-no-files-found: error
+
   release:
     name: publish GitHub prerelease
     runs-on: ubuntu-latest
-    needs: [tag, docker]
+    needs: [tag, control-plane]
     steps:
+      - name: Download verified pair manifest
+        uses: actions/download-artifact@v4
+        with:
+          name: beta-control-plane-${{ needs.tag.outputs.version }}
+
       - name: Publish prerelease after images are available
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TAG: ${{ needs.tag.outputs.version }}
         run: |
           set -euo pipefail
           if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
-            echo "GitHub prerelease ${TAG} already exists"
+            gh release upload "${TAG}" ftw-control-plane.json \
+              --repo "${GITHUB_REPOSITORY}" --clobber
+            echo "Verified GitHub prerelease ${TAG} already exists"
             exit 0
           fi
           gh release create "${TAG}" \
             --repo "${GITHUB_REPOSITORY}" \
             --title "${TAG}" \
             --prerelease \
-            --generate-notes
+            --generate-notes \
+            ftw-control-plane.json
diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml
@@ -54,6 +54,11 @@ jobs:
       tag:     ${{ steps.meta.outputs.tag }}
       version: ${{ steps.meta.outputs.version }}
     steps:
+      - name: Checkout stable tag
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
       - name: Derive tag/version from ref
         id: meta
         env:
@@ -81,6 +86,30 @@ jobs:
           echo "version=${VERSION}" >> "${GITHUB_OUTPUT}"
           echo "Building assets for ${TAG} (version ${VERSION})"
 
+      - name: Set up Docker Buildx for promotion gate
+        uses: docker/setup-buildx-action@v3
+
+      - name: Repeat beta-to-stable pair gate
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ steps.meta.outputs.tag }}
+        run: |
+          set -euo pipefail
+          TAG_COMMIT="$(git rev-list -n 1 "${TAG}")"
+          test "${TAG_COMMIT}" = "${GITHUB_SHA}"
+          BETA_TAG="$(git tag --list "${TAG}-beta.*" --sort=-v:refname | head -n 1)"
+          test -n "${BETA_TAG}"
+          BETA_COMMIT="$(git rev-list -n 1 "${BETA_TAG}")"
+          test "${BETA_COMMIT}" = "${GITHUB_SHA}"
+          RELEASE_JSON="$(gh release view "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft,isPrerelease)"
+          test "$(jq -r .isDraft <<<"${RELEASE_JSON}")" = false
+          test "$(jq -r .isPrerelease <<<"${RELEASE_JSON}")" = true
+          mkdir beta-control-plane
+          gh release download "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" \
+            --pattern ftw-control-plane.json --dir beta-control-plane
+          scripts/verify-control-plane-manifest.sh \
+            beta-control-plane/ftw-control-plane.json "${BETA_TAG}" "${GITHUB_SHA}"
+
   binaries:
     name: build + upload release binaries
     runs-on: ubuntu-latest
@@ -249,8 +278,28 @@ jobs:
           username: ${{ vars.SOURCEFUL_GHCR_USER || github.actor }}
           password: ${{ secrets.SOURCEFUL_GHCR_TOKEN || secrets.GITHUB_TOKEN }}
 
+      - name: Refuse to move an existing immutable tag
+        id: immutable
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IMAGE: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}:${{ needs.meta.outputs.tag }}
+          TAG: ${{ needs.meta.outputs.tag }}
+        run: |
+          set -euo pipefail
+          if ! scripts/inspect-image-digest.sh "${IMAGE}" >/dev/null 2>&1; then
+            echo "skip=false" >> "${GITHUB_OUTPUT}"
+            exit 0
+          fi
+          mkdir -p existing
+          gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \
+            --pattern ftw-control-plane.json --dir existing
+          scripts/verify-control-plane-manifest.sh \
+            existing/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}"
+          echo "skip=true" >> "${GITHUB_OUTPUT}"
+
       - name: Compute image metadata
         id: meta
+        if: steps.immutable.outputs.skip != 'true'
         uses: docker/metadata-action@v5
         with:
           images: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}
@@ -263,11 +312,13 @@ jobs:
             org.opencontainers.image.title=${{ matrix.title }}
             org.opencontainers.image.description=${{ matrix.description }}
             org.opencontainers.image.source=https://github.com/srcfl/ftw
+            org.opencontainers.image.revision=${{ github.sha }}
             org.opencontainers.image.version=${{ needs.meta.outputs.version }}
             org.opencontainers.image.licenses=Apache-2.0
 
       - name: Build and push
         id: build
+        if: steps.immutable.outputs.skip != 'true'
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -289,13 +340,15 @@ jobs:
       # canonical namespace, then copy that exact multi-platform manifest to
       # the temporary compatibility namespace under its own credentials.
       - name: Login to compatibility GHCR namespace
+        if: steps.immutable.outputs.skip != 'true'
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: frahlg
           password: ${{ secrets.LEGACY_GHCR_TOKEN || secrets.GITHUB_TOKEN }}
 
       - name: Mirror exact manifests to compatibility namespace
+        if: steps.immutable.outputs.skip != 'true'
         env:
           CANONICAL_TAGS: ${{ steps.meta.outputs.tags }}
           SOURCE_DIGEST: ${{ steps.build.outputs.digest }}
@@ -309,13 +362,55 @@ jobs:
             test "${SOURCE_DIGEST}" = "${legacy_digest}"
           done
 
+  control-plane:
+    name: verify pair and publish release
+    runs-on: ubuntu-latest
+    needs: [meta, binaries, docker, imager-metadata]
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout stable tag
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ needs.meta.outputs.tag }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create and verify immutable pair manifest
+        env:
+          TAG: ${{ needs.meta.outputs.tag }}
+        run: |
+          set -euo pipefail
+          scripts/create-control-plane-manifest.sh "${TAG}" "${GITHUB_SHA}" ftw-control-plane.json
+          scripts/verify-control-plane-manifest.sh ftw-control-plane.json "${TAG}" "${GITHUB_SHA}"
+
+      - name: Attach pair manifest and make draft public
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ needs.meta.outputs.tag }}
+        run: |
+          set -euo pipefail
+          gh release upload "${TAG}" ftw-control-plane.json \
+            --repo "${GITHUB_REPOSITORY}" --clobber
+          IS_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)"
+          if [ "${IS_DRAFT}" = true ]; then
+            gh release edit "${TAG}" --repo "${GITHUB_REPOSITORY}" --draft=false
+          fi
+          # Re-download what operators will consume and verify it one last time.
+          mkdir published
+          gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \
+            --pattern ftw-control-plane.json --dir published
+          scripts/verify-control-plane-manifest.sh \
+            published/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}"
+
   discord:
     name: announce release on Discord
     runs-on: ubuntu-latest
     # Wait for binaries + docker so the message links to assets that
     # actually exist. The independent rpi-installer channel is intentionally
     # outside the application release critical path.
-    needs: [meta, binaries, docker]
+    needs: [meta, control-plane]
     permissions:
       contents: read   # gh release view needs read access
     steps:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -2,8 +2,8 @@ name: release
 
 # Changesets-driven release. Pattern lifted from umara/u-front.
 #
-# This workflow ONLY handles the changesets flow + tag + GitHub Release
-# creation. The cross-platform binaries, docker images, and
+# This workflow ONLY handles the changesets flow + tag + draft GitHub Release
+# creation. The cross-platform binaries, docker images, pair verification and
 # Discord announcement live in `release-assets.yml` and are triggered
 # explicitly via `gh workflow run release-assets.yml --ref vX.Y.Z` at
 # the end of the publish step. Keeping the heavy build matrix out of
@@ -123,6 +123,10 @@ jobs:
           # needs broader scope, and a narrow token bounds the blast radius.
           GITHUB_TOKEN: ${{ secrets.CI_TOKEN }}
 
+      - name: Set up Docker Buildx for promotion verification
+        if: github.event_name == 'workflow_dispatch' && steps.changesets.outputs.hasChangesets == 'false'
+        uses: docker/setup-buildx-action@v3
+
       - name: Tag and create GitHub Release
         id: publish
         # Stable is an explicit promotion after real beta validation. Pushes
@@ -141,14 +145,16 @@ jobs:
             TAG_EXISTS=true
           fi
           RELEASE_EXISTS=false
+          RELEASE_DRAFT=false
           if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
             RELEASE_EXISTS=true
+            RELEASE_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)"
           fi
 
           # Already-done case: a previous run completed cleanly. No
           # new release was cut on THIS run, so the release-assets
           # dispatch below should skip — emit `published=false`.
-          if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ]; then
+          if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ] && [ "${RELEASE_DRAFT}" = "false" ]; then
             echo "Tag ${TAG} and GitHub Release already exist — nothing to publish."
             {
               echo "published=false"
@@ -157,6 +163,15 @@ jobs:
             exit 0
           fi
 
+          if [ "${TAG_EXISTS}" = "true" ]; then
+            git fetch --force origin "refs/tags/${TAG}:refs/tags/${TAG}"
+            TAG_COMMIT="$(git rev-list -n 1 "${TAG}")"
+            if [ "${TAG_COMMIT}" != "${GITHUB_SHA}" ]; then
+              echo "Existing ${TAG} points to ${TAG_COMMIT}, not ${GITHUB_SHA}; refusing to move it." >&2
+              exit 1
+            fi
+          fi
+
           # Stable is a promotion, never the first public channel for a commit.
           # beta.yml also checks that the prerelease base matches package.json.
           git fetch --force --tags origin
@@ -171,6 +186,18 @@ jobs:
             echo "Publish a beta from this exact commit before promotion." >&2
             exit 1
           fi
+          BETA_RELEASE_JSON="$(gh release view "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft,isPrerelease)"
+          if [ "$(jq -r .isDraft <<<"${BETA_RELEASE_JSON}")" != "false" ] || \
+             [ "$(jq -r .isPrerelease <<<"${BETA_RELEASE_JSON}")" != "true" ]; then
+            echo "${BETA_TAG} is not a published beta prerelease." >&2
+            exit 1
+          fi
+          rm -rf beta-control-plane
+          mkdir beta-control-plane
+          gh release download "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" \
+            --pattern ftw-control-plane.json --dir beta-control-plane
+          scripts/verify-control-plane-manifest.sh \
+            beta-control-plane/ftw-control-plane.json "${BETA_TAG}" "${GITHUB_SHA}"
           echo "Promoting tested ${BETA_TAG} to ${TAG}."
 
           # Build codename-annotated release notes. The script extracts
@@ -196,14 +223,18 @@ jobs:
             gh release create "${TAG}" \
               --repo "${GITHUB_REPOSITORY}" \
               --title "${TAG}" \
-              --notes-file release-notes.md
+              --notes-file release-notes.md \
+              --draft
+          elif [ "${RELEASE_DRAFT}" != "true" ]; then
+            echo "Existing release ${TAG} is public but was not recognized as complete." >&2
+            exit 1
           fi
 
           {
             echo "published=true"
             echo "version=${VERSION}"
           } >> "${GITHUB_OUTPUT}"
-          echo "Published ${TAG}"
+          echo "Prepared draft ${TAG}; release-assets will publish only after pair verification"
 
       - name: Dispatch release-assets workflow
         # GitHub does NOT fire downstream `push: tags` workflows from

diff --git a/Dockerfile b/Dockerfile
@@ -90,8 +90,11 @@ EXPOSE 8080
 # Without this the binary fails fast with "open state … unable to
 # open database file" because SQLite can't create state.db inside
 # a directory it doesn't own.
-HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=12 \
-  CMD wget -q -T 4 -O /dev/null http://127.0.0.1:8080/api/health || exit 1
+# Readiness, not the boot-phase health body, is the commit gate for Core.
+# State integrity checks and migrations can legitimately take close to 30 min
+# on large SD-card databases, so keep Docker in "starting" for that window.
+HEALTHCHECK --interval=10s --timeout=5s --start-period=30m --retries=12 \
+  CMD wget -q -T 4 -O /dev/null http://127.0.0.1:8080/api/status || exit 1
 
 ENTRYPOINT ["/app/ftw"]
 CMD ["-config", "/app/data/config.yaml", "-web", "/app/web", "-drivers", "/app/drivers", "-user-drivers", "/app/data/drivers"]