diff --git a/.changeset/atomic-control-plane-pair.md b/.changeset/atomic-control-plane-pair.md new file mode 100644 index 00000000..1f46f229 --- /dev/null +++ b/.changeset/atomic-control-plane-pair.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Make Core and updater an atomic versioned control-plane pair. Release checks now require a shared manifest with both immutable image digests, Core stays unready until the matching updater is present, and paired updates retain and restore both previous image IDs on failure. diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index aedf4a84..5be7b689 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -94,8 +94,31 @@ jobs: username: ${{ vars.SOURCEFUL_GHCR_USER || github.actor }} password: ${{ secrets.SOURCEFUL_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + - name: Refuse to move an existing immutable tag + id: immutable + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}:${{ needs.tag.outputs.version }} + TAG: ${{ needs.tag.outputs.version }} + run: | + set -euo pipefail + if ! scripts/inspect-image-digest.sh "${IMAGE}" >/dev/null 2>&1; then + echo "skip=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + # A complete prior run is idempotent. A partial exact tag has no + # authoritative pair manifest and must be removed/reviewed manually; + # rebuilding must never move it silently. + mkdir -p existing + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir existing + scripts/verify-control-plane-manifest.sh \ + existing/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + echo "skip=true" >> "${GITHUB_OUTPUT}" + - name: Build and publish beta image id: build + if: steps.immutable.outputs.skip != 'true' uses: docker/build-push-action@v6 with: context: . @@ -118,6 +141,7 @@ jobs: cache-to: type=gha,mode=max,scope=${{ matrix.target }} - name: Login to compatibility GHCR namespace + if: steps.immutable.outputs.skip != 'true' uses: docker/login-action@v3 with: registry: ghcr.io @@ -125,6 +149,7 @@ jobs: password: ${{ secrets.LEGACY_GHCR_TOKEN || secrets.GITHUB_TOKEN }} - name: Mirror exact beta manifests to compatibility namespace + if: steps.immutable.outputs.skip != 'true' env: IMAGE_SUFFIX: ${{ matrix.image_suffix }} SOURCE_DIGEST: ${{ steps.build.outputs.digest }} @@ -140,11 +165,46 @@ jobs: test "${SOURCE_DIGEST}" = "${legacy_digest}" done + control-plane: + name: verify immutable Core/updater pair + runs-on: ubuntu-latest + needs: [tag, docker] + steps: + - name: Checkout beta tag + uses: actions/checkout@v5 + with: + ref: ${{ needs.tag.outputs.version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build paired digest manifest + env: + TAG: ${{ needs.tag.outputs.version }} + run: scripts/create-control-plane-manifest.sh "${TAG}" "${GITHUB_SHA}" ftw-control-plane.json + + - name: Re-verify both immutable tags + env: + TAG: ${{ needs.tag.outputs.version }} + run: scripts/verify-control-plane-manifest.sh ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + + - name: Upload pair manifest for release job + uses: actions/upload-artifact@v4 + with: + name: beta-control-plane-${{ needs.tag.outputs.version }} + path: ftw-control-plane.json + if-no-files-found: error + release: name: publish GitHub prerelease runs-on: ubuntu-latest - needs: [tag, docker] + needs: [tag, control-plane] steps: + - name: Download verified pair manifest + uses: actions/download-artifact@v4 + with: + name: beta-control-plane-${{ needs.tag.outputs.version }} + - name: Publish prerelease after images are available env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -152,11 +212,14 @@ jobs: run: | set -euo pipefail if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then - echo "GitHub prerelease ${TAG} already exists" + gh release upload "${TAG}" ftw-control-plane.json \ + --repo "${GITHUB_REPOSITORY}" --clobber + echo "Verified GitHub prerelease ${TAG} already exists" exit 0 fi gh release create "${TAG}" \ --repo "${GITHUB_REPOSITORY}" \ --title "${TAG}" \ --prerelease \ - --generate-notes + --generate-notes \ + ftw-control-plane.json diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 1c4a0b7f..862dec09 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -54,6 +54,11 @@ jobs: tag: ${{ steps.meta.outputs.tag }} version: ${{ steps.meta.outputs.version }} steps: + - name: Checkout stable tag + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Derive tag/version from ref id: meta env: @@ -81,6 +86,30 @@ jobs: echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" echo "Building assets for ${TAG} (version ${VERSION})" + - name: Set up Docker Buildx for promotion gate + uses: docker/setup-buildx-action@v3 + + - name: Repeat beta-to-stable pair gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.meta.outputs.tag }} + run: | + set -euo pipefail + TAG_COMMIT="$(git rev-list -n 1 "${TAG}")" + test "${TAG_COMMIT}" = "${GITHUB_SHA}" + BETA_TAG="$(git tag --list "${TAG}-beta.*" --sort=-v:refname | head -n 1)" + test -n "${BETA_TAG}" + BETA_COMMIT="$(git rev-list -n 1 "${BETA_TAG}")" + test "${BETA_COMMIT}" = "${GITHUB_SHA}" + RELEASE_JSON="$(gh release view "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft,isPrerelease)" + test "$(jq -r .isDraft <<<"${RELEASE_JSON}")" = false + test "$(jq -r .isPrerelease <<<"${RELEASE_JSON}")" = true + mkdir beta-control-plane + gh release download "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir beta-control-plane + scripts/verify-control-plane-manifest.sh \ + beta-control-plane/ftw-control-plane.json "${BETA_TAG}" "${GITHUB_SHA}" + binaries: name: build + upload release binaries runs-on: ubuntu-latest @@ -249,8 +278,28 @@ jobs: username: ${{ vars.SOURCEFUL_GHCR_USER || github.actor }} password: ${{ secrets.SOURCEFUL_GHCR_TOKEN || secrets.GITHUB_TOKEN }} + - name: Refuse to move an existing immutable tag + id: immutable + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE: ghcr.io/srcfl/ftw${{ matrix.image_suffix }}:${{ needs.meta.outputs.tag }} + TAG: ${{ needs.meta.outputs.tag }} + run: | + set -euo pipefail + if ! scripts/inspect-image-digest.sh "${IMAGE}" >/dev/null 2>&1; then + echo "skip=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + mkdir -p existing + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir existing + scripts/verify-control-plane-manifest.sh \ + existing/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + echo "skip=true" >> "${GITHUB_OUTPUT}" + - name: Compute image metadata id: meta + if: steps.immutable.outputs.skip != 'true' uses: docker/metadata-action@v5 with: images: ghcr.io/srcfl/ftw${{ matrix.image_suffix }} @@ -263,11 +312,13 @@ jobs: org.opencontainers.image.title=${{ matrix.title }} org.opencontainers.image.description=${{ matrix.description }} org.opencontainers.image.source=https://github.com/srcfl/ftw + org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.version=${{ needs.meta.outputs.version }} org.opencontainers.image.licenses=Apache-2.0 - name: Build and push id: build + if: steps.immutable.outputs.skip != 'true' uses: docker/build-push-action@v6 with: context: . @@ -289,6 +340,7 @@ jobs: # canonical namespace, then copy that exact multi-platform manifest to # the temporary compatibility namespace under its own credentials. - name: Login to compatibility GHCR namespace + if: steps.immutable.outputs.skip != 'true' uses: docker/login-action@v3 with: registry: ghcr.io @@ -296,6 +348,7 @@ jobs: password: ${{ secrets.LEGACY_GHCR_TOKEN || secrets.GITHUB_TOKEN }} - name: Mirror exact manifests to compatibility namespace + if: steps.immutable.outputs.skip != 'true' env: CANONICAL_TAGS: ${{ steps.meta.outputs.tags }} SOURCE_DIGEST: ${{ steps.build.outputs.digest }} @@ -309,13 +362,55 @@ jobs: test "${SOURCE_DIGEST}" = "${legacy_digest}" done + control-plane: + name: verify pair and publish release + runs-on: ubuntu-latest + needs: [meta, binaries, docker, imager-metadata] + permissions: + contents: write + steps: + - name: Checkout stable tag + uses: actions/checkout@v5 + with: + ref: ${{ needs.meta.outputs.tag }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create and verify immutable pair manifest + env: + TAG: ${{ needs.meta.outputs.tag }} + run: | + set -euo pipefail + scripts/create-control-plane-manifest.sh "${TAG}" "${GITHUB_SHA}" ftw-control-plane.json + scripts/verify-control-plane-manifest.sh ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + + - name: Attach pair manifest and make draft public + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.meta.outputs.tag }} + run: | + set -euo pipefail + gh release upload "${TAG}" ftw-control-plane.json \ + --repo "${GITHUB_REPOSITORY}" --clobber + IS_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)" + if [ "${IS_DRAFT}" = true ]; then + gh release edit "${TAG}" --repo "${GITHUB_REPOSITORY}" --draft=false + fi + # Re-download what operators will consume and verify it one last time. + mkdir published + gh release download "${TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir published + scripts/verify-control-plane-manifest.sh \ + published/ftw-control-plane.json "${TAG}" "${GITHUB_SHA}" + discord: name: announce release on Discord runs-on: ubuntu-latest # Wait for binaries + docker so the message links to assets that # actually exist. The independent rpi-installer channel is intentionally # outside the application release critical path. - needs: [meta, binaries, docker] + needs: [meta, control-plane] permissions: contents: read # gh release view needs read access steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68732c5d..3042aaba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: release # Changesets-driven release. Pattern lifted from umara/u-front. # -# This workflow ONLY handles the changesets flow + tag + GitHub Release -# creation. The cross-platform binaries, docker images, and +# This workflow ONLY handles the changesets flow + tag + draft GitHub Release +# creation. The cross-platform binaries, docker images, pair verification and # Discord announcement live in `release-assets.yml` and are triggered # explicitly via `gh workflow run release-assets.yml --ref vX.Y.Z` at # the end of the publish step. Keeping the heavy build matrix out of @@ -123,6 +123,10 @@ jobs: # needs broader scope, and a narrow token bounds the blast radius. GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} + - name: Set up Docker Buildx for promotion verification + if: github.event_name == 'workflow_dispatch' && steps.changesets.outputs.hasChangesets == 'false' + uses: docker/setup-buildx-action@v3 + - name: Tag and create GitHub Release id: publish # Stable is an explicit promotion after real beta validation. Pushes @@ -141,14 +145,16 @@ jobs: TAG_EXISTS=true fi RELEASE_EXISTS=false + RELEASE_DRAFT=false if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then RELEASE_EXISTS=true + RELEASE_DRAFT="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)" fi # Already-done case: a previous run completed cleanly. No # new release was cut on THIS run, so the release-assets # dispatch below should skip — emit `published=false`. - if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ]; then + if [ "${TAG_EXISTS}" = "true" ] && [ "${RELEASE_EXISTS}" = "true" ] && [ "${RELEASE_DRAFT}" = "false" ]; then echo "Tag ${TAG} and GitHub Release already exist — nothing to publish." { echo "published=false" @@ -157,6 +163,15 @@ jobs: exit 0 fi + if [ "${TAG_EXISTS}" = "true" ]; then + git fetch --force origin "refs/tags/${TAG}:refs/tags/${TAG}" + TAG_COMMIT="$(git rev-list -n 1 "${TAG}")" + if [ "${TAG_COMMIT}" != "${GITHUB_SHA}" ]; then + echo "Existing ${TAG} points to ${TAG_COMMIT}, not ${GITHUB_SHA}; refusing to move it." >&2 + exit 1 + fi + fi + # Stable is a promotion, never the first public channel for a commit. # beta.yml also checks that the prerelease base matches package.json. git fetch --force --tags origin @@ -171,6 +186,18 @@ jobs: echo "Publish a beta from this exact commit before promotion." >&2 exit 1 fi + BETA_RELEASE_JSON="$(gh release view "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft,isPrerelease)" + if [ "$(jq -r .isDraft <<<"${BETA_RELEASE_JSON}")" != "false" ] || \ + [ "$(jq -r .isPrerelease <<<"${BETA_RELEASE_JSON}")" != "true" ]; then + echo "${BETA_TAG} is not a published beta prerelease." >&2 + exit 1 + fi + rm -rf beta-control-plane + mkdir beta-control-plane + gh release download "${BETA_TAG}" --repo "${GITHUB_REPOSITORY}" \ + --pattern ftw-control-plane.json --dir beta-control-plane + scripts/verify-control-plane-manifest.sh \ + beta-control-plane/ftw-control-plane.json "${BETA_TAG}" "${GITHUB_SHA}" echo "Promoting tested ${BETA_TAG} to ${TAG}." # Build codename-annotated release notes. The script extracts @@ -196,14 +223,18 @@ jobs: gh release create "${TAG}" \ --repo "${GITHUB_REPOSITORY}" \ --title "${TAG}" \ - --notes-file release-notes.md + --notes-file release-notes.md \ + --draft + elif [ "${RELEASE_DRAFT}" != "true" ]; then + echo "Existing release ${TAG} is public but was not recognized as complete." >&2 + exit 1 fi { echo "published=true" echo "version=${VERSION}" } >> "${GITHUB_OUTPUT}" - echo "Published ${TAG}" + echo "Prepared draft ${TAG}; release-assets will publish only after pair verification" - name: Dispatch release-assets workflow # GitHub does NOT fire downstream `push: tags` workflows from diff --git a/Dockerfile b/Dockerfile index 8d5164e3..fbdd6ff1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,8 +90,11 @@ EXPOSE 8080 # Without this the binary fails fast with "open state … unable to # open database file" because SQLite can't create state.db inside # a directory it doesn't own. -HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=12 \ - CMD wget -q -T 4 -O /dev/null http://127.0.0.1:8080/api/health || exit 1 +# Readiness, not the boot-phase health body, is the commit gate for Core. +# State integrity checks and migrations can legitimately take close to 30 min +# on large SD-card databases, so keep Docker in "starting" for that window. +HEALTHCHECK --interval=10s --timeout=5s --start-period=30m --retries=12 \ + CMD wget -q -T 4 -O /dev/null http://127.0.0.1:8080/api/status || exit 1 ENTRYPOINT ["/app/ftw"] CMD ["-config", "/app/data/config.yaml", "-web", "/app/web", "-drivers", "/app/drivers", "-user-drivers", "/app/data/drivers"] diff --git a/Makefile b/Makefile index 6fcdaad7..0e196c5f 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ # make dev — start sims + main app (hot-reload workflow) # make clean — remove all build artifacts -.PHONY: help test optimizer-install optimizer-test compose-migration-test container-boundary-test build build-arm64 build-amd64 build-windows-amd64 release \ +.PHONY: help test optimizer-install optimizer-test compose-migration-test container-boundary-test release-contract-test build build-arm64 build-amd64 build-windows-amd64 release \ run-sim dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions @@ -71,6 +71,9 @@ compose-migration-test: container-boundary-test: bash scripts/test-container-boundaries.sh +release-contract-test: + bash scripts/test-control-plane-release.sh + optimizer/.venv/.installed: optimizer/pyproject.toml $(MAKE) optimizer-install @@ -102,7 +105,7 @@ ci-hw-pi: # verify-all adds cross-compile checks for all release targets, catching # platform-specific syscall/import mistakes before push. -verify: test compose-migration-test container-boundary-test +verify: test compose-migration-test container-boundary-test release-contract-test cd go && go vet ./... cd go && go build ./... @echo "verify: vet + test + build clean" diff --git a/docs/self-update.md b/docs/self-update.md index 1ca8e75f..ab82c252 100644 --- a/docs/self-update.md +++ b/docs/self-update.md @@ -20,19 +20,27 @@ After that PR merges: 1. run `beta.yml` with `vX.Y.Z-beta.N`; 2. validate that immutable build on real sites; 3. manually dispatch `release.yml` from that same commit; -4. stable promotion verifies that a matching beta tag resolves to the exact - stable candidate commit; -5. release assets publish `vX.Y.Z` and move the stable aliases. +4. stable promotion verifies the published beta pair manifest, both live + immutable digests and the exact candidate commit; +5. stable is created as a draft; release assets publish `vX.Y.Z` and move the + stable aliases only after the Core/updater pair has been re-verified. Stable therefore cannot be the first public channel for new code. Beta and stable may have different tags but identify the same source commit. ## Immutable update targets -The checker uses GitHub Releases to select a released version and GHCR to prove -that its exact image tag exists. The updater installs the immutable tag, never -the moving `:latest` or `:beta` alias. This avoids the race where a release -exists before a moving image alias has advanced. +Every Core release carries `ftw-control-plane.json`. It names the release and +source commit and pins the immutable manifest digests for both +`ghcr.io/srcfl/ftw:` and `ghcr.io/srcfl/ftw-updater:`. The checker +announces an update only when the public GitHub Release contains this manifest +and both live tag digests still match it. A missing updater tag, missing +manifest or moved digest is fail-closed. + +The updater installs the immutable tag for both services, never the moving +`:latest` or `:beta` alias. Existing exact tags are not overwritten by release +reruns. Stable remains a draft and is invisible to update checkers until the +pair gate succeeds. Release notes are best-effort UI data. Failure to fetch notes does not weaken tag resolution or image verification. @@ -54,6 +62,12 @@ The main process checks versions and exposes update status. A separate pull/recreate operations over a Unix socket. Core never mounts the Docker socket. +Core and updater expose a versioned control-plane handshake. When self-update +is enabled, Core verifies the updater protocol and exact release version before +opening or migrating `state.db`; health and readiness remain 503 until they +match. This is the compatibility barrier that prevents an older updater from +installing a new Core by itself. + Before every Core update, Core creates a mandatory, consistent local rollback point for `state.db` and configuration. An older client request cannot skip it. These bounded points remain on the same disk and are deliberately labelled @@ -80,6 +94,13 @@ system component changes. Status is written atomically to the shared volume and is reconciled into the persistent component history after Core recreation. +A Core update captures both running image IDs, pulls both immutable targets, +then delegates the replacement to a detached transaction helper. The helper +survives updater recreation, starts the matching updater before Core, requires +full Core readiness, and writes `done` only after both running image references +match the requested release. Failure restores the previous updater first and +then the previous Core from the two retained image IDs. + The updater accepts only known components and `vX.Y.Z` or `vX.Y.Z-beta.N` targets. @@ -89,12 +110,17 @@ The version badge selects `stable` or `beta`, checks availability and starts an update. Changing channel does not deploy anything. A skipped version remains hidden only until a newer version appears. -For manual Core + updater operation: +For manual Core + updater reconciliation, select one verified release and use +the same immutable tag for both services (replace the example tag): ```bash cd ~/ftw +export FTW_IMAGE_TAG=v1.4.1-beta.1 +export FTW_UPDATER_IMAGE_TAG="$FTW_IMAGE_TAG" docker compose pull ftw ftw-updater -docker compose up -d --no-deps ftw ftw-updater +docker compose up -d --no-deps ftw-updater +docker compose up -d --no-deps ftw +curl -fsS http://127.0.0.1:8080/api/status ``` Manage Optimizer and Drivers independently in Update Center. A blanket @@ -103,6 +129,26 @@ Manage Optimizer and Drivers independently in Update Center. A blanket Use the [legacy upgrade guide](upgrade-from-legacy.md) before updating an older Compose layout with hard-coded or pre-FTW image names. +## Compatibility and first rollout gate + +| Running Core | Running updater | Result | +|---|---|---| +| v1.3.1 | v1.3.1 | Legacy behavior; cannot install the paired release atomically | +| v1.3.1 | first fixed release | Pair-capable updater can install the next matched pair | +| first fixed release | v1.3.1 | Core stays unready and does not migrate; the v1.3.1 update attempt times out and restores old Core | +| fixed version A | fixed version B | Core stays unready until both versions match | +| same fixed version | same fixed version | Paired in-app update, readiness gate and two-image rollback are supported | + +The first fixed release is a bootstrap boundary. Before stable promotion: + +1. publish a beta whose pair manifest and both digests pass the release gate; +2. verify a v1.3.1 in-app attempt restores old Core without migrating data; +3. manually reconcile a test site to the same beta tag for Core and updater; +4. test beta-to-beta in-app update with injected updater, Core and readiness + failures and confirm both prior image IDs are restored; +5. promote the exact tested beta commit. Do not make stable public when any of + these gates is missing. + ## Enabling The shipped Linux Compose topology sets `FTW_SELFUPDATE_ENABLED=1` and mounts diff --git a/docs/upgrade-from-legacy.md b/docs/upgrade-from-legacy.md index 3e97e76d..a8e85536 100644 --- a/docs/upgrade-from-legacy.md +++ b/docs/upgrade-from-legacy.md @@ -21,7 +21,9 @@ Kör från installationskatalogen: ```bash curl -fsSL https://raw.githubusercontent.com/srcfl/ftw/master/scripts/migrate-legacy-compose.sh \ -o /tmp/ftw-migrate.sh +RELEASE=v1.4.1-beta.1 # ersätt med den verifierade release som operatören anger bash /tmp/ftw-migrate.sh \ + --version "$RELEASE" \ --dir "$PWD" \ --backup-dir /media/$USER/FTW-BACKUP ``` @@ -36,7 +38,8 @@ migreringen. Skriptet letar annars i aktuell katalog, `~/ftw` och 1. **Full backup.** Den nya backuphjälparen öppnar den äldre databasen skrivskyddat, gör ingen schemamigrering, bygger en komplett `.ftwbak`, verifierar filhashar och SQLite och stoppar vid minsta fel. -2. **Core + updater.** Det parade kontrollplanet hämtas och återskapas med +2. **Core + updater.** Exakt samma oföränderliga release-tagg används för båda. + Updatern startas före Core, och det parade kontrollplanet återskapas med samma data-bind. Core måste både vara frisk på `/api/health` och helt startklar på `/api/status`; annars återställs Compose, tidigare oföränderliga image-ID:n och containrar automatiskt. @@ -87,7 +90,9 @@ Run from the installation directory: ```bash curl -fsSL https://raw.githubusercontent.com/srcfl/ftw/master/scripts/migrate-legacy-compose.sh \ -o /tmp/ftw-migrate.sh +RELEASE=v1.4.1-beta.1 # replace with the operator-approved verified release bash /tmp/ftw-migrate.sh \ + --version "$RELEASE" \ --dir "$PWD" \ --backup-dir /media/$USER/FTW-BACKUP ``` @@ -102,7 +107,8 @@ script can also discover the current directory, `~/ftw`, or 1. **Full backup.** The new helper opens the legacy database read-only, performs no schema migration, creates a complete archive, and verifies file hashes plus SQLite before any deployment change. -2. **Core + updater.** The paired control plane is recreated on the same data +2. **Core + updater.** The same immutable release tag is used for both and the + updater starts first. The paired control plane is recreated on the same data bind and must pass both `/api/health` and full readiness on `/api/status`. Failure restores Compose, the prior immutable image IDs, and the previous containers automatically. diff --git a/go/cmd/ftw-updater/main.go b/go/cmd/ftw-updater/main.go index cdc28fc2..cb0a7658 100644 --- a/go/cmd/ftw-updater/main.go +++ b/go/cmd/ftw-updater/main.go @@ -40,14 +40,23 @@ const ( canonicalMainServiceName = "ftw" legacyMainServiceName = "forty-two-watts" canonicalMainImage = "ghcr.io/srcfl/ftw" + updaterServiceName = "ftw-updater" + canonicalUpdaterImage = "ghcr.io/srcfl/ftw-updater" optimizerServiceName = "ftw-optimizer" canonicalOptimizerImage = "ghcr.io/srcfl/ftw-optimizer" + updaterProtocolVersion = 2 + controlPlaneCapability = "control-plane-pair-v1" ) +// Version is stamped from the same release tag as Core. A matching value is a +// readiness invariant, not display metadata: mixed control-plane versions are +// not allowed to migrate state or commit an update. +var Version = "dev" + // State mirrors selfupdate.UpdateStatus (we keep a local copy to avoid // importing the main module's internal package from this separate cmd). type State struct { - State string `json:"state"` // idle, pulling, restarting, restoring, done, failed + State string `json:"state"` // idle, pulling, transacting, restarting, restoring, done, failed Action string `json:"action,omitempty"` // update, restart, rollback (#152) Component string `json:"component,omitempty"` Target string `json:"target,omitempty"` @@ -76,6 +85,7 @@ type server struct { // main image. It lets immutable updates migrate safely without modifying // the read-only host Compose file or changing its service/data layout. updateOverrideFile string + socketPath string statusPath string stateMu sync.Mutex // skipPull is a dev-only escape hatch: when true, the "pulling" phase @@ -105,8 +115,11 @@ type server struct { // healthCheck waits for the recreated service to become healthy. Both are // injectable so the rollback path is testable without Docker. imageID func(ctx context.Context, service string) (string, error) + imageRef func(ctx context.Context, service string) (string, error) containerID func(ctx context.Context, service string) (string, error) healthCheck func(ctx context.Context, service string) error + updaterReady func(ctx context.Context, target string) error + launchTransaction func(target string, previous map[string]string, startedAt time.Time) error chownFile func(string, int, int) error checkSnapshotFile func(context.Context, string, string, string) error stageSnapshotFile func(context.Context, string, string, string, string) error @@ -217,6 +230,10 @@ func main() { compose := flag.String("compose", envOr("FTW_UPDATER_COMPOSE", "/compose/docker-compose.yml"), "Path to docker-compose.yml") mainService := flag.String("main-service", envOr("FTW_UPDATER_MAIN_SERVICE", ""), "Compose service for FTW (auto-detected when empty)") skipPull := flag.Bool("skip-pull", envOr("FTW_UPDATER_SKIP_PULL", "") != "", "Dev: skip docker compose pull (keeps local image)") + transactionTarget := flag.String("control-plane-transaction", "", "Internal: apply a paired Core/updater release") + transactionCoreImage := flag.String("previous-core-image", "", "Internal: previous Core image ID") + transactionUpdaterImage := flag.String("previous-updater-image", "", "Internal: previous updater image ID") + transactionStartedAt := flag.String("transaction-started-at", "", "Internal: RFC3339 transaction start") flag.Parse() slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))) @@ -237,6 +254,7 @@ func main() { srv := &server{ composeFile: *compose, + socketPath: *socket, statusPath: *statusPath, skipPull: *skipPull, pullRetryDelay: 60 * time.Second, @@ -258,6 +276,7 @@ func main() { } srv.mainServiceName = selectedService srv.imageID = srv.currentServiceImageID + srv.imageRef = srv.currentServiceImageRef srv.containerID = srv.serviceContainerID srv.healthCheck = srv.waitForServiceHealth srv.chownFile = os.Chown @@ -271,6 +290,16 @@ func main() { if *skipPull { slog.Warn("ftw-updater: skip-pull enabled — production deploys should leave this off") } + if *transactionTarget != "" { + startedAt, err := time.Parse(time.RFC3339Nano, *transactionStartedAt) + if err != nil { + startedAt = time.Now() + } + srv.runControlPlaneTransaction(*transactionTarget, map[string]string{ + "core": *transactionCoreImage, "updater": *transactionUpdaterImage, + }, startedAt) + return + } srv.recoverCrashedState() mux := http.NewServeMux() @@ -391,6 +420,14 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { http.Error(w, "action must be update, restart, rollback, or component_rollback", 400) return } + // The paired transaction is owned by a detached helper while this Compose + // service is replaced. runMu is process-local and therefore starts unlocked + // in the new updater; the shared transacting state is the cross-process lock + // that prevents a second mutation from racing the helper. + if st := s.readState(); st.State == "transacting" { + http.Error(w, "control-plane transaction already in progress", http.StatusConflict) + return + } if !s.runMu.TryLock() { http.Error(w, "update already in progress", 409) return @@ -422,7 +459,18 @@ func (s *server) handleUpdate(w http.ResponseWriter, r *http.Request) { func (s *server) handleStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") st := s.readState() - _ = json.NewEncoder(w).Encode(st) + response := struct { + State + ProtocolVersion int `json:"protocol_version"` + UpdaterVersion string `json:"updater_version"` + Capabilities []string `json:"capabilities"` + }{ + State: st, + ProtocolVersion: updaterProtocolVersion, + UpdaterVersion: Version, + Capabilities: []string{controlPlaneCapability}, + } + _ = json.NewEncoder(w).Encode(response) } // runJob executes a pull+up (or pull+up --force-recreate) sequence, @@ -439,6 +487,10 @@ func (s *server) runJob(action, target string) { } func (s *server) runComponentJob(action, target, component string, startedAt time.Time) { + if component == "core" && action == "update" { + s.runControlPlaneUpdate(target, startedAt) + return + } now := startedAt if now.IsZero() { now = time.Now() @@ -558,6 +610,324 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim s.writeState(State{State: "done", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: "compose up -d completed", PreviousImageID: previousImageID}) } +func controlPlaneEnv(target string) []string { + return []string{"FTW_IMAGE_TAG=" + target, "FTW_UPDATER_IMAGE_TAG=" + target} +} + +func (s *server) startControlPlaneHeartbeat(target string) func() { + stop := make(chan struct{}) + go func() { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-ticker.C: + st := s.readState() + if st.Action == "update" && st.Component == "core" && st.Target == target && (st.State == "pulling" || st.State == "transacting") { + st.UpdatedAt = time.Now() + s.writeState(st) + } + case <-stop: + return + } + } + }() + return func() { close(stop) } +} + +// runControlPlaneUpdate performs the non-destructive half of a paired update: +// preflight, capture both rollback identities and pull both immutable tags. It +// then starts a detached helper from the current updater image. The helper is +// outside the Compose service, so replacing ftw-updater cannot kill the owner +// of the transaction or its rollback path. +func (s *server) runControlPlaneUpdate(target string, startedAt time.Time) { + stopHeartbeat := s.startControlPlaneHeartbeat(target) + defer stopHeartbeat() + now := startedAt + if now.IsZero() { + now = time.Now() + } + base := State{State: "pulling", Action: "update", Component: "core", Target: target, StartedAt: now, UpdatedAt: time.Now()} + s.writeState(base) + cleanup, err := s.prepareControlPlaneImagePins() + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "compose preflight failed: "+err.Error(), time.Now() + s.writeState(base) + return + } + defer cleanup() + if err := s.validateControlPlaneImagePins(); err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "compose preflight failed: "+err.Error(), time.Now() + s.writeState(base) + return + } + specs, err := s.controlPlaneSpecs() + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", err.Error(), time.Now() + s.writeState(base) + return + } + previous := make(map[string]string, len(specs)) + for _, spec := range specs { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + imageID, captureErr := s.imageID(ctx, spec.service) + cancel() + if captureErr != nil || imageID == "" { + if captureErr == nil { + captureErr = errors.New("empty image ID") + } + base.State, base.Message, base.UpdatedAt = "failed", "cannot capture "+spec.name+" image for rollback: "+captureErr.Error(), time.Now() + s.writeState(base) + return + } + previous[spec.name] = imageID + } + base.PreviousImages = previous + base.PreviousImageID = previous["core"] + s.writeState(base) + + if !s.skipPull { + pullArgs := s.composeArgs("pull", s.mainServiceName, updaterServiceName) + var pullErr error + for attempt := 1; ; attempt++ { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + pullErr = s.runner(ctx, controlPlaneEnv(target), pullArgs...) + cancel() + if pullErr == nil { + break + } + if s.maxPullAttempts > 0 && attempt >= s.maxPullAttempts { + break + } + time.Sleep(s.pullRetryDelay) + } + if pullErr != nil { + base.State, base.Message, base.UpdatedAt = "failed", "paired pull failed: "+pullErr.Error(), time.Now() + s.writeState(base) + return + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + updaterContainer, err := s.containerID(ctx, updaterServiceName) + cancel() + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "cannot locate updater container: "+err.Error(), time.Now() + s.writeState(base) + return + } + base.State = "transacting" + base.Message = "paired images pulled; detached transaction helper is taking ownership" + base.UpdatedAt = time.Now() + s.writeState(base) + if s.launchTransaction != nil { + if err := s.launchTransaction(target, previous, now); err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "cannot start control-plane transaction helper: "+err.Error(), time.Now() + s.writeState(base) + } + return + } + helperName := fmt.Sprintf("ftw-control-plane-%d", time.Now().UnixNano()) + args := []string{ + "run", "--rm", "--detach", "--name", helperName, "--network", "none", + "--volumes-from", updaterContainer, + } + for _, pair := range []string{ + "FTW_UPDATER_COMPOSE=" + s.composeFile, + "FTW_UPDATER_MAIN_SERVICE=" + s.mainServiceName, + "FTW_UPDATER_SOCKET=" + s.socketPath, + "FTW_UPDATER_STATUS=" + s.statusPath, + "COMPOSE_PROJECT_NAME=" + os.Getenv("COMPOSE_PROJECT_NAME"), + } { + args = append(args, "--env", pair) + } + args = append(args, + previous["updater"], + "-control-plane-transaction", target, + "-previous-core-image", previous["core"], + "-previous-updater-image", previous["updater"], + "-transaction-started-at", now.Format(time.RFC3339Nano), + ) + launchCtx, cancelLaunch := context.WithTimeout(context.Background(), 2*time.Minute) + err = s.runner(launchCtx, nil, args...) + cancelLaunch() + if err != nil { + base.State, base.Message, base.UpdatedAt = "failed", "cannot start control-plane transaction helper: "+err.Error(), time.Now() + s.writeState(base) + } +} + +// runControlPlaneTransaction runs inside the detached helper. Updater is +// replaced first so the new Core sees a matching protocol/version before it +// opens state. Any failure restores updater first and then Core while the +// helper remains alive outside both Compose services. +func (s *server) runControlPlaneTransaction(target string, previous map[string]string, startedAt time.Time) { + stopHeartbeat := s.startControlPlaneHeartbeat(target) + defer stopHeartbeat() + base := State{ + State: "transacting", Action: "update", Component: "core", Target: target, + StartedAt: startedAt, UpdatedAt: time.Now(), PreviousImageID: previous["core"], PreviousImages: previous, + } + fail := func(cause error) { + s.rollbackControlPlane(base, cause) + } + if !isImmutableImageTag(target) || previous["core"] == "" || previous["updater"] == "" { + fail(errors.New("invalid paired transaction parameters")) + return + } + cleanup, err := s.prepareControlPlaneImagePins() + if err != nil { + fail(fmt.Errorf("compose preflight: %w", err)) + return + } + defer cleanup() + if err := s.validateControlPlaneImagePins(); err != nil { + fail(fmt.Errorf("compose preflight: %w", err)) + return + } + env := controlPlaneEnv(target) + base.Message = "replacing updater with matching release" + base.UpdatedAt = time.Now() + s.writeState(base) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + err = s.runner(ctx, env, s.composeArgs("up", "-d", updaterServiceName)...) + cancel() + if err != nil { + fail(fmt.Errorf("replace updater: %w", err)) + return + } + compatCtx, cancelCompat := context.WithTimeout(context.Background(), 3*time.Minute) + if s.updaterReady != nil { + err = s.updaterReady(compatCtx, target) + } else { + err = s.waitForUpdaterRelease(compatCtx, target) + } + cancelCompat() + if err != nil { + fail(err) + return + } + if err := s.requireServiceImageRef(updaterServiceName, canonicalUpdaterImage+":"+target); err != nil { + fail(err) + return + } + + base.Message = "matching updater ready; replacing Core" + base.UpdatedAt = time.Now() + s.writeState(base) + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) + err = s.runner(ctx, env, s.composeArgs("up", "-d", s.mainServiceName)...) + cancel() + if err != nil { + fail(fmt.Errorf("replace Core: %w", err)) + return + } + if s.healthCheck != nil { + healthCtx, cancelHealth := context.WithTimeout(context.Background(), componentHealthTimeout("core")) + err = s.healthCheck(healthCtx, s.mainServiceName) + cancelHealth() + if err != nil { + fail(fmt.Errorf("new Core readiness: %w", err)) + return + } + } + if err := s.requireServiceImageRef(s.mainServiceName, canonicalMainImage+":"+target); err != nil { + fail(err) + return + } + if err := s.requireServiceImageRef(updaterServiceName, canonicalUpdaterImage+":"+target); err != nil { + fail(err) + return + } + base.State = "done" + base.Message = "Core and updater committed as one release pair" + base.UpdatedAt = time.Now() + s.writeState(base) +} + +func (s *server) waitForUpdaterRelease(ctx context.Context, target string) error { + for { + cli := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", s.socketPath) + }}, + } + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/status", nil) + resp, err := cli.Do(req) + if err == nil { + var status struct { + ProtocolVersion int `json:"protocol_version"` + UpdaterVersion string `json:"updater_version"` + Capabilities []string `json:"capabilities"` + } + decodeErr := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&status) + resp.Body.Close() + if resp.StatusCode == http.StatusOK && decodeErr == nil && status.ProtocolVersion >= updaterProtocolVersion && status.UpdaterVersion == target { + for _, capability := range status.Capabilities { + if capability == controlPlaneCapability { + return nil + } + } + } + } + select { + case <-ctx.Done(): + return fmt.Errorf("matching updater %s did not become ready: %w", target, ctx.Err()) + case <-time.After(time.Second): + } + } +} + +func (s *server) requireServiceImageRef(service, want string) error { + if s.imageRef == nil { + return errors.New("image reference inspection is unavailable") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + got, err := s.imageRef(ctx, service) + cancel() + if err != nil { + return fmt.Errorf("inspect %s image reference: %w", service, err) + } + if got != want { + return fmt.Errorf("service %s runs %s, want immutable release %s", service, got, want) + } + return nil +} + +func (s *server) rollbackControlPlane(base State, cause error) { + base.State = "restoring" + base.Message = "paired update failed; restoring previous updater and Core: " + cause.Error() + base.UpdatedAt = time.Now() + s.writeState(base) + cleanup, preflightErr := s.prepareControlPlaneImagePins() + if preflightErr == nil { + defer cleanup() + } + var failures []string + if preflightErr != nil { + failures = append(failures, "rollback preflight: "+preflightErr.Error()) + } else { + updaterSpec, _ := s.componentSpec("updater") + coreSpec, _ := s.componentSpec("core") + if err := s.restorePreviousComponentImage(base.PreviousImages["updater"], updaterSpec); err != nil { + failures = append(failures, "updater: "+err.Error()) + } + if err := s.restorePreviousComponentImage(base.PreviousImages["core"], coreSpec); err != nil { + failures = append(failures, "Core: "+err.Error()) + } + } + base.State = "failed" + base.UpdatedAt = time.Now() + if len(failures) == 0 { + base.Message = "paired update failed; previous Core/updater image pair restored: " + cause.Error() + } else { + base.Message = "paired update failed and automatic pair rollback was incomplete: " + cause.Error() + "; " + strings.Join(failures, "; ") + } + s.writeState(base) +} + func (s *server) runComponentRollback(component string, startedAt time.Time) { now := startedAt if now.IsZero() { @@ -599,6 +969,13 @@ func (s *server) componentSpec(component string) (componentSpec, error) { return componentSpec{}, fmt.Errorf("compose service %q is unavailable", optimizerServiceName) } return componentSpec{name: "optimizer", service: optimizerServiceName, image: canonicalOptimizerImage, tagEnv: "FTW_OPTIMIZER_IMAGE_TAG", tagVariable: "FTW_OPTIMIZER_IMAGE_TAG"}, nil + case "updater": + if _, ok, err := serviceImageFromComposeFiles(s.composeFiles(), updaterServiceName); err != nil { + return componentSpec{}, err + } else if !ok { + return componentSpec{}, fmt.Errorf("compose service %q is unavailable", updaterServiceName) + } + return componentSpec{name: "updater", service: updaterServiceName, image: canonicalUpdaterImage, tagEnv: "FTW_UPDATER_IMAGE_TAG", tagVariable: "FTW_UPDATER_IMAGE_TAG"}, nil default: return componentSpec{}, fmt.Errorf("unsupported component %q", component) } @@ -662,6 +1039,96 @@ func (s *server) prepareUpdateImagePin() (func(), error) { return s.prepareComponentImagePin(spec) } +func (s *server) controlPlaneSpecs() ([]componentSpec, error) { + core, err := s.componentSpec("core") + if err != nil { + return nil, err + } + updater, err := s.componentSpec("updater") + if err != nil { + return nil, err + } + return []componentSpec{core, updater}, nil +} + +// prepareControlPlaneImagePins creates one transient override containing every +// legacy hard-coded service. A single file is important: two calls to the +// component helper would replace updateOverrideFile and silently unpin the +// first half of the pair. +func (s *server) prepareControlPlaneImagePins() (func(), error) { + specs, err := s.controlPlaneSpecs() + if err != nil { + return func() {}, err + } + type imageService struct { + Image string `yaml:"image"` + } + doc := struct { + Services map[string]imageService `yaml:"services"` + }{Services: make(map[string]imageService)} + for _, spec := range specs { + image, ok, err := serviceImageFromComposeFiles(s.composeFiles(), spec.service) + if err != nil { + return func() {}, err + } + if !ok { + return func() {}, fmt.Errorf("service %q is missing an image entry in compose files", spec.service) + } + if !strings.Contains(image, spec.tagVariable) { + doc.Services[spec.service] = imageService{Image: spec.image + ":${" + spec.tagVariable + ":-latest}"} + } + } + if len(doc.Services) == 0 { + return func() {}, nil + } + data, err := yaml.Marshal(doc) + if err != nil { + return func() {}, fmt.Errorf("build control-plane compatibility override: %w", err) + } + f, err := os.CreateTemp("", "ftw-control-plane-update-*.yml") + if err != nil { + return func() {}, fmt.Errorf("write control-plane compatibility override: %w", err) + } + path := f.Name() + removeOnError := func() { + _ = f.Close() + _ = os.Remove(path) + } + if _, err := f.Write(data); err != nil { + removeOnError() + return func() {}, err + } + if err := f.Sync(); err != nil { + removeOnError() + return func() {}, err + } + if err := f.Close(); err != nil { + _ = os.Remove(path) + return func() {}, err + } + s.updateOverrideFile = path + cleanup := func() { + s.updateOverrideFile = "" + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + slog.Warn("remove control-plane compatibility override", "path", path, "err", err) + } + } + return cleanup, nil +} + +func (s *server) validateControlPlaneImagePins() error { + specs, err := s.controlPlaneSpecs() + if err != nil { + return err + } + for _, spec := range specs { + if err := s.validateComponentImagePin(spec); err != nil { + return err + } + } + return nil +} + func (s *server) prepareComponentImagePin(spec componentSpec) (func(), error) { image, ok, err := serviceImageFromComposeFiles(s.composeFiles(), spec.service) if err != nil { @@ -1197,6 +1664,22 @@ func (s *server) currentServiceImageID(ctx context.Context, service string) (str return imageID, nil } +func (s *server) currentServiceImageRef(ctx context.Context, service string) (string, error) { + containerID, err := s.serviceContainerID(ctx, service) + if err != nil { + return "", err + } + out, err := dockerOutput(ctx, "inspect", "--format", "{{.Config.Image}}", containerID) + if err != nil { + return "", err + } + ref := strings.TrimSpace(out) + if ref == "" { + return "", errors.New("running container has no image reference") + } + return ref, nil +} + func (s *server) waitForServiceHealth(ctx context.Context, service string) error { containerID, err := s.serviceContainerID(ctx, service) if err != nil { diff --git a/go/cmd/ftw-updater/main_test.go b/go/cmd/ftw-updater/main_test.go index 2f2cadbb..48a770a3 100644 --- a/go/cmd/ftw-updater/main_test.go +++ b/go/cmd/ftw-updater/main_test.go @@ -69,14 +69,37 @@ func newTestServer(t *testing.T) (*server, *fakeRunner) { s := &server{ composeFile: filepath.Join(dir, "docker-compose.yml"), mainServiceName: canonicalMainServiceName, + socketPath: filepath.Join(dir, "sock"), statusPath: filepath.Join(dir, "state.json"), pullRetryDelay: time.Millisecond, runner: runner.run, healthCheck: func(context.Context, string) error { return nil }, imageID: func(context.Context, string) (string, error) { return "sha256:current", nil }, containerID: func(context.Context, string) (string, error) { return "ftw-container", nil }, + updaterReady: func(context.Context, string) error { return nil }, chownFile: func(string, int, int) error { return nil }, } + s.imageRef = func(_ context.Context, service string) (string, error) { + target := "" + for _, env := range runner.envSnapshot() { + for _, value := range env { + if strings.HasPrefix(value, "FTW_IMAGE_TAG=") { + target = strings.TrimPrefix(value, "FTW_IMAGE_TAG=") + } + } + } + if target == "" { + return "", errors.New("no target captured") + } + if service == updaterServiceName { + return canonicalUpdaterImage + ":" + target, nil + } + return canonicalMainImage + ":" + target, nil + } + s.launchTransaction = func(target string, previous map[string]string, startedAt time.Time) error { + s.runControlPlaneTransaction(target, previous, startedAt) + return nil + } s.checkSnapshotFile = func(_ context.Context, _ string, snapshotID, file string) error { _, err := os.Stat(filepath.Join(dir, "data", "snapshots", snapshotID, file)) return err @@ -93,6 +116,8 @@ func newTestServer(t *testing.T) (*server, *fakeRunner) { image: ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest} volumes: - ./data:/app/data + ftw-updater: + image: ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest} `) return s, runner } @@ -113,11 +138,11 @@ func TestSkipPull_BypassesPullStep(t *testing.T) { s.handleUpdate(rr, req) waitForState(t, s, "done") calls := runner.snapshot() - if len(calls) != 1 { - t.Fatalf("skip-pull should yield 1 call (up only), got %d: %v", len(calls), calls) + if len(calls) != 2 { + t.Fatalf("skip-pull should replace updater then Core, got %d calls: %v", len(calls), calls) } - if !strings.Contains(strings.Join(calls[0], " "), "up -d") { - t.Errorf("single call should be `up -d`: %v", calls[0]) + if calls[0][len(calls[0])-1] != updaterServiceName || calls[1][len(calls[1])-1] != canonicalMainServiceName { + t.Errorf("pair order must be updater then Core: %v", calls) } } @@ -203,15 +228,40 @@ func TestHandleUpdate_HappyPath(t *testing.T) { t.Errorf("unexpected final state: %+v", st) } calls := runner.snapshot() - if len(calls) != 2 { - t.Fatalf("expected 2 docker calls, got %d: %v", len(calls), calls) + if len(calls) != 3 { + t.Fatalf("expected pull + paired recreate, got %d: %v", len(calls), calls) } if calls[0][0] != "compose" || !strings.Contains(strings.Join(calls[0], " "), "pull") { t.Errorf("first call should be pull: %v", calls[0]) } - up := strings.Join(calls[1], " ") - if !strings.Contains(up, "up -d") || strings.Contains(up, "--force-recreate") { - t.Errorf("update path should NOT force-recreate: %v", calls[1]) + if calls[1][len(calls[1])-1] != updaterServiceName || calls[2][len(calls[2])-1] != canonicalMainServiceName { + t.Errorf("paired update must replace updater before Core: %v", calls) + } + for _, call := range calls[1:] { + if joined := strings.Join(call, " "); !strings.Contains(joined, "up -d") || strings.Contains(joined, "--force-recreate") { + t.Errorf("update path should recreate without force: %v", call) + } + } +} + +func TestControlPlaneUpdatePreservesBothPreviousImageIDs(t *testing.T) { + s, _ := newTestServer(t) + s.skipPull = true + s.imageID = func(_ context.Context, service string) (string, error) { + if service == updaterServiceName { + return "sha256:previous-updater", nil + } + return "sha256:previous-core", nil + } + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) + rr := httptest.NewRecorder() + s.handleUpdate(rr, req) + st := waitForState(t, s, "done") + if st.PreviousImages["core"] != "sha256:previous-core" || st.PreviousImages["updater"] != "sha256:previous-updater" { + t.Fatalf("paired rollback identities = %+v", st.PreviousImages) + } + if st.PreviousImageID != "sha256:previous-core" { + t.Fatalf("legacy Core rollback identity = %q", st.PreviousImageID) } } @@ -318,19 +368,22 @@ func TestHandleUpdate_PinsImageTagViaEnv(t *testing.T) { } waitForState(t, s, "done") envs := runner.envSnapshot() - if len(envs) != 2 { - t.Fatalf("expected 2 docker calls (pull, up); got %d", len(envs)) + if len(envs) != 3 { + t.Fatalf("expected pull + updater/Core recreate; got %d", len(envs)) } for i, env := range envs { - found := false + foundCore := false + foundUpdater := false for _, e := range env { if e == "FTW_IMAGE_TAG=v0.44.0" { - found = true - break + foundCore = true + } + if e == "FTW_UPDATER_IMAGE_TAG=v0.44.0" { + foundUpdater = true } } - if !found { - t.Errorf("call %d missing FTW_IMAGE_TAG=v0.44.0; env=%v", i, env) + if !foundCore || !foundUpdater { + t.Errorf("call %d missing paired immutable tags; env=%v", i, env) } } } @@ -342,6 +395,8 @@ func TestHandleUpdate_MigratesHardcodedImageWithTransientOverride(t *testing.T) image: forty-two-watts:optimizer-champion-recourse-b10acacd volumes: - ./data:/app/data + ftw-updater: + image: example.invalid/old-updater:latest `) s.mainServiceName = legacyMainServiceName @@ -353,17 +408,17 @@ func TestHandleUpdate_MigratesHardcodedImageWithTransientOverride(t *testing.T) } waitForState(t, s, "done") calls := runner.snapshot() - if len(calls) != 2 { - t.Fatalf("want pull + up, got %v", calls) + if len(calls) != 3 { + t.Fatalf("want pull + updater/Core recreate, got %v", calls) } for _, call := range calls { joined := strings.Join(call, " ") - if !strings.Contains(joined, "-f "+s.composeFile+" -f ") || !strings.Contains(joined, "ftw-compose-update-") { + if !strings.Contains(joined, "-f "+s.composeFile+" -f ") || !strings.Contains(joined, "ftw-control-plane-update-") { t.Fatalf("legacy update must append compatibility override after base file: %v", call) } - if call[len(call)-1] != legacyMainServiceName { - t.Fatalf("legacy service identity must be preserved: %v", call) - } + } + if calls[1][len(calls[1])-1] != updaterServiceName || calls[2][len(calls[2])-1] != legacyMainServiceName { + t.Fatalf("legacy pair order/service identity = %v", calls) } } @@ -667,6 +722,12 @@ func TestHandleUpdate_ConcurrentRejected(t *testing.T) { <-block return nil } + s.imageRef = func(_ context.Context, service string) (string, error) { + if service == updaterServiceName { + return canonicalUpdaterImage + ":v1.2.3", nil + } + return canonicalMainImage + ":v1.2.3", nil + } req1 := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) rr1 := httptest.NewRecorder() @@ -692,12 +753,38 @@ func TestHandleStatus_ReadsFile(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/status", nil) rr := httptest.NewRecorder() s.handleStatus(rr, req) - var st State + var st struct { + Status string `json:"state"` + ProtocolVersion int `json:"protocol_version"` + UpdaterVersion string `json:"updater_version"` + Capabilities []string `json:"capabilities"` + } if err := json.NewDecoder(rr.Body).Decode(&st); err != nil { t.Fatal(err) } - if st.State != "pulling" { - t.Errorf("state = %q", st.State) + if st.Status != "pulling" { + t.Errorf("state = %q", st.Status) + } + if st.ProtocolVersion != updaterProtocolVersion || st.UpdaterVersion != Version || len(st.Capabilities) != 1 || st.Capabilities[0] != controlPlaneCapability { + t.Fatalf("updater handshake = %+v", st) + } +} + +func TestDetachedControlPlaneTransactionRejectsConcurrentMutation(t *testing.T) { + s, runner := newTestServer(t) + s.writeState(State{ + State: "transacting", Action: "update", Component: "core", Target: "v1.2.3", + StartedAt: time.Now().Add(-time.Minute), UpdatedAt: time.Now(), + }) + + req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.4"}`)) + rr := httptest.NewRecorder() + s.handleUpdate(rr, req) + if rr.Code != http.StatusConflict { + t.Fatalf("concurrent transaction status = %d, want 409", rr.Code) + } + if calls := runner.snapshot(); len(calls) != 0 { + t.Fatalf("concurrent transaction must not call Docker: %v", calls) } } @@ -756,6 +843,8 @@ func TestUpdateHealthFailureRestoresPreviousImage(t *testing.T) { image: forty-two-watts:optimizer-champion-recourse-b10acacd volumes: - ./data:/app/data + ftw-updater: + image: example.invalid/old-updater:latest `) s.mainServiceName = legacyMainServiceName s.imageID = func(context.Context, string) (string, error) { return "sha256:previous", nil } @@ -772,20 +861,20 @@ func TestUpdateHealthFailureRestoresPreviousImage(t *testing.T) { rr := httptest.NewRecorder() s.handleUpdate(rr, req) st := waitForState(t, s, "failed") - if !strings.Contains(st.Message, "previous image restored") { + if !strings.Contains(st.Message, "previous Core/updater image pair restored") { t.Fatalf("state should report automatic rollback, got %+v", st) } calls := runner.snapshot() - if len(calls) != 4 { - t.Fatalf("want pull, new up, image tag, rollback up; got %v", calls) + if len(calls) != 7 { + t.Fatalf("want pull, updater/Core recreate and paired rollback; got %v", calls) } - if got := strings.Join(calls[2], " "); !strings.Contains(got, "image tag sha256:previous") { - t.Fatalf("third call should tag previous image, got %q", got) + if got := strings.Join(calls[3], " "); !strings.Contains(got, "image tag sha256:previous") { + t.Fatalf("rollback should first tag previous updater, got %q", got) } - if got := strings.Join(calls[2], " "); !strings.Contains(got, canonicalMainImage+":ftw-rollback-") { - t.Fatalf("previous legacy image should be retagged into canonical repository, got %q", got) + if got := strings.Join(calls[5], " "); !strings.Contains(got, canonicalMainImage+":ftw-rollback-") { + t.Fatalf("previous legacy Core should be retagged into canonical repository, got %q", got) } - if got := strings.Join(calls[3], " "); !strings.Contains(got, "ftw-compose-update-") || calls[3][len(calls[3])-1] != legacyMainServiceName { + if got := strings.Join(calls[6], " "); !strings.Contains(got, "ftw-control-plane-update-") || calls[6][len(calls[6])-1] != legacyMainServiceName { t.Fatalf("rollback must reuse transient pin and legacy service identity, got %q", got) } } diff --git a/go/cmd/ftw/boothealth.go b/go/cmd/ftw/boothealth.go index 946c60d3..5a09fda9 100644 --- a/go/cmd/ftw/boothealth.go +++ b/go/cmd/ftw/boothealth.go @@ -28,15 +28,18 @@ func (s *swappableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { (*s.h.Load()).ServeHTTP(w, r) } -// bootPhaseHandler answers health probes 200 while the process initializes, -// so a legitimately slow boot is distinguishable from a dead one. Everything -// else gets 503 + Retry-After so clients and the UI know to come back. +// bootPhaseHandler keeps the API port bound while state is opened or migrated, +// but deliberately fails both health and readiness. A control-plane updater +// must not commit a new Core image until the real mux is wired and /api/status +// is available; reporting 200 here would turn a long-running migration into a +// falsely successful update. func bootPhaseHandler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"status":"starting","phase":"initializing state"}`)) + w.Header().Set("Retry-After", "10") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"status":"starting","ready":false,"phase":"initializing state"}`)) }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/go/cmd/ftw/boothealth_test.go b/go/cmd/ftw/boothealth_test.go new file mode 100644 index 00000000..5f52b1fd --- /dev/null +++ b/go/cmd/ftw/boothealth_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestBootPhaseFailsHealthAndReadinessUntilStateIsReady(t *testing.T) { + h := bootPhaseHandler() + for _, path := range []string{"/api/health", "/api/status"} { + t.Run(path, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, path, nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + if rr.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", rr.Code) + } + if rr.Header().Get("Retry-After") == "" { + t.Fatal("boot response must tell clients to retry") + } + var body map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if got := body["status"]; path == "/api/health" && got != "starting" { + t.Fatalf("health status = %v, want starting", got) + } + }) + } +} diff --git a/go/cmd/ftw/bootstrap.go b/go/cmd/ftw/bootstrap.go index 4f505d7d..51c1f2ed 100644 --- a/go/cmd/ftw/bootstrap.go +++ b/go/cmd/ftw/bootstrap.go @@ -33,8 +33,10 @@ func runBootstrap(configPath, webDir, driverDir string) { } selfUpdater = selfupdate.New(selfupdate.Config{ CurrentVersion: current, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), + UpdaterVersion: Version, + PairedImage: "srcfl/ftw-updater", PairManifestAsset: "ftw-control-plane.json", + SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), + StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), }, nil) selfUpdater.Start(context.Background()) slog.Info("selfupdate enabled in bootstrap", "socket", envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock")) diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 0c127376..a436b010 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -165,8 +165,9 @@ func main() { // take 25+ minutes, and an unbound port for that long makes the Docker // healthcheck fail and the self-update sidecar judge the deploy failed — // observed 2026-07-16 as an auto-rollback in the middle of a VACUUM. - // Until the real mux is wired, /api/health answers 200 "starting" and - // everything else 503. + // Until the real mux is wired, health/readiness answer 503 "starting". + // The container stays alive, but no updater can mistake a migration for a + // successfully booted Core. apiHandler := newSwappableHandler(bootPhaseHandler()) httpSrv := &http.Server{ Addr: fmt.Sprintf(":%d", cfg.API.Port), @@ -180,6 +181,32 @@ func main() { } }() + // Core and updater are one versioned control plane. Check this before + // state.Open: a legacy updater can accept the request while replacing Core + // only, and the new Core must not run migrations or become ready in that + // mixed state. Keep the boot listener alive so an updater transaction can + // finish replacing its sidecar; the real mux is swapped in only after the + // exact release pair is present. + if envBool("FTW_SELFUPDATE_ENABLED") && Version != "dev" { + socketPath := envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock") + var lastCompatibilityLog time.Time + for { + probeCtx, cancelProbe := context.WithTimeout(context.Background(), 5*time.Second) + err := selfupdate.RequireUpdaterRelease(probeCtx, socketPath, Version) + cancelProbe() + if err == nil { + break + } + if time.Since(lastCompatibilityLog) >= 30*time.Second { + slog.Warn("Core readiness waiting for matching updater", + "core_version", Version, "socket", socketPath, "err", err) + lastCompatibilityLog = time.Now() + } + time.Sleep(2 * time.Second) + } + slog.Info("Core/updater release pair verified", "version", Version) + } + st, err := state.Open(statePath) if err != nil { slog.Error("open state", "err", err) @@ -1646,8 +1673,10 @@ func main() { } selfUpdater = selfupdate.New(selfupdate.Config{ CurrentVersion: current, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), + UpdaterVersion: Version, + PairedImage: "srcfl/ftw-updater", PairManifestAsset: "ftw-control-plane.json", + SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), + StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), // Publish events.UpdateAvailable when a new release lands so // the notifications service (or any other subscriber) can act // without polling the checker directly. @@ -1670,6 +1699,7 @@ func main() { Repo: "srcfl/ftw", Image: "srcfl/ftw-optimizer", ReleaseTagPrefix: "optimizer-", StoragePrefix: "optimizer.", CurrentVersion: optimizerCurrent, + UpdaterVersion: Version, SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), }, st) diff --git a/go/internal/api/api_selfupdate_test.go b/go/internal/api/api_selfupdate_test.go index b7e9c4cc..7443b15f 100644 --- a/go/internal/api/api_selfupdate_test.go +++ b/go/internal/api/api_selfupdate_test.go @@ -102,6 +102,14 @@ func startFakeSidecar(t *testing.T, statusCode int) string { t.Fatalf("listen fake sidecar: %v", err) } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/status" { + _ = json.NewEncoder(w).Encode(selfupdate.UpdaterRuntimeInfo{ + ProtocolVersion: selfupdate.UpdaterProtocolVersion, + Version: "v1.4.0", + Capabilities: []string{selfupdate.ControlPlanePairCapability}, + }) + return + } w.WriteHeader(statusCode) _, _ = w.Write([]byte(`{"status":"stub"}`)) })} diff --git a/go/internal/selfupdate/control_plane_release.go b/go/internal/selfupdate/control_plane_release.go new file mode 100644 index 00000000..9e720ae2 --- /dev/null +++ b/go/internal/selfupdate/control_plane_release.go @@ -0,0 +1,128 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strings" +) + +const controlPlaneManifestSchema = 1 + +var ( + sha256DigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + gitRevisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) +) + +type controlPlaneReleaseManifest struct { + SchemaVersion int `json:"schema_version"` + Release string `json:"release"` + Revision string `json:"revision"` + Components struct { + Core controlPlaneReleaseComponent `json:"core"` + Updater controlPlaneReleaseComponent `json:"updater"` + } `json:"components"` +} + +type controlPlaneReleaseComponent struct { + Image string `json:"image"` + Digest string `json:"digest"` +} + +func (c *Checker) verifyControlPlaneRelease(ctx context.Context, rel ghRelease, targetTag string) (bool, error) { + assetURL := "" + for _, asset := range rel.Assets { + if asset.Name == c.cfg.PairManifestAsset { + assetURL = asset.BrowserDownloadURL + break + } + } + if assetURL == "" { + return false, nil + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, assetURL, nil) + if err != nil { + return false, err + } + req.Header.Set("Accept", "application/octet-stream") + req.Header.Set("User-Agent", "FTW-selfupdate") + resp, err := c.cfg.HTTPClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + if resp.StatusCode >= 400 { + return false, fmt.Errorf("pair manifest HTTP %d", resp.StatusCode) + } + var manifest controlPlaneReleaseManifest + if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&manifest); err != nil { + return false, fmt.Errorf("decode pair manifest: %w", err) + } + if err := c.validateControlPlaneManifest(manifest, targetTag); err != nil { + return false, err + } + + coreProbe := ®istryProbe{httpClient: c.cfg.HTTPClient, base: c.cfg.RegistryBaseURL, repo: c.cfg.Image, service: c.cfg.RegistryService} + updaterProbe := ®istryProbe{httpClient: c.cfg.HTTPClient, base: c.cfg.RegistryBaseURL, repo: c.cfg.PairedImage, service: c.cfg.RegistryService} + coreDigest, err := coreProbe.manifestDigest(ctx, targetTag) + if err != nil { + return false, fmt.Errorf("Core digest: %w", err) + } + updaterDigest, err := updaterProbe.manifestDigest(ctx, targetTag) + if err != nil { + return false, fmt.Errorf("updater digest: %w", err) + } + if coreDigest == "" || updaterDigest == "" { + return false, nil + } + if coreDigest != manifest.Components.Core.Digest { + return false, fmt.Errorf("Core tag digest %s does not match release manifest %s", coreDigest, manifest.Components.Core.Digest) + } + if updaterDigest != manifest.Components.Updater.Digest { + return false, fmt.Errorf("updater tag digest %s does not match release manifest %s", updaterDigest, manifest.Components.Updater.Digest) + } + return true, nil +} + +func (c *Checker) validateControlPlaneManifest(manifest controlPlaneReleaseManifest, targetTag string) error { + if manifest.SchemaVersion != controlPlaneManifestSchema { + return fmt.Errorf("unsupported pair manifest schema %d", manifest.SchemaVersion) + } + if manifest.Release != targetTag { + return fmt.Errorf("pair manifest release %q does not match %q", manifest.Release, targetTag) + } + if !gitRevisionPattern.MatchString(manifest.Revision) { + return errors.New("pair manifest has invalid source revision") + } + wantCore := c.cfg.Image + ":" + targetTag + wantUpdater := c.cfg.PairedImage + ":" + targetTag + if trimRegistryHost(manifest.Components.Core.Image) != wantCore { + return fmt.Errorf("pair manifest Core image %q does not match %q", manifest.Components.Core.Image, wantCore) + } + if trimRegistryHost(manifest.Components.Updater.Image) != wantUpdater { + return fmt.Errorf("pair manifest updater image %q does not match %q", manifest.Components.Updater.Image, wantUpdater) + } + if !sha256DigestPattern.MatchString(manifest.Components.Core.Digest) || !sha256DigestPattern.MatchString(manifest.Components.Updater.Digest) { + return errors.New("pair manifest contains an invalid image digest") + } + return nil +} + +func trimRegistryHost(image string) string { + image = strings.TrimPrefix(image, "https://") + image = strings.TrimPrefix(image, "http://") + if slash := strings.IndexByte(image, '/'); slash >= 0 { + first := image[:slash] + if strings.Contains(first, ".") || strings.Contains(first, ":") || first == "localhost" { + return image[slash+1:] + } + } + return image +} diff --git a/go/internal/selfupdate/control_plane_release_test.go b/go/internal/selfupdate/control_plane_release_test.go new file mode 100644 index 00000000..5cf2c7f2 --- /dev/null +++ b/go/internal/selfupdate/control_plane_release_test.go @@ -0,0 +1,126 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const ( + testCoreDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testUpdaterDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + testRevision = "0123456789abcdef0123456789abcdef01234567" +) + +type pairReleaseFixture struct { + manifest controlPlaneReleaseManifest + coreDigest string + updaterDigest string +} + +func newPairReleaseFixture(tag string) pairReleaseFixture { + var manifest controlPlaneReleaseManifest + manifest.SchemaVersion = controlPlaneManifestSchema + manifest.Release = tag + manifest.Revision = testRevision + manifest.Components.Core = controlPlaneReleaseComponent{Image: "ghcr.io/srcfl/ftw:" + tag, Digest: testCoreDigest} + manifest.Components.Updater = controlPlaneReleaseComponent{Image: "ghcr.io/srcfl/ftw-updater:" + tag, Digest: testUpdaterDigest} + return pairReleaseFixture{manifest: manifest, coreDigest: testCoreDigest, updaterDigest: testUpdaterDigest} +} + +func pairReleaseServer(t *testing.T, tag string, fixture *pairReleaseFixture) *httptest.Server { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/release": + _ = json.NewEncoder(w).Encode(map[string]any{ + "tag_name": tag, "draft": false, "prerelease": false, + "assets": []map[string]string{{"name": "ftw-control-plane.json", "browser_download_url": srv.URL + "/asset"}}, + }) + case r.URL.Path == "/asset": + _ = json.NewEncoder(w).Encode(fixture.manifest) + case r.URL.Path == "/token": + _ = json.NewEncoder(w).Encode(map[string]string{"token": "test"}) + case r.URL.Path == "/v2/srcfl/ftw/manifests/"+tag: + if fixture.coreDigest == "" { + http.NotFound(w, r) + return + } + w.Header().Set("Docker-Content-Digest", fixture.coreDigest) + _, _ = w.Write([]byte(`{"schemaVersion":2}`)) + case r.URL.Path == "/v2/srcfl/ftw-updater/manifests/"+tag: + if fixture.updaterDigest == "" { + http.NotFound(w, r) + return + } + w.Header().Set("Docker-Content-Digest", fixture.updaterDigest) + _, _ = w.Write([]byte(`{"schemaVersion":2}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func pairChecker(tag string, srv *httptest.Server) *Checker { + return New(Config{ + Repo: "srcfl/ftw", Image: "srcfl/ftw", PairedImage: "srcfl/ftw-updater", + PairManifestAsset: "ftw-control-plane.json", CurrentVersion: "v1.3.1", + RegistryBaseURL: srv.URL, LatestReleaseURL: srv.URL + "/release", + HTTPClient: srv.Client(), + }, newMemStore()) +} + +func TestControlPlaneReleaseRequiresBothPinnedDigests(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + srv := pairReleaseServer(t, tag, &fixture) + info, err := pairChecker(tag, srv).Check(context.Background(), true) + if err != nil { + t.Fatal(err) + } + if !info.UpdateAvailable || info.Latest != tag { + t.Fatalf("pair release = %+v", info) + } +} + +func TestControlPlaneReleaseStaysHiddenUntilUpdaterDigestExists(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.updaterDigest = "" + srv := pairReleaseServer(t, tag, &fixture) + info, err := pairChecker(tag, srv).Check(context.Background(), true) + if err != nil { + t.Fatal(err) + } + if info.UpdateAvailable || info.Latest != "" { + t.Fatalf("partial pair must not be announced: %+v", info) + } +} + +func TestControlPlaneReleaseRejectsMovedTagDigest(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.coreDigest = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + srv := pairReleaseServer(t, tag, &fixture) + _, err := pairChecker(tag, srv).Check(context.Background(), true) + if err == nil || !strings.Contains(err.Error(), "does not match release manifest") { + t.Fatalf("Check error = %v", err) + } +} + +func TestControlPlaneReleaseRejectsWrongReleaseIdentity(t *testing.T) { + const tag = "v1.4.0" + fixture := newPairReleaseFixture(tag) + fixture.manifest.Release = "v1.4.1" + srv := pairReleaseServer(t, tag, &fixture) + _, err := pairChecker(tag, srv).Check(context.Background(), true) + if err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("Check error = %v", err) + } +} diff --git a/go/internal/selfupdate/registry.go b/go/internal/selfupdate/registry.go index 7aee0094..f9687590 100644 --- a/go/internal/selfupdate/registry.go +++ b/go/internal/selfupdate/registry.go @@ -2,6 +2,7 @@ package selfupdate import ( "context" + "crypto/sha256" "encoding/json" "fmt" "io" @@ -43,6 +44,52 @@ func (rp *registryProbe) hasTag(ctx context.Context, tag string) (bool, error) { return false, nil } +// manifestDigest resolves one immutable tag to its OCI/Docker content digest. +// A missing manifest is a normal publication race and returns "", nil. The +// release checker compares the value with the paired release manifest rather +// than trusting tag membership alone. +func (rp *registryProbe) manifestDigest(ctx context.Context, tag string) (string, error) { + tok, err := rp.token(ctx) + if err != nil { + return "", err + } + u := rp.base + "/v2/" + rp.repo + "/manifests/" + url.PathEscape(tag) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + req.Header.Set("Accept", strings.Join([]string{ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + }, ", ")) + resp, err := rp.httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + if resp.StatusCode >= 400 { + return "", fmt.Errorf("manifest %s: HTTP %d", tag, resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return "", err + } + digest := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest")) + if digest == "" { + // Distribution registries should send Docker-Content-Digest. Computing + // it from the exact response keeps compatible test/private registries + // usable without weakening the content-addressed comparison. + sum := sha256.Sum256(body) + digest = fmt.Sprintf("sha256:%x", sum[:]) + } + return digest, nil +} + func (rp *registryProbe) token(ctx context.Context) (string, error) { u := rp.base + "/token?service=" + url.QueryEscape(rp.service) + "&scope=" + url.QueryEscape("repository:"+rp.repo+":pull") diff --git a/go/internal/selfupdate/selfupdate.go b/go/internal/selfupdate/selfupdate.go index 2b24f82e..af9e3e69 100644 --- a/go/internal/selfupdate/selfupdate.go +++ b/go/internal/selfupdate/selfupdate.go @@ -39,7 +39,6 @@ import ( "fmt" "io" "log/slog" - "net" "net/http" "os" "strconv" @@ -97,6 +96,11 @@ type Config struct { // Image overrides the registry path if it differs from Repo (rare — // normally the GH repo and the GHCR image share a name). Defaults to Repo. Image string + // PairedImage and PairManifestAsset enable fail-closed Core/updater + // resolution. The manifest is attached to the GitHub release and pins both + // immutable GHCR tag digests plus their common source revision. + PairedImage string + PairManifestAsset string // ReleaseTagPrefix selects component-specific GitHub Releases while the // registry continues to use plain vX.Y.Z tags. Example: "optimizer-" // resolves GitHub tag optimizer-v1.2.0 to GHCR tag v1.2.0. @@ -112,6 +116,10 @@ type Config struct { // CurrentVersion is the running binary's version (from main.Version). CurrentVersion string + // UpdaterVersion is the Core release the sidecar must report before it is + // actionable. Optimizer checkers set this to the running Core version rather + // than their independently negotiated optimizer version. + UpdaterVersion string // CheckInterval is the probe cadence. 0 = 1 h. CheckInterval time.Duration // SocketPath is where the sidecar listens. Empty disables Trigger. @@ -175,7 +183,7 @@ const MaxReleaseBodyBytes = 16 * 1024 // through unchanged. The main service may also write early states before // handing off to the sidecar, e.g. starting/snapshotting. type UpdateStatus struct { - State string `json:"state"` // idle, starting, snapshotting, pulling, restarting, restoring, done, failed + State string `json:"state"` // idle, starting, snapshotting, pulling, transacting, restarting, restoring, done, failed Action string `json:"action,omitempty"` Component string `json:"component,omitempty"` Target string `json:"target,omitempty"` @@ -311,11 +319,19 @@ func (c *Checker) Check(ctx context.Context, force bool) (Info, error) { } targetTag := c.releaseTargetTag(rel.TagName) if targetTag != "" { - ok, err := rp.hasTag(ctx, targetTag) - if err != nil { - return c.recordErr(fmt.Errorf("registry probe: %w", err)) + if c.cfg.PairedImage != "" && c.cfg.PairManifestAsset != "" { + ok, err := c.verifyControlPlaneRelease(ctx, rel, targetTag) + if err != nil { + return c.recordErr(fmt.Errorf("control-plane release: %w", err)) + } + deployable = ok + } else { + ok, err := rp.hasTag(ctx, targetTag) + if err != nil { + return c.recordErr(fmt.Errorf("registry probe: %w", err)) + } + deployable = ok } - deployable = ok } c.mu.Lock() @@ -395,6 +411,10 @@ type ghRelease struct { PublishedAt time.Time `json:"published_at"` Draft bool `json:"draft"` Prerelease bool `json:"prerelease"` + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + } `json:"assets"` } // fetchLatestRelease asks GitHub for the most-recently-published @@ -507,10 +527,10 @@ func (c *Checker) releaseTargetTag(releaseTag string) string { // periodic Check. func (c *Checker) Info() Info { c.mu.Lock() - defer c.mu.Unlock() c.reloadSkipLocked() info := c.info - info.SidecarReady = c.sidecarReadyLocked() + c.mu.Unlock() + info.SidecarReady = c.sidecarReady() return info } @@ -557,19 +577,13 @@ func (c *Checker) SetChannel(channel Channel) error { return nil } -// sidecarReadyLocked reports whether the updater socket is present as an -// actual Unix socket. An empty SocketPath means the feature was never -// configured for this deploy — docker-compose sets it, native installs -// typically don't. -func (c *Checker) sidecarReadyLocked() bool { - if c.cfg.SocketPath == "" { - return false - } - fi, err := os.Stat(c.cfg.SocketPath) - if err != nil { - return false - } - return fi.Mode()&os.ModeSocket != 0 +// sidecarReady requires the explicit paired-control-plane handshake. This is +// intentionally stronger than a socket stat because the v1.3.1 updater uses +// the same socket and accepts update requests while replacing Core only. +func (c *Checker) sidecarReady() bool { + ctx, cancel := context.WithTimeout(context.Background(), updaterCompatibilityHTTPTimeout) + defer cancel() + return RequireUpdaterRelease(ctx, c.cfg.SocketPath, c.cfg.UpdaterVersion) == nil } func (c *Checker) reloadSkipLocked() { @@ -684,15 +698,10 @@ func (c *Checker) TriggerRollback(ctx context.Context, snapshotID string, files // endpoint. Shared by Trigger and TriggerRollback so the HTTP client // config (socket dialer + timeout) only lives in one place. func (c *Checker) postSidecar(ctx context.Context, body []byte) error { - cli := &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(ctx, "unix", c.cfg.SocketPath) - }, - }, + if err := RequireUpdaterRelease(ctx, c.cfg.SocketPath, c.cfg.UpdaterVersion); err != nil { + return err } + cli := updaterHTTPClient(c.cfg.SocketPath) req, _ := http.NewRequestWithContext(ctx, "POST", "http://unix/update", strings.NewReader(string(body))) req.Header.Set("Content-Type", "application/json") resp, err := cli.Do(req) @@ -785,7 +794,7 @@ func (c *Checker) WriteStatus(st UpdateStatus) error { func isInFlightState(state string) bool { switch state { - case "starting", "snapshotting", "pulling", "restarting", "restoring": + case "starting", "snapshotting", "pulling", "transacting", "restarting", "restoring": return true default: return false diff --git a/go/internal/selfupdate/updater_contract.go b/go/internal/selfupdate/updater_contract.go new file mode 100644 index 00000000..c53ee8e0 --- /dev/null +++ b/go/internal/selfupdate/updater_contract.go @@ -0,0 +1,93 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "slices" + "strings" + "time" +) + +const ( + UpdaterProtocolVersion = 2 + ControlPlanePairCapability = "control-plane-pair-v1" + updaterCompatibilityHTTPTimeout = 5 * time.Second +) + +// UpdaterRuntimeInfo is returned by the sidecar on GET /status. Version and +// capability are part of Core readiness: an updater from an older release may +// understand the JSON request while still updating only Core. +type UpdaterRuntimeInfo struct { + ProtocolVersion int `json:"protocol_version"` + Version string `json:"updater_version"` + Capabilities []string `json:"capabilities"` +} + +func updaterHTTPClient(socketPath string) *http.Client { + return &http.Client{ + Timeout: updaterCompatibilityHTTPTimeout, + Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socketPath) + }, + }, + } +} + +// ProbeUpdater reads the updater's explicit protocol handshake. Merely finding +// a socket is insufficient: v1.3.1 exposes the same socket and accepts the +// request, but replaces Core only. +func ProbeUpdater(ctx context.Context, socketPath string) (UpdaterRuntimeInfo, error) { + if socketPath == "" { + return UpdaterRuntimeInfo{}, errors.New("selfupdate: sidecar socket not configured") + } + fi, err := os.Stat(socketPath) + if err != nil { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater socket: %w", err) + } + if fi.Mode()&os.ModeSocket == 0 { + return UpdaterRuntimeInfo{}, errors.New("selfupdate: updater path is not a Unix socket") + } + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/status", nil) + resp, err := updaterHTTPClient(socketPath).Do(req) + if err != nil { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater handshake: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater handshake HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + var info UpdaterRuntimeInfo + if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&info); err != nil { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: decode updater handshake: %w", err) + } + if info.ProtocolVersion < UpdaterProtocolVersion || !slices.Contains(info.Capabilities, ControlPlanePairCapability) { + return UpdaterRuntimeInfo{}, fmt.Errorf("selfupdate: updater lacks %s protocol %d", ControlPlanePairCapability, UpdaterProtocolVersion) + } + if strings.TrimSpace(info.Version) == "" { + return UpdaterRuntimeInfo{}, errors.New("selfupdate: updater did not report its release version") + } + return info, nil +} + +// RequireUpdaterRelease proves that Core and updater are the same versioned +// control-plane release. Core calls this before state.Open, so a one-sided +// update never runs migrations or reaches readiness. +func RequireUpdaterRelease(ctx context.Context, socketPath, release string) error { + info, err := ProbeUpdater(ctx, socketPath) + if err != nil { + return err + } + if release != "" && info.Version != release { + return fmt.Errorf("selfupdate: updater release %s does not match Core release %s", info.Version, release) + } + return nil +} diff --git a/go/internal/selfupdate/updater_contract_test.go b/go/internal/selfupdate/updater_contract_test.go new file mode 100644 index 00000000..dfb79f59 --- /dev/null +++ b/go/internal/selfupdate/updater_contract_test.go @@ -0,0 +1,106 @@ +package selfupdate + +import ( + "context" + "encoding/json" + "net" + "net/http" + "os" + "strings" + "sync/atomic" + "testing" +) + +func serveUnixUpdater(t *testing.T, handler http.Handler) string { + t.Helper() + f, err := os.CreateTemp("/tmp", "ftw-updater-*.sock") + if err != nil { + t.Fatal(err) + } + path := f.Name() + _ = f.Close() + _ = os.Remove(path) + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + srv := &http.Server{Handler: handler} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { + _ = srv.Close() + _ = os.Remove(path) + }) + return path +} + +// v1.3.1 exposed GET /status and accepted the same update JSON, but ignored +// unknown fields and recreated Core only. This fixture deliberately mirrors +// that wire contract. New Core must reject it before POST and, together with +// the boot-phase 503 readiness gate, therefore cannot migrate in a mixed pair. +func TestV131UpdaterIsRejectedBeforeCoreUpdate(t *testing.T) { + var posts atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"state":"idle"}`)) + }) + mux.HandleFunc("POST /update", func(w http.ResponseWriter, _ *http.Request) { + posts.Add(1) + w.WriteHeader(http.StatusAccepted) + }) + socket := serveUnixUpdater(t, mux) + + c := New(Config{SocketPath: socket, UpdaterVersion: "v1.4.0"}, newMemStore()) + if c.Info().SidecarReady { + t.Fatal("v1.3.1 updater must not be actionable for a new Core") + } + err := c.Trigger(context.Background(), "update", "v1.4.1") + if err == nil || !strings.Contains(err.Error(), ControlPlanePairCapability) { + t.Fatalf("Trigger error = %v, want missing pair capability", err) + } + if got := posts.Load(); got != 0 { + t.Fatalf("legacy updater received %d POST(s); want zero", got) + } +} + +func TestMatchingUpdaterReleaseIsReadyAndAcceptsUpdate(t *testing.T) { + var posts atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(UpdaterRuntimeInfo{ + ProtocolVersion: UpdaterProtocolVersion, + Version: "v1.4.0", + Capabilities: []string{ControlPlanePairCapability}, + }) + }) + mux.HandleFunc("POST /update", func(w http.ResponseWriter, _ *http.Request) { + posts.Add(1) + w.WriteHeader(http.StatusAccepted) + }) + socket := serveUnixUpdater(t, mux) + + c := New(Config{SocketPath: socket, UpdaterVersion: "v1.4.0"}, newMemStore()) + if !c.Info().SidecarReady { + t.Fatal("matching updater should be ready") + } + if err := c.Trigger(context.Background(), "update", "v1.4.1"); err != nil { + t.Fatal(err) + } + if got := posts.Load(); got != 1 { + t.Fatalf("POST count = %d, want one", got) + } +} + +func TestUpdaterReleaseMismatchFailsClosed(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("GET /status", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(UpdaterRuntimeInfo{ + ProtocolVersion: UpdaterProtocolVersion, + Version: "v1.4.1", + Capabilities: []string{ControlPlanePairCapability}, + }) + }) + socket := serveUnixUpdater(t, mux) + if err := RequireUpdaterRelease(context.Background(), socket, "v1.4.0"); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("RequireUpdaterRelease error = %v", err) + } +} diff --git a/scripts/create-control-plane-manifest.sh b/scripts/create-control-plane-manifest.sh new file mode 100755 index 00000000..f102d2ae --- /dev/null +++ b/scripts/create-control-plane-manifest.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +tag="${1:?usage: create-control-plane-manifest.sh TAG REVISION OUTPUT}" +revision="${2:?usage: create-control-plane-manifest.sh TAG REVISION OUTPUT}" +output="${3:?usage: create-control-plane-manifest.sh TAG REVISION OUTPUT}" + +[[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]] || { + echo "invalid control-plane release tag: ${tag}" >&2 + exit 1 +} +[[ "${revision}" =~ ^[0-9a-f]{40}$ ]] || { + echo "invalid control-plane source revision: ${revision}" >&2 + exit 1 +} + +core_image="ghcr.io/srcfl/ftw:${tag}" +updater_image="ghcr.io/srcfl/ftw-updater:${tag}" +core_digest="$(scripts/inspect-image-digest.sh "${core_image}")" +updater_digest="$(scripts/inspect-image-digest.sh "${updater_image}")" + +for digest in "${core_digest}" "${updater_digest}"; do + [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "invalid immutable image digest: ${digest}" >&2 + exit 1 + } +done + +jq -n \ + --arg release "${tag}" \ + --arg revision "${revision}" \ + --arg core_image "${core_image}" \ + --arg core_digest "${core_digest}" \ + --arg updater_image "${updater_image}" \ + --arg updater_digest "${updater_digest}" \ + '{ + schema_version: 1, + release: $release, + revision: $revision, + components: { + core: {image: $core_image, digest: $core_digest}, + updater: {image: $updater_image, digest: $updater_digest} + } + }' >"${output}" diff --git a/scripts/migrate-legacy-compose.sh b/scripts/migrate-legacy-compose.sh index 1fb76a5c..787cdaa2 100755 --- a/scripts/migrate-legacy-compose.sh +++ b/scripts/migrate-legacy-compose.sh @@ -20,7 +20,7 @@ die() { usage() { cat <<'EOF' -Usage: migrate-legacy-compose.sh [--dir PATH] [--backup-dir PATH] +Usage: migrate-legacy-compose.sh --version vX.Y.Z[-beta.N] [--dir PATH] [--backup-dir PATH] Without --dir, the script uses the current directory when it contains docker-compose.yml, then tries ~/ftw and ~/forty-two-watts. @@ -33,6 +33,7 @@ EOF requested_dir="${FTW_DIR:-}" requested_full_backup_dir="${FTW_BACKUP_DIR:-}" +control_plane_version="${FTW_CONTROL_PLANE_VERSION:-}" while [ "$#" -gt 0 ]; do case "$1" in --dir) @@ -45,6 +46,11 @@ while [ "$#" -gt 0 ]; do requested_full_backup_dir="$2" shift 2 ;; + --version) + [ "$#" -ge 2 ] || die "--version requires an immutable release tag" + control_plane_version="$2" + shift 2 + ;; -h|--help) usage exit 0 @@ -55,6 +61,10 @@ while [ "$#" -gt 0 ]; do esac done +if [[ ! "$control_plane_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]]; then + die "--version must be an immutable vX.Y.Z or vX.Y.Z-beta.N release" +fi + [ "$(uname -s)" = Linux ] || die "automatic migration currently supports Linux Docker hosts only" command -v docker >/dev/null 2>&1 || die "docker is not installed" command -v curl >/dev/null 2>&1 || die "curl is not installed" @@ -130,7 +140,7 @@ if [ -n "$compose_project" ]; then fi compose() { if [ "$canonical_tags" = true ]; then - FTW_IMAGE_TAG=latest FTW_UPDATER_IMAGE_TAG=latest FTW_OPTIMIZER_IMAGE_TAG=latest \ + FTW_IMAGE_TAG="$control_plane_version" FTW_UPDATER_IMAGE_TAG="$control_plane_version" FTW_OPTIMIZER_IMAGE_TAG=latest \ "${compose_command[@]}" "$@" else "${compose_command[@]}" "$@" @@ -214,16 +224,25 @@ restore_after_failure() { # A pull can move a mutable tag such as :latest before any container is # recreated. Put every previous immutable image back under its original # reference before Compose restores the old service definitions. - if ! restore_image_reference "$previous_main_image_id" "$previous_main_image_ref"; then + if ! restore_image_reference "$previous_updater_image_id" "$previous_updater_image_ref"; then restore_ok=false fi - if ! restore_image_reference "$previous_updater_image_id" "$previous_updater_image_ref"; then + if ! restore_image_reference "$previous_main_image_id" "$previous_main_image_ref"; then restore_ok=false fi if ! restore_image_reference "$previous_optimizer_image_id" "$previous_optimizer_image_ref"; then restore_ok=false fi + # Restore the updater first. Fixed Core releases refuse readiness when their + # updater does not report the same release, so the reverse order can turn a + # healthy rollback into a restart loop. + if [ "$containers_changed" = true ]; then + if ! compose up -d --no-deps --force-recreate ftw-updater >/dev/null 2>&1; then + restore_ok=false + fi + fi + if [ -n "$renamed_container" ]; then # Remove any replacement by Compose service identity as well as by exact # ID/name. A Compose file without container_name uses a generated name. @@ -252,11 +271,6 @@ restore_after_failure() { restore_ok=false fi fi - if [ "$containers_changed" = true ]; then - if ! compose up -d --no-deps --force-recreate ftw-updater >/dev/null 2>&1; then - restore_ok=false - fi - fi if [ "$optimizer_container_changed" = true ] && [ "$optimizer_service_added" = false ]; then if ! compose up -d --no-deps --force-recreate ftw-optimizer >/dev/null 2>&1; then restore_ok=false @@ -563,26 +577,21 @@ rewrite_service_image() { for file in "${compose_files[@]}"; do rewrite_service_image "$file" "$main_service" 'ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest}' - rewrite_service_image "$file" 'ftw-updater' 'ghcr.io/srcfl/ftw-updater:latest' + rewrite_service_image "$file" 'ftw-updater' 'ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest}' rewrite_service_image "$file" 'ftw-optimizer' 'ghcr.io/srcfl/ftw-optimizer:${FTW_OPTIMIZER_IMAGE_TAG:-latest}' done -# A caller's shell or old .env file may contain a development tag. Use the -# canonical stable tag for this one-time migration; future in-app updates pass -# their own immutable FTW_IMAGE_TAG to Compose. +# A caller's shell or old .env file may contain a development tag. Pin both +# control-plane services to the explicit immutable release selected above. canonical_tags=true compose config >/dev/null effective_main_image="$(compose config --images "$main_service")" effective_updater_image="$(compose config --images ftw-updater)" effective_optimizer_image="$(compose config --images ftw-optimizer)" -case "$effective_main_image" in - ghcr.io/srcfl/ftw:*) ;; - *) die "the effective $main_service image is not ghcr.io/srcfl/ftw: $effective_main_image" ;; -esac -case "$effective_updater_image" in - ghcr.io/srcfl/ftw-updater:*) ;; - *) die "the effective ftw-updater image is not ghcr.io/srcfl/ftw-updater: $effective_updater_image" ;; -esac +[ "$effective_main_image" = "ghcr.io/srcfl/ftw:$control_plane_version" ] || \ + die "the effective $main_service image is not the selected release: $effective_main_image" +[ "$effective_updater_image" = "ghcr.io/srcfl/ftw-updater:$control_plane_version" ] || \ + die "the effective ftw-updater image is not the selected release: $effective_updater_image" case "$effective_optimizer_image" in ghcr.io/srcfl/ftw-optimizer:*) ;; *) die "the effective ftw-optimizer image is not ghcr.io/srcfl/ftw-optimizer: $effective_optimizer_image" ;; @@ -620,11 +629,11 @@ if docker container inspect "$main_service" >/dev/null 2>&1; then fi fi -log "starting $main_service from ghcr.io/srcfl/ftw" +log "starting ftw-updater from ghcr.io/srcfl/ftw-updater:$control_plane_version" containers_changed=true -compose up -d --no-deps --force-recreate "$main_service" -log "starting ftw-updater from ghcr.io/srcfl/ftw-updater" compose up -d --no-deps --force-recreate ftw-updater +log "starting $main_service from ghcr.io/srcfl/ftw:$control_plane_version" +compose up -d --no-deps --force-recreate "$main_service" new_main_id="$(compose ps -q --status running "$main_service" | tail -n 1)" [ -n "$new_main_id" ] || die "$main_service did not reach running state" @@ -633,14 +642,10 @@ updater_id="$(compose ps -q --status running ftw-updater | tail -n 1)" main_image="$(docker inspect "$new_main_id" --format '{{.Config.Image}}')" updater_image="$(docker inspect "$updater_id" --format '{{.Config.Image}}')" -case "$main_image" in - ghcr.io/srcfl/ftw:*) ;; - *) die "running main container uses unexpected image: $main_image" ;; -esac -case "$updater_image" in - ghcr.io/srcfl/ftw-updater:*) ;; - *) die "running updater uses unexpected image: $updater_image" ;; -esac +[ "$main_image" = "ghcr.io/srcfl/ftw:$control_plane_version" ] || \ + die "running main container is not the selected release: $main_image" +[ "$updater_image" = "ghcr.io/srcfl/ftw-updater:$control_plane_version" ] || \ + die "running updater is not the selected release: $updater_image" running_data_source="$(docker inspect "$new_main_id" --format '{{range .Mounts}}{{if eq .Destination "/app/data"}}{{.Source}}{{end}}{{end}}')" if [ -d "$running_data_source" ]; then running_data_source="$(cd "$running_data_source" && pwd -P)" @@ -651,7 +656,8 @@ fi health_url="${FTW_HEALTH_URL:-http://127.0.0.1:8080/api/health}" ready_url="${FTW_READY_URL:-${health_url%/api/health}/api/status}" healthy=false -health_deadline=$((SECONDS + 1800)) +health_timeout_s="${FTW_MIGRATION_HEALTH_TIMEOUT_S:-1800}" +health_deadline=$((SECONDS + health_timeout_s)) while [ "$SECONDS" -lt "$health_deadline" ]; do if curl -fsS --max-time 3 "$health_url" >/dev/null 2>&1 && \ curl -fsS --max-time 3 "$ready_url" >/dev/null 2>&1; then @@ -660,7 +666,7 @@ while [ "$SECONDS" -lt "$health_deadline" ]; do fi sleep 2 done -[ "$healthy" = true ] || die "FTW did not finish initialization at $ready_url within 30 minutes" +[ "$healthy" = true ] || die "FTW did not finish initialization at $ready_url within ${health_timeout_s}s" # Phase 3 deliberately starts only after Core + updater have passed their # health gate. Optimizer has its own release and compatibility handshake; a diff --git a/scripts/release.sh b/scripts/release.sh index 195758d0..c5ed2454 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,12 +1,12 @@ #!/bin/bash -# Build release archives and create (or attach to) a GitHub release. +# Build release archives and attach them to an already-gated draft release. # Usage: ./scripts/release.sh v0.2.0 # # Produces linux arm64/amd64 tarballs + windows amd64 zip via the # Makefile's `release` target, then uploads them to the GitHub release -# for the given tag. If the release doesn't exist yet it's created with -# auto-generated notes; if it does exist the assets are added with -# --clobber so a re-run safely replaces a broken archive. +# for the given tag. This script deliberately cannot create or publish a +# release: only release-assets.yml may make the draft public after verifying +# the immutable Core/updater pair manifest. set -euo pipefail @@ -14,6 +14,20 @@ VERSION=${1:?Usage: $0 } REPO="srcfl/ftw" echo "Building FTW ${VERSION} (linux arm64/amd64, windows amd64)…" + +RELEASE_JSON="$(gh release view "${VERSION}" --repo "${REPO}" --json isDraft,assets)" || { + echo "Release ${VERSION} does not exist. Create it through release.yml; local publishing is disabled." >&2 + exit 1 +} +if [ "$(jq -r .isDraft <<<"${RELEASE_JSON}")" != "true" ]; then + echo "Release ${VERSION} is not a draft; refusing local mutation." >&2 + exit 1 +fi +if ! jq -e '.assets | any(.name == "ftw-control-plane.json")' <<<"${RELEASE_JSON}" >/dev/null; then + echo "Draft ${VERSION} lacks the verified ftw-control-plane.json gate." >&2 + exit 1 +fi + make release ASSETS=( @@ -35,14 +49,7 @@ for f in "${ASSETS[@]}"; do [ -f "$f" ] || { echo "missing: $f"; exit 1; } done -if gh release view "${VERSION}" --repo "${REPO}" >/dev/null 2>&1; then - echo "Release ${VERSION} exists — uploading assets (clobbering duplicates)…" - gh release upload "${VERSION}" --repo "${REPO}" --clobber "${ASSETS[@]}" -else - echo "Creating GitHub release ${VERSION}…" - gh release create "${VERSION}" --repo "${REPO}" \ - --title "${VERSION}" --generate-notes \ - "${ASSETS[@]}" -fi +echo "Uploading assets to verified draft ${VERSION}…" +gh release upload "${VERSION}" --repo "${REPO}" --clobber "${ASSETS[@]}" -echo "Done! Release: https://github.com/${REPO}/releases/tag/${VERSION}" +echo "Assets attached to draft. Only release-assets.yml may publish it." diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index c11aa393..d3751a4f 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -15,5 +15,7 @@ grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile grep -q '^ ftw-optimizer:' docker-compose.yml grep -q 'FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' docker-compose.yml +grep -q 'http://127.0.0.1:8080/api/status' Dockerfile +grep -q -- '--start-period=30m' Dockerfile echo "container module boundaries verified" diff --git a/scripts/test-control-plane-release.sh b/scripts/test-control-plane-release.sh new file mode 100755 index 00000000..fa371243 --- /dev/null +++ b/scripts/test-control-plane-release.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +mkdir -p "${TMP}/bin" +cat >"${TMP}/bin/docker" <<'FAKE_DOCKER' +#!/usr/bin/env bash +set -euo pipefail +ref="${*: -1}" +case "${ref}" in + ghcr.io/srcfl/ftw:v1.4.0) + echo 'Name: test' + echo 'Digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + ;; + ghcr.io/srcfl/ftw-updater:v1.4.0) + echo 'Name: test' + echo "Digest: ${FAKE_UPDATER_DIGEST:-sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb}" + ;; + *) exit 1 ;; +esac +FAKE_DOCKER +chmod +x "${TMP}/bin/docker" + +cd "${ROOT}" +PATH="${TMP}/bin:${PATH}" scripts/create-control-plane-manifest.sh \ + v1.4.0 0123456789abcdef0123456789abcdef01234567 "${TMP}/manifest.json" +PATH="${TMP}/bin:${PATH}" scripts/verify-control-plane-manifest.sh \ + "${TMP}/manifest.json" v1.4.0 0123456789abcdef0123456789abcdef01234567 + +if PATH="${TMP}/bin:${PATH}" FAKE_UPDATER_DIGEST=sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc \ + scripts/verify-control-plane-manifest.sh "${TMP}/manifest.json" v1.4.0 \ + 0123456789abcdef0123456789abcdef01234567 >/dev/null 2>&1; then + echo "moved updater tag should fail pair verification" >&2 + exit 1 +fi + +if PATH="${TMP}/bin:${PATH}" scripts/verify-control-plane-manifest.sh \ + "${TMP}/manifest.json" v1.4.1 0123456789abcdef0123456789abcdef01234567 >/dev/null 2>&1; then + echo "wrong release identity should fail pair verification" >&2 + exit 1 +fi + +echo "control-plane release contract verified" diff --git a/scripts/test-modular-compose.sh b/scripts/test-modular-compose.sh index d841666c..1b3029bf 100755 --- a/scripts/test-modular-compose.sh +++ b/scripts/test-modular-compose.sh @@ -102,6 +102,7 @@ FAKE_UNAME cat >"$TMP/migrate/bin/curl" <<'FAKE_CURL' #!/usr/bin/env bash +[ "${FAKE_FAIL_READY:-}" != 1 ] || exit 1 exit 0 FAKE_CURL @@ -205,10 +206,10 @@ case "$command" in *Config.Image*) case "$subject" in ftw-id) - if [ -f "$state/canonical-ftw" ]; then echo 'ghcr.io/srcfl/ftw:latest'; else echo 'example.invalid/old-core:latest'; fi + if [ -f "$state/canonical-ftw" ]; then echo "ghcr.io/srcfl/ftw:$(cat "$state/canonical-ftw")"; else echo 'example.invalid/old-core:latest'; fi ;; ftw-updater-id) - if [ -f "$state/canonical-ftw-updater" ]; then echo 'ghcr.io/srcfl/ftw-updater:latest'; else echo 'example.invalid/old-updater:latest'; fi + if [ -f "$state/canonical-ftw-updater" ]; then echo "ghcr.io/srcfl/ftw-updater:$(cat "$state/canonical-ftw-updater")"; else echo 'example.invalid/old-updater:latest'; fi ;; ftw-optimizer-id) if [ -f "$state/canonical-ftw-optimizer" ]; then echo 'ghcr.io/srcfl/ftw-optimizer:latest'; else echo 'example.invalid/old-optimizer:latest'; fi @@ -256,8 +257,8 @@ case "$command" in ;; --images) case "${2:-}" in - ftw) echo 'ghcr.io/srcfl/ftw:latest' ;; - ftw-updater) echo 'ghcr.io/srcfl/ftw-updater:latest' ;; + ftw) echo "ghcr.io/srcfl/ftw:${FTW_IMAGE_TAG:-latest}" ;; + ftw-updater) echo "ghcr.io/srcfl/ftw-updater:${FTW_UPDATER_IMAGE_TAG:-latest}" ;; ftw-optimizer) echo 'ghcr.io/srcfl/ftw-optimizer:latest' ;; *) exit 1 ;; esac @@ -278,7 +279,12 @@ case "$command" in touch "$state/failure-consumed" exit 1 fi - touch "$state/$service" "$state/$service-id" "$state/canonical-$service" + touch "$state/$service" "$state/$service-id" + case "$service" in + ftw) printf '%s\n' "${FTW_IMAGE_TAG:-latest}" >"$state/canonical-$service" ;; + ftw-updater) printf '%s\n' "${FTW_UPDATER_IMAGE_TAG:-latest}" >"$state/canonical-$service" ;; + ftw-optimizer) printf '%s\n' "${FTW_OPTIMIZER_IMAGE_TAG:-latest}" >"$state/canonical-$service" ;; + esac ;; ps) service="${*: -1}" @@ -314,7 +320,7 @@ touch "$TMP/migrate/data/state.db" PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/migrate/state" \ FAKE_DATA_DIR="$TMP/migrate/data" \ -bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/migrate" +bash "$ROOT/scripts/migrate-legacy-compose.sh" --version v1.4.0 --dir "$TMP/migrate" grep -q '^ ftw-optimizer:' "$TMP/migrate/docker-compose.override.yml" test -f "$TMP/migrate/state/ftw" @@ -332,7 +338,7 @@ FAKE_STATE_DIR="$TMP/project/state" \ FAKE_DATA_DIR="$TMP/project/data" \ FAKE_PROJECT_NAME=custom-energy \ FAKE_INSTALL_DIR="$TMP/project" \ -bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/project" +bash "$ROOT/scripts/migrate-legacy-compose.sh" --version v1.4.0 --dir "$TMP/project" test -e "$TMP/project/state/project-custom-energy" if [ -n "$REAL_DOCKER" ]; then @@ -358,7 +364,7 @@ if PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/rollback/state" \ FAKE_DATA_DIR="$TMP/rollback/data" \ FAKE_FAIL_SERVICE=ftw \ - bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/rollback" \ + bash "$ROOT/scripts/migrate-legacy-compose.sh" --version v1.4.0 --dir "$TMP/rollback" \ >/dev/null 2>&1; then echo "expected a failed core recreate to roll the modular migration back" >&2 exit 1 @@ -392,7 +398,7 @@ if PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/rollback-existing/state" \ FAKE_DATA_DIR="$TMP/rollback-existing/data" \ FAKE_FAIL_SERVICE=ftw \ - bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/rollback-existing" \ + bash "$ROOT/scripts/migrate-legacy-compose.sh" --version v1.4.0 --dir "$TMP/rollback-existing" \ >/dev/null 2>&1; then echo "expected an existing modular migration failure to roll back" >&2 exit 1 @@ -403,6 +409,35 @@ grep -q '^sha256:old-core example.invalid/old-core:latest$' "$TMP/rollback-exist grep -q '^sha256:old-updater example.invalid/old-updater:latest$' "$TMP/rollback-existing/state/image-tags" grep -q '^sha256:old-optimizer example.invalid/old-optimizer:latest$' "$TMP/rollback-existing/state/image-tags" +# A failure after both candidates were recreated must still restore both old +# control-plane image IDs and never report migration success. +mkdir -p "$TMP/readiness-rollback/data" "$TMP/readiness-rollback/state" +touch "$TMP/readiness-rollback/data/state.db" +touch "$TMP/readiness-rollback/state/ftw" "$TMP/readiness-rollback/state/ftw-updater" +cat >"$TMP/readiness-rollback/docker-compose.yml" <<'YAML' +services: + ftw: + image: example.invalid/old-core:latest + volumes: + - ./data:/app/data + ftw-updater: + image: example.invalid/old-updater:latest +YAML +if PATH="$TMP/migrate/bin:$PATH" \ + FAKE_STATE_DIR="$TMP/readiness-rollback/state" \ + FAKE_DATA_DIR="$TMP/readiness-rollback/data" \ + FAKE_FAIL_READY=1 \ + FTW_MIGRATION_HEALTH_TIMEOUT_S=1 \ + bash "$ROOT/scripts/migrate-legacy-compose.sh" --version v1.4.0 --dir "$TMP/readiness-rollback" \ + >/dev/null 2>&1; then + echo "expected readiness failure to roll back the paired migration" >&2 + exit 1 +fi +grep -q 'example.invalid/old-core:latest' "$TMP/readiness-rollback/docker-compose.yml" +grep -q 'example.invalid/old-updater:latest' "$TMP/readiness-rollback/docker-compose.yml" +grep -q '^sha256:old-core example.invalid/old-core:latest$' "$TMP/readiness-rollback/state/image-tags" +grep -q '^sha256:old-updater example.invalid/old-updater:latest$' "$TMP/readiness-rollback/state/image-tags" + # Optimizer is an independent, optional phase. A failed optimizer candidate # must leave the newly healthy Core + updater online and must not fail the # migration or activate a driver. @@ -421,7 +456,7 @@ PATH="$TMP/migrate/bin:$PATH" \ FAKE_STATE_DIR="$TMP/optimizer-failure/state" \ FAKE_DATA_DIR="$TMP/optimizer-failure/data" \ FAKE_FAIL_SERVICE=ftw-optimizer \ -bash "$ROOT/scripts/migrate-legacy-compose.sh" --dir "$TMP/optimizer-failure" \ +bash "$ROOT/scripts/migrate-legacy-compose.sh" --version v1.4.0 --dir "$TMP/optimizer-failure" \ >/dev/null test -e "$TMP/optimizer-failure/state/ftw" test -e "$TMP/optimizer-failure/state/ftw-updater" diff --git a/scripts/verify-control-plane-manifest.sh b/scripts/verify-control-plane-manifest.sh new file mode 100755 index 00000000..d554ad0a --- /dev/null +++ b/scripts/verify-control-plane-manifest.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +manifest="${1:?usage: verify-control-plane-manifest.sh MANIFEST TAG REVISION}" +tag="${2:?usage: verify-control-plane-manifest.sh MANIFEST TAG REVISION}" +revision="${3:?usage: verify-control-plane-manifest.sh MANIFEST TAG REVISION}" + +jq -e \ + --arg release "${tag}" \ + --arg revision "${revision}" \ + --arg core "ghcr.io/srcfl/ftw:${tag}" \ + --arg updater "ghcr.io/srcfl/ftw-updater:${tag}" ' + .schema_version == 1 + and .release == $release + and .revision == $revision + and .components.core.image == $core + and .components.updater.image == $updater + and (.components.core.digest | test("^sha256:[0-9a-f]{64}$")) + and (.components.updater.digest | test("^sha256:[0-9a-f]{64}$")) + ' "${manifest}" >/dev/null + +core_ref="$(jq -r '.components.core.image' "${manifest}")" +updater_ref="$(jq -r '.components.updater.image' "${manifest}")" +expected_core="$(jq -r '.components.core.digest' "${manifest}")" +expected_updater="$(jq -r '.components.updater.digest' "${manifest}")" +actual_core="$(scripts/inspect-image-digest.sh "${core_ref}")" +actual_updater="$(scripts/inspect-image-digest.sh "${updater_ref}")" + +[ "${actual_core}" = "${expected_core}" ] || { + echo "Core tag moved: ${actual_core} != ${expected_core}" >&2 + exit 1 +} +[ "${actual_updater}" = "${expected_updater}" ] || { + echo "updater tag moved: ${actual_updater} != ${expected_updater}" >&2 + exit 1 +} diff --git a/web/release-metadata.test.mjs b/web/release-metadata.test.mjs index 1179637c..d0e6ce80 100644 --- a/web/release-metadata.test.mjs +++ b/web/release-metadata.test.mjs @@ -12,6 +12,18 @@ const releaseWorkflow = readFileSync( join(repoRoot, ".github", "workflows", "release.yml"), "utf8", ); +const betaWorkflow = readFileSync( + join(repoRoot, ".github", "workflows", "beta.yml"), + "utf8", +); +const releaseAssetsWorkflow = readFileSync( + join(repoRoot, ".github", "workflows", "release-assets.yml"), + "utf8", +); +const localReleaseScript = readFileSync( + join(repoRoot, "scripts", "release.sh"), + "utf8", +); const changesetCheckWorkflow = readFileSync( join(repoRoot, ".github", "workflows", "changeset-check.yml"), "utf8", @@ -40,4 +52,28 @@ describe("release metadata", () => { /changeset status --since=["']origin\/\$\{\{ github\.base_ref \}\}["']/, ); }); + + it("keeps stable draft-only until the immutable control-plane pair passes", () => { + assert.match(releaseWorkflow, /gh release create "\$\{TAG\}"[\s\S]+--draft/); + assert.match(releaseWorkflow, /verify-control-plane-manifest\.sh[\s\S]+"\$\{BETA_TAG\}"/); + assert.match(releaseAssetsWorkflow, /name: verify pair and publish release/); + assert.match(releaseAssetsWorkflow, /gh release edit "\$\{TAG\}"[\s\S]+--draft=false/); + assert.match(releaseAssetsWorkflow, /needs: \[meta, control-plane\]/); + }); + + it("requires the same-release Core and updater digests on beta and stable", () => { + for (const workflow of [betaWorkflow, releaseAssetsWorkflow]) { + assert.match(workflow, /create-control-plane-manifest\.sh/); + assert.match(workflow, /verify-control-plane-manifest\.sh/); + assert.match(workflow, /ftw-control-plane\.json/); + assert.match(workflow, /Refuse to move an existing immutable tag/); + } + }); + + it("does not let the local release helper create or publish releases", () => { + assert.doesNotMatch(localReleaseScript, /gh release create/); + assert.doesNotMatch(localReleaseScript, /gh release edit/); + assert.match(localReleaseScript, /isDraft/); + assert.match(localReleaseScript, /ftw-control-plane\.json/); + }); });