ci: one shellcheck gate, and it is the one that does not apt-install (#753)

.github/workflows/helm-ci.yaml

@@ -54,9 +54,17 @@ jobs:
 
       # helm is not on the runner image at a version we choose, so it is
       # installed here at the pin this repo standardises on. That is
-      # environment bootstrapping, not a duplicated command -- the same
-      # distinction that keeps the shellcheck apt install in
-      # standard-checks.yml. Both `helm-lint` and `helm-vocab` shell out to
+      # environment bootstrapping, not a duplicated command.
+      #
+      # This comment used to cite the shellcheck apt install in
+      # standard-checks.yml as the parallel case. That install is gone (#753):
+      # its premise was wrong -- shellcheck IS on the runner image -- so it was
+      # never the same distinction, and it hung CI for 35 minutes. helm is the
+      # genuine case: pinned, absent from the image, and installed by a
+      # sha-pinned action rather than a package manager, which is what a
+      # bootstrap step should look like.
+      #
+      # Both `helm-lint` and `helm-vocab` shell out to
       # `helm`, and chart-env-vocabulary.sh's helper-backstop cases branch on
       # the helm VERSION (they self-skip below 3.16), so an unpinned runner
       # helm would silently change which assertions run. drift-checks.yaml

.github/workflows/installer-tests.yaml

@@ -2,7 +2,9 @@ name: Installer tests
 
 # Validates the curl/PowerShell installer (scripts/) across the breadth of
 # environments a customer might actually have:
-#   • static    — shellcheck + bash -n + PSScriptAnalyzer
+#   • static    — bash -n + PSScriptAnalyzer + the repo's drift guards
+#                  (shellcheck is NOT here: it is the `quality / shellcheck`
+#                  required check — see the `static` job for why)
 #   • unit-bash  — bats (mocked) for the bash installer
 #   • unit-pester— Pester for the PowerShell installer, on Linux AND real Windows
 #   • distro-prereqs — runs the REAL Linux prerequisite-install path (package
@@ -66,22 +68,31 @@ jobs:
             | while IFS= read -r -d '' f; do bash -n "$f" || exit 1; done
           echo "all shell scripts parse"
 
-      - name: ShellCheck (libs + entrypoints)
-        run: |
-          sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck
-          shellcheck --version | grep version
-          # Gate at error severity. The libs are sourced together as one program,
-          # so single-file shellcheck reports SC2034 "unused" false positives for
-          # shared vars (CURL_SECURE, ARCH_DL, colours…) that are defined in
-          # common.sh and consumed in other sourced files. Warnings are printed
-          # below for visibility but don't fail the gate.
-          shellcheck --severity=error --shell=bash \
-            scripts/install.sh scripts/install-k8s.sh scripts/gen-manifest.sh scripts/check-facts.sh scripts/check-style.sh scripts/resolve-ingestor-digest.sh scripts/lib/*.sh \
-            scripts/tests/check-drift.sh scripts/tests/distro-prereqs.sh scripts/tests/e2e-auto-upgrade.sh scripts/tests/e2e-seal-check.sh scripts/tests/e2e-full-seal.sh scripts/tests/e2e-cluster.sh scripts/tests/e2e-journey.sh scripts/tests/e2e-proxy.sh scripts/tests/lib/e2e-common.sh scripts/tests/path-persist.sh scripts/tests/chart-env-vocabulary.sh scripts/tests/env-vocabulary-agreement.sh scripts/tests/telemetry-vocabulary-agreement.sh
-          echo "── shellcheck warnings (advisory, non-blocking) ──"
-          shellcheck --severity=warning --shell=bash \
-            scripts/install.sh scripts/install-k8s.sh scripts/gen-manifest.sh scripts/check-facts.sh scripts/check-style.sh scripts/resolve-ingestor-digest.sh scripts/lib/*.sh \
-            scripts/tests/check-drift.sh scripts/tests/distro-prereqs.sh scripts/tests/e2e-auto-upgrade.sh scripts/tests/e2e-seal-check.sh scripts/tests/e2e-full-seal.sh scripts/tests/e2e-cluster.sh scripts/tests/e2e-journey.sh scripts/tests/e2e-proxy.sh scripts/tests/lib/e2e-common.sh scripts/tests/path-persist.sh scripts/tests/chart-env-vocabulary.sh scripts/tests/env-vocabulary-agreement.sh scripts/tests/telemetry-vocabulary-agreement.sh || true
+      # There is no shellcheck step here any more, and adding one back is a
+      # regression. shellcheck ran in THREE places in this repo (#753): here,
+      # `Standard checks / Lint`, and `quality / shellcheck` from
+      # tracebloc/.github's code-quality.yml. This copy is the one with no
+      # reason to exist:
+      #
+      #   • It is not a REQUIRED check on develop (this whole job isn't), so it
+      #     could never block anything — it was advice, not a gate. Making the
+      #     R8 step below a gate is backend#2182 / #2183, a separate concern.
+      #   • It ENUMERATED its file set. Measured on develop 2026-08-19: the list
+      #     that used to sit here expanded to 34 files where deriving from the
+      #     tree yields 44 — it had drifted past eight real scripts, including
+      #     scripts/chart-version-guard.sh, scripts/check-digest-drift.sh and
+      #     scripts/index-invariants.sh, and #747 then had to hand-edit the list
+      #     in three places to add one new script. A parse error planted in
+      #     index-invariants.sh was invisible to this step.
+      #   • shellcheck is preinstalled on ubuntu-latest, so it needs no
+      #     `apt-get install` — the very step that hung this job to its
+      #     10-minute timeout twice on 2026-08-19 and reported as `cancelled`.
+      #
+      # The two survivors are complementary rather than duplicated, and both
+      # derive their file set: `Standard checks / Lint` runs `make lint` over
+      # the WHOLE TREE, and `quality / shellcheck` scans the PR DIFF. Same
+      # severity (error) and same SC1091 exclusion as the step deleted here, so
+      # no verdict changes.
 
       - name: Installer manifest is current (supply-chain, R8)
         # The bootstrap verifies each sub-script against scripts/manifest.sha256
@@ -128,13 +139,47 @@ jobs:
           Write-Host "no PSScriptAnalyzer errors"
 
   unit-bash:
-    timeout-minutes: 10
+    # 10 -> 15. At 10 the bounded install (390 s worst case) plus ~155 s of bats
+    # could still reach the cap and report `cancelled`, which is the #753 bug the
+    # bound exists to remove. Matches standard-checks.yml's `Unit tests`.
+    timeout-minutes: 15
     name: bats (bash unit, mocked)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: Install bats
-        run: sudo apt-get update -qq && sudo apt-get install -y -qq bats
+
+      # Bounded and retried, same shape as "Pull <distro> (bounded, retried)"
+      # below. This is not speculative hardening: this exact step stalled on
+      # 2026-08-19 at 11:58 and burned the whole `timeout-minutes: 10` budget,
+      # reporting `cancelled` with an empty log and `Run bats` skipped -- the
+      # #753 signature, on bats rather than shellcheck. shellcheck could be
+      # fixed by deleting the install (it is preinstalled); bats genuinely has
+      # to be installed, so it gets a bound instead.
+      #
+      # `sudo timeout`, not `timeout sudo`: timeout must run as root so it
+      # signals apt-get directly rather than signalling sudo and hoping the TERM
+      # is forwarded. `-qq` is dropped on the install so a slow mirror leaves
+      # something in the log, and `bats --version` is the proof-of-install
+      # marker whose absence identified the stall in the first place.
+      - name: Install bats (bounded, retried)
+        # Bounds sized against the job budget (see the twin step in
+        # standard-checks.yml for the arithmetic): worst case 3 x (60 + 60) +
+        # 30 s backoff = 390 s, plus ~155 s of bats, inside the 15 minutes this
+        # job now allows. `if`/`fi` not `[ ... ] &&` because the default shell is
+        # `bash -e` and a false test on the last iteration would abort before the
+        # ::error:: line printed.
+        run: |
+          for i in 1 2 3; do
+            if sudo timeout 60 apt-get update -qq \
+               && sudo timeout 60 apt-get install -y bats; then
+              bats --version && exit 0
+            fi
+            echo "::warning::bats install stalled or failed (attempt $i/3)"
+            if [ "$i" -lt 3 ]; then sleep $((i*10)); fi
+          done
+          echo "::error::could not install bats in 3 bounded attempts - runner-to-mirror connectivity, not this PR. Re-run this job."
+          exit 1
+
       - name: Run bats
         run: bats scripts/tests/*.bats
 

.github/workflows/standard-checks.yml

@@ -29,45 +29,90 @@ jobs:
   lint:
     name: Lint
     runs-on: ubuntu-latest
+    # #753: this job had no timeout at all, so when `Install shellcheck` stalled
+    # on 2026-08-19 it drifted toward GitHub's 360-minute default instead of
+    # failing fast. Every job in this file now has a bound.
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      # shellcheck itself is not on the runner image, so it is installed here.
-      # That is environment bootstrapping, not a duplicated command.
-      - name: Install shellcheck
-        run: |
-          sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck
-          shellcheck --version | grep version
+      # The `Install shellcheck` step that used to sit here is gone, and the
+      # premise it was written on ("shellcheck itself is not on the runner
+      # image") is simply false — it IS preinstalled on ubuntu-latest (0.9.0,
+      # measured in run 32232682350), which is how `quality / shellcheck` runs
+      # without installing anything. The apt call bought nothing and cost this
+      # job 35+ minutes on 2026-08-19 (#753).
 
       - name: make lint
-        # `make lint` (backend#1606). This job used to restate the parse loop AND
-        # spell out the shellcheck file list inline, while the Makefile kept the
-        # same list in SHELLCHECK_FILES. The two had ALREADY drifted, and in the
-        # direction nobody notices: the Makefile carried 19 entries and this file
-        # 9, so ten scripts were shellchecked on a contributor's machine and NOT
-        # at the merge gate --
+        # Calls the Makefile rather than restating the commands (backend#1606).
+        # That ticket's concern was a hand-written FILE LIST drifting between
+        # the Makefile and this workflow; it fixed the drift by re-copying the
+        # list, which only reset the clock — measured 2026-08-19, the list had
+        # drifted again, past eight scripts, and #747 then had to hand-edit it
+        # in three places to add one new script. `make lint` enumerates nothing:
+        # `parse` is a find, `shellcheck` derives from `git ls-files`.
         #
-        #   scripts/gen-manifest.sh          scripts/check-facts.sh
-        #   scripts/check-style.sh           scripts/lib/*.sh
-        #   scripts/tests/check-drift.sh     scripts/tests/e2e-full-seal.sh
-        #   scripts/tests/e2e-journey.sh     scripts/tests/path-persist.sh
-        #   scripts/tests/chart-env-vocabulary.sh
-        #   scripts/tests/env-vocabulary-agreement.sh
+        # WHOLE TREE, and that is the point of running it here (Arturo, #754
+        # review). `quality / shellcheck` is required and derives the same way,
+        # but on a PR it scans only the DIFF and exits 0 on an empty one. If
+        # this job dropped shellcheck, no CI job would run a whole-tree
+        # error-severity sweep at all — the caller has no `schedule:` and passes
+        # `all-files: false` — so the superset claim behind #753 would have been
+        # true set-wise and false scope-wise. The two are complementary, not
+        # duplicates: diff-scoped gate + whole-tree gate, neither installing
+        # anything, neither holding a file list.
         #
-        # gen-manifest.sh is the installer's integrity manifest generator, so a
-        # shell defect there was invisible to this gate. Measured green across
-        # all 19 before this landed, so it arms a green check rather than
-        # importing a backlog.
+        # Verified green on the 44-file derived set under the runner's 0.9.0
+        # (not just a newer local build) before arming it.
         run: make lint
 
   unit-tests:
     name: Unit tests
     runs-on: ubuntu-latest
+    # See the lint job: no job in this file had a timeout before #753.
+    # bats itself runs in ~2 min; the headroom is for the apt install.
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - name: Install bats
-        run: sudo apt-get update -qq && sudo apt-get install -y -qq bats
+      # bats genuinely has to be installed -- unlike shellcheck there is no
+      # preinstalled or reusable equivalent -- so it gets a bound rather than a
+      # deletion. Bounded and retried in the same shape as installer-tests.yaml's
+      # "Pull <distro> (bounded, retried)".
+      #
+      # This was left as plain apt in the first draft of #753, on the grounds
+      # that hardening installs was that ticket's own scope. Then the identical
+      # step in installer-tests.yaml's `unit-bash` stalled at 11:58 on
+      # 2026-08-19 and burned its whole 10-minute budget, reporting `cancelled`
+      # with an empty log. A required check is the wrong place to leave a known
+      # unbounded external call.
+      #
+      # `sudo timeout`, not `timeout sudo`: timeout must run as root to signal
+      # apt-get directly instead of signalling sudo and hoping TERM is
+      # forwarded. `-qq` dropped on the install so a stall is not silent, and
+      # `bats --version` is the proof-of-install marker.
+      - name: Install bats (bounded, retried)
+        # Bounds SIZED against the job budget, the way the distro-pull step
+        # sizes its own. Measured healthy: install 9 s, bats run 155 s. Worst
+        # case here is 3 x (60 + 60) + 10 + 20 = 390 s, plus ~155 s of bats =
+        # ~9 min, inside this job's 15. At the 120 s bound this first had, worst
+        # case was 780 s -- which would have blown a 10-minute budget and
+        # reported `cancelled` again, i.e. reintroduced the very bug.
+        #
+        # `if`/`fi` rather than `[ ... ] && sleep`: the default shell is
+        # `bash -e`, so a bare test returning false on the last iteration would
+        # abort the script before the ::error:: line ever printed.
+        run: |
+          for i in 1 2 3; do
+            if sudo timeout 60 apt-get update -qq \
+               && sudo timeout 60 apt-get install -y bats; then
+              bats --version && exit 0
+            fi
+            echo "::warning::bats install stalled or failed (attempt $i/3)"
+            if [ "$i" -lt 3 ]; then sleep $((i*10)); fi
+          done
+          echo "::error::could not install bats in 3 bounded attempts - runner-to-mirror connectivity, not this PR. Re-run this job."
+          exit 1
 
       - name: Run bats (bash unit suite, mocked)
         run: bats scripts/tests/*.bats