diff --git a/.github/workflows/drift-checks.yaml b/.github/workflows/drift-checks.yaml index b8bcd839..0770ae73 100644 --- a/.github/workflows/drift-checks.yaml +++ b/.github/workflows/drift-checks.yaml @@ -59,3 +59,14 @@ jobs: # helm-ci's `paths:` filter would have skipped entirely. - name: CLIENT_ENV vocabulary agreement (backend#1729) run: bash scripts/tests/env-vocabulary-agreement.sh + # The installer's telemetry vocabularies (backend#1907): four closed sets + # whose values are produced elsewhere -- the phases by install-k8s.sh's + # step_header calls, the client states by summary.sh, the script names by + # gen-manifest.sh's FILES array, the error classes by telemetry.sh's own + # classifier. A closed set that has drifted from its producer does not fail + # loudly; it reports `unknown` forever, on exactly the runs somebody added + # the new value for. It lives HERE for the same reason the CLIENT_ENV guard + # does: `Source-of-truth drift` is required, so this can block rather than + # advise. bash only, ~1s. + - name: Telemetry vocabulary agreement (backend#1907) + run: bash scripts/tests/telemetry-vocabulary-agreement.sh diff --git a/.github/workflows/installer-tests.yaml b/.github/workflows/installer-tests.yaml index f3904f71..73abcfeb 100644 --- a/.github/workflows/installer-tests.yaml +++ b/.github/workflows/installer-tests.yaml @@ -77,11 +77,11 @@ jobs: # below for visibility but don't fail the gate. shellcheck --severity=error --shell=bash \ scripts/install.sh scripts/install-k8s.sh scripts/gen-manifest.sh scripts/check-facts.sh scripts/check-style.sh scripts/resolve-ingestor-digest.sh scripts/lib/*.sh \ - scripts/tests/check-drift.sh scripts/tests/distro-prereqs.sh scripts/tests/e2e-auto-upgrade.sh scripts/tests/e2e-seal-check.sh scripts/tests/e2e-full-seal.sh scripts/tests/e2e-cluster.sh scripts/tests/e2e-journey.sh scripts/tests/e2e-proxy.sh scripts/tests/lib/e2e-common.sh scripts/tests/path-persist.sh scripts/tests/chart-env-vocabulary.sh scripts/tests/env-vocabulary-agreement.sh + scripts/tests/check-drift.sh scripts/tests/distro-prereqs.sh scripts/tests/e2e-auto-upgrade.sh scripts/tests/e2e-seal-check.sh scripts/tests/e2e-full-seal.sh scripts/tests/e2e-cluster.sh scripts/tests/e2e-journey.sh scripts/tests/e2e-proxy.sh scripts/tests/lib/e2e-common.sh scripts/tests/path-persist.sh scripts/tests/chart-env-vocabulary.sh scripts/tests/env-vocabulary-agreement.sh scripts/tests/telemetry-vocabulary-agreement.sh echo "── shellcheck warnings (advisory, non-blocking) ──" shellcheck --severity=warning --shell=bash \ scripts/install.sh scripts/install-k8s.sh scripts/gen-manifest.sh scripts/check-facts.sh scripts/check-style.sh scripts/resolve-ingestor-digest.sh scripts/lib/*.sh \ - scripts/tests/check-drift.sh scripts/tests/distro-prereqs.sh scripts/tests/e2e-auto-upgrade.sh scripts/tests/e2e-seal-check.sh scripts/tests/e2e-full-seal.sh scripts/tests/e2e-cluster.sh scripts/tests/e2e-journey.sh scripts/tests/e2e-proxy.sh scripts/tests/lib/e2e-common.sh scripts/tests/path-persist.sh scripts/tests/chart-env-vocabulary.sh scripts/tests/env-vocabulary-agreement.sh || true + scripts/tests/check-drift.sh scripts/tests/distro-prereqs.sh scripts/tests/e2e-auto-upgrade.sh scripts/tests/e2e-seal-check.sh scripts/tests/e2e-full-seal.sh scripts/tests/e2e-cluster.sh scripts/tests/e2e-journey.sh scripts/tests/e2e-proxy.sh scripts/tests/lib/e2e-common.sh scripts/tests/path-persist.sh scripts/tests/chart-env-vocabulary.sh scripts/tests/env-vocabulary-agreement.sh scripts/tests/telemetry-vocabulary-agreement.sh || true - name: Installer manifest is current (supply-chain, R8) # The bootstrap verifies each sub-script against scripts/manifest.sha256 diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..d3185b31 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,14 @@ +# Deliberate false positives for gitleaks live here. This file is auto-loaded from +# the repo root by tracebloc/.github's code-quality workflow and, unlike the +# baseline, is commit-INDEPENDENT — the right home for a canary fixture the scanner +# keeps re-finding across branch history. (The baseline is only for genuine +# historical exposure tracked for rotation.) +[extend] +useDefault = true + +[allowlist] +description = "The telemetry canary tests feed fabricated credentials (hunter2, the joke password) through curl -u / proxy URLs to prove the redaction guards strip them. These are never real secrets, so curl-auth-user hits on `$CANARY:hunter2` across this branch's history are false positives (backend#1907)." +regexTarget = "match" +regexes = [ + '''curl -u \$CANARY:hunter2''', +] diff --git a/Makefile b/Makefile index 3116ffb3..3b00d2d2 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,8 @@ SHELLCHECK_FILES := \ scripts/tests/lib/e2e-common.sh \ scripts/tests/path-persist.sh \ scripts/tests/chart-env-vocabulary.sh \ - scripts/tests/env-vocabulary-agreement.sh + scripts/tests/env-vocabulary-agreement.sh \ + scripts/tests/telemetry-vocabulary-agreement.sh # The bats total, DERIVED — never written down. It moves on most PRs that add a # test, nothing enforces it, and the help text had drifted from its hardcoded @@ -182,6 +183,7 @@ drift: scripts/check-facts.sh --check bash scripts/check-style.sh bash scripts/tests/env-vocabulary-agreement.sh + bash scripts/tests/telemetry-vocabulary-agreement.sh # digest-drift: the watcher on every mutable label that points at a pinned # digest (backend#1853). NOT in `check`: it needs the network and a docker diff --git a/scripts/gen-manifest.sh b/scripts/gen-manifest.sh index 0c3c48d1..abd60226 100755 --- a/scripts/gen-manifest.sh +++ b/scripts/gen-manifest.sh @@ -29,6 +29,7 @@ cd "$REPO_ROOT" FILES=( "scripts/install-k8s.sh" "scripts/lib/common.sh" + "scripts/lib/telemetry.sh" "scripts/lib/preflight.sh" "scripts/lib/detect-gpu.sh" "scripts/lib/gpu-nvidia.sh" diff --git a/scripts/install-k8s.sh b/scripts/install-k8s.sh index 4268c6b2..bbf27b54 100755 --- a/scripts/install-k8s.sh +++ b/scripts/install-k8s.sh @@ -58,6 +58,15 @@ LIB_DIR="${SCRIPT_DIR}/lib" # ── Source modules ─────────────────────────────────────────────────────────── source "${LIB_DIR}/common.sh" +# telemetry.sh (backend#1907) is sourced right after common.sh so the install +# clock starts before any work does, and so common.sh's step_header / +# install_cleanup hooks find their functions. Guarded like the other late +# additions: an older bootstrap (e.g. a not-yet-updated tracebloc.io/i.sh, whose +# FILES list is hand-maintained) may not have fetched it, and an installer that +# aborted because it could not report on itself would be a poor trade. +if [[ -f "${LIB_DIR}/telemetry.sh" ]]; then + source "${LIB_DIR}/telemetry.sh" +fi source "${LIB_DIR}/preflight.sh" source "${LIB_DIR}/detect-gpu.sh" source "${LIB_DIR}/gpu-nvidia.sh" @@ -149,6 +158,21 @@ main() { error "This installer build doesn't include prepare-host (stale bootstrap). Re-run: curl -fsSL https://tracebloc.io/i.sh | bash -s -- prepare-host" done + # Past this line the run is committed to installing, so it is the one that + # produces an outcome event (backend#1907). Everything above is terminal and + # touches nothing: --help exits 0, --diagnose clears the EXIT trap, and + # prepare-host swaps it for a lightweight reaper. Without this latch, a + # `--help` emitted install.run.succeeded and inflated the denominator of the + # failure rate the ticket exists to produce (Bugbot, client#747). + # + # NOT covered, deliberately: prepare-host. It is a different command with its + # own registry component (§10.1 gives `installer` the components install / + # preflight / upgrade), and reporting it as tracebloc.component=install would + # be mislabelling it rather than measuring it. + if declare -F telemetry_run_started >/dev/null 2>&1; then + telemetry_run_started + fi + # Run-modifying flags (unlike --help/--diagnose, which are terminal). --force / # --reinstall skips the stop-and-check gate below and re-runs every step. Also # honored via TRACEBLOC_FORCE_REINSTALL=1 for the curl|bash path (assess.sh diff --git a/scripts/install.sh b/scripts/install.sh index cb1580ae..cad2a203 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -265,6 +265,7 @@ mkdir -p "$TMPDIR/lib" FILES=( "scripts/install-k8s.sh" "scripts/lib/common.sh" + "scripts/lib/telemetry.sh" "scripts/lib/preflight.sh" "scripts/lib/detect-gpu.sh" "scripts/lib/gpu-nvidia.sh" diff --git a/scripts/lib/assess.sh b/scripts/lib/assess.sh index b990a85f..b8115d53 100644 --- a/scripts/lib/assess.sh +++ b/scripts/lib/assess.sh @@ -337,6 +337,13 @@ _assess_classify() { # `tracebloc` is somehow still unresolvable, fall back to a short status line so # a healthy re-run always ends cleanly at exit 0. _assess_handoff() { + # This exits 0 having run no install step, so the outcome event must say + # `skipped`, not `succeeded` (backend#1907) — otherwise the success count + # grows with every re-run on a machine nothing happened to, and the failure + # rate quietly falls for a reason that has nothing to do with installs. + if declare -F telemetry_run_skipped >/dev/null 2>&1; then + telemetry_run_skipped + fi success "Already set up on this machine — no need to run the installer again." export PATH="${HOME}/.local/bin:${PATH}" if has tracebloc; then diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 51a2a0fb..cfa94e01 100755 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -238,7 +238,21 @@ hint() { echo -e " ${DIM}$*${RESET}"; } # → " a) Checking your machine". Prints the header + a single trailing blank; the # blank-line gap BETWEEN steps comes from each step body ending with a blank line # (main() adds it), matching the run-through's spacing. -step_header() { echo -e " ${TB_HEADING}$1) $2${RESET}"; echo ""; } +# +# It is also where the install's phase clock turns over (backend#1907). Hooking +# THIS rather than adding a telemetry_phase_begin call to each of the six steps +# is the difference between phase timings that are correct by construction and +# phase timings that are correct for the steps somebody remembered — and the +# letters it is keyed on are the ones actually being printed, so nothing can +# drift. Guarded because telemetry.sh may be absent under an older bootstrap +# whose FILES list did not fetch it, and the `|| true` because printing a step +# header must never be able to end an install. +step_header() { + if declare -F telemetry_phase_begin >/dev/null 2>&1; then + telemetry_phase_begin "$1" || true + fi + echo -e " ${TB_HEADING}$1) $2${RESET}"; echo ""; +} # ── Utility ────────────────────────────────────────────────────────────────── has() { command -v "$1" &>/dev/null; } @@ -1095,6 +1109,16 @@ install_cleanup() { hint "If it keeps failing, re-run with --diagnose and send the bundle to tracebloc support." fi fi + + # One structured outcome event per install (backend#1907). Emitted LAST, from + # the EXIT trap, so it runs on every path — success, the re-run-required stop, + # Ctrl-C, and the fatal one — which is what §6.5 of the telemetry contract + # requires and what makes a failure RATE computable rather than just a count. + # Everything it reads (CLIENT_STATE, TB_ERR_*, the phase clock) is final by + # this point. Guarded for an older bootstrap that did not fetch telemetry.sh. + if declare -F telemetry_emit_outcome >/dev/null 2>&1; then + telemetry_emit_outcome "$exit_code" || true + fi } # Installer version shown in the banner's title (" · "). The curl|bash @@ -1185,6 +1209,16 @@ Advanced configuration (environment variables): Must be on a LOCAL disk — NFS/CIFS/SMB is rejected (the database corrupts on network storage). TRACEBLOC_ALLOW_NETWORK_FS=1 overrides. +Usage reporting: + This installer records ONE outcome event per run so we can see failures without + waiting for someone to report them: which step it reached, how long each step + took, the exit code, an error class, your OS and architecture, and the version. + It cannot record your arguments, any path, any file name, your username, your + hostname or your credentials — every field is a number or a value from a fixed + list, so there is nowhere for those to go. + TRACEBLOC_NO_TELEMETRY=1 Turn it off. + DO_NOT_TRACK=1 Also turns it off. + Windows: irm https://raw.githubusercontent.com/tracebloc/client/main/scripts/install.ps1 | iex diff --git a/scripts/lib/gpu-nvidia.sh b/scripts/lib/gpu-nvidia.sh index 4f2b8d9d..0b733c09 100755 --- a/scripts/lib/gpu-nvidia.sh +++ b/scripts/lib/gpu-nvidia.sh @@ -52,6 +52,15 @@ install_nvidia_drivers() { hint "After rebooting, re-run the installer. Driver steps will be skipped." if [[ -n "${TRACEBLOC_SKIP_REBOOT_PROMPT:-}" ]]; then log "TRACEBLOC_SKIP_REBOOT_PROMPT set — skipping reboot prompt." + # DECLARE THE HANDOFF, then exit. This is the installer's "complete this step + # and re-run" stop — the drivers went in fine and the machine needs a reboot — + # and telemetry keys `install.run.cancelled` on this marker rather than on the + # number 2, because grep/curl/tar all exit 2 too and one of those escaping + # under `set -e` must be counted as the failure it is. Without this line the + # run books itself as `unexpected_exit_2`, which is the safe direction to be + # wrong in but still wrong. Guarded for a stale bootstrap that did not fetch + # telemetry.sh (same shape as install_cleanup's emit call). + if declare -F telemetry_rerun_handoff >/dev/null 2>&1; then telemetry_rerun_handoff; fi exit 2 fi # Read the terminal directly: the main install path is `curl … | bash`, where diff --git a/scripts/lib/summary.sh b/scripts/lib/summary.sh index 21991451..30ff8e02 100755 --- a/scripts/lib/summary.sh +++ b/scripts/lib/summary.sh @@ -19,7 +19,10 @@ _log_cluster_status() { # returns we wait for the client's workloads to actually become Ready and set # CLIENT_STATE so the summary reports the truth instead of an unconditional # "installed successfully": -# connected | starting | bad_creds | image_pull | crash +# connected | starting | bad_creds | image_pull | image_pull_ca | crash +# (image_pull_ca is the TLS-inspecting-network case, #424. It was added to +# _diagnose_not_ready and not to this list; backend#1907's vocabulary-agreement +# guard derives the set from the function and caught the omission.) # Empty until wait_for_client_ready runs — so install_cleanup can distinguish an # early failure (before the readiness gate, CLIENT_STATE still empty) from a # reported outcome, and still print the "check the log / safe to re-run" hint. diff --git a/scripts/lib/telemetry.sh b/scripts/lib/telemetry.sh new file mode 100644 index 00000000..c8243e88 --- /dev/null +++ b/scripts/lib/telemetry.sh @@ -0,0 +1,962 @@ +#!/usr/bin/env bash +# ============================================================================= +# telemetry.sh — one structured outcome event per install (backend#1907) +# +# RFC-BACKEND-1872 D12's host-process path; the record shape is +# rfcs/specs/backend-1872-telemetry-contract.md, where this component is +# registered as service.name=installer / tracebloc.component=install (§10.1). +# +# WHY. The installer is the highest-variance, least-observed step in the +# product: it runs on machines we have never seen, under package managers, +# proxies and shells we do not control, and it reports to nobody. The +# backend#736 failures — the CLI landing in ~/.local/bin with PATH advice only +# printed, apt-get appearing hung because unattended-upgrades held the dpkg +# lock — were each invisible until a customer happened to mention one. +# +# "NO ARGUMENTS, NO PATHS, NO DATA" IS A SHAPE HERE, NOT A RULE. Every value +# that reaches the record goes through _telemetry_attr, which admits a string +# only if it matches ^[A-Za-z0-9._-]{1,64}$ and an integer only if it is one. +# A filesystem path contains '/', a proxy credential contains ':' and '@', a +# token is longer than 64 characters, a person's name contains a space — none +# of them can pass, on ANY input, whether or not anyone anticipated it. Values +# that fail are DROPPED, never trimmed or escaped: a redactor has to imagine +# what it is stripping; a shape only admits what it was told to. +# +# WHAT IS NOT HERE. The transport. The 17 Aug decision (rfcs#28) replaced the +# Collector gateway with an ingest endpoint on the backend — tracebloc/backend#1905, +# which does not exist yet. _telemetry_deliver therefore writes the event to +# the install log and to a bounded local spool, and posts nothing. See the +# comment on that function for the one thing #1905 changes. +# +# Opt-out (on by default): TRACEBLOC_NO_TELEMETRY=1 or DO_NOT_TRACK=1. +# ============================================================================= + +# ── Identity (contract §10.1) ──────────────────────────────────────────────── +# Constants, never derived from $0 or a hostname — deriving service identity +# from the process is the defect the contract exists to close (§2). +TB_TELEMETRY_SERVICE="installer" +TB_TELEMETRY_COMPONENT="install" + +# ── Phase vocabulary ───────────────────────────────────────────────────────── +# The letters are install-k8s.sh's own `step_header a..f` labels; the names are +# what a dashboard shows. step_header calls telemetry_phase_begin, so this map +# is keyed on the thing that actually runs the steps rather than on a parallel +# list somebody has to remember to extend — and +# scripts/tests/telemetry-vocabulary-agreement.sh parses install-k8s.sh to prove +# the two sets are identical. +# +# `bootstrap` is the phase before step a — and only the part of it that runs in +# THIS process: install-k8s.sh's preamble, validate_config, the leftover-data +# guard, the assess gate. Download and verify are NOT in it. They happen in +# install.sh, which never sources this file and whose EXIT trap is +# `rm -rf "$TMPDIR"`, not install_cleanup — so a fetch, manifest or cosign +# failure emits nothing at all, and `bootstrap` means "install-k8s.sh before +# step a", not "everything before step a". (saadqbal on client#747; verified — +# install.sh's only mention of telemetry.sh is the FILES list it downloads.) +# Extending coverage over the fetch is separate work, not this ticket. +# +# It has no letter because nothing in the a–f run-through covers it, and a run +# that dies there must not be filed under `preflight`. +TB_TELEMETRY_PHASES="a:preflight b:prerequisites c:cluster d:register e:helm f:connect" +TB_TELEMETRY_PHASE="bootstrap" + +# ── event.name vocabulary (contract §6.1, §6.4) ────────────────────────────── +# Every name this installer can emit. Three segments; `install` is a registered +# domain (§6.3); the third segment of each is a registered outcome verb (§6.4: +# started succeeded failed skipped rejected retried timed_out expired cancelled +# completed). §6.2 requires the set to be finite and enumerable by grep — this +# is that enumeration, and telemetry_render_event checks its answer against it. +# +# telemetry-vocabulary-agreement.sh derives the emitted names from +# telemetry_render_event's own case statement and from exercising the function +# over the exit codes the installer produces, then compares. It does NOT read +# this list twice: a list checked against itself is self-consistent and blind. +# +# What it cannot check from this repo is the §6.4 half — the verb registry lives +# in rfcs/specs/backend-1872-telemetry-contract.md, which is not checked out +# here, and a hand-copied second list of verbs would be the defect rather than +# the fix. Adding a name here is therefore a review question: is its third +# segment in §6.4? +TB_TELEMETRY_EVENT_NAMES="install.run.succeeded install.run.failed install.run.cancelled install.run.skipped" + +# ── Client-state vocabulary ────────────────────────────────────────────────── +# summary.sh's wait_for_client_ready + _diagnose_not_ready are the only writers +# of CLIENT_STATE. The agreement test derives THAT set from summary.sh and +# compares; anything not in this list is reported as `unknown` rather than +# passed through, because CLIENT_STATE is a shell variable and a shell variable +# is not a closed set until something closes it. +TB_TELEMETRY_CLIENT_STATES="connected starting bad_creds image_pull image_pull_ca crash" + +# ── error.type vocabulary for the `install` domain (§8.4) ──────────────────── +# The spec's open question 1 says each emitter ticket proposes its own — so a +# value added here is a decision this file gets to take, unlike an outcome verb +# (§6.4), which is a closed registry and a PR against the contract. +# +# This one is a function of (re-run handoff declared?, client state, phase +# reached) — all closed — so it is incapable of carrying anything else. Ordered +# most-specific first: an undeclared exit 2 says the STATUS itself was not ours +# to read, a readiness diagnosis names the actual fault, and a phase only names +# where the run stopped. +# +# `unexpected_exit_2` is its own row rather than `unclassified` or the phase's +# own bucket, because both of those hide it: `unclassified` already means "we +# cannot name the phase", and `prerequisites_failed` would make an ordinary +# tool's stray 2 indistinguishable from a real prerequisite failure. The phase +# is not lost by flattening it — tracebloc.install.phase is its own attribute on +# the same record — and flattening is what makes the class countable as one +# thing, which is the whole point of surfacing it. +TB_TELEMETRY_ERROR_CLASSES="unexpected_exit_2 bad_credentials image_pull_failed image_pull_untrusted_ca crash_loop not_ready bootstrap_failed preflight_failed prerequisites_failed cluster_create_failed registration_failed helm_install_failed unclassified" + +# ── Source-file vocabulary ─────────────────────────────────────────────────── +# The installer already records WHERE it died (common.sh's _record_err), and +# "died in setup-linux.sh at line 412" is the difference between an actionable +# failure and an unclassified one. The full TB_ERR_LOC is a PATH, though — +# under curl|bash it is a temp directory, which on macOS sits under +# /var/folders/ — so only the basename is emitted, and only if it is one +# of the installer's own scripts. That set is gen-manifest.sh's FILES array plus +# the bootstrap; the agreement test derives it from there. +# +# `install.sh` is in the set for derivation symmetry and is UNREACHABLE today: +# TB_ERR_LOC has exactly one writer, common.sh's _record_err (common.sh:988), and +# common.sh is only ever sourced inside install-k8s.sh's process. Nothing in the +# bootstrap can name itself here. Kept rather than special-cased out, because the +# day the bootstrap does get an emitter the name must already be admissible — +# but do not read its presence as coverage. (saadqbal on client#747.) +TB_TELEMETRY_SOURCES="install.sh install-k8s.sh common.sh preflight.sh detect-gpu.sh gpu-nvidia.sh gpu-amd.sh setup-macos.sh setup-linux.sh cluster.sh gpu-plugins.sh install-client-helm.sh install-cli.sh provision.sh assess.sh probe.sh summary.sh diagnose.sh telemetry.sh" + +# ── The value shapes ───────────────────────────────────────────────────────── +# This is the privacy boundary. Nothing else in this file is allowed to write to +# the record. +# +# MATCHED WITH `[[ =~ ]]`, NEVER `grep`. The anchors say whole string; grep says +# whole LINE, and the difference is a hole exactly one input shape wide. A value +# carrying an embedded newline gave grep a first line that matched and the record +# everything after it: +# +# TB_VERSION=$'v1.9.3\n","tracebloc.install.injected":"yes' +# → "service.version":"v1.9.3 +# ","tracebloc.install.injected":"yes",… +# +# — a forged attribute AND one record split across two lines of a `.jsonl` spool, +# so #1906's forwarder reads two malformed events. It is the one shape the +# "nowhere for a path to go" argument does not cover, because the value that +# lands is not a path. `[[ $v =~ $RE ]]` anchors at end of STRING (POSIX +# regexec, no REG_NEWLINE), so the same regex now refuses it. Reproduced on all +# four shape checks before fixing; found by saadqbal on client#747. +# +# The RHS must stay UNQUOTED — a quoted RHS is a literal string on bash 3.2+, +# which is the system bash on macOS. +TB_TELEMETRY_TOKEN_RE='^[A-Za-z0-9._-]{1,64}$' +TB_TELEMETRY_INT_RE='^-?[0-9]{1,15}$' +TB_TELEMETRY_KEY_RE='^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$' + +# service.version gets a TIGHTER shape than the generic token, and it is the +# bootstrap's own: install.sh refuses to fetch from anything that is not an +# immutable vX.Y.Z release tag, so a TB_VERSION that does not match that never +# came from a release. The generic token shape was not enough on its own — +# `v1.9.3-<64 arbitrary chars>` satisfies it, which makes the version column a +# 64-byte free-text channel. telemetry-vocabulary-agreement.sh checks this +# against install.sh's — byte-for-byte AND, since client#747, verdict-for-verdict +# over a corpus, each side evaluated by the operator its own file uses. The byte +# check alone was not enough and said it was: the two regexes were identical +# while install.sh's `[[ =~ ]]` refused an input this file's `grep -qE` admitted. +TB_TELEMETRY_VERSION_RE='^v[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.]+)?$' + +# How many events the local spool keeps. Bounded because it is a file on a +# customer's machine that nothing drains until #1905's forwarder exists. +TB_TELEMETRY_SPOOL_MAX="${TB_TELEMETRY_SPOOL_MAX:-50}" + +# ── Clock ──────────────────────────────────────────────────────────────────── +# Second resolution, multiplied up. BSD date (macOS, which is half the install +# base) has no %N, and adding a dependency to read a millisecond would be a poor +# trade for a signal whose interesting values are minutes: the dpkg-lock case +# this exists to surface is a phase taking twenty of them. +_telemetry_now_ms() { echo $(( $(date +%s) * 1000 )); } + +TB_TELEMETRY_STARTED_MS="$(_telemetry_now_ms)" +_TB_TELEMETRY_PHASE_STARTED_MS="$TB_TELEMETRY_STARTED_MS" +_TB_TELEMETRY_EMITTED="" + +# ── "an install actually ran" latch ────────────────────────────────────────── +# install_cleanup is the EXIT trap, so it fires for EVERY exit of install-k8s.sh +# — including the terminal, non-install commands. `--help` exits 0 without +# touching the machine, and it was emitting a full install.run.succeeded: a free +# success in the denominator of the exact failure RATE this whole ticket exists +# to produce, and the one people run most while a real install is broken. +# (Bugbot on client#747; reproduced — `install-k8s.sh --help` spooled an +# install.run.succeeded with phase `bootstrap`.) +# +# A latch rather than a phase test, because a genuine failure in the bootstrap +# phase — the leftover-data guard, validate_config — IS an install attempt and +# must still be reported. main() sets this once the terminal commands have had +# their chance to dispatch and the run is committed to installing. +_TB_TELEMETRY_RUN_STARTED="" +telemetry_run_started() { _TB_TELEMETRY_RUN_STARTED=1; return 0; } + +# ── "ran, but did nothing" ─────────────────────────────────────────────────── +# The stop-and-check gate hands a verifiably healthy machine to the home screen +# and exits 0 without running a single step. That is a real invocation and worth +# counting, but it is not a successful INSTALL: folding it into succeeded would +# make the success count grow with re-runs on machines nothing happened to. +# `skipped` is a registered outcome verb (contract §6.4), so it needs no new +# vocabulary — and "how often do people re-run an installer that was already +# done" is a question worth being able to ask. +# +# READ THE COUNT NARROWLY. It is NOT "re-runs on a healthy machine": on the +# `curl | bash` path install.sh:132 reaches a healthy machine first and +# `exec tracebloc`s at :144 — before install-k8s.sh has even been fetched — so +# assess.sh's gate never runs and this event is never emitted. What it counts is +# re-runs of `./install-k8s.sh` directly, plus curl|bash runs that reached the +# gate because the bootstrap's own health check did not bail (--force, +# TRACEBLOC_FORCE_REINSTALL, a pinned REF/BRANCH, or no `tracebloc` on PATH). +# (saadqbal on client#747; verified against install.sh's line order.) +_TB_TELEMETRY_SKIPPED="" +telemetry_run_skipped() { _TB_TELEMETRY_SKIPPED=1; return 0; } + +# ── "complete this step and re-run" marker ─────────────────────────────────── +# THE SENTINEL IS A MARKER, NOT THE NUMBER 2. +# +# `2` is a status the installer chose to mean "complete this step and re-run", +# and it is also a status ORDINARY TOOLS PRODUCE: grep exits 2 on a file error, +# curl on a failed init, tar on a fatal. Any bare one of those failing under +# `set -e` exits install-k8s.sh with 2 — and keying on the number rendered that +# as `cancelled` with NO error.type, i.e. a hard failure REMOVED FROM THE +# NUMERATOR of the rate this ticket exists to produce rather than misfiled +# inside it. Silence is worse than a wrong label, and this is the direction +# nobody notices. (saadqbal on client#747; reproduced end-to-end — a bare +# `grep -q x /nonexistent` in step b spooled install.run.cancelled, no +# error.type, under the installer's own trap wiring.) +# +# So the handoff is declared at the `exit 2` SITE and the emitter keys on that. +# An exit 2 nobody declared is a failure with its own error.type, which is the +# fail-closed direction: it lands in the numerator, and it is visible as +# "exited 2 for a reason we did not choose" instead of being folded into the +# phase bucket an ordinary failure would use. +# +# CLEARED AT SOURCE TIME, and that line is the point of it: this is a +# process-internal handshake, not an input. Were it read as `${VAR:-}` off the +# environment, `VAR=1` in a user's shell would turn every real failure into a +# cancel — the same fail-open hole one level up. telemetry.sh is sourced in +# install-k8s.sh's preamble, before main() runs, so an inherited value cannot +# survive to be read. A function rather than a documented variable name for the +# same reason: the producers call code, so there is one spelling of it. +# +# There is precedent for the drift this shape prevents: install_cleanup still +# reads TRACEBLOC_DOCKER_FIRST_RUN_EXIT, whose only producer (an +# `export` in setup-macos.sh) was deleted in 8c3a3d4 back in March — a marker +# read by a live branch that nothing has set since. telemetry-vocabulary- +# agreement.sh therefore fails closed when this marker has ZERO producers, +# rather than passing forever on a handoff that can no longer happen. +_TB_TELEMETRY_RERUN_HANDOFF="" +telemetry_rerun_handoff() { _TB_TELEMETRY_RERUN_HANDOFF=1; return 0; } + +# ── Opt-out ────────────────────────────────────────────────────────────────── +# Opt-OUT by design: telemetry only the already-convinced enable measures the +# wrong population, and the population this exists for is people whose install +# just failed. Anything other than the explicit "off" spellings counts as opting +# out — a user who typed TRACEBLOC_NO_TELEMETRY=please meant it, and guessing +# wrong in the other direction sends a record they declined. +TB_TELEMETRY_OPT_OUT_VARS="TRACEBLOC_NO_TELEMETRY DO_NOT_TRACK" +telemetry_enabled() { + local name value + for name in $TB_TELEMETRY_OPT_OUT_VARS; do + eval "value=\${$name:-}" + case "$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" in + ''|0|false) continue ;; + *) return 1 ;; + esac + done + return 0 +} + +# ── Phases ─────────────────────────────────────────────────────────────────── + +# telemetry_phase_name LETTER — the registered name, or empty for an unknown +# letter. Fail closed: a step_header letter this map has never seen must not +# invent a phase name. +telemetry_phase_name() { + local pair + for pair in $TB_TELEMETRY_PHASES; do + [ "${pair%%:*}" = "$1" ] && { printf '%s' "${pair#*:}"; return 0; } + done + return 1 +} + +# _telemetry_phase_names — every legal value of TB_TELEMETRY_PHASE: the six +# registered names plus the two that have no step letter. Derived from +# TB_TELEMETRY_PHASES so a phase added there needs no second edit. +_telemetry_phase_names() { + local pair out="bootstrap unknown" + for pair in $TB_TELEMETRY_PHASES; do out="$out ${pair#*:}"; done + printf '%s' "$out" +} + +# telemetry_phase_begin LETTER — close the running phase's timer, open the next. +# Called from step_header (common.sh), so the timings come from the code that +# actually prints the steps. Always returns 0: a timing bookkeeping error must +# never be able to end an install. +telemetry_phase_begin() { + local name now elapsed + now="$(_telemetry_now_ms)" + elapsed=$(( now - _TB_TELEMETRY_PHASE_STARTED_MS )) + # An `if` block, not `[ … ] && elapsed=0`. common.sh's _record_err records why: + # a bare AND-list whose test is FALSE returns 1, and as the last statement of a + # function that becomes the function's status — which under the installer's + # `set -e` is fatal at the call site. Clock-step clamps are exactly the lines + # where the test is normally false. + if [ "$elapsed" -lt 0 ]; then elapsed=0; fi + # Accumulate against the phase that is ENDING. Dynamic variable names are safe + # here precisely because the name came out of the closed map above. + printf -v "_TB_TELEMETRY_MS_${TB_TELEMETRY_PHASE}" '%s' "$elapsed" 2>/dev/null || true + + if name="$(telemetry_phase_name "$1")"; then + TB_TELEMETRY_PHASE="$name" + else + TB_TELEMETRY_PHASE="unknown" + fi + _TB_TELEMETRY_PHASE_STARTED_MS="$now" + return 0 +} + +# _telemetry_phase_ms NAME ACTIVE NOW — milliseconds against a phase. +# +# telemetry_phase_begin only closes a phase when the NEXT one starts, so the +# phase that is still running when the event is emitted has nothing recorded +# against it. That lost the most important number in the file: +# +# * on every SUCCESSFUL install, phase_connect_ms — the readiness wait, up to +# READY_TIMEOUT (600s), the single longest phase; +# * on every failure and every cancel, the duration of the phase named by +# tracebloc.install.phase — i.e. exactly the dpkg-lock shape this feature +# exists to surface, in its most likely real form: stuck twenty minutes in +# `prerequisites` and then killed or given up on. +# +# The remainder trick cannot recover it either, because bootstrap had no key at +# all. Found by Bugbot on client#747; reproduced before fixing. +# +# The live delta is ADDED here rather than written by a "close the phase" call in +# the emit path, so render stays idempotent — the tests call it repeatedly, and a +# render that mutated the accumulators would report different numbers each time. +_telemetry_phase_ms() { + local name="$1" active="${2:-}" now="${3:-0}" v live + eval "v=\${_TB_TELEMETRY_MS_${name}:-}" + if [ "$name" = "$active" ]; then + live=$(( now - _TB_TELEMETRY_PHASE_STARTED_MS )) + if [ "$live" -lt 0 ]; then live=0; fi + v=$(( ${v:-0} + live )) + fi + printf '%s' "$v" +} + +# ── Classification ─────────────────────────────────────────────────────────── + +# telemetry_error_class EXIT_CODE PHASE CLIENT_STATE HANDOFF — the closed error.type. +# +# Every input is a closed set (HANDOFF is a boolean), so this cannot see — and +# therefore cannot forward — an error message, a path or an argument. A readiness +# diagnosis wins over the phase because it names the actual fault rather than the +# place the run stopped. +# +# The exit code was accepted but deliberately unread, on the grounds that the +# installer's status is already its own attribute and folding it in here would +# give two attributes one meaning — with the note that a classification which DOES +# need it would be a body change, not a call-site change. This is that change: 2 +# WITHOUT the re-run marker is the one case where the status carries something the +# phase does not, namely that the status was not the installer's to mean anything +# by. It is read only in combination with the marker, never alone, so the "two +# attributes one meaning" objection still holds for every other code. +telemetry_error_class() { + local code="$1" phase="$2" state="$3" handoff="${4:-}" + # An exit 2 that no `exit 2` site declared: grep's file error, curl's failed + # init, tar's fatal, escaping under `set -e`. Most specific of all, because it + # says the exit STATUS is not a value we chose — so no other reading of it (the + # phase's bucket, a readiness diagnosis) is more trustworthy than saying so. + if [ "$code" = "2" ] && [ -z "$handoff" ]; then + printf 'unexpected_exit_2'; return 0 + fi + case "$state" in + bad_creds) printf 'bad_credentials'; return 0 ;; + image_pull) printf 'image_pull_failed'; return 0 ;; + image_pull_ca) printf 'image_pull_untrusted_ca'; return 0 ;; + crash) printf 'crash_loop'; return 0 ;; + starting) printf 'not_ready'; return 0 ;; + esac + case "$phase" in + bootstrap) printf 'bootstrap_failed' ;; + preflight) printf 'preflight_failed' ;; + prerequisites) printf 'prerequisites_failed' ;; + cluster) printf 'cluster_create_failed' ;; + register) printf 'registration_failed' ;; + helm) printf 'helm_install_failed' ;; + connect) printf 'not_ready' ;; + # Fail closed. A phase this function has not been taught is a countable + # "we cannot name it", never the phase string rendered into a new value + # that appears in the data on its own. + *) printf 'unclassified' ;; + esac + return 0 +} + +# telemetry_environment — deployment.environment, or empty for "do not export". +# +# tb_client_env is the installer's own alias reduction (backend#1745); calling it +# rather than re-deciding what `staging` means is what keeps this from becoming +# a fifth declaration of that vocabulary. An unrecognised value is NOT repaired +# and NOT guessed: §3.2 says a record filed under a value no query filters on is +# worse than no record, so the run reports nothing at all. +telemetry_environment() { + local env + if declare -F tb_client_env >/dev/null 2>&1; then + env="$(tb_client_env "${CLIENT_ENV:-prod}")" + else + env="${CLIENT_ENV:-prod}" + fi + case "$env" in + dev|stg|prod) printf '%s' "$env" ;; + *) return 1 ;; + esac +} + +# ── The record ─────────────────────────────────────────────────────────────── + +_TB_TELEMETRY_BUF="" + +_telemetry_reset() { _TB_TELEMETRY_BUF=""; } + +# _telemetry_attr KEY VALUE KIND — the single writer, and the whole privacy +# boundary. KIND is `str` or `int`. +# +# Every shape test is a `[[ =~ ]]`, not a grep. Two reasons, and the second one +# is a bug this file already had: +# +# * `[[ =~ ]]` matches the whole STRING; grep matches a LINE. See the shape +# declarations above for the newline bypass that cost. +# * grep is an external process invoked from an EXIT trap. The spelling that +# was here read its subject through a HERE-STRING rather than a pipe, +# because `grep -q` closes the pipe at its first hit and a +# `printf ... | grep -q` under the installer's `set -o pipefail` returns 141 +# (SIGPIPE) on a MATCH — a successful validation turned into a fatal that +# killed the whole run from inside the trap. Same defect, same fix as +# summary.sh's _diagnose_not_ready (backend#1778). `[[ =~ ]]` is a shell +# builtin with no pipe and no child, so that class cannot recur here at all. +# The bats test that drives install_cleanup for real is what caught it; every +# unit-level test of these functions passed throughout. +# +# Refuses and DROPS, in this order: a malformed key, an empty value, a value +# that is not the shape KIND promises. It never trims, escapes or truncates — +# a value that had to be repaired to be safe is a value we did not understand, +# and shipping our guess about it is how a redactor leaks. +_telemetry_attr() { + local key="$1" value="$2" kind="${3:-str}" + [[ $key =~ $TB_TELEMETRY_KEY_RE ]] || return 0 + [ -n "$value" ] || return 0 + case "$kind" in + int) + [[ $value =~ $TB_TELEMETRY_INT_RE ]] || return 0 + _TB_TELEMETRY_BUF="${_TB_TELEMETRY_BUF:+${_TB_TELEMETRY_BUF},}\"${key}\":${value}" + ;; + *) + [[ $value =~ $TB_TELEMETRY_TOKEN_RE ]] || return 0 + _TB_TELEMETRY_BUF="${_TB_TELEMETRY_BUF:+${_TB_TELEMETRY_BUF},}\"${key}\":\"${value}\"" + ;; + esac + return 0 +} + +# _telemetry_in_set VALUE SET — membership, the only way a shell variable +# becomes a closed vocabulary. +_telemetry_in_set() { + local needle="$1" set="$2" item + # Word-splitting $set is the point: these vocabularies are space-separated + # lists, and bash 3.2 (the system bash on macOS) has no associative arrays. + # shellcheck disable=SC2086 + for item in $set; do + [ "$item" = "$needle" ] && return 0 + done + return 1 +} + +# _telemetry_source_basename LOC — the script name out of a "file:line" +# location, if it is one of the installer's own. Everything else, including the +# directory the path came from, is discarded. +_telemetry_source_basename() { + local base="${1%%:*}" + base="${base##*/}" + _telemetry_in_set "$base" "$TB_TELEMETRY_SOURCES" || return 1 + printf '%s' "$base" +} + +# _telemetry_source_line LOC — the line number, if the location has one. +_telemetry_source_line() { + local line="${1##*:}" re='^[0-9]{1,7}$' + [[ $line =~ $re ]] || return 1 + printf '%s' "$line" +} + +# telemetry_render_event EXIT_CODE — echo one contract-shaped JSON object. +# +# Pure: reads the installer's state, writes nothing, touches no file. That is +# what lets the tests assert on the real payload instead of on a re-implementation +# of it. Returns 1 without printing when the environment is unrecognised (§3.2). +telemetry_render_event() { + local code="${1:-0}" env event state phase phase_ms name now total class src + env="$(telemetry_environment)" || return 1 + + # §6.1 — three segments, a registered domain, a registered outcome verb. The + # name is assembled from literals only; no runtime value appears in it. + # + # A DECLARED EXIT 2 IS NOT A FAILURE. It is the installer's "complete this step + # and re-run" handoff — install_cleanup has treated it as its own outcome + # ("Re-run required") since client#681, and its one live producer is + # gpu-nvidia.sh:55, which exits 2 after install_nvidia_drivers SUCCEEDED, to ask + # for a reboot. That call sits under step b, so folding it into failed booked + # every unattended GPU host's first install as `prerequisites_failed` — a + # fabricated prerequisite failure in the exact rate this ticket exists to + # produce. Same shape as the `--help` bug, opposite direction. (saadqbal on + # client#747; reproduced.) + # + # AN UNDECLARED ONE IS. The first version of this keyed on the number, and 2 is + # not only ours: grep exits 2 on a file error, curl on a failed init, tar on a + # fatal, and cluster.sh:1129 re-raises whatever k3d returned. Every one of those + # rendered `cancelled` with no error.type — a hard failure removed from the + # NUMERATOR rather than misfiled in it, which is the direction nobody notices. + # So the branch keys on _TB_TELEMETRY_RERUN_HANDOFF, set at the `exit 2` site + # itself, and an exit 2 nobody claimed falls through to failed with its own + # error.type (`unexpected_exit_2`). Fail closed toward counting it. (saadqbal on + # client#747; reproduced end-to-end before the change — see the marker's own + # comment above.) + # + # 130/143 are the signal exits: install-k8s.sh's `trap 'exit 130' INT` IS their + # declaration site, and there is no ordinary command whose 130 could reach here + # under `set -e` — bash reserves 128+n for signals, and a child killed by SIGINT + # takes this shell's own trap first. + # + # They ride `cancelled` rather than a verb of their own because §6.4's outcome + # verbs are a CLOSED list — started, succeeded, failed, skipped, rejected, + # retried, timed_out, expired, cancelled, completed — and adding one is a PR + # against the contract, not a decision an emitter takes unilaterally. Of those, + # `cancelled` is the only terminal verb that is true for a real interrupted + # install: the run stopped before completing, deliberately, without an error. + # The two causes stay separable because tracebloc.install.exit_code is already + # an attribute — exit_code=2 is the re-run handoff, 130/143 the user's own + # Ctrl-C — which is why the exit code is an attribute rather than something the + # name carries. + # + # BUT A SIGNAL ON A SKIPPED RUN IS STILL SKIPPED. An earlier version of this + # said 130/143 were "unconditional", and that was wrong in the direction that + # costs the most. _assess_handoff marks the run skipped and then hands the user + # the interactive `tracebloc` home screen before `exit 0` — so Ctrl-C on that + # screen, which is the most ordinary thing a user does there, took the INT trap + # and booked `cancelled` for a run that installed nothing. `cancelled` asserts + # an install was cancelled; on this path there was no install to cancel. It + # would land in the denominator of "how often do installs not complete", + # inflating it with runs that never attempted anything — the same distortion as + # the `--help` bug above and in the same direction: it makes the product look + # worse while telling nobody anything actionable. (Bugbot on client#747; + # reproduced — same skipped state, `exit 0` rendered skipped and Ctrl-C + # rendered cancelled.) + # + # SKIPPED DOES NOT WIN OVER A FAILURE, and that asymmetry is the point rather + # than an oversight. The flag is consulted only on the exits that mean nothing + # was installed — 0 and the two signals. A skipped run that then dies with a + # real non-zero (anything between telemetry_run_skipped and `exit 0` failing + # under errexit) is a genuine failure and must stay countable as one, so the + # `*)` branch deliberately does not look at the flag. Hoisting a blanket + # "skipped wins" ahead of this case would be shorter and would hide a failure. + case "$code" in + 0) if [ -n "$_TB_TELEMETRY_SKIPPED" ]; then + event="install.run.skipped" + else + event="install.run.succeeded" + fi ;; + 2) if [ -n "$_TB_TELEMETRY_RERUN_HANDOFF" ]; then + event="install.run.cancelled" + else + event="install.run.failed" + fi ;; + 130|143) if [ -n "$_TB_TELEMETRY_SKIPPED" ]; then + event="install.run.skipped" + else + event="install.run.cancelled" + fi ;; + *) event="install.run.failed" ;; + esac + # The name is checked against the declared set before it is written, and an + # unregistered one DROPS the record rather than filing it — same rule as §3.2's + # unrecognised environment, for the same reason: a record under a name no alert + # is written against is worse than no record, and it is how a closed namespace + # fills with rows nobody queries. This cannot fire on today's literals-only + # case; it is here for the edit that adds a branch, and + # telemetry-vocabulary-agreement.sh proves declaration and case agree by + # parsing this function rather than by reading the declaration twice. + _telemetry_in_set "$event" "$TB_TELEMETRY_EVENT_NAMES" || return 1 + + state="${CLIENT_STATE:-}" + _telemetry_in_set "$state" "$TB_TELEMETRY_CLIENT_STATES" || state="" + + # The phase is checked here as well as at the point it is set. telemetry_phase_begin + # is its only writer today and already closes the set — but TB_TELEMETRY_PHASE is + # a shell variable in a script that sources sixteen other files, and "the only + # writer is careful" is a property that stops being true silently. Checked at the + # boundary it cannot: a canary assigned straight to TB_TELEMETRY_PHASE reached the + # record while this line was missing, and it is shaped exactly like a legal value, + # so the token regex waved it through. + phase="$TB_TELEMETRY_PHASE" + _telemetry_in_set "$phase" "$(_telemetry_phase_names)" || phase="unknown" + + _telemetry_reset + _telemetry_attr "event.name" "$event" + _telemetry_attr "tracebloc.install.phase" "$phase" + _telemetry_attr "tracebloc.install.exit_code" "$code" int + # ONE clock read for the whole event: the per-phase numbers and the total are + # then exactly consistent, which is an invariant a test can assert (and does). + now="$(_telemetry_now_ms)" + total=$(( now - TB_TELEMETRY_STARTED_MS )) + if [ "$total" -lt 0 ]; then total=0; fi + _telemetry_attr "tracebloc.install.duration_ms" "$total" int + _telemetry_attr "tracebloc.install.client_state" "$state" + + # The #736 PATH case, as a number. install-cli.sh already computes this — it + # asks whether a FRESH login shell resolves `tracebloc` — and until now only + # ever printed advice about it. 1/0 rather than true/false so it sums. + case "${TB_CLI_ON_FRESH_PATH:-}" in + 0|1) _telemetry_attr "tracebloc.install.cli_on_path" "${TB_CLI_ON_FRESH_PATH}" int ;; + esac + + # Per-phase durations: a fixed, finite set of keys, one per phase name. + # This is what makes a step that took twenty minutes visible on a run that + # otherwise succeeded — the shape of the dpkg-lock failure, which never + # produces a non-zero exit at all. + # + # Iterating _telemetry_phase_names, NOT the letter map: `bootstrap` has no step + # letter, so the letter map skipped it and the download + verify + leftover + # guard + assess time was unattributable — it silently became the remainder + # nobody could name. A phase never entered still has no key (§1.2 omits an + # absent value); a phase that ran always has one, including 0. + for name in $(_telemetry_phase_names); do + phase_ms="$(_telemetry_phase_ms "$name" "$phase" "$now")" + _telemetry_attr "tracebloc.install.phase_${name}_ms" "$phase_ms" int + done + + if [ "$event" = "install.run.failed" ]; then + # §8.4 — a failure MUST carry error.type, or it cannot be grouped. The + # classifier's answer is checked against the declared vocabulary before it + # is written, so a future branch that returns something unregistered lands + # as a countable `unclassified` rather than opening a namespace of its own. + # (The declaration is not proved correct by this check — the agreement test + # calls telemetry_error_class over every phase x state pair and compares.) + class="$(telemetry_error_class "$code" "$phase" "$state" "$_TB_TELEMETRY_RERUN_HANDOFF")" + _telemetry_in_set "$class" "$TB_TELEMETRY_ERROR_CLASSES" || class="unclassified" + _telemetry_attr "error.type" "$class" + # Where the shell died, to the file and line — never the path that reached + # them. There is deliberately no exception.* set: bash has no stack trace, + # and TB_ERR_CMD is unexpanded command text, which is free text. + # + # ONE GATE FOR BOTH HALVES, and it is the source vocabulary's own answer. + # These were gated independently, so a location whose basename is not one of + # ours — but whose line is a number, which it always is (see below) — emitted + # `source_line` alone. A line number with no file is not a partial answer, it + # is a confident wrong one: it reads as information, sorts and groups like + # information, and points at line 9 of nothing. `?:118`, which the ERR trap + # produces whenever BASH_SOURCE is empty, rendered exactly that. + # (Bugbot on client#747; reproduced — `"tracebloc.install.source_line":9` + # with no source key beside it.) + # + # DERIVED, not restated: the condition is _telemetry_source_basename's own + # exit status, so TB_TELEMETRY_SOURCES stays the single declaration of what + # counts as one of our files. A second copy of that list here — or a + # hand-written "is it ours" test — would agree with itself and drift from the + # set the renderer actually uses. + # + # Deliberately NOT gated the other way (source kept, line missing): that + # cannot happen, and a branch for it would be belt and braces no test could + # redden. TB_ERR_LOC has exactly one writer in the whole tree, the ERR trap at + # install-k8s.sh:118, which always appends `:${LINENO}` — so the line half is + # always a number. And a file with no line is honest information anyway; a + # line with no file is the only one of the two that lies. + if [ -n "${TB_ERR_LOC:-}" ] && src="$(_telemetry_source_basename "$TB_ERR_LOC")"; then + _telemetry_attr "tracebloc.install.source" "$src" + _telemetry_attr "tracebloc.install.source_line" "$(_telemetry_source_line "$TB_ERR_LOC")" int + fi + fi + + printf '{"resource":{' + printf '"service.name":"%s",' "$TB_TELEMETRY_SERVICE" + printf '"tracebloc.component":"%s",' "$TB_TELEMETRY_COMPONENT" + printf '"service.version":"%s",' "$(_telemetry_version)" + printf '"deployment.environment":"%s",' "$env" + printf '"os.type":"%s",' "$(_telemetry_os)" + printf '"host.arch":"%s",' "$(_telemetry_arch)" + printf '"service.instance.id":"%s"' "$(_telemetry_instance_id)" + printf '},"attributes":{%s}}' "$_TB_TELEMETRY_BUF" +} + +# _telemetry_version — the release tag the bootstrap pinned. §4: unknown is a +# VALUE, not an omission, because 0.0.0-unknown is queryable and alertable while +# an absent key is neither. The tag is shape-checked like everything else: a +# TB_VERSION somebody set to a sentence must not become the version column. +_telemetry_version() { + local v="${TB_VERSION:-}" + [[ $v =~ $TB_TELEMETRY_VERSION_RE ]] || v="" + printf '%s' "${v:-0.0.0-unknown}" +} + +# _telemetry_os / _telemetry_arch — OpenTelemetry's own names for these (§1.1 +# forbids re-inventing them as tracebloc.os). Reduced to the closed sets OTel +# and Go use, so an exotic `uname` cannot open a new namespace. +_telemetry_os() { + case "${OS:-$(uname -s 2>/dev/null)}" in + Darwin) printf 'darwin' ;; + Linux) printf 'linux' ;; + *) printf 'unknown' ;; + esac +} +_telemetry_arch() { + case "${ARCH:-$(uname -m 2>/dev/null)}" in + arm64|aarch64) printf 'arm64' ;; + x86_64|amd64) printf 'amd64' ;; + *) printf 'unknown' ;; + esac +} + +# _telemetry_instance_id — §2 asks for a stable per-PROCESS uuid off-cluster. +# +# Not the hostname: field hostnames here are overwhelmingly "-macbook", +# and §7.3 forbids a person's name outright. Not persisted either — a durable +# machine id would be an identifier we then have to answer erasure requests +# about. Fresh per run, which is all this field is for off-cluster. +# Computed at SOURCE time, not memoised on first call: every reader here runs +# inside a command substitution, and a variable set in a subshell does not +# survive it — so a lazily-memoised id would silently be a fresh value on every +# read, which is not what "stable per process" means. +# +# Built from bash's own $RANDOM rather than /dev/urandom, and with no pipeline +# at all. `tr -dc … < /dev/urandom | head -c 16` is the obvious spelling and it +# is fatal here: head exits after 16 bytes, tr takes SIGPIPE, and under the +# installer's `set -o pipefail` the command substitution returns 141 — at SOURCE +# time, under `set -e`, which killed the entire installer before it printed a +# line. Four $RANDOM draws need no external process and cannot fail. This is an +# id for telling two concurrent runs apart, not a secret, so 60 bits is ample. +TB_TELEMETRY_INSTANCE_ID="$(printf '%04x%04x%04x%04x' \ + "$RANDOM" "$RANDOM" "$RANDOM" "$RANDOM")" +# Never a constant stand-in: that would silently fuse every affected run into +# one. +[ -n "$TB_TELEMETRY_INSTANCE_ID" ] || TB_TELEMETRY_INSTANCE_ID="unknown" + +_telemetry_instance_id() { printf '%s' "$TB_TELEMETRY_INSTANCE_ID"; } + +# ── Delivery ───────────────────────────────────────────────────────────────── + +_telemetry_spool_path() { + printf '%s/telemetry/pending.jsonl' "${HOST_DATA_DIR:-${HOME}/.tracebloc}" +} + +# _telemetry_deliver JSON — THE TRANSPORT SEAM (tracebloc/backend#1905). +# +# Today: the install log, plus a bounded local spool the forwarder (#1906) can +# drain once the ingest endpoint exists. Nothing is posted anywhere, because the +# endpoint the 17 Aug decision (rfcs#28) put this on does not exist yet — and +# building a client against an endpoint whose contract is still being written is +# how you ship two of them. +# +# Everything before this function is finished. This function is the change. +# _telemetry_fallback_dir — a directory whose contents survive this install. +# +# `${TMPDIR:-/tmp}` on its own is NOT that directory on the primary macOS path, +# and getting this wrong silently undid the whole pre-log fix. install.sh:238 +# does `TMPDIR="$(mktemp -d)"` and :239 traps `rm -rf "$TMPDIR"` on EXIT. A plain +# assignment to a name that is ALREADY EXPORTED keeps the export attribute — and +# TMPDIR is always exported on macOS (launchd sets a per-user one) and on plenty +# of Linux sessions — so `bash "$TMPDIR/install-k8s.sh"` at :571 inherits the +# bootstrap's scratch dir, this file writes the record into it, and the bootstrap +# deletes it the moment install-k8s.sh returns. The NFS refusal and every other +# pre-setup_log_file failure then produced no record anywhere, which is the exact +# hole the fallback was added to close. (saadqbal on client#747; reproduced +# end-to-end — the spooled file was gone after the bootstrap exited.) +# +# DERIVED, not agreed. The test is "is the running installer inside this +# directory?", which is true precisely when TMPDIR is the bootstrap's own scratch +# dir, because install.sh unpacks install-k8s.sh + lib/ into it and runs it from +# there. Asking install.sh to export its original TMPDIR under another name would +# work too and would be wrong here: install.sh is served from a URL the user may +# have curl'd months ago, so a fix that only works when the bootstrap is new is a +# fix that does not work on the machines this feature exists for. +# +# When TMPDIR is disqualified the record goes to $HOME — outside anything the +# bootstrap's trap owns, and NOT into $HOME/.tracebloc, which is HOST_DATA_DIR +# and which telemetry must never create (client#432, and the comment on +# _telemetry_deliver). /tmp is the last resort for a run with no usable HOME. +_TB_TELEMETRY_SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-.}")" 2>/dev/null && pwd -P)" || _TB_TELEMETRY_SRC_DIR="" +_telemetry_fallback_dir() { + local tmp="${TMPDIR:-/tmp}" real + tmp="${tmp%/}" + [ -n "$tmp" ] || tmp="/tmp" + # BOTH SIDES PHYSICAL, or the comparison misses the only platform that has the + # bug. macOS's /var is a symlink to /private/var, so `pwd -P` above resolves the + # installer's own directory to /private/var/folders/… while $TMPDIR keeps the + # /var/folders/… spelling it was exported with — compared as written, the + # prefix test never fires on a Mac. (Caught by re-running the reproduction + # against the fix rather than by reading it.) + real="$(cd "$tmp" 2>/dev/null && pwd -P)" || real="" + # A `case`, not a `[[ == ]]`: the pattern side must be the glob and the subject + # side must not be re-globbed. + if [ -n "$_TB_TELEMETRY_SRC_DIR" ] && [ -n "$real" ]; then + case "${_TB_TELEMETRY_SRC_DIR}/" in + "${real}"/*) tmp="" ;; + esac + fi + if [ -n "$tmp" ] && [ -d "$tmp" ] && [ -w "$tmp" ]; then printf '%s' "$tmp"; return 0; fi + if [ -n "${HOME:-}" ] && [ -d "$HOME" ] && [ -w "$HOME" ]; then printf '%s' "$HOME"; return 0; fi + printf '/tmp' + return 0 +} + +# _telemetry_fallback_spool — where an event goes when there is no data dir yet. +# +# mktemp, never a fixed name: /tmp is world-writable on Linux, so a predictable +# path is a symlink target for an append that may be running under sudo. mktemp +# creates with O_EXCL. Trailing X's with no suffix after them: BSD mktemp (macOS) +# requires the X's at the END of the template, exactly as _choose_log_file's own +# fallback does. +_telemetry_fallback_spool() { + mktemp "$(_telemetry_fallback_dir)/tracebloc-telemetry-XXXXXX" 2>/dev/null || return 1 +} + +_telemetry_deliver() { + local json="$1" spool dir + # The install log gets it WHERE THERE IS ONE. `log` is a no-op until + # setup_log_file sets LOG_FILE, and setup_log_file runs AFTER validate_config + # and early_data_dir_guard — deliberately, because #432 refuses a network data + # dir *before* logging starts. + # + # An earlier version of this function claimed the log "always" got the event. + # It did not, and the consequence was the worst possible one: a run refused for + # being on NFS is a real, actionable field failure, and it was the single case + # that produced no record anywhere — invisible to the very failure rate this + # feature exists to produce. (Found by Bugbot on client#747, which also spotted + # that this file's own NFS test masked it by setting LOG_FILE=/dev/null.) + log "telemetry: $json" + + # TELEMETRY MUST NOT CREATE HOST_DATA_DIR. An observer that changes the + # install's own preconditions is not an observer. + # + # This ran from the EXIT trap and did `mkdir -p "$HOST_DATA_DIR/telemetry"` + # unconditionally — including on the path where early_data_dir_guard had just + # REFUSED that directory for being on a network filesystem and called `error`. + # The guard skips an existing dir on purpose ("an EXISTING data dir has no + # at-risk mkdir here", client#441), so the next run saw the directory this + # trap had created, returned 0, and installed MySQL onto NFS — the exact + # corruption client#432 exists to prevent, reintroduced by the telemetry that + # was only supposed to watch. Found by Bugbot on client#747; reproduced. + # + # So: spool INTO the data dir only when it already exists — and when it does + # not, into a temp file rather than nowhere. #1906's forwarder reads both. + # + # AND A FAILED WRITE FALLS THROUGH, it does not return. Both of these steps + # used to be `|| return 0`, which put the fallback out of reach in the one case + # it exists for: HOST_DATA_DIR present but not writable is precisely when + # _choose_log_file has already fallen back to a mktemp log — and on curl|bash + # that log is inside the bootstrap's own doomed TMPDIR — so `log` above kept + # nothing either and the event was gone. The fallback existed and was + # unreachable exactly when it was needed. (Bugbot on client#747; reproduced both + # halves — a 0500 data dir with no telemetry/ yet, and a spool path that cannot + # be appended to. Neither produced a record anywhere.) + # + # WRITTEN AT MOST ONCE. The `return 0` below sits after a SUCCESSFUL append and + # nowhere else, so the fallback is reached only on a path that wrote nothing — + # there is no flag to keep in step and no ordering to get wrong. Everything + # after the append (chmod, trim, mv) can fail freely: the line is already on + # disk, and re-filing it in the fallback would turn one install into two rows. + if [ -n "${HOST_DATA_DIR:-}" ] && [ -d "$HOST_DATA_DIR" ]; then + spool="$(_telemetry_spool_path)" + dir="${spool%/*}" + if mkdir -p "$dir" 2>/dev/null; then + chmod 700 "$dir" 2>/dev/null || true + # `2>/dev/null` BEFORE the `>>`, not after it. Redirections are applied left + # to right, and a failing `>>` is reported by the SHELL, not by printf — so + # the old order printed `…/pending.jsonl: Is a directory`, complete with the + # customer's path, out of an EXIT trap. Ordered this way the diagnostic + # lands in /dev/null with the rest. + if printf '%s\n' "$json" 2>/dev/null >>"$spool"; then + chmod 600 "$spool" 2>/dev/null || true + _telemetry_trim_spool "$spool" + return 0 + fi + fi + # mkdir or append failed — say nothing here and let the fallback below have + # it. `chmod 700` failing is not a write failure and deliberately does not + # divert: the record is still about to go into a directory we own. + fi + + # No data dir, or a data dir that would not take the write: the pre-log + # failures — validate_config, early_data_dir_guard — which are exactly the ones + # the run-started latch was built to preserve. One file per run, which is the + # same footprint _choose_log_file already leaves on this path; no trimming, + # because there is only ever one line in it. + spool="$(_telemetry_fallback_spool)" || return 0 + printf '%s\n' "$json" 2>/dev/null >>"$spool" || return 0 + chmod 600 "$spool" 2>/dev/null || true + return 0 +} + +# _telemetry_trim_spool SPOOL — keep the data-dir spool bounded. +# +# Bounded because nothing drains it until #1906, and an unbounded append on a +# customer's disk is a defect we would be shipping on purpose. Split out of +# _telemetry_deliver so the append's success is the last thing in that branch: +# inline, the trim sat between the append and the `return 0`, which is what made +# it easy to write a `|| return` in here that silently skipped the fallback +# decision. Nothing in this function can lose the record — it is already +# appended — so every failure path is `|| true`. +_telemetry_trim_spool() { + local spool="$1" + if [ -s "$spool" ]; then + if tail -n "$TB_TELEMETRY_SPOOL_MAX" "$spool" > "${spool}.tmp" 2>/dev/null; then + # The trim REPLACES the file, so the mode has to be put on the thing that + # survives. `> "${spool}.tmp"` creates under the process umask and `mv` + # keeps the tmp file's mode, so a chmod that ran only before this landed + # a 0644 spool: common.sh's `umask 077` normally covers it, but + # _install_userspace_tools (setup-linux.sh:893) and its macOS twin + # (setup-macos.sh:418) set `umask 022` around install_{kubectl,k3d,helm} + # and restore it only afterwards — so an install that dies in one of + # those emits from the EXIT trap under 022. Nothing sensitive is in the + # record by construction, so this is defence in depth; it is here because + # a file this installer creates should not depend on which line it died + # on for its mode. (saadqbal on client#747; reproduced — spool 644.) + # BEFORE the mv, not after it: a chmod on the spool afterwards would + # leave a window in which the file is world-readable, and — the thing + # that matters more — a second chmod on the spool is unreachable belt and + # braces that no test can redden, which is how a guard nobody has watched + # fail gets shipped. One chmod, on the inode that survives. + chmod 600 "${spool}.tmp" 2>/dev/null || true + mv "${spool}.tmp" "$spool" 2>/dev/null || rm -f "${spool}.tmp" 2>/dev/null || true + else + rm -f "${spool}.tmp" 2>/dev/null || true + fi + fi + return 0 +} + +# telemetry_emit_outcome EXIT_CODE — the one event this install produces. +# +# Called from install_cleanup, the EXIT trap, so it runs on every path including +# the interrupted and the fatal one (§6.5). Emits at most once per process: +# "one structured outcome event per install" is the ticket's wording, and a +# second trap invocation must not be able to make it two. +# +# Always returns 0. An installer that failed because telemetry was unhappy would +# be a strictly worse installer. +telemetry_emit_outcome() { + local json + [ -z "$_TB_TELEMETRY_EMITTED" ] || return 0 + _TB_TELEMETRY_EMITTED=1 + # No latch, no event: this trap also fires for `--help`, which installs + # nothing. See _TB_TELEMETRY_RUN_STARTED above for why a false success is + # worse here than a missing one. + [ -n "$_TB_TELEMETRY_RUN_STARTED" ] || return 0 + telemetry_enabled || return 0 + json="$(telemetry_render_event "${1:-0}")" || return 0 + [ -n "$json" ] || return 0 + _telemetry_deliver "$json" + return 0 +} diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index d265ff6a..72c61aa3 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -1,8 +1,9 @@ -7d4d98379601ee2f243e5889e9f718ef8e581e4a3c5ed49ade6c3b0e7f939a82 scripts/install-k8s.sh -22c2a3e6c3868427ae3fd92d519e6d1ed867b029a34b9129a712030eb3b36347 scripts/lib/common.sh +ccc40075a4c6dde3750d378e8e9bd3df2796a98c2e4d2637a1a1c193b7980656 scripts/install-k8s.sh +50dfc2ef1a50155d3d860869f00599b4d1d57aeda3947c237316759ed8564b13 scripts/lib/common.sh +38b2a2e9424a8e1d0c97e56c1be4f15659a405e57d1a4d343562aa157d775958 scripts/lib/telemetry.sh 467afdc27d4d85676840cfc55f9a2d3f9935a569020b104b9e5d9a37dd75e74f scripts/lib/preflight.sh c6bf113c00d68fb94f7654f2fb34db296160acd990c6a612ed02ae30076f2fe3 scripts/lib/detect-gpu.sh -d8c29bc8bd1f4633300940894da0f6527ca0a1dd7a3cfcbc80aad19dfd4d88cb scripts/lib/gpu-nvidia.sh +5daa9f076d5fd8d0d8156639d10c619a2bd2aad9fb802d23448fa41a9b00779e scripts/lib/gpu-nvidia.sh b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c scripts/lib/gpu-amd.sh c2cf13159dc0f341e0241e6c619a36c59dfe9e62f5ef297242ce558077e87a62 scripts/lib/setup-macos.sh 47268af2c406e8b05c57d470469b71ce7358875fe1adda3ecbb566bfb9aed1cc scripts/lib/setup-linux.sh @@ -11,8 +12,8 @@ a57546ef6e59f539281b2d83969cd0193c3fa9e48f79319aba6baf58abade1b6 scripts/lib/cl 7f4bedfff5c24ea7ca7585a50bd89fb11a0157945f404a475387543e9b2c68e8 scripts/lib/install-client-helm.sh 61c1c887d158af52d4da4734b3bfa83205b2600ae7a291bfb3074daf3d9ffb55 scripts/lib/install-cli.sh ea2bbd9948ee9e31e93271e235c630ced50d746e51a5629b5622041d8a39df07 scripts/lib/provision.sh -ed458e63c06c2d86f9319ea3db227d5d183ca4e6975b1110618db500a4ae0292 scripts/lib/assess.sh +8d42658f7503a92cbf3cddd4bb4a0735bd400fa682af9b55d2c84e7769ab67e1 scripts/lib/assess.sh 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f scripts/lib/probe.sh -a9bb43caa3e7d156d48e7f9f4473c22da0b3b42c25d84fd49f35349b25956132 scripts/lib/summary.sh +c0f50789204569ff6f152694cc352148ce4b0b0cba425e8581db33776f1dd014 scripts/lib/summary.sh 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364 scripts/lib/diagnose.sh 478da84ef9be6cf62c3348c86ebcef53dc9e9ac9286d1a5631fbaa4a071c5207 scripts/install-k8s.ps1 diff --git a/scripts/testdata/golden/00-install.golden b/scripts/testdata/golden/00-install.golden index 21bdd1d0..baa9c379 100644 --- a/scripts/testdata/golden/00-install.golden +++ b/scripts/testdata/golden/00-install.golden @@ -91,6 +91,16 @@ Advanced configuration (environment variables): Must be on a LOCAL disk — NFS/CIFS/SMB is rejected (the database corrupts on network storage). TRACEBLOC_ALLOW_NETWORK_FS=1 overrides. +Usage reporting: + This installer records ONE outcome event per run so we can see failures without + waiting for someone to report them: which step it reached, how long each step + took, the exit code, an error class, your OS and architecture, and the version. + It cannot record your arguments, any path, any file name, your username, your + hostname or your credentials — every field is a number or a value from a fixed + list, so there is nowhere for those to go. + TRACEBLOC_NO_TELEMETRY=1 Turn it off. + DO_NOT_TRACK=1 Also turns it off. + Windows: irm https://raw.githubusercontent.com/tracebloc/client/main/scripts/install.ps1 | iex diff --git a/scripts/tests/install-bootstrap.bats b/scripts/tests/install-bootstrap.bats index a070dbda..07bd63e7 100644 --- a/scripts/tests/install-bootstrap.bats +++ b/scripts/tests/install-bootstrap.bats @@ -25,14 +25,31 @@ setup() { mkdir -p "$BIN" "$SERVE/scripts/lib" "$SERVE_REL" # ---- Populate the "repo" with stand-in sub-scripts the bootstrap fetches ---- - # Each is trivial but real bash; install-k8s.sh is the privileged entrypoint — - # it writes a sentinel so a test can prove it was (or was NOT) reached. - for rel in install-k8s.sh \ - lib/common.sh lib/preflight.sh lib/detect-gpu.sh lib/gpu-nvidia.sh \ - lib/gpu-amd.sh lib/setup-macos.sh lib/setup-linux.sh lib/cluster.sh \ - lib/gpu-plugins.sh lib/install-client-helm.sh lib/install-cli.sh \ - lib/provision.sh lib/assess.sh lib/probe.sh lib/summary.sh lib/diagnose.sh; do - printf '#!/usr/bin/env bash\n# stub %s\n' "$rel" > "$SERVE/scripts/$rel" + # The list is DERIVED from install.sh's own FILES array, not restated here. + # It used to be written out twice in this setup, and both copies had to be + # edited by hand whenever the installer gained a lib — so adding + # scripts/lib/telemetry.sh (backend#1907) turned ten unrelated supply-chain + # tests red for a reason that had nothing to do with them. gen-manifest.sh + # already reads the array this way, and cross-checks it against its own; this + # is the third reader of the same declaration and the first that used to + # disagree with it silently. + BOOT_FILES=() + while IFS= read -r _bf; do + [ -n "$_bf" ] && BOOT_FILES+=("$_bf") + done < <(awk '/^FILES=\(/{f=1;next} /^\)/{f=0} f' "$SCRIPTS_DIR/install.sh" \ + | sed -e 's/^[[:space:]]*"//' -e 's/"[[:space:]]*$//') + # Fail closed: an empty list would build an empty served tree AND an empty + # manifest, which verify against each other perfectly while testing nothing. + [ "${#BOOT_FILES[@]}" -ge 2 ] || { + echo "install-bootstrap: parsed ${#BOOT_FILES[@]} entries out of install.sh's FILES array — the parse is inert" >&2 + return 1 + } + # Each stub is trivial but real bash; install-k8s.sh is the privileged + # entrypoint — it writes a sentinel so a test can prove it was (or was NOT) + # reached. + for rel in "${BOOT_FILES[@]}"; do + mkdir -p "$SERVE/$(dirname "$rel")" + printf '#!/usr/bin/env bash\n# stub %s\n' "$rel" > "$SERVE/$rel" done cat > "$SERVE/scripts/install-k8s.sh" < "$SBX/k8s-ran" EOF # ---- Build a manifest.sha256 over exactly those files (real digests) ------- - ( cd "$SERVE" && for f in \ - scripts/install-k8s.sh scripts/lib/common.sh scripts/lib/preflight.sh \ - scripts/lib/detect-gpu.sh scripts/lib/gpu-nvidia.sh scripts/lib/gpu-amd.sh \ - scripts/lib/setup-macos.sh scripts/lib/setup-linux.sh scripts/lib/cluster.sh \ - scripts/lib/gpu-plugins.sh scripts/lib/install-client-helm.sh \ - scripts/lib/install-cli.sh scripts/lib/provision.sh scripts/lib/assess.sh \ - scripts/lib/probe.sh scripts/lib/summary.sh scripts/lib/diagnose.sh; do + ( cd "$SERVE" && for f in "${BOOT_FILES[@]}"; do printf '%s %s\n' "$(_real_sha "$SERVE/$f")" "$f" done ) > "$SERVE_REL/manifest.sha256" printf 'FAKE-SIG\n' > "$SERVE_REL/manifest.sha256.sig" diff --git a/scripts/tests/telemetry-vocabulary-agreement.sh b/scripts/tests/telemetry-vocabulary-agreement.sh new file mode 100755 index 00000000..9de1f33a --- /dev/null +++ b/scripts/tests/telemetry-vocabulary-agreement.sh @@ -0,0 +1,418 @@ +#!/usr/bin/env bash +# +# telemetry-vocabulary-agreement.sh — prove telemetry.sh's closed sets are the +# producers' sets (backend#1907). +# +# WHY THIS EXISTS +# --------------- +# telemetry.sh's whole privacy and quality argument rests on four closed +# vocabularies: the install phases, the client states, the installer's own +# script names, and the error classes. A closed set that has drifted from what +# actually produces its values does not fail loudly — it quietly reports +# `unknown`, forever, on the exact runs somebody added the new value for. That +# is backend#1729's class in its purest form: a mechanism that looks like it +# verifies something while being disconnected from the thing it claims to +# check. +# +# THIS SCRIPT DERIVES. It parses install-k8s.sh, summary.sh and gen-manifest.sh +# and compares them to telemetry.sh's declarations. It holds no fifth copy of +# any vocabulary — a fresh hand-written list is the defect, not the fix (the +# lesson env-vocabulary-agreement.sh was written for, in this same directory). +# +# The error-class check is different in kind and deliberately so: there is no +# second declaration to parse, so it EXERCISES telemetry_error_class over the +# full cross-product of the two closed input sets and checks (a) every answer +# is registered and (b) every registered class is reachable. Comparing the +# declaration to itself would be self-consistent and therefore blind. +# +# READ-ONLY. Exit 0 clean, 1 disagreement, 2 cannot tell (fail closed). +set -uo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../.." && pwd)" + +TELEMETRY="$root/scripts/lib/telemetry.sh" +INSTALL_K8S="$root/scripts/install-k8s.sh" +SUMMARY="$root/scripts/lib/summary.sh" +GEN_MANIFEST="$root/scripts/gen-manifest.sh" + +fail_closed() { printf 'ERROR: %s\n' "$1" >&2; exit 2; } + +# An unreadable file is not evidence of agreement: zero parsed values compare +# equal to zero parsed values, and the run would report clean having read +# nothing. Refuse instead. +for f in "$TELEMETRY" "$INSTALL_K8S" "$SUMMARY" "$GEN_MANIFEST"; do + [ -r "$f" ] || fail_closed "cannot read ${f#"$root"/} — refusing to report agreement between declarations one of which was not read" +done + +# shellcheck source=/dev/null +source "$TELEMETRY" + +status=0 +disagree() { printf ' [x] %s\n' "$1" >&2; status=1; } + +# sorted LIST — a space-separated vocabulary as sorted, unique lines. +sorted() { + # Word-splitting the argument is the point: these vocabularies are + # space-separated lists. + # shellcheck disable=SC2086 + printf '%s\n' $1 | sed '/^$/d' | LC_ALL=C sort -u +} + +compare() { # LABEL DECLARED DERIVED SOURCE_DESCRIPTION + local label="$1" declared="$2" derived="$3" src="$4" a b + a="$(sorted "$declared")" + b="$(sorted "$derived")" + if [ -z "$b" ]; then + fail_closed "parsed zero values for $label out of $src — the parse is inert, so this check proves nothing" + fi + if [ "$a" != "$b" ]; then + disagree "$label disagrees with $src:" + diff -u <(printf '%s\n' "$a") <(printf '%s\n' "$b") \ + | sed -e '1,2d' -e 's/^/ /' >&2 || true + return 1 + fi + printf ' ok: %s agrees with %s (%s values)\n' "$label" "$src" "$(printf '%s\n' "$a" | grep -c .)" +} + +echo "== telemetry vocabulary agreement ==" + +# --- 1. install phases ← install-k8s.sh's own step_header letters ------------ +# step_header IS the dispatcher for the phase clock (common.sh calls +# telemetry_phase_begin from it), so its letters are the input domain. A seventh +# step added to the run-through without a name here would be timed as `unknown`. +declared_letters="" +for pair in $TB_TELEMETRY_PHASES; do declared_letters="$declared_letters ${pair%%:*}"; done +derived_letters="$(sed -nE 's/^[[:space:]]*step_header[[:space:]]+([a-z])[[:space:]].*/\1/p' "$INSTALL_K8S")" +compare "phase letters" "$declared_letters" "$derived_letters" "install-k8s.sh's step_header calls" + +# --- 2. client states ← summary.sh's only two writers of CLIENT_STATE -------- +# wait_for_client_ready assigns the literal `connected`; _diagnose_not_ready +# printf's every other state. Note what this caught when it was written: the +# CLIENT_STATE docstring in summary.sh listed five states and the function +# produces six (image_pull_ca, added by #424, was never added to the comment). +derived_states="$( + { + sed -nE 's/^[[:space:]]*CLIENT_STATE="([a-z_]+)".*/\1/p' "$SUMMARY" + awk '/^_diagnose_not_ready\(\)/{f=1} f&&/^}/{f=0} f' "$SUMMARY" \ + | sed -nE "s/.*printf '([a-z_]+)'.*/\1/p" + } +)" +compare "client states" "$TB_TELEMETRY_CLIENT_STATES" "$derived_states" \ + "summary.sh's CLIENT_STATE writers" + +# --- 3. source basenames ← gen-manifest.sh's FILES array -------------------- +# The manifest already declares, and cosign already signs, the exact set of +# scripts this installer runs. Reusing it means a script added to the installer +# is reportable the day it lands, and a basename that is not one of ours cannot +# be emitted at all. +derived_sources="$( + { + awk '/^FILES=\(/{f=1;next} /^\)/{f=0} f' "$GEN_MANIFEST" \ + | sed -e 's/^[[:space:]]*"//' -e 's/"[[:space:]]*$//' -e 's|.*/||' + # install.sh is the bootstrap: it is the thing that VERIFIES the manifest, + # so it is legitimately not in it, and a failure inside it is still ours. + echo "install.sh" + } +)" +compare "source basenames" "$TB_TELEMETRY_SOURCES" "$derived_sources" \ + "gen-manifest.sh's FILES array + the bootstrap" + +# --- 4. error classes ← telemetry_error_class's actual behaviour ------------ +# No second declaration exists to parse, so this exercises the classifier over +# the full cross-product of its closed input sets. Both directions matter: +# an unregistered answer is a namespace opening on its own, and a registered +# class nothing can produce is a dashboard row that will never populate. +# +# The re-run marker is the classifier's fourth input and it is a BOOLEAN, so the +# full input domain over it is both values — enumerated here rather than left at +# the default, because a vocabulary gap is exactly what mutation coverage cannot +# see (workspace CLAUDE.md rule 6). Leaving it out would make `unexpected_exit_2` +# unreachable and this section would say so; that reading is the check working. +phase_names="bootstrap unknown" +for pair in $TB_TELEMETRY_PHASES; do phase_names="$phase_names ${pair#*:}"; done +produced="" +for phase in $phase_names; do + for state in "" $TB_TELEMETRY_CLIENT_STATES; do + for code in 1 2 42 130; do + for handoff in "" 1; do + cls="$(telemetry_error_class "$code" "$phase" "$state" "$handoff")" + produced="$produced $cls" + case " $TB_TELEMETRY_ERROR_CLASSES " in + *" $cls "*) ;; + *) disagree "telemetry_error_class($code, $phase, '$state', '$handoff') returned '$cls', which is not in TB_TELEMETRY_ERROR_CLASSES" ;; + esac + done + done + done +done +[ -n "$(sorted "$produced")" ] || fail_closed "the classifier produced nothing — the cross-product is inert" +for cls in $TB_TELEMETRY_ERROR_CLASSES; do + case " $(sorted "$produced" | tr '\n' ' ') " in + *" $cls "*) ;; + *) disagree "'$cls' is registered but no (phase, state) pair produces it — a dashboard row that can never populate" ;; + esac +done +[ "$status" -eq 0 ] && printf ' ok: every error class is reachable and every answer is registered\n' + +# --- 5. the version shape ← install.sh's own immutable-release-tag gate ------ +# service.version is the one resource field a shell variable feeds, and the +# generic token shape is not enough on its own: `v1.9.3-<64 arbitrary chars>` +# satisfies it, which would make the version column a 64-byte free-text channel. +# The bootstrap already decides what a release tag looks like — it refuses to +# fetch from anything else — so telemetry.sh reuses that exact regex. +# +# THIS CHECK USED TO COMPARE BYTES, AND BYTES WERE THE WRONG PROPERTY. It read +# the two regexes and asserted they were the same string, then reported "the +# service.version shape is install.sh's own release-tag gate" — a claim about +# BEHAVIOUR that byte-identity does not support, because the two sides did not +# match with the same operator. install.sh used `[[ =~ ]]` (whole string); +# telemetry.sh used `grep -qE` (whole LINE), so `v1.9.3\n` was refused +# by the bootstrap and admitted by the emitter — from identical regexes, with +# this check green throughout. Exactly backend#1729's class: a guard passing on a +# property it does not actually check. (saadqbal on client#747.) +# +# So it now checks both, in this order: +# (a) the two declarations are still byte-identical — cheap, and it localises +# the failure to "somebody edited one of them"; +# (b) they AGREE ON VERDICTS over a corpus of inputs, each side evaluated by +# the operator the file that owns it actually uses. (b) is the check the +# header sentence promises; (a) alone never was. +# +# The corpus is written down here rather than derived, and that is deliberate: +# it is the input domain, not a copy of either rule (workspace CLAUDE.md rule 6 — +# a vocabulary gap is invisible to mutation coverage, so the inputs have to be +# enumerated independently of the matcher). It must contain at least one input +# that separates line-matching from string-matching, or (b) degenerates into (a). +BOOTSTRAP="$root/scripts/install.sh" +[ -r "$BOOTSTRAP" ] || fail_closed "cannot read scripts/install.sh" +boot_re="$(sed -nE 's/.*! "\$REF" =~ (\^v.*\$)\ ?\]\].*/\1/p' "$BOOTSTRAP" | head -1)" +[ -n "$boot_re" ] || fail_closed "could not find install.sh's release-tag regex — the parse is inert, so this check proves nothing" +if [ "$boot_re" != "$TB_TELEMETRY_VERSION_RE" ]; then + disagree "TB_TELEMETRY_VERSION_RE and install.sh's release-tag gate are not the same regex:" + printf ' telemetry.sh: %s\n install.sh: %s\n' \ + "$TB_TELEMETRY_VERSION_RE" "$boot_re" >&2 +else + printf ' ok: the service.version regex is byte-identical to install.sh'"'"'s\n' +fi + +# The bootstrap's verdict, reached the way install.sh reaches it (:203). +_boot_admits() { [[ $1 =~ $boot_re ]]; } +# The emitter's verdict, reached the way telemetry.sh reaches it: through the +# real function, on the real variable. Not a re-implementation of the rule — +# a mutation of _telemetry_version has to redden this (workspace CLAUDE.md +# rule 9), which it cannot if this line spells the match out again. +_emitter_admits() { + local rendered + rendered="$( TB_VERSION="$1"; _telemetry_version )" + [ "$rendered" != "0.0.0-unknown" ] +} + +version_corpus=( + 'v1.9.3' # a plain release tag: both must admit + 'v1.9.3-rc.1' # the pre-release suffix the regex allows + 'v1.9.3.post1' # the dotted suffix it allows + 'main' # a branch name: both must refuse + 'v1.9' # two segments: both must refuse + 'v1.9.3-'"$(printf '%064d' 0)" # 64 trailing chars: both must refuse + 'v1.9.3 ; rm -rf /' # a space: both must refuse + $'v1.9.3\nmain' # THE SEPARATOR — a first line that matches and + # a second that does not. grep says yes, [[ =~ ]] + # says no. Without this input the behavioural + # check is just the byte check again. + $'main\nv1.9.3' # …and the other way round + $'v1.9.3\n","injected":"yes' # the actual forgery from the review +) +separator_seen=0 +for candidate in "${version_corpus[@]}"; do + case "$candidate" in *$'\n'*) separator_seen=1 ;; esac + if _boot_admits "$candidate"; then b=admit; else b=refuse; fi + if _emitter_admits "$candidate"; then e=admit; else e=refuse; fi + if [ "$b" != "$e" ]; then + disagree "the two version gates DISAGREE on $(printf '%q' "$candidate"): install.sh would $b, telemetry.sh does $e" + fi +done +# Fail closed on an inert corpus: without an embedded-newline input the +# behavioural check proves nothing the byte check did not already prove, and it +# would report clean forever. +[ "$separator_seen" -eq 1 ] || fail_closed "the version corpus contains no embedded-newline input — the behavioural check cannot separate line-matching from string-matching, so it proves nothing" +[ "$status" -eq 0 ] && printf ' ok: the two version gates agree on every input in the corpus (%s inputs), not just byte-for-byte\n' "${#version_corpus[@]}" + +# --- 6. event names ← telemetry_render_event's own case statement ------------ +# §6.2 requires the set of event names a service can emit to be finite and +# enumerable by grep. TB_TELEMETRY_EVENT_NAMES is that enumeration; this proves +# it is the set the renderer actually produces, from two independent directions. +# +# It cannot check the other half — that each third segment is a §6.4 registered +# outcome verb — because the verb registry lives in the rfcs repo, which is not +# checked out here. Copying the verbs into this file would be a fifth declaration +# of somebody else's vocabulary, i.e. the defect this script exists to prevent. +# The §6.1 GRAMMAR is checkable from here, so it is checked. +declared_names="$TB_TELEMETRY_EVENT_NAMES" + +# (a) the literals in the case statement. +parsed_names="$(sed -nE 's/.*[^A-Za-z_]event="(install\.[a-z0-9_.]+)".*/\1/p' "$TELEMETRY")" +compare "event names" "$declared_names" "$parsed_names" \ + "the literals in telemetry_render_event's case statement" + +# (b) what the function actually renders, over the exit codes the installer +# produces: 0, a bare 2, an ordinary failure, and both signals. 2 is swept here +# with the re-run marker UNSET, which is the ordinary-tool case — the marker's own +# effect on the render is (d5)'s job, because it is a behaviour and not a name. +# Read by the sourced telemetry.sh, not by this file — telemetry_render_event +# needs a recognised environment or it renders nothing at all (§3.2), which would +# make the comparison below inert. +# shellcheck disable=SC2034 +CLIENT_ENV=prod +# shellcheck disable=SC2034 +OS=Linux +# shellcheck disable=SC2034 +ARCH=x86_64 +# shellcheck disable=SC2034 +TB_VERSION=v1.9.3 +rendered_names="" +for code in 0 1 2 42 130 143; do + for skipped in "" 1; do + _TB_TELEMETRY_SKIPPED="$skipped" + ev="$(telemetry_render_event "$code" | sed -nE 's/.*"event\.name":"([^"]*)".*/\1/p')" + [ -n "$ev" ] || disagree "telemetry_render_event $code rendered no event.name" + rendered_names="$rendered_names $ev" + done +done +_TB_TELEMETRY_SKIPPED="" +compare "event names rendered" "$declared_names" "$rendered_names" \ + "telemetry_render_event exercised over the installer's exit codes" + +# (c) §6.1's grammar: exactly three segments, lowercase, no runtime value. +name_re='^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*){2}$' +for ev in $declared_names; do + [[ $ev =~ $name_re ]] || disagree "'$ev' is not a legal contract event name (§6.1: three segments, ^[a-z][a-z0-9_]*)" +done + +# (d) THE RE-RUN HANDOFF IS A MARKER, NOT THE NUMBER 2. +# +# gpu-nvidia.sh exits 2 after a SUCCESSFUL driver install to ask for a reboot, and +# install_cleanup has treated 2 as its own outcome since client#681 — so an exit 2 +# that renders `failed` fabricates a prerequisite failure on every unattended GPU +# host's first install. That much this check always proved. +# +# What it did not: `2` is also a status ORDINARY TOOLS produce (grep on a file +# error, curl on a failed init, tar on a fatal, and cluster.sh's `exit "$create_rc"` +# re-raising whatever k3d returned). Keying on the number filed those as +# `cancelled` with no error.type — removed from the numerator, not misfiled in it. +# So the checked property is now agreement between the marker's PRODUCERS, the +# emitter's BRANCH, and the RENDER, in that order. (saadqbal on client#747.) +# +# Still derived from install_cleanup's own branch as the premise: if common.sh +# stops treating 2 specially, this whole section retires itself loudly rather than +# guarding a rule that no longer exists. +COMMON="$root/scripts/lib/common.sh" +[ -r "$COMMON" ] || fail_closed "cannot read scripts/lib/common.sh" +grep -qE '^\s*if \[\[ \$exit_code -eq 2 \]\]; then' "$COMMON" \ + || fail_closed "could not find install_cleanup's exit-2 branch in common.sh — this check's premise is gone, so it proves nothing" + +# (d1) the variable the emitter's exit-2 branch actually tests, read out of the +# branch itself. Not written down here: a second spelling of the name is how this +# check would go on passing after a rename that broke the handoff. +handoff_var="$(awk '/^ case "\$code" in/{c=1} c&&/^ 2\)/{b=1} b&&match($0,/_TB_[A-Z_]+/){print substr($0,RSTART,RLENGTH); exit}' "$TELEMETRY")" +[ -n "$handoff_var" ] || fail_closed "could not find the variable telemetry_render_event's exit-2 branch tests — the parse is inert, so this check proves nothing" + +# (d2) the SETTER: the one-line function in telemetry.sh whose body assigns that +# variable — the same shape as its two sibling latches (telemetry_run_started, +# telemetry_run_skipped). Derived, so a rename moves both sides at once or reddens +# here. Deliberately NOT "the nearest preceding function header": the top-level +# `=""` init line would then be attributed to whichever latch was defined +# above it, and this check would go on green while pointing at the wrong function. +# Reformatting the setter across several lines makes this parse inert, and inert +# fails closed below rather than passing. +handoff_fn="$(sed -nE "s/^([a-z_]+)\(\) \{[^}]*${handoff_var}=.*/\1/p" "$TELEMETRY" | head -1)" +[ -n "$handoff_fn" ] || fail_closed "no function in telemetry.sh assigns $handoff_var — the marker has no setter, so nothing can declare a handoff" +declare -F "$handoff_fn" >/dev/null 2>&1 \ + || fail_closed "$handoff_fn was parsed out of telemetry.sh but is not defined after sourcing it" + +# (d3) the marker must be CLEARED at source time. This is the fail-open hole one +# level down: read as an inherited environment value, `=1` in a user's shell +# would turn every real failure into a cancel. +grep -qE "^${handoff_var}=(\"\"|'')?\$" "$TELEMETRY" \ + || disagree "$handoff_var is not cleared at telemetry.sh's top level — an inherited environment value could pose as a declared handoff" + +# (d4) EVERY deliberate `exit 2` in the installer runtime declares the handoff. +# The runtime is install-k8s.sh plus the libs it sources: those are the files that +# run under install_cleanup's EXIT trap, and therefore the only ones whose exit +# status telemetry ever sees. A NEW handoff site added without the setter books +# itself as a failure — the safe direction, but still wrong, and this is what +# says so. +# +# Fails closed on ZERO sites, because that is the shape this whole change exists +# to prevent: install_cleanup still reads TRACEBLOC_DOCKER_FIRST_RUN_EXIT, whose +# only producer was deleted in 8c3a3d4 (March) — a live branch keyed on a marker +# nothing sets, passing forever. Zero producers here means the marker has quietly +# become that, and "no sites to check" must never read as agreement. +handoff_sites=0 undeclared="" +while IFS= read -r hit; do + f="${hit%%:*}"; rest="${hit#*:}"; n="${rest%%:*}" + handoff_sites=$(( handoff_sites + 1 )) + # The declaration must be in the same block, immediately before the exit. Three + # lines is the window: the guarded call is one line, and a `log` line before it + # is the existing shape at gpu-nvidia.sh's site. + if ! awk -v s="$(( n > 3 ? n - 3 : 1 ))" -v e="$n" -v fn="$handoff_fn" \ + 'NR>=s && NR/dev/null || true) +if [ "$handoff_sites" -eq 0 ]; then + fail_closed "no \`exit 2\` site exists in the installer runtime, so nothing calls $handoff_fn — the marker is dead and the emitter's cancelled branch is unreachable (the TRACEBLOC_DOCKER_FIRST_RUN_EXIT shape)" +fi +if [ -n "$undeclared" ]; then + disagree "these \`exit 2\` sites do not call $handoff_fn, so telemetry will book them as failures:$undeclared" +else + printf ' ok: all %s deliberate `exit 2` site(s) declare the handoff via %s\n' "$handoff_sites" "$handoff_fn" +fi + +# (d5) and the render agrees with the marker, in BOTH directions — through the +# real functions, on the real variable, so a mutation of either has to redden this +# (workspace CLAUDE.md rule 9). The undeclared direction is the one the number- +# keyed version got wrong, and it is asserted first. +eval "$handoff_var=''" +ev2_bare="$(telemetry_render_event 2)" +case "$(printf '%s' "$ev2_bare" | sed -nE 's/.*"event\.name":"([^"]*)".*/\1/p')" in + '') disagree "an undeclared exit 2 rendered no event at all" ;; + *.failed) case "$ev2_bare" in + *'"error.type"'*) printf ' ok: an undeclared exit 2 is a failure and carries error.type (§8.4)\n' ;; + *) disagree "an undeclared exit 2 renders failed but carries no error.type — §8.4 requires one, and a failure that cannot be grouped is the reason this contract exists" ;; + esac ;; + *) disagree "an undeclared exit 2 renders '$(printf '%s' "$ev2_bare" | sed -nE 's/.*"event\.name":"([^"]*)".*/\1/p')' — an ordinary tool's status 2 escaping under \`set -e\` must land in the numerator, not be silently cancelled" ;; +esac + +"$handoff_fn" +ev2_marked="$(telemetry_render_event 2)" +case "$(printf '%s' "$ev2_marked" | sed -nE 's/.*"event\.name":"([^"]*)".*/\1/p')" in + *.failed) disagree "a DECLARED exit 2 renders failed — install_cleanup treats it as its own outcome (\"Re-run required\") and gpu-nvidia.sh exits 2 after a driver install SUCCEEDED" ;; + '') disagree "a declared exit 2 rendered no event at all" ;; + *) case "$ev2_marked" in + *'"error.type"'*) disagree "a declared exit 2 carries error.type — it is not a failure, and §8.4 attaches error.type to failures" ;; + *) printf ' ok: a declared exit 2 renders %s, not a failure\n' "$(printf '%s' "$ev2_marked" | sed -nE 's/.*"event\.name":"([^"]*)".*/\1/p')" ;; + esac ;; +esac +# The marker is process state and the sections below render more events: leave it +# the way telemetry.sh sourced it, or (b)'s name sweep silently runs half-marked. +eval "$handoff_var=''" + +# --- 7. the documented opt-out ← telemetry.sh's real variables -------------- +# A stale doc here is worse than none: a user who exports the variable +# `--help` names believes they have opted out, and nothing else would ever tell +# them otherwise. The declaration is in telemetry.sh and the promise is in +# common.sh's print_help, so neither file can be edited alone. +HELP_SRC="$root/scripts/lib/common.sh" +[ -r "$HELP_SRC" ] || fail_closed "cannot read scripts/lib/common.sh" +documented="$(awk '/^print_help\(\)/{f=1} f&&/^HELP$/{f=0} f' "$HELP_SRC" \ + | sed -nE 's/^[[:space:]]*([A-Z_]+)=1[[:space:]].*/\1/p')" +[ -n "$documented" ] || fail_closed "print_help names no opt-out variable — either the section was removed (then remove this check) or its shape changed and this parse is inert" +compare "opt-out variables" "$TB_TELEMETRY_OPT_OUT_VARS" "$documented" \ + "the environment variables print_help tells users to set" + +if [ "$status" -eq 0 ]; then + echo " ok: telemetry vocabularies agree with their producers" +fi +exit "$status" diff --git a/scripts/tests/telemetry.bats b/scripts/tests/telemetry.bats new file mode 100644 index 00000000..b13344d7 --- /dev/null +++ b/scripts/tests/telemetry.bats @@ -0,0 +1,1414 @@ +#!/usr/bin/env bats +# ============================================================================= +# telemetry.bats — the installer's one outcome event per install (backend#1907) +# +# The vocabularies are checked elsewhere, by derivation: +# scripts/tests/telemetry-vocabulary-agreement.sh parses install-k8s.sh, +# summary.sh and gen-manifest.sh and compares them to telemetry.sh's closed +# sets. This file checks the BEHAVIOUR — the event name, the required attribute +# set, and the privacy boundary, which is asserted against what the code +# actually renders rather than against a list of keys we hope nobody adds. +# ============================================================================= +bats_require_minimum_version 1.7.0 +load test_helper + +# A value that could only have arrived from the machine this ran on. Written +# down here, independently of anything the matcher inspects — a needle iterated +# out of the haystack finds itself and nothing else. +CANARY="CANARY-PATIENT-7" + +setup() { + load_lib telemetry.sh + CLIENT_ENV=prod + OS=Linux + ARCH=x86_64 + TB_VERSION=v1.9.3 + CLIENT_STATE="" + TB_ERR_LOC="" + HOST_DATA_DIR="$BATS_TEST_TMPDIR/data" + # The data dir must already exist for the spool to be written: telemetry never + # creates it (see the NFS-guard test below). A real install reaches + # _telemetry_deliver only after setup_log_file has made it. + mkdir -p "$HOST_DATA_DIR" + # main() sets this once the terminal commands (--help, --diagnose, + # prepare-host) have had their chance to dispatch. The tests that exercise + # delivery are standing in for a committed install run, so they set it too; + # the tests BELOW that assert on the latch clear it deliberately. + telemetry_run_started + unset TRACEBLOC_NO_TELEMETRY DO_NOT_TRACK 2>/dev/null || true +} + +# attr KEY — read one attribute out of a rendered event, without a JSON parser +# (the repo's installer scripts do not get to depend on jq). +attr() { + printf '%s' "$1" | sed -nE "s/.*\"$2\":\"?([^\",}]*)\"?.*/\1/p" +} + +# ── the event name ─────────────────────────────────────────────────────────── + +@test "the event name follows the exit code, from a literal (contract §6.1)" { + run telemetry_render_event 0 + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *'"event.name":"install.run.succeeded"'* ]] || return 1 + + run telemetry_render_event 1 + [[ "$output" == *'"event.name":"install.run.failed"'* ]] || return 1 + + # 130/143 are the SIGINT/SIGTERM routes install-k8s.sh installs. A user's + # Ctrl-C is not an installer failure, and counting it as one moves the rate + # D9's alerts are written against every time somebody changes their mind. + run telemetry_render_event 130 + [[ "$output" == *'"event.name":"install.run.cancelled"'* ]] || return 1 + run telemetry_render_event 143 + [[ "$output" == *'"event.name":"install.run.cancelled"'* ]] || return 1 +} + +@test "the DECLARED exit-2 re-run handoff is not a failure (saadqbal, client#747)" { + # gpu-nvidia.sh exits 2 after install_nvidia_drivers SUCCEEDED, to ask for a + # reboot. That call sits under step_header b, so folding 2 into `failed` booked + # a fabricated `prerequisites_failed` on every unattended GPU host's first + # install — in the very rate this ticket exists to produce. install_cleanup has + # treated 2 as its own outcome ("Re-run required") since client#681. + TB_TELEMETRY_PHASE=prerequisites + telemetry_rerun_handoff # what the `exit 2` site does before exiting + run telemetry_render_event 2 + [ "$status" -eq 0 ] || return 1 + [ "$(attr "$output" 'event.name')" = "install.run.cancelled" ] || return 1 + [[ "$output" != *'"error.type"'* ]] || return 1 + # The exit code is what separates the handoff from the user's own Ctrl-C, so it + # has to be on the record — the name deliberately does not carry it. + [ "$(attr "$output" 'tracebloc.install.exit_code')" = "2" ] || return 1 + + # The anchor: an ordinary failure in the same phase must STILL be a failure, or + # every assertion above is satisfied by a function that never says `failed`. + run telemetry_render_event 1 + [ "$(attr "$output" 'event.name')" = "install.run.failed" ] || return 1 + [ "$(attr "$output" 'error.type')" = "prerequisites_failed" ] || return 1 +} + +@test "an UNDECLARED exit 2 is a failure with its own error.type (saadqbal, client#747)" { + # 2 is a sentinel the installer chose AND a status ordinary tools produce: grep + # exits 2 on a file error, curl on a failed init, tar on a fatal, and + # cluster.sh:1129 re-raises whatever k3d returned. Keyed on the NUMBER, every one + # of those rendered `cancelled` with no error.type — a hard failure removed from + # the NUMERATOR of the rate this ticket exists to produce rather than misfiled + # inside it, which is the direction nobody notices. + # + # So: no marker, no handoff. Fail closed toward counting it. + TB_TELEMETRY_PHASE=prerequisites + [ -z "$_TB_TELEMETRY_RERUN_HANDOFF" ] || return 1 # nothing has declared one + run telemetry_render_event 2 + [ "$status" -eq 0 ] || return 1 + [ "$(attr "$output" 'event.name')" = "install.run.failed" ] || return 1 + # And it is DISTINGUISHABLE — not folded into the phase's own bucket, where a + # stray 2 would be indistinguishable from a real prerequisite failure, and not + # `unclassified`, which already means "we cannot name the phase". + [ "$(attr "$output" 'error.type')" = "unexpected_exit_2" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.exit_code')" = "2" ] || return 1 + # The phase is not lost by flattening the class: it is its own attribute. + [ "$(attr "$output" 'tracebloc.install.phase')" = "prerequisites" ] || return 1 + + # It wins over a readiness diagnosis too: an exit status that was not ours to + # read is a less trustworthy input than saying so, and client_state stays on the + # record either way. + TB_TELEMETRY_PHASE=connect + CLIENT_STATE=image_pull_ca + run telemetry_render_event 2 + [ "$(attr "$output" 'error.type')" = "unexpected_exit_2" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.client_state')" = "image_pull_ca" ] || return 1 + # The anchor: the same state on an ordinary failure still yields the diagnosis, + # or this test would pass against a classifier that only ever says one thing. + run telemetry_render_event 1 + [ "$(attr "$output" 'error.type')" = "image_pull_untrusted_ca" ] || return 1 +} + +@test "the marker cannot be inherited from the environment (fail closed)" { + # The marker is a process-internal handshake, not an input. Read off the + # environment, `_TB_TELEMETRY_RERUN_HANDOFF=1` in a user's shell would turn every + # real failure into a cancel — the same fail-open hole one level down. telemetry.sh + # clears it at source time, before main() runs. + run env _TB_TELEMETRY_RERUN_HANDOFF=1 CLIENT_ENV=prod OS=Linux ARCH=x86_64 \ + bash -c 'source "'"$LIB_DIR"'/common.sh"; source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null; telemetry_render_event 2' + [ "$status" -eq 0 ] || return 1 + [ "$(attr "$output" 'event.name')" = "install.run.failed" ] || return 1 + [ "$(attr "$output" 'error.type')" = "unexpected_exit_2" ] || return 1 + + # The anchor: the same harness DOES honour a handoff declared in-process, so the + # assertion above is about where the value came from and not about the harness + # being unable to produce a cancel at all. + run env CLIENT_ENV=prod OS=Linux ARCH=x86_64 \ + bash -c 'source "'"$LIB_DIR"'/common.sh"; source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null; telemetry_rerun_handoff; telemetry_render_event 2' + [ "$(attr "$output" 'event.name')" = "install.run.cancelled" ] || return 1 +} + +@test "every event name the renderer produces is one it declares" { + # §6.2: the set of names a service can emit must be finite and enumerable. The + # renderer's answer is checked against TB_TELEMETRY_EVENT_NAMES before it is + # written, and an unregistered one DROPS the record rather than opening a + # namespace of its own — the same rule §3.2 applies to the environment. + local code ev + for code in 0 1 2 42 130 143; do + ev="$(attr "$(telemetry_render_event "$code")" 'event.name')" + [ -n "$ev" ] || { printf 'exit %s rendered no event.name\n' "$code" >&2; return 1; } + _telemetry_in_set "$ev" "$TB_TELEMETRY_EVENT_NAMES" || { + printf 'exit %s rendered undeclared name %s\n' "$code" "$ev" >&2; return 1 + } + done + # And the guard is not vacuous: an undeclared name must be refused, not filed. + TB_TELEMETRY_EVENT_NAMES="install.run.succeeded" + run telemetry_render_event 1 + [ "$status" -ne 0 ] || { printf 'an undeclared event name was rendered anyway\n' >&2; return 1; } + [ -z "$output" ] || return 1 +} + +@test "a cancel carries no error.type; a failure must (contract §8.4)" { + run telemetry_render_event 130 + [[ "$output" != *'"error.type"'* ]] || return 1 + + run telemetry_render_event 1 + [[ "$output" == *'"error.type"'* ]] || return 1 +} + +@test "a failure's error.type comes from the phase and the client state" { + TB_TELEMETRY_PHASE=prerequisites + run telemetry_render_event 1 + [ "$(attr "$output" 'error.type')" = "prerequisites_failed" ] || return 1 + + # A readiness diagnosis wins over the phase: it names the actual fault, where + # the phase only names where the run stopped. + TB_TELEMETRY_PHASE=connect + CLIENT_STATE=image_pull_ca + run telemetry_render_event 1 + [ "$(attr "$output" 'error.type')" = "image_pull_untrusted_ca" ] || return 1 +} + +@test "an unregistered CLIENT_STATE or phase is dropped, not passed through" { + # CLIENT_STATE and TB_TELEMETRY_PHASE are shell variables in a script that + # sources sixteen files, and a shell variable is not a closed set until + # something closes it. + # + # Both subjects are SHAPE-SAFE on purpose. A value with a '/' in it is refused + # by _telemetry_attr's token shape whatever the vocabulary does, so testing + # with a path proves only the shape guard — under the mutation that deletes + # the set-membership check entirely, that version of this test stayed green. + # `degraded` and `verifying` are exactly what a future edit to summary.sh or + # install-k8s.sh would produce, and they are what must not appear. + CLIENT_STATE="degraded" + TB_TELEMETRY_PHASE="verifying" + run telemetry_render_event 1 + [ "$status" -eq 0 ] || return 1 + [[ "$output" != *"degraded"* ]] || return 1 + [[ "$output" != *"verifying"* ]] || return 1 + [[ "$output" != *'"tracebloc.install.client_state"'* ]] || return 1 + # The phase is REPORTED as unknown rather than omitted: "we saw a phase we + # cannot name" has to stay countable, or the drift is invisible. + [ "$(attr "$output" 'tracebloc.install.phase')" = "unknown" ] || return 1 + # …and the record still classifies rather than losing error.type entirely. + [ "$(attr "$output" 'error.type')" = "unclassified" ] || return 1 + + # The other half: registered values must still survive, or every assertion + # above is satisfied by a function that drops everything. + CLIENT_STATE="crash" + TB_TELEMETRY_PHASE="helm" + run telemetry_render_event 1 + [ "$(attr "$output" 'tracebloc.install.client_state')" = "crash" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.phase')" = "helm" ] || return 1 +} + +# ── the privacy boundary ───────────────────────────────────────────────────── + +@test "no path, credential or hostname can reach the record on ANY input" { + # Every string the installer holds that a record could plausibly pick up, set + # to something that would be a disclosure. This is the ticket's hard boundary, + # asserted over what the code renders rather than over a list of forbidden keys. + HOST_DATA_DIR="/Users/$CANARY/.tracebloc" + CLUSTER_NAME="$CANARY-cluster" + TB_NAMESPACE="$CANARY" + HTTPS_PROXY="http://$CANARY:hunter2@proxy.hospital.internal:3128" + HTTP_PROXY="$HTTPS_PROXY" + NO_PROXY="$CANARY.internal" + TB_VERSION="v1.9.3-$CANARY" + TB_ERR_LOC="/var/folders/qx/$CANARY/T/tmp.9k/scripts/lib/setup-linux.sh:412" + # TB_ERR_CMD is what common.sh's ERR trap records: the failing command, + # UNEXPANDED. It is still free text and still carries a path, which is why it + # is never emitted. (Deliberately not written as `curl -u user:pass` here — + # that spelling is a real credential pattern and gitleaks is right to flag it + # in a source file, canary or not. The credential half of this test is carried + # by HTTPS_PROXY above.) + TB_ERR_CMD="install_client_helm --values /Users/$CANARY/values.yaml" + CLIENT_STATE="$CANARY" + TB_TELEMETRY_PHASE="$CANARY" + USER="$CANARY" + HOSTNAME="$CANARY-macbook" + + run telemetry_render_event 1 + [ "$status" -eq 0 ] || return 1 + # The anchor: an empty render would pass "the canary is absent" trivially. + [[ "$output" == *'"event.name":"install.run.failed"'* ]] || return 1 + [[ "$output" != *"$CANARY"* ]] || return 1 + [[ "$output" != *"hunter2"* ]] || return 1 + [[ "$output" != *"hospital.internal"* ]] || return 1 + [[ "$output" != *"/var/folders"* ]] || return 1 +} + +@test "a value with an embedded newline is refused, not split across two records" { + # The one input shape the "nowhere for a path to go" argument does not cover: + # what lands is not a path, it is a SECOND LINE. `grep -qE` matched line by + # line, so the shape checks read the first line, said yes, and wrote the whole + # thing — forging an attribute and splitting one record across two lines of a + # `.jsonl` spool, which #1906's forwarder would read as two malformed events. + # (saadqbal on client#747; reproduced on all four checks before fixing.) + local nl=$'\n' + + # 1. the resource layer, through TB_VERSION and _telemetry_version. + TB_VERSION="v1.9.3${nl}\",\"tracebloc.install.injected\":\"yes" + run telemetry_render_event 0 + [ "$status" -eq 0 ] || return 1 + [[ "$output" != *"injected"* ]] || return 1 + [ "$(printf '%s' "$output" | grep -c .)" = "1" ] || { + printf 'the record was split across %s lines\n' "$(printf '%s' "$output" | grep -c .)" >&2 + return 1 + } + [ "$(attr "$output" 'service.version')" = "0.0.0-unknown" ] || return 1 + TB_VERSION=v1.9.3 + + # 2. the attribute writer — the str, int and key shapes, each on its own. + _telemetry_reset + _telemetry_attr "tracebloc.install.phase" "helm${nl}/etc/passwd" + [ -z "$_TB_TELEMETRY_BUF" ] || { printf 'str: %s\n' "$_TB_TELEMETRY_BUF" >&2; return 1; } + _telemetry_reset + _telemetry_attr "tracebloc.install.duration_ms" "12${nl},\"x\":\"y" int + [ -z "$_TB_TELEMETRY_BUF" ] || { printf 'int: %s\n' "$_TB_TELEMETRY_BUF" >&2; return 1; } + _telemetry_reset + _telemetry_attr "event.name${nl}bad key" "ok" + [ -z "$_TB_TELEMETRY_BUF" ] || { printf 'key: %s\n' "$_TB_TELEMETRY_BUF" >&2; return 1; } + + # 3. the source line number. + run _telemetry_source_line "cluster.sh:88${nl}/etc/shadow" + [ "$status" -ne 0 ] || { printf 'source_line admitted: %s\n' "$output" >&2; return 1; } + + # The anchor for all four: the legal spellings must still be ADMITTED, or a + # function that refuses everything passes this test. + _telemetry_reset + _telemetry_attr "tracebloc.install.phase" "helm" + [[ "$_TB_TELEMETRY_BUF" == *'"tracebloc.install.phase":"helm"'* ]] || return 1 + _telemetry_attr "tracebloc.install.duration_ms" "12" int + [[ "$_TB_TELEMETRY_BUF" == *'"tracebloc.install.duration_ms":12'* ]] || return 1 + [ "$(_telemetry_source_line "cluster.sh:88")" = "88" ] || return 1 + run telemetry_render_event 0 + [ "$(attr "$output" 'service.version')" = "v1.9.3" ] || return 1 +} + +@test "every rendered value is an int or a safe token — the derived guard" { + # Not a list of forbidden keys; that agrees with itself and says nothing about + # the twentieth attribute somebody adds. This walks what the code ACTUALLY + # renders and requires every value to be one of the two shapes telemetry.sh + # declares. A free-text channel of any kind fails it, whether or not anyone + # thought to forbid the thing travelling down it. + HOST_DATA_DIR="/Users/$CANARY/.tracebloc" + TB_ERR_LOC="/tmp/$CANARY/scripts/lib/cluster.sh:88" + CLIENT_STATE=crash + TB_TELEMETRY_PHASE=helm + TB_CLI_ON_FRESH_PATH=0 + + run telemetry_render_event 9 + [ "$status" -eq 0 ] || return 1 + + local pairs count=0 + # Flatten "key":value / "key":"value" pairs out of the rendered object. + pairs="$(printf '%s' "$output" | tr ',{}' '\n\n\n' | sed '/^$/d')" + while IFS= read -r pair; do + # `"resource":` and `"attributes":` flatten to a key with an empty value — + # they are the object wrappers, not attributes. + [ -n "$pair" ] || continue + case "$pair" in *:) continue ;; esac + local key="${pair%%:*}" value="${pair#*:}" + key="${key%\"}"; key="${key#\"}" + case "$value" in + \"*\") # a string: must be a safe token + value="${value%\"}"; value="${value#\"}" + printf '%s' "$value" | grep -qE "$TB_TELEMETRY_TOKEN_RE" || { + printf 'value for %s is not a safe token: %s\n' "$key" "$value" >&2 + return 1 + } + ;; + *) # a bare literal: must be an integer + printf '%s' "$value" | grep -qE "$TB_TELEMETRY_INT_RE" || { + printf 'value for %s is neither a quoted token nor an integer: %s\n' "$key" "$value" >&2 + return 1 + } + ;; + esac + printf '%s' "$key" | grep -qE "$TB_TELEMETRY_KEY_RE" || { + printf 'key is not a contract-shaped attribute key: %s\n' "$key" >&2 + return 1 + } + count=$(( count + 1 )) + done <<<"$pairs" + + # An inert loop over an empty payload reads exactly like a clean sweep. + [ "$count" -ge 12 ] || { printf 'only %s pairs inspected\n' "$count" >&2; return 1; } +} + +@test "_telemetry_attr drops an unsafe value rather than trimming it" { + # Dropping, not repairing, is the design: a value that had to be cleaned up to + # be safe is a value we did not understand, and shipping our guess about it is + # how a redactor leaks. + _telemetry_reset + _telemetry_attr "tracebloc.install.phase" "/etc/$CANARY/rows.csv" + [ -z "$_TB_TELEMETRY_BUF" ] || return 1 + + _telemetry_reset + _telemetry_attr "tracebloc.install.exit_code" "not-a-number" int + [ -z "$_TB_TELEMETRY_BUF" ] || return 1 + + # A key that is not contract-shaped is refused too (§1.1). + _telemetry_reset + _telemetry_attr "podName" "ok" + [ -z "$_TB_TELEMETRY_BUF" ] || return 1 + + # …and a legitimate pair still lands, or every assertion above is vacuous. + _telemetry_reset + _telemetry_attr "tracebloc.install.phase" "helm" + [ "$_TB_TELEMETRY_BUF" = '"tracebloc.install.phase":"helm"' ] || return 1 +} + +@test "the source is a basename from the installer's own scripts, never the path" { + TB_ERR_LOC="/var/folders/qx/$CANARY/T/scripts/lib/setup-linux.sh:412" + run telemetry_render_event 1 + [ "$(attr "$output" 'tracebloc.install.source')" = "setup-linux.sh" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.source_line')" = "412" ] || return 1 + [[ "$output" != *"$CANARY"* ]] || return 1 + + # A file that is not one of ours is dropped, not reported. + TB_ERR_LOC="/home/$CANARY/evil.sh:9" + run telemetry_render_event 1 + [[ "$output" != *'"tracebloc.install.source"'* ]] || return 1 + [[ "$output" != *"evil.sh"* ]] || return 1 + # …and the LINE goes with it. This assertion was the gap: the two halves were + # gated independently, so this same input emitted `"…source_line":9` on its own. + [[ "$output" != *'"tracebloc.install.source_line"'* ]] || return 1 +} + +@test "a line number is never emitted without the file it belongs to (Bugbot, client#747)" { + # `tracebloc.install.source` and `tracebloc.install.source_line` were two + # independent gates over ONE fact. A location whose basename is not one of ours + # dropped the file and kept the number, and a line number with no file is worse + # than no field at all: it looks like information, groups like information, and + # points at line 118 of nothing. + # + # The inputs are written down HERE, independently of the matcher, and the + # expectation for each is derived by asking TB_TELEMETRY_SOURCES — the closed + # vocabulary that is the declaration — rather than by restating which of these + # is ours. Get the pairing wrong in either direction and this reddens. + local loc base expect_source + for loc in \ + "/var/folders/qx/T/scripts/lib/cluster.sh:88" \ + "/opt/tracebloc/lib/install-cli.sh:7" \ + "/home/someone/evil.sh:9" \ + "/usr/lib/node_modules/npm/bin/npm-cli.js:60" \ + "?:118" \ + "install-k8s.sh:1" + do + TB_ERR_LOC="$loc" + base="${loc%%:*}"; base="${base##*/}" + if _telemetry_in_set "$base" "$TB_TELEMETRY_SOURCES"; then + expect_source="$base" + else + expect_source="" + fi + + run telemetry_render_event 1 + [ "$status" -eq 0 ] || { printf 'render died on %s: %s\n' "$loc" "$output" >&2; return 1; } + + # The anchor: whatever we conclude about the pair, this is still a failure + # event carrying an error.type. "No source keys" must not be reachable by + # having quietly lost the whole event. + [[ "$output" == *'"event.name":"install.run.failed"'* ]] || { + printf 'not a failure event for %s: %s\n' "$loc" "$output" >&2; return 1 + } + [[ "$output" == *'"error.type"'* ]] || { + printf 'no error.type for %s\n' "$loc" >&2; return 1 + } + + if [ -n "$expect_source" ]; then + [ "$(attr "$output" 'tracebloc.install.source')" = "$expect_source" ] || { + printf 'declared source %s not reported for %s: %s\n' "$expect_source" "$loc" "$output" >&2 + return 1 + } + [ "$(attr "$output" 'tracebloc.install.source_line')" = "${loc##*:}" ] || { + printf 'source without its line for %s: %s\n' "$loc" "$output" >&2; return 1 + } + else + [[ "$output" != *'"tracebloc.install.source"'* ]] || { + printf 'undeclared source %s reached the record: %s\n' "$base" "$output" >&2; return 1 + } + [[ "$output" != *'"tracebloc.install.source_line"'* ]] || { + printf 'ORPHAN LINE: %s has no source but its line was emitted: %s\n' "$loc" "$output" >&2 + return 1 + } + fi + done + + # The loop above is only worth anything if at least one input took each branch — + # six undeclared paths and no declared one would pass every assertion while + # proving only half the rule. + TB_ERR_LOC="/x/cluster.sh:88" + run telemetry_render_event 1 + [[ "$output" == *'"tracebloc.install.source_line":88'* ]] || { + printf 'ANCHOR: the positive case does not emit a line at all, so the negative cases prove nothing\n' >&2 + return 1 + } +} + +# ── the three field failures this ticket names ─────────────────────────────── + +@test "the #736 PATH case is a number: cli_on_path rides every event" { + # install-cli.sh already computes this — whether a FRESH login shell resolves + # `tracebloc` — and until now only ever printed advice about it. + TB_CLI_ON_FRESH_PATH=0 + run telemetry_render_event 0 + [ "$(attr "$output" 'tracebloc.install.cli_on_path')" = "0" ] || return 1 + + TB_CLI_ON_FRESH_PATH=1 + run telemetry_render_event 0 + [ "$(attr "$output" 'tracebloc.install.cli_on_path')" = "1" ] || return 1 + + # Unset (the CLI step was skipped) omits the key rather than guessing a 0, + # which would be indistinguishable from "installed, and unreachable". + unset TB_CLI_ON_FRESH_PATH + run telemetry_render_event 0 + [[ "$output" != *'"tracebloc.install.cli_on_path"'* ]] || return 1 +} + +@test "a slow phase is visible on a run that SUCCEEDED (the dpkg-lock shape)" { + # apt-get blocked on unattended-upgrades produces no non-zero exit at all — it + # just takes twenty minutes. Per-phase durations are the only thing that makes + # it visible, so they must ride the success event too. + step_header a "Checking your machine" >/dev/null + _TB_TELEMETRY_PHASE_STARTED_MS=$(( $(_telemetry_now_ms) - 1320000 )) + step_header b "Installing what tracebloc needs" >/dev/null + + run telemetry_render_event 0 + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *'"event.name":"install.run.succeeded"'* ]] || return 1 + local ms + ms="$(attr "$output" 'tracebloc.install.phase_preflight_ms')" + [ -n "$ms" ] || { printf 'no preflight duration on a success event\n' >&2; return 1; } + [ "$ms" -ge 1320000 ] || { printf 'preflight recorded %s ms, expected >= 1320000\n' "$ms" >&2; return 1; } +} + +@test "the STILL-RUNNING phase has a duration (Bugbot, client#747)" { + # The test above only ever measured a phase that a LATER step_header had + # closed. telemetry_phase_begin is the only writer, so the phase still running + # when the event fires had nothing recorded against it — and that is the one + # that matters: + # * the dpkg-lock case in its likeliest real form is stuck twenty minutes in + # `prerequisites` and then killed or given up on, never reaching step c; + # * every SUCCESSFUL install ends in `connect`, the readiness wait, up to + # READY_TIMEOUT (600s) — the single longest phase, and it had no key at all. + step_header a "x" >/dev/null + step_header b "x" >/dev/null + _TB_TELEMETRY_PHASE_STARTED_MS=$(( $(_telemetry_now_ms) - 1320000 )) + + # Cancelled while stuck in prerequisites. + run telemetry_render_event 130 + [ "$status" -eq 0 ] || return 1 + [ "$(attr "$output" 'tracebloc.install.phase')" = "prerequisites" ] || return 1 + local ms + ms="$(attr "$output" 'tracebloc.install.phase_prerequisites_ms')" + [ -n "$ms" ] || { printf 'the active phase had no duration\n' >&2; return 1; } + [ "$ms" -ge 1320000 ] || { printf 'active phase recorded %s ms\n' "$ms" >&2; return 1; } + + # And the success case: connect is always the active phase at emit time. + step_header f "x" >/dev/null + _TB_TELEMETRY_PHASE_STARTED_MS=$(( $(_telemetry_now_ms) - 400000 )) + run telemetry_render_event 0 + ms="$(attr "$output" 'tracebloc.install.phase_connect_ms')" + [ -n "$ms" ] || { printf 'a successful install had no connect duration\n' >&2; return 1; } + [ "$ms" -ge 400000 ] || return 1 +} + +@test "bootstrap time is attributed, not left as an unnamed remainder" { + # `bootstrap` has no step letter, so iterating the letter map skipped it and the + # download + verify + leftover-guard + assess time became a remainder nobody + # could name — which is also why subtracting the other keys from duration_ms + # could not recover the missing active phase. + TB_TELEMETRY_STARTED_MS=$(( $(_telemetry_now_ms) - 60000 )) + _TB_TELEMETRY_PHASE_STARTED_MS="$TB_TELEMETRY_STARTED_MS" + run telemetry_render_event 1 + local ms + ms="$(attr "$output" 'tracebloc.install.phase_bootstrap_ms')" + [ -n "$ms" ] || { printf 'bootstrap time is unattributed\n' >&2; return 1; } + [ "$ms" -ge 60000 ] || { printf 'bootstrap recorded %s ms\n' "$ms" >&2; return 1; } +} + +@test "the phase durations sum EXACTLY to duration_ms" { + # The invariant that makes the numbers trustworthy rather than merely present: + # every millisecond of the run is attributed to exactly one phase. A phase whose + # time vanishes — the Bugbot bug above — breaks it, and so does double-counting. + # + # Driven by a FAKE CLOCK, because the obvious fixture is wrong: winding + # _TB_TELEMETRY_PHASE_STARTED_MS backwards after the step_headers have already + # attributed that time invents milliseconds that never elapsed, and the first + # version of this test failed for exactly that reason (sum 1200000 vs total + # 900000). Overriding the clock is the only way to build a fixture the + # accounting can actually be true of. + local T=1000000 + _telemetry_now_ms() { echo "$T"; } + TB_TELEMETRY_STARTED_MS="$T" + _TB_TELEMETRY_PHASE_STARTED_MS="$T" + + T=$(( T + 5000 )); step_header a "x" >/dev/null # bootstrap 5 s + T=$(( T + 20000 )); step_header b "x" >/dev/null # preflight 20 s + T=$(( T + 1320000 )); step_header e "x" >/dev/null # prerequisites 22 min + T=$(( T + 300000 )) # helm still running, 5 min + # c and d were never entered, so they must carry no key at all. + + run telemetry_render_event 1 + [ "$status" -eq 0 ] || return 1 + + local total sum=0 name ms counted=0 + total="$(attr "$output" 'tracebloc.install.duration_ms')" + [ "$total" = "1645000" ] || { printf 'duration_ms=%s, want 1645000\n' "$total" >&2; return 1; } + # DERIVED: iterate the production phase-name list, not a list written here. + for name in $(_telemetry_phase_names); do + ms="$(attr "$output" "tracebloc.install.phase_${name}_ms")" + [ -n "$ms" ] || continue + sum=$(( sum + ms )) + counted=$(( counted + 1 )) + done + # Anchor: an inert loop finding no keys would sum to 0 and could still agree + # with a 0-length run's total. + [ "$counted" = "4" ] || { printf 'found %s phase keys, want 4\n' "$counted" >&2; return 1; } + [ "$sum" = "$total" ] || { + printf 'phases sum to %s but duration_ms is %s\n' "$sum" "$total" >&2; return 1 + } + # The individual attributions, so a compensating pair of errors cannot pass. + [ "$(attr "$output" 'tracebloc.install.phase_bootstrap_ms')" = "5000" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.phase_preflight_ms')" = "20000" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.phase_prerequisites_ms')" = "1320000" ] || return 1 + [ "$(attr "$output" 'tracebloc.install.phase_helm_ms')" = "300000" ] || return 1 + # A phase never entered carries no key (§1.2 omits an absent value). + [[ "$output" != *'"tracebloc.install.phase_cluster_ms"'* ]] || return 1 + [[ "$output" != *'"tracebloc.install.phase_register_ms"'* ]] || return 1 +} + +@test "step_header is what drives the phase clock, so no step can be forgotten" { + [ "$TB_TELEMETRY_PHASE" = "bootstrap" ] || return 1 + step_header c "Creating your secure environment" >/dev/null + [ "$TB_TELEMETRY_PHASE" = "cluster" ] || return 1 + step_header f "Connecting to the tracebloc network" >/dev/null + [ "$TB_TELEMETRY_PHASE" = "connect" ] || return 1 + + # A letter the map has never seen becomes `unknown` — a countable finding, + # not a guessed phase name. + step_header z "Something new" >/dev/null + [ "$TB_TELEMETRY_PHASE" = "unknown" ] || return 1 +} + +# ── environment ────────────────────────────────────────────────────────────── + +@test "an unrecognised CLIENT_ENV renders nothing at all (contract §3.2)" { + # `staging` is the classic near miss: it is the git branch name, and `stg` is + # the environment value. tb_client_env reduces the documented alias… + CLIENT_ENV=staging + run telemetry_render_event 0 + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *'"deployment.environment":"stg"'* ]] || return 1 + + # …but a value nothing reduces is refused outright rather than filed under a + # guess. A record no query filters on is worse than no record. + CLIENT_ENV=qa + run telemetry_render_event 0 + [ "$status" -ne 0 ] || return 1 + [ -z "$output" ] || return 1 +} + +@test "unset CLIENT_ENV is prod, matching _backend_url's own default" { + unset CLIENT_ENV + run telemetry_render_event 0 + [[ "$output" == *'"deployment.environment":"prod"'* ]] || return 1 +} + +@test "the resource layer carries the registered identity and OTel's own names" { + run telemetry_render_event 0 + [[ "$output" == *'"service.name":"installer"'* ]] || return 1 + [[ "$output" == *'"tracebloc.component":"install"'* ]] || return 1 + [[ "$output" == *'"os.type":"linux"'* ]] || return 1 + [[ "$output" == *'"host.arch":"amd64"'* ]] || return 1 + [[ "$output" == *'"service.version":"v1.9.3"'* ]] || return 1 + + # §4 — unknown is a VALUE, not an omission: queryable and alertable, where an + # absent key is neither. + unset TB_VERSION + run telemetry_render_event 0 + [[ "$output" == *'"service.version":"0.0.0-unknown"'* ]] || return 1 +} + +@test "service.instance.id is per-process and is not the hostname" { + # §7.3 forbids a person's name outright, and field hostnames here are + # overwhelmingly "-macbook". + HOSTNAME="$CANARY-macbook" + local a b + a="$(_telemetry_instance_id)" + b="$(_telemetry_instance_id)" + [ "$a" = "$b" ] || return 1 # stable within the process + [ "${#a}" -eq 16 ] || return 1 + [[ "$a" != *"$CANARY"* ]] || return 1 + printf '%s' "$a" | grep -qE '^[a-f0-9]{16}$' || return 1 +} + +# ── opt-out and delivery ───────────────────────────────────────────────────── + +@test "opt-out stops emission entirely, and only the opt-out spellings do" { + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + for var in $TB_TELEMETRY_OPT_OUT_VARS; do + for value in 1 true yes please; do + _TB_TELEMETRY_EMITTED="" + rm -f "$spool" + eval "export $var=$value" + telemetry_emit_outcome 0 + eval "unset $var" + [ ! -s "$spool" ] || { printf '%s=%s still emitted\n' "$var" "$value" >&2; return 1; } + done + done + + # The anchor for the loop above: if telemetry_enabled returned false + # unconditionally the whole feature would be dead and every case would pass. + for value in "" 0 false FALSE; do + _TB_TELEMETRY_EMITTED="" + rm -f "$spool" + export TRACEBLOC_NO_TELEMETRY="$value" + telemetry_emit_outcome 0 + unset TRACEBLOC_NO_TELEMETRY + [ -s "$spool" ] || { printf '%q disabled telemetry; only an opt-out should\n' "$value" >&2; return 1; } + done +} + +@test "exactly one event per install, however many times the trap fires" { + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + telemetry_emit_outcome 0 + telemetry_emit_outcome 1 + telemetry_emit_outcome 130 + [ "$(grep -c . "$spool")" = "1" ] || return 1 + grep -q '"event.name":"install.run.succeeded"' "$spool" || return 1 +} + +@test "the spool is bounded, 0600, and in a 0700 directory" { + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + TB_TELEMETRY_SPOOL_MAX=3 + local i + for i in 1 2 3 4 5 6; do + _TB_TELEMETRY_EMITTED="" + telemetry_emit_outcome "$i" + done + [ "$(grep -c . "$spool")" = "3" ] || return 1 + [ "$(_perm_of "$spool")" = "600" ] || return 1 + [ "$(_perm_of "$(dirname "$spool")")" = "700" ] || return 1 +} + +@test "the spool is 0600 under the umask the installer can actually be holding" { + # The test above could not see the defect it was written to pin, and the reason + # is in this file: load_lib sources common.sh, which sets `umask 077`, so every + # test ran under a umask that made the mode right by accident. The trim + # (`tail > "${spool}.tmp"` then `mv`) creates under the PROCESS umask and mv + # keeps the tmp file's mode, so a chmod that ran only before it left a 0644 + # spool the moment the umask was anything else — and _install_userspace_tools + # (setup-linux.sh:893) and its macOS twin (setup-macos.sh:418) set `umask 022` + # around install_{kubectl,k3d,helm} and restore it only afterwards, so an + # install that dies in one of those emits from the EXIT trap under 022. + # (saadqbal on client#747; reproduced — spool 644.) + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" saved + saved="$(umask)" + umask 022 + # Assert the mutation anchor applied: if common.sh has been re-sourced or the + # umask did not take, this test is measuring 077 again and proves nothing. + [ "$(umask)" = "0022" ] || { umask "$saved"; printf 'umask did not take\n' >&2; return 1; } + TB_TELEMETRY_SPOOL_MAX=2 + local i + for i in 1 2 3; do + _TB_TELEMETRY_EMITTED="" + telemetry_emit_outcome "$i" + done + local mode dir_mode lines + mode="$(_perm_of "$spool")" + dir_mode="$(_perm_of "$(dirname "$spool")")" + lines="$(grep -c . "$spool")" + umask "$saved" + [ "$mode" = "600" ] || { printf 'spool is %s under umask 022, not 600\n' "$mode" >&2; return 1; } + [ "$dir_mode" = "700" ] || return 1 + # The trim must still have run — a spool that was never rewritten would keep + # its 600 for the wrong reason and this test would pass vacuously. + [ "$lines" = "2" ] || { printf 'the trim did not run (%s lines)\n' "$lines" >&2; return 1; } +} + +@test "install_cleanup emits the outcome on every path, including a cancel" { + # The EXIT trap is where "a terminal event on every path" (§6.5) is actually + # honoured — a failure that exits under errexit never reaches any other line. + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + CLIENT_STATE="" + run bash -c ' + set -euo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null + HOST_DATA_DIR="'"$HOST_DATA_DIR"'" + CLIENT_ENV=prod + telemetry_run_started # main() does this once the run is committed + trap install_cleanup EXIT + exit 130 + ' + [ "$status" -eq 130 ] || return 1 + grep -q '"event.name":"install.run.cancelled"' "$spool" || return 1 +} + +@test "an ordinary tool's status 2 lands in the numerator, under the real trap" { + # THE REPRODUCTION, as a test. A bare `grep` on an unreadable file exits 2 — its + # own file-error status, nothing to do with the installer's sentinel — and under + # `set -e` that becomes install-k8s.sh's exit status. Keyed on the number, this + # spooled install.run.cancelled with no error.type: a hard failure REMOVED from + # the rate rather than misfiled in it. (saadqbal on client#747.) + # + # Driven through install_cleanup with install-k8s.sh's own trap wiring, not + # through telemetry_render_event, because the whole point is what an unmodified + # command failing somewhere in the middle of a real run produces. + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + run bash -c ' + set -euo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null + HOST_DATA_DIR="'"$HOST_DATA_DIR"'" + CLIENT_ENV=prod + telemetry_run_started + trap install_cleanup EXIT + set -E + trap '"'"'_record_err "${BASH_SOURCE[0]:-?}:${LINENO}" "$BASH_COMMAND"'"'"' ERR + telemetry_phase_begin b + grep -q anything /nonexistent/definitely/not/here + ' 3>&- + # The anchor: grep really did supply the 2, so the assertions below are about a + # tool's own status and not about a number this test picked. + [ "$status" -eq 2 ] || { printf 'expected exit 2 from grep, got %s\n' "$status" >&2; return 1; } + grep -q '"event.name":"install.run.failed"' "$spool" || { + printf 'the record was: %s\n' "$(cat "$spool")" >&2; return 1 + } + grep -q '"error.type":"unexpected_exit_2"' "$spool" || return 1 + # …and NOT the phase's ordinary bucket, which would hide it among real ones. + ! grep -q '"error.type":"prerequisites_failed"' "$spool" || return 1 +} + +@test "gpu-nvidia's reboot handoff still renders a cancel, through its real exit" { + # The other direction, and the one that must not regress: the marker has to be + # SET at the site, so this drives install_nvidia_drivers itself rather than + # re-implementing its exit. A mutation that drops the setter call from + # gpu-nvidia.sh has to redden here (workspace CLAUDE.md rule 9); asserting on a + # hand-written `telemetry_rerun_handoff; exit 2` could not see it. + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + run bash -c ' + set -euo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + source "'"$LIB_DIR"'/gpu-nvidia.sh" + LOG_FILE=/dev/null + HOST_DATA_DIR="'"$HOST_DATA_DIR"'" + CLIENT_ENV=prod + telemetry_run_started + trap install_cleanup EXIT + telemetry_phase_begin b + NVIDIA_DRIVER_OK=false + PM_UPDATE=true + PM_INSTALL=true + has() { return 0; } + sudo() { return 0; } + ubuntu-drivers() { return 0; } + TRACEBLOC_SKIP_REBOOT_PROMPT=1 + install_nvidia_drivers + ' 3>&- + [ "$status" -eq 2 ] || { printf 'expected the exit-2 handoff, got %s\n' "$status" >&2; return 1; } + grep -q '"event.name":"install.run.cancelled"' "$spool" || { + printf 'the record was: %s\n' "$(cat "$spool")" >&2; return 1 + } + ! grep -q '"error.type"' "$spool" || return 1 +} + +@test "a --help run installs nothing and must emit nothing (Bugbot, client#747)" { + # install_cleanup is the EXIT trap, so it fires for every exit of + # install-k8s.sh — including the terminal commands that touch no machine. + # `--help` was emitting a full install.run.succeeded: a free success in the + # denominator of the exact failure RATE this feature exists to produce, and + # the command people run most while a real install is broken. + # + # Driven through the REAL entrypoint, not the helper, because the bug was in + # which exits reach the trap — a unit test of telemetry_emit_outcome cannot + # see it, and none of the twenty tests above did. + local spool + for flag in --help -h; do + local dir="$BATS_TEST_TMPDIR/help-${flag#--}" + spool="$dir/telemetry/pending.jsonl" + run env HOST_DATA_DIR="$dir" CLIENT_ENV=prod \ + bash "$SCRIPTS_DIR/install-k8s.sh" "$flag" + [ "$status" -eq 0 ] || return 1 + # The anchor: prove --help actually ran, so an empty spool means "emitted + # nothing" and not "the command never started". + [[ "$output" == *"tracebloc"* ]] || return 1 + [ ! -s "$spool" ] || { + printf '%s emitted an event: %s\n' "$flag" "$(cat "$spool")" >&2 + return 1 + } + done +} + +@test "the already-set-up handoff is skipped, not succeeded" { + # The stop-and-check gate exits 0 having run no step. Counting that as a + # successful install makes the success count grow with re-runs on machines + # nothing happened to, and the failure rate fall for an unrelated reason. + # `skipped` is a registered outcome verb (§6.4), so this needs no new vocabulary. + telemetry_run_skipped + run telemetry_render_event 0 + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *'"event.name":"install.run.skipped"'* ]] || return 1 + + # …and a run that did NOT skip still reports success, or the branch above is + # simply relabelling every success. + _TB_TELEMETRY_SKIPPED="" + run telemetry_render_event 0 + [[ "$output" == *'"event.name":"install.run.succeeded"'* ]] || return 1 +} + +@test "Ctrl-C on the already-set-up screen is skipped, not cancelled (Bugbot, client#747)" { + # _assess_handoff marks the run skipped and THEN hands the user the interactive + # `tracebloc` home screen before `exit 0`, with install-k8s.sh's + # `trap 'exit 130' INT` live. Ctrl-C there — the most ordinary thing a user does + # on that screen — took the signal branch and booked `cancelled` for a run that + # installed nothing. `cancelled` asserts an install was cancelled; there was no + # install to cancel. It would inflate the "installs that did not complete" + # denominator with runs that never attempted anything. + local sig + for sig in 130 143; do + _TB_TELEMETRY_SKIPPED="" + telemetry_run_skipped + run telemetry_render_event "$sig" + [ "$status" -eq 0 ] || { printf 'render died on %s\n' "$sig" >&2; return 1; } + [[ "$output" == *'"event.name":"install.run.skipped"'* ]] || { + printf 'exit %s on a skipped run booked: %s\n' "$sig" \ + "$(attr "$output" 'event.name')" >&2 + return 1 + } + # The exit code still rides along, so the two remain separable downstream. + [ "$(attr "$output" 'tracebloc.install.exit_code')" = "$sig" ] || return 1 + done + + # THE POSITIVE CONTROL, and it is the half that makes the above mean anything: a + # genuine mid-install Ctrl-C — nothing skipped — must STILL book cancelled. + # Without this, "renders skipped" is satisfied by a change that never renders + # cancelled at all, which would delete the interrupted-install signal entirely. + for sig in 130 143; do + _TB_TELEMETRY_SKIPPED="" + run telemetry_render_event "$sig" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *'"event.name":"install.run.cancelled"'* ]] || { + printf 'a real interrupted install at %s no longer books cancelled: %s\n' \ + "$sig" "$(attr "$output" 'event.name')" >&2 + return 1 + } + done + + # …and the asymmetry is deliberate: skipped must NOT swallow a real failure. A + # skipped run that then dies with an ordinary non-zero is still a failure, so + # the flag is consulted on 0/130/143 and nowhere else. + _TB_TELEMETRY_SKIPPED="" + telemetry_run_skipped + run telemetry_render_event 1 + [[ "$output" == *'"event.name":"install.run.failed"'* ]] || { + printf 'a genuine failure on a skipped run was hidden as: %s\n' \ + "$(attr "$output" 'event.name')" >&2 + return 1 + } + [[ "$output" == *'"error.type"'* ]] || return 1 +} + +@test "a committed run still emits once the latch is set" { + # The anchor for the --help test: if the latch were never set, the whole + # feature would be dead and "--help emits nothing" would pass trivially. + local spool="$HOST_DATA_DIR/telemetry/pending.jsonl" + _TB_TELEMETRY_EMITTED=""; _TB_TELEMETRY_RUN_STARTED="" + telemetry_emit_outcome 0 + [ ! -s "$spool" ] || { printf 'emitted with no latch\n' >&2; return 1; } + + _TB_TELEMETRY_EMITTED="" + telemetry_run_started + telemetry_emit_outcome 0 + [ -s "$spool" ] || { printf 'the latch did not enable emission\n' >&2; return 1; } +} + +@test "a real run past the terminal commands DOES emit (the latch is wired)" { + # The anchor for the --help test, and it must go through the REAL entrypoint: + # deleting main()'s telemetry_run_started call kills the whole feature, and a + # unit test that sets the latch itself cannot notice. Nothing did, until this. + # + # HOST_DATA_DIR == $HOME is rejected by validate_config, which is AFTER the + # latch and BEFORE step a — so this also pins the case the latch had to + # preserve: a genuine failure in the bootstrap phase is still an install + # attempt and must still be reported. + local dir="$BATS_TEST_TMPDIR/real" + mkdir -p "$dir" + local spool="$dir/telemetry/pending.jsonl" + run env HOME="$dir" HOST_DATA_DIR="$dir" CLIENT_ENV=prod \ + TRACEBLOC_SKIP_LEFTOVER_GUARD=1 \ + bash "$SCRIPTS_DIR/install-k8s.sh" + [ "$status" -ne 0 ] || return 1 + [ -s "$spool" ] || { printf 'a committed run emitted nothing\n' >&2; return 1; } + [ "$(grep -c . "$spool")" = "1" ] || return 1 + grep -q '"event.name":"install.run.failed"' "$spool" || { + printf 'wrong event: %s\n' "$(cat "$spool")" >&2; return 1 + } + grep -q '"tracebloc.install.phase":"bootstrap"' "$spool" || return 1 + grep -q '"error.type":"bootstrap_failed"' "$spool" || return 1 +} + +@test "_assess_handoff itself marks the run skipped (the wiring, not the flag)" { + # Setting _TB_TELEMETRY_SKIPPED by hand proves the render branch and says + # nothing about whether anything calls it — deleting assess.sh's call reddened + # no test. Drive the real handoff, with `tracebloc` mocked so it exits instead + # of rendering a home screen. + local dir="$BATS_TEST_TMPDIR/handoff" + mkdir -p "$dir/bin" + printf '#!/usr/bin/env bash\nexit 0\n' > "$dir/bin/tracebloc" + chmod +x "$dir/bin/tracebloc" + local spool="$dir/telemetry/pending.jsonl" + + run bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + source "'"$LIB_DIR"'/assess.sh" + LOG_FILE=/dev/null + HOME="'"$dir"'"; HOST_DATA_DIR="'"$dir"'"; CLIENT_ENV=prod + PATH="'"$dir"'/bin:$PATH"; TB_TTY=/dev/null + telemetry_run_started # main() has committed to the run + trap install_cleanup EXIT + _assess_handoff + ' + [ "$status" -eq 0 ] || { printf 'handoff exited %s: %s\n' "$status" "$output" >&2; return 1; } + [ -s "$spool" ] || { printf 'the handoff emitted nothing\n' >&2; return 1; } + grep -q '"event.name":"install.run.skipped"' "$spool" || { + printf 'wrong event: %s\n' "$(cat "$spool")" >&2; return 1 + } +} + +@test "an unrecognised source location drops the field, never the event" { + # Bugbot (client#747) predicted the opposite: that the unprotected command + # substitutions feeding tracebloc.install.source would abort telemetry_render_event + # under `set -e` and take the whole outcome with them. That mechanism does not + # exist — a command substitution in an ARGUMENT position does not propagate its + # status to the enclosing command, verified on bash 3.2 (the system bash on + # macOS) and 5.x. The INVARIANT is worth pinning anyway: nothing about a + # location we cannot classify should cost us the event. + run bash -c ' + set -euo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null + HOST_DATA_DIR="'"$BATS_TEST_TMPDIR"'/src" + CLIENT_ENV=prod + TB_ERR_LOC="/home/someone/not-ours.sh:9" + telemetry_render_event 1 + ' + [ "$status" -eq 0 ] || { printf 'render died: %s\n' "$output" >&2; return 1; } + [[ "$output" == *'"event.name":"install.run.failed"'* ]] || return 1 + [[ "$output" == *'"error.type"'* ]] || return 1 + [[ "$output" != *'"tracebloc.install.source"'* ]] || return 1 + [[ "$output" != *"not-ours.sh"* ]] || return 1 +} + +@test "telemetry never creates HOST_DATA_DIR on a volume the installer refused" { + # early_data_dir_guard refuses a network filesystem BEFORE logging starts, + # because MySQL/InnoDB corrupts on NFS (client#432). It deliberately skips an + # existing directory ("an EXISTING data dir has no at-risk mkdir here", + # client#441) — so anything that creates that directory behind its back + # disarms it for every subsequent run. + # + # _telemetry_deliver did exactly that: it ran from the EXIT trap and mkdir -p'd + # $HOST_DATA_DIR/telemetry unconditionally, including on the path where the + # guard had just called `error`. Run 1 refused and created the dir; run 2 saw + # the dir, returned 0, and installed onto NFS. Found by Bugbot (client#747). + local vol="$BATS_TEST_TMPDIR/nfs-volume" + local target="$vol/.tracebloc" + local tmp="$BATS_TEST_TMPDIR/nfs-tmp" + mkdir -p "$tmp" + # LOG_FILE is deliberately NOT set. early_data_dir_guard runs BEFORE + # setup_log_file (#432 refuses a network data dir before logging starts), so on + # the real path there is no log yet — and an earlier version of this test set + # LOG_FILE=/dev/null, which masked the follow-on bug that the event went + # nowhere at all (Bugbot, client#747, round 4). + run env TMPDIR="$tmp" bash -c ' + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + source "'"$LIB_DIR"'/preflight.sh" + CLIENT_ENV=prod + HOST_DATA_DIR="'"$target"'" + _pf_fstype() { echo nfs; } # the target reads as a network filesystem + telemetry_run_started + trap install_cleanup EXIT + early_data_dir_guard + ' + # The anchor: prove the guard actually refused, so "no directory" cannot mean + # "the guard never ran". + [ "$status" -ne 0 ] || { printf 'the guard did not refuse; fixture is inert\n' >&2; return 1; } + [[ "$output" == *"network filesystem"* ]] || return 1 + + [ ! -d "$target" ] || { + printf 'telemetry created %s on the rejected volume — the guard is disarmed for the next run\n' "$target" >&2 + return 1 + } + + # …and the refusal is still REPORTED. It is a real, actionable field failure, + # and it was the one case that produced no record anywhere — invisible to the + # failure rate this feature exists to produce. + local fallback + fallback="$(ls "$tmp"/tracebloc-telemetry-* 2>/dev/null | head -1)" + [ -n "$fallback" ] || { printf 'the NFS refusal produced no record at all\n' >&2; return 1; } + grep -q '"event.name":"install.run.failed"' "$fallback" || return 1 + grep -q '"error.type":"bootstrap_failed"' "$fallback" || return 1 + [ "$(_perm_of "$fallback")" = "600" ] || return 1 + + # …and the other half: where the data dir legitimately exists, the spool is + # still written, or the fix has simply disabled the feature. + mkdir -p "$target" + run bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null + CLIENT_ENV=prod + HOST_DATA_DIR="'"$target"'" + telemetry_run_started + telemetry_emit_outcome 1 + ' + [ "$status" -eq 0 ] || return 1 + [ -s "$target/telemetry/pending.jsonl" ] || { + printf 'the spool is no longer written even into an existing data dir\n' >&2 + return 1 + } +} + +@test "the pre-log record survives the BOOTSTRAP, not just the process" { + # The test below hands the fallback a TMPDIR of its own, and that is exactly + # the shape that hid this: on the real `curl | bash` path TMPDIR is the + # bootstrap's own scratch dir, and the bootstrap deletes it. install.sh:238 + # does `TMPDIR="$(mktemp -d)"` — a plain assignment to a name that is ALREADY + # EXPORTED (always on macOS) keeps the export attribute — :239 traps + # `rm -rf "$TMPDIR"`, and :571 runs the installer out of it. So every pre-log + # record landed inside the doomed directory and was gone before anyone could + # read it, on the primary macOS path. (saadqbal on client#747; reproduced.) + # + # This test reproduces the bootstrap rather than describing it: it unpacks the + # libs into the scratch dir and sources them from THERE, because "where is the + # installer running from" is what the fix derives its answer from. + local boot="$BATS_TEST_TMPDIR/boot" home="$BATS_TEST_TMPDIR/home" + mkdir -p "$boot" "$home" + run env HOME="$home" TB_BOOT="$boot" bash -c ' + TMPDIR="$(mktemp -d "$TB_BOOT/tb-XXXXXX")" + export TMPDIR + trap '\''rm -rf "$TMPDIR"'\'' EXIT + mkdir -p "$TMPDIR/lib" + cp "'"$LIB_DIR"'"/*.sh "$TMPDIR/lib/" + bash -c '\'' + set -uo pipefail + source "$TMPDIR/lib/common.sh" + source "$TMPDIR/lib/telemetry.sh" + CLIENT_ENV=prod + HOST_DATA_DIR="$TB_BOOT/never-created" + unset LOG_FILE + telemetry_run_started + telemetry_emit_outcome 1 + '\'' + ' + [ "$status" -eq 0 ] || { printf 'the bootstrap stand-in died: %s\n' "$output" >&2; return 1; } + # The scratch dir is gone, as it is on a real run. + [ -z "$(ls "$boot" 2>/dev/null)" ] || { + printf 'the bootstrap stand-in did not clean up, so this proves nothing: %s\n' "$(ls "$boot")" >&2 + return 1 + } + local record + record="$(ls "$home"/tracebloc-telemetry-* 2>/dev/null | head -1)" + [ -n "$record" ] || { printf 'the record did not survive the bootstrap\n' >&2; return 1; } + grep -q '"event.name":"install.run.failed"' "$record" || return 1 + # Still never the data dir the installer refused. + [ ! -d "$boot/never-created" ] || return 1 +} + +@test "a pre-log failure is still reported (no log, no data dir)" { + # validate_config and early_data_dir_guard both run BEFORE setup_log_file, so + # on those paths `log` is a no-op AND (correctly) no data dir exists. Together + # those two facts silently discarded the event. This is the general case; the + # NFS test above is the specific one that made it matter. + # + # NOTE this test gives the fallback a TMPDIR that nothing deletes, so it says + # nothing about the curl|bash path — the test above is the one that does. + local tmp="$BATS_TEST_TMPDIR/prelog" + mkdir -p "$tmp" + run env TMPDIR="$tmp" bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + CLIENT_ENV=prod + HOST_DATA_DIR="'"$BATS_TEST_TMPDIR"'/never-created" + unset LOG_FILE + telemetry_run_started + telemetry_emit_outcome 1 + ' + [ "$status" -eq 0 ] || return 1 + [ ! -d "$BATS_TEST_TMPDIR/never-created" ] || return 1 + local fallback + fallback="$(ls "$tmp"/tracebloc-telemetry-* 2>/dev/null | head -1)" + [ -n "$fallback" ] || { printf 'a pre-log failure produced no record\n' >&2; return 1; } + grep -q '"event.name":"install.run.failed"' "$fallback" || return 1 + + # The fallback must be a FRESH file per run, not a predictable shared path: + # /tmp is world-writable on Linux and the installer runs privileged steps, so a + # fixed name is a symlink target. mktemp creates with O_EXCL. + run env TMPDIR="$tmp" bash -c ' + source "'"$LIB_DIR"'/common.sh"; source "'"$LIB_DIR"'/telemetry.sh" + CLIENT_ENV=prod; HOST_DATA_DIR="'"$BATS_TEST_TMPDIR"'/never-created"; unset LOG_FILE + telemetry_run_started; telemetry_emit_outcome 1 + ' + [ "$(ls "$tmp"/tracebloc-telemetry-* | wc -l | tr -d " ")" = "2" ] || { + printf 'the fallback reuses a predictable path\n' >&2; return 1 + } +} + +@test "a data dir that will not take the write falls through to the fallback (Bugbot, client#747)" { + # The data-dir spool's mkdir and append were both `|| return 0`, so a data dir + # that EXISTS but refuses the write ended the function with the record nowhere. + # That is not an exotic path: HOST_DATA_DIR present but unwritable is precisely + # when _choose_log_file has already fallen back to a mktemp log — and on + # curl|bash that log is inside the bootstrap's own doomed TMPDIR — so the `log` + # line kept nothing either. The fallback existed and was unreachable in exactly + # the case it was built for. + [[ "$(id -u)" -eq 0 ]] && skip "root bypasses filesystem permission bits" + local dd="$BATS_TEST_TMPDIR/ro/.tracebloc" tmp="$BATS_TEST_TMPDIR/ro/tmp" + mkdir -p "$dd" "$tmp" + chmod 500 "$dd" # exists; telemetry/ cannot be created in it + + # ANCHOR 1 — this fixture IS the _choose_log_file fallback case, checked by + # calling _choose_log_file rather than by asserting that it is. + local logf + logf="$(env TMPDIR="$tmp" bash -c ' + source "'"$LIB_DIR"'/common.sh"; HOST_DATA_DIR="'"$dd"'"; _choose_log_file + ' 2>/dev/null)" + case "$logf" in + "$dd"/*) printf 'ANCHOR: the log went into the data dir, so this is not the fallback case\n' >&2 + chmod 700 "$dd"; return 1 ;; + esac + + run env TMPDIR="$tmp" bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + CLIENT_ENV=prod + HOST_DATA_DIR="'"$dd"'" + LOG_FILE="'"$logf"'" + telemetry_run_started + telemetry_emit_outcome 1 + ' + [ "$status" -eq 0 ] || { chmod 700 "$dd"; printf 'emit died: %s\n' "$output" >&2; return 1; } + + # ANCHOR 2 — the mkdir really did fail. Without this, a fix that quietly made + # the directory writable would read identically to a fix that fell through. + [ ! -d "$dd/telemetry" ] || { + chmod 700 "$dd" + printf 'ANCHOR: telemetry/ was created after all, so nothing was diverted\n' >&2 + return 1 + } + + local fb + fb="$(ls "$tmp"/tracebloc-telemetry-* 2>/dev/null | head -1)" + [ -n "$fb" ] || { chmod 700 "$dd"; printf 'the record went nowhere\n' >&2; return 1; } + grep -q '"event.name":"install.run.failed"' "$fb" || { chmod 700 "$dd"; return 1; } + [ "$(_perm_of "$fb")" = "600" ] || { chmod 700 "$dd"; return 1; } + chmod 700 "$dd" + + # The other half of the same hole, and this one is not about permissions at all — + # it holds for root too: telemetry/ exists, but the spool path cannot be appended + # to. The old code returned 0 here as well. + local dd2="$BATS_TEST_TMPDIR/blocked/.tracebloc" tmp2="$BATS_TEST_TMPDIR/blocked/tmp" + mkdir -p "$dd2/telemetry/pending.jsonl" "$tmp2" # a DIRECTORY where the file goes + run env TMPDIR="$tmp2" bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + CLIENT_ENV=prod + HOST_DATA_DIR="'"$dd2"'" + LOG_FILE=/dev/null + telemetry_run_started + telemetry_emit_outcome 1 + ' + [ "$status" -eq 0 ] || { printf 'emit died on the blocked spool: %s\n' "$output" >&2; return 1; } + # ANCHOR — the append could not have succeeded, and the shell's own diagnostic + # about it must not have been printed at the user out of the EXIT trap. + [ -d "$dd2/telemetry/pending.jsonl" ] || { + printf 'ANCHOR: the spool path is no longer a directory, so the append was not blocked\n' >&2 + return 1 + } + [[ "$output" != *"Is a directory"* ]] || { + printf 'the shell leaked the spool path to the user from the trap: %s\n' "$output" >&2; return 1 + } + local fb2 + fb2="$(ls "$tmp2"/tracebloc-telemetry-* 2>/dev/null | head -1)" + [ -n "$fb2" ] || { printf 'a blocked append produced no record\n' >&2; return 1; } + grep -q '"event.name":"install.run.failed"' "$fb2" || return 1 +} + +@test "a spool that DID take the write is not filed twice (Bugbot, client#747)" { + # The other direction of the same fix, and the one that would be invisible: if + # the data-dir branch stopped returning on success, every ordinary install would + # write its outcome to the spool AND to a fallback file, doubling the denominator + # of the failure rate this feature exists to produce and littering TMPDIR on + # every run. + local dd="$BATS_TEST_TMPDIR/ok/.tracebloc" tmp="$BATS_TEST_TMPDIR/ok/tmp" + mkdir -p "$dd" "$tmp" + run env TMPDIR="$tmp" bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + CLIENT_ENV=prod + HOST_DATA_DIR="'"$dd"'" + LOG_FILE=/dev/null + telemetry_run_started + telemetry_emit_outcome 0 + ' + [ "$status" -eq 0 ] || { printf 'emit died: %s\n' "$output" >&2; return 1; } + # ANCHOR — the spool really was written, so "no fallback" cannot mean "nothing + # happened at all". + [ "$(wc -l < "$dd/telemetry/pending.jsonl" | tr -d ' ')" = "1" ] || { + printf 'the data-dir spool does not hold exactly one line: %s\n' \ + "$(cat "$dd/telemetry/pending.jsonl" 2>/dev/null)" >&2 + return 1 + } + [ "$(ls "$tmp"/tracebloc-telemetry-* 2>/dev/null | wc -l | tr -d ' ')" = "0" ] || { + printf 'the event was filed twice — once in the spool and once in the fallback\n' >&2 + return 1 + } + + # A failure AFTER the append must not divert either: the line is already on disk, + # so a broken trim is a bounding problem, never a second row. + # + # `tail` is broken with a shell FUNCTION, not with a PATH shim. The first version + # of this used a shim directory and proved nothing: common.sh:8 does + # `export PATH="/usr/local/sbin:…:/bin:${PATH}"`, which PREPENDS the system + # directories, so a shim prepended by the caller ends up behind /usr/bin and the + # real `tail` ran. Both assertions below passed against a trim that had worked + # perfectly. A function wins over PATH lookup outright and cannot be reordered. + # (Caught by mutating the trim's own cleanup and watching this test stay green.) + local spool="$dd/telemetry/pending.jsonl" + local i + for i in 3 4 5 6 7; do printf 'filler-%s\n' "$i" >> "$spool"; done + run env TMPDIR="$tmp" bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + tail() { return 1; } # beats PATH, whatever common.sh did to it + command -v tail >/dev/null && [ "$(type -t tail)" = "function" ] || { + echo "ANCHOR-TAIL-NOT-OVERRIDDEN"; exit 3; } + CLIENT_ENV=prod + TB_TELEMETRY_SPOOL_MAX=3 + HOST_DATA_DIR="'"$dd"'" + LOG_FILE=/dev/null + telemetry_run_started + telemetry_emit_outcome 1 + ' + [ "$status" -eq 0 ] || { printf 'a failing trim killed the emit: %s\n' "$output" >&2; return 1; } + [[ "$output" != *"ANCHOR-TAIL-NOT-OVERRIDDEN"* ]] || { + printf 'the trim was never actually broken, so this proves nothing\n' >&2; return 1 + } + # ANCHOR — the trim really did fail: with SPOOL_MAX=3 and 7 lines in the spool, a + # working trim leaves 3. Anything else and the fixture is inert. + [ "$(grep -c . "$spool")" = "7" ] || { + printf 'ANCHOR: the spool holds %s lines, so the trim ran after all\n' "$(grep -c . "$spool")" >&2 + return 1 + } + [ "$(ls "$tmp"/tracebloc-telemetry-* 2>/dev/null | wc -l | tr -d ' ')" = "0" ] || { + printf 'a failing trim diverted an already-written record into the fallback\n' >&2; return 1 + } + [ ! -e "${spool}.tmp" ] || { + printf 'the failed trim left its scratch file behind\n' >&2; return 1 + } + + # The trim's OTHER failure exit: tail succeeds, the mv does not. Its `|| rm -f` + # was the one line in this function no mutation could redden, because a fixture + # that breaks the directory cannot let tail write the .tmp in the first place. + # Overriding `mv` is the only way to construct it, so it is constructed rather + # than assumed harmless. + run env TMPDIR="$tmp" bash -c ' + set -uo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + mv() { return 1; } + [ "$(type -t mv)" = "function" ] || { echo "ANCHOR-MV-NOT-OVERRIDDEN"; exit 3; } + CLIENT_ENV=prod + TB_TELEMETRY_SPOOL_MAX=3 + HOST_DATA_DIR="'"$dd"'" + LOG_FILE=/dev/null + telemetry_run_started + telemetry_emit_outcome 1 + ' + [ "$status" -eq 0 ] || { printf 'a failing mv killed the emit: %s\n' "$output" >&2; return 1; } + [[ "$output" != *"ANCHOR-MV-NOT-OVERRIDDEN"* ]] || { + printf 'mv was never overridden, so this proves nothing\n' >&2; return 1 + } + # ANCHOR — the mv really did fail: a working one would have left SPOOL_MAX=3 + # lines. 8 means the trimmed copy never replaced the spool. + [ "$(grep -c . "$spool")" = "8" ] || { + printf 'ANCHOR: the spool holds %s lines, so the mv landed after all\n' "$(grep -c . "$spool")" >&2 + return 1 + } + [ ! -e "${spool}.tmp" ] || { + printf 'a failed mv left the trimmed copy behind\n' >&2; return 1 + } + [ "$(ls "$tmp"/tracebloc-telemetry-* 2>/dev/null | wc -l | tr -d ' ')" = "0" ] || { + printf 'a failing mv diverted an already-written record into the fallback\n' >&2; return 1 + } +} + +@test "nothing here can kill the installer under set -euo pipefail" { + # This class bit twice while writing the file, and both times every unit-level + # test stayed green: + # * `printf ... | grep -q` returns 141 on a MATCH, because grep -q closes the + # pipe (the backend#1778 shape, in summary.sh's own comment); + # * `tr -dc … < /dev/urandom | head -c 16` returns 141 for the same reason — + # at SOURCE time, so the installer died before printing a line. + # The installer runs under `set -euo pipefail` and calls this from an EXIT + # trap, so a non-zero anywhere in here is a fatal that a user would experience + # as the installer vanishing. Exercise the whole surface under those options. + run bash -c ' + set -euo pipefail + source "'"$LIB_DIR"'/common.sh" + source "'"$LIB_DIR"'/telemetry.sh" + LOG_FILE=/dev/null + HOST_DATA_DIR="'"$BATS_TEST_TMPDIR"'/euo" + CLIENT_ENV=prod; OS=Darwin; ARCH=arm64; TB_VERSION=v1.9.3 + telemetry_run_started + step_header a "x" >/dev/null + step_header f "x" >/dev/null + TB_CLI_ON_FRESH_PATH=0 + CLIENT_STATE=crash + TB_ERR_LOC="/tmp/scripts/lib/cluster.sh:88" + telemetry_render_event 0 >/dev/null + telemetry_render_event 1 >/dev/null + telemetry_render_event 130 >/dev/null + telemetry_emit_outcome 1 + echo SURVIVED + ' + [ "$status" -eq 0 ] || { printf "died with %s: %s\n" "$status" "$output" >&2; return 1; } + [[ "$output" == *"SURVIVED"* ]] || return 1 +} + +# _perm_of PATH — octal mode, portable across GNU and BSD stat. +_perm_of() { + stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" 2>/dev/null +}