From 6c962fc335425949ecae760c9389a342fb8002ac Mon Sep 17 00:00:00 2001 From: MulticastEngineer Date: Sat, 12 Sep 2026 10:14:21 +0000 Subject: [PATCH 01/11] BLO-33457: tunnels-ramp mode for the public-vantage probe Adds the ramp that measures the linux relay's control-plane tunnel-table ceiling. `mode` defaults to `oneshot`, so the BLO-21823 probe is unchanged. Target is 69.25.95.128:2268 = Service/blockcastd-amt-relay-linux (LoadBalancer), verified rather than assumed: the other relay -- Service/blockcastd-amt-relay, whose job="production-amt-relay" reports amt_relay_tunnel_limit=0 -- is ClusterIP-only with no public 2268, so a public-vantage ramp cannot accidentally measure the wrong one. Three things this guards, each of which would otherwise produce a confident wrong ceiling: - HOST CEILINGS ARE MEASURED FIRST, not assumed. `ulimit -n` (soft default 1024 on ubuntu-latest, an order of magnitude under AMT_MAX_TUNNELS=10000), the source-port range -- the relay keys tunnels by source (IP, port), so SNAT exhaustion reads exactly like a relay-side knee -- and conntrack. The step FAILS if max N exceeds the available port span, rather than producing a plausible number that is really the runner's limit. - N=1 POSITIVE CONTROL GATES THE RAMP. A relay that answers nothing yields 0 tunnels, 0 drops, no stop rule tripped, and reports "no knee through N=10000" -- maximal capacity for a relay that established nothing. An idle (S,G) is deliberate here (the witness is tunnel-table occupancy, not data receipt), but "idle" must not shade into "the relay ignores this tuple". - PER-STEP UTC TIMESTAMPS. The ramp must run from a public runner while Prometheus is in-cluster and unreachable from it, so AC 2's two halves cannot be captured by one process. The relay-side series is joined afterwards over the recorded window; without the timestamps the corroboration is unrecoverable after the fact. AC 4 is encoded in the verdict step: a clean completion at the cap prints "ceiling >= cap, config-bound", explicitly not "no state knee". Verdict logic exercised locally against clean-to-cap, mid-ramp knee, and zero-established. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 4c96728..e9666ee 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -30,6 +30,30 @@ on: packet_count: description: Packets to receive before reporting ok default: "3" + mode: + description: >- + oneshot = the BLO-21823 data-receipt probe (default, unchanged). + tunnels-ramp = the BLO-33457 tunnel-STATE occupancy ramp. + type: choice + options: [oneshot, tunnels-ramp] + default: oneshot + ramp_steps: + description: >- + tunnels-ramp only. Comma-separated N values, ascending. The cap is + AMT_MAX_TUNNELS=10000 on production-amt-relay-linux (read live from + amt_relay_tunnel_limit, 2026-09-12). + default: "1,8,64,256,1024,4096,8192,10000" + hold: + description: >- + tunnels-ramp only. Seconds to hold tunnels open after the last one is + established, before reading the survival verdict. Must exceed + --keepalive so every tunnel can emit >=1 keep-alive Membership Update + (BLO-33457 AC 3); amt-verify exits 2 rather than reporting a false + knee if it does not. + default: "25" + keepalive: + description: tunnels-ramp only. Keep-alive interval in seconds. + default: "10" push: branches: [blo-21823-public-amt-probe] @@ -39,6 +63,7 @@ permissions: jobs: probe: + if: ${{ (inputs.mode || 'oneshot') == 'oneshot' }} runs-on: ubuntu-latest env: RELAY: ${{ inputs.relay || '69.25.95.128' }} @@ -99,3 +124,178 @@ jobs: path: | report.json verbose.log + + # BLO-33457. Tunnel-STATE occupancy ramp: how many concurrent AMT tunnels can + # the linux relay hold in its tunnel table? This is the MX301-M leg of + # BLO-20175's sizing model, which is a STATE claim (tunnel-limit 16,384), not + # the throughput claim BLO-26574 measures. + # + # Target is 69.25.95.128:2268 = Service/blockcastd-amt-relay-linux + # (LoadBalancer, selector app.kubernetes.io/name=amt-relay-linux), verified + # 2026-09-12. The OTHER relay -- Service/blockcastd-amt-relay, the gw-* one + # whose job="production-amt-relay" reports amt_relay_tunnel_limit=0 -- is + # ClusterIP-only and has no public 2268, so a public-vantage ramp cannot + # accidentally measure it. Corroborate against + # amt_relay_active_tunnels{job="production-amt-relay-linux"} (limit 10000). + tunnels-ramp: + if: ${{ inputs.mode == 'tunnels-ramp' }} + runs-on: ubuntu-latest + env: + RELAY: ${{ inputs.relay || '69.25.95.128' }} + SOURCE: ${{ inputs.source || '69.25.95.192' }} + GROUP: ${{ inputs.group || '232.1.1.60' }} + STEPS: ${{ inputs.ramp_steps || '1,8,64,256,1024,4096,8192,10000' }} + HOLD: ${{ inputs.hold || '25' }} + KEEPALIVE: ${{ inputs.keepalive || '10' }} + steps: + - name: Vantage receipt + run: | + echo "utc=$(date -u +%FT%TZ)" + echo "public_ip=$(curl -fsS https://api.ipify.org)" + echo "target=$RELAY:2268 sg=($SOURCE,$GROUP) steps=$STEPS" + + # INSTRUMENT CEILINGS, MEASURED BEFORE THE FIRST STEP. + # + # Each of these would present as a relay-side state knee if it bound + # first, which is the specific way this measurement gets silently wrong: + # the ramp would report a ceiling that is really the runner's. + # + # nofile -- soft default on ubuntu-latest is 1024, an order of + # magnitude under AMT_MAX_TUNNELS=10000. One socket per + # tunnel, so this alone caps the ramp near ~1000. + # port range -- the relay keys tunnels by source (IP, port), so N + # tunnels need N distinct source ports. The usable span + # is a hard ceiling on N from a single vantage. + # conntrack -- NAT state table; N flows need N entries. + - name: Host ceilings (recorded, not assumed) + run: | + { + echo "nofile_soft=$(ulimit -Sn)" + echo "nofile_hard=$(ulimit -Hn)" + echo "ip_local_port_range=$(cat /proc/sys/net/ipv4/ip_local_port_range)" + lo=$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range) + hi=$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range) + echo "source_ports_available=$((hi-lo+1))" + echo "nf_conntrack_max=$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null || echo unavailable)" + echo "nproc=$(nproc)" + echo "mem_total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo)" + } | tee host-ceilings.txt + # The largest requested N must fit under every ceiling above, or the + # ramp is measuring this runner. Fail LOUDLY here rather than + # producing a plausible number. + max_n=$(echo "$STEPS" | tr ',' '\n' | sort -n | tail -1) + ports=$(sed -n 's/^source_ports_available=//p' host-ceilings.txt) + echo "max_requested_n=$max_n" + if [ "$max_n" -gt "$ports" ]; then + echo "::error::max N ($max_n) exceeds available source ports ($ports) -- the ramp would measure SNAT exhaustion, not relay state" + exit 1 + fi + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image receipt + run: | + docker pull -q ghcr.io/blockcast/amt-verify:main + { + echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/blockcast/amt-verify:main)" + echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' ghcr.io/blockcast/amt-verify:main)" + } | tee image-receipt.txt + + # POSITIVE CONTROL, and it gates the ramp. + # + # BLO-20175's liveness AC exists because a relay that answers NOTHING + # produces a clean-looking sweep: 0 tunnels, 0 drops, no stop rule ever + # trips, and the ramp reports "no knee through N=10000" -- i.e. maximal + # capacity -- for a relay that established nothing. An idle (S,G) is + # deliberate here (the witness is tunnel-table occupancy, not data + # receipt), but "idle" must not be allowed to shade into "the relay + # ignores this tuple". One tunnel must come up first, or the ramp is void. + - name: N=1 positive control (a ramp on a non-establishing tuple is void) + run: | + set +e + docker run --rm --network host --ulimit nofile=1048576:1048576 \ + ghcr.io/blockcast/amt-verify:main \ + --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ + --tunnels 1 --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ + >control.json 2>control.log + status=$? + set -e + echo "control_exit=$status"; cat control.json || true + alive=$(jq -r '.alive // 0' control.json 2>/dev/null || echo 0) + if [ "$alive" != "1" ]; then + echo "::error::N=1 control did not establish+survive (alive=$alive). Ramp aborted: a sweep from here would report a false ceiling." + cat control.log || true + exit 1 + fi + + # Per-step UTC start/end are load-bearing, not bookkeeping. The ramp runs + # from a PUBLIC runner (in-cluster clients get DISCOVERY logged and + # nothing back -- BLO-27413) while Prometheus is in-cluster and + # unreachable from here, so the two halves of AC 2 cannot be captured by + # one process. The relay-side series is pulled afterwards as a range query + # over this window and joined per step -- which is only possible if the + # timestamps were recorded while the ramp ran. + - name: Ramp + run: | + echo '[]' > ramp-index.json + for n in $(echo "$STEPS" | tr ',' ' '); do + t0=$(date -u +%FT%TZ) + echo "=== N=$n start=$t0 ===" + set +e + docker run --rm --network host --ulimit nofile=1048576:1048576 \ + ghcr.io/blockcast/amt-verify:main \ + --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ + --tunnels "$n" --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ + >"step-$n.json" 2>"step-$n.log" + rc=$? + set -e + t1=$(date -u +%FT%TZ) + echo "N=$n exit=$rc end=$t1"; cat "step-$n.json" || true + jq --argjson n "$n" --arg t0 "$t0" --arg t1 "$t1" --argjson rc "$rc" \ + --slurpfile r "step-$n.json" \ + '. + [{requested:$n, started_utc:$t0, ended_utc:$t1, exit:$rc, + report:($r[0] // null)}]' \ + ramp-index.json > tmp.json && mv tmp.json ramp-index.json + # A degraded step IS the measurement -- it is where the knee is -- + # so the loop does not stop on a non-zero exit. It stops only when + # nothing came up at all, which means every larger N is wasted. + est=$(jq -r '.established // 0' "step-$n.json" 2>/dev/null || echo 0) + if [ "$est" = "0" ]; then + echo "::warning::N=$n established 0 tunnels; stopping the ramp" + break + fi + done + echo '--- ramp-index.json ---'; jq . ramp-index.json + + # AC 4: a clean completion at the cap is "ceiling >= cap, config-bound", + # NOT "no state knee". Those are different findings and only one of them + # is honest about what was and was not shown. + - name: Verdict + run: | + cap=$(echo "$STEPS" | tr ',' '\n' | sort -n | tail -1) + top=$(jq -r '[.[]|select(.report.alive == .requested)]|last|.requested // 0' ramp-index.json) + echo "highest_fully_alive_N=$top cap=$cap" + if [ "$top" = "$cap" ]; then + echo "verdict=ceiling >= $cap, config-bound (NOT 'no state knee': the ramp ended at the configured cap, so no knee was reached or excluded above it)" | tee verdict.txt + else + echo "verdict=knee at N>$top (first degraded step above $top); see ramp-index.json for the named host-side vs relay-side split" | tee verdict.txt + fi + echo "NOTE: control-plane tunnel-table ceiling, not a loaded-state ceiling. Every step report carries the caveat inline." | tee -a verdict.txt + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: amt-tunnels-ramp + path: | + host-ceilings.txt + image-receipt.txt + control.json + control.log + ramp-index.json + verdict.txt + step-*.json + step-*.log From 1cdf214ebb3b372694a974b3573ee595ded3fbf1 Mon Sep 17 00:00:00 2001 From: MulticastEngineer Date: Sat, 12 Sep 2026 15:20:37 +0000 Subject: [PATCH 02/11] BLO-33457: receiver-side occupancy witness; validate ramp contract The AC's corroborating instrument is dead. amt_relay_active_tunnels has never been non-zero in 30d on production-amt-relay-linux, and has zero variance in 30d on the juniper arm (pinned at 1). A ramp joined against it reports 0 occupancy at every N, which reads as a relay defect rather than a dead dial. So `alive` -- how many of N reached Active and sustained it past a keep-alive interval, observed from the client -- is now the sole witness. An instrument only ever seen to agree with `requested` is indistinguishable from `alive = established`, so it gets a positive control: each deliberate failure mode (fatal_runtime_error / state_left_active / no_keepalive_sent) is driven through the extracted classify_held_tunnel() and asserted to reduce the count and name itself in not_alive. Verified to fail on a stubbed classifier (left: 3, right: 2). Caveat and teardown comment no longer send readers to the dead gauge. Ally review on #22, both Important findings: - ramp_steps was an unvalidated contract. The stop rule and the verdict both read the list in order, so unsorted input made both wrong without failing. Now rejected unless non-empty, integral, strictly ascending and within MAX_SUPPORTED_N. Exercised over 8 cases. - Host ceilings were recorded but only source_ports_available was compared. nofile (container value, which is the binder) and conntrack are now enforced; nproc/mem are labelled OBSERVATION with their failure signatures named, because neither has a defensible per-tunnel constant to threshold against. Verdict now takes the last step of the LEADING RUN of fully-alive steps. The previous `last` reported top=1024 on a ramp that had already failed at 256; the new expression reports 64 on the same input. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 170 ++++++++++++++--- src/bin/amt-verify.rs | 180 +++++++++++++++--- tests/tunnels_mode.rs | 7 +- 3 files changed, 294 insertions(+), 63 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index e9666ee..34dc4a1 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -135,8 +135,17 @@ jobs: # 2026-09-12. The OTHER relay -- Service/blockcastd-amt-relay, the gw-* one # whose job="production-amt-relay" reports amt_relay_tunnel_limit=0 -- is # ClusterIP-only and has no public 2268, so a public-vantage ramp cannot - # accidentally measure it. Corroborate against - # amt_relay_active_tunnels{job="production-amt-relay-linux"} (limit 10000). + # accidentally measure it. + # + # THE WITNESS IS RECEIVER-SIDE, and that is a correction, not a preference. + # The AC originally wanted amt_relay_active_tunnels{job="production-amt-relay-linux"} + # corroborating each step. That gauge is DEAD on both production relays + # (linux: never non-zero in 30d; juniper: zero variance in 30d, pinned at 1 -- + # BLO-33457). A ramp joined against it reports 0 occupancy at every N, which + # reads as a relay defect rather than a dead dial. So the witness is + # `alive` from the client: how many of N reached Active and sustained it past + # a keep-alive interval. Its positive control is in-tree + # (each_deliberate_failure_mode_reduces_the_alive_count). tunnels-ramp: if: ${{ inputs.mode == 'tunnels-ramp' }} runs-on: ubuntu-latest @@ -147,6 +156,13 @@ jobs: STEPS: ${{ inputs.ramp_steps || '1,8,64,256,1024,4096,8192,10000' }} HOLD: ${{ inputs.hold || '25' }} KEEPALIVE: ${{ inputs.keepalive || '10' }} + # Raised inside the container for every docker run below. Declared once so + # the value enforced against max N cannot drift from the value applied. + CONTAINER_NOFILE: '1048576' + # Instrument ceiling, not a relay limit: ~10 KB/tunnel measured to N=10000 + # in one process (tunnels_mode_scales_to_512_concurrent doc comment). + # Above this the RIG is unvalidated, so a knee there is unattributable. + MAX_SUPPORTED_N: '10000' steps: - name: Vantage receipt run: | @@ -154,43 +170,119 @@ jobs: echo "public_ip=$(curl -fsS https://api.ipify.org)" echo "target=$RELAY:2268 sg=($SOURCE,$GROUP) steps=$STEPS" + # THE STEP LIST IS A CONTRACT AND IT WAS NEVER CHECKED. + # + # Everything downstream assumes ascending order. The ramp stops at the + # first step that establishes nothing, and the verdict reads the run of + # fully-alive steps as a prefix. Fed `1024,8` those two disagree with each + # other and both report confidently: the ramp could stop below the + # configured cap, or the verdict could name a knee above an N that had + # already failed. Neither shows up as an error -- the numbers just mean + # something other than what they say. + - name: Validate ramp steps + run: | + steps=$(echo "$STEPS" | tr ',' ' ') + [ -n "$(echo "$steps" | tr -d ' ')" ] || { echo "::error::ramp_steps is empty"; exit 1; } + prev=0 + for n in $steps; do + case "$n" in + ''|*[!0-9]*) echo "::error::ramp step '$n' is not a non-negative integer"; exit 1 ;; + esac + [ "$n" -gt 0 ] || { echo "::error::ramp step '$n' must be > 0"; exit 1; } + if [ "$n" -le "$prev" ]; then + echo "::error::ramp steps must be STRICTLY ASCENDING ($n follows $prev). The stop rule and the verdict both read the list in order; unsorted input makes both wrong without failing." + exit 1 + fi + if [ "$n" -gt "$MAX_SUPPORTED_N" ]; then + echo "::error::ramp step $n exceeds MAX_SUPPORTED_N=$MAX_SUPPORTED_N -- the rig is only validated to that N, so a knee above it could not be attributed to the relay" + exit 1 + fi + prev=$n + done + echo "steps_validated=$steps" + # INSTRUMENT CEILINGS, MEASURED BEFORE THE FIRST STEP. # # Each of these would present as a relay-side state knee if it bound # first, which is the specific way this measurement gets silently wrong: # the ramp would report a ceiling that is really the runner's. # - # nofile -- soft default on ubuntu-latest is 1024, an order of - # magnitude under AMT_MAX_TUNNELS=10000. One socket per - # tunnel, so this alone caps the ramp near ~1000. - # port range -- the relay keys tunnels by source (IP, port), so N - # tunnels need N distinct source ports. The usable span - # is a hard ceiling on N from a single vantage. - # conntrack -- NAT state table; N flows need N entries. - - name: Host ceilings (recorded, not assumed) + # Every ceiling below is either ENFORCED against max N or explicitly + # labelled an OBSERVATION with its failure signature named. Recording a + # number and then not comparing it is the worst of both: it reads as a + # guard in the log and guards nothing. + # + # ENFORCED + # port range -- the relay keys tunnels by source (IP, port), so N + # tunnels need N distinct source ports. Hard ceiling on + # N from a single vantage; exhaustion is silent. + # nofile -- one socket per tunnel. The RUNNER's soft default is + # 1024, but the client runs in a container with + # --ulimit nofile=$CONTAINER_NOFILE, so the container + # value is the binder and is what gets compared. The + # runner's own limit is recorded as context only. + # conntrack -- NAT state table; N flows need N entries. Enforced + # when readable; when not, downgraded to an observation + # rather than silently skipped. + # + # OBSERVATIONS (recorded, NOT enforced -- no defensible per-tunnel + # constant to compare against, so a threshold here would be invented) + # nproc / mem -- these do not produce a clean shortfall. CPU + # starvation shows up as establishment TIMEOUTS + # (establish_errors, "timed out ... Discovering") and + # memory pressure as an OOM-killed container (step + # exit 137, established=0). Both are distinguishable + # from a relay knee in the artifact IF you look: a + # relay-side ceiling refuses new tunnels while the + # established ones stay alive, whereas a host-side one + # degrades the whole step. Read establish_errors and + # not_alive before attributing any knee. + - name: Host ceilings (recorded, then enforced or labelled) run: | { - echo "nofile_soft=$(ulimit -Sn)" - echo "nofile_hard=$(ulimit -Hn)" + echo "runner_nofile_soft=$(ulimit -Sn)" + echo "runner_nofile_hard=$(ulimit -Hn)" + echo "container_nofile=$CONTAINER_NOFILE" echo "ip_local_port_range=$(cat /proc/sys/net/ipv4/ip_local_port_range)" lo=$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range) hi=$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range) echo "source_ports_available=$((hi-lo+1))" echo "nf_conntrack_max=$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null || echo unavailable)" - echo "nproc=$(nproc)" - echo "mem_total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo)" + echo "nproc=$(nproc) # OBSERVATION" + echo "mem_total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) # OBSERVATION" } | tee host-ceilings.txt - # The largest requested N must fit under every ceiling above, or the - # ramp is measuring this runner. Fail LOUDLY here rather than - # producing a plausible number. + + # The largest requested N must fit under every ENFORCED ceiling, or + # the ramp is measuring this runner. Fail LOUDLY rather than produce + # a plausible number. max_n=$(echo "$STEPS" | tr ',' '\n' | sort -n | tail -1) + echo "max_requested_n=$max_n" | tee -a host-ceilings.txt + fail=0 + ports=$(sed -n 's/^source_ports_available=//p' host-ceilings.txt) - echo "max_requested_n=$max_n" if [ "$max_n" -gt "$ports" ]; then echo "::error::max N ($max_n) exceeds available source ports ($ports) -- the ramp would measure SNAT exhaustion, not relay state" - exit 1 + fail=1 + fi + + # One socket per tunnel plus the process's own handles. Headroom is + # deliberate: at the cap the shortfall would otherwise land exactly + # where the knee is expected. + if [ "$max_n" -ge "$CONTAINER_NOFILE" ]; then + echo "::error::max N ($max_n) is at or above the container nofile limit ($CONTAINER_NOFILE) -- 'Too many open files' would be recorded as a relay knee" + fail=1 fi + ct=$(sed -n 's/^nf_conntrack_max=//p' host-ceilings.txt) + if [ "$ct" = "unavailable" ]; then + echo "::warning::nf_conntrack_max unreadable -- DOWNGRADED to an observation. If a step degrades near a round number, suspect NAT state before the relay." + elif [ "$max_n" -gt "$ct" ]; then + echo "::error::max N ($max_n) exceeds nf_conntrack_max ($ct) -- NAT state exhaustion would be recorded as a relay knee" + fail=1 + fi + + [ "$fail" -eq 0 ] || exit 1 + - uses: docker/login-action@v3 with: registry: ghcr.io @@ -217,7 +309,8 @@ jobs: - name: N=1 positive control (a ramp on a non-establishing tuple is void) run: | set +e - docker run --rm --network host --ulimit nofile=1048576:1048576 \ + docker run --rm --network host \ + --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ ghcr.io/blockcast/amt-verify:main \ --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ --tunnels 1 --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ @@ -232,13 +325,11 @@ jobs: exit 1 fi - # Per-step UTC start/end are load-bearing, not bookkeeping. The ramp runs - # from a PUBLIC runner (in-cluster clients get DISCOVERY logged and - # nothing back -- BLO-27413) while Prometheus is in-cluster and - # unreachable from here, so the two halves of AC 2 cannot be captured by - # one process. The relay-side series is pulled afterwards as a range query - # over this window and joined per step -- which is only possible if the - # timestamps were recorded while the ramp ran. + # Per-step UTC start/end. The relay-side join these were originally for is + # off the table (amt_relay_active_tunnels is dead -- see the job header), + # but they stay: they are what lets a step be correlated with the relay + # pod's own CPU/memory series, with LB behaviour, or with anything else + # measured later. Unrecordable after the fact, cheap now. - name: Ramp run: | echo '[]' > ramp-index.json @@ -246,7 +337,8 @@ jobs: t0=$(date -u +%FT%TZ) echo "=== N=$n start=$t0 ===" set +e - docker run --rm --network host --ulimit nofile=1048576:1048576 \ + docker run --rm --network host \ + --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ ghcr.io/blockcast/amt-verify:main \ --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ --tunnels "$n" --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ @@ -274,17 +366,33 @@ jobs: # AC 4: a clean completion at the cap is "ceiling >= cap, config-bound", # NOT "no state knee". Those are different findings and only one of them # is honest about what was and was not shown. + # + # `top` is the last step of the LEADING RUN of fully-alive steps, not the + # largest fully-alive step anywhere in the list. With a degraded step at + # 256 and a clean one at 1024, `max` would report a ceiling above an N + # that had already failed. The leading run is the only reading that says + # "every N up to here held", which is the claim BLO-20175 sizes against. - name: Verdict run: | cap=$(echo "$STEPS" | tr ',' '\n' | sort -n | tail -1) - top=$(jq -r '[.[]|select(.report.alive == .requested)]|last|.requested // 0' ramp-index.json) - echo "highest_fully_alive_N=$top cap=$cap" + top=$(jq -r ' + (map((.report.alive // -1) == .requested) | index(false)) as $i + | (if $i == null then . else .[0:$i] end) + | if length == 0 then 0 else .[-1].requested end' ramp-index.json) + ran=$(jq -r 'length' ramp-index.json) + echo "highest_fully_alive_N=$top cap=$cap steps_run=$ran" if [ "$top" = "$cap" ]; then echo "verdict=ceiling >= $cap, config-bound (NOT 'no state knee': the ramp ended at the configured cap, so no knee was reached or excluded above it)" | tee verdict.txt else echo "verdict=knee at N>$top (first degraded step above $top); see ramp-index.json for the named host-side vs relay-side split" | tee verdict.txt fi - echo "NOTE: control-plane tunnel-table ceiling, not a loaded-state ceiling. Every step report carries the caveat inline." | tee -a verdict.txt + # A knee is only the RELAY's if nothing host-side bound first. The + # tool attributes these by name; surface them next to the verdict so + # the two are not read apart. + jq -r '[.[]|select((.report.establish_errors // {}) != {} or (.report.not_alive // {}) != {}) + |{requested, establish_errors:.report.establish_errors, not_alive:.report.not_alive}]' \ + ramp-index.json | tee -a verdict.txt + echo "NOTE: control-plane tunnel-table ceiling, not a loaded-state ceiling. Witness is RECEIVER-SIDE (client 'alive'); relay-side amt_relay_active_tunnels is dead and was deliberately not used. Every step report carries the caveat inline." | tee -a verdict.txt - uses: actions/upload-artifact@v4 if: always() diff --git a/src/bin/amt-verify.rs b/src/bin/amt-verify.rs index 017e3b4..5de511a 100644 --- a/src/bin/amt-verify.rs +++ b/src/bin/amt-verify.rs @@ -606,8 +606,12 @@ fn validate_families( const CONTROL_PLANE_CAVEAT: &str = "Idle tunnels: this is the relay's CONTROL-PLANE tunnel-table \ ceiling, not a loaded-state ceiling. An idle tunnel may cost a relay less state than an active \ one. Comparable to a configured control-plane limit (Junos tunnel-limit, Linux AMT_MAX_TUNNELS); \ -NOT comparable to a measured loaded ceiling. Client-side counts must be corroborated by relay-side \ -amt_relay_active_tunnels at each step."; +NOT comparable to a measured loaded ceiling. WITNESS IS RECEIVER-SIDE: `alive` is how many of \ +`requested` reached Active and sustained it past a keep-alive interval, observed from this client. \ +It is NOT corroborated by relay-side amt_relay_active_tunnels, which is dead on both production \ +relays (linux: never non-zero in 30d; juniper: zero variance in 30d, pinned at 1) -- see BLO-33457. \ +ATTRIBUTION: a shortfall here is the relay's tunnel table ONLY if establish_errors and not_alive \ +are empty of host-side reasons; otherwise the binder is this rig, not the relay."; /// Per-tunnel result. `establish_ms` is wall-clock from gateway construction to /// the gateway reporting `Active`. @@ -623,6 +627,46 @@ enum TunnelOutcome { Failed { error: String }, } +/// Verdict for one gateway at the end of the hold. Pure so each deliberate +/// failure mode can be driven in a unit test: an `alive` count that cannot be +/// shown to go DOWN is indistinguishable from one hardcoded to `established`, +/// and this whole measurement rests on it (BLO-33457 CTO condition 1). +/// +/// Order matters. `fatal_runtime_error` is checked first because a dead socket +/// leaves the published state stale, so a gateway whose runtime died can still +/// read `Active` — classifying that as relay behaviour is how a ramp turns the +/// INSTRUMENT's ceiling into a relay knee. +fn classify_held_tunnel( + has_fatal: bool, + state: GatewayState, + keepalives: u64, + establish_ms: u64, + rx: u64, +) -> TunnelOutcome { + if has_fatal { + TunnelOutcome::NotAlive { + reason: "fatal_runtime_error", + } + } else if state != GatewayState::Active { + TunnelOutcome::NotAlive { + reason: "state_left_active", + } + } else if keepalives == 0 { + // Held past one full keep-alive interval and emitted nothing: the + // tunnel was established but is not demonstrably maintained, so it + // must not be counted as live state. + TunnelOutcome::NotAlive { + reason: "no_keepalive_sent", + } + } else { + TunnelOutcome::Alive { + establish_ms, + keepalives, + rx, + } + } +} + #[derive(serde::Serialize)] struct TunnelsReport { outcome: &'static str, @@ -849,32 +893,13 @@ async fn run_tunnels( for (gw, establish_ms) in &up { let keepalives = gw.keepalives_sent(); let rx = gw.rx_datagrams(); - outcomes.push(if gw.has_fatal().await { - // Unrecoverable socket error in this gateway's runtime. Attributed - // separately from `state_left_active` because it is the INSTRUMENT - // failing, not the relay evicting state — reading a host-side send - // failure as a relay knee is how a ramp under-reports the ceiling. - TunnelOutcome::NotAlive { - reason: "fatal_runtime_error", - } - } else if gw.state() != GatewayState::Active { - TunnelOutcome::NotAlive { - reason: "state_left_active", - } - } else if keepalives == 0 { - // Held past one full keep-alive interval and emitted nothing: the - // tunnel was established but is not demonstrably maintained, so it - // must not be counted as live state. - TunnelOutcome::NotAlive { - reason: "no_keepalive_sent", - } - } else { - TunnelOutcome::Alive { - establish_ms: *establish_ms, - keepalives, - rx, - } - }); + outcomes.push(classify_held_tunnel( + gw.has_fatal().await, + gw.state(), + keepalives, + *establish_ms, + rx, + )); } let report = summarize( @@ -921,8 +946,9 @@ async fn run_tunnels( } // Tear down concurrently. Leaving state behind would inflate the next ramp - // step; confirm relay-side amt_relay_active_tunnels returns to baseline - // between steps rather than trusting this. + // step, and there is no relay-side gauge to confirm the table drained + // (amt_relay_active_tunnels is dead — see CONTROL_PLANE_CAVEAT), so the + // ramp driver must space steps rather than trust a readback. let mut teardown = tokio::task::JoinSet::new(); for (gw, _) in up { teardown.spawn(async move { finish_gateway(gw, group, source, shutdown_mode).await }); @@ -972,6 +998,100 @@ mod tests { const SRC: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 1)); const GROUP: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(232, 0, 0, 1)); + // ---- POSITIVE CONTROL for the receiver-side occupancy witness ---------- + // + // BLO-33457, CTO condition 1. `amt_relay_active_tunnels` was the AC's + // corroborating instrument and is dead on both production relays, so + // `alive` is now the sole witness for this measurement. An instrument that + // has only ever been seen to agree with `requested` is indistinguishable + // from `alive = established`. These drive each deliberate failure mode and + // assert the count goes DOWN. + + fn summarize_n(outcomes: Vec, requested: u32) -> TunnelsReport { + summarize( + outcomes, + requested, + "127.0.0.1".parse().unwrap(), + "v4", + GROUP, + SRC, + /* hold */ 30, + /* keepalive */ 10, + /* stagger */ 0, + ) + } + + fn alive_one() -> TunnelOutcome { + classify_held_tunnel(false, GatewayState::Active, 1, 5, 0) + } + + #[test] + fn a_healthy_held_tunnel_is_counted_alive() { + // The control's control: without this, a classifier that returned + // NotAlive unconditionally would "pass" every assertion below. + assert!(matches!(alive_one(), TunnelOutcome::Alive { .. })); + assert_eq!(summarize_n(vec![alive_one(), alive_one()], 2).alive, 2); + } + + #[test] + fn each_deliberate_failure_mode_reduces_the_alive_count() { + for (label, failed) in [ + // Socket died under us. Note `state` is still Active: a dead + // runtime leaves the published state stale, which is exactly the + // case that would otherwise be counted as live relay state. + ( + "fatal_runtime_error", + classify_held_tunnel(true, GatewayState::Active, 3, 5, 0), + ), + // Left Active during the hold. + ( + "state_left_active", + classify_held_tunnel(false, GatewayState::Discovering, 3, 5, 0), + ), + // Established but never maintained across a keep-alive interval. + ( + "no_keepalive_sent", + classify_held_tunnel(false, GatewayState::Active, 0, 5, 0), + ), + ] { + let r = summarize_n(vec![alive_one(), alive_one(), failed], 3); + assert_eq!(r.established, 3, "{label}: all three did establish"); + assert_eq!(r.alive, 2, "{label}: alive must drop below established"); + assert_eq!(r.outcome, "degraded", "{label}"); + assert_eq!( + r.not_alive.get(label), + Some(&1), + "{label} must be attributed by name, not folded into a bare shortfall: {:?}", + r.not_alive + ); + } + } + + #[test] + fn keepalives_min_exposes_a_single_unmaintained_tunnel() { + // The aggregate hides it: 2 alive either way. `keepalives_min` is what + // makes "survived an interval" a measurement rather than an assumption. + let r = summarize_n( + vec![ + classify_held_tunnel(false, GatewayState::Active, 9, 5, 0), + classify_held_tunnel(false, GatewayState::Active, 1, 5, 0), + ], + 2, + ); + assert_eq!(r.alive, 2); + assert_eq!(r.keepalives_min, 1); + } + + #[test] + fn the_caveat_travels_with_every_report() { + // AC 5, and the dead-gauge correction: the artifact must not send a + // reader to amt_relay_active_tunnels for corroboration. + let c = summarize_n(vec![alive_one()], 1).caveat; + assert!(c.contains("CONTROL-PLANE"), "{c}"); + assert!(c.contains("RECEIVER-SIDE"), "{c}"); + assert!(c.contains("ATTRIBUTION"), "{c}"); + } + // BLO-33456 review follow-up. A closed data broadcast and an expired // deadline both ended the data phase via the same `anyhow::Error`, so a // transport failure was reported as `outcome: "timeout"` — a deadline diff --git a/tests/tunnels_mode.rs b/tests/tunnels_mode.rs index 5ed2c2c..55a7857 100644 --- a/tests/tunnels_mode.rs +++ b/tests/tunnels_mode.rs @@ -92,10 +92,13 @@ async fn tunnels_mode_reports_per_tunnel_survival() { // BLO-33457 acceptance criterion: the control-plane-vs-loaded-state caveat // travels WITH the numbers, so a raw per-step artifact cannot be read as a - // loaded ceiling. + // loaded ceiling. It must ALSO name the witness as receiver-side: the + // relay's amt_relay_active_tunnels is dead on both production relays, so a + // reader sent there for corroboration reads 0 and calls it a relay defect. let caveat = v["caveat"].as_str().expect("caveat field"); assert!(caveat.contains("CONTROL-PLANE"), "{caveat}"); - assert!(caveat.contains("amt_relay_active_tunnels"), "{caveat}"); + assert!(caveat.contains("RECEIVER-SIDE"), "{caveat}"); + assert!(caveat.contains("ATTRIBUTION"), "{caveat}"); } /// A relay that never answers. The knee case: the report must say `degraded` From 945d069c15ecfc8f67ab271131b3a320a5b4b2e9 Mon Sep 17 00:00:00 2001 From: MulticastEngineer Date: Sat, 12 Sep 2026 20:34:51 +0000 Subject: [PATCH 03/11] BLO-33457: pin probe image to the checkout; sample a live occupancy instrument The workflow pulled ghcr.io/blockcast/amt-verify:main in all eight places, including the positive control and every ramp step. `:main` is not the ref the run checked out, so a ramp dispatched from a branch runs main's client while emitting the branch's strings: CONTROL_PLANE_CAVEAT, the ramp contract and the receiver-side witness classify_held_tunnel() would all be under test and none of them would execute. The artifact is what ships, and it would have carried the pre-ruling caveat pointing readers at a dead gauge. publish-amt-verify.yml already pushes amt-verify: on every main build, so pinning to github.sha is structural rather than an assertion: the tag IS the commit and cannot go stale. The Image receipt step additionally asserts the image's revision label equals the checkout and fails the job otherwise -- the label was already captured there and never compared, which is the same record-but-don't-enforce shape as the ramp contract and the host ceilings. Default ramp_steps drops 4096/8192/10000. Those rested on "the tunnel table is empty, so there is nothing to displace", whose only evidence was amt_relay_active_tunnels -- the gauge already ruled dead, read as a fact about the relay rather than about the gauge. Measured while looking for a replacement: the relay serves its own stats on :8080/_astats, and that surface is alive. Same pod, seconds apart, _astats amt.relay.active_tunnels=29 against exporter amt_relay_active_tunnels=0. So the ramp now samples _astats across each step and records the peak next to the requested N. It is an OBSERVATION, not a gate: enforcing an instrument on the run meant to qualify it is circular, and a zero would then void the ramp for the instrument's fault. Both amt.relay.active_tunnels and amt.interface.amtr.tunnels are kept because they disagreed when measured (29 vs 47) and which one tracks N is part of what the run establishes. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 128 ++++++++++++++++-- 1 file changed, 115 insertions(+), 13 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 34dc4a1..6cba4b3 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -41,8 +41,16 @@ on: description: >- tunnels-ramp only. Comma-separated N values, ascending. The cap is AMT_MAX_TUNNELS=10000 on production-amt-relay-linux (read live from - amt_relay_tunnel_limit, 2026-09-12). - default: "1,8,64,256,1024,4096,8192,10000" + amt_relay_tunnel_limit, 2026-09-12), but the DEFAULT stops at 1024: + 4096/8192/10000 are withheld pending a working relay-side occupancy + instrument (CTO ruling 2026-09-12). 1024 against a 10000 cap cannot + displace a live subscriber even if the table is occupied, which is why + it is safe without one; the larger steps rested on "the table is + empty", and the only evidence for that was amt_relay_active_tunnels, + which is the gauge the same ruling declared dead. Raise this only with + an occupancy reading that has been shown capable of returning a + positive. + default: "1,8,64,256,1024" hold: description: >- tunnels-ramp only. Seconds to hold tunnels open after the last one is @@ -61,6 +69,22 @@ permissions: contents: read packages: read +# THE IMAGE IS PINNED TO THE CHECKOUT, and that is the whole point. +# +# This was `:main` in all eight places. `:main` is whatever was last pushed to +# main, which is NOT the ref this run checked out -- so a run dispatched from a +# branch executes main's client while every string the job emits (the ramp +# contract, CONTROL_PLANE_CAVEAT, the receiver-side witness classify_held_tunnel) +# comes from the branch. The witness under test is then not the witness that +# runs, and the artifact reports the branch's caveat over main's measurement. +# +# publish-amt-verify.yml already pushes ghcr.io/blockcast/amt-verify: on +# every main build, so pinning to github.sha costs nothing and cannot go stale: +# the tag IS the commit. If no image exists for this ref the pull fails loudly, +# which is correct -- a ramp against code that was never built is void. +env: + IMAGE: ghcr.io/blockcast/amt-verify:${{ github.sha }} + jobs: probe: if: ${{ (inputs.mode || 'oneshot') == 'oneshot' }} @@ -84,13 +108,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: amt-verify (real client, public vantage) run: | - docker pull -q ghcr.io/blockcast/amt-verify:main - echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/blockcast/amt-verify:main)" + docker pull -q $IMAGE + echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' $IMAGE)" # BLO-21823 AC#3 wants the client commit alongside the artifact hash. - echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' ghcr.io/blockcast/amt-verify:main)" + echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" set +e t0=$(date +%s%3N) - docker run --rm --network host ghcr.io/blockcast/amt-verify:main \ + docker run --rm --network host $IMAGE \ --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ --timeout "$TIMEOUT" --packet-count "$PACKETS" --json --verbose \ >report.json 2>verbose.log @@ -289,13 +313,37 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # The tag pins by construction; this ASSERTS it. A receipt that is recorded + # and never compared is the same record-but-don't-enforce shape as the ramp + # contract and the host ceilings -- it reads as a guard in the artifact and + # guards nothing. Two lines, and it also catches the one case the tag alone + # cannot: a build that published under this sha with a different revision + # label. - name: Image receipt run: | - docker pull -q ghcr.io/blockcast/amt-verify:main + docker pull -q $IMAGE { - echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/blockcast/amt-verify:main)" - echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' ghcr.io/blockcast/amt-verify:main)" + echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' $IMAGE)" + echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" + echo "workflow_sha=${{ github.sha }}" } | tee image-receipt.txt + cc=$(sed -n 's/^client_commit=//p' image-receipt.txt) + [ "$cc" = "${{ github.sha }}" ] || { + echo "::error::image revision ($cc) != checkout (${{ github.sha }}) -- the client under test is not the client that would run"; exit 1; } + + # BASELINE OCCUPANCY, before the ramp adds anything. + # + # The withheld cap steps rested on "the linux relay has served zero tunnels + # for a week, so there is no live subscriber to displace". That read + # amt_relay_active_tunnels=0 as a fact about the relay when it is a fact + # about the gauge -- the same gauge a two-tunnel control had already shown + # stays flat at 0 while two tunnels were held. This records what the relay + # says about ITSELF instead, before the ramp perturbs it. Non-fatal: it is + # evidence for sizing the NEXT run, not a gate on this one. + - name: Baseline relay occupancy (_astats, pre-ramp) + run: | + curl -fsS -m 5 "http://$RELAY:8080/_astats" | tee astats-baseline.json || \ + echo "::warning::_astats unreachable from this public vantage -- baseline unknown, and the candidate occupancy instrument cannot be qualified from here" # POSITIVE CONTROL, and it gates the ramp. # @@ -311,7 +359,7 @@ jobs: set +e docker run --rm --network host \ --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ - ghcr.io/blockcast/amt-verify:main \ + $IMAGE \ --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ --tunnels 1 --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ >control.json 2>control.log @@ -330,6 +378,31 @@ jobs: # but they stay: they are what lets a step be correlated with the relay # pod's own CPU/memory series, with LB behaviour, or with anything else # measured later. Unrecordable after the fact, cheap now. + # + # CANDIDATE OCCUPANCY INSTRUMENT, SAMPLED UNDER QUALIFICATION. + # + # The relay serves its own stats as JSON on :8080/_astats, and that + # surface is NOT the dead gauge. Measured in-cluster 2026-09-12, same pod, + # seconds apart: _astats "amt.relay.active_tunnels"=29 while the Prometheus + # exporter's amt_relay_active_tunnels=0. So this one demonstrably returns a + # positive, which is exactly the property the gauge lacks. + # + # It is sampled here as an OBSERVATION and deliberately does NOT gate the + # step -- enforcing an instrument on the same run that is meant to qualify + # it is circular, and a 0 here would then void a ramp for the instrument's + # fault rather than the relay's. Qualification criterion, applied by hand + # to the artifact afterwards: does peak occupancy TRACK the requested N? + # If it moves to ~1024 while 1024 tunnels are held, it is qualified and it + # licenses the withheld cap steps. If it reads 0, it is a seventh dead dial. + # + # Both counters are kept because they are DIFFERENT FIELDS and were not + # equal when measured (relay.active_tunnels=29, interface.amtr.tunnels=47). + # Which of them tracks N is part of what this run establishes; assuming + # they are the same number is how a wrong one gets adopted. + # + # Public reachability of :8080 is unverified -- the pre-run read was via + # the LB IP from in-cluster, which kube-proxy may short-circuit. An + # unreachable endpoint records {} and costs the run nothing. - name: Ramp run: | echo '[]' > ramp-index.json @@ -339,17 +412,34 @@ jobs: set +e docker run --rm --network host \ --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ - ghcr.io/blockcast/amt-verify:main \ + $IMAGE \ --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ --tunnels "$n" --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ - >"step-$n.json" 2>"step-$n.log" + >"step-$n.json" 2>"step-$n.log" & + pid=$! + # Sample for the whole step rather than at one guessed instant: with + # establishment time unknown a single sample can land before the + # tunnels are up and read a false 0. + : > "astats-$n.ndjson" + while kill -0 $pid 2>/dev/null; do + curl -fsS -m 3 "http://$RELAY:8080/_astats" >> "astats-$n.ndjson" 2>/dev/null + echo >> "astats-$n.ndjson" + sleep 2 + done + wait $pid rc=$? set -e t1=$(date -u +%FT%TZ) - echo "N=$n exit=$rc end=$t1"; cat "step-$n.json" || true + peak_relay=$(jq -s 'map(.ats."amt.relay.active_tunnels"//empty)|max//null' "astats-$n.ndjson" 2>/dev/null || echo null) + peak_iface=$(jq -s 'map(.ats."amt.interface.amtr.tunnels"//empty)|max//null' "astats-$n.ndjson" 2>/dev/null || echo null) + echo "N=$n exit=$rc end=$t1 astats_peak_relay_active_tunnels=$peak_relay astats_peak_iface_tunnels=$peak_iface" + cat "step-$n.json" || true jq --argjson n "$n" --arg t0 "$t0" --arg t1 "$t1" --argjson rc "$rc" \ + --argjson pr "${peak_relay:-null}" --argjson pi "${peak_iface:-null}" \ --slurpfile r "step-$n.json" \ '. + [{requested:$n, started_utc:$t0, ended_utc:$t1, exit:$rc, + astats_peak_relay_active_tunnels:$pr, + astats_peak_iface_tunnels:$pi, report:($r[0] // null)}]' \ ramp-index.json > tmp.json && mv tmp.json ramp-index.json # A degraded step IS the measurement -- it is where the knee is -- @@ -393,6 +483,16 @@ jobs: |{requested, establish_errors:.report.establish_errors, not_alive:.report.not_alive}]' \ ramp-index.json | tee -a verdict.txt echo "NOTE: control-plane tunnel-table ceiling, not a loaded-state ceiling. Witness is RECEIVER-SIDE (client 'alive'); relay-side amt_relay_active_tunnels is dead and was deliberately not used. Every step report carries the caveat inline." | tee -a verdict.txt + # Occupancy-instrument QUALIFICATION, reported next to the verdict so + # the two are read together. This does not corroborate THIS ramp -- an + # unqualified instrument cannot corroborate anything -- it says whether + # _astats earned the right to corroborate the NEXT one. + { + echo "--- candidate occupancy instrument (_astats :8080), qualification ---" + echo "baseline_pre_ramp=$(jq -c '.ats|{relay:."amt.relay.active_tunnels",iface:."amt.interface.amtr.tunnels",limit:."amt.relay.tunnel_limit"}' astats-baseline.json 2>/dev/null || echo unreachable)" + jq -r '.[]|"requested=\(.requested) peak_relay_active_tunnels=\(.astats_peak_relay_active_tunnels) peak_iface_tunnels=\(.astats_peak_iface_tunnels) alive=\(.report.alive // "n/a")"' ramp-index.json + echo "QUALIFIED if a peak column rises with requested N; DEAD DIAL if it stays flat (that is the amt_relay_active_tunnels failure, and it would be the seventh). Read the two peak columns separately: they were NOT equal pre-ramp (relay=29, iface=47), so whichever tracks N is the occupancy counter and the other is something else." + } | tee -a verdict.txt - uses: actions/upload-artifact@v4 if: always() @@ -401,6 +501,8 @@ jobs: path: | host-ceilings.txt image-receipt.txt + astats-baseline.json + astats-*.ndjson control.json control.log ramp-index.json From 772247c3f87317aafc5eaec9b24d5c8297d737a3 Mon Sep 17 00:00:00 2001 From: MulticastEngineer Date: Sat, 12 Sep 2026 21:05:59 +0000 Subject: [PATCH 04/11] BLO-33457: build the probe image from the checkout when no SHA tag exists Ally (945d069, Important): the publisher only pushes amt-verify: on main, so pinning IMAGE to github.sha made this workflow undispatchable from the branch it is meant to verify -- docker pull fails before any measurement. Correct finding; I had flagged the same constraint as accepted, which left the ramp only runnable post-merge and the merge gated on the ramp. pull || build from the checkout dissolves it. main keeps the fast path (tag IS the commit); a branch builds the checked-out tree and stamps the revision label, so both paths run the client under review. Needs actions/checkout in both jobs -- neither had one. The receipt reports local-build where there is no registry digest rather than an empty field. Also: the STEPS env fallback still carried the pre-ruling cap list (...,4096,8192,10000) while the declared input default is 1,8,64,256,1024. Unreachable from the UI, reachable via a REST dispatch passing ramp_steps as an empty string -- i.e. the one path that skips the input default would have silently selected the four steps the CTO withheld. Matched to the default. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 6cba4b3..a02e46c 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -80,8 +80,17 @@ permissions: # # publish-amt-verify.yml already pushes ghcr.io/blockcast/amt-verify: on # every main build, so pinning to github.sha costs nothing and cannot go stale: -# the tag IS the commit. If no image exists for this ref the pull fails loudly, -# which is correct -- a ramp against code that was never built is void. +# the tag IS the commit. +# +# But that publisher only fires on main, so on a BRANCH no such tag exists and +# the pull fails -- which made this workflow undispatchable from the very PR it +# is supposed to verify (Ally, 945d069). So: pull, and on a miss BUILD from the +# checkout. Both paths yield the checkout's client, by tag identity on main and +# by construction on a branch. On the build path the revision assertion below is +# tautological -- provenance there rests on having built the checked-out tree, +# not on the label -- and the receipt says `local-build` rather than a registry +# digest, because there isn't one and an empty field would read as a missing +# measurement rather than an absent one. env: IMAGE: ghcr.io/blockcast/amt-verify:${{ github.sha }} @@ -101,6 +110,7 @@ jobs: echo "utc=$(date -u +%FT%TZ)" echo "public_ip=$(curl -fsS https://api.ipify.org)" echo "target=$RELAY:2268 sg=($SOURCE,$GROUP)" + - uses: actions/checkout@v4 - uses: docker/login-action@v3 with: registry: ghcr.io @@ -108,8 +118,9 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: amt-verify (real client, public vantage) run: | - docker pull -q $IMAGE - echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' $IMAGE)" + docker pull -q $IMAGE || docker build -q -t $IMAGE \ + --label org.opencontainers.image.revision=${{ github.sha }} . + echo "image_digest=$(docker inspect --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}local-build{{end}}' $IMAGE)" # BLO-21823 AC#3 wants the client commit alongside the artifact hash. echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" set +e @@ -177,7 +188,12 @@ jobs: RELAY: ${{ inputs.relay || '69.25.95.128' }} SOURCE: ${{ inputs.source || '69.25.95.192' }} GROUP: ${{ inputs.group || '232.1.1.60' }} - STEPS: ${{ inputs.ramp_steps || '1,8,64,256,1024,4096,8192,10000' }} + # Must MATCH the declared input default. It read + # `...,1024,4096,8192,10000` -- the pre-ruling list -- so a dispatch that + # supplied ramp_steps as an empty string (REST does; the UI cannot) would + # silently select the four steps the CTO withheld, via the one path that + # skips the input default entirely. + STEPS: ${{ inputs.ramp_steps || '1,8,64,256,1024' }} HOLD: ${{ inputs.hold || '25' }} KEEPALIVE: ${{ inputs.keepalive || '10' }} # Raised inside the container for every docker run below. Declared once so @@ -307,6 +323,7 @@ jobs: [ "$fail" -eq 0 ] || exit 1 + - uses: actions/checkout@v4 - uses: docker/login-action@v3 with: registry: ghcr.io @@ -321,9 +338,10 @@ jobs: # label. - name: Image receipt run: | - docker pull -q $IMAGE + docker pull -q $IMAGE || docker build -q -t $IMAGE \ + --label org.opencontainers.image.revision=${{ github.sha }} . { - echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' $IMAGE)" + echo "image_digest=$(docker inspect --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}local-build{{end}}' $IMAGE)" echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" echo "workflow_sha=${{ github.sha }}" } | tee image-receipt.txt From 627da00e5a77d24eea8b025b7125c5114031f6ff Mon Sep 17 00:00:00 2001 From: MulticastEngineer Date: Sat, 12 Sep 2026 22:27:23 +0000 Subject: [PATCH 05/11] ramp verdict: gate "knee" on relay attribution; stop laundering an unread instrument The 21:12Z ramp run produced verdict="knee at N>64" from a step whose only cause was `timed out after 30s in state Idle` x8. That string names no binder: it is equally consistent with the relay's tunnel table, public-path loss, and the runner's NAT/source-port limits. The attribution rule was in the caveat and in a comment, but the verdict did not read it -- the errors were printed next to the claim rather than gating it. A 3% shortfall at N=256 against a relay holding ~29 tunnels with AMT_MAX_TUNNELS=10000 is not a table-full condition, so this would have fed a wrong knee into BLO-20175's sizing model. Attribution is now a POSITIVE test for relay-side causes, not a blocklist of host-side symptoms, so an unrecognised error falls through to "not a relay knee" -- the safe direction. A blocklist spelling of this was written first and its own negative control caught it matching "refused" inside "relay refused: tunnel table full". Second defect, same family: `jq` exits 0 and prints nothing on an empty file, so `|| echo unreachable` was dead code. astats-baseline.json came back 0 bytes and both ndjson files empty, and the receipt rendered that as `baseline_pre_ramp=` -- an unreachable instrument shown in the same shape as one that answered zero. The sampler ran from the public runner; _astats is in-cluster only. Third: the qualification rubric offered QUALIFIED or DEAD DIAL and nothing else, so a column of nulls would have been recorded as the seventh dead dial. Nulls mean NOT SAMPLED. _astats was read live at relay=29/iface=47 from in-cluster hours earlier, so condemning it on a reading nobody took would have been wrong. Validated by replaying the real 21:12Z artifact plus four controls (relay-attributed -> knee; mixed -> not knee; clean-to-cap -> config-bound; novel error -> fails safe). Refs BLO-33457. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 104c6b2..9d93b23 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -519,14 +519,28 @@ jobs: | if length == 0 then 0 else .[-1].requested end' ramp-index.json) ran=$(jq -r 'length' ramp-index.json) echo "highest_fully_alive_N=$top cap=$cap steps_run=$ran" + # A shortfall is the RELAY's tunnel table only if its cause says so. + # This is a POSITIVE test for relay attribution, not a blocklist of + # host-side symptoms: an error string nobody has seen before must fall + # through to "not a relay knee", because that is the direction a capex + # number is safe to be wrong in. (An earlier blocklist spelling of this + # matched "refused" and so swallowed "relay refused: tunnel table + # full" -- the negative control below exists to keep that fixed.) + # Surfacing-without-enforcing is the fourth instance of that pattern in + # this file; the other three were the ramp contract, the host ceilings, + # and the image label. + causes=$(jq -r '.[]|select((.report.alive // -1) != .requested) + |(((.report.establish_errors // {})|keys[]?),((.report.not_alive // {})|keys[]?))' ramp-index.json) + n_relay=$(echo "$causes" | grep -cEi 'relay (refused|rejected)|tunnel[ -](table|limit)|no more tunnels' || true) + n_other=$(echo "$causes" | grep -cvEi 'relay (refused|rejected)|tunnel[ -](table|limit)|no more tunnels' || true) + [ -z "$causes" ] && n_relay=0 && n_other=0 if [ "$top" = "$cap" ]; then echo "verdict=ceiling >= $cap, config-bound (NOT 'no state knee': the ramp ended at the configured cap, so no knee was reached or excluded above it)" | tee verdict.txt + elif [ "$n_relay" -gt 0 ] && [ "$n_other" -eq 0 ]; then + echo "verdict=knee at N>$top (first degraded step above $top; every shortfall is relay-attributed -- see ramp-index.json)" | tee verdict.txt else - echo "verdict=knee at N>$top (first degraded step above $top); see ramp-index.json for the named host-side vs relay-side split" | tee verdict.txt + echo "verdict=degraded above N=$top, BINDER NOT ESTABLISHED ($n_other cause(s) not attributed to the relay). NOT a relay knee: a bare transport failure is equally consistent with the relay's tunnel table, public-path loss, and this runner's NAT/source-port or rate limiting. Do NOT size against this number." | tee verdict.txt fi - # A knee is only the RELAY's if nothing host-side bound first. The - # tool attributes these by name; surface them next to the verdict so - # the two are not read apart. jq -r '[.[]|select((.report.establish_errors // {}) != {} or (.report.not_alive // {}) != {}) |{requested, establish_errors:.report.establish_errors, not_alive:.report.not_alive}]' \ ramp-index.json | tee -a verdict.txt @@ -537,9 +551,26 @@ jobs: # _astats earned the right to corroborate the NEXT one. { echo "--- candidate occupancy instrument (_astats :8080), qualification ---" - echo "baseline_pre_ramp=$(jq -c '.ats|{relay:."amt.relay.active_tunnels",iface:."amt.interface.amtr.tunnels",limit:."amt.relay.tunnel_limit"}' astats-baseline.json 2>/dev/null || echo unreachable)" + # `jq` exits 0 and prints NOTHING on an empty file, so the old + # `|| echo unreachable` could never fire: an unreachable instrument + # was rendered as an empty string and read as "no data". Test the + # file has bytes first -- an instrument that was never read must not + # be reported in the same shape as one that answered. + if [ -s astats-baseline.json ]; then + echo "baseline_pre_ramp=$(jq -c '.ats|{relay:."amt.relay.active_tunnels",iface:."amt.interface.amtr.tunnels",limit:."amt.relay.tunnel_limit"}' astats-baseline.json 2>/dev/null || echo malformed)" + else + echo "baseline_pre_ramp=UNREACHABLE (no bytes returned; _astats is in-cluster only and this job runs from the public runner)" + fi jq -r '.[]|"requested=\(.requested) peak_relay_active_tunnels=\(.astats_peak_relay_active_tunnels) peak_iface_tunnels=\(.astats_peak_iface_tunnels) alive=\(.report.alive // "n/a")"' ramp-index.json - echo "QUALIFIED if a peak column rises with requested N; DEAD DIAL if it stays flat (that is the amt_relay_active_tunnels failure, and it would be the seventh). Read the two peak columns separately: they were NOT equal pre-ramp (relay=29, iface=47), so whichever tracks N is the occupancy counter and the other is something else." + # Three outcomes, not two. A column of nulls means the instrument was + # NOT SAMPLED -- it is not evidence the dial is dead, and recording it + # as such would condemn a counter measured live at relay=29/iface=47 + # from in-cluster hours earlier. + if jq -e '[.[]|.astats_peak_relay_active_tunnels,.astats_peak_iface_tunnels]|all(.==null)' ramp-index.json >/dev/null; then + echo "qualification=NOT SAMPLED -- every peak is null, i.e. no reading was taken. This is NOT the 'dead dial' verdict and must not be recorded as one: it says nothing about _astats, only that this vantage could not reach it. Re-run with the sampler in-cluster." + else + echo "qualification=QUALIFIED if a peak column rises with requested N; DEAD DIAL if it stays flat at a value (that is the amt_relay_active_tunnels failure, and it would be the seventh). Read the two peak columns separately: they were NOT equal pre-ramp (relay=29, iface=47), so whichever tracks N is the occupancy counter and the other is something else." + fi } | tee -a verdict.txt - uses: actions/upload-artifact@v4 From a660d4cd5304205306c01d826bab40bac5578264 Mon Sep 17 00:00:00 2001 From: MulticastEngineer Date: Sun, 13 Sep 2026 05:16:30 +0000 Subject: [PATCH 06/11] tunnels ramp: the witness cannot see relay state; gate the verdict on it `alive` never witnessed the relay. It is `state == Active && keepalives_sent >= 1`, and all three not-alive reasons (fatal_runtime_error, state_left_active, no_keepalive_sent) are client-local. keepalives_sent is incremented after this process's own send_to returns. `rx_datagrams` was collected and reported but was not in the predicate -- and could not be, since a relay owes an idle established gateway no unprompted traffic. That would be a weak witness on its own. What makes it an unusable one is the other half: run_tunnels builds every gateway with `builder(relay)`, so all N bind the host's default source address and differ only by ephemeral port, while linux-amt as deployed matches a tunnel on the outer source ADDRESS alone (`tunnel->addr.ip4 == iph->saddr` in amt_request_handler) and overwrites source_port from the newest Request. N gateways from one address are therefore ONE relay tunnel entry -- and every aliased gateway still reads Active and still sends keep-alives, because its Updates fail the relay's MAC check silently. So `alive` reports N while the relay holds 1, and no amount of reading the artifact reveals it. The existing positive control cannot catch this: deliberately failing a gateway stops it SENDING, which is the one thing the classifier can see. It exercises the plumbing, not the confound. The AC anticipated the shape -- "an alive only ever observed to equal requested is indistinguishable from alive = established" -- and it is worse than that: indistinguishable from alive = 1. Fix is a disqualifier the verdict gates on rather than more prose beside the number: report `distinct_outer_sources` (structurally 1 here), and make the ramp verdict check it FIRST, superseding every branch including clean-to-cap. A ramp that sails to the cap on one source address is the worst case, not the best -- it reports a ceiling having established one tunnel. An absent field defaults to shared, so an older artifact fails safe. Controls: shared+clean-to-cap fires; field-absent (the real 21:12Z shape, alive 248/256) fires; partial provisioning fires; distinct sources does NOT fire and falls through to the real verdict, so the gate is not clamped shut against a rig that outgrows it. tunnels_mode asserts distinct_outer_sources == 1 exactly, so giving the gateways distinct sources must fail this test and force the field to be set honestly. This mode cannot produce a relay tunnel-state ceiling. BLO-33636 fixes the keying; linux-amt kernel/selftests/amt_capacity.sh already provisions 129 distinct source IPs and takes ground truth from the relay's admit/refuse reply. Refs BLO-33457, BLO-33636. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 13 ++++++- src/bin/amt-verify.rs | 34 ++++++++++++++++++- tests/tunnels_mode.rs | 15 ++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 9d93b23..03e9c40 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -534,7 +534,18 @@ jobs: n_relay=$(echo "$causes" | grep -cEi 'relay (refused|rejected)|tunnel[ -](table|limit)|no more tunnels' || true) n_other=$(echo "$causes" | grep -cvEi 'relay (refused|rejected)|tunnel[ -](table|limit)|no more tunnels' || true) [ -z "$causes" ] && n_relay=0 && n_other=0 - if [ "$top" = "$cap" ]; then + # Does any step admit it could not tell N tunnels from one aliased N + # ways? This gate runs FIRST and supersedes every branch below, + # including the clean-to-cap one -- a ramp that sails to the cap on a + # single source address is the WORST case, not the best: it reports a + # ceiling having established one tunnel. `alive` is a send-side + # self-report, so nothing downstream of it can notice. + # `// .requested` defaults a report predating the field to "shared", + # i.e. an unknown fails to the safe side rather than passing the gate. + aliased=$(jq -r '[.[]|select(((.report.distinct_outer_sources // 1)) < .requested)]|length' ramp-index.json) + if [ "$aliased" -gt 0 ]; then + echo "verdict=WITNESS NOT ESTABLISHED ($aliased/$ran steps ran all gateways from ONE outer source address). NOT a ceiling, NOT a knee, and NOT 'no knee': linux-amt as deployed keys tunnel state on the outer ADDRESS alone, so N gateways from one address occupy ONE relay tunnel entry while every one still reads Active and sends keep-alives. An alive of N here is indistinguishable from one tunnel aliased N ways, so this run says nothing about the relay's tunnel table at any N. Do NOT size against this number. Needs distinct source addresses (linux-amt kernel/selftests/amt_capacity.sh) and/or the BLO-33636 keying fix." | tee verdict.txt + elif [ "$top" = "$cap" ]; then echo "verdict=ceiling >= $cap, config-bound (NOT 'no state knee': the ramp ended at the configured cap, so no knee was reached or excluded above it)" | tee verdict.txt elif [ "$n_relay" -gt 0 ] && [ "$n_other" -eq 0 ]; then echo "verdict=knee at N>$top (first degraded step above $top; every shortfall is relay-attributed -- see ramp-index.json)" | tee verdict.txt diff --git a/src/bin/amt-verify.rs b/src/bin/amt-verify.rs index a9c6166..8ebae0c 100644 --- a/src/bin/amt-verify.rs +++ b/src/bin/amt-verify.rs @@ -761,7 +761,19 @@ NOT comparable to a measured loaded ceiling. WITNESS IS RECEIVER-SIDE: `alive` i It is NOT corroborated by relay-side amt_relay_active_tunnels, which is dead on both production \ relays (linux: never non-zero in 30d; juniper: zero variance in 30d, pinned at 1) -- see BLO-33457. \ ATTRIBUTION: a shortfall here is the relay's tunnel table ONLY if establish_errors and not_alive \ -are empty of host-side reasons; otherwise the binder is this rig, not the relay."; +are empty of host-side reasons; otherwise the binder is this rig, not the relay. \ +DISQUALIFIER -- READ BEFORE CITING ANY NUMBER FROM THIS MODE: every gateway here binds the SAME \ +outer source address (distinct ephemeral ports only), so `distinct_outer_sources` is 1 regardless \ +of `requested`. RFC 7450 s4.2.2 keys a tunnel on the (address, port) endpoint, but linux-amt as \ +deployed matches on the outer source ADDRESS alone (`tunnel->addr.ip4 == iph->saddr`, amt.c \ +amt_request_handler), overwriting `source_port` from the newest Request. So N gateways from one \ +address occupy exactly ONE relay tunnel entry while every one of them still reads Active and still \ +sends keep-alives -- `alive` is a SEND-side self-report (state == Active && keepalives_sent >= 1; \ +all three not-alive reasons are client-local) and cannot witness relay state at all. An `alive` of \ +N is therefore indistinguishable from one tunnel aliased N ways. This mode CANNOT produce a relay \ +tunnel-state ceiling; see BLO-33636 for the keying fix and linux-amt \ +kernel/selftests/amt_capacity.sh for a rig that provisions distinct source addresses and takes \ +ground truth from the relay's own admit/refuse reply."; /// Per-tunnel result. `establish_ms` is wall-clock from gateway construction to /// the gateway reporting `Active`. @@ -850,6 +862,21 @@ struct TunnelsReport { /// recorded as a state knee, whereas `state_left_active` / /// `no_keepalive_sent` are the tunnel genuinely failing to survive. not_alive: BTreeMap, + /// How many DISTINCT outer source addresses the N gateways were spread + /// across. Structurally 1 here: `run_tunnels` builds every gateway with + /// `AsyncAmtGateway::builder(relay)`, which binds the host's default + /// source, so the tunnels differ only by ephemeral port. + /// + /// This is the field that makes the mode's central limitation machine- + /// checkable instead of prose. A relay that keys tunnel state on the outer + /// ADDRESS alone -- which is what linux-amt does as deployed -- collapses + /// all N onto one tunnel entry, and because `alive` is a send-side + /// self-report every aliased gateway still counts. So whenever this is + /// less than `requested`, no tunnel-state ceiling can be read off the run, + /// and the verdict must say so rather than print a number. When a rig that + /// provisions distinct source addresses lands, this becomes N and the gate + /// opens on its own. + distinct_outer_sources: u32, caveat: &'static str, } @@ -937,6 +964,11 @@ fn summarize( rx_datagrams_total: rx_total, establish_errors, not_alive, + // Every gateway in `run_tunnels` is built with `builder(relay)`, i.e. + // the host default source. Hardcoded rather than counted because there + // is exactly one call site and a counted-but-always-1 value would read + // as a measurement. + distinct_outer_sources: 1, caveat: CONTROL_PLANE_CAVEAT, } } diff --git a/tests/tunnels_mode.rs b/tests/tunnels_mode.rs index 55a7857..bab1524 100644 --- a/tests/tunnels_mode.rs +++ b/tests/tunnels_mode.rs @@ -99,6 +99,21 @@ async fn tunnels_mode_reports_per_tunnel_survival() { assert!(caveat.contains("CONTROL-PLANE"), "{caveat}"); assert!(caveat.contains("RECEIVER-SIDE"), "{caveat}"); assert!(caveat.contains("ATTRIBUTION"), "{caveat}"); + assert!(caveat.contains("DISQUALIFIER"), "{caveat}"); + + // The disqualifier must be a FIELD, not only prose in the caveat: the + // workflow verdict gates on it, and a reader who trusts `alive` is exactly + // the reader who will not finish the caveat. All 3 gateways bound the host + // default source, so on a relay keying tunnels by outer address alone they + // are ONE tunnel entry -- while `alive` says 3, because `alive` is a + // send-side self-report and every aliased gateway still sends. + // + // This asserting `1` rather than `<= requested` is deliberate: if someone + // gives the gateways distinct sources, this test must FAIL and make them + // set the field honestly, not pass silently and leave the verdict gate + // clamped shut on a rig that has outgrown it. + assert_eq!(v["distinct_outer_sources"], 1, "{out}"); + assert_eq!(v["alive"], 3, "{out}"); } /// A relay that never answers. The knee case: the report must say `degraded` From 6e2530216cfe1bfa003e1009b51384786ea9cf44 Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Mon, 21 Sep 2026 06:07:38 +0300 Subject: [PATCH 07/11] fix(probe): enforce available host capacity before ramp Measure current UDP source-port and conntrack occupancy before the ramp, fail closed when either reading is unavailable, and compare requested tunnels with the remaining capacity. This prevents runner resource exhaustion from being reported as a relay ceiling. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 64 +++++++++++++++---- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index a231e27..e73f2db 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -568,16 +568,18 @@ jobs: # # ENFORCED # port range -- the relay keys tunnels by source (IP, port), so N - # tunnels need N distinct source ports. Hard ceiling on - # N from a single vantage; exhaustion is silent. + # tunnels need N distinct source ports. We subtract + # ports already held by UDP sockets immediately before + # the ramp; the total range is not usable capacity. # nofile -- one socket per tunnel. The RUNNER's soft default is # 1024, but the client runs in a container with # --ulimit nofile=$CONTAINER_NOFILE, so the container # value is the binder and is what gets compared. The # runner's own limit is recorded as context only. - # conntrack -- NAT state table; N flows need N entries. Enforced - # when readable; when not, downgraded to an observation - # rather than silently skipped. + # conntrack -- NAT state table; N flows need N entries. We subtract + # current occupancy from the table maximum. If either + # value is unreadable, the ramp fails closed instead of + # producing a non-attributable ceiling. # # OBSERVATIONS (recorded, NOT enforced -- no defensible per-tunnel # constant to compare against, so a threshold here would be invented) @@ -600,12 +602,50 @@ jobs: echo "ip_local_port_range=$(cat /proc/sys/net/ipv4/ip_local_port_range)" lo=$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range) hi=$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range) - echo "source_ports_available=$((hi-lo+1))" + echo "source_ports_total=$((hi-lo+1))" + if command -v ss >/dev/null 2>&1 && ss -H -u -a -n >udp-sockets.txt 2>/dev/null; then + awk -v lo="$lo" -v hi="$hi" ' + { + if (NF < 4) { bad++; next } + raw = $4 + sub(/^.*:/, "", raw) + if (raw !~ /^[0-9]+$/) { bad++; next } + port = raw + 0 + if (port >= lo && port <= hi && !seen[port]++) used++ + } + END { printf "source_ports_used=%d\nsource_ports_parse_errors=%d\n", used + 0, bad + 0 } + ' udp-sockets.txt + else + echo "source_ports_used=unavailable" + echo "source_ports_parse_errors=unavailable" + fi echo "nf_conntrack_max=$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null || echo unavailable)" + echo "nf_conntrack_used=$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null || echo unavailable)" echo "nproc=$(nproc) # OBSERVATION" echo "mem_total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) # OBSERVATION" } | tee host-ceilings.txt + ports_total=$(sed -n 's/^source_ports_total=//p' host-ceilings.txt) + ports_used=$(sed -n 's/^source_ports_used=//p' host-ceilings.txt) + ports_bad=$(sed -n 's/^source_ports_parse_errors=//p' host-ceilings.txt) + if ! printf '%s\n' "$ports_total" "$ports_used" "$ports_bad" | grep -Eq '^[0-9]+$' || [ "$ports_bad" -ne 0 ]; then + echo "::error::could not establish current UDP source-port occupancy; the ramp is non-attributable" + exit 1 + fi + ports_available=$((ports_total - ports_used)) + [ "$ports_available" -ge 0 ] || { echo "::error::UDP source-port occupancy exceeds the configured range"; exit 1; } + echo "source_ports_available=$ports_available" | tee -a host-ceilings.txt + + ct=$(sed -n 's/^nf_conntrack_max=//p' host-ceilings.txt) + ct_used=$(sed -n 's/^nf_conntrack_used=//p' host-ceilings.txt) + if ! printf '%s\n' "$ct" "$ct_used" | grep -Eq '^[0-9]+$'; then + echo "::error::could not establish current conntrack occupancy; the ramp is non-attributable" + exit 1 + fi + ct_available=$((ct - ct_used)) + [ "$ct_available" -ge 0 ] || { echo "::error::conntrack occupancy exceeds nf_conntrack_max"; exit 1; } + echo "nf_conntrack_available=$ct_available" | tee -a host-ceilings.txt + # The largest requested N must fit under every ENFORCED ceiling, or # the ramp is measuring this runner. Fail LOUDLY rather than produce # a plausible number. @@ -614,8 +654,8 @@ jobs: fail=0 ports=$(sed -n 's/^source_ports_available=//p' host-ceilings.txt) - if [ "$max_n" -gt "$ports" ]; then - echo "::error::max N ($max_n) exceeds available source ports ($ports) -- the ramp would measure SNAT exhaustion, not relay state" + if [ "$max_n" -ge "$ports" ]; then + echo "::error::max N ($max_n) leaves no source-port headroom (only $ports currently free) -- the ramp would measure SNAT exhaustion, not relay state" fail=1 fi @@ -627,11 +667,9 @@ jobs: fail=1 fi - ct=$(sed -n 's/^nf_conntrack_max=//p' host-ceilings.txt) - if [ "$ct" = "unavailable" ]; then - echo "::warning::nf_conntrack_max unreadable -- DOWNGRADED to an observation. If a step degrades near a round number, suspect NAT state before the relay." - elif [ "$max_n" -gt "$ct" ]; then - echo "::error::max N ($max_n) exceeds nf_conntrack_max ($ct) -- NAT state exhaustion would be recorded as a relay knee" + ct=$(sed -n 's/^nf_conntrack_available=//p' host-ceilings.txt) + if [ "$max_n" -ge "$ct" ]; then + echo "::error::max N ($max_n) leaves no conntrack headroom (only $ct currently free) -- NAT state exhaustion would be recorded as a relay knee" fail=1 fi From 5a066a09c2ec9eb9871d58fe6902fd8b6d8f48fc Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Mon, 21 Sep 2026 10:08:27 +0300 Subject: [PATCH 08/11] fix(probe): validate host capacity inputs Reject malformed capacity readings before arithmetic and test the fail-closed guard. Deduplicate overlapping reserved-port ranges so available local headroom is not understated. Co-Authored-By: Paperclip --- .../workflows/amt-public-vantage-probe.yml | 252 ++++++++++++------ tests/probe_verdict_test.sh | 22 ++ 2 files changed, 190 insertions(+), 84 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index e73f2db..a2b5c30 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -555,7 +555,7 @@ jobs: done echo "steps_validated=$steps" - # INSTRUMENT CEILINGS, MEASURED BEFORE THE FIRST STEP. + # INSTRUMENT CEILINGS, MEASURED IMMEDIATELY BEFORE THE RAMP. # # Each of these would present as a relay-side state knee if it bound # first, which is the specific way this measurement gets silently wrong: @@ -566,20 +566,132 @@ jobs: # number and then not comparing it is the worst of both: it reads as a # guard in the log and guards nothing. # - # ENFORCED + # ENFORCED (local runner only; ss cannot establish upstream NAT headroom) # port range -- the relay keys tunnels by source (IP, port), so N # tunnels need N distinct source ports. We subtract # ports already held by UDP sockets immediately before - # the ramp; the total range is not usable capacity. + # the ramp, plus ip_local_reserved_ports. This is only + # local bind headroom; ss cannot establish upstream + # NAT capacity. # nofile -- one socket per tunnel. The RUNNER's soft default is # 1024, but the client runs in a container with # --ulimit nofile=$CONTAINER_NOFILE, so the container # value is the binder and is what gets compared. The # runner's own limit is recorded as context only. - # conntrack -- NAT state table; N flows need N entries. We subtract - # current occupancy from the table maximum. If either - # value is unreadable, the ramp fails closed instead of - # producing a non-attributable ceiling. + # conntrack -- this runner's conntrack table; N flows need N entries. + # We subtract current occupancy from the table maximum. + # It is local evidence only, not an upstream-NAT claim. + # If either value is unreadable, the ramp fails closed + # instead of producing a non-attributable ceiling. + # + # OBSERVATIONS (recorded, NOT enforced -- no defensible per-tunnel + # constant to compare against, so a threshold here would be invented) + # nproc / mem -- these do not produce a clean shortfall. CPU + # starvation shows up as establishment TIMEOUTS + # (establish_errors, "timed out ... Discovering") and + # memory pressure as an OOM-killed container (step + # exit 137, established=0). Both are distinguishable + # from a relay knee in the artifact IF you look: a + # relay-side ceiling refuses new tunnels while the + # established ones stay alive, whereas a host-side one + # degrades the whole step. Read establish_errors and + # not_alive before attributing any knee. + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The tag pins by construction; this ASSERTS it. A receipt that is recorded + # and never compared is the same record-but-don't-enforce shape as the ramp + # contract and the host ceilings -- it reads as a guard in the artifact and + # guards nothing. Two lines, and it also catches the one case the tag alone + # cannot: a build that published under this sha with a different revision + # label. + - name: Image receipt + run: | + docker pull -q $IMAGE || docker build -q -t $IMAGE \ + --label org.opencontainers.image.revision=${{ github.sha }} . + { + echo "image_digest=$(docker inspect --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}local-build{{end}}' $IMAGE)" + echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" + echo "workflow_sha=${{ github.sha }}" + } | tee image-receipt.txt + cc=$(sed -n 's/^client_commit=//p' image-receipt.txt) + [ "$cc" = "${{ github.sha }}" ] || { + echo "::error::image revision ($cc) != checkout (${{ github.sha }}) -- the client under test is not the client that would run"; exit 1; } + + # BASELINE OCCUPANCY, before the ramp adds anything. + # + # The withheld cap steps rested on "the linux relay has served zero tunnels + # for a week, so there is no live subscriber to displace". That read + # amt_relay_active_tunnels=0 as a fact about the relay when it is a fact + # about the gauge -- the same gauge a two-tunnel control had already shown + # stays flat at 0 while two tunnels were held. This records what the relay + # says about ITSELF instead, before the ramp perturbs it. Non-fatal: it is + # evidence for sizing the NEXT run, not a gate on this one. + - name: Baseline relay occupancy (_astats, pre-ramp) + run: | + curl -fsS -m 5 "http://$RELAY:8080/_astats" | tee astats-baseline.json || \ + echo "::warning::_astats unreachable from this public vantage -- baseline unknown, and the candidate occupancy instrument cannot be qualified from here" + + # POSITIVE CONTROL, and it gates the ramp. + # + # BLO-20175's liveness AC exists because a relay that answers NOTHING + # produces a clean-looking sweep: 0 tunnels, 0 drops, no stop rule ever + # trips, and the ramp reports "no knee through N=10000" -- i.e. maximal + # capacity -- for a relay that established nothing. An idle (S,G) is + # deliberate here (the witness is tunnel-table occupancy, not data + # receipt), but "idle" must not be allowed to shade into "the relay + # ignores this tuple". One tunnel must come up first, or the ramp is void. + - name: N=1 positive control (a ramp on a non-establishing tuple is void) + run: | + set +e + docker run --rm --network host \ + --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ + $IMAGE \ + --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ + --tunnels 1 --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ + >control.json 2>control.log + status=$? + set -e + echo "control_exit=$status"; cat control.json || true + alive=$(jq -r '.alive // 0' control.json 2>/dev/null || echo 0) + if [ "$alive" != "1" ]; then + echo "::error::N=1 control did not establish+survive (alive=$alive). Ramp aborted: a sweep from here would report a false ceiling." + cat control.log || true + exit 1 + fi + + # INSTRUMENT CEILINGS, MEASURED IMMEDIATELY BEFORE THE RAMP. + # + # Each of these would present as a relay-side state knee if it bound + # first, which is the specific way this measurement gets silently wrong: + # the ramp would report a ceiling that is really the runner's. + # + # Every ceiling below is either ENFORCED against max N or explicitly + # labelled an OBSERVATION with its failure signature named. Recording a + # number and then not comparing it is the worst of both: it reads as a + # guard in the log and guards nothing. + # + # ENFORCED (local runner only; ss cannot establish upstream NAT headroom) + # port range -- the relay keys tunnels by source (IP, port), so N + # tunnels need N distinct source ports. We subtract + # ports already held by UDP sockets immediately before + # the ramp, plus ip_local_reserved_ports. This is only + # local bind headroom; ss cannot establish upstream + # NAT capacity. + # nofile -- one socket per tunnel. The RUNNER's soft default is + # 1024, but the client runs in a container with + # --ulimit nofile=$CONTAINER_NOFILE, so the container + # value is the binder and is what gets compared. The + # runner's own limit is recorded as context only. + # conntrack -- this runner's conntrack table; N flows need N entries. + # We subtract current occupancy from the table maximum. + # It is local evidence only, not an upstream-NAT claim. + # If either value is unreadable, the ramp fails closed + # instead of producing a non-attributable ceiling. # # OBSERVATIONS (recorded, NOT enforced -- no defensible per-tunnel # constant to compare against, so a threshold here would be invented) @@ -603,6 +715,36 @@ jobs: lo=$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range) hi=$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range) echo "source_ports_total=$((hi-lo+1))" + reserved=$(cat /proc/sys/net/ipv4/ip_local_reserved_ports 2>/dev/null || true) + reserved_info=$(awk -v lo="$lo" -v hi="$hi" -v raw="$reserved" ' + BEGIN { + count = 0; bad = 0 + if (raw == "") { print "0 0"; exit } + n = split(raw, ranges, ",") + for (i = 1; i <= n; i++) { + token = ranges[i] + if (token !~ /^[0-9]+(-[0-9]+)?$/) { bad++; continue } + split(token, bounds, "-") + first = bounds[1] + 0 + last = (bounds[2] == "" ? first : bounds[2] + 0) + if (last < first) { bad++; continue } + if (first < lo) first = lo + if (last > hi) last = hi + if (first <= last) { + # The kernel normally emits sorted, merged ranges, but the + # contract does not require that. Deduplicate overlaps so + # reservations cannot be over-counted and turn a valid + # headroom reading into a false fail-closed result. + for (port = first; port <= last; port++) { + if (!seen[port]++) count++ + } + } + } + printf "%d %d\n", count, bad + }') + echo "ip_local_reserved_ports=${reserved:-none}" + echo "source_ports_reserved=${reserved_info%% *}" + echo "source_ports_reserved_parse_errors=${reserved_info#* }" if command -v ss >/dev/null 2>&1 && ss -H -u -a -n >udp-sockets.txt 2>/dev/null; then awk -v lo="$lo" -v hi="$hi" ' { @@ -625,25 +767,35 @@ jobs: echo "mem_total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) # OBSERVATION" } | tee host-ceilings.txt + is_uint() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; } ports_total=$(sed -n 's/^source_ports_total=//p' host-ceilings.txt) + ports_reserved=$(sed -n 's/^source_ports_reserved=//p' host-ceilings.txt) + reserved_bad=$(sed -n 's/^source_ports_reserved_parse_errors=//p' host-ceilings.txt) ports_used=$(sed -n 's/^source_ports_used=//p' host-ceilings.txt) ports_bad=$(sed -n 's/^source_ports_parse_errors=//p' host-ceilings.txt) - if ! printf '%s\n' "$ports_total" "$ports_used" "$ports_bad" | grep -Eq '^[0-9]+$' || [ "$ports_bad" -ne 0 ]; then - echo "::error::could not establish current UDP source-port occupancy; the ramp is non-attributable" + if ! is_uint "$ports_total" || ! is_uint "$ports_reserved" || \ + ! is_uint "$reserved_bad" || ! is_uint "$ports_used" || \ + ! is_uint "$ports_bad" || [ "$ports_bad" -ne 0 ] || \ + [ "$reserved_bad" -ne 0 ]; then + echo "::error::could not establish current local UDP source-port occupancy/reservations; the ramp is non-attributable" exit 1 fi - ports_available=$((ports_total - ports_used)) - [ "$ports_available" -ge 0 ] || { echo "::error::UDP source-port occupancy exceeds the configured range"; exit 1; } + ports_usable=$((ports_total - ports_reserved)) + ports_available=$((ports_usable - ports_used)) + [ "$ports_usable" -ge 0 ] && [ "$ports_available" -ge 0 ] || { + echo "::error::local UDP source-port occupancy/reservations exceed the configured range"; exit 1; + } + echo "source_ports_usable=$ports_usable" | tee -a host-ceilings.txt echo "source_ports_available=$ports_available" | tee -a host-ceilings.txt ct=$(sed -n 's/^nf_conntrack_max=//p' host-ceilings.txt) ct_used=$(sed -n 's/^nf_conntrack_used=//p' host-ceilings.txt) - if ! printf '%s\n' "$ct" "$ct_used" | grep -Eq '^[0-9]+$'; then - echo "::error::could not establish current conntrack occupancy; the ramp is non-attributable" + if ! is_uint "$ct" || ! is_uint "$ct_used"; then + echo "::error::could not establish current local conntrack occupancy; the ramp is non-attributable" exit 1 fi ct_available=$((ct - ct_used)) - [ "$ct_available" -ge 0 ] || { echo "::error::conntrack occupancy exceeds nf_conntrack_max"; exit 1; } + [ "$ct_available" -ge 0 ] || { echo "::error::local conntrack occupancy exceeds nf_conntrack_max"; exit 1; } echo "nf_conntrack_available=$ct_available" | tee -a host-ceilings.txt # The largest requested N must fit under every ENFORCED ceiling, or @@ -655,7 +807,7 @@ jobs: ports=$(sed -n 's/^source_ports_available=//p' host-ceilings.txt) if [ "$max_n" -ge "$ports" ]; then - echo "::error::max N ($max_n) leaves no source-port headroom (only $ports currently free) -- the ramp would measure SNAT exhaustion, not relay state" + echo "::error::max N ($max_n) leaves no local source-port headroom (only $ports currently free) -- this would measure local bind/SNAT behavior, not relay state" fail=1 fi @@ -669,80 +821,12 @@ jobs: ct=$(sed -n 's/^nf_conntrack_available=//p' host-ceilings.txt) if [ "$max_n" -ge "$ct" ]; then - echo "::error::max N ($max_n) leaves no conntrack headroom (only $ct currently free) -- NAT state exhaustion would be recorded as a relay knee" + echo "::error::max N ($max_n) leaves no local conntrack headroom (only $ct currently free) -- local NAT state exhaustion would be recorded as a relay knee" fail=1 fi [ "$fail" -eq 0 ] || exit 1 - - uses: actions/checkout@v4 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # The tag pins by construction; this ASSERTS it. A receipt that is recorded - # and never compared is the same record-but-don't-enforce shape as the ramp - # contract and the host ceilings -- it reads as a guard in the artifact and - # guards nothing. Two lines, and it also catches the one case the tag alone - # cannot: a build that published under this sha with a different revision - # label. - - name: Image receipt - run: | - docker pull -q $IMAGE || docker build -q -t $IMAGE \ - --label org.opencontainers.image.revision=${{ github.sha }} . - { - echo "image_digest=$(docker inspect --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}local-build{{end}}' $IMAGE)" - echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" - echo "workflow_sha=${{ github.sha }}" - } | tee image-receipt.txt - cc=$(sed -n 's/^client_commit=//p' image-receipt.txt) - [ "$cc" = "${{ github.sha }}" ] || { - echo "::error::image revision ($cc) != checkout (${{ github.sha }}) -- the client under test is not the client that would run"; exit 1; } - - # BASELINE OCCUPANCY, before the ramp adds anything. - # - # The withheld cap steps rested on "the linux relay has served zero tunnels - # for a week, so there is no live subscriber to displace". That read - # amt_relay_active_tunnels=0 as a fact about the relay when it is a fact - # about the gauge -- the same gauge a two-tunnel control had already shown - # stays flat at 0 while two tunnels were held. This records what the relay - # says about ITSELF instead, before the ramp perturbs it. Non-fatal: it is - # evidence for sizing the NEXT run, not a gate on this one. - - name: Baseline relay occupancy (_astats, pre-ramp) - run: | - curl -fsS -m 5 "http://$RELAY:8080/_astats" | tee astats-baseline.json || \ - echo "::warning::_astats unreachable from this public vantage -- baseline unknown, and the candidate occupancy instrument cannot be qualified from here" - - # POSITIVE CONTROL, and it gates the ramp. - # - # BLO-20175's liveness AC exists because a relay that answers NOTHING - # produces a clean-looking sweep: 0 tunnels, 0 drops, no stop rule ever - # trips, and the ramp reports "no knee through N=10000" -- i.e. maximal - # capacity -- for a relay that established nothing. An idle (S,G) is - # deliberate here (the witness is tunnel-table occupancy, not data - # receipt), but "idle" must not be allowed to shade into "the relay - # ignores this tuple". One tunnel must come up first, or the ramp is void. - - name: N=1 positive control (a ramp on a non-establishing tuple is void) - run: | - set +e - docker run --rm --network host \ - --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ - $IMAGE \ - --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ - --tunnels 1 --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ - >control.json 2>control.log - status=$? - set -e - echo "control_exit=$status"; cat control.json || true - alive=$(jq -r '.alive // 0' control.json 2>/dev/null || echo 0) - if [ "$alive" != "1" ]; then - echo "::error::N=1 control did not establish+survive (alive=$alive). Ramp aborted: a sweep from here would report a false ceiling." - cat control.log || true - exit 1 - fi - # Per-step UTC start/end. The relay-side join these were originally for is # off the table (amt_relay_active_tunnels is dead -- see the job header), # but they stay: they are what lets a step be correlated with the relay diff --git a/tests/probe_verdict_test.sh b/tests/probe_verdict_test.sh index 494b4f3..870a9cc 100755 --- a/tests/probe_verdict_test.sh +++ b/tests/probe_verdict_test.sh @@ -38,6 +38,28 @@ REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) WORKFLOW="$REPO_ROOT/.github/workflows/amt-public-vantage-probe.yml" WORK=$(mktemp -d); trap 'rm -rf "$WORK"' EXIT +# The ramp's host-capacity guard must validate every scalar before arithmetic. +# A single numeric-looking line is not enough: the old `printf ... | grep` +# check accepted a malformed sibling line because grep searched for *any* match. +# Keep this small contract test next to the workflow test so that a future edit +# cannot silently reintroduce arithmetic on `unavailable`/partial receipts. +is_uint() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; } +for value in 0 42 1048576; do + is_uint "$value" || { echo "FAIL: expected unsigned integer: $value"; exit 1; } +done +for value in '' unavailable '42\nnope' '1x' '-1'; do + if is_uint "$value"; then + echo "FAIL: malformed host-capacity value accepted: $value" + exit 1 + fi +done +grep -q 'is_uint()' "$WORKFLOW" || { + echo "FAIL: workflow has no fail-closed scalar validator"; exit 1; +} +if grep -q "printf '%s\\\\n'.*grep -Eq '^[0-9]" "$WORKFLOW"; then + echo "FAIL: workflow still validates a list with any-match grep"; exit 1 +fi + python3 - "$WORKFLOW" "$WORK/probe.sh" <<'PY' import sys, yaml wf, out = sys.argv[1], sys.argv[2] From 517e813ba5247246007ec967681ca173ea6f35fe Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Mon, 21 Sep 2026 17:19:14 +0000 Subject: [PATCH 09/11] fix(tests): make the is_uint newline fixture a real newline Ally review at head 5a066a0, Important (1): '42\nnope' was single-quoted, so the fixture was the literal 8-character string backslash-n and never exercised the multi-line case the comment claims to guard. Mutation-tested both ways: the old printf|grep any-match validator passed the suite as written and now fails it. Uses $'42\nnope' and adds $'42\n42', the every-line-numeric input the old check was surest about. Co-Authored-By: Claude Fable 5.1 --- tests/probe_verdict_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/probe_verdict_test.sh b/tests/probe_verdict_test.sh index 870a9cc..daa0b75 100755 --- a/tests/probe_verdict_test.sh +++ b/tests/probe_verdict_test.sh @@ -47,7 +47,10 @@ is_uint() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; } for value in 0 42 1048576; do is_uint "$value" || { echo "FAIL: expected unsigned integer: $value"; exit 1; } done -for value in '' unavailable '42\nnope' '1x' '-1'; do +# $'...' so the newline is real: a single-quoted '42\nnope' is the literal +# 8-char string backslash-n and never reached the multi-line case. $'42\n42' +# (every line numeric) is the input the old any-match grep was surest about. +for value in '' unavailable $'42\nnope' $'42\n42' '1x' '-1'; do if is_uint "$value"; then echo "FAIL: malformed host-capacity value accepted: $value" exit 1 From d3896d35de194bf541fe7652a9ffda5a40b64914 Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Tue, 22 Sep 2026 03:19:12 +0000 Subject: [PATCH 10/11] ci(probe): describe the aliasing gate's default in the direction the code takes The comment above the distinct_outer_sources gate named `// .requested` as the safe default, but the jq uses `// 1`, and only that direction fires the gate for a report predating the field. Say so, and say why the alternative would skip the gate for exactly those reports. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/amt-public-vantage-probe.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index a2b5c30..6e376df 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -946,8 +946,11 @@ jobs: # single source address is the WORST case, not the best: it reports a # ceiling having established one tunnel. `alive` is a send-side # self-report, so nothing downstream of it can notice. - # `// .requested` defaults a report predating the field to "shared", - # i.e. an unknown fails to the safe side rather than passing the gate. + # `// 1` defaults a report predating the field to ONE distinct + # source, so an unknown fires this gate rather than passing it: + # 1 < requested holds for every N > 1. (`// .requested` would read + # N < N, false, and skip the gate for exactly the legacy reports it + # exists to catch.) aliased=$(jq -r '[.[]|select(((.report.distinct_outer_sources // 1)) < .requested)]|length' ramp-index.json) if [ "$aliased" -gt 0 ]; then echo "verdict=WITNESS NOT ESTABLISHED ($aliased/$ran steps ran all gateways from ONE outer source address). NOT a ceiling, NOT a knee, and NOT 'no knee': linux-amt as deployed keys tunnel state on the outer ADDRESS alone, so N gateways from one address occupy ONE relay tunnel entry while every one still reads Active and sends keep-alives. An alive of N here is indistinguishable from one tunnel aliased N ways, so this run says nothing about the relay's tunnel table at any N. Do NOT size against this number. Needs distinct source addresses (linux-amt kernel/selftests/amt_capacity.sh) and/or the BLO-33636 keying fix." | tee verdict.txt From 367f691118b6656c478481460623401f09877eec Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Wed, 23 Sep 2026 02:01:34 +0000 Subject: [PATCH 11/11] fix(ramp): compare verdict cap numerically, strip blanks Ally review at head d3896d3, Important (1): the tunnels-ramp Verdict step derived `cap` from the raw STEPS text while `top` came from `jq -r`, and compared them as strings. `Validate ramp steps` word-splits, so `1, 8, 1024` is accepted, but the leading blank survived into `cap` and `[ "$top" = "$cap" ]` read `[ "1024" = " 1024" ]`. A fully clean ramp then fell through to "degraded above N=1024, BINDER NOT ESTABLISHED (0 cause(s))". Strip blanks before splitting (`tr -d '[:blank:]'`, not `[:space:]`, which would also delete the newlines `tr ',' '\n'` just inserted and collapse `1,8,1024` into `181024`) and compare with `-eq`. `max_n` at the host ceiling guard is left alone: it only reaches `-ge` arithmetic. tests/probe_verdict_test.sh now also extracts the tunnels-ramp Verdict run-block and drives it against synthetic ramp-index.json fixtures: clean-to-cap (bare and comma-space steps), aliased, legacy report without distinct_outer_sources, relay-attributed knee, unknown-cause degraded. Controls: `bash tests/probe_verdict_test.sh` ALL PASS with the fix; with the workflow change stashed the comma-space case FAILS and the script exits 1; restored, ALL PASS. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Omar Ramadan --- .../workflows/amt-public-vantage-probe.yml | 4 +- tests/probe_verdict_test.sh | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 6e376df..4cbec2c 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -918,7 +918,7 @@ jobs: # "every N up to here held", which is the claim BLO-20175 sizes against. - name: Verdict run: | - cap=$(echo "$STEPS" | tr ',' '\n' | sort -n | tail -1) + cap=$(echo "$STEPS" | tr -d '[:blank:]' | tr ',' '\n' | sort -n | tail -1) top=$(jq -r ' (map((.report.alive // -1) == .requested) | index(false)) as $i | (if $i == null then . else .[0:$i] end) @@ -954,7 +954,7 @@ jobs: aliased=$(jq -r '[.[]|select(((.report.distinct_outer_sources // 1)) < .requested)]|length' ramp-index.json) if [ "$aliased" -gt 0 ]; then echo "verdict=WITNESS NOT ESTABLISHED ($aliased/$ran steps ran all gateways from ONE outer source address). NOT a ceiling, NOT a knee, and NOT 'no knee': linux-amt as deployed keys tunnel state on the outer ADDRESS alone, so N gateways from one address occupy ONE relay tunnel entry while every one still reads Active and sends keep-alives. An alive of N here is indistinguishable from one tunnel aliased N ways, so this run says nothing about the relay's tunnel table at any N. Do NOT size against this number. Needs distinct source addresses (linux-amt kernel/selftests/amt_capacity.sh) and/or the BLO-33636 keying fix." | tee verdict.txt - elif [ "$top" = "$cap" ]; then + elif [ "$top" -eq "$cap" ]; then echo "verdict=ceiling >= $cap, config-bound (NOT 'no state knee': the ramp ended at the configured cap, so no knee was reached or excluded above it)" | tee verdict.txt elif [ "$n_relay" -gt 0 ] && [ "$n_other" -eq 0 ]; then echo "verdict=knee at N>$top (first degraded step above $top; every shortfall is relay-attributed -- see ramp-index.json)" | tee verdict.txt diff --git a/tests/probe_verdict_test.sh b/tests/probe_verdict_test.sh index daa0b75..bcfbe19 100755 --- a/tests/probe_verdict_test.sh +++ b/tests/probe_verdict_test.sh @@ -157,4 +157,53 @@ run_case "tunnels=abc rejected" abc "0" "1" 92 run_case "tunnels=257 over documented cap rejected" 257 "0" "1" 92 run_case "tunnels='' rejected" "" "0" "1" 92 + +# Ally review of #22 (head d3896d3), Important (1). The tunnels-ramp Verdict +# step derived `cap` from the raw STEPS text while `top` came from `jq -r`, and +# compared them as strings: `1, 8, 1024` (accepted by `Validate ramp steps`, +# which word-splits) left a leading blank in `cap`, so a fully clean ramp fell +# through to "degraded ... BINDER NOT ESTABLISHED (0 cause(s))". Drive the real +# Verdict run-block against synthetic ramp-index.json fixtures. It needs only +# `jq` and `tee`, so no docker stub. +python3 - "$WORKFLOW" "$WORK/verdict.sh" <<'PY' +import sys, yaml +wf, out = sys.argv[1], sys.argv[2] +steps = yaml.safe_load(open(wf))['jobs']['tunnels-ramp']['steps'] +open(out, 'w').write(next(s['run'] for s in steps if s.get('name') == 'Verdict')) +PY +bash -n "$WORK/verdict.sh" || { echo "FAIL tunnels-ramp Verdict step does not parse"; FAILED=1; } + +run_ramp_case() { + local name=$1 steps=$2 index=$3 want=$4 + local dir; dir=$(mktemp -d -p "$WORK") + printf '%s' "$index" >"$dir/ramp-index.json" + ( cd "$dir" && STEPS=$steps bash "$WORK/verdict.sh" ) >/dev/null 2>&1 + if grep -qF "$want" "$dir/verdict.txt" 2>/dev/null; then + echo "PASS $name" + else + echo "FAIL $name: want '$want', got: $(head -c 200 "$dir/verdict.txt" 2>/dev/null)"; FAILED=1 + fi + rm -rf "$dir" +} + +CLEAN='[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":1024,"distinct_outer_sources":1024}}]' +run_ramp_case "clean ramp, bare steps -> config-bound" "1,8,1024" "$CLEAN" \ + "verdict=ceiling >= 1024, config-bound" +# Negative control for the string-compare regression: the validator accepts +# this spelling, so the verdict must read it identically. +run_ramp_case "clean ramp, comma-space steps -> config-bound (regression: cap kept leading blank)" "1, 8, 1024" "$CLEAN" \ + "verdict=ceiling >= 1024, config-bound" +run_ramp_case "aliased step supersedes clean cap" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":1024,"distinct_outer_sources":1}}]' \ + "verdict=WITNESS NOT ESTABLISHED" +run_ramp_case "legacy report without distinct_outer_sources -> aliased" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8}},{"requested":1024,"report":{"alive":1024,"distinct_outer_sources":1024}}]' \ + "verdict=WITNESS NOT ESTABLISHED (1/3" +run_ramp_case "degraded relay-attributed -> knee" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":900,"distinct_outer_sources":1024,"establish_errors":{"relay refused: tunnel table full":124}}}]' \ + "verdict=knee at N>8" +run_ramp_case "degraded unknown cause -> binder not established" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":900,"distinct_outer_sources":1024,"establish_errors":{"connection reset":124}}}]' \ + "verdict=degraded above N=8, BINDER NOT ESTABLISHED (1 cause(s)" + [ "$FAILED" = 0 ] && { echo "ALL PASS"; exit 0; } || { echo "FAILURES"; exit 1; }