diff --git a/.github/workflows/amt-public-vantage-probe.yml b/.github/workflows/amt-public-vantage-probe.yml index 5c8afb0..4cbec2c 100644 --- a/.github/workflows/amt-public-vantage-probe.yml +++ b/.github/workflows/amt-public-vantage-probe.yml @@ -30,6 +30,38 @@ on: packet_count: description: Packets to receive before reporting ok default: "3" + mode: + description: >- + oneshot = the BLO-21823 data-receipt probe (default, unchanged). + tunnels-ramp = the BLO-33457 tunnel-STATE occupancy ramp. + type: choice + options: [oneshot, tunnels-ramp] + default: oneshot + ramp_steps: + description: >- + tunnels-ramp only. Comma-separated N values, ascending. The cap is + AMT_MAX_TUNNELS=10000 on production-amt-relay-linux (read live from + amt_relay_tunnel_limit, 2026-09-12), but the DEFAULT stops at 1024: + 4096/8192/10000 are withheld pending a working relay-side occupancy + instrument (CTO ruling 2026-09-12). 1024 against a 10000 cap cannot + displace a live subscriber even if the table is occupied, which is why + it is safe without one; the larger steps rested on "the table is + empty", and the only evidence for that was amt_relay_active_tunnels, + which is the gauge the same ruling declared dead. Raise this only with + an occupancy reading that has been shown capable of returning a + positive. + default: "1,8,64,256,1024" + hold: + description: >- + tunnels-ramp only. Seconds to hold tunnels open after the last one is + established, before reading the survival verdict. Must exceed + --keepalive so every tunnel can emit >=1 keep-alive Membership Update + (BLO-33457 AC 3); amt-verify exits 2 rather than reporting a false + knee if it does not. + default: "25" + keepalive: + description: tunnels-ramp only. Keep-alive interval in seconds. + default: "10" tunnels: description: >- BLO-26574 step 1: number of CONCURRENT data-receiving tunnels to run @@ -50,8 +82,34 @@ permissions: contents: read packages: read +# THE IMAGE IS PINNED TO THE CHECKOUT, and that is the whole point. +# +# This was `:main` in all eight places. `:main` is whatever was last pushed to +# main, which is NOT the ref this run checked out -- so a run dispatched from a +# branch executes main's client while every string the job emits (the ramp +# contract, CONTROL_PLANE_CAVEAT, the receiver-side witness classify_held_tunnel) +# comes from the branch. The witness under test is then not the witness that +# runs, and the artifact reports the branch's caveat over main's measurement. +# +# publish-amt-verify.yml already pushes ghcr.io/blockcast/amt-verify: on +# every main build, so pinning to github.sha costs nothing and cannot go stale: +# the tag IS the commit. +# +# But that publisher only fires on main, so on a BRANCH no such tag exists and +# the pull fails -- which made this workflow undispatchable from the very PR it +# is supposed to verify (Ally, 945d069). So: pull, and on a miss BUILD from the +# checkout. Both paths yield the checkout's client, by tag identity on main and +# by construction on a branch. On the build path the revision assertion below is +# tautological -- provenance there rests on having built the checked-out tree, +# not on the label -- and the receipt says `local-build` rather than a registry +# digest, because there isn't one and an empty field would read as a missing +# measurement rather than an absent one. +env: + IMAGE: ghcr.io/blockcast/amt-verify:${{ github.sha }} + jobs: probe: + if: ${{ (inputs.mode || 'oneshot') == 'oneshot' }} runs-on: ubuntu-latest env: RELAY: ${{ inputs.relay || '69.25.95.128' }} @@ -113,6 +171,7 @@ jobs: v6ok=no if curl -fsS -6 -m 5 -o /dev/null https://api6.ipify.org 2>/dev/null; then v6ok=yes; fi echo "ipv6_global_addrs=${v6n:-0} ipv6_egress=$v6ok" + - uses: actions/checkout@v4 - uses: docker/login-action@v3 with: registry: ghcr.io @@ -137,10 +196,11 @@ jobs: if [ "$TUNNELS" -lt 1 ] || [ "$TUNNELS" -gt "$MAX_TUNNELS" ]; then echo "::error::tunnels must be an integer in 1..$MAX_TUNNELS, got '$TUNNELS'"; exit 92 fi - docker pull -q ghcr.io/blockcast/amt-verify:main - echo "image_digest=$(docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/blockcast/amt-verify:main)" + docker pull -q $IMAGE || docker build -q -t $IMAGE \ + --label org.opencontainers.image.revision=${{ github.sha }} . + echo "image_digest=$(docker inspect --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}local-build{{end}}' $IMAGE)" # BLO-21823 AC#3 wants the client commit alongside the artifact hash. - echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' ghcr.io/blockcast/amt-verify:main)" + echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" set +e # BLO-33456. `mmtp.gaps` says packets went missing; it does NOT say # where. The kernel already counts the one candidate that is cheapest @@ -210,7 +270,7 @@ jobs: # to the single `docker run` it replaces, including the exit code. pids="" for i in $(seq 1 "$TUNNELS"); do - docker run --rm --network host ghcr.io/blockcast/amt-verify:main \ + docker run --rm --network host $IMAGE \ --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ --timeout "$TIMEOUT" --packet-count "$PACKETS" --json --verbose \ >"report-$i.json" 2>"verbose-$i.log" & @@ -413,3 +473,540 @@ jobs: path: | report*.json verbose*.log + + # BLO-33457. Tunnel-STATE occupancy ramp: how many concurrent AMT tunnels can + # the linux relay hold in its tunnel table? This is the MX301-M leg of + # BLO-20175's sizing model, which is a STATE claim (tunnel-limit 16,384), not + # the throughput claim BLO-26574 measures. + # + # Target is 69.25.95.128:2268 = Service/blockcastd-amt-relay-linux + # (LoadBalancer, selector app.kubernetes.io/name=amt-relay-linux), verified + # 2026-09-12. The OTHER relay -- Service/blockcastd-amt-relay, the gw-* one + # whose job="production-amt-relay" reports amt_relay_tunnel_limit=0 -- is + # ClusterIP-only and has no public 2268, so a public-vantage ramp cannot + # accidentally measure it. + # + # THE WITNESS IS RECEIVER-SIDE, and that is a correction, not a preference. + # The AC originally wanted amt_relay_active_tunnels{job="production-amt-relay-linux"} + # corroborating each step. That gauge is DEAD on both production relays + # (linux: never non-zero in 30d; juniper: zero variance in 30d, pinned at 1 -- + # BLO-33457). A ramp joined against it reports 0 occupancy at every N, which + # reads as a relay defect rather than a dead dial. So the witness is + # `alive` from the client: how many of N reached Active and sustained it past + # a keep-alive interval. Its positive control is in-tree + # (each_deliberate_failure_mode_reduces_the_alive_count). + tunnels-ramp: + if: ${{ inputs.mode == 'tunnels-ramp' }} + runs-on: ubuntu-latest + env: + RELAY: ${{ inputs.relay || '69.25.95.128' }} + SOURCE: ${{ inputs.source || '69.25.95.192' }} + GROUP: ${{ inputs.group || '232.1.1.60' }} + # Must MATCH the declared input default. It read + # `...,1024,4096,8192,10000` -- the pre-ruling list -- so a dispatch that + # supplied ramp_steps as an empty string (REST does; the UI cannot) would + # silently select the four steps the CTO withheld, via the one path that + # skips the input default entirely. + STEPS: ${{ inputs.ramp_steps || '1,8,64,256,1024' }} + HOLD: ${{ inputs.hold || '25' }} + KEEPALIVE: ${{ inputs.keepalive || '10' }} + # Raised inside the container for every docker run below. Declared once so + # the value enforced against max N cannot drift from the value applied. + CONTAINER_NOFILE: '1048576' + # Instrument ceiling, not a relay limit: ~10 KB/tunnel measured to N=10000 + # in one process (tunnels_mode_scales_to_512_concurrent doc comment). + # Above this the RIG is unvalidated, so a knee there is unattributable. + MAX_SUPPORTED_N: '10000' + steps: + - name: Vantage receipt + run: | + echo "utc=$(date -u +%FT%TZ)" + echo "public_ip=$(curl -fsS https://api.ipify.org)" + echo "target=$RELAY:2268 sg=($SOURCE,$GROUP) steps=$STEPS" + + # THE STEP LIST IS A CONTRACT AND IT WAS NEVER CHECKED. + # + # Everything downstream assumes ascending order. The ramp stops at the + # first step that establishes nothing, and the verdict reads the run of + # fully-alive steps as a prefix. Fed `1024,8` those two disagree with each + # other and both report confidently: the ramp could stop below the + # configured cap, or the verdict could name a knee above an N that had + # already failed. Neither shows up as an error -- the numbers just mean + # something other than what they say. + - name: Validate ramp steps + run: | + steps=$(echo "$STEPS" | tr ',' ' ') + [ -n "$(echo "$steps" | tr -d ' ')" ] || { echo "::error::ramp_steps is empty"; exit 1; } + prev=0 + for n in $steps; do + case "$n" in + ''|*[!0-9]*) echo "::error::ramp step '$n' is not a non-negative integer"; exit 1 ;; + esac + [ "$n" -gt 0 ] || { echo "::error::ramp step '$n' must be > 0"; exit 1; } + if [ "$n" -le "$prev" ]; then + echo "::error::ramp steps must be STRICTLY ASCENDING ($n follows $prev). The stop rule and the verdict both read the list in order; unsorted input makes both wrong without failing." + exit 1 + fi + if [ "$n" -gt "$MAX_SUPPORTED_N" ]; then + echo "::error::ramp step $n exceeds MAX_SUPPORTED_N=$MAX_SUPPORTED_N -- the rig is only validated to that N, so a knee above it could not be attributed to the relay" + exit 1 + fi + prev=$n + done + echo "steps_validated=$steps" + + # INSTRUMENT CEILINGS, MEASURED IMMEDIATELY BEFORE THE RAMP. + # + # Each of these would present as a relay-side state knee if it bound + # first, which is the specific way this measurement gets silently wrong: + # the ramp would report a ceiling that is really the runner's. + # + # Every ceiling below is either ENFORCED against max N or explicitly + # labelled an OBSERVATION with its failure signature named. Recording a + # number and then not comparing it is the worst of both: it reads as a + # guard in the log and guards nothing. + # + # ENFORCED (local runner only; ss cannot establish upstream NAT headroom) + # port range -- the relay keys tunnels by source (IP, port), so N + # tunnels need N distinct source ports. We subtract + # ports already held by UDP sockets immediately before + # the ramp, plus ip_local_reserved_ports. This is only + # local bind headroom; ss cannot establish upstream + # NAT capacity. + # nofile -- one socket per tunnel. The RUNNER's soft default is + # 1024, but the client runs in a container with + # --ulimit nofile=$CONTAINER_NOFILE, so the container + # value is the binder and is what gets compared. The + # runner's own limit is recorded as context only. + # conntrack -- this runner's conntrack table; N flows need N entries. + # We subtract current occupancy from the table maximum. + # It is local evidence only, not an upstream-NAT claim. + # If either value is unreadable, the ramp fails closed + # instead of producing a non-attributable ceiling. + # + # OBSERVATIONS (recorded, NOT enforced -- no defensible per-tunnel + # constant to compare against, so a threshold here would be invented) + # nproc / mem -- these do not produce a clean shortfall. CPU + # starvation shows up as establishment TIMEOUTS + # (establish_errors, "timed out ... Discovering") and + # memory pressure as an OOM-killed container (step + # exit 137, established=0). Both are distinguishable + # from a relay knee in the artifact IF you look: a + # relay-side ceiling refuses new tunnels while the + # established ones stay alive, whereas a host-side one + # degrades the whole step. Read establish_errors and + # not_alive before attributing any knee. + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The tag pins by construction; this ASSERTS it. A receipt that is recorded + # and never compared is the same record-but-don't-enforce shape as the ramp + # contract and the host ceilings -- it reads as a guard in the artifact and + # guards nothing. Two lines, and it also catches the one case the tag alone + # cannot: a build that published under this sha with a different revision + # label. + - name: Image receipt + run: | + docker pull -q $IMAGE || docker build -q -t $IMAGE \ + --label org.opencontainers.image.revision=${{ github.sha }} . + { + echo "image_digest=$(docker inspect --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}local-build{{end}}' $IMAGE)" + echo "client_commit=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' $IMAGE)" + echo "workflow_sha=${{ github.sha }}" + } | tee image-receipt.txt + cc=$(sed -n 's/^client_commit=//p' image-receipt.txt) + [ "$cc" = "${{ github.sha }}" ] || { + echo "::error::image revision ($cc) != checkout (${{ github.sha }}) -- the client under test is not the client that would run"; exit 1; } + + # BASELINE OCCUPANCY, before the ramp adds anything. + # + # The withheld cap steps rested on "the linux relay has served zero tunnels + # for a week, so there is no live subscriber to displace". That read + # amt_relay_active_tunnels=0 as a fact about the relay when it is a fact + # about the gauge -- the same gauge a two-tunnel control had already shown + # stays flat at 0 while two tunnels were held. This records what the relay + # says about ITSELF instead, before the ramp perturbs it. Non-fatal: it is + # evidence for sizing the NEXT run, not a gate on this one. + - name: Baseline relay occupancy (_astats, pre-ramp) + run: | + curl -fsS -m 5 "http://$RELAY:8080/_astats" | tee astats-baseline.json || \ + echo "::warning::_astats unreachable from this public vantage -- baseline unknown, and the candidate occupancy instrument cannot be qualified from here" + + # POSITIVE CONTROL, and it gates the ramp. + # + # BLO-20175's liveness AC exists because a relay that answers NOTHING + # produces a clean-looking sweep: 0 tunnels, 0 drops, no stop rule ever + # trips, and the ramp reports "no knee through N=10000" -- i.e. maximal + # capacity -- for a relay that established nothing. An idle (S,G) is + # deliberate here (the witness is tunnel-table occupancy, not data + # receipt), but "idle" must not be allowed to shade into "the relay + # ignores this tuple". One tunnel must come up first, or the ramp is void. + - name: N=1 positive control (a ramp on a non-establishing tuple is void) + run: | + set +e + docker run --rm --network host \ + --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ + $IMAGE \ + --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ + --tunnels 1 --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ + >control.json 2>control.log + status=$? + set -e + echo "control_exit=$status"; cat control.json || true + alive=$(jq -r '.alive // 0' control.json 2>/dev/null || echo 0) + if [ "$alive" != "1" ]; then + echo "::error::N=1 control did not establish+survive (alive=$alive). Ramp aborted: a sweep from here would report a false ceiling." + cat control.log || true + exit 1 + fi + + # INSTRUMENT CEILINGS, MEASURED IMMEDIATELY BEFORE THE RAMP. + # + # Each of these would present as a relay-side state knee if it bound + # first, which is the specific way this measurement gets silently wrong: + # the ramp would report a ceiling that is really the runner's. + # + # Every ceiling below is either ENFORCED against max N or explicitly + # labelled an OBSERVATION with its failure signature named. Recording a + # number and then not comparing it is the worst of both: it reads as a + # guard in the log and guards nothing. + # + # ENFORCED (local runner only; ss cannot establish upstream NAT headroom) + # port range -- the relay keys tunnels by source (IP, port), so N + # tunnels need N distinct source ports. We subtract + # ports already held by UDP sockets immediately before + # the ramp, plus ip_local_reserved_ports. This is only + # local bind headroom; ss cannot establish upstream + # NAT capacity. + # nofile -- one socket per tunnel. The RUNNER's soft default is + # 1024, but the client runs in a container with + # --ulimit nofile=$CONTAINER_NOFILE, so the container + # value is the binder and is what gets compared. The + # runner's own limit is recorded as context only. + # conntrack -- this runner's conntrack table; N flows need N entries. + # We subtract current occupancy from the table maximum. + # It is local evidence only, not an upstream-NAT claim. + # If either value is unreadable, the ramp fails closed + # instead of producing a non-attributable ceiling. + # + # OBSERVATIONS (recorded, NOT enforced -- no defensible per-tunnel + # constant to compare against, so a threshold here would be invented) + # nproc / mem -- these do not produce a clean shortfall. CPU + # starvation shows up as establishment TIMEOUTS + # (establish_errors, "timed out ... Discovering") and + # memory pressure as an OOM-killed container (step + # exit 137, established=0). Both are distinguishable + # from a relay knee in the artifact IF you look: a + # relay-side ceiling refuses new tunnels while the + # established ones stay alive, whereas a host-side one + # degrades the whole step. Read establish_errors and + # not_alive before attributing any knee. + - name: Host ceilings (recorded, then enforced or labelled) + run: | + { + echo "runner_nofile_soft=$(ulimit -Sn)" + echo "runner_nofile_hard=$(ulimit -Hn)" + echo "container_nofile=$CONTAINER_NOFILE" + echo "ip_local_port_range=$(cat /proc/sys/net/ipv4/ip_local_port_range)" + lo=$(awk '{print $1}' /proc/sys/net/ipv4/ip_local_port_range) + hi=$(awk '{print $2}' /proc/sys/net/ipv4/ip_local_port_range) + echo "source_ports_total=$((hi-lo+1))" + reserved=$(cat /proc/sys/net/ipv4/ip_local_reserved_ports 2>/dev/null || true) + reserved_info=$(awk -v lo="$lo" -v hi="$hi" -v raw="$reserved" ' + BEGIN { + count = 0; bad = 0 + if (raw == "") { print "0 0"; exit } + n = split(raw, ranges, ",") + for (i = 1; i <= n; i++) { + token = ranges[i] + if (token !~ /^[0-9]+(-[0-9]+)?$/) { bad++; continue } + split(token, bounds, "-") + first = bounds[1] + 0 + last = (bounds[2] == "" ? first : bounds[2] + 0) + if (last < first) { bad++; continue } + if (first < lo) first = lo + if (last > hi) last = hi + if (first <= last) { + # The kernel normally emits sorted, merged ranges, but the + # contract does not require that. Deduplicate overlaps so + # reservations cannot be over-counted and turn a valid + # headroom reading into a false fail-closed result. + for (port = first; port <= last; port++) { + if (!seen[port]++) count++ + } + } + } + printf "%d %d\n", count, bad + }') + echo "ip_local_reserved_ports=${reserved:-none}" + echo "source_ports_reserved=${reserved_info%% *}" + echo "source_ports_reserved_parse_errors=${reserved_info#* }" + if command -v ss >/dev/null 2>&1 && ss -H -u -a -n >udp-sockets.txt 2>/dev/null; then + awk -v lo="$lo" -v hi="$hi" ' + { + if (NF < 4) { bad++; next } + raw = $4 + sub(/^.*:/, "", raw) + if (raw !~ /^[0-9]+$/) { bad++; next } + port = raw + 0 + if (port >= lo && port <= hi && !seen[port]++) used++ + } + END { printf "source_ports_used=%d\nsource_ports_parse_errors=%d\n", used + 0, bad + 0 } + ' udp-sockets.txt + else + echo "source_ports_used=unavailable" + echo "source_ports_parse_errors=unavailable" + fi + echo "nf_conntrack_max=$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null || echo unavailable)" + echo "nf_conntrack_used=$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null || echo unavailable)" + echo "nproc=$(nproc) # OBSERVATION" + echo "mem_total_kb=$(awk '/MemTotal/{print $2}' /proc/meminfo) # OBSERVATION" + } | tee host-ceilings.txt + + is_uint() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; } + ports_total=$(sed -n 's/^source_ports_total=//p' host-ceilings.txt) + ports_reserved=$(sed -n 's/^source_ports_reserved=//p' host-ceilings.txt) + reserved_bad=$(sed -n 's/^source_ports_reserved_parse_errors=//p' host-ceilings.txt) + ports_used=$(sed -n 's/^source_ports_used=//p' host-ceilings.txt) + ports_bad=$(sed -n 's/^source_ports_parse_errors=//p' host-ceilings.txt) + if ! is_uint "$ports_total" || ! is_uint "$ports_reserved" || \ + ! is_uint "$reserved_bad" || ! is_uint "$ports_used" || \ + ! is_uint "$ports_bad" || [ "$ports_bad" -ne 0 ] || \ + [ "$reserved_bad" -ne 0 ]; then + echo "::error::could not establish current local UDP source-port occupancy/reservations; the ramp is non-attributable" + exit 1 + fi + ports_usable=$((ports_total - ports_reserved)) + ports_available=$((ports_usable - ports_used)) + [ "$ports_usable" -ge 0 ] && [ "$ports_available" -ge 0 ] || { + echo "::error::local UDP source-port occupancy/reservations exceed the configured range"; exit 1; + } + echo "source_ports_usable=$ports_usable" | tee -a host-ceilings.txt + echo "source_ports_available=$ports_available" | tee -a host-ceilings.txt + + ct=$(sed -n 's/^nf_conntrack_max=//p' host-ceilings.txt) + ct_used=$(sed -n 's/^nf_conntrack_used=//p' host-ceilings.txt) + if ! is_uint "$ct" || ! is_uint "$ct_used"; then + echo "::error::could not establish current local conntrack occupancy; the ramp is non-attributable" + exit 1 + fi + ct_available=$((ct - ct_used)) + [ "$ct_available" -ge 0 ] || { echo "::error::local conntrack occupancy exceeds nf_conntrack_max"; exit 1; } + echo "nf_conntrack_available=$ct_available" | tee -a host-ceilings.txt + + # The largest requested N must fit under every ENFORCED ceiling, or + # the ramp is measuring this runner. Fail LOUDLY rather than produce + # a plausible number. + max_n=$(echo "$STEPS" | tr ',' '\n' | sort -n | tail -1) + echo "max_requested_n=$max_n" | tee -a host-ceilings.txt + fail=0 + + ports=$(sed -n 's/^source_ports_available=//p' host-ceilings.txt) + if [ "$max_n" -ge "$ports" ]; then + echo "::error::max N ($max_n) leaves no local source-port headroom (only $ports currently free) -- this would measure local bind/SNAT behavior, not relay state" + fail=1 + fi + + # One socket per tunnel plus the process's own handles. Headroom is + # deliberate: at the cap the shortfall would otherwise land exactly + # where the knee is expected. + if [ "$max_n" -ge "$CONTAINER_NOFILE" ]; then + echo "::error::max N ($max_n) is at or above the container nofile limit ($CONTAINER_NOFILE) -- 'Too many open files' would be recorded as a relay knee" + fail=1 + fi + + ct=$(sed -n 's/^nf_conntrack_available=//p' host-ceilings.txt) + if [ "$max_n" -ge "$ct" ]; then + echo "::error::max N ($max_n) leaves no local conntrack headroom (only $ct currently free) -- local NAT state exhaustion would be recorded as a relay knee" + fail=1 + fi + + [ "$fail" -eq 0 ] || exit 1 + + # Per-step UTC start/end. The relay-side join these were originally for is + # off the table (amt_relay_active_tunnels is dead -- see the job header), + # but they stay: they are what lets a step be correlated with the relay + # pod's own CPU/memory series, with LB behaviour, or with anything else + # measured later. Unrecordable after the fact, cheap now. + # + # CANDIDATE OCCUPANCY INSTRUMENT, SAMPLED UNDER QUALIFICATION. + # + # The relay serves its own stats as JSON on :8080/_astats, and that + # surface is NOT the dead gauge. Measured in-cluster 2026-09-12, same pod, + # seconds apart: _astats "amt.relay.active_tunnels"=29 while the Prometheus + # exporter's amt_relay_active_tunnels=0. So this one demonstrably returns a + # positive, which is exactly the property the gauge lacks. + # + # It is sampled here as an OBSERVATION and deliberately does NOT gate the + # step -- enforcing an instrument on the same run that is meant to qualify + # it is circular, and a 0 here would then void a ramp for the instrument's + # fault rather than the relay's. Qualification criterion, applied by hand + # to the artifact afterwards: does peak occupancy TRACK the requested N? + # If it moves to ~1024 while 1024 tunnels are held, it is qualified and it + # licenses the withheld cap steps. If it reads 0, it is a seventh dead dial. + # + # Both counters are kept because they are DIFFERENT FIELDS and were not + # equal when measured (relay.active_tunnels=29, interface.amtr.tunnels=47). + # Which of them tracks N is part of what this run establishes; assuming + # they are the same number is how a wrong one gets adopted. + # + # Public reachability of :8080 is unverified -- the pre-run read was via + # the LB IP from in-cluster, which kube-proxy may short-circuit. An + # unreachable endpoint records {} and costs the run nothing. + - name: Ramp + run: | + echo '[]' > ramp-index.json + for n in $(echo "$STEPS" | tr ',' ' '); do + t0=$(date -u +%FT%TZ) + echo "=== N=$n start=$t0 ===" + set +e + docker run --rm --network host \ + --ulimit "nofile=$CONTAINER_NOFILE:$CONTAINER_NOFILE" \ + $IMAGE \ + --no-driad --relay "$RELAY" --source "$SOURCE" --group "$GROUP" \ + --tunnels "$n" --keepalive "$KEEPALIVE" --hold "$HOLD" --json \ + >"step-$n.json" 2>"step-$n.log" & + pid=$! + # Sample for the whole step rather than at one guessed instant: with + # establishment time unknown a single sample can land before the + # tunnels are up and read a false 0. + : > "astats-$n.ndjson" + while kill -0 $pid 2>/dev/null; do + curl -fsS -m 3 "http://$RELAY:8080/_astats" >> "astats-$n.ndjson" 2>/dev/null + echo >> "astats-$n.ndjson" + sleep 2 + done + wait $pid + rc=$? + set -e + t1=$(date -u +%FT%TZ) + peak_relay=$(jq -s 'map(.ats."amt.relay.active_tunnels"//empty)|max//null' "astats-$n.ndjson" 2>/dev/null || echo null) + peak_iface=$(jq -s 'map(.ats."amt.interface.amtr.tunnels"//empty)|max//null' "astats-$n.ndjson" 2>/dev/null || echo null) + echo "N=$n exit=$rc end=$t1 astats_peak_relay_active_tunnels=$peak_relay astats_peak_iface_tunnels=$peak_iface" + cat "step-$n.json" || true + jq --argjson n "$n" --arg t0 "$t0" --arg t1 "$t1" --argjson rc "$rc" \ + --argjson pr "${peak_relay:-null}" --argjson pi "${peak_iface:-null}" \ + --slurpfile r "step-$n.json" \ + '. + [{requested:$n, started_utc:$t0, ended_utc:$t1, exit:$rc, + astats_peak_relay_active_tunnels:$pr, + astats_peak_iface_tunnels:$pi, + report:($r[0] // null)}]' \ + ramp-index.json > tmp.json && mv tmp.json ramp-index.json + # A degraded step IS the measurement -- it is where the knee is -- + # so the loop does not stop on a non-zero exit. It stops only when + # nothing came up at all, which means every larger N is wasted. + est=$(jq -r '.established // 0' "step-$n.json" 2>/dev/null || echo 0) + if [ "$est" = "0" ]; then + echo "::warning::N=$n established 0 tunnels; stopping the ramp" + break + fi + done + echo '--- ramp-index.json ---'; jq . ramp-index.json + + # AC 4: a clean completion at the cap is "ceiling >= cap, config-bound", + # NOT "no state knee". Those are different findings and only one of them + # is honest about what was and was not shown. + # + # `top` is the last step of the LEADING RUN of fully-alive steps, not the + # largest fully-alive step anywhere in the list. With a degraded step at + # 256 and a clean one at 1024, `max` would report a ceiling above an N + # that had already failed. The leading run is the only reading that says + # "every N up to here held", which is the claim BLO-20175 sizes against. + - name: Verdict + run: | + cap=$(echo "$STEPS" | tr -d '[:blank:]' | tr ',' '\n' | sort -n | tail -1) + top=$(jq -r ' + (map((.report.alive // -1) == .requested) | index(false)) as $i + | (if $i == null then . else .[0:$i] end) + | if length == 0 then 0 else .[-1].requested end' ramp-index.json) + ran=$(jq -r 'length' ramp-index.json) + echo "highest_fully_alive_N=$top cap=$cap steps_run=$ran" + # A shortfall is the RELAY's tunnel table only if its cause says so. + # This is a POSITIVE test for relay attribution, not a blocklist of + # host-side symptoms: an error string nobody has seen before must fall + # through to "not a relay knee", because that is the direction a capex + # number is safe to be wrong in. (An earlier blocklist spelling of this + # matched "refused" and so swallowed "relay refused: tunnel table + # full" -- the negative control below exists to keep that fixed.) + # Surfacing-without-enforcing is the fourth instance of that pattern in + # this file; the other three were the ramp contract, the host ceilings, + # and the image label. + causes=$(jq -r '.[]|select((.report.alive // -1) != .requested) + |(((.report.establish_errors // {})|keys[]?),((.report.not_alive // {})|keys[]?))' ramp-index.json) + n_relay=$(echo "$causes" | grep -cEi 'relay (refused|rejected)|tunnel[ -](table|limit)|no more tunnels' || true) + n_other=$(echo "$causes" | grep -cvEi 'relay (refused|rejected)|tunnel[ -](table|limit)|no more tunnels' || true) + [ -z "$causes" ] && n_relay=0 && n_other=0 + # Does any step admit it could not tell N tunnels from one aliased N + # ways? This gate runs FIRST and supersedes every branch below, + # including the clean-to-cap one -- a ramp that sails to the cap on a + # single source address is the WORST case, not the best: it reports a + # ceiling having established one tunnel. `alive` is a send-side + # self-report, so nothing downstream of it can notice. + # `// 1` defaults a report predating the field to ONE distinct + # source, so an unknown fires this gate rather than passing it: + # 1 < requested holds for every N > 1. (`// .requested` would read + # N < N, false, and skip the gate for exactly the legacy reports it + # exists to catch.) + aliased=$(jq -r '[.[]|select(((.report.distinct_outer_sources // 1)) < .requested)]|length' ramp-index.json) + if [ "$aliased" -gt 0 ]; then + echo "verdict=WITNESS NOT ESTABLISHED ($aliased/$ran steps ran all gateways from ONE outer source address). NOT a ceiling, NOT a knee, and NOT 'no knee': linux-amt as deployed keys tunnel state on the outer ADDRESS alone, so N gateways from one address occupy ONE relay tunnel entry while every one still reads Active and sends keep-alives. An alive of N here is indistinguishable from one tunnel aliased N ways, so this run says nothing about the relay's tunnel table at any N. Do NOT size against this number. Needs distinct source addresses (linux-amt kernel/selftests/amt_capacity.sh) and/or the BLO-33636 keying fix." | tee verdict.txt + elif [ "$top" -eq "$cap" ]; then + echo "verdict=ceiling >= $cap, config-bound (NOT 'no state knee': the ramp ended at the configured cap, so no knee was reached or excluded above it)" | tee verdict.txt + elif [ "$n_relay" -gt 0 ] && [ "$n_other" -eq 0 ]; then + echo "verdict=knee at N>$top (first degraded step above $top; every shortfall is relay-attributed -- see ramp-index.json)" | tee verdict.txt + else + echo "verdict=degraded above N=$top, BINDER NOT ESTABLISHED ($n_other cause(s) not attributed to the relay). NOT a relay knee: a bare transport failure is equally consistent with the relay's tunnel table, public-path loss, and this runner's NAT/source-port or rate limiting. Do NOT size against this number." | tee verdict.txt + fi + jq -r '[.[]|select((.report.establish_errors // {}) != {} or (.report.not_alive // {}) != {}) + |{requested, establish_errors:.report.establish_errors, not_alive:.report.not_alive}]' \ + ramp-index.json | tee -a verdict.txt + echo "NOTE: control-plane tunnel-table ceiling, not a loaded-state ceiling. Witness is RECEIVER-SIDE (client 'alive'); relay-side amt_relay_active_tunnels is dead and was deliberately not used. Every step report carries the caveat inline." | tee -a verdict.txt + # Occupancy-instrument QUALIFICATION, reported next to the verdict so + # the two are read together. This does not corroborate THIS ramp -- an + # unqualified instrument cannot corroborate anything -- it says whether + # _astats earned the right to corroborate the NEXT one. + { + echo "--- candidate occupancy instrument (_astats :8080), qualification ---" + # `jq` exits 0 and prints NOTHING on an empty file, so the old + # `|| echo unreachable` could never fire: an unreachable instrument + # was rendered as an empty string and read as "no data". Test the + # file has bytes first -- an instrument that was never read must not + # be reported in the same shape as one that answered. + if [ -s astats-baseline.json ]; then + echo "baseline_pre_ramp=$(jq -c '.ats|{relay:."amt.relay.active_tunnels",iface:."amt.interface.amtr.tunnels",limit:."amt.relay.tunnel_limit"}' astats-baseline.json 2>/dev/null || echo malformed)" + else + echo "baseline_pre_ramp=UNREACHABLE (no bytes returned; _astats is in-cluster only and this job runs from the public runner)" + fi + jq -r '.[]|"requested=\(.requested) peak_relay_active_tunnels=\(.astats_peak_relay_active_tunnels) peak_iface_tunnels=\(.astats_peak_iface_tunnels) alive=\(.report.alive // "n/a")"' ramp-index.json + # Three outcomes, not two. A column of nulls means the instrument was + # NOT SAMPLED -- it is not evidence the dial is dead, and recording it + # as such would condemn a counter measured live at relay=29/iface=47 + # from in-cluster hours earlier. + if jq -e '[.[]|.astats_peak_relay_active_tunnels,.astats_peak_iface_tunnels]|all(.==null)' ramp-index.json >/dev/null; then + echo "qualification=NOT SAMPLED -- every peak is null, i.e. no reading was taken. This is NOT the 'dead dial' verdict and must not be recorded as one: it says nothing about _astats, only that this vantage could not reach it. Re-run with the sampler in-cluster." + else + echo "qualification=QUALIFIED if a peak column rises with requested N; DEAD DIAL if it stays flat at a value (that is the amt_relay_active_tunnels failure, and it would be the seventh). Read the two peak columns separately: they were NOT equal pre-ramp (relay=29, iface=47), so whichever tracks N is the occupancy counter and the other is something else." + fi + } | tee -a verdict.txt + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: amt-tunnels-ramp + path: | + host-ceilings.txt + image-receipt.txt + astats-baseline.json + astats-*.ndjson + control.json + control.log + ramp-index.json + verdict.txt + step-*.json + step-*.log + report*.json + verbose*.log diff --git a/src/bin/amt-verify.rs b/src/bin/amt-verify.rs index 384dcd9..a095c2a 100644 --- a/src/bin/amt-verify.rs +++ b/src/bin/amt-verify.rs @@ -849,8 +849,24 @@ fn validate_families( const CONTROL_PLANE_CAVEAT: &str = "Idle tunnels: this is the relay's CONTROL-PLANE tunnel-table \ ceiling, not a loaded-state ceiling. An idle tunnel may cost a relay less state than an active \ one. Comparable to a configured control-plane limit (Junos tunnel-limit, Linux AMT_MAX_TUNNELS); \ -NOT comparable to a measured loaded ceiling. Client-side counts must be corroborated by relay-side \ -amt_relay_active_tunnels at each step."; +NOT comparable to a measured loaded ceiling. WITNESS IS RECEIVER-SIDE: `alive` is how many of \ +`requested` reached Active and sustained it past a keep-alive interval, observed from this client. \ +It is NOT corroborated by relay-side amt_relay_active_tunnels, which is dead on both production \ +relays (linux: never non-zero in 30d; juniper: zero variance in 30d, pinned at 1) -- see BLO-33457. \ +ATTRIBUTION: a shortfall here is the relay's tunnel table ONLY if establish_errors and not_alive \ +are empty of host-side reasons; otherwise the binder is this rig, not the relay. \ +DISQUALIFIER -- READ BEFORE CITING ANY NUMBER FROM THIS MODE: every gateway here binds the SAME \ +outer source address (distinct ephemeral ports only), so `distinct_outer_sources` is 1 regardless \ +of `requested`. RFC 7450 s4.2.2 keys a tunnel on the (address, port) endpoint, but linux-amt as \ +deployed matches on the outer source ADDRESS alone (`tunnel->addr.ip4 == iph->saddr`, amt.c \ +amt_request_handler), overwriting `source_port` from the newest Request. So N gateways from one \ +address occupy exactly ONE relay tunnel entry while every one of them still reads Active and still \ +sends keep-alives -- `alive` is a SEND-side self-report (state == Active && keepalives_sent >= 1; \ +all three not-alive reasons are client-local) and cannot witness relay state at all. An `alive` of \ +N is therefore indistinguishable from one tunnel aliased N ways. This mode CANNOT produce a relay \ +tunnel-state ceiling; see BLO-33636 for the keying fix and linux-amt \ +kernel/selftests/amt_capacity.sh for a rig that provisions distinct source addresses and takes \ +ground truth from the relay's own admit/refuse reply."; /// Per-tunnel result. `establish_ms` is wall-clock from gateway construction to /// the gateway reporting `Active`. @@ -866,6 +882,46 @@ enum TunnelOutcome { Failed { error: String }, } +/// Verdict for one gateway at the end of the hold. Pure so each deliberate +/// failure mode can be driven in a unit test: an `alive` count that cannot be +/// shown to go DOWN is indistinguishable from one hardcoded to `established`, +/// and this whole measurement rests on it (BLO-33457 CTO condition 1). +/// +/// Order matters. `fatal_runtime_error` is checked first because a dead socket +/// leaves the published state stale, so a gateway whose runtime died can still +/// read `Active` — classifying that as relay behaviour is how a ramp turns the +/// INSTRUMENT's ceiling into a relay knee. +fn classify_held_tunnel( + has_fatal: bool, + state: GatewayState, + keepalives: u64, + establish_ms: u64, + rx: u64, +) -> TunnelOutcome { + if has_fatal { + TunnelOutcome::NotAlive { + reason: "fatal_runtime_error", + } + } else if state != GatewayState::Active { + TunnelOutcome::NotAlive { + reason: "state_left_active", + } + } else if keepalives == 0 { + // Held past one full keep-alive interval and emitted nothing: the + // tunnel was established but is not demonstrably maintained, so it + // must not be counted as live state. + TunnelOutcome::NotAlive { + reason: "no_keepalive_sent", + } + } else { + TunnelOutcome::Alive { + establish_ms, + keepalives, + rx, + } + } +} + #[derive(serde::Serialize)] struct TunnelsReport { outcome: &'static str, @@ -899,6 +955,21 @@ struct TunnelsReport { /// recorded as a state knee, whereas `state_left_active` / /// `no_keepalive_sent` are the tunnel genuinely failing to survive. not_alive: BTreeMap, + /// How many DISTINCT outer source addresses the N gateways were spread + /// across. Structurally 1 here: `run_tunnels` builds every gateway with + /// `AsyncAmtGateway::builder(relay)`, which binds the host's default + /// source, so the tunnels differ only by ephemeral port. + /// + /// This is the field that makes the mode's central limitation machine- + /// checkable instead of prose. A relay that keys tunnel state on the outer + /// ADDRESS alone -- which is what linux-amt does as deployed -- collapses + /// all N onto one tunnel entry, and because `alive` is a send-side + /// self-report every aliased gateway still counts. So whenever this is + /// less than `requested`, no tunnel-state ceiling can be read off the run, + /// and the verdict must say so rather than print a number. When a rig that + /// provisions distinct source addresses lands, this becomes N and the gate + /// opens on its own. + distinct_outer_sources: u32, caveat: &'static str, } @@ -986,6 +1057,11 @@ fn summarize( rx_datagrams_total: rx_total, establish_errors, not_alive, + // Every gateway in `run_tunnels` is built with `builder(relay)`, i.e. + // the host default source. Hardcoded rather than counted because there + // is exactly one call site and a counted-but-always-1 value would read + // as a measurement. + distinct_outer_sources: 1, caveat: CONTROL_PLANE_CAVEAT, } } @@ -1092,32 +1168,13 @@ async fn run_tunnels( for (gw, establish_ms) in &up { let keepalives = gw.keepalives_sent(); let rx = gw.rx_datagrams(); - outcomes.push(if gw.has_fatal().await { - // Unrecoverable socket error in this gateway's runtime. Attributed - // separately from `state_left_active` because it is the INSTRUMENT - // failing, not the relay evicting state — reading a host-side send - // failure as a relay knee is how a ramp under-reports the ceiling. - TunnelOutcome::NotAlive { - reason: "fatal_runtime_error", - } - } else if gw.state() != GatewayState::Active { - TunnelOutcome::NotAlive { - reason: "state_left_active", - } - } else if keepalives == 0 { - // Held past one full keep-alive interval and emitted nothing: the - // tunnel was established but is not demonstrably maintained, so it - // must not be counted as live state. - TunnelOutcome::NotAlive { - reason: "no_keepalive_sent", - } - } else { - TunnelOutcome::Alive { - establish_ms: *establish_ms, - keepalives, - rx, - } - }); + outcomes.push(classify_held_tunnel( + gw.has_fatal().await, + gw.state(), + keepalives, + *establish_ms, + rx, + )); } let report = summarize( @@ -1164,8 +1221,9 @@ async fn run_tunnels( } // Tear down concurrently. Leaving state behind would inflate the next ramp - // step; confirm relay-side amt_relay_active_tunnels returns to baseline - // between steps rather than trusting this. + // step, and there is no relay-side gauge to confirm the table drained + // (amt_relay_active_tunnels is dead — see CONTROL_PLANE_CAVEAT), so the + // ramp driver must space steps rather than trust a readback. let mut teardown = tokio::task::JoinSet::new(); for (gw, _) in up { teardown.spawn(async move { finish_gateway(gw, group, source, shutdown_mode).await }); @@ -1215,6 +1273,100 @@ mod tests { const SRC: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 1)); const GROUP: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(232, 0, 0, 1)); + // ---- POSITIVE CONTROL for the receiver-side occupancy witness ---------- + // + // BLO-33457, CTO condition 1. `amt_relay_active_tunnels` was the AC's + // corroborating instrument and is dead on both production relays, so + // `alive` is now the sole witness for this measurement. An instrument that + // has only ever been seen to agree with `requested` is indistinguishable + // from `alive = established`. These drive each deliberate failure mode and + // assert the count goes DOWN. + + fn summarize_n(outcomes: Vec, requested: u32) -> TunnelsReport { + summarize( + outcomes, + requested, + "127.0.0.1".parse().unwrap(), + "v4", + GROUP, + SRC, + /* hold */ 30, + /* keepalive */ 10, + /* stagger */ 0, + ) + } + + fn alive_one() -> TunnelOutcome { + classify_held_tunnel(false, GatewayState::Active, 1, 5, 0) + } + + #[test] + fn a_healthy_held_tunnel_is_counted_alive() { + // The control's control: without this, a classifier that returned + // NotAlive unconditionally would "pass" every assertion below. + assert!(matches!(alive_one(), TunnelOutcome::Alive { .. })); + assert_eq!(summarize_n(vec![alive_one(), alive_one()], 2).alive, 2); + } + + #[test] + fn each_deliberate_failure_mode_reduces_the_alive_count() { + for (label, failed) in [ + // Socket died under us. Note `state` is still Active: a dead + // runtime leaves the published state stale, which is exactly the + // case that would otherwise be counted as live relay state. + ( + "fatal_runtime_error", + classify_held_tunnel(true, GatewayState::Active, 3, 5, 0), + ), + // Left Active during the hold. + ( + "state_left_active", + classify_held_tunnel(false, GatewayState::Discovering, 3, 5, 0), + ), + // Established but never maintained across a keep-alive interval. + ( + "no_keepalive_sent", + classify_held_tunnel(false, GatewayState::Active, 0, 5, 0), + ), + ] { + let r = summarize_n(vec![alive_one(), alive_one(), failed], 3); + assert_eq!(r.established, 3, "{label}: all three did establish"); + assert_eq!(r.alive, 2, "{label}: alive must drop below established"); + assert_eq!(r.outcome, "degraded", "{label}"); + assert_eq!( + r.not_alive.get(label), + Some(&1), + "{label} must be attributed by name, not folded into a bare shortfall: {:?}", + r.not_alive + ); + } + } + + #[test] + fn keepalives_min_exposes_a_single_unmaintained_tunnel() { + // The aggregate hides it: 2 alive either way. `keepalives_min` is what + // makes "survived an interval" a measurement rather than an assumption. + let r = summarize_n( + vec![ + classify_held_tunnel(false, GatewayState::Active, 9, 5, 0), + classify_held_tunnel(false, GatewayState::Active, 1, 5, 0), + ], + 2, + ); + assert_eq!(r.alive, 2); + assert_eq!(r.keepalives_min, 1); + } + + #[test] + fn the_caveat_travels_with_every_report() { + // AC 5, and the dead-gauge correction: the artifact must not send a + // reader to amt_relay_active_tunnels for corroboration. + let c = summarize_n(vec![alive_one()], 1).caveat; + assert!(c.contains("CONTROL-PLANE"), "{c}"); + assert!(c.contains("RECEIVER-SIDE"), "{c}"); + assert!(c.contains("ATTRIBUTION"), "{c}"); + } + // BLO-33456 review follow-up. A closed data broadcast and an expired // deadline both ended the data phase via the same `anyhow::Error`, so a // transport failure was reported as `outcome: "timeout"` — a deadline diff --git a/tests/probe_verdict_test.sh b/tests/probe_verdict_test.sh index 494b4f3..bcfbe19 100755 --- a/tests/probe_verdict_test.sh +++ b/tests/probe_verdict_test.sh @@ -38,6 +38,31 @@ REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) WORKFLOW="$REPO_ROOT/.github/workflows/amt-public-vantage-probe.yml" WORK=$(mktemp -d); trap 'rm -rf "$WORK"' EXIT +# The ramp's host-capacity guard must validate every scalar before arithmetic. +# A single numeric-looking line is not enough: the old `printf ... | grep` +# check accepted a malformed sibling line because grep searched for *any* match. +# Keep this small contract test next to the workflow test so that a future edit +# cannot silently reintroduce arithmetic on `unavailable`/partial receipts. +is_uint() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; } +for value in 0 42 1048576; do + is_uint "$value" || { echo "FAIL: expected unsigned integer: $value"; exit 1; } +done +# $'...' so the newline is real: a single-quoted '42\nnope' is the literal +# 8-char string backslash-n and never reached the multi-line case. $'42\n42' +# (every line numeric) is the input the old any-match grep was surest about. +for value in '' unavailable $'42\nnope' $'42\n42' '1x' '-1'; do + if is_uint "$value"; then + echo "FAIL: malformed host-capacity value accepted: $value" + exit 1 + fi +done +grep -q 'is_uint()' "$WORKFLOW" || { + echo "FAIL: workflow has no fail-closed scalar validator"; exit 1; +} +if grep -q "printf '%s\\\\n'.*grep -Eq '^[0-9]" "$WORKFLOW"; then + echo "FAIL: workflow still validates a list with any-match grep"; exit 1 +fi + python3 - "$WORKFLOW" "$WORK/probe.sh" <<'PY' import sys, yaml wf, out = sys.argv[1], sys.argv[2] @@ -132,4 +157,53 @@ run_case "tunnels=abc rejected" abc "0" "1" 92 run_case "tunnels=257 over documented cap rejected" 257 "0" "1" 92 run_case "tunnels='' rejected" "" "0" "1" 92 + +# Ally review of #22 (head d3896d3), Important (1). The tunnels-ramp Verdict +# step derived `cap` from the raw STEPS text while `top` came from `jq -r`, and +# compared them as strings: `1, 8, 1024` (accepted by `Validate ramp steps`, +# which word-splits) left a leading blank in `cap`, so a fully clean ramp fell +# through to "degraded ... BINDER NOT ESTABLISHED (0 cause(s))". Drive the real +# Verdict run-block against synthetic ramp-index.json fixtures. It needs only +# `jq` and `tee`, so no docker stub. +python3 - "$WORKFLOW" "$WORK/verdict.sh" <<'PY' +import sys, yaml +wf, out = sys.argv[1], sys.argv[2] +steps = yaml.safe_load(open(wf))['jobs']['tunnels-ramp']['steps'] +open(out, 'w').write(next(s['run'] for s in steps if s.get('name') == 'Verdict')) +PY +bash -n "$WORK/verdict.sh" || { echo "FAIL tunnels-ramp Verdict step does not parse"; FAILED=1; } + +run_ramp_case() { + local name=$1 steps=$2 index=$3 want=$4 + local dir; dir=$(mktemp -d -p "$WORK") + printf '%s' "$index" >"$dir/ramp-index.json" + ( cd "$dir" && STEPS=$steps bash "$WORK/verdict.sh" ) >/dev/null 2>&1 + if grep -qF "$want" "$dir/verdict.txt" 2>/dev/null; then + echo "PASS $name" + else + echo "FAIL $name: want '$want', got: $(head -c 200 "$dir/verdict.txt" 2>/dev/null)"; FAILED=1 + fi + rm -rf "$dir" +} + +CLEAN='[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":1024,"distinct_outer_sources":1024}}]' +run_ramp_case "clean ramp, bare steps -> config-bound" "1,8,1024" "$CLEAN" \ + "verdict=ceiling >= 1024, config-bound" +# Negative control for the string-compare regression: the validator accepts +# this spelling, so the verdict must read it identically. +run_ramp_case "clean ramp, comma-space steps -> config-bound (regression: cap kept leading blank)" "1, 8, 1024" "$CLEAN" \ + "verdict=ceiling >= 1024, config-bound" +run_ramp_case "aliased step supersedes clean cap" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":1024,"distinct_outer_sources":1}}]' \ + "verdict=WITNESS NOT ESTABLISHED" +run_ramp_case "legacy report without distinct_outer_sources -> aliased" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8}},{"requested":1024,"report":{"alive":1024,"distinct_outer_sources":1024}}]' \ + "verdict=WITNESS NOT ESTABLISHED (1/3" +run_ramp_case "degraded relay-attributed -> knee" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":900,"distinct_outer_sources":1024,"establish_errors":{"relay refused: tunnel table full":124}}}]' \ + "verdict=knee at N>8" +run_ramp_case "degraded unknown cause -> binder not established" "1,8,1024" \ + '[{"requested":1,"report":{"alive":1,"distinct_outer_sources":1}},{"requested":8,"report":{"alive":8,"distinct_outer_sources":8}},{"requested":1024,"report":{"alive":900,"distinct_outer_sources":1024,"establish_errors":{"connection reset":124}}}]' \ + "verdict=degraded above N=8, BINDER NOT ESTABLISHED (1 cause(s)" + [ "$FAILED" = 0 ] && { echo "ALL PASS"; exit 0; } || { echo "FAILURES"; exit 1; } diff --git a/tests/tunnels_mode.rs b/tests/tunnels_mode.rs index 5ed2c2c..bab1524 100644 --- a/tests/tunnels_mode.rs +++ b/tests/tunnels_mode.rs @@ -92,10 +92,28 @@ async fn tunnels_mode_reports_per_tunnel_survival() { // BLO-33457 acceptance criterion: the control-plane-vs-loaded-state caveat // travels WITH the numbers, so a raw per-step artifact cannot be read as a - // loaded ceiling. + // loaded ceiling. It must ALSO name the witness as receiver-side: the + // relay's amt_relay_active_tunnels is dead on both production relays, so a + // reader sent there for corroboration reads 0 and calls it a relay defect. let caveat = v["caveat"].as_str().expect("caveat field"); assert!(caveat.contains("CONTROL-PLANE"), "{caveat}"); - assert!(caveat.contains("amt_relay_active_tunnels"), "{caveat}"); + assert!(caveat.contains("RECEIVER-SIDE"), "{caveat}"); + assert!(caveat.contains("ATTRIBUTION"), "{caveat}"); + assert!(caveat.contains("DISQUALIFIER"), "{caveat}"); + + // The disqualifier must be a FIELD, not only prose in the caveat: the + // workflow verdict gates on it, and a reader who trusts `alive` is exactly + // the reader who will not finish the caveat. All 3 gateways bound the host + // default source, so on a relay keying tunnels by outer address alone they + // are ONE tunnel entry -- while `alive` says 3, because `alive` is a + // send-side self-report and every aliased gateway still sends. + // + // This asserting `1` rather than `<= requested` is deliberate: if someone + // gives the gateways distinct sources, this test must FAIL and make them + // set the field honestly, not pass silently and leave the verdict gate + // clamped shut on a rig that has outgrown it. + assert_eq!(v["distinct_outer_sources"], 1, "{out}"); + assert_eq!(v["alive"], 3, "{out}"); } /// A relay that never answers. The knee case: the report must say `degraded`