diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..8e854d33 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,70 @@ +# Line-ending and binary rules for the LifeTrac repo. +# +# Why: this repo is mostly edited on Windows (core.autocrlf=true), but many +# files run on the Portenta X8 boards or inside Linux containers, where a CRLF +# breaks them ("/bin/bash^M: bad interpreter", systemd/udev/mosquitto parse +# errors, `source`d env files with a trailing \r in every value). The rules +# below make Git check those files out with LF on every OS and make +# `git archive` emit LF for them even when core.autocrlf=true. +# +# The index already stores these files with LF, so this file changes checkout +# behaviour only. An EXISTING working tree keeps its old copies until the files +# are checked out again. With a clean tree (commit or set aside your work +# first -- this rewrites every tracked file): +# git rm --cached -r -q . && git reset --hard +# On a Windows checkout made with core.autocrlf=true that refresh also repairs +# binaries that autocrlf had mangled on checkout (some *.pdf, *.hex, *.zip and +# bench-evidence *.bin files whose content Git guessed was text). +# Board-side deploys should still pass `-c core.autocrlf=false` to +# `git archive`, because files not listed here (.txt, .md, .json, .html, ...) +# follow core.autocrlf. +# +# Files not matched below keep the default behaviour (core.autocrlf). + +# --- Runs on the boards / in containers: always LF ------------------------- +*.sh text eol=lf +*.bash text eol=lf +*.py text eol=lf +*.service text eol=lf +*.timer text eol=lf +*.socket text eol=lf +*.rules text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +Dockerfile text eol=lf +Dockerfile.* text eol=lf +*.dockerfile text eol=lf +.env.example text eol=lf +*.env text eol=lf +*.env.example text eol=lf +requirements*.txt text eol=lf + +# --- Windows scripts: CRLF in the working tree ------------------------------ +# text=auto leaves anything Git detects as binary (e.g. a UTF-16 file) alone. +*.ps1 text=auto eol=crlf +*.bat text=auto eol=crlf + +# --- Binary: never convert, never text-diff/merge --------------------------- +*.bin binary +*.elf binary +*.hex binary +*.o binary +*.a binary +*.exe binary +*.dll binary +*.so binary +*.whl binary +*.tar binary +*.tgz binary +*.gz binary +*.xz binary +*.zip binary +*.jpg binary +*.jpeg binary +*.png binary +*.gif binary +*.pdf binary +*.rgb binary diff --git a/.github/workflows/l072-bench-md5.yml b/.github/workflows/l072-bench-md5.yml new file mode 100644 index 00000000..f6adbf42 --- /dev/null +++ b/.github/workflows/l072-bench-md5.yml @@ -0,0 +1,123 @@ +# L072 bench md5 -- NON-BLOCKING reproducibility check (bench kit, 2026-10-10). +# +# Rebuilds the Murata L072 firmware with the exact toolchain the bench PC uses +# (Arm GNU Toolchain 12.2.MPACBTI-Rel1, arm-none-eabi-gcc 12.2.1 20230214) and +# prints the md5s of the bench and production images next to the reference +# values in LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/README.md +# ("Expected md5s") and the committed build/firmware.bin. Mismatches are +# warnings, never failures: the job is continue-on-error and the md5 step +# always exits 0. The blocking link/size gate stays in arduino-ci.yml. +name: L072 bench md5 (non-blocking) + +on: + push: + paths: + - 'LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/**' + - '.github/workflows/l072-bench-md5.yml' + pull_request: + paths: + - 'LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/**' + - '.github/workflows/l072-bench-md5.yml' + workflow_dispatch: + +permissions: + contents: read + +env: + # Linux x86_64 host build of the release the bench PC uses. URL and sha256 + # checked 2026-10-10 against Arm's published .sha256asc (170 644 784 B). + ARM_TOOLCHAIN_URL: https://developer.arm.com/-/media/Files/downloads/gnu/12.2.mpacbti-rel1/binrel/arm-gnu-toolchain-12.2.mpacbti-rel1-x86_64-arm-none-eabi.tar.xz + ARM_TOOLCHAIN_SHA256: 17455a06c816031cc2c66243c117cba48463cd6a3a3fdfac7275b4e9c40eb314 + # murata_l072/README.md "Expected md5s" (main bb4a2071). Update both when the + # firmware source changes and the new pair is recorded. + EXPECTED_BENCH_MD5: 0c1bb0a9573f813137f941dfa47177d0 + EXPECTED_PROD_MD5: 589c120323c2d5e7ef9f459d7a4ba42d + L072_DIR: LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072 + +jobs: + l072-bench-md5: + name: L072 bench md5 (Arm 12.2.MPACBTI-Rel1) + runs-on: ubuntu-24.04 + continue-on-error: true + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Cache the Arm toolchain + uses: actions/cache@v4 + with: + path: ~/arm-gnu-toolchain-12.2.mpacbti-rel1 + key: arm-gnu-toolchain-12.2.mpacbti-rel1-x86_64-${{ env.ARM_TOOLCHAIN_SHA256 }} + + - name: Install Arm GNU Toolchain 12.2.MPACBTI-Rel1 + run: | + set -euo pipefail + TC="$HOME/arm-gnu-toolchain-12.2.mpacbti-rel1" + if [ ! -x "$TC/bin/arm-none-eabi-gcc" ]; then + curl -fsSL --retry 3 -o "$RUNNER_TEMP/tc.tar.xz" "$ARM_TOOLCHAIN_URL" + echo "$ARM_TOOLCHAIN_SHA256 $RUNNER_TEMP/tc.tar.xz" | sha256sum -c - + mkdir -p "$TC" + tar -xJf "$RUNNER_TEMP/tc.tar.xz" -C "$TC" --strip-components=1 + rm -f "$RUNNER_TEMP/tc.tar.xz" + fi + echo "$TC/bin" >> "$GITHUB_PATH" + "$TC/bin/arm-none-eabi-gcc" --version | head -1 + + - name: Record the committed production md5 + working-directory: ${{ env.L072_DIR }} + run: md5sum build/firmware.bin | cut -d' ' -f1 | tee "$RUNNER_TEMP/committed_prod.md5" + + - name: make bench + working-directory: ${{ env.L072_DIR }} + run: | + arm-none-eabi-gcc --version | head -1 + mkdir -p build # `bench` writes build/firmware_bench_diag.bin but does not create build/ + make bench + + - name: make (production) + working-directory: ${{ env.L072_DIR }} + run: make + + - name: Compare md5s with the README (warn only) + working-directory: ${{ env.L072_DIR }} + run: | + bench=$(md5sum build/firmware_bench_diag.bin | cut -d' ' -f1) + prod=$(md5sum build/firmware.bin | cut -d' ' -f1) + committed=$(cat "$RUNNER_TEMP/committed_prod.md5") + verdict() { [ "$1" = "$2" ] && echo "match" || echo "**differs**"; } + { + echo "## L072 md5 -- Arm GNU Toolchain 12.2.MPACBTI-Rel1 on Linux" + echo + echo "| image | built here | reference | |" + echo "|---|---|---|---|" + echo "| bench \`firmware_bench_diag.bin\` | \`$bench\` | README \`$EXPECTED_BENCH_MD5\` | $(verdict "$bench" "$EXPECTED_BENCH_MD5") |" + echo "| production \`firmware.bin\` | \`$prod\` | README \`$EXPECTED_PROD_MD5\` | $(verdict "$prod" "$EXPECTED_PROD_MD5") |" + echo "| production \`firmware.bin\` | \`$prod\` | committed \`$committed\` | $(verdict "$prod" "$committed") |" + echo + echo "A source change since bb4a2071 moves both README md5s; record the new" + echo "pair in murata_l072/README.md and in this workflow's env." + } >> "$GITHUB_STEP_SUMMARY" + echo "bench built=$bench expected=$EXPECTED_BENCH_MD5" + echo "production built=$prod expected=$EXPECTED_PROD_MD5 committed=$committed" + if [ "$bench" != "$EXPECTED_BENCH_MD5" ]; then + echo "::warning title=L072 bench md5 differs::firmware_bench_diag.bin md5 $bench != README $EXPECTED_BENCH_MD5 (source changed since bb4a2071, or this Linux build of 12.2.MPACBTI-Rel1 is not byte-identical to the bench PC build)" + fi + if [ "$prod" != "$EXPECTED_PROD_MD5" ]; then + echo "::warning title=L072 production md5 differs::firmware.bin md5 $prod != README $EXPECTED_PROD_MD5" + fi + if [ "$prod" != "$committed" ]; then + echo "::warning title=Committed firmware.bin not reproduced::the committed build/firmware.bin ($committed) differs from this build ($prod): rebuild and commit it with the firmware change, using the exact toolchain" + fi + exit 0 + + - name: Upload the built images + if: always() + uses: actions/upload-artifact@v4 + with: + name: l072-bench-md5-${{ github.sha }} + path: | + ${{ env.L072_DIR }}/build/firmware_bench_diag.bin + ${{ env.L072_DIR }}/build/firmware.bin + ${{ env.L072_DIR }}/build/firmware.map + if-no-files-found: warn + retention-days: 14 diff --git a/.vscode/tasks.json b/.vscode/tasks.json index be7ca0bb..0efd3768 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -701,9 +701,10 @@ "group": "test" }, { - "label": "Connect WiFi 5 star LAN", + "label": "WiFi connect: run by hand (no PSK in tasks)", + "detail": "The WiFi tasks that carried LIFETRAC_WIFI_PSK were removed: VS Code echoes the resolved command line, and the PSK also showed in the adb and sudo argv and in sudo's log. Run the repo-root WiFi script by hand instead (header of wifi_persist_and_reboot.sh). The bench tractor keeps WiFi off.", "type": "shell", - "command": "adb -s 2D0A1209DABC240B push wifi_connect.sh /tmp/wifi_connect.sh ; adb -s 2D0A1209DABC240B shell \\\"chmod +x /tmp/wifi_connect.sh ; echo fio | sudo -S /tmp/wifi_connect.sh 2>&1\\\"", + "command": "echo 'WiFi tasks removed (they put the PSK on a command line). By hand: adb -s SERIAL push wifi_persist_and_reboot.sh /tmp/wpr.sh, then in an interactive adb shell set LIFETRAC_WIFI_SSID / LIFETRAC_WIFI_PSK (read -r, no echo) and run: sudo --preserve-env=LIFETRAC_WIFI_SSID,LIFETRAC_WIFI_PSK sh /tmp/wpr.sh -- see the script header.'", "isBackground": false, "problemMatcher": [], "group": "test" @@ -724,46 +725,6 @@ "problemMatcher": [], "group": "test" }, - { - "label": "Wait and connect WiFi", - "type": "shell", - "command": "adb -s 2D0A1209DABC240B wait-for-device ; adb -s 2D0A1209DABC240B shell \\\"chmod +x /tmp/wifi_connect_fast.sh ; echo fio | sudo -S /tmp/wifi_connect_fast.sh 2>&1\\\"", - "isBackground": false, - "problemMatcher": [], - "group": "test" - }, - { - "label": "Push and connect WiFi v2", - "type": "shell", - "command": "adb -s 2D0A1209DABC240B wait-for-device ; adb -s 2D0A1209DABC240B push wifi_connect_fast.sh /tmp/wifi_connect_fast.sh ; adb -s 2D0A1209DABC240B shell \\\"chmod +x /tmp/wifi_connect_fast.sh ; echo fio | sudo -S /tmp/wifi_connect_fast.sh 2>&1\\\"", - "isBackground": false, - "problemMatcher": [], - "group": "test" - }, - { - "label": "Rescan then connect", - "type": "shell", - "command": "adb -s 2D0A1209DABC240B shell \\\"echo fio | sudo -S nmcli dev wifi rescan ifname wlan0 ; sleep 6 ; echo '--- visible nets ---' ; echo fio | sudo -S nmcli -t -f SSID,SIGNAL dev wifi list ifname wlan0 | head -n 15 ; echo '--- connect ---' ; echo fio | sudo -S nmcli dev wifi connect \\\\\\\"I'm a 5 star LAN!\\\\\\\" password 9182603083 ifname wlan0 ; sleep 3 ; echo '--- status ---' ; nmcli -t -f NAME,DEVICE,STATE con show --active ; ip -4 addr show wlan0\\\"", - "isBackground": false, - "problemMatcher": [], - "group": "test" - }, - { - "label": "Rescan connect via script", - "type": "shell", - "command": "adb -s 2D0A1209DABC240B push wifi_rescan_connect.sh /tmp/wrc.sh ; adb -s 2D0A1209DABC240B shell \\\"chmod +x /tmp/wrc.sh ; echo fio | sudo -S /tmp/wrc.sh 2>&1\\\"", - "isBackground": false, - "problemMatcher": [], - "group": "test" - }, - { - "label": "Persist profile and reboot", - "type": "shell", - "command": "adb -s 2D0A1209DABC240B push wifi_persist_and_reboot.sh /tmp/wpr.sh ; adb -s 2D0A1209DABC240B shell \\\"chmod +x /tmp/wpr.sh ; echo fio | sudo -S /tmp/wpr.sh 2>&1\\\"", - "isBackground": false, - "problemMatcher": [], - "group": "test" - }, { "label": "Verify autoconnect after reboot", "type": "shell", diff --git a/LifeTrac-v25/BUILD-CONTROLLER/05_firmware_installation.md b/LifeTrac-v25/BUILD-CONTROLLER/05_firmware_installation.md index 6797bf32..7d8b2ddf 100644 --- a/LifeTrac-v25/BUILD-CONTROLLER/05_firmware_installation.md +++ b/LifeTrac-v25/BUILD-CONTROLLER/05_firmware_installation.md @@ -61,6 +61,27 @@ This prints something like `9f3a8c4e2b1d7506a8e3f4c5b9d2e1f0`. Copy it into a fi This is the radio + arbitration + Modbus master firmware that runs on the X8's onboard STM32H747. +> [!WARNING] +> **On a Portenta X8 this step overwrites the stock x8h7 firmware.** The +> H747's M7 core (flash bank 1) normally runs Arduino's stock **x8h7** +> firmware: the bridge through which X8 Linux reaches the H7-side peripherals +> (the `x8h7_*` kernel modules for GPIO, ADC, CAN, UART, PWM and RTC, and +> `m4-proxy` for M4 sketches). Arduino's own X8 workflow only uploads user +> sketches to the M4 core for this reason. Uploading `tractor_h7` as +> *Portenta H7 (M7 core)* replaces the bridge, and everything Linux does +> through it stops working until the stock image is back. +> `stm32h7-program.service` can restore it from +> `/usr/arduino/extra/STM32H747AII6_CM7.bin`, but whether it does so on its +> own depends on the LmP image. +> +> LifeTrac did this to both bench X8s on 2026-05-04; both have run stock +> x8h7 again since 2026-05-13. The current radio bench does **not** use +> `tractor_h7` at all: the LoRa radio is the Max Carrier's Murata L072, +> driven from X8 Linux over `/dev/ttymxc3`. See +> [`BENCH_BOARDS.md`](../DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_BOARDS.md) +> before flashing anything on an X8, and skip this step unless you mean to +> replace the bridge and know how to put the stock image back. + 1. Open `firmware/tractor_h7/tractor_h7.ino` in Arduino IDE 2. 2. Select **Tools → Board → Arduino Mbed OS Portenta Boards → Portenta H7 (M7 core)**. (The X8's onboard H747 enumerates the same way as a standalone H7 for the toolchain — same M7 binary.) 3. Select **Tools → Port** → the X8's USB-C serial port. diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/Dockerfile b/LifeTrac-v25/DESIGN-CONTROLLER/Dockerfile index b53064f6..2a1462ca 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/Dockerfile +++ b/LifeTrac-v25/DESIGN-CONTROLLER/Dockerfile @@ -9,7 +9,10 @@ # (time/params/logger) — the firmware/tractor_x8 sources are mounted in # from the host so the image stays node-agnostic. -FROM python:3.11-slim +# PYTHON_IMAGE is overridable only to pin the flown base image by digest +# (see base_station/requirements.lock.txt). The default is unchanged. +ARG PYTHON_IMAGE=python:3.11-slim +FROM ${PYTHON_IMAGE} # Mosquitto client + a few diagnostic tools so an operator shell-in is # useful (mosquitto_pub/sub for poking topics by hand). @@ -22,8 +25,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app # Layer the deps separately so source edits do not invalidate the pip cache. -COPY base_station/requirements.txt /app/base_station/requirements.txt -RUN pip install --no-cache-dir -r /app/base_station/requirements.txt +# +# Optional LOCKED=1 (`docker build --build-arg LOCKED=1 ...`) installs the +# wheel versions of the flown bench image: base_station/requirements.lock.txt +# is applied as a pip constraints file on top of requirements.txt. The +# default (LOCKED unset or 0) is the unpinned resolve this file always did. +# The bench deploy script passes LOCKED=1 +# (firmware/x8_lora_bootloader_helper/bench_tools/deploy_base.sh). +ARG LOCKED=0 +COPY base_station/requirements.txt base_station/requirements.lock.txt /app/base_station/ +RUN if [ "$LOCKED" = "1" ]; then \ + pip install --no-cache-dir -r /app/base_station/requirements.txt \ + -c /app/base_station/requirements.lock.txt; \ + else \ + pip install --no-cache-dir -r /app/base_station/requirements.txt; \ + fi # Application code. firmware/common/lora_proto/ is also copied so # lora_bridge.py can import the shared protocol header constants in the diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/FIRMWARE_UPDATES.md b/LifeTrac-v25/DESIGN-CONTROLLER/FIRMWARE_UPDATES.md index eba23d54..b7579b01 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/FIRMWARE_UPDATES.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/FIRMWARE_UPDATES.md @@ -50,13 +50,18 @@ The release-key public half is baked into each node's boot loader at manufacture 3. Watch `journalctl -fu lifetrac-base` for ≥1 minute to confirm the new bridge handshakes with the tractor. - **Rollback:** `sudo /usr/local/bin/lifetrac-update vX.Y.` — same wrapper, previous tag. -### Handheld — Portenta H747 (M7 firmware) + secure element +### Handheld — Arduino MKR WAN 1310 (SAMD21 firmware, `firmware/handheld_mkr`) -- **Artifact:** `lifetrac-handheld-h747-vX.Y.Z.bin` for application code; secure-element provisioning is **separate** and only happens at manufacture or pair. +> The handheld is an MKR WAN 1310 (SAMD21 MCU + Murata LoRa module), not a +> Portenta H747 — see [HANDHELD_REMOTE.md](HANDHELD_REMOTE.md) and +> [firmware/handheld_mkr/](firmware/handheld_mkr/) (FQBN +> `arduino:samd:mkrwan1310`, [ARDUINO_CI.md](ARDUINO_CI.md)). + +- **Artifact:** `lifetrac-handheld-mkr-vX.Y.Z.bin` for application code; key provisioning (secure element or key store) is **separate** and only happens at manufacture or pair. - **Channel:** USB-C from the base-station laptop. **Never OTA** — handheld is the operator's safety-critical surface. - **Procedure:** - 1. Connect handheld via USB-C while it is **powered off**. The bootloader enumerates as `Arduino_Bootloader` on USB. - 2. `python tools/flash_handheld.py --image lifetrac-handheld-h747-vX.Y.Z.bin --port COM7`. + 1. Connect the handheld via USB-C and double-tap the MKR's RESET button: the SAMD21 bootloader starts (LED pulses) and enumerates as its own COM port. + 2. `python tools/flash_handheld.py --image lifetrac-handheld-mkr-vX.Y.Z.bin --port COM7` (planned tool; until it exists, compile the sketch as in [ARDUINO_CI.md](ARDUINO_CI.md) and flash it with `arduino-cli upload --fqbn arduino:samd:mkrwan1310 -p `). 3. Tool verifies signature, flashes, reboots. Power up the handheld and confirm the OLED displays the new firmware version on the splash screen. 4. **Re-run pairing** — the secure-element key is preserved across firmware updates by design, but verify pairing still works (cf. [OPERATIONS_MANUAL.md § 2](OPERATIONS_MANUAL.md)). - **Rollback:** keep the previous `.bin` on the maintenance laptop; reflash with the previous version via the same procedure. diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/README.md b/LifeTrac-v25/DESIGN-CONTROLLER/README.md index 3f5e0739..16a67ce7 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/README.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/README.md @@ -46,6 +46,7 @@ This folder contains the **primary controller design** for the LifeTrac v25 trac - **[VIDEO_OPTIONS.md](VIDEO_OPTIONS.md)** — wireless video streaming analysis (LoRa thumbnails on the air; WiFi/cellular alternatives are archived) - **[ARDUINO_CI.md](ARDUINO_CI.md)** — Arduino-CLI continuous integration setup - **[firmware/murata_l072/BRINGUP_MAX_CARRIER.md](firmware/murata_l072/BRINGUP_MAX_CARRIER.md)** — Murata L072 flash + bench bring-up runbook (SWD/OpenOCD/J-Link/DFU) +- **[Radio bench quickstart](firmware/x8_lora_bootloader_helper/bench_tools/BENCH_QUICKSTART.md)** — how to reproduce the two-board LoRa bench (the RS-12/RS-13 campaigns): PC and board setup, L072 firmware, deploy, legs, analysis, park - **[arduino_libraries.txt](arduino_libraries.txt)** — pinned library versions - **[RESEARCH-CONTROLLER/WIRELESS_OPTIONS.md](RESEARCH-CONTROLLER/WIRELESS_OPTIONS.md)** *(archived)* — historical comparison of wireless technologies considered before LoRa was selected diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/ADB_TIPS_AND_TRICKS.md b/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/ADB_TIPS_AND_TRICKS.md index a13180b9..879eaf37 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/ADB_TIPS_AND_TRICKS.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/ADB_TIPS_AND_TRICKS.md @@ -137,7 +137,8 @@ Each of these has been documented as actively destabilizing the bench. & $adb -s reverse --list ``` -- Observed (2026-05-24, board 2E2C): `reverse tcp:1883 tcp:1883` returns +- Observed (2026-05-24, board 2D0A -- the base, the board that runs 934-91; + 2E2C runs the factory 674-88 image): `reverse tcp:1883 tcp:1883` returns exit 0 but the tunnel is never installed. From the X8, `exec 9<>/dev/tcp/127.0.0.1/1883` returns "Connection refused". - `reverse --list` returns `error: protocol fault (couldn't read status diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md b/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md index ec1bf636..e2bb6d04 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md @@ -6,62 +6,193 @@ - T2 cold power cycle did not recover (boot itself wedges). - Suspected eMMC corruption (rootfs or OTA partitions damaged). - Want a known-good baseline before re-running diagnostics. +- Bringing up a new bench board: this is step 4 of + [BENCH_SETUP.md](../../firmware/x8_lora_bootloader_helper/bench_tools/BENCH_SETUP.md). **Why it always works:** SDP (Serial Download Protocol) is in i.MX8MM mask ROM, not eMMC, so it runs even if the entire on-disk OS is destroyed. `uuu` walks the SDP USB endpoint and re-images eMMC end-to-end. +**What it erases:** the whole eMMC: OS, `/etc` changes, docker images, +`/home/fio` and the base's deployed tree and `secrets/`. Capture first if +anything matters (`bench_tools/pull_board_state.sh`). It does **not** touch +the L072 firmware, which lives on the carrier's Murata module. After a +reflash, run the board through BENCH_SETUP steps 5–7 again (first boot, +`provision_bench_board.sh`, verify), then redeploy. + +## Image and tool (pinned) + +### Which image + +| image | kernel | URL | on this bench | +|---|---|---|---| +| **LmP 4.0.11-934-91** | 6.1.24 | `https://downloads.arduino.cc/portentax8image/934.tar.gz`: 1,097,240,589 B, Last-Modified 2025-08-11 | The base (`2D0A1209DABC240B`), flashed 2026-05-13 and 2026-05-24. **Use this one.** | +| LmP 4.0.3-674-88 | 5.10.93 | `https://downloads.arduino.cc/portentax8image/674.tar.gz`: 854,998,055 B | The tractor's (`2E2C1209DABC240B`) factory image; that board was never reflashed. Hosted, but never downloaded or byte-verified here, so there is no sha256 on record. | +| `image-latest.tar.gz` | — | `https://downloads.arduino.cc/portentax8image/image-latest.tar.gz` | **Not the bench image.** The URL moves to newer builds. On 2026-05-08 it still served 934, and the bench bundle was downloaded under that name. | + +### Download and verify (Git Bash) + +```bash +mkdir -p ~/portenta-x8-reflash && cd ~/portenta-x8-reflash +curl -fLO https://downloads.arduino.cc/portentax8image/934.tar.gz +sha256sum 934.tar.gz +tar xzf 934.tar.gz # creates 934/ +cd 934 +tar xzf mfgtool-files-portenta-x8.tar.gz # creates mfgtool-files-portenta-x8/ +gunzip -k lmp-factory-image-portenta-x8.wic.gz # 3,222,240,256 B .wic; -k keeps the .gz +cd .. +sha256sum -c 934.sha256 # the list below, saved as 934.sha256 +``` + +The bench bundle's outer tarball, saved as `image-latest.tar.gz` on +2026-05-08, has sha256 +`2f2065c7b10ce33d8444a138353187c507727b5109ddca325506ecaea1efe221`. Today's +`934.tar.gz` has the same size and Last-Modified date. Nobody has compared +it byte for byte, so it is expected to match but not proven. If your hash +differs, the files inside still decide. Save this list as `934.sha256` next +to the `934/` folder; it holds the sha256 of every file in the bundle used +for both base reflashes, computed 2026-10-04: + +```text +9cd4dbf7acdb4c4737cc50ad2d23d7deeb136920d37ba42f6ac064b5aaa531df *./934/imx-boot-portenta-x8 +8bc5938a61418c02665baf0d721800045c2caf9818ff1e2c2ab33ca44cad9cc7 *./934/lmp-factory-image-portenta-x8.wic +a9571a611922170ea7a350a250abd45493652e21393ec6a6806ca1365da73df0 *./934/lmp-factory-image-portenta-x8.wic.gz +ddeaa6f239feb5cfe9b88ae67a2fe8ef586d52faa8dad609320e338ed596ab2a *./934/mfgtool-files-portenta-x8.tar.gz +fbb6e417a215d1dfc07de6f069d3b65b7621efa7f1c33fda4d00148e99782421 *./934/mfgtool-files-portenta-x8/bootloader.uuu +339e63cb0013b8876df643c163440f04fcc2b2a4d4571257660e6d5f25b1936a *./934/mfgtool-files-portenta-x8/erase_emmc.uuu +e624aa9720debc2914c11d598cbaf5b6b965e7865ea8bdfb3031db7de6b7f4a6 *./934/mfgtool-files-portenta-x8/fitImage-portenta-x8-mfgtool +3ff0a055c799eb1e5d0c1e4af870d764c968bff0ebbd21c8b6b2da1c15e49af7 *./934/mfgtool-files-portenta-x8/full_image.uuu +f97e3afaa8491ce4b7023470975d5639afd5302ac79db00c92f508148d9fc300 *./934/mfgtool-files-portenta-x8/imx-boot-mfgtool +a63a73ec50c455dc4666df3b4bc5161ffcbd2c9e9201638a6327fba81c5a582a *./934/mfgtool-files-portenta-x8/linux_initramfs.uuu +01e04a0a0e85a99ee0050c7377ca9bf83048a0612409be233b61771eabac3212 *./934/mfgtool-files-portenta-x8/probe_emmc.uuu +a4f945f5e81255fe52a97143424274e50da01358c83df55ef8a1b02e18a2da4a *./934/mfgtool-files-portenta-x8/probe_sdcard.uuu +20d6aa39aa525d6325e879853bc40fd0ed751da8aacfc60bd9025de0de7d9bc5 *./934/mfgtool-files-portenta-x8/test_ram.uuu +51d4f5f95316e3a5026e988f255200f0d7862f97bd9e1063273a37e4b7a50b51 *./934/mfgtool-files-portenta-x8/u-boot-mfgtool.itb +d5568f967a686dcef4f60932baba2dc8393b79e43dab5c1437226b2a34658a40 *./934/mfgtool-files-portenta-x8/uuu +ae5dd2b1a6575d1a2416a51ab849009bcaf230d803692c46ade95266b6dcc08e *./934/mfgtool-files-portenta-x8/uuu.exe +2e37d04e7e4435e25dbe0f9463efe672324e1718b0a272b2d6627b90e13cb4fa *./934/mfgtool-files-portenta-x8/uuu_mac +c8afc30f003ec18c278ef8da3307f075ffad0bd744565feb95e09f2e8f3d16da *./934/sit-portenta-x8.bin +b4c055ed4fe6aa7f553fdf2cdcefbfea528d5917abf740a8b55510b52682af08 *./934/u-boot-portenta-x8.itb +``` + +The source of this list is +`LifeTrac-bench-archive\board_state_2026-10-04\pc_only\lmp_934_bundle_sha256.txt` +in the private archive (PC only). + +### `uuu` version: 1.5.243 + +Use **`uuu` 1.5.243** from NXP's mfgtools releases +(, tag `uuu_1.5.243`). The +bench's copy of `uuu.exe` has sha256 +`f6b76a6246befabeadfebdc1cbfe58f35939596caf7b78717ceab599b0c85027`. +**Do not use the `uuu.exe` inside `mfgtool-files-portenta-x8/`.** That one +is 1.5.109 (sha256 `ae5dd2b1…`), and it was unstable on Windows 11 with this +image. Both bench reflashes ran 1.5.243 (12/12 stages). + +### Extracted layout + +`full_image.uuu` refers to the image files as `../`, so run `uuu` from +inside `mfgtool-files-portenta-x8/`: + +```text +portenta-x8-reflash/ +├── 934.tar.gz +├── 934.sha256 +└── 934/ + ├── imx-boot-portenta-x8 + ├── lmp-factory-image-portenta-x8.wic (from the .wic.gz) + ├── lmp-factory-image-portenta-x8.wic.gz + ├── mfgtool-files-portenta-x8.tar.gz + ├── mfgtool-files-portenta-x8/ (from the .tar.gz) + │ ├── full_image.uuu <- run uuu here + │ ├── imx-boot-mfgtool, u-boot-mfgtool.itb, fitImage-portenta-x8-mfgtool + │ └── ... (other .uuu scripts; bundled uuu 1.5.109, do not use) + ├── sit-portenta-x8.bin + └── u-boot-portenta-x8.itb +``` + ## Pre-flight -1. Run T2.5 first — confirm `VID_1366&PID_0105` (Max Carrier BMP) is - enumerating. If not, the carrier is unpowered and SDP won't appear either. -2. Have a known-good `full_image.uuu` and image set on the host. +1. Run T2.5 first: confirm `VID_1366&PID_0105` (the Max Carrier's on-board + J-Link) is enumerating. If not, the carrier is unpowered and SDP won't + appear either. +2. Have the image downloaded, verified and extracted as above, and + `uuu` 1.5.243. +3. Use a USB-A to USB-C data cable into the **X8 module's** USB-C port, + plugged straight into the PC. No hub, and no USB-C to USB-C cable. ## Procedure -1. Power Board down (USB-C **and** 12V). -2. Set Max Carrier DIP switches. **The required position is board-dependent** - — try (a) first; if SDP does not enumerate after a 12V+USB-C re-attach, - power down and try (b): - - (a) `BOOT SEL` → **ON** only (`BOOT` → OFF) *— worked on Board-2 (2D0A) 2026-05-24* - - (b) `BOOT SEL` → **ON** and `BOOT` → **ON** *— worked on Board-1 2026-05-13* -3. Reconnect 12V, then USB-C. -4. Confirm SDP enumeration. The mask-ROM PID also varies between silicon revisions: +1. Power the board down (USB-C **and** 12 V). +2. Set the Max Carrier DIP switches: **`BOOT SEL` ON** (`BOOT` OFF). If SDP + does not appear at step 5, power down and **add `BOOT` ON**. + - Both reflashes on this bench, `2D0A1209DABC240B` on 2026-05-13 and + 2026-05-24, entered SDP with `BOOT SEL` ON only. + - `BOOT SEL` + `BOOT` both ON is what Arduino's uuu guide and the + 2026-05-12 recovery plan describe. It was never needed here. +3. Start `uuu` so it is waiting, in PowerShell: + `cd ...\934\mfgtool-files-portenta-x8; & \uuu.exe full_image.uuu`. + It prints `Waiting for Known USB Device Appear...`. +4. Power up: connect 12 V, then USB-C. On 2026-05-13 SDP appeared on USB-C + alone; on 2026-05-24 it needed USB-C and 12 V. Connect both. +5. Confirm SDP enumeration, in another PowerShell window: ```powershell Get-PnpDevice -PresentOnly | Where-Object { $_.InstanceId -match 'VID_1FC9&PID_(012B|0134)' } | Select-Object Status, FriendlyName, InstanceId | Format-List ``` - Observed PIDs: `012B` (Board-1, 2026-05-13) and `0134` (Board-2, 2026-05-24). - Either is a valid i.MX SDP endpoint and `uuu` 1.5.243+ accepts both. -5. Run `uuu full_image.uuu` (or equivalent OE recipe). -6. Power down. Return both DIP switches to **OFF**. -7. Power up; allow ~60 s for first-boot. -8. Run HC-01 → HC-02 → HC-04. + Both bench reflashes showed `PID_0134`. `PID_012B` is the PID that older + i.MX mask-ROM revisions use; it has not been seen on this bench. + `uuu` 1.5.243 accepts both. +6. `uuu` pushes the boot loaders, then writes the full wic image. That takes + about 60–80 minutes (≈75 min on 2026-05-24). Success is 12/12 stages + and `Success 1 Failure 0`. +7. Power down and set **both DIP switches OFF**. +8. First boot (next section). +9. Run HC-01, then HC-02. Run HC-04 only for a flash-pipeline session. -## Pass criteria +## First boot -- SDP enumerated as `VID_1FC9&PID_012B` **or** `VID_1FC9&PID_0134` at step 4. -- `uuu` reports successful flash + verify (full wic write ≈ 60–80 min). -- After reboot: HC-01..HC-04 all PASS. +1. With both DIP switches OFF, connect 12 V, then USB-C. Allow about 60 s. + If the LEDs go red or adb has not appeared after 2 minutes, power-cycle + once more; the 2026-05-13 reflash needed two cycles. +2. `adb devices -l` must list the serial as `device`. Then + `adb -s shell cat /etc/os-release` must show + `VERSION="4.0.11-934-91"`, and `uname -r` must show `6.1.24-lmp-standard`. +3. Continue with BENCH_SETUP step 5: password, sudoers drop-in, base + network and ssh key, then `provision_bench_board.sh`. > **First-boot caveat (LmP 934-91):** the freshly imaged X8 has no Wi-Fi > credentials and no SSH password. The only post-flash channel is `adb` over -> USB-C. **Do NOT run `adb kill-server`** — first-boot adbd is fragile and a -> kill-server leaves the board adb-invisible until a USB-C unplug/replug. See -> `T0_adb_daemon_kick.md` and the `lifetrac-x8-adb-reverse-broken` repo memory. +> USB-C. **Do NOT run `adb kill-server`**: first-boot adbd is fragile, and a +> kill-server leaves the board adb-invisible until a USB-C unplug and replug. +> See `T0_adb_daemon_kick.md` and the `lifetrac-x8-adb-reverse-broken` repo +> memory. + +## Pass criteria + +- SDP enumerated as `VID_1FC9&PID_0134` (or `VID_1FC9&PID_012B` on older + silicon) at step 5. +- `uuu` reports successful flash and verify (full wic write ≈ 60–80 min). +- After reboot: `os-release` shows the pinned version, and HC-01 and HC-02 + PASS. ## Common failure modes | Symptom | Cause | Action | |---|---|---| -| `1FC9:012B` does not appear with DIPs ON | DIP switches not actually flipped, carrier under-powered, or hardware fault | Re-check DIPs; T2.5 | -| `uuu` errors mid-flash | Cable / hub flakiness | Use direct host port, not a hub; retry | -| Flash succeeds but board still boots into hung state | Uncommon — image itself corrupt or pre-flash hardware fault | Try a different image build | +| No `1FC9:0134` (or `012B`) with `BOOT SEL` ON | DIP switches not actually flipped, `BOOT` also needed, carrier under-powered, or a hardware fault | Re-check the DIP switches; add `BOOT` ON; connect 12 V as well as USB-C; T2.5 | +| `uuu` errors mid-flash | Cable or hub flakiness, or the bundled `uuu` 1.5.109 | Use a direct host port, not a hub; use `uuu` 1.5.243; retry | +| `full_image.uuu` cannot find the `.wic` | `.wic.gz` not decompressed, or `uuu` not run from `mfgtool-files-portenta-x8/` | See *Extracted layout* | +| Flash succeeds, but the board boots a different LmP version | Wrong bundle (for example `image-latest.tar.gz`) | Check `/etc/os-release`; reflash with the pinned bundle | +| Flash succeeds but board still boots into hung state | Uncommon: the image itself is corrupt, or a pre-flash hardware fault | Verify the sha256 list; try again | ## Verdict matrix +The May notes call `2D0A1209DABC240B` "Board 1" on 2026-05-13 and "Board-2" +on 2026-05-24. Go by the serial. + | Date | Board | Outcome | Notes | |---|---|---|---| -| 2026-05-13 | Board-1 (2E2C1209DABC240B) | PASS | DIP = `BOOT`+`BOOT SEL` both ON. SDP enumerated as `VID_1FC9&PID_012B`. `uuu` 1.5.243 EXIT=0. | +| 2026-05-13 | `2D0A1209DABC240B` (then "Board 1") | PASS | DIP = `BOOT SEL` ON **only**. USB-C only. SDP enumerated as `VID_1FC9&PID_0134`. `uuu` 1.5.243 ran `full_image.uuu`: 12/12 stages, Success 1 / Failure 0. First boot needed 2 power cycles. Fresh `4.0.11-934-91` / `6.1.24-lmp-standard`. Source: [log/2026-05-13_board1_T3a_postreflash_v1.md](../log/2026-05-13_board1_T3a_postreflash_v1.md). *Corrected 2026-10-10 against that log. The row used to say "Board-1 (2E2C1209DABC240B)", "`BOOT`+`BOOT SEL` both ON" and "`PID_012B`". `2E2C` (the tractor) has never been reflashed.* | | 2026-05-24 | Board-2 (2D0A1209DABC240B) | PASS | DIP = `BOOT SEL` ON **only** (`BOOT` OFF). SDP enumerated as `VID_1FC9&PID_0134` (NOT `012B`). USB-C + 12V both required to enter SDP. `uuu` 1.5.243 EXIT=0; full wic write ≈75 min total. First-boot adbd later wedged after host-side `adb kill-server`; recovery requires USB-C unplug/replug (no software-only path on a factory image with no Wi-Fi/SSH). | diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/routines/HC-02_linux_bridge_full.md b/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/routines/HC-02_linux_bridge_full.md index 2c65a4ab..6beb0d91 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/routines/HC-02_linux_bridge_full.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/X8_HEALTH_AND_RECOVERY/routines/HC-02_linux_bridge_full.md @@ -10,18 +10,20 @@ SWD work. ## Procedure ```powershell -$script = "LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/board1_healthcheck.sh" +$script = "LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/x8_max_carrier_healthcheck.sh" foreach ($s in '2D0A1209DABC240B','2E2C1209DABC240B') { $tag = if ($s -like '2D0A*') {'board1'} else {'board2'} $log = "LifeTrac-v25/AI NOTES/$(Get-Date -Format yyyy-MM-dd)_${tag}_healthcheck_v1_0.log" - adb -s $s push $script /tmp/board1_healthcheck.sh 2>&1 | Out-Null - adb -s $s exec-out "sh /tmp/board1_healthcheck.sh" 2>&1 | + adb -s $s push $script /tmp/x8_max_carrier_healthcheck.sh 2>&1 | Out-Null + adb -s $s exec-out "sh /tmp/x8_max_carrier_healthcheck.sh" 2>&1 | Out-File -FilePath $log -Encoding utf8 Get-Content $log } ``` -Helper: [board1_healthcheck.sh](../../firmware/x8_lora_bootloader_helper/board1_healthcheck.sh) +Helper: [x8_max_carrier_healthcheck.sh](../../firmware/x8_lora_bootloader_helper/x8_max_carrier_healthcheck.sh) +(renamed from `board1_healthcheck.sh` on 2026-05-15; older notes and the +2026-05-13 logs still use the old name). ## Pass criteria diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/base_station/requirements.lock.txt b/LifeTrac-v25/DESIGN-CONTROLLER/base_station/requirements.lock.txt new file mode 100644 index 00000000..b4e8cc6d --- /dev/null +++ b/LifeTrac-v25/DESIGN-CONTROLLER/base_station/requirements.lock.txt @@ -0,0 +1,67 @@ +# LifeTrac v25 base station: the wheels in the FLOWN bench image. +# +# Image: lifetrac-v25:latest 4623980c2dac +# (sha256:4623980c2daca116eb68a80fc18285fa273b43545aac498557bcc349871161a6), +# built natively (linux/arm64) on the bench base X8 from main d3751286 +# on 2026-09-15. It is the image the RS-13 legs flew with. +# Evidence (LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/board_state_2026-10-04/base/files/home_fio/): +# - docker_build_d3751286.log, the build of 4623980c2dac, does NOT list the +# wheels: its pip step is "Using cache ---> 9e3efb0b5438". +# - docker_build.log (2026-09-12, image 12fe97dece57) is the build that +# CREATED layer 9e3efb0b5438, on top of the same requirements.txt layer +# 1d0767921ca1. docker_build_3a0cb524.log (ed7c058774f4) reuses it too. +# - The list below is docker_build.log's "Successfully installed" line, +# which is every package that pip step installed. pip, setuptools and +# wheel come from the python image and are not listed. +# Python: FROM python:3.11-slim = image 2fe5ff938d2b, +# python@sha256:db3ff2e1800a8581e2c48a27c3995339d47bdf046da21c7627accd3d51053a93 +# (pip 24.0). The apt step was cached before 2026-09-12, so its +# Debian package versions were never logged. +# +# Use: docker build --build-arg LOCKED=1 -t lifetrac-v25:latest . +# The Dockerfile then applies this file as a pip CONSTRAINTS file on +# top of requirements.txt. A dependency added to requirements.txt +# later still installs, unpinned. A floor above a pin here fails the +# build loudly; then refresh this file or build without LOCKED. +# To pin the base image as well, add: +# --build-arg PYTHON_IMAGE=python:3.11-slim@sha256:db3ff2e1800a8581e2c48a27c3995339d47bdf046da21c7627accd3d51053a93 +# Without LOCKED (the default, as in plain `docker compose build`) +# this file is ignored and pip resolves requirements.txt unpinned. +# CI does not build the image; it installs requirements-dev.txt, +# which pins fastapi 0.136.1 (the flown image has 0.141.1). +# The bench deploy (firmware/x8_lora_bootloader_helper/bench_tools/ +# deploy_base.sh, see DEPLOY.md there) builds with LOCKED=1 by default. +# Format: name==version only. Constraints files cannot carry extras, so +# uvicorn[standard]'s extras appear as their own packages +# (httptools, python-dotenv, pyyaml, uvloop, watchfiles, websockets). +# Refresh: after a deliberate unlocked rebuild, regenerate from +# `docker run --rm --entrypoint python3 lifetrac-v25:latest -m pip freeze` +# (deploy_base.sh saves this as /home/fio/pip_freeze_lifetrac-v25_.txt) +# and update the header. +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +cffi==2.1.1 +click==8.5.0 +cryptography==50.0.1 +fastapi==0.141.1 +h11==0.16.0 +httptools==0.8.0 +idna==3.19 +jinja2==3.1.6 +MarkupSafe==3.0.3 +paho-mqtt==2.1.0 +Pillow==12.3.0 +pycparser==3.0 +pydantic==2.13.5 +pydantic-core==2.46.5 +pyserial==3.5 +python-dotenv==1.2.3 +pyyaml==6.0.3 +starlette==1.6.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +uvicorn==0.52.4 +uvloop==0.22.1 +watchfiles==1.2.0 +websockets==17.1 diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/RS_13_vector_scene_2026-09-26/scripts/step1_pass_fix.sh b/LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/RS_13_vector_scene_2026-09-26/scripts/step1_pass_fix.sh index c69f35e5..52f6cf46 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/RS_13_vector_scene_2026-09-26/scripts/step1_pass_fix.sh +++ b/LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/RS_13_vector_scene_2026-09-26/scripts/step1_pass_fix.sh @@ -22,7 +22,7 @@ $T "$S docker rm -f camera_svc rs13_cap >/dev/null 2>&1" >/dev/null 2>&1 # leg (the tractor daemon republishes the base's 0x63 commands here) would flip the # camera off VECTOR or onto another profile's budget (2026-10-03 ABORTED1) $T "$S docker rm -f bench_mqtt >/dev/null 2>&1; $S docker run -d --name bench_mqtt --network=host -v /tmp/lifetrac_strict/bench_mqtt.conf:/mosquitto/config/mosquitto.conf eclipse-mosquitto:2 >/dev/null; sleep 2" >/dev/null 2>&1 -echo -n "retained on the tractor broker (must be empty): "; $T "$S docker exec bench_mqtt timeout 2 mosquitto_sub -h 127.0.0.1 -t 'lifetrac/#' -v --retained-only 2>/dev/null | wc -l" | tr -d ' ' +echo -n "retained on the tractor broker (must be empty): "; $T "$S docker exec bench_mqtt timeout 2 mosquitto_sub -h 127.0.0.1 -t 'lifetrac/#' -v --retained-only 2>/dev/null | wc -l" | tr -d '\r' # 1. capture first (the base-station store from /work too: same tree as the encoder) $T "$S docker run --rm --name rs13_cap --network=host -v /tmp/lifetrac_strict:/work -w /work \ diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/common/setup_x8_mqtt_link.ps1 b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/common/setup_x8_mqtt_link.ps1 index 2b62d572..e79c91d7 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/common/setup_x8_mqtt_link.ps1 +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/common/setup_x8_mqtt_link.ps1 @@ -5,8 +5,9 @@ .DESCRIPTION The historical recipe used 'adb -s reverse tcp:1883 tcp:1883' to - expose the host broker as 127.0.0.1:1883 on the device. On LmP factory - image 934-91 (verified 2026-05-24 on board 2E2C1209DABC240B) that command + expose the host broker as 127.0.0.1:1883 on the device. On LmP image + 4.0.11-934-91 (verified 2026-05-24 on board 2D0A1209DABC240B, the base; + the tractor 2E2C runs the as-shipped 4.0.3-674-88 image) that command returns exit=0 but does NOT actually install the tunnel: adb reverse tcp:1883 tcp:1883 -> exit=0 (silent failure) diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/README.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/README.md index fef448b1..22002835 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/README.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072/README.md @@ -63,16 +63,79 @@ Future A/B layout (Phase 6, N-26) splits APP: Stack is reserved at the top (size set in [include/memory_map.h](include/memory_map.h) as `MM_STACK_SIZE`, default 2.5 KB per Claude review §2.3 recommendation). -## Build +## Building + +### Toolchain (byte-identical builds need exactly this) + +| tool | version | notes | +|---|---|---| +| cross compiler | **Arm GNU Toolchain 12.2.MPACBTI-Rel1**: `arm-none-eabi-gcc --version` prints `arm-none-eabi-gcc (Arm GNU Toolchain 12.2.MPACBTI-Rel1 (Build arm-12-mpacbti.34)) 12.2.1 20230214` | From developer.arm.com, *Arm GNU Toolchain Downloads*, release 12.2.MPACBTI-Rel1, "AArch32 bare-metal target (arm-none-eabi)". Bench PC install: `C:\Program Files (x86)\Arm GNU Toolchain arm-none-eabi\12.2 mpacbti-rel1\bin` on `PATH`. Linux x86_64 tarball: `https://developer.arm.com/-/media/Files/downloads/gnu/12.2.mpacbti-rel1/binrel/arm-gnu-toolchain-12.2.mpacbti-rel1-x86_64-arm-none-eabi.tar.xz` (170 644 784 B, sha256 `17455a06c816031cc2c66243c117cba48463cd6a3a3fdfac7275b4e9c40eb314`, as published in Arm's `.sha256asc`). | +| make | Windows: WinLibs `mingw32-make` (winget `BrechtSanders.WinLibs.POSIX.UCRT`), **run from PowerShell only**. Linux: GNU make. | On Windows the Makefile uses `cmd` syntax for `mkdir`/`rmdir`, so it fails from Git Bash. | +| host compiler (`make check` only) | any gcc/cc (WinLibs `gcc` on Windows) | `HOST_CC` overrides. | + +Other compilers build **working firmware with different bytes**: +- Arduino's bundled `arm-none-eabi-gcc` 7-2017q4. [`build.ps1`](build.ps1) + picks it first when it is installed, so do not use `build.ps1` for a build + you will compare. +- Ubuntu's `gcc-arm-none-eabi`. The `L072 cross-compile` job in + `arduino-ci.yml` uses it for its link + size-budget gate. + +A matching md5 therefore proves both the source and the toolchain. + +### Targets ``` -make # build firmware.elf, firmware.bin, firmware.hex -make size # show region usage against the budget +mingw32-make # = all: build/firmware.elf/.bin/.hex + region sizes (Linux: make) +mingw32-make bench # build_bench/ with -DHOST_ALLOW_REG_WRITE_DIAG=1, copied to build/firmware_bench_diag.bin +mingw32-make check # host-compiler unit vectors + memory-map static asserts (no cross toolchain) +mingw32-make size # region usage against the budget +mingw32-make clean # removes build/ (including the committed production bin; restore it with `mingw32-make`) python3 tools/check_size_budget.py build/firmware.elf -make clean ``` -A real toolchain (`arm-none-eabi-gcc`) is required to link. CI now runs a full cross-compile gate on pinned `ubuntu-24.04` with Ubuntu's `gcc-arm-none-eabi` package, publishes the ELF/BIN/HEX/MAP artifacts, and enforces APP/RAM budgets via [tools/check_size_budget.py](tools/check_size_budget.py) using constants from [include/memory_map.h](include/memory_map.h). +- `all` makes the **production** image `build/firmware.bin`, which is + committed (`HOST_ALLOW_REG_WRITE_DIAG=0`). It refuses the bench's + diagnostic register writes, such as the DTS carrier pin `-ForceFrfHz`. + Rebuild and commit it with every firmware PR. +- `bench` makes `build/firmware_bench_diag.bin`, the image **both bench + boards** run. It is untracked: add firmware files to commits by path, never + with `git add `. +- `bench` writes into `build/` but does not create it. After a `clean`, run + `mingw32-make` (or create `build/`) before `mingw32-make bench`. +- Run `check` before flashing anything. Flashing is covered in + [`../x8_lora_bootloader_helper/FLASH_RUNBOOK.md`](../x8_lora_bootloader_helper/FLASH_RUNBOOK.md). + +### Expected md5s + +At `main` `bb4a2071` (re-checked 2026-10-04; `firmware/murata_l072` unchanged +since), built with the toolchain above and the default flags: + +| image | md5 | size | +|---|---|---| +| `build/firmware_bench_diag.bin` (bench, on both boards since 2026-09-15) | `0c1bb0a9573f813137f941dfa47177d0` | 24 860 B | +| `build/firmware.bin` (production, committed) | `589c120323c2d5e7ef9f459d7a4ba42d` | 24 860 B | + +Check with `certutil -hashfile build\firmware_bench_diag.bin MD5` +(PowerShell) or `md5sum build/firmware_bench_diag.bin`. +- Any source change moves both md5s. Record the new pair in the PR. +- Older flashed builds and their md5s are listed in + [`bench-evidence/RS_13_vector_scene_2026-09-26/firmware/README.md`](../../bench-evidence/RS_13_vector_scene_2026-09-26/firmware/README.md). + +### CI + +- `arduino-ci.yml` → `L072 cross-compile` (blocking): links with Ubuntu's + `gcc-arm-none-eabi`. It publishes the ELF/BIN/HEX/MAP artifacts and + enforces the APP/RAM budgets through + [tools/check_size_budget.py](tools/check_size_budget.py), using constants + from [include/memory_map.h](include/memory_map.h). Its bytes differ from + the bench build by design. +- `l072-bench-md5.yml` → `L072 bench md5 (Arm 12.2.MPACBTI-Rel1)` + (**non-blocking**): downloads the exact Arm release above for Linux, runs + `make bench` and `make`, and prints both md5s against this table and + against the committed `build/firmware.bin`, with a warning on mismatch. + The Windows bench PC builds are the reference. Whether a Linux build of + the same release is byte-identical is what this job tests; it has not been + checked by hand. ## Not yet present diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/Dockerfile b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/Dockerfile index 571f4abb..a1bcfbe3 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/Dockerfile +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/Dockerfile @@ -1,5 +1,8 @@ # LifeTrac tractor X8 camera/image pipeline Dockerfile -FROM python:3.11-slim-bookworm +# PYTHON_IMAGE is overridable only to pin the flown base image by digest +# (see requirements.lock.txt). The default is unchanged. +ARG PYTHON_IMAGE=python:3.11-slim-bookworm +FROM ${PYTHON_IMAGE} # Install system dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -9,10 +12,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* -# Install Python dependencies -COPY requirements.txt /app/requirements.txt +# Install Python dependencies. +# Optional LOCKED=1 (`docker build --build-arg LOCKED=1 ...`) installs the +# wheel versions of the flown bench image: requirements.lock.txt is applied +# as a pip constraints file on top of requirements.txt. The default (LOCKED +# unset or 0) is the unpinned resolve this file always did. The bench build +# script passes LOCKED=1 +# (firmware/x8_lora_bootloader_helper/bench_tools/build_tractor_image.sh). +ARG LOCKED=0 +COPY requirements.txt requirements.lock.txt /app/ WORKDIR /app -RUN pip install --no-cache-dir -r requirements.txt +RUN if [ "$LOCKED" = "1" ]; then \ + pip install --no-cache-dir -r requirements.txt -c requirements.lock.txt; \ + else \ + pip install --no-cache-dir -r requirements.txt; \ + fi # Copy camera/image pipeline code COPY . /app diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/requirements.lock.txt b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/requirements.lock.txt new file mode 100644 index 00000000..8492fa78 --- /dev/null +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8/requirements.lock.txt @@ -0,0 +1,59 @@ +# LifeTrac v25 tractor X8: the wheels in the FLOWN bench image. +# +# Image: lifetrac-tractor-x8:latest / :rs13-65869517 2727dfd36f9f +# (sha256:2727dfd36f9fb2dd58af00694fd6010e2dd8404643470dff3be6ff06aaf01d77), +# built natively (linux/arm64) on the bench base X8 on 2026-09-26 from +# origin/main 65869517:LifeTrac-v25/DESIGN-CONTROLLER/firmware/tractor_x8, +# then loaded on the tractor. It is the image the RS-13 legs flew with. +# Evidence: +# - LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/board_state_2026-10-04/base/files/home_fio/rs13_build.log +# is the full build log of 2727dfd36f9f. Its pip step ran uncached; the +# list below is its "Successfully installed" line, which is every package +# that step installed. pip, setuptools and wheel come from the python +# image and are not listed. All were binary wheels: no sdist was built. +# - LifeTrac-v25/DESIGN-CONTROLLER/bench-evidence/RS_13_vector_scene_2026-09-26/legs/step0_image_build.txt +# is the leg summary of the same build (4 of these wheels, same versions). +# Python: FROM python:3.11-slim-bookworm = image aad78d4fecb5, +# python@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b +# (pip 24.0). +# Debian packages from the same log (the apt step is unpinned too): +# ffmpeg 7:5.1.9-0+deb12u1, libglib2.0-0 2.74.6-2+deb12u9, +# libusb-1.0-0 2:1.0.26-1, mosquitto-clients 2.0.11-1.2+deb12u2. +# +# Use: docker build --build-arg LOCKED=1 -t lifetrac-tractor-x8:latest . +# (in firmware/tractor_x8). The Dockerfile then applies this file as a +# pip CONSTRAINTS file on top of requirements.txt. A dependency added +# to requirements.txt later still installs, unpinned. A floor above a +# pin here fails the build loudly; then refresh this file or build +# without LOCKED. To pin the base image as well, add: +# --build-arg PYTHON_IMAGE=python:3.11-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b +# Without LOCKED (the default) this file is ignored and pip resolves +# requirements.txt unpinned. Note that CI pins OpenCV 4.14.0.94 for +# the tests (base_station/requirements-dev.txt), while the flown +# tractor image runs OpenCV 5.0.0.93. +# The bench build (firmware/x8_lora_bootloader_helper/bench_tools/ +# build_tractor_image.sh, see DEPLOY.md there) uses LOCKED=1 by default. +# Format: name==version only (a pip constraints file allows no extras). +# Refresh: after a deliberate unlocked rebuild, use the pip freeze that +# build_tractor_image.sh saves next to the image tarball +# (lifetrac-tractor-x8_.pip-freeze.txt) and update the header. +adafruit-blinka==9.2.0 +adafruit-circuitpython-bno08x==1.3.3 +adafruit-circuitpython-busdevice==5.2.17 +Adafruit-Circuitpython-ConnectionManager==3.1.8 +adafruit-circuitpython-gps==4.0.0 +adafruit-circuitpython-requests==4.1.17 +adafruit-circuitpython-typing==1.12.3 +Adafruit-PlatformDetect==3.89.1 +Adafruit-PureIO==1.1.12 +binho-host-adapter==0.1.6 +hidapi==0.15.0 +numpy==2.4.6 +opencv-python-headless==5.0.0.93 +paho-mqtt==2.1.0 +Pillow==12.3.0 +pyftdi==0.57.2 +pyserial==3.5 +pyusb==1.3.1 +sysv_ipc==1.2.0 +typing-extensions==4.16.0 diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/FLASH_RUNBOOK.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/FLASH_RUNBOOK.md index 76d3d3cd..8f54b40b 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/FLASH_RUNBOOK.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/FLASH_RUNBOOK.md @@ -6,6 +6,18 @@ openocd holds BOOT0/NRST through the H7's GPIOs. Written 2026-09-12 after a session that met every trap below; the scripts live in this directory and are pushed to `/tmp/lifetrac_p0c` on the board. +- **Scripted form (2026-10-10):** [`bench_tools/flash_l072.sh`](bench_tools/flash_l072.sh) + does sections 1–2 from the PC. It reads the serials, base address, ssh key + and sudo password from `bench_tools/bench.env` (template + `bench_tools/bench.env.example`); without that file it falls back to the + original bench's serials and base address, so create it first. + Without `--go` it only stages and preflights; `--go` flashes. The manual + commands in sections 1–2 are what the script does, for reference. +- **A board that has never been flashed** (stock MKRWAN AT firmware on the + L072): read [section 5](#5-first-flash-from-a-factory-board) first. +- **A flash is a radio-on event.** The L072 boots into RXCONT. Flash only on + the operator's explicit GO for that round. + ## 0. Which binary | build | command | binary | use | @@ -17,20 +29,72 @@ Run `mingw32-make check` before flashing anything; `check-stats-layout` pins the STATS wire against `host_types.h` (an additive-tail slip shifted four fields by one slot on 2026-09-12 — the test exists because of it). +### Byte-identical builds and the expected md5s + +The build is reproducible byte for byte, but only with the exact toolchain: +**Arm GNU Toolchain 12.2.MPACBTI-Rel1** (`arm-none-eabi-gcc --version` → +`12.2.1 20230214`). On the bench PC it is installed at +`C:\Program Files (x86)\Arm GNU Toolchain arm-none-eabi\12.2 mpacbti-rel1\`, +with WinLibs `mingw32-make` run from PowerShell. Expected md5s at `main` +`bb4a2071` (`firmware/murata_l072` unchanged since): + +| build | md5 | size | +|---|---|---| +| bench `build/firmware_bench_diag.bin` (on both boards since 2026-09-15) | `0c1bb0a9573f813137f941dfa47177d0` | 24 860 B | +| production `build/firmware.bin` (committed) | `589c120323c2d5e7ef9f459d7a4ba42d` | 24 860 B | + +Other compilers (Arduino's bundled 7-2017q4, which `build.ps1` picks first; +Ubuntu's `gcc-arm-none-eabi`, which the main CI job uses) build working +firmware with **different bytes**, so a matching md5 also proves the +toolchain. Details and make targets: +[`../murata_l072/README.md`](../murata_l072/README.md#building). The +non-blocking CI job `.github/workflows/l072-bench-md5.yml` rebuilds both +images on Linux with the same Arm release and prints the md5s against these +values. `flash_l072.sh` names every recorded md5 before it stages. The older +builds are listed in +[`bench-evidence/RS_13_vector_scene_2026-09-26/firmware/README.md`](../../bench-evidence/RS_13_vector_scene_2026-09-26/firmware/README.md). + ## 1. Push the tooling (LF!) -git autocrlf rewrites `*.sh`/`*.cfg` to CRLF in the working tree whenever -git touches them (a pull that changed them, a fresh worktree). Bash on the +**Use the script.** From this directory, in Git Bash: + +```bash +bash bench_tools/flash_l072.sh ../murata_l072/build/firmware_bench_diag.bin +``` + +It does all of this section: LF-clean copies of the nine pipeline files +below plus the bin to `/tmp/lifetrac_p0c`, the three wrapper files to +`/home/fio`, then the wrapper's preflight (`FLASH_PREFLIGHT_ONLY=1`). +`PREFLIGHT-OK` means the staging is complete. The serials, base address, +ssh key and sudo password come from `bench_tools/bench.env`. The wrapper +itself refuses to start — before touching openocd, the UART or the +watchdog — when a pipeline file or the image is missing or a `.sh`/`.cfg` +has CRLF (`PREFLIGHT-FAIL`, exit 3). + +Why LF matters: git autocrlf rewrites `*.sh`/`*.cfg` to CRLF in the working +tree whenever git touches them (a pull that changed them, a fresh worktree, +a checkout made before the repo's `.gitattributes` landed). Bash on the board then sees `\r` in every path (`/tmp/lifetrac_p0c\r/pipeline.log: No -such file`). Normalize before pushing: +such file`). + +**By hand** (what the script does; the `<...>` values are the ones in your +`bench.env`): ```bash -mkdir -p /tmp/p0c_lf && for f in full_flash_pipeline.sh run_flash_l072.sh prep_bridge.sh revive_bridge.sh wdt_pet.sh stm32_an3155_flasher.py 07_assert_pa11_pf4_long.cfg 08_boot_user_app.cfg 99_release_and_reset.cfg; do sed 's/\r$//' "$f" > /tmp/p0c_lf/$f; done +rm -rf /tmp/p0c_lf && mkdir -p /tmp/p0c_lf/p0c /tmp/p0c_lf/home +for f in full_flash_pipeline.sh run_flash_l072.sh prep_bridge.sh revive_bridge.sh wdt_pet.sh stm32_an3155_flasher.py 07_assert_pa11_pf4_long.cfg 08_boot_user_app.cfg 99_release_and_reset.cfg; do sed 's/\r$//' "$f" > /tmp/p0c_lf/p0c/$f; done +for f in run_flash_bench.sh stamp.py kmsg_log.py; do sed 's/\r$//' "bench_tools/$f" > /tmp/p0c_lf/home/$f; done +cp ../murata_l072/build/firmware_bench_diag.bin /tmp/p0c_lf/p0c/ # binary: copy, never sed +# base (ethernet) +ssh -i fio@ "mkdir -p /tmp/lifetrac_p0c" +scp -i /tmp/p0c_lf/p0c/* fio@:/tmp/lifetrac_p0c/ +scp -i /tmp/p0c_lf/home/* fio@:/home/fio/ +# tractor (adb: MSYS_NO_PATHCONV=1 and Windows-style local paths) +export MSYS_NO_PATHCONV=1 +for f in /tmp/p0c_lf/p0c/*; do adb -s push "$(cygpath -m "$f")" /tmp/lifetrac_p0c/; done +for f in /tmp/p0c_lf/home/*; do adb -s push "$(cygpath -m "$f")" /home/fio/; done ``` -Base (ethernet): `scp /tmp/p0c_lf/* firmware_bench_diag.bin fio@192.168.1.117:/tmp/lifetrac_p0c/` -Tractor (adb, Windows-style local paths with `MSYS_NO_PATHCONV=1`): -`adb -s 2E2C1209DABC240B push C:/…/firmware_bench_diag.bin /tmp/lifetrac_p0c/` Verify on the board: `grep -l $'\r' /tmp/lifetrac_p0c/*.sh | wc -l` → 0. The instrumented wrapper `run_flash_bench.sh` plus `stamp.py` and @@ -40,13 +104,36 @@ evidence. ## 2. Flash +**Only on the operator's GO.** From the PC, one board at a time; each run +stages, preflights, then flashes (on the tractor it stops the camera unit +and container first): + +```bash +bash bench_tools/flash_l072.sh tractor ../murata_l072/build/firmware_bench_diag.bin --go +bash bench_tools/flash_l072.sh base ../murata_l072/build/firmware_bench_diag.bin --go +``` + +**By hand**, after the by-hand staging of section 1 (`<...>` from your +`bench.env`; `sudo -n` relies on the sudoers drop-in that provisioning +installs, see `bench_tools/BENCH_SETUP.md`): + ```bash # tractor (adb) — stop the unit AND the container first, or the UART is stolen -adb -s 2E2C1209DABC240B shell "echo fio | sudo -S -p '' systemctl stop lifetrac-camera.service; echo fio | sudo -S -p '' docker stop tractor-camera; echo fio | sudo -S -p '' env REVIVE_MODE=reboot bash /home/fio/run_flash_bench.sh /tmp/lifetrac_p0c/firmware_bench_diag.bin" +adb -s shell "sudo -n systemctl stop lifetrac-camera.service; sudo -n docker stop tractor-camera; sudo -n env REVIVE_MODE=reboot bash /home/fio/run_flash_bench.sh /tmp/lifetrac_p0c/firmware_bench_diag.bin" # base (ssh) — root's SSH shell has no sbin on PATH; the wrapper exports it -ssh -i ~/.ssh/lifetrac_base_ed25519 fio@192.168.1.117 "REVIVE_MODE=reboot bash /home/fio/run_flash_bench.sh /tmp/lifetrac_p0c/firmware_bench_diag.bin" +ssh -i fio@ "REVIVE_MODE=reboot bash /home/fio/run_flash_bench.sh /tmp/lifetrac_p0c/firmware_bench_diag.bin" ``` +**`REVIVE_MODE` now defaults to `reboot`** (2026-10-10) in both +`run_flash_bench.sh` and `full_flash_pipeline.sh`; it used to default to +`full`, so every flash had to pass `REVIVE_MODE=reboot` by hand. Passing it +explicitly, as above, still works. `REVIVE_MODE=full` remains selectable for +a 5.10 board where a reboot is unwanted; on the base's 6.1 kernel it oopses +(below). `revive_bridge.sh` treats `reboot` as `reset_run_only`, and it warns +before a module reload on any 6.x kernel. Other wrapper variables: +`FLASH_VERIFY_ONLY=1` (read back and compare, write nothing), +`BENCH_SUDO_PW` and `BENCH_HOME` (defaults `fio`, `/home/fio`). + What happens: `wdt_pet start` (pets `/dev/watchdog0` every 10 s, fsync'd log) → `prep_bridge` (unbind consumers, rmmod `x8h7_*`) → `run_flash_l072` (openocd holds BOOT0 high + NRST, AN3155 write + `Verify OK`, ~45 s) → @@ -59,7 +146,8 @@ Why the reboot is deliberate: the old revive re-inserted the `x8h7` modules, which OOPSes the base's 6.1.24 kernel in `spi_probe → of_irq_get` and drops the board anyway (twice on 2026-09-12, petter alive 2 s before). `REVIVE_MODE=full` keeps the old behaviour for the tractor's 5.10 -kernel if ever wanted; nothing needs it. +kernel if ever wanted; nothing needs it, which is why `reboot` is now the +default. ## 3. Watchdog facts (why the scripts look like this) @@ -71,7 +159,14 @@ kernel if ever wanted; nothing needs it. with no petter → reboot 60 s later (tractor, 5.10.93, 2026-09-12). - No `/sys/class/watchdog/*` attributes, no `devmem`, host python has no `mmap`: read registers with `wdog_regs.py` inside the docker image - (`--privileged -v /dev/mem:/dev/mem`). + (`--privileged -v /dev/mem:/dev/mem`). The script now lives in the repo at + [`bench_tools/wdog_regs.py`](bench_tools/wdog_regs.py). Until 2026-10-10 + it existed only in `/home/fio` on the boards; it was recovered from the + board captures. It is read-only (`O_RDONLY`, `PROT_READ`). Push it to + `/tmp/lifetrac_strict`, then: + `echo fio | sudo -S -p '' docker run --rm --privileged -v /dev/mem:/dev/mem -v /tmp/lifetrac_strict:/work --entrypoint python3 lifetrac-v25:latest /work/wdog_regs.py` + (on the tractor use the `arduino-ootb-python-devel:738bc44` image). + Expected: `WCR=0x773d WDE=1 WT=119 (60.0 s)`. ## 4. After the reboot @@ -94,3 +189,135 @@ bracket silently parses only the old fields. Park at the end of the session: `radio_park.py` → `PARK_OK {"opmode_readback": "0x80"}` on both boards. + +## 5. First flash from a factory board + +This section collects guidance from the 2026-05 bring-up notes that was not +in one place before. Both bench L072s have been flashed many times since +May. Nobody has run this first-flash procedure from scratch since then, so +treat it as consolidated guidance and record what you see. + +### What a factory board has + +- The L072 inside the Murata CMWX1ZZABZ runs Arduino's stock **MKRWAN AT + modem firmware**. The repo keeps 1.2.3 as the factory restore image, + [`../../../tools/mlm32l07x01.bin`](../../../tools/mlm32l07x01.bin) + (83 032 B). The file of the same name in this directory is misnamed: it is + LifeTrac "hello v0.2". +- **Nothing on the carrier needs to change for this path.** The X8 reaches + the L072's ROM bootloader with parts it already has. `/dev/ttymxc3` (i.MX + UART4, wired straight to the L072, not through the H7 bridge) carries + AN3155 at **19200 8E1**. The X8's own openocd + (`/usr/arduino/extra/openocd_script-imx_gpio.cfg`) halts the H747 and + drives the L072's **BOOT0 = H7 PA11** and **NRST = H7 PF4**. + [`07_assert_pa11_pf4_long.cfg`](07_assert_pa11_pf4_long.cfg) holds BOOT0 + high for a 600 s window. [`stm32_an3155_flasher.py`](stm32_an3155_flasher.py) + is stdlib-only, because LmP's python has no `termios`; `stty` sets up the + UART in the shell. Pin map: [Phase 0 crack note](../../../AI%20NOTES/2026-05-07_Portenta_X8_LoRa_Phase0_Crack_and_Phase1_Options_Copilot_v1_0.md). + First end-to-end flash, which wrote the MKRWAN image itself: + [Method G Phase 1 note](../../../AI%20NOTES/2026-05-08_Method_G_Phase1_End_to_End_Flash_Success_Copilot_v1_0.md). +- No J-Link or ST-Link is needed. The carrier's L072 SWD header `CN2` is + **not populated** by default + ([BRINGUP_MAX_CARRIER.md](../murata_l072/BRINGUP_MAX_CARRIER.md) §2); it + only matters for the last-resort recovery below. + +### Before the first flash + +1. Set up the X8 first ([`bench_tools/BENCH_SETUP.md`](bench_tools/BENCH_SETUP.md)): + the sudoers drop-in, docker usable, and on the tractor + `lifetrac-camera.service` / `tractor-camera` stopped. Both grab + `/dev/ttymxc3`, and `run_flash_l072.sh` refuses while anything holds it + (exit 90). +2. **Freshly reflashed X8 (uuu):** `gpio10` (H7 NRST) boots LOW, so openocd + reports `cannot read IDR` (README_P0C TT-1). `run_flash_l072.sh` already + exports gpio8/10/15, drives gpio10 high and unexports them before + openocd attaches. If attach still fails, compare against a good board + ([HC-03](../../X8_HEALTH_AND_RECOVERY/routines/HC-03_h7_swd_attach_diff.md)). + `SWD DPIDR 0xdeadbeef` on a newer openocd is the sysfs/mmap race, **not a + bricked L072** ([T4](../../X8_HEALTH_AND_RECOVERY/recovery/T4_lora_jlink_hardware_unbrick.md) + "When NOT to use"). +3. Build with the exact toolchain and check the md5 (section 0). + +### The first flash, step by step + +1. **Read-only first contact.** Run + `bash bench_tools/flash_l072.sh ../../../tools/mlm32l07x01.bin --go --verify-only`. + It enters the ROM bootloader, reads the flash back and compares it with + the stock image without writing anything, then reboots the X8. + - `READY: L072 in STM32 ROM bootloader` (openocd) followed by the + flasher's `Bootloader version: 31` means ROM entry and the UART path + work. + - `Verify-only OK` means the chip carries stock 1.2.3. + - A mismatch only means a different stock version. The real question is + whether ROM entry works. + - This is also how to capture what was on the chip before LifeTrac + touched it. +2. **Flash the bench build:** + `bash bench_tools/flash_l072.sh ../murata_l072/build/firmware_bench_diag.bin --go`. + - Erase: the flasher first tries Extended Erase mass erase (`0x44`, + `0xFFFF`). On the Murata factory part that NACKs, so it falls back to + **page-by-page erase** of only the pages the image needs. The rest of + the flash and the option bytes are untouched. + - Success means `Verify OK`, `flash_rc=0` and `PIPELINE-EXIT=0` in + `/home/fio/pipeline_stamped.log`. The X8 reboots + (`REVIVE_MODE=reboot`, section 2). +3. After the reboot, follow section 4: re-stage `/tmp`, stop the tractor + camera, run `rs116_health_probe.py`, and park the radio. + +### Option bytes and readout protection: do not touch them + +- **Never send these AN3155 commands from the X8:** Write Unprotect `0x73`, + Readout Protect `0x82`, Readout Unprotect `0x92`, or any Write Memory to + the option bytes at `0x1FF80000`. The pipeline's flasher sends none of + them. + - On 2026-05-14, `flash_l072_via_uart.py wunprot` (through + `recover_l072_opt.sh`) was ACKed, and then the chip went silent. Its + option-byte complement pairs no longer matched, and it would not run its + flash again without SWD + ([timeline](../../../AI%20NOTES/2026-05-14_W2-02_Board2_L072_wunprot_Brick_Timeline_Copilot_v1_0.md), + [root cause](../../../AI%20NOTES/2026-05-14_W2-02_Board2_L072_OPT_Bytes_Root_Cause_Copilot_v1_0.md)). + - That script, `flash_l072_via_uart.py`, `l072_unprotect.py`, + `run_l072_unprotect.sh` and `recover_l072_opt.sh` are still in this + directory **as history. Do not run them.** +- Readout Protect sets RDP level 1. The ROM then refuses reads, so the + pipeline can no longer verify. Going back to level 0 mass-erases the chip. + Level 2 is permanent. Neither has any use on the bench. +- An option-byte change needs a full power-on reset to load; NRST alone is + not enough (root-cause note §4.2). +- Factory-clean option bytes read `0x807800AA` at `0x1FF80000` (RDP `0xAA` = + level 0). Source: the + [J-Link note §3a](../../../AI%20NOTES/2026-05-17_Murata_L072_LoRa_Unbrick_JLink_Hardware.md). + The bench L072s' current option bytes were never recorded. Reading them + (AN3155 Read Memory, read-only) is an open item in + [`BENCH_BOARDS.md`](bench_tools/BENCH_BOARDS.md), to be done only on a GO. + +### When it goes wrong (2026-05 lessons) + +- **Openocd never prints `READY: L072 in STM32 ROM bootloader`.** This is + the H7 side (gpio10, the openocd race, or a wedged bridge), not the L072. + Power-cycle the X8 and carrier with the 12 V barrel and USB-C unplugged + for 10 s + ([T2](../../X8_HEALTH_AND_RECOVERY/recovery/T2_cold_power_cycle.md)) + before you suspect the chip. +- **Do not use the PWM/gpio163 "SWD bypass"** + (`swd_bypass_pa11_pf4_launcher.sh`, sysfs PWM4 + gpio163). It was shown + on 2026-05-17 not to enter the ROM. Only the openocd path does. +- **Last resort: an external J-Link on `CN2`** + ([T4](../../X8_HEALTH_AND_RECOVERY/recovery/T4_lora_jlink_hardware_unbrick.md), + [J-Link note](../../../AI%20NOTES/2026-05-17_Murata_L072_LoRa_Unbrick_JLink_Hardware.md)). + 1. Solder a 2×5 1.27 mm header. Use the standard Cortex 10-pin zig-zag + cable; the Segger needle adapter does not fit. + 2. VTref (~3.3 V) needs the 12 V barrel jack **and** the X8 seated and + powered. + 3. Run `JLink.exe -device STM32L072CZ -if SWD -speed 1000 -autoconnect 1`. + 4. `erase`: a mass erase that also restores the option-byte defaults. + 5. `mem32 0x1FF80000 4` must show `0x807800AA`. + 6. `loadbin , 0x08000000`, then `r` and `go`. + + Once the option bytes are clean, the X8 pipeline above works again. +- **Which board was bricked is uncertain.** The 05-14 brick ran on adb + serial `2E2C…`, but the 05-17 unbrick note says "Board 1 (`2D0A…`)". The + notes disagree, which [`BENCH_BOARDS.md`](bench_tools/BENCH_BOARDS.md) + lists as an open item. +- Do not accept J-Link OB firmware-update prompts. An X8 OS reflash (T3a) + never fixes an L072 problem. diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/README.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/README.md new file mode 100644 index 00000000..4907c214 --- /dev/null +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/README.md @@ -0,0 +1,52 @@ +# x8_lora_bootloader_helper + +Tools that run on a Portenta X8 to reflash and talk to the Murata L072 +(the LoRa radio module on the Max Carrier, on `/dev/ttymxc3`), plus the +PC-side harness for radio bench legs. + +Where to start: + +- Flashing the L072: [FLASH_RUNBOOK.md](FLASH_RUNBOOK.md). +- Running the radio bench from scratch: + [bench_tools/BENCH_QUICKSTART.md](bench_tools/BENCH_QUICKSTART.md). +- What is on each bench board today: + [bench_tools/BENCH_BOARDS.md](bench_tools/BENCH_BOARDS.md). + +## Binaries in this folder: read before you flash + +The file names here are misleading. None of these files is the firmware +the bench boards run. + +| file | size | md5 | what it really is | +|---|---|---|---| +| `mlm32l07x01.bin` | 4,808 B | `1e3d4328dd74e371e4d583e6fe84f856` | **Misnamed.** A LifeTrac test build ("LIFETRAC L072 hello v0.2 (USART1+LPUART1)"): it prints a banner and a tick counter. It is byte-identical to `../murata_l072/build_hello/hello_world.bin`, built from [`../murata_l072/hello_world.c`](../murata_l072/hello_world.c). It is **not** the Murata/Arduino stock firmware, even though it has the stock file name. | +| `hello.bin` | 677 B | `972340b8486fb8589ff99fed3dbf4d29` | An earlier, smaller hello build with the same banner. It is byte-identical to `../murata_l072_hello/hello.bin`, built from [`../murata_l072_hello/main.c`](../murata_l072_hello/main.c). | +| `board2_l072_flash_dump.bin` | 16,440 B | `16e12477f904245f54a694eb7e31fc4c` | A **readback**, not a build. It holds the first 16,440 bytes of L072 flash from `0x08000000`, read in May 2026 (committed 2026-05-13) with [`dump_l072_flash.py`](dump_l072_flash.py) from the board the May notes call "Board 2" (2E2C under that naming). The length is just the size of the `murata_l072` build of that week. It is an early LifeTrac build. It matches no tracked build byte for byte and is not the factory image. Keep it as diagnostic evidence and never flash it. | + +### Why the hello build has the stock name + +The early flash scripts default to an image called `mlm32l07x01.bin`. For +example, [`run_flash_l072.sh`](run_flash_l072.sh) uses +`IMAGE=${1:-/tmp/lifetrac_p0c/mlm32l07x01.bin}`. The hello build was saved +under that name in May 2026, most likely so those defaults would pick it up +(the 2026-05-08 hello-world notes push it as +`/tmp/lifetrac_p0c/mlm32l07x01.bin`). The file is not renamed here because +old notes and scripts push it by that path. Pass an explicit image path +instead of relying on the default. + +### The real stock image and the real bench firmware + +- **Stock (factory) L072 firmware:** + [`LifeTrac-v25/tools/mlm32l07x01.bin`](../../../tools/mlm32l07x01.bin) + (83,032 B, md5 `33b8a001fac12396c6b7211a463a9cab`). This is Arduino's + MKRWAN AT-modem firmware: its strings include the `+VER`/`+DEV` AT + commands, `ARD-078` and version `1.2.3`. It was extracted from the MKRWAN + library's `MKRWANFWUpdate_standalone/fw.h`. Use it to put a module back to + factory state; the LifeTrac bench software does not work with it. +- **LifeTrac bench firmware (both bench boards since 2026-09-15):** build + it with `mingw32-make bench` in [`../murata_l072/`](../murata_l072/). That + gives `build/firmware_bench_diag.bin`, md5 + `0c1bb0a9573f813137f941dfa47177d0`; toolchain and expected hashes are in + [`../murata_l072/README.md`](../murata_l072/README.md). Saved copies of + every flashed bench build are in + [`bench-evidence/RS_13_vector_scene_2026-09-26/firmware/`](../../bench-evidence/RS_13_vector_scene_2026-09-26/firmware/README.md). diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/.gitignore b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/.gitignore new file mode 100644 index 00000000..c4bede8e --- /dev/null +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/.gitignore @@ -0,0 +1,3 @@ +# Per-bench settings (serials, addresses, the day's DTS carrier); start from +# bench.env.example. Never commit the real file. +bench.env diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_BOARDS.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_BOARDS.md index 53800d9f..b571a934 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_BOARDS.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_BOARDS.md @@ -8,13 +8,21 @@ captures: The rest comes from a review of AI NOTES, bench-evidence, X8_HEALTH_AND_RECOVERY and git history. +**To bring up a new pair to this state, see [BENCH_SETUP.md](BENCH_SETUP.md).** +It covers hardware, the OS image pinned by URL and sha256, the reflash, +first boot, verification and the L072 firmware. +[`provision_bench_board.sh`](provision_bench_board.sh) applies the `/etc` +and docker changes listed under *LifeTrac software living in the X8 Linux* +below, except the production units, which it never installs. `--check` +reports how far a board is from that state. + ## The two boards | | base | tractor | |---|---|---| | serial (adb) | `2D0A1209DABC240B` | `2E2C1209DABC240B` | | hardware | Portenta X8 on a Portenta Max Carrier | same | -| OS (LmP) | **4.0.11-934-91**, kernel 6.1.24-lmp-standard (uuu reflash 2026-05-13 and 05-24; bundle sha256 in the private archive) | **4.0.3-674-88**, kernel 5.10.93-lmp-standard (factory, never reflashed). **No copy of the 674 image exists.** A reflash would move it to 934/6.1.24. | +| OS (LmP) | **4.0.11-934-91**, kernel 6.1.24-lmp-standard (uuu reflash 2026-05-13 and 05-24). The bundle is `https://downloads.arduino.cc/portentax8image/934.tar.gz`; its sha256 list is in [T3a](../../../X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md). | **4.0.3-674-88**, kernel 5.10.93-lmp-standard (factory, never reflashed). Arduino still hosts the image at `https://downloads.arduino.cc/portentax8image/674.tar.gz` (854,998,055 B). Nobody here has downloaded or byte-verified it, so there is no sha256 on record. Reflashing to 934 would move it to 6.1.24; BENCH_SETUP §3 compares the two. | | access | adb; ethernet 192.168.1.117 with eth0 pinned to 100BASE-TX full (marginal cable); ssh key `~/.ssh/lifetrac_base_ed25519`. On 2026-10-10 it did not enumerate on USB after a power cycle, but ssh worked. | adb only (WiFi off by policy) | | power-down | `systemctl poweroff` did **not** keep it off: it booted again about 4.5 min later (2026-10-04). Remove power to keep it down. | stayed off after `systemctl poweroff` | | clock | NTP | no time source; it lags by hours to days | @@ -114,8 +122,11 @@ Repo L072 binaries that are on no board: - `docker/key.json`. All of these were installed by `provision_x8.sh` (2026-05-15) and the May - WiFi-off steps. The `disable-wifi.service` unit and the wpa mask are not in - git. + WiFi-off steps. The `disable-wifi.service` unit is now in git as + [`units/disable-wifi.service`](units/disable-wifi.service), copied from + the capture. `provision_bench_board.sh tractor` installs and enables it, + masks `wpa_supplicant`, and reruns `provision_x8.sh` when the camera + files differ. - **WiFi:** soft-blocked and disabled in NetworkManager. **A stored `5star.nmconnection` holds the WiFi password** (not copied). - **`/var/rootdirs/opt/lifetrac`:** @@ -144,7 +155,9 @@ Repo L072 binaries that are on no board: - **Repo:** - L072 source and bench binaries (PR #140); - - provisioning scripts, unit files and the sudoers installer; + - provisioning scripts, unit files and the sudoers installer, now driven + by [`provision_bench_board.sh`](provision_bench_board.sh) and + [BENCH_SETUP.md](BENCH_SETUP.md); - every sketch source; - the base capture (this folder's sibling); - the capture tools [`capture_board_state.sh`](capture_board_state.sh) @@ -159,8 +172,9 @@ Repo L072 binaries that are on no board: - `base/`: the full capture tarball and the deployed tree without secrets. - `pc_only/`: the patched `libmbed_x8.a`, the LmP 934 bundle's sha256 - list, and copies of the Claude and Copilot memory notes, which are the - only written record of several provisioning steps. + list (now also in T3a), and copies of the Claude and Copilot memory + notes. Those notes were the only written record of several provisioning + steps, which are now in BENCH_SETUP.md. - `scripts/`. **Make a second copy on an external disk or NAS.** @@ -177,7 +191,10 @@ Repo L072 binaries that are on no board: 2. Stage only the probe tools (no `push_fix_to_board.sh`). Then run `radio_state.py`. At boot the L072 sits in **RXCONT**, listening only; `radio_park.py` puts it to `0x80` SLEEP. -3. Run `bash pull_board_state.sh [--images]`. Review the +3. Run `bash pull_board_state.sh [--images]` (adb only). It + writes to `ARCHIVE`, else `ARCHIVE_DIR` from `bench.env`: any folder + outside git, in any path form (the script converts it for adb; + [BENCH_QUICKSTART.md](BENCH_QUICKSTART.md) step 9). Review the reports for secrets before copying them into `bench-evidence/`; the 2026-10-04 run caught a Dropbear host key, and the filter now excludes it. @@ -190,17 +207,27 @@ Repo L072 binaries that are on no board: - [ ] **Decide on the tractor's `lifetrac-tractor-compose.service`.** It is enabled and fails at every boot, like the base's units. - [ ] Delete the stored `5star` WiFi profile on the tractor once the password - is rotated (`nmcli con delete 5star`). + is rotated (`nmcli con delete 5star`). `provision_bench_board.sh tractor` + backs up the stored WiFi profiles to a root-only directory on the board, + then deletes them and prints only their names. Pass `--keep-wifi-profiles` + to keep them until then. - [ ] **H747 M4 bank 2: identify, then decide.** An openocd `dump_image` of banks 1 and 2 halts the H7 and drops the x8h7 bridge, and needs a deliberate reboot (FLASH_RUNBOOK precautions; never re-insmod x8h7 on the base). **Explicit GO only.** Then upload an empty M4 sketch or erase bank 2, and record which. -- [ ] **Rotate the WiFi password.** It is committed in seven repo-root +- [ ] **Rotate the WiFi password.** It was committed in seven repo-root scripts on the public `main` (`connect_wpa.sh`, `do_wifi.sh`, `setup_5star_wifi.sh`, `wifi_connect.sh`, `wifi_connect_fast.sh`, - `wifi_persist_and_reboot.sh`, `wifi_rescan_connect.sh`). Then replace the - literal with an environment variable. + `wifi_persist_and_reboot.sh`, `wifi_rescan_connect.sh`) and in + `.vscode/tasks.json`. + - [x] ~~Replace the literal with an environment variable.~~ Done in the + bench kit (2026-10-10): the scripts take the SSID and passphrase from + `LIFETRAC_WIFI_SSID` / `LIFETRAC_WIFI_PSK`, and the VS Code tasks that + carried the passphrase were removed (it would appear in the task echo). + - [ ] **Change the passphrase on the access point.** The old value is + still in the public git history, so treat it as compromised. Rewriting a + public history with forks is not practical; rotation is the fix. - [ ] **Decide on the base's `lifetrac-base.service` / `lifetrac-base-compose.service`.** Both are enabled and fail at every boot; they would start `lora_bridge` on the radio UART if they ever succeeded. @@ -209,7 +236,9 @@ Repo L072 binaries that are on no board: - [ ] Record the carrier ↔ module pairing, both carriers' DIP switch positions and the J-Link OB serials (1078222309 / 1078180658). - [ ] Fix the doc errors the review found: - - T3a verdict row :66 says 2E2C for the 05-13 reflash; it was 2D0A. + - ~~T3a verdict row :66 says 2E2C for the 05-13 reflash; it was 2D0A.~~ + Fixed 2026-10-10. The row now matches the session log: `BOOT SEL` only, + USB-C only, PID 0134. - `setup_x8_mqtt_link.ps1:9` names the wrong board. - HC-02:13 uses the old name `board1_healthcheck.sh`. - FIRMWARE_UPDATES.md:53 calls the handheld a Portenta H747. diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_QUICKSTART.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_QUICKSTART.md new file mode 100644 index 00000000..f9bbec44 --- /dev/null +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_QUICKSTART.md @@ -0,0 +1,422 @@ +# Radio bench quickstart + +Start here to reproduce the LifeTrac v25 LoRa bench from GitHub: two +Portenta X8 boards on Max Carriers talking over their Murata L072 radios, as +in the RS-12 and RS-13 campaigns. This page lists the steps in order with the +commands. The linked documents explain each step. + +**The GO rule.** Radios run only on an explicit operator GO, given for each +round. Without that GO, nothing that opens `/dev/ttymxc3` runs: no flash +(the L072 boots into RXCONT, so a flash turns the receiver on), no probe +(the L072 section of the HC-02 health check writes to that UART too, +[BENCH_SETUP.md](BENCH_SETUP.md) §7), no channel spot-check, no leg. There is +one exception: `radio_state.py`, which reads whether a radio is asleep, and +`radio_park.py`, which puts it to sleep. The power-up guard runs both on its +own (step 5.1). Apart from the guard's `radio_state.py` / `radio_park.py`, +power-up, staging, deploys and image builds do not touch the radios. + +## What you need + +- **Boards.** Two Arduino Portenta X8 (ABX00049), each on a Portenta Max + Carrier (ABX00043). The carrier holds the Murata CMWX1ZZABZ (L072 + SX1276). + You also need a 12 V supply per carrier and a USB-A to USB-C data cable from + each X8 straight to the PC: no hub, and no USB-C to USB-C cable + ([BENCH_SETUP.md](BENCH_SETUP.md) §1). +- **Antennas.** A 915 MHz antenna on each carrier's LoRa SMA (the bench uses + Taoglas TD.95.6H31). Never key a radio without one. +- **Network.** Ethernet for the **base** to a LAN with DHCP and internet; it + builds the Docker images. The **tractor** has WiFi off and is reached over + adb only. +- **Camera legs.** A USB UVC camera on the tractor (it enumerates as + `/dev/video1`), aimed at the PC screen. +- **PC.** Windows 11 on the same LAN as the base. Set it up with + [PC_SETUP.md](PC_SETUP.md). + +On the original bench the **base** is `2D0A1209DABC240B` and the **tractor** +is `2E2C1209DABC240B`; those are the values in `bench.env.example`. Your +boards' serials go into `bench.env` (step 1). +[BENCH_BOARDS.md](BENCH_BOARDS.md) lists what is on every chip of the +original pair. + +## 0. PC setup + +Follow [PC_SETUP.md](PC_SETUP.md): Git for Windows (long paths), Python +through `py -3` plus [`requirements-pc.txt`](requirements-pc.txt), the Arm GNU +Toolchain 12.2.MPACBTI-Rel1, WinLibs `mingw32-make`, adb, a PowerShell +execution policy that lets the kit's `.ps1` scripts run +([PC_SETUP.md](PC_SETUP.md#powershell-execution-policy)), and optionally +`uuu` and arduino-cli. + +## 1. Configure `bench.env` + +Every bench script reads `bench_tools/bench.env`: provisioning, the L072 +flash, the deploys and the leg scripts. Create it now, before any script +touches a board. A script that finds no `bench.env` falls back to the +original bench's serials and base address (`192.168.1.117`), which on your +LAN may be some other host. + +The commands on this page run in **Git Bash from the repo root**, except +those marked PowerShell. Start every Git Bash window with: + +```bash +export MSYS_NO_PATHCONV=1 # adb from Git Bash: no MSYS path rewriting +DC=LifeTrac-v25/DESIGN-CONTROLLER +BT=$DC/firmware/x8_lora_bootloader_helper/bench_tools +``` + +Create the file once, then edit it: + +```bash +cp "$BT/bench.env.example" "$BT/bench.env" # gitignored; never commit it +``` + +Fill in, following the comments in the file: + +- `BASE_SERIAL`, `TRACTOR_SERIAL`: the adb serials. On new boards you read + them at first boot ([BENCH_SETUP.md](BENCH_SETUP.md) §5, before + provisioning). +- `BASE_HOST`: the base's reserved DHCP lease. `BASE_SSH_KEY` only if your + key is not `~/.ssh/lifetrac_base_ed25519`. +- `BENCH_SUDO_PW` if you changed the `fio` password. +- `PC_HOST` (**required** for legs): this PC's LAN IPv4 address as the boards + see it. The harness gets it as `-HostIp` and the base rx daemon uses it as + its control broker. +- `BASE_TRANSPORT`: `adb` (radio legs need the base on USB) or `ssh`. + +Check what the scripts will use: + +```bash +bash -c '. "$0/lib/bench_env.sh" && bench_show_env' "$BT" +``` + +Then load the board values into this shell. The commands below use these +variables. Paste the block into each new Git Bash window, and again after you +edit `bench.env`: + +```bash +# board values from bench.env, read in a subshell (the loader is not meant for an interactive shell) +eval "$(bash -c '. "$0/lib/bench_env.sh" || exit 1 + for v in BASE_SERIAL TRACTOR_SERIAL BASE_HOST BASE_SSH_USER BASE_SSH_KEY BASE_IMAGE TRACTOR_PROBE_IMAGE ARCHIVE_DIR; do + printf "%s=%q\n" "$v" "${!v}"; done' "$BT")" +BASE="ssh -i $BASE_SSH_KEY $BASE_SSH_USER@$BASE_HOST" # base: ethernet + ssh +TRAC="adb -s $TRACTOR_SERIAL shell" # tractor: adb only +R="docker run --rm --network=host --entrypoint python3 --device=/dev/ttymxc3 -v /tmp/lifetrac_strict:/work -w /work -e PYTHONPATH=/work:/work/paho" +BI=$BASE_IMAGE # base daemon image +TI=$TRACTOR_PROBE_IMAGE # tractor probe image +``` + +## 2. Boards (once per board) + +[BENCH_SETUP.md](BENCH_SETUP.md) covers this step: + +- the OS image, pinned by URL and sha256 in T3a: LmP 4.0.11-934-91 on both + boards. The original tractor still runs its factory 4.0.3-674-88; keep 674 + only to match that pair exactly (BENCH_SETUP §3); +- reflashing through + [T3a](../../../X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md), + brand-new boards included, because the factory image depends on the + production batch (BENCH_SETUP §4); +- first boot, where the serials and the base address go into `bench.env`; +- [`provision_bench_board.sh`](provision_bench_board.sh), which installs the + sudoers drop-in, masks the compose early-start units on the base **before + any docker use**, loads the probe image from the repo-root + `foundries_python.tar` on both boards, and on the tractor adds WiFi-off + ([`units/disable-wifi.service`](units/disable-wifi.service)), the camera + unit handling and `eclipse-mosquitto:2` (`--mosquitto-from-base`). + +## 3. Radio firmware (L072) + +Both boards run the bench build **`0c1bb0a9`**. The production +`build/firmware.bin` (`589c1203`) refuses the bench carrier pin, so do not put +it on a bench board. + +```powershell +# PowerShell (mingw32-make does not work from Git Bash) +cd LifeTrac-v25\DESIGN-CONTROLLER\firmware\murata_l072 +mingw32-make check +mingw32-make bench +(Get-FileHash build\firmware_bench_diag.bin -Algorithm MD5).Hash # 0C1BB0A9573F813137F941DFA47177D0 +``` + +A different md5 means a different toolchain; see the toolchain section of +[murata_l072/README.md](../../murata_l072/README.md). If you have no +toolchain, the same bytes are committed as +[`firmware_bench_diag_0c1bb0a9.bin`](../../../bench-evidence/RS_13_vector_scene_2026-09-26/firmware/). + +Flash with [`flash_l072.sh`](flash_l072.sh), which is +[FLASH_RUNBOOK.md](../FLASH_RUNBOOK.md) §1–§2 as one script and reads +`bench.env`. One board at a time: + +```bash +FW=$DC/firmware/murata_l072/build/firmware_bench_diag.bin +bash "$BT/flash_l072.sh" tractor "$FW" # stage LF-clean + preflight only: want PREFLIGHT-OK +bash "$BT/flash_l072.sh" tractor "$FW" --go # NEEDS THE GO: stops the camera unit, then flashes +``` + +Then the same with `base`. Without `--go` the script copies files and runs +the wrapper's preflight; it touches no radio. + +- **New carriers** still run Arduino's stock MKRWAN firmware. Work through + [FLASH_RUNBOOK.md §5](../FLASH_RUNBOOK.md#5-first-flash-from-a-factory-board), + "First flash from a factory board", first. It starts with a read-only + first contact (also on the GO): + `bash "$BT/flash_l072.sh" LifeTrac-v25/tools/mlm32l07x01.bin --go --verify-only`, + then flashes the bench build as above. +- **Success** is `Verify OK` and `flash_rc=0` in + `/home/fio/pipeline_stamped.log`. The flash reboots the board + (`REVIVE_MODE=reboot`), which wipes `/tmp`, and the L072 comes back + listening in RXCONT. +- **Afterwards** run the health probe and park of FLASH_RUNBOOK §4; the + commands are in [BENCH_SETUP.md](BENCH_SETUP.md) §8, step 3. + `bash "$BT/legs/power_up_guard.sh" --only ` does the park part on + its own: it stages the probe tools, reads the radio and parks it. + +## 4. Board software + +[DEPLOY.md](DEPLOY.md) explains both scripts and their arguments: + +- [`deploy_base.sh`](deploy_base.sh) puts a commit's `DESIGN-CONTROLLER` + subset on the base and builds `lifetrac-v25:latest` there. It never starts + `lora_bridge`. The base broker `design-controller-mosquitto-1` must be + running. +- [`build_tractor_image.sh`](build_tractor_image.sh) builds + `lifetrac-tractor-x8` natively on the base, then loads it onto the offline + tractor. The two images the tractor cannot pull came with provisioning + (step 2): `provision_bench_board.sh tractor` loads the probe image from the + repo-root `foundries_python.tar` and, with `--mosquitto-from-base`, + `eclipse-mosquitto:2`. `build_tractor_image.sh --with-mosquitto` can reload + mosquitto. + +Wheel versions are pinned in the `requirements.lock.txt` files next to each +`requirements.txt`. + +## 5. Every session + +1. **Power-up radio safety.** Arm the guard, then switch the boards on: + + ```bash + bash "$BT/legs/power_up_guard.sh" --wait-power-cycle # boards already up: leave out the flag + ``` + + As each board comes up, the guard stops the tractor's + `lifetrac-camera.service` and the `tractor-camera` container (both grab + `/dev/ttymxc3` at every boot) and reports any UART holder. It then stages + the probe tools, reads each L072 with `radio_state.py`, and parks any + radio that is not in `0x80` SLEEP with `radio_park.py` (3 tries, 60 s + apart). Those two scripts are the GO-rule exception; the guard never + transmits. Each L072 boots into RXCONT, which only listens, and the guard + is what puts it to sleep. `--no-park` only reads; `--check-only` is the + end-of-session check (step 8). +2. **Staging.** + + ```bash + bash "$BT/legs/stage_boards.sh" + ``` + + This re-pushes `/tmp/lifetrac_strict` on both boards. Run it after any + reboot (and every flash reboots) or when the last push is 5 days old, + because `systemd-tmpfiles` ages `/tmp`. Afterwards, + `ls /tmp/lifetrac_strict | wc -l` must be at least 19. The flash staging + in `/tmp/lifetrac_p0c` is separate: `flash_l072.sh ` without + `--go` re-stages it and runs the wrapper's preflight, and `PREFLIGHT-OK` + means it is complete. +3. **Moving scene (camera legs).** Play the railroad video in a **normal** + Firefox window of a separate profile, with the player's own fullscreen + (Esc leaves it). **Never use `firefox --kiosk`**: a kiosk locked the + operator out of the PC. Before **every** camera run, check the scene: at + least 8 % of pixels must change over 10 s (`legs/scene_check.sh`; see + [legs/README.md](legs/README.md)). +4. **Channel spot-check (needs the GO; receive only) and the DTS carrier + pin.** The tractor must be parked. The guard parked it at power-up; park + it again if anything has woken it since (a probe, a flash, a leg prep): + + ```bash + $TRAC "sudo -n $R $TI -u /work/radio_park.py" # want PARK_OK, both readbacks 0x80 + ``` + + Then sniff the candidate channel on the base, where the known emitter is + about 20 dB hotter. `CAND` is the carrier you want to try, in Hz: inside + 902250000..927750000, and not 915000000. + + ```bash + CAND= + scp -i "$BASE_SSH_KEY" $DC/firmware/x8_lora_bootloader_helper/channel_survey_sniff.py "$BASE_SSH_USER@$BASE_HOST:/tmp/lifetrac_strict/" + $BASE "sudo docker rm -f rx_smoke 2>/dev/null; sudo $R -e LIFETRAC_REG_PROFILE=2 $BI -u /work/channel_survey_sniff.py --start-hz $CAND --stop-hz $CAND --step-hz 500000 --dwell-s 45 --interval-s 0.05" | tee spot_$(date +%F).txt + py -3 $DC/tools/survey_compare.py spot_$(date +%F).txt + ``` + + A hot sample rules a channel out. A clean one does not prove the link; + only the leg does. To sweep instead of spot-checking, use + `--start-hz 902500000 --stop-hz 927500000 --dwell-s 30`, which takes about + 26 minutes. Picks hold for hours, not days. + + **Write the pick into `bench.env`** and record it in RESULTS: + + ```bash + DTS_CARRIER_HZ= + DTS_CARRIER_DATE= + ``` + + `leg_prep.sh` refuses a profile-2 leg without `DTS_CARRIER_HZ`, on + 915.000 MHz, and on any day other than `DTS_CARRIER_DATE`. It puts the + pick into the harness command as `-ForceFrfHz`. The harness itself + refuses `-RegProfile 2` without `-ForceFrfHz`, because the profile + default, 915.000 MHz, is the channel of an external emitter (RS-11.6). + `-AllowDefaultCarrier` overrides that refusal; use it only for a + deliberate 915 MHz control leg. + +## 6. Run a leg (needs the GO) + +[legs/README.md](legs/README.md) lists the per-leg scripts and their +arguments. They are parameterised copies of RS-13's `leg_prep.sh` and +`leg_post.sh`: + +- **prep:** scene check, health probes, `clear_retained.py`, pre-brackets, + base-side capture. It ends by writing the harness command for this leg; +- **post:** post-brackets, reports, park. + +RS-13 leg 2a, VECTOR at boot on DTS, goes like this: + +```bash +bash "$BT/legs/leg_prep.sh" 2a image_bw500 # prints the wrapper path (--print-harness: preview, touches no board) +``` + +```powershell +# PowerShell, right away: the base capture is already listening. +# leg_prep.sh printed the exact path of the wrapper. +& "/leg2a_harness.ps1" +``` + +```bash +bash "$BT/legs/link_sample.sh" 2a # while frames flow +bash "$BT/legs/leg_post.sh" 2a # after the harness prints "[EVIDENCE] archived to ..." +``` + +The wrapper runs +[`run_live_radio_monitor.ps1`](../run_live_radio_monitor.ps1) from the +helper directory with the serials, `-HostIp` and `-ForceFrfHz` from +`bench.env`. It tees the transcript and records the archive folder for +`leg_post.sh`. If PowerShell says that running scripts is disabled, see +[PC_SETUP.md](PC_SETUP.md#powershell-execution-policy). + +If you must type the harness command yourself, it is the line below, with +every `<...>` replaced by your `bench.env` values. Typed by hand, it skips +`leg_prep.sh`'s carrier gate and the archive record, so pass the archive to +`leg_post.sh 2a `: + +```powershell +cd LifeTrac-v25\DESIGN-CONTROLLER\firmware\x8_lora_bootloader_helper +.\run_live_radio_monitor.ps1 -TxAdbSerial -RxAdbSerial -HostIp ` + -TxFeed camera -RegProfile 2 -ForceFrfHz -DurationS 300 -SynthFps 2 ` + -KfRequestDisable 1 -ProbeEcho 0 -LogFragArrivals 1 -TxBatch 0 ` + -CamExtraEnv "-e LIFETRAC_ENCODE_MODE=9 -e LIFETRAC_VECTOR_DETAIL=80" -Archive +``` + +The other RS-13 legs (legs/README.md, "The four RS-13 legs"): + +- 2b is the same leg on FHSS: `leg_prep.sh 2b image_bw250` (`-RegProfile 1`, + no `-ForceFrfHz`). +- 2c is the `mono_g4` control: `leg_prep.sh 2c image_bw500` + (`-CamExtraEnv "-e LIFETRAC_ENCODE_MODE=6"`). +- 2d is the over-the-air switch: `leg_prep.sh 2d image_bw500`, then + `leg2d_switch.sh 2d` right after the harness launch. + +**Not flying after the prep after all** (no GO, the harness refused, the +round was aborted)? `leg_prep.sh` has woken both receivers and left the base +capture running. `bash "$BT/legs/power_up_guard.sh" --stop-leg-daemons` +stops the leftover containers and parks both radios. + +[RS13_VECTOR_LEG.md](RS13_VECTOR_LEG.md) gives each leg and the pass criteria +P1–P8. Start the session's `RESULTS.md` from +[RS13_RESULTS_TEMPLATE.md](RS13_RESULTS_TEMPLATE.md). For RS-12-style tile +legs, see [BENCH_RUNBOOK.md](BENCH_RUNBOOK.md), "A leg". The archive goes to +`DESIGN-CONTROLLER/bench-evidence/radio_monitor__/`, and its +`params.txt` records what actually ran. + +## 7. Analyse + +`leg_post.sh` runs the reports for you. To run them by hand, the tools are +[`rs12_leg_report.py`](../../../tools/rs12_leg_report.py), +[`frag_gap_report.py`](frag_gap_report.py), +[`vector_dry_run.py`](../../../tools/vector_dry_run.py) and +[`legs/leg_replay.py`](legs/leg_replay.py). Run them from +`DESIGN-CONTROLLER`. From Git Bash, give them Windows-style `C:/...` paths, +not `/c/...`: + +```powershell +py -3 tools/rs12_leg_report.py --pre /leg_pre_base.txt --post /leg_post_base.txt --capture /leg_base.jsonl +py -3 firmware/x8_lora_bootloader_helper/bench_tools/frag_gap_report.py +py -3 tools/vector_dry_run.py replay /leg_base.jsonl --profile image_bw500 +py -3 tools/vector_dry_run.py tractor-log /camera_service.log +py -3 firmware/x8_lora_bootloader_helper/bench_tools/legs/leg_replay.py image_bw500 --evidence-dir +``` + +`leg_replay.py [--evidence-dir DIR | --capture FILE]` +gives the post-lock sequence gaps and the P1 resync episodes; it reads +`/leg_base.jsonl`. + +The loss of record is the report's `capture seq gaps` line. The counter +`loss` line can be stale (RS13_VECTOR_LEG A14) or frozen by a dead stats +thread (RS-12.17). + +## 8. Park and power down + +About 2 minutes after the daemons stop, park both radios with +[`radio_park.py`](radio_park.py). The firmware's scan needs that long to fail +out. `leg_post.sh` does this at the end of a leg; by hand: + +```bash +$BASE "sudo $R $BI -u /work/radio_park.py" # want PARK_OK, both readbacks 0x80 +$TRAC "sudo -n $R $TI -u /work/radio_park.py" +``` + +`PARK_TRANSIENT` (exit 5) means the scan was still walking: wait 60 s and +repeat. The park readback does not guarantee the radio stays off, so before +you call the bench off, re-check both boards with the read-only +[`radio_state.py`](radio_state.py) (same command line), or run +`bash "$BT/legs/power_up_guard.sh" --check-only`. + +To power down: + +- **tractor:** `$TRAC "sudo -n systemctl poweroff"`; it stays off. +- **base:** `$BASE "sudo systemctl poweroff"`, then **remove its power**. + Otherwise it boots again about 4.5 minutes later. + +## 9. Capture board state + +```bash +ARCHIVE=$ARCHIVE_DIR bash "$BT/pull_board_state.sh" tractor # add --images for docker saves +ARCHIVE=$ARCHIVE_DIR bash "$BT/pull_board_state.sh" base +``` + +`ARCHIVE_DIR` comes from `bench.env` through the block in step 1, already in +the `C:/...` form adb needs. To write somewhere else, set `ARCHIVE` to any +folder outside git (`C:/...`, `~/...` or `/c/...` all work: the script +converts it with `cygpath -m`). The capture is adb-only, so the base must be +on USB. + +The capture is read-only and withholds secrets. First stop the tractor's +camera unit (step 5.1). Review the reports before you copy text into +`bench-evidence/`, and never commit images or secrets. See +[BENCH_BOARDS.md](BENCH_BOARDS.md), "Capture at a power-up". + +## Troubleshooting + +[BENCH_RUNBOOK.md](BENCH_RUNBOOK.md) has the board facts and the full trap +list. These are the common symptoms: + +| symptom | cause | fix | +|---|---|---| +| probe or daemon gets no HostLink answer although the L072 transmits | tractor camera unit or container holds `/dev/ttymxc3` | `systemctl stop lifetrac-camera.service; docker stop tractor-camera`; `fuser /dev/ttymxc3` must be empty | +| flash stops with `PREFLIGHT-FAIL: missing or empty` or `PREFLIGHT-FAIL: CRLF line endings in:` | `/tmp/lifetrac_p0c` aged out or was wiped by a reboot, or a script went up with CRLF | re-run `flash_l072.sh ` without `--go`: it re-stages LF-clean and repeats the preflight; `PREFLIGHT-OK` means the staging is complete | +| `ERR_PROTO FORBIDDEN detail=6` on `-ForceFrfHz` or a survey | production firmware on the board | flash the bench build (step 3) | +| harness throws `Refusing a profile-2 (DTS) leg with -ForceFrfHz 0` | a DTS leg without a carrier pin | spot-check (step 5.4) and launch the wrapper `leg_prep.sh` writes | +| `leg_prep.sh` prints `REFUSED: DTS_CARRIER_HZ ...` | no pick in `bench.env`, 915.000 MHz, or a pick from another day | spot-check today and set `DTS_CARRIER_HZ` / `DTS_CARRIER_DATE` (step 5.4) | +| DTS loss folds on a ~7.08 s period | the leg flew on 915.000 MHz | spot-check and pin `-ForceFrfHz` (step 5.4) | +| `running scripts is disabled on this system` in PowerShell | execution policy `Restricted` | [PC_SETUP.md](PC_SETUP.md#powershell-execution-policy) | +| camera changes mode or a profile is commanded mid-leg | retained control topics | `clear_retained.py` on the base broker **and** `clear_retained_host.py` on the PC broker | +| report says ~99.8 % loss and `published=1` | stats thread died (RS-12.17) | use `capture seq gaps` / count `published frame_id` lines | +| docker images vanish on the base | `compose-apps-early-start-recovery.service` is unmasked: it runs `rm -rf /var/lib/docker` every 60 s | mask it ([BENCH_SETUP.md](BENCH_SETUP.md)) | +| board resets about 60 s into a long operation | the U-Boot-armed X8 watchdog | FLASH_RUNBOOK §3; close `/dev/watchdog0` with `V` | +| a harness parameter seems ignored | misspelled name | the harness now rejects unknown names; read the archive's `params.txt` | +| adb lost the board after `adb kill-server` | the board's adbd hung | run `adb -s wait-for-device` once; if that fails, replug USB-C or press RESET. Avoid `kill-server` while the boards are enumerating ([ADB_TIPS_AND_TRICKS.md](../../../X8_HEALTH_AND_RECOVERY/recovery/ADB_TIPS_AND_TRICKS.md)) | diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_RUNBOOK.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_RUNBOOK.md index d3922990..ec350c9a 100644 --- a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_RUNBOOK.md +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_RUNBOOK.md @@ -5,6 +5,17 @@ next session starts from a checklist instead of memory. Everything that keys a radio needs the operator's GO; radios are parked (LoRa SLEEP, `0x80` readback) whenever a session ends. +**New to the bench?** Start at [BENCH_QUICKSTART.md](BENCH_QUICKSTART.md). +It walks through PC and board setup, radio firmware, deploy, a first leg, +analysis and park, in order, with the commands. This runbook is the +per-session reference and the trap list. The rest of the bench kit: + +- [PC_SETUP.md](PC_SETUP.md) +- [BENCH_SETUP.md](BENCH_SETUP.md) +- [DEPLOY.md](DEPLOY.md) +- [`bench.env.example`](bench.env.example) +- [legs/README.md](legs/README.md) + ## What is here | file | runs on | purpose | @@ -31,16 +42,19 @@ the harness (`run_live_radio_monitor.ps1`) live one directory up. Leg reports: * **Powering down.** `systemctl poweroff` keeps the tractor off, but the base booted again about 4.5 min later (2026-10-04). Remove power to keep the base down. -* **Powering up.** The L072 boots into RXCONT (listening). Park it with - `radio_park.py` if the radios must stay off. +* **Powering up.** The L072 boots into RXCONT (listening). + `legs/power_up_guard.sh` reads each radio with `radio_state.py` and parks + any that is listening with `radio_park.py` ([legs/README.md](legs/README.md)). * **PC toolchain.** arm-none-eabi-gcc 12.2.1 (Arm GNU Toolchain 12.2.MPACBTI-Rel1) gives byte-identical L072 builds. Also: WinLibs `mingw32-make` (PowerShell only), arduino-cli with `arduino:mbed_portenta` 4.5.0, `py -3`. The PC has no Docker; images are - built on the base X8. -* Base `192.168.1.117` via ssh key `~/.ssh/lifetrac_base_ed25519`; tractor + built on the base X8. Installation steps: [PC_SETUP.md](PC_SETUP.md). +* Original bench (the `bench.env.example` values; yours are in `bench.env`): + base `192.168.1.117` via ssh key `~/.ssh/lifetrac_base_ed25519`; tractor ONLY via `adb -s 2E2C1209DABC240B` (WiFi stays off); base adb - `2D0A1209DABC240B`. Password `fio` → `echo fio | sudo -S -p ''`. + `2D0A1209DABC240B`. Password `fio` (LmP default) → `echo fio | sudo -S -p ''`; + after provisioning, the sudoers drop-in makes `sudo -n` work without it. * USB is for programming and debug logs only. Never run the harness or `mingw32-make` from bash — PowerShell only. * `/tmp` is tmpfs on both boards: **every reboot (every flash) wipes @@ -53,12 +67,17 @@ the harness (`run_live_radio_monitor.ps1`) live one directory up. Leg reports: 19 h uptime, no reboot, yet `/tmp/lifetrac_p0c` held only the bin pushed the night before — the flash scripts had aged out. A flash attempted then fails as `run_flash_bench.sh: line 5: 1: image` (rc 127) and touches no - radio. **Count the staging before trusting a flash:** - `ls /tmp/lifetrac_p0c | wc -l` must be 11, and `ls /tmp/lifetrac_strict | - wc -l` ≥ 19. + radio. **Check the staging before trusting a flash:** `flash_l072.sh` + re-stages the pipeline and `run_flash_bench.sh` now refuses to run on an + incomplete `/tmp/lifetrac_p0c` (preflight); for legs, `ls /tmp/lifetrac_strict | + wc -l` ≥ 19 (`legs/stage_boards.sh` re-stages it). * The L072 boots into RXCONT (`sx1276_rx_arm()`), so a flash brings the receiver up — a flash IS a radio-on event. A probe HostLink connect also auto-wakes it. +* The HC-02 health check (`../x8_max_carrier_healthcheck.sh`) touches the + radio UART too: its L072 GET probe sets `/dev/ttymxc3` to 19200 8E1 and + writes two bytes (`00 FF`) to it. Run it only with the UART free, and on a + GO like any probe ([BENCH_SETUP.md](BENCH_SETUP.md) §7). * The tractor's camera unit/container come back on reboot and steal the radio UART: `systemctl stop lifetrac-camera.service; docker stop tractor-camera` before any probe. Confirm `fuser /dev/ttymxc3` is empty. @@ -80,7 +99,8 @@ the harness (`run_live_radio_monitor.ps1`) live one directory up. Leg reports: 1. `git archive` the helper tooling to `/tmp/lifetrac_strict` on both boards (probes, `lora_proto.py`, and everything in this directory except the `/home/fio` files). `adb push` needs `MSYS_NO_PATHCONV=1` and a - Windows-style `C:/...` source path. + Windows-style `C:/...` source path. `legs/stage_boards.sh` does this + step from `bench.env` ([legs/README.md](legs/README.md)). 2. Flash staging: push the helper `.sh`/`.cfg`/`.py` pipeline files plus the bins to `/tmp/lifetrac_p0c` on both boards **LF-clean** (`tr -d '\r'`); a CRLF script fails silently as `1: image` from the wrapper. `scp` to the @@ -108,8 +128,10 @@ the harness (`run_live_radio_monitor.ps1`) live one directory up. Leg reports: fragment and does not exercise the break. 8. DTS (profile 2) legs: run a same-day receive-only channel spot-check with the tractor parked (`../channel_survey_sniff.py` or `../hunt_sniff.ps1`, - then `tools/survey_compare.py`). Pass `-ForceFrfHz ` on every - profile-2 leg. `-ForceFrfHz 0` pins 915.000 MHz, where the RS-11.6 + then `tools/survey_compare.py`). Put the pick in `bench.env` + (`DTS_CARRIER_HZ`, `DTS_CARRIER_DATE`) and pass `-ForceFrfHz ` + on every profile-2 leg; the harness refuses a profile-2 leg without it. + `-ForceFrfHz 0` pins 915.000 MHz, where the RS-11.6 external emitter sends ~25 ms bursts at −43 to −45 dBm every ~7.08 s. All RS-13.1 DTS legs flew there by accident and their losses fold on its period (RS-13.1 A19). The band changes from day to day, so re-check rather @@ -118,10 +140,18 @@ the harness (`run_live_radio_monitor.ps1`) live one directory up. Leg reports: ## A leg ``` -.\run_live_radio_monitor.ps1 -TxFeed camera|local -RegProfile 1|2 -DurationS 300 ` +.\run_live_radio_monitor.ps1 -TxAdbSerial -RxAdbSerial -HostIp ` + -TxFeed camera|local -RegProfile 1|2 [-ForceFrfHz ] -DurationS 300 ` [-SynthFps 2 -SynthBudgetB 3000] -KfRequestDisable 0 -ProbeEcho 0 ` -NoParkLast 0 -LogFragArrivals 1 -IdleDrainQuietS 1.5 -CmdStreamMinGapS 1.0 -Archive ``` +The `<...>` values come from `bench.env`; without them the harness falls +back to the original bench's serials and PC address. `-ForceFrfHz ` +is **required with `-RegProfile 2`** (prep 8): the harness refuses a +profile-2 leg without it, and `-AllowDefaultCarrier` is only for a deliberate +915 MHz control. Leave it out with `-RegProfile 1`. For an RS-13 leg, +`legs/leg_prep.sh` writes the complete line ([legs/README.md](legs/README.md)). + On the first `published frame_id` line start the injector on the base (`kf_inject.py 15 20` or `dual_inject.py 250`). After `archived to ...`: post-brackets both boards, `rs12_leg_report.py`, `frag_gap_report.py`, @@ -137,9 +167,72 @@ leg V), but that is because of the timing above, not because the write is authoritative. RXCONT is receive-only — the TX path only fires on a host request, and there is none with the daemons down. +## Bench web UI on the base (optional) + +The operator console (`base_station/web_ui.py`) is not part of the harness +loop. Run it on the base only when a leg needs it, for example the RS-1.4 +Auto selector (leg V) or a look at the tiles. While it runs it adds base +commands to the control plane (TILE_STALE scans, profile switches), so say in +RESULTS that it ran. This recipe comes from the F10/F11 sessions and leg V. + +1. **Push** `web_ui.py`, `lora_proto.py`, `image_pipeline/` and `web/` from + the checkout to `/tmp/lifetrac_strict` on the base. The harness pushes + `lora_proto.py` and `image_pipeline/` at every launch, but after a reboot + nothing is there until you push it. If `scp` cannot overwrite + `image_pipeline/`, remove its root-owned `__pycache__` first. + + ```bash + K="-i $HOME/.ssh/lifetrac_base_ed25519"; B=fio@192.168.1.117 + S=LifeTrac-v25/DESIGN-CONTROLLER/base_station + ssh $K $B "sudo rm -rf /tmp/lifetrac_strict/image_pipeline/__pycache__" + scp $K -r $S/web_ui.py $S/lora_proto.py $S/image_pipeline $S/web $B:/tmp/lifetrac_strict/ + ``` + +2. **Run** it in `lifetrac-v25:latest` with host networking: uvicorn on port + **8090**, away from the production compose's 8080, + MQTT to the base broker at `127.0.0.1`, and PIN **2525**. That PIN is for + the bench only and is never a field PIN; production reads its PIN from + `secrets/lifetrac_pin`. + + ```bash + ssh $K $B "sudo docker rm -f bench_webui 2>/dev/null; sudo docker run -d --name bench_webui --network=host \ + -v /tmp/lifetrac_strict:/work -w /work -e PYTHONPATH=/work:/work/paho \ + -e LIFETRAC_MQTT_HOST=127.0.0.1 -e LIFETRAC_PIN=2525 \ + --entrypoint python3 lifetrac-v25:latest -m uvicorn web_ui:app --host 0.0.0.0 --port 8090" + ``` + + Browse to `http://:8090`. Some modules, such as `audit_log`, + `build_config` and `settings_store`, are imported lazily, and web_ui only + degrades when they are missing. If a feature needs one, append + `:/app/base_station` to `PYTHONPATH` so the image's deployed copy fills + the gap. + +3. **Record what it did** with an MQTT tap. web_ui has no root logger, so its + info lines never reach `docker logs`: + + ```bash + ssh $K $B "sudo docker exec design-controller-mosquitto-1 mosquitto_sub -h 127.0.0.1 -F '%U %t %p' \ + -t lifetrac/v25/control/radio_profile -t 'lifetrac/v25/status/radio_profile/#' -t lifetrac/v25/video/link_stats" | tee webui_tap.txt + ``` + +4. **Auto selector.** Arm Auto only **after** the daemons are up. With no + daemon running, Auto pins FHSS within 60 s (the stale-link rule). At the + end of the session, set a concrete profile (e.g. 2) before you stop. A + retained `auto` otherwise re-commands the tractor at the next start. Then + run `clear_retained.py`. + +5. **Stop it** at the end of the session: + `ssh $K $B "sudo docker rm -f bench_webui"`. Port 8090 then closes. + ## Traps that cost time this campaign * Bash-tool heredocs mangle backslashes — write patch scripts to files. +* **Misspelled harness parameters were ignored without a warning.** + `run_live_radio_monitor.ps1` used to put an unknown name into `$args`, so + `-DurationSeconds 300` ran the 30 s default of `-DurationS`. The harness + now refuses unknown names at launch. Even so, read the archive's + `params.txt` for the values that actually ran (`duration_s`, + `force_frf_hz`, `reg_profile`, ...) before you quote a leg. * `git add ` sweeps the untracked bench binary (`build/firmware_bench_diag.bin`) into the commit; add firmware by path. * The committed `build/firmware.bin` is the PRODUCTION image (no diag flag); diff --git a/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_SETUP.md b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_SETUP.md new file mode 100644 index 00000000..e2eca321 --- /dev/null +++ b/LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools/BENCH_SETUP.md @@ -0,0 +1,371 @@ +# Bench setup: from two new boards to a bench-ready pair + +This page takes two new Portenta X8 boards on Max Carriers to the state the +LifeTrac radio bench runs in: OS image, board settings, firmware and docker +images. Work through it in order. Each step names the doc that has the details. + +It reproduces the state measured on the original pair. The base was captured +in [`board_state_2026-10-04/base/`](../../../bench-evidence/board_state_2026-10-04/base/) +and the tractor in [`board_state_2026-10-10/tractor/`](../../../bench-evidence/board_state_2026-10-10/tractor/); +[BENCH_BOARDS.md](BENCH_BOARDS.md) describes both captures. + +| step | what | where | +|---|---|---| +| 0 | Radio safety rules | below | +| 1 | Hardware | below | +| 2 | DIP switches | below | +| 3 | Choose the OS image | below | +| 4 | Reflash both boards | [T3a](../../../X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md) | +| 5 | First boot: adb, password, base network, ssh key | below | +| 6 | Provision: `provision_bench_board.sh base`, then `tractor` | below | +| 7 | Verify: HC-01, HC-02, `--check` | below | +| 8 | L072 radio firmware, health probe, park | [FLASH_RUNBOOK.md](../FLASH_RUNBOOK.md) | +| 9 | Deploy the base tree, the images and the broker | [DEPLOY.md](DEPLOY.md) | +| 10 | Staging and radio legs | [BENCH_QUICKSTART.md](BENCH_QUICKSTART.md) | + +You need a Windows PC set up as in [PC_SETUP.md](PC_SETUP.md): adb, Git Bash, +PowerShell, `py -3`, the Arm GCC toolchain, `mingw32-make` and `uuu`. +Run the shell commands on this page in **Git Bash**, from this directory +(`LifeTrac-v25/DESIGN-CONTROLLER/firmware/x8_lora_bootloader_helper/bench_tools`), +unless a step says PowerShell. + +Who is the base and who is the tractor: the **base** is on ethernet and is +reached over ssh or adb. The **tractor** has its WiFi off for good, has the +camera, and is reached over adb only (USB). Put the serials in `bench.env` +(step 5). + +--- + +## 0. Radio safety (read this first) + +The bench has two LoRa radios in the 902–928 MHz ISM band. The repo's +references are US rules (FCC §15.247); check your local rules before any +radio test. + +1. **Antennas first.** Screw the antenna onto each carrier's LoRa connector + before you power the board. Never let a radio transmit without one. +2. **The LifeTrac L072 firmware boots listening.** It powers up in RXCONT, + which is receive-only. It transmits only when a host program on the X8 + asks it to: the bench daemons, the harness, or a probe that sends TX + requests. If the radios must stay off, park them with + `radio_park.py` (step 8). Its `0x80` readback is not a lasting guarantee + while the firmware's scan is still running, so confirm a few minutes + later with the read-only `radio_state.py`. +3. **Flashing the L072 turns its receiver on.** A probe that connects to + it (rs116, radio_park, radio_state) also wakes it. +4. **The tractor's camera unit takes the radio UART.** At every boot, and + whenever the camera is plugged in, `lifetrac-camera.service` and its + `tractor-camera` container grab `/dev/ttymxc3`. Stop both before any + probe or flash: + `sudo -n systemctl stop lifetrac-camera.service; sudo -n docker stop tractor-camera`. + `sudo -n fuser /dev/ttymxc3` must then print nothing. +5. **The base does not stay off.** After `systemctl poweroff` the base + booted again about 4.5 minutes later (2026-10-04). To keep it down, + remove its power. The tractor stays off after `systemctl poweroff`. +6. **Never start the production units on the bench.** These are + `lifetrac-base.service`, `lifetrac-base-compose.service` and + `lifetrac-tractor-compose.service`. They would start `lora_bridge` or the + tractor stack on the radio UART. This kit does not install them. The + original pair has them enabled, and they fail at every boot. +7. **Radio tests need a GO.** On a shared bench, every round of radio + tests needs the operator's explicit GO, and both radios are parked + after it. + +## 1. Hardware + +| item | qty | notes | +|---|---|---| +| Arduino Portenta X8 (ABX00049) | 2 | The OS, eMMC, H747 and SE050 belong to the module. | +| Arduino Portenta Max Carrier (ABX00043) | 2 | Carries the Murata CMWX1ZZABZ LoRa module (STM32L072 + SX1276), the BOOT DIP switches and an on-board J-Link. The L072 firmware belongs to the **carrier**. | +| 915 MHz antenna | 2 | Bench parts: Taoglas TD.95.6H31 Blade (868/915 MHz dual-band dipole, SMA), one on each carrier's LoRa antenna connector. | +| 12 V DC supply | 2 | One per carrier (barrel jack). | +| USB-A to USB-C data cable | 2 | Goes into the X8 module's own USB-C port. For a reflash, plug it straight into the PC: no hub, and no USB-C to USB-C cable. | +| Ethernet cable + LAN with DHCP | 1 | For the base. A marginal cable at gigabit forced the optional 100BASE-TX pin (step 6). | +| Kurokesu C2 USB camera (16d0:0ed4) | 1 | Goes on the tractor carrier's USB-A host port. Needed only for camera legs. | +| Windows 10/11 PC | 1 | Set up as in [PC_SETUP.md](PC_SETUP.md). The PC has no Docker: images are built on the base X8. | +| Optional: a second USB cable per carrier | 2 | For the carrier's own USB-C (on-board J-Link, `VID_1366&PID_0105`). Only the T2.5 and T4 recovery steps use it. | + +Label each module and each carrier with its serial, and write down which +module sits on which carrier. Recovery depends on that pairing: the L072 +firmware, its option bytes and the DIP switches stay with the carrier. + +## 2. DIP switches + +Each Max Carrier has a `BOOT SEL` and a `BOOT` DIP switch. + +- **Normal boot: both OFF.** Check this on new carriers before first power. +- **Reflash (SDP mode):** set `BOOT SEL` ON. If the SDP device does not + appear, power down and set `BOOT` ON as well. Both reflashes on this bench + needed only `BOOT SEL`. See [T3a](../../../X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md). +- After the reflash, set both back to OFF. + +Write down each carrier's switch positions, as BENCH_BOARDS asks. + +## 3. Choose the OS image + +| image | kernel | URL | status | +|---|---|---|---| +| **LmP 4.0.11-934-91** | 6.1.24 | `https://downloads.arduino.cc/portentax8image/934.tar.gz` (1,097,240,589 B, Last-Modified 2025-08-11) | **The base image.** Flashed with `uuu` 2026-05-13 and 05-24. Pinned by sha256 in T3a. **Use this on both boards.** | +| LmP 4.0.3-674-88 | 5.10.93 | `https://downloads.arduino.cc/portentax8image/674.tar.gz` (854,998,055 B) | The tractor's factory image; that board was never reflashed. It is still hosted at this URL, but nobody here has downloaded or byte-checked it, so there is no sha256 on record. | +| `image-latest.tar.gz` | — | `https://downloads.arduino.cc/portentax8image/image-latest.tar.gz` | **Not the bench image.** This URL moves to newer builds. The bench bundle was downloaded under this name on 2026-05-08, when it still served 934. | + +The 934 bundle used on the base has this sha256: +`2f2065c7b10ce33d8444a138353187c507727b5109ddca325506ecaea1efe221`. +It was saved as `image-latest.tar.gz`. Today's `934.tar.gz` has the same +size and date, but nobody has compared it byte for byte. If your download +hashes the same, it is identical. If not, check the extracted files against +the full list in T3a; those files are what `uuu` writes. + +**Recommendation for a new bench: 934 on both boards.** That gives one image +to keep and one set of quirks. Be aware that **the tractor role has only ever +run on 674**. Treat the first tractor-on-934 legs as a new configuration and +record it in the evidence. The 934 differences that matter for the tractor +are known and handled: + +- The flash wrapper must reboot instead of re-inserting `x8h7`, which crashes + the 6.1 kernel. `REVIVE_MODE` now defaults to `reboot` in the flash wrapper, and + FLASH_RUNBOOK passes it explicitly too. +- The compose-apps recovery unit wipes `/var/lib/docker`. + `provision_bench_board.sh` masks it. +- The factory docker images differ. The probe image comes from the repo's + `foundries_python.tar`, which `provision_bench_board.sh` loads. + +To match the original pair exactly, keep the tractor on its factory 674. +That works only if the board still runs 674; check +`cat /etc/os-release` once you have adb. + +## 4. Reflash both boards + +Follow [T3a_sdp_uuu_reflash.md](../../../X8_HEALTH_AND_RECOVERY/recovery/T3a_sdp_uuu_reflash.md). +It covers the download and sha256 check, the extracted layout, `uuu` 1.5.243, +the DIP switches, SDP detection and the first boot. Budget 60–80 minutes per +board for the eMMC write. + +Reflash even brand-new boards: the factory image depends on the production +batch. A reflash erases the whole eMMC (OS, `/etc`, docker images, +`/home/fio`). It does not touch the L072 firmware on the carrier. + +## 5. First boot + +1. Set both DIP switches OFF. Connect 12 V, then USB-C. First boot takes + about 60 s. If the LEDs go red or adb has not appeared after 2 minutes, + power-cycle once more; the 2026-05-13 reflash needed two cycles. +2. Check the board is visible: `adb devices -l` must list its serial as + `device`. **Do not run `adb kill-server`.** On a first-boot board adbd is + fragile, and a kill-server can leave it invisible until you replug the + USB-C ([ADB_TIPS_AND_TRICKS.md](../../../X8_HEALTH_AND_RECOVERY/recovery/ADB_TIPS_AND_TRICKS.md)). +3. **Record the serials.** Copy [`bench.env.example`](bench.env.example) to + `bench.env` in this directory and set `BASE_SERIAL`, `TRACTOR_SERIAL` + and `BASE_HOST`, plus `BENCH_SUDO_PW` if you change the password, and + `PC_HOST` (this PC's LAN IPv4 address; radio legs need it). Do this + before any kit script runs: without `bench.env` the scripts fall back to + the original bench's serials and base address. + `bench.env` is your local file: keep it out of git. +4. **Password.** The image's user is `fio` with password `fio`; adb opens a + shell as `fio` without asking for it. Changing it is recommended on the + base, which sits on your LAN with ssh. Run `adb -s shell`, then + `passwd`. If you change it, put it in `bench.env` as `BENCH_SUDO_PW`. + The provisioning script uses it once, to install the sudoers drop-in; + after that `sudo -n` needs no password. +5. **Sudo without a password.** Step 6 installs this with the repo's + [`install_lifetrac_nopasswd.sh`](../install_lifetrac_nopasswd.sh), which + writes `/etc/sudoers.d/99-lifetrac-bench-nopasswd` = + `fio ALL=(ALL) NOPASSWD: ALL`. It is bench-only: it gives anyone with a + shell as `fio` full root. To install it by hand instead: + + ```bash + export MSYS_NO_PATHCONV=1 + sed 's/\r$//' ../install_lifetrac_nopasswd.sh > /tmp/install_nopasswd.sh # LF only + adb -s push "$(cygpath -m /tmp/install_nopasswd.sh)" /tmp/install_lifetrac_nopasswd.sh + adb -s shell # then on the board: sudo sh /tmp/install_lifetrac_nopasswd.sh + ``` + +6. **Base network.** Plug the base into the LAN and read its address: + `adb -s shell ip -br addr show eth0`. Reserve that DHCP + lease in your router (the original bench uses `192.168.1.117`) and set + `BASE_HOST` in `bench.env`. +7. **Base ssh key.** Create a key on the PC, in Git Bash: + + ```bash + ssh-keygen -t ed25519 -f ~/.ssh/lifetrac_base_ed25519 -C lifetrac-bench-pc + ``` + + Step 6 appends `~/.ssh/lifetrac_base_ed25519.pub` to the base's + `/home/fio/.ssh/authorized_keys`. Afterwards this must work: + `ssh -i ~/.ssh/lifetrac_base_ed25519 fio@ uptime`. Back up + the private key in a password manager, never in git. +8. **Tractor network: none, on purpose.** Leave its ethernet unplugged. + Step 6 turns its WiFi off for good. + +## 6. Provision + +[`provision_bench_board.sh`](provision_bench_board.sh) runs on the PC and +applies every board-side customization the original pair has. It reads +`bench.env` and talks to one board over adb. Every step checks first, so it +is safe to re-run. `--check` reports without changing anything. Do the base +first, because the tractor's broker image comes from it. + +```bash +bash provision_bench_board.sh base --check # what is missing +bash provision_bench_board.sh base # add --nic-100 only for a marginal cable +bash provision_bench_board.sh tractor --check +bash provision_bench_board.sh tractor --mosquitto-from-base +``` + +What it applies, and why: + +| step | board | why | +|---|---|---| +| sudoers drop-in (`install_lifetrac_nopasswd.sh`) | both | Every bench script uses `sudo -n`. | +| mask `compose-apps-early-start-recovery.service` and `compose-apps-early-start.service` | base, and the tractor on any image other than 674 | On 934 `compose-apps-early-start` fails at boot. Its `OnFailure` recovery unit then runs `systemctl stop docker` and deletes `/var/lib/docker`, about every 60 s, so every image you load disappears. Mask both before relying on docker. On 674 the unit succeeds and is left alone (`--mask-compose` masks it anyway). | +| optional `--nic-100`: `Wired connection 1` set to speed 100, duplex full, auto-negotiate yes | base | The original base's cable loses about half its RX frames at gigabit and none at 100. Auto-negotiation stays on, as on the base, so it only advertises 100/full. Skip this on a good cable. To undo it: `sudo nmcli connection modify 'Wired connection 1' 802-3-ethernet.speed 0 802-3-ethernet.duplex '' 802-3-ethernet.auto-negotiate yes`. | +| PC ssh key appended to `~/.ssh/authorized_keys` | base | Base work goes over ssh. Its USB has failed to enumerate after a power cycle (2026-10-10) while ssh kept working. | +| [`units/disable-wifi.service`](units/disable-wifi.service) installed and enabled, which runs `rfkill block wifi` at boot | tractor | The tractor has no IP path by policy. This is the unit text from the tractor capture. | +| `wpa_supplicant.service` masked; `nmcli radio wifi off` | tractor | Same reason. | +| stored NetworkManager WiFi profiles backed up to a root-only directory on the board, then deleted (only their names are printed) | tractor | The original tractor stores the `5star` profile with its password. `--keep-wifi-profiles` keeps them. | +| camera USB provisioning: runs the repo's [`provision_x8.sh`](../provision_x8.sh) unchanged | tractor | Installs the `snd_usb_audio` blacklist, the C2 udev rules (autosuspend off, `/dev/lifetrac-c2`) and `lifetrac-camera.service`, and adds `fio` to `video`. The script skips this when all three files already match. It refuses when a camera compose app is installed and the C2 is attached, because `provision_x8.sh` would restart the camera, whose container maps the radio UART; pass `--allow-camera-restart` to accept that. | +| docker running; probe image `hub.foundries.io/arduino/arduino-ootb-python-devel:738bc44` loaded from the repo-root `foundries_python.tar` (image id `9a454afe48f7`) | both | `radio_park.py`, `radio_state.py` and `rs116_health_probe.py` run in it. The tractor cannot pull images, and the base's own `lifetrac-v25` image exists only after step 9. | +| `eclipse-mosquitto:2` | base: `docker pull`. Tractor: `--mosquitto-from-base` (docker save on the base, then through the PC) or `--mosquitto-tar FILE` | The base broker and the tractor's local broker. The tag moves, so note the image id the script prints. | +| `lifetrac-camera.service` and `tractor-camera` stopped | tractor | Keeps the radio UART free (rule 4). Both come back at the next boot. | + +It never opens `/dev/ttymxc3` and never starts the production units. It +reports any of them it finds, and who holds the radio UART. It ends with a +checklist of what is left. Exit codes: 0 done, 1 a step failed, 2 usage or +board unreachable, 3 `--check` found work to do. Run +`bash provision_bench_board.sh --help` for all options. + +It does not touch the L072 firmware or option bytes, the H747, the deployed +tree or the production units. It does not rotate the WiFi password either. +**That password must be treated as compromised:** it was committed in seven +repo-root WiFi scripts, and although the bench kit replaced the literals +with the `LIFETRAC_WIFI_SSID` / `LIFETRAC_WIFI_PSK` environment variables, it +is still readable in the public git history. Rewriting a public history with +forks is not practical, so change the passphrase on the access point; see +the open items in [BENCH_BOARDS.md](BENCH_BOARDS.md). + +## 7. Verify + +1. **HC-01 (adb and USB enumeration, PowerShell):** + [HC-01](../../../X8_HEALTH_AND_RECOVERY/routines/HC-01_adb_and_usb_enumeration.md). + Each serial must show `device`, stable over 30 s. +2. **HC-02 (Linux, x8h7 bridge, GPIO, UART):** + [HC-02](../../../X8_HEALTH_AND_RECOVERY/routines/HC-02_linux_bridge_full.md). + The helper is [`x8_max_carrier_healthcheck.sh`](../x8_max_carrier_healthcheck.sh). + On the tractor, stop the camera unit first (rule 4). + + ```bash + export MSYS_NO_PATHCONV=1 + sed 's/\r$//' ../x8_max_carrier_healthcheck.sh > /tmp/hc.sh + adb -s push "$(cygpath -m /tmp/hc.sh)" /tmp/x8_max_carrier_healthcheck.sh + adb -s shell "sh /tmp/x8_max_carrier_healthcheck.sh" + ``` + + **Its L072 section touches the radio UART.** The "L072 GET probe" sets + `/dev/ttymxc3` to 19200 8E1 and writes two bytes to it (`00 FF`, the + AN3155 GET command), then listens for 0.4 s to see whether the L072 sits + in its ROM bootloader. A running LifeTrac firmware talks HostLink at + 921600 baud in COBS frames with a CRC-16, so it should discard the two + bytes, and the probe sends nothing over the air. It is still a write to + the radio UART: run it only when nothing else holds `/dev/ttymxc3` + (`sudo -n fuser /dev/ttymxc3` prints nothing), and on a shared bench + treat it like a probe, on the operator's GO. Expected on a fresh board: the x8h7 + modules load (934 lists 10), `m4-proxy` and `stm32h7-program` are active, + and `/dev/ttymxc3` and `/dev/watchdog0` exist. A + `/sys/kernel/x8h7_firmware/version` timeout is normal. `gpio8/10/15` are + exported only after a flash-pipeline run. +3. `bash provision_bench_board.sh --check` must report no TODO. + After a reboot, the tractor's camera line reads TODO again until you stop + the camera. + +## 8. L072 radio firmware + +A new carrier's L072 normally runs Arduino's stock AT firmware. +`LifeTrac-v25/tools/mlm32l07x01.bin` is that image (MKRWAN 1.2.3), kept for +a factory restore. The same-named file in `x8_lora_bootloader_helper/` is a +different build. The stock firmware does not answer the LifeTrac probes +until you flash the LifeTrac build. + +1. **Build** in PowerShell, in `LifeTrac-v25/DESIGN-CONTROLLER/firmware/murata_l072`: + `mingw32-make bench` makes `build/firmware_bench_diag.bin`, md5 + `0c1bb0a9573f813137f941dfa47177d0`. Use arm-none-eabi-gcc 12.2.1 + (Arm GNU Toolchain 12.2.MPACBTI-Rel1); the toolchain and the expected md5 + are in [`murata_l072/README.md`](../../murata_l072/README.md). The + committed `build/firmware.bin` (md5 `589c120323c2d5e7ef9f459d7a4ba42d`) + is the production image. Do not put it on a bench board: it refuses the + bench's diagnostic register writes. +2. **Flash** with [`flash_l072.sh`](flash_l072.sh), which does + [FLASH_RUNBOOK.md](../FLASH_RUNBOOK.md) §1–§2 from the PC and reads + `bench.env`: without `--go` it stages and preflights, with `--go` it runs + `run_flash_bench.sh` (`REVIVE_MODE=reboot`). On a new carrier that still + runs the stock firmware, start with FLASH_RUNBOOK §5, + [First flash from a factory board](../FLASH_RUNBOOK.md#5-first-flash-from-a-factory-board): + a read-only `--go --verify-only` first contact against + `LifeTrac-v25/tools/mlm32l07x01.bin`, then the bench build. U-Boot arms + the X8 watchdog (60 s, then a PMIC power-cycle); the runbook says why the + scripts pet it and reboot. **Flashing turns the receiver on** (rule 3). +3. **Health probe and park.** Stage the probe files to + `/tmp/lifetrac_strict` and run them in the probe image. The scripts in + [`legs/`](legs/README.md) automate this; by hand, for the tractor: + + ```bash + export MSYS_NO_PATHCONV=1; S=; H=$(cygpath -m "$PWD/..") + adb -s $S shell "sudo -n systemctl stop lifetrac-camera.service; sudo -n docker stop tractor-camera; mkdir -p /tmp/lifetrac_strict" + for f in method_g_stage1_probe.py method_h_stage2_tx_probe_v2.py rs116_health_probe.py bench_tools/radio_park.py bench_tools/radio_state.py; do + adb -s $S push "$H/$f" /tmp/lifetrac_strict/ >/dev/null; done + R="sudo -n docker run --rm --network=host --entrypoint python3 --device=/dev/ttymxc3 -v /tmp/lifetrac_strict:/work -w /work -e PYTHONPATH=/work hub.foundries.io/arduino/arduino-ootb-python-devel:738bc44" + adb -s $S shell "$R -u /work/rs116_health_probe.py" # STATS-OK, RS115-INSTRUMENTED-FIRMWARE=YES, RS12-URC-COUNTERS=YES, radio_state=4 + adb -s $S shell "$R -u /work/radio_park.py" # PARK_OK {"opmode_readback": "0x80"} + adb -s $S shell "$R -u /work/radio_state.py" # read-only; repeat a few minutes later + ``` + + Do the same on the base, over adb or ssh; it has no camera to stop. + `/tmp` is tmpfs: a reboot (and so every flash) wipes the staging, and + `systemd-tmpfiles` also ages it out after 5 days. + +## 9. Deploy + +[DEPLOY.md](DEPLOY.md) covers it: + +- **Base** (`deploy_base.sh`): it copies a `git archive` subset of a commit + to `/var/rootdirs/opt/lifetrac/DESIGN-CONTROLLER`, builds `lifetrac-v25` + there, and starts the broker `design-controller-mosquitto-1` on + `127.0.0.1:1883`. It never starts `lora_bridge`. +- **Tractor** (`build_tractor_image.sh`): it builds `lifetrac-tractor-x8` + natively on the base X8, then moves it with `docker save` and + `docker load`. The PC has no Docker. + +## 10. Staging and radio legs + +Continue with [BENCH_QUICKSTART.md](BENCH_QUICKSTART.md), which covers +staging, a first leg and parking. The other leg docs are: + +- the parameterised leg scripts and the power-up guard: [`legs/README.md`](legs/README.md); +- the per-session checklist: [BENCH_RUNBOOK.md](BENCH_RUNBOOK.md); +- the RS-13 procedure: [RS13_VECTOR_LEG.md](RS13_VECTOR_LEG.md). + +## After every power-up + +[`legs/power_up_guard.sh`](legs/README.md) does the tractor and radio steps +for you and stages the probe tools it needs; arm it with +`--wait-power-cycle` before you switch the boards on. `legs/stage_boards.sh` +does the full staging. + +- **Tractor:** stop `lifetrac-camera.service` and `tractor-camera`, then + check that `sudo -n fuser /dev/ttymxc3` prints nothing. +- **Both:** `/tmp` staging is gone; stage again before any probe or flash. +- **Radios:** read them with `radio_state.py`. If they must stay off and + the readback is not `0x80`, run `radio_park.py`. +- **Base:** if it does not show up on USB, use ssh + (`provision_bench_board.sh base --ssh --check` works over ssh). + +## Troubleshooting + +| symptom | cause | fix | +|---|---|---| +| Docker images disappear about every minute on a 934 board | The compose-apps recovery unit deletes `/var/lib/docker` | `provision_bench_board.sh ` masks it | +| A probe prints `timeout waiting for response` on the tractor | The camera unit or container holds `/dev/ttymxc3` | Rule 4 | +| A flash stops with `PREFLIGHT-FAIL: missing or empty` or `PREFLIGHT-FAIL: CRLF line endings in:` (older wrappers: `1: image`, rc 127) | Staging that aged out of `/tmp` or was wiped by a reboot, or a script pushed with CRLF | Re-run `flash_l072.sh ` without `--go`: it re-stages LF-clean and repeats the preflight; `PREFLIGHT-OK` means the staging is complete ([FLASH_RUNBOOK.md](../FLASH_RUNBOOK.md) §1) | +| `adb devices` is empty right after a first boot | adbd is fragile on first boot | Wait, then replug the USB-C; do not `adb kill-server` ([T0](../../../X8_HEALTH_AND_RECOVERY/recovery/T0_adb_daemon_kick.md)) | +| `provision_bench_board.sh` stops at the sudoers step | Wrong `BENCH_SUDO_PW` | Fix it in `bench.env` | +| `$'\r': command not found` when you start a kit script on the PC | git `core.autocrlf` checked the script out with CRLF | The repo's `.gitattributes` keeps `*.sh` LF. On an older checkout, run `git add --renormalize .` or `sed -i 's/\r$//'