Adopt upstream radiometric calibration QC filter and retrain

README.md

@@ -50,6 +50,12 @@ spatial split, censoring/detection settings, season exclusions):
 uv run snakemake --cores 4 --rerun-triggers=mtime -- model_all
 ```
 
+Each rule depends on a stamp of the config section its run_id hashes
+(`outputs/config_stamps/`, written by the Snakefile at parse time), so editing
+the grid section rebuilds the grid and everything after it, editing a split or
+train setting reruns only those stages, and a comment-only edit or a
+re-pinned snapshot triggers exactly the stages it affects.
+
 This builds the full-ice-sheet covariate grid, the spatially-blocked
 train/test split, trains the configured models with cross-validation, and
 writes per-model outputs to `outputs/model/<model>/`:

Snakefile

@@ -13,20 +13,56 @@
 # the filenames, so every rule can target a static path directly.
 
 STORE = config.get("store", "ase")
-CONFIG_PATH = config.get("config_path", f"config/{STORE}.yaml")
 OUT_DIR = config.get("out_dir", "outputs")
 
 # Cross-store modeling pipeline (grid -> split -> train -> benchmark), driven by
 # a single config that names its input stores and models. Invoked without store=:
 #   uv run snakemake --cores 4 model_all
 MODEL_CONFIG_PATH = config.get("model_config_path", "config/model.yaml")
-import yaml as _yaml
-with open(MODEL_CONFIG_PATH) as _f:
-    _model_cfg = _yaml.safe_load(_f) or {}
-MODEL_STORES = sorted({s for stores in _model_cfg.get(
-    "inputs", {"antarctic": ["ase", "utig"], "greenland": ["greenland"]}).values() for s in stores})
-MODELS = [m["name"] if isinstance(m, dict) else m
-          for m in _model_cfg.get("train", {}).get("models", [{"name": "linear"}])]
+import os as _os
+from pathlib import Path as _Path
+from radar_postproc.config import config_hash, load_config, load_model_config
+
+_model_cfg = load_model_config(MODEL_CONFIG_PATH)
+MODEL_STORES = sorted({s for stores in _model_cfg["inputs"].values() for s in stores})
+MODELS = [m["name"] for m in _model_cfg["train"]["models"]]
+
+
+def config_stamp(name: str, section: dict) -> str:
+    """Path of a stamp file that changes exactly when `section` changes.
+
+    Each stage hashes only its own config section into its run_id, so each
+    rule depends on a stamp of that same section rather than on the whole
+    config file: editing train settings does not rebuild the grid, and a
+    comment-only edit rebuilds nothing. Written at parse time, only when the
+    hash differs (so mtime moves only on a real change); a stamp created for
+    the first time gets an epoch mtime, so migrating existing outputs does
+    not trigger a rebuild.
+    """
+    path = _Path(OUT_DIR) / "config_stamps" / f"{name}.hash"
+    digest = config_hash(section)
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(digest)
+        _os.utime(path, (0, 0))
+    elif path.read_text() != digest:
+        path.write_text(digest)
+    return str(path)
+
+
+def augment_stamp(store: str) -> str:
+    """Stamp of a store's full augment config (what its run_id hashes)."""
+    path = config.get("config_path", f"config/{store}.yaml")
+    return config_stamp(f"augment_{store}", load_config(path))
+
+
+# Section hashes below mirror grid.py / split.py / train.py exactly.
+_tcfg = _model_cfg["train"]
+STAMP_GRID = config_stamp("grid", {"inputs": _model_cfg["inputs"], "grid": _model_cfg["grid"]})
+STAMP_SPLIT = config_stamp("split", {"inputs": _model_cfg["inputs"], "split": _model_cfg["split"]})
+STAMP_TRAIN = {m["name"]: config_stamp(f"train_{m['name']}",
+                                       {"train": {**_tcfg, "models": [m]}, "model": m["name"]})
+               for m in _tcfg["models"]}
 
 # Which trained model the mission design tool ships. Lives in its own config
 # section so it never enters a stage's section hash (and so never perturbs a
@@ -44,6 +80,11 @@ rule all:
 
 
 rule run:
+    # Depends on a stamp of the store's config, so re-pinning
+    # icechunk.snapshot_id (or any other augment change) re-runs the
+    # extraction without --forcerun.
+    input:
+        stamp=lambda w: augment_stamp(w.store),
     output:
         parquet=f"{OUT_DIR}/{{store}}/{{store}}.parquet",
         manifest=f"{OUT_DIR}/{{store}}/{{store}}.manifest.json",
@@ -52,10 +93,8 @@ rule run:
 
         # Derive the config from the wildcard (not the module-level CONFIG_PATH)
         # so cross-store DAGs (e.g. model_all) build the right store.
-        run_pipeline(
-            config.get("config_path", f"config/{wildcards.store}.yaml"),
-            out_dir=OUT_DIR,
-        )
+        run_pipeline(config.get("config_path", f"config/{wildcards.store}.yaml"),
+                     out_dir=OUT_DIR)
 
 
 rule csv:
@@ -94,7 +133,9 @@ rule model_all:
 
 rule grid:
     # Full-ice-sheet covariate grid (the prediction domain). Network + slow;
-    # only re-runs when config/model.yaml's grid section changes.
+    # only re-runs when config/model.yaml's inputs/grid sections change.
+    input:
+        STAMP_GRID,
     output:
         parquet=f"{OUT_DIR}/model/grid.parquet",
         manifest=f"{OUT_DIR}/model/grid.manifest.json",
@@ -108,6 +149,7 @@ rule split:
     # Attach the radar target to grid points, assign blocking cells + folds,
     # emit helper maps for hand-picking test cells.
     input:
+        STAMP_SPLIT,
         grid=f"{OUT_DIR}/model/grid.parquet",
         stores=expand(f"{OUT_DIR}/{{s}}/{{s}}.parquet", s=MODEL_STORES),
     output:
@@ -122,6 +164,7 @@ rule split:
 
 rule train:
     input:
+        lambda w: STAMP_TRAIN[w.model],
         f"{OUT_DIR}/model/split.parquet",
     output:
         metrics=f"{OUT_DIR}/model/{{model}}/metrics.json",

agent_notes/20260901-calibration-qc-results.md

@@ -0,0 +1,124 @@
+# Calibration QC filter — results (2026-09-01)
+
+Branch `qc-calibration-filter`. Plan: `agent_plans/20260901-calibration-qc-filter.md`.
+All numbers reproducible with `uv run python scripts/qc_filter_comparison.py`
+(needs `outputs/baseline_20260807/`, a copy of the pre-QC `outputs/model` +
+augment parquets) and `scripts/season_crossover_matrix.py [--calibration-qc]
+--out-dir outputs/qc_filter/crossover`. Full tables: `outputs/qc_filter/comparison.md`.
+
+## Data
+
+Pinned to the 2026-09-03 15:51 UTC method-0.4.1 snapshots (antarctica
+087JBD7NTAE8BTBTEYSG, greenland ACA8WY61ZF9W6VSBA3HG, ase F6TXWSEQ9RQCD1H7MSMG).
+Upstream verified no science values changed, so baseline vs new differs only
+by the filter. The 0.4.1 refresh (from the 2026-09-02 0.4.0 snapshots first
+used here) changed exactly one thing that reaches the model: 72 traces of
+2018_Antarctica_DC8 lost their at-ceiling flag when its rising img2 envelope
+stopped counting as a ceiling — all 72 were already rejected by the img2
+rule, so the split parquet is identical apart from run_ids and the model
+retrains to the same posterior (fixed seed).
+
+Share of traces entering the split stage (after the two season exclusions)
+rejected, exclusive attribution:
+
+| rule | Antarctica | Greenland |
+|---|---|---|
+| seam \|Δ\| ≥ 3 dB | 9.1% | 8.9% |
+| surface from img2+ | 3.1% | 3.0% |
+| margin < 2 dB | 2.2% | 0.7% |
+| any | 14.4% | 12.7% |
+
+Worst seasons: 2016_Greenland_P3 51% (49% seam), 2017_Antarctica_Basler 46%
+(already excluded), 2013_Antarctica_Basler 33%, 2017_Antarctica_P3 31%,
+2019_Antarctica_GV 24% (22% at its −30 dB ceiling), 2016_Antarctica_DC8 22%.
+Unmeasured seams (check could not run): 92% of 2014_Greenland_P3, 61% of
+2012_Antarctica_DC8 → `drop_unmeasured_seam: true` would reject 35% / 50% of
+all traces, i.e. whole seasons; kept false. Threshold sensitivity: seam 2 dB →
+27% / 18% rejected, 5 dB → 12% / 7%; margin 1–3 dB moves totals by ±2 pts;
+dropping the img2 rule saves 2.6 / 2.5 pts. img2-sourced traces have median
+RSSNR 8–17 dB below img1 traces of the same DC8 season (2012: 46.7 vs 38.4,
+2014: 59.5 vs 42.1, 2016: 46.7 vs 35.9, 2018: 65.3 vs 50.2 dB) — the bias the
+changelog warns of is visible in the target, so the rule stays.
+
+Training grid points 19,272 → 17,447 (−9.5%); 714 kept points now match a
+different, QC-passing trace. Median training target rises 61.9 → 64.6 dB
+(Antarctica) and 73.6 → 74.0 dB (Greenland): the filter preferentially
+removes low-RSSNR traces (saturated surface ⇒ RSSNR biased low).
+
+## Model fit
+
+| | atten_refl baseline | atten_refl QC | linear baseline | linear QC |
+|---|---|---|---|---|
+| CV RMSE [dB] (fold range) | 13.02 (11.9–14.4) | 12.87 (11.8–14.3) | 13.92 | 13.80 |
+| CV coverage 1σ | 0.680 | 0.679 | 0.704 | 0.702 |
+| CV log score [dB] | −4.019 | −4.009 | −4.103 | −4.096 |
+| test RMSE (own test set) | 12.67 | 12.72 | 13.95 | 13.98 |
+| σ residual [dB] | 12.91 | 12.71 | 14.33 | 14.15 |
+| θ / τ [dB] | −1.22 / 3.01 | −1.08 / 2.79 | | |
+| divergences / R̂max | 0 / 1.0025 | 0 / 1.0049 | | |
+
+Same points, both posteriors (the fair comparison — the CV populations differ):
+on the QC test set (1,330 uncensored) atten_refl baseline 12.75 dB vs QC 12.72
+dB; on the baseline test set (1,433) 12.67 vs 12.64 dB. Bias +1.9 → +2.0 dB.
+Differences of 0.03–0.05 dB are noise.
+
+Posteriors (`posterior_comparison.png`): every parameter within ~2 sd of its
+baseline. Largest moves: β_a[greenland] −2.1 → −3.6 dB/km, −β_r[greenland]
+−16.0 → −17.2 dB, −α_r +6.05 → +5.66 dB, −β_r[T_air] +0.25 → +0.28 dB/K,
+σ 12.9 → 12.7 dB, τ 3.0 → 2.8 dB. The sheet-level attenuation and
+reflectivity offsets move against each other, so predictions barely change.
+
+Predictions (`prediction_difference.png`): new − baseline posterior mean over
+the full grid, median +0.01 dB (Antarctica) / +0.07 dB (Greenland), 5th–95th
+percentile −0.2…+0.6 / −0.3…+0.6 dB; predictive std ratio 0.985. Largest
+positive shifts on the Siple Coast / Ross ice streams and NW Greenland
+(≈ +0.6–1 dB); slightly negative in interior SW Greenland.
+
+## Model-free check: season crossovers
+
+`outputs/qc_filter/crossover/crossover_summary.md` (off-diagonal cells present
+both before and after):
+
+| | mean \|median Δ\| pre → QC | mean sd pre → QC | diagonal sd pre → QC |
+|---|---|---|---|
+| Antarctica (18 cells) | 11.65 → 11.90 dB | 9.84 → 9.69 dB | 8.61 → 7.71 dB |
+| Greenland (15 cells) | 7.13 → 6.53 dB | 9.03 → 7.55 dB | 7.06 → 6.92 dB |
+
+- **2016_Greenland_P3** (49% seam rejects) is the clear success: vs
+  2014/2017/2018/2019 it goes from −4.3/+2.1/+2.5/−1.2 dB with sd 11–13 dB to
+  −0.2/−1.2/−0.6/−3.3 dB with sd 7.7–8.8 dB — i.e. it now looks like every
+  other P3 season, and its pairs with 2013 drop from −18.8 to −14.9 dB.
+- Within-season repeatability tightens where seams were flagged:
+  2013_Antarctica_Basler 10.4 → 8.4 dB, 2017_Antarctica_P3 8.2 → 5.6 dB,
+  2014_Antarctica_DC8 9.4 → 8.4 dB, 2017_Antarctica_Basler 17.9 → 13.7 dB.
+- **Season-level offsets do not move**: 2012_Antarctica_DC8 stays ~12 dB below
+  the 2014–2018 DC8/P3 seasons; 2017_Antarctica_Basler stays ~30 dB below its
+  neighbours (and 9–13 dB below the 2022/23 BaslerMKB); 2013_Antarctica_P3 vs
+  2019_GV stays +15 dB; 2013_Greenland_P3 stays 16–19 dB low. The QC catches
+  2% of 2013_Greenland_P3 and 46% of 2017_Antarctica_Basler, so
+  `exclude_collections` must stay as it is.
+
+Out-of-fold residuals by season (`scripts/residual_audit.py`, now in
+`docs/figures/residuals_by_season.png`): per-season medians move by <= 1.4 dB
+(2012 DC8 +7.7 -> +7.9, 2013 P3 -5.3 -> -6.7, 2019 GV +11.2 -> +12.2,
+2017 P3 -9.2 -> -9.6); the same seasons stay offset.
+
+## Assessment
+
+The suggested filter is sound hygiene and cheap (≈13% of traces, no
+retuning), and it demonstrably repairs the one season whose problem *is*
+trace-level (2016 Greenland seams). But the required-SNR model was already
+insensitive to it: at the 5 km grid / 1 km nearest-trace matching scale the
+rejected traces were a minority whose biases partly average out, and the
+dominant residual scatter (σ ≈ 12.7 dB) and the season-level calibration
+offsets are untouched. Adopt it as the default, keep the two exclusions, and
+treat season-level (crossover-derived) calibration as the next lever — not
+QC threshold tuning, which the sensitivity table shows only trades data
+volume for no measurable fit change.
+
+Open follow-ups: 2012_Antarctica_DC8's img1 ceiling is now fit_ok at −29.7 dB
+(4% of traces at the ceiling) but its ~12 dB offset to later DC8 seasons is
+a whole-season effect; 2019_Antarctica_GV loses 22% to its −30 dB ceiling
+(pileup 0.11, span 0.35 — a credible fit, worth a look upstream);
+2023_Antarctica_BaslerMKB's ceiling (pileup 0.01) is a weak fit that removes
+0.7% — harmless but not evidence of saturation.

agent_plans/20260901-calibration-qc-filter.md

@@ -0,0 +1,50 @@
+# Radiometric calibration QC filter (2026-09-01)
+
+Branch: `qc-calibration-filter`.
+
+## Context
+
+Upstream `radar-return-statistics` shipped calibration method 0.4.0 (refreshed to 0.4.1 on 2026-09-03: 2018_Antarctica_DC8's rising img2 envelope no longer counts as a ceiling, 49 Antarctic frames backfilled, variable attrs added)
+(`docs/dataset_changelog.md`, 2026-09): four per-trace diagnostics —
+`img_comb_offset_dB`, `img_comb_pair`, `surface_source_image_index`,
+`surface_ceiling_margin_dB` — plus a season-keyed `saturation` root attr. No
+pre-existing values changed (verified byte-identical upstream), so a model
+retrained on the re-pinned snapshots differs from the 2026-08-07 baseline only
+through the QC filter.
+
+Stores landed 2026-09-02 00:36 UTC (0.4.0: antarctica SYAKG11X8AFFY0H9H65G,
+greenland JWDABR34HPM816P70FD0, ase WQCXS05H226PX1KGRZ2G) and were refreshed
+2026-09-03 15:51 UTC (0.4.1, the pinned ones: antarctica 087JBD7NTAE8BTBTEYSG,
+greenland ACA8WY61ZF9W6VSBA3HG, ase F6TXWSEQ9RQCD1H7MSMG). utig / crosssystem not updated (not model inputs).
+
+## Plan
+
+1. Carry the four calibration columns through augment (`extract.carry_columns`
+   default; warn-skipped on stores without them).
+2. Implement the suggested filter as `split.calibration_qc` (split stage, next to
+   `exclude_collections`) so thresholds can be varied without re-running augment,
+   and so it applies to observations AND non-detections before matching:
+   - seam: `|img_comb_offset_dB| >= 3 dB` -> drop; NaN passes unless
+     `drop_unmeasured_seam` (dropping NaN would remove 93% of 2014_Greenland_P3
+     — `insufficient_overlap` is a geometry limitation, not evidence of a step).
+   - img2: `surface_source_image_index >= 2` -> drop (-1 unknown passes). Kept
+     as suggested: img2-sourced traces have median RSSNR 8–17 dB lower than
+     img1 traces of the same DC8 season (surface-power low bias), and the rule
+     costs only 3% of Antarctic / 4% of Greenland traces.
+   - saturated: `surface_ceiling_margin_dB < 2 dB` -> drop; NaN (no credible
+     season ceiling) passes.
+3. Back up the pre-QC outputs (`outputs/baseline_20260807/`: model/ + augment
+   parquets), re-pin snapshots, `snakemake model_all`.
+4. Compare with `scripts/qc_filter_comparison.py` (per-season removal, threshold
+   sensitivity, benchmark side by side, both posteriors on the SAME test points,
+   posterior overlays, prediction-difference maps) and
+   `scripts/season_crossover_matrix.py --calibration-qc` (model-free check).
+5. Keep `exclude_collections` unchanged for the main comparison (one change at a
+   time); report how much of each excluded season the QC alone would catch.
+
+## Status
+
+- [x] 1–2 implemented + unit tests (`tests/unit/test_calibration_qc.py`).
+- [x] 3 pipeline run (2026-09-02 on 0.4.0; re-run 2026-09-03 16:13–16:27 UTC on 0.4.1 — identical posteriors; run_ids split 4ff998437f3b, atten_refl af5514ae3b72, linear c0b385764662)
+- [x] 4 comparison + assessment (`agent_notes/20260901-calibration-qc-results.md`)
+- [x] docs (`docs/2_input_data.md`); Snakefile: store + model configs are now rule inputs

config/antarctica.yaml

@@ -8,7 +8,7 @@ store:
 
 icechunk:
   branch: "main"
-  snapshot_id: "VE13DY3F546ZE1J9KC60"  # pinned 2026-08-03: reprocessed with missing bed picks + pick-free noise stats
+  snapshot_id: "087JBD7NTAE8BTBTEYSG"  # pinned 2026-09-03: calibration method 0.4.1 refresh, no science values changed
 
 extract:
   qc_only: true

config/ase.yaml

@@ -7,7 +7,7 @@ store:
 
 icechunk:
   branch: "main"
-  snapshot_id: "CS9HEDFGE8FXDHVC9KM0"  # pinned 2026-07-30: tail noise window shifted off the end-of-record taper
+  snapshot_id: "F6TXWSEQ9RQCD1H7MSMG"  # pinned 2026-09-03: calibration method 0.4.1 refresh, no science values changed
 
 extract:
   qc_only: true

config/greenland.yaml

@@ -7,7 +7,7 @@ store:
 
 icechunk:
   branch: "main"
-  snapshot_id: "GEAMAHQ7BRVPG9SQPK20"  # pinned 2026-07-31: reprocessed with missing bed picks + new noise metrics
+  snapshot_id: "ACA8WY61ZF9W6VSBA3HG"  # pinned 2026-09-03: calibration method 0.4.1 refresh, no science values changed
 
 extract:
   qc_only: true

config/model.yaml

@@ -41,6 +41,15 @@ split:
   seed: 42
   test_cells: ["ant:-5:1", "ant:-3:-2", "ant:1:-3", "ant:2:-2", "ant:2:0", "grl:0:-6", "grl:1:-4", "grl:0:-2"] # pick from outputs/model/cell_maps/, e.g. ["ant:-3:1", "grl:0:-5"]
   exclude_collections: [2017_Antarctica_Basler, 2013_Greenland_P3]  # crossover-identified calibration outliers
+  # Per-trace radiometric QC from the upstream calibration fields (dataset
+  # changelog 2026-09, method 0.4.1). Rules pass where their input is
+  # unmeasured (NaN / unknown source image) — see docs/2_input_data.md.
+  calibration_qc:
+    enabled: true
+    max_seam_offset_dB: 3.0      # image-combine seam step |img_comb_offset_dB| >= this -> drop
+    drop_unmeasured_seam: false  # NaN offsets (check not run) are kept
+    require_img1_surface: true   # surface sampled from a higher-gain image (index >= 2) -> drop
+    min_ceiling_margin_dB: 2.0   # surface within this of its season's clip level -> drop
 
 train:
   seed: 42