From ffc192b06be22595ef981657c3dbcd23282e8df2 Mon Sep 17 00:00:00 2001 From: faridkurdnezhad Date: Wed, 26 Aug 2026 19:04:20 +0200 Subject: [PATCH 1/2] Prepare QQ v1.0.1 metadata, provenance, and documentation --- .gitattributes | 11 + .github/workflows/release.yml | 42 ++- .github/workflows/test.yml | 1 + Dockerfile | 30 +- README.md | 56 +++- deploy/deploy.sh | 18 +- documentations/ARCHITECTURE.md | 451 +++++++++++++++++++++++++++ documentations/CHANGELOG.md | 91 ++++++ documentations/METHODOLOGY_v1.0.0.md | 262 ++++++++++++++++ documentations/VERSIONING.md | 109 +++++++ pyproject.toml | 49 ++- qq/__init__.py | 9 +- qq/constants.py | 44 ++- qq/metadata.py | 181 +++++++++++ qq/output_netcdf.py | 108 +++++-- requirements.txt | 2 +- run_qq.py | 5 +- tests/test_pipeline.py | 69 +++- 18 files changed, 1450 insertions(+), 88 deletions(-) create mode 100644 .gitattributes create mode 100644 documentations/ARCHITECTURE.md create mode 100644 documentations/CHANGELOG.md create mode 100644 documentations/METHODOLOGY_v1.0.0.md create mode 100644 documentations/VERSIONING.md create mode 100644 qq/metadata.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fa7f440 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +* text=auto + +*.py text eol=lf +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.tf text eol=lf +*.md text eol=lf +*.txt text eol=lf +Dockerfile text eol=lf diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6985850..c5687e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,31 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Verify release tag matches package version + shell: bash + run: | + PACKAGE_VERSION=$(python - <<'PY' + import pathlib + import tomllib + + data = tomllib.loads( + pathlib.Path("pyproject.toml").read_text(encoding="utf-8") + ) + + print(data["project"]["version"]) + PY + ) + + if [ "v${PACKAGE_VERSION}" != "${GITHUB_REF_NAME}" ]; then + echo "ERROR: tag ${GITHUB_REF_NAME} does not match pyproject version v${PACKAGE_VERSION}" + exit 1 + fi + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -26,12 +51,17 @@ jobs: - name: Build and push Docker image env: - REGISTRY: ${{ steps.login-ecr.outputs.registry }} - REPOSITORY: swot-confluence-qq - IMAGE_TAG: ${{ github.ref_name }} + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: swot-confluence-qq + IMAGE_TAG: ${{ github.ref_name }} + GIT_COMMIT: ${{ github.sha }} run: | - docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . - docker tag $REGISTRY/$REPOSITORY:$IMAGE_TAG $REGISTRY/$REPOSITORY:latest + docker build \ + --build-arg GIT_COMMIT=$GIT_COMMIT \ + --build-arg GIT_DESCRIBE=$IMAGE_TAG \ + -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . + + docker tag $REGISTRY/$REPOSITORY:$IMAGE_TAG $REGISTRY/$REPOSITORY:latest docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG docker push $REGISTRY/$REPOSITORY:latest @@ -52,4 +82,4 @@ jobs: -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" \ -backend-config="key=${{ secrets.CONFLUENCE_PREFIX }}/qq/terraform.tfstate" \ -backend-config="region=${{ secrets.AWS_REGION }}" - terraform apply -auto-approve + terraform apply -auto-approve \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5959e6e..9bc2d07 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -e . pip install pytest pytest-cov - name: Run tests diff --git a/Dockerfile b/Dockerfile index ec30eaf..5aca6c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,6 @@ FROM python:3.11-slim AS base -LABEL maintainer="SWOT-Confluence" -LABEL description="QQ FLPE algorithm: quantile-quantile WSE-to-discharge mapping" WORKDIR /app @@ -28,13 +26,39 @@ COPY requirements.txt ./ RUN pip install --no-cache-dir --upgrade pip \ && pip install --no-cache-dir -r requirements.txt -# Copy the algorithm package and entry point +# # Copy the algorithm package and entry point +# COPY qq/ ./qq/ +# COPY run_qq.py ./ + + +# Copy package metadata and source +COPY pyproject.toml ./ +COPY README.md ./ COPY qq/ ./qq/ COPY run_qq.py ./ +# Install QQ itself so Python package metadata is available at runtime. +RUN pip install --no-cache-dir --no-deps . + + + + # Default production paths are baked into config.py as constants; # they can be overridden at runtime via --input_dir / --output_dir. ENV PYTHONUNBUFFERED=1 + +# Git provenance is injected by the release workflow. +ARG GIT_COMMIT=unknown +ARG GIT_DESCRIBE=unknown + +ENV QQ_GIT_COMMIT=${GIT_COMMIT} +ENV QQ_GIT_DESCRIBE=${GIT_DESCRIBE} + +LABEL org.opencontainers.image.revision=${GIT_COMMIT} +LABEL org.opencontainers.image.version=${GIT_DESCRIBE} + + + ENTRYPOINT ["python", "run_qq.py"] CMD ["--help"] diff --git a/README.md b/README.md index 4633d2e..35a5a9e 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ A reach-based Discharge Estimation algorithm for the [SWOT-Confluence](https://g - [Repository Structure](#repository-structure) - [Configuration](#configuration) - [Testing](#testing) -- [Deployment](#deployment) -- [SWOT-Confluence Integration](#swot-confluence-integration) - [Documentation](#documentation) - [Contributing](#contributing) - [License](#license) @@ -31,20 +29,21 @@ QQ (Quantile-Quantile) matching is a statistical, prior-based discharge estimati 1. Ranks clean, quality-filtered SWOT WSE observations to build an **empirical non-exceedance probability curve**. 2. Resamples that curve onto a standardized **deliverable probability grid** for NetCDF output. -3. Maps each observation's probability to a discharge value using the **SOS Flow Duration Curve** at the same probability level. +3. Maps each observation's probability to a discharge value using the **SOS Flow Duration Curve** at the same probability level, and creates a semi-rating-curve. 4. Writes a per-reach NetCDF discharge time series, always — including a fill-value file for reaches that fail quality gates. Unlike hydraulic FLPE algorithms, QQ requires no channel geometry, no Manning's roughness coefficient, and no rating-curve calibration. It depends only on the assumption that WSE rank approximates discharge rank over the observation period, and that the SOS FDC is representative of that period. - +See [`documentations/METHODOLOGY_v1.0.0.md`](documentations/METHODOLOGY_v1.0.0.md) for the full scientific description of the initial release (version 1.0.0). The changes records are registered in [`documentations/CHANGELOG.md`](documentations/CHANGELOG.md). the versioning naming convention definition is found in [`documentations/VERSIONING.md`](documentations/VERSIONING.md) --- ## Requirements -- Python **3.11+** +- Python **>= 3.10** - `netCDF4`, `numpy`, `pandas` (see [`requirements.txt`](requirements.txt)) - Git +- Production Docker/CI currently use Python **3.11** --- @@ -73,7 +72,7 @@ python -m pip install --upgrade pip pip install -r requirements.txt ``` -### 5. Editable install (optional, for development) +### 5. Install QQ ```bash python -m pip install -e . ``` @@ -135,7 +134,7 @@ pytest tests/ -v ├── swot/ │ └── _SWOT.nc └── sos/ - └── _SOS.nc + └── _sword_v17c_SOS_priors.nc ``` **`reaches.json`** — one entry per reach: @@ -145,7 +144,7 @@ pytest tests/ -v "reach_id": "21101200141", "swot": "21101200141_SWOT.nc", "sos": "eu_sword_v17c_SOS_priors.nc", - "sword": "eu_sword_v17b.nc" + "sword": "eu_sword_v17c.nc" } ] ``` @@ -186,7 +185,7 @@ Group "lookup_table" Fill / missing values: `f8` → `-999999999999.0`, `i2` flags → `-999`, `i4` scalars → `-999999999`. (The WSE-Q lookup table reuses these same values — no new fill-value convention was introduced.) -Full schema in [`docs/architecture.md`](docs/architecture.md). +Full schema in [`documentations/ARCHITECTURE.md`](documentations/ARCHITECTURE.md). --- @@ -202,7 +201,7 @@ Options: -i, --index INT 0-based reach index (default: 0) Overridden by $AWS_BATCH_JOB_ARRAY_INDEX --input_dir DIR Root input directory (default: /mnt/data/input) - --output_dir DIR Output directory (default: /mnt/data/output) + --output_dir DIR Output directory (default: /mnt/data/flpe/qq) --mode {RUN,DEBUG,AUDIT} RUN = production; errors non-fatal, fill-value NC written DEBUG = raises immediately on error @@ -229,15 +228,16 @@ QQ/ │ ├── input_json.py Manifest reading, path resolution │ ├── input_swot.py SWOT read → clean → filter → gate │ ├── input_sos.py SOS FDC extraction +│ ├── metadata.py +│ ├── lookup_table.py │ ├── wse_quantile.py Empirical + deliverable WSE quantile │ ├── quantile_matching.py Core WSE → probability → discharge │ ├── output_arrays.py Final output array preparation │ ├── output_netcdf.py NetCDF writer + log saver │ ├── diagnostics.py Optional Plotly plots │ └── pipeline.py Orchestrator -├── tests/ 60 automated tests -├── docs/ Full documentation suite -├── confluence/templates/modules/qq.sh.j2 run-confluence-locally SLURM template +├── tests/ Automated test suite +├── documentations/ Full documentation suite ├── deploy/deploy.sh 5-argument deploy script ├── terraform/ AWS Batch + ECR infrastructure ├── .github/workflows/ CI (test.yml) + CD (release.yml) @@ -267,6 +267,8 @@ Runtime settings (paths, index, mode) are resolved separately in `qq/config.py` ## Testing +Run the full automated test suite with: + ```bash pytest tests/ -v ``` @@ -277,10 +279,36 @@ pytest tests/ -v | `test_helpers.py` | Interpolation utilities | | `test_pipeline.py` | End-to-end pipeline, valid + invalid reach, CLI exit codes | -60 tests total, all passing on Python 3.11+. +--- + +## Documentation + +Detailed QQ Project documentation is available in: + +- [`documentations/METHODOLOGY_v1.0.0.md`](documentations/METHODOLOGY_v1.0.0.md) +- [`documentations/ARCHITECTURE.md`](documentations/ARCHITECTURE.md) +- [`documentations/VERSIONING.md`](documentations/VERSIONING.md) +- [`documentations/CHANGELOG.md`](documentations/CHANGELOG.md) + + +--- + +## Contributing + +Changes should normally be developed on a dedicated branch, tested with the +full automated test suite, and merged into `main` after review. --- +## Maintainers + +Canonical project authorship and maintainer information and repository metadata are declared in +[`pyproject.toml`](pyproject.toml). + +Repository ownership and access are managed through the +[SWOT-Confluence GitHub organization](https://github.com/SWOT-Confluence). + +--- ## License diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 2f1bcf3..42567a3 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -35,11 +35,27 @@ echo " S3_BUCKET : ${S3_BUCKET}" echo " PROFILE : ${PROFILE}" echo "" + +GIT_COMMIT="$( + git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || echo unknown +)" + +GIT_DESCRIBE="$( + git -C "${REPO_ROOT}" describe --tags --always --dirty 2>/dev/null || echo unknown +)" + + # ----------------------------------------------------------------------- # 1. Build Docker image # ----------------------------------------------------------------------- echo "[1/4] Building Docker image..." -docker build -t "${REGISTRY}/${REPOSITORY}:latest" "${REPO_ROOT}" +# docker build -t "${REGISTRY}/${REPOSITORY}:latest" "${REPO_ROOT}" + +docker build \ + --build-arg GIT_COMMIT="${GIT_COMMIT}" \ + --build-arg GIT_DESCRIBE="${GIT_DESCRIBE}" \ + -t "${REGISTRY}/${REPOSITORY}:latest" \ + "${REPO_ROOT}" # ----------------------------------------------------------------------- # 2. Authenticate to ECR and push diff --git a/documentations/ARCHITECTURE.md b/documentations/ARCHITECTURE.md new file mode 100644 index 0000000..5e11cd3 --- /dev/null +++ b/documentations/ARCHITECTURE.md @@ -0,0 +1,451 @@ +# QQ Architecture + +## Scope + +This document describes the software architecture of the SWOT-Confluence QQ discharge-estimation package. + +The scientific method of the initial release is documented separately in +[`METHODOLOGY_v1.0.0.md`](METHODOLOGY_v1.0.0.md). Version 1.0.1 changes +documentation, package metadata, and software provenance; the core QQ +scientific algorithm remains the v1.0.0 method. + +--- + +## 1. High-level architecture + +QQ is a **single-reach package**, written for SWOT-Confluence pipeline. v1.0.0 is written as QQ is as a FLPE Algorithm within the pipeline. One invocation selects one reach from `reaches.json`, reads its SWOT observations and SOS flow-duration-curve +prior, performs quantile matching, and writes one QQ NetCDF product. + +```mermaid +flowchart LR + CLI["run_qq.py
CLI / AWS Batch index"] + CFG["QQConfig
runtime configuration"] + STATE["QQState
per-reach mutable state"] + JSON["reaches.json"] + SWOT["SWOT reach NetCDF"] + SOS["SOS prior NetCDF"] + PIPE["qq.pipeline.run()"] + QC["SWOT cleaning / filtering / gates"] + WSEQ["Empirical + deliverable WSE quantiles"] + FDC["SOS FDC extraction / quality control"] + LUT["WSE-Q lookup table"] + QM["Quantile matching"] + ARR["Output-array preparation"] + NC["_qq.nc"] + LOG["_qq.log"] + + CLI --> CFG + CFG --> PIPE + PIPE --> STATE + JSON --> PIPE + SWOT --> PIPE + SOS --> PIPE + PIPE --> QC --> WSEQ + WSEQ --> FDC --> LUT --> QM --> ARR --> NC + PIPE --> LOG +``` + +QQ does not use hydraulic-model calibration or channel-geometry parameters. +In the current implementation, the manifest `sword` field is resolved for +ecosystem compatibility, but the SWORD NetCDF is not read by the scientific +calculation in the v1.0.0 but will be used in future versions of QQ. + +--- + +## 2. Package layers + +### 2.1 Entry point and runtime configuration + +**`run_qq.py`** + +- parses CLI arguments; +- resolves the reach manifest and runtime paths; +- initializes logging; +- builds `QQConfig`; +- invokes `qq.pipeline.run()`; +- returns process exit status. + +**`qq/config.py`** + +Defines `QQConfig`, which contains runtime controls such as: + +- input/output directories; +- manifest index; +- run mode (`RUN`, `DEBUG`, `AUDIT`); +- console/file logging; +- optional diagnostics. + +Runtime configuration is deliberately separate from scientific constants. + +--- + +### 2.2 Canonical project metadata and provenance + +**`pyproject.toml`** + +Canonical source for static package/project metadata: + +- distribution name; +- release version; +- short description; +- authors/maintainers; +- project URLs; +- Python requirement; +- package dependencies. + +**`qq/metadata.py`** + +Runtime bridge to the installed package metadata plus execution provenance: + +- package name/version/description; +- authors/maintainers; +- repository URL; +- UTC creation time; +- Git commit; +- Git describe/tag/dirty state. + +Static metadata should not be redefined here. + +**`.github/workflows/release.yml` + `Dockerfile`** + +Inject Git release provenance into the production container, where the +repository `.git` directory is not expected to be available. + +--- + +### 2.3 Scientific and ecosystem constants + +**`qq/constants.py`** + +Contains algorithmic, schema, flag, fill-value, variable-name and +probability-grid constants. + +It should not be the authoritative source for package version, authorship, +project URLs or other project identity metadata. + +--- + +### 2.4 Per-reach state and fault handling + +**`qq/state.py`** + +`QQState` is the mutable state object shared across pipeline stages. It holds: + +- resolved input/output paths; +- read SWOT/SOS data; +- cleaned/filtered tables; +- empirical WSE quantiles; +- SOS FDC; +- lookup-table products; +- matched discharge; +- final output arrays; +- warnings/errors and invalid-reach codes. + +**`qq/logger.py`** + +Provides: + +- structured logging; +- section/log helpers; +- warnings; +- fault-survival behavior; +- detailed-to-summary invalid-reach flag mapping. + +In `RUN` and `AUDIT`, recoverable scientific/data failures are represented in +state and in the output NetCDF rather than necessarily terminating the process. +`DEBUG` raises failures immediately. + +--- + +## 3. Input architecture + +### `qq/input_json.py` + +`setup_inputs_and_defaults()` + +- creates safe state defaults; +- resolves logical input/output locations. + +`read_json()` + +- reads `reaches.json`; +- selects the configured zero-based index; +- extracts `reach_id`, SWOT, SOS and SWORD filenames; +- resolves per-reach paths. + +### `qq/input_swot.py` + +`read_swot()` + +- opens the reach-level SWOT NetCDF; +- extracts required reach variables and NetCDF metadata; +- builds the SWOT DataFrame. + +`clean_swot()` + +- applies internal validity cleaning. + +`filter_swot()` + +- applies optional quality filtering such as the current `reach_q` criterion. + +`control_wse_count()` + +- applies the minimum clean/filtered observation gate. + +### `qq/input_sos.py` + +`read_sos()` + +- locates the selected reach in the continental SOS prior file; +- reads probability and flow-duration discharge; +- converts probabilities to the internal non-exceedance convention; +- applies FDC quality gates; +- builds the reach FDC table. + +--- + +## 4. Scientific transformation architecture + +### `qq/wse_quantile.py` + +`empirical_wse_quantile()` + +- ranks retained WSE values; +- constructs the empirical WSE/non-exceedance-probability relation. + +`deliverable_wse_quantile()` + +- resamples the empirical relation to the standard probability grid. + +`deliverable_wse_flag()` + +- records whether the standardized WSE product represents upsampling, + downsampling, equal-size sampling, or missing output. + +### `qq/lookup_table.py` + +`build_lookup_table()` + +- determines the common empirical-WSE/SOS-FDC probability interval; +- builds the regular lookup probability grid; +- interpolates WSE and discharge on that overlap; +- does not extrapolate beyond the overlap. + +The lookup table is an auxiliary deliverable and does not itself invalidate an +otherwise valid reach. + +### `qq/quantile_matching.py` + +`quantile_matching()` + +Implements the core mapping: + +```text +SWOT WSE + ↓ +empirical WSE non-exceedance probability + ↓ +SOS FDC at the same probability + ↓ +QQ discharge +``` + +With the current default configuration, discharge is estimated only within +the probability range supported by the SOS FDC. + +### `qq/helpers.py` + +Contains reusable low-level utilities for: + +- NetCDF character/string conversion; +- NetCDF variable metadata extraction; +- fail-safe empty DataFrames; +- WSE-quantile interpolation/extrapolation; +- probability/value interpolation. + +--- + +## 5. Output architecture + +### `qq/output_arrays.py` + +`prepare_output_arrays()` + +Builds the final arrays and flags written to NetCDF, including: + +- `QQ_time`; +- `QQ_q`; +- `QQ_q_status_flag`; +- standardized WSE quantiles; +- lookup-table arrays. + +It also defines fail-safe arrays for invalid reaches. + +### `qq/output_netcdf.py` + +`output_paths()` + +Creates the per-reach output and log paths. + +`write_nc()` + +Writes the complete NetCDF product for both valid and recoverably invalid +reaches. + +Current logical structure: + +```text +Root +├── QQ_time +├── QQ_invalid_reach_detailed_flag +├── QQ_invalid_reach_summary_flag +├── global dataset/software/provenance attributes +├── q/ +│ ├── QQ_q +│ └── QQ_q_status_flag +├── wse_quantile/ +│ ├── QQ_wse_quant_prob +│ ├── QQ_wse_quant_wse +│ └── QQ_wse_quant_flag +└── lookup_table/ + ├── QQ_lookup_table_prob + ├── QQ_lookup_table_wse + ├── QQ_lookup_table_q + └── QQ_lookup_table_flag +``` + +Software provenance includes package version and Git information without +changing the scientific variable/group contract. + +`save_log()` + +Writes the accumulated run log unless disabled by runtime configuration. + +### `qq/diagnostics.py` + +Optional Plotly diagnostics. These products are not required by the production +scientific output contract. + +--- + +## 6. Orchestration order + +`qq.pipeline.run()` is the central orchestrator. + +The current execution order is: + +```text +1. setup_inputs_and_defaults +2. read_json +3. read_swot +4. clean_swot +5. filter_swot +6. control_wse_count +7. empirical_wse_quantile +8. deliverable_wse_quantile +9. deliverable_wse_flag +10. read_sos +11. build_lookup_table +12. quantile_matching +13. prepare_output_arrays +14. prepare_plot_limits +15. output_paths +16. write_nc +17. make_plots +18. save_log +``` + +This ordering is part of the current implementation contract: later stages +consume state populated by earlier stages. + +--- + +## 7. Repository architecture + +```text +QQ/ +├── qq/ +│ ├── __init__.py +│ ├── metadata.py +│ ├── constants.py +│ ├── config.py +│ ├── state.py +│ ├── logger.py +│ ├── helpers.py +│ ├── input_json.py +│ ├── input_swot.py +│ ├── input_sos.py +│ ├── wse_quantile.py +│ ├── lookup_table.py +│ ├── quantile_matching.py +│ ├── output_arrays.py +│ ├── output_netcdf.py +│ ├── diagnostics.py +│ ├── pipeline.py +│ +├── documentations/ +│ ├── METHODOLOGY_v1.0.0.md +│ ├── VERSIONING.md +│ ├── CHANGELOG.md +│ ├── ARCHITECTURE.md +│ +├── tests/ +│ +├── deploy/ +├── terraform/ +├── .github/workflows/ +├── Dockerfile +├── requirements.txt +├── pyproject.toml +├── README.md +│ +└── run_qq.py +``` + +--- + +## 8. Deployment architecture + +```mermaid +flowchart LR + TAG["Git tag vX.Y.Z"] + GH["GitHub Actions release workflow"] + IMG["Docker image"] + ECR["Amazon ECR"] + TF["Terraform"] + BATCH["AWS Batch job definition"] + RUN["QQ container run"] + + TAG --> GH + GH --> IMG + IMG --> ECR + GH --> TF + TF --> BATCH + ECR --> RUN + BATCH --> RUN +``` + +The release workflow passes the release tag and Git commit into the Docker +build. `qq/metadata.py` reads these environment values in production so every +QQ NetCDF can retain the software provenance even when `.git` is absent from +the container. + +--- + +## 9. Compatibility boundary + +The stable downstream contract is primarily: + +- CLI/input manifest expectations; +- per-reach output filename `_qq.nc`; +- NetCDF dimensions, groups and variables; +- fill values and flags; +- scientific meaning of the output variables. + +Adding new global provenance attributes is backward-compatible for normal +NetCDF readers because existing groups and variables are unchanged. + +Changes that rename/remove required inputs or outputs, change required +dimensions incompatibly, or fundamentally replace the scientific method should +be treated according to the repository versioning policy. diff --git a/documentations/CHANGELOG.md b/documentations/CHANGELOG.md new file mode 100644 index 0000000..a55b659 --- /dev/null +++ b/documentations/CHANGELOG.md @@ -0,0 +1,91 @@ +# QQ Change Record + +This file records user-visible and scientifically relevant changes between QQ versions. + + +## [1.0.0] — Initial release + +### Included development commits + +#### **1. July 17, 2026 — Initial production package (`30ce4ba`):** +- Initial commit: SWOT-QQ Discharge Estimation algorithm production package +- https://github.com/SWOT-Confluence/QQ/commit/30ce4ba8be1f5a769be316014af3aa174079181d + +**Scientific method** + +- Introduced reach-based quantile-quantile discharge estimation. +- Constructed an empirical non-exceedance distribution from cleaned and filtered SWOT WSE observations. +- Matched SWOT WSE probability to discharge through the reach-specific SOS FDC. +- Restricted default discharge matching to the probability range available in the SOS FDC. +- Added a standardized WSE quantile product on the default 0–1 probability grid. +- Added an auxiliary WSE-Q lookup table over the empirical-WSE/SOS-FDC probability overlap. + +**Quality control** + +- Added SWOT cleaning and optional `reach_q` filtering. +- Added a minimum clean/filtered SWOT WSE count. +- Added SOS FDC missing-data and minimum-length quality gates. +- Added detailed and summary invalid-reach flags. + +**Outputs** + +- Added per-reach QQ discharge NetCDF output. +- Added discharge status flags. +- Added standardized WSE quantile outputs and resampling flag. +- Added WSE-Q lookup-table outputs and flag. +- Added run logging and optional diagnostic plots. + +**Execution** + +- Added `RUN`, `DEBUG`, and `AUDIT` modes. +- Added local/container CLI execution and AWS Batch index support. +- Added automated tests for constants, helper/interpolation logic, lookup-table behavior, end-to-end valid/invalid reaches, and CLI exit behavior. + + +#### **2. July 19, 2026 — Added WSE-Q lookup table (`bf02287`):** + +- Add Lookup Table +- https://github.com/SWOT-Confluence/QQ/commit/bf022871f8f805c96a45e72e0f9f7e51f25e7496 + + +## [1.0.1] — Aug 27, 2026 + +**Metadata and provenance** + +- Centralized static project metadata in `pyproject.toml`. +- Added `qq/metadata.py` for runtime package metadata and Git provenance. +- Added QQ software version and Git provenance to NetCDF global attributes. +- Added dataset creation timestamp, software repository, authorship, + source, history, references, and description metadata. +- Added Git commit and release-tag provenance to production Docker builds. + +**Documentation** + +- Added v1.0.0 methodology documentation. +- Added architecture documentation. +- Added semantic versioning policy. +- Added this change record. +- Updated README metadata, input examples, documentation links, and output descriptions. + +**Testing and deployment** + +- Added package installation to CI. +- Added metadata/provenance tests. +- Added release-tag/package-version consistency validation. + +**Scientific behavior** + +- No QQ scientific algorithm change. +- No scientific constant/default change. +- No quantile-matching change. +- No FDC-processing change. +- No change to the scientific NetCDF variables, dimensions, groups, flags, or fill values. + +**Compatibility** + +- Added backward-compatible NetCDF global attributes. +- Existing QQ scientific output variables and file naming remain unchanged. + + + +## [1.1.0] — Unreleased: Under Development diff --git a/documentations/METHODOLOGY_v1.0.0.md b/documentations/METHODOLOGY_v1.0.0.md new file mode 100644 index 0000000..366ed5a --- /dev/null +++ b/documentations/METHODOLOGY_v1.0.0.md @@ -0,0 +1,262 @@ +# QQ Methodology — Version 1.0.0 + +## 1. Scope + +QQ is a reach-based SWOT-Confluence FLPE algorithm that estimates river discharge from SWOT water-surface-elevation (WSE) observations by quantile matching against the SOS flow-duration curve (FDC). + +Version 1.0.0 is a **single-reach, prior-based statistical method**. It does not use a hydraulic model, channel geometry, calibration parameters, or upstream/downstream reach information. + +The implementation described here follows the code in `qq/pipeline.py` and its called modules. + +## 2. Core idea + +For one river reach: + +1. Read and quality-control the SWOT WSE time series. +2. Rank the retained WSE observations and assign empirical non-exceedance probabilities. +3. Read the SOS discharge FDC for the same reach. +4. Convert each retained SWOT WSE to a non-exceedance probability. +5. Convert that probability to discharge using the SOS FDC. +6. Write the discharge time series and supporting diagnostic products to NetCDF. + +The scientific assumption is that **WSE rank is a useful proxy for discharge rank**, and that the SOS FDC is representative for the reach. + +--- + +# 3. Inputs + +QQ selects one entry from `reaches.json` using a zero-based index. + +Each entry must provide: + +```json +{ + "reach_id": 12221500011, + "swot": "12221500011_SWOT.nc", + "sos": "af_sword_v17c_SOS_priors.nc", + "sword": "af_sword_v17c.nc" +} +``` + +QQ v1.0.0 reads: + +- the reach-specific SWOT NetCDF; +- the continental SOS prior NetCDF. + +The `sword` field is required by the manifest schema and its path is constructed, but the SWORD NetCDF is not used in the v1.0.0 calculation. + +--- + +# 4. High-level methodology + +## 4.1 SWOT preparation + +QQ reads the SWOT `reach` group and requires: + +- `reach_id` +- `time` +- `time_str` +- `wse` +- `n_good_nod` + +`reach_q` is optional. + +The default processing then: + +- removes rows with missing/invalid WSE or time information; +- if `reach_q` exists, removes rows with `reach_q == 3`; +- requires at least **50** clean and filtered WSE observations. + +A reach failing a required quality gate is marked invalid. + +## 4.2 Empirical WSE distribution + +The retained WSE values are sorted in ascending order. + +For `n` observations, their empirical non-exceedance probabilities are assigned linearly from: + +```text +0.0 → 1.0 +``` + +including both endpoints. + +Thus, for the default v1.0.0 probability convention: + +```text +smallest retained WSE → p = 0 +largest retained WSE → p = 1 +``` + +## 4.3 Standard WSE quantile product + +The empirical WSE curve is resampled onto the standard probability grid: + +```text +0.00, 0.01, 0.02, ..., 1.00 +``` + +for a total of **101 probability levels**. + +Interior values are linearly interpolated. Edge extrapolation is enabled by default, although with the default empirical probability range of 0–1 a valid empirical table already spans the full deliverable range. + +## 4.4 SOS flow-duration curve + +For the selected reach, QQ reads: + +```text +/reaches/reach_id +/model/probability +/model/flow_duration_q +``` + +SOS probabilities are stored as percentages and converted to `[0, 1]`. + +The FDC must pass the default quality controls: + +- missing FDC values ≤ 50%; +- longest consecutive missing gap ≤ 25 entries; +- at least 2 valid FDC values. + +Valid FDC rows are sorted by non-exceedance probability. + +## 4.5 Quantile matching + +For every clean and filtered SWOT observation: + +```text +WSE + ↓ +empirical WSE distribution + ↓ +non-exceedance probability p + ↓ +SOS FDC + ↓ +discharge Q +``` + +The WSE-to-probability and probability-to-discharge mappings use linear interpolation. + +In v1.0.0 default settings: + +- no fixed 5–95% probability clipping is applied; +- matching **is restricted to the probability range covered by the SOS FDC**; +- no discharge is estimated outside that FDC probability range. + +## 4.6 WSE-Q lookup table + +QQ also creates an auxiliary lookup table over the probability overlap between: + +- the empirical WSE distribution; and +- the SOS FDC. + +The default grid spacing is 1%. + +Both WSE and Q are linearly interpolated on this common probability grid, with **no extrapolation** outside the shared range. + +Failure of this auxiliary lookup table does not by itself invalidate the reach. + +--- + +# 5. Low-level pipeline order + +`qq.pipeline.run()` executes these steps in this exact order: + +1. `setup_inputs_and_defaults()` +2. `read_json()` +3. `read_swot()` +4. `clean_swot()` +5. `filter_swot()` +6. `control_wse_count()` +7. `empirical_wse_quantile()` +8. `deliverable_wse_quantile()` +9. `deliverable_wse_flag()` +10. `read_sos()` +11. `build_lookup_table()` +12. `quantile_matching()` +13. `prepare_output_arrays()` +14. `prepare_plot_limits()` +15. `output_paths()` +16. `write_nc()` +17. `make_plots()` +18. `save_log()` + +The mutable per-reach state is carried through these steps by `QQState`. + +Scientific and ecosystem constants are centralized in `qq/constants.py`; run-specific paths, index, mode, logging and plotting settings are handled by `QQConfig`. + +--- + +# 6. Output behavior + +The main file is: + +```text +/_qq.nc +``` + +Main products are: + +```text +Root +├── QQ_time +├── QQ_invalid_reach_detailed_flag +├── QQ_invalid_reach_summary_flag +├── q/ +│ ├── QQ_q +│ └── QQ_q_status_flag +├── wse_quantile/ +│ ├── QQ_wse_quant_prob +│ ├── QQ_wse_quant_wse +│ └── QQ_wse_quant_flag +└── lookup_table/ + ├── QQ_lookup_table_prob + ├── QQ_lookup_table_wse + ├── QQ_lookup_table_q + └── QQ_lookup_table_flag +``` + +Default discharge/time output uses the **cleaned and filtered SWOT time dimension**. + +Missing discharge values are retained as fill values and accompanied by `QQ_q_status_flag`. + +--- + +# 7. Fault handling + +QQ v1.0.0 is designed to be fault-surviving in `RUN` and `AUDIT` modes. + +A recoverable scientific/data failure: + +- marks the reach invalid; +- records detailed and summary failure codes; +- continues far enough to produce a NetCDF output, generally with fill values. + +In `DEBUG` mode, failures raise immediately to support development. + +The CLI therefore returns success for both: + +- a scientifically valid reach; and +- a recoverably invalid reach for which the expected fill-value output was written. + +--- + +# 8. Main v1.0.0 scientific defaults + +| Setting | v1.0.0 default | +|---|---:| +| Minimum clean/filtered SWOT WSE observations | `50` | +| WSE probability range | `0–100%` | +| WSE probability step | `1%` | +| Standard WSE quantile length | `101` | +| Remove `reach_q == 3` when available | Yes | +| Maximum missing SOS FDC percentage | `50%` | +| Maximum consecutive missing FDC gap | `25` | +| Minimum valid SOS FDC values | `2` | +| Fixed 5–95% clipping | Disabled | +| Restrict matching to SOS FDC probability range | Yes | +| Lookup-table probability step | `1%` | +| Lookup-table extrapolation | No | + +For the exact implementation and all flags/fill values, `qq/constants.py` remains the authoritative source for v1.0.0. diff --git a/documentations/VERSIONING.md b/documentations/VERSIONING.md new file mode 100644 index 0000000..5d4da73 --- /dev/null +++ b/documentations/VERSIONING.md @@ -0,0 +1,109 @@ +# QQ Versioning Policy + +QQ uses **Semantic Versioning**: + +```text +MAJOR.MINOR.PATCH +``` + +Example: + +```text +1.0.0 +``` + +Git release tags use: + +```text +v1.0.0 +``` + +The repository release workflow is triggered by tags matching `v*`. + +## 1. PATCH — `x.y.Z` + +QQ uses a patch release when the intended scientific method and public data contract do not change. + +Examples: + +- documentation additions or corrections; +- tests; +- logging or comments; +- metadata-only corrections; +- internal refactoring intended to preserve numerical results; +- backward-compatible bug fixes that restore the intended behavior. + +Example: + +```text +1.0.0 → 1.0.1 +``` + +## 2. MINOR — `x.Y.z` + +QQ uses a minor release for a backward-compatible scientific or functional change that may change results. + +Examples: + +- changing a scientific threshold or default constant; +- changing how the SOS FDC is screened, constructed, interpolated, or used; +- adding a new optional scientific mode; +- adding a new CLI/configuration capability while preserving existing usage; +- changing quantile-matching behavior while keeping the same basic input/output contract. + +Example: + +```text +1.0.1 → 1.1.0 +``` + +For scientific reproducibility, QQ considers a default-value change that can change discharge results as a **MINOR**, not **PATCH**. + +## 3. MAJOR — `X.y.z` + +QQ uses a major release for an incompatible contract or a fundamental / architectural redesign. + +Examples: + +- requiring new input data that old workflows do not provide; +- changing the NetCDF schema incompatibly; +- removing or renaming required outputs; +- replacing the core QQ method with a substantially different methodology; +- making upstream/downstream reach information a required part of the algorithm. + +Example: + +```text +1.x.x → 2.0.0 +``` + +--- + +# 4. Version source + +The canonical QQ software version is defined only in: + +```text +pyproject.toml +``` + +--- + +# 5. Release checklist for the maintainers + +For each release: + +1. Decide `MAJOR.MINOR.PATCH`. +2. Update `pyproject.toml`. +3. Update `documentations/CHANGELOG.md`. +4. if the methodology is updated, release an updated METHODOLOGY_vX.y.z.md document. +5. Reinstall/refresh the editable package locally. +6. Run the full test suite. +7. Commit the release changes. +8. Merge the approved branch into `main`. +9. Tag the exact release commit: matching vX.Y.Z tag. for example: + +```bash +git tag -a v1.0.1 -m "QQ v1.0.1" +git push origin v1.0.1 +``` diff --git a/pyproject.toml b/pyproject.toml index 25132be..71c015f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,20 +3,59 @@ requires = ["setuptools>=61", "wheel"] build-backend = "setuptools.build_meta" + [project] name = "swot-confluence-qq" -version = "1.0.0" -description = "SWOT-Confluence QQ FLPE algorithm: quantile-quantile discharge estimation" +version = "1.0.1" +description = "SWOT-Confluence QQ discharge estimation algorithm: reach-based quantile-quantile mapping" +readme = "README.md" requires-python = ">=3.10" + +authors = [ + { name = "SWOT-Confluence" }, + { name = "Mohammadjavad Tourian", email = "tourian@gis.uni-stuttgart.de" }, +] + +maintainers = [ + { name = "Farid Kurdnezhad", email = "farid.kurdnezhad@unibo.it" }, + { name = "Siqi Ke", email = "siqi.ke@gis.uni-stuttgart.de" } +] + + + +keywords = [ + "SWOT", + "SWOT-Confluence", + "hydrology", + "river discharge", + "quantile-quantile", + "flow duration curve" +] + dependencies = [ - "netCDF4>=1.6.5", - "numpy>=1.26.0", - "pandas>=2.1.0", + "netCDF4>=1.6.5,<2.0", + "numpy>=1.26.0,<2.0", + "pandas>=2.1.0,<3.0" ] +[project.urls] +Homepage = "https://github.com/SWOT-Confluence/QQ" +Repository = "https://github.com/SWOT-Confluence/QQ" +Issues = "https://github.com/SWOT-Confluence/QQ/issues" +Documentation = "https://github.com/SWOT-Confluence/QQ/tree/main/documentations" +Architecture = "https://github.com/SWOT-Confluence/QQ/blob/main/documentations/ARCHITECTURE.md" +Methodology_v100 = "https://github.com/SWOT-Confluence/QQ/blob/main/documentations/METHODOLOGY_v1.0.0.md" +Versioning_convention = "https://github.com/SWOT-Confluence/QQ/blob/main/documentations/VERSIONING.md" +Changelog = "https://github.com/SWOT-Confluence/QQ/blob/main/documentations/CHANGELOG.md" +Readme = "https://github.com/SWOT-Confluence/QQ/blob/main/README.md" + + [project.scripts] run_qq = "run_qq:main" +[tool.setuptools] +py-modules = ["run_qq"] + [tool.setuptools.packages.find] where = ["."] include = ["qq*"] diff --git a/qq/__init__.py b/qq/__init__.py index ffcfa92..ed04c99 100644 --- a/qq/__init__.py +++ b/qq/__init__.py @@ -1,9 +1,12 @@ """ -qq — SWOT-Confluence FLPE algorithm: QQ discharge estimation. +QQ discharge estimation algorithm. Quantile-Quantile mapping from SWOT WSE observations to discharge using the SOS Flow Duration Curve. """ -__version__ = "1.0.0" -__author__ = "SWOT-Confluence" + +from qq.metadata import ( + PROJECT_AUTHOR as __author__, + PROJECT_VERSION as __version__, +) diff --git a/qq/constants.py b/qq/constants.py index 92f4c4e..8c19afa 100644 --- a/qq/constants.py +++ b/qq/constants.py @@ -1,7 +1,7 @@ """ qq/constants.py =============== -All algorithm-level constants for the SWOT-QQ FLPE algorithm. +All algorithm-level constants for the SWOT-QQ Discharge Estimation algorithm. These constants are extracted directly from build_config() in the prototype notebook and must not be changed without scientific review. They are intentionally verbose and @@ -209,16 +209,16 @@ def _build_wse_probability_grid() -> np.ndarray: WSE_QUANT_FLAG_DEFINITION: dict[int, str] = { 0: "same_length: standard_table_N == n_empirical_obs", - 1: "downsampled_10pct: standard_N > empirical by 0-10%", - 2: "downsampled_25pct: standard_N > empirical by 10-25%", - 3: "downsampled_50pct: standard_N > empirical by 25-50%", - 4: "downsampled_75pct: standard_N > empirical by 50-75%", - 5: "downsampled_extreme: standard_N > empirical by >75%", - -1: "upsampled_10pct: standard_N < empirical by 0-10%", - -2: "upsampled_25pct: standard_N < empirical by 10-25%", - -3: "upsampled_50pct: standard_N < empirical by 25-50%", - -4: "upsampled_75pct: standard_N < empirical by 50-75%", - -5: "upsampled_extreme: standard_N < empirical by >75%", + 1: "downsampled_10pct: standard_N < empirical by 0-10%", + 2: "downsampled_25pct: standard_N < empirical by 10-25%", + 3: "downsampled_50pct: standard_N < empirical by 25-50%", + 4: "downsampled_75pct: standard_N < empirical by 50-75%", + 5: "downsampled_extreme: standard_N < empirical by >75%", + -1: "upsampled_10pct: standard_N > empirical by 0-10%", + -2: "upsampled_25pct: standard_N > empirical by 10-25%", + -3: "upsampled_50pct: standard_N > empirical by 25-50%", + -4: "upsampled_75pct: standard_N > empirical by 50-75%", + -5: "upsampled_extreme: standard_N > empirical by >75%", -999: "all_missing: WSE quantile table could not be produced", } @@ -252,16 +252,16 @@ def _build_wse_probability_grid() -> np.ndarray: LOOKUP_TABLE_FLAG_DEFINITION: dict[int, str] = { 0: "same_length: lookup_table_N == n_empirical_obs", - 1: "downsampled_10pct: lookup_N > empirical by 0-10%", - 2: "downsampled_25pct: lookup_N > empirical by 10-25%", - 3: "downsampled_50pct: lookup_N > empirical by 25-50%", - 4: "downsampled_75pct: lookup_N > empirical by 50-75%", - 5: "downsampled_extreme: lookup_N > empirical by >75%", - -1: "upsampled_10pct: lookup_N < empirical by 0-10%", - -2: "upsampled_25pct: lookup_N < empirical by 10-25%", - -3: "upsampled_50pct: lookup_N < empirical by 25-50%", - -4: "upsampled_75pct: lookup_N < empirical by 50-75%", - -5: "upsampled_extreme: lookup_N < empirical by >75%", + 1: "downsampled_10pct: lookup_N < empirical by 0-10%", + 2: "downsampled_25pct: lookup_N < empirical by 10-25%", + 3: "downsampled_50pct: lookup_N < empirical by 25-50%", + 4: "downsampled_75pct: lookup_N < empirical by 50-75%", + 5: "downsampled_extreme: lookup_N < empirical by >75%", + -1: "upsampled_10pct: lookup_N > empirical by 0-10%", + -2: "upsampled_25pct: lookup_N > empirical by 10-25%", + -3: "upsampled_50pct: lookup_N > empirical by 25-50%", + -4: "upsampled_75pct: lookup_N > empirical by 50-75%", + -5: "upsampled_extreme: lookup_N > empirical by >75%", -999: "all_missing: lookup table could not be produced", } @@ -449,8 +449,6 @@ def _build_wse_probability_grid() -> np.ndarray: # NetCDF root-level metadata # --------------------------------------------------------------------------- -OUTPUT_NC_ROOT_TITLE: str = "QQ discharge output" -OUTPUT_NC_ROOT_INSTITUTION: str = "SWOT-Confluence" OUTPUT_NC_ROOT_CONVENTIONS: str = "CF-1.8" OUTPUT_NC_ROOT_DIM_NT_NAME: str = "nt" diff --git a/qq/metadata.py b/qq/metadata.py new file mode 100644 index 0000000..0ba1300 --- /dev/null +++ b/qq/metadata.py @@ -0,0 +1,181 @@ +""" +Runtime access to QQ project metadata and software provenance. + +Static project metadata comes from the installed distribution metadata, +whose authoritative source is pyproject.toml. + +Git information and creation timestamps are determined at runtime. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from importlib.metadata import ( + PackageNotFoundError, + metadata as distribution_metadata, + packages_distributions, +) +import os +from pathlib import Path +import subprocess +from email.utils import getaddresses + + +def _load_project_metadata(): + """ + Locate the installed distribution that provides the `qq` package. + + This avoids repeating the distribution name here. + """ + distributions = sorted( + set(packages_distributions().get("qq", [])) + ) + + if len(distributions) != 1: + raise RuntimeError( + "Could not uniquely identify the installed distribution " + f"providing package 'qq': {distributions}. " + "Install QQ with `python -m pip install -e .`." + ) + + try: + return distribution_metadata(distributions[0]) + except PackageNotFoundError as exc: + raise RuntimeError( + "QQ package metadata is unavailable. " + "Install QQ with `python -m pip install -e .`." + ) from exc + + +_PROJECT_METADATA = _load_project_metadata() + + +PROJECT_NAME = _PROJECT_METADATA.get("Name", "") +PROJECT_VERSION = _PROJECT_METADATA.get("Version", "") +PROJECT_DESCRIPTION = _PROJECT_METADATA.get("Summary", "") + +def _read_people(name_field: str, email_field: str) -> list[dict[str, str]]: + people: list[dict[str, str]] = [] + + # Entries having a name but no email. + for raw in _PROJECT_METADATA.get_all(name_field) or []: + for name in raw.split(","): + name = name.strip() + if name: + people.append({ + "name": name, + "email": "", + }) + + # Entries having an email, optionally with a name. + for name, email in getaddresses( + _PROJECT_METADATA.get_all(email_field) or [] + ): + people.append({ + "name": name.strip(), + "email": email.strip(), + }) + + return people + + +PROJECT_AUTHORS = _read_people( + "Author", + "Author-email", +) + +PROJECT_MAINTAINERS = _read_people( + "Maintainer", + "Maintainer-email", +) + +PROJECT_AUTHOR = ", ".join( + p["name"] or p["email"] + for p in PROJECT_AUTHORS +) + +PROJECT_MAINTAINER = ", ".join( + p["name"] or p["email"] + for p in PROJECT_MAINTAINERS +) + +# QQ project convention: +# first author entry is the owning organization. +PROJECT_OWNER = ( + PROJECT_AUTHORS[0]["name"] + if PROJECT_AUTHORS + else "" +) + + +def _read_project_urls() -> dict[str, str]: + urls: dict[str, str] = {} + + for item in _PROJECT_METADATA.get_all("Project-URL") or []: + if "," not in item: + continue + + label, url = item.split(",", 1) + urls[label.strip().lower()] = url.strip() + + return urls + + +PROJECT_URLS = _read_project_urls() + +REPOSITORY_URL = ( + PROJECT_URLS.get("repository") + or PROJECT_URLS.get("homepage") + or "" +) + + +def utc_now_iso() -> str: + """Current UTC timestamp in ISO-8601 format.""" + return ( + datetime.now(timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z") + ) + + +def _run_git(*args: str) -> str: + """Run a Git command in the repository root.""" + repo_root = Path(__file__).resolve().parents[1] + + try: + result = subprocess.run( + ["git", "-C", str(repo_root), *args], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + return result.stdout.strip() + except Exception: + return "" + + +def get_git_commit() -> str: + """ + Exact Git commit used for the run. + + Deployment can inject QQ_GIT_COMMIT when `.git` is not available + inside the container. + """ + return ( + os.getenv("QQ_GIT_COMMIT") + or _run_git("rev-parse", "HEAD") + or "unknown" + ) + + +def get_git_describe() -> str: + """ + Human-readable Git version/tag and dirty state. + """ + return ( + os.getenv("QQ_GIT_DESCRIBE") + or _run_git("describe", "--tags", "--always", "--dirty") + or "unknown" + ) \ No newline at end of file diff --git a/qq/output_netcdf.py b/qq/output_netcdf.py index 2b913cc..35f7a4f 100644 --- a/qq/output_netcdf.py +++ b/qq/output_netcdf.py @@ -24,11 +24,35 @@ Variables: nt(nt), nwseq(nwseq), QQ_time(nt), QQ_invalid_reach_detailed_flag (scalar), QQ_invalid_reach_summary_flag (scalar) - Global attrs: title, institution, algorithm, reach_id, is_valid, - Conventions, source_swot_file, source_sos_file, - warnings, errors, run_mode, invalid_reach_detailed_code, - invalid_reach_summary_code, invalid_reach_detailed_codes, - invalid_reach_messages + + Global attrs: + title + institution + algorithm + software_name + software_version + software_git_commit + software_git_describe + software_repository + software_authors + software_maintainers + date_created + source + history + references + comment + reach_id + is_valid + Conventions + source_swot_file + source_sos_file + warnings + errors + run_mode + invalid_reach_detailed_code + invalid_reach_summary_code + invalid_reach_detailed_codes + invalid_reach_messages Group "q" Variables: QQ_q(nt), QQ_q_status_flag(nt) @@ -51,6 +75,8 @@ import numpy as np from qq import constants as C +from qq import metadata as M + from qq.config import QQConfig from qq.logger import fail, log, log_vars, section from qq.state import QQState @@ -143,6 +169,11 @@ def write_nc(config: QQConfig, state: QQState) -> None: """ section(config, state, "4-1 WRITE QQ OUTPUT NETCDF") + # For every generated NetCDF, QQ determines: + created_utc = M.utc_now_iso() # when it was created + software_git_commit = M.get_git_commit() # which exact Git commit produced it + software_git_describe = M.get_git_describe() # which tag/description/dirty state produced it + # ----------------------------------------------------------------------- # Run-specific root metadata (computed here, not in constants) # ----------------------------------------------------------------------- @@ -182,15 +213,46 @@ def write_nc(config: QQConfig, state: QQState) -> None: with Dataset(state.output_nc_path, "w", format="NETCDF4") as qq_nc: + # ---------------------------------------------------------------- # GLOBAL METADATA # ---------------------------------------------------------------- - qq_nc.title = C.OUTPUT_NC_ROOT_TITLE - qq_nc.institution = C.OUTPUT_NC_ROOT_INSTITUTION - qq_nc.algorithm = output_nc_root_algorithm - qq_nc.reach_id = output_nc_root_reach_id - qq_nc.is_valid = output_nc_root_is_valid - qq_nc.Conventions = C.OUTPUT_NC_ROOT_CONVENTIONS + + # Dataset identity + qq_nc.title = f"{C.ALGO_NAME.upper()} discharge output" + # qq_nc.institution = M.PROJECT_AUTHOR + qq_nc.institution = M.PROJECT_OWNER + qq_nc.software_authors = M.PROJECT_AUTHOR + if M.PROJECT_MAINTAINER: + qq_nc.software_maintainers = M.PROJECT_MAINTAINER + + qq_nc.algorithm = output_nc_root_algorithm + + # Software identity / reproducibility + qq_nc.software_name = M.PROJECT_NAME + qq_nc.software_version = M.PROJECT_VERSION + qq_nc.software_git_commit = software_git_commit + qq_nc.software_git_describe = software_git_describe + qq_nc.software_repository = M.REPOSITORY_URL + + # Creation provenance + qq_nc.date_created = created_utc + qq_nc.source = f"{M.PROJECT_NAME} {M.PROJECT_VERSION}" + qq_nc.history = ( + f"{created_utc}: generated by " + f"{M.PROJECT_NAME} {M.PROJECT_VERSION}; " + f"git_commit={software_git_commit}" + ) + qq_nc.references = M.REPOSITORY_URL + qq_nc.comment = M.PROJECT_DESCRIPTION + + # Data conventions + qq_nc.Conventions = C.OUTPUT_NC_ROOT_CONVENTIONS + + # Reach/run metadata + qq_nc.reach_id = output_nc_root_reach_id + qq_nc.is_valid = output_nc_root_is_valid + qq_nc.source_swot_file = output_nc_root_source_swot_file qq_nc.source_sos_file = output_nc_root_source_sos_file qq_nc.warnings = output_nc_root_warnings @@ -213,12 +275,12 @@ def write_nc(config: QQConfig, state: QQState) -> None: # nt_var = qq_nc.createVariable( # C.OUTPUT_NC_ROOT_DIM_NT_NAME, "i4", (C.OUTPUT_NC_ROOT_DIM_NT_NAME,) # ) - + nt_var = qq_nc.createVariable( C.OUTPUT_NC_ROOT_DIM_NT_NAME, "i4", (C.OUTPUT_NC_ROOT_DIM_NT_NAME,), fill_value=np.int32(-999999999), ) - + nt_var.long_name = "time_step_index" nt_var.units = "1" nt_var[:] = np.arange(len(state.QQ_time_out), dtype=np.int32) @@ -229,12 +291,12 @@ def write_nc(config: QQConfig, state: QQState) -> None: # nwseq_var = qq_nc.createVariable( # C.OUTPUT_NC_ROOT_DIM_NWSEQ_NAME, "i4", (C.OUTPUT_NC_ROOT_DIM_NWSEQ_NAME,) # ) - + nwseq_var = qq_nc.createVariable( C.OUTPUT_NC_ROOT_DIM_NWSEQ_NAME, "i4", (C.OUTPUT_NC_ROOT_DIM_NWSEQ_NAME,), fill_value=np.int32(-999999999), ) - + nwseq_var.long_name = "wse_quantile_index" nwseq_var.units = "1" nwseq_var[:] = np.arange(len(state.QQ_wse_quant_prob_out), dtype=np.int32) @@ -267,12 +329,12 @@ def write_nc(config: QQConfig, state: QQState) -> None: # det_flag_var = qq_nc.createVariable( # C.SWOT_QQ_DELIVERABLE_INVALID_REACH_DETAILED_FLAG_NAME, "i4", () # ) - + det_flag_var = qq_nc.createVariable( C.SWOT_QQ_DELIVERABLE_INVALID_REACH_DETAILED_FLAG_NAME, "i4", (), fill_value=np.int32(-999999999), ) - + det_flag_var.long_name = "QQ_invalid_reach_detailed_reason_flag" det_flag_var.units = "1" det_flag_var.flag_definition = json.dumps(C.INVALID_REACH_DETAILED_FLAG_DICT) @@ -284,12 +346,12 @@ def write_nc(config: QQConfig, state: QQState) -> None: # sum_flag_var = qq_nc.createVariable( # C.SWOT_QQ_DELIVERABLE_INVALID_REACH_SUMMARY_FLAG_NAME, "i4", () # ) - + sum_flag_var = qq_nc.createVariable( C.SWOT_QQ_DELIVERABLE_INVALID_REACH_SUMMARY_FLAG_NAME, "i4", (), fill_value=np.int32(-999999999), ) - + sum_flag_var.long_name = "QQ_invalid_reach_summary_reason_flag" sum_flag_var.units = "1" sum_flag_var.flag_definition = json.dumps(C.INVALID_REACH_SUMMARY_FLAG_DICT) @@ -335,7 +397,7 @@ def write_nc(config: QQConfig, state: QQState) -> None: qs_var.flag_meanings = C.QQ_Q_STATUS_FLAG_MEANINGS qs_var.flag_definition = json.dumps(C.QQ_Q_STATUS_FLAG_DEFINITION) qs_var[:] = state.QQ_q_status_flag_out - + # ---------------------------------------------------------------- # GROUP "wse_quantile" # ---------------------------------------------------------------- @@ -373,7 +435,7 @@ def write_nc(config: QQConfig, state: QQState) -> None: # "0_same_length positive_downsampled negative_upsampled -999_all_missing" # ) # flag_var.assignValue(state.QQ_wse_quant_flag_out) - + flag_var = wq_gp.createVariable( C.SWOT_QQ_DELIVERABLE_WSE_QUANT_FLAG_NAME, "i2", (), fill_value=C.WSE_QUANT_FLAG_ALL_MISSING_VALUE, @@ -383,8 +445,8 @@ def write_nc(config: QQConfig, state: QQState) -> None: flag_var.missing_value = C.WSE_QUANT_FLAG_ALL_MISSING_VALUE flag_var.flag_definition = json.dumps(C.WSE_QUANT_FLAG_DEFINITION) flag_var.assignValue(state.QQ_wse_quant_flag_out) - - + + # ---------------------------------------------------------------- # GROUP "lookup_table" — WSE-Q lookup table (auxiliary deliverable) # ---------------------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index 1314558..13a4cf6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# SWOT-Confluence QQ FLPE algorithm — Python dependencies +# SWOT-Confluence QQ discharge estimation algorithm — Python dependencies # # Pin major.minor to match the SWOT-Confluence production environment. # Update pins only after compatibility testing. diff --git a/run_qq.py b/run_qq.py index 2915f14..3a47072 100644 --- a/run_qq.py +++ b/run_qq.py @@ -2,7 +2,7 @@ """ run_qq.py ========= -SWOT-Confluence FLPE algorithm: QQ discharge estimation. +SWOT-Confluence Discharge Estimation algorithm: QQ discharge estimation. CLI entry point — compatible with run-confluence-locally and AWS Batch. Production invocation (inside container): @@ -36,12 +36,13 @@ import argparse import sys from pathlib import Path +from qq.metadata import PROJECT_DESCRIPTION def _build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="run_qq.py", - description="SWOT-Confluence QQ FLPE algorithm — per-reach discharge estimation.", + description=PROJECT_DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter, ) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c4849e0..cda9683 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -11,6 +11,7 @@ import pytest from netCDF4 import Dataset +from qq import metadata as M from qq import constants as C from qq.config import QQConfig from tests.conftest import N_OBS, REACH_ID @@ -63,8 +64,8 @@ def test_q_and_time_same_length(self, state): def test_q_status_flag_same_length_as_q(self, state): assert len(state.QQ_q_status_flag_out) == len(state.QQ_q_out) - - + + def test_quantile_matching_uses_only_fdc_range(self, state): """With predefined extremes disabled, every valid Q must have p within FDC range.""" import pandas as pd @@ -91,16 +92,16 @@ def test_quantile_matching_uses_only_fdc_range(self, state): # def test_wse_quant_wse_length_is_100(self, state): # assert len(state.QQ_wse_quant_wse_out) == C.DELIVERABLE_WSE_QUANTILE_TABLE_N - - + + def test_wse_quant_prob_length_matches_grid(self, state): assert len(state.QQ_wse_quant_prob_out) == len(C.DELIVERABLE_WSE_PROBABILITY_GRID) def test_wse_quant_wse_length_matches_grid(self, state): assert len(state.QQ_wse_quant_wse_out) == len(C.DELIVERABLE_WSE_PROBABILITY_GRID) - - + + @@ -148,6 +149,60 @@ def test_nc_conventions(self, state): with open_nc(state) as nc: assert nc.Conventions == C.OUTPUT_NC_ROOT_CONVENTIONS + def test_nc_software_name(self, state): + with open_nc(state) as nc: + assert nc.software_name == M.PROJECT_NAME + + + def test_nc_software_version(self, state): + with open_nc(state) as nc: + assert nc.software_version == M.PROJECT_VERSION + + + def test_nc_git_commit_exists(self, state): + with open_nc(state) as nc: + assert isinstance(nc.software_git_commit, str) + assert len(nc.software_git_commit) > 0 + + + def test_nc_git_describe_exists(self, state): + with open_nc(state) as nc: + assert isinstance(nc.software_git_describe, str) + assert len(nc.software_git_describe) > 0 + + + def test_nc_date_created_exists(self, state): + with open_nc(state) as nc: + assert isinstance(nc.date_created, str) + assert nc.date_created.endswith("Z") + + + def test_nc_repository(self, state): + with open_nc(state) as nc: + assert nc.software_repository == M.REPOSITORY_URL + + + def test_package_version_single_source(self): + import qq + assert qq.__version__ == M.PROJECT_VERSION + + def test_nc_institution(self, state): + with open_nc(state) as nc: + assert nc.institution == M.PROJECT_OWNER + + + + + + + + + + + + + + def test_nc_root_dimensions(self, state): with open_nc(state) as nc: assert C.OUTPUT_NC_ROOT_DIM_NT_NAME in nc.dimensions @@ -161,7 +216,7 @@ def test_nc_root_dimensions(self, state): def test_nc_nwseq_dimension_matches_grid(self, state): with open_nc(state) as nc: assert nc.dimensions[C.OUTPUT_NC_ROOT_DIM_NWSEQ_NAME].size == len(C.DELIVERABLE_WSE_PROBABILITY_GRID) - + def test_nc_time_variable_exists(self, state): From bcdb2f67441eda19014052f3b6e11863e7079c0e Mon Sep 17 00:00:00 2001 From: faridKurdnezhadUnibo <127771837+faridKurdnezhadUnibo@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:01:26 +0200 Subject: [PATCH 2/2] fix: clean up Dockerfile, fix stale comments and ARCHITECTURE grammar --- Dockerfile | 10 +++--- README.md | 23 ++++++++++++ .../run-confluence-locally/config.py | 19 ++++++++++ .../run-confluence-locally/dir_structure.py | 24 +++++++++++++ .../run-confluence-locally/module_names.py | 36 +++++++++++++++++++ .../run-confluence-locally/qq.sh.j2 | 24 +++++++++++++ documentations/ARCHITECTURE.md | 3 +- documentations/CHANGELOG.md | 4 +++ qq/output_netcdf.py | 9 +++-- 9 files changed, 140 insertions(+), 12 deletions(-) create mode 100644 _other_modules/run-confluence-locally/config.py create mode 100644 _other_modules/run-confluence-locally/dir_structure.py create mode 100644 _other_modules/run-confluence-locally/module_names.py create mode 100644 _other_modules/run-confluence-locally/qq.sh.j2 diff --git a/Dockerfile b/Dockerfile index 5aca6c5..5d913af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ # -v /path/to/mnt/flpe/qq:/mnt/data/flpe/qq \ # qq:latest /mnt/data/input/reaches.json --index 0 -FROM python:3.11-slim AS base +# FROM python:3.11-slim AS base +FROM python:3.11-slim WORKDIR /app @@ -26,10 +27,6 @@ COPY requirements.txt ./ RUN pip install --no-cache-dir --upgrade pip \ && pip install --no-cache-dir -r requirements.txt -# # Copy the algorithm package and entry point -# COPY qq/ ./qq/ -# COPY run_qq.py ./ - # Copy package metadata and source COPY pyproject.toml ./ @@ -55,6 +52,9 @@ ARG GIT_DESCRIBE=unknown ENV QQ_GIT_COMMIT=${GIT_COMMIT} ENV QQ_GIT_DESCRIBE=${GIT_DESCRIBE} +LABEL maintainer="SWOT-Confluence" +LABEL description="SWOT-Confluence QQ discharge estimation algorithm: reach-based quantile-quantile mapping" + LABEL org.opencontainers.image.revision=${GIT_COMMIT} LABEL org.opencontainers.image.version=${GIT_DESCRIBE} diff --git a/README.md b/README.md index 35a5a9e..dc361cb 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,29 @@ pytest tests/ -v --- +## Deployment + +### Docker / AWS Batch + +```bash +bash deploy/deploy.sh +``` + +This builds the Docker image (injecting `GIT_COMMIT` and `GIT_DESCRIBE` as +build-args for provenance), pushes it to ECR, and applies the Terraform +configuration in `terraform/`. The Terraform state bucket and AWS credentials +must be configured in advance. See `deploy/deploy.sh` and `terraform/` for +full details. + +### HPC / run-confluence-locally + +QQ runs via +[run-confluence-locally](https://github.com/SWOT-Confluence/run-confluence-locally). +Add `qq` to `modules_to_run` in your configuration YAML. The Apptainer SIF +image is built automatically from this repository's `Dockerfile`. + +--- + ## Documentation Detailed QQ Project documentation is available in: diff --git a/_other_modules/run-confluence-locally/config.py b/_other_modules/run-confluence-locally/config.py new file mode 100644 index 0000000..f70984b --- /dev/null +++ b/_other_modules/run-confluence-locally/config.py @@ -0,0 +1,19 @@ +# To be implemented in Repository: "run-confluence-locally": +# Modification in existing file: +# confluence/utils/config.py + + +FLPE_MODULES: ClassVar[set[str]] = { + "metroman", + "metroman_consolidation", + "unconstrained_momma", + "busboi", + "sad", + "hivdi", + "sic4dvar", + "consensus", + "qq", +} + + + diff --git a/_other_modules/run-confluence-locally/dir_structure.py b/_other_modules/run-confluence-locally/dir_structure.py new file mode 100644 index 0000000..52e7dd3 --- /dev/null +++ b/_other_modules/run-confluence-locally/dir_structure.py @@ -0,0 +1,24 @@ +# To be implemented in Repository: "run-confluence-locally": +# Modification in existing file: +# confluence/utils/dir_structure.py + + +# in _create_directory_structure, modify the mnt_dir_list: + + + mnt_dir_list = [ + "diagnostics/prediagnostics", + "diagnostics/postdiagnostics/basin", + "diagnostics/postdiagnostics/reach", + "flpe/busboi", + "flpe/consensus", + "flpe/hivdi", + "flpe/metroman/sets", + "flpe/momma", + "flpe/qq", # ← ADD THIS LINE + "flpe/sad", + "flpe/sic4dvar", + ... + ] + + diff --git a/_other_modules/run-confluence-locally/module_names.py b/_other_modules/run-confluence-locally/module_names.py new file mode 100644 index 0000000..c0a82c0 --- /dev/null +++ b/_other_modules/run-confluence-locally/module_names.py @@ -0,0 +1,36 @@ +# To be implemented in Repository: "run-confluence-locally": +# Modification in existing file: +# confluence/utils/module_names.py + + + + +# get_repo_name("qq") returns "qq" (lowercase), +# but the GitHub repository is SWOT-Confluence/QQ (uppercase). +# The _clone_worker in module_images.py uses get_repo_name(name) to form the git clone URL: + +# url = f"https://github.com/{github_name}/{repo_name}.git" + +# With repo_name = "qq", the clone would target SWOT-Confluence/qq.git, +# which does not exist. GitHub clone will fail silently or with a 404. + +# one entry must be added to REPO_NAME_MAP: + +REPO_NAME_MAP = { + "offline": "offline-discharge-data-product-creation", + "moi": "MOI", + "validation": "Validation", + "hivdi": "h2ivdi", + "busboi": "BUSBOI", + "lakeflow": "LakeFlow_Confluence", + "qq": "QQ", +} + + +# No change to IMAGE_NAME_MAP is needed — the SIF and image will be named qq (lowercase), which is correct and consistent with the {{ sif_dir }}/qq.sif reference in the template above. + + + + + + diff --git a/_other_modules/run-confluence-locally/qq.sh.j2 b/_other_modules/run-confluence-locally/qq.sh.j2 new file mode 100644 index 0000000..0a027b4 --- /dev/null +++ b/_other_modules/run-confluence-locally/qq.sh.j2 @@ -0,0 +1,24 @@ +# To be implemented in Repository: "run-confluence-locally": +# NEW FILE: +# confluence/templates/modules/qq.sh.j2 + + +START_INDEX=$(( ${OFFSET:-0} + SLURM_ARRAY_TASK_ID * ${INDEX_RANGE:-1} )) +END_INDEX=$(( START_INDEX + ${INDEX_RANGE:-1} - 1 )) + +if [[ -n "$MAX_LIMIT" ]]; then + (( END_INDEX >= MAX_LIMIT )) && END_INDEX=$(( MAX_LIMIT - 1 )) +fi + +for (( idx=START_INDEX; idx<=END_INDEX; idx++ )); do + {{ container_cmd.run }} run \ + {{ container_cmd.bind }} {{ mnt_dir }}/input:/mnt/data/input \ + {{ container_cmd.bind }} {{ mnt_dir }}/flpe/qq:/mnt/data/flpe/qq \ + {{ optional_binds | join(' \\\n ') }} \ + {{ sif_dir }}/qq.sif \ + /mnt/data/input/reaches.json \ + --input_dir /mnt/data/input \ + --output_dir /mnt/data/flpe/qq \ + --mode RUN \ + -i ${idx} +done \ No newline at end of file diff --git a/documentations/ARCHITECTURE.md b/documentations/ARCHITECTURE.md index 5e11cd3..fbfe3ad 100644 --- a/documentations/ARCHITECTURE.md +++ b/documentations/ARCHITECTURE.md @@ -13,8 +13,7 @@ scientific algorithm remains the v1.0.0 method. ## 1. High-level architecture -QQ is a **single-reach package**, written for SWOT-Confluence pipeline. v1.0.0 is written as QQ is as a FLPE Algorithm within the pipeline. One invocation selects one reach from `reaches.json`, reads its SWOT observations and SOS flow-duration-curve -prior, performs quantile matching, and writes one QQ NetCDF product. +QQ is a **single-reach package** written for the SWOT-Confluence pipeline. In v1.0.0, QQ operates as a FLPE Algorithm within the pipeline. One invocation selects one reach from `reaches.json`, reads its SWOT observations and SOS flow-duration-curve prior, performs quantile matching, and writes one QQ NetCDF product. ```mermaid flowchart LR diff --git a/documentations/CHANGELOG.md b/documentations/CHANGELOG.md index a55b659..d7730dd 100644 --- a/documentations/CHANGELOG.md +++ b/documentations/CHANGELOG.md @@ -80,6 +80,10 @@ This file records user-visible and scientifically relevant changes between QQ ve - No quantile-matching change. - No FDC-processing change. - No change to the scientific NetCDF variables, dimensions, groups, flags, or fill values. +- Corrected text descriptions of the WSE-quantile resampling flag and the WSE-Q + lookup-table resampling flag in `qq/constants.py`; the downsampled/upsampled + direction labels were textually inverted in v1.0.0. No numeric flag value, + algorithm behavior, or output array changed. **Compatibility** diff --git a/qq/output_netcdf.py b/qq/output_netcdf.py index 35f7a4f..73da392 100644 --- a/qq/output_netcdf.py +++ b/qq/output_netcdf.py @@ -102,15 +102,14 @@ def output_paths(config: QQConfig, state: QQState) -> None: # Write directly into output_dir — no extra qq/ subfolder. # # Production mount convention (run-confluence-locally j2 template): - # --bind mnt_dir/flpe/qq:/mnt/data/output (or --bind .../flpe/qq:/mnt/data/flpe/qq) - # --output_dir /mnt/data/output (or /mnt/data/flpe/qq) + # --bind mnt_dir/flpe/qq:/mnt/data/flpe/qq + # --output_dir /mnt/data/flpe/qq # # Files land at: # /_qq.nc # /logs/_qq.log - # - # This avoids double-nesting (/qq/_qq.nc) that would - # result if the j2 template already binds the qq-specific directory. + + output_path = state.output_dir output_path.mkdir(parents=True, exist_ok=True)