Docs/update documentation

.gitattributes

@@ -0,0 +1,11 @@
+* text=auto
+
+*.py text eol=lf
+*.sh text eol=lf
+*.yml text eol=lf
+*.yaml text eol=lf
+*.toml text eol=lf
+*.tf text eol=lf
+*.md text eol=lf
+*.txt text eol=lf
+Dockerfile text eol=lf

.github/workflows/release.yml

@@ -13,6 +13,31 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Verify release tag matches package version
+        shell: bash
+        run: |
+          PACKAGE_VERSION=$(python - <<'PY'
+          import pathlib
+          import tomllib
+
+          data = tomllib.loads(
+              pathlib.Path("pyproject.toml").read_text(encoding="utf-8")
+          )
+
+          print(data["project"]["version"])
+          PY
+          )
+
+          if [ "v${PACKAGE_VERSION}" != "${GITHUB_REF_NAME}" ]; then
+            echo "ERROR: tag ${GITHUB_REF_NAME} does not match pyproject version v${PACKAGE_VERSION}"
+            exit 1
+          fi
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -26,12 +51,17 @@ jobs:
 
       - name: Build and push Docker image
         env:
-          REGISTRY:   ${{ steps.login-ecr.outputs.registry }}
-          REPOSITORY: swot-confluence-qq
-          IMAGE_TAG:  ${{ github.ref_name }}
+          REGISTRY:    ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY:  swot-confluence-qq
+          IMAGE_TAG:   ${{ github.ref_name }}
+          GIT_COMMIT:  ${{ github.sha }}
         run: |
-          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
-          docker tag  $REGISTRY/$REPOSITORY:$IMAGE_TAG $REGISTRY/$REPOSITORY:latest
+          docker build \
+            --build-arg GIT_COMMIT=$GIT_COMMIT \
+            --build-arg GIT_DESCRIBE=$IMAGE_TAG \
+            -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
+
+          docker tag $REGISTRY/$REPOSITORY:$IMAGE_TAG $REGISTRY/$REPOSITORY:latest
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
           docker push $REGISTRY/$REPOSITORY:latest
 
@@ -52,4 +82,4 @@ jobs:
             -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" \
             -backend-config="key=${{ secrets.CONFLUENCE_PREFIX }}/qq/terraform.tfstate" \
             -backend-config="region=${{ secrets.AWS_REGION }}"
-          terraform apply -auto-approve
+          terraform apply -auto-approve

.github/workflows/test.yml

@@ -22,6 +22,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          pip install -e .
           pip install pytest pytest-cov
 
       - name: Run tests

Dockerfile

@@ -16,10 +16,9 @@
 #     -v /path/to/mnt/flpe/qq:/mnt/data/flpe/qq \
 #     qq:latest /mnt/data/input/reaches.json --index 0
 
-FROM python:3.11-slim AS base
+# FROM python:3.11-slim AS base
+FROM python:3.11-slim
 
-LABEL maintainer="SWOT-Confluence"
-LABEL description="QQ FLPE algorithm: quantile-quantile WSE-to-discharge mapping"
 
 WORKDIR /app
 
@@ -28,13 +27,38 @@ COPY requirements.txt ./
 RUN pip install --no-cache-dir --upgrade pip \
  && pip install --no-cache-dir -r requirements.txt
 
-# Copy the algorithm package and entry point
+
+# Copy package metadata and source
+COPY pyproject.toml ./
+COPY README.md ./
 COPY qq/ ./qq/
 COPY run_qq.py ./
 
+# Install QQ itself so Python package metadata is available at runtime.
+RUN pip install --no-cache-dir --no-deps .
+
+
+
+
 # Default production paths are baked into config.py as constants;
 # they can be overridden at runtime via --input_dir / --output_dir.
 ENV PYTHONUNBUFFERED=1
 
+
+# Git provenance is injected by the release workflow.
+ARG GIT_COMMIT=unknown
+ARG GIT_DESCRIBE=unknown
+
+ENV QQ_GIT_COMMIT=${GIT_COMMIT}
+ENV QQ_GIT_DESCRIBE=${GIT_DESCRIBE}
+
+LABEL maintainer="SWOT-Confluence"
+LABEL description="SWOT-Confluence QQ discharge estimation algorithm: reach-based quantile-quantile mapping"
+
+LABEL org.opencontainers.image.revision=${GIT_COMMIT}
+LABEL org.opencontainers.image.version=${GIT_DESCRIBE}
+
+
+
 ENTRYPOINT ["python", "run_qq.py"]
 CMD ["--help"]

README.md

@@ -16,8 +16,6 @@ A reach-based Discharge Estimation algorithm for the [SWOT-Confluence](https://g
 - [Repository Structure](#repository-structure)
 - [Configuration](#configuration)
 - [Testing](#testing)
-- [Deployment](#deployment)
-- [SWOT-Confluence Integration](#swot-confluence-integration)
 - [Documentation](#documentation)
 - [Contributing](#contributing)
 - [License](#license)
@@ -31,20 +29,21 @@ QQ (Quantile-Quantile) matching is a statistical, prior-based discharge estimati
 
 1. Ranks clean, quality-filtered SWOT WSE observations to build an **empirical non-exceedance probability curve**.
 2. Resamples that curve onto a standardized **deliverable probability grid** for NetCDF output.
-3. Maps each observation's probability to a discharge value using the **SOS Flow Duration Curve** at the same probability level.
+3. Maps each observation's probability to a discharge value using the **SOS Flow Duration Curve** at the same probability level, and creates a semi-rating-curve.
 4. Writes a per-reach NetCDF discharge time series, always — including a fill-value file for reaches that fail quality gates.
 
 Unlike hydraulic FLPE algorithms, QQ requires no channel geometry, no Manning's roughness coefficient, and no rating-curve calibration. It depends only on the assumption that WSE rank approximates discharge rank over the observation period, and that the SOS FDC is representative of that period.
 
-<!-- See [`docs/methodology.md`](docs/methodology.md) for the full scientific description. -->
+See [`documentations/METHODOLOGY_v1.0.0.md`](documentations/METHODOLOGY_v1.0.0.md) for the full scientific description of the initial release (version 1.0.0). The changes records are registered in [`documentations/CHANGELOG.md`](documentations/CHANGELOG.md). the versioning naming convention definition is found in [`documentations/VERSIONING.md`](documentations/VERSIONING.md)
 
 ---
 
 ## Requirements
 
-- Python **3.11+**
+- Python **>= 3.10**
 - `netCDF4`, `numpy`, `pandas` (see [`requirements.txt`](requirements.txt))
 - Git
+- Production Docker/CI currently use Python **3.11**
 
 ---
 
@@ -73,7 +72,7 @@ python -m pip install --upgrade pip
 pip install -r requirements.txt
 ```
 
-### 5. Editable install (optional, for development)
+### 5. Install QQ
 ```bash
 python -m pip install -e .
 ```
@@ -135,7 +134,7 @@ pytest tests/ -v
 ├── swot/
 │   └── <reach_id>_SWOT.nc
 └── sos/
-    └── <continent>_SOS.nc
+    └── <continent>_sword_v17c_SOS_priors.nc
 ```
 
 **`reaches.json`** — one entry per reach:
@@ -145,7 +144,7 @@ pytest tests/ -v
     "reach_id": "21101200141",
     "swot":  "21101200141_SWOT.nc",
     "sos":   "eu_sword_v17c_SOS_priors.nc",
-    "sword": "eu_sword_v17b.nc"
+    "sword": "eu_sword_v17c.nc"
   }
 ]
 ```
@@ -186,7 +185,7 @@ Group "lookup_table"
 Fill / missing values: `f8` → `-999999999999.0`, `i2` flags → `-999`, `i4` scalars → `-999999999`.
 (The WSE-Q lookup table reuses these same values — no new fill-value convention was introduced.)
 
-Full schema in [`docs/architecture.md`](docs/architecture.md).
+Full schema in [`documentations/ARCHITECTURE.md`](documentations/ARCHITECTURE.md).
 
 ---
 
@@ -202,7 +201,7 @@ Options:
   -i, --index INT       0-based reach index (default: 0)
                         Overridden by $AWS_BATCH_JOB_ARRAY_INDEX
   --input_dir DIR       Root input directory (default: /mnt/data/input)
-  --output_dir DIR      Output directory (default: /mnt/data/output)
+  --output_dir DIR      Output directory (default: /mnt/data/flpe/qq)
   --mode {RUN,DEBUG,AUDIT}
                         RUN   = production; errors non-fatal, fill-value NC written
                         DEBUG = raises immediately on error
@@ -229,15 +228,16 @@ QQ/
 │   ├── input_json.py             Manifest reading, path resolution
 │   ├── input_swot.py             SWOT read → clean → filter → gate
 │   ├── input_sos.py              SOS FDC extraction
+│   ├── metadata.py
+│   ├── lookup_table.py
 │   ├── wse_quantile.py           Empirical + deliverable WSE quantile
 │   ├── quantile_matching.py      Core WSE → probability → discharge
 │   ├── output_arrays.py          Final output array preparation
 │   ├── output_netcdf.py          NetCDF writer + log saver
 │   ├── diagnostics.py            Optional Plotly plots
 │   └── pipeline.py               Orchestrator
-├── tests/                       60 automated tests
-├── docs/                        Full documentation suite
-├── confluence/templates/modules/qq.sh.j2   run-confluence-locally SLURM template
+├── tests/                        Automated test suite
+├── documentations/            Full documentation suite
 ├── deploy/deploy.sh              5-argument deploy script
 ├── terraform/                    AWS Batch + ECR infrastructure
 ├── .github/workflows/            CI (test.yml) + CD (release.yml)
@@ -267,6 +267,8 @@ Runtime settings (paths, index, mode) are resolved separately in `qq/config.py`
 
 ## Testing
 
+Run the full automated test suite with:
+
 ```bash
 pytest tests/ -v
 ```
@@ -277,10 +279,59 @@ pytest tests/ -v
 | `test_helpers.py` | Interpolation utilities |
 | `test_pipeline.py` | End-to-end pipeline, valid + invalid reach, CLI exit codes |
 
-60 tests total, all passing on Python 3.11+.
+---
+
+## Deployment
+
+### Docker / AWS Batch
+
+```bash
+bash deploy/deploy.sh <registry> <repository> <prefix> <s3_state_bucket> <profile>
+```
+
+This builds the Docker image (injecting `GIT_COMMIT` and `GIT_DESCRIBE` as
+build-args for provenance), pushes it to ECR, and applies the Terraform
+configuration in `terraform/`. The Terraform state bucket and AWS credentials
+must be configured in advance. See `deploy/deploy.sh` and `terraform/` for
+full details.
+
+### HPC / run-confluence-locally
+
+QQ runs via
+[run-confluence-locally](https://github.com/SWOT-Confluence/run-confluence-locally).
+Add `qq` to `modules_to_run` in your configuration YAML. The Apptainer SIF
+image is built automatically from this repository's `Dockerfile`.
+
+---
+
+## Documentation
+
+Detailed QQ Project documentation is available in:
+
+- [`documentations/METHODOLOGY_v1.0.0.md`](documentations/METHODOLOGY_v1.0.0.md)
+- [`documentations/ARCHITECTURE.md`](documentations/ARCHITECTURE.md)
+- [`documentations/VERSIONING.md`](documentations/VERSIONING.md)
+- [`documentations/CHANGELOG.md`](documentations/CHANGELOG.md)
+
+
+---
+
+## Contributing
+
+Changes should normally be developed on a dedicated branch, tested with the
+full automated test suite, and merged into `main` after review.
 
 ---
 
+## Maintainers
+
+Canonical project authorship and maintainer information and repository metadata are declared in
+[`pyproject.toml`](pyproject.toml).
+
+Repository ownership and access are managed through the
+[SWOT-Confluence GitHub organization](https://github.com/SWOT-Confluence).
+
+---
 
 ## License
 

_other_modules/run-confluence-locally/config.py

@@ -0,0 +1,19 @@
+# To be implemented in Repository: "run-confluence-locally":
+# Modification in existing file:
+# confluence/utils/config.py
+
+
+FLPE_MODULES: ClassVar[set[str]] = {
+    "metroman",
+    "metroman_consolidation",
+    "unconstrained_momma",
+    "busboi",
+    "sad",
+    "hivdi",
+    "sic4dvar",
+    "consensus",
+    "qq",
+}
+
+
+

_other_modules/run-confluence-locally/dir_structure.py

@@ -0,0 +1,24 @@
+# To be implemented in Repository: "run-confluence-locally":
+# Modification in existing file:
+# confluence/utils/dir_structure.py
+
+
+# in _create_directory_structure, modify the mnt_dir_list:
+
+
+    mnt_dir_list = [
+        "diagnostics/prediagnostics",
+        "diagnostics/postdiagnostics/basin",
+        "diagnostics/postdiagnostics/reach",
+        "flpe/busboi",
+        "flpe/consensus",
+        "flpe/hivdi",
+        "flpe/metroman/sets",
+        "flpe/momma",
+        "flpe/qq",          # ← ADD THIS LINE
+        "flpe/sad",
+        "flpe/sic4dvar",
+        ...
+    ]
+
+

_other_modules/run-confluence-locally/module_names.py

@@ -0,0 +1,36 @@
+# To be implemented in Repository: "run-confluence-locally":
+# Modification in existing file:
+# confluence/utils/module_names.py
+
+
+
+
+# get_repo_name("qq") returns "qq" (lowercase),
+# but the GitHub repository is SWOT-Confluence/QQ (uppercase).
+# The _clone_worker in module_images.py uses get_repo_name(name) to form the git clone URL:
+
+# url = f"https://github.com/{github_name}/{repo_name}.git"
+
+# With repo_name = "qq", the clone would target SWOT-Confluence/qq.git,
+# which does not exist. GitHub clone will fail silently or with a 404.
+
+# one entry must be added to REPO_NAME_MAP:
+
+REPO_NAME_MAP = {
+    "offline": "offline-discharge-data-product-creation",
+    "moi": "MOI",
+    "validation": "Validation",
+    "hivdi": "h2ivdi",
+    "busboi": "BUSBOI",
+    "lakeflow": "LakeFlow_Confluence",
+    "qq": "QQ",
+}
+
+
+# No change to IMAGE_NAME_MAP is needed — the SIF and image will be named qq (lowercase), which is correct and consistent with the {{ sif_dir }}/qq.sif reference in the template above.
+
+
+
+
+
+