diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cdd1f08..f40b51b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -83,7 +83,15 @@ jobs: CONFIG_URL="https://${{ vars.PROJECT }}-config.eoapi.dev" echo "Fetching database credentials from: $CONFIG_URL" - CONFIG_RESPONSE=$(curl -s -H "Authorization: Bearer ${{ vars.WORKSHOP_TOKEN }}" "$CONFIG_URL") + # -f so an auth failure surfaces here rather than as a null jq result and a + # confusing psql error several lines later. The usual cause is the + # WORKSHOP_TOKEN variable disagreeing with the token the config Lambda holds: + # either it is unset (config.py then generates one nobody knows) or it was + # rotated in the variable without a deploy to carry it through. + if ! CONFIG_RESPONSE=$(curl -sf -H "Authorization: Bearer ${{ vars.WORKSHOP_TOKEN }}" "$CONFIG_URL"); then + echo "::error::Could not read database credentials from $CONFIG_URL. Check that the WORKSHOP_TOKEN variable is set and matches the deployed config Lambda." + exit 1 + fi export PGHOST=$(echo "$CONFIG_RESPONSE" | jq -r '.pghost') export PGPORT=$(echo "$CONFIG_RESPONSE" | jq -r '.pgport') diff --git a/.github/workflows/reset-data.yml b/.github/workflows/reset-data.yml new file mode 100644 index 0000000..985aefc --- /dev/null +++ b/.github/workflows/reset-data.yml @@ -0,0 +1,149 @@ +name: Reset Workshop Data + +# Removes the STAC collections that workshop participants created, leaving only the +# fixtures that `CDK Deploy` loads. Does not touch the stack, the database itself, or the +# `features.ecoregions` table -- for those, see `CDK Destroy` and `CDK Deploy`. +# +# Defaults to a dry run. Applying requires typing the project name. + +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or tag to use" + required: true + default: "main" + type: string + environment: + description: "Deployment environment name" + required: true + default: "dev" + type: string + mode: + description: "dry-run lists what would be deleted; apply deletes it" + required: true + default: "dry-run" + type: choice + options: + - dry-run + - apply + confirm: + description: "To apply, type the PROJECT name exactly. Ignored for dry-run." + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + reset: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref }} + + - name: Confirm destructive run + if: inputs.mode == 'apply' + env: + CONFIRM: ${{ inputs.confirm }} + PROJECT: ${{ vars.PROJECT }} + run: | + if [ "$CONFIRM" != "$PROJECT" ]; then + echo "::error::mode=apply requires 'confirm' to be exactly '$PROJECT'." + exit 1 + fi + echo "Confirmed: deleting non-fixture collections from '$PROJECT'." + + - name: Install PostgreSQL client + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client + + - name: Reset collections + env: + MODE: ${{ inputs.mode }} + CONFIG_URL: https://${{ vars.PROJECT }}-config.eoapi.dev + WORKSHOP_TOKEN: ${{ vars.WORKSHOP_TOKEN }} + run: | + set -euo pipefail + + # A rotated WORKSHOP_TOKEN that has not been deployed yet shows up here as a + # 401, because the variable has the new value while the config Lambda still + # holds the old one. Say so, rather than failing on a bare curl exit code. + if ! CONFIG_RESPONSE=$(curl -sf -H "Authorization: Bearer $WORKSHOP_TOKEN" "$CONFIG_URL"); then + echo "::error::Could not read database credentials from $CONFIG_URL. If WORKSHOP_TOKEN was just rotated, run CDK Deploy first so the config Lambda picks up the new value." + exit 1 + fi + export PGHOST=$(echo "$CONFIG_RESPONSE" | jq -r '.pghost') + export PGPORT=$(echo "$CONFIG_RESPONSE" | jq -r '.pgport') + export PGDATABASE=$(echo "$CONFIG_RESPONSE" | jq -r '.pgdatabase') + export PGUSER=$(echo "$CONFIG_RESPONSE" | jq -r '.pguser') + export PGPASSWORD=$(echo "$CONFIG_RESPONSE" | jq -r '.pgpassword') + echo "Connected to $PGDATABASE on $PGHOST" + + # Collections to preserve, discovered from data/ plus the remote fixtures. + python3 scripts/fixture_collections.py > /tmp/keep.txt + echo "Fixture collections to keep:" + sed 's/^/ /' /tmp/keep.txt + + # Anything in the catalog that is not a fixture. Subtracted client-side rather + # than with a TEMP table joined in SQL: the database is reached through + # pgbouncer in transaction pooling mode, where temp tables leak between pooled + # backends and a second run fails with `relation "keep" already exists`. + psql -X -Atq -v ON_ERROR_STOP=1 \ + -c "SELECT id FROM pgstac.collections ORDER BY 1;" > /tmp/all.txt + # grep exits 1 when nothing is selected, which is a valid "no strays" result. + grep -Fxv -f /tmp/keep.txt /tmp/all.txt > /tmp/strays.txt || true + + COUNT=$(wc -l < /tmp/strays.txt | tr -d ' ') + echo "Non-fixture collections: $COUNT" + sed 's/^/ /' /tmp/strays.txt + + { + echo "## Reset Workshop Data (${MODE})" + echo "- fixtures kept: $(wc -l < /tmp/keep.txt | tr -d ' ')" + echo "- non-fixture collections found: $COUNT" + } >> "$GITHUB_STEP_SUMMARY" + + if [ "$MODE" != "apply" ]; then + echo "Dry run: nothing deleted. Re-run with mode=apply to delete these." + exit 0 + fi + + if [ "$COUNT" -eq 0 ]; then + echo "Nothing to delete." + exit 0 + fi + + while IFS= read -r cid; do + [ -z "$cid" ] && continue + echo "deleting $cid" + # Items first: pgstac.delete_collection only removes the collection row, so + # do not rely on a cascade to clear the collection's items. + psql -X -q -v ON_ERROR_STOP=1 -v cid="$cid" \ + -c "DELETE FROM pgstac.items WHERE collection = :'cid';" + psql -X -q -v ON_ERROR_STOP=1 -v cid="$cid" \ + -c "SELECT pgstac.delete_collection(:'cid');" + done < /tmp/strays.txt + + REMAINING=$(psql -X -Atq -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pgstac.collections;") + ORPHANS=$(psql -X -Atq -v ON_ERROR_STOP=1 \ + -c "SELECT count(*) FROM pgstac.items i + LEFT JOIN pgstac.collections c ON c.id = i.collection + WHERE c.id IS NULL;") + + echo "Deleted $COUNT collections. Collections remaining: $REMAINING. Orphaned items: $ORPHANS" + { + echo "- deleted: $COUNT" + echo "- collections remaining: $REMAINING" + echo "- orphaned items: $ORPHANS" + } >> "$GITHUB_STEP_SUMMARY" + + if [ "$ORPHANS" -ne 0 ]; then + echo "::warning::$ORPHANS item(s) reference a collection that no longer exists." + fi diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 2f3dc85..bf144f2 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -72,7 +72,7 @@ The easiest way to deploy is using the GitHub Actions workflow, which automatica - `VPC_ID` - **Required** - VPC ID where resources will be deployed - `HOSTED_ZONE_ID` - **Required** - Route53 hosted zone ID for `eoapi.dev` domain - `CERTIFICATE_ARN` - **Required** - ACM certificate ARN for `*.eoapi.dev` wildcard certificate - - `WORKSHOP_TOKEN` - Bearer token for workshop config (optional, auto-generated if not provided) + - `WORKSHOP_TOKEN` - **Required** - Bearer token for the workshop config Lambda. `config.py` will generate one when this is unset, but the value never reaches you: the deploy's own data-loading step and the **Reset Workshop Data** workflow both authenticate with this variable, so an unset variable means they send an empty token to a Lambda holding a generated one, and a freshly generated token on every deploy - `PGSTAC_VERSION` - pgstac version (optional, defaults to `0.9.8`) 3. **IAM Role Setup** @@ -300,3 +300,52 @@ docker run --rm ghcr.io/osgeo/gdal:alpine-small-latest ogr2ogr -f "PostgreSQL" \ ``` Once loaded, this data persists in the database and is available for all workshop variants. + +## Resetting the Workshop Database + +Participants create collections as they work through the notebooks, so the catalog +accumulates between workshops. The **Reset Workshop Data** workflow deletes every STAC +collection that is not a fixture, returning the catalog to its freshly-deployed state. + +A *fixture* is any collection the **CDK Deploy** workflow loads. Everything else is a +participant's leftover. The keep-list is derived at run time rather than maintained by +hand — `scripts/fixture_collections.py` discovers collection ids from `data/*.json` and +`data/*.ndjson`, plus the ids in its `REMOTE_FIXTURES` set for fixtures fetched from a +remote STAC API. To see the current list: + +```bash +python3 scripts/fixture_collections.py --check +``` + +### Running it + +Actions → **Reset Workshop Data** → Run workflow: + +- **ref** — the branch or tag to use. Use the same one you deployed; see the warning below +- **environment** — `dev` +- **mode** — `dry-run` (the default) lists what would be deleted without touching + anything; `apply` deletes it +- **confirm** — required for `apply`: type the `PROJECT` name exactly, or the run fails + +Always dry-run first and read the list. Deletion is irreversible; anything a participant +wants to keep should be exported beforehand. + +> [!WARNING] +> **Run it with the same `ref` you deployed.** The keep-list comes from the checked out +> `data/` directory, so running from a branch that is missing a fixture file will classify +> that fixture as a leftover and delete it. + +### What it does not touch + +- **The stack, database, or credentials.** To tear those down, use **CDK Destroy**. To + rotate `WORKSHOP_TOKEN`, see [When to Redeploy the Stack](#when-to-redeploy-the-stack) — + update the variable and run **CDK Deploy**. Do that *before* resetting: this workflow + authenticates to the config Lambda with `WORKSHOP_TOKEN`, so between changing the + variable and deploying, the variable and the Lambda disagree and the run fails. +- **The `features.ecoregions` table** behind the vector API. +- **Fixtures themselves.** They are upserted on every **CDK Deploy**, so re-running the + deploy restores any fixture that was removed by hand. + +Because fixtures are re-upserted on deploy and preserved by the reset, the two workflows +compose: **Reset Workshop Data** returns the catalog to fixtures only, and **CDK Deploy** +puts back anything missing. diff --git a/scripts/fixture_collections.py b/scripts/fixture_collections.py new file mode 100644 index 0000000..47d994e --- /dev/null +++ b/scripts/fixture_collections.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Print the STAC collection ids that a database reset must preserve. + +The "fixtures" are whatever the Load Workshop Data step of `.github/workflows/deploy.yml` +puts in the database. Everything else in the catalog is a workshop participant's leftover +and is what `reset-data.yml` removes. + +Most fixtures come from files in `data/`, so they are discovered rather than listed -- +adding a fixture file is enough, with nothing to keep in sync here. Fixtures fetched from +a remote STAC API have no local file, so those ids are listed in REMOTE_FIXTURES below. + +Usage: + python scripts/fixture_collections.py # one id per line + python scripts/fixture_collections.py --check # also report where each came from +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = REPO_ROOT / "data" + +# Fixtures loaded from a remote STAC API, so there is no file in data/ to discover them +# from. Keep in step with the `pypgstac load` calls in .github/workflows/deploy.yml. +REMOTE_FIXTURES = { + "glad-global-forest-change-1.11", # fetched from https://stac.maap-project.org +} + + +def _records(path: Path): + """Yield the JSON objects in a .json or newline-delimited .ndjson file.""" + text = path.read_text().strip() + if not text: + return + if path.suffix == ".ndjson": + for line in text.splitlines(): + line = line.strip() + if line: + yield json.loads(line) + return + parsed = json.loads(text) + if isinstance(parsed, list): + yield from parsed + else: + yield parsed + + +def discover(data_dir: Path = DATA_DIR) -> dict[str, str]: + """Map collection id -> where it was found.""" + found: dict[str, str] = {cid: "deploy.yml (remote)" for cid in REMOTE_FIXTURES} + + for path in sorted(data_dir.glob("*.json")) + sorted(data_dir.glob("*.ndjson")): + for record in _records(path): + if not isinstance(record, dict): + continue + # A Collection names itself; an Item names the collection it belongs to. + if record.get("type") == "Collection" and record.get("id"): + found.setdefault(record["id"], path.name) + elif record.get("collection"): + found.setdefault(record["collection"], path.name) + + return found + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", action="store_true", help="show where each id was found" + ) + args = parser.parse_args() + + found = discover() + if not found: + print("no fixture collections discovered", file=sys.stderr) + return 1 + + for cid in sorted(found): + print(f"{cid}\t{found[cid]}" if args.check else cid) + return 0 + + +def demo() -> None: + """Self-check: `python scripts/fixture_collections.py --selftest`.""" + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) + (d / "c.json").write_text(json.dumps({"type": "Collection", "id": "fix-a"})) + (d / "i.ndjson").write_text( + json.dumps({"type": "Feature", "id": "x", "collection": "fix-b"}) + + "\n" + + json.dumps({"type": "Feature", "id": "y", "collection": "fix-b"}) + + "\n" + ) + (d / "empty.ndjson").write_text("\n") + + found = discover(d) + + assert "fix-a" in found, found + assert "fix-b" in found, found + assert REMOTE_FIXTURES <= set(found), found + # Nothing invented: only what the files and REMOTE_FIXTURES name. + assert set(found) == {"fix-a", "fix-b"} | REMOTE_FIXTURES, found + print("fixture_collections: all checks passed") + + +if __name__ == "__main__": + if "--selftest" in sys.argv: + demo() + else: + raise SystemExit(main())