Add a workflow to reset the deployed catalog to fixtures

.github/workflows/deploy.yml

@@ -83,7 +83,15 @@ jobs:
           CONFIG_URL="https://${{ vars.PROJECT }}-config.eoapi.dev"
 
           echo "Fetching database credentials from: $CONFIG_URL"
-          CONFIG_RESPONSE=$(curl -s -H "Authorization: Bearer ${{ vars.WORKSHOP_TOKEN }}" "$CONFIG_URL")
+          # -f so an auth failure surfaces here rather than as a null jq result and a
+          # confusing psql error several lines later. The usual cause is the
+          # WORKSHOP_TOKEN variable disagreeing with the token the config Lambda holds:
+          # either it is unset (config.py then generates one nobody knows) or it was
+          # rotated in the variable without a deploy to carry it through.
+          if ! CONFIG_RESPONSE=$(curl -sf -H "Authorization: Bearer ${{ vars.WORKSHOP_TOKEN }}" "$CONFIG_URL"); then
+            echo "::error::Could not read database credentials from $CONFIG_URL. Check that the WORKSHOP_TOKEN variable is set and matches the deployed config Lambda."
+            exit 1
+          fi
 
           export PGHOST=$(echo "$CONFIG_RESPONSE" | jq -r '.pghost')
           export PGPORT=$(echo "$CONFIG_RESPONSE" | jq -r '.pgport')

.github/workflows/reset-data.yml

@@ -0,0 +1,149 @@
+name: Reset Workshop Data
+
+# Removes the STAC collections that workshop participants created, leaving only the
+# fixtures that `CDK Deploy` loads. Does not touch the stack, the database itself, or the
+# `features.ecoregions` table -- for those, see `CDK Destroy` and `CDK Deploy`.
+#
+# Defaults to a dry run. Applying requires typing the project name.
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Branch or tag to use"
+        required: true
+        default: "main"
+        type: string
+      environment:
+        description: "Deployment environment name"
+        required: true
+        default: "dev"
+        type: string
+      mode:
+        description: "dry-run lists what would be deleted; apply deletes it"
+        required: true
+        default: "dry-run"
+        type: choice
+        options:
+          - dry-run
+          - apply
+      confirm:
+        description: "To apply, type the PROJECT name exactly. Ignored for dry-run."
+        required: false
+        default: ""
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  reset:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Confirm destructive run
+        if: inputs.mode == 'apply'
+        env:
+          CONFIRM: ${{ inputs.confirm }}
+          PROJECT: ${{ vars.PROJECT }}
+        run: |
+          if [ "$CONFIRM" != "$PROJECT" ]; then
+            echo "::error::mode=apply requires 'confirm' to be exactly '$PROJECT'."
+            exit 1
+          fi
+          echo "Confirmed: deleting non-fixture collections from '$PROJECT'."
+
+      - name: Install PostgreSQL client
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y postgresql-client
+
+      - name: Reset collections
+        env:
+          MODE: ${{ inputs.mode }}
+          CONFIG_URL: https://${{ vars.PROJECT }}-config.eoapi.dev
+          WORKSHOP_TOKEN: ${{ vars.WORKSHOP_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          # A rotated WORKSHOP_TOKEN that has not been deployed yet shows up here as a
+          # 401, because the variable has the new value while the config Lambda still
+          # holds the old one. Say so, rather than failing on a bare curl exit code.
+          if ! CONFIG_RESPONSE=$(curl -sf -H "Authorization: Bearer $WORKSHOP_TOKEN" "$CONFIG_URL"); then
+            echo "::error::Could not read database credentials from $CONFIG_URL. If WORKSHOP_TOKEN was just rotated, run CDK Deploy first so the config Lambda picks up the new value."
+            exit 1
+          fi
+          export PGHOST=$(echo "$CONFIG_RESPONSE" | jq -r '.pghost')
+          export PGPORT=$(echo "$CONFIG_RESPONSE" | jq -r '.pgport')
+          export PGDATABASE=$(echo "$CONFIG_RESPONSE" | jq -r '.pgdatabase')
+          export PGUSER=$(echo "$CONFIG_RESPONSE" | jq -r '.pguser')
+          export PGPASSWORD=$(echo "$CONFIG_RESPONSE" | jq -r '.pgpassword')
+          echo "Connected to $PGDATABASE on $PGHOST"
+
+          # Collections to preserve, discovered from data/ plus the remote fixtures.
+          python3 scripts/fixture_collections.py > /tmp/keep.txt
+          echo "Fixture collections to keep:"
+          sed 's/^/  /' /tmp/keep.txt
+
+          # Anything in the catalog that is not a fixture. Subtracted client-side rather
+          # than with a TEMP table joined in SQL: the database is reached through
+          # pgbouncer in transaction pooling mode, where temp tables leak between pooled
+          # backends and a second run fails with `relation "keep" already exists`.
+          psql -X -Atq -v ON_ERROR_STOP=1 \
+            -c "SELECT id FROM pgstac.collections ORDER BY 1;" > /tmp/all.txt
+          # grep exits 1 when nothing is selected, which is a valid "no strays" result.
+          grep -Fxv -f /tmp/keep.txt /tmp/all.txt > /tmp/strays.txt || true
+
+          COUNT=$(wc -l < /tmp/strays.txt | tr -d ' ')
+          echo "Non-fixture collections: $COUNT"
+          sed 's/^/  /' /tmp/strays.txt
+
+          {
+            echo "## Reset Workshop Data (${MODE})"
+            echo "- fixtures kept: $(wc -l < /tmp/keep.txt | tr -d ' ')"
+            echo "- non-fixture collections found: $COUNT"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ "$MODE" != "apply" ]; then
+            echo "Dry run: nothing deleted. Re-run with mode=apply to delete these."
+            exit 0
+          fi
+
+          if [ "$COUNT" -eq 0 ]; then
+            echo "Nothing to delete."
+            exit 0
+          fi
+
+          while IFS= read -r cid; do
+            [ -z "$cid" ] && continue
+            echo "deleting $cid"
+            # Items first: pgstac.delete_collection only removes the collection row, so
+            # do not rely on a cascade to clear the collection's items.
+            psql -X -q -v ON_ERROR_STOP=1 -v cid="$cid" \
+              -c "DELETE FROM pgstac.items WHERE collection = :'cid';"
+            psql -X -q -v ON_ERROR_STOP=1 -v cid="$cid" \
+              -c "SELECT pgstac.delete_collection(:'cid');"
+          done < /tmp/strays.txt
+
+          REMAINING=$(psql -X -Atq -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pgstac.collections;")
+          ORPHANS=$(psql -X -Atq -v ON_ERROR_STOP=1 \
+            -c "SELECT count(*) FROM pgstac.items i
+                 LEFT JOIN pgstac.collections c ON c.id = i.collection
+                WHERE c.id IS NULL;")
+
+          echo "Deleted $COUNT collections. Collections remaining: $REMAINING. Orphaned items: $ORPHANS"
+          {
+            echo "- deleted: $COUNT"
+            echo "- collections remaining: $REMAINING"
+            echo "- orphaned items: $ORPHANS"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ "$ORPHANS" -ne 0 ]; then
+            echo "::warning::$ORPHANS item(s) reference a collection that no longer exists."
+          fi

DEPLOYMENT.md

@@ -72,7 +72,7 @@ The easiest way to deploy is using the GitHub Actions workflow, which automatica
    - `VPC_ID` - **Required** - VPC ID where resources will be deployed
    - `HOSTED_ZONE_ID` - **Required** - Route53 hosted zone ID for `eoapi.dev` domain
    - `CERTIFICATE_ARN` - **Required** - ACM certificate ARN for `*.eoapi.dev` wildcard certificate
-   - `WORKSHOP_TOKEN` - Bearer token for workshop config (optional, auto-generated if not provided)
+   - `WORKSHOP_TOKEN` - **Required** - Bearer token for the workshop config Lambda. `config.py` will generate one when this is unset, but the value never reaches you: the deploy's own data-loading step and the **Reset Workshop Data** workflow both authenticate with this variable, so an unset variable means they send an empty token to a Lambda holding a generated one, and a freshly generated token on every deploy
    - `PGSTAC_VERSION` - pgstac version (optional, defaults to `0.9.8`)
 
 3. **IAM Role Setup**
@@ -300,3 +300,52 @@ docker run --rm ghcr.io/osgeo/gdal:alpine-small-latest ogr2ogr -f "PostgreSQL" \
 ```
 
 Once loaded, this data persists in the database and is available for all workshop variants.
+
+## Resetting the Workshop Database
+
+Participants create collections as they work through the notebooks, so the catalog
+accumulates between workshops. The **Reset Workshop Data** workflow deletes every STAC
+collection that is not a fixture, returning the catalog to its freshly-deployed state.
+
+A *fixture* is any collection the **CDK Deploy** workflow loads. Everything else is a
+participant's leftover. The keep-list is derived at run time rather than maintained by
+hand — `scripts/fixture_collections.py` discovers collection ids from `data/*.json` and
+`data/*.ndjson`, plus the ids in its `REMOTE_FIXTURES` set for fixtures fetched from a
+remote STAC API. To see the current list:
+
+```bash
+python3 scripts/fixture_collections.py --check
+```
+
+### Running it
+
+Actions → **Reset Workshop Data** → Run workflow:
+
+- **ref** — the branch or tag to use. Use the same one you deployed; see the warning below
+- **environment** — `dev`
+- **mode** — `dry-run` (the default) lists what would be deleted without touching
+  anything; `apply` deletes it
+- **confirm** — required for `apply`: type the `PROJECT` name exactly, or the run fails
+
+Always dry-run first and read the list. Deletion is irreversible; anything a participant
+wants to keep should be exported beforehand.
+
+> [!WARNING]
+> **Run it with the same `ref` you deployed.** The keep-list comes from the checked out
+> `data/` directory, so running from a branch that is missing a fixture file will classify
+> that fixture as a leftover and delete it.
+
+### What it does not touch
+
+- **The stack, database, or credentials.** To tear those down, use **CDK Destroy**. To
+  rotate `WORKSHOP_TOKEN`, see [When to Redeploy the Stack](#when-to-redeploy-the-stack) —
+  update the variable and run **CDK Deploy**. Do that *before* resetting: this workflow
+  authenticates to the config Lambda with `WORKSHOP_TOKEN`, so between changing the
+  variable and deploying, the variable and the Lambda disagree and the run fails.
+- **The `features.ecoregions` table** behind the vector API.
+- **Fixtures themselves.** They are upserted on every **CDK Deploy**, so re-running the
+  deploy restores any fixture that was removed by hand.
+
+Because fixtures are re-upserted on deploy and preserved by the reset, the two workflows
+compose: **Reset Workshop Data** returns the catalog to fixtures only, and **CDK Deploy**
+puts back anything missing.

scripts/fixture_collections.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Print the STAC collection ids that a database reset must preserve.
+
+The "fixtures" are whatever the Load Workshop Data step of `.github/workflows/deploy.yml`
+puts in the database. Everything else in the catalog is a workshop participant's leftover
+and is what `reset-data.yml` removes.
+
+Most fixtures come from files in `data/`, so they are discovered rather than listed --
+adding a fixture file is enough, with nothing to keep in sync here. Fixtures fetched from
+a remote STAC API have no local file, so those ids are listed in REMOTE_FIXTURES below.
+
+Usage:
+    python scripts/fixture_collections.py            # one id per line
+    python scripts/fixture_collections.py --check    # also report where each came from
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DATA_DIR = REPO_ROOT / "data"
+
+# Fixtures loaded from a remote STAC API, so there is no file in data/ to discover them
+# from. Keep in step with the `pypgstac load` calls in .github/workflows/deploy.yml.
+REMOTE_FIXTURES = {
+    "glad-global-forest-change-1.11",  # fetched from https://stac.maap-project.org
+}
+
+
+def _records(path: Path):
+    """Yield the JSON objects in a .json or newline-delimited .ndjson file."""
+    text = path.read_text().strip()
+    if not text:
+        return
+    if path.suffix == ".ndjson":
+        for line in text.splitlines():
+            line = line.strip()
+            if line:
+                yield json.loads(line)
+        return
+    parsed = json.loads(text)
+    if isinstance(parsed, list):
+        yield from parsed
+    else:
+        yield parsed
+
+
+def discover(data_dir: Path = DATA_DIR) -> dict[str, str]:
+    """Map collection id -> where it was found."""
+    found: dict[str, str] = {cid: "deploy.yml (remote)" for cid in REMOTE_FIXTURES}
+
+    for path in sorted(data_dir.glob("*.json")) + sorted(data_dir.glob("*.ndjson")):
+        for record in _records(path):
+            if not isinstance(record, dict):
+                continue
+            # A Collection names itself; an Item names the collection it belongs to.
+            if record.get("type") == "Collection" and record.get("id"):
+                found.setdefault(record["id"], path.name)
+            elif record.get("collection"):
+                found.setdefault(record["collection"], path.name)
+
+    return found
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--check", action="store_true", help="show where each id was found"
+    )
+    args = parser.parse_args()
+
+    found = discover()
+    if not found:
+        print("no fixture collections discovered", file=sys.stderr)
+        return 1
+
+    for cid in sorted(found):
+        print(f"{cid}\t{found[cid]}" if args.check else cid)
+    return 0
+
+
+def demo() -> None:
+    """Self-check: `python scripts/fixture_collections.py --selftest`."""
+    import tempfile
+
+    with tempfile.TemporaryDirectory() as tmp:
+        d = Path(tmp)
+        (d / "c.json").write_text(json.dumps({"type": "Collection", "id": "fix-a"}))
+        (d / "i.ndjson").write_text(
+            json.dumps({"type": "Feature", "id": "x", "collection": "fix-b"})
+            + "\n"
+            + json.dumps({"type": "Feature", "id": "y", "collection": "fix-b"})
+            + "\n"
+        )
+        (d / "empty.ndjson").write_text("\n")
+
+        found = discover(d)
+
+    assert "fix-a" in found, found
+    assert "fix-b" in found, found
+    assert REMOTE_FIXTURES <= set(found), found
+    # Nothing invented: only what the files and REMOTE_FIXTURES name.
+    assert set(found) == {"fix-a", "fix-b"} | REMOTE_FIXTURES, found
+    print("fixture_collections: all checks passed")
+
+
+if __name__ == "__main__":
+    if "--selftest" in sys.argv:
+        demo()
+    else:
+        raise SystemExit(main())