Harden motor database release pipeline

.github/workflows/tests.yml

@@ -11,18 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.10"
 
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r scripts/requirements.txt
-          pip install pytest cryptography
 
       - name: Run Tests
         run: pytest

.github/workflows/update-motors.yml

@@ -2,69 +2,131 @@ name: Update Motor Database
 
 on:
   schedule:
-    # Run at 02:00 every Sunday
+    # Run at 02:00 every Sunday.
     - cron: '0 2 * * 0'
   workflow_dispatch:
 
+concurrency:
+  group: motor-database-release
+  cancel-in-progress: false
+
 permissions:
-  contents: write
+  contents: read
 
 jobs:
-  update-and-publish:
+  build-and-validate:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.10'
 
-      - name: Install Dependencies
-        run: pip install -r scripts/requirements.txt
+      - name: Preserve Previous Build Baseline
+        run: cp state/last_build.json previous-build.json
 
       - name: Fetch New Motors
         run: python scripts/fetch_updates.py
 
       - name: Build Database
         run: python scripts/build_database.py
 
-      - name: Install Crypto Lib
-        run: pip install cryptography
+      - name: Validate Unsigned Release
+        run: >-
+          python scripts/validate_release.py motors.db.gz metadata.json
+          --baseline previous-build.json
+
+      - name: Commit Raw Data Changes
+        # This saves downloaded motor files and state back to the repository.
+        uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0
+        with:
+          commit_message: "Auto-update motor cache [skip ci]"
+          file_pattern: "data/ state/"
+
+      - name: Prepare Unsigned Release
+        run: |
+          mkdir unsigned-release
+          cp motors.db.gz metadata.json unsigned-release/
+
+      - name: Upload Validated Unsigned Release
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: unsigned-motor-database
+          path: unsigned-release/
+          if-no-files-found: error
+          retention-days: 1
+
+  sign:
+    needs: build-and-validate
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout Trusted Signing Code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Download Validated Unsigned Release
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: unsigned-motor-database
+          path: release
+
+      - name: Revalidate Before Signing
+        run: python scripts/validate_release.py release/motors.db.gz release/metadata.json
 
       - name: Sign Database Update
         env:
           MOTOR_DB_PRIVATE_KEY_BASE64: ${{ secrets.MOTOR_DB_PRIVATE_KEY_BASE64 }}
           MOTOR_DB_KEY_ID: ${{ secrets.MOTOR_DB_KEY_ID }}
-        run: python scripts/sign_database.py motors.db.gz metadata.json
+        run: python scripts/sign_database.py release/motors.db.gz release/metadata.json
 
-      - name: Commit Raw Data Changes
-        # This saves the downloaded .eng files back to the repo
-        uses: stefanzweifel/git-auto-commit-action@v5
+      - name: Validate Signed Release
+        run: >-
+          python scripts/validate_release.py release/motors.db.gz release/metadata.json
+          --require-signature
+          --public-key security/motor-database-signing-public-key.pem
+
+      - name: Upload Signed Release
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          commit_message: "Auto-update motor cache [skip ci]"
-          file_pattern: "data/ state/"
+          name: signed-motor-database
+          path: release/
+          if-no-files-found: error
+          retention-days: 1
 
-      - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+  publish:
+    needs: sign
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout Validation Code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./
-          keep_files: false
-          # We only want to publish the artifacts, not the whole repo
-          exclude_assets: "scripts,schema,data,.github"
-          # Or better, move artifacts to a 'public' folder and publish that:
+          ref: ${{ github.sha }}
 
-      - name: Prepare Publish Folder
-        run: |
-          mkdir public
-          mv motors.db.gz public/
-          mv metadata.json public/
+      - name: Download Signed Release
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: signed-motor-database
+          path: release
+
+      - name: Final Release Validation
+        run: >-
+          python scripts/validate_release.py release/motors.db.gz release/metadata.json
+          --require-signature
+          --public-key security/motor-database-signing-public-key.pem
 
       - name: Deploy Artifacts
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./public
+          publish_dir: ./release
           force_orphan: true

README.md

@@ -205,7 +205,9 @@ Variant report filters:
 
 ## Signing
 
-Signing is done in CI after the database build completes.
+Signing is done in an isolated CI job after the database build completes and passes release validation. The signing
+job runs on a fresh runner, uses only Python's standard library and the runner-provided OpenSSL executable, and never
+runs the data-fetch or deployment actions with the private key available.
 
 What is signed:
 - Canonical message: `openrocket-motordb-v1\n{database_version}\n{sha256_gz}\n`
@@ -217,9 +219,12 @@ What gets added to `metadata.json` by the signing step:
 - `key_id` (optional): identifier for key rotation
 
 How CI handles it:
-- `.github/workflows/update-motors.yml` installs `cryptography`
-- It runs `python scripts/sign_database.py motors.db.gz metadata.json`
-- The private key is provided via secrets
+- The build job checks SQLite integrity and foreign keys, schema and metadata consistency, minimum row counts,
+  unexpected count drops, thrust-point bounds, and sufficient time coverage for every curve.
+- A fresh signing job revalidates the artifact and runs `python scripts/sign_database.py motors.db.gz metadata.json`.
+- The signer delegates Ed25519 operations to OpenSSL, so no third-party Python package is loaded with the private key.
+- A separate publishing job verifies the signature again before deploying only `motors.db.gz` and `metadata.json`.
+- All workflow actions are pinned to immutable commit SHAs.
 
 Set the private key in:
 
@@ -231,8 +236,7 @@ Manual signing: `python scripts/sign_database.py motors.db.gz metadata.json`
 ## Unit Tests
 
 1.  `pip install -r scripts/requirements.txt`
-2.  `pip install pytest cryptography`
-3.  `pytest`
+2.  `pytest`
 
 ## Data Attribution & License
 The motor data in this repository is cached from [ThrustCurve.org](https://www.thrustcurve.org). 

scripts/build_database.py

@@ -795,6 +795,7 @@ def build(force=False):
             "motor_count": build_state["motor_count"],
             "curve_count": build_state["curve_count"],
             "sha256": build_state["sha256"],
+            "sha256_gz": build_state["sha256"],
             "last_checked": last_checked,
             "download_url": "https://openrocket.github.io/motor-database/motors.db.gz"
         }
@@ -1185,6 +1186,7 @@ def get_source_priority(source_dir):
         "motor_count": motor_count,
         "curve_count": curve_count,
         "sha256": sha256_hex,
+        "sha256_gz": sha256_hex,
         "last_checked": last_checked,
         "download_url": "https://openrocket.github.io/motor-database/motors.db.gz"
     }

scripts/fetch_updates.py

@@ -1,9 +1,10 @@
 import os
 import json
-import requests
 import time
 import base64
 from datetime import datetime
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
 
 # Config
 DATA_DIR = "data/thrustcurve.org"
@@ -22,6 +23,28 @@
     'Content-Type': 'application/json'
 }
 
+REQUEST_TIMEOUT_SECONDS = 30
+MAX_API_RESPONSE_BYTES = 25 * 1024 * 1024
+
+
+def request_json(url, payload, method):
+    """Send a bounded JSON request and return ``(status_code, response_json)``."""
+    request_data = json.dumps(payload).encode("utf-8")
+    request = Request(url, data=request_data, headers=HEADERS, method=method)
+
+    try:
+        with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
+            content_length = response.headers.get("Content-Length")
+            if content_length and int(content_length) > MAX_API_RESPONSE_BYTES:
+                raise ValueError(f"API response exceeds {MAX_API_RESPONSE_BYTES} bytes")
+
+            response_data = response.read(MAX_API_RESPONSE_BYTES + 1)
+            if len(response_data) > MAX_API_RESPONSE_BYTES:
+                raise ValueError(f"API response exceeds {MAX_API_RESPONSE_BYTES} bytes")
+            return response.status, json.loads(response_data.decode("utf-8"))
+    except HTTPError as error:
+        return error.code, None
+
 
 def load_state():
     # Handle empty or corrupt JSON files gracefully
@@ -122,9 +145,8 @@ def get_manufacturers():
             "availability": "all"
         }
 
-        resp = requests.get(TC_API_METADATA, json=payload, headers=HEADERS)
-        if resp.status_code == 200:
-            data = resp.json()
+        status_code, data = request_json(TC_API_METADATA, payload, "GET")
+        if status_code == 200:
             manufacturers = data.get('manufacturers', [])
 
             # Save the full manufacturers list for use in build_database.py
@@ -151,12 +173,11 @@ def download_motor_data(motor_id, mfr_name, motor_name, simfile_mapping):
     }
 
     try:
-        resp = requests.post(TC_API_DOWNLOAD, json=payload, headers=HEADERS)
-        if resp.status_code != 200:
-            print(f"  [Error] Download failed for {motor_id}: Status {resp.status_code}")
+        status_code, data = request_json(TC_API_DOWNLOAD, payload, "POST")
+        if status_code != 200:
+            print(f"  [Error] Download failed for {motor_id}: Status {status_code}")
             return 0, []
 
-        data = resp.json()
         results = data.get('results', [])
 
         saved_count = 0
@@ -244,12 +265,12 @@ def fetch_motors():
         }
 
         try:
-            resp = requests.post(TC_API_SEARCH, json=search_payload, headers=HEADERS)
-            if resp.status_code != 200:
-                print(f"Failed to search {mfr}: {resp.status_code}")
+            status_code, data = request_json(TC_API_SEARCH, search_payload, "POST")
+            if status_code != 200:
+                print(f"Failed to search {mfr}: {status_code}")
                 continue
 
-            results = resp.json().get('results', [])
+            results = data.get('results', [])
 
             # Client-side filtering for dates (since API search criteria is limited)
             motors_to_update = []
@@ -356,9 +377,8 @@ def rebuild_simfile_mapping():
         }
 
         try:
-            resp = requests.post(TC_API_DOWNLOAD, json=payload, headers=HEADERS)
-            if resp.status_code == 200:
-                data = resp.json()
+            status_code, data = request_json(TC_API_DOWNLOAD, payload, "POST")
+            if status_code == 200:
                 results = data.get('results', [])
 
                 for res in results: