diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f521064..79f0723 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" @@ -22,7 +22,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r scripts/requirements.txt - pip install pytest cryptography - name: Run Tests run: pytest diff --git a/.github/workflows/update-motors.yml b/.github/workflows/update-motors.yml index 75e2daf..e312453 100644 --- a/.github/workflows/update-motors.yml +++ b/.github/workflows/update-motors.yml @@ -2,27 +2,33 @@ name: Update Motor Database on: schedule: - # Run at 02:00 every Sunday + # Run at 02:00 every Sunday. - cron: '0 2 * * 0' workflow_dispatch: +concurrency: + group: motor-database-release + cancel-in-progress: false + permissions: - contents: write + contents: read jobs: - update-and-publish: + build-and-validate: runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout Code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.10' - - name: Install Dependencies - run: pip install -r scripts/requirements.txt + - name: Preserve Previous Build Baseline + run: cp state/last_build.json previous-build.json - name: Fetch New Motors run: python scripts/fetch_updates.py @@ -30,41 +36,97 @@ jobs: - name: Build Database run: python scripts/build_database.py - - name: Install Crypto Lib - run: pip install cryptography + - name: Validate Unsigned Release + run: >- + python scripts/validate_release.py motors.db.gz metadata.json + --baseline previous-build.json + + - name: Commit Raw Data Changes + # This saves downloaded motor files and state back to the repository. + uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0 + with: + commit_message: "Auto-update motor cache [skip ci]" + file_pattern: "data/ state/" + + - name: Prepare Unsigned Release + run: | + mkdir unsigned-release + cp motors.db.gz metadata.json unsigned-release/ + + - name: Upload Validated Unsigned Release + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: unsigned-motor-database + path: unsigned-release/ + if-no-files-found: error + retention-days: 1 + + sign: + needs: build-and-validate + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout Trusted Signing Code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + + - name: Download Validated Unsigned Release + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: unsigned-motor-database + path: release + + - name: Revalidate Before Signing + run: python scripts/validate_release.py release/motors.db.gz release/metadata.json - name: Sign Database Update env: MOTOR_DB_PRIVATE_KEY_BASE64: ${{ secrets.MOTOR_DB_PRIVATE_KEY_BASE64 }} MOTOR_DB_KEY_ID: ${{ secrets.MOTOR_DB_KEY_ID }} - run: python scripts/sign_database.py motors.db.gz metadata.json + run: python scripts/sign_database.py release/motors.db.gz release/metadata.json - - name: Commit Raw Data Changes - # This saves the downloaded .eng files back to the repo - uses: stefanzweifel/git-auto-commit-action@v5 + - name: Validate Signed Release + run: >- + python scripts/validate_release.py release/motors.db.gz release/metadata.json + --require-signature + --public-key security/motor-database-signing-public-key.pem + + - name: Upload Signed Release + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - commit_message: "Auto-update motor cache [skip ci]" - file_pattern: "data/ state/" + name: signed-motor-database + path: release/ + if-no-files-found: error + retention-days: 1 - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 + publish: + needs: sign + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout Validation Code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./ - keep_files: false - # We only want to publish the artifacts, not the whole repo - exclude_assets: "scripts,schema,data,.github" - # Or better, move artifacts to a 'public' folder and publish that: + ref: ${{ github.sha }} - - name: Prepare Publish Folder - run: | - mkdir public - mv motors.db.gz public/ - mv metadata.json public/ + - name: Download Signed Release + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: signed-motor-database + path: release + + - name: Final Release Validation + run: >- + python scripts/validate_release.py release/motors.db.gz release/metadata.json + --require-signature + --public-key security/motor-database-signing-public-key.pem - name: Deploy Artifacts - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./public + publish_dir: ./release force_orphan: true diff --git a/README.md b/README.md index 3525dd9..f675bd5 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,9 @@ Variant report filters: ## Signing -Signing is done in CI after the database build completes. +Signing is done in an isolated CI job after the database build completes and passes release validation. The signing +job runs on a fresh runner, uses only Python's standard library and the runner-provided OpenSSL executable, and never +runs the data-fetch or deployment actions with the private key available. What is signed: - Canonical message: `openrocket-motordb-v1\n{database_version}\n{sha256_gz}\n` @@ -217,9 +219,12 @@ What gets added to `metadata.json` by the signing step: - `key_id` (optional): identifier for key rotation How CI handles it: -- `.github/workflows/update-motors.yml` installs `cryptography` -- It runs `python scripts/sign_database.py motors.db.gz metadata.json` -- The private key is provided via secrets +- The build job checks SQLite integrity and foreign keys, schema and metadata consistency, minimum row counts, + unexpected count drops, thrust-point bounds, and sufficient time coverage for every curve. +- A fresh signing job revalidates the artifact and runs `python scripts/sign_database.py motors.db.gz metadata.json`. +- The signer delegates Ed25519 operations to OpenSSL, so no third-party Python package is loaded with the private key. +- A separate publishing job verifies the signature again before deploying only `motors.db.gz` and `metadata.json`. +- All workflow actions are pinned to immutable commit SHAs. Set the private key in: @@ -231,8 +236,7 @@ Manual signing: `python scripts/sign_database.py motors.db.gz metadata.json` ## Unit Tests 1. `pip install -r scripts/requirements.txt` -2. `pip install pytest cryptography` -3. `pytest` +2. `pytest` ## Data Attribution & License The motor data in this repository is cached from [ThrustCurve.org](https://www.thrustcurve.org). diff --git a/scripts/build_database.py b/scripts/build_database.py index 67d450f..77f463b 100644 --- a/scripts/build_database.py +++ b/scripts/build_database.py @@ -795,6 +795,7 @@ def build(force=False): "motor_count": build_state["motor_count"], "curve_count": build_state["curve_count"], "sha256": build_state["sha256"], + "sha256_gz": build_state["sha256"], "last_checked": last_checked, "download_url": "https://openrocket.github.io/motor-database/motors.db.gz" } @@ -1185,6 +1186,7 @@ def get_source_priority(source_dir): "motor_count": motor_count, "curve_count": curve_count, "sha256": sha256_hex, + "sha256_gz": sha256_hex, "last_checked": last_checked, "download_url": "https://openrocket.github.io/motor-database/motors.db.gz" } diff --git a/scripts/fetch_updates.py b/scripts/fetch_updates.py index cbfda01..d5f4a10 100644 --- a/scripts/fetch_updates.py +++ b/scripts/fetch_updates.py @@ -1,9 +1,10 @@ import os import json -import requests import time import base64 from datetime import datetime +from urllib.error import HTTPError +from urllib.request import Request, urlopen # Config DATA_DIR = "data/thrustcurve.org" @@ -22,6 +23,28 @@ 'Content-Type': 'application/json' } +REQUEST_TIMEOUT_SECONDS = 30 +MAX_API_RESPONSE_BYTES = 25 * 1024 * 1024 + + +def request_json(url, payload, method): + """Send a bounded JSON request and return ``(status_code, response_json)``.""" + request_data = json.dumps(payload).encode("utf-8") + request = Request(url, data=request_data, headers=HEADERS, method=method) + + try: + with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > MAX_API_RESPONSE_BYTES: + raise ValueError(f"API response exceeds {MAX_API_RESPONSE_BYTES} bytes") + + response_data = response.read(MAX_API_RESPONSE_BYTES + 1) + if len(response_data) > MAX_API_RESPONSE_BYTES: + raise ValueError(f"API response exceeds {MAX_API_RESPONSE_BYTES} bytes") + return response.status, json.loads(response_data.decode("utf-8")) + except HTTPError as error: + return error.code, None + def load_state(): # Handle empty or corrupt JSON files gracefully @@ -122,9 +145,8 @@ def get_manufacturers(): "availability": "all" } - resp = requests.get(TC_API_METADATA, json=payload, headers=HEADERS) - if resp.status_code == 200: - data = resp.json() + status_code, data = request_json(TC_API_METADATA, payload, "GET") + if status_code == 200: manufacturers = data.get('manufacturers', []) # Save the full manufacturers list for use in build_database.py @@ -151,12 +173,11 @@ def download_motor_data(motor_id, mfr_name, motor_name, simfile_mapping): } try: - resp = requests.post(TC_API_DOWNLOAD, json=payload, headers=HEADERS) - if resp.status_code != 200: - print(f" [Error] Download failed for {motor_id}: Status {resp.status_code}") + status_code, data = request_json(TC_API_DOWNLOAD, payload, "POST") + if status_code != 200: + print(f" [Error] Download failed for {motor_id}: Status {status_code}") return 0, [] - data = resp.json() results = data.get('results', []) saved_count = 0 @@ -244,12 +265,12 @@ def fetch_motors(): } try: - resp = requests.post(TC_API_SEARCH, json=search_payload, headers=HEADERS) - if resp.status_code != 200: - print(f"Failed to search {mfr}: {resp.status_code}") + status_code, data = request_json(TC_API_SEARCH, search_payload, "POST") + if status_code != 200: + print(f"Failed to search {mfr}: {status_code}") continue - results = resp.json().get('results', []) + results = data.get('results', []) # Client-side filtering for dates (since API search criteria is limited) motors_to_update = [] @@ -356,9 +377,8 @@ def rebuild_simfile_mapping(): } try: - resp = requests.post(TC_API_DOWNLOAD, json=payload, headers=HEADERS) - if resp.status_code == 200: - data = resp.json() + status_code, data = request_json(TC_API_DOWNLOAD, payload, "POST") + if status_code == 200: results = data.get('results', []) for res in results: diff --git a/scripts/keygen.py b/scripts/keygen.py index 4fedc53..4f5b609 100644 --- a/scripts/keygen.py +++ b/scripts/keygen.py @@ -1,38 +1,41 @@ -from cryptography.hazmat.primitives.asymmetric import ed25519 -from cryptography.hazmat.primitives import serialization +"""Generate an Ed25519 signing key pair using the system OpenSSL executable.""" + import base64 +import os +import subprocess +import tempfile + + +def run_openssl(arguments): + """Run OpenSSL and raise a concise error if key generation fails.""" + try: + subprocess.run(["openssl", *arguments], check=True, capture_output=True) + except (OSError, subprocess.CalledProcessError) as error: + raise RuntimeError("OpenSSL could not generate the Ed25519 key pair") from error + + +def main(): + with tempfile.TemporaryDirectory(prefix="openrocket-motordb-keygen-") as temp_dir: + private_pem_path = os.path.join(temp_dir, "private.pem") + private_der_path = os.path.join(temp_dir, "private.der") + public_der_path = os.path.join(temp_dir, "public.der") + + run_openssl(["genpkey", "-algorithm", "Ed25519", "-out", private_pem_path]) + run_openssl(["pkey", "-in", private_pem_path, "-outform", "DER", "-out", private_der_path]) + run_openssl([ + "pkey", "-in", private_pem_path, "-pubout", "-outform", "DER", "-out", public_der_path, + ]) + + with open(private_der_path, "rb") as private_file: + private_key_b64 = base64.b64encode(private_file.read()).decode("utf-8") + with open(public_der_path, "rb") as public_file: + public_key_b64 = base64.b64encode(public_file.read()).decode("utf-8") + + print("=== COPY TO GITHUB SECRETS (Private Key) ===") + print(private_key_b64) + print("\n=== COPY TO OPENROCKET JAVA CODE (Public Key) ===") + print(public_key_b64) + -## Helper script to generate an Ed25519 keypair for OpenRocket update signing - -# 1. Generate the private key -private_key = ed25519.Ed25519PrivateKey.generate() - -# 2. Extract Private Key bytes (PKCS8 format is standard and easy to handle) -priv_bytes = private_key.private_bytes( - encoding=serialization.Encoding.DER, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption() -) - -# 3. Extract Public Key bytes (SubjectPublicKeyInfo format) -public_key = private_key.public_key() -pub_bytes = public_key.public_bytes( - encoding=serialization.Encoding.PEM, # Helper to get the bytes, we strip headers later if needed - format=serialization.PublicFormat.SubjectPublicKeyInfo -) - -# Convert to pure Base64 strings (stripping PEM headers for easier copy-pasting) -priv_b64 = base64.b64encode(priv_bytes).decode('utf-8') - -# For the public key, let's get the raw bytes then base64 encode them -# so it looks like "MCowBQYDK2VwAyEA..." -raw_pub_bytes = public_key.public_bytes( - encoding=serialization.Encoding.DER, - format=serialization.PublicFormat.SubjectPublicKeyInfo -) -pub_b64 = base64.b64encode(raw_pub_bytes).decode('utf-8') - -print("=== COPY TO GITHUB SECRETS (Private Key) ===") -print(priv_b64) -print("\n=== COPY TO OPENROCKET JAVA CODE (Public Key) ===") -print(pub_b64) \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 2001f3e..8271f69 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,2 +1,2 @@ -requests -pytest +pytest==7.4.2 +requests==2.31.0 diff --git a/scripts/sign_database.py b/scripts/sign_database.py index ea57cbe..0793237 100644 --- a/scripts/sign_database.py +++ b/scripts/sign_database.py @@ -2,11 +2,9 @@ import hashlib import json import os +import subprocess import sys - -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ed25519 - +import tempfile ENV_PRIVATE_KEY = "MOTOR_DB_PRIVATE_KEY_BASE64" ENV_KEY_ID = "MOTOR_DB_KEY_ID" @@ -26,21 +24,47 @@ def compute_sha256_hex(path): def load_private_key_from_b64(private_key_b64): try: - key_bytes = base64.b64decode(private_key_b64) + key_bytes = base64.b64decode(private_key_b64, validate=True) except (ValueError, TypeError) as e: raise ValueError("Invalid base64 for private key.") from e - loaders = (serialization.load_der_private_key, serialization.load_pem_private_key) - last_error = None - for loader in loaders: + if not key_bytes: + raise ValueError("Private key is empty.") + return key_bytes + + +def sign_message(private_key_bytes, message_bytes): + """Sign a message with OpenSSL without loading third-party Python code.""" + with tempfile.TemporaryDirectory(prefix="openrocket-motordb-sign-") as temp_dir: + key_path = os.path.join(temp_dir, "signing-key") + message_path = os.path.join(temp_dir, "message") + signature_path = os.path.join(temp_dir, "signature") + + with open(key_path, "wb") as key_file: + key_file.write(private_key_bytes) + os.chmod(key_path, 0o600) + with open(message_path, "wb") as message_file: + message_file.write(message_bytes) + + command = [ + "openssl", "pkeyutl", "-sign", "-rawin", + "-inkey", key_path, + "-in", message_path, + "-out", signature_path, + ] + if not private_key_bytes.lstrip().startswith(b"-----BEGIN"): + command.extend(["-keyform", "DER"]) + try: - key = loader(key_bytes, password=None) - if isinstance(key, ed25519.Ed25519PrivateKey): - return key - except Exception as e: - last_error = e + subprocess.run(command, check=True, capture_output=True) + except (OSError, subprocess.CalledProcessError) as error: + detail = getattr(error, "stderr", b"") + if isinstance(detail, bytes): + detail = detail.decode("utf-8", errors="replace").strip() + raise ValueError(f"Unsupported private key format or signing failure: {detail}") from error - raise ValueError("Unsupported private key format or type.") from last_error + with open(signature_path, "rb") as signature_file: + return signature_file.read() def sign_metadata(db_file, metadata_file, private_key_b64=None, key_id=None): @@ -63,8 +87,8 @@ def sign_metadata(db_file, metadata_file, private_key_b64=None, key_id=None): message_str = f"{MESSAGE_PREFIX}\n{db_version}\n{sha256_gz}\n" message_bytes = message_str.encode("utf-8") - private_key = load_private_key_from_b64(private_key_b64) - signature = private_key.sign(message_bytes) + private_key_bytes = load_private_key_from_b64(private_key_b64) + signature = sign_message(private_key_bytes, message_bytes) sig_b64 = base64.b64encode(signature).decode("utf-8") metadata["sha256"] = sha256_gz diff --git a/scripts/validate_release.py b/scripts/validate_release.py new file mode 100644 index 0000000..7875ea0 --- /dev/null +++ b/scripts/validate_release.py @@ -0,0 +1,278 @@ +"""Fail-closed validation for a motor database release artifact.""" + +import argparse +import base64 +import gzip +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import subprocess +import tempfile +from urllib.parse import urlparse + + +MESSAGE_PREFIX = "openrocket-motordb-v1" +MAX_COMPRESSED_BYTES = 50 * 1024 * 1024 +MAX_DATABASE_BYTES = 200 * 1024 * 1024 +MIN_MOTOR_COUNT = 1_000 +MIN_CURVE_COUNT = 1_000 +MIN_THRUST_POINT_COUNT = 10_000 +MAX_COUNT_DROP_RATIO = 0.15 +ALLOWED_DOWNLOAD_HOSTS = {"openrocket.info", "openrocket.github.io"} + +REQUIRED_COLUMNS = { + "meta": {"key", "value"}, + "manufacturers": {"id", "name", "abbrev"}, + "motors": { + "id", "manufacturer_id", "tc_motor_id", "designation", "common_name", + "impulse_class", "diameter", "length", "total_impulse", "avg_thrust", + "max_thrust", "burn_time", "propellant_weight", "total_weight", "type", + "delays", "case_info", "prop_info", "sparky", "info_url", "data_files", + "updated_on", + }, + "thrust_curves": { + "id", "motor_id", "tc_simfile_id", "source", "format", "license", + "info_url", "data_url", "total_impulse", "avg_thrust", "max_thrust", + "burn_time", + }, + "thrust_data": {"id", "curve_id", "time_seconds", "force_newtons"}, +} + + +class ValidationError(Exception): + """Raised when a release artifact is unsafe to publish or install.""" + + +def sha256_file(path): + """Return the lowercase SHA-256 digest for a file.""" + digest = hashlib.sha256() + with open(path, "rb") as input_file: + for chunk in iter(lambda: input_file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_metadata(metadata_path): + """Load and validate release metadata fields that are independent of SQLite.""" + with open(metadata_path, "r", encoding="utf-8") as metadata_file: + metadata = json.load(metadata_file) + + required = {"schema_version", "database_version", "motor_count", "curve_count", "sha256_gz", "download_url"} + missing = sorted(required.difference(metadata)) + if missing: + raise ValidationError(f"metadata.json is missing fields: {', '.join(missing)}") + + for key in ("schema_version", "database_version", "motor_count", "curve_count"): + if not isinstance(metadata[key], int) or isinstance(metadata[key], bool): + raise ValidationError(f"metadata field {key} must be an integer") + if metadata["database_version"] <= 0: + raise ValidationError("database_version must be positive") + + sha256_gz = str(metadata["sha256_gz"]).strip().lower() + if len(sha256_gz) != 64 or any(character not in "0123456789abcdef" for character in sha256_gz): + raise ValidationError("sha256_gz must be 64 lowercase hexadecimal characters") + metadata["sha256_gz"] = sha256_gz + + parsed_url = urlparse(str(metadata["download_url"])) + if parsed_url.scheme.lower() != "https" or parsed_url.hostname not in ALLOWED_DOWNLOAD_HOSTS: + raise ValidationError("download_url must use HTTPS and an approved OpenRocket host") + return metadata + + +def decompress_database(database_gz_path, output_path): + """Decompress a database while enforcing compressed and expanded size limits.""" + if os.path.getsize(database_gz_path) > MAX_COMPRESSED_BYTES: + raise ValidationError("compressed database exceeds the release size limit") + + expanded_bytes = 0 + try: + with gzip.open(database_gz_path, "rb") as compressed_file, open(output_path, "wb") as database_file: + while True: + chunk = compressed_file.read(1024 * 1024) + if not chunk: + break + expanded_bytes += len(chunk) + if expanded_bytes > MAX_DATABASE_BYTES: + raise ValidationError("expanded database exceeds the release size limit") + database_file.write(chunk) + except (OSError, EOFError) as error: + raise ValidationError(f"invalid gzip database: {error}") from error + + +def table_columns(connection, table_name): + """Return the column names for an SQLite table.""" + return {row[1] for row in connection.execute(f'PRAGMA table_info("{table_name}")')} + + +def read_single_count(connection, table_name): + """Read a table count using a fixed, internally supplied table name.""" + return connection.execute(f'SELECT count(*) FROM "{table_name}"').fetchone()[0] + + +def validate_database(database_path, metadata, minimum_motors, minimum_curves, minimum_points): + """Validate integrity, schema, metadata consistency, and core physical invariants.""" + database_uri = Path(database_path).resolve().as_uri() + "?mode=ro&immutable=1" + connection = sqlite3.connect(database_uri, uri=True) + try: + integrity_rows = [row[0] for row in connection.execute("PRAGMA integrity_check")] + if integrity_rows != ["ok"]: + raise ValidationError(f"SQLite integrity_check failed: {integrity_rows[:3]}") + + foreign_key_error = connection.execute("PRAGMA foreign_key_check").fetchone() + if foreign_key_error is not None: + raise ValidationError(f"SQLite foreign_key_check failed: {foreign_key_error}") + + for table_name, required_columns in REQUIRED_COLUMNS.items(): + present_columns = table_columns(connection, table_name) + missing_columns = sorted(required_columns.difference(present_columns)) + if missing_columns: + raise ValidationError(f"table {table_name} is missing columns: {', '.join(missing_columns)}") + + database_metadata = dict(connection.execute("SELECT key, value FROM meta")) + for key in ("schema_version", "database_version", "motor_count", "curve_count"): + if key not in database_metadata: + raise ValidationError(f"SQLite metadata is missing {key}") + if int(database_metadata[key]) != metadata[key]: + raise ValidationError(f"SQLite and release metadata disagree on {key}") + + motor_count = read_single_count(connection, "motors") + curve_count = read_single_count(connection, "thrust_curves") + point_count = read_single_count(connection, "thrust_data") + if motor_count != metadata["motor_count"] or curve_count != metadata["curve_count"]: + raise ValidationError("declared motor/curve counts do not match the SQLite tables") + if motor_count < minimum_motors or curve_count < minimum_curves or point_count < minimum_points: + raise ValidationError( + f"release is unexpectedly small: {motor_count} motors, {curve_count} curves, {point_count} points" + ) + + invalid_point = connection.execute( + "SELECT id FROM thrust_data " + "WHERE time_seconds IS NULL OR force_newtons IS NULL " + "OR typeof(time_seconds) NOT IN ('integer', 'real') " + "OR typeof(force_newtons) NOT IN ('integer', 'real') " + "OR time_seconds < 0 OR force_newtons < 0 " + "OR abs(time_seconds) > 1000000 OR abs(force_newtons) > 1000000000 LIMIT 1" + ).fetchone() + if invalid_point is not None: + raise ValidationError(f"invalid thrust point found at row {invalid_point[0]}") + + incomplete_curve = connection.execute( + "SELECT thrust_curves.id FROM thrust_curves " + "LEFT JOIN thrust_data ON thrust_data.curve_id = thrust_curves.id " + "GROUP BY thrust_curves.id " + "HAVING count(thrust_data.id) < 2 OR max(time_seconds) <= min(time_seconds) LIMIT 1" + ).fetchone() + if incomplete_curve is not None: + raise ValidationError(f"thrust curve has insufficient time coverage: {incomplete_curve[0]}") + except (sqlite3.DatabaseError, TypeError, ValueError) as error: + if isinstance(error, ValidationError): + raise + raise ValidationError(f"invalid SQLite database: {error}") from error + finally: + connection.close() + + return {"motor_count": motor_count, "curve_count": curve_count, "point_count": point_count} + + +def validate_baseline(counts, baseline_path): + """Reject unexpectedly large count drops relative to the previous successful build.""" + if baseline_path is None: + return + with open(baseline_path, "r", encoding="utf-8") as baseline_file: + baseline = json.load(baseline_file) + for count_key in ("motor_count", "curve_count"): + old_count = int(baseline.get(count_key, 0)) + if old_count > 0 and counts[count_key] < old_count * (1 - MAX_COUNT_DROP_RATIO): + raise ValidationError( + f"{count_key} dropped from {old_count} to {counts[count_key]} (more than {MAX_COUNT_DROP_RATIO:.0%})" + ) + + +def verify_signature(metadata, public_key_path): + """Verify the release signature with the public key embedded by OpenRocket.""" + signature_text = metadata.get("sig") + if not signature_text: + raise ValidationError("signed metadata is missing sig") + try: + signature = base64.b64decode(signature_text, validate=True) + except (TypeError, ValueError) as error: + raise ValidationError("sig is not valid base64") from error + if len(signature) != 64: + raise ValidationError("Ed25519 signature must be 64 bytes") + + message = ( + f"{MESSAGE_PREFIX}\n{metadata['database_version']}\n{metadata['sha256_gz']}\n" + ).encode("utf-8") + with tempfile.TemporaryDirectory(prefix="openrocket-motordb-verify-") as temp_dir: + message_path = os.path.join(temp_dir, "message") + signature_path = os.path.join(temp_dir, "signature") + with open(message_path, "wb") as message_file: + message_file.write(message) + with open(signature_path, "wb") as signature_file: + signature_file.write(signature) + command = [ + "openssl", "pkeyutl", "-verify", "-pubin", "-rawin", + "-inkey", str(public_key_path), "-in", message_path, "-sigfile", signature_path, + ] + try: + subprocess.run(command, check=True, capture_output=True) + except (OSError, subprocess.CalledProcessError) as error: + raise ValidationError("release signature verification failed") from error + + +def validate_release(database_gz_path, metadata_path, baseline_path=None, require_signature=False, + public_key_path=None, minimum_motors=MIN_MOTOR_COUNT, + minimum_curves=MIN_CURVE_COUNT, minimum_points=MIN_THRUST_POINT_COUNT): + """Validate a complete release and return its verified row counts.""" + metadata = load_metadata(metadata_path) + actual_sha256 = sha256_file(database_gz_path) + if actual_sha256 != metadata["sha256_gz"]: + raise ValidationError("motors.db.gz SHA-256 does not match metadata.json") + if "sha256" in metadata and str(metadata["sha256"]).lower() != actual_sha256: + raise ValidationError("legacy sha256 field does not match motors.db.gz") + + with tempfile.TemporaryDirectory(prefix="openrocket-motordb-validate-") as temp_dir: + database_path = os.path.join(temp_dir, "motors.db") + decompress_database(database_gz_path, database_path) + counts = validate_database(database_path, metadata, minimum_motors, minimum_curves, minimum_points) + + validate_baseline(counts, baseline_path) + if require_signature: + if public_key_path is None: + raise ValidationError("a public key is required for signature verification") + verify_signature(metadata, public_key_path) + return counts + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("database", help="path to motors.db.gz") + parser.add_argument("metadata", help="path to metadata.json") + parser.add_argument("--baseline", help="previous build state used for count-drop detection") + parser.add_argument("--require-signature", action="store_true", help="require and verify the Ed25519 signature") + parser.add_argument("--public-key", help="PEM public key used with --require-signature") + args = parser.parse_args() + + try: + counts = validate_release( + args.database, + args.metadata, + baseline_path=args.baseline, + require_signature=args.require_signature, + public_key_path=args.public_key, + ) + except (OSError, json.JSONDecodeError, ValidationError) as error: + print(f"Release validation failed: {error}", file=os.sys.stderr) + return 1 + + print( + f"Release validated: {counts['motor_count']} motors, " + f"{counts['curve_count']} curves, {counts['point_count']} thrust points" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/security/motor-database-signing-public-key.pem b/security/motor-database-signing-public-key.pem new file mode 100644 index 0000000..fbc1e20 --- /dev/null +++ b/security/motor-database-signing-public-key.pem @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA6CLrOzMhnpYon+H01z3uzRp6sjEHii608GdtTbCLRGs= +-----END PUBLIC KEY----- diff --git a/tests/test_fetch_updates.py b/tests/test_fetch_updates.py index 87f12fb..5b12ad6 100644 --- a/tests/test_fetch_updates.py +++ b/tests/test_fetch_updates.py @@ -12,15 +12,6 @@ spec.loader.exec_module(fetch_updates) -class DummyResponse: - def __init__(self, status_code, payload): - self.status_code = status_code - self._payload = payload - - def json(self): - return self._payload - - def test_load_state_handles_missing_and_corrupt(tmp_path, monkeypatch): state_file = tmp_path / "state.json" monkeypatch.setattr(fetch_updates, "STATE_LAST_UPDATE_FILE", str(state_file)) @@ -48,11 +39,12 @@ def test_get_manufacturers_saves_list(tmp_path, monkeypatch): manuf_file = tmp_path / "data" / "manufacturers.json" monkeypatch.setattr(fetch_updates, "MANUFACTURERS_FILE", str(manuf_file)) - def fake_get(url, json=None, headers=None): + def fake_request(url, payload, method): assert url == fetch_updates.TC_API_METADATA - return DummyResponse(200, {"manufacturers": [{"name": "Acme", "abbrev": "AC"}]}) + assert method == "GET" + return 200, {"manufacturers": [{"name": "Acme", "abbrev": "AC"}]} - monkeypatch.setattr(fetch_updates.requests, "get", fake_get) + monkeypatch.setattr(fetch_updates, "request_json", fake_request) names = fetch_updates.get_manufacturers() @@ -68,9 +60,10 @@ def test_download_motor_data_writes_files_and_mapping(tmp_path, monkeypatch): content = "F32 29 124 0 0.05 0.07 Test Motors\n0 5\n0.5 0\n" encoded = base64.b64encode(content.encode("utf-8")).decode("utf-8") - def fake_post(url, json=None, headers=None): + def fake_request(url, payload, method): assert url == fetch_updates.TC_API_DOWNLOAD - return DummyResponse( + assert method == "POST" + return ( 200, { "results": [ @@ -87,7 +80,7 @@ def fake_post(url, json=None, headers=None): }, ) - monkeypatch.setattr(fetch_updates.requests, "post", fake_post) + monkeypatch.setattr(fetch_updates, "request_json", fake_request) mapping = {} saved_count, simfile_ids = fetch_updates.download_motor_data( @@ -114,10 +107,11 @@ def test_download_motor_data_writes_multiple_formats(tmp_path, monkeypatch): "" ) - def fake_post(url, json=None, headers=None): + def fake_request(url, payload, method): assert url == fetch_updates.TC_API_DOWNLOAD - assert "format" not in json - return DummyResponse( + assert "format" not in payload + assert method == "POST" + return ( 200, { "results": [ @@ -137,7 +131,7 @@ def fake_post(url, json=None, headers=None): }, ) - monkeypatch.setattr(fetch_updates.requests, "post", fake_post) + monkeypatch.setattr(fetch_updates, "request_json", fake_request) mapping = {} saved_count, simfile_ids = fetch_updates.download_motor_data( @@ -176,16 +170,16 @@ def test_fetch_motors_saves_metadata_mapping_and_state(tmp_path, monkeypatch): str(data_dir / "manufacturers.json"), ) - def fake_get(url, json=None, headers=None): - assert url == fetch_updates.TC_API_METADATA - return DummyResponse(200, {"manufacturers": [{"name": "Acme", "abbrev": "AC"}]}) - content = "G64 29 150 0 0.08 0.1 Acme\n0 10\n0.6 0\n" encoded = base64.b64encode(content.encode("utf-8")).decode("utf-8") - def fake_post(url, json=None, headers=None): + def fake_request(url, payload, method): + if url == fetch_updates.TC_API_METADATA: + assert method == "GET" + return 200, {"manufacturers": [{"name": "Acme", "abbrev": "AC"}]} if url == fetch_updates.TC_API_SEARCH: - return DummyResponse( + assert method == "POST" + return ( 200, { "results": [ @@ -217,8 +211,9 @@ def fake_post(url, json=None, headers=None): }, ) if url == fetch_updates.TC_API_DOWNLOAD: - assert "format" not in json - return DummyResponse( + assert "format" not in payload + assert method == "POST" + return ( 200, { "results": [ @@ -236,8 +231,7 @@ def fake_post(url, json=None, headers=None): ) raise AssertionError(f"Unexpected URL: {url}") - monkeypatch.setattr(fetch_updates.requests, "get", fake_get) - monkeypatch.setattr(fetch_updates.requests, "post", fake_post) + monkeypatch.setattr(fetch_updates, "request_json", fake_request) monkeypatch.setattr(fetch_updates.time, "sleep", lambda _: None) fetch_updates.fetch_motors() diff --git a/tests/test_sign_database.py b/tests/test_sign_database.py index d192d06..f0788f4 100644 --- a/tests/test_sign_database.py +++ b/tests/test_sign_database.py @@ -3,10 +3,9 @@ import importlib.util import json from pathlib import Path +import subprocess import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ed25519 MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "sign_database.py" @@ -17,14 +16,27 @@ spec.loader.exec_module(sign_db) -def _generate_private_key_b64(): - private_key = ed25519.Ed25519PrivateKey.generate() - private_key_bytes = private_key.private_bytes( - encoding=serialization.Encoding.DER, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), +def _generate_private_key_b64(tmp_path): + private_key_path = tmp_path / "private-key.pem" + private_key_der_path = tmp_path / "private-key.der" + public_key_path = tmp_path / "public-key.pem" + subprocess.run( + ["openssl", "genpkey", "-algorithm", "Ed25519", "-out", str(private_key_path)], + check=True, + capture_output=True, ) - return private_key, base64.b64encode(private_key_bytes).decode("utf-8") + subprocess.run( + ["openssl", "pkey", "-in", str(private_key_path), "-outform", "DER", "-out", str(private_key_der_path)], + check=True, + capture_output=True, + ) + subprocess.run( + ["openssl", "pkey", "-in", str(private_key_path), "-pubout", "-out", str(public_key_path)], + check=True, + capture_output=True, + ) + private_key_b64 = base64.b64encode(private_key_der_path.read_bytes()).decode("utf-8") + return private_key_b64, public_key_path def test_compute_sha256_hex(tmp_path): @@ -41,7 +53,7 @@ def test_load_private_key_rejects_invalid_base64(): def test_sign_metadata_writes_signature(tmp_path): - private_key, key_b64 = _generate_private_key_b64() + key_b64, public_key_path = _generate_private_key_b64(tmp_path) gz_path = tmp_path / "motors.db.gz" meta_path = tmp_path / "metadata.json" gz_path.write_bytes(b"payload") @@ -59,12 +71,22 @@ def test_sign_metadata_writes_signature(tmp_path): assert metadata["key_id"] == "1" message = f"{sign_db.MESSAGE_PREFIX}\n{metadata['database_version']}\n{sha}\n" - signature = base64.b64decode(metadata["sig"]) - private_key.public_key().verify(signature, message.encode("utf-8")) + message_path = tmp_path / "message" + signature_path = tmp_path / "signature" + message_path.write_text(message) + signature_path.write_bytes(base64.b64decode(metadata["sig"])) + subprocess.run( + [ + "openssl", "pkeyutl", "-verify", "-pubin", "-rawin", + "-inkey", str(public_key_path), "-in", str(message_path), "-sigfile", str(signature_path), + ], + check=True, + capture_output=True, + ) def test_sign_metadata_missing_database_version(tmp_path): - _, key_b64 = _generate_private_key_b64() + key_b64, _ = _generate_private_key_b64(tmp_path) gz_path = tmp_path / "motors.db.gz" meta_path = tmp_path / "metadata.json" gz_path.write_bytes(b"payload") diff --git a/tests/test_validate_release.py b/tests/test_validate_release.py new file mode 100644 index 0000000..73bf152 --- /dev/null +++ b/tests/test_validate_release.py @@ -0,0 +1,173 @@ +import base64 +import gzip +import hashlib +import importlib.util +import json +from pathlib import Path +import sqlite3 +import subprocess + +import pytest + + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "validate_release.py" +spec = importlib.util.spec_from_file_location("validate_release", MODULE_PATH) +if spec is None or spec.loader is None: + raise RuntimeError("Unable to load validate_release module") +validator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validator) + + +def _create_release(tmp_path, database_version=20240101010101): + database_path = tmp_path / "motors.db" + connection = sqlite3.connect(database_path) + connection.executescript( + """ + PRAGMA foreign_keys = ON; + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE manufacturers (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, abbrev TEXT); + CREATE TABLE motors ( + id INTEGER PRIMARY KEY, manufacturer_id INTEGER NOT NULL, tc_motor_id TEXT, + designation TEXT NOT NULL, common_name TEXT, impulse_class TEXT, diameter REAL, + length REAL, total_impulse REAL, avg_thrust REAL, max_thrust REAL, burn_time REAL, + propellant_weight REAL, total_weight REAL, type TEXT, delays TEXT, case_info TEXT, + prop_info TEXT, sparky INTEGER, info_url TEXT, data_files INTEGER, updated_on TEXT, + FOREIGN KEY (manufacturer_id) REFERENCES manufacturers(id) + ); + CREATE TABLE thrust_curves ( + id INTEGER PRIMARY KEY, motor_id INTEGER NOT NULL, tc_simfile_id TEXT, source TEXT, + format TEXT, license TEXT, info_url TEXT, data_url TEXT, total_impulse REAL, + avg_thrust REAL, max_thrust REAL, burn_time REAL, + FOREIGN KEY (motor_id) REFERENCES motors(id) + ); + CREATE TABLE thrust_data ( + id INTEGER PRIMARY KEY, curve_id INTEGER NOT NULL, time_seconds REAL NOT NULL, + force_newtons REAL NOT NULL, FOREIGN KEY (curve_id) REFERENCES thrust_curves(id) + ); + INSERT INTO manufacturers VALUES (1, 'Test Motors', 'TM'); + INSERT INTO motors (id, manufacturer_id, designation) VALUES (1, 1, 'A1'); + INSERT INTO thrust_curves (id, motor_id) VALUES (1, 1); + INSERT INTO thrust_data VALUES (1, 1, 0.0, 0.0); + INSERT INTO thrust_data VALUES (2, 1, 1.0, 1.0); + """ + ) + metadata_values = { + "schema_version": 2, + "database_version": database_version, + "motor_count": 1, + "curve_count": 1, + } + connection.executemany( + "INSERT INTO meta (key, value) VALUES (?, ?)", + [(key, str(value)) for key, value in metadata_values.items()], + ) + connection.commit() + connection.close() + + compressed_path = tmp_path / "motors.db.gz" + with open(database_path, "rb") as database_file, gzip.open(compressed_path, "wb") as compressed_file: + compressed_file.write(database_file.read()) + sha256_gz = hashlib.sha256(compressed_path.read_bytes()).hexdigest() + metadata = { + **metadata_values, + "sha256": sha256_gz, + "sha256_gz": sha256_gz, + "download_url": "https://openrocket.github.io/motor-database/motors.db.gz", + } + metadata_path = tmp_path / "metadata.json" + metadata_path.write_text(json.dumps(metadata)) + return compressed_path, metadata_path + + +def _validate_small_release(compressed_path, metadata_path, **kwargs): + return validator.validate_release( + compressed_path, + metadata_path, + minimum_motors=1, + minimum_curves=1, + minimum_points=2, + **kwargs, + ) + + +def _generate_signing_key(tmp_path): + private_key_path = tmp_path / "private.pem" + private_key_der_path = tmp_path / "private.der" + public_key_path = tmp_path / "public.pem" + subprocess.run( + ["openssl", "genpkey", "-algorithm", "Ed25519", "-out", str(private_key_path)], + check=True, + capture_output=True, + ) + subprocess.run( + ["openssl", "pkey", "-in", str(private_key_path), "-outform", "DER", "-out", str(private_key_der_path)], + check=True, + capture_output=True, + ) + subprocess.run( + ["openssl", "pkey", "-in", str(private_key_path), "-pubout", "-out", str(public_key_path)], + check=True, + capture_output=True, + ) + return base64.b64encode(private_key_der_path.read_bytes()).decode("utf-8"), public_key_path + + +def test_validate_release_accepts_consistent_database(tmp_path): + compressed_path, metadata_path = _create_release(tmp_path) + + counts = _validate_small_release(compressed_path, metadata_path) + + assert counts == {"motor_count": 1, "curve_count": 1, "point_count": 2} + + +def test_validate_release_rejects_metadata_database_version_mismatch(tmp_path): + compressed_path, metadata_path = _create_release(tmp_path) + metadata = json.loads(metadata_path.read_text()) + metadata["database_version"] += 1 + metadata_path.write_text(json.dumps(metadata)) + + with pytest.raises(validator.ValidationError, match="database_version"): + _validate_small_release(compressed_path, metadata_path) + + +def test_validate_release_rejects_large_count_drop(tmp_path): + compressed_path, metadata_path = _create_release(tmp_path) + baseline_path = tmp_path / "baseline.json" + baseline_path.write_text(json.dumps({"motor_count": 2, "curve_count": 2})) + + with pytest.raises(validator.ValidationError, match="dropped"): + _validate_small_release(compressed_path, metadata_path, baseline_path=baseline_path) + + +def test_validate_release_verifies_signature(tmp_path): + compressed_path, metadata_path = _create_release(tmp_path) + private_key_b64, public_key_path = _generate_signing_key(tmp_path) + + sign_path = Path(__file__).resolve().parents[1] / "scripts" / "sign_database.py" + sign_spec = importlib.util.spec_from_file_location("sign_database_for_validator", sign_path) + sign_module = importlib.util.module_from_spec(sign_spec) + sign_spec.loader.exec_module(sign_module) + sign_module.sign_metadata(compressed_path, metadata_path, private_key_b64=private_key_b64) + + _validate_small_release( + compressed_path, + metadata_path, + require_signature=True, + public_key_path=public_key_path, + ) + + +def test_validate_release_rejects_invalid_signature(tmp_path): + compressed_path, metadata_path = _create_release(tmp_path) + metadata = json.loads(metadata_path.read_text()) + metadata["sig"] = base64.b64encode(bytes(64)).decode("utf-8") + metadata_path.write_text(json.dumps(metadata)) + _, public_key_path = _generate_signing_key(tmp_path) + + with pytest.raises(validator.ValidationError, match="signature verification"): + _validate_small_release( + compressed_path, + metadata_path, + require_signature=True, + public_key_path=public_key_path, + )