diff --git a/.github/workflows/main_base.yml b/.github/workflows/main_base.yml index 3efe4a7868..6203c549d9 100644 --- a/.github/workflows/main_base.yml +++ b/.github/workflows/main_base.yml @@ -31,37 +31,6 @@ jobs: with: toolchain: ${{ inputs.toolchain }} - log-reference: - name: Log index — log-reference.md in sync with source - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - submodules: recursive - - - name: Check log-reference.md is up to date - id: check - run: python3 .mise/scripts/docs/update_log_index.py --check --no-color - - - name: How to fix - if: failure() - run: | - echo "" - echo "════════════════════════════════════════════════════════════" - echo " log-reference.md is out of sync with the source code." - echo "" - echo " Fix it locally by running:" - echo "" - echo " python3 .mise/scripts/docs/update_log_index.py --non-interactive --no-color" - echo "" - echo " Then review the diff, stage, and commit:" - echo "" - echo " git diff documentation/docs/configuration/log-reference.md" - echo " git add documentation/docs/configuration/log-reference.md" - echo " git commit -m 'docs: sync log-reference.md'" - echo "" - echo "════════════════════════════════════════════════════════════" - forward-proxy: uses: ./.github/workflows/forward_proxy.yml with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1f35ddf6e9..3ee89f3850 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,8 +2,9 @@ name: Nightly Packaging # Triggered nightly (schedule), manually, on tag pushes, or on push to -# develop/main. Bot commits from nix-update-hashes.yml carry [skip ci] so -# they do NOT re-trigger this workflow. +# develop/main. Bot commits from nix-update-hashes.yml and +# update-log-index.yml carry [skip ci] so they do NOT re-trigger this +# workflow. on: push: @@ -21,7 +22,8 @@ jobs: # ───────────────────────────────────────────────────────────────────────── # Non-tag refs (branches, schedule, workflow_dispatch): # 1. Recompute Nix vendor hashes on the current branch and commit them. - # 2. Dispatch packaging.yml once both platform jobs have finished. + # 2. Sync log-reference.md with source call-sites and commit it. + # 3. Dispatch packaging.yml once both hash-update platform jobs have finished. # # Packaging is triggered by nix-update-hashes.yml (trigger_packaging=true) # so this workflow does NOT call packaging.yml directly for the branch case. @@ -34,6 +36,17 @@ jobs: trigger_packaging: true secrets: inherit + # ───────────────────────────────────────────────────────────────────────── + # Non-tag refs only: sync log-reference.md with source call-sites and + # commit the result (tag refs are already in sync via release.yml). + # ───────────────────────────────────────────────────────────────────────── + update-log-index: + if: "!startsWith(github.ref, 'refs/tags/')" + uses: ./.github/workflows/update-log-index.yml + with: + ref: ${{ github.ref_name }} + secrets: inherit + # ───────────────────────────────────────────────────────────────────────── # Tag refs only: # Nix hashes are already committed by release.yml — skip the hash update diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2898615947..e25f71173c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -236,6 +236,23 @@ jobs: secrets: PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + # ───────────────────────────────────────────────────────────────────────────── + # Job 2b — Sync log-reference.md with source call-sites. + # + # Delegated to the local reusable workflow update-log-index.yml. Sequenced + # after update-nix-hashes so the two do not push to the release branch + # concurrently (both still rebase before pushing, but this avoids the race + # in the common case). + # ───────────────────────────────────────────────────────────────────────────── + update-log-index: + name: Update log index + needs: [prepare, update-nix-hashes] + uses: ./.github/workflows/update-log-index.yml + with: + ref: ${{ needs.prepare.outputs.release_branch }} + secrets: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + # ───────────────────────────────────────────────────────────────────────────── # Job 3 — Trigger packaging CI on the release branch and wait for completion # @@ -248,7 +265,7 @@ jobs: # ───────────────────────────────────────────────────────────────────────────── trigger-packaging: name: Trigger & await packaging CI - needs: [prepare, update-nix-hashes] + needs: [prepare, update-nix-hashes, update-log-index] # nix-update-hashes.yml runs a Linux+macOS matrix internally; GitHub waits # for all instances before starting trigger-packaging. runs-on: ubuntu-latest diff --git a/.github/workflows/test_all.yml b/.github/workflows/test_all.yml index a770d42a66..a37fcf604b 100644 --- a/.github/workflows/test_all.yml +++ b/.github/workflows/test_all.yml @@ -23,7 +23,6 @@ jobs: - mariadb - psql - otel - - google-cse - redis - pykmip - wasm @@ -38,9 +37,6 @@ jobs: - iris - db2 - ase - - secret_vault - - secret_aws - - secret_azure - secret_cosmian_kms - spire - kmip-go @@ -86,13 +82,8 @@ jobs: - type: ase features: fips # secret_cosmian_kms runs against a local KMS server — works with both fips and non-fips - # secret_vault, secret_aws, secret_azure require external services — run non-fips only - - type: secret_vault - features: fips - - type: secret_aws - features: fips - - type: secret_azure - features: fips + # google-cse, secret_vault, secret_aws and secret_azure need upstream-only secrets: + # they run in the `test-nix-upstream` job, which is skipped on forks. # spire relies on the Vault API which is non-fips only - type: spire features: fips @@ -209,15 +200,75 @@ jobs: set -ex mise run test:${{ matrix.type }} --variant ${{ matrix.features }} + # Test types that depend on upstream-only secrets / external services. Kept out of + # `test-nix` because a job-level `if` cannot read the `matrix` context: the whole + # job is skipped on forks, where those secrets do not exist. + test-nix-upstream: + name: Test on ${{ matrix.type }} - ${{ matrix.features }} + runs-on: ubuntu-latest + if: github.repository == 'Cosmian/kms' + strategy: + fail-fast: false + matrix: + type: + - google-cse + - secret_vault + - secret_aws + - secret_azure + features: [fips, non-fips] + exclude: + # secret_vault, secret_aws, secret_azure require external services — run non-fips only + - type: secret_vault + features: fips + - type: secret_aws + features: fips + - type: secret_azure + features: fips + + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + + - uses: ./.github/actions/cleanup-runner + + - uses: ./.github/actions/setup-nix + + - uses: ./.github/actions/install-mise + + - name: Test + env: + # Google variables (google-cse test type) + TEST_GOOGLE_OAUTH_CLIENT_ID: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_ID }} + TEST_GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_SECRET }} + TEST_GOOGLE_OAUTH_REFRESH_TOKEN: ${{ secrets.TEST_GOOGLE_OAUTH_REFRESH_TOKEN }} + GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY }} + + # AWS secret backend variables (secret_aws test type) + AWS_ACCESS_KEY_ID: ${{ secrets.KMS_CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.KMS_CI_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.KMS_CI_AWS_REGION }} + + # Azure Key Vault secret backend variables (secret_azure test type) + AZURE_TENANT_ID: ${{ secrets.KMS_CI_AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.KMS_CI_AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.KMS_CI_AZURE_CLIENT_SECRET }} + AZURE_KV_NAME: ${{ secrets.KMS_CI_AZURE_KV_NAME }} + + # Provide an authenticated token so mise can install tools from + # GitHub releases without hitting the unauthenticated rate limit. + GITHUB_TOKEN: ${{ github.token }} + run: | + set -ex + mise run test:${{ matrix.type }} --variant ${{ matrix.features }} + hsm: name: HSM ${{ matrix.hsm-type }} - ${{ matrix.features }} runs-on: ubuntu-latest - # proteccio and crypt2pay hardware do not support concurrent connections from multiple CI runs; - # give them fixed concurrency groups so only one job runs at a time across all PRs. - # utimaco and softhsm2 use a per-run group so they are never blocked by other PRs. + # Software/simulated HSMs: a per-run group so they are never blocked by other PRs. + # Hardware HSMs (proteccio, crypt2pay, aws-cloudhsm) run in the `hsm-upstream` job. concurrency: - group: ${{ (matrix.hsm-type == 'proteccio' && 'hsm-proteccio') || (matrix.hsm-type == 'crypt2pay' && 'hsm-crypt2pay') || format('hsm-{0}-{1}', matrix.hsm-type, - github.run_id) }} + group: ${{ format('hsm-{0}-{1}', matrix.hsm-type, github.run_id) }} queue: max cancel-in-progress: false strategy: @@ -225,17 +276,8 @@ jobs: matrix: hsm-type: - utimaco - - proteccio - softhsm2 - - crypt2pay features: [fips, non-fips] - exclude: - # parallel connections on proteccio is not supported - - hsm-type: proteccio - features: fips - # Not required - testing non-fips is sufficient - - hsm-type: crypt2pay - features: fips steps: - uses: actions/checkout@v7 @@ -250,18 +292,74 @@ jobs: - name: Test env: - # HSM - PROTECCIO_IP: ${{ secrets.PROTECCIO_IP }} - PROTECCIO_PASSWORD: ${{ secrets.PROTECCIO_PASSWORD }} - PROTECCIO_SLOT: ${{ secrets.PROTECCIO_SLOT }} - CRYPT2PAY_PASSWORD: ${{ secrets.CRYPT2PAY_PASSWORD }} # Google variables TEST_GOOGLE_OAUTH_CLIENT_ID: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_ID }} TEST_GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_SECRET }} TEST_GOOGLE_OAUTH_REFRESH_TOKEN: ${{ secrets.TEST_GOOGLE_OAUTH_REFRESH_TOKEN }} GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY }} - # OVPN for Crypt2Pay HSM tests + run: | + mise run test:hsm-${{ matrix.hsm-type }} --variant ${{ matrix.features }} + + # Hardware HSMs reachable only with upstream-only secrets and infrastructure (Proteccio + # network HSM, Crypt2Pay over OpenVPN, the AWS CloudHSM CI cluster). They live in their + # own job (a job-level `if` cannot read the `matrix` context) and are skipped on forks. + hsm-upstream: + name: HSM ${{ matrix.hsm-type }} - ${{ matrix.features }} + runs-on: ubuntu-latest + if: github.repository == 'Cosmian/kms' + # This hardware does not support concurrent connections from multiple CI runs: a fixed + # group per HSM so only one job runs at a time across all PRs. + concurrency: + group: ${{ format('hsm-{0}', matrix.hsm-type) }} + queue: max + cancel-in-progress: false + strategy: + fail-fast: false + matrix: + hsm-type: + - proteccio + - crypt2pay + - aws-cloudhsm + # non-fips only: parallel connections on proteccio are not supported, non-fips is + # sufficient for crypt2pay, and aws-cloudhsm FIPS-mode compatibility is not yet + # confirmed against the cluster's supported mechanism list. + features: [non-fips] + + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + + - uses: ./.github/actions/cleanup-runner + + - uses: ./.github/actions/setup-nix + + - uses: ./.github/actions/install-mise + + - name: Test + env: + # Proteccio + PROTECCIO_IP: ${{ secrets.PROTECCIO_IP }} + PROTECCIO_PASSWORD: ${{ secrets.PROTECCIO_PASSWORD }} + PROTECCIO_SLOT: ${{ secrets.PROTECCIO_SLOT }} + # Crypt2Pay (reached through OpenVPN) + CRYPT2PAY_PASSWORD: ${{ secrets.CRYPT2PAY_PASSWORD }} OVPN_CONF: ${{ secrets.OVPN_CONF }} + # AWS CloudHSM: persistent CI cluster (see crate/hsm/aws_cloudhsm/README.md) + AWS_CLOUDHSM_CLUSTER_ID: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CLUSTER_ID }} + AWS_CLOUDHSM_CU_USERNAME: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CU_USERNAME }} + AWS_CLOUDHSM_CU_PASSWORD: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CU_PASSWORD }} + AWS_CLOUDHSM_SLOT_ID: ${{ secrets.KMS_CI_AWS_CLOUDHSM_SLOT_ID }} + AWS_CLOUDHSM_CA_CERT: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CA_CERT }} + AWS_CLOUDHSM_HSM_IPS: ${{ secrets.KMS_CI_AWS_CLOUDHSM_HSM_IPS }} + # AWS Client VPN profile (.ovpn, cert-based mutual auth): GitHub-hosted + # runners have no route to the HSM's private VPC IP; prepare_aws_cloudhsm.sh + # starts this tunnel only if the HSM is not already directly reachable. + AWS_CLOUDHSM_OVPN_CONF: ${{ secrets.KMS_CI_AWS_CLOUDHSM_OVPN_CONF }} + # Reused generic AWS credentials (read-only `cloudhsm:DescribeClusters` is enough) + AWS_ACCESS_KEY_ID: ${{ secrets.KMS_CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.KMS_CI_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.KMS_CI_AWS_REGION }} run: | mise run test:hsm-${{ matrix.hsm-type }} --variant ${{ matrix.features }} @@ -302,7 +400,9 @@ jobs: permissions: contents: read environment: xks-remote-approval + # Upstream only: needs the XKS test server and its secrets, which forks do not have. if: >- + github.repository == 'Cosmian/kms' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) @@ -361,8 +461,13 @@ jobs: cargo-publish: needs: - test-nix + - test-nix-upstream - hsm + - hsm-upstream - helm + # The upstream-only jobs are skipped on forks: a skipped dependency must not skip the + # publish dry-run, but any failed or cancelled dependency still blocks it. + if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/cargo-publish.yml with: toolchain: 1.97.0 diff --git a/.github/workflows/update-log-index.yml b/.github/workflows/update-log-index.yml new file mode 100644 index 0000000000..61f1aa4bef --- /dev/null +++ b/.github/workflows/update-log-index.yml @@ -0,0 +1,81 @@ +--- +# Local reusable workflow — Update log-reference.md. +# +# Runs `mise run docs:log-index` (non-interactive, deletes stale entries) and +# commits the result back to `ref`. Mirrors nix-update-hashes.yml's +# checkout → run → commit-if-changed → rebase → push flow. +# +# Callers / triggers +# ────────────────── +# nightly.yml — non-tag refs only (branches/schedule/workflow_dispatch); tag +# refs are already in sync via release.yml. +# release.yml — Job: runs on the release branch after `prepare` (and after +# `update-nix-hashes`, to avoid two independent workflows +# pushing to the same release branch at once). + +name: Update log index + +on: + workflow_dispatch: + inputs: + ref: + description: > + Branch to update (leave empty to use the dispatched branch). + required: false + type: string + default: '' + + workflow_call: + inputs: + ref: + description: > + Git ref (branch name) to check out and push the updated + log-reference.md to. + required: true + type: string + secrets: + PAT_TOKEN: + description: > + Personal Access Token with `repo` + `workflow` scopes. + Required so that the push re-triggers other workflows + (GITHUB_TOKEN cannot do this). + required: true + +jobs: + update-log-index: + name: Update log index + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.ref_name }} + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.PAT_TOKEN }} + + - name: Configure git identity + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - uses: ./.github/actions/install-mise + + - name: Update log-reference.md + run: mise run docs:log-index + + - name: Commit updated log index + run: | + REF="${{ inputs.ref || github.ref_name }}" + git add documentation/docs/configuration/log-reference.md + if git diff --cached --quiet; then + echo "log-reference.md is already up-to-date, nothing to commit." + else + # [skip ci] prevents push-triggered workflows (e.g. nightly.yml) + # from re-running on this automated commit and creating a loop. + git commit -m "docs: sync log-reference.md [skip ci]" + # Pull after committing to avoid "index contains uncommitted + # changes" errors if another job pushed to the same ref meanwhile. + git pull --rebase origin "$REF" + git push origin "$REF" + fi diff --git a/CHANGELOG/fix_postgresql.md b/CHANGELOG/fix_postgresql.md index c9bdcb4311..ba31346cd9 100644 --- a/CHANGELOG/fix_postgresql.md +++ b/CHANGELOG/fix_postgresql.md @@ -47,7 +47,9 @@ planner starts from the rarest tag and probes `UNIQUE (id, tag)` for the others), the owner/grant check is an `EXISTS` probe of `read_access` instead of a `LEFT JOIN` plus `DISTINCT`, and the `Locate` path pushes the `MaximumItems`/server cap (`LIMIT`) and the destroyed-object exclusion into the -query so the database stops after the requested page. +query so the database stops after the requested page. The searched tags are +joined in sorted order so that the same search always produces the same SQL +text (prepared-statement cache reuse on SQLite). ## Features @@ -56,4 +58,6 @@ query so the database stops after the requested page. - Add `metrics_count_interval_secs` server setting (`--metrics-count-interval-secs`, default 30) controlling how often the `kms.objects.total` and `kms.keys.active.count` metrics are refreshed from full COUNT queries; `0` - disables both the startup seed and the periodic refresh. + disables both the startup seed and the periodic refresh. The option is + documented in `resources/kms.toml`, `crate/server/kms_template.toml` and + `pkg/kms.toml`. diff --git a/crate/server/kms_template.toml b/crate/server/kms_template.toml index ea4e57dbaa..dd82933da1 100644 --- a/crate/server/kms_template.toml +++ b/crate/server/kms_template.toml @@ -93,6 +93,12 @@ info = false # are promoted to the `CryptoOfficer` role automatically on startup. # privileged_users = ["", ""] +# Interval in seconds between background refreshes of the `kms.objects.total` +# and `kms.keys.active.count` metrics. Each refresh runs a full COUNT over the +# objects table, which is expensive on very large databases. +# Set to 0 to disable both the startup seed and the periodic refresh. +# metrics_count_interval_secs = 30 + # Check the database configuration documentation pages for more information [db] # The main database of the KMS server that holds default cryptographic objects and permissions. diff --git a/crate/server_database/src/stores/sql/locate_query.rs b/crate/server_database/src/stores/sql/locate_query.rs index cdc9ae69a7..8848f4f7d0 100644 --- a/crate/server_database/src/stores/sql/locate_query.rs +++ b/crate/server_database/src/stores/sql/locate_query.rs @@ -486,7 +486,12 @@ fn select_from_objects( FROM objects" ); if let Some(attributes) = attributes { - for (i, tag) in attributes.get_tags(vendor_id).into_iter().enumerate() { + // `get_tags` returns a `HashSet`: sort the tags so that the same search + // always yields the same SQL text, which keeps prepared-statement caches + // (e.g. SQLite `prepare_cached`) effective and query logs comparable. + let mut tags: Vec = attributes.get_tags(vendor_id).into_iter().collect(); + tags.sort_unstable(); + for (i, tag) in tags.into_iter().enumerate() { let tag = qb.bind_text(tag); let _ = write!( query, @@ -706,7 +711,9 @@ mod tests { use cosmian_kmip::kmip_2_1::{extra::tagging::VENDOR_ID_COSMIAN, kmip_attributes::Attributes}; use cosmian_kms_interfaces::FindOptions; - use super::{MySqlPlaceholder, PgSqlPlaceholder, SqlitePlaceholder, query_from_attributes}; + use super::{ + LocateParam, MySqlPlaceholder, PgSqlPlaceholder, SqlitePlaceholder, query_from_attributes, + }; fn tagged_attributes() -> Attributes { let mut attributes = Attributes::default(); @@ -754,6 +761,31 @@ mod tests { assert!(!query.sql.contains("NOT IN"), "{}", query.sql); } + #[test] + fn tag_joins_are_bound_in_sorted_order() { + let query = query_from_attributes::( + Some(&tagged_attributes()), + None, + "alice", + false, + VENDOR_ID_COSMIAN, + &FindOptions::default(), + ); + // Tags are bound first (t0, t1, …), in sorted order whatever the + // iteration order of the tag set. + assert_eq!( + query.params.get(..2), + Some( + &[ + LocateParam::Text("a".to_owned()), + LocateParam::Text("b".to_owned()) + ][..] + ), + "{}", + query.sql + ); + } + #[test] fn mysql_query_uses_question_placeholders() { let query = query_from_attributes::( diff --git a/pkg/kms.toml b/pkg/kms.toml index ea4e57dbaa..dd82933da1 100644 --- a/pkg/kms.toml +++ b/pkg/kms.toml @@ -93,6 +93,12 @@ info = false # are promoted to the `CryptoOfficer` role automatically on startup. # privileged_users = ["", ""] +# Interval in seconds between background refreshes of the `kms.objects.total` +# and `kms.keys.active.count` metrics. Each refresh runs a full COUNT over the +# objects table, which is expensive on very large databases. +# Set to 0 to disable both the startup seed and the periodic refresh. +# metrics_count_interval_secs = 30 + # Check the database configuration documentation pages for more information [db] # The main database of the KMS server that holds default cryptographic objects and permissions.