fix(db): review follow-ups for Cosmian/kms#1224 — deterministic tag-join SQL, metrics option in TOML templates

.github/workflows/main_base.yml

@@ -31,37 +31,6 @@ jobs:
     with:
       toolchain: ${{ inputs.toolchain }}
 
-  log-reference:
-    name: Log index — log-reference.md in sync with source
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          submodules: recursive
-
-      - name: Check log-reference.md is up to date
-        id: check
-        run: python3 .mise/scripts/docs/update_log_index.py --check --no-color
-
-      - name: How to fix
-        if: failure()
-        run: |
-          echo ""
-          echo "════════════════════════════════════════════════════════════"
-          echo "  log-reference.md is out of sync with the source code."
-          echo ""
-          echo "  Fix it locally by running:"
-          echo ""
-          echo "    python3 .mise/scripts/docs/update_log_index.py --non-interactive --no-color"
-          echo ""
-          echo "  Then review the diff, stage, and commit:"
-          echo ""
-          echo "    git diff documentation/docs/configuration/log-reference.md"
-          echo "    git add documentation/docs/configuration/log-reference.md"
-          echo "    git commit -m 'docs: sync log-reference.md'"
-          echo ""
-          echo "════════════════════════════════════════════════════════════"
-
   forward-proxy:
     uses: ./.github/workflows/forward_proxy.yml
     with:

.github/workflows/nightly.yml

@@ -2,8 +2,9 @@
 name: Nightly Packaging
 
 # Triggered nightly (schedule), manually, on tag pushes, or on push to
-# develop/main.  Bot commits from nix-update-hashes.yml carry [skip ci] so
-# they do NOT re-trigger this workflow.
+# develop/main.  Bot commits from nix-update-hashes.yml and
+# update-log-index.yml carry [skip ci] so they do NOT re-trigger this
+# workflow.
 
 on:
   push:
@@ -21,7 +22,8 @@ jobs:
   # ─────────────────────────────────────────────────────────────────────────
   # Non-tag refs (branches, schedule, workflow_dispatch):
   #   1. Recompute Nix vendor hashes on the current branch and commit them.
-  #   2. Dispatch packaging.yml once both platform jobs have finished.
+  #   2. Sync log-reference.md with source call-sites and commit it.
+  #   3. Dispatch packaging.yml once both hash-update platform jobs have finished.
   #
   # Packaging is triggered by nix-update-hashes.yml (trigger_packaging=true)
   # so this workflow does NOT call packaging.yml directly for the branch case.
@@ -34,6 +36,17 @@ jobs:
       trigger_packaging: true
     secrets: inherit
 
+  # ─────────────────────────────────────────────────────────────────────────
+  # Non-tag refs only: sync log-reference.md with source call-sites and
+  # commit the result (tag refs are already in sync via release.yml).
+  # ─────────────────────────────────────────────────────────────────────────
+  update-log-index:
+    if: "!startsWith(github.ref, 'refs/tags/')"
+    uses: ./.github/workflows/update-log-index.yml
+    with:
+      ref: ${{ github.ref_name }}
+    secrets: inherit
+
   # ─────────────────────────────────────────────────────────────────────────
   # Tag refs only:
   #   Nix hashes are already committed by release.yml — skip the hash update

.github/workflows/release.yml

@@ -236,6 +236,23 @@ jobs:
     secrets:
       PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
 
+  # ─────────────────────────────────────────────────────────────────────────────
+  # Job 2b — Sync log-reference.md with source call-sites.
+  #
+  # Delegated to the local reusable workflow update-log-index.yml. Sequenced
+  # after update-nix-hashes so the two do not push to the release branch
+  # concurrently (both still rebase before pushing, but this avoids the race
+  # in the common case).
+  # ─────────────────────────────────────────────────────────────────────────────
+  update-log-index:
+    name: Update log index
+    needs: [prepare, update-nix-hashes]
+    uses: ./.github/workflows/update-log-index.yml
+    with:
+      ref: ${{ needs.prepare.outputs.release_branch }}
+    secrets:
+      PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
+
   # ─────────────────────────────────────────────────────────────────────────────
   # Job 3 — Trigger packaging CI on the release branch and wait for completion
   #
@@ -248,7 +265,7 @@ jobs:
   # ─────────────────────────────────────────────────────────────────────────────
   trigger-packaging:
     name: Trigger & await packaging CI
-    needs: [prepare, update-nix-hashes]
+    needs: [prepare, update-nix-hashes, update-log-index]
     # nix-update-hashes.yml runs a Linux+macOS matrix internally; GitHub waits
     # for all instances before starting trigger-packaging.
     runs-on: ubuntu-latest

.github/workflows/test_all.yml

@@ -23,7 +23,6 @@ jobs:
           - mariadb
           - psql
           - otel
-          - google-cse
           - redis
           - pykmip
           - wasm
@@ -38,9 +37,6 @@ jobs:
           - iris
           - db2
           - ase
-          - secret_vault
-          - secret_aws
-          - secret_azure
           - secret_cosmian_kms
           - spire
           - kmip-go
@@ -86,13 +82,8 @@ jobs:
           - type: ase
             features: fips
           # secret_cosmian_kms runs against a local KMS server — works with both fips and non-fips
-          # secret_vault, secret_aws, secret_azure require external services — run non-fips only
-          - type: secret_vault
-            features: fips
-          - type: secret_aws
-            features: fips
-          - type: secret_azure
-            features: fips
+          # google-cse, secret_vault, secret_aws and secret_azure need upstream-only secrets:
+          # they run in the `test-nix-upstream` job, which is skipped on forks.
           # spire relies on the Vault API which is non-fips only
           - type: spire
             features: fips
@@ -209,33 +200,84 @@ jobs:
           set -ex
           mise run test:${{ matrix.type }} --variant ${{ matrix.features }}
 
+  # Test types that depend on upstream-only secrets / external services. Kept out of
+  # `test-nix` because a job-level `if` cannot read the `matrix` context: the whole
+  # job is skipped on forks, where those secrets do not exist.
+  test-nix-upstream:
+    name: Test on ${{ matrix.type }} - ${{ matrix.features }}
+    runs-on: ubuntu-latest
+    if: github.repository == 'Cosmian/kms'
+    strategy:
+      fail-fast: false
+      matrix:
+        type:
+          - google-cse
+          - secret_vault
+          - secret_aws
+          - secret_azure
+        features: [fips, non-fips]
+        exclude:
+          # secret_vault, secret_aws, secret_azure require external services — run non-fips only
+          - type: secret_vault
+            features: fips
+          - type: secret_aws
+            features: fips
+          - type: secret_azure
+            features: fips
+
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - uses: ./.github/actions/cleanup-runner
+
+      - uses: ./.github/actions/setup-nix
+
+      - uses: ./.github/actions/install-mise
+
+      - name: Test
+        env:
+          # Google variables (google-cse test type)
+          TEST_GOOGLE_OAUTH_CLIENT_ID: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_ID }}
+          TEST_GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_SECRET }}
+          TEST_GOOGLE_OAUTH_REFRESH_TOKEN: ${{ secrets.TEST_GOOGLE_OAUTH_REFRESH_TOKEN }}
+          GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY }}
+
+          # AWS secret backend variables (secret_aws test type)
+          AWS_ACCESS_KEY_ID: ${{ secrets.KMS_CI_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.KMS_CI_AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.KMS_CI_AWS_REGION }}
+
+          # Azure Key Vault secret backend variables (secret_azure test type)
+          AZURE_TENANT_ID: ${{ secrets.KMS_CI_AZURE_TENANT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.KMS_CI_AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.KMS_CI_AZURE_CLIENT_SECRET }}
+          AZURE_KV_NAME: ${{ secrets.KMS_CI_AZURE_KV_NAME }}
+
+          # Provide an authenticated token so mise can install tools from
+          # GitHub releases without hitting the unauthenticated rate limit.
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          set -ex
+          mise run test:${{ matrix.type }} --variant ${{ matrix.features }}
+
   hsm:
     name: HSM ${{ matrix.hsm-type }} - ${{ matrix.features }}
     runs-on: ubuntu-latest
-    # proteccio and crypt2pay hardware do not support concurrent connections from multiple CI runs;
-    # give them fixed concurrency groups so only one job runs at a time across all PRs.
-    # utimaco and softhsm2 use a per-run group so they are never blocked by other PRs.
+    # Software/simulated HSMs: a per-run group so they are never blocked by other PRs.
+    # Hardware HSMs (proteccio, crypt2pay, aws-cloudhsm) run in the `hsm-upstream` job.
     concurrency:
-      group: ${{ (matrix.hsm-type == 'proteccio' && 'hsm-proteccio') || (matrix.hsm-type == 'crypt2pay' && 'hsm-crypt2pay') || format('hsm-{0}-{1}', matrix.hsm-type,
-        github.run_id) }}
+      group: ${{ format('hsm-{0}-{1}', matrix.hsm-type, github.run_id) }}
       queue: max
       cancel-in-progress: false
     strategy:
       fail-fast: false
       matrix:
         hsm-type:
           - utimaco
-          - proteccio
           - softhsm2
-          - crypt2pay
         features: [fips, non-fips]
-        exclude:
-          # parallel connections on proteccio is not supported
-          - hsm-type: proteccio
-            features: fips
-          # Not required - testing non-fips is sufficient
-          - hsm-type: crypt2pay
-            features: fips
 
     steps:
       - uses: actions/checkout@v7
@@ -250,18 +292,74 @@ jobs:
 
       - name: Test
         env:
-          # HSM
-          PROTECCIO_IP: ${{ secrets.PROTECCIO_IP }}
-          PROTECCIO_PASSWORD: ${{ secrets.PROTECCIO_PASSWORD }}
-          PROTECCIO_SLOT: ${{ secrets.PROTECCIO_SLOT }}
-          CRYPT2PAY_PASSWORD: ${{ secrets.CRYPT2PAY_PASSWORD }}
           # Google variables
           TEST_GOOGLE_OAUTH_CLIENT_ID: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_ID }}
           TEST_GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.TEST_GOOGLE_OAUTH_CLIENT_SECRET }}
           TEST_GOOGLE_OAUTH_REFRESH_TOKEN: ${{ secrets.TEST_GOOGLE_OAUTH_REFRESH_TOKEN }}
           GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY }}
-          # OVPN for Crypt2Pay HSM tests
+        run: |
+          mise run test:hsm-${{ matrix.hsm-type }} --variant ${{ matrix.features }}
+
+  # Hardware HSMs reachable only with upstream-only secrets and infrastructure (Proteccio
+  # network HSM, Crypt2Pay over OpenVPN, the AWS CloudHSM CI cluster). They live in their
+  # own job (a job-level `if` cannot read the `matrix` context) and are skipped on forks.
+  hsm-upstream:
+    name: HSM ${{ matrix.hsm-type }} - ${{ matrix.features }}
+    runs-on: ubuntu-latest
+    if: github.repository == 'Cosmian/kms'
+    # This hardware does not support concurrent connections from multiple CI runs: a fixed
+    # group per HSM so only one job runs at a time across all PRs.
+    concurrency:
+      group: ${{ format('hsm-{0}', matrix.hsm-type) }}
+      queue: max
+      cancel-in-progress: false
+    strategy:
+      fail-fast: false
+      matrix:
+        hsm-type:
+          - proteccio
+          - crypt2pay
+          - aws-cloudhsm
+        # non-fips only: parallel connections on proteccio are not supported, non-fips is
+        # sufficient for crypt2pay, and aws-cloudhsm FIPS-mode compatibility is not yet
+        # confirmed against the cluster's supported mechanism list.
+        features: [non-fips]
+
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - uses: ./.github/actions/cleanup-runner
+
+      - uses: ./.github/actions/setup-nix
+
+      - uses: ./.github/actions/install-mise
+
+      - name: Test
+        env:
+          # Proteccio
+          PROTECCIO_IP: ${{ secrets.PROTECCIO_IP }}
+          PROTECCIO_PASSWORD: ${{ secrets.PROTECCIO_PASSWORD }}
+          PROTECCIO_SLOT: ${{ secrets.PROTECCIO_SLOT }}
+          # Crypt2Pay (reached through OpenVPN)
+          CRYPT2PAY_PASSWORD: ${{ secrets.CRYPT2PAY_PASSWORD }}
           OVPN_CONF: ${{ secrets.OVPN_CONF }}
+          # AWS CloudHSM: persistent CI cluster (see crate/hsm/aws_cloudhsm/README.md)
+          AWS_CLOUDHSM_CLUSTER_ID: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CLUSTER_ID }}
+          AWS_CLOUDHSM_CU_USERNAME: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CU_USERNAME }}
+          AWS_CLOUDHSM_CU_PASSWORD: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CU_PASSWORD }}
+          AWS_CLOUDHSM_SLOT_ID: ${{ secrets.KMS_CI_AWS_CLOUDHSM_SLOT_ID }}
+          AWS_CLOUDHSM_CA_CERT: ${{ secrets.KMS_CI_AWS_CLOUDHSM_CA_CERT }}
+          AWS_CLOUDHSM_HSM_IPS: ${{ secrets.KMS_CI_AWS_CLOUDHSM_HSM_IPS }}
+          # AWS Client VPN profile (.ovpn, cert-based mutual auth): GitHub-hosted
+          # runners have no route to the HSM's private VPC IP; prepare_aws_cloudhsm.sh
+          # starts this tunnel only if the HSM is not already directly reachable.
+          AWS_CLOUDHSM_OVPN_CONF: ${{ secrets.KMS_CI_AWS_CLOUDHSM_OVPN_CONF }}
+          # Reused generic AWS credentials (read-only `cloudhsm:DescribeClusters` is enough)
+          AWS_ACCESS_KEY_ID: ${{ secrets.KMS_CI_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.KMS_CI_AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.KMS_CI_AWS_REGION }}
         run: |
           mise run test:hsm-${{ matrix.hsm-type }} --variant ${{ matrix.features }}
 
@@ -302,7 +400,9 @@ jobs:
     permissions:
       contents: read
     environment: xks-remote-approval
+    # Upstream only: needs the XKS test server and its secrets, which forks do not have.
     if: >-
+      github.repository == 'Cosmian/kms' &&
       github.actor != 'dependabot[bot]' &&
       (github.event_name != 'pull_request' ||
        github.event.pull_request.head.repo.full_name == github.repository)
@@ -361,8 +461,13 @@ jobs:
   cargo-publish:
     needs:
       - test-nix
+      - test-nix-upstream
       - hsm
+      - hsm-upstream
       - helm
+    # The upstream-only jobs are skipped on forks: a skipped dependency must not skip the
+    # publish dry-run, but any failed or cancelled dependency still blocks it.
+    if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/cargo-publish.yml
     with:
       toolchain: 1.97.0