diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffb5fcb4f1..c1fe570752 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,9 +57,12 @@ jobs: # deliberately unprivileged: it may select ordinary test work, but runner, # action, permission, and command authority remain in this workflow. # - # The first rollout only publishes the matrices and review artifact. The - # existing handwritten matrices remain authoritative until later commits - # consume these outputs after shadow validation on real workflow runs. + # The ordinary build and Miri jobs consume these matrices directly. The + # jobs consume only the checked selectors from each matrix cell and pass them + # back to `cargo-zerocopy`, which reconstructs and executes the complete + # command without interpreting matrix data as shell text. Keep the output + # names coordinated with `tools/zc/src/github.rs` and the two `fromJSON` + # expressions below. plan_ci: name: Plan ordinary CI work runs-on: ubuntu-latest @@ -107,7 +110,7 @@ jobs: # the matrix runs in parallel, so without this pre-computation, most jobs # would duplicate the work of downloading crates from the internet. # Pre-computing ensures that this work only happens once. - needs: build_docker_env + needs: [build_docker_env, plan_ci] permissions: contents: read defaults: @@ -121,231 +124,11 @@ jobs: # development because it means that errors need to be encountered and # fixed one at a time. fail-fast: false - matrix: - # See `INTERNAL.md` for an explanation of these pinned toolchain - # versions. - toolchain: [ - "msrv", - "stable", - "nightly", - - # These are the names of specific Rust versions detected in - # `build.rs`. Each of these represents the minimum Rust version for - # which a particular feature is supported. - "no-zerocopy-simd-x86-avx12-1-89-0", - "no-zerocopy-core-error-1-81-0", - "no-zerocopy-diagnostic-on-unimplemented-1-78-0", - "no-zerocopy-generic-bounds-in-const-fn-1-61-0", - "no-zerocopy-target-has-atomics-1-60-0", - "no-zerocopy-aarch64-simd-1-59-0", - "no-zerocopy-aarch64-simd-be-1-87-0", - "no-zerocopy-panic-in-const-and-vec-try-reserve-1-57-0" - ] - target: &build_targets [ - "i686-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "arm-unknown-linux-gnueabi", - "aarch64-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-pc-windows-msvc", - "thumbv6m-none-eabi", - "wasm32-unknown-unknown" - ] - feature_profile: &feature_profiles ["default", "stable", "all"] - crate: &build_crates ["zerocopy", "zerocopy-derive"] - event_name: [ "${{ github.event_name }}" ] - exclude: - # Exclude any combination which uses a non-nightly toolchain but - # enables nightly features. - - toolchain: "msrv" - feature_profile: "all" - - toolchain: "stable" - feature_profile: "all" - - toolchain: "no-zerocopy-simd-x86-avx12-1-89-0" - feature_profile: "all" - - toolchain: "no-zerocopy-core-error-1-81-0" - feature_profile: "all" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - feature_profile: "all" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - feature_profile: "all" - - toolchain: "no-zerocopy-target-has-atomics-1-60-0" - feature_profile: "all" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - feature_profile: "all" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - feature_profile: "all" - - toolchain: "no-zerocopy-panic-in-const-and-vec-try-reserve-1-57-0" - feature_profile: "all" - # Exclude any combination for the zerocopy-derive crate which - # uses zerocopy features. - - crate: "zerocopy-derive" - feature_profile: "stable" - - crate: "zerocopy-derive" - feature_profile: "all" - # Exclue any combination of zerocopy-derive and any toolchain version - # other than "msrv", "stable", and "nightly". These other versions - # exist to exercise zerocopy behavior which differs by toolchain; - # zerocopy-derive doesn't behave differently on these toolchains. - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-simd-x86-avx12-1-89-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-core-error-1-81-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-target-has-atomics-1-60-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - - crate: "zerocopy-derive" - toolchain: "no-zerocopy-panic-in-const-and-vec-try-reserve-1-57-0" - # Exclude stable/wasm since wasm is no longer provided via rustup on - # stable. - - toolchain: "stable" - target: "wasm32-unknown-unknown" - # Exclude non-aarch64 targets from the `no-zerocopy-aarch64-simd-1-59-0` - # toolchain. - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "i686-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "x86_64-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "arm-unknown-linux-gnueabi" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "powerpc-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "powerpc64-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "riscv64gc-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "s390x-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "x86_64-pc-windows-msvc" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "thumbv6m-none-eabi" - - toolchain: "no-zerocopy-aarch64-simd-1-59-0" - target: "wasm32-unknown-unknown" - # Exclude non-aarch64 targets from the `no-zerocopy-aarch64-simd-be-1-87-0` - # toolchain. - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "i686-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "x86_64-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "arm-unknown-linux-gnueabi" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "powerpc-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "powerpc64-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "riscv64gc-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "s390x-unknown-linux-gnu" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "x86_64-pc-windows-msvc" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "thumbv6m-none-eabi" - - toolchain: "no-zerocopy-aarch64-simd-be-1-87-0" - target: "wasm32-unknown-unknown" - # Exclude most targets from the `no-zerocopy-core-error-1-81-0` - # toolchain since the `no-zerocopy-core-error-1-81-0` feature is unrelated to - # compilation target. This only leaves i686 and x86_64 targets. - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "arm-unknown-linux-gnueabi" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "aarch64-unknown-linux-gnu" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "powerpc-unknown-linux-gnu" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "powerpc64-unknown-linux-gnu" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "riscv64gc-unknown-linux-gnu" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "s390x-unknown-linux-gnu" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "x86_64-pc-windows-msvc" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "thumbv6m-none-eabi" - - toolchain: "no-zerocopy-core-error-1-81-0" - target: "wasm32-unknown-unknown" - # Exclude most targets from the - # `no-zerocopy-diagnostic-on-unimplemented-1-78-0` toolchain since the - # `no-zerocopy-diagnostic-on-unimplemented-1-78-0` feature is unrelated to - # compilation target. This only leaves i686 and x86_64 targets. - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "arm-unknown-linux-gnueabi" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "aarch64-unknown-linux-gnu" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "powerpc-unknown-linux-gnu" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "powerpc64-unknown-linux-gnu" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "riscv64gc-unknown-linux-gnu" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "s390x-unknown-linux-gnu" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "x86_64-pc-windows-msvc" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "thumbv6m-none-eabi" - - toolchain: "no-zerocopy-diagnostic-on-unimplemented-1-78-0" - target: "wasm32-unknown-unknown" - # Exclude most targets from the - # `no-zerocopy-generic-bounds-in-const-fn-1-61-0` toolchain since the - # `no-zerocopy-generic-bounds-in-const-fn-1-61-0` feature is unrelated to - # compilation target. This only leaves i686 and x86_64 targets. - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "arm-unknown-linux-gnueabi" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "aarch64-unknown-linux-gnu" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "powerpc-unknown-linux-gnu" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "powerpc64-unknown-linux-gnu" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "riscv64gc-unknown-linux-gnu" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "s390x-unknown-linux-gnu" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "x86_64-pc-windows-msvc" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "thumbv6m-none-eabi" - - toolchain: "no-zerocopy-generic-bounds-in-const-fn-1-61-0" - target: "wasm32-unknown-unknown" - # Exclude `thumbv6m-none-eabi` combined with any feature that implies - # the `std` feature since `thumbv6m-none-eabi` does not include a - # pre-compiled std. - - target: "thumbv6m-none-eabi" - feature_profile: "stable" - - target: "thumbv6m-none-eabi" - feature_profile: "all" - # Exclude most targets during PR development, but allow them in the - # merge queue. This speeds up our development flow, while still - # ensuring that errors on these targets are caught before a PR is - # merged to main. - - target: "arm-unknown-linux-gnueabi" - event_name: "pull_request" - - target: "aarch64-unknown-linux-gnu" - event_name: "pull_request" - - target: "powerpc-unknown-linux-gnu" - event_name: "pull_request" - - target: "powerpc64-unknown-linux-gnu" - event_name: "pull_request" - - target: "riscv64gc-unknown-linux-gnu" - event_name: "pull_request" - - target: "s390x-unknown-linux-gnu" - event_name: "pull_request" - - target: "thumbv6m-none-eabi" - event_name: "pull_request" - - target: "wasm32-unknown-unknown" - event_name: "pull_request" + # `plan_ci` emits a complete `include` object, including an empty one if + # no cells are selected. Do not add handwritten axes or exclusions here: + # coverage belongs to `ci/zc.toml`, and the typed executor rejects any + # selector which is not present in the checked plan for this event. + matrix: ${{ fromJSON(needs.plan_ci.outputs.build_matrix) }} name: Build & Test (${{ matrix.crate }} / ${{ matrix.toolchain }} / ${{ matrix.feature_profile }} / ${{ matrix.target }}) @@ -353,6 +136,10 @@ jobs: - &matrix_checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + # `Prepare cargo-semver-checks` reads the pull request head commit by + # its explicit SHA. Keep this depth large enough to include that parent + # of GitHub's synthetic pull request merge commit. A missing object + # makes `git log` fail rather than checking a different message. fetch-depth: 2 persist-credentials: false @@ -414,8 +201,9 @@ jobs: -v /home/runner/.docker-cargo/git:/root/.cargo/git \ -e GITHUB_ENV -e GITHUB_PATH -e GITHUB_STEP_SUMMARY -e GITHUB_OUTPUT -e GITHUB_WORKSPACE \ -e CI -e GITHUB_ACTIONS -e GITHUB_ACTOR -e GITHUB_REPOSITORY -e GITHUB_SHA -e GITHUB_REF -e GITHUB_EVENT_NAME \ - -e TOOLCHAIN -e CRATE -e TARGET -e FEATURE_PROFILE -e FEATURES -e ZC_TOOLCHAIN \ - -e RUSTFLAGS -e RUSTDOCFLAGS -e MIRIFLAGS -e MIRI_MODEL_FLAGS \ + -e TOOLCHAIN -e CRATE -e TARGET -e FEATURE_PROFILE \ + -e MIRI_MODEL -e ZC_TOOLCHAIN -e PR_HEAD_SHA \ + -e RUSTFLAGS -e RUSTDOCFLAGS -e MIRIFLAGS \ -e CARGO_NET_RETRY -e RUSTUP_MAX_RETRIES \ -e ZC_NIGHTLY_RUSTFLAGS -e ZC_NIGHTLY_MIRIFLAGS \ -e ZC_SKIP_CARGO_SEMVER_CHECKS \ @@ -423,171 +211,77 @@ jobs: EOF chmod +x /tmp/docker-shell.sh - - &configure_matrix_environment - name: Configure environment variables - env: - TOOLCHAIN: ${{ matrix.toolchain }} - FEATURE_PROFILE: ${{ matrix.feature_profile }} - run: | - set -euo pipefail - MIRIFLAGS="${MIRIFLAGS:-}" - - case "$FEATURE_PROFILE" in - default) FEATURES='' ;; - stable) - FEATURES='--no-default-features --features __internal_use_only_features_that_work_on_stable' - ;; - all) FEATURES='--all-features' ;; - *) echo "unknown feature profile: $FEATURE_PROFILE" >&2; exit 1 ;; - esac - echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" - - # We use toolchain descriptors ("msrv", "stable", "nightly", and values - # from the "metadata.build-rs" key in Cargo.toml) in the matrix. This - # step converts the current descriptor to a particular toolchain version - # by looking up the corresponding key in `Cargo.toml`. It sets the - # `ZC_TOOLCHAIN` environment variable for use in the next step - # (toolchain installation) because GitHub variable interpolation doesn't - # support running arbitrary commands. In other words, we can't rewrite: - # - # toolchain: $ {{ env.ZC_TOOLCHAIN }} - # - # ...to: - # - # toolchain: $ {{ ./cargo.sh --version matrix.toolchain }} # hypothetical syntax - ZC_TOOLCHAIN="$(./cargo.sh --version $TOOLCHAIN)" - echo "Found that the '$TOOLCHAIN' toolchain is $ZC_TOOLCHAIN" | tee -a $GITHUB_STEP_SUMMARY - echo "ZC_TOOLCHAIN=$ZC_TOOLCHAIN" >> $GITHUB_ENV - - if [[ "$TOOLCHAIN" == 'nightly' ]]; then - RUSTFLAGS="$RUSTFLAGS $ZC_NIGHTLY_RUSTFLAGS" - MIRIFLAGS="$MIRIFLAGS $ZC_NIGHTLY_MIRIFLAGS" - echo "Using nightly toolchain; setting RUSTFLAGS='$RUSTFLAGS' and MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY - echo "RUSTFLAGS=$RUSTFLAGS" >> $GITHUB_ENV - echo "MIRIFLAGS=$MIRIFLAGS" >> $GITHUB_ENV - else - echo "Using non-nightly toolchain; not modifying RUSTFLAGS='$RUSTFLAGS' or MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY - fi - - # Native targets run every test that Cargo selects by default in one pass. - # Cargo skips the UI test without `derive` and runs the codegen test in its - # dedicated CI job. - - name: Test native target - env: - TOOLCHAIN: ${{ matrix.toolchain }} - CRATE: ${{ matrix.crate }} - TARGET: ${{ matrix.target }} - run: | - ./cargo.sh +$TOOLCHAIN test \ - --package $CRATE \ - --target $TARGET \ - $FEATURES \ - --verbose - if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'i686-unknown-linux-gnu' - - # Cross targets cannot execute tests on this Linux x86_64 runner, so check - # test-only code and build library code in separate passes. - - name: Check cross target + # The matrix values are data, not shell fragments. `cargo-zerocopy` + # validates these selectors against the plan for this event, reconstructs + # the typed argv and environment, and executes the complete cell. Keep this + # invocation coordinated with `tools/zc/src/cli.rs`; adding a command here + # would bypass the checked execution model. + - name: Execute checked build cell env: TOOLCHAIN: ${{ matrix.toolchain }} CRATE: ${{ matrix.crate }} + FEATURE_PROFILE: ${{ matrix.feature_profile }} TARGET: ${{ matrix.target }} run: | - ./cargo.sh +$TOOLCHAIN check --tests \ - --package $CRATE --target $TARGET $FEATURES --verbose - ./cargo.sh +$TOOLCHAIN build \ - --package $CRATE --target $TARGET $FEATURES --verbose - # This excludes the Linux x86 targets, whose test binaries run in the - # native-target step; other non-thumb targets compile their tests here. - if: matrix.target != 'thumbv6m-none-eabi' && matrix.target != 'x86_64-unknown-linux-gnu' && matrix.target != 'i686-unknown-linux-gnu' - - # `memchr` in thumb's dev-dependency graph prevents `cargo check --tests` - # on `thumbv6m-none-eabi`, so check only the library. - - name: Check thumb library - env: - TOOLCHAIN: ${{ matrix.toolchain }} - CRATE: ${{ matrix.crate }} - TARGET: ${{ matrix.target }} - run: ./cargo.sh +$TOOLCHAIN check --package $CRATE --target $TARGET $FEATURES --verbose - if: matrix.target == 'thumbv6m-none-eabi' - - # On the `thumbv6m-none-eabi` target, we can't run `cargo clippy --tests` - # due to the `memchr` crate, so we just do `cargo clippy` instead. - - name: Clippy - env: - TOOLCHAIN: ${{ matrix.toolchain }} - CRATE: ${{ matrix.crate }} - TARGET: ${{ matrix.target }} - run: ./cargo.sh +$TOOLCHAIN clippy --package $CRATE --target $TARGET $FEATURES --verbose - if: matrix.toolchain == 'nightly' && matrix.target == 'thumbv6m-none-eabi' - - - name: Clippy tests - env: - TOOLCHAIN: ${{ matrix.toolchain }} - CRATE: ${{ matrix.crate }} - TARGET: ${{ matrix.target }} - run: ./cargo.sh +$TOOLCHAIN clippy --package $CRATE --target $TARGET $FEATURES --tests --verbose - # Clippy improves the accuracy of lints over time, and fixes bugs. Only - # running Clippy on nightly allows us to avoid having to write code which - # is compatible with older versions of Clippy, which sometimes requires - # hacks to work around limitations that are fixed in more recent versions. - if: matrix.toolchain == 'nightly' && matrix.target != 'thumbv6m-none-eabi' - - - name: Cargo doc - # We pass --document-private-items and --document-hidden items to ensure that - # documentation always builds even for these items. This makes future changes to - # make those items public/non-hidden more painless. Note that - # --document-hidden-items is unstable; if a future release breaks or removes it, - # we can just update CI to no longer pass that flag. + set -euo pipefail + ./cargo.sh ci execute-build-cell \ + --event "$GITHUB_EVENT_NAME" \ + --package "$CRATE" \ + --toolchain "$TOOLCHAIN" \ + --feature-profile "$FEATURE_PROFILE" \ + --target "$TARGET" + + # The semver check must remain a literal `uses` step, so it cannot run + # inside the typed executor. This workflow-owned preparation resolves the + # one action input which GitHub expressions cannot compute and handles the + # documented skip marker. Keep its selector condition coordinated with the + # action condition and the semver policy in `ci/zc.toml`; a later audit + # replaces this manual cross-file contract with a checked one. + - name: Prepare cargo-semver-checks env: TOOLCHAIN: ${{ matrix.toolchain }} - CRATE: ${{ matrix.crate }} - NIGHTLY_FLAG: ${{ matrix.toolchain == 'nightly' && '-Z unstable-options --document-hidden-items $METADATA_DOCS_RS_RUSTDOC_ARGS'|| '' }} - run: | - # Include arguments passed during docs.rs deployments to make sure those - # work properly. - set -eo pipefail - METADATA_DOCS_RS_RUSTDOC_ARGS="$(env -u RUSTFLAGS -u RUSTDOCFLAGS ./cargo.sh +stable metadata --manifest-path Cargo.toml --no-deps --format-version 1 | \ - jq -r ".packages[] | select(.name == \"zerocopy\").metadata.docs.rs.\"rustdoc-args\"[]" | tr '\n' ' ')" - if [[ "$TOOLCHAIN" == "nightly" ]]; then - export RUSTDOCFLAGS="-Z unstable-options --document-hidden-items $METADATA_DOCS_RS_RUSTDOC_ARGS $RUSTDOCFLAGS" - fi - ./cargo.sh +$TOOLCHAIN doc --no-deps --document-private-items --package $CRATE $FEATURES - - # If the commit message contains the line `SKIP_CARGO_SEMVER_CHECKS=1`, then - # skip the cargo-semver-checks step. - - name: Check whether to skip cargo-semver-checks - env: - EVENT_NAME: ${{ github.event_name }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - SHA: ${{ github.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - set -eo pipefail + set -euo pipefail - if [ "$EVENT_NAME" == "pull_request" ]; then - # Invoked from a PR - get the PR body directly - MESSAGE="$(git log -1 --pretty=%B $HEAD_SHA)" + ZC_TOOLCHAIN="$(./cargo.sh --version "$TOOLCHAIN")" + printf "Resolved the '%s' toolchain to %s\n" \ + "$TOOLCHAIN" "$ZC_TOOLCHAIN" | tee -a "$GITHUB_STEP_SUMMARY" + printf 'ZC_TOOLCHAIN=%s\n' "$ZC_TOOLCHAIN" >> "$GITHUB_ENV" + + # Pull request jobs check the head commit rather than GitHub's synthetic + # merge commit. `PR_HEAD_SHA`, the depth-2 checkout above, and this + # lookup are one contract: if checkout stops fetching that object, + # `git log` fails instead of silently inspecting another message. + if [[ "$GITHUB_EVENT_NAME" == 'pull_request' ]]; then + MESSAGE="$(git log -1 --pretty=%B "$PR_HEAD_SHA")" + MESSAGE_SOURCE='pull request head commit message' else - # Invoked from the merge queue - get the commit message - MESSAGE="$(git log -1 --pretty=%B $SHA)" + MESSAGE="$(git log -1 --pretty=%B HEAD)" + MESSAGE_SOURCE='commit message' fi - if echo "$MESSAGE" | grep '^\s*SKIP_CARGO_SEMVER_CHECKS=1\s*$' > /dev/null; then - echo "Found 'SKIP_CARGO_SEMVER_CHECKS=1' in commit message; skipping cargo-semver-checks..." | tee -a $GITHUB_STEP_SUMMARY - echo "ZC_SKIP_CARGO_SEMVER_CHECKS=1" >> $GITHUB_ENV + if grep -Eq \ + '^[[:space:]]*SKIP_CARGO_SEMVER_CHECKS=1[[:space:]]*$' \ + <<< "$MESSAGE"; then + printf "Found 'SKIP_CARGO_SEMVER_CHECKS=1' in the %s; " \ + "$MESSAGE_SOURCE" | tee -a "$GITHUB_STEP_SUMMARY" + printf 'skipping cargo-semver-checks.\n' | \ + tee -a "$GITHUB_STEP_SUMMARY" + printf 'ZC_SKIP_CARGO_SEMVER_CHECKS=1\n' >> "$GITHUB_ENV" + else + # FIXME(#2906): cargo-semver-checks fetches the latest Zerocopy from + # crates.io, but the vendored-source configuration cannot resolve + # that package. This exact file removal affects only a semver matrix + # cell. Switch to --baseline-rev before removing this workaround. + rm .cargo/config.toml fi - - # FIXME(#2906): We do this because `cargo semver-checks` fetches the latest - # zerocopy from crates.io, but `.cargo/config.toml` causes that to resolve - # in our vendor directory, and we don't vendor zerocopy. Removing this file - # has the effect of causing the subsequent build to use crates.io rather - # than vendored dependencies, which is fine since we only run this on the - # stable toolchain. Eventually, we should update this job to use the - # `--baseline-rev` option to use a previous git commit as the baseline for - # checking compatibility (rather than the most recent published version), - # which will make it unnecessary to remove `.cargo/config.toml`. - - name: Remove Cargo config and vendored dependencies - run: rm -rf .cargo/config.toml + # TODO(#1565): Run on wasm32-unknown-unknown. + if: | + matrix.crate == 'zerocopy' && + matrix.feature_profile == 'stable' && + matrix.toolchain == 'stable' && + matrix.target != 'wasm32-unknown-unknown' # Check semver compatibility with the most recently-published version on # crates.io. We do this in the matrix rather than in its own job so that it @@ -631,7 +325,7 @@ jobs: miri: if: github.event_name != 'pull_request' runs-on: ubuntu-latest - needs: build_docker_env + needs: [build_docker_env, plan_ci] permissions: contents: read defaults: @@ -640,64 +334,38 @@ jobs: working-directory: zerocopy strategy: fail-fast: false - matrix: - toolchain: ["nightly"] - target: *build_targets - feature_profile: *feature_profiles - crate: *build_crates - miri_model: - - name: stacked - flags: "" - - name: tree - flags: "-Zmiri-tree-borrows" - exclude: - # These targets are not currently supported by Miri in this project. - - target: "riscv64gc-unknown-linux-gnu" - - target: "thumbv6m-none-eabi" - - target: "wasm32-unknown-unknown" - # zerocopy-derive has no package-local feature profiles. - - crate: "zerocopy-derive" - feature_profile: "stable" - - crate: "zerocopy-derive" - feature_profile: "all" + # Pull requests skip this job before matrix expansion. Full events consume + # exactly the Miri cells selected by the same checked plan used by the + # executor below. + matrix: ${{ fromJSON(needs.plan_ci.outputs.miri_matrix) }} - name: Miri (${{ matrix.crate }} / ${{ matrix.feature_profile }} / ${{ matrix.miri_model.name }} / ${{ matrix.target }}) + name: Miri (${{ matrix.crate }} / ${{ matrix.feature_profile }} / ${{ matrix.miri_model }} / ${{ matrix.target }}) steps: - *matrix_checkout - *download_ci_image - *load_ci_image - *create_docker_shell - - *configure_matrix_environment - - name: Run tests under Miri + # As with ordinary cells, the workflow passes only selectors. Model flags, + # feature arguments, the Cargo configuration transaction, and target + # workarounds belong to the typed executor rather than this shell adapter. + - name: Execute checked Miri cell env: - TARGET: ${{ matrix.target }} TOOLCHAIN: ${{ matrix.toolchain }} CRATE: ${{ matrix.crate }} - MIRI_MODEL_FLAGS: ${{ matrix.miri_model.flags }} + FEATURE_PROFILE: ${{ matrix.feature_profile }} + TARGET: ${{ matrix.target }} + MIRI_MODEL: ${{ matrix.miri_model }} run: | set -euo pipefail - - # FIXME(#2906): Miri builds std from source, whose dependencies are not - # included in the vendored Cargo source. Temporarily bypass vendoring - # and restore the configuration on every exit path. - mv .cargo/config.toml .cargo/config.toml.bak - trap 'mv .cargo/config.toml.bak .cargo/config.toml' EXIT - - # Work around https://github.com/rust-lang/miri/issues/3125. - [ "$TARGET" == "aarch64-unknown-linux-gnu" ] && cargo clean - - THREADS=$(echo "$(nproc) * 2" | bc) - echo "Running Miri tests with $THREADS threads" | tee -a "$GITHUB_STEP_SUMMARY" - - MIRIFLAGS="$MIRIFLAGS $MIRI_MODEL_FLAGS" ./cargo.sh +$TOOLCHAIN \ - miri nextest run --locked \ - --ignore-default-filter \ - --test-threads "$THREADS" \ - --package $CRATE \ - --target $TARGET \ - $FEATURES + ./cargo.sh ci execute-miri-cell \ + --event "$GITHUB_EVENT_NAME" \ + --package "$CRATE" \ + --toolchain "$TOOLCHAIN" \ + --feature-profile "$FEATURE_PROFILE" \ + --target "$TARGET" \ + --miri-model "$MIRI_MODEL" codegen: runs-on: ubuntu-latest @@ -947,20 +615,6 @@ jobs: - name: Check MSRV is minimal run: ./ci/check_msrv_is_minimal.sh - check-all-toolchains-tested: - runs-on: ubuntu-latest - name: Check that all toolchains listed in Cargo.toml are tested in CI - steps: - - name: Install yq (for YAML parsing) - # FIXME(https://github.com/mikefarah/yq/issues/2587): Remove - # `GONOSUMDB` once this bug is fixed. - run: GONOSUMDB=github.com/mikefarah/yq/v4 go install github.com/mikefarah/yq/v4@v4.44.1 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Run check - run: cd zerocopy && ./ci/check_all_toolchains_tested.sh - check-job-dependencies: runs-on: ubuntu-latest name: Check all-jobs-succeeded depends on all jobs @@ -1106,7 +760,7 @@ jobs: # success in branch protection, so this aggregation must fail closed. if: ${{ always() }} runs-on: ubuntu-latest - needs: [build_test, miri, codegen, coverage, kani, check_be_aarch64, check_avr_atmega, check_fmt, check_tools, check_actions, check_readme, check_versions, check_msrv_is_minimal, check_stale_stderr, check-all-toolchains-tested, check-job-dependencies, check-todo, run-git-hooks, zizmor, build_docker_env, plan_ci] + needs: [build_test, miri, codegen, coverage, kani, check_be_aarch64, check_avr_atmega, check_fmt, check_tools, check_actions, check_readme, check_versions, check_msrv_is_minimal, check_stale_stderr, check-job-dependencies, check-todo, run-git-hooks, zizmor, build_docker_env, plan_ci] steps: - name: Reject workflow cancellation if: ${{ cancelled() }} diff --git a/ci/workflow-jobs.tsv b/ci/workflow-jobs.tsv index 8b9111bf38..9d2b2207ba 100644 --- a/ci/workflow-jobs.tsv +++ b/ci/workflow-jobs.tsv @@ -29,7 +29,6 @@ workflow job role .github/workflows/ci.yml all-jobs-succeed aggregate .github/workflows/ci.yml build_docker_env static-ci .github/workflows/ci.yml build_test planned -.github/workflows/ci.yml check-all-toolchains-tested static-ci .github/workflows/ci.yml check-job-dependencies static-ci .github/workflows/ci.yml check-todo static-ci .github/workflows/ci.yml check_actions static-ci diff --git a/ci/zc.toml b/ci/zc.toml index ffcee0ee17..e4d07f8bdc 100644 --- a/ci/zc.toml +++ b/ci/zc.toml @@ -393,8 +393,9 @@ reason = "cargo-semver-checks does not yet support this target" [baselines] # These files are an independent observation of the workflow on main, not -# generated truth from this policy. Later shadow planning must match them before -# Actions consumes a plan. Update them only for an intentional coverage change. +# generated truth from this policy. Planning must match them before Actions can +# consume a plan or execute a selected cell. Update them only for an intentional +# coverage change. manifest = "ci/baselines/manifest.tsv" build_reduced = "ci/baselines/build-pr.tsv" build_full = "ci/baselines/build-full.tsv" diff --git a/githooks/pre-push b/githooks/pre-push index f2c487c3a1..0c9b2d4f64 100755 --- a/githooks/pre-push +++ b/githooks/pre-push @@ -111,7 +111,6 @@ if [[ "$TOOLCHAINS_READY" -eq 1 ]]; then ./ci/check_fmt.sh & FMT_PID=$! ./ci/check_job_dependencies.sh >/dev/null & JOB_DEPS_PID=$! ./ci/check_tools.sh & TOOLS_PID=$! - ./zerocopy/ci/check_all_toolchains_tested.sh >/dev/null & TOOLCHAINS_PID=$! ./zerocopy/ci/check_readme.sh >/dev/null & README_PID=$! ./zerocopy/ci/check_stale_stderr.sh >/dev/null & STALE_STDERR_PID=$! ./zerocopy/ci/check_versions.sh >/dev/null & VERSIONS_PID=$! @@ -125,8 +124,6 @@ if [[ "$TOOLCHAINS_READY" -eq 1 ]]; then wait_for_check "ci/check_fmt.sh" "$FMT_PID" wait_for_check "ci/check_job_dependencies.sh" "$JOB_DEPS_PID" wait_for_check "ci/check_tools.sh" "$TOOLS_PID" - wait_for_check \ - "zerocopy/ci/check_all_toolchains_tested.sh" "$TOOLCHAINS_PID" wait_for_check "zerocopy/ci/check_readme.sh" "$README_PID" wait_for_check "zerocopy/ci/check_stale_stderr.sh" "$STALE_STDERR_PID" wait_for_check "zerocopy/ci/check_versions.sh" "$VERSIONS_PID" @@ -141,9 +138,9 @@ fi # catch obvious errors. Also note that this entire hook is a nice-to-have - # failures that aren't caught here will still be caught in CI. # -# This was added because, in #728, we added -# `zerocopy/ci/check_all_toolchains_tested.sh` without calling it from this -# script. +# This inventory was added after a check script was once introduced without a +# corresponding hook invocation. It deliberately inventories the files which +# exist now, so deleting a retired check also removes that hook obligation. shopt -s extglob GLOBIGNORE="./*/@(release_crate_version|check_todo|release_anneal_version).sh" # We don't want to run these for f in ./ci/*.sh; do diff --git a/githooks/test_pre_push.py b/githooks/test_pre_push.py index 03ca23245d..7fd652f349 100755 --- a/githooks/test_pre_push.py +++ b/githooks/test_pre_push.py @@ -34,7 +34,6 @@ # Keep this fixture coordinated with the explicit fan-out in pre-push. # Omitting a real check here would leave its child handling untested. "ci/check_tools.sh", - "zerocopy/ci/check_all_toolchains_tested.sh", "zerocopy/ci/check_readme.sh", "zerocopy/ci/check_stale_stderr.sh", "zerocopy/ci/check_versions.sh", diff --git a/tools/zc/src/execution.rs b/tools/zc/src/execution.rs index 4f67a6006f..4f52db2c36 100644 --- a/tools/zc/src/execution.rs +++ b/tools/zc/src/execution.rs @@ -17,10 +17,12 @@ //! `.github/workflows/ci.yml`. //! //! The operation builders below are the single semantic source for both parity -//! checking and local execution. Every place where their command spelling or -//! setup remains duplicated in `ci.yml` is called out explicitly. The -//! independent files under `ci/baselines/` are comparison evidence only: this -//! module never reads a baseline row to construct proposed behavior. In +//! checking and local execution. `ci.yml` passes complete cell selectors back +//! to this module instead of reproducing Cargo or Miri commands. The semver +//! action remains an explicit workflow-owned exception because GitHub requires +//! a literal `uses` value. The independent files under `ci/baselines/` are +//! comparison evidence only: this module never reads a baseline row to +//! construct proposed behavior. In //! particular, the legacy comparison covers the repository state named by the //! baseline manifest. It is not an inventory of control-plane validation jobs //! added after that frozen source commit. Live workflow jobs and pre-push @@ -76,13 +78,11 @@ pub const EXECUTION_CONTEXT_ENV: &str = "ZEROCOPY_INTERNAL_EXECUTION_CONTEXT"; #[doc(hidden)] pub const MIRI_REPOSITORY_ROOT_CONTEXT: &str = "miri-repository-root"; -// These environment values are executor-owned command behavior, not policy. -// Until `.github/workflows/ci.yml` delegates matrix commands to this executor, -// the base values remain duplicated in its top-level `env` block and the -// nightly additions in its "Configure environment variables" step. The frozen -// command goldens prove that this model matches independently captured main; -// they do not inspect that temporary live-YAML duplication. Migrated workflow -// jobs must not reproduce these values in YAML. +// These environment values are executor-owned behavior for planned matrix +// cells, not policy. The workflow has similar top-level values for handwritten +// static jobs, but planned cells set this complete map directly and do not +// inherit changes to those jobs. The frozen command goldens prove that this +// model matches independently captured main. const BASE_RUSTFLAGS: &str = "-Dwarnings"; const BASE_RUSTDOCFLAGS: &str = "-Dwarnings --cfg=zerocopy_unstable_ptr"; const NIGHTLY_RUSTFLAGS: &str = "-Zrandomize-layout"; @@ -1430,20 +1430,16 @@ fn docs_operation(docs_rs_rustdoc_args: &[String], cell: &BuildCellSemantics) -> argv.extend(cell.features.cargo_args()); // Repository inventory obtains this ordered sequence from the canonical - // Zerocopy package's `package.metadata.docs.rs.rustdoc-args`. `ci.yml` - // independently performs the same Cargo metadata lookup before invoking - // Cargo doc; keep that workflow adapter coordinated until a typed executor - // owns the invocation itself. Inventory rejects whitespace inside an - // element, so joining with one space preserves every argument boundary - // understood by RUSTDOCFLAGS. + // Zerocopy package's `package.metadata.docs.rs.rustdoc-args`. Inventory + // rejects whitespace inside an element, so joining with one space + // preserves every argument boundary understood by RUSTDOCFLAGS. let docs_rs_rustdoc_args = docs_rs_rustdoc_args.join(" "); // Cargo doc inherits the same ordinary matrix environment as every other - // command, then its step replaces RUSTDOCFLAGS. Keep this complete map - // coordinated with the Cargo doc step in ci.yml and the representative - // nightly-docs command golden. The golden used to omit inherited - // RUSTFLAGS and MIRIFLAGS; retaining that omission in executable behavior - // would make this typed executor silently differ from CI. + // command, then replaces RUSTDOCFLAGS. Keep this complete map coordinated + // with the representative nightly-docs command golden. The golden used to + // omit inherited RUSTFLAGS and MIRIFLAGS; retaining that omission in + // executable behavior would silently differ from the captured workflow. let mut environment = ordinary_environment(cell.pinned_nightly); let rustdocflags = if cell.pinned_nightly { format!( @@ -1457,8 +1453,8 @@ fn docs_operation(docs_rs_rustdoc_args: &[String], cell: &BuildCellSemantics) -> kind, // `cargo doc` intentionally has no `--target`; all target cells for // the same package/toolchain/profile normalize to one obligation with - // an occurrence count. Keep this coupled to the Cargo doc step in - // `ci.yml` and the comment in logical-obligations.tsv. + // an occurrence count. Keep this coupled to the comment in + // logical-obligations.tsv. logical: matrix_logical_spec( kind, &cell.package, @@ -1960,6 +1956,10 @@ const LEGACY_STANDALONE_SPECS: &[StandaloneSpec] = &[ condition: ALWAYS, golden: None, }, + // These two operations belong to the immutable source commit named by the + // baseline manifest. The live shell check was retired after typed + // inventory and policy validation took ownership of this invariant; keep + // the historical operations here so parity still proves what it replaced. StandaloneSpec { obligation: "check-toolchains", job: "check-all-toolchains-tested", diff --git a/tools/zc/src/plan.rs b/tools/zc/src/plan.rs index 0554f36a35..1646678371 100644 --- a/tools/zc/src/plan.rs +++ b/tools/zc/src/plan.rs @@ -19,11 +19,11 @@ //! choices, or shell commands. Keep those security-sensitive concerns in the //! small hand-written workflows. //! -//! The types below record the intended semantics that a later workflow -//! projection must preserve. Before a workflow consumes a plan, that -//! projection and its command behavior must be validated separately. Matrix -//! membership equality alone cannot prove, for example, that native tests are -//! executed, cross-target tests are only compiled, or Miri tests are +//! The types below record the intended semantics preserved by the review +//! artifact and typed executor. The workflow consumes only complete selectors; +//! the executor resolves their command behavior again from this checked plan. +//! Matrix membership equality alone cannot prove, for example, that native +//! tests are executed, cross-target tests are only compiled, or Miri tests are //! interpreted. //! //! The selectors below coordinate three independently reviewed sources: @@ -88,7 +88,7 @@ const LEGACY_EVENT_CLASSES: [(&str, EventClass); 4] = [ ("workflow_dispatch", EventClass::Full), ]; -/// How a later workflow projection must handle an ordinary compilation target. +/// How the typed executor must handle an ordinary compilation target. #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum ExecutionMode { /// Build and execute tests on the runner. @@ -133,11 +133,8 @@ pub enum FeatureSelection { impl FeatureSelection { /// Returns the exact ordered Cargo arguments for this selection. /// - /// The vector preserves argument boundaries; callers must pass its entries - /// as arguments rather than joining them into shell text. Until the typed - /// executor replaces the live adapter, keep this exhaustive translation - /// coordinated with the `FEATURE_PROFILE` case statement in - /// `.github/workflows/ci.yml`. + /// The vector preserves argument boundaries; the executor passes its + /// entries directly to a process rather than joining them into shell text. pub fn cargo_args(&self) -> Vec { match self { Self::Default => Vec::new(), diff --git a/zerocopy/ci/check_all_toolchains_tested.sh b/zerocopy/ci/check_all_toolchains_tested.sh deleted file mode 100755 index 2f77af082e..0000000000 --- a/zerocopy/ci/check_all_toolchains_tested.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2024 The Fuchsia Authors -# -# Licensed under a BSD-style license , Apache License, Version 2.0 -# , or the MIT -# license , at your option. -# This file may not be copied, modified, or distributed except according to -# those terms. - -set -eo pipefail -cd "$(dirname "$0")/.." - -# Check whether the set of toolchains tested in this file (other than -# 'msrv', 'stable', and 'nightly') is equal to the set of toolchains -# listed in the 'package.metadata.build-rs' section of Cargo.toml. -# -# If the inputs to `diff` are not identical, `diff` exits with a -# non-zero error code, which causes this script to fail (thanks to -# `set -e`). -diff \ - <(yq -r '.jobs.build_test.strategy.matrix.toolchain | .[]' ../.github/workflows/ci.yml | \ - sort -u | grep -v '^\(msrv\|stable\|nightly\)$') \ - <(cargo metadata -q --format-version 1 | \ - jq -r ".packages[] | select(.name == \"zerocopy\").metadata.\"build-rs\" | keys | .[]" | \ - sort -u) >&2