diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml new file mode 100644 index 00000000..e089dff9 --- /dev/null +++ b/.github/actions/debusine-build/action.yml @@ -0,0 +1,126 @@ +name: Build in Debusine +description: > + Generate the Debian source package from the prepared tree and submit it to + Debusine, returning the CI workspace it was built in. + +# Shared by the two Debian workflows, which differ only in what they do with +# the result: build-kernel-debian.yml downloads the .deb files and uploads them +# to S3, release-kernel-debian.yml promotes them to a target workspace. Holding +# these steps in a composite action lets each workflow declare its own runner, +# container and environment while keeping one copy of the build itself. +# +# The caller must have checked out pkg-linux-qcom into the workspace first, +# which is also where this action comes from. Runs inside the +# debusine-pkg-builder container. +# +# vars and secrets contexts do not reach a composite action, so the Debusine +# host, scope and credentials come in as inputs from the calling workflow. + +inputs: + suite: + description: 'Target Debian suite (trixie, forky, sid)' + required: true + flavour: + description: 'Kernel flavour, isolating artifacts and Debusine workspaces' + required: true + debusine-host: + description: 'Debusine instance hostname' + required: true + debusine-scope: + description: 'Debusine scope' + required: true + debusine-user: + description: 'Debusine account used to submit the build' + required: true + debusine-token: + description: 'Debusine token for the CI child workspace' + required: true + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + default: 'qli-ci' + extra-build-dep-workspaces: + description: > + Workspaces the build resolves its build-dependencies from, space + separated. A staging build may depend on a package version that has only + reached qli-staging; a release build reads qli alone, so a released + kernel is never built against something unreleased. + default: 'qli' + debusine-action-ref: + description: 'Ref of qualcomm-linux/debusine-action to use' + default: 'main' + +outputs: + workspace: + description: 'Debusine CI workspace the package was built in' + value: ${{ steps.build-debusine.outputs.workspace }} + workspace_url: + description: 'Browser URL of that workspace' + value: ${{ steps.build-debusine.outputs.workspace_url }} + srcpkg_version: + description: 'Version of the generated source package' + value: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} + +runs: + using: composite + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: ${{ inputs.debusine-action-ref }} + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Download prepared source tree + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} + path: /tmp/srcpkg-artifact + + - name: Extract prepared source tree + shell: bash + run: | + mkdir srcpkg + tar xzf /tmp/srcpkg-artifact/*.tar.gz \ + -C srcpkg --strip-components=1 + + - name: Generate source package + id: generate-srcpkg + shell: bash + env: + SUITE: ${{ inputs.suite }} + DEBUSINE_ASSEMBLE_ORIG: "true" + run: | + SUITE="$SUITE" debusine-action/lib/generate-source-package + + - name: Build in Debusine + id: build-debusine + shell: bash + env: + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_INDEX: ${{ inputs.flavour }}-${{ inputs.suite }} + DEBUSINE_HOST: ${{ inputs.debusine-host }} + DEBUSINE_SCOPE: ${{ inputs.debusine-scope }} + DEBUSINE_USER: ${{ inputs.debusine-user }} + DEBUSINE_TOKEN: ${{ inputs.debusine-token }} + DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} + SUITE: ${{ inputs.suite }} + EXTRA_BUILD_DEP_WORKSPACES: ${{ inputs.extra-build-dep-workspaces }} + run: | + # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. + set -euo pipefail + debusine-action/lib/build + workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") + echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" + + - name: Note Debusine workspace URL + shell: bash + env: + WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} + run: | + echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml new file mode 100644 index 00000000..1aba18a2 --- /dev/null +++ b/.github/actions/prepare-kernel-source/action.yml @@ -0,0 +1,445 @@ +name: Prepare kernel source +description: > + Resolve the kernel ref, clone it, apply any PR patches, inject debian/ and + generate the source package, then upload the prepared tree as an artifact. + +# Every build takes this path, whichever family it belongs to, so it lives in a +# composite action rather than a reusable workflow: the two family workflows +# call it as a step of their own prepare job, which keeps one copy of these +# steps without adding a nesting level to every job name in the checks list. +# +# The caller must have checked out pkg-linux-qcom into the workspace first: +# debian/ and ci/ come from that checkout, and so does this action. +# +# Runs on the self-hosted arm64 runner, and leaves nothing behind in the job +# environment: values pass between these steps as step outputs, each consumer +# naming what it reads. Writing them to GITHUB_ENV instead would put them in +# the environment of every later step in the job, including steps outside this +# action, and a newline in one would let the value that carried it define +# environment variables of its own. Several of these values are not ours: the +# kernel ref comes from a remote's tag list, and on pr-build.yml the inputs +# come from the build matrix as the pull request wrote it. + +inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + required: true + suite: + description: 'Target suite' + required: true + registry-token: + description: > + Token used to pull the pkg-builder image from ghcr.io, which is not a + public package. Defaults to the job's own GITHUB_TOKEN, which needs the + packages: read permission the calling workflows declare. The secrets + context does not reach a composite action, hence an input. + default: ${{ github.token }} + expect-family: + description: > + Build family the calling workflow builds, debian or ubuntu. The suite is + a free-text matrix value, so this catches one handed to the workflow for + the other family before anything is cloned or built. + required: true + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + kernel-url: + description: 'Kernel repository URL' + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + default: 'latest_tag' + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag' + default: 'qcom-next-*' + srcpkg: + description: 'Debian source package name' + default: 'linux-qcom-next' + binpkg: + description: 'Kernel image metapackage name' + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + default: '' + debian-revision: + description: 'Debian revision override (empty takes the Daily revision the matrix gives this build)' + default: '' + localversion: + description: 'LOCALVERSION override (auto-derived from the resolved ref if empty)' + default: '' + kver-extra: + description: 'Extra package-version suffix (e.g. -ci42)' + default: '' + debug-build: + description: 'Enable the debug build configuration' + default: 'false' + qcom-next-pr: + description: 'qcom-next PR numbers to merge, space-separated' + default: '' + kernel-topics-pr: + description: 'kernel-topics PR numbers to apply as patches, space-separated' + default: '' + +runs: + using: composite + steps: + - name: Resolve build environment + id: env + shell: bash + env: + SUITE_INPUT: ${{ inputs.suite }} + EXPECT_FAMILY: ${{ inputs.expect-family }} + run: | + set -euo pipefail + # Family and docker image in one place, because they answer the same + # question about a suite. The family list here is the one in + # ci/scripts/resolve-matrix.py, which routes an entry to the workflow + # that calls this action. + # + # Ubuntu-family suites use an image of their own suite. Every + # Debian-family suite uses the trixie image, forky and sid included: + # the registry publishes no image for either, and docker-pkg-build has + # no Dockerfile to build one from, so they borrow trixie's until one + # exists. That costs nothing today because Debusine performs the actual + # suite-specific build; this container only has to run + # prepare-source.sh, which generates packaging rather than compiling + # anything. + case "$SUITE_INPUT" in + trixie|forky|sid|unstable) FAMILY=debian; DOCKER_DISTRO=trixie ;; + *) FAMILY=ubuntu; DOCKER_DISTRO="$SUITE_INPUT" ;; + esac + + if [[ "$FAMILY" != "$EXPECT_FAMILY" ]]; then + echo "::error::Suite $SUITE_INPUT is $FAMILY-family, but this workflow builds the $EXPECT_FAMILY family." + echo "::error::Dispatch the $FAMILY build workflow for it instead." + exit 1 + fi + + # Pull the images qualcomm-linux/docker-pkg-build publishes, and build + # the rest here. Its container-build-and-upload.yml pushes trixie and + # resolute (and noble) on every push to main and weekly, having + # validated each with a test package build, so rebuilding them from the + # same Dockerfile on every job repeats work already done. Anything it + # does not publish still has to be built on the runner. + case "$DOCKER_DISTRO" in + trixie|resolute) IMAGE_SOURCE=pull ;; + *) IMAGE_SOURCE=build ;; + esac + + echo "distro=$SUITE_INPUT" >> "$GITHUB_OUTPUT" + echo "docker_distro=$DOCKER_DISTRO" >> "$GITHUB_OUTPUT" + echo "image_source=$IMAGE_SOURCE" >> "$GITHUB_OUTPUT" + echo "Suite $SUITE_INPUT is $FAMILY-family and uses the $DOCKER_DISTRO image ($IMAGE_SOURCE)" + + - name: Pull builder image + if: ${{ steps.env.outputs.image_source == 'pull' }} + shell: bash + env: + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + REGISTRY_TOKEN: ${{ inputs.registry-token }} + run: | + # No -x: REGISTRY_TOKEN is in env. + set -euo pipefail + # ghcr.io/qualcomm-linux/pkg-builder is not a public package, so the + # pull needs a login even though the image is only ever read. + echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + docker pull "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" + docker image ls + + # Only for a suite with no published image: the pull path needs neither + # this checkout nor the build below. + - name: Checkout docker-pkg-build + if: ${{ steps.env.outputs.image_source == 'build' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/docker-pkg-build + ref: main + path: docker-pkg-build + + - name: Build builder image + if: ${{ steps.env.outputs.image_source == 'build' }} + shell: bash + env: + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + run: | + set -euo pipefail + ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" + docker image ls + + - name: Resolve kernel ref + id: ref + shell: bash + env: + KERNEL_URL_INPUT: ${{ inputs.kernel-url }} + BRANCH_INPUT: ${{ inputs.kernel-branch }} + REF_STRATEGY_INPUT: ${{ inputs.ref-strategy }} + TAG_PATTERN_INPUT: ${{ inputs.tag-pattern }} + BUILD_INPUT: ${{ inputs.build }} + run: | + set -euo pipefail + [[ "$BUILD_INPUT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "::error::build name must use lowercase letters, digits, and internal hyphens" + exit 1 + } + case "$REF_STRATEGY_INPUT" in + latest_tag) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL_INPUT" \ + --latest-tag "$TAG_PATTERN_INPUT") + ;; + branch_tip|pinned_ref) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL_INPUT" --ref "$BRANCH_INPUT") + ;; + *) + echo "::error::unsupported ref strategy: $REF_STRATEGY_INPUT" + exit 1 + ;; + esac + + # A ref carrying a newline would otherwise define outputs of its own. + # The remote chooses this value, so check it before writing it out. + [[ "$KERNEL_REF" == *$'\n'* ]] && { + echo "::error::resolved kernel ref contains a newline" + exit 1 + } + + echo "kernel_ref=$KERNEL_REF" >> "$GITHUB_OUTPUT" + echo "Build: $BUILD_INPUT" + echo "Ref strategy: $REF_STRATEGY_INPUT" + echo "Resolved kernel ref: $KERNEL_REF" + + - name: Clone kernel source + id: clone + shell: bash + env: + KERNEL_URL: ${{ inputs.kernel-url }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + run: | + set -euo pipefail + KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" + echo "kernel_dir=$KERNEL_DIR" >> "$GITHUB_OUTPUT" + echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." + git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ + "$KERNEL_URL" "$KERNEL_DIR" + KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) + echo "kernel_sha=$KERNEL_SHA" >> "$GITHUB_OUTPUT" + echo "Kernel HEAD SHA: $KERNEL_SHA" + + # Committer date of HEAD, normalised to UTC. Committer rather than + # author date: an author date can be months old on a backported patch, + # which would date the build by when someone first wrote the code + # rather than by when this snapshot came to exist. + KERNEL_DATE=$(TZ=UTC git -C "$KERNEL_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + [[ "$KERNEL_DATE" =~ ^[0-9]{8}$ ]] || { + echo "::error::could not read a YYYYMMDD commit date from $KERNEL_REF (got '$KERNEL_DATE')" + exit 1 + } + echo "kernel_date=$KERNEL_DATE" >> "$GITHUB_OUTPUT" + echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" + + - name: Derive version fields + id: localversion + shell: bash + env: + LOCALVERSION_INPUT: ${{ inputs.localversion }} + FLAVOUR_INPUT: ${{ inputs.flavour }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} + KERNEL_DATE: ${{ steps.clone.outputs.kernel_date }} + run: | + set -euo pipefail + if [[ -n "$LOCALVERSION_INPUT" ]]; then + # An override supplies no snapshot, and the ref it was chosen to + # replace cannot be trusted to describe it. The Debian version then + # carries no +git; prepare-source.sh warns about the same. + echo "Using explicit LOCALVERSION override: $LOCALVERSION_INPUT" + echo "localversion=$LOCALVERSION_INPUT" >> "$GITHUB_OUTPUT" + echo "snapshot=" >> "$GITHUB_OUTPUT" + echo "gitsha=" >> "$GITHUB_OUTPUT" + else + # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines. They are read + # here rather than recovered from LOCALVERSION later: that string + # also carries a flavour name, and a hex SHA can end in eight digits + # of its own. + # + # --date is only consulted for branch-tip builds, where the ref + # carries no snapshot. It comes from the commit rather than the + # clock, so rebuilding a commit reproduces its version instead of + # inventing a higher one. + FIELDS=$(ci/scripts/derive-localversion.sh \ + --flavour "$FLAVOUR_INPUT" \ + --ref "$KERNEL_REF" \ + --sha "$KERNEL_SHA" \ + --date "$KERNEL_DATE") + echo "Derived version fields:" + echo "$FIELDS" + echo "$FIELDS" | while IFS='=' read -r key value; do + case "$key" in + LOCALVERSION) echo "localversion=$value" ;; + SNAPSHOT) echo "snapshot=$value" ;; + GITSHA) echo "gitsha=$value" ;; + *) echo "::error::unexpected field '$key' from derive-localversion.sh"; exit 1 ;; + esac + done >> "$GITHUB_OUTPUT" + fi + + - name: Resolve Debian revision + # Callers driven by the matrix pass the entry's own debian-revision + # straight through. A direct workflow_dispatch that leaves it empty looks + # the same entry up itself, so a manual build of a configured build gets + # exactly the revision the daily build would have used. The lookup is + # always the Daily entry: direct dispatch is build-only and never + # promotes. + id: revision + shell: bash + env: + DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} + BUILD_INPUT: ${{ inputs.build }} + DISTRO: ${{ steps.env.outputs.distro }} + run: | + set -euo pipefail + if [[ -n "$DEBIAN_REVISION_INPUT" ]]; then + DEBIAN_REVISION="$DEBIAN_REVISION_INPUT" + echo "Using supplied Debian revision: $DEBIAN_REVISION" + else + DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ + --type Daily \ + --build "$BUILD_INPUT" \ + --suite "$DISTRO" \ + --field debian_revision) || { + echo "::error::No Daily matrix entry named $BUILD_INPUT for suite $DISTRO." + echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." + exit 1 + } + echo "Matrix Debian revision for $BUILD_INPUT/$DISTRO: $DEBIAN_REVISION" + fi + echo "debian_revision=$DEBIAN_REVISION" >> "$GITHUB_OUTPUT" + + - name: Merge qcom-next PR patches + if: ${{ inputs.qcom-next-pr != '' }} + shell: bash + env: + QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + run: | + cd "$KERNEL_DIR" + echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" + for pr in $QCOM_NEXT_PR; do + [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } + echo "::group::Merging qcom-next PR #$pr" + git fetch --no-tags origin "pull/$pr/head:pr-$pr" + if ! git merge --no-ff --no-commit "pr-$pr"; then + echo "ERROR: Merge conflict while merging PR #$pr. Aborting." + git merge --abort || true + exit 1 + fi + if ! git diff --cached --quiet; then + git commit -m "Merged qcom-next PR #$pr" + echo "PR #$pr merged successfully." + else + echo "PR #$pr already present; nothing to merge." + fi + echo "::endgroup::" + done + + - name: Apply kernel-topics PR patches + if: ${{ inputs.kernel-topics-pr != '' }} + shell: bash + env: + KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + run: | + cd "$KERNEL_DIR" + echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" + for pr in $KERNEL_TOPICS_PR; do + [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } + echo "::group::Applying kernel-topics PR #$pr" + wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" + if ! git am "$pr.patch"; then + echo "ERROR: Patch application failed for PR #$pr. Aborting." + git am --abort || true + exit 1 + fi + echo "PR #$pr applied successfully." + echo "::endgroup::" + done + + - name: Prepare source + shell: bash + env: + SRCPKG_INPUT: ${{ inputs.srcpkg }} + BINPKG_INPUT: ${{ inputs.binpkg }} + KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} + KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} + DKMS_INPUT: ${{ inputs.dkms }} + DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + DISTRO: ${{ steps.env.outputs.distro }} + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + LOCALVERSION: ${{ steps.localversion.outputs.localversion }} + SNAPSHOT: ${{ steps.localversion.outputs.snapshot }} + DEBIAN_REVISION: ${{ steps.revision.outputs.debian_revision }} + KERNEL_URL: ${{ inputs.kernel-url }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} + run: | + # No -x here: keep xtrace off for this step as it assembles the + # prepare-source.sh argument list which may include sensitive paths. + set -euo pipefail + + ARGS=( + --source-dir "$KERNEL_DIR" + --distro "$DISTRO" + --srcpkg "$SRCPKG_INPUT" + --binpkg "$BINPKG_INPUT" + --debian-revision "$DEBIAN_REVISION" + ) + + # The version fields all come from derive-localversion.sh, which + # composed them from the ref and HEAD together. Nothing here appends to + # them: the SHA is already inside LOCALVERSION, and SNAPSHOT and the + # SHA are passed separately for the Debian version, which joins them + # differently. + [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") + [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") + # Full SHA: prepare-source.sh takes the first 12 for the version + # strings and records the whole thing in the changelog, alongside the + # repository and ref that the version strings do not name. + ARGS+=(--git-sha "$KERNEL_SHA" --git-clone "$KERNEL_URL" --git-ref "$KERNEL_REF") + [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") + [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") + # Always passed, unlike the optional inputs above: an empty list is a + # genuine "bundle nothing" rather than a request for some default. + ARGS+=(--dkms "$DKMS_INPUT") + [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) + + docker run -i --rm \ + -v "$PWD:$PWD" --workdir="$PWD" \ + "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ + ./prepare-source.sh "${ARGS[@]}" + + - name: Pack prepared source tree + shell: bash + env: + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + run: | + set -euo pipefail + tar czf /tmp/kernel-srcpkg.tar.gz \ + -C "$(dirname "$KERNEL_DIR")" \ + "$(basename "$KERNEL_DIR")" + + - name: Upload prepared source tree artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} + path: /tmp/kernel-srcpkg.tar.gz + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml deleted file mode 100644 index d31dd3ba..00000000 --- a/.github/workflows/build-kernel-deb.yml +++ /dev/null @@ -1,496 +0,0 @@ -name: build-kernel-deb - -on: - workflow_dispatch: - inputs: - kernel-variant: - description: 'Kernel variant identifier' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target suite' - required: true - default: 'trixie' - type: string - ref-strategy: - description: 'Kernel ref strategy' - required: true - default: 'latest_tag' - type: choice - options: - - latest_tag - - branch_tip - - pinned_ref - kernel-branch: - description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' - required: false - default: 'qcom-next' - tag-pattern: - description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' - required: false - default: 'qcom-next-*' - kernel-url: - description: 'Advanced: custom kernel repository URL (empty uses qualcomm-linux/kernel)' - required: false - default: '' - srcpkg: - description: 'Advanced: Debian source package name' - required: false - default: 'linux-qcom-next' - binpkg: - description: 'Advanced: kernel image metapackage name' - required: false - default: 'linux-image-qcom-next' - kernel-config: - description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - required: false - default: '' - dkms: - description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - required: false - default: '' - debian-version-stub: - description: 'Advanced: Debian version stub; the selected suite''s mapped suffix and a Daily-style trailing ~ are applied automatically' - required: false - default: '0qli1' - localversion: - description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' - required: false - default: '' - kver-extra: - description: 'Advanced: extra KVER suffix (e.g. -ci42), appended to the kernel release verbatim' - required: false - default: '' - debug-build: - description: 'Advanced: enable the debug build configuration' - type: boolean - required: false - default: false - qcom-next-pr: - description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' - type: string - required: false - default: '' - kernel-topics-pr: - description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' - type: string - required: false - default: '' - - # Called by daily.yml and release.yml as a reusable workflow. - workflow_call: - inputs: - kernel-variant: - description: 'Kernel variant identifier' - type: string - default: 'qcom-next' - suite: - description: 'Target suite (trixie, forky, noble, questing, resolute, sid)' - type: string - default: 'trixie' - kernel-branch: - description: 'Source branch or pinned tag used by branch_tip or pinned_ref' - type: string - default: 'qcom-next' - git-clone: - description: 'Kernel repository URL' - type: string - default: 'https://github.com/qualcomm-linux/kernel' - ref-strategy: - description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' - type: string - default: 'latest_tag' - tag-pattern: - description: 'Tag glob used when ref-strategy=latest_tag' - type: string - default: 'qcom-next-*' - srcpkg: - description: 'Source package name' - type: string - default: 'linux-qcom-next' - binpkg: - description: 'Binary metapackage name' - type: string - default: 'linux-image-qcom-next' - kernel-config: - description: 'Extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - type: string - default: '' - dkms: - description: 'Out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - type: string - default: '' - debian-revision: - description: 'Debian revision component of the package version' - type: string - default: '0qli1~' - localversion: - description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' - type: string - default: '' - kver-extra: - description: 'Extra KVER suffix, appended to the kernel release verbatim' - type: string - default: '' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: '' - target-workspace: - description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' - type: string - default: '' - secrets: - DEBUSINE_USER: - description: 'Debusine account used to submit CI and release builds' - required: false - DEBUSINE_TOKEN: - description: 'Debusine token for the CI child workspace' - required: false - DEBUSINE_RELEASE_TOKEN: - description: 'Debusine token for package-publish promotion to the release target workspace' - required: false - -permissions: - contents: read - packages: read - -jobs: - # --------------------------------------------------------------------------- - # resolve: classify the target suite as 'debian' or 'ubuntu'. - # --------------------------------------------------------------------------- - resolve: - name: Resolve suite family - runs-on: ubuntu-latest - outputs: - family: ${{ steps.classify.outputs.family }} - steps: - - name: Classify suite - id: classify - shell: bash - env: - SUITE_INPUT: ${{ inputs.suite || 'trixie' }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} - run: | - suite="$SUITE_INPUT" - kernel_variant="$KERNEL_VARIANT_INPUT" - [[ "$kernel_variant" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: kernel variant must use lowercase letters, digits, and internal hyphens" >&2 - exit 1 - } - case "$suite" in - trixie|forky|sid|unstable|bookworm) - family=debian ;; - *) - family=ubuntu ;; - esac - echo "family=$family" >> "$GITHUB_OUTPUT" - echo "Suite '$suite' classified as family=$family" - - # --------------------------------------------------------------------------- - # prepare: clone kernel source, inject packaging, run prepare-source.sh. - # --------------------------------------------------------------------------- - prepare: - name: Prepare kernel source - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - # TEMPORARY, with the DKMS override in "Prepare source" below. - needs: resolve - - env: - KERNEL_URL: ${{ inputs.git-clone || inputs.kernel-url || 'https://github.com/qualcomm-linux/kernel' }} - DISTRO: ${{ inputs.suite || 'trixie' }} - # For Debian-family suites (trixie, forky, sid) use trixie as the docker - # image distro: docker-pkg-build and pkg-builder only support trixie for - # Debian. Debusine handles the actual suite-specific build internally. - # For Ubuntu-family suites (noble, questing, resolute) use the real suite. - DOCKER_DISTRO: ${{ (inputs.suite == 'forky' || inputs.suite == 'sid' || inputs.suite == 'unstable') && 'trixie' || inputs.suite || 'trixie' }} - - defaults: - run: - shell: bash - - steps: - - name: Checkout pkg-linux-qcom - # debian/ and ci/ live on the same branch, so one checkout supplies - # both the packaging and resolve-kernel-ref.sh, derive-localversion.sh, - # derive-debian-revision.sh and the suite_suffix_mapping. - uses: actions/checkout@v4 - - - name: Checkout docker-pkg-build - uses: actions/checkout@v4 - with: - repository: qualcomm-linux/docker-pkg-build - ref: main - path: docker-pkg-build - - - name: Build docker image - run: | - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" - docker image ls - - - name: Resolve kernel ref - id: kernel-ref - env: - BRANCH_INPUT: ${{ inputs.kernel-branch || 'qcom-next' }} - REF_STRATEGY_INPUT: ${{ inputs.ref-strategy || 'latest_tag' }} - TAG_PATTERN_INPUT: ${{ inputs.tag-pattern || 'qcom-next-*' }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} - run: | - set -euo pipefail - BRANCH="$BRANCH_INPUT" - REF_STRATEGY="$REF_STRATEGY_INPUT" - TAG_PATTERN="$TAG_PATTERN_INPUT" - - case "$REF_STRATEGY" in - latest_tag) - KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ - --url "$KERNEL_URL" \ - --latest-tag "$TAG_PATTERN") - ;; - branch_tip|pinned_ref) - KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh --url "$KERNEL_URL" --ref "$BRANCH") - ;; - *) - echo "ERROR: unsupported ref strategy: $REF_STRATEGY" >&2 - exit 1 - ;; - esac - - echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" - echo "Kernel variant: $KERNEL_VARIANT_INPUT" - echo "Ref strategy: $REF_STRATEGY" - echo "Resolved kernel ref: $KERNEL_REF" - - - name: Clone kernel source - run: | - KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" - echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" - echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." - git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ - "$KERNEL_URL" "$KERNEL_DIR" - KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) - echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" - echo "Kernel HEAD SHA: $KERNEL_SHA" - - # Committer date of HEAD, normalised to UTC. Committer rather than - # author date: an author date can be months old on a backported - # patch, which would date the build by when someone first wrote the - # code rather than by when this snapshot came to exist. - KERNEL_DATE=$(TZ=UTC git -C "$KERNEL_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) - [[ "$KERNEL_DATE" =~ ^[0-9]{8}$ ]] || { - echo "::error::could not read a YYYYMMDD commit date from $KERNEL_REF (got '$KERNEL_DATE')" - exit 1 - } - echo "KERNEL_DATE=$KERNEL_DATE" >> "$GITHUB_ENV" - echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" - - - name: Derive version fields - env: - LOCALVERSION_INPUT: ${{ inputs.localversion }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} - run: | - set -euo pipefail - OVERRIDE_LV="$LOCALVERSION_INPUT" - if [[ -n "$OVERRIDE_LV" ]]; then - # An override supplies no snapshot, and the ref it was chosen to - # replace cannot be trusted to describe it. The Debian version then - # carries no +git; prepare-source.sh warns about the same. - echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" - echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" - echo "SNAPSHOT=" >> "$GITHUB_ENV" - echo "GITSHA=" >> "$GITHUB_ENV" - else - # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines, already in - # GITHUB_ENV form. - # - # --date is only consulted for branch-tip builds, where the ref - # carries no snapshot of its own. It comes from the commit rather - # than the clock, so rebuilding a commit reproduces its version - # instead of inventing a higher one, and it lands in the same space - # as upstream's tag dates. - FIELDS=$(ci/scripts/derive-localversion.sh \ - --variant "$KERNEL_VARIANT_INPUT" \ - --ref "$KERNEL_REF" \ - --sha "$KERNEL_SHA" \ - --date "$KERNEL_DATE") - echo "$FIELDS" >> "$GITHUB_ENV" - echo "Derived version fields:" - echo "$FIELDS" - fi - - - name: Resolve Debian revision - # workflow_call callers (daily.yml, release.yml) pass an already - # fully-derived debian-revision from resolve-matrix.sh. Direct - # workflow_dispatch runs pass only debian-version-stub and get the - # selected suite's mapped suffix applied automatically, always using - # Daily semantics since direct dispatch is build-only and - # non-promoting. - env: - DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - DEBIAN_VERSION_STUB_INPUT: ${{ inputs.debian-version-stub || '0qli1' }} - run: | - set -euo pipefail - INPUT_REVISION="$DEBIAN_REVISION_INPUT" - if [[ -n "$INPUT_REVISION" ]]; then - DEBIAN_REVISION="$INPUT_REVISION" - echo "Using matrix-derived Debian revision: $DEBIAN_REVISION" - else - DEBIAN_REVISION=$(ci/scripts/derive-debian-revision.sh \ - --stub "$DEBIAN_VERSION_STUB_INPUT" \ - --suite "$DISTRO" \ - --delivery-type Daily \ - --matrix-file ci/build-matrix.json) - echo "Derived Debian revision for direct dispatch: $DEBIAN_REVISION" - fi - echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" - - - name: Merge qcom-next PR patches - if: ${{ inputs.qcom-next-pr != '' }} - env: - QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} - run: | - cd "$KERNEL_DIR" - echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" - for pr in $QCOM_NEXT_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } - echo "::group::Merging qcom-next PR #$pr" - git fetch --no-tags origin "pull/$pr/head:pr-$pr" - if ! git merge --no-ff --no-commit "pr-$pr"; then - echo "ERROR: Merge conflict while merging PR #$pr. Aborting." - git merge --abort || true - exit 1 - fi - if ! git diff --cached --quiet; then - git commit -m "Merged qcom-next PR #$pr" - echo "PR #$pr merged successfully." - else - echo "PR #$pr already present; nothing to merge." - fi - echo "::endgroup::" - done - - - name: Apply kernel-topics PR patches - if: ${{ inputs.kernel-topics-pr != '' }} - env: - KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} - run: | - cd "$KERNEL_DIR" - echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" - for pr in $KERNEL_TOPICS_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } - echo "::group::Applying kernel-topics PR #$pr" - wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - if ! git am "$pr.patch"; then - echo "ERROR: Patch application failed for PR #$pr. Aborting." - git am --abort || true - exit 1 - fi - echo "PR #$pr applied successfully." - echo "::endgroup::" - done - - - name: Prepare source - env: - SRCPKG_INPUT: ${{ inputs.srcpkg || 'linux-qcom-next' }} - BINPKG_INPUT: ${{ inputs.binpkg || 'linux-image-qcom-next' }} - KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} - KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} - DKMS_INPUT: ${{ inputs.dkms }} - # TEMPORARY, consumed by the DKMS override below. - SUITE_FAMILY: ${{ needs.resolve.outputs.family }} - DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} - run: | - # No -x here: keep xtrace off for this step as it assembles the - # prepare-source.sh argument list which may include sensitive paths. - set -euo pipefail - - DKMS_LIST="$DKMS_INPUT" - - # TEMPORARY: camx and iris-vpu are bundled on non-Ubuntu builds only. - # A listed module is a presence contract, so an Ubuntu leg that asked - # for them would hard-fail rather than quietly ship without them. - # - # This is delivery policy and belongs in ci/build-matrix.json, which - # cannot express a per-suite dkms list yet. Revert this override — - # and the "needs: resolve"/SUITE_FAMILY plumbing that feeds it — once - # the matrix can. - if [[ "$SUITE_FAMILY" == "ubuntu" ]]; then - DKMS_LIST="kgsl" - echo "Ubuntu build: DKMS list overridden to '$DKMS_LIST' (was '$DKMS_INPUT')" - fi - - ARGS=( - --source-dir "$KERNEL_DIR" - --distro "$DISTRO" - --srcpkg "$SRCPKG_INPUT" - --binpkg "$BINPKG_INPUT" - --debian-revision "$DEBIAN_REVISION" - ) - - # All three version fields come from derive-localversion.sh, which - # composed them from the ref and HEAD together. Nothing here appends - # to them: the SHA is already inside LOCALVERSION, and GITSHA carries - # it separately for the Debian version, which joins it differently. - [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") - [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") - # Full SHA: prepare-source.sh takes the first 12 for the version - # strings and records the whole thing in the changelog, alongside the - # repository and ref that the version strings do not name. - ARGS+=(--git-sha "$KERNEL_SHA" --git-clone "$KERNEL_URL" --git-ref "$KERNEL_REF") - [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") - [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") - # Always passed, unlike the optional inputs above: an empty list is a - # genuine "bundle nothing" rather than a request for some default. - ARGS+=(--dkms "$DKMS_LIST") - [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) - - docker run -i --rm \ - -v "$PWD:$PWD" --workdir="$PWD" \ - "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ - ./prepare-source.sh "${ARGS[@]}" - - - name: Upload prepared source tree - run: | - tar czf /tmp/kernel-srcpkg.tar.gz \ - -C "$(dirname "$KERNEL_DIR")" \ - "$(basename "$KERNEL_DIR")" - - - name: Upload prepared source tree artifact - uses: actions/upload-artifact@v4 - with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} - path: /tmp/kernel-srcpkg.tar.gz - if-no-files-found: error - retention-days: 7 - - # --------------------------------------------------------------------------- - # debusine-build: Debian suites only (trixie, forky, sid). - # --------------------------------------------------------------------------- - debusine-build: - name: Build in Debusine - needs: [resolve, prepare] - if: ${{ needs.resolve.outputs.family == 'debian' }} - uses: ./.github/workflows/build-kernel-debusine.yml - with: - suite: ${{ inputs.suite || 'trixie' }} - kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} - srcpkg: ${{ inputs.srcpkg || 'linux-qcom-next' }} - debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} - target-workspace: ${{ inputs.target-workspace || '' }} - secrets: - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - - # --------------------------------------------------------------------------- - # ubuntu-build: Ubuntu suites only (noble, questing, resolute). - # --------------------------------------------------------------------------- - ubuntu-build: - name: Build (Ubuntu, docker) - needs: [resolve, prepare] - if: ${{ needs.resolve.outputs.family == 'ubuntu' }} - uses: ./.github/workflows/build-kernel-ubuntu.yml - with: - distro: ${{ inputs.suite || 'resolute' }} - kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml new file mode 100644 index 00000000..e6dcf24d --- /dev/null +++ b/.github/workflows/build-kernel-debian.yml @@ -0,0 +1,341 @@ +name: build-kernel-debian + +# Build a kernel package for a Debian-family suite and publish it to S3. +# +# Called per matrix entry by daily.yml and pr-build.yml, which select the +# Debian entries with resolve-matrix.py --family debian and call only this +# workflow for them. Nothing here is conditional on the suite: a caller that +# reaches this file has already decided the build belongs on the Debusine +# path, so every job in it runs. +# +# release-kernel-debian.yml is the same build with the release tail: it +# promotes to a target Debusine workspace instead of publishing to S3. +# +# Jobs: +# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# build (debusine-pkg-builder container): submit to Debusine. +# publish (self-hosted): download the .deb files and upload them to S3. + +on: + workflow_call: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + default: 'qcom-next-trixie' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + default: 'qcom-next' + suite: + description: 'Target Debian suite (trixie, forky, sid)' + type: string + default: 'trixie' + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + default: 'latest_tag' + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' + type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli~' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debug-build: + description: 'Enable the debug build configuration' + type: boolean + default: false + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + type: string + default: '' + qcom-next-pr: + description: 'qcom-next PR numbers to merge (e.g. "42 43")' + type: string + default: '' + kernel-topics-pr: + description: 'kernel-topics PR numbers to apply as patches' + type: string + default: '' + secrets: + DEBUSINE_USER: + description: 'Debusine account used to submit CI builds' + required: false + DEBUSINE_TOKEN: + description: 'Debusine token for the CI child workspace' + required: false + + # Ad-hoc build of one Debian suite. Release promotion is not offered here: + # a manual build never promotes, so it always takes the S3 publish path. + workflow_dispatch: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + required: true + default: 'qcom-next-trixie' + type: string + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + default: 'qcom-next' + type: string + suite: + description: 'Target Debian suite (trixie, forky, sid). An Ubuntu suite here is rejected before anything is built; dispatch build-kernel-ubuntu for those.' + required: true + default: 'trixie' + type: string + ref-strategy: + description: 'Kernel ref strategy' + required: true + default: 'latest_tag' + type: choice + options: + - latest_tag + - branch_tip + - pinned_ref + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + required: false + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' + required: false + default: 'qcom-next-*' + git-clone: + description: 'Advanced: custom kernel repository URL' + required: false + default: 'https://github.com/qualcomm-linux/kernel' + srcpkg: + description: 'Advanced: Debian source package name' + required: false + default: 'linux-qcom-next' + binpkg: + description: 'Advanced: kernel image metapackage name' + required: false + default: 'linux-image-qcom-next' + kernel-config: + description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' + required: false + default: '' + dkms: + description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' + required: false + default: '' + debian-revision: + description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this build and suite)' + required: false + default: '' + localversion: + description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' + required: false + default: '' + kver-extra: + description: 'Advanced: extra package-version suffix (e.g. -ci42)' + required: false + default: '' + debug-build: + description: 'Advanced: enable the debug build configuration' + type: boolean + required: false + default: false + qcom-next-pr: + description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' + type: string + required: false + default: '' + kernel-topics-pr: + description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' + type: string + required: false + default: '' + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- + prepare: + name: Generate source package + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder + steps: + # debian/ and ci/ live on the same branch, so one checkout supplies both + # the packaging and the scripts, the delivery matrix they read, and the + # prepare-kernel-source action below. + # + # An empty ref leaves actions/checkout on github.sha, the commit this run + # was dispatched from or the one the calling workflow runs at. + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: debian + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + debug-build: ${{ inputs.debug-build }} + qcom-next-pr: ${{ inputs.qcom-next-pr }} + kernel-topics-pr: ${{ inputs.kernel-topics-pr }} + + # --------------------------------------------------------------------------- + # build: generate the Debian source package and submit it to Debusine. + # --------------------------------------------------------------------------- + build: + name: Build package (Debusine) + needs: prepare + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + environment: Staging + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + outputs: + workspace: ${{ steps.debusine.outputs.workspace }} + steps: + # Only for the debusine-build action itself: the package being built + # comes from the prepared-source artifact, not from this checkout. + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build in Debusine + id: debusine + uses: ./.github/actions/debusine-build + with: + suite: ${{ inputs.suite }} + flavour: ${{ inputs.flavour }} + debusine-host: ${{ vars.DEBUSINE_HOST }} + debusine-scope: ${{ vars.DEBUSINE_SCOPE }} + debusine-user: ${{ secrets.DEBUSINE_USER }} + debusine-token: ${{ secrets.DEBUSINE_TOKEN }} + debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + # A staging build: also reads qli-staging, which carries package + # versions that have not been released to qli yet. + extra-build-dep-workspaces: qli qli-staging + + # --------------------------------------------------------------------------- + # publish: download the .deb files from Debusine and upload them to S3. + # Runs on the self-hosted runner, which has direct IAM access. + # --------------------------------------------------------------------------- + publish: + name: Publish to S3 + needs: build + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Generate Debusine apt configuration + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} + SUITE: ${{ inputs.suite }} + run: | + debusine-action/lib/generate-apt-config + + - name: Download .deb packages from Debusine workspace + run: | + set -euxo pipefail + sudo apt-get install -y --no-install-recommends devscripts + mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" + chdist create debusine-workspace + rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list + install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d + install -d ~/.chdist/debusine-workspace/etc/apt/auth.conf.d + install -m 0644 debusine-ci.sources \ + ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ + install -m 0600 debusine-ci-auth.conf \ + ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ + chdist apt-get debusine-workspace update + packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') + [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } + echo "Packages to download: $packages" + cd "$GITHUB_WORKSPACE/deb-artifacts" + # shellcheck disable=SC2086 + chdist apt-get debusine-workspace download $packages + + - name: Upload .deb packages to S3 + # Keep flavour and suite in the destination so concurrent matrix legs + # cannot overwrite or mix package outputs. Consumers must select the + # flavour and suite they intend to install. + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: deb-artifacts + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml deleted file mode 100644 index 7b8cc17a..00000000 --- a/.github/workflows/build-kernel-debusine.yml +++ /dev/null @@ -1,242 +0,0 @@ -name: build-debusine - -# Reusable workflow: Debian kernel build via Debusine, then publish. -# -# Called by build-kernel-deb.yml for Debian-family suites (trixie, forky, sid). -# The kernel source is already prepared by the caller's prepare job and shared -# as the kernel-srcpkg artifact. Artifact and workspace identity include both -# kernel_variant and suite so parallel variants cannot share inputs or outputs. -# -# Publish path is determined by target-workspace: -# target-workspace == '' -> daily path: download .deb files, upload to S3. -# target-workspace != '' -> release path: promote from CI workspace to the -# target Debusine workspace via debusine-action lib/release. -# -# Jobs: -# build (debusine-pkg-builder container): generates .dsc, submits to Debusine. -# publish (self-hosted runner): S3 upload (daily path only). -# release (debusine-pkg-builder container): Debusine promotion (release path only). - -on: - workflow_call: - inputs: - suite: - description: 'Target Debian suite (trixie, forky, sid)' - type: string - required: true - kernel-variant: - description: 'Kernel variant identifier used to isolate artifacts and Debusine workspaces' - type: string - required: true - srcpkg: - description: 'Source package name (e.g. linux-qcom-next)' - type: string - default: 'linux-qcom-next' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: 'qli-ci' - target-workspace: - description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' - type: string - default: '' - secrets: - DEBUSINE_USER: - required: true - DEBUSINE_TOKEN: - required: true - DEBUSINE_RELEASE_TOKEN: - required: false - -permissions: - contents: read - packages: read - -env: - DEBUSINE_ACTION_REF: main - -jobs: - # --------------------------------------------------------------------------- - # build: generate Debian source package and submit to Debusine. - # --------------------------------------------------------------------------- - build: - name: Build (Debusine) - runs-on: ubuntu-latest - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - outputs: - workspace: ${{ steps.build-debusine.outputs.workspace }} - workspace_url: ${{ steps.build-debusine.outputs.workspace_url }} - srcpkg_version: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} - defaults: - run: - shell: bash - environment: ${{ inputs.target-workspace != '' && 'Production' || 'Staging' }} - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Download prepared source tree - uses: actions/download-artifact@v4 - with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} - path: /tmp/srcpkg-artifact - - - name: Extract prepared source tree - run: | - mkdir srcpkg - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C srcpkg --strip-components=1 - - - name: Generate source package - id: generate-srcpkg - env: - SUITE: ${{ inputs.suite }} - DEBUSINE_ASSEMBLE_ORIG: "true" - run: | - SUITE="$SUITE" debusine-action/lib/generate-source-package - - - name: Build in Debusine - id: build-debusine - env: - GITHUB_REPOSITORY_ID: ${{ github.repository_id }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_INDEX: ${{ inputs.kernel-variant }}-${{ inputs.suite }} - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} - SUITE: ${{ inputs.suite }} - EXTRA_BUILD_DEP_WORKSPACES: qli - run: | - # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. - set -euo pipefail - debusine-action/lib/build - workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") - echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" - - - name: Note Debusine workspace URL - env: - WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} - run: | - echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" - - # --------------------------------------------------------------------------- - # publish: download .deb files from Debusine and upload to S3. - # Daily path only (target-workspace is empty). - # --------------------------------------------------------------------------- - publish: - name: Publish to S3 - needs: build - if: ${{ needs.build.result == 'success' && inputs.target-workspace == '' }} - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Generate Debusine apt configuration - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} - SUITE: ${{ inputs.suite }} - run: | - debusine-action/lib/generate-apt-config - - - name: Download .deb packages from Debusine workspace - run: | - set -euxo pipefail - sudo apt-get install -y --no-install-recommends devscripts - mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" - chdist create debusine-workspace - rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list - install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d - install -d ~/.chdist/debusine-workspace/etc/apt/auth.conf.d - install -m 0644 debusine-ci.sources \ - ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ - install -m 0600 debusine-ci-auth.conf \ - ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ - chdist apt-get debusine-workspace update - packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') - [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } - echo "Packages to download: $packages" - cd "$GITHUB_WORKSPACE/deb-artifacts" - # shellcheck disable=SC2086 - chdist apt-get debusine-workspace download $packages - - - name: Upload .deb packages to S3 - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 - with: - s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} - path: deb-artifacts - destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ - - # --------------------------------------------------------------------------- - # release: promote packages from CI workspace to target Debusine workspace. - # Release path only (target-workspace is non-empty). - # --------------------------------------------------------------------------- - release: - name: Release to Debusine - needs: build - if: ${{ needs.build.result == 'success' && inputs.target-workspace != '' }} - runs-on: ubuntu-latest - environment: Production - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - persist-credentials: false - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Promote packages to target workspace - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} - DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} - SRCPKG_NAME: ${{ inputs.srcpkg }} - SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} - SUITE: ${{ inputs.suite }} - run: | - set -ex - debusine-action/lib/release diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index f62317ec..270c15b3 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -1,66 +1,279 @@ name: build-kernel-ubuntu -# Reusable workflow: Ubuntu kernel build via build-kernel.sh. +# Build a kernel package for an Ubuntu-family suite and publish it to S3. # -# Called by build-kernel-deb.yml when the target suite is Ubuntu-family -# (noble, questing, resolute). The kernel source has already been cloned, -# patched, and prepared (debian/ injected, debian/control + debian/changelog -# generated) by the prepare job in the caller workflow. This workflow -# downloads the kernel-srcpkg artifact and runs build-kernel.sh --skip-prepare -# to go directly to the build step. +# Called per matrix entry by daily.yml and pr-build.yml, which select the +# Ubuntu entries with resolve-matrix.py --family ubuntu and call only this +# workflow for them. The package is built in a suite-matched pkg-builder +# container on the self-hosted runner rather than in Debusine, which builds +# only the Debian family. # -# The artifact is shared from the caller via github.run_id. Build and S3 upload -# run on the self-hosted runner, which has direct IAM access. +# Ubuntu has no release path: promotion runs through Debusine, so a Release +# entry for an Ubuntu suite is rejected by resolve-matrix.py rather than built +# here and quietly not promoted. +# +# Jobs: +# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# build (self-hosted): build the .deb files and upload them to S3. on: workflow_call: inputs: - distro: - description: Target Ubuntu suite (noble, questing, resolute) + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + default: 'qcom-next-resolute' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + default: 'qcom-next' + suite: + description: 'Target Ubuntu suite (noble, resolute)' + type: string + default: 'resolute' + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + default: 'latest_tag' + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli~' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debug-build: + description: 'Enable the debug build configuration' + type: boolean + default: false + qcom-next-pr: + description: 'qcom-next PR numbers to merge (e.g. "42 43")' + type: string + default: '' + kernel-topics-pr: + description: 'kernel-topics PR numbers to apply as patches' + type: string + default: '' + + # Ad-hoc build of one Ubuntu suite. + workflow_dispatch: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + required: true + default: 'qcom-next-resolute' + type: string + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + default: 'qcom-next' + type: string + suite: + description: 'Target Ubuntu suite (noble, resolute). A Debian suite here is rejected before anything is built; dispatch build-kernel-debian for those.' required: true - kernel-variant: - description: 'Kernel variant identifier used to isolate prepared-source artifacts and published packages' + default: 'resolute' type: string + ref-strategy: + description: 'Kernel ref strategy' required: true + default: 'latest_tag' + type: choice + options: + - latest_tag + - branch_tip + - pinned_ref + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + required: false + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' + required: false + default: 'qcom-next-*' + git-clone: + description: 'Advanced: custom kernel repository URL' + required: false + default: 'https://github.com/qualcomm-linux/kernel' + srcpkg: + description: 'Advanced: Debian source package name' + required: false + default: 'linux-qcom-next' + binpkg: + description: 'Advanced: kernel image metapackage name' + required: false + default: 'linux-image-qcom-next' + kernel-config: + description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' + required: false + default: '' + dkms: + description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' + required: false + default: '' + debian-revision: + description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this build and suite)' + required: false + default: '' + localversion: + description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' + required: false + default: '' + kver-extra: + description: 'Advanced: extra package-version suffix (e.g. -ci42)' + required: false + default: '' + debug-build: + description: 'Advanced: enable the debug build configuration' + type: boolean + required: false + default: false + qcom-next-pr: + description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' + type: string + required: false + default: '' + kernel-topics-pr: + description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' + type: string + required: false + default: '' permissions: contents: read packages: read jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- + prepare: + name: Generate source package + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder + steps: + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: ubuntu + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + debug-build: ${{ inputs.debug-build }} + qcom-next-pr: ${{ inputs.qcom-next-pr }} + kernel-topics-pr: ${{ inputs.kernel-topics-pr }} + + # --------------------------------------------------------------------------- + # build: build the prepared source in a suite-matched container, then publish. + # + # Named for where the build happens, as the Debian workflows' Build package + # (Debusine) is: this one runs in a suite-matched pkg-builder container on + # the runner rather than being submitted to a build service. + # --------------------------------------------------------------------------- build: - name: Build kernel package + name: Build package (Docker) + needs: prepare runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder env: - DISTRO: ${{ inputs.distro }} + DISTRO: ${{ inputs.suite }} defaults: run: shell: bash - steps: - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false # Pinned to @main while native kernel-build support is still landing # upstream in qualcomm-linux/docker-pkg-build. - name: Checkout docker-pkg-build - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: false repository: qualcomm-linux/docker-pkg-build ref: main path: docker-pkg-build - - name: Build docker image + - name: Fetch builder image + env: + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # --rebuild forces a fresh suite-matched pkg-builder image. - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DISTRO" + # No -x: REGISTRY_TOKEN is in env. + set -euo pipefail + # Same split as prepare-kernel-source: pull what docker-pkg-build + # publishes, build what it does not. Nothing is forced here, because + # docker_deb_build.py builds the image itself when the build below + # finds it missing -- so an unpublished suite needs no step of its + # own, only the absence of a pull. + case "$DISTRO" in + resolute) + echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + docker pull "ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" + ;; + *) + echo "No published pkg-builder image for $DISTRO; it will be built on demand." + ;; + esac docker image ls - name: Download prepared source tree - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.distro }} + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/srcpkg-artifact - name: Extract prepared source tree @@ -74,12 +287,12 @@ jobs: - name: Build kernel package run: | - # Kernel source was cloned, patched, and prepared by the caller's - # prepare job. Pass --local-source and --skip-prepare so - # build-kernel.sh goes directly to the build step. - # debian/control, debian/changelog, and all config fragments are - # already baked into the extracted source tree; no re-derivation - # of LOCALVERSION, kver-extra, or debug config is needed here. + # Kernel source was cloned, patched, and prepared by the prepare job. + # Pass --local-source and --skip-prepare so build-kernel.sh goes + # directly to the build step. debian/control, debian/changelog, and + # all config fragments are already baked into the extracted source + # tree; no re-derivation of LOCALVERSION, kver-extra, or debug config + # is needed here. ./build-kernel.sh \ --build-mode docker \ --distro "$DISTRO" \ @@ -89,11 +302,11 @@ jobs: - name: Upload .deb packages to S3 # build-kernel.sh outputs to kernel-build// by default. - # Keep kernel variant and suite in the destination so concurrent matrix - # legs cannot overwrite or mix package outputs. Consumers must select - # the variant and suite they intend to install. - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 + # Keep flavour and suite in the destination so concurrent matrix legs + # cannot overwrite or mix package outputs. Consumers must select the + # flavour and suite they intend to install. + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} - path: kernel-build/${{ inputs.distro }} - destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.distro }}/${{ github.run_id }}-${{ github.run_attempt }}/ + path: kernel-build/${{ inputs.suite }} + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index a3022f7e..12eda419 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,20 +1,20 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel -# build per Daily entry in ci/build-matrix.json after suite flattening. -# Manual dispatch can run the full Daily matrix, every suite for one kernel -# variant, or one kernel variant and suite. Variant and suite are matrix values, -# so future variants need no workflow edit. +# build per Daily entry in ci/build-matrix.yaml. Manual dispatch can run the +# full Daily matrix, a named set of builds, or every suite of one flavour. +# Build names and flavours are matrix values, so new ones need no workflow edit. # -# Matrix schema: ci/build-matrix.json -# Each row with "type": "Daily" is expanded by ci/scripts/resolve-matrix.sh: -# the "suites" array is flattened into one job per suite. All other fields -# (kernel_variant, srcpkg, binpkg, kernel_config, dkms, debian_revision, -# git_clone, branch_or_tag, ref_strategy) are passed through to -# build-kernel-deb.yml as inputs. +# Matrix schema: ci/build-matrix.yaml +# One entry is one generated package, so ci/scripts/resolve-matrix.py selects +# the entries with type: Daily and passes them through as they stand. Every +# field (name, suite, flavour, srcpkg, binpkg, kernel_config, dkms, +# debian_revision, git_clone, branch_or_tag, ref_strategy) becomes an input +# of the build workflow for the entry's family. name labels the job; flavour +# is what reaches the built kernel's version and its published paths. # -# To add a daily build target: add a row to ci/build-matrix.json with -# "type": "Daily". No workflow changes needed. +# To add a daily build target: add an entry to ci/build-matrix.yaml with +# type: Daily. No workflow changes needed. on: schedule: @@ -23,23 +23,23 @@ on: workflow_dispatch: inputs: build-scope: - description: 'Choose all variants, all suites for one variant, or one variant and suite' + description: 'Choose the whole matrix, named build(s), or every suite of one flavour' type: choice required: true - default: 'Selected variant and suite' + default: 'Selected builds' options: - 'Full matrix' - - 'Selected variant (all suites)' - - 'Selected variant and suite' - kernel-variant: - description: 'Kernel variant to build (ignored for Full matrix)' + - 'Selected builds' + - 'Selected flavour (all suites)' + builds: + description: 'Comma-separated build names, e.g. qcom-next-trixie,qcom-next-debug-forky (Selected builds)' required: true - default: 'qcom-next' + default: 'qcom-next-trixie' type: string - suite: - description: 'Target suite to build (used only for Selected variant and suite)' + flavour: + description: 'Kernel flavour to build in every suite it targets (Selected flavour)' required: true - default: 'trixie' + default: 'qcom-next' type: string permissions: @@ -48,27 +48,32 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Daily rows from ci/build-matrix.json. + # configure-matrix: select the Daily entries from ci/build-matrix.yaml. # - # Delegates to ci/scripts/resolve-matrix.sh which filters type=Daily rows - # and expands each row's "suites" array into one flat entry per suite. - # Manual dispatch selects the complete matrix, every suite for a variant, or - # one variant and suite. + # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix + # and returns the entries with type: Daily. Manual dispatch selects the + # complete matrix, a comma-separated list of build names, or one flavour + # across every suite it targets. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + debian: ${{ steps.set-matrix.outputs.debian }} + ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Generate build matrix id: set-matrix env: BUILD_SCOPE_INPUT: ${{ github.event.inputs.build-scope }} - INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} - INPUT_SUITE: ${{ github.event.inputs.suite }} + INPUT_BUILDS: ${{ github.event.inputs.builds }} + INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} run: | set -euo pipefail @@ -80,46 +85,68 @@ jobs: case "$BUILD_SCOPE" in "Full matrix") - echo "Source: ci/build-matrix.json (full Daily matrix)" + echo "Source: ci/build-matrix.yaml (full Daily matrix)" args=(--type Daily) ;; - "Selected variant (all suites)") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT") + "Selected builds") + echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" + args=(--type Daily --build "$INPUT_BUILDS") ;; - "Selected variant and suite") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") + "Selected flavour (all suites)") + echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" + args=(--type Daily --flavour "$INPUT_FLAVOUR") ;; *) echo "ERROR: unsupported build scope: $BUILD_SCOPE" >&2 exit 1 ;; esac - MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + # One selection, split by build path. The two families are built by + # different workflows, so each gets the entries it builds and neither + # run starts a job for the other. A narrowed dispatch can leave one + # side empty, which --allow-empty reports as [] rather than as a + # failure; a name that matches nothing still fails, in both calls. + for family in debian ubuntu; do + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}" \ + --family "$family" --allow-empty) + echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$family entries: $(echo "$MATRIX" | jq length)" + done # --------------------------------------------------------------------------- - # build: one parallel job per flat matrix entry. + # build-debian / build-ubuntu: one parallel job per matrix entry. + # + # Two jobs rather than one because the families are built by different + # workflows, on different runners, and published from different places. + # Splitting the matrix by family means each entry starts exactly the jobs + # that build it: a run shows no skipped job for the path it did not take. # - # Each entry calls build-kernel-deb.yml as a reusable workflow. The full - # set of matrix fields (suite, srcpkg, binpkg, kernel_config, dkms, etc.) is - # threaded through as inputs. fail-fast is disabled so a failed trixie - # build does not cancel the forky build. + # Both are named ${{ matrix.name }}, so the checks list reads the same as + # one job would -- the split is in what each entry calls, not in what it is + # called. fail-fast is disabled so a failed trixie build does not cancel the + # forky build. + # + # The if: guards only matter for a narrowed dispatch that selects nothing on + # one path; an empty matrix is an error rather than an empty job list. The + # scheduled full-matrix run has entries on both. # --------------------------------------------------------------------------- - build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + build-debian: + name: ${{ matrix.name }} needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.debian != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} @@ -135,4 +162,31 @@ jobs: secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + + build-ubuntu: + name: ${{ matrix.name }} + needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.ubuntu != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-ubuntu.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: ${{ matrix.kver_extra || '' }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 4f9e5e3a..8dfeea50 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -1,6 +1,10 @@ name: PR Build # Build-test PRs through the same pipeline the daily builds use. +# +# Packaging and CI now live on one branch, so a PR is tested entirely against +# its own commit: its ci/build-matrix.yaml defines the legs, and its debian/ +# tree and ci/scripts build them. Nothing is read from main. on: pull_request: @@ -16,7 +20,7 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Daily rows from the PR's build-matrix.json. + # configure-matrix: select the Daily entries from the PR's build-matrix.yaml. # # The PR head, not the merge commit: a push mid-run must not change what this # run tested. A SHA also resolves for fork PRs, whose head branch does not @@ -25,10 +29,13 @@ jobs: configure-matrix: name: Configure build matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + debian: ${{ steps.set-matrix.outputs.debian }} + ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false @@ -37,27 +44,46 @@ jobs: id: set-matrix run: | set -euo pipefail - MATRIX=$(ci/scripts/resolve-matrix.sh --type Daily) - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + # Split by build path: the families are built by different workflows, + # so a PR gets one job per leg it actually builds and no skipped job + # for the path that leg does not take. + for family in debian ubuntu; do + MATRIX=$(ci/scripts/resolve-matrix.py --type Daily \ + --family "$family" --allow-empty) + echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$family entries: $(echo "$MATRIX" | jq length)" + done # --------------------------------------------------------------------------- - # build: one parallel job per flat matrix entry, mirroring daily.yml. + # build-debian / build-ubuntu: one parallel job per matrix entry, mirroring + # daily.yml. + # + # A PR gets one check per build leg and nothing else: neither workflow has a + # branch to skip, and a PR builds the whole Daily matrix, so both families + # have entries and neither guard fires. # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. + # + # Neither takes DEBUSINE_RELEASE_TOKEN: a PR build never promotes, so it + # calls the S3-publishing workflow and not release-kernel-debian.yml. # --------------------------------------------------------------------------- - build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + build-debian: + name: ${{ matrix.name }} needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.debian != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} @@ -72,7 +98,33 @@ jobs: # in the shared Debusine workspace. kver-extra: -pr${{ github.event.number }} secrets: - # No DEBUSINE_RELEASE_TOKEN: PR builds never promote to a release - # workspace, so they leave target-workspace empty. DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + + build-ubuntu: + name: ${{ matrix.name }} + needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.ubuntu != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-ubuntu.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: -pr${{ github.event.number }} diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml new file mode 100644 index 00000000..f4eeb40a --- /dev/null +++ b/.github/workflows/release-kernel-debian.yml @@ -0,0 +1,223 @@ +name: release-kernel-debian + +# Build a kernel package for a Debian-family suite and promote it to a release +# workspace in Debusine. +# +# Called per matrix entry by release.yml. Identical to build-kernel-debian.yml +# up to the Debusine build -- both share the prepare-kernel-source and +# debusine-build actions -- and differs only in the tail: this one promotes the +# built package to target-workspace rather than publishing it to the daily S3 +# path. Which tail a run wants is decided by the workflow that calls it, so +# neither file carries the other's tail as a job to be skipped. +# +# Only the Debian family has a release path: promotion runs through Debusine, +# and resolve-matrix.py rejects a Release entry for any other suite. +# +# Jobs: +# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# build (debusine-pkg-builder container): submit to Debusine. +# release (debusine-pkg-builder container): promote to the target workspace. + +on: + workflow_call: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + default: 'qcom-next-trixie' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + default: 'qcom-next' + suite: + description: 'Target Debian suite (trixie, forky, sid)' + type: string + default: 'trixie' + kernel-branch: + description: 'Immutable ref to build, from the entry branch_or_tag' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: a Release entry always pins one' + type: string + default: 'pinned_ref' + tag-pattern: + description: 'Tag glob, unused by pinned_ref and carried for symmetry with the Daily inputs' + type: string + default: '' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' + type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + type: string + default: '' + target-workspace: + description: 'Debusine workspace to promote the built package into' + type: string + required: true + secrets: + DEBUSINE_USER: + description: 'Debusine account used to submit release builds' + required: false + DEBUSINE_TOKEN: + description: 'Debusine token for the CI child workspace' + required: false + DEBUSINE_RELEASE_TOKEN: + description: 'Debusine token for promotion to the release target workspace' + required: false + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- + prepare: + name: Generate source package + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder + steps: + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: debian + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + + # --------------------------------------------------------------------------- + # build: generate the Debian source package and submit it to Debusine. + # --------------------------------------------------------------------------- + build: + name: Build package (Debusine) + needs: prepare + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + environment: Production + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + outputs: + workspace: ${{ steps.debusine.outputs.workspace }} + srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} + steps: + - name: Checkout pkg-linux-qcom + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build in Debusine + id: debusine + uses: ./.github/actions/debusine-build + with: + suite: ${{ inputs.suite }} + flavour: ${{ inputs.flavour }} + debusine-host: ${{ vars.DEBUSINE_HOST }} + debusine-scope: ${{ vars.DEBUSINE_SCOPE }} + debusine-user: ${{ secrets.DEBUSINE_USER }} + debusine-token: ${{ secrets.DEBUSINE_TOKEN }} + debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + # A release build reads qli alone, so a released kernel is never + # built against something that has not been released alongside it. + extra-build-dep-workspaces: qli + + # --------------------------------------------------------------------------- + # release: promote the built package from the CI workspace to the target. + # --------------------------------------------------------------------------- + release: + name: Release to Debusine + needs: build + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + environment: Production + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Promote packages to target workspace + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + SUITE: ${{ inputs.suite }} + run: | + set -ex + debusine-action/lib/release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32eddcf5..a96f4caa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,14 +2,15 @@ name: release # Release kernel package builds. Triggered manually only. # -# Reads Release rows from ci/build-matrix.json, flattens suites, and builds -# each entry using the matrix-selected ref strategy and branch_or_tag. After a -# successful build, packages are promoted from the ephemeral Debusine CI -# workspace to the stable target workspace (e.g. qli) via debusine-action -# lib/release. +# Reads the Release entries from ci/build-matrix.yaml and builds each one using +# the matrix-selected ref strategy and branch_or_tag. After a successful build, +# packages are promoted from the ephemeral Debusine CI workspace to the stable +# target workspace (e.g. qli) via debusine-action lib/release. # -# To update a release ref: open a PR that changes the Release row for that -# kernel variant in ci/build-matrix.json. +# To update a release ref: open a PR that changes branch_or_tag on every +# Release entry for that flavour in ci/build-matrix.yaml. A flavour's Release +# entries must agree on the ref, so resolve-matrix.py rejects the matrix if +# only some of them are updated. # # This workflow intentionally has no schedule trigger. Releases are always # initiated manually after validation. @@ -18,22 +19,22 @@ on: workflow_dispatch: inputs: release-scope: - description: 'Choose all suites for one variant or one variant and suite' + description: 'Choose every suite of one flavour, or named build(s)' type: choice required: true - default: 'Selected variant (all suites)' + default: 'Selected flavour (all suites)' options: - - 'Selected variant (all suites)' - - 'Selected variant and suite' - kernel-variant: - description: 'Kernel variant to release' + - 'Selected flavour (all suites)' + - 'Selected builds' + flavour: + description: 'Kernel flavour to release in every suite it targets (Selected flavour)' required: true default: 'qcom-next' type: string - suite: - description: 'Target suite to release (used only for Selected variant and suite)' + builds: + description: 'Comma-separated build names, e.g. qcom-next-trixie,qcom-next-forky (Selected builds)' required: true - default: 'trixie' + default: 'qcom-next-trixie' type: string permissions: @@ -42,15 +43,17 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Release rows from ci/build-matrix.json. + # configure-matrix: select the Release entries from ci/build-matrix.yaml. # --------------------------------------------------------------------------- configure-matrix: name: Configure release matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -58,50 +61,60 @@ jobs: id: set-matrix env: RELEASE_SCOPE_INPUT: ${{ github.event.inputs.release-scope }} - INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} - INPUT_SUITE: ${{ github.event.inputs.suite }} + INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} + INPUT_BUILDS: ${{ github.event.inputs.builds }} run: | set -euo pipefail RELEASE_SCOPE="$RELEASE_SCOPE_INPUT" case "$RELEASE_SCOPE" in - "Selected variant (all suites)") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" - args=(--type Release --kernel-variant "$INPUT_VARIANT") + "Selected flavour (all suites)") + echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" + args=(--type Release --flavour "$INPUT_FLAVOUR") ;; - "Selected variant and suite") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Release --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") + "Selected builds") + echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" + args=(--type Release --build "$INPUT_BUILDS") ;; *) echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 exit 1 ;; esac - MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # build: one parallel job per flat Release matrix entry. + # build: one parallel job per Release matrix entry. # # Uses the ref strategy and branch_or_tag selected by the matrix. - # Passes target_workspace so build-kernel-debusine.yml triggers the Debusine - # release path (lib/release) instead of the S3 publish path. + # release-kernel-debian.yml builds exactly as the daily workflow does and + # then promotes to target_workspace, rather than publishing to the daily S3 + # path. Choosing the workflow here is what makes that the whole of the run: + # no job in it exists to publish the other way and be skipped. + # + # There is no Ubuntu counterpart. Promotion runs through Debusine, which + # builds only the Debian family, so resolve-matrix.py rejects a Release entry + # for any other suite and this matrix is Debian by construction. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + name: ${{ matrix.name }} needs: configure-matrix strategy: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/release-kernel-debian.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} diff --git a/.gitignore b/.gitignore index 2583b54a..d2fc2efd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ debian/linux-image-*.prerm # Default kernel source and build output directories kernel-source/ kernel-build/ + +# Python bytecode cache from ci/scripts/ +__pycache__/ diff --git a/README.md b/README.md index 294e6f48..3fefeeb2 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,8 @@ kernel variants, each with independent source/package identity, kernel source and ref strategy, configuration fragments, Debian revision, target suites, and release destination. -Every kernel variant owns exactly two complete matrix rows: one `Daily` row and -one `Release` row. The resolver expands every suite in those rows into an -isolated `kernel_variant + suite` build leg. +Every entry in the matrix is one isolated build leg producing one package. A +kernel flavour owns as many entries as it has suites and delivery types. ### Configured variants @@ -25,14 +24,22 @@ isolated `kernel_variant + suite` build leg. |---------|----------------|-------------------|--------------|----------------|-------| | `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | trixie, forky | Standard kernel | | `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | - -Both build the same kernel ref. `derive-localversion.sh` folds the variant name -into LOCALVERSION, so each produces a distinct kernel release -(`+qcom-next--g` and `+qcom-next-debug--g`) and therefore a -distinct versioned image package that can be installed alongside the other. See +| `qcom-arduino` | `linux-qcom-arduino` | `linux-image-qcom-arduino` | trixie, forky | none | Arduino hardware-enablement topic branch (`early/hwe/arduino` of `kernel-topics`) | +| `mainline` | `linux-mainline` | `linux-image-mainline` | trixie, forky | none | Tip of Linus's tree, tracked for early warning of upstream breakage. No DKMS modules | +| `next` | `linux-next` | `linux-image-next` | trixie, forky | none | Newest `next-YYYYMMDD` tag of linux-next. No DKMS modules | + +`derive-localversion.sh` folds the *flavour* into LOCALVERSION, so each +produces a distinct kernel release (`+qcom-next--g`, +`+qcom-next-debug--g`, and so on) and therefore a distinct versioned +image package that can be installed alongside the others. The flavour is what +the kernel is; a build's `name` is only what CI calls it. See [docs/version.md](docs/version.md) for how the version strings are composed. -`ci/build-matrix.json` is the source of truth; this table is a summary. +The last three track a moving upstream — a branch tip, or a tag cut every +night — and so have no `Release` entries: a release must name an immutable ref +to promote. They are built and published daily and never promoted to `qli`. + +`ci/build-matrix.yaml` is the source of truth; this table is a summary. Two entry points use the same reusable build pipeline: @@ -41,56 +48,57 @@ Two entry points use the same reusable build pipeline: - **Release** uses a pinned matrix ref and promotes successful Debian packages to the selected production Debusine workspace. -The final Production matrix is conceptually: - -```json -{ - "suite_suffix_mapping": { - "trixie": "~bpo13+1", - "forky": "", - "resolute": "~26.04.1" - }, - "deliveries": [ - { - "kernel_variant": "qcom-next", - "type": "Daily", - "suites": ["trixie", "forky", "resolute"], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next", - "ref_strategy": "latest_tag", - "tag_pattern": "qcom-next-*", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli1", - "debian_version_suffix": "~" - }, - { - "kernel_variant": "qcom-next", - "type": "Release", - "suites": ["trixie", "forky"], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "", - "ref_strategy": "pinned_ref", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli1", - "debian_version_suffix": "", - "target_workspace": "qli" - } - ] -} +One entry in `builds` is one generated package: a single `name`, a single +`type`, and a single `suite`. The `name` labels that one build and nothing +else — it is the Actions job name and what a manual dispatch asks for. Nothing +is expanded or derived at resolve time, so what an entry says is what gets +built: + +```yaml +builds: + - name: qcom-next-trixie + type: Daily + suite: trixie + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1~' + + - name: qcom-next-trixie + type: Release + suite: trixie + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli ``` -`suite_suffix_mapping` is matrix-wide policy, not duplicated per row: every -suite referenced by any row's `suites` must have an entry here, and every -delivery for a variant derives its final `debian_revision` as -`debian_version_stub + suite_suffix_mapping[suite] + delivery_suffix`, where -`delivery_suffix` is `~` for Daily and empty for Release. For the values -above: +Entries are written out in full rather than sharing YAML anchors, so each one +can be read, grepped, and changed on its own. `resolve-matrix.py` enforces the +consistency that duplication would otherwise put at risk — see +[Matrix Model](#matrix-model). + +Each entry states its `debian_revision` outright. The configured values are: | Suite | Daily | Release | | --- | --- | --- | @@ -99,19 +107,21 @@ above: | Resolute | `0qli1~26.04.1~` | (not a configured Release suite) | `~` always sorts below the same prefix without it in Debian version -ordering, so Daily always sorts below Release for the same suite and stub. -Ordering across *different* suites depends entirely on the configured -suffixes: with the mapping above, Resolute < Trixie < Forky for the same +ordering, so a suite's Daily always sorts below its Release; `resolve-matrix.py` +requires a Daily revision to be exactly its Release revision plus a trailing +`~`. Ordering across *different* suites depends entirely on the configured +revisions: with the values above, Resolute < Trixie < Forky for the same delivery type, matching a Debian-backports-then-unstable promotion chain. This is a deliberate ordering policy, not an automatic guarantee — adding a -suite means choosing a suffix that sorts where that suite belongs relative to -the others. One nuance to be aware of: because Forky's suffix is empty, its -Daily revision ends immediately after the trailing `~`, so Trixie Daily does -not sort below Forky Daily even though Trixie Release sorts below Forky -Release. This does not affect the supported Release-to-Release upgrade path. +suite means choosing a revision that sorts where that suite belongs relative to +the others. One nuance to be aware of: because Forky's revision carries no +suite component, its Daily revision ends immediately after the trailing `~`, so +Trixie Daily does not sort below Forky Daily even though Trixie Release sorts +below Forky Release. This does not affect the supported Release-to-Release +upgrade path. -`ci/build-matrix.json` is the authoritative configuration. Adding a kernel -variant is a two-row matrix change, not a workflow redesign. +`ci/build-matrix.yaml` is the authoritative configuration. Adding a kernel +variant is a matrix change, not a workflow redesign. ## Workflows @@ -119,9 +129,24 @@ variant is a two-row matrix change, not a workflow redesign. | --- | --- | --- | | `daily.yml` | Resolves and runs the Daily matrix. | Scheduled daily at `23:00 UTC`, or manual dispatch. | | `release.yml` | Resolves and runs the Release matrix. | Manual dispatch only. | -| `build-kernel-deb.yml` | Reusable orchestrator for one kernel variant and suite. | Manual dispatch or called by Daily and Release. | -| `build-kernel-debusine.yml` | Builds Debian suites in Debusine and either publishes Daily artifacts or promotes Releases. | Called by `build-kernel-deb.yml`. | -| `build-kernel-ubuntu.yml` | Builds Ubuntu-family suites with the Docker path. | Called by `build-kernel-deb.yml`. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and publishes it to S3. | Manual dispatch or called by Daily and PR build. | +| `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Manual dispatch or called by Daily and PR build. | +| `release-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and promotes it to the release workspace. | Called by Release. | + +The three build workflows share their steps through two composite actions +rather than through a common orchestrator workflow: + +| Action | Used by | +| --- | --- | +| `.github/actions/prepare-kernel-source` | All three, as the `prepare` job. | +| `.github/actions/debusine-build` | The two Debian workflows, as the `build` job. | + +Which of them a build leg calls is decided by the caller, from the entry's +suite: `resolve-matrix.py --family debian|ubuntu` splits the selection, and +each family's entries call only the workflow that builds them. Nothing inside a +build workflow is conditional on the suite or on whether the run releases, so a +run starts exactly the jobs it needs and shows no skipped job for a path it did +not take. ### Daily @@ -130,8 +155,10 @@ Daily is the recurring build and artifact-publication path. - The scheduled run resolves the full `Daily` matrix. - A manual run selects one **Build scope**: - **Full matrix** builds every configured variant and suite. - - **Selected variant (all suites)** builds every configured suite for one variant. - - **Selected variant and suite** builds one isolated matrix leg. + - **Selected builds** builds a comma-separated list of build names, e.g. + `qcom-next-trixie,qcom-next-debug-forky`. + - **Selected flavour (all suites)** builds one flavour in every suite it + targets. - `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves the configured branch directly. - Debian suites build in Debusine, then their `.deb` outputs are downloaded and @@ -144,13 +171,13 @@ Daily is the recurring build and artifact-publication path. Release is the controlled promotion path. - It is manual only and uses one **Release scope** for a kernel variant: - - **Selected variant (all suites)** is the normal release action and promotes - every configured Release suite for that variant. - - **Selected variant and suite** promotes one configured Release suite for - that variant when a targeted action is required. + - **Selected flavour (all suites)** is the normal release action and promotes + every configured Release suite for that flavour. + - **Selected builds** promotes a comma-separated list of build names when a + targeted action is required. - It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it never resolves a newest tag. -- Debian source and binary artifacts are built in per-variant, per-suite +- Debian source and binary artifacts are built in per-flavour, per-suite Debusine CI workspaces. - Successful builds are promoted with Debusine's `package-publish` workflow to the `qli` workspace, where they are available through the production Debusine @@ -159,88 +186,107 @@ Release is the controlled promotion path. the release credential and enforces the required approval gate before promotion to `qli`. -Direct `build-kernel-deb.yml` dispatches are build-only. Release promotion is +Direct `build-kernel-debian.yml` and `build-kernel-ubuntu.yml` dispatches are +build-only: neither has a promotion path to offer. Release promotion is initiated exclusively by `release.yml`, which owns the target workspace and -production release controls. +production release controls, and is the only caller of +`release-kernel-debian.yml`. + +Only the Debian family has a release path at all, because promotion runs +through Debusine. `resolve-matrix.py` rejects a `Release` entry for any other +suite rather than letting it build and then silently not promote. ## Matrix Model -`ci/build-matrix.json` is an object with two top-level keys: `deliveries` -(the matrix rows) and `suite_suffix_mapping` (matrix-wide Debian suffix -policy, shared by every variant and delivery type). `ci/scripts/resolve-matrix.sh` -validates the document, requires each `kernel_variant` to have exactly one -`Daily` and one `Release` row in `deliveries`, filters by delivery type, and -flattens each `suites` array into independent suite legs. Each leg carries -its own values for: +`ci/build-matrix.yaml` is a mapping with exactly one top-level key, `builds`. +One entry in it is one generated package, so there is no +expansion step: `ci/scripts/resolve-matrix.py` validates the whole document, +selects the entries matching the requested type, variant, and suite, and hands +them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | -| `kernel_variant` | Stable identifier for a separately packaged kernel variant. Lowercase letters, digits, and internal hyphens only. | +| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only. | | `type` | `Daily` or `Release`. | -| `suites` | Suites to flatten into individual build legs. Each must have a `suite_suffix_mapping` entry. | +| `suite` | The one suite this entry builds for. | +| `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | | `git_clone` | Kernel source repository. | | `branch_or_tag` | Source branch or pinned tag, according to `ref_strategy`. | | `ref_strategy` | `latest_tag`, `branch_tip`, or `pinned_ref`. | | `tag_pattern` | Required only for `latest_tag`; matching tags must end in `-YYYYMMDD`, which determines newest-first ordering. | | `srcpkg` | Debian source package name. | | `binpkg` | Kernel image metapackage name. | -| `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per array element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.sh` joins it into the comma-separated `kernel-config` workflow input. | -| `dkms` | Out-of-tree DKMS modules built against this kernel and bundled into its `linux-image` package, one per array element, each named as the stem of its `-dkms` package (e.g. `kgsl`). Empty bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.sh` joins it into the comma-separated `dkms` workflow input. | -| `debian_version_stub` | Base Debian revision, shared by a variant's Daily and Release rows. Must not end in `~`; the suite suffix is derived, not stored here. | -| `debian_version_suffix` | `~` for Daily rows, empty for Release rows. Documents the delivery-type half of the revision formula on the row itself; `resolve-matrix.sh` rejects a row where this disagrees with `type`, but derivation always computes this suffix from `type`, never reads this field. | +| `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per list element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.py` joins it into the comma-separated `kernel-config` workflow input. | +| `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). Each needs a `-dkms` package in the suite being built for, so this varies between suites. An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | +| `debian_revision` | The Debian revision this package is built at, stated outright. Daily revisions end in `~`; Release revisions do not. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | | `target_workspace` | Debusine destination for Release entries only. | -`dkms` currently has one exception the matrix cannot express: `build-kernel-deb.yml` -replaces the resolved list with `kgsl` on Ubuntu-family legs, so `camx` and -`iris-vpu` are bundled on Debian suites only. That override is temporary and goes -away once the matrix gains per-suite `dkms` lists. - -`target_workspace` is required for `Release` and rejected for `Daily`. -`tag_pattern` is required for `latest_tag` and rejected for other strategies. -The resolver selects the most recent trailing `YYYYMMDD` date, and rejects -duplicate suites and malformed variant identifiers before any build jobs -start. It also rejects a matrix where any configured suite has no -`suite_suffix_mapping` entry, where two suites share the same suffix, where a -suffix is non-empty and doesn't start with `~`, where a variant's Daily -and Release rows disagree on `debian_version_stub`, or where a row's -`debian_version_suffix` doesn't match what its `type` implies — all before -any build job starts. - -Each flattened leg's final `debian_revision` is derived by -`ci/scripts/derive-debian-revision.sh` from `debian_version_stub`, -`suite_suffix_mapping[suite]`, and the delivery type -(`stub + suffix + "~"` for Daily, `stub + suffix` for Release). This script is -the single implementation of the formula: `resolve-matrix.sh` calls it once -per flattened leg, and `build-kernel-deb.yml`'s direct-dispatch path (which -has no full-matrix context) calls the same script for the one suite it was -given. - -Each leg has a distinct prepared-source artifact, Debusine child workspace, and -S3 path keyed by `kernel_variant + suite`. This prevents two variants that both +`resolve-matrix.py` rejects the matrix — before any build job starts — where an +entry has an unknown field or a missing required one, a malformed variant or +suite identifier, a `ref_strategy` its `type` does not allow (`Daily` must +track something moving, `Release` must be pinned), a `tag_pattern` without +`latest_tag`, a `target_workspace` on a `Daily` entry or none on a `Release` +one, a `kernel_config` fragment that escapes the kernel source root or collides +with another fragment's filename, a `dkms` entry that is not a package name +stem or repeats, or a `debian_revision` that is not a valid Debian revision or +carries the wrong trailing `~` for its `type`. + +Because entries are written out in full, the resolver also checks the +invariants that span them, which is what makes the duplication safe to read at +face value: + +- No two entries of one `type` share a `name` — a name identifies exactly one + build, so a run cannot produce two jobs with one name and a dispatch cannot + be ambiguous. +- A flavour's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those + decide what the package *is*, and the entries differ only in where it goes. +- A flavour's entries for one suite agree on `dkms`. The module set depends on + which `-dkms` packages the target archive carries, so it varies between + suites — but a suite's Daily and Release must match, or the Daily is not + testing the module set the Release will ship. +- A flavour's entries of one `type` agree on `git_clone`, `branch_or_tag`, + `ref_strategy` and `tag_pattern`, so a forgotten suite cannot quietly ship a + different kernel from its siblings after a release ref bump. +- A flavour with a `Release` entry also has a `Daily` one, so nothing is + promoted that the daily build has not tested. The converse is allowed: a + flavour tracking a moving upstream is built daily and never released. +- No `srcpkg` or `binpkg` is shared between flavours, and no two entries build + the same `srcpkg` at the same `debian_revision`. +- Where a suite has both, its `Daily` revision is its `Release` revision plus a + trailing `~`. + +A direct dispatch of a build workflow has no matrix context of its own, so when +its `debian-revision` input is empty the `prepare-kernel-source` action looks up +the `Daily` entry for the build name and suite it was given (`resolve-matrix.py +--field debian_revision`) and builds at the revision the daily build would have +used. + +Each entry has a distinct prepared-source artifact, Debusine child workspace, +and S3 path keyed by `flavour + suite`. This prevents two flavours that both build, for example, `trixie` from consuming or publishing each other's inputs -or outputs. +or outputs. They key on `flavour`, not on the build's `name`, so renaming a +build never moves a published artifact. Daily S3 outputs use these layouts, where `` is `-`: ```text -/pkg/debusine///// -/pkg/temp///// +/pkg/debusine///// +/pkg/temp///// ``` The first layout is for Debian/Debusine builds; the second is for Ubuntu Docker -builds. Consumers must select the intended kernel variant and suite. +builds. Consumers must select the intended flavour and suite. Supporting scripts keep workflow YAML small and testable: | Script | Responsibility | | --- | --- | -| `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | +| `ci/scripts/resolve-matrix.py` | Validates the delivery matrix and selects the entries to build. Needs PyYAML (`python3-yaml`). | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | -| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | -| `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | +| `ci/scripts/derive-localversion.sh` | Derives the version fields from the flavour, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | ## Architecture @@ -253,16 +299,22 @@ This document covers the CI generator. For the packaging internals: `debian/rule targets, the config fragment merge pipeline, DKMS module bundling and the produced package layout see [debian/README.md](debian/README.md). +Both branches below are taken in the caller, when the matrix is resolved: the +family from the entry's suite, the tail from which workflow is running. By the +time a build workflow starts, there is nothing left to decide. + ```mermaid flowchart LR - IN["Matrix variant + suite input"] --> R{"Resolve suite family"} + IN["Matrix entries"] --> R{"resolve-matrix.py\n--family"} - R -->|"trixie · forky"| DEB["Debian path\nbuild-kernel-debusine.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] - R -->|"resolute"| UBU["Ubuntu path\nbuild-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] + R -->|"debian: trixie · forky"| DT{"Which caller"} + R -->|"ubuntu: resolute"| UBU["build-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] - DEB --> DOUT{"Build type"} - DOUT -->|Daily| S3["Download .deb files\nPublish to S3"] - DOUT -->|Release| QLI["Promote source and binaries\nto qli"] + DT -->|"daily.yml · pr-build.yml"| DEB["build-kernel-debian.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] + DT -->|"release.yml"| REL["release-kernel-debian.yml\nSame build, release tail"] + + DEB --> S3["Download .deb files\nPublish to S3"] + REL --> QLI["Promote source and binaries\nto qli"] UBU --> US3["Publish .deb files to S3"] ``` @@ -274,23 +326,26 @@ flowchart LR flowchart TD subgraph triggers[Triggers] A1["daily.yml\nScheduled full matrix"] - A2["daily.yml\nManual full or filtered variant + suite"] - A3["release.yml\nManual full or filtered variant + suite"] - A4["build-kernel-deb.yml\nManual one-off build"] + A2["daily.yml\nManual full or filtered builds"] + A5["pr-build.yml\nFull Daily matrix on every PR"] + A3["release.yml\nManual full or filtered builds"] + A4["build-kernel-debian.yml\nbuild-kernel-ubuntu.yml\nManual one-off build"] end subgraph matrix[Matrix entry points] - B1["Daily configure-matrix\nFlatten Daily rows"] - B2["Daily variant + suite legs\nqcom-next / trixie · forky · resolute\nqcom-next-debug / trixie · forky"] - B3["Release configure-matrix\nFlatten Release rows"] - B4["Release variant + suite legs\nqcom-next / trixie · forky\nqcom-next-debug / trixie · forky"] + B1["configure-matrix\nDaily entries, split by family"] + B2["build-debian legs\nqcom-next · qcom-next-debug · qcom-arduino\nmainline · next / trixie · forky"] + B5["build-ubuntu legs\nqcom-next / resolute"] + B3["configure-matrix\nRelease entries, Debian by construction"] + B4["build legs\nqcom-next · qcom-next-debug / trixie · forky"] end - subgraph orchestrator[build-kernel-deb.yml] - C1["resolve\nClassify suite family"] - C2["prepare\nClone selected kernel ref\nRun prepare-source.sh\nUpload kernel-srcpkg-variant-suite"] - C3["debusine-build\nDebian suites only"] - C4["ubuntu-build\nUbuntu suites only"] + subgraph build[One build workflow per leg] + C2["prepare\nprepare-kernel-source action\nClone ref, run prepare-source.sh\nUpload kernel-srcpkg-flavour-suite"] + C3["build\ndebusine-build action"] + C4["build\nbuild-kernel.sh in Docker"] + C5["publish\nDownload .deb files, upload to S3"] + C6["release\nPromote to target workspace"] end subgraph outputs[Outputs] @@ -300,16 +355,25 @@ flowchart TD A1 --> B1 A2 --> B1 + A5 --> B1 A3 --> B3 - B1 --> B2 --> C1 - B3 --> B4 --> C1 - A4 --> C1 - C1 --> C2 + B1 --> B2 & B5 + B3 --> B4 + B2 --> C2 + B5 --> C2 + B4 --> C2 + A4 --> C2 C2 --> C3 & C4 - C3 --> D1 & D2 + C3 --> C5 & C6 C4 --> D1 + C5 --> D1 + C6 --> D2 ``` +Every leg runs every job drawn under it. `build-debian` and `build-ubuntu` legs +reach different build workflows, and `publish` and `release` belong to +different ones, so no leg starts a job it will skip. + ### Prepare stage ```mermaid @@ -428,21 +492,25 @@ out-of-tree module builds are required. ## Manual Builds -Use **Actions** → **build-kernel-deb** for a one-off build. It is an explicit -override workflow, not a matrix-derived delivery flow: use `daily.yml` and -`release.yml` for normal Daily and Release operations. +Use **Actions** → **build-kernel-debian** or **build-kernel-ubuntu** for a +one-off build, picking the one that builds the suite you want. These are +explicit override workflows, not matrix-derived delivery flows: use `daily.yml` +and `release.yml` for normal Daily and Release operations. Neither promotes. -`kernel-variant`, `suite`, and `ref-strategy` are the required build selection. +`build`, `suite`, and `ref-strategy` are the required build selection. All remaining package, configuration, and PR inputs are advanced overrides for -validation or debugging. Variant and suite are free-text matrix values rather -than static dropdowns, so adding a matrix entry never requires editing the -workflow UI. +validation or debugging. Build name and suite are free-text matrix values +rather than static dropdowns, so adding a matrix entry never requires editing +the workflow UI. A suite belonging to the other family is rejected by the +`prepare` job before anything is cloned or built, so the only cost of picking +the wrong workflow is a fast failure. The available inputs are: | Input | Default | Purpose | | --- | --- | --- | -| `kernel-variant` | `qcom-next` | Stable variant identifier used in artifact and workspace identity. | +| `build` | `qcom-next-trixie` | Names this build; also selects the matrix entry a blank `debian-revision` is taken from. | +| `flavour` | `qcom-next` | Kernel flavour: the LOCALVERSION suffix, and so the kernel release identity. | | `suite` | `trixie` | Target suite. | | `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | | `kernel-branch` | `qcom-next` | Branch for `branch_tip`, or immutable ref for `pinned_ref`; ignored by `latest_tag`. | @@ -451,7 +519,8 @@ The available inputs are: | `srcpkg` | `linux-qcom-next` | Advanced source package identity override. | | `binpkg` | `linux-image-qcom-next` | Advanced image metapackage identity override. | | `kernel-config` | Empty | Advanced extra fragments applied on top of all of `debian/config-available/`, e.g. `intree:arch/arm64/configs/qcom_debug.config`. | -| `debian-version-stub` | `0qli1` | Advanced Debian version stub. The selected suite's mapped suffix and a Daily-style trailing `~` are applied automatically; direct builds always use Daily semantics since they are build-only and non-promoting. | +| `dkms` | Empty | Advanced comma-separated DKMS modules bundled into the image package, each without the `-dkms` suffix. Empty bundles none. | +| `debian-revision` | The matrix Daily revision | Advanced Debian revision override. Left empty, the build takes the `debian_revision` of the matrix's `Daily` entry for the selected variant and suite; direct builds always use the Daily entry since they are build-only and non-promoting. | | `localversion` | Auto-derived | Advanced explicit `LOCALVERSION` override. | | `kver-extra` | Empty | Advanced kernel-release suffix. | | `debug-build` | `false` | Advanced debug configuration toggle. | @@ -487,37 +556,43 @@ and keeps production approval controls in the workflow path. To add a kernel variant: -1. Add exactly two rows to `deliveries` with the same `kernel_variant`: one - `Daily` and one `Release`. -2. Define all package identity, source/ref strategy, configuration, - `debian_version_stub`, and suite values in both rows. Do not rely on - another variant's values. `srcpkg`, `binpkg`, and `debian_version_stub` - must remain identical across the pair. Set `debian_version_suffix` to `~` - on the Daily row and `""` on the Release row; `resolve-matrix.sh` rejects - the pair if either disagrees with its row's `type`. +1. Add one entry to `builds` per package the variant should produce: one per + Daily suite and one per Release suite, each spelling out all of its own + fields, and each with a `name` unique within its delivery type. Do not rely + on another variant's values. +2. Give them all the same `flavour`, distinct from every other flavour's — it + becomes the LOCALVERSION suffix, so this is what lets the new kernel install + alongside the existing ones. Keep `srcpkg`, `binpkg` and `kernel_config` + identical across every entry for the flavour, and the ref fields identical + across its entries of one `type`. `resolve-matrix.py` rejects the matrix if + they drift apart. 3. Use `latest_tag` with a dated tag glob or `branch_tip` for Daily. Use `pinned_ref` for Release, and update that ref through a reviewed PR. -4. Give the variant distinct `srcpkg` and `binpkg` values. Set the Release - `target_workspace` explicitly. -5. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use +4. Give the variant distinct `srcpkg` and `binpkg` values. Set + `target_workspace` on each Release entry. +5. Give each entry a `debian_revision`: the Daily one is the Release one for + the same suite plus a trailing `~`. +6. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use the Docker path. -6. Run a filtered Daily validation for the new variant, then its full Daily and +7. Run a filtered Daily validation for the new variant, then its full Daily and Release flows. To add a new suite (for an existing or new variant): -1. Add an entry for it to the shared top-level `suite_suffix_mapping`, empty - or starting with `~`, and distinct from every other suite's suffix. -2. Add the suite to the `suites` array of the relevant Daily and/or Release - rows. `resolve-matrix.sh` rejects any configured suite with no mapping - entry before any build job starts. -3. Choose the suffix so the suite sorts where it belongs relative to the +1. Add one entry per delivery type the suite should get, copying the variant's + existing entry for that type and changing `suite` and `debian_revision`. +2. Choose the revision so the suite sorts where it belongs relative to the others for the same delivery type (see the ordering discussion in - [Overview](#overview)). + [Overview](#overview)), and so it does not collide with another entry + building the same `srcpkg`. No workflow dispatch choices need to be updated: manual Daily and Release inputs accept matrix-defined variant and suite strings. +Run `ci/scripts/resolve-matrix.py --type Daily` and `--type Release` locally to +validate a matrix change before pushing it; both validate the whole document, +so either one catches a mistake in the other's entries. + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for branch, review, and DCO diff --git a/build-kernel.sh b/build-kernel.sh index 23375748..a0dd5668 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -8,9 +8,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DEFAULT_REPO="git@github.com:qualcomm-linux/kernel.git" +DEFAULT_REPO="https://github.com/qualcomm-linux/kernel" DEFAULT_BRANCH="qcom-next" DEFAULT_DISTRO="trixie" +DEFAULT_FLAVOUR="qcom-next" DEFAULT_BUILD_MODE="docker" KERNEL_DIR="$SCRIPT_DIR/kernel-source" OUTPUT_BASE_DIR="$SCRIPT_DIR/kernel-build" @@ -36,13 +37,28 @@ OPTIONS: -l, --latest-tag Select the latest qcom-next-* tag automatically -b, --branch BRANCH Branch to use (default: $DEFAULT_BRANCH) -r, --repo URL Kernel repository URL (default: $DEFAULT_REPO) + --dsc FILE Build from an existing Debian source package + instead of a kernel tree: no clone, no prepare. + The .orig.tar.gz and .debian.tar.xz the .dsc + names must sit beside it, as build-source-package.sh + and CI leave them. + + What to build: + --source-package Stop after building the Debian source package + (.orig.tar.gz, .dsc, .changes) into the output + directory, with build-source-package.sh. The orig + tarball is reproducible: see that script. Build + binaries from it later with --dsc. Version control: - --localversion TAG LOCALVERSION suffix (e.g. qcom-next-20260312) - Auto-detected from git tag when using --local-source + --flavour NAME Kernel flavour carried in LOCALVERSION + (default: $DEFAULT_FLAVOUR). Ignored with --localversion. + --localversion SUFFIX LOCALVERSION suffix (e.g. +qcom-next-20260312-g07f50dc44edd) + Derived from the checked-out tag or branch if not given, + by prepare-source.sh, the same way CI derives it. --kver-extra SUFFIX Extra suffix appended to the derived KVER, e.g.: --kver-extra -mybuild - Results in: 7.0.0-rc2-qcom-next-20260312-mybuild + Results in: 7.0.0-rc2+qcom-next-20260312-g07f50dc44edd-mybuild Useful for CI build IDs or local user builds. Build control: @@ -82,6 +98,8 @@ EXAMPLES: $0 --local-source /path/to/kernel --kver-extra -mybuild $0 --latest-tag --kernel-config docker,systemd-boot $0 --latest-tag --dkms kgsl + $0 --latest-tag --source-package + $0 --dsc kernel-build/trixie/linux-qcom-next_*.dsc DISTRIBUTIONS: noble Ubuntu 24.04 LTS @@ -95,10 +113,10 @@ EOF # Defaults TAG=""; LATEST_TAG=false; BRANCH="$DEFAULT_BRANCH"; REPO="$DEFAULT_REPO" -DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE" +DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE"; FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION=""; KVER_EXTRA=""; PROFILES=""; CLEAN=false LOCAL_SOURCE=""; ENABLE_CONFIGS="squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can"; SKIP_PREPARE=false -DKMS_MODULES="" +DKMS_MODULES=""; DSC=""; SOURCE_PACKAGE=false while [[ $# -gt 0 ]]; do case $1 in @@ -111,7 +129,10 @@ while [[ $# -gt 0 ]]; do --debian-dir) DEBIAN_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; --local-source) LOCAL_SOURCE="$2"; shift 2 ;; + --dsc) DSC="$2"; shift 2 ;; + --source-package) SOURCE_PACKAGE=true; shift ;; --docker-build) DOCKER_PKG_BUILD="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --profiles) PROFILES="$2"; shift 2 ;; @@ -142,20 +163,35 @@ VALID_MODES=(docker native sbuild) exit 1 } -# Locate docker_deb_build.py (docker mode) +# --dsc replaces the tree: everything that selects or prepares one is +# meaningless beside it, and so is asking for the source package it already is. +if [[ -n "$DSC" ]]; then + [[ -z "$LOCAL_SOURCE$TAG" && "$LATEST_TAG" == false && "$SKIP_PREPARE" == false && "$SOURCE_PACKAGE" == false ]] || { + log_error "--dsc cannot be combined with --local-source, --tag, --latest-tag, --skip-prepare or --source-package" + exit 1 + } + [[ -f "$DSC" ]] || { log_error "Source package not found: $DSC"; exit 1; } + DSC="$(cd "$(dirname "$DSC")" && pwd)/$(basename "$DSC")" +fi + +# Locate docker_deb_build.py (docker mode). Building binaries from a tree goes +# through it, so it must exist for that. Building from a .dsc runs sbuild in +# the image directly and only needs it to build the image when the image is +# missing, and --source-package builds no binaries at all, so for those it is +# looked for and not required. if [[ "$BUILD_MODE" == "docker" && -z "$DOCKER_PKG_BUILD" ]]; then for p in "$HOME/docker-pkg-build/docker_deb_build.py" \ "$SCRIPT_DIR/docker-pkg-build/docker_deb_build.py" \ "$(which docker_deb_build.py 2>/dev/null || true)"; do [[ -x "$p" ]] && { DOCKER_PKG_BUILD="$p"; break; } done - [[ -z "$DOCKER_PKG_BUILD" ]] && { + [[ -z "$DOCKER_PKG_BUILD" && -z "$DSC" && "$SOURCE_PACKAGE" == false ]] && { log_error "docker_deb_build.py not found. Use --docker-build, set DOCKER_PKG_BUILD, or use --build-mode native." exit 1 } fi [[ "$BUILD_MODE" == "docker" && -d "$DOCKER_PKG_BUILD" ]] && DOCKER_PKG_BUILD="$DOCKER_PKG_BUILD/docker_deb_build.py" -[[ "$BUILD_MODE" == "docker" && ! -x "$DOCKER_PKG_BUILD" ]] && { log_error "Not executable: $DOCKER_PKG_BUILD"; exit 1; } +[[ "$BUILD_MODE" == "docker" && -n "$DOCKER_PKG_BUILD" && ! -x "$DOCKER_PKG_BUILD" ]] && { log_error "Not executable: $DOCKER_PKG_BUILD"; exit 1; } # Handle local source if [[ -n "$LOCAL_SOURCE" ]]; then @@ -165,39 +201,39 @@ if [[ -n "$LOCAL_SOURCE" ]]; then fi log_step "Configuration:" -[[ -n "$LOCAL_SOURCE" ]] && log_info " Source: local ($KERNEL_DIR)" \ - || log_info " Repo: $REPO branch: $BRANCH" +if [[ -n "$DSC" ]]; then + log_info " Source: $DSC" +elif [[ -n "$LOCAL_SOURCE" ]]; then + log_info " Source: local ($KERNEL_DIR)" +else + log_info " Repo: $REPO branch: $BRANCH" +fi log_info " Output: $OUTPUT_DIR" +[[ "$SOURCE_PACKAGE" == true ]] && log_info " Building: source package only" log_info " Distro: $DISTRO mode: $BUILD_MODE" -[[ "$BUILD_MODE" == "docker" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" -[[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" -[[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" +[[ "$BUILD_MODE" == "docker" && -n "$DOCKER_PKG_BUILD" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" [[ -n "$PROFILES" ]] && log_info " Profiles: $PROFILES" -[[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" -[[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" -[[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" +# The rest describes preparing a tree, which a .dsc has already had done. +if [[ -z "$DSC" ]]; then + [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" \ + || log_info " Flavour: $FLAVOUR" + [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" + [[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" + [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" + [[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" +fi echo -# ── Helper: derive LOCALVERSION from a tag name ────────────────────────────── -# qcom-next-6.19-rc8-20260210 → qcom-next-20260210 -_auto_localversion() { - local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then - echo "${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" - else - echo "$tag" - fi -} - # ── Git operations: resolve ref → sync → checkout ──────────────────────────── -if [[ -z "$LOCAL_SOURCE" ]]; then - # Resolve the latest tag remotely before any network I/O (avoids fetching all tags) +if [[ -n "$DSC" ]]; then + : # No tree: the source package is the source. +elif [[ -z "$LOCAL_SOURCE" ]]; then + # Resolve the latest tag remotely before any network I/O (avoids fetching + # all tags). The same script CI uses, so "latest" means the same thing + # here: the newest trailing date, not the highest kernel version. if [[ "$LATEST_TAG" == true ]]; then log_step "Finding latest qcom-next-* tag from remote..." - TAG=$(git ls-remote --tags "$REPO" 'refs/tags/qcom-next-*' \ - | awk '{print $2}' | sed 's|refs/tags/||' | grep -v '\^{}' \ - | sort -V | tail -1) - [[ -n "$TAG" ]] || { log_error "No qcom-next-* tags found in $REPO"; exit 1; } + TAG=$("$SCRIPT_DIR/ci/scripts/resolve-kernel-ref.sh" --url "$REPO" --latest-tag 'qcom-next-*') log_info "Latest tag: $TAG" fi @@ -229,39 +265,25 @@ if [[ -z "$LOCAL_SOURCE" ]]; then git -C "$KERNEL_DIR" checkout -B "$BRANCH" FETCH_HEAD fi fi - - # Auto-detect LOCALVERSION from tag (applies to both clone and update paths) - if [[ -n "$TAG" && -z "$LOCALVERSION" ]]; then - LOCALVERSION="$(_auto_localversion "$TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION'" - fi -fi - -cd "$KERNEL_DIR" - -# ── Local source: LOCALVERSION detection ───────────────────────────────────── -if [[ -n "$LOCAL_SOURCE" ]]; then +else log_info "Using local source as-is (skipping git checkout)" - if [[ -z "$LOCALVERSION" ]]; then - GIT_TAG=$(git describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - LOCALVERSION="$(_auto_localversion "$GIT_TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' from tag '$GIT_TAG'" - else - log_warn "LOCALVERSION not set and no exact git tag found." - log_warn "Package will be named linux-image--qcom (no branch/ABI suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion qcom-next-20260312" - fi - fi fi +[[ -n "$DSC" ]] || cd "$KERNEL_DIR" + # ── Source preparation ──────────────────────────────────────────────────────── # Delegates to prepare-source.sh, which is the single source of truth for -# debian/ injection, config fragment activation, and debian/rules prepare. +# debian/ injection, config fragment activation, version derivation, and +# debian/rules prepare. Nothing about the version is decided here: an explicit +# --localversion is passed through, and otherwise prepare-source.sh derives +# it from the checkout the way CI does. # Skipped when --skip-prepare is set (CI mode: prepare-source.sh already ran -# as a dedicated prior step). -if [[ "$SKIP_PREPARE" != true ]]; then - PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR") +# as a dedicated prior step), and when there is no tree to prepare. +if [[ -n "$DSC" ]]; then + : +elif [[ "$SKIP_PREPARE" != true ]]; then + PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR" + --flavour "$FLAVOUR") [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS+=(--localversion "$LOCALVERSION") [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS+=(--kver-extra "$KVER_EXTRA") [[ -n "$ENABLE_CONFIGS" ]] && PREPARE_ARGS+=(--kernel-config "$ENABLE_CONFIGS") @@ -279,12 +301,26 @@ else } fi -# ── Build ──────────────────────────────────────────────────────────────────── +# ── Source package ─────────────────────────────────────────────────────────── mkdir -p "$OUTPUT_DIR" +if [[ "$SOURCE_PACKAGE" == true ]]; then + log_step "Building the source package into $OUTPUT_DIR..." + "$SCRIPT_DIR/build-source-package.sh" --source-dir "$KERNEL_DIR" --output-dir "$OUTPUT_DIR" + echo + log_info "Build binaries from it with:" + log_info " $0 --dsc $OUTPUT_DIR/$(cd "$OUTPUT_DIR" && ls -- *.dsc | tail -1) --distro $DISTRO" + exit 0 +fi + +# ── Build ──────────────────────────────────────────────────────────────────── log_step "Building kernel package (mode: $BUILD_MODE)..." [[ -n "$PROFILES" ]] && log_info "Build profiles: $PROFILES" echo +# Every mode builds either the prepared tree or the source package. The +# source-package path is the one CI takes for every suite, so it is the one to +# use when a local build must produce what CI produced: same .dsc in, same +# binaries out. case "$BUILD_MODE" in docker) USE_SUDO="" @@ -294,26 +330,70 @@ case "$BUILD_MODE" in [[ $REPLY =~ ^[Yy]$ ]] || { log_error "Aborted."; exit 1; } USE_SUDO="sudo" } - BUILD_CMD=("$DOCKER_PKG_BUILD" - --skip-gbp - --no-update-check - --source-dir "$KERNEL_DIR" - --output-dir "$OUTPUT_DIR" - --distro "$DISTRO") - [[ -n "$PROFILES" ]] && BUILD_CMD+=(--profiles "$PROFILES") - ${USE_SUDO:+sudo} "${BUILD_CMD[@]}" + if [[ -n "$DSC" ]]; then + # docker_deb_build.py builds from a tree only, so the .dsc is + # handed to sbuild inside the same builder image directly, with + # the same sbuild flags docker_deb_build.py uses. The image is + # named as docker-pkg-build names it; when it is missing, + # docker_deb_build.py --rebuild builds it from its Dockerfile. + DOCKER_IMAGE="ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" + ${USE_SUDO:+sudo} docker image inspect "$DOCKER_IMAGE" >/dev/null 2>&1 || { + [[ -n "$DOCKER_PKG_BUILD" ]] || { + log_error "Builder image $DOCKER_IMAGE is not present, and docker_deb_build.py was not found to build it." + log_error "Pull the image, or point --docker-build at a docker-pkg-build checkout." + exit 1 + } + log_info "Builder image $DOCKER_IMAGE not present; building it with docker-pkg-build..." + ${USE_SUDO:+sudo} "$DOCKER_PKG_BUILD" --no-update-check --rebuild -d "$DISTRO" + } + # The .dsc's directory is mounted read-only so sbuild can read the + # tarballs it names; results go to the output mount. + SBUILD_CMD="sbuild --chroot-mode=unshare --build-dep-resolver=aptitude" + SBUILD_CMD+=" --no-clean-source --build-dir=/workspace/output" + SBUILD_CMD+=" --host=arm64 --build=arm64 --dist=$DISTRO --no-run-lintian" + [[ -n "$PROFILES" ]] && SBUILD_CMD+=" --profiles=$PROFILES" + SBUILD_CMD+=" /workspace/source/$(basename "$DSC")" + ${USE_SUDO:+sudo} docker run --rm --privileged \ + -v "$(dirname "$DSC"):/workspace/source:ro" \ + -v "$OUTPUT_DIR:/workspace/output:Z" \ + -w /workspace/output \ + "$DOCKER_IMAGE" bash -c "$SBUILD_CMD" + else + BUILD_CMD=("$DOCKER_PKG_BUILD" + --skip-gbp + --no-update-check + --source-dir "$KERNEL_DIR" + --output-dir "$OUTPUT_DIR" + --distro "$DISTRO") + [[ -n "$PROFILES" ]] && BUILD_CMD+=(--profiles "$PROFILES") + ${USE_SUDO:+sudo} "${BUILD_CMD[@]}" + fi ;; native) log_info "Running dpkg-buildpackage on host..." [[ -n "$PROFILES" ]] && export DEB_BUILD_PROFILES="$PROFILES" - dpkg-buildpackage -us -uc -b - find "$(dirname "$KERNEL_DIR")" -maxdepth 1 -name "*.deb" -exec mv -v {} "$OUTPUT_DIR/" \; + if [[ -n "$DSC" ]]; then + # Unpack into the output directory, so dpkg-buildpackage's ../ + # is the output directory and the .deb files land there. + BUILD_TREE="$OUTPUT_DIR/$(basename "$DSC" .dsc)" + rm -rf "$BUILD_TREE" + dpkg-source -x "$DSC" "$BUILD_TREE" + (cd "$BUILD_TREE" && dpkg-buildpackage -us -uc -b) + else + dpkg-buildpackage -us -uc -b + find "$(dirname "$KERNEL_DIR")" -maxdepth 1 -name "*.deb" -exec mv -v {} "$OUTPUT_DIR/" \; + fi ;; sbuild) log_info "Running sbuild for $DISTRO..." - SBUILD_CMD=(sbuild --dist "$DISTRO" --arch arm64 --no-source) + SBUILD_CMD=(sbuild --dist "$DISTRO" --arch arm64) [[ -n "$PROFILES" ]] && SBUILD_CMD+=(--profiles "$PROFILES") - "${SBUILD_CMD[@]}" + if [[ -n "$DSC" ]]; then + # sbuild takes a .dsc directly and writes beside its cwd. + (cd "$OUTPUT_DIR" && "${SBUILD_CMD[@]}" "$DSC") + else + "${SBUILD_CMD[@]}" --no-source + fi ;; esac diff --git a/build-source-package.sh b/build-source-package.sh new file mode 100755 index 00000000..12ba8a0a --- /dev/null +++ b/build-source-package.sh @@ -0,0 +1,236 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +set -euo pipefail + +# Build a Debian source package from a prepared kernel tree. +# +# Takes a kernel checkout that prepare-source.sh has already injected debian/ +# into, and produces in an output directory the .orig.tar.gz, .debian.tar.xz, +# .dsc and .changes that a binary build -- local, Debusine, or sbuild in a +# container -- starts from. +# +# The orig tarball is a function of the commit and nothing else. It is written +# with git archive, which sets every entry's mtime to the commit time and its +# owner to root, and compressed with gzip -n, which writes no timestamp. Two +# runs on one commit therefore produce one tarball, byte for byte, and two +# matrix entries that share a source package and upstream version -- trixie +# and forky, differing only in Debian revision -- share one orig, which is +# what an archive holding both requires. +# +# The upstream version names the commit (the ~g field), so the archived +# commit is checked against it: an orig named for one commit must hold that +# commit's tree. The tree is also checked against the commit before the +# package is built, because dpkg-source would otherwise fold any stray file +# into an automatic patch and the package would no longer describe the +# commit its version names. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $*"; } + +usage() { + cat </dev/null || { log_error "$tool not found; install dpkg-dev and git"; exit 1; } +done + +git -C "$SOURCE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { + log_error "$SOURCE_DIR is not a git checkout; the orig tarball is archived from a commit." + exit 1 +} + +# ── Read the package identity from the changelog ───────────────────────────── +# The changelog is what prepare-source.sh generated and what the .dsc will be +# built from, so it is the only source of the name and version here. The sed +# filters keep the values to what Debian policy allows in a package name and +# version, as debusine-action does, so they are safe in filenames. +PKG=$(dpkg-parsechangelog -l "$SOURCE_DIR/debian/changelog" -SSource | sed 's/[^a-z0-9.+-]//g') +VER=$(dpkg-parsechangelog -l "$SOURCE_DIR/debian/changelog" -SVersion | sed 's/[^A-Za-z0-9.+~:-]//g') +[[ -n "$PKG" && -n "$VER" ]] || { log_error "Could not read Source and Version from debian/changelog"; exit 1; } +UPSTREAM_VER="${VER%-*}" +[[ "$UPSTREAM_VER" != "$VER" ]] || { + log_error "Version $VER has no Debian revision; a 3.0 (quilt) package needs one." + exit 1 +} + +# ── Resolve the commit and check it against the version ────────────────────── +[[ -n "$COMMIT" ]] || COMMIT=HEAD +COMMIT_SHA=$(git -C "$SOURCE_DIR" rev-parse --verify "${COMMIT}^{commit}") || { + log_error "Not a commit in $SOURCE_DIR: $COMMIT" + exit 1 +} + +# debian/rules spells the commit into the upstream version as ~g<12 hex>. An +# orig named for that commit must hold that commit's tree, so the two are +# checked against each other rather than trusted to agree. A version without +# the field (an explicit --localversion with no snapshot) names no commit, in +# which case there is nothing to check and the tarball is only as +# reproducible as the caller's choice of commit. +if [[ "$UPSTREAM_VER" =~ ~g([0-9a-f]{12}) ]]; then + VERSION_SHA="${BASH_REMATCH[1]}" + [[ "$COMMIT_SHA" == "$VERSION_SHA"* ]] || { + log_error "Version $VER names commit $VERSION_SHA, but $COMMIT is $COMMIT_SHA." + log_error "Re-run prepare-source.sh on this checkout, or pass --commit $VERSION_SHA." + exit 1 + } +else + log_warn "Version $VER names no commit; the orig tarball is archived from $COMMIT_SHA." +fi + +# ── Check the tree against the commit ──────────────────────────────────────── +# dpkg-source compares the tree with the orig, so anything outside debian/ +# that differs from the commit -- an in-tree build's objects, an edit not +# committed -- would become an automatic patch or abort the build. Refuse it +# here with the list and the command that cleans it, rather than let +# dpkg-source report it one file at a time. +log_step "Checking $SOURCE_DIR matches $COMMIT_SHA outside debian/..." +if [[ "$COMMIT_SHA" != "$(git -C "$SOURCE_DIR" rev-parse HEAD)" ]]; then + log_error "HEAD of $SOURCE_DIR is not $COMMIT; check out the commit the package names." + exit 1 +fi +STRAY=$(git -C "$SOURCE_DIR" status --porcelain --ignored -- . ':(exclude)debian' | head -20) +[[ -z "$STRAY" ]] || { + log_error "$SOURCE_DIR differs from commit $COMMIT_SHA outside debian/:" + echo "$STRAY" + log_error "The source package must describe the commit its version names." + log_error "Clean the tree first, keeping the injected packaging:" + log_error " git -C $SOURCE_DIR checkout -- . && git -C $SOURCE_DIR clean -xdf -- . ':(exclude)debian'" + exit 1 +} + +# ── Write the orig tarball ─────────────────────────────────────────────────── +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)" +ORIG="$OUTPUT_DIR/${PKG}_${UPSTREAM_VER}.orig.tar.gz" +DSC="$OUTPUT_DIR/${PKG}_${VER}.dsc" +CHANGES="$OUTPUT_DIR/${PKG}_${VER}_source.changes" + +PREVIOUS_ORIG_SHA256="" +[[ -f "$ORIG" ]] && PREVIOUS_ORIG_SHA256=$(sha256sum "$ORIG" | cut -d' ' -f1) + +log_step "Writing $ORIG from commit $COMMIT_SHA..." +# --format=tar piped through gzip rather than --format=tar.gz: git's own +# gzip is also deterministic, but this spells out which compressor decides +# the bytes. The bytes depend on the gzip implementation, so every build +# that must agree on a checksum runs this in the same builder image. +git -C "$SOURCE_DIR" archive --format=tar --prefix="${PKG}-${UPSTREAM_VER}/" "$COMMIT_SHA" \ + | gzip -9n > "$ORIG.tmp" +mv "$ORIG.tmp" "$ORIG" +ORIG_SHA256=$(sha256sum "$ORIG" | cut -d' ' -f1) +log_info " sha256: $ORIG_SHA256" + +if [[ -n "$PREVIOUS_ORIG_SHA256" && "$PREVIOUS_ORIG_SHA256" != "$ORIG_SHA256" ]]; then + # The file that was there carried the same name and so claimed the same + # commit. It cannot have been the same tarball, so something about how + # it was made differs from this run -- which is what this script exists + # to rule out. + log_error "An orig tarball for $PKG $UPSTREAM_VER already existed with sha256 $PREVIOUS_ORIG_SHA256" + log_error "and rebuilding it from $COMMIT_SHA gave $ORIG_SHA256." + log_error "Two files with this name must be one file; find what changed between the two builds." + exit 1 +fi + +# ── Build the .dsc and .changes ────────────────────────────────────────────── +# dpkg-source -b writes into the current directory and finds the orig there, +# so it runs from the output directory. dpkg-genchanges reads debian/ from +# the source tree and is told where the files are and where to write. +log_step "Building the source package..." +rm -f "$DSC" "$OUTPUT_DIR/${PKG}_${VER}.debian.tar."* "$CHANGES" +(cd "$OUTPUT_DIR" && dpkg-source -b "$SOURCE_DIR") +(cd "$SOURCE_DIR" && dpkg-genchanges -S -sa -u"$OUTPUT_DIR" -O"$CHANGES" >/dev/null) + +[[ -f "$DSC" && -f "$CHANGES" ]] || { + log_error "Expected $DSC and $CHANGES to exist after the build" + exit 1 +} + +echo +log_step "Source package complete." +log_info " $CHANGES" +grep -A100 '^Checksums-Sha256:' "$CHANGES" | sed -n '2,/^[A-Z]/{/^ /p}' | sed 's/^/ /' + +if [[ -n "$FIELDS_FILE" ]]; then + cat > "$FIELDS_FILE" <-dkms packages the +# target archive carries: the Debian suites bundle camx and iris-vpu, which +# Ubuntu does not package. +# +# A flavour tracking a moving upstream is Daily only. A Release entry must name +# an immutable ref, and a branch tip or a tag cut every night is not one, so +# mainline, next and the topic branches are built and published daily and never +# promoted. A Release entry still requires the Daily that tests it. +# +# debian_revision is stated outright, not derived. Daily revisions end in "~" +# so they sort below the Release revision for the same suite; the rest of the +# revision places the suite where it belongs relative to the other suites (see +# the ordering discussion in README.md). No two entries may produce the same +# srcpkg at the same debian_revision. + +builds: + # --------------------------------------------------------------------------- + # qcom-next — the standard kernel. + # --------------------------------------------------------------------------- + - name: qcom-next-trixie + type: Daily + suite: trixie + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1~' + + - name: qcom-next-forky + type: Daily + suite: forky + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~' + + - name: qcom-next-resolute + type: Daily + suite: resolute + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. + dkms: + - kgsl + debian_revision: '0qli1~26.04.1~' + + - name: qcom-next-trixie + type: Release + suite: trixie + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli + + - name: qcom-next-forky + type: Release + suite: forky + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1' + target_workspace: qli + + # --------------------------------------------------------------------------- + # qcom-next-debug — the same kernel ref built with the in-tree debug + # fragments, packaged separately so it installs alongside qcom-next. + # --------------------------------------------------------------------------- + - name: qcom-next-debug-trixie + type: Daily + suite: trixie + flavour: qcom-next-debug + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1~' + + - name: qcom-next-debug-forky + type: Daily + suite: forky + flavour: qcom-next-debug + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~' + + - name: qcom-next-debug-trixie + type: Release + suite: trixie + flavour: qcom-next-debug + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli + + - name: qcom-next-debug-forky + type: Release + suite: forky + flavour: qcom-next-debug + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1' + target_workspace: qli + + # --------------------------------------------------------------------------- + # qcom-arduino — the Arduino hardware-enablement topic branch. + # + # Daily only: a topic branch has no immutable ref to promote from. + # --------------------------------------------------------------------------- + - name: qcom-arduino-trixie + type: Daily + suite: trixie + flavour: qcom-arduino + git_clone: https://github.com/qualcomm-linux/kernel-topics + branch_or_tag: early/hwe/arduino + ref_strategy: branch_tip + srcpkg: linux-qcom-arduino + binpkg: linux-image-qcom-arduino + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + debian_revision: '0qli~bpo13+1~' + + - name: qcom-arduino-forky + type: Daily + suite: forky + flavour: qcom-arduino + git_clone: https://github.com/qualcomm-linux/kernel-topics + branch_or_tag: early/hwe/arduino + ref_strategy: branch_tip + srcpkg: linux-qcom-arduino + binpkg: linux-image-qcom-arduino + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + debian_revision: '0qli~' + + # --------------------------------------------------------------------------- + # mainline — Linus's tree, tracked for early warning of upstream breakage. + # + # Daily only, and no DKMS: the out-of-tree modules target the Qualcomm tree, + # and a listed module is a presence contract that would fail the build here. + # --------------------------------------------------------------------------- + - name: mainline-trixie + type: Daily + suite: trixie + flavour: mainline + git_clone: https://github.com/torvalds/linux + branch_or_tag: master + ref_strategy: branch_tip + srcpkg: linux-mainline + binpkg: linux-image-mainline + kernel_config: [] + dkms: [] + debian_revision: '0qli~bpo13+1~' + + - name: mainline-forky + type: Daily + suite: forky + flavour: mainline + git_clone: https://github.com/torvalds/linux + branch_or_tag: master + ref_strategy: branch_tip + srcpkg: linux-mainline + binpkg: linux-image-mainline + kernel_config: [] + dkms: [] + debian_revision: '0qli~' + + # --------------------------------------------------------------------------- + # next — linux-next, the integration tree, one step ahead of mainline. + # + # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of + # them the same way qcom-next-* does. Daily only, and no DKMS, as above. + # --------------------------------------------------------------------------- + - name: next-trixie + type: Daily + suite: trixie + flavour: next + git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + branch_or_tag: master + ref_strategy: latest_tag + tag_pattern: 'next-*' + srcpkg: linux-next + binpkg: linux-image-next + kernel_config: [] + dkms: [] + debian_revision: '0qli~bpo13+1~' + + - name: next-forky + type: Daily + suite: forky + flavour: next + git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + branch_or_tag: master + ref_strategy: latest_tag + tag_pattern: 'next-*' + srcpkg: linux-next + binpkg: linux-image-next + kernel_config: [] + dkms: [] + debian_revision: '0qli~' diff --git a/ci/scripts/derive-debian-revision.sh b/ci/scripts/derive-debian-revision.sh deleted file mode 100755 index 008e993e..00000000 --- a/ci/scripts/derive-debian-revision.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause-Clear -set -euo pipefail - -# Derive the suite-specific Debian revision for one delivery leg. -# -# Formula: -# debian_revision = stub + suite_suffix_mapping[suite] + delivery_suffix -# delivery_suffix: Daily -> "~", Release -> "" -# -# This is the single implementation of the formula. It is called both by -# resolve-matrix.sh (once per flattened Daily/Release leg) and by -# build-kernel-deb.yml's direct-dispatch path (one suite, no full matrix -# context), so the derivation and its validation live in exactly one place. -# -# Usage: -# ci/scripts/derive-debian-revision.sh --stub 0qli1 --suite trixie --delivery-type Daily -# ci/scripts/derive-debian-revision.sh --stub 0qli1 --suite forky --delivery-type Release --matrix-file ci/build-matrix.json -# -# Options: -# --stub STUB Debian version stub, e.g. 0qli1. Must end in a digit: -# that digit is the packaging revision, bumped for a -# rebuild of an unchanged kernel snapshot. Must not -# end in ~ (the delivery suffix supplies any -# trailing ~). Required. -# --suite SUITE Target suite; must have an entry in -# suite_suffix_mapping. Required. -# --delivery-type TYPE Daily or Release. Required. -# --matrix-file FILE Path to the matrix JSON containing -# suite_suffix_mapping -# (default: ci/build-matrix.json relative to CWD). -# -# Output: -# Final Debian revision printed to stdout. -# -# Exit codes: -# 0 Success. -# 1 Error (invalid args, malformed or missing suite_suffix_mapping, -# unmapped suite, unsupported delivery type). - -STUB="" -SUITE="" -DELIVERY_TYPE="" -MATRIX_FILE="ci/build-matrix.json" - -usage() { - sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' - exit 1 -} - -while [[ $# -gt 0 ]]; do - case $1 in - --stub) STUB="$2"; shift 2 ;; - --suite) SUITE="$2"; shift 2 ;; - --delivery-type) DELIVERY_TYPE="$2"; shift 2 ;; - --matrix-file) MATRIX_FILE="$2"; shift 2 ;; - -h|--help) usage ;; - *) echo "ERROR: Unknown option: $1" >&2; usage ;; - esac -done - -[[ -n "$STUB" ]] || { echo "ERROR: --stub is required" >&2; exit 1; } -[[ -n "$SUITE" ]] || { echo "ERROR: --suite is required" >&2; exit 1; } -[[ -n "$DELIVERY_TYPE" ]] || { echo "ERROR: --delivery-type is required" >&2; exit 1; } -[[ "$STUB" != *"~" ]] || { echo "ERROR: --stub must not end in ~ (got '$STUB')" >&2; exit 1; } -# The trailing digit is the packaging revision: it is the only field left to -# bump when the kernel snapshot is unchanged but the packaging is rebuilt. -# suite_suffix_mapping is a per-suite constant and cannot carry it, and the -# delivery suffix is the Daily/Release marker, so a stub without a digit leaves -# a rebuild with nowhere to go. -[[ "$STUB" =~ [0-9]$ ]] || { - echo "ERROR: --stub must end in a digit, the packaging revision (got '$STUB'; use '${STUB}1')" >&2 - exit 1 -} -[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } - -jq empty "$MATRIX_FILE" 2>/dev/null \ - || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } - -mapping_errors=$(jq -r ' - .suite_suffix_mapping as $mapping | - if ($mapping | type) != "object" - then "suite_suffix_mapping is missing or not an object" - else - ( - [$mapping | to_entries[] | select(.value | type != "string") | "suite_suffix_mapping[" + .key + "] must be a string"] - + - [ - $mapping - | to_entries[] - | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) - | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" - ] - + - [ - $mapping - | to_entries - | group_by(.value) - | map(select(length > 1)) - | .[]? - | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" - ] - ) | .[] - end -' "$MATRIX_FILE") - -if [[ -n "$mapping_errors" ]]; then - echo "ERROR: Invalid suite_suffix_mapping in $MATRIX_FILE:" >&2 - while IFS= read -r error; do - [[ -n "$error" ]] && echo " - $error" >&2 - done <<< "$mapping_errors" - exit 1 -fi - -SUFFIX=$(jq -r --arg suite "$SUITE" '.suite_suffix_mapping[$suite] // "__MISSING__"' "$MATRIX_FILE") -[[ "$SUFFIX" != "__MISSING__" ]] || { - echo "ERROR: no suite_suffix_mapping entry for suite '$SUITE'" >&2 - exit 1 -} - -case "$DELIVERY_TYPE" in - Daily) DELIVERY_SUFFIX="~" ;; - Release) DELIVERY_SUFFIX="" ;; - *) - echo "ERROR: --delivery-type must be Daily or Release (got '$DELIVERY_TYPE')" >&2 - exit 1 - ;; -esac - -echo "${STUB}${SUFFIX}${DELIVERY_SUFFIX}" diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index eca3c989..0517e10e 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -3,23 +3,29 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear set -euo pipefail -# Derive the version fields for a build from a kernel variant and resolved ref. +# Derive the version fields for a build from a kernel flavour and resolved ref. +# +# The flavour is the kernel's own identity, the part of the kernel release +# that distinguishes two kernels built from the same ref with different +# configuration, so that their linux-image packages install alongside each +# other. It is not the CI identifier for the build: a flavour is built for +# several suites, and all of those builds produce the same kernel release. # # Emits LOCALVERSION (the kernel release suffix), SNAPSHOT (the dated component # of the Debian version) and GITSHA, all derived from the ref in one place. # SNAPSHOT and GITSHA are emitted alongside rather than recovered from # LOCALVERSION later: reading them back out means guessing where each field ends -# in a string that also carries a variant name, and a hex SHA can end in eight +# in a string that also carries a flavour name, and a hex SHA can end in eight # digits of its own. # # For dated tag builds (ref ends in -YYYYMMDD, optionally .): -# Produces +-[.]-g<12 hex>. +# Produces +-[.]-g<12 hex>. # Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g07f50dc44edd # qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g07f50dc44edd # # The respin ordinal distinguishes a second tag cut on the same day. It is # carried verbatim rather than normalised, so the first tag of a day stays -# plain +-: systemd compares the separator before the chunk +# plain +-: systemd compares the separator before the chunk # behind it, so an absent ordinal already sorts below a present one and no # build has to spell a ".0". # @@ -61,11 +67,11 @@ set -euo pipefail # which spells the release candidate ~rcN and orders correctly either way. # # Usage: -# ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd -# ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd --date 20260904 +# ci/scripts/derive-localversion.sh --flavour qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd +# ci/scripts/derive-localversion.sh --flavour arduino --ref main --sha 07f50dc44edd --date 20260904 # # Options: -# --variant VARIANT Kernel variant identifier. Defaults to qcom-next. +# --flavour FLAVOUR Kernel flavour. Defaults to qcom-next. # --ref REF Kernel ref (tag name or branch name). Required. # --sha SHA Commit SHA, truncated to 12 hex characters. Required. # --date DATE HEAD commit date as YYYYMMDD or YYYYMMDD.N. Required for @@ -86,7 +92,7 @@ set -euo pipefail # 0 Success. # 1 Error (invalid args, malformed --sha, branch tip without --date). -VARIANT="qcom-next" +FLAVOUR="qcom-next" REF="" SHA="" DATE="" @@ -98,7 +104,7 @@ usage() { while [[ $# -gt 0 ]]; do case $1 in - --variant) VARIANT="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --ref) REF="$2"; shift 2 ;; --sha) SHA="$2"; shift 2 ;; --date) DATE="$2"; shift 2 ;; @@ -108,8 +114,8 @@ while [[ $# -gt 0 ]]; do done [[ -n "$REF" ]] || { echo "ERROR: --ref is required" >&2; exit 1; } -[[ "$VARIANT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: --variant must use lowercase letters, digits, and internal hyphens" >&2 +[[ "$FLAVOUR" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "ERROR: --flavour must use lowercase letters, digits, and internal hyphens" >&2 exit 1 } # Every build identifies its commit, so --sha is required for all of them, not @@ -123,11 +129,11 @@ done GITSHA="${SHA:0:12}" # Dated tags use a trailing YYYYMMDD snapshot, optionally followed by a respin -# ordinal. The matrix selects the tag set; the variant supplies the stable -# package identity used in LOCALVERSION. +# ordinal. The matrix selects the tag set; the flavour supplies the stable +# kernel identity used in LOCALVERSION. if [[ "$REF" =~ -([0-9]{8}(\.[0-9]+)?)$ ]]; then SNAPSHOT="${BASH_REMATCH[1]}" - LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" + LOCALVERSION="+${FLAVOUR}-${SNAPSHOT}-g${GITSHA}" else # Branch-tip build: the ref carries no date, so the commit date supplies # one. Without it these builds had no snapshot at all, which put their @@ -141,7 +147,7 @@ else exit 1 } SNAPSHOT="$DATE" - LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" + LOCALVERSION="+${FLAVOUR}-${SNAPSHOT}-g${GITSHA}" fi echo "LOCALVERSION=${LOCALVERSION}" diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py new file mode 100755 index 00000000..edadb6a9 --- /dev/null +++ b/ci/scripts/resolve-matrix.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +"""Validate and select entries from the kernel delivery matrix. + +ci/build-matrix.yaml holds one entry per generated package: a single named +build, one delivery type, and one suite. Nothing here +expands or derives anything -- the matrix is already flat, and each entry +states its own debian_revision. This script validates the whole document, +selects the entries a caller asked for, and prints them. + +The document is validated in full on every invocation, not just the selected +entries, so a typo in a Release entry fails the daily build too rather than +lying in wait until someone runs a release. + +Usage: + ci/scripts/resolve-matrix.py --type Daily + ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie + ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie,qcom-next-forky + ci/scripts/resolve-matrix.py --type Release --flavour qcom-next + ci/scripts/resolve-matrix.py --type Daily --family ubuntu --allow-empty + ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie \\ + --field debian_revision + +Options: + --type TYPE Delivery type to select (Daily or Release). + Required. + --build NAMES Select only these builds by name, comma-separated. + A name identifies one build, so this is the way + to ask for a specific set of them. + --flavour FLAVOURS Select only these flavours, comma-separated. A + flavour spans its suites, so this asks for one + kernel everywhere it is built. + --suite SUITES Select only these suites, comma-separated. + --family FAMILY Select only the builds taking one build path, + debian or ubuntu. Derived from suite, so it + selects by how a build is built rather than by + naming every suite that is built that way. + --allow-empty Print [] rather than failing when the filters + select nothing. For a caller asking each family + for the same selection, where one of them + having nothing to build is an ordinary answer. + --field NAME Print just this field of the single selected + entry, unquoted. Errors unless exactly one + entry matches. + --matrix-file FILE Matrix path (default: ci/build-matrix.yaml + relative to CWD). + + Filters combine: an entry must match every filter given. Every name in a + filter must match at least one entry of the selected type, so a typo or a + stale name fails instead of quietly narrowing the build set. That check is + per filter, so --allow-empty still rejects a name that matches nothing. + +Output: + Without --field, a compact JSON array of the selected entries, ready for a + GitHub Actions matrix `include`. Each entry carries a derived family field + naming its build path. kernel_config and dkms are joined into the + comma-separated strings that the build workflows' kernel-config and dkms + inputs, and prepare-source.sh's --kernel-config and --dkms, expect; every + other field is passed through as written. + + With --field, the named field's value alone, so a workflow step can capture + it directly. + +Exit codes: + 0 Success, at least one entry selected. + 1 Error (invalid arguments, matrix validation failure, no matching entry, + or --field matching more than one entry). +""" + +import argparse +import json +import re +import sys +from collections import defaultdict + +try: + import yaml +except ImportError: + sys.exit( + "ERROR: PyYAML is required to read the delivery matrix.\n" + " Install it with 'apt-get install python3-yaml' or 'pip install pyyaml'." + ) + +DEFAULT_MATRIX_FILE = "ci/build-matrix.yaml" + +DELIVERY_TYPES = ("Daily", "Release") +REF_STRATEGIES = ("latest_tag", "branch_tip", "pinned_ref") + +# Which build path a suite takes. Debian-family suites are built by Debusine; +# everything else is built in a suite-matched docker container on the +# self-hosted runner. The two paths are different workflows with different +# runners, containers and publish steps, so a caller selects entries by family +# and calls the one workflow that builds them -- rather than every build leg +# starting both and skipping one. +DEBIAN_SUITES = ("trixie", "forky", "sid", "unstable") +FAMILIES = ("debian", "ubuntu") + + +def family_for(suite): + """Return the build family a suite belongs to.""" + return "debian" if suite in DEBIAN_SUITES else "ubuntu" + +# A delivery type constrains how its kernel ref is chosen: a Daily build tracks +# something moving, a Release build is pinned to an immutable ref. +REF_STRATEGIES_FOR_TYPE = { + "Daily": ("latest_tag", "branch_tip"), + "Release": ("pinned_ref",), +} + +REQUIRED_STRING_FIELDS = ( + "name", + "type", + "suite", + "flavour", + "git_clone", + "branch_or_tag", + "ref_strategy", + "srcpkg", + "binpkg", + "debian_revision", +) + +OPTIONAL_STRING_FIELDS = ( + "tag_pattern", + "target_workspace", + "localversion", + "kver_extra", + "debusine_parent_workspace", +) + +KNOWN_FIELDS = frozenset( + REQUIRED_STRING_FIELDS + OPTIONAL_STRING_FIELDS + ("kernel_config", "dkms") +) + +# Everything about what is built is anchored on flavour, not on the build's +# name. The flavour is the kernel's own identity: it becomes the LOCALVERSION +# suffix, so two flavours built from one ref produce distinct kernel releases +# whose linux-image packages install alongside each other. A build's name only +# labels it in CI -- the Actions job, and what a dispatch asks for -- and +# carries no packaging meaning. + +# Fields that identify the flavour itself rather than one of its build legs. +# Every entry for a flavour must agree on them, because they decide what the +# package is; the entries only differ in where it is delivered. +FLAVOUR_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config") + +# Fields a flavour may vary between suites but not within one. The out-of-tree +# module set depends on which -dkms packages the target archive has, so +# it is a property of the flavour in a suite rather than of the flavour. A +# suite's Daily and Release must still agree: a Daily that bundles a different +# module set from the Release it precedes is not testing what will ship. +SUITE_IDENTITY_FIELDS = ("dkms",) + +# Fields deciding which kernel tree is built. All entries for one flavour and +# delivery type build the same source, so a stale suite cannot quietly ship a +# different kernel from its siblings. +REF_FIELDS = ("git_clone", "branch_or_tag", "ref_strategy", "tag_pattern") + +NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") + +# A Debian revision: no hyphen (that would start a new revision component) and +# none of the characters dpkg rejects in a version. +REVISION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+~]*$") + +# An "intree:" entry names a fragment shipped by the kernel source, as a path +# relative to the kernel source root. A bare entry names a fragment in +# debian/config-available/, with or without its .config extension. +INTREE_PATH_RE = re.compile(r"^([A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\.config$") +BARE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + +# A dkms entry is a package name stem: the build wants -dkms available, +# and generates both the Build-Depends entry and the debian/dkms-modules +# manifest from it. Same shape debian/rules enforces at prepare time, checked +# here so a typo fails before a runner is claimed rather than mid-build. +DKMS_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9+.-]*$") + + +def fragment_basename(fragment): + """Filename a fragment lands under in debian/config/, minus .config. + + Every fragment is copied into debian/config/ under its basename, so + arch/arm64/configs/hardening.config and kernel/configs/hardening.config + collide there even though the entries differ. + """ + return fragment.removeprefix("intree:").rsplit("/", 1)[-1].removesuffix(".config") + + +def check_kernel_config(entry, report): + """Validate one entry's kernel_config list.""" + fragments = entry.get("kernel_config") + if not isinstance(fragments, list): + report("kernel_config must be a list") + return + if any(not isinstance(f, str) or not f for f in fragments): + report("kernel_config must contain only non-empty strings") + return + if any("," in f for f in fragments): + report( + "kernel_config entries must not contain commas; " + "use one list element per fragment" + ) + return + if len(set(fragments)) != len(fragments): + report("kernel_config must not contain duplicates") + + for fragment in fragments: + if fragment.startswith("intree:"): + path = fragment.removeprefix("intree:") + traversal = path == ".." or path.startswith("../") or "/../" in path + if not INTREE_PATH_RE.match(path) or traversal or path.endswith("/.."): + report( + f"intree: entry '{fragment}' must be a kernel-source-relative " + "path ending in .config " + "(e.g. intree:arch/arm64/configs/qcom_debug.config)" + ) + elif not BARE_FRAGMENT_RE.match(fragment): + report( + f"kernel_config entry '{fragment}' must be a fragment name from " + "debian/config-available/ or an intree: path" + ) + + basenames = [fragment_basename(f) for f in fragments] + if len(set(basenames)) != len(basenames): + report("kernel_config entries must not resolve to the same fragment filename") + + +def check_dkms(entry, report): + """Validate one entry's dkms list. + + An empty list is meaningful: it bundles nothing, as opposed to leaving the + field out, which the matrix does not allow. + """ + modules = entry.get("dkms") + if not isinstance(modules, list): + report("dkms must be a list") + return + if any(not isinstance(m, str) or not m for m in modules): + report("dkms must contain only non-empty strings") + return + if len(set(modules)) != len(modules): + report("dkms must not contain duplicates") + + for module in modules: + if module.endswith("-dkms"): + report( + f"dkms entry '{module}' must omit the -dkms suffix " + f"(use '{module.removesuffix('-dkms')}')" + ) + elif not DKMS_NAME_RE.match(module): + report(f"dkms entry '{module}' must be a package name stem, e.g. kgsl") + + +def check_entry(entry, report): + """Validate one delivery entry in isolation.""" + for field in REQUIRED_STRING_FIELDS: + value = entry.get(field) + if not isinstance(value, str) or not value: + report(f"missing or invalid {field}") + + for field in OPTIONAL_STRING_FIELDS: + if field in entry and not isinstance(entry[field], str): + report(f"invalid {field}") + + for field in sorted(set(entry) - KNOWN_FIELDS): + report(f"unknown field {field}") + + for field in ("name", "suite", "flavour"): + value = entry.get(field) + if isinstance(value, str) and not NAME_RE.match(value): + report(f"{field} must use lowercase letters, digits, and internal hyphens") + + check_kernel_config(entry, report) + check_dkms(entry, report) + + delivery_type = entry.get("type") + if delivery_type not in DELIVERY_TYPES: + report("type must be Daily or Release") + + ref_strategy = entry.get("ref_strategy") + if ref_strategy not in REF_STRATEGIES: + report("ref_strategy must be " + ", ".join(REF_STRATEGIES)) + elif delivery_type in REF_STRATEGIES_FOR_TYPE: + allowed = REF_STRATEGIES_FOR_TYPE[delivery_type] + if ref_strategy not in allowed: + report( + f"{delivery_type} entries must use " + + " or ".join(f"ref_strategy={s}" for s in allowed) + ) + + if ref_strategy == "latest_tag": + if not entry.get("tag_pattern"): + report("missing or invalid tag_pattern") + elif "tag_pattern" in entry: + report("tag_pattern is only valid with ref_strategy=latest_tag") + + if delivery_type == "Release": + if not entry.get("target_workspace"): + report("missing or invalid target_workspace") + # Promotion runs through Debusine, and only the Debian family is built + # there. An Ubuntu Release entry would build the package and then have + # no workspace to promote it from, publishing to the daily S3 path and + # reporting success without ever releasing anything. + suite = entry.get("suite") + if isinstance(suite, str) and family_for(suite) != "debian": + report( + f"Release entries must target a Debian suite, not {suite}; " + "promotion runs through Debusine, which builds " + + ", ".join(DEBIAN_SUITES) + ) + elif "target_workspace" in entry: + report("target_workspace is only valid for Release") + + revision = entry.get("debian_revision") + if isinstance(revision, str) and revision: + if not REVISION_RE.match(revision): + report( + f'debian_revision "{revision}" is not a valid Debian revision ' + "(letters, digits, and . + ~ only, starting with a letter or digit)" + ) + # A trailing ~ sorts a version below the same version without it, so a + # Daily always sorts below the Release it will be superseded by. + elif delivery_type == "Daily" and not revision.endswith("~"): + report(f'debian_revision "{revision}" must end in ~ for a Daily entry') + elif delivery_type == "Release" and revision.endswith("~"): + report(f'debian_revision "{revision}" must not end in ~ for a Release entry') + + +def describe(entry, index): + """Label an entry in an error message by what identifies it to a reader.""" + if not isinstance(entry, dict): + return f"entry {index}" + parts = [ + str(entry[field]) + for field in ("name", "type", "suite") + if isinstance(entry.get(field), str) + ] + return f"entry {index} ({'/'.join(parts)})" if parts else f"entry {index}" + + +def check_consistency(builds, errors): + """Validate the invariants that span entries. + + Entries are written out in full, so the matrix can state a flavour twice + and disagree with itself. These checks are what makes that duplication + safe to read at face value. + + Everything about the package is grouped by flavour. A build's name is + only checked for the one thing it is used for: identifying that build + uniquely. + """ + by_leg = defaultdict(list) + by_flavour = defaultdict(list) + by_flavour_type = defaultdict(list) + by_flavour_suite = defaultdict(list) + by_package_version = defaultdict(list) + flavours_by_package = defaultdict(set) + + for entry in builds: + if not isinstance(entry, dict): + continue + flavour = entry.get("flavour") + delivery_type = entry.get("type") + suite = entry.get("suite") + if not isinstance(flavour, str): + continue + + by_flavour[flavour].append(entry) + by_flavour_type[(flavour, delivery_type)].append(entry) + by_flavour_suite[(flavour, suite)].append(entry) + by_leg[(delivery_type, entry.get("name"))].append(entry) + + for field in ("srcpkg", "binpkg"): + if isinstance(entry.get(field), str) and entry[field]: + flavours_by_package[(field, entry[field])].add(flavour) + + if isinstance(entry.get("srcpkg"), str) and isinstance( + entry.get("debian_revision"), str + ): + by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) + + # A name identifies one build: it is the Actions job name and what a + # workflow dispatch asks for. It only has to be unique within a delivery + # type, because a run only ever resolves one type, and that lets a Daily and + # its Release share a name. Two entries sharing both would give a run two + # identically named jobs and make the dispatch filter ambiguous. + for (delivery_type, name), entries in sorted( + by_leg.items(), key=lambda item: str(item[0]) + ): + if len(entries) > 1: + suites = ", ".join(sorted(str(e.get("suite")) for e in entries)) + errors.append( + f"name {name} is used by {len(entries)} {delivery_type} entries " + f"(suites {suites}); a name identifies exactly one build" + ) + + for flavour, entries in sorted(by_flavour.items()): + for field in FLAVOUR_IDENTITY_FIELDS: + values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} + if len(values) > 1: + errors.append( + f"flavour {flavour} must use one {field} across all its " + "entries (got " + ", ".join(sorted(values)) + ")" + ) + # A Release must be preceded by the Daily that tests it, but the + # converse does not hold: a flavour tracking a moving upstream has + # nothing immutable to pin, so it is built daily and never promoted. + types = {entry.get("type") for entry in entries} + if "Release" in types and "Daily" not in types: + errors.append( + f"flavour {flavour} has a Release entry but no Daily entry; " + "a release must be preceded by the daily build that tests it" + ) + + for (flavour, suite), entries in sorted( + by_flavour_suite.items(), key=lambda item: str(item[0]) + ): + for field in SUITE_IDENTITY_FIELDS: + values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} + if len(values) > 1: + errors.append( + f"flavour {flavour} must use one {field} across its " + f"{suite} entries (got " + ", ".join(sorted(values)) + ")" + ) + + for (flavour, delivery_type), entries in sorted( + by_flavour_type.items(), key=lambda item: str(item[0]) + ): + for field in REF_FIELDS: + values = {entry.get(field) for entry in entries} + if len(values) > 1: + rendered = ", ".join(sorted(str(v) for v in values)) + errors.append( + f"flavour {flavour} must build one {field} across its " + f"{delivery_type} entries (got {rendered})" + ) + + # Two flavours sharing a package name would overwrite each other in the + # archive; the whole point of a second flavour is a second package. + for (field, package), flavours in sorted(flavours_by_package.items()): + if len(flavours) > 1: + errors.append( + f"{field} {package} is shared by flavours " + ", ".join(sorted(flavours)) + ) + + for (srcpkg, revision), entries in sorted( + by_package_version.items(), key=lambda item: str(item[0]) + ): + if len(entries) > 1: + suites = ", ".join(sorted(str(entry.get("suite")) for entry in entries)) + errors.append( + f"srcpkg {srcpkg} is built at debian_revision {revision} for " + f"suites {suites}; each entry needs a revision of its own" + ) + + # A suite's Daily and Release differ only by the Daily's trailing ~, so the + # Daily reliably sorts below the Release that supersedes it. + for flavour, entries in sorted(by_flavour.items()): + revisions = { + (entry.get("type"), entry.get("suite")): entry.get("debian_revision") + for entry in entries + } + for (delivery_type, suite), revision in sorted( + revisions.items(), key=lambda item: str(item[0]) + ): + if delivery_type != "Daily" or not isinstance(revision, str): + continue + release = revisions.get(("Release", suite)) + if isinstance(release, str) and revision != release + "~": + errors.append( + f"flavour {flavour} suite {suite}: Daily " + f'debian_revision "{revision}" must be the Release revision ' + f'"{release}" with a trailing ~' + ) + + +def validate(builds): + """Return every problem found in the matrix, as a list of messages.""" + errors = [] + for index, entry in enumerate(builds): + if not isinstance(entry, dict): + errors.append(f"{describe(entry, index)}: delivery entries must be mappings") + continue + label = describe(entry, index) + check_entry(entry, lambda message, label=label: errors.append(f"{label}: {message}")) + check_consistency(builds, errors) + return errors + + +def load_matrix(path): + """Read, parse, and validate the matrix, returning its builds.""" + try: + with open(path, encoding="utf-8") as handle: + document = yaml.safe_load(handle) + except FileNotFoundError: + sys.exit(f"ERROR: Matrix file not found: {path}") + except OSError as error: + sys.exit(f"ERROR: Cannot read {path}: {error}") + except yaml.YAMLError as error: + sys.exit(f"ERROR: Invalid YAML in {path}: {error}") + + if not isinstance(document, dict): + sys.exit(f"ERROR: {path}: matrix root must be a mapping with a builds key") + + # builds is the whole schema. Anything else at the root is a leftover from + # an older matrix (suite_suffix_mapping, say) that would otherwise sit + # there looking authoritative while nothing read it. + unknown_root = sorted(set(document) - {"builds"}) + if unknown_root: + sys.exit( + f"ERROR: {path}: unknown top-level key(s) {', '.join(unknown_root)}; " + "builds is the only one" + ) + + builds = document.get("builds") + if not isinstance(builds, list): + sys.exit(f"ERROR: {path}: builds must be a list") + if not builds: + sys.exit(f"ERROR: {path}: builds must contain at least one entry") + + errors = validate(builds) + if errors: + sys.exit( + f"ERROR: Invalid kernel delivery matrix in {path}:\n" + + "\n".join(f" - {error}" for error in errors) + ) + + # Derived, never written: family follows from suite, so the matrix cannot + # state one that disagrees with the suite it is built for. Attached after + # validation, which rejects family as an unknown field on an entry. + for entry in builds: + entry["family"] = family_for(entry["suite"]) + + return builds + + +def parse_filter(value): + """Split a comma-separated filter into names, or None when unset.""" + if not value: + return None + return [name.strip() for name in value.split(",") if name.strip()] + + +def select(builds, delivery_type, filters): + """Return the entries matching the requested filters, in matrix order. + + filters maps a field name to the list of values allowed for it, or to + None when that filter was not given. + """ + return [ + entry + for entry in builds + if entry["type"] == delivery_type + and all( + wanted is None or entry[field] in wanted + for field, wanted in filters.items() + ) + ] + + +def unmatched_filters(builds, delivery_type, filters): + """Names asked for that no entry of this delivery type offers. + + family is exempt: it routes a selection to a build path rather than naming + something in the matrix, argparse already restricts it to a real family, + and a type having no builds on one path is an ordinary state of the matrix + rather than a mistake in the request. + """ + missing = [] + for field, wanted in filters.items(): + if wanted is None or field == "family": + continue + available = { + entry[field] for entry in builds if entry["type"] == delivery_type + } + for name in wanted: + if name not in available: + missing.append( + f"no {delivery_type} entry has {field} {name} " + f"(available: {', '.join(sorted(available))})" + ) + return missing + + +def describe_selection(delivery_type, filters): + """Render the active filters for an error message.""" + parts = [f"type={delivery_type}"] + parts += [ + f"{field}={','.join(wanted)}" + for field, wanted in filters.items() + if wanted is not None + ] + return " ".join(parts) + + +def for_workflow(entry): + """Shape one entry the way build-kernel-deb.yml's inputs expect it.""" + return { + **entry, + "kernel_config": ",".join(entry["kernel_config"]), + "dkms": ",".join(entry["dkms"]), + } + + +def main(): + parser = argparse.ArgumentParser( + description="Validate and select entries from the kernel delivery matrix.", + epilog="See the module docstring in this file for full documentation.", + ) + parser.add_argument( + "--type", required=True, choices=DELIVERY_TYPES, help="delivery type to select" + ) + parser.add_argument( + "--build", + default="", + help="select only these builds by name, comma-separated", + ) + parser.add_argument( + "--flavour", default="", help="select only these flavours, comma-separated" + ) + parser.add_argument( + "--suite", default="", help="select only these suites, comma-separated" + ) + parser.add_argument( + "--family", + default="", + choices=("",) + FAMILIES, + help="select only the builds taking this build path", + ) + parser.add_argument( + "--allow-empty", + action="store_true", + help="print [] instead of failing when the filters select nothing", + ) + parser.add_argument( + "--field", + default="", + help="print just this field of the single selected entry", + ) + parser.add_argument( + "--matrix-file", default=DEFAULT_MATRIX_FILE, help="path to the matrix YAML" + ) + args = parser.parse_args() + + builds = load_matrix(args.matrix_file) + filters = { + "name": parse_filter(args.build), + "flavour": parse_filter(args.flavour), + "suite": parse_filter(args.suite), + "family": parse_filter(args.family), + } + what = describe_selection(args.type, filters) + + # Report a name that matches nothing before reporting an empty selection: + # "no Daily entry has name qcom-next" says what to fix, where + # "no entries found" leaves the reader to work out which filter was wrong. + unmatched = unmatched_filters(builds, args.type, filters) + if unmatched: + sys.exit( + f"ERROR: Nothing to build for {what}:\n" + + "\n".join(f" - {problem}" for problem in unmatched) + ) + + selected = select(builds, args.type, filters) + if not selected: + # A caller splitting one dispatch across both build paths asks each + # family for the same selection, and one of them legitimately has + # nothing to build. Every name in the request still had to match + # something above, so this is an empty intersection, not a typo. + if args.allow_empty and not args.field: + print("[]") + return + sys.exit(f"ERROR: No matrix entries found for {what}") + + if not args.field: + print(json.dumps([for_workflow(entry) for entry in selected], separators=(",", ":"))) + return + + if len(selected) > 1: + sys.exit( + f"ERROR: --field {args.field} needs exactly one entry, but {what} " + f"selects {len(selected)}; narrow it with --build" + ) + + entry = for_workflow(selected[0]) + if args.field not in entry: + sys.exit( + f"ERROR: {what} has no field {args.field}; " + "available: " + ", ".join(sorted(entry)) + ) + print(entry[args.field]) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/resolve-matrix.sh b/ci/scripts/resolve-matrix.sh deleted file mode 100755 index 4c120641..00000000 --- a/ci/scripts/resolve-matrix.sh +++ /dev/null @@ -1,400 +0,0 @@ -#!/bin/bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause-Clear -set -euo pipefail - -# Resolve and flatten the kernel delivery matrix for a given delivery type. -# -# The matrix root is an object with two top-level keys: -# - "suite_suffix_mapping": a suite -> Debian suffix map shared by every -# kernel variant and delivery type (e.g. "trixie": "~bpo13+1"). -# - "deliveries": the matrix rows. Each kernel_variant owns exactly one -# Daily row and one Release row. A row declares every input needed by -# that delivery, including a debian_version_stub. Three fields are -# list-valued: suites, which is expanded into isolated legs; -# kernel_config, which is one config fragment per element; and dkms, -# which is one out-of-tree module per element. A fragment is -# either a bare name from debian/config-available/ or an "intree:" entry -# naming a path relative to the kernel source root -# (e.g. intree:arch/arm64/configs/qcom_debug.config). A dkms entry is the -# stem of a -dkms package, e.g. "kgsl". -# -# Each flattened leg's final debian_revision is derived from -# debian_version_stub, suite_suffix_mapping[suite], and the delivery type via -# ci/scripts/derive-debian-revision.sh, so the formula has exactly one -# implementation shared with build-kernel-deb.yml's direct-dispatch path. Each -# row also carries debian_version_suffix ("~" for Daily, "" for Release) as a -# visible, validated record of that same delivery-type mapping; it is checked -# against the row's type but never fed into derivation, so a copy/paste error -# here fails fast instead of silently drifting from the formula's single -# implementation. -# -# Usage: -# ci/scripts/resolve-matrix.sh --type Daily -# ci/scripts/resolve-matrix.sh --type Release -# ci/scripts/resolve-matrix.sh --type Daily --single-suite trixie -# ci/scripts/resolve-matrix.sh --type Daily --kernel-variant qcom-next -# ci/scripts/resolve-matrix.sh --type Daily --matrix-file path/to/matrix.json -# -# Options: -# --type TYPE Delivery type to filter (Daily or Release). -# Required. -# --single-suite SUITE Emit only entries for this suite. -# --kernel-variant VARIANT Emit only entries for this kernel variant. -# --matrix-file FILE Path to the matrix JSON file -# (default: ci/build-matrix.json relative to CWD). -# -# Output: -# Compact JSON array to stdout. Every entry has a single suite, the -# kernel_variant that scopes its artifacts, Debusine workspace, and logs, -# and a suite-specific debian_revision (debian_version_stub and -# debian_version_suffix are consumed and removed). kernel_config and dkms are -# joined into the comma-separated strings that build-kernel-deb.yml's -# kernel-config and dkms inputs — and in turn prepare-source.sh's -# --kernel-config and --dkms — expect. -# -# Exit codes: -# 0 Success, at least one entry emitted. -# 1 Error (invalid arguments, matrix validation failure, no matching -# entry, revision derivation failure). - -TYPE="" -SINGLE_SUITE="" -KERNEL_VARIANT="" -MATRIX_FILE="ci/build-matrix.json" - -usage() { - sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' - exit 1 -} - -while [[ $# -gt 0 ]]; do - case $1 in - --type) TYPE="$2"; shift 2 ;; - --single-suite) SINGLE_SUITE="$2"; shift 2 ;; - --kernel-variant) KERNEL_VARIANT="$2"; shift 2 ;; - --matrix-file) MATRIX_FILE="$2"; shift 2 ;; - -h|--help) usage ;; - *) echo "ERROR: Unknown option: $1" >&2; usage ;; - esac -done - -[[ "$TYPE" == "Daily" || "$TYPE" == "Release" ]] || { - echo "ERROR: --type must be Daily or Release" >&2 - exit 1 -} -[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } - -jq empty "$MATRIX_FILE" 2>/dev/null \ - || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } - -validation_errors=$(jq -r ' - def required_string($field): - if (has($field) and (.[$field] | type == "string") and (.[$field] | length > 0)) - then empty - else "missing or invalid " + $field - end; - - def optional_string($field): - if (has($field) | not) or (.[$field] | type == "string") - then empty - else "invalid " + $field - end; - - def variant_name_valid: - if (.kernel_variant | type) != "string" - then empty - elif (.kernel_variant | test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$")) - then empty - else "kernel_variant must use lowercase letters, digits, and internal hyphens" - end; - - # An "intree:" entry names a fragment shipped by the kernel source, as a - # path relative to the kernel source root (e.g. - # intree:arch/arm64/configs/qcom_debug.config). Reject absolute paths, - # ".." traversal, and anything not ending in .config so a typo fails here - # rather than deep inside prepare-source.sh. - def intree_path_valid: - .[7:] as $path | - ($path | test("^([A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\\.config$")) - and (($path | test("(^|/)\\.\\.(/|$)")) | not); - - # Every fragment lands in debian/config/ under its basename, so two entries - # sharing one basename (e.g. arch/arm64/configs/hardening.config and - # kernel/configs/hardening.config) collide there. - def fragment_basename: - sub("^intree:"; "") | sub("^.*/"; "") | sub("\\.config$"; ""); - - def kernel_config_valid: - if (.kernel_config | type) != "array" - then "kernel_config must be an array" - elif any(.kernel_config[]; type != "string" or length == 0) - then "kernel_config must contain only non-empty strings" - elif any(.kernel_config[]; test(",")) - then "kernel_config entries must not contain commas; use one array element per fragment" - elif ([.kernel_config[]] | unique | length) != (.kernel_config | length) - then "kernel_config must not contain duplicates" - elif any(.kernel_config[]; startswith("intree:") and (intree_path_valid | not)) - then "intree: entries must be a kernel-source-relative path ending in .config (e.g. intree:arch/arm64/configs/qcom_debug.config)" - elif ([.kernel_config[] | fragment_basename] | unique | length) != (.kernel_config | length) - then "kernel_config entries must not resolve to the same fragment filename" - else empty - end; - - # Out-of-tree modules to build and bundle, each named as the stem of its - # -dkms package. The same shape rules debian/rules prepare enforces, - # applied here so a typo fails before any build job starts rather than in - # the middle of one. - def dkms_valid: - if (.dkms | type) != "array" - then "dkms must be an array" - elif any(.dkms[]; type != "string" or length == 0) - then "dkms must contain only non-empty strings" - elif any(.dkms[]; test(",")) - then "dkms entries must not contain commas; use one array element per module" - elif any(.dkms[]; endswith("-dkms")) - then "dkms entries must omit the -dkms suffix (e.g. kgsl, not kgsl-dkms)" - elif any(.dkms[]; test("^[a-z0-9][a-z0-9+.-]*$") | not) - then "dkms entries must be package name stems (lowercase letters, digits, + . -)" - elif ([.dkms[]] | unique | length) != (.dkms | length) - then "dkms must not contain duplicates" - else empty - end; - - def suites_valid: - if (.suites | type) != "array" or (.suites | length) == 0 - then "suites must be a non-empty array" - elif any(.suites[]; type != "string" or length == 0) - then "suites must contain only non-empty strings" - elif any(.suites[]; test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") | not) - then "suites must use lowercase letters, digits, and internal hyphens" - elif ([.suites[]] | unique | length) != (.suites | length) - then "suites must not contain duplicates" - else empty - end; - - def row_errors($index): - if type != "object" - then "row " + ($index | tostring) + ": matrix entries must be objects" - else - . as $row | - [ - required_string("kernel_variant"), - required_string("type"), - required_string("git_clone"), - required_string("branch_or_tag"), - required_string("ref_strategy"), - required_string("srcpkg"), - required_string("binpkg"), - required_string("debian_version_stub"), - optional_string("debusine_parent_workspace"), - optional_string("localversion"), - optional_string("kver_extra"), - variant_name_valid, - suites_valid, - kernel_config_valid, - dkms_valid, - if (.debian_version_stub | type) == "string" and (.debian_version_stub | test("~$")) - then "debian_version_stub must not end in ~" - else empty end, - if (has("debian_version_suffix") | not) or (.debian_version_suffix | type) != "string" - then "missing or invalid debian_version_suffix" - elif .type == "Daily" and .debian_version_suffix != "~" - then "debian_version_suffix must be \"~\" for Daily rows (got \"" + (.debian_version_suffix | tostring) + "\")" - elif .type == "Release" and .debian_version_suffix != "" - then "debian_version_suffix must be \"\" for Release rows (got \"" + (.debian_version_suffix | tostring) + "\")" - else empty end, - if (.type == "Daily" or .type == "Release") - then empty else "type must be Daily or Release" end, - if (.ref_strategy == "latest_tag" or .ref_strategy == "branch_tip" or .ref_strategy == "pinned_ref") - then empty else "ref_strategy must be latest_tag, branch_tip, or pinned_ref" end, - if .type == "Daily" and (.ref_strategy != "latest_tag" and .ref_strategy != "branch_tip") - then "Daily rows must use ref_strategy=latest_tag or ref_strategy=branch_tip" - elif .type == "Release" and .ref_strategy != "pinned_ref" - then "Release rows must use ref_strategy=pinned_ref" - else empty - end, - if .ref_strategy == "latest_tag" - then required_string("tag_pattern") - elif has("tag_pattern") - then "tag_pattern is only valid with ref_strategy=latest_tag" - else empty - end, - if .type == "Release" - then required_string("target_workspace") - elif has("target_workspace") - then "target_workspace is only valid for Release" - else empty - end - ] | .[] | "row " + ($index | tostring) + " (" + (($row.kernel_variant // "unknown") | tostring) + "): " + . - end; - - if type != "object" - then "matrix root must be an object with suite_suffix_mapping and deliveries" - elif (.deliveries | type) != "array" - then "deliveries must be an array" - elif (.deliveries | length) == 0 - then "deliveries must contain at least one row" - elif (.suite_suffix_mapping | type) != "object" - then "suite_suffix_mapping is missing or not an object" - else - .deliveries as $matrix | - .suite_suffix_mapping as $mapping | - ( - [range(0; ($matrix | length)) as $index | $matrix[$index] | row_errors($index)] - + - [ - [$matrix[] | select(type == "object")] - | group_by(.kernel_variant) - | .[] - | . as $rows - | (($rows[0].kernel_variant // "unknown") | tostring) as $variant - | ([ $rows[].type ] | sort) as $types - | ([ $rows[].srcpkg ] | unique) as $srcpkgs - | ([ $rows[].binpkg ] | unique) as $binpkgs - | ([ $rows[].debian_version_stub ] | unique) as $stubs - | if ($rows | length) != 2 - then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" - elif $types != ["Daily", "Release"] - then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" - elif ($srcpkgs | length) != 1 - then "kernel_variant " + $variant + " must use one srcpkg across its Daily and Release rows" - elif ($binpkgs | length) != 1 - then "kernel_variant " + $variant + " must use one binpkg across its Daily and Release rows" - elif ($stubs | length) != 1 - then "kernel_variant " + $variant + " must use one debian_version_stub across its Daily and Release rows" - else empty - end - ] - + - [ - [ - $matrix[] - | select(type == "object") - | select((.kernel_variant | type) == "string") - | select((.srcpkg | type) == "string" and (.srcpkg | length) > 0) - | {package: .srcpkg, kernel_variant: .kernel_variant} - ] - | group_by(.package)[] - | ([.[].kernel_variant] | unique) as $variants - | select($variants | length > 1) - | "srcpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) - ] - + - [ - [ - $matrix[] - | select(type == "object") - | select((.kernel_variant | type) == "string") - | select((.binpkg | type) == "string" and (.binpkg | length) > 0) - | {package: .binpkg, kernel_variant: .kernel_variant} - ] - | group_by(.package)[] - | ([.[].kernel_variant] | unique) as $variants - | select($variants | length > 1) - | "binpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) - ] - + - [ - $mapping | to_entries[] | select(.value | type != "string") - | "suite_suffix_mapping[" + .key + "] must be a string" - ] - + - [ - $mapping - | to_entries[] - | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) - | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" - ] - + - [ - $mapping - | to_entries - | group_by(.value) - | map(select(length > 1)) - | .[]? - | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" - ] - + - [ - [$matrix[] | select(type == "object") | select((.suites | type) == "array") | .suites[]] - | unique - | .[] as $suite - | select(($mapping | has($suite)) | not) - | "suite " + $suite + " has no suite_suffix_mapping entry" - ] - ) | .[] - end -' "$MATRIX_FILE") - -if [[ -n "$validation_errors" ]]; then - echo "ERROR: Invalid kernel delivery matrix:" >&2 - while IFS= read -r error; do - [[ -n "$error" ]] && echo " - $error" >&2 - done <<< "$validation_errors" - exit 1 -fi - -result=$(jq -c \ - --arg type "$TYPE" \ - --arg single_suite "$SINGLE_SUITE" \ - --arg kernel_variant "$KERNEL_VARIANT" ' - [ - .deliveries[] - | select(.type == $type) - | select($kernel_variant == "" or .kernel_variant == $kernel_variant) - | . as $row - | ( - if $single_suite == "" - then .suites - elif (.suites | index($single_suite)) != null - then [$single_suite] - else [] - end - )[] as $suite - | $row - | del(.suites) - | . + { - "suite": $suite, - "kernel_config": ($row.kernel_config | join(",")), - "dkms": ($row.dkms | join(",")) - } - ] - | if length == 0 - then error( - "no matrix entries found for type=" + $type - + (if $kernel_variant != "" then " kernel_variant=" + $kernel_variant else "" end) - + (if $single_suite != "" then " suite=" + $single_suite else "" end) - ) - else . - end - ' "$MATRIX_FILE") || { - echo "ERROR: Matrix resolution failed for type=$TYPE${KERNEL_VARIANT:+ kernel_variant=$KERNEL_VARIANT}${SINGLE_SUITE:+ suite=$SINGLE_SUITE}" >&2 - exit 1 -} - -# Derive each leg's final debian_revision from debian_version_stub, -# suite_suffix_mapping, and its delivery type. derive-debian-revision.sh is -# the single implementation of the formula; build-kernel-deb.yml's direct -# dispatch path calls the same script for the one-suite, no-matrix case. -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -final="[]" -while IFS= read -r leg; do - suite=$(jq -r '.suite' <<< "$leg") - stub=$(jq -r '.debian_version_stub' <<< "$leg") - delivery_type=$(jq -r '.type' <<< "$leg") - variant=$(jq -r '.kernel_variant' <<< "$leg") - - revision=$("$script_dir/derive-debian-revision.sh" \ - --stub "$stub" --suite "$suite" --delivery-type "$delivery_type" \ - --matrix-file "$MATRIX_FILE") || { - echo "ERROR: Failed to derive Debian revision for kernel_variant=$variant suite=$suite type=$delivery_type" >&2 - exit 1 - } - - leg=$(jq -c --arg rev "$revision" '(. + {debian_revision: $rev}) | del(.debian_version_stub, .debian_version_suffix)' <<< "$leg") - final=$(jq -c --argjson leg "$leg" '. + [$leg]' <<< "$final") -done < <(jq -c '.[]' <<< "$result") - -echo "$final" diff --git a/debian/README.md b/debian/README.md index bdc1e2e6..0863f55c 100644 --- a/debian/README.md +++ b/debian/README.md @@ -49,6 +49,8 @@ for dependency resolution. ``` pkg-linux-qcom/ ├── build-kernel.sh ← Build orchestrator (clone → prepare → build) +├── prepare-source.sh ← Inject debian/, derive the version, run 'prepare' +├── build-source-package.sh ← Prepared tree → reproducible .orig.tar.gz, .dsc, .changes ├── debian/ │ ├── control.in ← Source-of-truth template (version-controlled) │ ├── changelog.in ← Source-of-truth template (version-controlled) @@ -184,17 +186,14 @@ KVER = + LOCALVERSION passed via `--localversion`) 3. Optionally appending `KVER_EXTRA` (passed via `--kver-extra`) -The LOCALVERSION suffix encodes the branch name and snapshot/ABI date. For -tagged builds, `build-kernel.sh` auto-extracts this from the tag name. +The LOCALVERSION suffix encodes the flavour, the snapshot date and the +commit. `prepare-source.sh` derives it from the checked-out tag or branch +with `ci/scripts/derive-localversion.sh`, the same script CI uses, so a local +build of a commit gets the version CI would give it. See +[docs/version.md](../docs/version.md) for the format. -`debian/rules` recovers LOCALVERSION from the package name at build time by -stripping the `linux-image-` prefix and the trailing `-qcom` flavour suffix: - -``` -linux-image-7.2.0-qcom-next-20260826 -→ strip "linux-image-7.0.0-rc2" → -qcom-next-20260826-qcom -→ strip "-qcom" suffix → -qcom-next-20260826 (= LOCALVERSION) -``` +`debian/rules` reads LOCALVERSION back at build time from `debian/localversion`, +which `prepare` writes, rather than recovering it from a package name. --- @@ -242,7 +241,7 @@ clone → prepare → build. Run it from the repo root. ```bash # 1. Set up kernel source -git clone git@github.com:qualcomm-linux/kernel.git kernel-source +git clone https://github.com/qualcomm-linux/kernel kernel-source cd kernel-source git checkout qcom-next @@ -274,6 +273,50 @@ python3 docker_deb_build.py \ `build-kernel.sh` calls this automatically after running `prepare`. +### Building a source package + +CI does not build binaries from a tree. It builds a Debian source package +first and hands that to the builder — Debusine for the Debian suites, sbuild +in the builder container for Ubuntu — so the thing that gets built is a +`.dsc` whose checksums are recorded in a `.changes`. The same is available +locally: + +```bash +# Clone, prepare, and stop after the source package (no kernel build) +./build-kernel.sh --latest-tag --source-package +# -> kernel-build/trixie/linux-qcom-next_.orig.tar.gz +# kernel-build/trixie/linux-qcom-next_-.dsc +# kernel-build/trixie/linux-qcom-next_-_source.changes + +# Build binaries from it, in any mode +./build-kernel.sh --dsc kernel-build/trixie/linux-qcom-next_*.dsc +./build-kernel.sh --dsc kernel-build/trixie/linux-qcom-next_*.dsc --build-mode native + +# Or step by step, from a prepared tree +./prepare-source.sh --source-dir kernel-source --distro trixie +./build-source-package.sh --source-dir kernel-source --output-dir kernel-build/trixie +``` + +The `.orig.tar.gz` is a function of the kernel commit and nothing else. It +is written with `git archive`, which gives every entry the commit's timestamp +and root ownership, and compressed with `gzip -n`, which writes no timestamp, +so two builds of one commit produce one tarball byte for byte — and so do +two suites that differ only in Debian revision, which share the tarball an +archive requires them to share. `build-source-package.sh` checks that the +commit it archives is the one the version names (`~g`), and refuses a +tree that differs from that commit outside `debian/`, since `dpkg-source` +would otherwise fold the difference into an automatic patch. An in-tree +build leaves such differences; clean them with + +```bash +git -C kernel-source checkout -- . && git -C kernel-source clean -xdf -- . ':(exclude)debian' +``` + +The debian tarball and `.dsc` are reproducible too: the changelog is dated +from the commit, and `dpkg-source` takes the tarball's timestamps from it. +Only the `gzip` implementation is outside the commit's control, which is why +CI writes every orig in one builder image. + --- ## Packages produced @@ -394,7 +437,7 @@ A listed module is a presence contract — if it fails to build, or its `BUILD_EXCLUSIVE` gates exclude this kernel, the package build fails rather than shipping a kernel image without it. To stop bundling a module, drop it from the list its build was given (for CI builds, the `dkms` field in -`ci/build-matrix.json`). +`ci/build-matrix.yaml`). ### Standalone developer use diff --git a/debian/changelog.in b/debian/changelog.in index 802a600a..4a67cd49 100644 --- a/debian/changelog.in +++ b/debian/changelog.in @@ -6,4 +6,4 @@ * Commit: @GITSHA@ * Includes kernel image, modules, DTBs, headers, and debug symbols - -- Bjordis Collaku Fri, 14 Nov 2025 10:00:00 -0800 + -- Bjordis Collaku @DATE@ diff --git a/debian/rules b/debian/rules index 5f8798fc..37405e8a 100755 --- a/debian/rules +++ b/debian/rules @@ -72,6 +72,15 @@ KREL_FILE := debian/kernel.release # GIT_SHA Full kernel HEAD commit SHA, recorded in the changelog. # GITSHA above is the same commit at 12 characters, for the # version strings; this is the unabbreviated value. +# CHANGELOG_DATE Date of the changelog entry, RFC 2822 (date -R form). +# prepare-source.sh passes the kernel commit's committer +# date, so the entry is dated by the source it describes +# and two builds of one commit write one changelog. +# dpkg-source takes the debian tarball's mtimes from this +# date and dpkg-buildpackage exports it as +# SOURCE_DATE_EPOCH, so it is also what makes the source +# package reproducible. Defaults to now, for a prepare run +# by hand. # # Outputs (generated files, all listed in debian/clean): # debian/control Substituted from debian/control.in @@ -101,6 +110,7 @@ DKMS_MODULES ?= GIT_CLONE ?= unknown GIT_REF ?= unknown GIT_SHA ?= unknown +CHANGELOG_DATE ?= $(shell date -R) .PHONY: prepare prepare: @@ -241,6 +251,7 @@ prepare: -e "s|@GITCLONE@|$(GIT_CLONE)|g" \ -e "s|@BRANCH@|$(GIT_REF)|g" \ -e "s|@GITSHA@|$(GIT_SHA)|g" \ + -e "s|@DATE@|$(CHANGELOG_DATE)|g" \ debian/changelog.in > debian/changelog; \ \ # ── Write runtime state files ───────────────────────────────────────────── diff --git a/prepare-source.sh b/prepare-source.sh index c5594c35..34007d50 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -20,6 +20,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_DISTRO="trixie" +DEFAULT_FLAVOUR="qcom-next" DEFAULT_SRCPKG="linux-qcom-next" DEFAULT_BINPKG="linux-image-qcom-next" DEFAULT_DEBIAN_REVISION="0qcom1" @@ -51,16 +52,20 @@ OPTIONS: Version control: -d, --distro DISTRO Target suite: trixie|forky|sid|noble|questing|resolute (default: $DEFAULT_DISTRO) + --flavour NAME Kernel flavour, the identity LOCALVERSION carries + (default: $DEFAULT_FLAVOUR). Only consulted when + --localversion is not given. --localversion SUFFIX LOCALVERSION suffix appended to the base kernel - version (e.g. +qcom-next-20260722). - Auto-detected from git tag if not specified. + version (e.g. +qcom-next-20260722-g07f50dc44edd). + Derived from the checked-out tag or branch by + ci/scripts/derive-localversion.sh if not given. --snapshot SNAPSHOT Dated component of the Debian version: YYYYMMDD with an optional . ordinal (e.g. - 20260722 or 20260722.1). Auto-detected from the - git tag, or from the HEAD commit date when the - ref carries no date of its own; pass it - explicitly whenever --localversion is passed - explicitly. + 20260722 or 20260722.1). Derived alongside + --localversion, from the git tag if it carries a + date or otherwise from the HEAD commit date; + pass it explicitly whenever --localversion is + passed explicitly. --git-sha SHA Full commit SHA the build was cut from. Its first 12 characters discriminate two builds of one snapshot (a moved tag) in the version @@ -69,8 +74,14 @@ OPTIONS: --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). --git-clone URL Kernel repository URL, recorded in the changelog. + Defaults to the checkout's origin remote. --git-ref REF Resolved kernel ref (tag or branch), recorded in - the changelog. + the changelog. Defaults to the exact tag HEAD + sits on, or the branch it is the tip of. + --changelog-date DATE Date of the changelog entry, RFC 2822 (date -R + form). Defaults to HEAD's committer date, so + the entry is dated by the source and the + source package is reproducible. Package naming: --srcpkg NAME Source package name (default: $DEFAULT_SRCPKG) @@ -110,13 +121,14 @@ OPTIONS: -h, --help Show this help EXAMPLES: - # Minimal: auto-detect LOCALVERSION from git tag, default package names + # Minimal: derive LOCALVERSION from the checkout, default package names $0 --source-dir /path/to/kernel # Full CI invocation with all options $0 --source-dir /path/to/kernel \\ --distro trixie \\ - --localversion +qcom-next-20260722 \\ + --localversion +qcom-next-20260722-g07f50dc44edd \\ + --snapshot 20260722 \\ --srcpkg linux-qcom-next \\ --binpkg linux-image-qcom-next \\ --debian-revision 0qcom1 \\ @@ -129,6 +141,7 @@ EOF # Defaults SOURCE_DIR="" DISTRO="$DEFAULT_DISTRO" +FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION="" SNAPSHOT="" KVER_EXTRA="" @@ -140,17 +153,20 @@ DKMS_MODULES="" GIT_CLONE="" GIT_REF="" GIT_SHA="" +CHANGELOG_DATE="" while [[ $# -gt 0 ]]; do case $1 in -s|--source-dir) SOURCE_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --snapshot) SNAPSHOT="$2"; shift 2 ;; --git-sha) GIT_SHA="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --git-clone) GIT_CLONE="$2"; shift 2 ;; --git-ref) GIT_REF="$2"; shift 2 ;; + --changelog-date) CHANGELOG_DATE="$2"; shift 2 ;; --srcpkg) SRCPKG="$2"; shift 2 ;; --binpkg) BINPKG="$2"; shift 2 ;; --debian-revision) DEBIAN_REVISION="$2"; shift 2 ;; @@ -182,62 +198,58 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -n "$GIT_SHA" ]] || GIT_SHA=$(git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || true) GITSHA="${GIT_SHA:0:12}" -# ── Helper: the HEAD commit date, for refs that carry no date of their own ─── -# The COMMIT date, not the build date: rebuilding a commit then reproduces its -# version instead of inventing a higher one, and the date describes the source -# rather than when the build happened to run. +# ── Name the checkout ──────────────────────────────────────────────────────── +# The ref HEAD answers to: the exact tag it sits on, else the branch it is the +# tip of. A detached HEAD reports HEAD, which is treated like any undated ref. +# This names the checkout for the version derivation below and, when the +# caller gave none, for the provenance recorded in the changelog -- so a local +# build says where it came from just as a CI build does. # -# Committer date rather than author date, because an author date can be months -# old on a backported patch. Normalised to UTC, since an unnormalised date -# renders in each committer's timezone and one commit would yield different -# snapshots on different hosts. -_head_commit_date() { - TZ=UTC git -C "$SOURCE_DIR" log -1 --format=%cd --date=format-local:%Y%m%d \ - 2>/dev/null || true -} - -# ── Helper: derive LOCALVERSION, SNAPSHOT and GITSHA from a tag name ───────── -# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g / 20260722 -# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g / 20260722.1 -# -# The trailing component is a YYYYMMDD snapshot with an optional respin ordinal -# for a second tag cut on the same day. Matching the date width explicitly (and -# not just "trailing digits") keeps the ordinal attached to it. -# -# All three fields come out of the tag and HEAD together. Recovering them from -# LOCALVERSION afterwards would mean parsing a string that also holds a variant -# name and a hex SHA that can end in eight digits. -_auto_version_fields() { - local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]{8}(\.[0-9]+)?)$ ]]; then - SNAPSHOT="${BASH_REMATCH[2]}" - LOCALVERSION="+${BASH_REMATCH[1]}-${SNAPSHOT}-g${GITSHA}" - else - # A tag carrying no date of its own is dated by its commit, exactly as - # derive-localversion.sh dates a branch tip. Leaving the snapshot empty - # would drop the +git and sort the build below every dated one. - LOCALVERSION="+$tag" - SNAPSHOT=$(_head_commit_date) - fi -} - -# ── Auto-detect LOCALVERSION, SNAPSHOT and GITSHA from git (if not provided) ── +# Every git call here tolerates failure. CI passes all of these values in and +# runs this script in a container as a different user from the one owning +# the checkout, where git refuses to read the repository at all; nothing it +# was given is then asked of git. +_git() { git -C "$SOURCE_DIR" "$@" 2>/dev/null || true; } +CHECKOUT_REF="" +if [[ -z "$GIT_REF" || -z "$LOCALVERSION" ]]; then + CHECKOUT_REF=$(_git describe --tags --exact-match) + [[ -n "$CHECKOUT_REF" ]] || CHECKOUT_REF=$(_git rev-parse --abbrev-ref HEAD) +fi +[[ -n "$GIT_CLONE" ]] || GIT_CLONE=$(_git remote get-url origin) +[[ -n "$GIT_REF" ]] || GIT_REF="$CHECKOUT_REF" +# Committer date, like the snapshot: it describes the source rather than the +# build, so rebuilding a commit rewrites the same changelog entry. +[[ -n "$CHANGELOG_DATE" ]] || CHANGELOG_DATE=$(_git log -1 --format=%cD) + +# ── Derive LOCALVERSION, SNAPSHOT and GITSHA from the checkout (if not given) ── +# The same derivation CI performs, by the same script: the ref is the exact +# tag HEAD sits on, or the branch it is the tip of, and the date is HEAD's +# committer date, used whenever that ref carries no date of its own (a +# branch tip, or a tag with no trailing -YYYYMMDD). A local build of a commit +# therefore produces the version CI would give it, which is what lets a +# source package built here stand in for one built there. if [[ -z "$LOCALVERSION" ]]; then - GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - _auto_version_fields "$GIT_TAG" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from tag '$GIT_TAG'" + if [[ -z "$GIT_SHA" || -z "$CHECKOUT_REF" ]]; then + log_warn "LOCALVERSION not set and $SOURCE_DIR is not a readable git checkout." + log_warn "Package will be named linux-image- (no flavour/date suffix)." + log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722-g07f50dc44edd" else - # Branch tip: no ref to name the kernel release after, but HEAD still - # dates the build, so the Debian version keeps its +git and stays - # in sequence with the dated builds instead of below all of them. - SNAPSHOT=$(_head_commit_date) - log_warn "LOCALVERSION not set and no exact git tag found." - log_warn "Package will be named linux-image- (no branch/date suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722" - if [[ -n "$SNAPSHOT" ]]; then - log_warn "Debian version dated by the HEAD commit: +git${SNAPSHOT}." - fi + # Committer date, normalised to UTC, as CI does. + DERIVE_DATE=$(TZ=UTC git -C "$SOURCE_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + FIELDS=$("$SCRIPT_DIR/ci/scripts/derive-localversion.sh" \ + --flavour "$FLAVOUR" \ + --ref "$CHECKOUT_REF" \ + --sha "$GIT_SHA" \ + --date "$DERIVE_DATE") + while IFS='=' read -r key value; do + case "$key" in + LOCALVERSION) LOCALVERSION="$value" ;; + SNAPSHOT) SNAPSHOT="$value" ;; + GITSHA) GITSHA="$value" ;; + *) log_error "Unexpected field '$key' from derive-localversion.sh"; exit 1 ;; + esac + done <<< "$FIELDS" + log_info "Derived LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from $CHECKOUT_REF" fi elif [[ -z "$SNAPSHOT" ]]; then # An explicit --localversion is not parsed for a snapshot; say so rather @@ -249,12 +261,14 @@ fi log_step "Configuration:" log_info " Source dir: $SOURCE_DIR" log_info " Distro: $DISTRO" +log_info " Flavour: $FLAVOUR" log_info " Source package: $SRCPKG" log_info " Binary metapkg: $BINPKG" log_info " Debian revision: $DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" [[ -n "$SNAPSHOT" ]] && log_info " SNAPSHOT: $SNAPSHOT" [[ -n "$GITSHA" ]] && log_info " GITSHA: $GITSHA" +[[ -n "$GIT_CLONE" ]] && log_info " Source: $GIT_CLONE $GIT_REF" [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" [[ -n "$KERNEL_CONFIG" ]] && log_info " Kernel config: $KERNEL_CONFIG" [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" @@ -377,11 +391,16 @@ PREPARE_ARGS="DISTRO=$DISTRO SRCPKG=$SRCPKG BINPKG=$BINPKG DEBIAN_REVISION=$DEBI [[ -n "$GIT_CLONE" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_CLONE=$GIT_CLONE" [[ -n "$GIT_REF" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_REF=$GIT_REF" [[ -n "$GIT_SHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_SHA=$GIT_SHA" +# Quoted separately: an RFC 2822 date has spaces, which the string-built +# argument list above would split. Empty means "now", which debian/rules +# decides for itself. +CHANGELOG_DATE_ARG=() +[[ -n "$CHANGELOG_DATE" ]] && CHANGELOG_DATE_ARG=("CHANGELOG_DATE=$CHANGELOG_DATE") # Spaces are stripped so a list written as "kgsl, camx" stays a single make # argument; debian/rules validates the names it is given. [[ -n "$DKMS_MODULES" ]] && PREPARE_ARGS="$PREPARE_ARGS DKMS_MODULES=$(tr -d ' ' <<< "$DKMS_MODULES")" # shellcheck disable=SC2086 -make -f "$SOURCE_DIR/debian/rules" -C "$SOURCE_DIR" prepare $PREPARE_ARGS +make -f "$SOURCE_DIR/debian/rules" -C "$SOURCE_DIR" prepare $PREPARE_ARGS "${CHANGELOG_DATE_ARG[@]}" echo log_step "Source preparation complete."