From ed056417a30affa7859b327ab52d5b92cf41d0f4 Mon Sep 17 00:00:00 2001 From: Jim Wu Date: Tue, 18 Aug 2026 13:17:44 -0600 Subject: [PATCH 1/6] ci: re-enable ubuntu-22-rocm release build with larger ccache Revert #26969 (which disabled the ROCm Ubuntu release job) and address the underlying cause rather than just turning the job back on. Root cause of the 2h+ builds: the ROCm 7.14 switch (#25775) doubled the Linux GPU target count from 11 to 22. Each HIP translation unit is compiled once per target arch, so both build time and ccache size roughly doubled. The ggml-org/ccache-action defaults to max-size 500MB, which was enough for the 11 targets of ROCm 7.2.1 but too small for 22: CI logs from the 7.14 era show the cache pinned at the 500MB cap (78% full, 10 evictions mid-build, only ~45% hit rate), so it never went warm and every run took ~90-135 min. Fix: re-enable the ubuntu-22-rocm job and its ccache steps, and raise the ccache max-size to 2G so the 22-target cache fits and builds stay warm across runs. Also restores the job in the release job's needs: list and the release-notes download link. --- .github/workflows/release.yml | 239 +++++++++++++++++----------------- 1 file changed, 120 insertions(+), 119 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 82a8364e435..d38a2c766f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1285,123 +1285,124 @@ jobs: path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz name: llama-bin-ubuntu-sycl-${{ matrix.build }}-x64.tar.gz -# ubuntu-22-rocm: -# needs: [check-release, get-version] -# if: ${{ needs.check-release.outputs.should_release == 'true' }} - -# runs-on: ubuntu-22.04 - -# permissions: -# actions: write - -# strategy: -# matrix: -# include: -# - ROCM_VERSION: "7.14.0" -# gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201" -# build: 'x64' - -# steps: -# - name: Clone -# id: checkout -# uses: actions/checkout@v6 -# with: -# fetch-depth: 0 - -# - name: Setup Node.js -# uses: actions/setup-node@v6 -# with: -# node-version: "24" -# cache: "npm" -# cache-dependency-path: "tools/ui/package-lock.json" - -# - name: Free up disk space -# uses: ggml-org/free-disk-space@v1.3.1 -# with: -# tool-cache: true - -# # - name: ccache -# # uses: ggml-org/ccache-action@v1.2.21 -# # with: -# # key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} - -# - name: Dependencies -# id: depends -# run: | -# sudo apt install -y build-essential git cmake wget - -# - name: Setup TheRock with Wheels -# id: therock_env -# run: | -# # Create Python virtual environment -# python3 -m venv .venv -# source .venv/bin/activate - -# # Install ROCm wheels for build -# # libraries = HIP runtime and CMake configs needed for linking -# # devel = compilers, headers, static libs -# python -m pip install --upgrade pip -# python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}" - -# # Get ROCm installation paths using the rocm-sdk CLI tool -# ROCM_PATH=$(rocm-sdk path --root) -# CMAKE_PATH=$(rocm-sdk path --cmake) -# BIN_PATH=$(rocm-sdk path --bin) -# echo "ROCM_PATH=$ROCM_PATH" -# echo "CMAKE_PATH=$CMAKE_PATH" -# echo "BIN_PATH=$BIN_PATH" - -# # Set environment variables -# echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV -# echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV -# echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV -# echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV -# echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV - -# # Keep venv activated for subsequent steps -# echo "$(pwd)/.venv/bin" >> $GITHUB_PATH - -# - name: Build with native CMake HIP support -# id: cmake_build -# run: | -# cmake -B build -S . \ -# -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ -# -DCMAKE_BUILD_TYPE=Release \ -# -DGGML_BACKEND_DL=ON \ -# -DGGML_NATIVE=OFF \ -# -DCMAKE_INSTALL_RPATH='$ORIGIN' \ -# -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ -# -DGGML_CPU_ALL_VARIANTS=ON \ -# -DGPU_TARGETS="${{ matrix.gpu_targets }}" \ -# -DGGML_HIP=ON \ -# -DHIP_PLATFORM=amd \ -# -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ -# ${{ env.CMAKE_ARGS }} -# cmake --build build --config Release -j $(nproc) - -# # - name: ccache-clear -# # uses: ./.github/actions/ccache-clear -# # with: -# # key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} - -# - name: Determine tag name -# id: tag -# uses: ./.github/actions/get-tag-name - -# - name: Get ROCm short version -# run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV - -# - name: Pack artifacts -# id: pack_artifacts -# run: | -# cp LICENSE ./build/bin/ -# tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . - -# - name: Upload artifacts -# uses: actions/upload-artifact@v6 -# with: -# path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz -# name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz + ubuntu-22-rocm: + needs: [check-release, get-version] + if: ${{ needs.check-release.outputs.should_release == 'true' }} + + runs-on: ubuntu-22.04 + + permissions: + actions: write + + strategy: + matrix: + include: + - ROCM_VERSION: "7.14.0" + gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201" + build: 'x64' + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: Free up disk space + uses: ggml-org/free-disk-space@v1.3.1 + with: + tool-cache: true + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} + max-size: "2G" + + - name: Dependencies + id: depends + run: | + sudo apt install -y build-essential git cmake wget + + - name: Setup TheRock with Wheels + id: therock_env + run: | + # Create Python virtual environment + python3 -m venv .venv + source .venv/bin/activate + + # Install ROCm wheels for build + # libraries = HIP runtime and CMake configs needed for linking + # devel = compilers, headers, static libs + python -m pip install --upgrade pip + python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}" + + # Get ROCm installation paths using the rocm-sdk CLI tool + ROCM_PATH=$(rocm-sdk path --root) + CMAKE_PATH=$(rocm-sdk path --cmake) + BIN_PATH=$(rocm-sdk path --bin) + echo "ROCM_PATH=$ROCM_PATH" + echo "CMAKE_PATH=$CMAKE_PATH" + echo "BIN_PATH=$BIN_PATH" + + # Set environment variables + echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV + echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV + echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV + echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + + # Keep venv activated for subsequent steps + echo "$(pwd)/.venv/bin" >> $GITHUB_PATH + + - name: Build with native CMake HIP support + id: cmake_build + run: | + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGPU_TARGETS="${{ matrix.gpu_targets }}" \ + -DGGML_HIP=ON \ + -DHIP_PLATFORM=amd \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Get ROCm short version + run: echo "ROCM_VERSION_SHORT=$(echo '${{ matrix.ROCM_VERSION }}' | cut -d '.' -f 1,2)" >> $GITHUB_ENV + + - name: Pack artifacts + id: pack_artifacts + run: | + cp LICENSE ./build/bin/ + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin . + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz + name: llama-bin-ubuntu-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.tar.gz ios-xcode: needs: [check-release, get-version] @@ -1578,7 +1579,7 @@ jobs: #- windows-sycl - windows-rocm - windows-openvino - #- ubuntu-22-rocm + - ubuntu-22-rocm - ubuntu-cpu - ubuntu-vulkan - ubuntu-24-openvino @@ -1688,7 +1689,7 @@ jobs: - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz) - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz) - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz) - - Ubuntu x64 (ROCm 7.14)[DISABLED](https://github.com/ggml-org/llama.cpp/pull/26969) + - [Ubuntu x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.14-x64.tar.gz) - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz) - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz) - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz) From fe844adc2a33e8544213229827ab35021a5cb2fd Mon Sep 17 00:00:00 2001 From: Jim Wu Date: Tue, 18 Aug 2026 15:51:55 -0600 Subject: [PATCH 2/6] ci: add temporary ROCm build-timing workflow [DO NOT MERGE] Standalone workflow_dispatch job that runs only the ubuntu-22-rocm build (22 targets, ROCm 7.14, ccache max-size 2G) so build time and cold->warm ccache behavior can be measured on stock GitHub runners without triggering the full release matrix or publishing a release. Prints ccache -s before/after and BUILD_SECONDS to the job summary. Remove before merging the parent change. --- .github/workflows/rocm-build-timing.yml | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/rocm-build-timing.yml diff --git a/.github/workflows/rocm-build-timing.yml b/.github/workflows/rocm-build-timing.yml new file mode 100644 index 00000000000..e023f0f71e4 --- /dev/null +++ b/.github/workflows/rocm-build-timing.yml @@ -0,0 +1,81 @@ +# TEMP eval workflow to measure ubuntu-22-rocm build time (cold vs warm +# ccache) on stock GitHub runners. Runs ONLY the ROCm build, on manual dispatch. +# Do NOT merge - remove before this PR lands. +name: ROCm build timing (temp) + +on: + workflow_dispatch: + +env: + CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON" + +jobs: + ubuntu-22-rocm-timing: + runs-on: ubuntu-22.04 + + strategy: + matrix: + include: + - ROCM_VERSION: "7.14.0" + gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201" + build: 'x64' + + steps: + - name: Clone + uses: actions/checkout@v6 + + - name: Free up disk space + uses: ggml-org/free-disk-space@v1.3.1 + with: + tool-cache: true + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} + max-size: "2G" + + - name: Dependencies + run: | + sudo apt install -y build-essential git cmake wget + + - name: Setup TheRock with Wheels + run: | + python3 -m venv .venv + source .venv/bin/activate + python -m pip install --upgrade pip + python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}" + ROCM_PATH=$(rocm-sdk path --root) + CMAKE_PATH=$(rocm-sdk path --cmake) + BIN_PATH=$(rocm-sdk path --bin) + echo "ROCM_PATH=$ROCM_PATH" >> $GITHUB_ENV + echo "CMAKE_PREFIX_PATH=$CMAKE_PATH" >> $GITHUB_ENV + echo "HIP_PATH=$ROCM_PATH" >> $GITHUB_ENV + echo "PATH=$BIN_PATH:${PATH}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "$(pwd)/.venv/bin" >> $GITHUB_PATH + + - name: ccache stats (before build) + run: ccache -s + + - name: Build with native CMake HIP support + run: | + start=$(date +%s) + cmake -B build -S . \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_BACKEND_DL=ON \ + -DGGML_NATIVE=OFF \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGPU_TARGETS="${{ matrix.gpu_targets }}" \ + -DGGML_HIP=ON \ + -DHIP_PLATFORM=amd \ + ${{ env.CMAKE_ARGS }} + cmake --build build --config Release -j $(nproc) + end=$(date +%s) + echo "BUILD_SECONDS=$((end - start))" | tee -a $GITHUB_STEP_SUMMARY + + - name: ccache stats (after build) + run: ccache -s | tee -a $GITHUB_STEP_SUMMARY From 5d0212ec2ae77f57faede8cd0eb850abb5275320 Mon Sep 17 00:00:00 2001 From: Jim Wu Date: Tue, 18 Aug 2026 16:19:14 -0600 Subject: [PATCH 3/6] ci: trigger temp ROCm timing workflow on branch push [DO NOT MERGE] Add a push trigger scoped to this branch so the timing job runs without needing workflow_dispatch registration on the default branch. --- .github/workflows/rocm-build-timing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/rocm-build-timing.yml b/.github/workflows/rocm-build-timing.yml index e023f0f71e4..ecea1d61a92 100644 --- a/.github/workflows/rocm-build-timing.yml +++ b/.github/workflows/rocm-build-timing.yml @@ -5,6 +5,9 @@ name: ROCm build timing (temp) on: workflow_dispatch: + push: + branches: + - jimwu.restore-ubuntu-rocm-release env: CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON" From dc2c89949257d8262d6442542af03cec551cf8c5 Mon Sep 17 00:00:00 2001 From: Jim Wu Date: Tue, 18 Aug 2026 21:59:28 -0600 Subject: [PATCH 4/6] ci: trigger warm ccache timing run [DO NOT MERGE] From 6645469cd022c9a16bfe7a166ccce551ad85abe0 Mon Sep 17 00:00:00 2001 From: Jim Wu Date: Tue, 18 Aug 2026 23:32:14 -0600 Subject: [PATCH 5/6] ci: set ccache compiler_check=content for ROCm build The ROCm toolchain is pip-installed fresh on every run, so the clang binary's mtime changes each time. With ccache's default compiler_check=mtime that invalidates the whole cache and warm builds only reached ~70% hits. Hash the compiler contents instead so the cache survives toolchain reinstalls. Also applied to the temp timing workflow for measurement. --- .github/workflows/release.yml | 9 +++++++++ .github/workflows/rocm-build-timing.yml | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d38a2c766f9..d4b46032ba2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1326,6 +1326,15 @@ jobs: key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} max-size: "2G" + - name: Tune ccache for reinstalled ROCm toolchain + run: | + # ROCm is pip-installed fresh each run, so the clang binary's mtime + # changes every time. With the default compiler_check=mtime that + # invalidates the cache; hash compiler contents instead so warm + # builds hit. + ccache --set-config=compiler_check=content + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime + - name: Dependencies id: depends run: | diff --git a/.github/workflows/rocm-build-timing.yml b/.github/workflows/rocm-build-timing.yml index ecea1d61a92..17496dcd8c8 100644 --- a/.github/workflows/rocm-build-timing.yml +++ b/.github/workflows/rocm-build-timing.yml @@ -38,6 +38,16 @@ jobs: key: release-ubuntu-22.04-rocm-${{ matrix.ROCM_VERSION }} max-size: "2G" + - name: Tune ccache for reinstalled ROCm toolchain + run: | + # The ROCm toolchain is pip-installed fresh each run, so the clang + # binary's mtime changes every time. With the default + # compiler_check=mtime that invalidates the whole cache. Hash the + # compiler contents instead so warm builds actually hit. + ccache --set-config=compiler_check=content + ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime + ccache -p | grep -E "compiler_check|sloppiness" + - name: Dependencies run: | sudo apt install -y build-essential git cmake wget From d60e10cc4d5f46403007a908ddc560d0013a06c7 Mon Sep 17 00:00:00 2001 From: Jim Wu Date: Wed, 19 Aug 2026 08:57:24 -0600 Subject: [PATCH 6/6] ci: trigger warm run with compiler_check=content [DO NOT MERGE]