From 0b94f8a54c7a5b00098e94b233141246b6c740f0 Mon Sep 17 00:00:00 2001 From: ravi9 Date: Wed, 24 Jun 2026 02:14:21 +0530 Subject: [PATCH 01/20] Update to OV 2026.2.1, Make OV release packages self-contained --- .devops/openvino.Dockerfile | 12 ++-- .../actions/linux-setup-openvino/action.yml | 4 +- .github/workflows/build-cache.yml | 8 +-- .github/workflows/build-openvino.yml | 8 +-- .github/workflows/build-self-hosted.yml | 4 +- .github/workflows/release.yml | 64 ++++++++++++++++--- docs/backend/OPENVINO.md | 12 ++-- 7 files changed, 78 insertions(+), 34 deletions(-) diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile index fec72b1c7deb..bcaea16cc3f1 100644 --- a/.devops/openvino.Dockerfile +++ b/.devops/openvino.Dockerfile @@ -1,12 +1,12 @@ -ARG OPENVINO_VERSION_MAJOR=2026.2 -ARG OPENVINO_VERSION_FULL=2026.2.0.21903.52ddc073857 +ARG OPENVINO_VERSION_MAJOR=2026.2.1 +ARG OPENVINO_VERSION_FULL=2026.2.1.21919.ede283a88e3 ARG UBUNTU_VERSION=24.04 # Intel GPU driver versions. https://github.com/intel/compute-runtime/releases -ARG IGC_VERSION=v2.34.4 -ARG IGC_VERSION_FULL=2_2.34.4+21428 -ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1 -ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0 +ARG IGC_VERSION=v2.36.3 +ARG IGC_VERSION_FULL=2_2.36.3+21719 +ARG COMPUTE_RUNTIME_VERSION=26.22.38646.4 +ARG COMPUTE_RUNTIME_VERSION_FULL=26.22.38646.4-0 ARG IGDGMM_VERSION=22.10.0 # Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases diff --git a/.github/actions/linux-setup-openvino/action.yml b/.github/actions/linux-setup-openvino/action.yml index 46a659a827c4..fe3006ac8764 100644 --- a/.github/actions/linux-setup-openvino/action.yml +++ b/.github/actions/linux-setup-openvino/action.yml @@ -5,10 +5,10 @@ inputs: description: "Installation path" required: true version_major: - description: "OpenVINO major version (e.g., 2025.3)" + description: "OpenVINO major version (e.g., 2026.2.1)" required: true version_full: - description: "OpenVINO full version (e.g., 2025.3.0.19807.44526285f24)" + description: "OpenVINO full version (e.g., 2026.2.1.21919.ede283a88e3)" required: true runs: diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index b36c6e1ea89b..327f71978bf1 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -68,8 +68,8 @@ jobs: env: # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Clone @@ -96,8 +96,8 @@ jobs: env: # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Clone diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index 49ab13695cbf..938cde3f20ff 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -39,8 +39,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Clone @@ -96,8 +96,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Clone diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index c4366ece3e59..1a71ed827729 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -266,8 +266,8 @@ jobs: env: # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Clone diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7b67e49255f..220cce718b3f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -446,8 +446,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Set OpenVINO version output @@ -506,8 +506,11 @@ jobs: cmake -B build/ReleaseOV -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DGGML_OPENVINO=ON \ - -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} - cmake --build build/ReleaseOV --config Release -j $(nproc) + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \ + ${{ env.CMAKE_ARGS }} + cmake --build build/ReleaseOV --config Release --parallel - name: ccache-clear uses: ./.github/actions/ccache-clear @@ -521,8 +524,26 @@ jobs: - name: Pack artifacts id: pack_artifacts run: | - cp LICENSE ./build/ReleaseOV/bin/ - tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/ReleaseOV/bin . + dest=./build/ReleaseOV/bin + OPENVINO_ROOT=./openvino_toolkit + ov_lib="$OPENVINO_ROOT/runtime/lib/intel64" + + # Bundle OpenVINO runtime libs + TBB. Binaries built with RPATH=$ORIGIN + # load these siblings without setupvars.sh / LD_LIBRARY_PATH. + cp -P "$ov_lib"/libopenvino.so* \ + "$ov_lib"/libopenvino_c.so* \ + "$ov_lib"/libopenvino_*_plugin.so \ + "$ov_lib"/libopenvino_intel_npu_compiler*.so \ + "$OPENVINO_ROOT/runtime/3rdparty/tbb/lib"/*.so* \ + "$dest"/ + cp -P /usr/lib/x86_64-linux-gnu/libOpenCL.so.1* "$dest" 2>/dev/null || true + cp "$ov_lib/cache.json" "$dest" 2>/dev/null || true + + # OpenVINO licensing + cp -r "$OPENVINO_ROOT/docs/licensing" "$dest/openvino-licensing" + + cp LICENSE "$dest" + tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C "$dest" . - name: Upload artifacts uses: actions/upload-artifact@v6 @@ -538,8 +559,8 @@ jobs: env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.2" - OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + OPENVINO_VERSION_MAJOR: "2026.2.1" + OPENVINO_VERSION_FULL: "2026.2.1.21919.ede283a88e3" steps: - name: Set OpenVINO version output @@ -607,7 +628,9 @@ jobs: -A x64 ^ -DCMAKE_BUILD_TYPE=Release ^ -DGGML_OPENVINO=ON ^ - -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^ + ${{ env.CMAKE_ARGS }} cmake --build build\ReleaseOV --config Release -- /m @@ -624,7 +647,28 @@ jobs: id: pack_artifacts shell: powershell run: | - Copy-Item LICENSE .\build\ReleaseOV\bin\ + # Locate the extracted OpenVINO toolkit root (same pattern as the Build step). + $OPENVINO_ROOT = (Get-ChildItem -Directory openvino_toolkit | Select-Object -First 1).FullName + if (-not $OPENVINO_ROOT) { + Write-Error "OpenVINO toolkit folder not found under .\openvino_toolkit" + exit 1 + } + + $dest = ".\build\ReleaseOV\bin\Release" + + $ovBin = Join-Path $OPENVINO_ROOT 'runtime\bin\intel64\Release' + Copy-Item -Path (Join-Path $ovBin '*.dll') -Destination $dest -Force + Copy-Item -Path (Join-Path $ovBin 'cache.json') -Destination $dest -Force + + $tbbBin = Join-Path $OPENVINO_ROOT 'runtime\3rdparty\tbb\bin' + Copy-Item -Path (Join-Path $tbbBin 'tbb*.dll') -Destination $dest -Force + + # OpenVINO licensing + $licensingDest = Join-Path $dest 'openvino-licensing' + New-Item -ItemType Directory -Force -Path $licensingDest | Out-Null + Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force + + Copy-Item LICENSE $dest 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip .\build\ReleaseOV\bin\* - name: Upload artifacts diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 631d4bc3bf78..d5c6f46e299d 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -237,8 +237,8 @@ chmod +x ubuntu-llamacpp-ov-install.sh # ============================================ set -euo pipefail -OPENVINO_VERSION_MAJOR="2026.2" -OPENVINO_VERSION_FULL="2026.2.0.21903.52ddc073857" +OPENVINO_VERSION_MAJOR="2026.2.1" +OPENVINO_VERSION_FULL="2026.2.1.21919.ede283a88e3" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}" @@ -334,7 +334,7 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf" ``` > [!NOTE] -> The script pins OpenVINO `2026.2` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. +> The script pins OpenVINO `2026.2.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. @@ -364,8 +364,8 @@ REM ============================================ REM llama.cpp OpenVINO Build Script (Ninja) REM ============================================ -set "OPENVINO_VERSION_MAJOR=2026.2" -set "OPENVINO_VERSION_FULL=2026.2.0.21903.52ddc073857" +set "OPENVINO_VERSION_MAJOR=2026.2.1" +set "OPENVINO_VERSION_FULL=2026.2.1.21919.ede283a88e3" set "SCRIPT_DIR=%~dp0" set "VCPKG_DIR=C:\vcpkg" @@ -547,7 +547,7 @@ endlocal ``` > [!NOTE] -> The script pins OpenVINO `2026.2` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**. +> The script pins OpenVINO `2026.2.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**. From 02d37fc9b27e271174787cf78dcd9fc8066f82e9 Mon Sep 17 00:00:00 2001 From: ravi9 Date: Wed, 24 Jun 2026 03:13:57 +0530 Subject: [PATCH 02/20] Update to OV 2026.2.1, Make OV release packages self-contained --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 220cce718b3f..15ec339815d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -669,7 +669,7 @@ jobs: Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force Copy-Item LICENSE $dest - 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip .\build\ReleaseOV\bin\* + 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip .\build\ReleaseOV\bin\Release\* - name: Upload artifacts uses: actions/upload-artifact@v6 From b247c73d8542f3c9d80a229344614f0eee0280e9 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Tue, 23 Jun 2026 02:41:57 +0300 Subject: [PATCH 03/20] OpenVINO Backend: Remove compute_op_type hardcoded sets (#222) * OpenVINO Backend: Remove compute_op_type hardcoded sets * revert get_op_type removal --- ggml/src/ggml-openvino/ggml-decoder.cpp | 69 ++----------------------- 1 file changed, 3 insertions(+), 66 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index b6df4f0fbb7a..48c63e4d70fa 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1270,77 +1270,14 @@ void GgmlOvDecoder::visit_subgraph(std::function ops = { - {GGML_OP_NONE, "GGML_OP_NONE" }, - {GGML_OP_ACC, "GGML_OP_ACC" }, - {GGML_OP_ADD, "GGML_OP_ADD" }, - {GGML_OP_ADD1, "GGML_OP_ADD1" }, - {GGML_OP_ADD_ID, "GGML_OP_ADD_ID" }, - {GGML_OP_CONCAT, "GGML_OP_CONCAT" }, - {GGML_OP_CONT, "GGML_OP_CONT" }, - {GGML_OP_DIV, "GGML_OP_DIV" }, - {GGML_OP_DUP, "GGML_OP_DUP" }, - {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS" }, - {GGML_OP_MUL, "GGML_OP_MUL" }, - {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT" }, - {GGML_OP_MUL_MAT_ID, "GGML_OP_MUL_MAT_ID" }, - {GGML_OP_PERMUTE, "GGML_OP_PERMUTE" }, - {GGML_OP_RESHAPE, "GGML_OP_RESHAPE" }, - {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM" }, - {GGML_OP_NORM, "GGML_OP_NORM" }, - {GGML_OP_ROPE, "GGML_OP_ROPE" }, - {GGML_OP_SCALE, "GGML_OP_SCALE" }, - {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX" }, - {GGML_OP_SUM_ROWS, "GGML_OP_SUM_ROWS" }, - {GGML_OP_SUB, "GGML_OP_SUB" }, - {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE" }, - {GGML_OP_VIEW, "GGML_OP_VIEW" }, - {GGML_OP_SET_ROWS, "GGML_OP_SET_ROWS" }, - {GGML_OP_CPY, "GGML_OP_CPY" }, - {GGML_OP_FLASH_ATTN_EXT, "GGML_OP_FLASH_ATTN_EXT" }, - {GGML_OP_L2_NORM, "GGML_OP_L2_NORM" }, - {GGML_OP_CLAMP, "GGML_OP_CLAMP" }, - {GGML_OP_PAD, "GGML_OP_PAD" }, - {GGML_OP_SSM_CONV, "GGML_OP_SSM_CONV" }, - {GGML_OP_GATED_DELTA_NET, "GGML_OP_GATED_DELTA_NET"}, - {GGML_OP_ARGSORT, "GGML_OP_ARGSORT" }, - {GGML_OP_REPEAT, "GGML_OP_REPEAT" }, - {GGML_OP_IM2COL, "GGML_OP_IM2COL" } - }; - static const std::map unary_ops = { - {GGML_UNARY_OP_ABS, "GGML_UNARY_OP_ABS" }, - {GGML_UNARY_OP_SGN, "GGML_UNARY_OP_SGN" }, - {GGML_UNARY_OP_NEG, "GGML_UNARY_OP_NEG" }, - {GGML_UNARY_OP_STEP, "GGML_UNARY_OP_STEP" }, - {GGML_UNARY_OP_TANH, "GGML_UNARY_OP_TANH" }, - {GGML_UNARY_OP_ELU, "GGML_UNARY_OP_ELU" }, - {GGML_UNARY_OP_RELU, "GGML_UNARY_OP_RELU" }, - {GGML_UNARY_OP_SIGMOID, "GGML_UNARY_OP_SIGMOID" }, - {GGML_UNARY_OP_GELU, "GGML_UNARY_OP_GELU" }, - {GGML_UNARY_OP_GELU_QUICK, "GGML_UNARY_OP_GELU_QUICK" }, - {GGML_UNARY_OP_SILU, "GGML_UNARY_OP_SILU" }, - {GGML_UNARY_OP_SOFTPLUS, "GGML_UNARY_OP_SOFTPLUS" }, - {GGML_UNARY_OP_HARDSWISH, "GGML_UNARY_OP_HARDSWISH" }, - {GGML_UNARY_OP_HARDSIGMOID, "GGML_UNARY_OP_HARDSIGMOID"}, - {GGML_UNARY_OP_EXP, "GGML_UNARY_OP_EXP" }, - {GGML_UNARY_OP_COUNT, "GGML_UNARY_OP_COUNT" } - }; - static const std::map glu_ops = { - {GGML_GLU_OP_SWIGLU, "GGML_GLU_OP_SWIGLU"}, - {GGML_GLU_OP_GEGLU, "GGML_GLU_OP_GEGLU" }, - {GGML_GLU_OP_REGLU, "GGML_GLU_OP_REGLU" } - }; - switch (node->op) { case GGML_OP_UNARY: - return unary_ops.at(ggml_get_unary_op(node)); + return std::string("GGML_UNARY_OP_") + ggml_unary_op_name(ggml_get_unary_op(node)); case GGML_OP_GLU: - return glu_ops.at(ggml_get_glu_op(node)); + return std::string("GGML_GLU_OP_") + ggml_glu_op_name(ggml_get_glu_op(node)); default: - return ops.at(node->op); + return std::string("GGML_OP_") + ggml_op_name(node->op); } - static const std::string unknown_op = "UNKNOWN_GGML_OP"; - return unknown_op; } const std::string & GgmlOvDecoder::get_op_type(int node_idx) const { From 96951e1711644b766d79e892723e8832d18cdb52 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Mon, 15 Jun 2026 16:24:30 +0800 Subject: [PATCH 04/20] OpenVINO backend: enable softmax with sink input --- .../src/ggml-openvino/openvino/op/softmax.cpp | 70 +++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index 287faedbb531..b391d3f91075 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -6,12 +6,16 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include +#include #include #include @@ -20,12 +24,31 @@ namespace frontend { namespace ggml { namespace op { +static bool is_static_one(const ov::Dimension & dim) { + return dim.is_static() && dim.get_length() == 1; +} + +static bool same_static_dim(const ov::Dimension & lhs, const ov::Dimension & rhs) { + return lhs.is_static() && rhs.is_static() && lhs.get_length() == rhs.get_length(); +} + +static bool is_attention_sinks_input_shape(const ov::PartialShape & candidate, const ov::PartialShape & logits_shape) { + if (candidate.rank().is_dynamic() || logits_shape.rank().is_dynamic() || candidate.rank().get_length() != 4 || + logits_shape.rank().get_length() != 4) { + return false; + } + + return is_static_one(candidate[0]) && is_static_one(candidate[1]) && is_static_one(candidate[2]) && + same_static_dim(candidate[3], logits_shape[1]); +} + // Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend: // 1) logits = src0 * scale // 2) logits += mask (if provided) -// 3) softmax over the last dimension +// 3) append attention sinks as hidden logits (if provided) +// 4) softmax over the last dimension and remove the hidden sink column OutputVector translate_soft_max(const NodeContext & context) { - num_inputs_check(context, 1, 2); + num_inputs_check(context, 1, 3); float scale = 1.0f; float max_bias = 0.0f; @@ -33,6 +56,11 @@ OutputVector translate_soft_max(const NodeContext & context) { memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float)); ov::Output logits = context.get_input(0); + const bool second_input_is_sinks = + context.get_input_size() == 2 && is_attention_sinks_input_shape(context.get_input_shape(1), context.get_output_shape()); + const bool has_mask = context.get_input_size() > 1 && !second_input_is_sinks; + const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2; + const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2; // Apply scale first: logits = src0 * scale if (scale != 1.0f) { @@ -41,12 +69,12 @@ OutputVector translate_soft_max(const NodeContext & context) { logits = std::make_shared(logits, scale_const); } - FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && context.get_input_size() < 2), + FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && !has_mask), "OpenVINO softmax ALiBi path requires mask input"); // Optional mask add: logits += mask // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. - if (context.get_input_size() > 1) { + if (has_mask) { ov::Output mask = context.get_input(1); // For stateful @@ -94,8 +122,40 @@ OutputVector translate_soft_max(const NodeContext & context) { logits = std::make_shared(logits, mask); } + ov::Output softmax_input = logits; + if (has_sinks) { + ov::Output sinks = context.get_input(sinks_input_idx); + if (sinks.get_element_type() != logits.get_element_type()) { + sinks = std::make_shared(sinks, logits.get_element_type()); + } + + auto sink_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, {1, -1, 1, 1}); + auto sinks_4d = std::make_shared(sinks, sink_shape, false); + + auto logits_shape = std::make_shared(logits, ov::element::i64); + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto four = ov::op::v0::Constant::create(ov::element::i64, {1}, {4}); + auto shape_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + + auto sink_prefix_shape = std::make_shared(logits_shape, zero, three, one, shape_axis); + auto sink_last_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto sink_broadcast_shape = std::make_shared( + ov::OutputVector{sink_prefix_shape, sink_last_dim}, 0); + auto sink_column = std::make_shared(sinks_4d, sink_broadcast_shape, + ov::op::BroadcastType::BIDIRECTIONAL); + softmax_input = std::make_shared(ov::OutputVector{logits, sink_column}, 3); + + auto softmax_with_sink = std::make_shared(softmax_input, -1); + auto original_last_dim = std::make_shared(logits_shape, three, four, one, shape_axis); + auto res = std::make_shared(softmax_with_sink, zero, original_last_dim, one, three); + + return rename_outputs_with_suffix({res}, context.get_name()); + } + // Softmax along last dimension (equivalent to ggml softmax over ne[0]). - auto res = std::make_shared(logits, -1); + auto res = std::make_shared(softmax_input, -1); return rename_outputs_with_suffix({res}, context.get_name()); } From 34edec6bf854b2a3833caa301551792804d28c04 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Tue, 23 Jun 2026 16:35:49 +0800 Subject: [PATCH 05/20] OpenVINO backend: opt mul_mat_id convert process for large size --- .../ggml-openvino/openvino/op/mul_mat_id.cpp | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index 09e29d4cce2a..6df2784c2e45 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -2,23 +2,135 @@ #include "../op_table.h" #include "../utils.h" +#include +#include +#include #include +#include +#include #include #include #include #include #include #include +#include #include #include -#include +#include #include +#include namespace ov { namespace frontend { namespace ggml { namespace op { +namespace { + +std::shared_ptr const_i64(const std::vector & values) { + return ov::op::v0::Constant::create(ov::element::i64, ov::Shape{values.size()}, values); +} + +ov::Output slice_axis(const ov::Output & input, int64_t axis, int64_t begin, int64_t end) { + return std::make_shared(input, const_i64({begin}), const_i64({end}), const_i64({1}), + const_i64({axis})); +} + +ov::Output translate_mul_mat_id_mxfp4_packed(const NodeContext & context, + ov::Output expert_weights, + ov::Output activations, + ov::Output ids) { + auto packed_shape = expert_weights.get_partial_shape().to_shape(); + FRONT_END_OP_CONVERSION_CHECK(packed_shape.size() == 5 && packed_shape[4] == 17, + "Expected packed MXFP4 expert weights with shape [1, n_expert, m, k_blocks, 17]"); + + const int64_t n_expert = static_cast(packed_shape[1]); + const int64_t rows = static_cast(packed_shape[2]); + const int64_t k_blocks = static_cast(packed_shape[3]); + const int64_t qk = 32; + const int64_t cols = k_blocks * qk; + + auto packed_shape_4d = const_i64({n_expert, rows, k_blocks, 17}); + expert_weights = std::make_shared(expert_weights, packed_shape_4d, false); + + auto activations_shape_4d = std::make_shared(activations, ov::element::i64); + auto ids_shape_4d = std::make_shared(ids, ov::element::i64); + auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); + auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); + + activations = std::make_shared(activations, activations_shape_3d, false); + ids = std::make_shared(ids, ids_shape_2d, false); + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); + + static const std::vector f4e2m1_lut = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f}; + std::vector e8m0_lut(256); + for (size_t i = 0; i < e8m0_lut.size(); ++i) { + uint32_t bits = static_cast(i) << 23; + memcpy(&e8m0_lut[i], &bits, sizeof(float)); + } + e8m0_lut[0] = std::numeric_limits::min() / 2.0f; + e8m0_lut[255] = std::numeric_limits::quiet_NaN(); + + auto f4_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{f4e2m1_lut.size()}, f4e2m1_lut); + auto scale_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{e8m0_lut.size()}, e8m0_lut); + + auto selected_packed_weights = std::make_shared(expert_weights, ids, gather_axis); + auto scale_byte = slice_axis(selected_packed_weights, 4, 0, 1); + auto qs = slice_axis(selected_packed_weights, 4, 1, 17); + auto low = std::make_shared( + qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {0x0F}), ov::op::AutoBroadcastType::NUMPY); + auto high_shift = std::make_shared( + qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {4}), ov::op::AutoBroadcastType::NUMPY); + auto nibbles = std::make_shared(ov::OutputVector{low, high_shift}, 4); + auto nibble_indices = std::make_shared(nibbles, ov::element::i32); + auto weights_f32 = std::make_shared(f4_lut, nibble_indices, gather_axis); + + auto scale_indices = std::make_shared(scale_byte, ov::element::i32); + auto scales_f32 = std::make_shared(scale_lut, scale_indices, gather_axis); + ov::Output selected_weights = std::make_shared(weights_f32, scales_f32, + ov::op::AutoBroadcastType::NUMPY); + + auto ids_shape = std::make_shared(ids, ov::element::i64); + auto selected_weights_target_dims = std::make_shared( + ov::OutputVector{get_dimensions(ids_shape, {0, 1}), const_i64({rows, cols})}, 0); + selected_weights = std::make_shared(selected_weights, selected_weights_target_dims, false); + + auto activations_shape = std::make_shared(activations, ov::element::i64); + ov::Output acts_target_dims = std::make_shared( + ov::OutputVector{ + get_dimensions(activations_shape, {0}), + get_dimensions(ids_shape, {1}), + get_dimensions(activations_shape, {2}), + }, + 0); + ov::Output acts_broadcasted = + std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); + + auto activations_expanded = std::make_shared(acts_broadcasted, const_i64({2})); + ov::Output result = + std::make_shared(activations_expanded, selected_weights, false, true); + + auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {rows}); + auto result_target_dims = std::make_shared( + ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0); + result = std::make_shared(result, result_target_dims, false); + + const auto output_type = context.get_output_type(); + if (result.get_element_type() != output_type) { + result = std::make_shared(result, output_type); + } + return result; +} + +} // namespace + OutputVector translate_mul_mat_id(const NodeContext & context) { num_inputs_check(context, 3, 3); @@ -26,6 +138,12 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { auto activations = process_view_input_new(context, 1); auto ids = process_view_input_new(context, 2); + if (expert_weights.get_element_type() == ov::element::u8 && expert_weights.get_partial_shape().rank().is_static() && + expert_weights.get_partial_shape().rank().get_length() == 5) { + return rename_outputs_with_suffix({translate_mul_mat_id_mxfp4_packed(context, expert_weights, activations, ids)}, + context.get_name()); + } + // OpenVINO sees GGML tensors in reversed dimension order: // weights: [1, n_expert, m, k] // activations: [1, n_tokens, n_used_or_1, k] From 61fd9749bde90c3e88f8c46ce0783dabda7897df Mon Sep 17 00:00:00 2001 From: Xuejun Date: Tue, 23 Jun 2026 16:32:42 +0800 Subject: [PATCH 06/20] OpenVINO backend: Modify add_id to support 2D/4D --- ggml/src/ggml-openvino/openvino/op/add_id.cpp | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp index c8bf08152242..e54d700d421a 100644 --- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -17,6 +17,22 @@ namespace frontend { namespace ggml { namespace op { +static ov::Output reshape_add_id_input_to_2d(const ov::Output & input, + const ov::PartialShape & input_shape, + const std::vector & dims) { + const auto actual_shape = input.get_partial_shape(); + if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) { + return input; + } + + if (input_shape.rank().is_static() && input_shape.rank().get_length() == 2) { + return input; + } + + auto shape = std::make_shared(input, ov::element::i64); + return std::make_shared(input, get_dimensions(shape, dims), false); +} + OutputVector translate_add_id(const NodeContext & context) { num_inputs_check(context, 3, 3); @@ -28,11 +44,9 @@ OutputVector translate_add_id(const NodeContext & context) { // input: [1, n_token, n_used, n_embd] // bias: [1, 1, n_expert, n_embd] // ids: [1, 1, n_token, n_used] - auto bias_shape_4d = std::make_shared(bias, ov::element::i64); - auto ids_shape_4d = std::make_shared(ids, ov::element::i64); - - bias = std::make_shared(bias, get_dimensions(bias_shape_4d, {2, 3}), false); - ids = std::make_shared(ids, get_dimensions(ids_shape_4d, {2, 3}), false); + // Model bias constants may already be stored as [n_expert, n_embd]. + bias = reshape_add_id_input_to_2d(bias, context.get_input_shape(1), {2, 3}); + ids = reshape_add_id_input_to_2d(ids, context.get_input_shape(2), {2, 3}); if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { ids = std::make_shared(ids, ov::element::i32); From a4a7e3622f05e9c7e5a4f4eca4cddec9b7d20652 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Tue, 23 Jun 2026 16:28:05 +0800 Subject: [PATCH 07/20] OpenVINO Backend: Add glu_swiglu_oai --- .../ggml-openvino/openvino/op/glu_swiglu.cpp | 32 ++++++++++++++++++- ggml/src/ggml-openvino/openvino/op_table.cpp | 1 + ggml/src/ggml-openvino/openvino/op_table.h | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index 5c46e071375e..d220f2f584a5 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -3,8 +3,11 @@ #include "../utils.h" #include +#include #include #include +#include +#include #include #include #include @@ -15,7 +18,7 @@ namespace frontend { namespace ggml { namespace op { -OutputVector translate_glu_swiglu(const NodeContext & context) { +static std::pair, ov::Output> get_glu_inputs(const NodeContext & context) { num_inputs_check(context, 1, 2); ov::Output src0; @@ -52,6 +55,12 @@ OutputVector translate_glu_swiglu(const NodeContext & context) { std::swap(src0, src1); } + return {src0, src1}; +} + +OutputVector translate_glu_swiglu(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + auto sigmoid = std::make_shared(src0); auto silu = std::make_shared(src0, sigmoid); auto res = std::make_shared(silu, src1); @@ -59,6 +68,27 @@ OutputVector translate_glu_swiglu(const NodeContext & context) { return rename_outputs_with_suffix({res}, context.get_name()); } +OutputVector translate_glu_swiglu_oai(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + + const int32_t * params = context.get_output_op_params(); + const float alpha = reinterpret_cast(params)[2]; + const float limit = reinterpret_cast(params)[3]; + + auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); + auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); + auto scaled_gate = std::make_shared(gate, alpha_const); + auto sigmoid = std::make_shared(scaled_gate); + auto out_glu = std::make_shared(gate, sigmoid); + + auto up = std::make_shared(src1, -limit, limit); + auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); + auto up_plus_one = std::make_shared(up, one); + auto res = std::make_shared(out_glu, up_plus_one); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + } // namespace op } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index f84a1bf931ae..59fd26df8cd5 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -47,6 +47,7 @@ std::unordered_map get_supported_ops() { {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, {"GGML_OP_VIEW", op::translate_view }, {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, + {"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai }, {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, {"GGML_OP_SET_ROWS", op::translate_set_rows }, {"GGML_OP_CPY", op::translate_cpy }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index c90ff8377908..1d695fa12588 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -32,6 +32,7 @@ GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); GGML_OP_CONVERTER(translate_view); GGML_OP_CONVERTER(translate_glu_swiglu); +GGML_OP_CONVERTER(translate_glu_swiglu_oai); GGML_OP_CONVERTER(translate_glu_geglu); GGML_OP_CONVERTER(translate_set_rows); GGML_OP_CONVERTER(translate_cpy); From 3064c2b18dcb0790eaa0773f71d57ab1dfef6b1f Mon Sep 17 00:00:00 2001 From: ravi9 Date: Wed, 24 Jun 2026 20:10:03 +0530 Subject: [PATCH 08/20] PR review: fix paths --- .github/actions/linux-setup-openvino/action.yml | 4 ++-- .github/workflows/release.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/linux-setup-openvino/action.yml b/.github/actions/linux-setup-openvino/action.yml index fe3006ac8764..46a659a827c4 100644 --- a/.github/actions/linux-setup-openvino/action.yml +++ b/.github/actions/linux-setup-openvino/action.yml @@ -5,10 +5,10 @@ inputs: description: "Installation path" required: true version_major: - description: "OpenVINO major version (e.g., 2026.2.1)" + description: "OpenVINO major version (e.g., 2025.3)" required: true version_full: - description: "OpenVINO full version (e.g., 2026.2.1.21919.ede283a88e3)" + description: "OpenVINO full version (e.g., 2025.3.0.19807.44526285f24)" required: true runs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15ec339815d2..8524262462b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -524,7 +524,7 @@ jobs: - name: Pack artifacts id: pack_artifacts run: | - dest=./build/ReleaseOV/bin + dest=./build/ReleaseOV/bin/ OPENVINO_ROOT=./openvino_toolkit ov_lib="$OPENVINO_ROOT/runtime/lib/intel64" @@ -535,7 +535,7 @@ jobs: "$ov_lib"/libopenvino_*_plugin.so \ "$ov_lib"/libopenvino_intel_npu_compiler*.so \ "$OPENVINO_ROOT/runtime/3rdparty/tbb/lib"/*.so* \ - "$dest"/ + "$dest" cp -P /usr/lib/x86_64-linux-gnu/libOpenCL.so.1* "$dest" 2>/dev/null || true cp "$ov_lib/cache.json" "$dest" 2>/dev/null || true @@ -669,7 +669,7 @@ jobs: Copy-Item -Path (Join-Path $OPENVINO_ROOT 'docs\licensing\*') -Destination $licensingDest -Recurse -Force Copy-Item LICENSE $dest - 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip .\build\ReleaseOV\bin\Release\* + 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip $dest\* - name: Upload artifacts uses: actions/upload-artifact@v6 From d61dbf23ae74aefd0f7873f27fe75108bdda34f9 Mon Sep 17 00:00:00 2001 From: ravi9 Date: Thu, 25 Jun 2026 01:08:17 +0530 Subject: [PATCH 09/20] PR review: fix path consistency --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8524262462b1..eb7e1f20d468 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -524,7 +524,7 @@ jobs: - name: Pack artifacts id: pack_artifacts run: | - dest=./build/ReleaseOV/bin/ + dest=./build/ReleaseOV/bin OPENVINO_ROOT=./openvino_toolkit ov_lib="$OPENVINO_ROOT/runtime/lib/intel64" @@ -534,13 +534,13 @@ jobs: "$ov_lib"/libopenvino_c.so* \ "$ov_lib"/libopenvino_*_plugin.so \ "$ov_lib"/libopenvino_intel_npu_compiler*.so \ - "$OPENVINO_ROOT/runtime/3rdparty/tbb/lib"/*.so* \ + "$OPENVINO_ROOT"/runtime/3rdparty/tbb/lib/*.so* \ "$dest" cp -P /usr/lib/x86_64-linux-gnu/libOpenCL.so.1* "$dest" 2>/dev/null || true - cp "$ov_lib/cache.json" "$dest" 2>/dev/null || true + cp "$ov_lib"/cache.json "$dest" 2>/dev/null || true # OpenVINO licensing - cp -r "$OPENVINO_ROOT/docs/licensing" "$dest/openvino-licensing" + cp -r "$OPENVINO_ROOT"/docs/licensing "$dest"/openvino-licensing cp LICENSE "$dest" tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C "$dest" . From ac7d478efee8c164e2b048464baec4b4cd2d3299 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Tue, 23 Jun 2026 16:36:48 +0800 Subject: [PATCH 10/20] OpenVINO backend: 1) enable gpt-oss moe on OV bk; 2) enable mxfp4 support --- ggml/src/ggml-openvino/ggml-decoder.cpp | 2 +- .../src/ggml-openvino/ggml-openvino-extra.cpp | 23 +++- ggml/src/ggml-openvino/ggml-openvino.cpp | 80 ++--------- ggml/src/ggml-openvino/ggml-quants.cpp | 124 +++++++++++++++++- ggml/src/ggml-openvino/ggml-quants.h | 7 + 5 files changed, 160 insertions(+), 76 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 48c63e4d70fa..faa857944054 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -834,7 +834,7 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name); static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K, - GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}; + GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4}; if (weight_types.find(tensor->type) == weight_types.end()) { throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " + ggml_type_name(tensor->type)); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index d9ad7be734d1..18df24c77e64 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -252,14 +252,24 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // Only handle 2D weight tensors - if (tensor->ne[2] != 1 || tensor->ne[3] != 1) { + // Most quantized weights use the existing 2D extraction path. MXFP4 also + // appears as 3D expert weights for MUL_MAT_ID, so allow that type through. + if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) { return layout; } int64_t n_elements = ggml_nelements(tensor); const size_t alignment = 64; // Good for SIMD + if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) { + layout.weights_per_block = 32; + layout.is_symmetric = true; + layout.weights_size = ggml_nbytes(tensor); + layout.weights_offset = 0; + layout.total_size = layout.weights_size; + return layout; + } + // Check if requantization is needed (NPU-specific) auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias); if (requant_type.has_value()) { @@ -334,6 +344,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.is_symmetric = false; switch (tensor->type) { + case GGML_TYPE_MXFP4: + layout.is_u4 = true; + layout.is_symmetric = true; + break; + case GGML_TYPE_Q4_0: layout.is_u4 = true; layout.is_symmetric = true; @@ -369,9 +384,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements; - // Scales: F16 per block + // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block. int64_t n_blocks = n_elements / layout.weights_per_block; - layout.scales_size = n_blocks * sizeof(uint16_t); // F16 = 2 bytes + layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t)); // For symmetric quantization, no zp needed (weights stored as signed) if (layout.is_symmetric) { layout.zp_size = 0; diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 943aef864535..e1fcf2493bbe 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -237,8 +237,9 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr); // 2D tensor (typical weight shape) bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1); + bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4; - if (is_weight_buffer && is_full_tensor_set && is_2d) { + if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) { try { auto result = process_weight_tensor(tensor, data, tensor->data); result.weight_node->set_friendly_name(tensor->name); @@ -458,8 +459,9 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff const ggml_tensor * tensor) { GGML_UNUSED(buft); - // For quantized 2D tensors (weights), we need extra space for extracted data - if (ggml_is_quantized(tensor->type) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + // For quantized weight tensors, we need extra space for extracted data. + if (ggml_is_quantized(tensor->type) && + ((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) { ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor); if (layout.total_size > 0) { // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n", @@ -901,17 +903,10 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { return true; } - // Keep the MoE routing weights gather on CPU for GPU runs. Splitting - // only at the later SUM/CLAMP/DIV nodes still leaves this routing path - // numerically unstable for arctic-style MoE graphs. - if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { - return true; - } break; } case GGML_OP_RESHAPE: { - if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 || - strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { + if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { return true; } break; @@ -958,49 +953,15 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { return true; } - // qwen3next MoE weight normalization is numerically sensitive on the GPU - // path. Keep the normalization divide on CPU to match the reference. - if (strncmp(op->name, "ffn_moe_weights_norm", sizeof("ffn_moe_weights_norm") - 1) == 0) { - return true; - } - break; - } - case GGML_OP_SOFT_MAX: { - if (op->src[2] != nullptr) { - // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n"); - return true; - } - - if (strncmp(op->name, "ffn_moe_probs", sizeof("ffn_moe_probs") - 1) == 0) { - return true; - } - - // GPU execution of the MoE routing weights softmax is numerically unstable - // when fused with the surrounding GET_ROWS/reshape path. Keep this softmax - // on CPU so the scheduler splits at the same boundary that restores parity. - if (op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && op->src[0]->src[0] != nullptr && - strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { - return true; - } break; } case GGML_OP_SUM_ROWS: { - if (strncmp(op->name, "ffn_moe_weights_sum", sizeof("ffn_moe_weights_sum") - 1) == 0) { - return true; - } - // if the input is PERMUTE skip if (op->src[0]->op == GGML_OP_PERMUTE) { return true; } break; } - case GGML_OP_CLAMP: { - if (strncmp(op->name, "ffn_moe_weights_sum_clamped", sizeof("ffn_moe_weights_sum_clamped") - 1) == 0) { - return true; - } - break; - } case GGML_OP_FLASH_ATTN_EXT: { float scale = 1.0f; float max_bias = 0.0f; @@ -1056,12 +1017,6 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT: { - if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX && - op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr && - op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr && - op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) { - return true; - } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return true; } @@ -1071,12 +1026,8 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT_ID: { - if (strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 || - strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0) { - return true; - } - - if (mul_mat_id_requires_large_tmp(op)) { + if (mul_mat_id_requires_large_tmp(op) && + !(op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4)) { return true; } break; @@ -1153,14 +1104,6 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { // Keep this op on CPU until the OpenVINO implementation is fixed. return true; } - case GGML_OP_VIEW: { - // Skip TOPK_MOE fused tests until it is fully supported - // the argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe - if (strcmp(op->name, "selected_experts") == 0) { - return true; - } - break; - } default: break; } @@ -1172,7 +1115,8 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con static std::unordered_set supported_types{ GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, - GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; + GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, + GGML_TYPE_MXFP4}; // derive supported op sets from the op_table map, keys in // the map use the full macro name (e.g. "GGML_OP_ADD"), while @@ -1270,7 +1214,9 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type)); return false; } - if (ggml_is_quantized(src->type) && src->ne[2] != 1) { + const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 && + src->type == GGML_TYPE_MXFP4; + if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) { // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n"); return false; } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 275b95428273..d4e4d8f660b0 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -18,7 +18,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -44,6 +46,38 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) { } } +static constexpr size_t MXFP4_BLOCK_SIZE = 32; +static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; +static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; + +static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { + for (int j = 0; j < static_cast(MXFP4_BLOCK_QS_SIZE); j += 2) { + const uint8_t v0 = data[j] & 0x0F; + const uint8_t v1 = (data[j + 1] & 0x0F) << 4; + const uint8_t v16 = data[j] >> 4; + const uint8_t v17 = data[j + 1] & 0xF0; + dst[j / 2] = v0 | v1; + dst[MXFP4_BLOCK_SIZE / 4 + j / 2] = v16 | v17; + } +} + +void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) { + GGML_ASSERT(tensor->type == GGML_TYPE_MXFP4); + GGML_ASSERT(weights_arr.get_element_type() == ov::element::f4e2m1); + GGML_ASSERT(scales_arr.get_element_type() == ov::element::f8e8m0); + + const auto * data = static_cast(tensor->data); + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + const size_t n_blocks = scales_arr.get_size(); + + ov::parallel_for(n_blocks, [&](size_t i) { + const uint8_t * block = data + i * MXFP4_BLOCK_BYTES; + pack_32_mxfp4_for_openvino(block + sizeof(uint8_t), weights + i * MXFP4_BLOCK_QS_SIZE); + scales[i] = ov::float8_e8m0::from_bits(block[0]); + }); +} + // Extracts (weight, scales, zp) from Q4_0 tensors. // Data layout is: |16 bit scale|32 x 4bit weights|. // When zp_arr is empty (symmetric), weights are stored as signed i4 (value - 8). @@ -617,6 +651,42 @@ ov::Output make_int4_weights(ov::Tensor & weight, return std::make_shared(result, ov::element::f32); } +ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales) { + const ov::Shape final_shape = weight.get_shape(); + GGML_ASSERT(!final_shape.empty()); + GGML_ASSERT(final_shape.back() % MXFP4_BLOCK_SIZE == 0); + + ov::Shape packed_shape = final_shape; + packed_shape.back() /= MXFP4_BLOCK_SIZE; + packed_shape.push_back(MXFP4_BLOCK_SIZE); + + ov::Shape scale_shape = packed_shape; + scale_shape.back() = 1; + scales.set_shape(scale_shape); + + auto weights_node = std::make_shared(ov::element::f4e2m1, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f32 = std::make_shared(weights_node, ov::element::f32); + + auto scales_node = std::make_shared(scales); + auto scales_f32 = std::make_shared(scales_node, ov::element::f32); + ov::Output result = + std::make_shared(weights_f32, scales_f32, ov::op::AutoBroadcastType::NUMPY); + + auto final_shape_node = + std::make_shared(ov::element::i64, ov::Shape{final_shape.size()}, final_shape); + return std::make_shared(result, final_shape_node, false); +} + +ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight) { + auto weights_node = std::make_shared(ov::element::u8, weight.get_shape(), + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + weights_node->get_rt_info()["__ggml_openvino_mxfp4_moe_packed"] = true; + return weights_node; +} + // Extract quantized weights from tensor and create weight subgraph std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, const void * data, @@ -628,6 +698,13 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, ggml_tensor temp_tensor = *tensor; temp_tensor.data = const_cast(data); + if (tensor->type == GGML_TYPE_MXFP4) { + extract_mxfp4_data(&temp_tensor, weights, scales); + auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr(); + result->set_friendly_name(tensor->name); + return result; + } + // Determine block size based on tensor type int64_t weights_per_block; bool is_u4; @@ -788,6 +865,27 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type)); } + const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1); + if (is_3d_mxfp4_moe) { + ov::Shape packed_shape = {static_cast(tensor->ne[3]), + static_cast(tensor->ne[2]), + static_cast(tensor->ne[1]), + static_cast(tensor->ne[0] / MXFP4_BLOCK_SIZE), + MXFP4_BLOCK_BYTES}; + const size_t tensor_bytes = ggml_nbytes(tensor); + if (output_base_ptr) { + auto * buf_base = static_cast(output_base_ptr); + memcpy(buf_base + layout.weights_offset, data, tensor_bytes); + result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset); + } else { + result.weights = ov::Tensor(ov::element::u8, packed_shape); + memcpy(result.weights.data(), data, tensor_bytes); + } + result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr(); + result.weight_node->set_friendly_name(tensor->name); + return result; + } + if (use_bias) { OPENVINO_ASSERT(!layout.is_requant, "use_bias is only used for test-backend-ops, which should not have requantization"); @@ -812,14 +910,31 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // Quantized path (normal extraction or quantized requant) // Create weight/scale/zp tensors - shared between both paths // For symmetric quantization, use signed types (i4/i8) and no ZP tensor - ov::element::Type weight_type = layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : - (layout.is_u4 ? ov::element::u4 : ov::element::u8); + ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ? + ov::element::f4e2m1 : + (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : + (layout.is_u4 ? ov::element::u4 : ov::element::u8)); ov::Shape scale_shape = {node_shape[0], node_shape[1] / layout.weights_per_block}; + if (tensor->type == GGML_TYPE_MXFP4) { + if (tensor->ne[2] == 1 && tensor->ne[3] == 1) { + node_shape = {static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; + } else { + node_shape.clear(); + for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) { + node_shape.push_back(static_cast(tensor->ne[i])); + } + } + + scale_shape = node_shape; + scale_shape.back() /= layout.weights_per_block; + } + if (output_base_ptr) { uint8_t * buf_base = static_cast(output_base_ptr); result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset); - result.scales = ov::Tensor(ov::element::f16, scale_shape, buf_base + layout.scales_offset); + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); if (!layout.is_symmetric) { ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); @@ -827,7 +942,8 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // else: result.zp remains default-constructed (empty) for symmetric } else { result.weights = ov::Tensor(weight_type, node_shape); - result.scales = ov::Tensor(ov::element::f16, scale_shape); + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + result.scales = ov::Tensor(scale_type, scale_shape); if (!layout.is_symmetric) { if (use_bias) { result.zp = ov::Tensor(ov::element::f16, scale_shape); diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index 28b7c1213be2..1b89fd887e16 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -4,6 +4,7 @@ #include #include +#include #include void unpack_32_4(const uint8_t * data, uint8_t * dst); @@ -49,6 +50,8 @@ void extract_q6_k_data(const ggml_tensor * tensor, ov::Tensor & scales_arr, ov::Tensor & zp_arr); +void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr); + static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32; ov::Output make_int8_weights(ov::Tensor & weight, @@ -63,6 +66,10 @@ ov::Output make_int4_weights(ov::Tensor & weight, size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, bool use_bias = false); +ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales); + +ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight); + // Extract quantized weights from tensor and create weight subgraph // If weights/scales/zp are provided (non-empty), uses them as output buffers // Otherwise allocates new ov::Tensors internally From 66ddab584382fbebc3acc4bc8b21440e0e03e3f4 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Wed, 24 Jun 2026 16:54:08 +0800 Subject: [PATCH 11/20] OpenVINO backend: disable TOPK_MOE op test --- ggml/src/ggml-openvino/ggml-openvino.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index e1fcf2493bbe..5e14fc0f4d3b 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1104,6 +1104,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { // Keep this op on CPU until the OpenVINO implementation is fixed. return true; } + case GGML_OP_VIEW: { + // Skip TOPK_MOE fused tests until it is fully supported. + // The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe. + if (strcmp(op->name, "selected_experts") == 0) { + return true; + } + break; + } default: break; } From 34b0e98ad30661f106050e5c396860ac462a1f70 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Wed, 27 May 2026 14:06:43 +0800 Subject: [PATCH 12/20] OpenVINO backend: 1) ensure unique node names for OpenVINO; 2) add org_src to recorde the src ggml tensor for OpenVINO dynamic shape infer --- ggml/include/ggml.h | 4 +++- ggml/src/ggml-backend.cpp | 23 +++++++++++++++++++++++ ggml/src/ggml.c | 1 + tests/test-llama-archs.cpp | 9 ++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d6807b6dd47a..184c3d59285d 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -695,7 +695,9 @@ extern "C" { void * extra; // extra things e.g. for ggml-cuda.cu - char padding[8]; + char padding[16]; + // add a struct ggml_tensor * named org_src, initialized to NULL, for keeping track of original source tensors in case of in-place operations + struct ggml_tensor * org_src; }; static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 87615921c09b..9a13f50f58a4 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1242,6 +1242,28 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra GGML_ASSERT(*cur_backend_id != -1); } + // OpenVINO currently uses ggml tensor names as graph indices. Some models (e.g. gpt-oss and + // llama4) can contain duplicate ggml tensor names, so we append node ids here to keep names + // unique. This is a temporary workaround and will be further optimized away in the future. + { + bool has_openvino_backend = false; + for (int i = 0; i < sched->n_backends; i++) { + if (strcmp(ggml_backend_name(sched->backends[i]), "OPENVINO") == 0) { + has_openvino_backend = true; + break; + } + } + + if (has_openvino_backend) { + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + char new_name[128]; + snprintf(new_name, sizeof(new_name), "%s#%d", node->name, i); + ggml_format_name(node, "%s", new_name); + } + } + } + // pass 5: split graph, find tensors that need to be copied { int i_split = 0; @@ -1360,6 +1382,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra ggml_set_input(tensor_copy); ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor } + tensor_copy->org_src = src; tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy; SET_CAUSE(tensor_copy, "4.cpy"); } diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 0f682fd1856c..5af88af449d3 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1781,6 +1781,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( /*.name =*/ { 0 }, /*.extra =*/ NULL, /*.padding =*/ { 0 }, + /*.org_src =*/ NULL, }; // TODO: this should not be needed as long as we don't rely on aligned SIMD loads diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 524971ae4b30..f086358e1318 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -509,6 +509,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg size_t max_device_label_length = 4; { std::vector devices_meta; + bool has_openvino = false; { const size_t device_count = ggml_backend_dev_count(); for (size_t i = 0; i < device_count; i++) { @@ -516,6 +517,10 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg dev_configs.emplace_back(std::vector{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER); max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length()); + if (strncmp(ggml_backend_dev_name(dev), "OPENVINO", 8) == 0) { + has_openvino = true; + } + // cpu-based devices cannot be used in tensor split mode if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) { devices_meta.push_back(dev); @@ -523,7 +528,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg } } - dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR); + if (!has_openvino) { + dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR); + } } size_t max_arch_name_length = 0; From e84bb748b9433d45539190fd729c6602e7098d2e Mon Sep 17 00:00:00 2001 From: Xuejun Date: Wed, 27 May 2026 14:07:31 +0800 Subject: [PATCH 13/20] OpenVINO backend: enable fallback for openVINO to CPU backend --- ggml/src/ggml-openvino/ggml-decoder.cpp | 10 +++++----- ggml/src/ggml-openvino/ggml-decoder.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index faa857944054..32f23963d327 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1307,10 +1307,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { if (src == nullptr) { continue; } - struct ggml_tensor * root_src = nullptr; - // if (src->org_src) { - // root_src = src->org_src; - // } + ggml_tensor * root_src = nullptr; + if (src->org_src) { + root_src = src->org_src; + } if (root_src) { if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) { m_node_dynamic_dims[root_src] = 0; @@ -1388,7 +1388,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { // identifies the dynamic dim even when two dims share the same size. m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[0]] != -1) { - if (node->src[0]->op == GGML_OP_NONE) { + if (node->src[0]->op == GGML_OP_NONE && node->src[0]->org_src == nullptr) { m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; break; } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index ae545f47e5fe..20db049e740e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -267,7 +267,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void update_io(ggml_cgraph * cgraph); inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; + return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE && op->src[0]->org_src == nullptr; } inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { From d3cba10022f778e94282c805f73a4676b6dac809 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Tue, 23 Jun 2026 16:40:12 +0800 Subject: [PATCH 14/20] print cgraph --- ggml/src/ggml-backend.cpp | 56 ++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 9a13f50f58a4..980a98400690 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -942,42 +942,61 @@ static char * fmt_size(size_t size) { return buffer; } +static void ggml_backend_debug_print_tensor_info(const struct ggml_tensor * tensor, const char * backend_name) { + printf("%20.20s [%5.5s] shape=[%lld,%lld,%lld,%lld] stride=[%zu,%zu,%zu,%zu] offset=%zu", + tensor->name, + backend_name ? backend_name : "NULL", + (long long) tensor->ne[0], (long long) tensor->ne[1], (long long) tensor->ne[2], (long long) tensor->ne[3], + tensor->nb[0], tensor->nb[1], tensor->nb[2], tensor->nb[3], + tensor->view_offs); +} + +static void ggml_backend_debug_print_tensor_srcs(const struct ggml_tensor * tensor, const char * backend_name) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + const struct ggml_tensor * src = tensor->src[i]; + if (src == NULL) { + continue; + } + printf(" "); + ggml_backend_debug_print_tensor_info(src, backend_name); + } +} + static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { int cur_split = 0; - for (int i = 0; i < graph->n_nodes; i++) { + for (int i = 0; i < 200; i++) { if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) { ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id]; - GGML_LOG_DEBUG("\n## SPLIT #%d: %s # %d inputs", cur_split, ggml_backend_name(split_backend), + printf("\n## SPLIT #%d: %s # %d inputs", cur_split, ggml_backend_name(split_backend), sched->splits[cur_split].n_inputs); for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) { if (j == 0) { - GGML_LOG_DEBUG(": "); + printf(": "); } - GGML_LOG_DEBUG("[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name, - fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j]))); + printf("[%s] ", sched->splits[cur_split].inputs[j]->name); + } - GGML_LOG_DEBUG("\n"); + printf("\n"); cur_split++; } struct ggml_tensor * node = graph->nodes[i]; - if (ggml_is_view_op(node->op)) { - continue; - } - if (sched->debug > 1) { - ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node); - GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_desc(node), node->name, - fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node), - graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0); + if (2 > 1) { + + + + printf("node #%3d (%10.10s): %20.20s shape=[%lld,%lld,%lld,%lld] offset=%zu:", i, ggml_op_name(node->op), node->name, + (long long) node->ne[0], (long long) node->ne[1], (long long) node->ne[2], (long long) node->ne[3], + node->view_offs); for (int j = 0; j < GGML_MAX_SRC; j++) { struct ggml_tensor * src = node->src[j]; if (src == NULL) { continue; } - ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src); - GGML_LOG_DEBUG(" %20.20s (%5.5s) [%5.5s %8.8s]", src->name, - fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src)); + printf(" %20.20s shape=[%lld,%lld,%lld,%lld] offset=%zu", src->name, + (long long) src->ne[0], (long long) src->ne[1], (long long) src->ne[2], (long long) src->ne[3], + src->view_offs); } - GGML_LOG_DEBUG("\n"); + printf("\n"); } } } @@ -1398,6 +1417,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra sched->n_splits = i_split + 1; } + // ggml_backend_sched_print_assignments(sched, graph); if (sched->debug) { ggml_backend_sched_print_assignments(sched, graph); } From aab952acd2f93bef4b05edc4e29ad8764c4bf434 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Thu, 25 Jun 2026 13:37:17 +0800 Subject: [PATCH 15/20] OpenVINO Backend: Add op FILL support --- ggml/src/ggml-openvino/openvino/op/fill.cpp | 47 ++++++++++++++++++++ ggml/src/ggml-openvino/openvino/op_table.cpp | 1 + ggml/src/ggml-openvino/openvino/op_table.h | 1 + 3 files changed, 49 insertions(+) create mode 100644 ggml/src/ggml-openvino/openvino/op/fill.cpp diff --git a/ggml/src/ggml-openvino/openvino/op/fill.cpp b/ggml/src/ggml-openvino/openvino/op/fill.cpp new file mode 100644 index 000000000000..1450b70be23d --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/fill.cpp @@ -0,0 +1,47 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_fill(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "FILL requires output op params"); + + float value; + std::memcpy(&value, op_params, sizeof(float)); + + auto scalar = ov::op::v0::Constant::create(context.get_output_type(), ov::Shape{}, {value}); + + ov::Output target_shape; + const auto output_shape = context.get_output_shape(); + if (output_shape.rank().is_static() && output_shape.is_static()) { + const auto static_shape = output_shape.to_shape(); + std::vector shape_values(static_shape.begin(), static_shape.end()); + target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape_values.size()}, shape_values); + } else { + auto input = process_view_input_new(context, 0); + target_shape = std::make_shared(input, ov::element::i64); + } + + auto res = std::make_shared(scalar, target_shape); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 59fd26df8cd5..47fa6f68dc9e 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -24,6 +24,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_CONCAT", op::translate_concat }, {"GGML_OP_CONT", op::translate_cont }, {"GGML_OP_DIV", op::translate_div }, + {"GGML_OP_FILL", op::translate_fill }, {"GGML_OP_GET_ROWS", op::translate_get_rows }, {"GGML_OP_IM2COL", op::translate_im2col }, {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index 1d695fa12588..921008482367 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -14,6 +14,7 @@ GGML_OP_CONVERTER(translate_cont); GGML_OP_CONVERTER(translate_concat); GGML_OP_CONVERTER(translate_add_id); GGML_OP_CONVERTER(translate_div); +GGML_OP_CONVERTER(translate_fill); GGML_OP_CONVERTER(translate_get_rows); GGML_OP_CONVERTER(translate_im2col); GGML_OP_CONVERTER(translate_mulmat); From c43c52130beec6b5bba357e2cc1a01c73b1b101e Mon Sep 17 00:00:00 2001 From: Xuejun Date: Thu, 25 Jun 2026 13:37:55 +0800 Subject: [PATCH 16/20] OpenVINO backend: enable set rows with multi dims --- .../ggml-openvino/openvino/op/set_rows.cpp | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 18643371e329..0fe8e0a8d067 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -29,20 +31,17 @@ OutputVector translate_set_rows(const NodeContext & context) { num_inputs_check(context, 3, 3); auto data = process_view_input_new(context, 0); - auto indices = context.get_input(1); - auto dst = context.get_input(2); + auto indices = process_view_input_new(context, 1); + auto dst = process_view_input_new(context, 2); data = std::make_shared(data, context.get_output_type()); - auto row_size = context.get_input_shape(2)[3].get_length(); + const auto indices_shape = context.get_input_shape(1); + const bool multidim_indices = indices_shape.rank().is_static() && + indices_shape.rank().get_length() == 4 && + ((indices_shape[1].is_static() && indices_shape[1].get_length() > 1) || + (indices_shape[2].is_static() && indices_shape[2].get_length() > 1)); - auto ind_squeezed = - std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); - auto data_reshaped = std::make_shared( - data, - ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), - false); auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); Output res; @@ -53,11 +52,31 @@ OutputVector translate_set_rows(const NodeContext & context) { data = std::make_shared( data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false); res = std::make_shared(OutputVector{dst, data}, concat_axis); + } else if (multidim_indices) { + auto updates_shape = std::make_shared(data, ov::element::i64); + + auto indices_rank3 = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto indices_rank4_shape = std::make_shared(OutputVector{get_dimensions(updates_shape, {0, 1, 2}), one}, 0); + auto indices_rank4 = std::make_shared(indices_rank3, indices_rank4_shape, false); + auto broadcasted_indices = std::make_shared(indices_rank4, updates_shape); + + res = std::make_shared(dst, broadcasted_indices, data, axes); } else { + auto row_size = context.get_input_shape(2)[3].get_length(); + auto ind_squeezed = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); + auto data_reshaped = std::make_shared( + data, + ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), + false); res = std::make_shared(dst, ind_squeezed, data_reshaped, axes); } - if (auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr())) { + auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr()); + if (!multidim_indices && dst_reshape) { // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb] // ctx_per_seq is not fixed due to llama-bench compatibility auto dst_shape_partial = dst_reshape->get_input_partial_shape(0); From a3e9df423fc406a07aed273e9dc1a943f64d699c Mon Sep 17 00:00:00 2001 From: Xuejun Date: Thu, 25 Jun 2026 13:38:22 +0800 Subject: [PATCH 17/20] fix the name missmatch in setrow + view --- ggml/src/ggml-openvino/ggml-decoder.cpp | 16 ++++++++-------- ggml/src/ggml-openvino/ggml-decoder.h | 2 ++ ggml/src/ggml-openvino/openvino/decoder.h | 2 ++ .../ggml-openvino/openvino/translate_session.cpp | 7 +++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 32f23963d327..0cd69eeaaa4e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -106,14 +106,6 @@ void GgmlOvDecoder::set_input_output() { auto node_name = std::string(node->name); auto node_output_name = node_name; auto * node_output = node; - if (node->op == GGML_OP_SET_ROWS) { - // SET_ROWS updates the tensor in place. For later ov op that uses the - // the view_src of SET_ROWS, we need to make sure they get the updated tensor - // by putting the view_src name in the tensor_map in - // /src/frontends/ggml/src/translate_session.cpp - node_output_name = std::string(node->view_src->name); - node_output = node->view_src; - } current_node_info.node = node; current_node_info.node_name = node_name; @@ -1231,6 +1223,14 @@ std::vector GgmlOvDecoder::get_output_names(int node_idx) const { return {m_node_info_list[node_idx].node_output_name}; } +std::vector GgmlOvDecoder::get_output_aliases(int node_idx) const { + const auto * node = m_node_info_list[node_idx].node; + if (node != nullptr && node->op == GGML_OP_SET_ROWS && node->view_src != nullptr) { + return {std::string(node->view_src->name)}; + } + return {}; +} + const std::string & GgmlOvDecoder::get_op_name() const { static const std::string unknown_name = "UNKNOWN_OP_NAME"; return unknown_name; diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 20db049e740e..695676acd6ba 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -156,6 +156,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual std::vector get_output_names(int node_idx) const override; + virtual std::vector get_output_aliases(int node_idx) const override; + virtual const std::string & get_op_type() const override; virtual const std::string & get_op_type(int node_idx) const override; diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h index 9d64fe575c4c..3b429078c343 100644 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ b/ggml/src/ggml-openvino/openvino/decoder.h @@ -75,6 +75,8 @@ class GgmlDecoder : public DecoderBase { virtual std::vector get_output_names(int node_idx) const = 0; + virtual std::vector get_output_aliases(int node_idx) const = 0; + virtual const std::string & get_op_type() const = 0; virtual const std::string & get_op_type(int node_idx) const = 0; diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index d00c438e2a1f..4e981c5cff7d 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -216,6 +216,13 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo (*tensor_map)[output_name] = converted_outputs[i]; } } + + const auto & node_output_aliases = decoder->get_output_aliases(node_idx); + for (const auto & output_alias : node_output_aliases) { + if (!converted_outputs.empty() && converted_outputs[0].get_node_shared_ptr() != nullptr) { + (*tensor_map)[output_alias] = converted_outputs[0]; + } + } }; if (!m_naive) { From bea28f0458c7ef2f4b56f36414045295adc2e97a Mon Sep 17 00:00:00 2001 From: Xuejun Date: Thu, 25 Jun 2026 15:01:05 +0800 Subject: [PATCH 18/20] OpenVINO backend: enable op GGML_UNARY_OP_SIGMOID --- ggml/src/ggml-openvino/openvino/op_table.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 47fa6f68dc9e..6b8e589b9350 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -43,6 +44,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, {"GGML_OP_TRANSPOSE", op::translate_transpose }, {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, From 035f50cfe43a4213423a341e46da013d3af220f4 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Thu, 25 Jun 2026 16:49:06 +0800 Subject: [PATCH 19/20] OpenVINO Backend: enable SQR & SQRT --- ggml/src/ggml-openvino/openvino/op/sqr.cpp | 35 ++++++++++++++++++++ ggml/src/ggml-openvino/openvino/op_table.cpp | 2 ++ ggml/src/ggml-openvino/openvino/op_table.h | 2 ++ 3 files changed, 39 insertions(+) create mode 100644 ggml/src/ggml-openvino/openvino/op/sqr.cpp diff --git a/ggml/src/ggml-openvino/openvino/op/sqr.cpp b/ggml/src/ggml-openvino/openvino/op/sqr.cpp new file mode 100644 index 000000000000..9ea886e73567 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/sqr.cpp @@ -0,0 +1,35 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_sqr(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, input); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_sqrt(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 6b8e589b9350..cca448a7cec1 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -39,6 +39,8 @@ std::unordered_map get_supported_ops() { {"GGML_OP_SUM_ROWS", op::translate_sum_rows }, {"GGML_OP_ROPE", op::translate_rope }, {"GGML_OP_SCALE", op::translate_scale }, + {"GGML_OP_SQR", op::translate_sqr }, + {"GGML_OP_SQRT", op::translate_sqrt }, {"GGML_OP_SOFT_MAX", op::translate_soft_max }, {"GGML_OP_ARGSORT", op::translate_argsort }, {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index 921008482367..cd35e1429ec7 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -25,8 +25,10 @@ GGML_OP_CONVERTER(translate_rms_norm); GGML_OP_CONVERTER(translate_norm); GGML_OP_CONVERTER(translate_l2_norm); GGML_OP_CONVERTER(translate_sum_rows); +GGML_OP_CONVERTER(translate_sqr); GGML_OP_CONVERTER(translate_rope); GGML_OP_CONVERTER(translate_scale); +GGML_OP_CONVERTER(translate_sqrt); GGML_OP_CONVERTER(translate_unary_silu); GGML_OP_CONVERTER(translate_unary_softplus); GGML_OP_CONVERTER(translate_soft_max); From 98f83bc43e3addf04ad05243ee4c33aafc990448 Mon Sep 17 00:00:00 2001 From: Xuejun Date: Tue, 30 Jun 2026 15:19:06 +0800 Subject: [PATCH 20/20] OpenVINO backend: fix accurace issue in gemma3n arch test --- ggml/src/ggml-openvino/ggml-openvino.cpp | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 5e14fc0f4d3b..e0b030a18daa 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -932,29 +932,6 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { } break; } - case GGML_OP_DIV: { - bool requires_broadcast = false; - for (int i = 0; i < 4; i++) { - if (op->src[0]->ne[i] == op->src[1]->ne[i]) { - continue; - } - - if (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1) { - return true; - } - - requires_broadcast = true; - } - - // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path - // and produce infs for per-channel scale vectors. Keep those DIVs on CPU - // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt) - if (requires_broadcast && ggml_openvino_get_device_name() == "GPU") { - return true; - } - - break; - } case GGML_OP_SUM_ROWS: { // if the input is PERMUTE skip if (op->src[0]->op == GGML_OP_PERMUTE) {