From c827393bf76f2b0889acbfe47e9a41a3e8822182 Mon Sep 17 00:00:00 2001 From: ochougul Date: Tue, 8 Sep 2026 15:04:15 +0530 Subject: [PATCH 1/2] fix: return decode QPC key for CB decode-only compile Signed-off-by: ochougul --- .../transformers/models/modeling_auto.py | 4 +- .../unit_test/models/test_model_quickcheck.py | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/QEfficient/transformers/models/modeling_auto.py b/QEfficient/transformers/models/modeling_auto.py index 213986d41e..21255475f5 100755 --- a/QEfficient/transformers/models/modeling_auto.py +++ b/QEfficient/transformers/models/modeling_auto.py @@ -2203,9 +2203,7 @@ def compile( else: specializations = lang_specs[:1] qpc_key = "lang_prefill_qpc_path" - elif prefill_seq_len == 1 and not ( - self.continuous_batching and full_batch_size is not None and full_batch_size != batch_size - ): + elif prefill_seq_len == 1: if self.comp_ctx_lengths_decode is not None: specializations = lang_specs[-len(self.comp_ctx_lengths_decode) :] else: diff --git a/tests/unit_test/models/test_model_quickcheck.py b/tests/unit_test/models/test_model_quickcheck.py index 5fe9217794..a49ac40a9e 100644 --- a/tests/unit_test/models/test_model_quickcheck.py +++ b/tests/unit_test/models/test_model_quickcheck.py @@ -3481,6 +3481,46 @@ def test_layerwise_compile_hydrates_outer_qpc_paths(monkeypatch, tmp_path): assert model.lang_model.qpc_path == qpc_path +@pytest.mark.llm_model +def test_dual_qpc_decode_only_continuous_batching_returns_decode_qpc_key(monkeypatch): + from QEfficient.transformers.models import modeling_auto + from QEfficient.transformers.models.modeling_auto import _QEffAutoModelForImageTextToTextDualQPC + + model = object.__new__(_QEffAutoModelForImageTextToTextDualQPC) + model.continuous_batching = True + model.ccl_enabled = False + model.comp_ctx_lengths_prefill = None + model.comp_ctx_lengths_decode = None + model.transform = lambda **kwargs: None + model.model = type( + "Model", + (), + { + "config": type("Config", (), {"torch_dtype": torch.float32, "model_type": "test"})(), + "get_output_names": lambda self, **kwargs: {"vision": [], "lang": []}, + "get_specializations": lambda self, **kwargs: ({"vision": [], "lang": [{"seq_len": 1}]}, {}), + }, + )() + model.lang_model = type( + "LanguageModel", + (), + {"onnx_path": "language.onnx", "_compile": staticmethod(lambda **kwargs: "decode.qpc")}, + )() + + monkeypatch.setattr(modeling_auto, "_filter_custom_io_for_onnx", lambda custom_io, onnx_path: custom_io) + + result = model.compile( + prefill_seq_len=1, + ctx_len=16, + batch_size=1, + full_batch_size=4, + skip_vision=True, + lang_onnx_path="language.onnx", + ) + + assert result == {"lang_decode_qpc_path": "decode.qpc"} + + @pytest.mark.llm_model def test_layerwise_compile_rejects_unsupported_model(): """End-to-end smoke: invoking layerwise=True on llama bubbles the guard error.""" From b15995d82e51ca1c813039eaa0a79dbdbbe1671d Mon Sep 17 00:00:00 2001 From: ochougul Date: Thu, 17 Sep 2026 01:20:34 +0530 Subject: [PATCH 2/2] fix: restore Kimi deterministic test helpers Signed-off-by: ochougul --- .../unit_test/models/test_model_quickcheck.py | 28 +++++++++++++++++++ tests/utils/load_kimi_utils.py | 7 +++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/unit_test/models/test_model_quickcheck.py b/tests/unit_test/models/test_model_quickcheck.py index f879317d0c..f5e194b484 100644 --- a/tests/unit_test/models/test_model_quickcheck.py +++ b/tests/unit_test/models/test_model_quickcheck.py @@ -1051,6 +1051,34 @@ def test_kimi_k25_quickcheck_hf_qeff_vision_logits_parity(): assert np.allclose(hf_logits, qeff_logits, atol=1e-4, rtol=1e-4) +def test_kimi_k25_test_config_accepts_deterministic_seed(monkeypatch): + import inspect + + from tests.utils import load_kimi_utils + + seeded = [] + monkeypatch.setattr(load_kimi_utils, "set_deterministic", seeded.append) + + text_config = type("TextConfig", (), {})() + vision_config = type("VisionConfig", (), {})() + config = type( + "Config", + (), + {"_name_or_path": "kimi", "text_config": text_config, "vision_config": vision_config}, + )() + monkeypatch.setattr(load_kimi_utils.AutoConfig, "from_pretrained", lambda *args, **kwargs: config) + monkeypatch.setattr(load_kimi_utils, "load_kimi_k25_class", lambda *args: None) + + assert ( + load_kimi_utils.get_kimi_k25_test_config( + "kimi", {"kimi": {"additional_params": {"text_config": {}, "vision_config": {}}}}, seed=1234 + ) + is config + ) + assert seeded == [1234] + assert "seed" in inspect.signature(load_kimi_utils.load_kimi_k25_model_from_config).parameters + + @pytest.mark.llm_model @pytest.mark.parametrize( ("model_type", "model_id"), diff --git a/tests/utils/load_kimi_utils.py b/tests/utils/load_kimi_utils.py index 2a05669021..49dcc5539c 100644 --- a/tests/utils/load_kimi_utils.py +++ b/tests/utils/load_kimi_utils.py @@ -161,7 +161,8 @@ def prepare_config(model_path: Path): return config -def get_kimi_k25_test_config(model_name: str, model_config_dict): +def get_kimi_k25_test_config(model_name: str, model_config_dict, *, seed: int = 42): + set_deterministic(seed) config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) config._attn_implementation = "eager" config.torch_dtype = torch.float32 @@ -220,14 +221,16 @@ def _simulate_kimi_k25_quantized_experts(model): _attach_fake_gptq_weight(expert.down_proj) -def load_kimi_k25_model_from_config(config): +def load_kimi_k25_model_from_config(config, *, seed: int = 42): kimi_cls = load_kimi_k25_class(config._name_or_path) + set_deterministic(seed) model = kimi_cls._from_config(config) torch_dtype = getattr(model.config, "torch_dtype", None) if torch_dtype == torch.bfloat16 or torch_dtype == torch.float16: model = model.to(torch.float32) _simulate_kimi_k25_quantized_experts(model) model.vision_tower.patch_embed.pos_emb.interpolation_mode = "bilinear" + model.language_model.lm_head.weight.data.zero_() model.eval() tokenizer = AutoTokenizer.from_pretrained(KIMI_K25_MODEL_NAME, trust_remote_code=True) processor = AutoProcessor.from_pretrained(KIMI_K25_MODEL_NAME, trust_remote_code=True)