From 7b887c026ecf76246f96ca1be098690edf370908 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Fri, 11 Sep 2026 05:35:05 +0200 Subject: [PATCH 1/4] feat: add top_k, min_p, penalty sampling params ChatCompletionInferenceParams only had temperature, top_p and max_tokens, and extra="forbid" rejected anything else, so sampling knobs had to be hand-written into extra_body. Add presence_penalty, top_k, min_p and repetition_penalty as optional, range-checked fields. presence_penalty is sent top-level; the other three are not OpenAI parameters, so they go through extra_body, where explicit extra_body keys still win. Drop the new keys from the config fingerprint while unset, so existing configs keep their stored hash and resume still works. Signed-off-by: ChethanUK --- .../concepts/models/inference-parameters.mdx | 13 +++- .../src/data_designer/config/fingerprint.py | 14 +++- .../src/data_designer/config/models.py | 16 ++++ .../tests/config/test_fingerprint.py | 31 ++++++++ .../tests/config/test_models.py | 52 +++++++++++++ .../tests/engine/models/test_facade.py | 74 +++++++++++++++++++ 6 files changed, 196 insertions(+), 4 deletions(-) diff --git a/fern/versions/latest/pages/concepts/models/inference-parameters.mdx b/fern/versions/latest/pages/concepts/models/inference-parameters.mdx index 90131dc44..94f19bda3 100644 --- a/fern/versions/latest/pages/concepts/models/inference-parameters.mdx +++ b/fern/versions/latest/pages/concepts/models/inference-parameters.mdx @@ -20,13 +20,24 @@ The `ChatCompletionInferenceParams` class controls how models generate text comp | `temperature` | `float` or `Distribution` | No | Controls randomness in generation (0.0 to 2.0). Higher values = more creative/random | | `top_p` | `float` or `Distribution` | No | Nucleus sampling parameter (0.0 to 1.0). Controls diversity by filtering low-probability tokens | | `max_tokens` | `int` | No | Maximum number of tokens to generate in the response (≥ 1) | +| `presence_penalty` | `float` | No | Penalizes tokens that already appeared in the output (-2.0 to 2.0) | +| `top_k` | `int` | No | Samples only from the k most likely tokens (≥ 1). Sent in `extra_body` | +| `min_p` | `float` | No | Drops tokens below this probability, relative to the most likely token (0.0 to 1.0). Sent in `extra_body` | +| `repetition_penalty` | `float` | No | Multiplicative penalty for repeated tokens (> 0; 1.0 means no penalty). Sent in `extra_body` | | `max_parallel_requests` | `int` | No | Maximum concurrent API requests to this model (default: 4, ≥ 1). See [Concurrency Control](#concurrency-control) below. | | `timeout` | `int` | No | API request timeout in seconds (≥ 1) | | `extra_body` | `dict[str, Any]` | No | Additional parameters to include in the API request body | Default Values -If `temperature`, `top_p`, or `max_tokens` are not provided, the model provider's default values will be used. Different providers and models may have different defaults. +If `temperature`, `top_p`, `max_tokens`, or any of the sampling parameters above are not provided, the model provider's default values will be used. Different providers and models may have different defaults. + + + +Provider Support for Sampling Parameters +`top_k`, `min_p`, and `repetition_penalty` are not [OpenAI Chat Completions](https://platform.openai.com/docs/api-reference/chat/create) parameters, so they are merged into the request body through `extra_body`, and the endpoint must accept them. [vLLM](https://github.com/vllm-project/vllm/blob/main/vllm/sampling_params.py) and [SGLang](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/sampling/sampling_params.py) accept all three; SGLang also caps `repetition_penalty` at 2.0. Keys you set in `extra_body` yourself override these fields, and a model provider's `extra_body` overrides both. + +For Anthropic providers, `presence_penalty` is not sent. The [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) supports `top_k` but has no `min_p` or `repetition_penalty`, so leave those two unset. diff --git a/packages/data-designer-config/src/data_designer/config/fingerprint.py b/packages/data-designer-config/src/data_designer/config/fingerprint.py index 4cb32090b..421a394c0 100644 --- a/packages/data-designer-config/src/data_designer/config/fingerprint.py +++ b/packages/data-designer-config/src/data_designer/config/fingerprint.py @@ -64,6 +64,10 @@ ) _TOOL_CONFIG_OPTIONAL_COLLECTIONS: frozenset[str] = frozenset({"allow_tools"}) +# Optional sampling params added without a hash version bump. Dropped while unset, +# so configs that don't use them keep their stored hash and can still resume. +_INFERENCE_OPTIONAL_SCALARS: frozenset[str] = frozenset({"presence_penalty", "top_k", "min_p", "repetition_penalty"}) + # --------------------------------------------------------------------------- # Public API @@ -82,7 +86,8 @@ def fingerprint_config(config: DataDesignerConfig) -> dict[str, str | int]: * `columns` - names, types, generator params, processors, validators, skip/drop flags. Column order is part of identity (DAG ordering). * `model_configs` - alias, model, provider, sampling-relevant inference - params (temperature, top_p, max_tokens, extra_body). Sorted by alias. + params (temperature, top_p, max_tokens, presence_penalty, top_k, min_p, + repetition_penalty, extra_body). Sorted by alias. * `tool_configs` - alias, providers, allow_tools, max_tool_call_turns (the set of MCP tools shapes generation). Sorted by tool_alias. * `seed_config` - source path, sampling strategy, selection strategy. @@ -133,7 +138,8 @@ def _drop_empty_optional(source: dict[str, Any], keys: Iterable[str]) -> dict[st """Drop keys whose value is `None` or an empty list. `None` and `[]` are user-equivalent for optional collection fields; this - collapses both to "absent" before hashing. + collapses both to "absent" before hashing. For optional scalars only the + `None` case applies. """ keyset = set(keys) return {k: v for k, v in source.items() if not (k in keyset and (v is None or v == []))} @@ -143,7 +149,9 @@ def _normalize_model_config(model_config: dict[str, Any]) -> dict[str, Any]: normalized = _drop_keys(model_config, _EXCLUDED_MODEL_KEYS) inference_params = normalized.get("inference_parameters") if isinstance(inference_params, dict): - normalized["inference_parameters"] = _drop_keys(inference_params, _EXCLUDED_INFERENCE_KEYS) + normalized["inference_parameters"] = _drop_empty_optional( + _drop_keys(inference_params, _EXCLUDED_INFERENCE_KEYS), _INFERENCE_OPTIONAL_SCALARS + ) return normalized diff --git a/packages/data-designer-config/src/data_designer/config/models.py b/packages/data-designer-config/src/data_designer/config/models.py index e6ead1123..8b8f66209 100644 --- a/packages/data-designer-config/src/data_designer/config/models.py +++ b/packages/data-designer-config/src/data_designer/config/models.py @@ -497,12 +497,20 @@ class ChatCompletionInferenceParams(BaseInferenceParams): temperature: Sampling temperature (0.0-2.0). Can be a fixed value or a distribution for dynamic sampling. top_p: Nucleus sampling probability (0.0-1.0). Can be a fixed value or a distribution for dynamic sampling. max_tokens: Maximum number of tokens to generate in the response. + presence_penalty: Penalty (-2.0 to 2.0) for tokens that already appeared in the output. + top_k: Sample only from the k most likely tokens. Sent via `extra_body`. + min_p: Minimum token probability (0.0-1.0), relative to the most likely token. Sent via `extra_body`. + repetition_penalty: Multiplicative penalty (> 0) for repeated tokens; 1.0 means none. Sent via `extra_body`. """ generation_type: Literal[GenerationType.CHAT_COMPLETION] = GenerationType.CHAT_COMPLETION temperature: float | DistributionT | None = None top_p: float | DistributionT | None = None max_tokens: int | None = Field(default=None, ge=1) + presence_penalty: float | None = Field(default=None, ge=-2.0, le=2.0) + top_k: int | None = Field(default=None, ge=1) + min_p: float | None = Field(default=None, ge=0.0, le=1.0) + repetition_penalty: float | None = Field(default=None, gt=0.0) @property def generate_kwargs(self) -> dict[str, Any]: @@ -515,6 +523,14 @@ def generate_kwargs(self) -> dict[str, Any]: result["top_p"] = self.top_p.sample() if hasattr(self.top_p, "sample") else self.top_p if self.max_tokens is not None: result["max_tokens"] = self.max_tokens + if self.presence_penalty is not None: + result["presence_penalty"] = self.presence_penalty + # Not OpenAI params; the engine drops unknown top-level kwargs, so send them via extra_body. + # Keys the user set in extra_body themselves take precedence. + sampling_params = {"top_k": self.top_k, "min_p": self.min_p, "repetition_penalty": self.repetition_penalty} + extra_params = {key: value for key, value in sampling_params.items() if value is not None} + if extra_params: + result["extra_body"] = {**extra_params, **(result.get("extra_body") or {})} return result @model_validator(mode="after") diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py index a02ca3603..334683489 100644 --- a/packages/data-designer-config/tests/config/test_fingerprint.py +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -149,6 +149,37 @@ def test_changing_temperature_changes_hash() -> None: assert _compute_hash(a) != _compute_hash(b) +@pytest.mark.parametrize( + "sampling_param", + [{"presence_penalty": 0.5}, {"top_k": 40}, {"min_p": 0.05}, {"repetition_penalty": 1.1}], + ids=["presence_penalty", "top_k", "min_p", "repetition_penalty"], +) +def test_changing_sampling_param_changes_hash(sampling_param: dict[str, float]) -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + provider="some-provider", + inference_parameters=ChatCompletionInferenceParams( + temperature=0.5, top_p=0.9, max_tokens=128, **sampling_param + ), + ) + ], + ) + assert _compute_hash(a) != _compute_hash(b) + + +def test_unset_sampling_params_keep_existing_hash() -> None: + """Configs that leave the optional sampling params unset must keep their stored hash so resume still works. + + The literal was computed from this same config before those fields existed; only a literal catches drift. + """ + expected = "sha256:b21a98e06ea3c84c8fac9fe5231e68b0bfcffe03283952b03e807baba17139dc" + assert _compute_hash(_make_minimal_config()) == expected + + def test_changing_column_order_changes_hash() -> None: """Column order is part of identity (DAG ordering).""" cols_a = [ diff --git a/packages/data-designer-config/tests/config/test_models.py b/packages/data-designer-config/tests/config/test_models.py index 2083229b4..093b5faec 100644 --- a/packages/data-designer-config/tests/config/test_models.py +++ b/packages/data-designer-config/tests/config/test_models.py @@ -440,6 +440,56 @@ def test_inference_parameters_generate_kwargs(): assert inference_parameters_kwargs["top_p"] is not None +@pytest.mark.parametrize( + ("params", "expected"), + [ + ({"presence_penalty": 0.5}, {"presence_penalty": 0.5}), + ({"presence_penalty": -2.0}, {"presence_penalty": -2.0}), + ( + {"top_k": 50, "min_p": 0.1, "repetition_penalty": 1.1}, + {"extra_body": {"top_k": 50, "min_p": 0.1, "repetition_penalty": 1.1}}, + ), + ({"top_k": 1, "min_p": 0.0}, {"extra_body": {"top_k": 1, "min_p": 0.0}}), + ( + {"top_k": 50, "extra_body": {"reasoning_effort": "high"}}, + {"extra_body": {"top_k": 50, "reasoning_effort": "high"}}, + ), + ({"top_k": 50, "extra_body": {"top_k": 20}}, {"extra_body": {"top_k": 20}}), + ], + ids=[ + "presence-penalty-top-level", + "presence-penalty-lower-bound", + "non-openai-params-in-extra-body", + "falsy-but-set-values-sent", + "merged-with-config-extra-body", + "config-extra-body-wins", + ], +) +def test_inference_parameters_routes_sampling_params(params: dict, expected: dict) -> None: + caller_extra_body = params.get("extra_body") + caller_extra_body_before = dict(caller_extra_body) if caller_extra_body is not None else None + + assert ChatCompletionInferenceParams(**params).generate_kwargs == expected + assert caller_extra_body == caller_extra_body_before + + +@pytest.mark.parametrize( + "params", + [ + {"presence_penalty": 2.1}, + {"presence_penalty": -2.1}, + {"top_k": 0}, + {"min_p": -0.1}, + {"min_p": 1.1}, + {"repetition_penalty": 0}, + {"repetition_penalty": -1}, + ], +) +def test_inference_parameters_rejects_out_of_range_sampling_params(params: dict) -> None: + with pytest.raises(ValidationError): + ChatCompletionInferenceParams(**params) + + def test_uniform_distribution_low_lt_high_validation(): with pytest.raises(ValueError, match="`low` must be less than `high`"): UniformDistribution(params=UniformDistributionParams(low=0.8, high=0.8)) @@ -729,6 +779,7 @@ def test_chat_completion_params_format_for_display_all_params(): max_tokens=2048, max_parallel_requests=4, timeout=60, + top_k=40, ) result = params.format_for_display() assert "generation_type=chat-completion" in result @@ -737,6 +788,7 @@ def test_chat_completion_params_format_for_display_all_params(): assert "max_tokens=2048" in result assert "max_parallel_requests=4" in result assert "timeout=60" in result + assert "top_k=40" in result def test_chat_completion_params_format_for_display_partial_params(): diff --git a/packages/data-designer-engine/tests/engine/models/test_facade.py b/packages/data-designer-engine/tests/engine/models/test_facade.py index 758ea256f..0df01abec 100644 --- a/packages/data-designer-engine/tests/engine/models/test_facade.py +++ b/packages/data-designer-engine/tests/engine/models/test_facade.py @@ -8,7 +8,11 @@ import pytest +from data_designer.config.models import ChatCompletionInferenceParams from data_designer.engine.mcp.errors import MCPConfigurationError, MCPToolError +from data_designer.engine.models.clients.adapters.anthropic import AnthropicClient +from data_designer.engine.models.clients.adapters.http_model_client import ClientConcurrencyMode +from data_designer.engine.models.clients.adapters.openai_compatible import OpenAICompatibleClient from data_designer.engine.models.clients.errors import ProviderError, ProviderErrorKind from data_designer.engine.models.clients.types import ( AssistantMessage, @@ -32,6 +36,7 @@ from data_designer.engine.models.usage_events import TokenUsageEvent, subscribe_token_usage from data_designer.engine.models.utils import ChatMessage from data_designer.engine.testing import StubMCPFacade, StubMCPRegistry, make_stub_completion_response +from tests.engine.models.clients.conftest import make_mock_sync_client def _make_response( @@ -223,6 +228,75 @@ def test_generate_drops_configured_extra_body_n_from_single_result_request( ) +_OPENAI_TEXT_RESPONSE = { + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello!"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, +} +_ANTHROPIC_TEXT_RESPONSE = { + "content": [{"type": "text", "text": "Hello!"}], + "usage": {"input_tokens": 10, "output_tokens": 5}, + "stop_reason": "end_turn", +} + + +@pytest.mark.parametrize( + ("client_cls", "response_json", "provider_extra_body", "expected", "absent"), + [ + ( + OpenAICompatibleClient, + _OPENAI_TEXT_RESPONSE, + None, + {"presence_penalty": 0.5, "top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, + {"extra_body"}, + ), + ( + OpenAICompatibleClient, + _OPENAI_TEXT_RESPONSE, + {"top_k": 20, "presence_penalty": 0.1}, + {"presence_penalty": 0.1, "top_k": 20, "min_p": 0.05, "repetition_penalty": 1.1}, + {"extra_body"}, + ), + ( + AnthropicClient, + _ANTHROPIC_TEXT_RESPONSE, + None, + {"top_k": 40}, + {"presence_penalty", "extra_body"}, + ), + ], + ids=["openai-compatible", "provider-extra-body-wins", "anthropic-drops-presence-penalty"], +) +def test_generate_sends_sampling_params_in_body( + stub_model_configs: list[Any], + stub_model_provider_registry: Any, + client_cls: type[OpenAICompatibleClient] | type[AnthropicClient], + response_json: dict[str, Any], + provider_extra_body: dict[str, Any] | None, + expected: dict[str, Any], + absent: set[str], +) -> None: + model_config = stub_model_configs[0] + model_config.inference_parameters = ChatCompletionInferenceParams( + presence_penalty=0.5, top_k=40, min_p=0.05, repetition_penalty=1.1 + ) + http_client = make_mock_sync_client(response_json) + client = client_cls( + provider_name="stub-model-provider", + endpoint="https://api.example.com/v1", + api_key="sk-test", + concurrency_mode=ClientConcurrencyMode.SYNC, + sync_client=http_client, + ) + facade = ModelFacade(model_config, stub_model_provider_registry, client=client) + facade.model_provider.extra_body = provider_extra_body + + facade.generate(prompt="does not matter", parser=lambda x: x) + + payload = http_client.post.call_args.kwargs["json"] + assert {key: payload.get(key) for key in expected} == expected + assert absent.isdisjoint(payload) + + @pytest.mark.asyncio async def test_agenerate_drops_n_from_single_result_request( stub_model_facade: ModelFacade, From 146a63aa4cd248f15b6fb7a39f01e1a5f57be098 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Sat, 12 Sep 2026 01:28:02 +0200 Subject: [PATCH 2/4] docs: state how Anthropic handles unsupported sampling params The provider-support note said the Anthropic Messages API has no min_p or repetition_penalty and to leave them unset, but not what happens if they are set anyway. The adapter's exclude set only covers presence_penalty, and extra_body keys are merged into the request body after that set is applied, so min_p and repetition_penalty are still sent and the API rejects the request. Say so on the page, and assert it in the facade wire test, so the behavior is pinned by a test instead of left implied. Signed-off-by: ChethanUK --- .../latest/pages/concepts/models/inference-parameters.mdx | 2 +- .../data-designer-engine/tests/engine/models/test_facade.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fern/versions/latest/pages/concepts/models/inference-parameters.mdx b/fern/versions/latest/pages/concepts/models/inference-parameters.mdx index 94f19bda3..0c47644e5 100644 --- a/fern/versions/latest/pages/concepts/models/inference-parameters.mdx +++ b/fern/versions/latest/pages/concepts/models/inference-parameters.mdx @@ -37,7 +37,7 @@ If `temperature`, `top_p`, `max_tokens`, or any of the sampling parameters above Provider Support for Sampling Parameters `top_k`, `min_p`, and `repetition_penalty` are not [OpenAI Chat Completions](https://platform.openai.com/docs/api-reference/chat/create) parameters, so they are merged into the request body through `extra_body`, and the endpoint must accept them. [vLLM](https://github.com/vllm-project/vllm/blob/main/vllm/sampling_params.py) and [SGLang](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/sampling/sampling_params.py) accept all three; SGLang also caps `repetition_penalty` at 2.0. Keys you set in `extra_body` yourself override these fields, and a model provider's `extra_body` overrides both. -For Anthropic providers, `presence_penalty` is not sent. The [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) supports `top_k` but has no `min_p` or `repetition_penalty`, so leave those two unset. +For Anthropic providers, `presence_penalty` is not sent, because the Anthropic adapter excludes it. The [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) supports `top_k` but has no `min_p` or `repetition_penalty`, and the adapter does not filter those two: if you set them they are still merged into the request body, and the API rejects the request. Leave them unset for Anthropic, as you would for any key in `extra_body` that the endpoint does not accept. diff --git a/packages/data-designer-engine/tests/engine/models/test_facade.py b/packages/data-designer-engine/tests/engine/models/test_facade.py index 0df01abec..41f8ec8d3 100644 --- a/packages/data-designer-engine/tests/engine/models/test_facade.py +++ b/packages/data-designer-engine/tests/engine/models/test_facade.py @@ -260,11 +260,13 @@ def test_generate_drops_configured_extra_body_n_from_single_result_request( AnthropicClient, _ANTHROPIC_TEXT_RESPONSE, None, - {"top_k": 40}, + # The adapter excludes presence_penalty; min_p and repetition_penalty are not + # Messages API parameters but reach the body like any other extra_body key. + {"top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, {"presence_penalty", "extra_body"}, ), ], - ids=["openai-compatible", "provider-extra-body-wins", "anthropic-drops-presence-penalty"], + ids=["openai-compatible", "provider-extra-body-wins", "anthropic-forwards-extra-body-minus-presence-penalty"], ) def test_generate_sends_sampling_params_in_body( stub_model_configs: list[Any], From 452237ff0fb57b62f9c530370138be53097ead4c Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Sat, 12 Sep 2026 01:41:47 +0200 Subject: [PATCH 3/4] test: cover the async path for the sampling params The wire test only proved synchronous serialization, so a regression confined to agenerate would have passed. The async adapters assemble the request separately from the sync ones, even though both go through the same consolidate_kwargs and TransportKwargs.from_request. Lift the three cases into a shared table and run them through agenerate as well, so both public paths are pinned by the same assertions. Signed-off-by: ChethanUK --- .../tests/engine/models/test_facade.py | 125 ++++++++++++------ 1 file changed, 86 insertions(+), 39 deletions(-) diff --git a/packages/data-designer-engine/tests/engine/models/test_facade.py b/packages/data-designer-engine/tests/engine/models/test_facade.py index 41f8ec8d3..c70e58ad6 100644 --- a/packages/data-designer-engine/tests/engine/models/test_facade.py +++ b/packages/data-designer-engine/tests/engine/models/test_facade.py @@ -36,7 +36,7 @@ from data_designer.engine.models.usage_events import TokenUsageEvent, subscribe_token_usage from data_designer.engine.models.utils import ChatMessage from data_designer.engine.testing import StubMCPFacade, StubMCPRegistry, make_stub_completion_response -from tests.engine.models.clients.conftest import make_mock_sync_client +from tests.engine.models.clients.conftest import make_mock_async_client, make_mock_sync_client def _make_response( @@ -239,66 +239,113 @@ def test_generate_drops_configured_extra_body_n_from_single_result_request( } -@pytest.mark.parametrize( - ("client_cls", "response_json", "provider_extra_body", "expected", "absent"), - [ - ( - OpenAICompatibleClient, - _OPENAI_TEXT_RESPONSE, - None, - {"presence_penalty": 0.5, "top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, - {"extra_body"}, - ), - ( - OpenAICompatibleClient, - _OPENAI_TEXT_RESPONSE, - {"top_k": 20, "presence_penalty": 0.1}, - {"presence_penalty": 0.1, "top_k": 20, "min_p": 0.05, "repetition_penalty": 1.1}, - {"extra_body"}, - ), - ( - AnthropicClient, - _ANTHROPIC_TEXT_RESPONSE, - None, - # The adapter excludes presence_penalty; min_p and repetition_penalty are not - # Messages API parameters but reach the body like any other extra_body key. - {"top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, - {"presence_penalty", "extra_body"}, - ), - ], - ids=["openai-compatible", "provider-extra-body-wins", "anthropic-forwards-extra-body-minus-presence-penalty"], -) -def test_generate_sends_sampling_params_in_body( +_SAMPLING_PARAM_FIELDS = ("client_cls", "response_json", "provider_extra_body", "expected", "absent") +_SAMPLING_PARAM_CASES = [ + ( + OpenAICompatibleClient, + _OPENAI_TEXT_RESPONSE, + None, + {"presence_penalty": 0.5, "top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, + {"extra_body"}, + ), + ( + OpenAICompatibleClient, + _OPENAI_TEXT_RESPONSE, + {"top_k": 20, "presence_penalty": 0.1}, + {"presence_penalty": 0.1, "top_k": 20, "min_p": 0.05, "repetition_penalty": 1.1}, + {"extra_body"}, + ), + ( + AnthropicClient, + _ANTHROPIC_TEXT_RESPONSE, + None, + # The adapter excludes presence_penalty; min_p and repetition_penalty are not + # Messages API parameters but reach the body like any other extra_body key. + {"top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, + {"presence_penalty", "extra_body"}, + ), +] +_SAMPLING_PARAM_IDS = [ + "openai-compatible", + "provider-extra-body-wins", + "anthropic-forwards-extra-body-minus-presence-penalty", +] + + +def _make_sampling_params_facade( stub_model_configs: list[Any], stub_model_provider_registry: Any, client_cls: type[OpenAICompatibleClient] | type[AnthropicClient], - response_json: dict[str, Any], + http_client: MagicMock, provider_extra_body: dict[str, Any] | None, - expected: dict[str, Any], - absent: set[str], -) -> None: + *, + is_async: bool = False, +) -> ModelFacade: + """Facade over a mocked HTTP transport, with all four sampling params set.""" model_config = stub_model_configs[0] model_config.inference_parameters = ChatCompletionInferenceParams( presence_penalty=0.5, top_k=40, min_p=0.05, repetition_penalty=1.1 ) - http_client = make_mock_sync_client(response_json) client = client_cls( provider_name="stub-model-provider", endpoint="https://api.example.com/v1", api_key="sk-test", - concurrency_mode=ClientConcurrencyMode.SYNC, - sync_client=http_client, + concurrency_mode=ClientConcurrencyMode.ASYNC if is_async else ClientConcurrencyMode.SYNC, + sync_client=None if is_async else http_client, + async_client=http_client if is_async else None, ) facade = ModelFacade(model_config, stub_model_provider_registry, client=client) facade.model_provider.extra_body = provider_extra_body + return facade - facade.generate(prompt="does not matter", parser=lambda x: x) +def _assert_sampling_params_payload(http_client: MagicMock, expected: dict[str, Any], absent: set[str]) -> None: payload = http_client.post.call_args.kwargs["json"] assert {key: payload.get(key) for key in expected} == expected assert absent.isdisjoint(payload) +@pytest.mark.parametrize(_SAMPLING_PARAM_FIELDS, _SAMPLING_PARAM_CASES, ids=_SAMPLING_PARAM_IDS) +def test_generate_sends_sampling_params_in_body( + stub_model_configs: list[Any], + stub_model_provider_registry: Any, + client_cls: type[OpenAICompatibleClient] | type[AnthropicClient], + response_json: dict[str, Any], + provider_extra_body: dict[str, Any] | None, + expected: dict[str, Any], + absent: set[str], +) -> None: + http_client = make_mock_sync_client(response_json) + facade = _make_sampling_params_facade( + stub_model_configs, stub_model_provider_registry, client_cls, http_client, provider_extra_body + ) + + facade.generate(prompt="does not matter", parser=lambda x: x) + + _assert_sampling_params_payload(http_client, expected, absent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(_SAMPLING_PARAM_FIELDS, _SAMPLING_PARAM_CASES, ids=_SAMPLING_PARAM_IDS) +async def test_agenerate_sends_sampling_params_in_body( + stub_model_configs: list[Any], + stub_model_provider_registry: Any, + client_cls: type[OpenAICompatibleClient] | type[AnthropicClient], + response_json: dict[str, Any], + provider_extra_body: dict[str, Any] | None, + expected: dict[str, Any], + absent: set[str], +) -> None: + http_client = make_mock_async_client(response_json) + facade = _make_sampling_params_facade( + stub_model_configs, stub_model_provider_registry, client_cls, http_client, provider_extra_body, is_async=True + ) + + await facade.agenerate(prompt="does not matter", parser=lambda x: x) + + _assert_sampling_params_payload(http_client, expected, absent) + + @pytest.mark.asyncio async def test_agenerate_drops_n_from_single_result_request( stub_model_facade: ModelFacade, From 0cbfce80efdaf6b92c4ca6da43d3bc7c0be1a111 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Wed, 16 Sep 2026 17:56:31 +0200 Subject: [PATCH 4/4] fix: send top_k, min_p, repetition_penalty as request fields A per-call extra_body no longer drops them, and the Anthropic adapter excludes min_p and repetition_penalty, which the Messages API rejects. Signed-off-by: ChethanUK --- .../concepts/models/inference-parameters.mdx | 10 ++--- .../src/data_designer/config/models.py | 17 +++------ .../tests/config/test_models.py | 14 +++---- .../models/clients/adapters/anthropic.py | 2 + .../engine/models/clients/types.py | 3 ++ .../src/data_designer/engine/models/facade.py | 3 ++ .../tests/engine/models/test_facade.py | 38 ++++++++++++------- 7 files changed, 49 insertions(+), 38 deletions(-) diff --git a/fern/versions/latest/pages/concepts/models/inference-parameters.mdx b/fern/versions/latest/pages/concepts/models/inference-parameters.mdx index 0c47644e5..42d5a4f10 100644 --- a/fern/versions/latest/pages/concepts/models/inference-parameters.mdx +++ b/fern/versions/latest/pages/concepts/models/inference-parameters.mdx @@ -21,9 +21,9 @@ The `ChatCompletionInferenceParams` class controls how models generate text comp | `top_p` | `float` or `Distribution` | No | Nucleus sampling parameter (0.0 to 1.0). Controls diversity by filtering low-probability tokens | | `max_tokens` | `int` | No | Maximum number of tokens to generate in the response (≥ 1) | | `presence_penalty` | `float` | No | Penalizes tokens that already appeared in the output (-2.0 to 2.0) | -| `top_k` | `int` | No | Samples only from the k most likely tokens (≥ 1). Sent in `extra_body` | -| `min_p` | `float` | No | Drops tokens below this probability, relative to the most likely token (0.0 to 1.0). Sent in `extra_body` | -| `repetition_penalty` | `float` | No | Multiplicative penalty for repeated tokens (> 0; 1.0 means no penalty). Sent in `extra_body` | +| `top_k` | `int` | No | Samples only from the k most likely tokens (≥ 1) | +| `min_p` | `float` | No | Drops tokens below this probability, relative to the most likely token (0.0 to 1.0) | +| `repetition_penalty` | `float` | No | Multiplicative penalty for repeated tokens (> 0; 1.0 means no penalty) | | `max_parallel_requests` | `int` | No | Maximum concurrent API requests to this model (default: 4, ≥ 1). See [Concurrency Control](#concurrency-control) below. | | `timeout` | `int` | No | API request timeout in seconds (≥ 1) | | `extra_body` | `dict[str, Any]` | No | Additional parameters to include in the API request body | @@ -35,9 +35,9 @@ If `temperature`, `top_p`, `max_tokens`, or any of the sampling parameters above Provider Support for Sampling Parameters -`top_k`, `min_p`, and `repetition_penalty` are not [OpenAI Chat Completions](https://platform.openai.com/docs/api-reference/chat/create) parameters, so they are merged into the request body through `extra_body`, and the endpoint must accept them. [vLLM](https://github.com/vllm-project/vllm/blob/main/vllm/sampling_params.py) and [SGLang](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/sampling/sampling_params.py) accept all three; SGLang also caps `repetition_penalty` at 2.0. Keys you set in `extra_body` yourself override these fields, and a model provider's `extra_body` overrides both. +`top_k`, `min_p`, and `repetition_penalty` are not [OpenAI Chat Completions](https://platform.openai.com/docs/api-reference/chat/create) parameters. They are sent as top-level request fields, so the endpoint must accept them. [vLLM](https://github.com/vllm-project/vllm/blob/main/vllm/sampling_params.py) and [SGLang](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/sampling/sampling_params.py) accept all three; SGLang also caps `repetition_penalty` at 2.0. A key with the same name in `extra_body` overrides the field. -For Anthropic providers, `presence_penalty` is not sent, because the Anthropic adapter excludes it. The [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) supports `top_k` but has no `min_p` or `repetition_penalty`, and the adapter does not filter those two: if you set them they are still merged into the request body, and the API rejects the request. Leave them unset for Anthropic, as you would for any key in `extra_body` that the endpoint does not accept. +For Anthropic providers, only `top_k` is sent. The [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) has no `presence_penalty`, `min_p`, or `repetition_penalty`, so the Anthropic adapter drops them. diff --git a/packages/data-designer-config/src/data_designer/config/models.py b/packages/data-designer-config/src/data_designer/config/models.py index 8b8f66209..e83fdeed5 100644 --- a/packages/data-designer-config/src/data_designer/config/models.py +++ b/packages/data-designer-config/src/data_designer/config/models.py @@ -498,9 +498,9 @@ class ChatCompletionInferenceParams(BaseInferenceParams): top_p: Nucleus sampling probability (0.0-1.0). Can be a fixed value or a distribution for dynamic sampling. max_tokens: Maximum number of tokens to generate in the response. presence_penalty: Penalty (-2.0 to 2.0) for tokens that already appeared in the output. - top_k: Sample only from the k most likely tokens. Sent via `extra_body`. - min_p: Minimum token probability (0.0-1.0), relative to the most likely token. Sent via `extra_body`. - repetition_penalty: Multiplicative penalty (> 0) for repeated tokens; 1.0 means none. Sent via `extra_body`. + top_k: Sample only from the k most likely tokens. + min_p: Minimum token probability (0.0-1.0), relative to the most likely token. + repetition_penalty: Multiplicative penalty (> 0) for repeated tokens; 1.0 means none. """ generation_type: Literal[GenerationType.CHAT_COMPLETION] = GenerationType.CHAT_COMPLETION @@ -523,14 +523,9 @@ def generate_kwargs(self) -> dict[str, Any]: result["top_p"] = self.top_p.sample() if hasattr(self.top_p, "sample") else self.top_p if self.max_tokens is not None: result["max_tokens"] = self.max_tokens - if self.presence_penalty is not None: - result["presence_penalty"] = self.presence_penalty - # Not OpenAI params; the engine drops unknown top-level kwargs, so send them via extra_body. - # Keys the user set in extra_body themselves take precedence. - sampling_params = {"top_k": self.top_k, "min_p": self.min_p, "repetition_penalty": self.repetition_penalty} - extra_params = {key: value for key, value in sampling_params.items() if value is not None} - if extra_params: - result["extra_body"] = {**extra_params, **(result.get("extra_body") or {})} + for key in ("presence_penalty", "top_k", "min_p", "repetition_penalty"): + if (value := getattr(self, key)) is not None: + result[key] = value return result @model_validator(mode="after") diff --git a/packages/data-designer-config/tests/config/test_models.py b/packages/data-designer-config/tests/config/test_models.py index 093b5faec..a42b85e62 100644 --- a/packages/data-designer-config/tests/config/test_models.py +++ b/packages/data-designer-config/tests/config/test_models.py @@ -447,22 +447,20 @@ def test_inference_parameters_generate_kwargs(): ({"presence_penalty": -2.0}, {"presence_penalty": -2.0}), ( {"top_k": 50, "min_p": 0.1, "repetition_penalty": 1.1}, - {"extra_body": {"top_k": 50, "min_p": 0.1, "repetition_penalty": 1.1}}, + {"top_k": 50, "min_p": 0.1, "repetition_penalty": 1.1}, ), - ({"top_k": 1, "min_p": 0.0}, {"extra_body": {"top_k": 1, "min_p": 0.0}}), + ({"top_k": 1, "min_p": 0.0}, {"top_k": 1, "min_p": 0.0}), ( {"top_k": 50, "extra_body": {"reasoning_effort": "high"}}, - {"extra_body": {"top_k": 50, "reasoning_effort": "high"}}, + {"top_k": 50, "extra_body": {"reasoning_effort": "high"}}, ), - ({"top_k": 50, "extra_body": {"top_k": 20}}, {"extra_body": {"top_k": 20}}), ], ids=[ - "presence-penalty-top-level", + "presence-penalty", "presence-penalty-lower-bound", - "non-openai-params-in-extra-body", + "non-openai-params", "falsy-but-set-values-sent", - "merged-with-config-extra-body", - "config-extra-body-wins", + "extra-body-kept-separate", ], ) def test_inference_parameters_routes_sampling_params(params: dict, expected: dict) -> None: diff --git a/packages/data-designer-engine/src/data_designer/engine/models/clients/adapters/anthropic.py b/packages/data-designer-engine/src/data_designer/engine/models/clients/adapters/anthropic.py index 1f4d5081d..b117241c6 100644 --- a/packages/data-designer-engine/src/data_designer/engine/models/clients/adapters/anthropic.py +++ b/packages/data-designer-engine/src/data_designer/engine/models/clients/adapters/anthropic.py @@ -49,6 +49,8 @@ class AnthropicClient(HttpModelClient): "response_format", "frequency_penalty", "presence_penalty", + "min_p", + "repetition_penalty", "seed", } ) diff --git a/packages/data-designer-engine/src/data_designer/engine/models/clients/types.py b/packages/data-designer-engine/src/data_designer/engine/models/clients/types.py index d28f5f16b..7f45ad72a 100644 --- a/packages/data-designer-engine/src/data_designer/engine/models/clients/types.py +++ b/packages/data-designer-engine/src/data_designer/engine/models/clients/types.py @@ -70,6 +70,9 @@ class ChatCompletionRequest: response_format: dict[str, Any] | None = None frequency_penalty: float | None = None presence_penalty: float | None = None + top_k: int | None = None + min_p: float | None = None + repetition_penalty: float | None = None timeout: float | None = None extra_body: dict[str, Any] | None = None extra_headers: dict[str, str] | None = None diff --git a/packages/data-designer-engine/src/data_designer/engine/models/facade.py b/packages/data-designer-engine/src/data_designer/engine/models/facade.py index 96bb22eb8..f170367bd 100644 --- a/packages/data-designer-engine/src/data_designer/engine/models/facade.py +++ b/packages/data-designer-engine/src/data_designer/engine/models/facade.py @@ -106,6 +106,9 @@ def _build_generation_validation_error( "response_format", "frequency_penalty", "presence_penalty", + "top_k", + "min_p", + "repetition_penalty", "timeout", "tools", "extra_body", diff --git a/packages/data-designer-engine/tests/engine/models/test_facade.py b/packages/data-designer-engine/tests/engine/models/test_facade.py index c70e58ad6..d596aedfd 100644 --- a/packages/data-designer-engine/tests/engine/models/test_facade.py +++ b/packages/data-designer-engine/tests/engine/models/test_facade.py @@ -239,36 +239,42 @@ def test_generate_drops_configured_extra_body_n_from_single_result_request( } -_SAMPLING_PARAM_FIELDS = ("client_cls", "response_json", "provider_extra_body", "expected", "absent") +_SAMPLING_PARAM_FIELDS = ("client_cls", "response_json", "provider_extra_body", "call_extra_body", "expected", "absent") +_ALL_SAMPLING_PARAMS = {"presence_penalty": 0.5, "top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1} _SAMPLING_PARAM_CASES = [ + (OpenAICompatibleClient, _OPENAI_TEXT_RESPONSE, None, None, _ALL_SAMPLING_PARAMS, {"extra_body"}), ( OpenAICompatibleClient, _OPENAI_TEXT_RESPONSE, + {"top_k": 20, "presence_penalty": 0.1}, None, - {"presence_penalty": 0.5, "top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, + {"presence_penalty": 0.1, "top_k": 20, "min_p": 0.05, "repetition_penalty": 1.1}, {"extra_body"}, ), ( + # A per-call extra_body replaces the configured one; the typed params must survive it. OpenAICompatibleClient, _OPENAI_TEXT_RESPONSE, - {"top_k": 20, "presence_penalty": 0.1}, - {"presence_penalty": 0.1, "top_k": 20, "min_p": 0.05, "repetition_penalty": 1.1}, + None, + {"reasoning_effort": "high"}, + {**_ALL_SAMPLING_PARAMS, "reasoning_effort": "high"}, {"extra_body"}, ), ( + # Messages API has top_k but no presence_penalty, min_p or repetition_penalty. AnthropicClient, _ANTHROPIC_TEXT_RESPONSE, None, - # The adapter excludes presence_penalty; min_p and repetition_penalty are not - # Messages API parameters but reach the body like any other extra_body key. - {"top_k": 40, "min_p": 0.05, "repetition_penalty": 1.1}, - {"presence_penalty", "extra_body"}, + None, + {"top_k": 40}, + {"presence_penalty", "min_p", "repetition_penalty", "extra_body"}, ), ] _SAMPLING_PARAM_IDS = [ "openai-compatible", "provider-extra-body-wins", - "anthropic-forwards-extra-body-minus-presence-penalty", + "per-call-extra-body-keeps-params", + "anthropic-sends-only-top-k", ] @@ -283,9 +289,7 @@ def _make_sampling_params_facade( ) -> ModelFacade: """Facade over a mocked HTTP transport, with all four sampling params set.""" model_config = stub_model_configs[0] - model_config.inference_parameters = ChatCompletionInferenceParams( - presence_penalty=0.5, top_k=40, min_p=0.05, repetition_penalty=1.1 - ) + model_config.inference_parameters = ChatCompletionInferenceParams(**_ALL_SAMPLING_PARAMS) client = client_cls( provider_name="stub-model-provider", endpoint="https://api.example.com/v1", @@ -299,6 +303,10 @@ def _make_sampling_params_facade( return facade +def _call_kwargs(call_extra_body: dict[str, Any] | None) -> dict[str, Any]: + return {} if call_extra_body is None else {"extra_body": call_extra_body} + + def _assert_sampling_params_payload(http_client: MagicMock, expected: dict[str, Any], absent: set[str]) -> None: payload = http_client.post.call_args.kwargs["json"] assert {key: payload.get(key) for key in expected} == expected @@ -312,6 +320,7 @@ def test_generate_sends_sampling_params_in_body( client_cls: type[OpenAICompatibleClient] | type[AnthropicClient], response_json: dict[str, Any], provider_extra_body: dict[str, Any] | None, + call_extra_body: dict[str, Any] | None, expected: dict[str, Any], absent: set[str], ) -> None: @@ -320,7 +329,7 @@ def test_generate_sends_sampling_params_in_body( stub_model_configs, stub_model_provider_registry, client_cls, http_client, provider_extra_body ) - facade.generate(prompt="does not matter", parser=lambda x: x) + facade.generate(prompt="does not matter", parser=lambda x: x, **_call_kwargs(call_extra_body)) _assert_sampling_params_payload(http_client, expected, absent) @@ -333,6 +342,7 @@ async def test_agenerate_sends_sampling_params_in_body( client_cls: type[OpenAICompatibleClient] | type[AnthropicClient], response_json: dict[str, Any], provider_extra_body: dict[str, Any] | None, + call_extra_body: dict[str, Any] | None, expected: dict[str, Any], absent: set[str], ) -> None: @@ -341,7 +351,7 @@ async def test_agenerate_sends_sampling_params_in_body( stub_model_configs, stub_model_provider_registry, client_cls, http_client, provider_extra_body, is_async=True ) - await facade.agenerate(prompt="does not matter", parser=lambda x: x) + await facade.agenerate(prompt="does not matter", parser=lambda x: x, **_call_kwargs(call_extra_body)) _assert_sampling_params_payload(http_client, expected, absent)