From de979e8f34af6bc980bc09e777e161b2ab3bc5a7 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 15:50:04 +0500 Subject: [PATCH 01/15] Init pydantic_compat tests --- .../_internal/pydantic_compat/__init__.py | 0 .../_internal/pydantic_compat/compare.py | 62 ++++++++++ .../_internal/pydantic_compat/conftest.py | 15 +++ .../_internal/pydantic_compat/factories.py | 90 ++++++++++++++ .../pydantic_compat/fixtures/api/fleet.json | 86 +++++++++++++ .../fixtures/db/job_provisioning_data.json | 38 ++++++ .../pydantic_compat/fixtures/db/run_spec.json | 114 ++++++++++++++++++ .../pydantic_compat/test_serialization.py | 76 ++++++++++++ 8 files changed, 481 insertions(+) create mode 100644 src/tests/_internal/pydantic_compat/__init__.py create mode 100644 src/tests/_internal/pydantic_compat/compare.py create mode 100644 src/tests/_internal/pydantic_compat/conftest.py create mode 100644 src/tests/_internal/pydantic_compat/factories.py create mode 100644 src/tests/_internal/pydantic_compat/fixtures/api/fleet.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/db/job_provisioning_data.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/db/run_spec.json create mode 100644 src/tests/_internal/pydantic_compat/test_serialization.py diff --git a/src/tests/_internal/pydantic_compat/__init__.py b/src/tests/_internal/pydantic_compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tests/_internal/pydantic_compat/compare.py b/src/tests/_internal/pydantic_compat/compare.py new file mode 100644 index 0000000000..75b602c2dc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/compare.py @@ -0,0 +1,62 @@ +""" +Fixture comparison for the pydantic v1 → v2 migration. + +Fixtures are generated under pydantic v1 and committed. The same tests then run under v2, so a +mismatch means the two versions disagree about the wire format. Regenerating a fixture IS the act +of accepting a wire change — every accepted diff shows up in review as a fixture diff. +""" + +import difflib +import json +from pathlib import Path +from typing import Union + +import pytest + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + +_REGEN_HINT = ( + "Fix the regression, or if the change is intended, accept it with" + " `pytest src/tests/_internal/pydantic_compat --regen-wire-fixtures`." +) + + +def canonicalize(payload: Union[bytes, str]) -> str: + """ + Re-render a JSON payload so that only meaningful differences survive. + + `sort_keys` drops key-order noise, which the v2 serializer is free to change. + + Comparing the re-rendered *text* rather than the parsed objects is deliberate: `==` on parsed + JSON would miss the drift this suite exists to catch, because Python compares `16 == 16.0` and + `True == 1` as equal. `Memory` is a `float` subclass and `Duration` an `int` subclass, so an + int/float representation change is one of the likeliest v2 diffs — and `json.dumps` renders + `16` and `16.0` distinctly where `==` cannot tell them apart. + """ + return json.dumps(json.loads(payload), indent=2, sort_keys=True) + "\n" + + +def assert_matches_fixture(kind: str, name: str, payload: Union[bytes, str], regen: bool) -> None: + """Compare a serialized payload against `fixtures//.json`.""" + path = FIXTURES_DIR / kind / f"{name}.json" + actual = canonicalize(payload) + + if regen: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(actual) + return + + if not path.exists(): + pytest.fail(f"no fixture at {path.relative_to(FIXTURES_DIR.parent)}. {_REGEN_HINT}") + + expected = path.read_text() + if actual != expected: + diff = "".join( + difflib.unified_diff( + expected.splitlines(keepends=True), + actual.splitlines(keepends=True), + fromfile=f"{kind}/{name}.json (committed)", + tofile=f"{kind}/{name}.json (actual)", + ) + ) + pytest.fail(f"{kind}/{name} serialization changed:\n{diff}\n{_REGEN_HINT}") diff --git a/src/tests/_internal/pydantic_compat/conftest.py b/src/tests/_internal/pydantic_compat/conftest.py new file mode 100644 index 0000000000..bdd4f16f58 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/conftest.py @@ -0,0 +1,15 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--regen-wire-fixtures", + action="store_true", + default=False, + help="Rewrite the pydantic_compat fixtures instead of asserting against them", + ) + + +@pytest.fixture +def regen(request) -> bool: + return request.config.getoption("--regen-wire-fixtures") diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py new file mode 100644 index 0000000000..f10e00860d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -0,0 +1,90 @@ +""" +Deterministic model instances for the comparison fixtures. + +Every value is pinned — no `uuid4()`, no `now()` — so a fixture only changes when serialization +changes. Builds on `dstack._internal.server.testing.common` where a factory already exists, so +these stay in sync with the shapes the rest of the suite uses. +""" + +import uuid +from datetime import datetime, timezone + +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.configurations import DevEnvironmentConfiguration +from dstack._internal.core.models.fleets import Fleet, FleetStatus +from dstack._internal.core.models.instances import Instance, InstanceStatus +from dstack._internal.core.models.profiles import Profile +from dstack._internal.core.models.resources import Memory, Range, ResourcesSpec +from dstack._internal.core.models.runs import JobProvisioningData, RunSpec +from dstack._internal.server.testing.common import ( + get_fleet_spec, + get_job_provisioning_data, + get_run_spec, +) + +_ID = uuid.UUID("11111111-1111-1111-1111-111111111111") +_CREATED_AT = datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + + +def job_provisioning_data() -> JobProvisioningData: + """Stored in `JobModel.job_provisioning_data` and `InstanceModel.job_provisioning_data`.""" + return get_job_provisioning_data( + dockerized=True, + availability_zone="us-east-1a", + gpu_count=1, + ) + + +def run_spec() -> RunSpec: + """ + Stored in `RunModel.run_spec` — the largest blob, and the one carrying the type zoo. + + `max_duration`/`idle_duration` are `Duration` (an `int` subclass) and `resources` carries + `Memory` and `Range[T]`, so this fixture pins how the custom types render. Those are the types + whose `__get_validators__` → `__get_pydantic_core_schema__` port is riskiest. + + Values are given in their already-parsed form rather than as YAML shorthand (`"2h"`, + `"16GB.."`): shorthand only type-checks because a `pre=True` validator widens the input, and + pinning the *parse* side is the job of the parse fixtures, not these. + """ + return get_run_spec( + repo_id="test-repo", + profile=Profile(name="default", max_duration=7200, idle_duration=300), + configuration=DevEnvironmentConfiguration( + ide="vscode", + resources=ResourcesSpec( + cpu=Range[int](min=2, max=8), + memory=Range[Memory](min=Memory(16), max=None), + ), + ), + ) + + +def fleet() -> Fleet: + """ + Returned by `/api/project/{name}/fleets/list` through `CustomORJSONResponse`. + + The default `FleetNodesSpec` has `target == min`, which is what makes `FleetNodesSpec.dict()` + drop `target` — the old-client compat hack from #3066. That override becomes a + `@model_serializer` in v2, so this fixture is the thing that proves the hack survived. + """ + return Fleet( + id=_ID, + name="test-fleet", + project_name="test-project", + spec=get_fleet_spec(), + created_at=_CREATED_AT, + status=FleetStatus.ACTIVE, + instances=[ + Instance( + id=_ID, + project_name="test-project", + name="test-instance", + instance_num=0, + status=InstanceStatus.IDLE, + created=_CREATED_AT, + backend=BackendType.AWS, + region="us-east-1", + ) + ], + ) diff --git a/src/tests/_internal/pydantic_compat/fixtures/api/fleet.json b/src/tests/_internal/pydantic_compat/fixtures/api/fleet.json new file mode 100644 index 0000000000..93947bee55 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/api/fleet.json @@ -0,0 +1,86 @@ +{ + "created_at": "2024-01-02T03:04:05+00:00", + "id": "11111111-1111-1111-1111-111111111111", + "instances": [ + { + "availability_zone": null, + "backend": "aws", + "busy_blocks": 0, + "created": "2024-01-02T03:04:05+00:00", + "finished_at": null, + "fleet_id": null, + "fleet_name": null, + "health_status": "healthy", + "hostname": null, + "id": "11111111-1111-1111-1111-111111111111", + "instance_num": 0, + "instance_type": null, + "job_name": null, + "name": "test-instance", + "price": null, + "project_name": "test-project", + "region": "us-east-1", + "status": "idle", + "termination_reason": null, + "termination_reason_message": null, + "total_blocks": null, + "unreachable": false + } + ], + "name": "test-fleet", + "project_name": "test-project", + "spec": { + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": null, + "max_price": null, + "name": "test-fleet", + "nodes": { + "max": 1, + "min": 1 + }, + "placement": null, + "regions": null, + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": null, + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": "fleet.dstack.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } + }, + "status": "active", + "status_message": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/db/job_provisioning_data.json b/src/tests/_internal/pydantic_compat/fixtures/db/job_provisioning_data.json new file mode 100644 index 0000000000..6f8c681be2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/db/job_provisioning_data.json @@ -0,0 +1,38 @@ +{ + "availability_zone": "us-east-1a", + "backend": "aws", + "backend_data": null, + "base_backend": null, + "dockerized": true, + "hostname": "127.0.0.4", + "instance_id": "instance_id", + "instance_network": null, + "instance_type": { + "name": "instance", + "resources": { + "cpu_arch": null, + "cpus": 1, + "description": "", + "disk": { + "size_mib": 102400 + }, + "gpus": [ + { + "memory_mib": 16384, + "name": "T4", + "vendor": "nvidia" + } + ], + "memory_mib": 512, + "spot": false + } + }, + "internal_ip": "127.0.0.4", + "price": 10.5, + "public_ip_enabled": true, + "region": "us-east-1", + "reservation": null, + "ssh_port": 22, + "ssh_proxy": null, + "username": "ubuntu" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/db/run_spec.json b/src/tests/_internal/pydantic_compat/fixtures/db/run_spec.json new file mode 100644 index 0000000000..6a236e634c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/db/run_spec.json @@ -0,0 +1,114 @@ +{ + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "home_dir": "/root", + "ide": "vscode", + "idle_duration": null, + "image": null, + "inactivity_duration": null, + "init": [], + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": null, + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "dev-environment", + "user": null, + "utilization_policy": null, + "version": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": "dstack.yaml", + "file_archives": [], + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": 300, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + }, + "repo_code_hash": null, + "repo_data": { + "repo_dir": "/", + "repo_type": "local" + }, + "repo_dir": null, + "repo_id": "test-repo", + "run_name": "test-run", + "ssh_key_pub": "user_ssh_key", + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py new file mode 100644 index 0000000000..600b4a613b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -0,0 +1,76 @@ +""" +Serialization stability for DB blobs and API responses. + +Each model is serialized through the *production* path — `.json()` for anything written to a +`Text` column, `CustomORJSONResponse` for anything returned from a router — and compared against a +fixture generated under pydantic v1. On v1 these are regression tests; on the v2 branch they are +the compat assertion. + +Nothing here may reference the duality API (`__request__` / `__response__`): these tests have to +run unchanged on both versions, and duality is gone in v2. + +Disposable: this package is deleted once the v2 release is verified in prod, except for a curated +subset of the `db/` fixtures, which outlive it because stored rows do. +""" + +from typing import Callable + +import pytest + +from dstack._internal.core.models.common import CoreModel +from dstack._internal.core.models.fleets import FleetNodesSpec +from dstack._internal.server.utils.routers import CustomORJSONResponse +from tests._internal.pydantic_compat import factories +from tests._internal.pydantic_compat.compare import assert_matches_fixture + +# Written to a `Text` column via `.json()`. +DB_BLOBS: dict[str, Callable[[], CoreModel]] = { + "job_provisioning_data": factories.job_provisioning_data, + "run_spec": factories.run_spec, +} + +# Returned from a router via `CustomORJSONResponse`. +API_RESPONSES: dict[str, Callable[[], CoreModel]] = { + "fleet": factories.fleet, +} + + +class TestDbBlobSerialization: + @pytest.mark.parametrize("name", sorted(DB_BLOBS)) + def test_matches_fixture(self, name, regen): + payload = DB_BLOBS[name]().json() + assert_matches_fixture("db", name, payload, regen=regen) + + +class TestApiResponseSerialization: + @pytest.mark.parametrize("name", sorted(API_RESPONSES)) + def test_matches_fixture(self, name, regen): + # Response bodies go through orjson with a `default=` hook, not through `.json()`, so the + # two paths can drift apart. Serialize the way the router does. + payload = bytes(CustomORJSONResponse(API_RESPONSES[name]()).body) + assert_matches_fixture("api", name, payload, regen=regen) + + +class TestFleetNodesTargetCompatHack: + """ + Pins the #3066 old-client hack explicitly, not just via the fixture bytes. + + `FleetNodesSpec.dict()` drops `target` when it equals `min`. A fixture would catch the change + but not explain it; naming the invariant gives the v2 `@model_serializer` rewrite something + unambiguous to satisfy. + + Instances are built fresh here rather than reached out of `factories.fleet()`: the `nodes` + default in `get_fleet_configuration` is a single shared `FleetNodesSpec`, so mutating one + reached through the fixture corrupts every later caller in the session. + """ + + def test_target_is_omitted_when_it_equals_min(self): + assert "target" not in FleetNodesSpec(min=1, target=1, max=1).dict() + + def test_target_is_kept_when_it_differs_from_min(self): + assert FleetNodesSpec(min=1, target=5, max=5).dict()["target"] == 5 + + def test_the_api_fixture_exercises_the_hack(self): + nodes = factories.fleet().spec.configuration.nodes + assert nodes is not None + assert nodes.min == nodes.target, "fixture must hit the target == min branch" From 61783f517113016d62f73823959f5781c5b39b15 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 16:45:00 +0500 Subject: [PATCH 02/15] Add parsing tests --- .../_internal/pydantic_compat/compare.py | 45 ++++++- src/tests/_internal/pydantic_compat/compat.py | 32 +++++ .../_internal/pydantic_compat/conftest.py | 4 +- .../create_volume_request.input.json | 9 ++ .../create_volume_request.types.json | 6 + .../create_volume_request.values.json | 13 ++ .../parsing/api_response/fleet.input.json | 14 +++ .../parsing/api_response/fleet.types.json | 13 ++ .../parsing/api_response/fleet.values.json | 61 +++++++++ .../fixtures/parsing/db/run_spec.input.json | 16 +++ .../fixtures/parsing/db/run_spec.types.json | 23 ++++ .../fixtures/parsing/db/run_spec.values.json | 119 ++++++++++++++++++ .../api_response}/fleet.json | 0 .../db/job_provisioning_data.json | 0 .../{ => serialization}/db/run_spec.json | 0 .../_internal/pydantic_compat/test_parsing.py | 89 +++++++++++++ .../pydantic_compat/test_serialization.py | 4 +- 17 files changed, 442 insertions(+), 6 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/compat.py create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.values.json rename src/tests/_internal/pydantic_compat/fixtures/{api => serialization/api_response}/fleet.json (100%) rename src/tests/_internal/pydantic_compat/fixtures/{ => serialization}/db/job_provisioning_data.json (100%) rename src/tests/_internal/pydantic_compat/fixtures/{ => serialization}/db/run_spec.json (100%) create mode 100644 src/tests/_internal/pydantic_compat/test_parsing.py diff --git a/src/tests/_internal/pydantic_compat/compare.py b/src/tests/_internal/pydantic_compat/compare.py index 75b602c2dc..31768fb890 100644 --- a/src/tests/_internal/pydantic_compat/compare.py +++ b/src/tests/_internal/pydantic_compat/compare.py @@ -9,15 +9,21 @@ import difflib import json from pathlib import Path -from typing import Union +from typing import Any, Union import pytest +from pydantic import BaseModel FIXTURES_DIR = Path(__file__).parent / "fixtures" +# Types JSON represents faithfully. Anything else — a `Duration` (int subclass), a `Memory` +# (float subclass), an enum, or a model standing in for a union arm — renders identically to its +# base type, so the value dump cannot show which one parsing produced. +_JSON_NATIVE = (str, int, float, bool, type(None)) + _REGEN_HINT = ( "Fix the regression, or if the change is intended, accept it with" - " `pytest src/tests/_internal/pydantic_compat --regen-wire-fixtures`." + " `pytest src/tests/_internal/pydantic_compat --regen-fixtures`." ) @@ -60,3 +66,38 @@ def assert_matches_fixture(kind: str, name: str, payload: Union[bytes, str], reg ) ) pytest.fail(f"{kind}/{name} serialization changed:\n{diff}\n{_REGEN_HINT}") + + +def type_map(value: Any, path: str = "", out: Union[dict, None] = None) -> dict: + """ + Map JSON pointer -> the concrete class parsing produced, wherever JSON erases it. + + `Duration(7200)` and `7200` serialize identically, as do `Memory(16.0)` and `16.0`, and + `PythonVersion.PY310` and `"3.10"`. Losing the subclass is therefore invisible in the value + dump while still being a real regression — `Memory.__str__` is `"16GB"`, and that string + reaches the CLI. Models are recorded as well as recursed into, so a union that resolves to a + different arm shows up here even when both arms happen to serialize the same. + """ + out = {} if out is None else out + if isinstance(value, BaseModel): + out[path or "/"] = _class_name(value) + for name, attr in value.__dict__.items(): + type_map(attr, f"{path}/{name}", out) + elif isinstance(value, dict): + for key, attr in value.items(): + type_map(attr, f"{path}/{key}", out) + elif isinstance(value, (list, tuple)): + for i, attr in enumerate(value): + type_map(attr, f"{path}/{i}", out) + elif type(value) not in _JSON_NATIVE: + out[path] = _class_name(value) + return out + + +def _class_name(value: Any) -> str: + # pydantic-duality names the concrete classes `XRequest` / `XResponse`. Those suffixes vanish + # in v2, so strip them or every line of every type map diffs on the migration branch. + name = type(value).__name__ + for suffix in ("Request", "Response"): + name = name.removesuffix(suffix) + return name diff --git a/src/tests/_internal/pydantic_compat/compat.py b/src/tests/_internal/pydantic_compat/compat.py new file mode 100644 index 0000000000..fd9ee43993 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/compat.py @@ -0,0 +1,32 @@ +""" +The only place in this package allowed to branch on pydantic version. + +Every test here has to run unchanged on v1 and v2, which rules out touching the duality API +directly. Strict parsing needs no help — `parse_obj` is strict in both versions. Permissive +parsing does: v1 spells it `X.__response__`, and v2 will spell it `validate_extra_ignore`. +""" + +from typing import Any + +import pydantic +from pydantic import BaseModel + +PYDANTIC_V1 = pydantic.VERSION.startswith("1.") + + +def parse_strict(model: Any, data: Any) -> BaseModel: + """Parse the way the server validates a request body: unknown fields are an error.""" + return model.parse_obj(data) + + +def parse_permissive(model: Any, data: Any) -> BaseModel: + """ + Parse the way a stored blob or a peer's response is read: unknown fields are ignored. + + This is what lets an older reader survive a newer writer, so it is the behaviour the whole + migration has to preserve. + """ + if PYDANTIC_V1: + return model.__response__.parse_obj(data) + # v2: from dstack._internal.core.models.common import validate_extra_ignore + raise NotImplementedError("wire up validate_extra_ignore when the v2 branch lands") diff --git a/src/tests/_internal/pydantic_compat/conftest.py b/src/tests/_internal/pydantic_compat/conftest.py index bdd4f16f58..b4421ecb10 100644 --- a/src/tests/_internal/pydantic_compat/conftest.py +++ b/src/tests/_internal/pydantic_compat/conftest.py @@ -3,7 +3,7 @@ def pytest_addoption(parser): parser.addoption( - "--regen-wire-fixtures", + "--regen-fixtures", action="store_true", default=False, help="Rewrite the pydantic_compat fixtures instead of asserting against them", @@ -12,4 +12,4 @@ def pytest_addoption(parser): @pytest.fixture def regen(request) -> bool: - return request.config.getoption("--regen-wire-fixtures") + return request.config.getoption("--regen-fixtures") diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.input.json new file mode 100644 index 0000000000..52ab35ad1c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.input.json @@ -0,0 +1,9 @@ +{ + "configuration": { + "type": "volume", + "name": "my-volume", + "backend": "aws", + "region": "us-east-1", + "size": "100GB" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.types.json new file mode 100644 index 0000000000..213adaae53 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.types.json @@ -0,0 +1,6 @@ +{ + "/": "CreateVolumeRequest", + "/configuration": "AWSVolumeConfiguration", + "/configuration/backend": "BackendType", + "/configuration/size": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.values.json new file mode 100644 index 0000000000..41eceb2751 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/create_volume_request.values.json @@ -0,0 +1,13 @@ +{ + "configuration": { + "auto_cleanup_duration": null, + "availability_zone": null, + "backend": "aws", + "name": "my-volume", + "region": "us-east-1", + "size": 100.0, + "tags": null, + "type": "volume", + "volume_id": null + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.input.json new file mode 100644 index 0000000000..2f4fce6acf --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.input.json @@ -0,0 +1,14 @@ +{ + "id": "11111111-1111-1111-1111-111111111111", + "name": "test-fleet", + "project_name": "test-project", + "spec": { + "configuration": {"type": "fleet", "name": "test-fleet", "nodes": {"min": 1, "max": 1}}, + "configuration_path": "fleet.dstack.yml", + "profile": {"name": "default"} + }, + "created_at": "2024-01-02T03:04:05+00:00", + "status": "active", + "instances": [], + "field_added_by_a_newer_server": "ignored" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.types.json new file mode 100644 index 0000000000..7c2ed2280f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.types.json @@ -0,0 +1,13 @@ +{ + "/": "Fleet", + "/created_at": "datetime", + "/id": "UUID", + "/spec": "FleetSpec", + "/spec/configuration": "FleetConfiguration", + "/spec/configuration/env": "Env", + "/spec/configuration/nodes": "FleetNodesSpec", + "/spec/merged_profile": "Profile", + "/spec/merged_profile/spot_policy": "SpotPolicy", + "/spec/profile": "Profile", + "/status": "FleetStatus" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.values.json new file mode 100644 index 0000000000..fc93513d22 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet.values.json @@ -0,0 +1,61 @@ +{ + "created_at": "2024-01-02T03:04:05+00:00", + "id": "11111111-1111-1111-1111-111111111111", + "instances": [], + "name": "test-fleet", + "project_name": "test-project", + "spec": { + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": null, + "max_price": null, + "name": "test-fleet", + "nodes": { + "max": 1, + "min": 1 + }, + "placement": null, + "regions": null, + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": null, + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": "fleet.dstack.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } + }, + "status": "active", + "status_message": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.input.json new file mode 100644 index 0000000000..0f302086b6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.input.json @@ -0,0 +1,16 @@ +{ + "run_name": "test-run", + "repo_id": "test-repo", + "repo_data": {"repo_type": "local", "repo_dir": "/"}, + "configuration_path": "dstack.yaml", + "configuration": { + "type": "task", + "commands": ["echo hi"], + "python": "3.10", + "resources": {"cpu": "2..8", "memory": "16GB.."}, + "env": ["A=1", "B"] + }, + "profile": {"name": "default", "max_duration": "2h"}, + "ssh_key_pub": "user_ssh_key", + "unknown_field_from_a_newer_server": {"nested": [1, 2]} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.types.json new file mode 100644 index 0000000000..e11749d6c6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.types.json @@ -0,0 +1,23 @@ +{ + "/": "RunSpec", + "/configuration": "TaskConfiguration", + "/configuration/env": "Env", + "/configuration/env/__root__/B": "EnvSentinel", + "/configuration/python": "PythonVersion", + "/configuration/resources": "ResourcesSpec", + "/configuration/resources/cpu": "CPUSpec", + "/configuration/resources/cpu/count": "Range[int]", + "/configuration/resources/disk": "DiskSpec", + "/configuration/resources/disk/size": "Range[Memory]", + "/configuration/resources/disk/size/min": "Memory", + "/configuration/resources/gpu": "GPUSpec", + "/configuration/resources/gpu/count": "Range[int]", + "/configuration/resources/memory": "Range[Memory]", + "/configuration/resources/memory/min": "Memory", + "/merged_profile": "Profile", + "/merged_profile/creation_policy": "CreationPolicy", + "/merged_profile/max_duration": "Duration", + "/profile": "Profile", + "/profile/max_duration": "Duration", + "/repo_data": "LocalRunRepoData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.values.json new file mode 100644 index 0000000000..21b6661191 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/run_spec.values.json @@ -0,0 +1,119 @@ +{ + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "commands": [ + "echo hi" + ], + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": { + "A": "1", + "B": { + "key": "B" + } + }, + "files": [], + "fleets": null, + "home_dir": "/root", + "idle_duration": null, + "image": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nodes": 1, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": "3.10", + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "task", + "user": null, + "utilization_policy": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": "dstack.yaml", + "file_archives": [], + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + }, + "repo_code_hash": null, + "repo_data": { + "repo_dir": "/", + "repo_type": "local" + }, + "repo_dir": null, + "repo_id": "test-repo", + "run_name": "test-run", + "ssh_key_pub": "user_ssh_key", + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/api/fleet.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/api/fleet.json rename to src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/db/job_provisioning_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_provisioning_data.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/db/job_provisioning_data.json rename to src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_provisioning_data.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/db/run_spec.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/run_spec.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/db/run_spec.json rename to src/tests/_internal/pydantic_compat/fixtures/serialization/db/run_spec.json diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py new file mode 100644 index 0000000000..368a48c142 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -0,0 +1,89 @@ +""" +Parse equality for stored blobs, request bodies, and client-side response parsing. + +Each case is a frozen input plus two expectations: the values parsing produced, and the concrete +classes it chose. Both are needed — `Duration(7200)` and `7200` are the same JSON, so the value +fixture alone cannot show that a custom type survived. + +Inputs are deliberately NOT canonical dumps. Feeding a model its own output back makes the +expected values identical to the input and the test proves nothing; every input here differs from +its output in some way a real writer could produce — an unknown field from a newer version, an +absent field that now defaults, or a sugared value. + +Inputs are hand-written and must never be regenerated. `--regen-fixtures` rewrites +`*.values.json` and `*.types.json` only. +""" + +import json +from typing import Any + +import pytest + +from dstack._internal.core.models.fleets import Fleet +from dstack._internal.core.models.runs import RunSpec +from dstack._internal.server.schemas.volumes import CreateVolumeRequest +from tests._internal.pydantic_compat.compare import ( + FIXTURES_DIR, + assert_matches_fixture, + type_map, +) +from tests._internal.pydantic_compat.compat import parse_permissive, parse_strict + +# Read from a `Text` column, permissively, so a row written by a newer server still loads. +DB_BLOBS: dict[str, Any] = { + "run_spec": RunSpec, +} + +# Validated from a request body, strictly: an unknown field is a user-facing error, not noise. +REQUEST_BODIES: dict[str, Any] = { + "create_volume_request": CreateVolumeRequest, +} + +# Parsed by the API client from a server response, permissively, so an older CLI works. +CLIENT_RESPONSES: dict[str, Any] = { + "fleet": Fleet, +} + + +class TestDbBlobParsing: + @pytest.mark.parametrize("name", sorted(DB_BLOBS)) + def test_parses_to_expected_values_and_types(self, name, regen): + _assert_parses( + "db", name, parse_permissive(DB_BLOBS[name], _load_input("db", name)), regen + ) + + +class TestRequestBodyParsing: + @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_strict(REQUEST_BODIES[name], _load_input("api_request", name)) + _assert_parses("api_request", name, model, regen) + + @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) + def test_unknown_field_is_still_rejected(self, name): + """ + Strictness is the feature here: it is what makes `dstack apply` report a typo'd key + instead of silently ignoring it. The v2 `CoreModel` is strict with a per-call + `extra="ignore"` override, so the regression to guard against is that override leaking + onto a request path. + """ + body = {**_load_input("api_request", name), "definitely_not_a_field": 1} + with pytest.raises(Exception, match="(?i)extra"): + parse_strict(REQUEST_BODIES[name], body) + + +class TestClientResponseParsing: + @pytest.mark.parametrize("name", sorted(CLIENT_RESPONSES)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_permissive(CLIENT_RESPONSES[name], _load_input("api_response", name)) + _assert_parses("api_response", name, model, regen) + + +def _assert_parses(surface: str, name: str, model, regen: bool) -> None: + kind = f"parsing/{surface}" + assert_matches_fixture(kind, f"{name}.values", model.json(), regen=regen) + assert_matches_fixture(kind, f"{name}.types", json.dumps(type_map(model)), regen=regen) + + +def _load_input(surface: str, name: str) -> Any: + return json.loads((FIXTURES_DIR / "parsing" / surface / f"{name}.input.json").read_text()) diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 600b4a613b..95e6d2ade9 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -39,7 +39,7 @@ class TestDbBlobSerialization: @pytest.mark.parametrize("name", sorted(DB_BLOBS)) def test_matches_fixture(self, name, regen): payload = DB_BLOBS[name]().json() - assert_matches_fixture("db", name, payload, regen=regen) + assert_matches_fixture("serialization/db", name, payload, regen=regen) class TestApiResponseSerialization: @@ -48,7 +48,7 @@ def test_matches_fixture(self, name, regen): # Response bodies go through orjson with a `default=` hook, not through `.json()`, so the # two paths can drift apart. Serialize the way the router does. payload = bytes(CustomORJSONResponse(API_RESPONSES[name]()).body) - assert_matches_fixture("api", name, payload, regen=regen) + assert_matches_fixture("serialization/api_response", name, payload, regen=regen) class TestFleetNodesTargetCompatHack: From ad87c9de38a5b5a6c9c8754caa5bd60bcdb12be8 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 17:02:53 +0500 Subject: [PATCH 03/15] Test api request serialization --- src/tests/_internal/pydantic_compat/compat.py | 26 ++++-- .../_internal/pydantic_compat/factories.py | 33 ++++--- .../config/fleet_nodes_range.input.yml | 6 ++ .../config/fleet_nodes_range.types.json | 19 ++++ .../config/fleet_nodes_range.values.json | 54 +++++++++++ .../config/profiles_durations.input.yml | 6 ++ .../config/profiles_durations.types.json | 7 ++ .../config/profiles_durations.values.json | 28 ++++++ .../parsing/config/task_sugared.input.yml | 17 ++++ .../parsing/config/task_sugared.types.json | 18 ++++ .../parsing/config/task_sugared.values.json | 83 +++++++++++++++++ .../api_request/apply_fleet_plan_request.json | 58 ++++++++++++ .../api_request/delete_fleets_request.json | 6 ++ .../_internal/pydantic_compat/test_parsing.py | 49 +++++++--- .../pydantic_compat/test_rejection.py | 89 +++++++++++++++++++ .../pydantic_compat/test_serialization.py | 14 +++ 16 files changed, 482 insertions(+), 31 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/delete_fleets_request.json create mode 100644 src/tests/_internal/pydantic_compat/test_rejection.py diff --git a/src/tests/_internal/pydantic_compat/compat.py b/src/tests/_internal/pydantic_compat/compat.py index fd9ee43993..2a67c1b3e2 100644 --- a/src/tests/_internal/pydantic_compat/compat.py +++ b/src/tests/_internal/pydantic_compat/compat.py @@ -2,8 +2,13 @@ The only place in this package allowed to branch on pydantic version. Every test here has to run unchanged on v1 and v2, which rules out touching the duality API -directly. Strict parsing needs no help — `parse_obj` is strict in both versions. Permissive -parsing does: v1 spells it `X.__response__`, and v2 will spell it `validate_extra_ignore`. +directly. Forbidding extra fields needs no help — `parse_obj` forbids them in both versions. +Ignoring them does: v1 spells it `X.__response__`, and v2 will spell it `validate_extra_ignore`. + +Both helpers are named for the `extra` setting they apply, deliberately avoiding the word +"strict": pydantic's `strict` is an unrelated axis that turns off type coercion, and a migration +that may well want real strict mode later should not have the term already spent on something +else. """ from typing import Any @@ -14,17 +19,22 @@ PYDANTIC_V1 = pydantic.VERSION.startswith("1.") -def parse_strict(model: Any, data: Any) -> BaseModel: - """Parse the way the server validates a request body: unknown fields are an error.""" +def parse_forbid_extra(model: Any, data: Any) -> BaseModel: + """ + Parse with `extra="forbid"` — the way the server validates a request body or a user's YAML. + + An unknown field is an error, which is what makes `dstack apply` report a typo'd key instead + of silently ignoring it. + """ return model.parse_obj(data) -def parse_permissive(model: Any, data: Any) -> BaseModel: +def parse_ignore_extra(model: Any, data: Any) -> BaseModel: """ - Parse the way a stored blob or a peer's response is read: unknown fields are ignored. + Parse with `extra="ignore"` — the way a stored blob or a peer's response is read. - This is what lets an older reader survive a newer writer, so it is the behaviour the whole - migration has to preserve. + Unknown fields are dropped, which is what lets an older reader survive a newer writer, so it + is the behaviour the whole migration has to preserve. """ if PYDANTIC_V1: return model.__response__.parse_obj(data) diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index f10e00860d..a97fb15689 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -16,6 +16,11 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.resources import Memory, Range, ResourcesSpec from dstack._internal.core.models.runs import JobProvisioningData, RunSpec +from dstack._internal.server.schemas.fleets import ( + ApplyFleetPlanInput, + ApplyFleetPlanRequest, + DeleteFleetsRequest, +) from dstack._internal.server.testing.common import ( get_fleet_spec, get_job_provisioning_data, @@ -27,7 +32,6 @@ def job_provisioning_data() -> JobProvisioningData: - """Stored in `JobModel.job_provisioning_data` and `InstanceModel.job_provisioning_data`.""" return get_job_provisioning_data( dockerized=True, availability_zone="us-east-1a", @@ -37,15 +41,9 @@ def job_provisioning_data() -> JobProvisioningData: def run_spec() -> RunSpec: """ - Stored in `RunModel.run_spec` — the largest blob, and the one carrying the type zoo. - - `max_duration`/`idle_duration` are `Duration` (an `int` subclass) and `resources` carries - `Memory` and `Range[T]`, so this fixture pins how the custom types render. Those are the types - whose `__get_validators__` → `__get_pydantic_core_schema__` port is riskiest. - Values are given in their already-parsed form rather than as YAML shorthand (`"2h"`, `"16GB.."`): shorthand only type-checks because a `pre=True` validator widens the input, and - pinning the *parse* side is the job of the parse fixtures, not these. + pinning the *parse* side is the job of the parsing fixtures, not these. """ return get_run_spec( repo_id="test-repo", @@ -62,11 +60,9 @@ def run_spec() -> RunSpec: def fleet() -> Fleet: """ - Returned by `/api/project/{name}/fleets/list` through `CustomORJSONResponse`. - The default `FleetNodesSpec` has `target == min`, which is what makes `FleetNodesSpec.dict()` drop `target` — the old-client compat hack from #3066. That override becomes a - `@model_serializer` in v2, so this fixture is the thing that proves the hack survived. + `@model_serializer` in v2, so this fixture is what proves the hack survived. """ return Fleet( id=_ID, @@ -88,3 +84,18 @@ def fleet() -> Fleet: ) ], ) + + +def delete_fleets_request() -> DeleteFleetsRequest: + return DeleteFleetsRequest(names=["fleet-a", "fleet-b"]) + + +def apply_fleet_plan_request() -> ApplyFleetPlanRequest: + """ + Chosen over a simpler request body because it carries a whole `FleetSpec`, so it covers the + client side of the #3066 `target`-dropping hack too. + """ + return ApplyFleetPlanRequest( + plan=ApplyFleetPlanInput(spec=get_fleet_spec(), current_resource=None), + force=False, + ) diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.input.yml new file mode 100644 index 0000000000..0e2a48f119 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.input.yml @@ -0,0 +1,6 @@ +type: fleet +name: test-fleet +nodes: 1..2 +idle_duration: 2h +resources: + gpu: 24GB.. diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.types.json new file mode 100644 index 0000000000..57665a1fde --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.types.json @@ -0,0 +1,19 @@ +{ + "/": "DstackConfiguration", + "/__root__": "FleetConfiguration", + "/__root__/env": "Env", + "/__root__/idle_duration": "Duration", + "/__root__/nodes": "FleetNodesSpec", + "/__root__/resources": "ResourcesSpec", + "/__root__/resources/cpu": "CPUSpec", + "/__root__/resources/cpu/count": "Range[int]", + "/__root__/resources/disk": "DiskSpec", + "/__root__/resources/disk/size": "Range[Memory]", + "/__root__/resources/disk/size/min": "Memory", + "/__root__/resources/gpu": "GPUSpec", + "/__root__/resources/gpu/count": "Range[int]", + "/__root__/resources/gpu/memory": "Range[Memory]", + "/__root__/resources/gpu/memory/min": "Memory", + "/__root__/resources/memory": "Range[Memory]", + "/__root__/resources/memory/min": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.values.json new file mode 100644 index 0000000000..66cac17673 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.values.json @@ -0,0 +1,54 @@ +{ + "availability_zones": null, + "backend_options": null, + "backends": null, + "blocks": 1, + "env": {}, + "idle_duration": 7200, + "instance_types": null, + "max_price": null, + "name": "test-fleet", + "nodes": { + "max": 2, + "min": 1 + }, + "placement": null, + "regions": null, + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 1 + }, + "memory": { + "max": null, + "min": 24.0 + }, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "retry": null, + "spot_policy": null, + "ssh_config": null, + "tags": null, + "type": "fleet" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.input.yml new file mode 100644 index 0000000000..3f4a1a3761 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.input.yml @@ -0,0 +1,6 @@ +profiles: + - name: default + max_duration: 2h + stop_duration: off + idle_duration: 300 + spot_policy: auto diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.types.json new file mode 100644 index 0000000000..93d04a9e6d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.types.json @@ -0,0 +1,7 @@ +{ + "/": "ProfilesConfig", + "/profiles/0": "Profile", + "/profiles/0/idle_duration": "Duration", + "/profiles/0/max_duration": "Duration", + "/profiles/0/spot_policy": "SpotPolicy" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.values.json new file mode 100644 index 0000000000..dead41fb3a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.values.json @@ -0,0 +1,28 @@ +{ + "profiles": [ + { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": 300, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": "auto", + "startup_order": null, + "stop_criteria": null, + "stop_duration": "off", + "tags": null, + "utilization_policy": null + } + ] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.input.yml new file mode 100644 index 0000000000..3f5a530be8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.input.yml @@ -0,0 +1,17 @@ +# Sugared forms a user actually types. Every one of these depends on a `pre=True` validator or +# on number->str coercion, so this file is the highest-value parse input in the suite. +type: task +commands: + - echo hi +python: 3.10 # a YAML float, not a string +nodes: 2 +max_duration: 2h +idle_duration: off +resources: + cpu: 2..8 + memory: 16GB.. + gpu: A100:2 + disk: 100GB.. +env: + - A=1 + - B # no value -> EnvSentinel diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.types.json new file mode 100644 index 0000000000..a3d384fa38 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.types.json @@ -0,0 +1,18 @@ +{ + "/": "DstackConfiguration", + "/__root__": "TaskConfiguration", + "/__root__/env": "Env", + "/__root__/env/__root__/B": "EnvSentinel", + "/__root__/max_duration": "Duration", + "/__root__/python": "PythonVersion", + "/__root__/resources": "ResourcesSpec", + "/__root__/resources/cpu": "CPUSpec", + "/__root__/resources/cpu/count": "Range[int]", + "/__root__/resources/disk": "DiskSpec", + "/__root__/resources/disk/size": "Range[Memory]", + "/__root__/resources/disk/size/min": "Memory", + "/__root__/resources/gpu": "GPUSpec", + "/__root__/resources/gpu/count": "Range[int]", + "/__root__/resources/memory": "Range[Memory]", + "/__root__/resources/memory/min": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.values.json new file mode 100644 index 0000000000..d6cf38dd13 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.values.json @@ -0,0 +1,83 @@ +{ + "availability_zones": null, + "backend_options": null, + "backends": null, + "commands": [ + "echo hi" + ], + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": { + "A": "1", + "B": { + "key": "B" + } + }, + "files": [], + "fleets": null, + "home_dir": "/root", + "idle_duration": -1, + "image": null, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": null, + "nodes": 2, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": "3.10", + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": 2, + "min": 2 + }, + "memory": null, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "task", + "user": null, + "utilization_policy": null, + "volumes": [], + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json new file mode 100644 index 0000000000..5bafec8547 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json @@ -0,0 +1,58 @@ +{ + "force": false, + "plan": { + "current_resource": null, + "spec": { + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": null, + "max_price": null, + "name": "test-fleet", + "nodes": { + "max": 1, + "min": 1 + }, + "placement": null, + "regions": null, + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": null, + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": "fleet.dstack.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/delete_fleets_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/delete_fleets_request.json new file mode 100644 index 0000000000..477d428797 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/delete_fleets_request.json @@ -0,0 +1,6 @@ +{ + "names": [ + "fleet-a", + "fleet-b" + ] +} diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index 368a48c142..6130aebfda 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -18,8 +18,11 @@ from typing import Any import pytest +import yaml +from dstack._internal.core.models.configurations import DstackConfiguration from dstack._internal.core.models.fleets import Fleet +from dstack._internal.core.models.profiles import ProfilesConfig from dstack._internal.core.models.runs import RunSpec from dstack._internal.server.schemas.volumes import CreateVolumeRequest from tests._internal.pydantic_compat.compare import ( @@ -27,58 +30,76 @@ assert_matches_fixture, type_map, ) -from tests._internal.pydantic_compat.compat import parse_permissive, parse_strict +from tests._internal.pydantic_compat.compat import parse_forbid_extra, parse_ignore_extra -# Read from a `Text` column, permissively, so a row written by a newer server still loads. +# Read from a `Text` column with extra="ignore", so a row written by a newer server loads. DB_BLOBS: dict[str, Any] = { "run_spec": RunSpec, } -# Validated from a request body, strictly: an unknown field is a user-facing error, not noise. +# Validated from a request body with extra="forbid": an unknown field is a user-facing error. REQUEST_BODIES: dict[str, Any] = { "create_volume_request": CreateVolumeRequest, } -# Parsed by the API client from a server response, permissively, so an older CLI works. +# Parsed by the API client from a server response with extra="ignore", so an older CLI works. CLIENT_RESPONSES: dict[str, Any] = { "fleet": Fleet, } +# Parsed from user-authored YAML with extra="forbid". The only surface whose inputs are `.yml`, +# because the sugar pinned here is a YAML phenomenon: `python: 3.10` is a float that only survives via +# number->str coercion, and `16GB..` / `2..8` / an env list are shorthands people type by hand. +# `DstackConfiguration` dispatches every `.dstack.yml` type through its `__root__` union, so one +# entry point covers task, service, dev-environment, fleet, volume, and gateway. +CONFIGS: dict[str, Any] = { + "task_sugared": DstackConfiguration, + "fleet_nodes_range": DstackConfiguration, + "profiles_durations": ProfilesConfig, +} + class TestDbBlobParsing: @pytest.mark.parametrize("name", sorted(DB_BLOBS)) def test_parses_to_expected_values_and_types(self, name, regen): _assert_parses( - "db", name, parse_permissive(DB_BLOBS[name], _load_input("db", name)), regen + "db", name, parse_ignore_extra(DB_BLOBS[name], _load_input("db", name)), regen ) class TestRequestBodyParsing: @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) def test_parses_to_expected_values_and_types(self, name, regen): - model = parse_strict(REQUEST_BODIES[name], _load_input("api_request", name)) + model = parse_forbid_extra(REQUEST_BODIES[name], _load_input("api_request", name)) _assert_parses("api_request", name, model, regen) @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) def test_unknown_field_is_still_rejected(self, name): """ - Strictness is the feature here: it is what makes `dstack apply` report a typo'd key - instead of silently ignoring it. The v2 `CoreModel` is strict with a per-call - `extra="ignore"` override, so the regression to guard against is that override leaking - onto a request path. + Forbidding extra fields is the feature here: it is what makes `dstack apply` report a + typo'd key instead of silently ignoring it. The v2 `CoreModel` forbids them by default + with a per-call `extra="ignore"` override, so the regression to guard against is that + override leaking onto a request path. """ body = {**_load_input("api_request", name), "definitely_not_a_field": 1} with pytest.raises(Exception, match="(?i)extra"): - parse_strict(REQUEST_BODIES[name], body) + parse_forbid_extra(REQUEST_BODIES[name], body) class TestClientResponseParsing: @pytest.mark.parametrize("name", sorted(CLIENT_RESPONSES)) def test_parses_to_expected_values_and_types(self, name, regen): - model = parse_permissive(CLIENT_RESPONSES[name], _load_input("api_response", name)) + model = parse_ignore_extra(CLIENT_RESPONSES[name], _load_input("api_response", name)) _assert_parses("api_response", name, model, regen) +class TestConfigParsing: + @pytest.mark.parametrize("name", sorted(CONFIGS)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_forbid_extra(CONFIGS[name], _load_yaml_input("config", name)) + _assert_parses("config", name, model, regen) + + def _assert_parses(surface: str, name: str, model, regen: bool) -> None: kind = f"parsing/{surface}" assert_matches_fixture(kind, f"{name}.values", model.json(), regen=regen) @@ -87,3 +108,7 @@ def _assert_parses(surface: str, name: str, model, regen: bool) -> None: def _load_input(surface: str, name: str) -> Any: return json.loads((FIXTURES_DIR / "parsing" / surface / f"{name}.input.json").read_text()) + + +def _load_yaml_input(surface: str, name: str) -> Any: + return yaml.safe_load((FIXTURES_DIR / "parsing" / surface / f"{name}.input.yml").read_text()) diff --git a/src/tests/_internal/pydantic_compat/test_rejection.py b/src/tests/_internal/pydantic_compat/test_rejection.py new file mode 100644 index 0000000000..30dce4c174 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/test_rejection.py @@ -0,0 +1,89 @@ +""" +Rejection parity: input that pydantic v1 refuses must still be refused under v2. + +Two rules: + +- Assert only that validation raised. Never assert on the message text: v2 rewords errors by + design, so a message assertion would fail for a reason we do not care about. The `dstack apply` + error UX gets reviewed by hand instead. +- Inputs live inline, not in fixture files. Nothing is generated from them, they are one or two + keys each, and a file per case would add indirection with no payoff. +""" + +from typing import Any, Callable + +import pytest +import yaml +from pydantic import ValidationError + +from dstack._internal.core.models.configurations import DstackConfiguration +from dstack._internal.server.schemas.volumes import CreateVolumeRequest +from tests._internal.pydantic_compat.compat import parse_forbid_extra + +_VALID_TASK = "type: task\ncommands: [echo hi]\n" + + +def _task(extra_yaml: str) -> Any: + return yaml.safe_load(_VALID_TASK + extra_yaml) + + +# name -> (parser, input). Each input is rejected by v1 today; the test pins that it stays rejected. +REJECTED_INPUTS: dict[str, tuple[Callable, Any]] = { + # A mistyped key must not be silently ignored — this is the whole point of `extra="forbid"`, + # and the v2 `CoreModel` reaches it through a per-call override that could leak. + "config_unknown_key": (DstackConfiguration.parse_obj, _task("comands: [oops]\n")), + "request_unknown_key": ( + CreateVolumeRequest.parse_obj, + { + "configuration": {"type": "volume", "name": "v", "backend": "aws", "region": "r"}, + "unexpected": 1, + }, + ), + # Reversed ranges: caught by a validator, not by the type, so a dropped validator during the + # `__get_pydantic_core_schema__` port would make these start passing. + "resources_cpu_reversed_range": ( + DstackConfiguration.parse_obj, + _task("resources:\n cpu: 8..2\n"), + ), + "resources_memory_reversed_range": ( + DstackConfiguration.parse_obj, + _task("resources:\n memory: 32GB..8GB\n"), + ), + # Custom-type parsing: `Duration.parse` rejects unknown units and non-numeric input. + "duration_bad_unit": (DstackConfiguration.parse_obj, _task("max_duration: 5 years\n")), + # An unknown discriminator tag. After Phase 0 item 2 these unions are discriminated, so the + # error should name the tag rather than accumulate one failure per arm. + "unknown_config_type": (DstackConfiguration.parse_obj, {"type": "not-a-real-type"}), +} + + +class TestRejectionParity: + @pytest.mark.parametrize("name", sorted(REJECTED_INPUTS)) + def test_still_rejected(self, name): + parser, data = REJECTED_INPUTS[name] + with pytest.raises(ValidationError): + parser(data) + + +class TestExtraHandlingIsNotAccidentallyRelaxed: + """ + A guard on the mechanism rather than on any one model. + + `parse_forbid_extra` must keep forbidding. If the v2 rewrite routes request bodies through + `parse_ignore_extra` by mistake, every case above would still pass while user typos quietly + stopped being reported — so assert the forbidding helper actually rejects. + """ + + def test_forbid_extra_rejects_an_unknown_field(self): + body = { + "configuration": { + "type": "volume", + "name": "v", + "backend": "aws", + "region": "r", + "size": "1GB", + }, + "unexpected": 1, + } + with pytest.raises(ValidationError): + parse_forbid_extra(CreateVolumeRequest, body) diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 95e6d2ade9..3492cc5641 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -34,6 +34,13 @@ "fleet": factories.fleet, } +# Sent by the API client as a request body — `body=X.json()`, 40 call sites in `api/server/`. +# This is the new-CLI-against-old-server direction, which nothing else in the suite covers. +API_REQUESTS: dict[str, Callable[[], CoreModel]] = { + "delete_fleets_request": factories.delete_fleets_request, + "apply_fleet_plan_request": factories.apply_fleet_plan_request, +} + class TestDbBlobSerialization: @pytest.mark.parametrize("name", sorted(DB_BLOBS)) @@ -51,6 +58,13 @@ def test_matches_fixture(self, name, regen): assert_matches_fixture("serialization/api_response", name, payload, regen=regen) +class TestApiRequestSerialization: + @pytest.mark.parametrize("name", sorted(API_REQUESTS)) + def test_matches_fixture(self, name, regen): + payload = API_REQUESTS[name]().json() + assert_matches_fixture("serialization/api_request", name, payload, regen=regen) + + class TestFleetNodesTargetCompatHack: """ Pins the #3066 old-client hack explicitly, not just via the fixture bytes. From 9b21fe13973859600df3a6554b621bace53f6159 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 17:21:16 +0500 Subject: [PATCH 04/15] Scale out db serialization tests --- .../_internal/pydantic_compat/compare.py | 10 + .../_internal/pydantic_compat/factories.py | 215 +++++++++++++++++- ..._nodes_range.input.yml => fleet.input.yml} | 0 ...odes_range.types.json => fleet.types.json} | 0 ...es_range.values.json => fleet.values.json} | 0 ...durations.input.yml => profiles.input.yml} | 0 ...rations.types.json => profiles.types.json} | 0 ...tions.values.json => profiles.values.json} | 0 .../fixtures/parsing/config/service.input.yml | 6 + .../parsing/config/service.types.json | 17 ++ .../parsing/config/service.values.json | 93 ++++++++ ...{task_sugared.input.yml => task.input.yml} | 0 ...ask_sugared.types.json => task.types.json} | 0 ...k_sugared.values.json => task.values.json} | 0 .../fixtures/parsing/config/volume.input.yml | 5 + .../fixtures/parsing/config/volume.types.json | 7 + .../parsing/config/volume.values.json | 11 + .../fixtures/parsing/db/aws_creds.input.json | 6 + .../fixtures/parsing/db/aws_creds.types.json | 4 + .../fixtures/parsing/db/aws_creds.values.json | 5 + .../fixtures/parsing/db/job_spec.input.json | 8 + .../fixtures/parsing/db/job_spec.types.json | 14 ++ .../fixtures/parsing/db/job_spec.values.json | 72 ++++++ .../fixtures/serialization/db/aws_creds.json | 5 + .../db/compute_group_provisioning_data.json | 9 + .../fixtures/serialization/db/fleet_spec.json | 52 +++++ .../db/gateway_compute_configuration.json | 12 + .../db/gateway_configuration.json | 16 ++ .../serialization/db/image_pull_progress.json | 6 + .../db/instance_configuration.json | 10 + .../serialization/db/instance_offer.json | 25 ++ .../serialization/db/job_runtime_data.json | 14 ++ .../fixtures/serialization/db/job_spec.json | 80 +++++++ .../db/placement_group_configuration.json | 5 + .../db/placement_group_provisioning_data.json | 4 + .../fixtures/serialization/db/profile.json | 29 +++ .../db/remote_connection_info.json | 14 ++ .../serialization/db/requirements.json | 43 ++++ .../fixtures/serialization/db/resources.json | 17 ++ .../serialization/db/service_spec.json | 5 + .../db/volume_attachment_data.json | 3 + .../db/volume_configuration.json | 11 + .../db/volume_provisioning_data.json | 10 + .../_internal/pydantic_compat/test_parsing.py | 29 ++- .../pydantic_compat/test_serialization.py | 20 ++ 45 files changed, 881 insertions(+), 11 deletions(-) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{fleet_nodes_range.input.yml => fleet.input.yml} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{fleet_nodes_range.types.json => fleet.types.json} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{fleet_nodes_range.values.json => fleet.values.json} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{profiles_durations.input.yml => profiles.input.yml} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{profiles_durations.types.json => profiles.types.json} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{profiles_durations.values.json => profiles.values.json} (100%) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{task_sugared.input.yml => task.input.yml} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{task_sugared.types.json => task.types.json} (100%) rename src/tests/_internal/pydantic_compat/fixtures/parsing/config/{task_sugared.values.json => task.values.json} (100%) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/compute_group_provisioning_data.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/image_pull_progress.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_offer.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_spec.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_configuration.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_provisioning_data.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/remote_connection_info.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/requirements.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/resources.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/service_spec.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_attachment_data.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_configuration.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_provisioning_data.json diff --git a/src/tests/_internal/pydantic_compat/compare.py b/src/tests/_internal/pydantic_compat/compare.py index 31768fb890..6170f2be18 100644 --- a/src/tests/_internal/pydantic_compat/compare.py +++ b/src/tests/_internal/pydantic_compat/compare.py @@ -4,6 +4,16 @@ Fixtures are generated under pydantic v1 and committed. The same tests then run under v2, so a mismatch means the two versions disagree about the wire format. Regenerating a fixture IS the act of accepting a wire change — every accepted diff shows up in review as a fixture diff. + +Layout is `fixtures///[.].`: + +- direction: `serialization` or `parsing` +- surface: `db`, `api_request`, `api_response`, `config` +- variant: omitted while a model has only one case; added to *every* case for that model as soon + as a second one exists, so `volume.input.yml` becomes `volume.size.input.yml` and + `volume.kubernetes.input.yml` together +- role: `input` for hand-written parse inputs, `values` / `types` for generated expectations, and + a bare `.json` for serialization fixtures, which need no input """ import difflib diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index a97fb15689..5408c38381 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -9,28 +9,131 @@ import uuid from datetime import datetime, timezone +from dstack._internal.core.backends.aws.models import AWSCreds from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.compute_groups import ComputeGroupProvisioningData from dstack._internal.core.models.configurations import DevEnvironmentConfiguration -from dstack._internal.core.models.fleets import Fleet, FleetStatus -from dstack._internal.core.models.instances import Instance, InstanceStatus -from dstack._internal.core.models.profiles import Profile -from dstack._internal.core.models.resources import Memory, Range, ResourcesSpec -from dstack._internal.core.models.runs import JobProvisioningData, RunSpec +from dstack._internal.core.models.envs import Env +from dstack._internal.core.models.fleets import Fleet, FleetSpec, FleetStatus +from dstack._internal.core.models.gateways import ( + GatewayComputeConfiguration, + GatewayConfiguration, +) +from dstack._internal.core.models.instances import ( + Disk, + Gpu, + Instance, + InstanceConfiguration, + InstanceOffer, + InstanceStatus, + RemoteConnectionInfo, + Resources, +) +from dstack._internal.core.models.placement import ( + PlacementGroupConfiguration, + PlacementGroupProvisioningData, +) +from dstack._internal.core.models.profiles import ( + Profile, + ProfileRetry, + RetryEvent, + SpotPolicy, +) +from dstack._internal.core.models.resources import ( + ComputeCapability, + GPUSpec, + Memory, + Range, + ResourcesSpec, +) +from dstack._internal.core.models.runs import ( + ImagePullProgress, + JobProvisioningData, + JobRuntimeData, + JobSpec, + Requirements, + RunSpec, + ServiceSpec, +) +from dstack._internal.core.models.volumes import ( + VolumeAttachmentData, + VolumeConfiguration, + VolumeProvisioningData, +) from dstack._internal.server.schemas.fleets import ( ApplyFleetPlanInput, ApplyFleetPlanRequest, DeleteFleetsRequest, ) from dstack._internal.server.testing.common import ( + get_compute_group_provisioning_data, get_fleet_spec, + get_gateway_compute_configuration, + get_instance_configuration, + get_instance_offer_with_availability, get_job_provisioning_data, + get_job_runtime_data, + get_placement_group_configuration, + get_placement_group_provisioning_data, + get_remote_connection_info, get_run_spec, + get_volume_configuration, + get_volume_provisioning_data, ) _ID = uuid.UUID("11111111-1111-1111-1111-111111111111") _CREATED_AT = datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc) +# --- DB blobs ------------------------------------------------------------------------ +# Written to a `Text` column with `.json()`. Rows outlive any release, so each stored model +# is an independent liability and gets its own fixture rather than standing in for a shared +# shape — unlike the API surfaces below, which are covered by feature instead. + + +def aws_creds() -> AWSCreds: + """A custom-root discriminated union, read from `BackendModel.auth`.""" + return AWSCreds.parse_obj({"type": "access_key", "access_key": "AK", "secret_key": "SK"}) + + +def compute_group_provisioning_data() -> ComputeGroupProvisioningData: + return get_compute_group_provisioning_data() + + +def fleet_spec() -> FleetSpec: + return get_fleet_spec() + + +def gateway_compute_configuration() -> GatewayComputeConfiguration: + return get_gateway_compute_configuration() + + +def gateway_configuration() -> GatewayConfiguration: + return GatewayConfiguration( + name="test-gateway", + backend=BackendType.AWS, + region="us-east-1", + domain="example.com", + ) + + +def image_pull_progress() -> ImagePullProgress: + return ImagePullProgress( + downloaded_bytes=1024, + extracted_bytes=512, + total_bytes=4096, + is_total_bytes_final=True, + ) + + +def instance_configuration() -> InstanceConfiguration: + return get_instance_configuration() + + +def instance_offer() -> InstanceOffer: + return get_instance_offer_with_availability() + + def job_provisioning_data() -> JobProvisioningData: return get_job_provisioning_data( dockerized=True, @@ -39,6 +142,80 @@ def job_provisioning_data() -> JobProvisioningData: ) +def job_runtime_data() -> JobRuntimeData: + """`ports` is a `dict[int, int]` — the only non-str dict keys in the models.""" + data = get_job_runtime_data() + data.ports = {8080: 30080, 8081: 30081} + return data + + +def job_spec() -> JobSpec: + return JobSpec( + job_num=0, + job_name="test-run-0-0", + commands=["/bin/bash", "-i", "-c", "echo hi"], + env=Env.parse_obj({"A": "1"}), + image_name="dstackai/base:latest", + requirements=requirements(), + max_duration=7200, + working_dir="/workflow", + ) + + +def placement_group_configuration() -> PlacementGroupConfiguration: + return get_placement_group_configuration() + + +def placement_group_provisioning_data() -> PlacementGroupProvisioningData: + return get_placement_group_provisioning_data() + + +def profile() -> Profile: + return Profile( + name="default", + max_duration=7200, + stop_duration=300, + idle_duration=600, + spot_policy=SpotPolicy.AUTO, + retry=ProfileRetry(on_events=[RetryEvent.NO_CAPACITY], duration=3600), + ) + + +def remote_connection_info() -> RemoteConnectionInfo: + return get_remote_connection_info() + + +def requirements() -> Requirements: + """Carries `ComputeCapability`, a `Tuple[int, int]` subclass nothing else covers.""" + return Requirements( + resources=ResourcesSpec( + cpu=Range[int](min=2, max=8), + memory=Range[Memory](min=Memory(16), max=None), + gpu=GPUSpec( + name=["A100"], + count=Range[int](min=1, max=1), + memory=Range[Memory](min=Memory(40), max=None), + compute_capability=ComputeCapability((8, 0)), + ), + ), + max_price=10.5, + spot=False, + reservation="test-reservation", + ) + + +def resources() -> Resources: + """`Resources.dict()` rewrites `cpu` for old clients — the other custom serializer.""" + return Resources( + cpus=8, + memory_mib=16384, + gpus=[Gpu(name="A100", memory_mib=40960)], + spot=False, + disk=Disk(size_mib=102400), + description="8xCPU, 16GB, 1xA100", + ) + + def run_spec() -> RunSpec: """ Values are given in their already-parsed form rather than as YAML shorthand (`"2h"`, @@ -58,6 +235,26 @@ def run_spec() -> RunSpec: ) +def service_spec() -> ServiceSpec: + return ServiceSpec(url="/proxy/services/test-project/test-run/", model=None, options={}) + + +def volume_attachment_data() -> VolumeAttachmentData: + return VolumeAttachmentData(device_name="/dev/sdb") + + +def volume_configuration() -> VolumeConfiguration: + return get_volume_configuration() + + +def volume_provisioning_data() -> VolumeProvisioningData: + return get_volume_provisioning_data() + + +# --- API responses ------------------------------------------------------------------- +# Returned from a router through `CustomORJSONResponse`. + + def fleet() -> Fleet: """ The default `FleetNodesSpec` has `target == min`, which is what makes `FleetNodesSpec.dict()` @@ -86,8 +283,8 @@ def fleet() -> Fleet: ) -def delete_fleets_request() -> DeleteFleetsRequest: - return DeleteFleetsRequest(names=["fleet-a", "fleet-b"]) +# --- API request bodies -------------------------------------------------------------- +# Serialized by the API client as `body=X.json()`. def apply_fleet_plan_request() -> ApplyFleetPlanRequest: @@ -99,3 +296,7 @@ def apply_fleet_plan_request() -> ApplyFleetPlanRequest: plan=ApplyFleetPlanInput(spec=get_fleet_spec(), current_resource=None), force=False, ) + + +def delete_fleets_request() -> DeleteFleetsRequest: + return DeleteFleetsRequest(names=["fleet-a", "fleet-b"]) diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.input.yml similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.input.yml rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.input.yml diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.types.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.types.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.types.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.values.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet_nodes_range.values.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.values.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles.input.yml similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.input.yml rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles.input.yml diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles.types.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.types.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles.types.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles.values.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles_durations.values.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/profiles.values.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.input.yml new file mode 100644 index 0000000000..92deca264a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.input.yml @@ -0,0 +1,6 @@ +# `model: ` is the shorthand a discriminator cannot read on its own; it survives only +# because `convert_model` is a pre-validator. Phase 0 item 2 added that discriminator. +type: service +commands: [echo hi] +port: 8000 +model: llama diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json new file mode 100644 index 0000000000..5bef881a81 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json @@ -0,0 +1,17 @@ +{ + "/": "DstackConfiguration", + "/__root__": "ServiceConfiguration", + "/__root__/env": "Env", + "/__root__/model": "OpenAIChatModel", + "/__root__/port": "PortMapping", + "/__root__/resources": "ResourcesSpec", + "/__root__/resources/cpu": "CPUSpec", + "/__root__/resources/cpu/count": "Range[int]", + "/__root__/resources/disk": "DiskSpec", + "/__root__/resources/disk/size": "Range[Memory]", + "/__root__/resources/disk/size/min": "Memory", + "/__root__/resources/gpu": "GPUSpec", + "/__root__/resources/gpu/count": "Range[int]", + "/__root__/resources/memory": "Range[Memory]", + "/__root__/resources/memory/min": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json new file mode 100644 index 0000000000..cf387dcd93 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.values.json @@ -0,0 +1,93 @@ +{ + "auth": true, + "availability_zones": null, + "backend_options": null, + "backends": null, + "commands": [ + "echo hi" + ], + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "gateway": null, + "home_dir": "/root", + "https": null, + "idle_duration": null, + "image": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "model": { + "format": "openai", + "name": "llama", + "prefix": "/v1", + "type": "chat" + }, + "name": null, + "nvcc": null, + "port": { + "container_port": 8000, + "local_port": 80 + }, + "priority": null, + "privileged": false, + "probes": null, + "python": null, + "rate_limits": [], + "regions": null, + "registry_auth": null, + "replicas": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "retry": null, + "router": null, + "scaling": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "strip_prefix": true, + "tags": null, + "type": "service", + "user": null, + "utilization_policy": null, + "volumes": [], + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.input.yml similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.input.yml rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.input.yml diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.types.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.types.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.types.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.values.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/config/task_sugared.values.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.values.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.input.yml new file mode 100644 index 0000000000..44612a3076 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.input.yml @@ -0,0 +1,5 @@ +type: volume +name: my-volume +backend: aws +region: us-east-1 +size: 100GB diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json new file mode 100644 index 0000000000..beb7d52a31 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json @@ -0,0 +1,7 @@ +{ + "/": "DstackConfiguration", + "/__root__": "VolumeConfiguration", + "/__root__/__root__": "AWSVolumeConfiguration", + "/__root__/__root__/backend": "BackendType", + "/__root__/__root__/size": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.values.json new file mode 100644 index 0000000000..17b6be5701 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.values.json @@ -0,0 +1,11 @@ +{ + "auto_cleanup_duration": null, + "availability_zone": null, + "backend": "aws", + "name": "my-volume", + "region": "us-east-1", + "size": 100.0, + "tags": null, + "type": "volume", + "volume_id": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json new file mode 100644 index 0000000000..30e4a69d8f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json @@ -0,0 +1,6 @@ +{ + "type": "access_key", + "access_key": "AK", + "secret_key": "SK", + "rotation_hint_from_a_newer_server": "2027-01-01" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.types.json new file mode 100644 index 0000000000..ed38da62cb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.types.json @@ -0,0 +1,4 @@ +{ + "/": "AWSCreds", + "/__root__": "AWSAccessKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json new file mode 100644 index 0000000000..2a8ac73adb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json @@ -0,0 +1,5 @@ +{ + "access_key": "AK", + "secret_key": "SK", + "type": "access_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.input.json new file mode 100644 index 0000000000..ce49ff3851 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.input.json @@ -0,0 +1,8 @@ +{ + "job_num": 0, + "job_name": "test-run-0-0", + "commands": ["/bin/bash", "-i", "-c", "echo hi"], + "env": {"A": "1"}, + "image_name": "dstackai/base:latest", + "requirements": {"resources": {"cpu": "2..8", "memory": "16GB.."}} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.types.json new file mode 100644 index 0000000000..7b682bf8fd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.types.json @@ -0,0 +1,14 @@ +{ + "/": "JobSpec", + "/requirements": "Requirements", + "/requirements/resources": "ResourcesSpec", + "/requirements/resources/cpu": "CPUSpec", + "/requirements/resources/cpu/count": "Range[int]", + "/requirements/resources/disk": "DiskSpec", + "/requirements/resources/disk/size": "Range[Memory]", + "/requirements/resources/disk/size/min": "Memory", + "/requirements/resources/gpu": "GPUSpec", + "/requirements/resources/gpu/count": "Range[int]", + "/requirements/resources/memory": "Range[Memory]", + "/requirements/resources/memory/min": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.values.json new file mode 100644 index 0000000000..3c95dddd3c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_spec.values.json @@ -0,0 +1,72 @@ +{ + "app_specs": null, + "commands": [ + "/bin/bash", + "-i", + "-c", + "echo hi" + ], + "env": { + "A": "1" + }, + "file_archives": [], + "home_dir": null, + "image_name": "dstackai/base:latest", + "job_name": "test-run-0-0", + "job_num": 0, + "jobs_per_replica": 1, + "max_duration": null, + "privileged": false, + "probes": [], + "registry_auth": null, + "replica_group": "0", + "replica_num": 0, + "repo_code_hash": null, + "repo_data": null, + "repo_dir": "/workflow", + "repo_exists_action": null, + "requirements": { + "backend_options": null, + "max_price": null, + "multinode": null, + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "spot": null + }, + "retry": null, + "service_port": null, + "single_branch": null, + "ssh_key": null, + "stop_duration": null, + "user": null, + "utilization_policy": null, + "volumes": null, + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json new file mode 100644 index 0000000000..2a8ac73adb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json @@ -0,0 +1,5 @@ +{ + "access_key": "AK", + "secret_key": "SK", + "type": "access_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/compute_group_provisioning_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/compute_group_provisioning_data.json new file mode 100644 index 0000000000..e6b9fdb5d6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/compute_group_provisioning_data.json @@ -0,0 +1,9 @@ +{ + "backend": "runpod", + "backend_data": null, + "base_backend": null, + "compute_group_id": "test_compute_group", + "compute_group_name": "test_compute_group", + "job_provisioning_datas": [], + "region": "US" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json new file mode 100644 index 0000000000..bd38276d3e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json @@ -0,0 +1,52 @@ +{ + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": null, + "max_price": null, + "name": "test-fleet", + "nodes": { + "max": 1, + "min": 1 + }, + "placement": null, + "regions": null, + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": null, + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": "fleet.dstack.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json new file mode 100644 index 0000000000..7c78b5dd14 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_compute_configuration.json @@ -0,0 +1,12 @@ +{ + "backend": "aws", + "certificate": null, + "instance_name": "test-instance", + "instance_type": null, + "project_name": "test-project", + "public_ip": true, + "region": "us", + "router": null, + "ssh_key_pub": "", + "tags": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json new file mode 100644 index 0000000000..37a4ff056a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/gateway_configuration.json @@ -0,0 +1,16 @@ +{ + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "default": false, + "domain": "example.com", + "instance_type": null, + "name": "test-gateway", + "public_ip": true, + "region": "us-east-1", + "replicas": null, + "router": null, + "tags": null, + "type": "gateway" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/image_pull_progress.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/image_pull_progress.json new file mode 100644 index 0000000000..470d6c4ee7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/image_pull_progress.json @@ -0,0 +1,6 @@ +{ + "downloaded_bytes": 1024, + "extracted_bytes": 512, + "is_total_bytes_final": true, + "total_bytes": 4096 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json new file mode 100644 index 0000000000..39b0c11e78 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json @@ -0,0 +1,10 @@ +{ + "instance_id": null, + "instance_name": "test-instance", + "project_name": "test-project", + "reservation": null, + "ssh_keys": [], + "tags": null, + "user": "dstack-user", + "volumes": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_offer.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_offer.json new file mode 100644 index 0000000000..67af5aa0af --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_offer.json @@ -0,0 +1,25 @@ +{ + "availability": "available", + "availability_zones": null, + "backend": "aws", + "backend_data": {}, + "blocks": 1, + "instance": { + "name": "instance", + "resources": { + "cpu_arch": null, + "cpus": 2, + "description": "", + "disk": { + "size_mib": 102400 + }, + "gpus": [], + "memory_mib": 12288, + "spot": false + } + }, + "instance_runtime": "shim", + "price": 1.0, + "region": "eu-west", + "total_blocks": 1 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json new file mode 100644 index 0000000000..c0c39d48fb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json @@ -0,0 +1,14 @@ +{ + "cpu": null, + "gpu": null, + "memory": null, + "network_mode": "host", + "offer": null, + "ports": { + "8080": 30080, + "8081": 30081 + }, + "username": null, + "volume_names": null, + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_spec.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_spec.json new file mode 100644 index 0000000000..5f4662b985 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_spec.json @@ -0,0 +1,80 @@ +{ + "app_specs": null, + "commands": [ + "/bin/bash", + "-i", + "-c", + "echo hi" + ], + "env": { + "A": "1" + }, + "file_archives": [], + "home_dir": null, + "image_name": "dstackai/base:latest", + "job_name": "test-run-0-0", + "job_num": 0, + "jobs_per_replica": 1, + "max_duration": 7200, + "privileged": false, + "probes": [], + "registry_auth": null, + "replica_group": "0", + "replica_num": 0, + "repo_code_hash": null, + "repo_data": null, + "repo_dir": "/workflow", + "repo_exists_action": null, + "requirements": { + "backend_options": null, + "max_price": 10.5, + "multinode": null, + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": [ + 8, + 0 + ], + "count": { + "max": 1, + "min": 1 + }, + "memory": { + "max": null, + "min": 40.0 + }, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "spot": false + }, + "retry": null, + "service_port": null, + "single_branch": null, + "ssh_key": null, + "stop_duration": null, + "user": null, + "utilization_policy": null, + "volumes": null, + "working_dir": "/workflow" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_configuration.json new file mode 100644 index 0000000000..b8644bd368 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_configuration.json @@ -0,0 +1,5 @@ +{ + "backend": "aws", + "placement_strategy": "cluster", + "region": "eu-central-1" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_provisioning_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_provisioning_data.json new file mode 100644 index 0000000000..2dec8d05c0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/placement_group_provisioning_data.json @@ -0,0 +1,4 @@ +{ + "backend": "aws", + "backend_data": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json new file mode 100644 index 0000000000..f1bd7642ac --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json @@ -0,0 +1,29 @@ +{ + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": 600, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": null, + "spot_policy": "auto", + "startup_order": null, + "stop_criteria": null, + "stop_duration": 300, + "tags": null, + "utilization_policy": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/remote_connection_info.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/remote_connection_info.json new file mode 100644 index 0000000000..cc971014f8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/remote_connection_info.json @@ -0,0 +1,14 @@ +{ + "env": {}, + "host": "10.0.0.10", + "port": 22, + "ssh_keys": [ + { + "private": "\n -----BEGIN OPENSSH PRIVATE KEY-----\n b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n QyNTUxOQAAACDupicVWzbZtM14DC7wcq4VyQpiUb3kqycSwefzs3MgpAAAAJCiWa5Volmu\n VQAAAAtzc2gtZWQyNTUxOQAAACDupicVWzbZtM14DC7wcq4VyQpiUb3kqycSwefzs3MgpA\n AAAEAncHi4AhS6XdMp5Gzd+IMse/4ekyQ54UngByf0Sp0uH+6mJxVbNtm0zXgMLvByrhXJ\n CmJRveSrJxLB5/OzcyCkAAAACWRlZkBkZWZwYwECAwQ=\n -----END OPENSSH PRIVATE KEY-----\n ", + "public": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO6mJxVbNtm0zXgMLvByrhXJCmJRveSrJxLB5/OzcyCk" + } + ], + "ssh_proxy": null, + "ssh_proxy_keys": null, + "ssh_user": "ubuntu" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/requirements.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/requirements.json new file mode 100644 index 0000000000..24b8cb2a39 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/requirements.json @@ -0,0 +1,43 @@ +{ + "backend_options": null, + "max_price": 10.5, + "multinode": null, + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": [ + 8, + 0 + ], + "count": { + "max": 1, + "min": 1 + }, + "memory": { + "max": null, + "min": 40.0 + }, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "spot": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/resources.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/resources.json new file mode 100644 index 0000000000..e7939811c7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/resources.json @@ -0,0 +1,17 @@ +{ + "cpu_arch": null, + "cpus": 8, + "description": "8xCPU, 16GB, 1xA100", + "disk": { + "size_mib": 102400 + }, + "gpus": [ + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + } + ], + "memory_mib": 16384, + "spot": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/service_spec.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/service_spec.json new file mode 100644 index 0000000000..1e83e84c64 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/service_spec.json @@ -0,0 +1,5 @@ +{ + "model": null, + "options": {}, + "url": "/proxy/services/test-project/test-run/" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_attachment_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_attachment_data.json new file mode 100644 index 0000000000..16a0a181e7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_attachment_data.json @@ -0,0 +1,3 @@ +{ + "device_name": "/dev/sdb" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_configuration.json new file mode 100644 index 0000000000..46bfcf0c65 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_configuration.json @@ -0,0 +1,11 @@ +{ + "auto_cleanup_duration": null, + "availability_zone": null, + "backend": "aws", + "name": "test-volume", + "region": "eu-west-1", + "size": 100.0, + "tags": null, + "type": "volume", + "volume_id": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_provisioning_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_provisioning_data.json new file mode 100644 index 0000000000..da00383650 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/volume_provisioning_data.json @@ -0,0 +1,10 @@ +{ + "attachable": true, + "availability_zone": null, + "backend": null, + "backend_data": null, + "detachable": true, + "price": 1.0, + "size_gb": 100, + "volume_id": "vol-1234" +} diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index 6130aebfda..d70845631e 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -20,20 +20,25 @@ import pytest import yaml +from dstack._internal.core.backends.aws.models import AWSCreds from dstack._internal.core.models.configurations import DstackConfiguration from dstack._internal.core.models.fleets import Fleet from dstack._internal.core.models.profiles import ProfilesConfig -from dstack._internal.core.models.runs import RunSpec +from dstack._internal.core.models.runs import JobSpec, RunSpec from dstack._internal.server.schemas.volumes import CreateVolumeRequest from tests._internal.pydantic_compat.compare import ( FIXTURES_DIR, assert_matches_fixture, + canonicalize, type_map, ) from tests._internal.pydantic_compat.compat import parse_forbid_extra, parse_ignore_extra +from tests._internal.pydantic_compat.test_serialization import DB_BLOBS as SERIALIZED_DB_BLOBS # Read from a `Text` column with extra="ignore", so a row written by a newer server loads. DB_BLOBS: dict[str, Any] = { + "aws_creds": AWSCreds, + "job_spec": JobSpec, "run_spec": RunSpec, } @@ -53,9 +58,11 @@ # `DstackConfiguration` dispatches every `.dstack.yml` type through its `__root__` union, so one # entry point covers task, service, dev-environment, fleet, volume, and gateway. CONFIGS: dict[str, Any] = { - "task_sugared": DstackConfiguration, - "fleet_nodes_range": DstackConfiguration, - "profiles_durations": ProfilesConfig, + "fleet": DstackConfiguration, + "profiles": ProfilesConfig, + "service": DstackConfiguration, + "task": DstackConfiguration, + "volume": DstackConfiguration, } @@ -67,6 +74,20 @@ def test_parses_to_expected_values_and_types(self, name, regen): ) +class TestDbBlobExtraFieldTolerance: + """ + Every stored model must survive a row extended by a newer writer. + """ + + @pytest.mark.parametrize("name", sorted(SERIALIZED_DB_BLOBS)) + def test_unknown_field_is_dropped(self, name): + committed = (FIXTURES_DIR / "serialization" / "db" / f"{name}.json").read_text() + perturbed = {**json.loads(committed), "unknown_from_a_newer_writer": {"x": [1]}} + model = type(SERIALIZED_DB_BLOBS[name]()) + parsed = parse_ignore_extra(model, perturbed) + assert canonicalize(parsed.json()) == canonicalize(committed) + + class TestRequestBodyParsing: @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) def test_parses_to_expected_values_and_types(self, name, regen): diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 3492cc5641..aaf4119750 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -25,8 +25,28 @@ # Written to a `Text` column via `.json()`. DB_BLOBS: dict[str, Callable[[], CoreModel]] = { + "aws_creds": factories.aws_creds, + "compute_group_provisioning_data": factories.compute_group_provisioning_data, + "fleet_spec": factories.fleet_spec, + "gateway_compute_configuration": factories.gateway_compute_configuration, + "gateway_configuration": factories.gateway_configuration, + "image_pull_progress": factories.image_pull_progress, + "instance_configuration": factories.instance_configuration, + "instance_offer": factories.instance_offer, "job_provisioning_data": factories.job_provisioning_data, + "job_runtime_data": factories.job_runtime_data, + "job_spec": factories.job_spec, + "placement_group_configuration": factories.placement_group_configuration, + "placement_group_provisioning_data": factories.placement_group_provisioning_data, + "profile": factories.profile, + "remote_connection_info": factories.remote_connection_info, + "requirements": factories.requirements, + "resources": factories.resources, "run_spec": factories.run_spec, + "service_spec": factories.service_spec, + "volume_attachment_data": factories.volume_attachment_data, + "volume_configuration": factories.volume_configuration, + "volume_provisioning_data": factories.volume_provisioning_data, } # Returned from a router via `CustomORJSONResponse`. From d38150ce4d4755e9e2f26dd34b776e59eb60c0ab Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 17:29:54 +0500 Subject: [PATCH 05/15] Scale out api response serialization tests --- .../_internal/pydantic_compat/factories.py | 145 ++++++++++- .../serialization/api_response/fleet.json | 4 +- .../api_response/fleet_plan.json | 88 +++++++ .../serialization/api_response/gateway.json | 32 +++ .../serialization/api_response/project.json | 41 +++ .../serialization/api_response/run_plan.json | 234 ++++++++++++++++++ .../serialization/api_response/secret.json | 5 + .../api_response/server_info.json | 3 + .../api_response/user_with_creds.json | 16 ++ .../serialization/api_response/volume.json | 29 +++ .../pydantic_compat/test_serialization.py | 8 + 11 files changed, 601 insertions(+), 4 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/project.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/secret.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/server_info.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/user_with_creds.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/volume.json diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index 5408c38381..5e2882f26d 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -11,13 +11,16 @@ from dstack._internal.core.backends.aws.models import AWSCreds from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.common import ApplyAction from dstack._internal.core.models.compute_groups import ComputeGroupProvisioningData from dstack._internal.core.models.configurations import DevEnvironmentConfiguration from dstack._internal.core.models.envs import Env -from dstack._internal.core.models.fleets import Fleet, FleetSpec, FleetStatus +from dstack._internal.core.models.fleets import Fleet, FleetPlan, FleetSpec, FleetStatus from dstack._internal.core.models.gateways import ( + Gateway, GatewayComputeConfiguration, GatewayConfiguration, + GatewayStatus, ) from dstack._internal.core.models.instances import ( Disk, @@ -39,6 +42,12 @@ RetryEvent, SpotPolicy, ) +from dstack._internal.core.models.projects import ( + Member, + MemberPermissions, + Project, + ProjectRole, +) from dstack._internal.core.models.resources import ( ComputeCapability, GPUSpec, @@ -48,14 +57,26 @@ ) from dstack._internal.core.models.runs import ( ImagePullProgress, + JobPlan, JobProvisioningData, JobRuntimeData, JobSpec, Requirements, + RunPlan, RunSpec, ServiceSpec, ) +from dstack._internal.core.models.secrets import Secret +from dstack._internal.core.models.server import ServerInfo +from dstack._internal.core.models.users import ( + GlobalRole, + User, + UserPermissions, + UserTokenCreds, + UserWithCreds, +) from dstack._internal.core.models.volumes import ( + Volume, VolumeAttachmentData, VolumeConfiguration, VolumeProvisioningData, @@ -77,11 +98,13 @@ get_placement_group_provisioning_data, get_remote_connection_info, get_run_spec, + get_volume, get_volume_configuration, get_volume_provisioning_data, ) -_ID = uuid.UUID("11111111-1111-1111-1111-111111111111") +# Version-4 shaped: several models annotate their id as `UUID4`, which validates the version. +_ID = uuid.UUID("11111111-1111-4111-8111-111111111111") _CREATED_AT = datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc) @@ -283,6 +306,124 @@ def fleet() -> Fleet: ) +def run_plan() -> RunPlan: + """ + The single richest response tree — 67 of the 129 model classes reachable from any response + model are reachable only through here. + """ + return RunPlan( + project_name="test-project", + user="test-user", + run_spec=run_spec(), + job_plans=[ + JobPlan( + job_spec=job_spec(), + offers=[get_instance_offer_with_availability()], + total_offers=1, + max_price=10.5, + ) + ], + current_resource=None, + action=ApplyAction.CREATE, + ) + + +def fleet_plan() -> FleetPlan: + return FleetPlan( + project_name="test-project", + user="test-user", + spec=get_fleet_spec(), + effective_spec=None, + current_resource=None, + offers=[get_instance_offer_with_availability()], + total_offers=1, + max_offer_price=10.5, + action=ApplyAction.CREATE, + ) + + +def project() -> Project: + return Project( + project_id=_ID, + project_name="test-project", + owner=user(), + backends=[], + members=[ + Member( + user=user(), + project_role=ProjectRole.ADMIN, + permissions=MemberPermissions(can_manage_ssh_fleets=True), + ) + ], + is_public=False, + ) + + +def user() -> User: + return User( + id=_ID, + username="test-user", + created_at=_CREATED_AT, + global_role=GlobalRole.USER, + email=None, + active=True, + permissions=UserPermissions(can_create_projects=True), + ) + + +def user_with_creds() -> UserWithCreds: + """ + The `SerializeAsAny` case. v2 drops `creds`/`ssh_private_key` from any `User`-typed field, and + that drop is desired — so this fixture pins that the *top-level* response still carries them. + """ + return UserWithCreds( + id=_ID, + username="test-user", + created_at=_CREATED_AT, + global_role=GlobalRole.USER, + email=None, + active=True, + permissions=UserPermissions(can_create_projects=True), + creds=UserTokenCreds(token="test-token"), + ) + + +def volume() -> Volume: + return get_volume( + id_=_ID, + name="test-volume", + project_name="test-project", + created_at=_CREATED_AT, + last_processed_at=_CREATED_AT, + ) + + +def gateway() -> Gateway: + return Gateway( + id=_ID, + name="test-gateway", + project_name="test-project", + backend=BackendType.AWS, + region="us-east-1", + created_at=_CREATED_AT, + status=GatewayStatus.RUNNING, + status_message=None, + hostname="gateway.example.com", + wildcard_domain=None, + default=True, + replicas=[], + configuration=gateway_configuration(), + ) + + +def secret() -> Secret: + return Secret(id=_ID, name="test-secret", value=None) + + +def server_info() -> ServerInfo: + return ServerInfo(server_version="0.20.0") + + # --- API request bodies -------------------------------------------------------------- # Serialized by the API client as `body=X.json()`. diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json index 93947bee55..f6455670ed 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json @@ -1,6 +1,6 @@ { "created_at": "2024-01-02T03:04:05+00:00", - "id": "11111111-1111-1111-1111-111111111111", + "id": "11111111-1111-4111-8111-111111111111", "instances": [ { "availability_zone": null, @@ -12,7 +12,7 @@ "fleet_name": null, "health_status": "healthy", "hostname": null, - "id": "11111111-1111-1111-1111-111111111111", + "id": "11111111-1111-4111-8111-111111111111", "instance_num": 0, "instance_type": null, "job_name": null, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json new file mode 100644 index 0000000000..c48f98aff5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json @@ -0,0 +1,88 @@ +{ + "action": "create", + "current_resource": null, + "effective_spec": null, + "max_offer_price": 10.5, + "offers": [ + { + "availability": "available", + "availability_zones": null, + "backend": "aws", + "backend_data": {}, + "blocks": 1, + "instance": { + "name": "instance", + "resources": { + "cpu_arch": null, + "cpus": 2, + "description": "", + "disk": { + "size_mib": 102400 + }, + "gpus": [], + "memory_mib": 12288, + "spot": false + } + }, + "instance_runtime": "shim", + "price": 1.0, + "region": "eu-west", + "total_blocks": 1 + } + ], + "project_name": "test-project", + "spec": { + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": null, + "max_price": null, + "name": "test-fleet", + "nodes": { + "max": 1, + "min": 1 + }, + "placement": null, + "regions": null, + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": null, + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": "fleet.dstack.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } + }, + "total_offers": 1, + "user": "test-user" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json new file mode 100644 index 0000000000..0a6bee732b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/gateway.json @@ -0,0 +1,32 @@ +{ + "backend": "aws", + "configuration": { + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "default": false, + "domain": "example.com", + "instance_type": null, + "name": "test-gateway", + "public_ip": true, + "region": "us-east-1", + "replicas": null, + "router": null, + "tags": null, + "type": "gateway" + }, + "created_at": "2024-01-02T03:04:05+00:00", + "default": true, + "hostname": "gateway.example.com", + "id": "11111111-1111-4111-8111-111111111111", + "instance_id": null, + "ip_address": null, + "name": "test-gateway", + "project_name": "test-project", + "region": "us-east-1", + "replicas": [], + "status": "running", + "status_message": null, + "wildcard_domain": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/project.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/project.json new file mode 100644 index 0000000000..b4a6f0a4c3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/project.json @@ -0,0 +1,41 @@ +{ + "backends": [], + "created_at": null, + "is_public": false, + "members": [ + { + "permissions": { + "can_manage_secrets": false, + "can_manage_ssh_fleets": true + }, + "project_role": "admin", + "user": { + "active": true, + "created_at": "2024-01-02T03:04:05+00:00", + "email": null, + "global_role": "user", + "id": "11111111-1111-4111-8111-111111111111", + "permissions": { + "can_create_projects": true + }, + "ssh_public_key": null, + "username": "test-user" + } + } + ], + "owner": { + "active": true, + "created_at": "2024-01-02T03:04:05+00:00", + "email": null, + "global_role": "user", + "id": "11111111-1111-4111-8111-111111111111", + "permissions": { + "can_create_projects": true + }, + "ssh_public_key": null, + "username": "test-user" + }, + "project_id": "11111111-1111-4111-8111-111111111111", + "project_name": "test-project", + "templates_repo": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json new file mode 100644 index 0000000000..f0ba552507 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json @@ -0,0 +1,234 @@ +{ + "action": "create", + "current_resource": null, + "effective_run_spec": null, + "job_plans": [ + { + "job_spec": { + "app_specs": null, + "commands": [ + "/bin/bash", + "-i", + "-c", + "echo hi" + ], + "env": { + "A": "1" + }, + "file_archives": [], + "home_dir": null, + "image_name": "dstackai/base:latest", + "job_name": "test-run-0-0", + "job_num": 0, + "jobs_per_replica": 1, + "max_duration": 7200, + "privileged": false, + "probes": [], + "registry_auth": null, + "replica_group": "0", + "replica_num": 0, + "repo_code_hash": null, + "repo_data": null, + "repo_dir": "/workflow", + "repo_exists_action": null, + "requirements": { + "backend_options": null, + "max_price": 10.5, + "multinode": null, + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": [ + 8, + 0 + ], + "count": { + "max": 1, + "min": 1 + }, + "memory": { + "max": null, + "min": 40.0 + }, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "spot": false + }, + "retry": null, + "service_port": null, + "single_branch": null, + "ssh_key": null, + "stop_duration": null, + "user": null, + "utilization_policy": null, + "volumes": null, + "working_dir": "/workflow" + }, + "max_price": 10.5, + "offers": [ + { + "availability": "available", + "availability_zones": null, + "backend": "aws", + "backend_data": {}, + "blocks": 1, + "instance": { + "name": "instance", + "resources": { + "cpu_arch": null, + "cpus": 2, + "description": "", + "disk": { + "size_mib": 102400 + }, + "gpus": [], + "memory_mib": 12288, + "spot": false + } + }, + "instance_runtime": "shim", + "price": 1.0, + "region": "eu-west", + "total_blocks": 1 + } + ], + "total_offers": 1 + } + ], + "project_name": "test-project", + "run_spec": { + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "home_dir": "/root", + "ide": "vscode", + "idle_duration": null, + "image": null, + "inactivity_duration": null, + "init": [], + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": null, + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "dev-environment", + "user": null, + "utilization_policy": null, + "version": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": "dstack.yaml", + "file_archives": [], + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": 300, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + }, + "repo_code_hash": null, + "repo_data": { + "repo_dir": "/", + "repo_type": "local" + }, + "repo_dir": null, + "repo_id": "test-repo", + "run_name": "test-run", + "ssh_key_pub": "user_ssh_key", + "working_dir": null + }, + "user": "test-user" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/secret.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/secret.json new file mode 100644 index 0000000000..f2d05ce0c5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/secret.json @@ -0,0 +1,5 @@ +{ + "id": "11111111-1111-4111-8111-111111111111", + "name": "test-secret", + "value": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/server_info.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/server_info.json new file mode 100644 index 0000000000..0284360557 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/server_info.json @@ -0,0 +1,3 @@ +{ + "server_version": "0.20.0" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/user_with_creds.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/user_with_creds.json new file mode 100644 index 0000000000..97c4e2f379 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/user_with_creds.json @@ -0,0 +1,16 @@ +{ + "active": true, + "created_at": "2024-01-02T03:04:05+00:00", + "creds": { + "token": "test-token" + }, + "email": null, + "global_role": "user", + "id": "11111111-1111-4111-8111-111111111111", + "permissions": { + "can_create_projects": true + }, + "ssh_private_key": null, + "ssh_public_key": null, + "username": "test-user" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/volume.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/volume.json new file mode 100644 index 0000000000..80ec497bad --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/volume.json @@ -0,0 +1,29 @@ +{ + "attachment_data": null, + "attachments": [], + "configuration": { + "auto_cleanup_duration": null, + "availability_zone": null, + "backend": "aws", + "name": "test-volume", + "region": "eu-west-1", + "size": 100.0, + "tags": null, + "type": "volume", + "volume_id": null + }, + "cost": 0, + "created_at": "2024-01-02T03:04:05+00:00", + "deleted": false, + "deleted_at": null, + "external": false, + "id": "11111111-1111-4111-8111-111111111111", + "last_processed_at": "2024-01-02T03:04:05+00:00", + "name": "test-volume", + "project_name": "test-project", + "provisioning_data": null, + "status": "active", + "status_message": null, + "user": "test_user", + "volume_id": null +} diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index aaf4119750..829de9d5bf 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -52,6 +52,14 @@ # Returned from a router via `CustomORJSONResponse`. API_RESPONSES: dict[str, Callable[[], CoreModel]] = { "fleet": factories.fleet, + "fleet_plan": factories.fleet_plan, + "gateway": factories.gateway, + "project": factories.project, + "run_plan": factories.run_plan, + "secret": factories.secret, + "server_info": factories.server_info, + "user_with_creds": factories.user_with_creds, + "volume": factories.volume, } # Sent by the API client as a request body — `body=X.json()`, 40 call sites in `api/server/`. From e6e51ca0e639dc8a1d01cfaf6d40bfb543322ede Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 17:42:02 +0500 Subject: [PATCH 06/15] Scale out api request serialization tests --- .../_internal/pydantic_compat/factories.py | 63 ++++++++- .../apply_gateway_plan_request.json | 25 ++++ .../api_request/apply_run_plan_request.json | 120 ++++++++++++++++++ .../api_request/create_volume_request.json | 13 ++ .../api_request/save_repo_creds_request.json | 12 ++ .../pydantic_compat/test_rejection.py | 4 +- .../pydantic_compat/test_serialization.py | 4 + 7 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/create_volume_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/save_repo_creds_request.json diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index 5e2882f26d..780a09f19b 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -20,6 +20,7 @@ Gateway, GatewayComputeConfiguration, GatewayConfiguration, + GatewaySpec, GatewayStatus, ) from dstack._internal.core.models.instances import ( @@ -48,6 +49,7 @@ Project, ProjectRole, ) +from dstack._internal.core.models.repos.remote import RemoteRepoCreds, RemoteRepoInfo from dstack._internal.core.models.resources import ( ComputeCapability, GPUSpec, @@ -86,6 +88,16 @@ ApplyFleetPlanRequest, DeleteFleetsRequest, ) +from dstack._internal.server.schemas.gateways import ( + ApplyGatewayPlanInput, + ApplyGatewayPlanRequest, +) +from dstack._internal.server.schemas.repos import SaveRepoCredsRequest +from dstack._internal.server.schemas.runs import ( + ApplyRunPlanInput, + ApplyRunPlanRequest, +) +from dstack._internal.server.schemas.volumes import CreateVolumeRequest from dstack._internal.server.testing.common import ( get_compute_group_provisioning_data, get_fleet_spec, @@ -109,9 +121,7 @@ # --- DB blobs ------------------------------------------------------------------------ -# Written to a `Text` column with `.json()`. Rows outlive any release, so each stored model -# is an independent liability and gets its own fixture rather than standing in for a shared -# shape — unlike the API surfaces below, which are covered by feature instead. +# Written to a `Text` column with `.json()`. def aws_creds() -> AWSCreds: @@ -275,7 +285,10 @@ def volume_provisioning_data() -> VolumeProvisioningData: # --- API responses ------------------------------------------------------------------- -# Returned from a router through `CustomORJSONResponse`. +# Returned from a router through `CustomORJSONResponse`. Chosen by greedy set cover so that +# between them they reach every model class reachable from any response model — 129 of 129. +# `run` and `instance` are absent on purpose: `run_plan` and `fleet` already reach everything +# they would add. def fleet() -> Fleet: @@ -425,7 +438,11 @@ def server_info() -> ServerInfo: # --- API request bodies -------------------------------------------------------------- -# Serialized by the API client as `body=X.json()`. +# Serialized by the API client as `body=X.json()`. Picked by the same greedy cover as the +# responses, over the 75 client->server schemas: these six reach 98 of the 171 nested model +# classes. Coverage plateaus there because most of the remainder are wrappers like +# `{names: list[str]}` whose only contribution is their own class, which a fixture pins no +# better than the annotation does. def apply_fleet_plan_request() -> ApplyFleetPlanRequest: @@ -439,5 +456,41 @@ def apply_fleet_plan_request() -> ApplyFleetPlanRequest: ) +def apply_run_plan_request() -> ApplyRunPlanRequest: + """The largest client-sent body: it carries a whole `RunSpec`.""" + return ApplyRunPlanRequest( + plan=ApplyRunPlanInput(run_spec=run_spec(), current_resource=None), + force=False, + ) + + +def apply_gateway_plan_request() -> ApplyGatewayPlanRequest: + return ApplyGatewayPlanRequest( + plan=ApplyGatewayPlanInput( + spec=GatewaySpec( + configuration=gateway_configuration(), + configuration_path="gateway.dstack.yml", + ), + current_resource=None, + ), + force=False, + ) + + +def create_volume_request() -> CreateVolumeRequest: + return CreateVolumeRequest(configuration=get_volume_configuration()) + + +def save_repo_creds_request() -> SaveRepoCredsRequest: + return SaveRepoCredsRequest( + repo_id="test-repo", + repo_info=RemoteRepoInfo(repo_name="dstack"), + repo_creds=RemoteRepoCreds( + clone_url="https://github.com/dstackai/dstack.git", + oauth_token="test-token", + ), + ) + + def delete_fleets_request() -> DeleteFleetsRequest: return DeleteFleetsRequest(names=["fleet-a", "fleet-b"]) diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json new file mode 100644 index 0000000000..16bdd691c8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_gateway_plan_request.json @@ -0,0 +1,25 @@ +{ + "force": false, + "plan": { + "current_resource": null, + "spec": { + "configuration": { + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "default": false, + "domain": "example.com", + "instance_type": null, + "name": "test-gateway", + "public_ip": true, + "region": "us-east-1", + "replicas": null, + "router": null, + "tags": null, + "type": "gateway" + }, + "configuration_path": "gateway.dstack.yml" + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json new file mode 100644 index 0000000000..167f303ad2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json @@ -0,0 +1,120 @@ +{ + "force": false, + "plan": { + "current_resource": null, + "run_spec": { + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "home_dir": "/root", + "ide": "vscode", + "idle_duration": null, + "image": null, + "inactivity_duration": null, + "init": [], + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": null, + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "dev-environment", + "user": null, + "utilization_policy": null, + "version": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": "dstack.yaml", + "file_archives": [], + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": 300, + "instance_types": null, + "instances": null, + "max_duration": 7200, + "max_price": null, + "name": "default", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + }, + "repo_code_hash": null, + "repo_data": { + "repo_dir": "/", + "repo_type": "local" + }, + "repo_dir": null, + "repo_id": "test-repo", + "run_name": "test-run", + "ssh_key_pub": "user_ssh_key", + "working_dir": null + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/create_volume_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/create_volume_request.json new file mode 100644 index 0000000000..1b1221de5f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/create_volume_request.json @@ -0,0 +1,13 @@ +{ + "configuration": { + "auto_cleanup_duration": null, + "availability_zone": null, + "backend": "aws", + "name": "test-volume", + "region": "eu-west-1", + "size": 100.0, + "tags": null, + "type": "volume", + "volume_id": null + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/save_repo_creds_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/save_repo_creds_request.json new file mode 100644 index 0000000000..18f4a1cf0d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/save_repo_creds_request.json @@ -0,0 +1,12 @@ +{ + "repo_creds": { + "clone_url": "https://github.com/dstackai/dstack.git", + "oauth_token": "test-token", + "private_key": null + }, + "repo_id": "test-repo", + "repo_info": { + "repo_name": "dstack", + "repo_type": "remote" + } +} diff --git a/src/tests/_internal/pydantic_compat/test_rejection.py b/src/tests/_internal/pydantic_compat/test_rejection.py index 30dce4c174..d644ce218a 100644 --- a/src/tests/_internal/pydantic_compat/test_rejection.py +++ b/src/tests/_internal/pydantic_compat/test_rejection.py @@ -51,8 +51,8 @@ def _task(extra_yaml: str) -> Any: ), # Custom-type parsing: `Duration.parse` rejects unknown units and non-numeric input. "duration_bad_unit": (DstackConfiguration.parse_obj, _task("max_duration: 5 years\n")), - # An unknown discriminator tag. After Phase 0 item 2 these unions are discriminated, so the - # error should name the tag rather than accumulate one failure per arm. + # An unknown discriminator tag. `AnyDstackConfiguration` declares `discriminator="type"`, so + # the error names the tag instead of accumulating one failure per arm. "unknown_config_type": (DstackConfiguration.parse_obj, {"type": "not-a-real-type"}), } diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 829de9d5bf..79103bd3ee 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -66,7 +66,11 @@ # This is the new-CLI-against-old-server direction, which nothing else in the suite covers. API_REQUESTS: dict[str, Callable[[], CoreModel]] = { "delete_fleets_request": factories.delete_fleets_request, + "save_repo_creds_request": factories.save_repo_creds_request, "apply_fleet_plan_request": factories.apply_fleet_plan_request, + "apply_gateway_plan_request": factories.apply_gateway_plan_request, + "apply_run_plan_request": factories.apply_run_plan_request, + "create_volume_request": factories.create_volume_request, } From ab6f3f6b6bf87dea25a57bf8f8dd9929225df37b Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 17:53:43 +0500 Subject: [PATCH 07/15] Test runner and gateway models --- .../_internal/pydantic_compat/compare.py | 21 +- .../_internal/pydantic_compat/factories.py | 77 +++++++ .../register_replica_request.input.json | 7 + .../register_replica_request.types.json | 3 + .../register_replica_request.values.json | 11 + .../register_service_request.input.json | 20 ++ .../register_service_request.types.json | 6 + .../register_service_request.values.json | 21 ++ .../runner/healthcheck_response.input.json | 1 + .../runner/healthcheck_response.types.json | 3 + .../runner/healthcheck_response.values.json | 4 + .../runner/metrics_response.input.json | 8 + .../runner/metrics_response.types.json | 4 + .../runner/metrics_response.values.json | 12 + .../serialization/gateway/service_stats.json | 10 + .../serialization/runner/submit_body.json | 205 ++++++++++++++++++ .../_internal/pydantic_compat/test_parsing.py | 35 +++ .../pydantic_compat/test_serialization.py | 27 +++ 18 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/gateway/service_stats.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json diff --git a/src/tests/_internal/pydantic_compat/compare.py b/src/tests/_internal/pydantic_compat/compare.py index 6170f2be18..988b8fcc5d 100644 --- a/src/tests/_internal/pydantic_compat/compare.py +++ b/src/tests/_internal/pydantic_compat/compare.py @@ -105,9 +105,20 @@ def type_map(value: Any, path: str = "", out: Union[dict, None] = None) -> dict: def _class_name(value: Any) -> str: - # pydantic-duality names the concrete classes `XRequest` / `XResponse`. Those suffixes vanish - # in v2, so strip them or every line of every type map diffs on the migration branch. - name = type(value).__name__ - for suffix in ("Request", "Response"): - name = name.removesuffix(suffix) + """ + The model's name with pydantic-duality's generated suffix removed. + + Duality names its concrete classes `XRequest` / `XResponse`, and those suffixes vanish in v2, + so leaving them in would make every line of every type map diff on the migration branch. + + Strip only for classes duality actually generated, which is what `__response__` identifies. + Plenty of models are genuinely *named* `...Request` — every gateway registry schema, for one — + and those are plain `BaseModel`, so stripping there would report `RegisterService` for a class + called `RegisterServiceRequest`. + """ + cls = type(value) + name = cls.__name__ + if hasattr(cls, "__response__"): + for suffix in ("Request", "Response"): + name = name.removesuffix(suffix) return name diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index 780a09f19b..1383e85216 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -59,13 +59,18 @@ ) from dstack._internal.core.models.runs import ( ImagePullProgress, + Job, JobPlan, JobProvisioningData, JobRuntimeData, JobSpec, + JobStatus, + JobSubmission, Requirements, + Run, RunPlan, RunSpec, + RunStatus, ServiceSpec, ) from dstack._internal.core.models.secrets import Secret @@ -83,6 +88,7 @@ VolumeConfiguration, VolumeProvisioningData, ) +from dstack._internal.proxy.gateway.schemas.stats import ServiceStats, Stat from dstack._internal.server.schemas.fleets import ( ApplyFleetPlanInput, ApplyFleetPlanRequest, @@ -93,6 +99,7 @@ ApplyGatewayPlanRequest, ) from dstack._internal.server.schemas.repos import SaveRepoCredsRequest +from dstack._internal.server.schemas.runner import HealthcheckResponse, SubmitBody from dstack._internal.server.schemas.runs import ( ApplyRunPlanInput, ApplyRunPlanRequest, @@ -494,3 +501,73 @@ def save_repo_creds_request() -> SaveRepoCredsRequest: def delete_fleets_request() -> DeleteFleetsRequest: return DeleteFleetsRequest(names=["fleet-a", "fleet-b"]) + + +# --- Runner API ------------------------------------------------------------------------ +# The server<->runner (shim) protocol. The server is the client here, so `serialization` holds +# the bodies it sends and `parsing` holds the responses it reads back. Unlike the public API +# there is no negotiation on this boundary: a server talks to whatever runner version is baked +# into the running instance's image, so both directions have to stay compatible. + + +def run() -> Run: + return Run( + id=_ID, + project_name="test-project", + user="test-user", + submitted_at=_CREATED_AT, + last_processed_at=_CREATED_AT, + status=RunStatus.SUBMITTED, + run_spec=run_spec(), + jobs=[Job(job_spec=job_spec(), job_submissions=[job_submission()])], + ) + + +def job_submission() -> JobSubmission: + return JobSubmission( + id=_ID, + submission_num=0, + submitted_at=_CREATED_AT, + last_processed_at=_CREATED_AT, + status=JobStatus.SUBMITTED, + job_provisioning_data=job_provisioning_data(), + job_runtime_data=job_runtime_data(), + ) + + +def submit_body() -> SubmitBody: + """The largest body on any boundary — it carries a whole `Run` plus the job spec.""" + return SubmitBody( + run=run(), + job_spec=job_spec(), + job_submission=job_submission(), + run_spec=run_spec(), + ) + + +def healthcheck_response() -> HealthcheckResponse: + return HealthcheckResponse(service="dstack-shim", version="0.20.0") + + +# No `PullResponse` factory: `LogEvent.message` is `bytes`, which the project's orjson dumper +# refuses outright, so the model has no working `.json()` at all. Harmless today because the +# server only ever parses it (`services/runner/client.py`), but it means that write path has +# never run — worth knowing before the serializer is swapped. + + +# --- Gateway API ----------------------------------------------------------------------- +# The server<->gateway protocol. Note the server builds these payloads as hand-written dicts +# rather than dumping a model (see `services/gateways/client.py`), so only the gateway's own +# parsing side is model-driven — hence the request shapes live in `parsing/gateway` with +# hand-written inputs, and only the stats response is serialized from a model here. +# +# These schemas are plain `BaseModel`, not `CoreModel`: they already default to extra="ignore" +# in both pydantic versions, so they need neither the duality shim nor a strictness test. + + +def service_stats() -> ServiceStats: + return ServiceStats( + project_name="test-project", + run_name="test-run", + stats={60: Stat(requests=10, request_time=0.125)}, + ) diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json new file mode 100644 index 0000000000..753ade2fa6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json @@ -0,0 +1,7 @@ +{ + "job_id": "11111111-1111-4111-8111-111111111111", + "app_port": 8000, + "ssh_host": "ubuntu@10.0.0.1", + "ssh_port": 22, + "ssh_proxy": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.types.json new file mode 100644 index 0000000000..ae84c2c6d3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.types.json @@ -0,0 +1,3 @@ +{ + "/": "RegisterReplicaRequest" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json new file mode 100644 index 0000000000..00fbd3f0d6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json @@ -0,0 +1,11 @@ +{ + "app_port": 8000, + "internal_ip": null, + "job_id": "11111111-1111-4111-8111-111111111111", + "ssh_head_proxy": null, + "ssh_head_proxy_private_key": null, + "ssh_host": "ubuntu@10.0.0.1", + "ssh_port": 22, + "ssh_proxy": null, + "ssh_proxy_private_key": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.input.json new file mode 100644 index 0000000000..d07b174386 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.input.json @@ -0,0 +1,20 @@ +{ + "run_name": "test-run", + "domain": "test-run.example.com", + "https": true, + "auth": false, + "client_max_body_size": 65536, + "options": { + "openai": { + "model": { + "type": "chat", + "name": "llama", + "format": "openai", + "prefix": "/v1" + } + } + }, + "rate_limits": [], + "ssh_private_key": "PRIVATE-KEY", + "field_added_by_a_newer_server": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.types.json new file mode 100644 index 0000000000..bac6d2aafc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.types.json @@ -0,0 +1,6 @@ +{ + "/": "RegisterServiceRequest", + "/options": "Options", + "/options/openai": "OpenAIOptions", + "/options/openai/model": "OpenAIChatModel" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.values.json new file mode 100644 index 0000000000..9b6daadb8b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_service_request.values.json @@ -0,0 +1,21 @@ +{ + "auth": false, + "client_max_body_size": 65536, + "domain": "test-run.example.com", + "has_router_replica": false, + "https": true, + "options": { + "openai": { + "model": { + "format": "openai", + "name": "llama", + "prefix": "/v1", + "type": "chat" + } + } + }, + "rate_limits": [], + "router": null, + "run_name": "test-run", + "ssh_private_key": "PRIVATE-KEY" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.input.json new file mode 100644 index 0000000000..d97a22a7d3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.input.json @@ -0,0 +1 @@ +{"service": "dstack-shim", "version": "0.20.0", "capabilities_added_later": ["gpu-metrics"]} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.types.json new file mode 100644 index 0000000000..3ad335c9a7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.types.json @@ -0,0 +1,3 @@ +{ + "/": "HealthcheckResponse" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.values.json new file mode 100644 index 0000000000..4511a74de3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/healthcheck_response.values.json @@ -0,0 +1,4 @@ +{ + "service": "dstack-shim", + "version": "0.20.0" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.input.json new file mode 100644 index 0000000000..c54c391615 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.input.json @@ -0,0 +1,8 @@ +{ + "timestamp_micro": 1700000000000000, + "cpu_usage_micro": 12345, + "memory_usage_bytes": 1048576, + "memory_working_set_bytes": 524288, + "gpus": [{"gpu_memory_usage_bytes": 2097152, "gpu_util_percent": 42}], + "field_added_by_a_newer_runner": "ignored" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.types.json new file mode 100644 index 0000000000..1715e71cac --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.types.json @@ -0,0 +1,4 @@ +{ + "/": "MetricsResponse", + "/gpus/0": "GPUMetrics" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.values.json new file mode 100644 index 0000000000..90854463f4 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/metrics_response.values.json @@ -0,0 +1,12 @@ +{ + "cpu_usage_micro": 12345, + "gpus": [ + { + "gpu_memory_usage_bytes": 2097152, + "gpu_util_percent": 42 + } + ], + "memory_usage_bytes": 1048576, + "memory_working_set_bytes": 524288, + "timestamp_micro": 1700000000000000 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/gateway/service_stats.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/gateway/service_stats.json new file mode 100644 index 0000000000..a19bb5d4ef --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/gateway/service_stats.json @@ -0,0 +1,10 @@ +{ + "project_name": "test-project", + "run_name": "test-run", + "stats": { + "60": { + "request_time": 0.125, + "requests": 10 + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json new file mode 100644 index 0000000000..b3a35e76a5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json @@ -0,0 +1,205 @@ +{ + "cluster_info": null, + "job_spec": { + "commands": [ + "/bin/bash", + "-i", + "-c", + "echo hi" + ], + "env": { + "A": "1" + }, + "file_archives": [], + "job_num": 0, + "jobs_per_replica": 1, + "max_duration": 7200, + "replica_num": 0, + "repo_data": null, + "repo_dir": "/workflow", + "repo_exists_action": null, + "single_branch": null, + "ssh_key": null, + "user": null, + "working_dir": "/workflow" + }, + "job_submission": { + "id": "11111111-1111-4111-8111-111111111111" + }, + "log_quota_hour": null, + "repo_credentials": null, + "run": { + "id": "11111111-1111-4111-8111-111111111111", + "run_spec": { + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "home_dir": "/root", + "ide": "vscode", + "idle_duration": null, + "image": null, + "inactivity_duration": null, + "init": [], + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": null, + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "dev-environment", + "user": null, + "utilization_policy": null, + "version": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": "dstack.yaml", + "repo_data": { + "repo_dir": "/", + "repo_type": "local" + }, + "repo_id": "test-repo", + "run_name": "test-run" + } + }, + "run_spec": { + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "home_dir": "/root", + "ide": "vscode", + "idle_duration": null, + "image": null, + "inactivity_duration": null, + "init": [], + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": null, + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "dev-environment", + "user": null, + "utilization_policy": null, + "version": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": "dstack.yaml", + "repo_data": { + "repo_dir": "/", + "repo_type": "local" + }, + "repo_id": "test-repo", + "run_name": "test-run" + }, + "secrets": null +} diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index d70845631e..400a368308 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -25,6 +25,11 @@ from dstack._internal.core.models.fleets import Fleet from dstack._internal.core.models.profiles import ProfilesConfig from dstack._internal.core.models.runs import JobSpec, RunSpec +from dstack._internal.proxy.gateway.schemas.registry import ( + RegisterReplicaRequest, + RegisterServiceRequest, +) +from dstack._internal.server.schemas.runner import HealthcheckResponse, MetricsResponse from dstack._internal.server.schemas.volumes import CreateVolumeRequest from tests._internal.pydantic_compat.compare import ( FIXTURES_DIR, @@ -74,6 +79,36 @@ def test_parses_to_expected_values_and_types(self, name, regen): ) +# Responses the server reads back from the runner (shim), permissively: a newer runner may add +# fields, and the server has no say over which runner version an instance is running. +RUNNER_RESPONSES: dict[str, Any] = { + "healthcheck_response": HealthcheckResponse, + "metrics_response": MetricsResponse, +} + +# Request payloads the gateway parses from the server. These schemas are plain `BaseModel`, so +# `parse_obj` already ignores unknown fields in both pydantic versions — no shim, and nothing to +# assert about strictness. +GATEWAY_REQUESTS: dict[str, Any] = { + "register_replica_request": RegisterReplicaRequest, + "register_service_request": RegisterServiceRequest, +} + + +class TestRunnerResponseParsing: + @pytest.mark.parametrize("name", sorted(RUNNER_RESPONSES)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_ignore_extra(RUNNER_RESPONSES[name], _load_input("runner", name)) + _assert_parses("runner", name, model, regen) + + +class TestGatewayRequestParsing: + @pytest.mark.parametrize("name", sorted(GATEWAY_REQUESTS)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = GATEWAY_REQUESTS[name].parse_obj(_load_input("gateway", name)) + _assert_parses("gateway", name, model, regen) + + class TestDbBlobExtraFieldTolerance: """ Every stored model must survive a row extended by a newer writer. diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 79103bd3ee..93665ebb07 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -16,6 +16,7 @@ from typing import Callable import pytest +from pydantic import BaseModel from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.fleets import FleetNodesSpec @@ -97,6 +98,32 @@ def test_matches_fixture(self, name, regen): assert_matches_fixture("serialization/api_request", name, payload, regen=regen) +# Sent by the server to the runner (shim) as a request body. +RUNNER_REQUESTS: dict[str, Callable[[], CoreModel]] = { + "submit_body": factories.submit_body, +} + +# Returned by the gateway to the server. The request direction is not model-driven — the server +# hand-builds those payloads — so it is covered on the parsing side instead. +GATEWAY_RESPONSES: dict[str, Callable[[], BaseModel]] = { + "service_stats": factories.service_stats, +} + + +class TestRunnerRequestSerialization: + @pytest.mark.parametrize("name", sorted(RUNNER_REQUESTS)) + def test_matches_fixture(self, name, regen): + payload = RUNNER_REQUESTS[name]().json() + assert_matches_fixture("serialization/runner", name, payload, regen=regen) + + +class TestGatewayResponseSerialization: + @pytest.mark.parametrize("name", sorted(GATEWAY_RESPONSES)) + def test_matches_fixture(self, name, regen): + payload = GATEWAY_RESPONSES[name]().json() + assert_matches_fixture("serialization/gateway", name, payload, regen=regen) + + class TestFleetNodesTargetCompatHack: """ Pins the #3066 old-client hack explicitly, not just via the fixture bytes. From b6b674dd5d5d733f0473e93960e8277a5ef1ebfb Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Wed, 29 Jul 2026 18:04:54 +0500 Subject: [PATCH 08/15] Test proxy models --- .../_internal/pydantic_compat/factories.py | 24 +++++ .../proxy/chat_completions_chunk.input.json | 4 + .../proxy/chat_completions_chunk.types.json | 4 + .../proxy/chat_completions_chunk.values.json | 18 ++++ .../proxy/chat_completions_request.input.json | 7 ++ .../proxy/chat_completions_request.types.json | 4 + .../chat_completions_request.values.json | 23 ++++ .../chat_completions_response.input.json | 9 ++ .../chat_completions_response.types.json | 6 ++ .../chat_completions_response.values.json | 22 ++++ .../proxy/chat_completions_request.json | 13 +++ .../_internal/pydantic_compat/test_parsing.py | 100 +++++++++++------- .../pydantic_compat/test_serialization.py | 84 +++++++++------ 13 files changed, 248 insertions(+), 70 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/proxy/chat_completions_request.json diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index 1383e85216..27aa0c6346 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -89,6 +89,10 @@ VolumeProvisioningData, ) from dstack._internal.proxy.gateway.schemas.stats import ServiceStats, Stat +from dstack._internal.proxy.lib.schemas.model_proxy import ( + ChatCompletionsRequest, + ChatMessage, +) from dstack._internal.server.schemas.fleets import ( ApplyFleetPlanInput, ApplyFleetPlanRequest, @@ -571,3 +575,23 @@ def service_stats() -> ServiceStats: run_name="test-run", stats={60: Stat(requests=10, request_time=0.125)}, ) + + +# --- Model proxy --------------------------------------------------------------------------- +# The OpenAI-compatible proxy. Both directions are model-driven: a request is parsed from the +# caller and then forwarded upstream, and the upstream's reply is parsed back. +# +# The forward leg is the only place in the codebase that dumps with `exclude_unset=True` +# (`proxy/lib/services/model_proxy/clients/openai.py`), which makes `__fields_set__` load-bearing +# here and nowhere else: if a field the caller never set starts counting as set, the proxy begins +# sending keys the caller did not ask for, and the upstream model behaves differently. + + +def chat_completions_request() -> ChatCompletionsRequest: + """Only some fields set on purpose — `exclude_unset` is the point.""" + return ChatCompletionsRequest( + model="llama", + messages=[ChatMessage(role="user", content="hi")], + temperature=0.7, + stop=["\n"], + ) diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.input.json new file mode 100644 index 0000000000..5a0c29599d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.input.json @@ -0,0 +1,4 @@ +{ + "choices": [{"delta": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": null}], + "model": "llama" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.types.json new file mode 100644 index 0000000000..be2b0c22d7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.types.json @@ -0,0 +1,4 @@ +{ + "/": "ChatCompletionsChunk", + "/choices/0": "ChatCompletionsChunkChoice" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.values.json new file mode 100644 index 0000000000..c889dfc5f2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_chunk.values.json @@ -0,0 +1,18 @@ +{ + "choices": [ + { + "delta": { + "content": "hi", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": {} + } + ], + "created": null, + "id": null, + "model": "llama", + "object": "chat.completion.chunk", + "system_fingerprint": "" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.input.json new file mode 100644 index 0000000000..52f3e6a800 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.input.json @@ -0,0 +1,7 @@ +{ + "model": "llama", + "messages": [{"role": "user", "content": "hi"}], + "stop": "\n", + "tool_choice": "auto", + "extra_param_a_newer_caller_sent": 1 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.types.json new file mode 100644 index 0000000000..515dd411a0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.types.json @@ -0,0 +1,4 @@ +{ + "/": "ChatCompletionsRequest", + "/messages/0": "ChatMessage" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.values.json new file mode 100644 index 0000000000..848cc2d2db --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_request.values.json @@ -0,0 +1,23 @@ +{ + "frequency_penalty": 0.0, + "logit_bias": {}, + "max_tokens": null, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "llama", + "n": 1, + "presence_penalty": 0.0, + "response_format": null, + "seed": null, + "stop": "\n", + "stream": false, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "user": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.input.json new file mode 100644 index 0000000000..3808c3ecf1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.input.json @@ -0,0 +1,9 @@ +{ + "id": "chatcmpl-1", + "choices": [{"finish_reason": "stop", "index": 0, "message": {"role": "assistant", "content": "hi"}}], + "created": 1700000000, + "model": "llama", + "usage": {"completion_tokens": 2, "prompt_tokens": 3, "total_tokens": 5}, + "system_fingerprint": "fp_1", + "field_added_by_a_newer_upstream": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.types.json new file mode 100644 index 0000000000..d46add4f78 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.types.json @@ -0,0 +1,6 @@ +{ + "/": "ChatCompletionsResponse", + "/choices/0": "ChatCompletionsChoice", + "/choices/0/message": "ChatMessage", + "/usage": "ChatCompletionsUsage" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.values.json new file mode 100644 index 0000000000..7658d631e0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/proxy/chat_completions_response.values.json @@ -0,0 +1,22 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi", + "role": "assistant" + } + } + ], + "created": 1700000000, + "id": "chatcmpl-1", + "model": "llama", + "object": "chat.completion", + "system_fingerprint": "fp_1", + "usage": { + "completion_tokens": 2, + "prompt_tokens": 3, + "total_tokens": 5 + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy/chat_completions_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy/chat_completions_request.json new file mode 100644 index 0000000000..e750ca73ca --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy/chat_completions_request.json @@ -0,0 +1,13 @@ +{ + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "llama", + "stop": [ + "\n" + ], + "temperature": 0.7 +} diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index 400a368308..acfbf82a11 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -1,5 +1,5 @@ """ -Parse equality for stored blobs, request bodies, and client-side response parsing. +Parse equality across every boundary that reads somebody else's payload. Each case is a frozen input plus two expectations: the values parsing produced, and the concrete classes it chose. Both are needed — `Duration(7200)` and `7200` are the same JSON, so the value @@ -12,6 +12,9 @@ Inputs are hand-written and must never be regenerated. `--regen-fixtures` rewrites `*.values.json` and `*.types.json` only. + +Registries and test classes below follow the same surface order as `test_serialization.py`: +db, api_request, api_response, config, runner, gateway, proxy. """ import json @@ -29,6 +32,11 @@ RegisterReplicaRequest, RegisterServiceRequest, ) +from dstack._internal.proxy.lib.schemas.model_proxy import ( + ChatCompletionsChunk, + ChatCompletionsRequest, + ChatCompletionsResponse, +) from dstack._internal.server.schemas.runner import HealthcheckResponse, MetricsResponse from dstack._internal.server.schemas.volumes import CreateVolumeRequest from tests._internal.pydantic_compat.compare import ( @@ -48,18 +56,18 @@ } # Validated from a request body with extra="forbid": an unknown field is a user-facing error. -REQUEST_BODIES: dict[str, Any] = { +API_REQUESTS: dict[str, Any] = { "create_volume_request": CreateVolumeRequest, } # Parsed by the API client from a server response with extra="ignore", so an older CLI works. -CLIENT_RESPONSES: dict[str, Any] = { +API_RESPONSES: dict[str, Any] = { "fleet": Fleet, } # Parsed from user-authored YAML with extra="forbid". The only surface whose inputs are `.yml`, -# because the sugar pinned here is a YAML phenomenon: `python: 3.10` is a float that only survives via -# number->str coercion, and `16GB..` / `2..8` / an env list are shorthands people type by hand. +# because the sugar pinned here is a YAML phenomenon: `python: 3.10` is a float that survives only +# via number->str coercion, and `16GB..` / `2..8` / an env list are shorthands people type by hand. # `DstackConfiguration` dispatches every `.dstack.yml` type through its `__root__` union, so one # entry point covers task, service, dev-environment, fleet, volume, and gateway. CONFIGS: dict[str, Any] = { @@ -70,15 +78,6 @@ "volume": DstackConfiguration, } - -class TestDbBlobParsing: - @pytest.mark.parametrize("name", sorted(DB_BLOBS)) - def test_parses_to_expected_values_and_types(self, name, regen): - _assert_parses( - "db", name, parse_ignore_extra(DB_BLOBS[name], _load_input("db", name)), regen - ) - - # Responses the server reads back from the runner (shim), permissively: a newer runner may add # fields, and the server has no say over which runner version an instance is running. RUNNER_RESPONSES: dict[str, Any] = { @@ -87,49 +86,57 @@ def test_parses_to_expected_values_and_types(self, name, regen): } # Request payloads the gateway parses from the server. These schemas are plain `BaseModel`, so -# `parse_obj` already ignores unknown fields in both pydantic versions — no shim, and nothing to -# assert about strictness. +# `parse_obj` already ignores unknown fields in both pydantic versions — no shim needed, and +# nothing to assert about strictness. GATEWAY_REQUESTS: dict[str, Any] = { "register_replica_request": RegisterReplicaRequest, "register_service_request": RegisterServiceRequest, } - -class TestRunnerResponseParsing: - @pytest.mark.parametrize("name", sorted(RUNNER_RESPONSES)) - def test_parses_to_expected_values_and_types(self, name, regen): - model = parse_ignore_extra(RUNNER_RESPONSES[name], _load_input("runner", name)) - _assert_parses("runner", name, model, regen) +# The model proxy. The request comes from a caller and the response from an upstream model server, +# so both are somebody else's payload and both are read permissively. `stop` and `tool_choice` are +# two of the unions with no common Literal to discriminate on, which makes them the clearest +# smart-union cases in the codebase. +PROXY_PAYLOADS: dict[str, Any] = { + "chat_completions_chunk": ChatCompletionsChunk, + "chat_completions_request": ChatCompletionsRequest, + "chat_completions_response": ChatCompletionsResponse, +} -class TestGatewayRequestParsing: - @pytest.mark.parametrize("name", sorted(GATEWAY_REQUESTS)) +class TestDbBlobParsing: + @pytest.mark.parametrize("name", sorted(DB_BLOBS)) def test_parses_to_expected_values_and_types(self, name, regen): - model = GATEWAY_REQUESTS[name].parse_obj(_load_input("gateway", name)) - _assert_parses("gateway", name, model, regen) + model = parse_ignore_extra(DB_BLOBS[name], _load_input("db", name)) + _assert_parses("db", name, model, regen) class TestDbBlobExtraFieldTolerance: """ Every stored model must survive a row extended by a newer writer. + + This needs no hand-written input: the serialization fixture already *is* a valid row, so + injecting one unknown key into it produces the exact situation an older reader faces. Covers + all of `serialization/db` rather than only the models with an interesting input above, because + failing to read a `VolumeProvisioningData` row is a distinct bug from failing to read a + `RunSpec` row. """ @pytest.mark.parametrize("name", sorted(SERIALIZED_DB_BLOBS)) def test_unknown_field_is_dropped(self, name): committed = (FIXTURES_DIR / "serialization" / "db" / f"{name}.json").read_text() perturbed = {**json.loads(committed), "unknown_from_a_newer_writer": {"x": [1]}} - model = type(SERIALIZED_DB_BLOBS[name]()) - parsed = parse_ignore_extra(model, perturbed) + parsed = parse_ignore_extra(type(SERIALIZED_DB_BLOBS[name]()), perturbed) assert canonicalize(parsed.json()) == canonicalize(committed) -class TestRequestBodyParsing: - @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) +class TestApiRequestParsing: + @pytest.mark.parametrize("name", sorted(API_REQUESTS)) def test_parses_to_expected_values_and_types(self, name, regen): - model = parse_forbid_extra(REQUEST_BODIES[name], _load_input("api_request", name)) + model = parse_forbid_extra(API_REQUESTS[name], _load_input("api_request", name)) _assert_parses("api_request", name, model, regen) - @pytest.mark.parametrize("name", sorted(REQUEST_BODIES)) + @pytest.mark.parametrize("name", sorted(API_REQUESTS)) def test_unknown_field_is_still_rejected(self, name): """ Forbidding extra fields is the feature here: it is what makes `dstack apply` report a @@ -139,13 +146,13 @@ def test_unknown_field_is_still_rejected(self, name): """ body = {**_load_input("api_request", name), "definitely_not_a_field": 1} with pytest.raises(Exception, match="(?i)extra"): - parse_forbid_extra(REQUEST_BODIES[name], body) + parse_forbid_extra(API_REQUESTS[name], body) -class TestClientResponseParsing: - @pytest.mark.parametrize("name", sorted(CLIENT_RESPONSES)) +class TestApiResponseParsing: + @pytest.mark.parametrize("name", sorted(API_RESPONSES)) def test_parses_to_expected_values_and_types(self, name, regen): - model = parse_ignore_extra(CLIENT_RESPONSES[name], _load_input("api_response", name)) + model = parse_ignore_extra(API_RESPONSES[name], _load_input("api_response", name)) _assert_parses("api_response", name, model, regen) @@ -156,6 +163,27 @@ def test_parses_to_expected_values_and_types(self, name, regen): _assert_parses("config", name, model, regen) +class TestRunnerResponseParsing: + @pytest.mark.parametrize("name", sorted(RUNNER_RESPONSES)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_ignore_extra(RUNNER_RESPONSES[name], _load_input("runner", name)) + _assert_parses("runner", name, model, regen) + + +class TestGatewayRequestParsing: + @pytest.mark.parametrize("name", sorted(GATEWAY_REQUESTS)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = GATEWAY_REQUESTS[name].parse_obj(_load_input("gateway", name)) + _assert_parses("gateway", name, model, regen) + + +class TestProxyPayloadParsing: + @pytest.mark.parametrize("name", sorted(PROXY_PAYLOADS)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_ignore_extra(PROXY_PAYLOADS[name], _load_input("proxy", name)) + _assert_parses("proxy", name, model, regen) + + def _assert_parses(surface: str, name: str, model, regen: bool) -> None: kind = f"parsing/{surface}" assert_matches_fixture(kind, f"{name}.values", model.json(), regen=regen) diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 93665ebb07..72cecad1e9 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -1,18 +1,21 @@ """ -Serialization stability for DB blobs and API responses. +Serialization stability across every boundary that writes a payload somebody else reads. -Each model is serialized through the *production* path — `.json()` for anything written to a -`Text` column, `CustomORJSONResponse` for anything returned from a router — and compared against a -fixture generated under pydantic v1. On v1 these are regression tests; on the v2 branch they are -the compat assertion. +Each model is serialized through the *production* path — there are four distinct ones, noted on +each registry below — and compared against a fixture generated under pydantic v1. On v1 these are +regression tests; on the v2 branch they are the compat assertion. Nothing here may reference the duality API (`__request__` / `__response__`): these tests have to run unchanged on both versions, and duality is gone in v2. Disposable: this package is deleted once the v2 release is verified in prod, except for a curated subset of the `db/` fixtures, which outlive it because stored rows do. + +Registries and test classes follow the same surface order as `test_parsing.py`: +db, api_request, api_response, runner, gateway, proxy. """ +import json from typing import Callable import pytest @@ -50,7 +53,19 @@ "volume_provisioning_data": factories.volume_provisioning_data, } -# Returned from a router via `CustomORJSONResponse`. +# Sent by the API client as a request body — `body=X.json()`, 40 call sites in `api/server/`. +# This is the new-CLI-against-old-server direction. +API_REQUESTS: dict[str, Callable[[], CoreModel]] = { + "apply_fleet_plan_request": factories.apply_fleet_plan_request, + "apply_gateway_plan_request": factories.apply_gateway_plan_request, + "apply_run_plan_request": factories.apply_run_plan_request, + "create_volume_request": factories.create_volume_request, + "delete_fleets_request": factories.delete_fleets_request, + "save_repo_creds_request": factories.save_repo_creds_request, +} + +# Returned from a router via `CustomORJSONResponse` — orjson with a `default=` hook rather than +# `.json()`, so this path can drift away from the one above. API_RESPONSES: dict[str, Callable[[], CoreModel]] = { "fleet": factories.fleet, "fleet_plan": factories.fleet_plan, @@ -63,15 +78,23 @@ "volume": factories.volume, } -# Sent by the API client as a request body — `body=X.json()`, 40 call sites in `api/server/`. -# This is the new-CLI-against-old-server direction, which nothing else in the suite covers. -API_REQUESTS: dict[str, Callable[[], CoreModel]] = { - "delete_fleets_request": factories.delete_fleets_request, - "save_repo_creds_request": factories.save_repo_creds_request, - "apply_fleet_plan_request": factories.apply_fleet_plan_request, - "apply_gateway_plan_request": factories.apply_gateway_plan_request, - "apply_run_plan_request": factories.apply_run_plan_request, - "create_volume_request": factories.create_volume_request, +# Sent by the server to the runner (shim) as a request body, via `.json()`. +RUNNER_REQUESTS: dict[str, Callable[[], CoreModel]] = { + "submit_body": factories.submit_body, +} + +# Returned by the gateway to the server. The request direction is not model-driven — the server +# hand-builds those payloads — so it is covered on the parsing side instead. +GATEWAY_RESPONSES: dict[str, Callable[[], BaseModel]] = { + "service_stats": factories.service_stats, +} + +# Forwarded upstream by the model proxy. Dumped with `exclude_unset=True`, matching +# `proxy/lib/services/model_proxy/clients/openai.py`, and encoded with stdlib json the way httpx +# does for a `json=` body — the fourth distinct dump path, and the only one where +# `__fields_set__` decides what goes on the wire. +PROXY_REQUESTS: dict[str, Callable[[], CoreModel]] = { + "chat_completions_request": factories.chat_completions_request, } @@ -82,15 +105,6 @@ def test_matches_fixture(self, name, regen): assert_matches_fixture("serialization/db", name, payload, regen=regen) -class TestApiResponseSerialization: - @pytest.mark.parametrize("name", sorted(API_RESPONSES)) - def test_matches_fixture(self, name, regen): - # Response bodies go through orjson with a `default=` hook, not through `.json()`, so the - # two paths can drift apart. Serialize the way the router does. - payload = bytes(CustomORJSONResponse(API_RESPONSES[name]()).body) - assert_matches_fixture("serialization/api_response", name, payload, regen=regen) - - class TestApiRequestSerialization: @pytest.mark.parametrize("name", sorted(API_REQUESTS)) def test_matches_fixture(self, name, regen): @@ -98,16 +112,11 @@ def test_matches_fixture(self, name, regen): assert_matches_fixture("serialization/api_request", name, payload, regen=regen) -# Sent by the server to the runner (shim) as a request body. -RUNNER_REQUESTS: dict[str, Callable[[], CoreModel]] = { - "submit_body": factories.submit_body, -} - -# Returned by the gateway to the server. The request direction is not model-driven — the server -# hand-builds those payloads — so it is covered on the parsing side instead. -GATEWAY_RESPONSES: dict[str, Callable[[], BaseModel]] = { - "service_stats": factories.service_stats, -} +class TestApiResponseSerialization: + @pytest.mark.parametrize("name", sorted(API_RESPONSES)) + def test_matches_fixture(self, name, regen): + payload = bytes(CustomORJSONResponse(API_RESPONSES[name]()).body) + assert_matches_fixture("serialization/api_response", name, payload, regen=regen) class TestRunnerRequestSerialization: @@ -124,6 +133,13 @@ def test_matches_fixture(self, name, regen): assert_matches_fixture("serialization/gateway", name, payload, regen=regen) +class TestProxyRequestSerialization: + @pytest.mark.parametrize("name", sorted(PROXY_REQUESTS)) + def test_matches_fixture(self, name, regen): + payload = json.dumps(PROXY_REQUESTS[name]().dict(exclude_unset=True)) + assert_matches_fixture("serialization/proxy", name, payload, regen=regen) + + class TestFleetNodesTargetCompatHack: """ Pins the #3066 old-client hack explicitly, not just via the fixture bytes. From fbdb641cd4d1a4dff3d9b83df985343c57525553 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 10:17:10 +0500 Subject: [PATCH 09/15] Add more tests --- src/tests/_internal/pydantic_compat/compat.py | 5 + .../_internal/pydantic_compat/factories.py | 230 ++++++++++++++++- .../apply_fleet_plan_request.input.json | 17 ++ .../apply_fleet_plan_request.types.json | 16 ++ .../apply_fleet_plan_request.values.json | 60 +++++ .../apply_gateway_plan_request.input.json | 16 ++ .../apply_gateway_plan_request.types.json | 8 + .../apply_gateway_plan_request.values.json | 25 ++ .../apply_run_plan_request.input.json | 24 ++ .../apply_run_plan_request.types.json | 30 +++ .../apply_run_plan_request.values.json | 133 ++++++++++ .../delete_fleets_request.input.json | 1 + .../delete_fleets_request.types.json | 3 + .../delete_fleets_request.values.json | 6 + .../save_repo_creds_request.input.json | 8 + .../save_repo_creds_request.types.json | 5 + .../save_repo_creds_request.values.json | 12 + .../api_response/fleet_plan.input.json | 38 +++ .../api_response/fleet_plan.types.json | 25 ++ .../api_response/fleet_plan.values.json | 100 ++++++++ .../parsing/api_response/gateway.input.json | 19 ++ .../parsing/api_response/gateway.types.json | 10 + .../parsing/api_response/gateway.values.json | 32 +++ .../parsing/api_response/project.input.json | 40 +++ .../parsing/api_response/project.types.json | 23 ++ .../parsing/api_response/project.values.json | 60 +++++ .../parsing/api_response/run_plan.input.json | 54 ++++ .../parsing/api_response/run_plan.types.json | 46 ++++ .../parsing/api_response/run_plan.values.json | 242 ++++++++++++++++++ .../parsing/api_response/secret.input.json | 1 + .../parsing/api_response/secret.types.json | 4 + .../parsing/api_response/secret.values.json | 5 + .../api_response/server_info.input.json | 1 + .../api_response/server_info.types.json | 3 + .../api_response/server_info.values.json | 3 + .../api_response/user_with_creds.input.json | 10 + .../api_response/user_with_creds.types.json | 8 + .../api_response/user_with_creds.values.json | 16 ++ .../parsing/api_response/volume.input.json | 27 ++ .../parsing/api_response/volume.types.json | 12 + .../parsing/api_response/volume.values.json | 38 +++ .../parsing/config/dev_environment.input.yml | 11 + .../parsing/config/dev_environment.types.json | 21 ++ .../config/dev_environment.values.json | 81 ++++++ ...compute_group_provisioning_data.input.json | 8 + ...compute_group_provisioning_data.types.json | 4 + ...ompute_group_provisioning_data.values.json | 9 + .../fixtures/parsing/db/fleet_spec.input.json | 17 ++ .../fixtures/parsing/db/fleet_spec.types.json | 27 ++ .../parsing/db/fleet_spec.values.json | 87 +++++++ .../gateway_compute_configuration.input.json | 9 + .../gateway_compute_configuration.types.json | 5 + .../gateway_compute_configuration.values.json | 14 + .../db/gateway_configuration.input.json | 8 + .../db/gateway_configuration.types.json | 5 + .../db/gateway_configuration.values.json | 16 ++ .../parsing/db/image_pull_progress.input.json | 6 + .../parsing/db/image_pull_progress.types.json | 3 + .../db/image_pull_progress.values.json | 6 + .../db/instance_configuration.input.json | 9 + .../db/instance_configuration.types.json | 4 + .../db/instance_configuration.values.json | 17 ++ .../parsing/db/instance_offer.input.json | 17 ++ .../parsing/db/instance_offer.types.json | 11 + .../parsing/db/instance_offer.values.json | 31 +++ .../db/job_provisioning_data.input.json | 33 +++ .../db/job_provisioning_data.types.json | 23 ++ .../db/job_provisioning_data.values.json | 73 ++++++ .../parsing/db/job_runtime_data.input.json | 10 + .../parsing/db/job_runtime_data.types.json | 5 + .../parsing/db/job_runtime_data.values.json | 16 ++ .../placement_group_configuration.input.json | 1 + .../placement_group_configuration.types.json | 5 + .../placement_group_configuration.values.json | 5 + ...acement_group_provisioning_data.input.json | 1 + ...acement_group_provisioning_data.types.json | 4 + ...cement_group_provisioning_data.values.json | 4 + .../fixtures/parsing/db/profile.input.json | 13 + .../fixtures/parsing/db/profile.types.json | 13 + .../fixtures/parsing/db/profile.values.json | 42 +++ .../db/remote_connection_info.input.json | 7 + .../db/remote_connection_info.types.json | 5 + .../db/remote_connection_info.values.json | 16 ++ .../parsing/db/requirements.input.json | 17 ++ .../parsing/db/requirements.types.json | 17 ++ .../parsing/db/requirements.values.json | 44 ++++ .../fixtures/parsing/db/resources.input.json | 9 + .../fixtures/parsing/db/resources.types.json | 7 + .../fixtures/parsing/db/resources.values.json | 17 ++ .../parsing/db/service_spec.input.json | 5 + .../parsing/db/service_spec.types.json | 4 + .../parsing/db/service_spec.values.json | 18 ++ .../db/volume_attachment_data.input.json | 1 + .../db/volume_attachment_data.types.json | 3 + .../db/volume_attachment_data.values.json | 3 + .../db/volume_configuration.input.json | 8 + .../db/volume_configuration.types.json | 5 + .../db/volume_configuration.values.json | 11 + .../db/volume_provisioning_data.input.json | 8 + .../db/volume_provisioning_data.types.json | 4 + .../db/volume_provisioning_data.values.json | 10 + .../register_entrypoint_request.input.json | 1 + .../register_entrypoint_request.types.json | 3 + .../register_entrypoint_request.values.json | 4 + .../register_replica_request.input.json | 4 +- .../register_replica_request.values.json | 4 +- .../parsing/gateway/service_stats.input.json | 9 + .../parsing/gateway/service_stats.types.json | 6 + .../parsing/gateway/service_stats.values.json | 18 ++ .../instance_health_response.input.json | 4 + .../instance_health_response.types.json | 5 + .../instance_health_response.values.json | 6 + .../runner/job_info_response.input.json | 1 + .../runner/job_info_response.types.json | 3 + .../runner/job_info_response.values.json | 4 + .../runner/task_info_response.input.json | 7 + .../runner/task_info_response.types.json | 4 + .../runner/task_info_response.values.json | 8 + .../api_request/apply_fleet_plan_request.json | 122 ++++++--- .../api_request/apply_run_plan_request.json | 77 ++++-- .../serialization/api_response/fleet.json | 122 ++++++--- .../api_response/fleet_plan.json | 122 ++++++--- .../serialization/api_response/run_plan.json | 77 ++++-- .../fixtures/serialization/db/fleet_spec.json | 122 ++++++--- .../db/instance_configuration.json | 15 +- .../serialization/db/job_runtime_data.json | 14 +- .../fixtures/serialization/db/profile.json | 46 +++- .../fixtures/serialization/db/run_spec.json | 77 ++++-- .../chat_completions_chunk.json | 18 ++ .../proxy_response/models_response.json | 11 + .../runner/component_install_request.json | 4 + .../runner/legacy_submit_body.json | 17 ++ .../runner/shutdown_request.json | 3 + .../serialization/runner/submit_body.json | 36 +-- .../runner/task_submit_request.json | 25 ++ .../runner/task_terminate_request.json | 5 + .../_internal/pydantic_compat/test_parsing.py | 129 +++++++--- .../pydantic_compat/test_serialization.py | 74 +++--- 138 files changed, 3203 insertions(+), 313 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/chat_completions_chunk.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/models_response.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/runner/component_install_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/runner/legacy_submit_body.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/runner/shutdown_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_submit_request.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_terminate_request.json diff --git a/src/tests/_internal/pydantic_compat/compat.py b/src/tests/_internal/pydantic_compat/compat.py index 2a67c1b3e2..94ccb78f1d 100644 --- a/src/tests/_internal/pydantic_compat/compat.py +++ b/src/tests/_internal/pydantic_compat/compat.py @@ -36,6 +36,11 @@ def parse_ignore_extra(model: Any, data: Any) -> BaseModel: Unknown fields are dropped, which is what lets an older reader survive a newer writer, so it is the behaviour the whole migration has to preserve. """ + if not hasattr(model, "__response__"): + # Plain `BaseModel` rather than `CoreModel` — the proxy and gateway schemas. Their default + # is already extra="ignore" in both pydantic versions, so `parse_obj` is the ignore path + # and there is no duality variant to reach for. + return model.parse_obj(data) if PYDANTIC_V1: return model.__response__.parse_obj(data) # v2: from dstack._internal.core.models.common import validate_extra_ignore diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index 27aa0c6346..85f6a24bf4 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -11,11 +11,24 @@ from dstack._internal.core.backends.aws.models import AWSCreds from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.common import ApplyAction +from dstack._internal.core.models.common import ( + ApplyAction, + EntityReference, + NetworkMode, +) from dstack._internal.core.models.compute_groups import ComputeGroupProvisioningData -from dstack._internal.core.models.configurations import DevEnvironmentConfiguration +from dstack._internal.core.models.configurations import ( + DevEnvironmentConfiguration, + PythonVersion, +) from dstack._internal.core.models.envs import Env -from dstack._internal.core.models.fleets import Fleet, FleetPlan, FleetSpec, FleetStatus +from dstack._internal.core.models.fleets import ( + Fleet, + FleetPlan, + FleetSpec, + FleetStatus, + InstanceGroupPlacement, +) from dstack._internal.core.models.gateways import ( Gateway, GatewayComputeConfiguration, @@ -32,16 +45,21 @@ InstanceStatus, RemoteConnectionInfo, Resources, + SSHKey, ) from dstack._internal.core.models.placement import ( PlacementGroupConfiguration, PlacementGroupProvisioningData, ) from dstack._internal.core.models.profiles import ( + CreationPolicy, Profile, ProfileRetry, RetryEvent, + Schedule, SpotPolicy, + StartupOrder, + StopCriteria, ) from dstack._internal.core.models.projects import ( Member, @@ -52,6 +70,7 @@ from dstack._internal.core.models.repos.remote import RemoteRepoCreds, RemoteRepoInfo from dstack._internal.core.models.resources import ( ComputeCapability, + DiskSpec, GPUSpec, Memory, Range, @@ -88,10 +107,15 @@ VolumeConfiguration, VolumeProvisioningData, ) +from dstack._internal.proxy.gateway.schemas.registry import RegisterEntrypointRequest from dstack._internal.proxy.gateway.schemas.stats import ServiceStats, Stat from dstack._internal.proxy.lib.schemas.model_proxy import ( + ChatCompletionsChunk, + ChatCompletionsChunkChoice, ChatCompletionsRequest, ChatMessage, + Model, + ModelsResponse, ) from dstack._internal.server.schemas.fleets import ( ApplyFleetPlanInput, @@ -103,7 +127,20 @@ ApplyGatewayPlanRequest, ) from dstack._internal.server.schemas.repos import SaveRepoCredsRequest -from dstack._internal.server.schemas.runner import HealthcheckResponse, SubmitBody +from dstack._internal.server.schemas.runner import ( + ComponentInstallRequest, + ComponentName, + HealthcheckResponse, + InstanceHealthResponse, + JobInfoResponse, + LegacySubmitBody, + ShutdownRequest, + SubmitBody, + TaskInfoResponse, + TaskStatus, + TaskSubmitRequest, + TaskTerminateRequest, +) from dstack._internal.server.schemas.runs import ( ApplyRunPlanInput, ApplyRunPlanRequest, @@ -111,6 +148,7 @@ from dstack._internal.server.schemas.volumes import CreateVolumeRequest from dstack._internal.server.testing.common import ( get_compute_group_provisioning_data, + get_fleet_configuration, get_fleet_spec, get_gateway_compute_configuration, get_instance_configuration, @@ -145,7 +183,31 @@ def compute_group_provisioning_data() -> ComputeGroupProvisioningData: def fleet_spec() -> FleetSpec: - return get_fleet_spec() + """ + Fills the optional configuration fields: unfilled they serialize as ~34 nulls, and a null + cannot drift, so the fixture would pin almost nothing. + """ + spec = get_fleet_spec( + conf=get_fleet_configuration( + backends=[BackendType.AWS], + placement=InstanceGroupPlacement.CLUSTER, + ), + profile=profile(), + ) + conf = spec.configuration + conf.regions = ["us-east-1"] + conf.availability_zones = ["us-east-1a"] + conf.instance_types = ["p4d.24xlarge"] + conf.reservation = "test-reservation" + conf.spot_policy = SpotPolicy.AUTO + conf.idle_duration = 600 + conf.max_price = 25.5 + conf.tags = {"env": "test"} + conf.resources = ResourcesSpec( + cpu=Range[int](min=2, max=8), + memory=Range[Memory](min=Memory(16), max=None), + ) + return spec def gateway_compute_configuration() -> GatewayComputeConfiguration: @@ -171,7 +233,12 @@ def image_pull_progress() -> ImagePullProgress: def instance_configuration() -> InstanceConfiguration: - return get_instance_configuration() + conf = get_instance_configuration() + conf.instance_id = "i-1234567890abcdef0" + conf.reservation = "test-reservation" + conf.tags = {"env": "test"} + conf.ssh_keys = [SSHKey(public="ssh-rsa PUBLIC", private=None)] + return conf def instance_offer() -> InstanceOffer: @@ -190,6 +257,13 @@ def job_runtime_data() -> JobRuntimeData: """`ports` is a `dict[int, int]` — the only non-str dict keys in the models.""" data = get_job_runtime_data() data.ports = {8080: 30080, 8081: 30081} + data.network_mode = NetworkMode.HOST + data.cpu = 2.0 + data.gpu = 1 + data.memory = float(16 * 1024**3) + data.volume_names = ["test-volume"] + data.working_dir = "/workflow" + data.username = "ubuntu" return data @@ -217,11 +291,23 @@ def placement_group_provisioning_data() -> PlacementGroupProvisioningData: def profile() -> Profile: return Profile( name="default", + backends=[BackendType.AWS, BackendType.GCP], + regions=["us-east-1", "eu-west-1"], + availability_zones=["us-east-1a"], + instance_types=["p4d.24xlarge"], + reservation="test-reservation", + spot_policy=SpotPolicy.AUTO, + retry=ProfileRetry(on_events=[RetryEvent.NO_CAPACITY], duration=3600), max_duration=7200, stop_duration=300, idle_duration=600, - spot_policy=SpotPolicy.AUTO, - retry=ProfileRetry(on_events=[RetryEvent.NO_CAPACITY], duration=3600), + max_price=25.5, + creation_policy=CreationPolicy.REUSE_OR_CREATE, + stop_criteria=StopCriteria.ALL_DONE, + startup_order=StartupOrder.MASTER_FIRST, + fleets=[EntityReference(project=None, name="test-fleet")], + tags={"env": "test", "team": "core"}, + schedule=Schedule(cron=["0 0 * * *"]), ) @@ -268,12 +354,24 @@ def run_spec() -> RunSpec: """ return get_run_spec( repo_id="test-repo", - profile=Profile(name="default", max_duration=7200, idle_duration=300), + profile=profile(), configuration=DevEnvironmentConfiguration( + name="test-dev", ide="vscode", + version="1.85.0", + user="ubuntu", + privileged=False, + # `image` is deliberately absent: it is mutually exclusive with `python`, and `python` + # is the more valuable of the two to pin because it is a str enum fed by a YAML float. + python=PythonVersion.PY311, + env=Env.parse_obj({"HF_TOKEN": "secret"}), + working_dir="/workflow", + inactivity_duration=3600, resources=ResourcesSpec( cpu=Range[int](min=2, max=8), memory=Range[Memory](min=Memory(16), max=None), + shm_size=Memory(1024), + disk=DiskSpec(size=Range[Memory](min=Memory(100), max=None)), ), ), ) @@ -312,7 +410,7 @@ def fleet() -> Fleet: id=_ID, name="test-fleet", project_name="test-project", - spec=get_fleet_spec(), + spec=fleet_spec(), created_at=_CREATED_AT, status=FleetStatus.ACTIVE, instances=[ @@ -356,7 +454,7 @@ def fleet_plan() -> FleetPlan: return FleetPlan( project_name="test-project", user="test-user", - spec=get_fleet_spec(), + spec=fleet_spec(), effective_spec=None, current_resource=None, offers=[get_instance_offer_with_availability()], @@ -462,7 +560,7 @@ def apply_fleet_plan_request() -> ApplyFleetPlanRequest: client side of the #3066 `target`-dropping hack too. """ return ApplyFleetPlanRequest( - plan=ApplyFleetPlanInput(spec=get_fleet_spec(), current_resource=None), + plan=ApplyFleetPlanInput(spec=fleet_spec(), current_resource=None), force=False, ) @@ -559,6 +657,84 @@ def healthcheck_response() -> HealthcheckResponse: # never run — worth knowing before the serializer is swapped. +def task_submit_request() -> TaskSubmitRequest: + """The newer per-task API; `TaskSubmitRequest` and `SubmitBody` are separate protocols.""" + return TaskSubmitRequest( + id=str(_ID), + name="test-task", + registry_username="", + registry_password="", + image_name="dstackai/base:latest", + container_user="root", + privileged=False, + gpu=1, + cpu=2.0, + memory=16 * 1024**3, + shm_size=1024**3, + network_mode=NetworkMode.HOST, + volumes=[], + volume_mounts=[], + instance_mounts=[], + gpu_devices=[], + host_ssh_user="ubuntu", + host_ssh_keys=["ssh-rsa HOST"], + container_ssh_keys=["ssh-rsa CONTAINER"], + ) + + +def legacy_submit_body() -> LegacySubmitBody: + """Kept because old runners are still in the wild; its shape must not drift either.""" + return LegacySubmitBody( + username="", + password="", + image_name="dstackai/base:latest", + privileged=False, + container_name="test-container", + container_user="root", + shm_size=1024**3, + public_keys=["ssh-rsa PUBLIC"], + ssh_user="ubuntu", + ssh_key="ssh-rsa SSH", + mounts=[], + volumes=[], + instance_mounts=[], + ) + + +def shutdown_request() -> ShutdownRequest: + return ShutdownRequest(force=True) + + +def component_install_request() -> ComponentInstallRequest: + return ComponentInstallRequest(name=ComponentName.SHIM, url="https://example.com/shim.tar.gz") + + +def task_terminate_request() -> TaskTerminateRequest: + return TaskTerminateRequest( + termination_reason="MAX_DURATION_EXCEEDED", + termination_message="max duration exceeded", + timeout=10, + ) + + +def job_info_response() -> JobInfoResponse: + return JobInfoResponse(working_dir="/workflow", username="ubuntu") + + +def task_info_response() -> TaskInfoResponse: + return TaskInfoResponse( + id=str(_ID), + status=TaskStatus.RUNNING, + termination_reason="", + termination_message="", + ) + + +def instance_health_response() -> InstanceHealthResponse: + """All fields optional; the empty shape is what a runner without DCGM reports.""" + return InstanceHealthResponse() + + # --- Gateway API ----------------------------------------------------------------------- # The server<->gateway protocol. Note the server builds these payloads as hand-written dicts # rather than dumping a model (see `services/gateways/client.py`), so only the gateway's own @@ -595,3 +771,33 @@ def chat_completions_request() -> ChatCompletionsRequest: temperature=0.7, stop=["\n"], ) + + +def register_entrypoint_request() -> RegisterEntrypointRequest: + return RegisterEntrypointRequest(domain="gateway.example.com", https=True) + + +# --- Model proxy responses ------------------------------------------------------------------- +# Returned to the caller of the OpenAI-compatible API. `ChatCompletionsChunk` is serialized one +# chunk at a time into an SSE stream (`proxy/lib/routers/model_proxy.py`), which is a fifth dump +# path: `f"data:{chunk.json()}"` rather than a response body. + + +def models_response() -> ModelsResponse: + return ModelsResponse( + data=[Model(object="model", id="llama", created=1700000000, owned_by="dstack")] + ) + + +def chat_completions_chunk() -> ChatCompletionsChunk: + return ChatCompletionsChunk( + id="chatcmpl-1", + choices=[ + ChatCompletionsChunkChoice( + delta={"role": "assistant", "content": "hi"}, index=0, finish_reason=None + ) + ], + created=1700000000, + model="llama", + system_fingerprint="fp_1", + ) diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.input.json new file mode 100644 index 0000000000..7a5f5ffffa --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.input.json @@ -0,0 +1,17 @@ +{ + "plan": { + "spec": { + "configuration": { + "type": "fleet", + "name": "research-cluster", + "nodes": {"min": 2, "max": 8}, + "placement": "cluster", + "backends": ["aws"], + "spot_policy": "on-demand" + }, + "configuration_path": ".dstack/fleets/research.yml", + "profile": {"name": "prod", "spot_policy": "on-demand"} + } + }, + "force": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.types.json new file mode 100644 index 0000000000..d8be84d01a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.types.json @@ -0,0 +1,16 @@ +{ + "/": "ApplyFleetPlanRequest", + "/plan": "ApplyFleetPlanInput", + "/plan/spec": "FleetSpec", + "/plan/spec/configuration": "FleetConfiguration", + "/plan/spec/configuration/backends/0": "BackendType", + "/plan/spec/configuration/env": "Env", + "/plan/spec/configuration/nodes": "FleetNodesSpec", + "/plan/spec/configuration/placement": "InstanceGroupPlacement", + "/plan/spec/configuration/spot_policy": "SpotPolicy", + "/plan/spec/merged_profile": "Profile", + "/plan/spec/merged_profile/backends/0": "BackendType", + "/plan/spec/merged_profile/spot_policy": "SpotPolicy", + "/plan/spec/profile": "Profile", + "/plan/spec/profile/spot_policy": "SpotPolicy" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.values.json new file mode 100644 index 0000000000..121008fe9a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_fleet_plan_request.values.json @@ -0,0 +1,60 @@ +{ + "force": false, + "plan": { + "current_resource": null, + "spec": { + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": [ + "aws" + ], + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": null, + "max_price": null, + "name": "research-cluster", + "nodes": { + "max": 8, + "min": 2 + }, + "placement": "cluster", + "regions": null, + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": "on-demand", + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": ".dstack/fleets/research.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "prod", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": "on-demand", + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.input.json new file mode 100644 index 0000000000..ce788c829a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.input.json @@ -0,0 +1,16 @@ +{ + "plan": { + "spec": { + "configuration": { + "type": "gateway", + "name": "prod-gateway", + "backend": "aws", + "region": "eu-west-1", + "domain": "inference.example.com", + "certificate": {"type": "lets-encrypt"} + }, + "configuration_path": ".dstack/gateway.yml" + } + }, + "force": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.types.json new file mode 100644 index 0000000000..77ed3fb9bd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.types.json @@ -0,0 +1,8 @@ +{ + "/": "ApplyGatewayPlanRequest", + "/plan": "ApplyGatewayPlanInput", + "/plan/spec": "GatewaySpec", + "/plan/spec/configuration": "GatewayConfiguration", + "/plan/spec/configuration/backend": "BackendType", + "/plan/spec/configuration/certificate": "LetsEncryptGatewayCertificate" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json new file mode 100644 index 0000000000..431ae169f4 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_gateway_plan_request.values.json @@ -0,0 +1,25 @@ +{ + "force": false, + "plan": { + "current_resource": null, + "spec": { + "configuration": { + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "default": false, + "domain": "inference.example.com", + "instance_type": null, + "name": "prod-gateway", + "public_ip": true, + "region": "eu-west-1", + "replicas": null, + "router": null, + "tags": null, + "type": "gateway" + }, + "configuration_path": ".dstack/gateway.yml" + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.input.json new file mode 100644 index 0000000000..32e3feddd1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.input.json @@ -0,0 +1,24 @@ +{ + "plan": { + "run_spec": { + "run_name": "llama-70b-sft", + "repo_id": "3f8a1c2e9b7d4a5f", + "repo_data": {"repo_type": "remote", "repo_name": "llm-training", "repo_branch": "main", "repo_hash": "9f2c1a7"}, + "repo_code_hash": "4b8e2d1f6a3c9075", + "configuration_path": ".dstack/train.yml", + "configuration": { + "type": "task", + "name": "llama-70b-sft", + "commands": ["accelerate launch train.py --model llama-3.1-70b"], + "nodes": 2, + "python": "3.11", + "env": ["HF_TOKEN", "WANDB_PROJECT=llm-training"], + "resources": {"gpu": "A100:8", "memory": "512GB..", "disk": "1TB.."}, + "max_duration": "3d" + }, + "profile": {"name": "prod", "backends": ["aws"], "spot_policy": "on-demand", "max_duration": "3d"}, + "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 alice@example.com" + } + }, + "force": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.types.json new file mode 100644 index 0000000000..50646cd45b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.types.json @@ -0,0 +1,30 @@ +{ + "/": "ApplyRunPlanRequest", + "/plan": "ApplyRunPlanInput", + "/plan/run_spec": "RunSpec", + "/plan/run_spec/configuration": "TaskConfiguration", + "/plan/run_spec/configuration/env": "Env", + "/plan/run_spec/configuration/env/__root__/HF_TOKEN": "EnvSentinel", + "/plan/run_spec/configuration/max_duration": "Duration", + "/plan/run_spec/configuration/python": "PythonVersion", + "/plan/run_spec/configuration/resources": "ResourcesSpec", + "/plan/run_spec/configuration/resources/cpu": "CPUSpec", + "/plan/run_spec/configuration/resources/cpu/count": "Range[int]", + "/plan/run_spec/configuration/resources/disk": "DiskSpec", + "/plan/run_spec/configuration/resources/disk/size": "Range[Memory]", + "/plan/run_spec/configuration/resources/disk/size/min": "Memory", + "/plan/run_spec/configuration/resources/gpu": "GPUSpec", + "/plan/run_spec/configuration/resources/gpu/count": "Range[int]", + "/plan/run_spec/configuration/resources/memory": "Range[Memory]", + "/plan/run_spec/configuration/resources/memory/min": "Memory", + "/plan/run_spec/merged_profile": "Profile", + "/plan/run_spec/merged_profile/backends/0": "BackendType", + "/plan/run_spec/merged_profile/creation_policy": "CreationPolicy", + "/plan/run_spec/merged_profile/max_duration": "Duration", + "/plan/run_spec/merged_profile/spot_policy": "SpotPolicy", + "/plan/run_spec/profile": "Profile", + "/plan/run_spec/profile/backends/0": "BackendType", + "/plan/run_spec/profile/max_duration": "Duration", + "/plan/run_spec/profile/spot_policy": "SpotPolicy", + "/plan/run_spec/repo_data": "RemoteRunRepoData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.values.json new file mode 100644 index 0000000000..a2e4c0247d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/apply_run_plan_request.values.json @@ -0,0 +1,133 @@ +{ + "force": false, + "plan": { + "current_resource": null, + "run_spec": { + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "commands": [ + "accelerate launch train.py --model llama-3.1-70b" + ], + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": { + "HF_TOKEN": { + "key": "HF_TOKEN" + }, + "WANDB_PROJECT": "llm-training" + }, + "files": [], + "fleets": null, + "home_dir": "/root", + "idle_duration": null, + "image": null, + "instance_types": null, + "instances": null, + "max_duration": 259200, + "max_price": null, + "name": "llama-70b-sft", + "nodes": 2, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": "3.11", + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 1024.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": 8, + "min": 8 + }, + "memory": null, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 512.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "task", + "user": null, + "utilization_policy": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": ".dstack/train.yml", + "file_archives": [], + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": [ + "aws" + ], + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": 259200, + "max_price": null, + "name": "prod", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": "on-demand", + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + }, + "repo_code_hash": "4b8e2d1f6a3c9075", + "repo_data": { + "repo_branch": "main", + "repo_config_email": null, + "repo_config_name": null, + "repo_hash": "9f2c1a7", + "repo_name": "llm-training", + "repo_type": "remote" + }, + "repo_dir": null, + "repo_id": "3f8a1c2e9b7d4a5f", + "run_name": "llama-70b-sft", + "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 alice@example.com", + "working_dir": null + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.input.json new file mode 100644 index 0000000000..158b25cdd7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.input.json @@ -0,0 +1 @@ +{"names": ["research-cluster", "inference-pool"]} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.types.json new file mode 100644 index 0000000000..b634b6a0c7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.types.json @@ -0,0 +1,3 @@ +{ + "/": "DeleteFleetsRequest" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.values.json new file mode 100644 index 0000000000..373fd61de4 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/delete_fleets_request.values.json @@ -0,0 +1,6 @@ +{ + "names": [ + "research-cluster", + "inference-pool" + ] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.input.json new file mode 100644 index 0000000000..42d907de36 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.input.json @@ -0,0 +1,8 @@ +{ + "repo_id": "3f8a1c2e9b7d4a5f", + "repo_info": {"repo_type": "remote", "repo_name": "llm-training"}, + "repo_creds": { + "clone_url": "https://github.com/example-org/llm-training.git", + "oauth_token": "gho_16C7e42F292c6912E7710c838347Ae178B4a" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.types.json new file mode 100644 index 0000000000..b7690d7033 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.types.json @@ -0,0 +1,5 @@ +{ + "/": "SaveRepoCredsRequest", + "/repo_creds": "RemoteRepoCreds", + "/repo_info": "RemoteRepoInfo" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.values.json new file mode 100644 index 0000000000..b842cc5d6f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_request/save_repo_creds_request.values.json @@ -0,0 +1,12 @@ +{ + "repo_creds": { + "clone_url": "https://github.com/example-org/llm-training.git", + "oauth_token": "gho_16C7e42F292c6912E7710c838347Ae178B4a", + "private_key": null + }, + "repo_id": "3f8a1c2e9b7d4a5f", + "repo_info": { + "repo_name": "llm-training", + "repo_type": "remote" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.input.json new file mode 100644 index 0000000000..42f7af9ffb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.input.json @@ -0,0 +1,38 @@ +{ + "project_name": "main", + "user": "alice", + "spec": { + "configuration": { + "type": "fleet", + "name": "research-cluster", + "nodes": {"min": 2, "max": 8}, + "placement": "cluster", + "backends": ["aws"], + "regions": ["us-east-1"], + "instance_types": ["p4d.24xlarge"], + "spot_policy": "on-demand" + }, + "configuration_path": ".dstack/fleets/research.yml", + "profile": {"name": "prod", "spot_policy": "on-demand"} + }, + "offers": [ + { + "backend": "aws", + "instance": { + "name": "p4d.24xlarge", + "resources": { + "cpus": 96, "memory_mib": 1179648, + "gpus": [{"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}], + "spot": false, "disk": {"size_mib": 8388608}, + "description": "96xCPU, 1152GB, 8xA100 (40GB)" + } + }, + "region": "us-east-1", + "price": 32.7726, + "availability": "available" + } + ], + "total_offers": 1, + "max_offer_price": 32.7726, + "action": "create" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.types.json new file mode 100644 index 0000000000..7452de9c8b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.types.json @@ -0,0 +1,25 @@ +{ + "/": "FleetPlan", + "/action": "ApplyAction", + "/offers/0": "InstanceOfferWithAvailability", + "/offers/0/availability": "InstanceAvailability", + "/offers/0/backend": "BackendType", + "/offers/0/instance": "InstanceType", + "/offers/0/instance/resources": "Resources", + "/offers/0/instance/resources/disk": "Disk", + "/offers/0/instance/resources/gpus/0": "Gpu", + "/offers/0/instance/resources/gpus/0/vendor": "AcceleratorVendor", + "/offers/0/instance_runtime": "InstanceRuntime", + "/spec": "FleetSpec", + "/spec/configuration": "FleetConfiguration", + "/spec/configuration/backends/0": "BackendType", + "/spec/configuration/env": "Env", + "/spec/configuration/nodes": "FleetNodesSpec", + "/spec/configuration/placement": "InstanceGroupPlacement", + "/spec/configuration/spot_policy": "SpotPolicy", + "/spec/merged_profile": "Profile", + "/spec/merged_profile/backends/0": "BackendType", + "/spec/merged_profile/spot_policy": "SpotPolicy", + "/spec/profile": "Profile", + "/spec/profile/spot_policy": "SpotPolicy" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.values.json new file mode 100644 index 0000000000..95647117d5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/fleet_plan.values.json @@ -0,0 +1,100 @@ +{ + "action": "create", + "current_resource": null, + "effective_spec": null, + "max_offer_price": 32.7726, + "offers": [ + { + "availability": "available", + "availability_zones": null, + "backend": "aws", + "backend_data": {}, + "blocks": 1, + "instance": { + "name": "p4d.24xlarge", + "resources": { + "cpu_arch": null, + "cpus": 96, + "description": "96xCPU, 1152GB, 8xA100 (40GB)", + "disk": { + "size_mib": 8388608 + }, + "gpus": [ + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + } + ], + "memory_mib": 1179648, + "spot": false + } + }, + "instance_runtime": "shim", + "price": 32.7726, + "region": "us-east-1", + "total_blocks": 1 + } + ], + "project_name": "main", + "spec": { + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": [ + "aws" + ], + "blocks": 1, + "env": {}, + "idle_duration": null, + "instance_types": [ + "p4d.24xlarge" + ], + "max_price": null, + "name": "research-cluster", + "nodes": { + "max": 8, + "min": 2 + }, + "placement": "cluster", + "regions": [ + "us-east-1" + ], + "reservation": null, + "resources": null, + "retry": null, + "spot_policy": "on-demand", + "ssh_config": null, + "tags": null, + "type": "fleet" + }, + "configuration_path": ".dstack/fleets/research.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "prod", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": "on-demand", + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } + }, + "total_offers": 1, + "user": "alice" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.input.json new file mode 100644 index 0000000000..772ee5b306 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.input.json @@ -0,0 +1,19 @@ +{ + "id": "b41d9e77-3c8a-4f21-9d6e-5a7b8c9d0e1f", + "name": "prod-gateway", + "project_name": "main", + "backend": "aws", + "region": "eu-west-1", + "created_at": "2025-03-14T09:26:53+00:00", + "status": "running", + "hostname": "gateway.inference.example.com", + "wildcard_domain": "*.inference.example.com", + "default": true, + "configuration": { + "backend": "aws", + "name": "prod-gateway", + "region": "eu-west-1", + "domain": "inference.example.com", + "certificate": {"type": "lets-encrypt"} + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.types.json new file mode 100644 index 0000000000..96771fe897 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.types.json @@ -0,0 +1,10 @@ +{ + "/": "Gateway", + "/backend": "BackendType", + "/configuration": "GatewayConfiguration", + "/configuration/backend": "BackendType", + "/configuration/certificate": "LetsEncryptGatewayCertificate", + "/created_at": "datetime", + "/id": "UUID", + "/status": "GatewayStatus" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json new file mode 100644 index 0000000000..0e2961b328 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/gateway.values.json @@ -0,0 +1,32 @@ +{ + "backend": "aws", + "configuration": { + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "default": false, + "domain": "inference.example.com", + "instance_type": null, + "name": "prod-gateway", + "public_ip": true, + "region": "eu-west-1", + "replicas": null, + "router": null, + "tags": null, + "type": "gateway" + }, + "created_at": "2025-03-14T09:26:53+00:00", + "default": true, + "hostname": "gateway.inference.example.com", + "id": "b41d9e77-3c8a-4f21-9d6e-5a7b8c9d0e1f", + "instance_id": null, + "ip_address": null, + "name": "prod-gateway", + "project_name": "main", + "region": "eu-west-1", + "replicas": [], + "status": "running", + "status_message": null, + "wildcard_domain": "*.inference.example.com" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.input.json new file mode 100644 index 0000000000..06f82ca9b5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.input.json @@ -0,0 +1,40 @@ +{ + "project_id": "1a2b3c4d-5e6f-4071-8293-a4b5c6d7e8f9", + "project_name": "main", + "owner": { + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "username": "alice", + "created_at": "2025-01-08T11:02:41+00:00", + "global_role": "admin", + "email": "alice@example.com", + "active": true, + "permissions": {"can_create_projects": true} + }, + "backends": [], + "members": [ + { + "user": { + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "username": "alice", + "global_role": "admin", + "email": "alice@example.com", + "active": true, + "permissions": {"can_create_projects": true} + }, + "project_role": "admin", + "permissions": {"can_manage_ssh_fleets": true} + }, + { + "user": { + "id": "5d6e7f80-9a1b-4c2d-8e3f-4a5b6c7d8e9f", + "username": "bob", + "global_role": "user", + "email": null, + "active": true, + "permissions": {"can_create_projects": false} + }, + "project_role": "user", + "permissions": {"can_manage_ssh_fleets": false} + } + ] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.types.json new file mode 100644 index 0000000000..ea9569dbde --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.types.json @@ -0,0 +1,23 @@ +{ + "/": "Project", + "/members/0": "Member", + "/members/0/permissions": "MemberPermissions", + "/members/0/project_role": "ProjectRole", + "/members/0/user": "User", + "/members/0/user/global_role": "GlobalRole", + "/members/0/user/id": "UUID", + "/members/0/user/permissions": "UserPermissions", + "/members/1": "Member", + "/members/1/permissions": "MemberPermissions", + "/members/1/project_role": "ProjectRole", + "/members/1/user": "User", + "/members/1/user/global_role": "GlobalRole", + "/members/1/user/id": "UUID", + "/members/1/user/permissions": "UserPermissions", + "/owner": "User", + "/owner/created_at": "datetime", + "/owner/global_role": "GlobalRole", + "/owner/id": "UUID", + "/owner/permissions": "UserPermissions", + "/project_id": "UUID" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.values.json new file mode 100644 index 0000000000..60e523ab83 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/project.values.json @@ -0,0 +1,60 @@ +{ + "backends": [], + "created_at": null, + "is_public": false, + "members": [ + { + "permissions": { + "can_manage_secrets": false, + "can_manage_ssh_fleets": true + }, + "project_role": "admin", + "user": { + "active": true, + "created_at": null, + "email": "alice@example.com", + "global_role": "admin", + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "permissions": { + "can_create_projects": true + }, + "ssh_public_key": null, + "username": "alice" + } + }, + { + "permissions": { + "can_manage_secrets": false, + "can_manage_ssh_fleets": false + }, + "project_role": "user", + "user": { + "active": true, + "created_at": null, + "email": null, + "global_role": "user", + "id": "5d6e7f80-9a1b-4c2d-8e3f-4a5b6c7d8e9f", + "permissions": { + "can_create_projects": false + }, + "ssh_public_key": null, + "username": "bob" + } + } + ], + "owner": { + "active": true, + "created_at": "2025-01-08T11:02:41+00:00", + "email": "alice@example.com", + "global_role": "admin", + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "permissions": { + "can_create_projects": true + }, + "ssh_public_key": null, + "username": "alice" + }, + "project_id": "1a2b3c4d-5e6f-4071-8293-a4b5c6d7e8f9", + "project_name": "main", + "templates_repo": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.input.json new file mode 100644 index 0000000000..c5f45d5e3c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.input.json @@ -0,0 +1,54 @@ +{ + "project_name": "main", + "user": "alice", + "run_spec": { + "run_name": "llama-70b-sft", + "repo_id": "3f8a1c2e9b7d4a5f", + "repo_data": {"repo_type": "remote", "repo_name": "llm-training", "repo_branch": "main", "repo_hash": "9f2c1a7"}, + "configuration_path": ".dstack/train.yml", + "configuration": { + "type": "task", + "name": "llama-70b-sft", + "commands": ["accelerate launch train.py"], + "nodes": 2, + "python": "3.11", + "resources": {"gpu": "A100:8"} + }, + "profile": {"name": "prod", "backends": ["aws"]}, + "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 alice@example.com" + }, + "job_plans": [ + { + "job_spec": { + "job_num": 0, + "job_name": "llama-70b-sft-0-0", + "commands": ["/bin/bash", "-i", "-c", "accelerate launch train.py"], + "env": {"WANDB_PROJECT": "llm-training"}, + "image_name": "dstackai/base:py3.11-0.9-cuda-12.1", + "requirements": {"resources": {"gpu": {"name": ["A100"], "count": {"min": 8, "max": 8}}}, "spot": false}, + "working_dir": "/workflow", + "max_duration": 259200 + }, + "offers": [ + { + "backend": "aws", + "instance": { + "name": "p4d.24xlarge", + "resources": { + "cpus": 96, "memory_mib": 1179648, + "gpus": [{"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}], + "spot": false, "disk": {"size_mib": 8388608}, + "description": "96xCPU, 1152GB, 8xA100 (40GB)" + } + }, + "region": "us-east-1", + "price": 32.7726, + "availability": "available" + } + ], + "total_offers": 1, + "max_price": 32.7726 + } + ], + "action": "create" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.types.json new file mode 100644 index 0000000000..a12c791e5f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.types.json @@ -0,0 +1,46 @@ +{ + "/": "RunPlan", + "/action": "ApplyAction", + "/job_plans/0": "JobPlan", + "/job_plans/0/job_spec": "JobSpec", + "/job_plans/0/job_spec/requirements": "Requirements", + "/job_plans/0/job_spec/requirements/resources": "ResourcesSpec", + "/job_plans/0/job_spec/requirements/resources/cpu": "CPUSpec", + "/job_plans/0/job_spec/requirements/resources/cpu/count": "Range[int]", + "/job_plans/0/job_spec/requirements/resources/disk": "DiskSpec", + "/job_plans/0/job_spec/requirements/resources/disk/size": "Range[Memory]", + "/job_plans/0/job_spec/requirements/resources/disk/size/min": "Memory", + "/job_plans/0/job_spec/requirements/resources/gpu": "GPUSpec", + "/job_plans/0/job_spec/requirements/resources/gpu/count": "Range[int]", + "/job_plans/0/job_spec/requirements/resources/memory": "Range[Memory]", + "/job_plans/0/job_spec/requirements/resources/memory/min": "Memory", + "/job_plans/0/offers/0": "InstanceOfferWithAvailability", + "/job_plans/0/offers/0/availability": "InstanceAvailability", + "/job_plans/0/offers/0/backend": "BackendType", + "/job_plans/0/offers/0/instance": "InstanceType", + "/job_plans/0/offers/0/instance/resources": "Resources", + "/job_plans/0/offers/0/instance/resources/disk": "Disk", + "/job_plans/0/offers/0/instance/resources/gpus/0": "Gpu", + "/job_plans/0/offers/0/instance/resources/gpus/0/vendor": "AcceleratorVendor", + "/job_plans/0/offers/0/instance_runtime": "InstanceRuntime", + "/run_spec": "RunSpec", + "/run_spec/configuration": "TaskConfiguration", + "/run_spec/configuration/env": "Env", + "/run_spec/configuration/python": "PythonVersion", + "/run_spec/configuration/resources": "ResourcesSpec", + "/run_spec/configuration/resources/cpu": "CPUSpec", + "/run_spec/configuration/resources/cpu/count": "Range[int]", + "/run_spec/configuration/resources/disk": "DiskSpec", + "/run_spec/configuration/resources/disk/size": "Range[Memory]", + "/run_spec/configuration/resources/disk/size/min": "Memory", + "/run_spec/configuration/resources/gpu": "GPUSpec", + "/run_spec/configuration/resources/gpu/count": "Range[int]", + "/run_spec/configuration/resources/memory": "Range[Memory]", + "/run_spec/configuration/resources/memory/min": "Memory", + "/run_spec/merged_profile": "Profile", + "/run_spec/merged_profile/backends/0": "BackendType", + "/run_spec/merged_profile/creation_policy": "CreationPolicy", + "/run_spec/profile": "Profile", + "/run_spec/profile/backends/0": "BackendType", + "/run_spec/repo_data": "RemoteRunRepoData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.values.json new file mode 100644 index 0000000000..df8614566e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/run_plan.values.json @@ -0,0 +1,242 @@ +{ + "action": "create", + "current_resource": null, + "effective_run_spec": null, + "job_plans": [ + { + "job_spec": { + "app_specs": null, + "commands": [ + "/bin/bash", + "-i", + "-c", + "accelerate launch train.py" + ], + "env": { + "WANDB_PROJECT": "llm-training" + }, + "file_archives": [], + "home_dir": null, + "image_name": "dstackai/base:py3.11-0.9-cuda-12.1", + "job_name": "llama-70b-sft-0-0", + "job_num": 0, + "jobs_per_replica": 1, + "max_duration": 259200, + "privileged": false, + "probes": [], + "registry_auth": null, + "replica_group": "0", + "replica_num": 0, + "repo_code_hash": null, + "repo_data": null, + "repo_dir": "/workflow", + "repo_exists_action": null, + "requirements": { + "backend_options": null, + "max_price": null, + "multinode": null, + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": 8, + "min": 8 + }, + "memory": null, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "spot": false + }, + "retry": null, + "service_port": null, + "single_branch": null, + "ssh_key": null, + "stop_duration": null, + "user": null, + "utilization_policy": null, + "volumes": null, + "working_dir": "/workflow" + }, + "max_price": 32.7726, + "offers": [ + { + "availability": "available", + "availability_zones": null, + "backend": "aws", + "backend_data": {}, + "blocks": 1, + "instance": { + "name": "p4d.24xlarge", + "resources": { + "cpu_arch": null, + "cpus": 96, + "description": "96xCPU, 1152GB, 8xA100 (40GB)", + "disk": { + "size_mib": 8388608 + }, + "gpus": [ + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + } + ], + "memory_mib": 1179648, + "spot": false + } + }, + "instance_runtime": "shim", + "price": 32.7726, + "region": "us-east-1", + "total_blocks": 1 + } + ], + "total_offers": 1 + } + ], + "project_name": "main", + "run_spec": { + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "commands": [ + "accelerate launch train.py" + ], + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": {}, + "files": [], + "fleets": null, + "home_dir": "/root", + "idle_duration": null, + "image": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "llama-70b-sft", + "nodes": 2, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": "3.11", + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": 8, + "min": 8 + }, + "memory": null, + "name": [ + "A100" + ], + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "task", + "user": null, + "utilization_policy": null, + "volumes": [], + "working_dir": null + }, + "configuration_path": ".dstack/train.yml", + "file_archives": [], + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": [ + "aws" + ], + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": null, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "prod", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + }, + "repo_code_hash": null, + "repo_data": { + "repo_branch": "main", + "repo_config_email": null, + "repo_config_name": null, + "repo_hash": "9f2c1a7", + "repo_name": "llm-training", + "repo_type": "remote" + }, + "repo_dir": null, + "repo_id": "3f8a1c2e9b7d4a5f", + "run_name": "llama-70b-sft", + "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 alice@example.com", + "working_dir": null + }, + "user": "alice" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.input.json new file mode 100644 index 0000000000..1e1a86e321 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.input.json @@ -0,0 +1 @@ +{"id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", "name": "HF_TOKEN"} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.types.json new file mode 100644 index 0000000000..c1ec77c29c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.types.json @@ -0,0 +1,4 @@ +{ + "/": "Secret", + "/id": "UUID" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.values.json new file mode 100644 index 0000000000..fb43f09241 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/secret.values.json @@ -0,0 +1,5 @@ +{ + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "name": "HF_TOKEN", + "value": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.input.json new file mode 100644 index 0000000000..a1b25620ea --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.input.json @@ -0,0 +1 @@ +{"server_version": "0.19.14"} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.types.json new file mode 100644 index 0000000000..681369804f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.types.json @@ -0,0 +1,3 @@ +{ + "/": "ServerInfo" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.values.json new file mode 100644 index 0000000000..e879b48807 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/server_info.values.json @@ -0,0 +1,3 @@ +{ + "server_version": "0.19.14" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.input.json new file mode 100644 index 0000000000..6252ed8e22 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.input.json @@ -0,0 +1,10 @@ +{ + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "username": "alice", + "created_at": "2025-03-14T09:26:53+00:00", + "global_role": "user", + "email": "alice@example.com", + "active": true, + "permissions": {"can_create_projects": true}, + "creds": {"token": "0a1b2c3d-4e5f-6071-8293-a4b5c6d7e8f9"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.types.json new file mode 100644 index 0000000000..e57c2c4fb3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.types.json @@ -0,0 +1,8 @@ +{ + "/": "UserWithCreds", + "/created_at": "datetime", + "/creds": "UserTokenCreds", + "/global_role": "GlobalRole", + "/id": "UUID", + "/permissions": "UserPermissions" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.values.json new file mode 100644 index 0000000000..c2778861e1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/user_with_creds.values.json @@ -0,0 +1,16 @@ +{ + "active": true, + "created_at": "2025-03-14T09:26:53+00:00", + "creds": { + "token": "0a1b2c3d-4e5f-6071-8293-a4b5c6d7e8f9" + }, + "email": "alice@example.com", + "global_role": "user", + "id": "9c2f7b31-5e4a-4d8c-9f01-2b3c4d5e6f70", + "permissions": { + "can_create_projects": true + }, + "ssh_private_key": null, + "ssh_public_key": null, + "username": "alice" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.input.json new file mode 100644 index 0000000000..c1457b9ce5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.input.json @@ -0,0 +1,27 @@ +{ + "id": "e7f80912-3a4b-4c5d-9e6f-708192a3b4c5", + "name": "training-data", + "user": "alice", + "project_name": "main", + "configuration": { + "backend": "aws", + "name": "training-data", + "region": "us-east-1", + "size": 500.0 + }, + "external": false, + "created_at": "2025-04-02T16:41:07+00:00", + "last_processed_at": "2025-04-02T16:41:22+00:00", + "status": "active", + "deleted": false, + "volume_id": "vol-0fe1a2b3c4d5e6f78", + "provisioning_data": { + "backend": "aws", + "volume_id": "vol-0fe1a2b3c4d5e6f78", + "size_gb": 500, + "availability_zone": "us-east-1a", + "attachable": true, + "detachable": true + }, + "cost": 42.5 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.types.json new file mode 100644 index 0000000000..5062b0d42c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.types.json @@ -0,0 +1,12 @@ +{ + "/": "Volume", + "/configuration": "AWSVolumeConfiguration", + "/configuration/backend": "BackendType", + "/configuration/size": "Memory", + "/created_at": "datetime", + "/id": "UUID", + "/last_processed_at": "datetime", + "/provisioning_data": "VolumeProvisioningData", + "/provisioning_data/backend": "BackendType", + "/status": "VolumeStatus" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.values.json new file mode 100644 index 0000000000..5a3344d2eb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/api_response/volume.values.json @@ -0,0 +1,38 @@ +{ + "attachment_data": null, + "attachments": null, + "configuration": { + "auto_cleanup_duration": null, + "availability_zone": null, + "backend": "aws", + "name": "training-data", + "region": "us-east-1", + "size": 500.0, + "tags": null, + "type": "volume", + "volume_id": null + }, + "cost": 42.5, + "created_at": "2025-04-02T16:41:07+00:00", + "deleted": false, + "deleted_at": null, + "external": false, + "id": "e7f80912-3a4b-4c5d-9e6f-708192a3b4c5", + "last_processed_at": "2025-04-02T16:41:22+00:00", + "name": "training-data", + "project_name": "main", + "provisioning_data": { + "attachable": true, + "availability_zone": "us-east-1a", + "backend": "aws", + "backend_data": null, + "detachable": true, + "price": null, + "size_gb": 500, + "volume_id": "vol-0fe1a2b3c4d5e6f78" + }, + "status": "active", + "status_message": null, + "user": "alice", + "volume_id": "vol-0fe1a2b3c4d5e6f78" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.input.yml new file mode 100644 index 0000000000..923741d0b3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.input.yml @@ -0,0 +1,11 @@ +# The sixth `.dstack.yml` type. `ide` is a Literal union and `inactivity_duration` a Duration +# shorthand, neither of which the other config inputs exercise. +type: dev-environment +ide: vscode +python: 3.11 +inactivity_duration: 1h +resources: + gpu: 24GB..80GB + shm_size: 16GB +env: + HF_TOKEN: secret diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json new file mode 100644 index 0000000000..4de8822cb2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json @@ -0,0 +1,21 @@ +{ + "/": "DstackConfiguration", + "/__root__": "DevEnvironmentConfiguration", + "/__root__/env": "Env", + "/__root__/inactivity_duration": "Duration", + "/__root__/python": "PythonVersion", + "/__root__/resources": "ResourcesSpec", + "/__root__/resources/cpu": "CPUSpec", + "/__root__/resources/cpu/count": "Range[int]", + "/__root__/resources/disk": "DiskSpec", + "/__root__/resources/disk/size": "Range[Memory]", + "/__root__/resources/disk/size/min": "Memory", + "/__root__/resources/gpu": "GPUSpec", + "/__root__/resources/gpu/count": "Range[int]", + "/__root__/resources/gpu/memory": "Range[Memory]", + "/__root__/resources/gpu/memory/max": "Memory", + "/__root__/resources/gpu/memory/min": "Memory", + "/__root__/resources/memory": "Range[Memory]", + "/__root__/resources/memory/min": "Memory", + "/__root__/resources/shm_size": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.values.json new file mode 100644 index 0000000000..c79cf2d918 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.values.json @@ -0,0 +1,81 @@ +{ + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "docker": null, + "dstack": false, + "entrypoint": null, + "env": { + "HF_TOKEN": "secret" + }, + "files": [], + "fleets": null, + "home_dir": "/root", + "ide": "vscode", + "idle_duration": null, + "image": null, + "inactivity_duration": 3600, + "init": [], + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": null, + "nvcc": null, + "ports": [], + "priority": null, + "privileged": false, + "python": "3.11", + "regions": null, + "registry_auth": null, + "repos": [], + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 1 + }, + "memory": { + "max": 80.0, + "min": 24.0 + }, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": 16.0 + }, + "retry": null, + "schedule": null, + "setup": [], + "shell": null, + "single_branch": null, + "spot_policy": null, + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "type": "dev-environment", + "user": null, + "utilization_policy": null, + "version": null, + "volumes": [], + "working_dir": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.input.json new file mode 100644 index 0000000000..98db12d295 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.input.json @@ -0,0 +1,8 @@ +{ + "compute_group_id": "cr-0abc123def4567890", + "compute_group_name": "research-cluster-cg", + "backend": "aws", + "region": "us-east-1", + "job_provisioning_datas": [], + "backend_data": "{\"capacity_reservation_id\": \"cr-0abc123def4567890\"}" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.types.json new file mode 100644 index 0000000000..eea2526fb1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.types.json @@ -0,0 +1,4 @@ +{ + "/": "ComputeGroupProvisioningData", + "/backend": "BackendType" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.values.json new file mode 100644 index 0000000000..c998988934 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/compute_group_provisioning_data.values.json @@ -0,0 +1,9 @@ +{ + "backend": "aws", + "backend_data": "{\"capacity_reservation_id\": \"cr-0abc123def4567890\"}", + "base_backend": null, + "compute_group_id": "cr-0abc123def4567890", + "compute_group_name": "research-cluster-cg", + "job_provisioning_datas": [], + "region": "us-east-1" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.input.json new file mode 100644 index 0000000000..15946b3c90 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.input.json @@ -0,0 +1,17 @@ +{ + "configuration": { + "type": "fleet", + "name": "research-cluster", + "nodes": {"min": 2, "max": 8}, + "placement": "cluster", + "backends": ["aws"], + "regions": ["us-east-1"], + "instance_types": ["p4d.24xlarge"], + "spot_policy": "on-demand", + "idle_duration": 1800, + "resources": {"cpu": {"min": 8}, "memory": {"min": 64.0}}, + "tags": {"team": "research"} + }, + "configuration_path": ".dstack/fleets/research.yml", + "profile": {"name": "prod", "spot_policy": "on-demand", "idle_duration": 1800} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.types.json new file mode 100644 index 0000000000..d24311bc90 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.types.json @@ -0,0 +1,27 @@ +{ + "/": "FleetSpec", + "/configuration": "FleetConfiguration", + "/configuration/backends/0": "BackendType", + "/configuration/env": "Env", + "/configuration/idle_duration": "Duration", + "/configuration/nodes": "FleetNodesSpec", + "/configuration/placement": "InstanceGroupPlacement", + "/configuration/resources": "ResourcesSpec", + "/configuration/resources/cpu": "CPUSpec", + "/configuration/resources/cpu/count": "Range[int]", + "/configuration/resources/disk": "DiskSpec", + "/configuration/resources/disk/size": "Range[Memory]", + "/configuration/resources/disk/size/min": "Memory", + "/configuration/resources/gpu": "GPUSpec", + "/configuration/resources/gpu/count": "Range[int]", + "/configuration/resources/memory": "Range[Memory]", + "/configuration/resources/memory/min": "Memory", + "/configuration/spot_policy": "SpotPolicy", + "/merged_profile": "Profile", + "/merged_profile/backends/0": "BackendType", + "/merged_profile/idle_duration": "Duration", + "/merged_profile/spot_policy": "SpotPolicy", + "/profile": "Profile", + "/profile/idle_duration": "Duration", + "/profile/spot_policy": "SpotPolicy" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.values.json new file mode 100644 index 0000000000..18cf830b07 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/fleet_spec.values.json @@ -0,0 +1,87 @@ +{ + "autocreated": false, + "configuration": { + "availability_zones": null, + "backend_options": null, + "backends": [ + "aws" + ], + "blocks": 1, + "env": {}, + "idle_duration": 1800, + "instance_types": [ + "p4d.24xlarge" + ], + "max_price": null, + "name": "research-cluster", + "nodes": { + "max": 8, + "min": 2 + }, + "placement": "cluster", + "regions": [ + "us-east-1" + ], + "reservation": null, + "resources": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 64.0 + }, + "shm_size": null + }, + "retry": null, + "spot_policy": "on-demand", + "ssh_config": null, + "tags": { + "team": "research" + }, + "type": "fleet" + }, + "configuration_path": ".dstack/fleets/research.yml", + "profile": { + "availability_zones": null, + "backend_options": null, + "backends": null, + "creation_policy": null, + "default": false, + "fleets": null, + "idle_duration": 1800, + "instance_types": null, + "instances": null, + "max_duration": null, + "max_price": null, + "name": "prod", + "regions": null, + "reservation": null, + "retry": null, + "schedule": null, + "spot_policy": "on-demand", + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": null, + "utilization_policy": null + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.input.json new file mode 100644 index 0000000000..02ad4e13e1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.input.json @@ -0,0 +1,9 @@ +{ + "project_name": "main", + "instance_name": "prod-gateway", + "backend": "aws", + "region": "eu-west-1", + "public_ip": true, + "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 gateway@example.com", + "certificate": {"type": "lets-encrypt"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.types.json new file mode 100644 index 0000000000..12d3ef3955 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.types.json @@ -0,0 +1,5 @@ +{ + "/": "GatewayComputeConfiguration", + "/backend": "BackendType", + "/certificate": "LetsEncryptGatewayCertificate" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json new file mode 100644 index 0000000000..fea497cc02 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_compute_configuration.values.json @@ -0,0 +1,14 @@ +{ + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "instance_name": "prod-gateway", + "instance_type": null, + "project_name": "main", + "public_ip": true, + "region": "eu-west-1", + "router": null, + "ssh_key_pub": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 gateway@example.com", + "tags": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.input.json new file mode 100644 index 0000000000..b86c0d3521 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.input.json @@ -0,0 +1,8 @@ +{ + "backend": "aws", + "name": "prod-gateway", + "region": "eu-west-1", + "domain": "inference.example.com", + "certificate": {"type": "lets-encrypt"}, + "public_ip": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.types.json new file mode 100644 index 0000000000..9bad75fa91 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.types.json @@ -0,0 +1,5 @@ +{ + "/": "GatewayConfiguration", + "/backend": "BackendType", + "/certificate": "LetsEncryptGatewayCertificate" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json new file mode 100644 index 0000000000..2828b2021e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/gateway_configuration.values.json @@ -0,0 +1,16 @@ +{ + "backend": "aws", + "certificate": { + "type": "lets-encrypt" + }, + "default": false, + "domain": "inference.example.com", + "instance_type": null, + "name": "prod-gateway", + "public_ip": true, + "region": "eu-west-1", + "replicas": null, + "router": null, + "tags": null, + "type": "gateway" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.input.json new file mode 100644 index 0000000000..5e56e4f744 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.input.json @@ -0,0 +1,6 @@ +{ + "downloaded_bytes": 3221225472, + "extracted_bytes": 2147483648, + "total_bytes": 4294967296, + "is_total_bytes_final": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.types.json new file mode 100644 index 0000000000..637fff7fab --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.types.json @@ -0,0 +1,3 @@ +{ + "/": "ImagePullProgress" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.values.json new file mode 100644 index 0000000000..6b9a66dfdf --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/image_pull_progress.values.json @@ -0,0 +1,6 @@ +{ + "downloaded_bytes": 3221225472, + "extracted_bytes": 2147483648, + "is_total_bytes_final": true, + "total_bytes": 4294967296 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.input.json new file mode 100644 index 0000000000..23340ee693 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.input.json @@ -0,0 +1,9 @@ +{ + "project_name": "main", + "instance_name": "main-fleet-0", + "instance_id": "i-0abc123def4567890", + "user": "alice", + "ssh_keys": [{"public": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 alice@example.com"}], + "reservation": "cr-0abc123def4567890", + "tags": {"team": "research"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.types.json new file mode 100644 index 0000000000..41f39211cb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.types.json @@ -0,0 +1,4 @@ +{ + "/": "InstanceConfiguration", + "/ssh_keys/0": "SSHKey" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.values.json new file mode 100644 index 0000000000..fe27055e3f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_configuration.values.json @@ -0,0 +1,17 @@ +{ + "instance_id": "i-0abc123def4567890", + "instance_name": "main-fleet-0", + "project_name": "main", + "reservation": "cr-0abc123def4567890", + "ssh_keys": [ + { + "private": null, + "public": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 alice@example.com" + } + ], + "tags": { + "team": "research" + }, + "user": "alice", + "volumes": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.input.json new file mode 100644 index 0000000000..054414921b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.input.json @@ -0,0 +1,17 @@ +{ + "backend": "gcp", + "instance": { + "name": "a2-highgpu-1g", + "resources": { + "cpus": 12, + "memory_mib": 87040, + "gpus": [{"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}], + "spot": true, + "disk": {"size_mib": 102400}, + "description": "12xCPU, 85GB, 1xA100 (40GB), 100GB (disk)" + } + }, + "region": "europe-west4", + "price": 1.1213, + "availability": "available" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.types.json new file mode 100644 index 0000000000..92228d3649 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.types.json @@ -0,0 +1,11 @@ +{ + "/": "InstanceOfferWithAvailability", + "/availability": "InstanceAvailability", + "/backend": "BackendType", + "/instance": "InstanceType", + "/instance/resources": "Resources", + "/instance/resources/disk": "Disk", + "/instance/resources/gpus/0": "Gpu", + "/instance/resources/gpus/0/vendor": "AcceleratorVendor", + "/instance_runtime": "InstanceRuntime" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.values.json new file mode 100644 index 0000000000..4658dc6e68 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/instance_offer.values.json @@ -0,0 +1,31 @@ +{ + "availability": "available", + "availability_zones": null, + "backend": "gcp", + "backend_data": {}, + "blocks": 1, + "instance": { + "name": "a2-highgpu-1g", + "resources": { + "cpu_arch": null, + "cpus": 12, + "description": "12xCPU, 85GB, 1xA100 (40GB), 100GB (disk)", + "disk": { + "size_mib": 102400 + }, + "gpus": [ + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + } + ], + "memory_mib": 87040, + "spot": true + } + }, + "instance_runtime": "shim", + "price": 1.1213, + "region": "europe-west4", + "total_blocks": 1 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.input.json new file mode 100644 index 0000000000..70ea735655 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.input.json @@ -0,0 +1,33 @@ +{ + "backend": "aws", + "instance_type": { + "name": "p4d.24xlarge", + "resources": { + "cpus": 96, + "memory_mib": 1179648, + "gpus": [ + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"}, + {"name": "A100", "memory_mib": 40960, "vendor": "nvidia"} + ], + "spot": false, + "disk": {"size_mib": 8388608}, + "description": "96xCPU, 1152GB, 8xA100 (40GB)" + } + }, + "instance_id": "i-0abc123def4567890", + "hostname": "54.221.13.207", + "internal_ip": "172.31.24.11", + "region": "us-east-1", + "availability_zone": "us-east-1a", + "price": 32.7726, + "username": "ubuntu", + "ssh_port": 22, + "dockerized": true, + "backend_data": "{\"boot_disk_id\": \"vol-0fe1a2b3c4d5e6f78\"}" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.types.json new file mode 100644 index 0000000000..4142f264d8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.types.json @@ -0,0 +1,23 @@ +{ + "/": "JobProvisioningData", + "/backend": "BackendType", + "/instance_type": "InstanceType", + "/instance_type/resources": "Resources", + "/instance_type/resources/disk": "Disk", + "/instance_type/resources/gpus/0": "Gpu", + "/instance_type/resources/gpus/0/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/1": "Gpu", + "/instance_type/resources/gpus/1/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/2": "Gpu", + "/instance_type/resources/gpus/2/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/3": "Gpu", + "/instance_type/resources/gpus/3/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/4": "Gpu", + "/instance_type/resources/gpus/4/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/5": "Gpu", + "/instance_type/resources/gpus/5/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/6": "Gpu", + "/instance_type/resources/gpus/6/vendor": "AcceleratorVendor", + "/instance_type/resources/gpus/7": "Gpu", + "/instance_type/resources/gpus/7/vendor": "AcceleratorVendor" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.values.json new file mode 100644 index 0000000000..46b4cac601 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_provisioning_data.values.json @@ -0,0 +1,73 @@ +{ + "availability_zone": "us-east-1a", + "backend": "aws", + "backend_data": "{\"boot_disk_id\": \"vol-0fe1a2b3c4d5e6f78\"}", + "base_backend": null, + "dockerized": true, + "hostname": "54.221.13.207", + "instance_id": "i-0abc123def4567890", + "instance_network": null, + "instance_type": { + "name": "p4d.24xlarge", + "resources": { + "cpu_arch": null, + "cpus": 96, + "description": "96xCPU, 1152GB, 8xA100 (40GB)", + "disk": { + "size_mib": 8388608 + }, + "gpus": [ + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + }, + { + "memory_mib": 40960, + "name": "A100", + "vendor": "nvidia" + } + ], + "memory_mib": 1179648, + "spot": false + } + }, + "internal_ip": "172.31.24.11", + "price": 32.7726, + "public_ip_enabled": true, + "region": "us-east-1", + "reservation": null, + "ssh_port": 22, + "ssh_proxy": null, + "username": "ubuntu" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.input.json new file mode 100644 index 0000000000..b70094692f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.input.json @@ -0,0 +1,10 @@ +{ + "network_mode": "host", + "gpu": 8, + "cpu": 96.0, + "memory": 1266013306880.0, + "ports": {"8080": 32768, "6006": 32769}, + "volume_names": ["training-data"], + "working_dir": "/workflow", + "username": "ubuntu" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.types.json new file mode 100644 index 0000000000..d5afaebd27 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.types.json @@ -0,0 +1,5 @@ +{ + "/": "JobRuntimeData", + "/memory": "Memory", + "/network_mode": "NetworkMode" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.values.json new file mode 100644 index 0000000000..03314631b8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/job_runtime_data.values.json @@ -0,0 +1,16 @@ +{ + "cpu": 96.0, + "gpu": 8, + "memory": 1266013306880.0, + "network_mode": "host", + "offer": null, + "ports": { + "6006": 32769, + "8080": 32768 + }, + "username": "ubuntu", + "volume_names": [ + "training-data" + ], + "working_dir": "/workflow" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.input.json new file mode 100644 index 0000000000..ca5529281e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.input.json @@ -0,0 +1 @@ +{"backend": "aws", "region": "us-east-1", "placement_strategy": "cluster"} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.types.json new file mode 100644 index 0000000000..b7f6917052 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.types.json @@ -0,0 +1,5 @@ +{ + "/": "PlacementGroupConfiguration", + "/backend": "BackendType", + "/placement_strategy": "PlacementStrategy" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.values.json new file mode 100644 index 0000000000..16f28dfd7a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_configuration.values.json @@ -0,0 +1,5 @@ +{ + "backend": "aws", + "placement_strategy": "cluster", + "region": "us-east-1" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.input.json new file mode 100644 index 0000000000..cc1da7cd46 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.input.json @@ -0,0 +1 @@ +{"backend": "aws", "backend_data": "{\"name\": \"main-fleet-pg\"}"} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.types.json new file mode 100644 index 0000000000..b6b3db122b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.types.json @@ -0,0 +1,4 @@ +{ + "/": "PlacementGroupProvisioningData", + "/backend": "BackendType" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.values.json new file mode 100644 index 0000000000..95b2e43bbb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/placement_group_provisioning_data.values.json @@ -0,0 +1,4 @@ +{ + "backend": "aws", + "backend_data": "{\"name\": \"main-fleet-pg\"}" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.input.json new file mode 100644 index 0000000000..57c2ba2fe3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.input.json @@ -0,0 +1,13 @@ +{ + "name": "prod", + "backends": ["aws", "gcp"], + "regions": ["us-east-1", "europe-west4"], + "instance_types": ["p4d.24xlarge", "a2-highgpu-8g"], + "spot_policy": "auto", + "retry": {"on_events": ["no-capacity", "interruption"], "duration": 21600}, + "max_duration": 259200, + "idle_duration": 900, + "max_price": 40.0, + "creation_policy": "reuse-or-create", + "tags": {"team": "research", "cost-center": "ml-platform"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.types.json new file mode 100644 index 0000000000..d8193ccef6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.types.json @@ -0,0 +1,13 @@ +{ + "/": "Profile", + "/backends/0": "BackendType", + "/backends/1": "BackendType", + "/creation_policy": "CreationPolicy", + "/idle_duration": "Duration", + "/max_duration": "Duration", + "/retry": "ProfileRetry", + "/retry/duration": "Duration", + "/retry/on_events/0": "RetryEvent", + "/retry/on_events/1": "RetryEvent", + "/spot_policy": "SpotPolicy" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.values.json new file mode 100644 index 0000000000..7982ecc785 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/profile.values.json @@ -0,0 +1,42 @@ +{ + "availability_zones": null, + "backend_options": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", + "default": false, + "fleets": null, + "idle_duration": 900, + "instance_types": [ + "p4d.24xlarge", + "a2-highgpu-8g" + ], + "instances": null, + "max_duration": 259200, + "max_price": 40.0, + "name": "prod", + "regions": [ + "us-east-1", + "europe-west4" + ], + "reservation": null, + "retry": { + "duration": 21600, + "on_events": [ + "no-capacity", + "interruption" + ] + }, + "schedule": null, + "spot_policy": "auto", + "startup_order": null, + "stop_criteria": null, + "stop_duration": null, + "tags": { + "cost-center": "ml-platform", + "team": "research" + }, + "utilization_policy": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.input.json new file mode 100644 index 0000000000..3b180ff895 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.input.json @@ -0,0 +1,7 @@ +{ + "host": "192.168.100.14", + "port": 22, + "ssh_user": "ubuntu", + "ssh_keys": [{"public": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 ops@example.com"}], + "env": {"CUDA_VISIBLE_DEVICES": "0,1"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.types.json new file mode 100644 index 0000000000..d96492202b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.types.json @@ -0,0 +1,5 @@ +{ + "/": "RemoteConnectionInfo", + "/env": "Env", + "/ssh_keys/0": "SSHKey" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.values.json new file mode 100644 index 0000000000..f1983d7f0b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/remote_connection_info.values.json @@ -0,0 +1,16 @@ +{ + "env": { + "CUDA_VISIBLE_DEVICES": "0,1" + }, + "host": "192.168.100.14", + "port": 22, + "ssh_keys": [ + { + "private": null, + "public": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0000000000000000000000000000000000000000000 ops@example.com" + } + ], + "ssh_proxy": null, + "ssh_proxy_keys": null, + "ssh_user": "ubuntu" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.input.json new file mode 100644 index 0000000000..c7e4895d1e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.input.json @@ -0,0 +1,17 @@ +{ + "resources": { + "cpu": {"min": 8, "max": 64}, + "memory": {"min": 64.0, "max": 512.0}, + "gpu": { + "vendor": "nvidia", + "name": ["A100", "H100"], + "count": {"min": 4, "max": 8}, + "memory": {"min": 40.0}, + "compute_capability": [8, 0] + }, + "disk": {"size": {"min": 500.0}} + }, + "max_price": 40.0, + "spot": false, + "reservation": "cr-0abc123def4567890" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.types.json new file mode 100644 index 0000000000..7392325e80 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.types.json @@ -0,0 +1,17 @@ +{ + "/": "Requirements", + "/resources": "ResourcesSpec", + "/resources/cpu": "CPUSpec", + "/resources/cpu/count": "Range[int]", + "/resources/disk": "DiskSpec", + "/resources/disk/size": "Range[Memory]", + "/resources/disk/size/min": "Memory", + "/resources/gpu": "GPUSpec", + "/resources/gpu/count": "Range[int]", + "/resources/gpu/memory": "Range[Memory]", + "/resources/gpu/memory/min": "Memory", + "/resources/gpu/vendor": "AcceleratorVendor", + "/resources/memory": "Range[Memory]", + "/resources/memory/max": "Memory", + "/resources/memory/min": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.values.json new file mode 100644 index 0000000000..41fc54a769 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/requirements.values.json @@ -0,0 +1,44 @@ +{ + "backend_options": null, + "max_price": 40.0, + "multinode": null, + "reservation": "cr-0abc123def4567890", + "resources": { + "cpu": { + "max": 64, + "min": 8 + }, + "disk": { + "size": { + "max": null, + "min": 500.0 + } + }, + "gpu": { + "compute_capability": [ + 8, + 0 + ], + "count": { + "max": 8, + "min": 4 + }, + "memory": { + "max": null, + "min": 40.0 + }, + "name": [ + "A100", + "H100" + ], + "total_memory": null, + "vendor": "nvidia" + }, + "memory": { + "max": 512.0, + "min": 64.0 + }, + "shm_size": null + }, + "spot": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.input.json new file mode 100644 index 0000000000..4be515d04f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.input.json @@ -0,0 +1,9 @@ +{ + "cpus": 96, + "memory_mib": 1179648, + "gpus": [{"name": "H100", "memory_mib": 81920, "vendor": "nvidia"}], + "spot": false, + "disk": {"size_mib": 8388608}, + "description": "96xCPU, 1152GB, 1xH100 (80GB)", + "cpu_arch": "x86" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.types.json new file mode 100644 index 0000000000..c388a888af --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.types.json @@ -0,0 +1,7 @@ +{ + "/": "Resources", + "/cpu_arch": "CPUArchitecture", + "/disk": "Disk", + "/gpus/0": "Gpu", + "/gpus/0/vendor": "AcceleratorVendor" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.values.json new file mode 100644 index 0000000000..e2e67e2797 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/resources.values.json @@ -0,0 +1,17 @@ +{ + "cpu_arch": "x86", + "cpus": 96, + "description": "96xCPU, 1152GB, 1xH100 (80GB)", + "disk": { + "size_mib": 8388608 + }, + "gpus": [ + { + "memory_mib": 81920, + "name": "H100", + "vendor": "nvidia" + } + ], + "memory_mib": 1179648, + "spot": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.input.json new file mode 100644 index 0000000000..a51377a742 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.input.json @@ -0,0 +1,5 @@ +{ + "url": "/proxy/services/main/llama-70b/", + "model": {"name": "llama-3.1-70b", "base_url": "https://inference.example.com/proxy/models/main", "type": "chat"}, + "options": {"openai": {"model": {"type": "chat", "name": "llama-3.1-70b", "format": "openai", "prefix": "/v1"}}} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.types.json new file mode 100644 index 0000000000..c6e8ff48f0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.types.json @@ -0,0 +1,4 @@ +{ + "/": "ServiceSpec", + "/model": "ServiceModelSpec" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.values.json new file mode 100644 index 0000000000..66028bcd2a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/service_spec.values.json @@ -0,0 +1,18 @@ +{ + "model": { + "base_url": "https://inference.example.com/proxy/models/main", + "name": "llama-3.1-70b", + "type": "chat" + }, + "options": { + "openai": { + "model": { + "format": "openai", + "name": "llama-3.1-70b", + "prefix": "/v1", + "type": "chat" + } + } + }, + "url": "/proxy/services/main/llama-70b/" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.input.json new file mode 100644 index 0000000000..af4b4cde17 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.input.json @@ -0,0 +1 @@ +{"device_name": "/dev/nvme1n1"} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.types.json new file mode 100644 index 0000000000..9d74794f49 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.types.json @@ -0,0 +1,3 @@ +{ + "/": "VolumeAttachmentData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.values.json new file mode 100644 index 0000000000..126bdad94e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_attachment_data.values.json @@ -0,0 +1,3 @@ +{ + "device_name": "/dev/nvme1n1" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.input.json new file mode 100644 index 0000000000..1909560f2f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.input.json @@ -0,0 +1,8 @@ +{ + "backend": "aws", + "name": "training-data", + "region": "us-east-1", + "availability_zone": "us-east-1a", + "size": 500.0, + "volume_id": "vol-0fe1a2b3c4d5e6f78" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.types.json new file mode 100644 index 0000000000..075e76beb0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.types.json @@ -0,0 +1,5 @@ +{ + "/": "AWSVolumeConfiguration", + "/backend": "BackendType", + "/size": "Memory" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.values.json new file mode 100644 index 0000000000..1bef341c8d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_configuration.values.json @@ -0,0 +1,11 @@ +{ + "auto_cleanup_duration": null, + "availability_zone": "us-east-1a", + "backend": "aws", + "name": "training-data", + "region": "us-east-1", + "size": 500.0, + "tags": null, + "type": "volume", + "volume_id": "vol-0fe1a2b3c4d5e6f78" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.input.json new file mode 100644 index 0000000000..b762ce6e2a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.input.json @@ -0,0 +1,8 @@ +{ + "backend": "aws", + "volume_id": "vol-0fe1a2b3c4d5e6f78", + "size_gb": 500, + "availability_zone": "us-east-1a", + "attachable": true, + "detachable": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.types.json new file mode 100644 index 0000000000..272a1772ef --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.types.json @@ -0,0 +1,4 @@ +{ + "/": "VolumeProvisioningData", + "/backend": "BackendType" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.values.json new file mode 100644 index 0000000000..f43d06fe7e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/volume_provisioning_data.values.json @@ -0,0 +1,10 @@ +{ + "attachable": true, + "availability_zone": "us-east-1a", + "backend": "aws", + "backend_data": null, + "detachable": true, + "price": null, + "size_gb": 500, + "volume_id": "vol-0fe1a2b3c4d5e6f78" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.input.json new file mode 100644 index 0000000000..c48f5a369f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.input.json @@ -0,0 +1 @@ +{"domain": "gateway.inference.example.com", "https": true} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.types.json new file mode 100644 index 0000000000..b88b1681ed --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.types.json @@ -0,0 +1,3 @@ +{ + "/": "RegisterEntrypointRequest" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.values.json new file mode 100644 index 0000000000..52d46d33da --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_entrypoint_request.values.json @@ -0,0 +1,4 @@ +{ + "domain": "gateway.inference.example.com", + "https": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json index 753ade2fa6..b221a2f6a5 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.input.json @@ -1,7 +1,7 @@ { - "job_id": "11111111-1111-4111-8111-111111111111", + "job_id": "7d1e4f60-8a2b-4c3d-9e5f-0a1b2c3d4e5f", "app_port": 8000, - "ssh_host": "ubuntu@10.0.0.1", + "ssh_host": "ubuntu@54.221.13.207", "ssh_port": 22, "ssh_proxy": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json index 00fbd3f0d6..68d78d9d37 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/register_replica_request.values.json @@ -1,10 +1,10 @@ { "app_port": 8000, "internal_ip": null, - "job_id": "11111111-1111-4111-8111-111111111111", + "job_id": "7d1e4f60-8a2b-4c3d-9e5f-0a1b2c3d4e5f", "ssh_head_proxy": null, "ssh_head_proxy_private_key": null, - "ssh_host": "ubuntu@10.0.0.1", + "ssh_host": "ubuntu@54.221.13.207", "ssh_port": 22, "ssh_proxy": null, "ssh_proxy_private_key": null diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.input.json new file mode 100644 index 0000000000..0c59eed70d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.input.json @@ -0,0 +1,9 @@ +{ + "project_name": "main", + "run_name": "llama-70b", + "stats": { + "30": {"requests": 412, "request_time": 0.184}, + "60": {"requests": 847, "request_time": 0.201}, + "300": {"requests": 4103, "request_time": 0.223} + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.types.json new file mode 100644 index 0000000000..2c7742b9fd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.types.json @@ -0,0 +1,6 @@ +{ + "/": "ServiceStats", + "/stats/30": "Stat", + "/stats/300": "Stat", + "/stats/60": "Stat" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.values.json new file mode 100644 index 0000000000..ce9fb0ff85 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/gateway/service_stats.values.json @@ -0,0 +1,18 @@ +{ + "project_name": "main", + "run_name": "llama-70b", + "stats": { + "30": { + "request_time": 0.184, + "requests": 412 + }, + "300": { + "request_time": 0.223, + "requests": 4103 + }, + "60": { + "request_time": 0.201, + "requests": 847 + } + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.input.json new file mode 100644 index 0000000000..9691b9b096 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.input.json @@ -0,0 +1,4 @@ +{ + "dcgm": {"overall_health": 0, "incidents": []}, + "future_probe": "ignored" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.types.json new file mode 100644 index 0000000000..ecb09e58f2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.types.json @@ -0,0 +1,5 @@ +{ + "/": "InstanceHealthResponse", + "/dcgm": "DCGMHealthResponse", + "/dcgm/overall_health": "DCGMHealthResult" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.values.json new file mode 100644 index 0000000000..177aaaea36 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/instance_health_response.values.json @@ -0,0 +1,6 @@ +{ + "dcgm": { + "incidents": [], + "overall_health": 0 + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.input.json new file mode 100644 index 0000000000..e7f9823d8d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.input.json @@ -0,0 +1 @@ +{"working_dir": "/workflow", "username": "ubuntu", "added_by_a_newer_runner": 1} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.types.json new file mode 100644 index 0000000000..ce733d5062 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.types.json @@ -0,0 +1,3 @@ +{ + "/": "JobInfoResponse" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.values.json new file mode 100644 index 0000000000..c799132012 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/job_info_response.values.json @@ -0,0 +1,4 @@ +{ + "username": "ubuntu", + "working_dir": "/workflow" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.input.json new file mode 100644 index 0000000000..aec4a1d9b7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.input.json @@ -0,0 +1,7 @@ +{ + "id": "11111111-1111-4111-8111-111111111111", + "status": "running", + "termination_reason": "", + "termination_message": "", + "extra_from_newer_shim": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.types.json new file mode 100644 index 0000000000..de61485e9d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.types.json @@ -0,0 +1,4 @@ +{ + "/": "TaskInfoResponse", + "/status": "TaskStatus" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.values.json new file mode 100644 index 0000000000..63e6e8398e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/runner/task_info_response.values.json @@ -0,0 +1,8 @@ +{ + "id": "11111111-1111-4111-8111-111111111111", + "image_pull_progress": null, + "ports": [], + "status": "running", + "termination_message": "", + "termination_reason": "" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json index 5bafec8547..3a61a13fa7 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_fleet_plan_request.json @@ -5,52 +5,116 @@ "spec": { "autocreated": false, "configuration": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, + "backends": [ + "aws" + ], "blocks": 1, "env": {}, - "idle_duration": null, - "instance_types": null, - "max_price": null, + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], + "max_price": 25.5, "name": "test-fleet", "nodes": { "max": 1, "min": 1 }, - "placement": null, - "regions": null, - "reservation": null, - "resources": null, + "placement": "cluster", + "regions": [ + "us-east-1" + ], + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, "retry": null, - "spot_policy": null, + "spot_policy": "auto", "ssh_config": null, - "tags": null, + "tags": { + "env": "test" + }, "type": "fleet" }, "configuration_path": "fleet.dstack.yml", "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": null, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, - "max_duration": null, - "max_price": null, - "name": "", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "max_duration": 7200, + "max_price": 25.5, + "name": "default", + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null } } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json index 167f303ad2..1703c61aee 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_request/apply_run_plan_request.json @@ -11,25 +11,27 @@ "docker": null, "dstack": false, "entrypoint": null, - "env": {}, + "env": { + "HF_TOKEN": "secret" + }, "files": [], "fleets": null, "home_dir": "/root", "ide": "vscode", "idle_duration": null, "image": null, - "inactivity_duration": null, + "inactivity_duration": 3600, "init": [], "instance_types": null, "instances": null, "max_duration": null, "max_price": null, - "name": null, + "name": "test-dev", "nvcc": null, "ports": [], "priority": null, "privileged": false, - "python": null, + "python": "3.11", "regions": null, "registry_auth": null, "repos": [], @@ -60,7 +62,7 @@ "max": null, "min": 16.0 }, - "shm_size": null + "shm_size": 1024.0 }, "retry": null, "schedule": null, @@ -73,36 +75,63 @@ "stop_duration": null, "tags": null, "type": "dev-environment", - "user": null, + "user": "ubuntu", "utilization_policy": null, - "version": null, + "version": "1.85.0", "volumes": [], - "working_dir": null + "working_dir": "/workflow" }, "configuration_path": "dstack.yaml", "file_archives": [], "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": 300, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, "max_duration": 7200, - "max_price": null, + "max_price": 25.5, "name": "default", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null }, "repo_code_hash": null, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json index f6455670ed..2b0b2d6f78 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet.json @@ -32,52 +32,116 @@ "spec": { "autocreated": false, "configuration": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, + "backends": [ + "aws" + ], "blocks": 1, "env": {}, - "idle_duration": null, - "instance_types": null, - "max_price": null, + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], + "max_price": 25.5, "name": "test-fleet", "nodes": { "max": 1, "min": 1 }, - "placement": null, - "regions": null, - "reservation": null, - "resources": null, + "placement": "cluster", + "regions": [ + "us-east-1" + ], + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, "retry": null, - "spot_policy": null, + "spot_policy": "auto", "ssh_config": null, - "tags": null, + "tags": { + "env": "test" + }, "type": "fleet" }, "configuration_path": "fleet.dstack.yml", "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": null, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, - "max_duration": null, - "max_price": null, - "name": "", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "max_duration": 7200, + "max_price": 25.5, + "name": "default", + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null } }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json index c48f98aff5..b9f81fa330 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/fleet_plan.json @@ -34,52 +34,116 @@ "spec": { "autocreated": false, "configuration": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, + "backends": [ + "aws" + ], "blocks": 1, "env": {}, - "idle_duration": null, - "instance_types": null, - "max_price": null, + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], + "max_price": 25.5, "name": "test-fleet", "nodes": { "max": 1, "min": 1 }, - "placement": null, - "regions": null, - "reservation": null, - "resources": null, + "placement": "cluster", + "regions": [ + "us-east-1" + ], + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, "retry": null, - "spot_policy": null, + "spot_policy": "auto", "ssh_config": null, - "tags": null, + "tags": { + "env": "test" + }, "type": "fleet" }, "configuration_path": "fleet.dstack.yml", "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": null, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, - "max_duration": null, - "max_price": null, - "name": "", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "max_duration": 7200, + "max_price": 25.5, + "name": "default", + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null } }, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json index f0ba552507..e5a910e815 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/api_response/run_plan.json @@ -125,25 +125,27 @@ "docker": null, "dstack": false, "entrypoint": null, - "env": {}, + "env": { + "HF_TOKEN": "secret" + }, "files": [], "fleets": null, "home_dir": "/root", "ide": "vscode", "idle_duration": null, "image": null, - "inactivity_duration": null, + "inactivity_duration": 3600, "init": [], "instance_types": null, "instances": null, "max_duration": null, "max_price": null, - "name": null, + "name": "test-dev", "nvcc": null, "ports": [], "priority": null, "privileged": false, - "python": null, + "python": "3.11", "regions": null, "registry_auth": null, "repos": [], @@ -174,7 +176,7 @@ "max": null, "min": 16.0 }, - "shm_size": null + "shm_size": 1024.0 }, "retry": null, "schedule": null, @@ -187,36 +189,63 @@ "stop_duration": null, "tags": null, "type": "dev-environment", - "user": null, + "user": "ubuntu", "utilization_policy": null, - "version": null, + "version": "1.85.0", "volumes": [], - "working_dir": null + "working_dir": "/workflow" }, "configuration_path": "dstack.yaml", "file_archives": [], "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": 300, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, "max_duration": 7200, - "max_price": null, + "max_price": 25.5, "name": "default", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null }, "repo_code_hash": null, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json index bd38276d3e..b746b99f27 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/fleet_spec.json @@ -1,52 +1,116 @@ { "autocreated": false, "configuration": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, + "backends": [ + "aws" + ], "blocks": 1, "env": {}, - "idle_duration": null, - "instance_types": null, - "max_price": null, + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], + "max_price": 25.5, "name": "test-fleet", "nodes": { "max": 1, "min": 1 }, - "placement": null, - "regions": null, - "reservation": null, - "resources": null, + "placement": "cluster", + "regions": [ + "us-east-1" + ], + "reservation": "test-reservation", + "resources": { + "cpu": { + "max": 8, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 16.0 + }, + "shm_size": null + }, "retry": null, - "spot_policy": null, + "spot_policy": "auto", "ssh_config": null, - "tags": null, + "tags": { + "env": "test" + }, "type": "fleet" }, "configuration_path": "fleet.dstack.yml", "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": null, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, - "max_duration": null, - "max_price": null, - "name": "", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "max_duration": 7200, + "max_price": 25.5, + "name": "default", + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null } } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json index 39b0c11e78..e6049263e4 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/instance_configuration.json @@ -1,10 +1,17 @@ { - "instance_id": null, + "instance_id": "i-1234567890abcdef0", "instance_name": "test-instance", "project_name": "test-project", - "reservation": null, - "ssh_keys": [], - "tags": null, + "reservation": "test-reservation", + "ssh_keys": [ + { + "private": null, + "public": "ssh-rsa PUBLIC" + } + ], + "tags": { + "env": "test" + }, "user": "dstack-user", "volumes": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json index c0c39d48fb..58bdfcb4c3 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/job_runtime_data.json @@ -1,14 +1,16 @@ { - "cpu": null, - "gpu": null, - "memory": null, + "cpu": 2.0, + "gpu": 1, + "memory": 17179869184.0, "network_mode": "host", "offer": null, "ports": { "8080": 30080, "8081": 30081 }, - "username": null, - "volume_names": null, - "working_dir": null + "username": "ubuntu", + "volume_names": [ + "test-volume" + ], + "working_dir": "/workflow" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json index f1bd7642ac..5f01662b59 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/profile.json @@ -1,29 +1,51 @@ { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], "idle_duration": 600, - "instance_types": null, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, "max_duration": 7200, - "max_price": null, + "max_price": 25.5, "name": "default", - "regions": null, - "reservation": null, + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", "retry": { "duration": 3600, "on_events": [ "no-capacity" ] }, - "schedule": null, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, "spot_policy": "auto", - "startup_order": null, - "stop_criteria": null, + "startup_order": "master-first", + "stop_criteria": "all-done", "stop_duration": 300, - "tags": null, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null } diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/run_spec.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/run_spec.json index 6a236e634c..8de08fc8f5 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/run_spec.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/run_spec.json @@ -7,25 +7,27 @@ "docker": null, "dstack": false, "entrypoint": null, - "env": {}, + "env": { + "HF_TOKEN": "secret" + }, "files": [], "fleets": null, "home_dir": "/root", "ide": "vscode", "idle_duration": null, "image": null, - "inactivity_duration": null, + "inactivity_duration": 3600, "init": [], "instance_types": null, "instances": null, "max_duration": null, "max_price": null, - "name": null, + "name": "test-dev", "nvcc": null, "ports": [], "priority": null, "privileged": false, - "python": null, + "python": "3.11", "regions": null, "registry_auth": null, "repos": [], @@ -56,7 +58,7 @@ "max": null, "min": 16.0 }, - "shm_size": null + "shm_size": 1024.0 }, "retry": null, "schedule": null, @@ -69,36 +71,63 @@ "stop_duration": null, "tags": null, "type": "dev-environment", - "user": null, + "user": "ubuntu", "utilization_policy": null, - "version": null, + "version": "1.85.0", "volumes": [], - "working_dir": null + "working_dir": "/workflow" }, "configuration_path": "dstack.yaml", "file_archives": [], "profile": { - "availability_zones": null, + "availability_zones": [ + "us-east-1a" + ], "backend_options": null, - "backends": null, - "creation_policy": null, + "backends": [ + "aws", + "gcp" + ], + "creation_policy": "reuse-or-create", "default": false, - "fleets": null, - "idle_duration": 300, - "instance_types": null, + "fleets": [ + { + "name": "test-fleet", + "project": null + } + ], + "idle_duration": 600, + "instance_types": [ + "p4d.24xlarge" + ], "instances": null, "max_duration": 7200, - "max_price": null, + "max_price": 25.5, "name": "default", - "regions": null, - "reservation": null, - "retry": null, - "schedule": null, - "spot_policy": null, - "startup_order": null, - "stop_criteria": null, - "stop_duration": null, - "tags": null, + "regions": [ + "us-east-1", + "eu-west-1" + ], + "reservation": "test-reservation", + "retry": { + "duration": 3600, + "on_events": [ + "no-capacity" + ] + }, + "schedule": { + "cron": [ + "0 0 * * *" + ] + }, + "spot_policy": "auto", + "startup_order": "master-first", + "stop_criteria": "all-done", + "stop_duration": 300, + "tags": { + "env": "test", + "team": "core" + }, "utilization_policy": null }, "repo_code_hash": null, diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/chat_completions_chunk.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/chat_completions_chunk.json new file mode 100644 index 0000000000..37c7f97963 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/chat_completions_chunk.json @@ -0,0 +1,18 @@ +{ + "choices": [ + { + "delta": { + "content": "hi", + "role": "assistant" + }, + "finish_reason": null, + "index": 0, + "logprobs": {} + } + ], + "created": 1700000000, + "id": "chatcmpl-1", + "model": "llama", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_1" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/models_response.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/models_response.json new file mode 100644 index 0000000000..cc5eedf825 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/proxy_response/models_response.json @@ -0,0 +1,11 @@ +{ + "data": [ + { + "created": 1700000000, + "id": "llama", + "object": "model", + "owned_by": "dstack" + } + ], + "object": "list" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/component_install_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/component_install_request.json new file mode 100644 index 0000000000..1d8cfb1828 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/component_install_request.json @@ -0,0 +1,4 @@ +{ + "name": "dstack-shim", + "url": "https://example.com/shim.tar.gz" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/legacy_submit_body.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/legacy_submit_body.json new file mode 100644 index 0000000000..65c5af89d1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/legacy_submit_body.json @@ -0,0 +1,17 @@ +{ + "container_name": "test-container", + "container_user": "root", + "image_name": "dstackai/base:latest", + "instance_mounts": [], + "mounts": [], + "password": "", + "privileged": false, + "public_keys": [ + "ssh-rsa PUBLIC" + ], + "shm_size": 1073741824, + "ssh_key": "ssh-rsa SSH", + "ssh_user": "ubuntu", + "username": "", + "volumes": [] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/shutdown_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/shutdown_request.json new file mode 100644 index 0000000000..9bbd353f57 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/shutdown_request.json @@ -0,0 +1,3 @@ +{ + "force": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json index b3a35e76a5..f429b9f548 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/submit_body.json @@ -39,25 +39,27 @@ "docker": null, "dstack": false, "entrypoint": null, - "env": {}, + "env": { + "HF_TOKEN": "secret" + }, "files": [], "fleets": null, "home_dir": "/root", "ide": "vscode", "idle_duration": null, "image": null, - "inactivity_duration": null, + "inactivity_duration": 3600, "init": [], "instance_types": null, "instances": null, "max_duration": null, "max_price": null, - "name": null, + "name": "test-dev", "nvcc": null, "ports": [], "priority": null, "privileged": false, - "python": null, + "python": "3.11", "regions": null, "registry_auth": null, "repos": [], @@ -88,7 +90,7 @@ "max": null, "min": 16.0 }, - "shm_size": null + "shm_size": 1024.0 }, "retry": null, "schedule": null, @@ -101,11 +103,11 @@ "stop_duration": null, "tags": null, "type": "dev-environment", - "user": null, + "user": "ubuntu", "utilization_policy": null, - "version": null, + "version": "1.85.0", "volumes": [], - "working_dir": null + "working_dir": "/workflow" }, "configuration_path": "dstack.yaml", "repo_data": { @@ -125,25 +127,27 @@ "docker": null, "dstack": false, "entrypoint": null, - "env": {}, + "env": { + "HF_TOKEN": "secret" + }, "files": [], "fleets": null, "home_dir": "/root", "ide": "vscode", "idle_duration": null, "image": null, - "inactivity_duration": null, + "inactivity_duration": 3600, "init": [], "instance_types": null, "instances": null, "max_duration": null, "max_price": null, - "name": null, + "name": "test-dev", "nvcc": null, "ports": [], "priority": null, "privileged": false, - "python": null, + "python": "3.11", "regions": null, "registry_auth": null, "repos": [], @@ -174,7 +178,7 @@ "max": null, "min": 16.0 }, - "shm_size": null + "shm_size": 1024.0 }, "retry": null, "schedule": null, @@ -187,11 +191,11 @@ "stop_duration": null, "tags": null, "type": "dev-environment", - "user": null, + "user": "ubuntu", "utilization_policy": null, - "version": null, + "version": "1.85.0", "volumes": [], - "working_dir": null + "working_dir": "/workflow" }, "configuration_path": "dstack.yaml", "repo_data": { diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_submit_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_submit_request.json new file mode 100644 index 0000000000..42e17adab8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_submit_request.json @@ -0,0 +1,25 @@ +{ + "container_ssh_keys": [ + "ssh-rsa CONTAINER" + ], + "container_user": "root", + "cpu": 2.0, + "gpu": 1, + "gpu_devices": [], + "host_ssh_keys": [ + "ssh-rsa HOST" + ], + "host_ssh_user": "ubuntu", + "id": "11111111-1111-4111-8111-111111111111", + "image_name": "dstackai/base:latest", + "instance_mounts": [], + "memory": 17179869184, + "name": "test-task", + "network_mode": "host", + "privileged": false, + "registry_password": "", + "registry_username": "", + "shm_size": 1073741824, + "volume_mounts": [], + "volumes": [] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_terminate_request.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_terminate_request.json new file mode 100644 index 0000000000..b47bc3db2c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/runner/task_terminate_request.json @@ -0,0 +1,5 @@ +{ + "termination_message": "max duration exceeded", + "termination_reason": "MAX_DURATION_EXCEEDED", + "timeout": 10 +} diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index acfbf82a11..ddb80b12ea 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -23,22 +23,27 @@ import pytest import yaml -from dstack._internal.core.backends.aws.models import AWSCreds from dstack._internal.core.models.configurations import DstackConfiguration -from dstack._internal.core.models.fleets import Fleet from dstack._internal.core.models.profiles import ProfilesConfig -from dstack._internal.core.models.runs import JobSpec, RunSpec from dstack._internal.proxy.gateway.schemas.registry import ( + RegisterEntrypointRequest, RegisterReplicaRequest, RegisterServiceRequest, ) +from dstack._internal.proxy.gateway.schemas.stats import ServiceStats from dstack._internal.proxy.lib.schemas.model_proxy import ( ChatCompletionsChunk, ChatCompletionsRequest, ChatCompletionsResponse, ) -from dstack._internal.server.schemas.runner import HealthcheckResponse, MetricsResponse -from dstack._internal.server.schemas.volumes import CreateVolumeRequest +from dstack._internal.server.schemas.runner import ( + HealthcheckResponse, + InstanceHealthResponse, + JobInfoResponse, + MetricsResponse, + TaskInfoResponse, +) +from tests._internal.pydantic_compat import test_serialization as ser from tests._internal.pydantic_compat.compare import ( FIXTURES_DIR, assert_matches_fixture, @@ -46,24 +51,32 @@ type_map, ) from tests._internal.pydantic_compat.compat import parse_forbid_extra, parse_ignore_extra -from tests._internal.pydantic_compat.test_serialization import DB_BLOBS as SERIALIZED_DB_BLOBS -# Read from a `Text` column with extra="ignore", so a row written by a newer server loads. -DB_BLOBS: dict[str, Any] = { - "aws_creds": AWSCreds, - "job_spec": JobSpec, - "run_spec": RunSpec, +# Parse cases are derived from the serialization surfaces rather than listed again: every model we +# write, we also read, and a model added on one side must not silently lack coverage on the other. +# +# The input for each case is the payload v1 produced, frozen on disk. Where a hand-written input +# already exists it is kept instead — those are strictly better, because they carry shapes a +# canonical dump cannot show: a row missing fields that now default, or YAML shorthand. +# +# `extra` policy follows the reader, not the model: the server forbids unknown fields in a request +# body and ignores them in a stored row. +READ_SURFACES: dict[str, str] = { + "db": "ignore", + "api_request": "forbid", + "api_response": "ignore", + "gateway": "ignore", } -# Validated from a request body with extra="forbid": an unknown field is a user-facing error. -API_REQUESTS: dict[str, Any] = { - "create_volume_request": CreateVolumeRequest, -} -# Parsed by the API client from a server response with extra="ignore", so an older CLI works. -API_RESPONSES: dict[str, Any] = { - "fleet": Fleet, -} +def _derived_registry(surface: str) -> dict[str, Any]: + registry, _ = ser.SURFACES[surface] + return {name: type(factory()) for name, factory in registry.items()} + + +DB_BLOBS = _derived_registry("db") +API_REQUESTS = _derived_registry("api_request") +API_RESPONSES = _derived_registry("api_response") # Parsed from user-authored YAML with extra="forbid". The only surface whose inputs are `.yml`, # because the sugar pinned here is a YAML phenomenon: `python: 3.10` is a float that survives only @@ -71,6 +84,7 @@ # `DstackConfiguration` dispatches every `.dstack.yml` type through its `__root__` union, so one # entry point covers task, service, dev-environment, fleet, volume, and gateway. CONFIGS: dict[str, Any] = { + "dev_environment": DstackConfiguration, "fleet": DstackConfiguration, "profiles": ProfilesConfig, "service": DstackConfiguration, @@ -82,15 +96,20 @@ # fields, and the server has no say over which runner version an instance is running. RUNNER_RESPONSES: dict[str, Any] = { "healthcheck_response": HealthcheckResponse, + "instance_health_response": InstanceHealthResponse, + "job_info_response": JobInfoResponse, "metrics_response": MetricsResponse, + "task_info_response": TaskInfoResponse, } # Request payloads the gateway parses from the server. These schemas are plain `BaseModel`, so # `parse_obj` already ignores unknown fields in both pydantic versions — no shim needed, and # nothing to assert about strictness. -GATEWAY_REQUESTS: dict[str, Any] = { +GATEWAY_PAYLOADS: dict[str, Any] = { + "register_entrypoint_request": RegisterEntrypointRequest, "register_replica_request": RegisterReplicaRequest, "register_service_request": RegisterServiceRequest, + "service_stats": ServiceStats, } # The model proxy. The request comes from a caller and the response from an upstream model server, @@ -104,6 +123,10 @@ } +def _case_id(value: Any) -> str: + return str(value) + + class TestDbBlobParsing: @pytest.mark.parametrize("name", sorted(DB_BLOBS)) def test_parses_to_expected_values_and_types(self, name, regen): @@ -111,23 +134,59 @@ def test_parses_to_expected_values_and_types(self, name, regen): _assert_parses("db", name, model, regen) -class TestDbBlobExtraFieldTolerance: +# Derived coverage: a serialization fixture is already a valid payload, so injecting one unknown +# key into it reproduces exactly what a reader faces when the writer is a newer version. No +# hand-written input needed, so this scales to every model on the surface for free. +# +# Only listed where *we* are the reader. `serialization/runner` is read by the Go runner and +# `serialization/proxy` by an upstream model server — we cannot assert anything about how those parse. +TOLERANT_SURFACES = ("db", "api_response", "gateway", "proxy_response") + +# Same trick, opposite expectation: these are read with extra="forbid", so the injected key must +# be an error rather than be dropped. +FORBIDDING_SURFACES = ("api_request",) + +_TOLERANCE_CASES = [c for c in ser._CASES if c[0] in TOLERANT_SURFACES] +_FORBID_CASES = [c for c in ser._CASES if c[0] in FORBIDDING_SURFACES] + + +class TestUnknownFieldTolerance: + """Failing to read one model is a distinct bug from failing to read another, so cover all.""" + + @pytest.mark.parametrize(("surface", "name"), _TOLERANCE_CASES, ids=_case_id) + def test_unknown_field_is_dropped(self, surface, name): + # The payload is produced from the factory rather than read from the committed fixture: + # the fixture is already pinned by `test_serialization`, and reading it here would make + # this test depend on that one having run first — which it has not, since `test_parsing` + # collects earlier. + # + # Compare perturbed against the *same payload parsed without* the extra key rather than + # against the payload itself. Parsing validates, and validation coerces defaults that were + # never validated on the way out — `Volume.cost: float = 0` holds an int until something + # parses it — so comparing to the input would fail for unrelated reasons. + registry, dump = ser.SURFACES[surface] + model = type(registry[name]()) + payload = json.loads(ser.serialize(surface, name)) + baseline = parse_ignore_extra(model, payload) + perturbed = parse_ignore_extra(model, {**payload, "unknown_from_a_newer_writer": {"x": 1}}) + assert canonicalize(dump(perturbed)) == canonicalize(dump(baseline)) + + +class TestUnknownFieldRejection: """ - Every stored model must survive a row extended by a newer writer. + The mirror image, and the reason the two lists are separate. - This needs no hand-written input: the serialization fixture already *is* a valid row, so - injecting one unknown key into it produces the exact situation an older reader faces. Covers - all of `serialization/db` rather than only the models with an interesting input above, because - failing to read a `VolumeProvisioningData` row is a distinct bug from failing to read a - `RunSpec` row. + Forbidding extra fields is what makes `dstack apply` report a typo'd key instead of ignoring + it. The v2 `CoreModel` forbids by default with a per-call `extra="ignore"` override, so the + regression to guard against is that override leaking onto a surface listed here. """ - @pytest.mark.parametrize("name", sorted(SERIALIZED_DB_BLOBS)) - def test_unknown_field_is_dropped(self, name): - committed = (FIXTURES_DIR / "serialization" / "db" / f"{name}.json").read_text() - perturbed = {**json.loads(committed), "unknown_from_a_newer_writer": {"x": [1]}} - parsed = parse_ignore_extra(type(SERIALIZED_DB_BLOBS[name]()), perturbed) - assert canonicalize(parsed.json()) == canonicalize(committed) + @pytest.mark.parametrize(("surface", "name"), _FORBID_CASES, ids=_case_id) + def test_unknown_field_is_rejected(self, surface, name): + registry, _ = ser.SURFACES[surface] + perturbed = {**json.loads(ser.serialize(surface, name)), "definitely_not_a_field": 1} + with pytest.raises(Exception, match="(?i)extra"): + parse_forbid_extra(type(registry[name]()), perturbed) class TestApiRequestParsing: @@ -171,9 +230,9 @@ def test_parses_to_expected_values_and_types(self, name, regen): class TestGatewayRequestParsing: - @pytest.mark.parametrize("name", sorted(GATEWAY_REQUESTS)) + @pytest.mark.parametrize("name", sorted(GATEWAY_PAYLOADS)) def test_parses_to_expected_values_and_types(self, name, regen): - model = GATEWAY_REQUESTS[name].parse_obj(_load_input("gateway", name)) + model = parse_ignore_extra(GATEWAY_PAYLOADS[name], _load_input("gateway", name)) _assert_parses("gateway", name, model, regen) diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 72cecad1e9..ee8667b33a 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -16,7 +16,7 @@ """ import json -from typing import Callable +from typing import Any, Callable, Union import pytest from pydantic import BaseModel @@ -80,7 +80,12 @@ # Sent by the server to the runner (shim) as a request body, via `.json()`. RUNNER_REQUESTS: dict[str, Callable[[], CoreModel]] = { + "component_install_request": factories.component_install_request, + "legacy_submit_body": factories.legacy_submit_body, + "shutdown_request": factories.shutdown_request, "submit_body": factories.submit_body, + "task_submit_request": factories.task_submit_request, + "task_terminate_request": factories.task_terminate_request, } # Returned by the gateway to the server. The request direction is not model-driven — the server @@ -89,6 +94,14 @@ "service_stats": factories.service_stats, } +# Returned to the caller of the OpenAI-compatible API. A chunk is dumped one at a time into an SSE +# stream (`f"data:{chunk.json()}"` in `proxy/lib/routers/model_proxy.py`) rather than as a body, +# which makes it a fifth production dump path. +PROXY_RESPONSES: dict[str, Callable[[], CoreModel]] = { + "chat_completions_chunk": factories.chat_completions_chunk, + "models_response": factories.models_response, +} + # Forwarded upstream by the model proxy. Dumped with `exclude_unset=True`, matching # `proxy/lib/services/model_proxy/clients/openai.py`, and encoded with stdlib json the way httpx # does for a `json=` body — the fourth distinct dump path, and the only one where @@ -98,46 +111,35 @@ } -class TestDbBlobSerialization: - @pytest.mark.parametrize("name", sorted(DB_BLOBS)) - def test_matches_fixture(self, name, regen): - payload = DB_BLOBS[name]().json() - assert_matches_fixture("serialization/db", name, payload, regen=regen) - - -class TestApiRequestSerialization: - @pytest.mark.parametrize("name", sorted(API_REQUESTS)) - def test_matches_fixture(self, name, regen): - payload = API_REQUESTS[name]().json() - assert_matches_fixture("serialization/api_request", name, payload, regen=regen) - - -class TestApiResponseSerialization: - @pytest.mark.parametrize("name", sorted(API_RESPONSES)) - def test_matches_fixture(self, name, regen): - payload = bytes(CustomORJSONResponse(API_RESPONSES[name]()).body) - assert_matches_fixture("serialization/api_response", name, payload, regen=regen) - +# Surface -> (models, the production call that serializes them). Keeping the dump here rather than +# inside each test matters because `test_parsing` re-serializes these same fixtures to check +# unknown-field tolerance: it has to use the identical path, or it compares orjson output against +# `.json()` output and fails for reasons that have nothing to do with parsing. +SURFACES: dict[str, tuple[dict[str, Callable[[], Any]], Callable[[Any], Union[bytes, str]]]] = { + "db": (DB_BLOBS, lambda model: model.json()), + "api_request": (API_REQUESTS, lambda model: model.json()), + "api_response": (API_RESPONSES, lambda model: bytes(CustomORJSONResponse(model).body)), + "runner": (RUNNER_REQUESTS, lambda model: model.json()), + "gateway": (GATEWAY_RESPONSES, lambda model: model.json()), + "proxy": (PROXY_REQUESTS, lambda model: json.dumps(model.dict(exclude_unset=True))), + "proxy_response": (PROXY_RESPONSES, lambda model: model.json()), +} -class TestRunnerRequestSerialization: - @pytest.mark.parametrize("name", sorted(RUNNER_REQUESTS)) - def test_matches_fixture(self, name, regen): - payload = RUNNER_REQUESTS[name]().json() - assert_matches_fixture("serialization/runner", name, payload, regen=regen) +_CASES = [(surface, name) for surface, (reg, _) in SURFACES.items() for name in sorted(reg)] -class TestGatewayResponseSerialization: - @pytest.mark.parametrize("name", sorted(GATEWAY_RESPONSES)) - def test_matches_fixture(self, name, regen): - payload = GATEWAY_RESPONSES[name]().json() - assert_matches_fixture("serialization/gateway", name, payload, regen=regen) +def serialize(surface: str, name: str) -> Union[bytes, str]: + """Build the model for a case and serialize it the way production does.""" + registry, dump = SURFACES[surface] + return dump(registry[name]()) -class TestProxyRequestSerialization: - @pytest.mark.parametrize("name", sorted(PROXY_REQUESTS)) - def test_matches_fixture(self, name, regen): - payload = json.dumps(PROXY_REQUESTS[name]().dict(exclude_unset=True)) - assert_matches_fixture("serialization/proxy", name, payload, regen=regen) +class TestSerialization: + @pytest.mark.parametrize(("surface", "name"), _CASES, ids=lambda v: str(v)) + def test_matches_fixture(self, surface, name, regen): + assert_matches_fixture( + f"serialization/{surface}", name, serialize(surface, name), regen=regen + ) class TestFleetNodesTargetCompatHack: From 49fbc2bcf63e2b03e8d0c5f222604898cae7ab95 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 10:26:22 +0500 Subject: [PATCH 10/15] Fix extra parsing tests --- .../_internal/pydantic_compat/factories.py | 4 -- .../_internal/pydantic_compat/test_parsing.py | 72 ++++++++----------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index 85f6a24bf4..c7c03fbba7 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -1,9 +1,5 @@ """ Deterministic model instances for the comparison fixtures. - -Every value is pinned — no `uuid4()`, no `now()` — so a fixture only changes when serialization -changes. Builds on `dstack._internal.server.testing.common` where a factory already exists, so -these stay in sync with the shapes the rest of the suite uses. """ import uuid diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index ddb80b12ea..9c0ea6349b 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -52,21 +52,14 @@ ) from tests._internal.pydantic_compat.compat import parse_forbid_extra, parse_ignore_extra -# Parse cases are derived from the serialization surfaces rather than listed again: every model we -# write, we also read, and a model added on one side must not silently lack coverage on the other. -# -# The input for each case is the payload v1 produced, frozen on disk. Where a hand-written input -# already exists it is kept instead — those are strictly better, because they carry shapes a -# canonical dump cannot show: a row missing fields that now default, or YAML shorthand. +# The model *list* for each surface is taken from the serialization side rather than repeated here, +# so a model added on one side cannot silently lack coverage on the other. That is the only thing +# borrowed — every payload below is a hand-written input, because a canonical dump can never carry +# the shapes that stress a parser: fields absent as an older writer left them, values in the +# shorthand a user or CLI actually sends. # # `extra` policy follows the reader, not the model: the server forbids unknown fields in a request -# body and ignores them in a stored row. -READ_SURFACES: dict[str, str] = { - "db": "ignore", - "api_request": "forbid", - "api_response": "ignore", - "gateway": "ignore", -} +# body and ignores them in a stored row, and the same model can appear on both sides. def _derived_registry(surface: str) -> dict[str, Any]: @@ -134,20 +127,26 @@ def test_parses_to_expected_values_and_types(self, name, regen): _assert_parses("db", name, model, regen) -# Derived coverage: a serialization fixture is already a valid payload, so injecting one unknown -# key into it reproduces exactly what a reader faces when the writer is a newer version. No -# hand-written input needed, so this scales to every model on the surface for free. +# Every case above earns a second assertion: inject an unknown key into the same input and check the +# reader's `extra` policy holds. That covers what happens when the writer is a newer version. # -# Only listed where *we* are the reader. `serialization/runner` is read by the Go runner and -# `serialization/proxy` by an upstream model server — we cannot assert anything about how those parse. -TOLERANT_SURFACES = ("db", "api_response", "gateway", "proxy_response") - -# Same trick, opposite expectation: these are read with extra="forbid", so the injected key must -# be an error rather than be dropped. -FORBIDDING_SURFACES = ("api_request",) +# Only the surfaces we read are listed. Runner request bodies are read by the Go runner and proxy +# requests by an upstream model server, so there is nothing here we could assert about how they +# parse — they are absent rather than passing vacuously. +_REGISTRIES = { + "db": DB_BLOBS, + "api_request": API_REQUESTS, + "api_response": API_RESPONSES, + "gateway": GATEWAY_PAYLOADS, +} -_TOLERANCE_CASES = [c for c in ser._CASES if c[0] in TOLERANT_SURFACES] -_FORBID_CASES = [c for c in ser._CASES if c[0] in FORBIDDING_SURFACES] +# Readers that ignore unknown fields, versus the one that must reject them. +_TOLERANCE_CASES = [ + (surface, name) + for surface in ("db", "api_response", "gateway") + for name in sorted(_REGISTRIES[surface]) +] +_FORBID_CASES = [("api_request", name) for name in sorted(API_REQUESTS)] class TestUnknownFieldTolerance: @@ -155,21 +154,13 @@ class TestUnknownFieldTolerance: @pytest.mark.parametrize(("surface", "name"), _TOLERANCE_CASES, ids=_case_id) def test_unknown_field_is_dropped(self, surface, name): - # The payload is produced from the factory rather than read from the committed fixture: - # the fixture is already pinned by `test_serialization`, and reading it here would make - # this test depend on that one having run first — which it has not, since `test_parsing` - # collects earlier. - # - # Compare perturbed against the *same payload parsed without* the extra key rather than - # against the payload itself. Parsing validates, and validation coerces defaults that were - # never validated on the way out — `Volume.cost: float = 0` holds an int until something - # parses it — so comparing to the input would fail for unrelated reasons. - registry, dump = ser.SURFACES[surface] - model = type(registry[name]()) - payload = json.loads(ser.serialize(surface, name)) + # Compare against the *same input parsed without* the extra key, not against the input + # itself: parsing fills defaults, so the two are not expected to match. + model = _REGISTRIES[surface][name] + payload = _load_input(surface, name) baseline = parse_ignore_extra(model, payload) perturbed = parse_ignore_extra(model, {**payload, "unknown_from_a_newer_writer": {"x": 1}}) - assert canonicalize(dump(perturbed)) == canonicalize(dump(baseline)) + assert canonicalize(perturbed.json()) == canonicalize(baseline.json()) class TestUnknownFieldRejection: @@ -183,10 +174,9 @@ class TestUnknownFieldRejection: @pytest.mark.parametrize(("surface", "name"), _FORBID_CASES, ids=_case_id) def test_unknown_field_is_rejected(self, surface, name): - registry, _ = ser.SURFACES[surface] - perturbed = {**json.loads(ser.serialize(surface, name)), "definitely_not_a_field": 1} + perturbed = {**_load_input(surface, name), "definitely_not_a_field": 1} with pytest.raises(Exception, match="(?i)extra"): - parse_forbid_extra(type(registry[name]()), perturbed) + parse_forbid_extra(_REGISTRIES[surface][name], perturbed) class TestApiRequestParsing: From e06a18dd98b52742f68134ac1b2a70c69bc64bd1 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 11:04:00 +0500 Subject: [PATCH 11/15] Test backend models --- .../pydantic_compat/backend_factories.py | 651 ++++++++++++++++++ .../_internal/pydantic_compat/compare.py | 8 +- .../_internal/pydantic_compat/factories.py | 6 - .../backend_config/amddevcloud.input.json | 5 + .../backend_config/amddevcloud.types.json | 4 + .../backend_config/amddevcloud.values.json | 9 + .../parsing/backend_config/aws.input.json | 14 + .../parsing/backend_config/aws.types.json | 4 + .../parsing/backend_config/aws.values.json | 30 + .../parsing/backend_config/azure.input.json | 13 + .../parsing/backend_config/azure.types.json | 4 + .../parsing/backend_config/azure.values.json | 21 + .../backend_config/cloudrift.input.json | 5 + .../backend_config/cloudrift.types.json | 4 + .../backend_config/cloudrift.values.json | 10 + .../parsing/backend_config/crusoe.input.json | 9 + .../parsing/backend_config/crusoe.types.json | 4 + .../parsing/backend_config/crusoe.values.json | 10 + .../backend_config/datacrunch.input.json | 9 + .../backend_config/datacrunch.types.json | 4 + .../backend_config/datacrunch.values.json | 12 + .../backend_config/digitalocean.input.json | 6 + .../backend_config/digitalocean.types.json | 4 + .../backend_config/digitalocean.values.json | 13 + .../parsing/backend_config/gcp.input.json | 16 + .../parsing/backend_config/gcp.types.json | 4 + .../parsing/backend_config/gcp.values.json | 27 + .../backend_config/hotaisle.input.json | 6 + .../backend_config/hotaisle.types.json | 4 + .../backend_config/hotaisle.values.json | 11 + .../backend_config/jarvislabs.input.json | 4 + .../backend_config/jarvislabs.types.json | 4 + .../backend_config/jarvislabs.values.json | 8 + .../backend_config/kubernetes.input.json | 12 + .../backend_config/kubernetes.types.json | 6 + .../backend_config/kubernetes.values.json | 19 + .../parsing/backend_config/lambda.input.json | 5 + .../parsing/backend_config/lambda.types.json | 4 + .../parsing/backend_config/lambda.values.json | 12 + .../parsing/backend_config/nebius.input.json | 12 + .../parsing/backend_config/nebius.types.json | 4 + .../parsing/backend_config/nebius.values.json | 23 + .../parsing/backend_config/oci.input.json | 17 + .../parsing/backend_config/oci.types.json | 4 + .../parsing/backend_config/oci.values.json | 22 + .../parsing/backend_config/runpod.input.json | 6 + .../parsing/backend_config/runpod.types.json | 4 + .../parsing/backend_config/runpod.values.json | 13 + .../parsing/backend_config/slurm.input.json | 25 + .../parsing/backend_config/slurm.types.json | 8 + .../parsing/backend_config/slurm.values.json | 39 ++ .../parsing/backend_config/vastai.input.json | 6 + .../parsing/backend_config/vastai.types.json | 4 + .../parsing/backend_config/vastai.values.json | 13 + .../parsing/backend_config/verda.input.json | 9 + .../parsing/backend_config/verda.types.json | 4 + .../parsing/backend_config/verda.values.json | 11 + .../parsing/backend_config/vultr.input.json | 5 + .../parsing/backend_config/vultr.types.json | 4 + .../parsing/backend_config/vultr.values.json | 12 + .../backend_creds/aws.access_key.input.json | 5 + .../aws.access_key.types.json} | 0 .../backend_creds/aws.access_key.values.json | 5 + .../backend_creds/aws.default.input.json | 3 + .../backend_creds/aws.default.types.json | 4 + .../backend_creds/aws.default.values.json | 3 + .../backend_creds/azure.client.input.json | 5 + .../backend_creds/azure.client.types.json | 4 + .../backend_creds/azure.client.values.json | 6 + .../backend_creds/azure.default.input.json | 3 + .../backend_creds/azure.default.types.json | 4 + .../backend_creds/azure.default.values.json | 3 + .../backend_creds/cloudrift.input.json | 4 + .../backend_creds/cloudrift.types.json | 3 + .../backend_creds/cloudrift.values.json | 4 + .../parsing/backend_creds/crusoe.input.json | 5 + .../parsing/backend_creds/crusoe.types.json | 3 + .../parsing/backend_creds/crusoe.values.json | 5 + .../backend_creds/digitalocean.input.json | 4 + .../backend_creds/digitalocean.types.json | 3 + .../backend_creds/digitalocean.values.json | 4 + .../backend_creds/gcp.default.input.json | 3 + .../backend_creds/gcp.default.types.json | 4 + .../backend_creds/gcp.default.values.json | 3 + .../gcp.service_account.input.json | 4 + .../gcp.service_account.types.json | 4 + .../gcp.service_account.values.json | 5 + .../parsing/backend_creds/hotaisle.input.json | 4 + .../parsing/backend_creds/hotaisle.types.json | 3 + .../backend_creds/hotaisle.values.json | 4 + .../backend_creds/jarvislabs.input.json | 4 + .../backend_creds/jarvislabs.types.json | 3 + .../backend_creds/jarvislabs.values.json | 4 + .../parsing/backend_creds/lambda.input.json | 4 + .../parsing/backend_creds/lambda.types.json | 3 + .../parsing/backend_creds/lambda.values.json | 4 + .../parsing/backend_creds/nebius.input.json | 6 + .../parsing/backend_creds/nebius.types.json | 3 + .../parsing/backend_creds/nebius.values.json | 8 + .../backend_creds/oci.client.input.json | 8 + .../backend_creds/oci.client.types.json | 4 + .../backend_creds/oci.client.values.json | 10 + .../backend_creds/oci.default.input.json | 3 + .../backend_creds/oci.default.types.json | 4 + .../backend_creds/oci.default.values.json | 5 + .../parsing/backend_creds/runpod.input.json | 4 + .../parsing/backend_creds/runpod.types.json | 3 + .../parsing/backend_creds/runpod.values.json | 4 + .../parsing/backend_creds/vastai.input.json | 4 + .../parsing/backend_creds/vastai.types.json | 3 + .../parsing/backend_creds/vastai.values.json | 4 + .../parsing/backend_creds/verda.input.json | 5 + .../parsing/backend_creds/verda.types.json | 3 + .../parsing/backend_creds/verda.values.json | 5 + .../parsing/backend_creds/vultr.input.json | 4 + .../parsing/backend_creds/vultr.types.json | 3 + .../parsing/backend_creds/vultr.values.json | 4 + .../backend_data/aws_gateway.input.json | 5 + .../backend_data/aws_gateway.types.json | 3 + .../backend_data/aws_gateway.values.json | 6 + .../backend_data/aws_instance.input.json | 1 + .../backend_data/aws_instance.types.json | 3 + .../backend_data/aws_instance.values.json | 3 + .../backend_data/aws_volume.input.json | 4 + .../backend_data/aws_volume.types.json | 3 + .../backend_data/aws_volume.values.json | 4 + .../backend_data/crusoe_instance.input.json | 1 + .../backend_data/crusoe_instance.types.json | 3 + .../backend_data/crusoe_instance.values.json | 3 + .../crusoe_placement_group.input.json | 3 + .../crusoe_placement_group.types.json | 3 + .../crusoe_placement_group.values.json | 4 + .../parsing/backend_data/gcp_offer.input.json | 1 + .../parsing/backend_data/gcp_offer.types.json | 3 + .../backend_data/gcp_offer.values.json | 3 + .../backend_data/gcp_volume_disk.input.json | 3 + .../backend_data/gcp_volume_disk.types.json | 3 + .../backend_data/gcp_volume_disk.values.json | 4 + .../backend_data/hotaisle_instance.input.json | 4 + .../backend_data/hotaisle_instance.types.json | 3 + .../hotaisle_instance.values.json | 3 + .../backend_data/hotaisle_offer.input.json | 9 + .../backend_data/hotaisle_offer.types.json | 3 + .../backend_data/hotaisle_offer.values.json | 17 + .../jarvislabs_instance.input.json | 1 + .../jarvislabs_instance.types.json | 3 + .../jarvislabs_instance.values.json | 3 + .../backend_data/kubernetes.input.json | 6 + .../backend_data/kubernetes.types.json | 3 + .../backend_data/kubernetes.values.json | 5 + .../backend_data/nebius_cluster.input.json | 5 + .../backend_data/nebius_cluster.types.json | 3 + .../backend_data/nebius_cluster.values.json | 4 + .../backend_data/nebius_instance.input.json | 4 + .../backend_data/nebius_instance.types.json | 3 + .../backend_data/nebius_instance.values.json | 3 + .../nebius_placement_group.input.json | 7 + .../nebius_placement_group.types.json | 4 + .../nebius_placement_group.values.json | 6 + .../backend_data/runpod_offer.input.json | 1 + .../backend_data/runpod_offer.types.json | 3 + .../backend_data/runpod_offer.values.json | 3 + .../backend_data/vastai_offer.input.json | 3 + .../backend_data/vastai_offer.types.json | 3 + .../backend_data/vastai_offer.values.json | 3 + .../backend_data/verda_instance.input.json | 3 + .../backend_data/verda_instance.types.json | 3 + .../backend_data/verda_instance.values.json | 4 + .../fixtures/parsing/db/aws_creds.input.json | 6 - .../fixtures/parsing/db/aws_creds.values.json | 5 - .../backend_config/amddevcloud.json | 7 + .../serialization/backend_config/aws.json | 29 + .../serialization/backend_config/azure.json | 21 + .../backend_config/cloudrift.json | 6 + .../serialization/backend_config/crusoe.json | 7 + .../backend_config/datacrunch.json | 6 + .../backend_config/digitalocean.json | 8 + .../serialization/backend_config/gcp.json | 25 + .../backend_config/hotaisle.json | 7 + .../backend_config/jarvislabs.json | 6 + .../backend_config/kubernetes.json | 22 + .../serialization/backend_config/lambda.json | 7 + .../serialization/backend_config/nebius.json | 15 + .../serialization/backend_config/oci.json | 10 + .../serialization/backend_config/runpod.json | 8 + .../serialization/backend_config/slurm.json | 27 + .../serialization/backend_config/vastai.json | 8 + .../serialization/backend_config/verda.json | 6 + .../serialization/backend_config/vultr.json | 7 + .../backend_creds/aws.access_key.json | 5 + .../backend_creds/aws.default.json | 3 + .../backend_creds/azure.client.json | 6 + .../backend_creds/azure.default.json | 3 + .../backend_creds/cloudrift.json | 4 + .../serialization/backend_creds/crusoe.json | 5 + .../backend_creds/digitalocean.json | 4 + .../backend_creds/gcp.default.json | 3 + .../backend_creds/gcp.service_account.json | 5 + .../serialization/backend_creds/hotaisle.json | 4 + .../backend_creds/jarvislabs.json | 4 + .../serialization/backend_creds/lambda.json | 4 + .../serialization/backend_creds/nebius.json | 8 + .../backend_creds/oci.client.json | 10 + .../backend_creds/oci.default.json | 5 + .../serialization/backend_creds/runpod.json | 4 + .../serialization/backend_creds/vastai.json | 4 + .../serialization/backend_creds/verda.json | 5 + .../serialization/backend_creds/vultr.json | 4 + .../backend_data/aws_gateway.json | 6 + .../backend_data/aws_instance.json | 3 + .../backend_data/aws_volume.json | 4 + .../backend_data/crusoe_instance.json | 3 + .../backend_data/crusoe_placement_group.json | 4 + .../serialization/backend_data/gcp_offer.json | 3 + .../backend_data/gcp_volume_disk.json | 4 + .../backend_data/hotaisle_instance.json | 3 + .../backend_data/hotaisle_offer.json | 12 + .../backend_data/jarvislabs_instance.json | 6 + .../backend_data/kubernetes.json | 5 + .../backend_data/nebius_cluster.json | 4 + .../backend_data/nebius_instance.json | 3 + .../backend_data/nebius_placement_group.json | 6 + .../backend_data/runpod_offer.json | 8 + .../backend_data/vastai_offer.json | 3 + .../backend_data/verda_instance.json | 6 + .../fixtures/serialization/db/aws_creds.json | 5 - .../_internal/pydantic_compat/test_parsing.py | 77 ++- .../pydantic_compat/test_serialization.py | 15 +- 228 files changed, 2152 insertions(+), 31 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/backend_factories.py create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.input.json rename src/tests/_internal/pydantic_compat/fixtures/parsing/{db/aws_creds.types.json => backend_creds/aws.access_key.types.json} (100%) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.input.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.values.json delete mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json delete mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/amddevcloud.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/aws.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/azure.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/cloudrift.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/crusoe.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/datacrunch.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/digitalocean.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/gcp.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/hotaisle.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/jarvislabs.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/kubernetes.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/lambda.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/nebius.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/oci.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/runpod.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/slurm.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vastai.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/verda.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vultr.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.access_key.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.default.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.client.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.default.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/cloudrift.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/crusoe.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/digitalocean.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.default.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.service_account.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/hotaisle.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/jarvislabs.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/lambda.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/nebius.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.client.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.default.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/runpod.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vastai.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/verda.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vultr.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_gateway.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_instance.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_volume.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_instance.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_placement_group.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_offer.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_volume_disk.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_instance.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_offer.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/jarvislabs_instance.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/kubernetes.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_cluster.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_instance.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_placement_group.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/runpod_offer.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/vastai_offer.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/verda_instance.json delete mode 100644 src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json diff --git a/src/tests/_internal/pydantic_compat/backend_factories.py b/src/tests/_internal/pydantic_compat/backend_factories.py new file mode 100644 index 0000000000..7cd16ef9a2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/backend_factories.py @@ -0,0 +1,651 @@ +""" +Deterministic instances for the backend layer of the DB surface. + +Kept out of `factories.py` because the backend models are one repetitive family — 19 backend +types plus 18 `*BackendData` blobs — and interleaving them with the core models would bury both. + +Three `Text` columns are involved, one registry each: + +- `BackendModel.config` holds `XStoredConfig(...).json()`, read back as the splice + `XConfig(**json.loads(config), creds=XCreds.parse_raw(auth))`. The registries below keep the + two halves apart the way the columns do, so a fixture matches one column's bytes exactly. +- `BackendModel.auth` holds `XCreds(...).json()`. +- `InstanceModel.backend_data` and `VolumeModel.backend_data` hold a `*BackendData` blob. +""" + +from typing import Any, Callable + +from dstack._internal.core.backends.aws.compute import ( + AWSGatewayBackendData, + AWSInstanceBackendData, + AWSVolumeBackendData, +) +from dstack._internal.core.backends.aws.models import ( + AWSAccessKeyCreds, + AWSConfig, + AWSCreds, + AWSDefaultCreds, + AWSOSImage, + AWSOSImageConfig, + AWSStoredConfig, +) +from dstack._internal.core.backends.azure.models import ( + AzureClientCreds, + AzureConfig, + AzureCreds, + AzureDefaultCreds, + AzureStoredConfig, +) +from dstack._internal.core.backends.cloudrift.models import ( + CloudRiftConfig, + CloudRiftCreds, + CloudRiftStoredConfig, +) +from dstack._internal.core.backends.crusoe.compute import ( + CrusoeInstanceBackendData, + CrusoePlacementGroupBackendData, +) +from dstack._internal.core.backends.crusoe.models import ( + CrusoeConfig, + CrusoeCreds, + CrusoeStoredConfig, +) +from dstack._internal.core.backends.digitalocean_base.models import ( + BaseDigitalOceanConfig, + BaseDigitalOceanCreds, + BaseDigitalOceanStoredConfig, +) +from dstack._internal.core.backends.gcp.compute import ( + GCPOfferBackendData, + GCPVolumeDiskBackendData, +) +from dstack._internal.core.backends.gcp.models import ( + GCPConfig, + GCPCreds, + GCPDefaultCreds, + GCPServiceAccountCreds, + GCPStoredConfig, +) +from dstack._internal.core.backends.hotaisle.compute import ( + HotAisleInstanceBackendData, + HotAisleOfferBackendData, +) +from dstack._internal.core.backends.hotaisle.models import ( + HotAisleConfig, + HotAisleCreds, + HotAisleStoredConfig, +) +from dstack._internal.core.backends.jarvislabs.compute import JarvisLabsInstanceBackendData +from dstack._internal.core.backends.jarvislabs.models import ( + JarvisLabsConfig, + JarvisLabsCreds, + JarvisLabsStoredConfig, +) +from dstack._internal.core.backends.kubernetes.compute import KubernetesBackendData +from dstack._internal.core.backends.kubernetes.models import ( + KubeconfigConfig, + KubernetesConfig, + KubernetesContextConfig, + KubernetesProxyJumpConfig, + KubernetesStoredConfig, +) +from dstack._internal.core.backends.lambdalabs.models import ( + LambdaConfig, + LambdaCreds, + LambdaStoredConfig, +) +from dstack._internal.core.backends.nebius.compute import ( + NebiusClusterBackendData, + NebiusInstanceBackendData, + NebiusPlacementGroupBackendData, +) +from dstack._internal.core.backends.nebius.models import ( + NebiusConfig, + NebiusCreds, + NebiusStoredConfig, +) +from dstack._internal.core.backends.oci.models import ( + OCIClientCreds, + OCIConfig, + OCICreds, + OCIDefaultCreds, + OCIStoredConfig, +) +from dstack._internal.core.backends.runpod.compute import RunpodOfferBackendData +from dstack._internal.core.backends.runpod.models import ( + RunpodConfig, + RunpodCreds, + RunpodStoredConfig, +) +from dstack._internal.core.backends.slurm.models import ( + SlurmClusterConfigWithCreds, + SlurmConfig, + SlurmGPUPartitionConfig, + SlurmPrivateKeyConfig, + SlurmStoredConfig, +) +from dstack._internal.core.backends.vastai.compute import VastAIOfferBackendData +from dstack._internal.core.backends.vastai.models import ( + VastAIConfig, + VastAICreds, + VastAIStoredConfig, +) +from dstack._internal.core.backends.verda.compute import VerdaInstanceBackendData +from dstack._internal.core.backends.verda.models import ( + VerdaConfig, + VerdaCreds, + VerdaStoredConfig, +) +from dstack._internal.core.backends.vultr.models import ( + VultrConfig, + VultrCreds, + VultrStoredConfig, +) + +# A PEM-shaped placeholder. Several backends store a private key inline in the `auth` column, and +# a one-word stand-in would not exercise the newlines that make those values awkward to encode. +_PRIVATE_KEY = ( + "-----BEGIN PRIVATE KEY-----\nMIIBVgIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----\n" +) +_SERVICE_ACCOUNT_JSON = ( + '{"type": "service_account", "project_id": "dstack-prod",' + ' "private_key_id": "0123456789abcdef0123456789abcdef01234567"}' +) + + +# --- `BackendModel.config`: `XStoredConfig` ------------------------------------------------- +# What the write path dumps into the column. The read side reconstructs an `XConfig` from these +# bytes plus the `auth` column, so it needs the class rather than an instance — see +# `BACKEND_CONFIG_MODELS` at the bottom. + + +def aws_stored_config() -> AWSStoredConfig: + return AWSStoredConfig( + regions=["us-east-1", "us-west-2"], + vpc_name=None, + vpc_ids={"us-east-1": "vpc-0a1b2c3d4e5f67890"}, + default_vpcs=False, + public_ips=True, + iam_instance_profile="dstack-instance-profile", + tags={"env": "prod", "team": "ml"}, + os_images=AWSOSImageConfig( + nvidia=AWSOSImage( + name="dstack-nvidia-ubuntu-22.04", owner="123456789012", user="ubuntu" + ), + ), + experimental_instance_types=["p5.48xlarge"], + ) + + +def azure_stored_config() -> AzureStoredConfig: + return AzureStoredConfig( + tenant_id="11111111-2222-3333-4444-555555555555", + subscription_id="66666666-7777-8888-9999-000000000000", + resource_group="dstack-rg", + regions=["eastus", "westeurope"], + vpc_ids={"eastus": "dstack-vnet"}, + subnet_ids={"eastus": "dstack-subnet"}, + public_ips=True, + vm_managed_identity="dstack-identity", + tags={"env": "prod"}, + ) + + +def amddevcloud_stored_config() -> BaseDigitalOceanStoredConfig: + return BaseDigitalOceanStoredConfig( + type="amddevcloud", project_name="dstack", regions=["tor1"] + ) + + +def cloudrift_stored_config() -> CloudRiftStoredConfig: + return CloudRiftStoredConfig(regions=["us-east-nc-nrp-1"]) + + +def crusoe_stored_config() -> CrusoeStoredConfig: + return CrusoeStoredConfig( + project_id="a1b2c3d4-5678-4abc-9def-000000000000", regions=["us-east1"] + ) + + +def datacrunch_stored_config() -> VerdaStoredConfig: + """`VerdaStoredConfig.type` is a two-value Literal, so both tags get a fixture.""" + return VerdaStoredConfig(type="datacrunch", regions=["FIN-01"]) + + +def digitalocean_stored_config() -> BaseDigitalOceanStoredConfig: + return BaseDigitalOceanStoredConfig( + type="digitalocean", project_name="dstack", regions=["nyc3", "ams3"] + ) + + +def gcp_stored_config() -> GCPStoredConfig: + return GCPStoredConfig( + project_id="dstack-prod", + regions=["us-central1", "europe-west4"], + vpc_name="dstack-vpc", + extra_vpcs=["dstack-extra-vpc"], + roce_vpcs=["dstack-roce-vpc"], + vpc_project_id="dstack-network", + public_ips=True, + nat_check=False, + vm_service_account="dstack@dstack-prod.iam.gserviceaccount.com", + tags={"env": "prod"}, + preview_features=["g4"], + ) + + +def hotaisle_stored_config() -> HotAisleStoredConfig: + return HotAisleStoredConfig(team_handle="dstack-team", regions=["us-michigan-1"]) + + +def jarvislabs_stored_config() -> JarvisLabsStoredConfig: + return JarvisLabsStoredConfig(regions=["in-north-1"]) + + +def kubernetes_stored_config() -> KubernetesStoredConfig: + """ + One of the two backends whose creds live in the `config` column instead of `auth`, which the + configurator writes as `auth=""`. So the kubeconfig is here rather than in a creds fixture. + """ + return KubernetesStoredConfig( + contexts=[ + KubernetesContextConfig( + name="prod-cluster", + proxy_jump=KubernetesProxyJumpConfig(hostname="10.0.0.1", port=22), + ), + "staging-cluster", + ], + proxy_jump=KubernetesProxyJumpConfig(hostname="bastion.internal", port=2222), + namespace="dstack", + kubeconfig=KubeconfigConfig(filename="", data="apiVersion: v1\nkind: Config\n"), + ) + + +def lambdalabs_stored_config() -> LambdaStoredConfig: + return LambdaStoredConfig(regions=["us-east-1", "us-west-2"]) + + +def nebius_stored_config() -> NebiusStoredConfig: + return NebiusStoredConfig( + projects=["project-e00dstack"], + regions=["eu-north1"], + fabrics=["fabric-3"], + tags={"env": "prod"}, + ) + + +def oci_stored_config() -> OCIStoredConfig: + return OCIStoredConfig( + regions=["eu-frankfurt-1"], + compartment_id="ocid1.compartment.oc1..aaaaaaaadstack", + subnet_ids_per_region={"eu-frankfurt-1": "ocid1.subnet.oc1.eu-frankfurt-1.aaaaaaaadstack"}, + ) + + +def runpod_stored_config() -> RunpodStoredConfig: + return RunpodStoredConfig(regions=["US-KS-2", "EU-RO-1"], community_cloud=True) + + +def slurm_stored_config() -> SlurmStoredConfig: + """The other creds-in-`config` backend: the SSH private key is part of the cluster config.""" + return SlurmStoredConfig( + clusters=[ + SlurmClusterConfigWithCreds( + name="hpc-1", + gpu_partitions=[ + SlurmGPUPartitionConfig(gpu="H100", partitions=["gpu", "gpu-long"]) + ], + cpu_partitions=["cpu"], + hostname="login.hpc.example.com", + port=22, + user="dstack", + private_key=SlurmPrivateKeyConfig(path="", content=_PRIVATE_KEY), + ) + ] + ) + + +def vastai_stored_config() -> VastAIStoredConfig: + return VastAIStoredConfig(regions=["Poland", "Sweden"], community_cloud=False) + + +def verda_stored_config() -> VerdaStoredConfig: + return VerdaStoredConfig(type="verda", regions=["ICE-01"]) + + +def vultr_stored_config() -> VultrStoredConfig: + return VultrStoredConfig(regions=["ewr", "ams"]) + + +# --- `BackendModel.auth`: `XCreds` ------------------------------------------------------------ +# A fixture PER UNION ARM for the four custom-root unions. Building the union once only ever +# exercises the arm that happens to match first, and which arm matches is exactly what changes +# when v2 resolves unions in smart mode instead of left to right. +# +# `kubernetes` and `slurm` are absent on purpose: their configurators write `auth=""` and keep +# creds in the `config` column, so there is no blob here to be compatible with. + + +def aws_creds_access_key() -> AWSCreds: + return AWSCreds.parse_obj( + {"type": "access_key", "access_key": "AKIAIOSFODNN7EXAMPLE", "secret_key": "wJalrXUtnFEMI"} + ) + + +def aws_creds_default() -> AWSCreds: + return AWSCreds.parse_obj({"type": "default"}) + + +def azure_creds_client() -> AzureCreds: + return AzureCreds.parse_obj( + { + "type": "client", + "client_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "client_secret": "azure-client-secret", + "tenant_id": "11111111-2222-3333-4444-555555555555", + } + ) + + +def azure_creds_default() -> AzureCreds: + return AzureCreds.parse_obj({"type": "default"}) + + +def gcp_creds_service_account() -> GCPCreds: + return GCPCreds.parse_obj( + {"type": "service_account", "filename": "", "data": _SERVICE_ACCOUNT_JSON} + ) + + +def gcp_creds_default() -> GCPCreds: + return GCPCreds.parse_obj({"type": "default"}) + + +def oci_creds_client() -> OCICreds: + return OCICreds.parse_obj( + { + "type": "client", + "user": "ocid1.user.oc1..aaaaaaaadstack", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaadstack", + "key_content": _PRIVATE_KEY, + "fingerprint": "12:34:56:78:9a:bc:de:f0:12:34:56:78:9a:bc:de:f0", + "region": "eu-frankfurt-1", + } + ) + + +def oci_creds_default() -> OCICreds: + return OCICreds.parse_obj({"type": "default"}) + + +def cloudrift_creds() -> CloudRiftCreds: + return CloudRiftCreds(api_key="rift-api-key") + + +def crusoe_creds() -> CrusoeCreds: + return CrusoeCreds(access_key="crusoe-access-key", secret_key="crusoe-secret-key") + + +def digitalocean_creds() -> BaseDigitalOceanCreds: + """Shared by the `digitalocean` and `amddevcloud` backends, which store the same shape.""" + return BaseDigitalOceanCreds(api_key="dop_v1_digitalocean") + + +def hotaisle_creds() -> HotAisleCreds: + return HotAisleCreds(api_key="hotaisle-api-key") + + +def jarvislabs_creds() -> JarvisLabsCreds: + return JarvisLabsCreds(api_key="jarvislabs-api-key") + + +def lambdalabs_creds() -> LambdaCreds: + return LambdaCreds(api_key="lambda-api-key") + + +def nebius_creds() -> NebiusCreds: + return NebiusCreds( + service_account_id="serviceaccount-e00dstack", + public_key_id="publickey-e00dstack", + private_key_content=_PRIVATE_KEY, + ) + + +def runpod_creds() -> RunpodCreds: + return RunpodCreds(api_key="runpod-api-key") + + +def vastai_creds() -> VastAICreds: + return VastAICreds(api_key="vastai-api-key") + + +def verda_creds() -> VerdaCreds: + """Also covers `datacrunch`, which shares the creds model and differs only in config `type`.""" + return VerdaCreds(client_id="verda-client-id", client_secret="verda-secret") + + +def vultr_creds() -> VultrCreds: + return VultrCreds(api_key="vultr-api-key") + + +# --- `backend_data` columns: `*BackendData` --------------------------------------------------- +# `InstanceModel.backend_data` and `VolumeModel.backend_data`. These are the smallest models in +# the codebase and the least reviewed, which is why they get a fixture each rather than a +# representative sample: a wrong read here silently loses the handle to a live cloud resource. +# +# `NebiusOfferBackendData` is absent — its `fabrics: set[str]` cannot be dumped at all today. See +# `TestNebiusOfferBackendDataSetField` in the test modules. + + +def aws_gateway_backend_data() -> AWSGatewayBackendData: + return AWSGatewayBackendData( + lb_arn="arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/net/dstack/1a2b", + tg_arn="arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/dstack/3c4d", + listener_arn="arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/net/dstack/5e6f", + http_listener_arn=None, + ) + + +def aws_instance_backend_data() -> AWSInstanceBackendData: + return AWSInstanceBackendData(eip_allocation_id="eipalloc-0a1b2c3d4e5f67890") + + +def aws_volume_backend_data() -> AWSVolumeBackendData: + return AWSVolumeBackendData(volume_type="gp3", iops=3000) + + +def crusoe_instance_backend_data() -> CrusoeInstanceBackendData: + return CrusoeInstanceBackendData(data_disk_id="a1b2c3d4-5678-4abc-9def-000000000001") + + +def crusoe_placement_group_backend_data() -> CrusoePlacementGroupBackendData: + return CrusoePlacementGroupBackendData( + ib_partition_id="a1b2c3d4-5678-4abc-9def-000000000002", + ib_network_id="a1b2c3d4-5678-4abc-9def-000000000003", + ) + + +def gcp_offer_backend_data() -> GCPOfferBackendData: + return GCPOfferBackendData(is_dws_calendar_mode=True) + + +def gcp_volume_disk_backend_data() -> GCPVolumeDiskBackendData: + return GCPVolumeDiskBackendData(disk_type="pd-balanced") + + +def hotaisle_instance_backend_data() -> HotAisleInstanceBackendData: + return HotAisleInstanceBackendData(ip_address="203.0.113.10") + + +def hotaisle_offer_backend_data() -> HotAisleOfferBackendData: + """`vm_specs: Mapping[str, Any]` — an untyped passthrough, so the values ride through as-is.""" + return HotAisleOfferBackendData( + vm_specs={"cpu": {"count": 26}, "gpu": {"count": 1, "model": "MI300X"}, "ram": 224} + ) + + +def jarvislabs_instance_backend_data() -> JarvisLabsInstanceBackendData: + return JarvisLabsInstanceBackendData(ssh_key_ids=["1234", "5678"]) + + +def kubernetes_backend_data() -> KubernetesBackendData: + return KubernetesBackendData( + jump_pod_name="dstack-jump-pod", + jump_pod_service_name="dstack-jump-pod-service", + user_ssh_public_key="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD dstack@localhost", + ) + + +def nebius_cluster_backend_data() -> NebiusClusterBackendData: + return NebiusClusterBackendData(id="computecluster-e00dstack", fabric="fabric-3") + + +def nebius_instance_backend_data() -> NebiusInstanceBackendData: + return NebiusInstanceBackendData(boot_disk_id="computedisk-e00dstack") + + +def nebius_placement_group_backend_data() -> NebiusPlacementGroupBackendData: + return NebiusPlacementGroupBackendData(cluster=nebius_cluster_backend_data()) + + +def runpod_offer_backend_data() -> RunpodOfferBackendData: + return RunpodOfferBackendData(pod_counts=[1, 2, 4, 8]) + + +def vastai_offer_backend_data() -> VastAIOfferBackendData: + return VastAIOfferBackendData(min_bid=0.1234) + + +def verda_instance_backend_data() -> VerdaInstanceBackendData: + return VerdaInstanceBackendData( + startup_script_id="a1b2c3d4-5678-4abc-9def-000000000004", + ssh_key_ids=["a1b2c3d4-5678-4abc-9def-000000000005"], + ) + + +# --- Registries ------------------------------------------------------------------------------- +# Keys are the fixture names. A backend appears under its `BackendType` value rather than its +# module name, so `lambdalabs` reads as `lambda` and `digitalocean_base` splits into the two +# backends that share it. + +BACKEND_STORED_CONFIGS: dict[str, Callable[[], Any]] = { + "amddevcloud": amddevcloud_stored_config, + "aws": aws_stored_config, + "azure": azure_stored_config, + "cloudrift": cloudrift_stored_config, + "crusoe": crusoe_stored_config, + "datacrunch": datacrunch_stored_config, + "digitalocean": digitalocean_stored_config, + "gcp": gcp_stored_config, + "hotaisle": hotaisle_stored_config, + "jarvislabs": jarvislabs_stored_config, + "kubernetes": kubernetes_stored_config, + "lambda": lambdalabs_stored_config, + "nebius": nebius_stored_config, + "oci": oci_stored_config, + "runpod": runpod_stored_config, + "slurm": slurm_stored_config, + "vastai": vastai_stored_config, + "verda": verda_stored_config, + "vultr": vultr_stored_config, +} + +BACKEND_CREDS: dict[str, Callable[[], Any]] = { + "aws.access_key": aws_creds_access_key, + "aws.default": aws_creds_default, + "azure.client": azure_creds_client, + "azure.default": azure_creds_default, + "cloudrift": cloudrift_creds, + "crusoe": crusoe_creds, + "digitalocean": digitalocean_creds, + "gcp.default": gcp_creds_default, + "gcp.service_account": gcp_creds_service_account, + "hotaisle": hotaisle_creds, + "jarvislabs": jarvislabs_creds, + "lambda": lambdalabs_creds, + "nebius": nebius_creds, + "oci.client": oci_creds_client, + "oci.default": oci_creds_default, + "runpod": runpod_creds, + "vastai": vastai_creds, + "verda": verda_creds, + "vultr": vultr_creds, +} + +BACKEND_DATA: dict[str, Callable[[], Any]] = { + "aws_gateway": aws_gateway_backend_data, + "aws_instance": aws_instance_backend_data, + "aws_volume": aws_volume_backend_data, + "crusoe_instance": crusoe_instance_backend_data, + "crusoe_placement_group": crusoe_placement_group_backend_data, + "gcp_offer": gcp_offer_backend_data, + "gcp_volume_disk": gcp_volume_disk_backend_data, + "hotaisle_instance": hotaisle_instance_backend_data, + "hotaisle_offer": hotaisle_offer_backend_data, + "jarvislabs_instance": jarvislabs_instance_backend_data, + "kubernetes": kubernetes_backend_data, + "nebius_cluster": nebius_cluster_backend_data, + "nebius_instance": nebius_instance_backend_data, + "nebius_placement_group": nebius_placement_group_backend_data, + "runpod_offer": runpod_offer_backend_data, + "vastai_offer": vastai_offer_backend_data, + "verda_instance": verda_instance_backend_data, +} + +# The model each `backend_config` fixture is read back as. Parsing needs the class, and taking it +# off the factory would tie the read to whichever arm the factory picked. +BACKEND_CONFIG_MODELS: dict[str, Any] = { + "amddevcloud": BaseDigitalOceanConfig, + "aws": AWSConfig, + "azure": AzureConfig, + "cloudrift": CloudRiftConfig, + "crusoe": CrusoeConfig, + "datacrunch": VerdaConfig, + "digitalocean": BaseDigitalOceanConfig, + "gcp": GCPConfig, + "hotaisle": HotAisleConfig, + "jarvislabs": JarvisLabsConfig, + "kubernetes": KubernetesConfig, + "lambda": LambdaConfig, + "nebius": NebiusConfig, + "oci": OCIConfig, + "runpod": RunpodConfig, + "slurm": SlurmConfig, + "vastai": VastAIConfig, + "verda": VerdaConfig, + "vultr": VultrConfig, +} + +BACKEND_CREDS_MODELS: dict[str, Any] = { + "aws.access_key": AWSCreds, + "aws.default": AWSCreds, + "azure.client": AzureCreds, + "azure.default": AzureCreds, + "cloudrift": CloudRiftCreds, + "crusoe": CrusoeCreds, + "digitalocean": BaseDigitalOceanCreds, + "gcp.default": GCPCreds, + "gcp.service_account": GCPCreds, + "hotaisle": HotAisleCreds, + "jarvislabs": JarvisLabsCreds, + "lambda": LambdaCreds, + "nebius": NebiusCreds, + "oci.client": OCICreds, + "oci.default": OCICreds, + "runpod": RunpodCreds, + "vastai": VastAICreds, + "verda": VerdaCreds, + "vultr": VultrCreds, +} + +# Named so the arm tests can assert on the resolved arm rather than only on the dump. +CREDS_ARMS: dict[str, Any] = { + "aws.access_key": AWSAccessKeyCreds, + "aws.default": AWSDefaultCreds, + "azure.client": AzureClientCreds, + "azure.default": AzureDefaultCreds, + "gcp.default": GCPDefaultCreds, + "gcp.service_account": GCPServiceAccountCreds, + "oci.client": OCIClientCreds, + "oci.default": OCIDefaultCreds, +} diff --git a/src/tests/_internal/pydantic_compat/compare.py b/src/tests/_internal/pydantic_compat/compare.py index 988b8fcc5d..f8b46ce5ce 100644 --- a/src/tests/_internal/pydantic_compat/compare.py +++ b/src/tests/_internal/pydantic_compat/compare.py @@ -90,7 +90,7 @@ def type_map(value: Any, path: str = "", out: Union[dict, None] = None) -> dict: """ out = {} if out is None else out if isinstance(value, BaseModel): - out[path or "/"] = _class_name(value) + out[path or "/"] = class_name(value) for name, attr in value.__dict__.items(): type_map(attr, f"{path}/{name}", out) elif isinstance(value, dict): @@ -100,11 +100,11 @@ def type_map(value: Any, path: str = "", out: Union[dict, None] = None) -> dict: for i, attr in enumerate(value): type_map(attr, f"{path}/{i}", out) elif type(value) not in _JSON_NATIVE: - out[path] = _class_name(value) + out[path] = class_name(value) return out -def _class_name(value: Any) -> str: +def class_name(value: Any) -> str: """ The model's name with pydantic-duality's generated suffix removed. @@ -116,7 +116,7 @@ def _class_name(value: Any) -> str: and those are plain `BaseModel`, so stripping there would report `RegisterService` for a class called `RegisterServiceRequest`. """ - cls = type(value) + cls = value if isinstance(value, type) else type(value) name = cls.__name__ if hasattr(cls, "__response__"): for suffix in ("Request", "Response"): diff --git a/src/tests/_internal/pydantic_compat/factories.py b/src/tests/_internal/pydantic_compat/factories.py index c7c03fbba7..2e3c76f8e6 100644 --- a/src/tests/_internal/pydantic_compat/factories.py +++ b/src/tests/_internal/pydantic_compat/factories.py @@ -5,7 +5,6 @@ import uuid from datetime import datetime, timezone -from dstack._internal.core.backends.aws.models import AWSCreds from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import ( ApplyAction, @@ -169,11 +168,6 @@ # Written to a `Text` column with `.json()`. -def aws_creds() -> AWSCreds: - """A custom-root discriminated union, read from `BackendModel.auth`.""" - return AWSCreds.parse_obj({"type": "access_key", "access_key": "AK", "secret_key": "SK"}) - - def compute_group_provisioning_data() -> ComputeGroupProvisioningData: return get_compute_group_provisioning_data() diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.input.json new file mode 100644 index 0000000000..1d7145182a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.input.json @@ -0,0 +1,5 @@ +{ + "type": "amddevcloud", + "project_name": "dstack-amd", + "creds": {"type": "api_key", "api_key": "dop_v1_amd9c2e8b1d47065f3a9c3e5f7b2d8a4c6e"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.types.json new file mode 100644 index 0000000000..ca22cf2c7b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.types.json @@ -0,0 +1,4 @@ +{ + "/": "BaseDigitalOceanConfig", + "/creds": "BaseDigitalOceanAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.values.json new file mode 100644 index 0000000000..6363a2099f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/amddevcloud.values.json @@ -0,0 +1,9 @@ +{ + "creds": { + "api_key": "dop_v1_amd9c2e8b1d47065f3a9c3e5f7b2d8a4c6e", + "type": "api_key" + }, + "project_name": "dstack-amd", + "regions": null, + "type": "amddevcloud" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.input.json new file mode 100644 index 0000000000..dc604e43ad --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.input.json @@ -0,0 +1,14 @@ +{ + "type": "aws", + "regions": ["us-east-1", "us-west-2", "eu-central-1"], + "vpc_ids": {"us-east-1": "vpc-0a1b2c3d4e5f67890", "us-west-2": "vpc-0f9e8d7c6b5a43210"}, + "public_ips": false, + "iam_instance_profile": "dstack-instance-profile", + "tags": {"env": "prod", "cost-center": "ml-research"}, + "experimental_instance_types": ["p5.48xlarge", "p4d.24xlarge"], + "creds": { + "type": "access_key", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.types.json new file mode 100644 index 0000000000..53f6db372b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.types.json @@ -0,0 +1,4 @@ +{ + "/": "AWSConfig", + "/creds": "AWSAccessKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.values.json new file mode 100644 index 0000000000..3e9d5f6a9a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/aws.values.json @@ -0,0 +1,30 @@ +{ + "creds": { + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "type": "access_key" + }, + "default_vpcs": null, + "experimental_instance_types": [ + "p5.48xlarge", + "p4d.24xlarge" + ], + "iam_instance_profile": "dstack-instance-profile", + "os_images": null, + "public_ips": false, + "regions": [ + "us-east-1", + "us-west-2", + "eu-central-1" + ], + "tags": { + "cost-center": "ml-research", + "env": "prod" + }, + "type": "aws", + "vpc_ids": { + "us-east-1": "vpc-0a1b2c3d4e5f67890", + "us-west-2": "vpc-0f9e8d7c6b5a43210" + }, + "vpc_name": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.input.json new file mode 100644 index 0000000000..d9dabf8868 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.input.json @@ -0,0 +1,13 @@ +{ + "type": "azure", + "tenant_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "subscription_id": "9c2e8b1d-4706-4a9c-8e5f-7b2d8a4c6e1f", + "regions": ["eastus", "westeurope"], + "public_ips": true, + "creds": { + "type": "client", + "client_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "client_secret": "Abc8Q~fakeAzureClientSecretValue_0123", + "tenant_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.types.json new file mode 100644 index 0000000000..ab8bf5fc5b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.types.json @@ -0,0 +1,4 @@ +{ + "/": "AzureConfig", + "/creds": "AzureClientCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.values.json new file mode 100644 index 0000000000..d40d072e5c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/azure.values.json @@ -0,0 +1,21 @@ +{ + "creds": { + "client_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "client_secret": "Abc8Q~fakeAzureClientSecretValue_0123", + "tenant_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "type": "client" + }, + "public_ips": true, + "regions": [ + "eastus", + "westeurope" + ], + "resource_group": "", + "subnet_ids": null, + "subscription_id": "9c2e8b1d-4706-4a9c-8e5f-7b2d8a4c6e1f", + "tags": null, + "tenant_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "type": "azure", + "vm_managed_identity": null, + "vpc_ids": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.input.json new file mode 100644 index 0000000000..78b27c8baf --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.input.json @@ -0,0 +1,5 @@ +{ + "type": "cloudrift", + "regions": ["us-east-nc-nrp-1"], + "creds": {"type": "api_key", "api_key": "rift_5f3a9c2e8b1d4706a9c3e5f7b2d8a4c6"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.types.json new file mode 100644 index 0000000000..bc2faf340f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.types.json @@ -0,0 +1,4 @@ +{ + "/": "CloudRiftConfig", + "/creds": "CloudRiftAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.values.json new file mode 100644 index 0000000000..2be969672b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/cloudrift.values.json @@ -0,0 +1,10 @@ +{ + "creds": { + "api_key": "rift_5f3a9c2e8b1d4706a9c3e5f7b2d8a4c6", + "type": "api_key" + }, + "regions": [ + "us-east-nc-nrp-1" + ], + "type": "cloudrift" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.input.json new file mode 100644 index 0000000000..b1774585f5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.input.json @@ -0,0 +1,9 @@ +{ + "type": "crusoe", + "project_id": "7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "creds": { + "type": "access_key", + "access_key": "CRUSOE7K3JVQZHX2MFAKE", + "secret_key": "wJ4lrXUtnFEMI7K3MDENGbPxRfiCYcrusoeKEY" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.types.json new file mode 100644 index 0000000000..3fa50c4a6a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.types.json @@ -0,0 +1,4 @@ +{ + "/": "CrusoeConfig", + "/creds": "CrusoeAccessKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.values.json new file mode 100644 index 0000000000..2ea674f54d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/crusoe.values.json @@ -0,0 +1,10 @@ +{ + "creds": { + "access_key": "CRUSOE7K3JVQZHX2MFAKE", + "secret_key": "wJ4lrXUtnFEMI7K3MDENGbPxRfiCYcrusoeKEY", + "type": "access_key" + }, + "project_id": "7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "regions": null, + "type": "crusoe" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.input.json new file mode 100644 index 0000000000..23db3fdc5a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.input.json @@ -0,0 +1,9 @@ +{ + "type": "datacrunch", + "regions": ["FIN-01", "ICE-01"], + "creds": { + "type": "api_key", + "client_id": "dc-7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "client_secret": "dc_0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.types.json new file mode 100644 index 0000000000..beb4677290 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.types.json @@ -0,0 +1,4 @@ +{ + "/": "VerdaConfig", + "/creds": "VerdaAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.values.json new file mode 100644 index 0000000000..039a728e23 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/datacrunch.values.json @@ -0,0 +1,12 @@ +{ + "creds": { + "client_id": "dc-7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "client_secret": "dc_0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d", + "type": "api_key" + }, + "regions": [ + "FIN-01", + "ICE-01" + ], + "type": "datacrunch" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.input.json new file mode 100644 index 0000000000..9a6a91f3bc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.input.json @@ -0,0 +1,6 @@ +{ + "type": "digitalocean", + "project_name": "dstack", + "regions": ["nyc3", "ams3", "sfo3"], + "creds": {"type": "api_key", "api_key": "dop_v1_9c2e8b1d47065f3a9c3e5f7b2d8a4c6e"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.types.json new file mode 100644 index 0000000000..ca22cf2c7b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.types.json @@ -0,0 +1,4 @@ +{ + "/": "BaseDigitalOceanConfig", + "/creds": "BaseDigitalOceanAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.values.json new file mode 100644 index 0000000000..dae704e136 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/digitalocean.values.json @@ -0,0 +1,13 @@ +{ + "creds": { + "api_key": "dop_v1_9c2e8b1d47065f3a9c3e5f7b2d8a4c6e", + "type": "api_key" + }, + "project_name": "dstack", + "regions": [ + "nyc3", + "ams3", + "sfo3" + ], + "type": "digitalocean" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.input.json new file mode 100644 index 0000000000..585a5857f5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.input.json @@ -0,0 +1,16 @@ +{ + "type": "gcp", + "project_id": "dstack-prod", + "regions": ["us-central1", "europe-west4", "asia-northeast1"], + "vpc_name": "dstack-vpc", + "extra_vpcs": ["dstack-extra-vpc"], + "public_ips": true, + "nat_check": true, + "vm_service_account": "dstack@dstack-prod.iam.gserviceaccount.com", + "tags": {"env": "prod"}, + "creds": { + "type": "service_account", + "filename": "", + "data": "{\"type\": \"service_account\", \"project_id\": \"dstack-prod\", \"private_key_id\": \"7b1e4f2a9c8d6e5f4a3b2c1d0e9f8a7b6c5d4e3f\"}" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.types.json new file mode 100644 index 0000000000..48183e7a1e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.types.json @@ -0,0 +1,4 @@ +{ + "/": "GCPConfig", + "/creds": "GCPServiceAccountCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.values.json new file mode 100644 index 0000000000..f1f25602b8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/gcp.values.json @@ -0,0 +1,27 @@ +{ + "creds": { + "data": "{\"type\": \"service_account\", \"project_id\": \"dstack-prod\", \"private_key_id\": \"7b1e4f2a9c8d6e5f4a3b2c1d0e9f8a7b6c5d4e3f\"}", + "filename": "", + "type": "service_account" + }, + "extra_vpcs": [ + "dstack-extra-vpc" + ], + "nat_check": true, + "preview_features": null, + "project_id": "dstack-prod", + "public_ips": true, + "regions": [ + "us-central1", + "europe-west4", + "asia-northeast1" + ], + "roce_vpcs": null, + "tags": { + "env": "prod" + }, + "type": "gcp", + "vm_service_account": "dstack@dstack-prod.iam.gserviceaccount.com", + "vpc_name": "dstack-vpc", + "vpc_project_id": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.input.json new file mode 100644 index 0000000000..3337fa5ae6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.input.json @@ -0,0 +1,6 @@ +{ + "type": "hotaisle", + "team_handle": "dstack-team", + "regions": ["us-michigan-1"], + "creds": {"type": "api_key", "api_key": "ha_2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d2e"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.types.json new file mode 100644 index 0000000000..a02b4aef10 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.types.json @@ -0,0 +1,4 @@ +{ + "/": "HotAisleConfig", + "/creds": "HotAisleAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.values.json new file mode 100644 index 0000000000..5f08d6bb99 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/hotaisle.values.json @@ -0,0 +1,11 @@ +{ + "creds": { + "api_key": "ha_2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d2e", + "type": "api_key" + }, + "regions": [ + "us-michigan-1" + ], + "team_handle": "dstack-team", + "type": "hotaisle" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.input.json new file mode 100644 index 0000000000..38a44d5752 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.input.json @@ -0,0 +1,4 @@ +{ + "type": "jarvislabs", + "creds": {"type": "api_key", "api_key": "jl_7b2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.types.json new file mode 100644 index 0000000000..5f0fd2cba4 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.types.json @@ -0,0 +1,4 @@ +{ + "/": "JarvisLabsConfig", + "/creds": "JarvisLabsAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.values.json new file mode 100644 index 0000000000..4ff971fe98 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/jarvislabs.values.json @@ -0,0 +1,8 @@ +{ + "creds": { + "api_key": "jl_7b2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d", + "type": "api_key" + }, + "regions": null, + "type": "jarvislabs" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.input.json new file mode 100644 index 0000000000..dd3324c8ac --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.input.json @@ -0,0 +1,12 @@ +{ + "type": "kubernetes", + "contexts": [ + "staging-cluster", + {"name": "prod-cluster", "proxy_jump": {"hostname": "10.0.0.1", "port": 22}} + ], + "namespace": "dstack", + "kubeconfig": { + "filename": "", + "data": "apiVersion: v1\nkind: Config\nclusters:\n- name: prod-cluster\n" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.types.json new file mode 100644 index 0000000000..ca2dd2fe5f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.types.json @@ -0,0 +1,6 @@ +{ + "/": "KubernetesConfig", + "/contexts/1": "KubernetesContextConfig", + "/contexts/1/proxy_jump": "KubernetesProxyJumpConfig", + "/kubeconfig": "KubeconfigConfig" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.values.json new file mode 100644 index 0000000000..d2e6b9cb9a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/kubernetes.values.json @@ -0,0 +1,19 @@ +{ + "contexts": [ + "staging-cluster", + { + "name": "prod-cluster", + "proxy_jump": { + "hostname": "10.0.0.1", + "port": 22 + } + } + ], + "kubeconfig": { + "data": "apiVersion: v1\nkind: Config\nclusters:\n- name: prod-cluster\n", + "filename": "" + }, + "namespace": "dstack", + "proxy_jump": null, + "type": "kubernetes" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.input.json new file mode 100644 index 0000000000..f85e20c6ad --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.input.json @@ -0,0 +1,5 @@ +{ + "type": "lambda", + "regions": ["us-east-1", "us-west-2", "us-south-1"], + "creds": {"type": "api_key", "api_key": "secret_dstack_9c2e8b1d47065f3a9c3e5f7b2d8a4c6e"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.types.json new file mode 100644 index 0000000000..90ec9b55ac --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.types.json @@ -0,0 +1,4 @@ +{ + "/": "LambdaConfig", + "/creds": "LambdaAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.values.json new file mode 100644 index 0000000000..fac12f171c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/lambda.values.json @@ -0,0 +1,12 @@ +{ + "creds": { + "api_key": "secret_dstack_9c2e8b1d47065f3a9c3e5f7b2d8a4c6e", + "type": "api_key" + }, + "regions": [ + "us-east-1", + "us-west-2", + "us-south-1" + ], + "type": "lambda" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.input.json new file mode 100644 index 0000000000..7610dc6f62 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.input.json @@ -0,0 +1,12 @@ +{ + "type": "nebius", + "projects": ["project-e00fake9c2e8b1d4706", "project-e01fake5f3a9c3e5f7b"], + "regions": ["eu-north1", "eu-west1"], + "tags": {"env": "prod"}, + "creds": { + "type": "service_account", + "service_account_id": "serviceaccount-e00fake9c2e8b1d4706", + "public_key_id": "publickey-e00fake5f3a9c3e5f7b2d", + "private_key_content": "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIF9mYWtlTmViaXVzS2V5Q29udGVudEZvclRlc3Rz\n-----END PRIVATE KEY-----\n" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.types.json new file mode 100644 index 0000000000..19421acd1e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.types.json @@ -0,0 +1,4 @@ +{ + "/": "NebiusConfig", + "/creds": "NebiusServiceAccountCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.values.json new file mode 100644 index 0000000000..5cabe488cd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/nebius.values.json @@ -0,0 +1,23 @@ +{ + "creds": { + "filename": null, + "private_key_content": "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIF9mYWtlTmViaXVzS2V5Q29udGVudEZvclRlc3Rz\n-----END PRIVATE KEY-----\n", + "private_key_file": null, + "public_key_id": "publickey-e00fake5f3a9c3e5f7b2d", + "service_account_id": "serviceaccount-e00fake9c2e8b1d4706", + "type": "service_account" + }, + "fabrics": null, + "projects": [ + "project-e00fake9c2e8b1d4706", + "project-e01fake5f3a9c3e5f7b" + ], + "regions": [ + "eu-north1", + "eu-west1" + ], + "tags": { + "env": "prod" + }, + "type": "nebius" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.input.json new file mode 100644 index 0000000000..ae090338fc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.input.json @@ -0,0 +1,17 @@ +{ + "type": "oci", + "regions": ["eu-frankfurt-1", "uk-london-1"], + "compartment_id": "ocid1.compartment.oc1..aaaaaaaa7k3jvqzhx2mfakecompartment", + "subnet_ids_per_region": { + "eu-frankfurt-1": "ocid1.subnet.oc1.eu-frankfurt-1.aaaaaaaafakesubnetfra", + "uk-london-1": "ocid1.subnet.oc1.uk-london-1.aaaaaaaafakesubnetlhr" + }, + "creds": { + "type": "client", + "user": "ocid1.user.oc1..aaaaaaaa7k3jvqzhx2mfakeuserocid", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaaq4d9wmfaketenancyocid", + "key_content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC\n-----END PRIVATE KEY-----\n", + "fingerprint": "a1:b2:c3:d4:e5:f6:07:18:29:3a:4b:5c:6d:7e:8f:90", + "region": "eu-frankfurt-1" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.types.json new file mode 100644 index 0000000000..effc2aef32 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.types.json @@ -0,0 +1,4 @@ +{ + "/": "OCIConfig", + "/creds": "OCIClientCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.values.json new file mode 100644 index 0000000000..458537a3cf --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/oci.values.json @@ -0,0 +1,22 @@ +{ + "compartment_id": "ocid1.compartment.oc1..aaaaaaaa7k3jvqzhx2mfakecompartment", + "creds": { + "fingerprint": "a1:b2:c3:d4:e5:f6:07:18:29:3a:4b:5c:6d:7e:8f:90", + "key_content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC\n-----END PRIVATE KEY-----\n", + "key_file": null, + "pass_phrase": null, + "region": "eu-frankfurt-1", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaaq4d9wmfaketenancyocid", + "type": "client", + "user": "ocid1.user.oc1..aaaaaaaa7k3jvqzhx2mfakeuserocid" + }, + "regions": [ + "eu-frankfurt-1", + "uk-london-1" + ], + "subnet_ids_per_region": { + "eu-frankfurt-1": "ocid1.subnet.oc1.eu-frankfurt-1.aaaaaaaafakesubnetfra", + "uk-london-1": "ocid1.subnet.oc1.uk-london-1.aaaaaaaafakesubnetlhr" + }, + "type": "oci" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.input.json new file mode 100644 index 0000000000..e65645df36 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.input.json @@ -0,0 +1,6 @@ +{ + "type": "runpod", + "regions": ["US-KS-2", "EU-RO-1", "CA-MTL-1"], + "community_cloud": true, + "creds": {"type": "api_key", "api_key": "rpa_5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5C7D9E2F"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.types.json new file mode 100644 index 0000000000..830092c6d2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.types.json @@ -0,0 +1,4 @@ +{ + "/": "RunpodConfig", + "/creds": "RunpodAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.values.json new file mode 100644 index 0000000000..7cd16eaa50 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/runpod.values.json @@ -0,0 +1,13 @@ +{ + "community_cloud": true, + "creds": { + "api_key": "rpa_5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5C7D9E2F", + "type": "api_key" + }, + "regions": [ + "US-KS-2", + "EU-RO-1", + "CA-MTL-1" + ], + "type": "runpod" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.input.json new file mode 100644 index 0000000000..84a7ec79d0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.input.json @@ -0,0 +1,25 @@ +{ + "type": "slurm", + "clusters": [ + { + "name": "hpc-1", + "gpu_partitions": [{"gpu": "H100", "partitions": ["gpu", "gpu-long"]}], + "cpu_partitions": ["cpu"], + "hostname": "login1.hpc.example.com", + "port": 2222, + "user": "dstack", + "private_key": { + "path": "", + "content": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAAB\n-----END OPENSSH PRIVATE KEY-----\n" + } + }, + { + "name": "hpc-2", + "hostname": "login2.hpc.example.com", + "user": "dstack", + "private_key": { + "content": "-----BEGIN OPENSSH PRIVATE KEY-----\nZmFrZVNsdXJtU2Vjb25kQ2x1c3RlcktleUZvclRlc3RzAAAAAAAB\n-----END OPENSSH PRIVATE KEY-----\n" + } + } + ] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.types.json new file mode 100644 index 0000000000..12cceee3c6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.types.json @@ -0,0 +1,8 @@ +{ + "/": "SlurmConfig", + "/clusters/0": "SlurmClusterConfigWithCreds", + "/clusters/0/gpu_partitions/0": "SlurmGPUPartitionConfig", + "/clusters/0/private_key": "SlurmPrivateKeyConfig", + "/clusters/1": "SlurmClusterConfigWithCreds", + "/clusters/1/private_key": "SlurmPrivateKeyConfig" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.values.json new file mode 100644 index 0000000000..21bec8b965 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/slurm.values.json @@ -0,0 +1,39 @@ +{ + "clusters": [ + { + "cpu_partitions": [ + "cpu" + ], + "gpu_partitions": [ + { + "gpu": "H100", + "partitions": [ + "gpu", + "gpu-long" + ] + } + ], + "hostname": "login1.hpc.example.com", + "name": "hpc-1", + "port": 2222, + "private_key": { + "content": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAAB\n-----END OPENSSH PRIVATE KEY-----\n", + "path": "" + }, + "user": "dstack" + }, + { + "cpu_partitions": null, + "gpu_partitions": null, + "hostname": "login2.hpc.example.com", + "name": "hpc-2", + "port": null, + "private_key": { + "content": "-----BEGIN OPENSSH PRIVATE KEY-----\nZmFrZVNsdXJtU2Vjb25kQ2x1c3RlcktleUZvclRlc3RzAAAAAAAB\n-----END OPENSSH PRIVATE KEY-----\n", + "path": "" + }, + "user": "dstack" + } + ], + "type": "slurm" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.input.json new file mode 100644 index 0000000000..613fa34e6c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.input.json @@ -0,0 +1,6 @@ +{ + "type": "vastai", + "regions": ["Poland", "Sweden", "United States"], + "community_cloud": false, + "creds": {"type": "api_key", "api_key": "1f0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.types.json new file mode 100644 index 0000000000..c3f4b0ce46 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.types.json @@ -0,0 +1,4 @@ +{ + "/": "VastAIConfig", + "/creds": "VastAIAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.values.json new file mode 100644 index 0000000000..69cd4bd9bd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vastai.values.json @@ -0,0 +1,13 @@ +{ + "community_cloud": false, + "creds": { + "api_key": "1f0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a", + "type": "api_key" + }, + "regions": [ + "Poland", + "Sweden", + "United States" + ], + "type": "vastai" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.input.json new file mode 100644 index 0000000000..90b5512131 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.input.json @@ -0,0 +1,9 @@ +{ + "type": "verda", + "regions": ["ICE-01"], + "creds": { + "type": "api_key", + "client_id": "verda-7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "client_secret": "vd_0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.types.json new file mode 100644 index 0000000000..beb4677290 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.types.json @@ -0,0 +1,4 @@ +{ + "/": "VerdaConfig", + "/creds": "VerdaAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.values.json new file mode 100644 index 0000000000..40e6f5e4a9 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/verda.values.json @@ -0,0 +1,11 @@ +{ + "creds": { + "client_id": "verda-7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "client_secret": "vd_0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d", + "type": "api_key" + }, + "regions": [ + "ICE-01" + ], + "type": "verda" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.input.json new file mode 100644 index 0000000000..957cd2f0a6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.input.json @@ -0,0 +1,5 @@ +{ + "type": "vultr", + "regions": ["ewr", "ams", "nrt"], + "creds": {"type": "api_key", "api_key": "5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5"} +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.types.json new file mode 100644 index 0000000000..0959ed9254 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.types.json @@ -0,0 +1,4 @@ +{ + "/": "VultrConfig", + "/creds": "VultrAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.values.json new file mode 100644 index 0000000000..cbeb56ffda --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_config/vultr.values.json @@ -0,0 +1,12 @@ +{ + "creds": { + "api_key": "5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5", + "type": "api_key" + }, + "regions": [ + "ewr", + "ams", + "nrt" + ], + "type": "vultr" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.input.json new file mode 100644 index 0000000000..e37cb3f6ad --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.input.json @@ -0,0 +1,5 @@ +{ + "type": "access_key", + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.types.json similarity index 100% rename from src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.types.json rename to src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.types.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.values.json new file mode 100644 index 0000000000..4b85167105 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.access_key.values.json @@ -0,0 +1,5 @@ +{ + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "type": "access_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.input.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.input.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.types.json new file mode 100644 index 0000000000..361c6cc505 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.types.json @@ -0,0 +1,4 @@ +{ + "/": "AWSCreds", + "/__root__": "AWSDefaultCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.values.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/aws.default.values.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.input.json new file mode 100644 index 0000000000..bd95d5950b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.input.json @@ -0,0 +1,5 @@ +{ + "type": "client", + "client_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "client_secret": "Abc8Q~fakeAzureClientSecretValue_0123" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.types.json new file mode 100644 index 0000000000..c5fd8d1a97 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.types.json @@ -0,0 +1,4 @@ +{ + "/": "AzureCreds", + "/__root__": "AzureClientCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.values.json new file mode 100644 index 0000000000..6ddc8749fc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.client.values.json @@ -0,0 +1,6 @@ +{ + "client_id": "3f8c1d2e-4b5a-6c7d-8e9f-0a1b2c3d4e5f", + "client_secret": "Abc8Q~fakeAzureClientSecretValue_0123", + "tenant_id": null, + "type": "client" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.input.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.input.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.types.json new file mode 100644 index 0000000000..387b7147bd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.types.json @@ -0,0 +1,4 @@ +{ + "/": "AzureCreds", + "/__root__": "AzureDefaultCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.values.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/azure.default.values.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.input.json new file mode 100644 index 0000000000..4a2dc6ee95 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "rift_5f3a9c2e8b1d4706a9c3e5f7b2d8a4c6" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.types.json new file mode 100644 index 0000000000..288bdad78a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.types.json @@ -0,0 +1,3 @@ +{ + "/": "CloudRiftAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.values.json new file mode 100644 index 0000000000..7568291d70 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/cloudrift.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "rift_5f3a9c2e8b1d4706a9c3e5f7b2d8a4c6", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.input.json new file mode 100644 index 0000000000..c49ac05dad --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.input.json @@ -0,0 +1,5 @@ +{ + "type": "access_key", + "access_key": "CRUSOE7K3JVQZHX2MFAKE", + "secret_key": "wJ4lrXUtnFEMI7K3MDENGbPxRfiCYcrusoeKEY" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.types.json new file mode 100644 index 0000000000..a2eb15b13d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.types.json @@ -0,0 +1,3 @@ +{ + "/": "CrusoeAccessKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.values.json new file mode 100644 index 0000000000..01ca6acc5d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/crusoe.values.json @@ -0,0 +1,5 @@ +{ + "access_key": "CRUSOE7K3JVQZHX2MFAKE", + "secret_key": "wJ4lrXUtnFEMI7K3MDENGbPxRfiCYcrusoeKEY", + "type": "access_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.input.json new file mode 100644 index 0000000000..c62d974e92 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "test-digitalocean-api-key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.types.json new file mode 100644 index 0000000000..6cc4680120 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.types.json @@ -0,0 +1,3 @@ +{ + "/": "BaseDigitalOceanAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.values.json new file mode 100644 index 0000000000..dbd897b2da --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/digitalocean.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "test-digitalocean-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.input.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.input.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.types.json new file mode 100644 index 0000000000..b51f633e4c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.types.json @@ -0,0 +1,4 @@ +{ + "/": "GCPCreds", + "/__root__": "GCPDefaultCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.values.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.default.values.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.input.json new file mode 100644 index 0000000000..1a891185a7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.input.json @@ -0,0 +1,4 @@ +{ + "type": "service_account", + "data": "{\"type\": \"service_account\", \"project_id\": \"dstack-prod\", \"private_key_id\": \"7b1e4f2a9c8d6e5f4a3b2c1d0e9f8a7b6c5d4e3f\", \"client_email\": \"dstack@dstack-prod.iam.gserviceaccount.com\"}" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.types.json new file mode 100644 index 0000000000..5225a9485c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.types.json @@ -0,0 +1,4 @@ +{ + "/": "GCPCreds", + "/__root__": "GCPServiceAccountCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.values.json new file mode 100644 index 0000000000..eb2e5689d2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/gcp.service_account.values.json @@ -0,0 +1,5 @@ +{ + "data": "{\"type\": \"service_account\", \"project_id\": \"dstack-prod\", \"private_key_id\": \"7b1e4f2a9c8d6e5f4a3b2c1d0e9f8a7b6c5d4e3f\", \"client_email\": \"dstack@dstack-prod.iam.gserviceaccount.com\"}", + "filename": "", + "type": "service_account" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.input.json new file mode 100644 index 0000000000..02f09ea092 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "ha_2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d2e" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.types.json new file mode 100644 index 0000000000..f3cdf8e8ca --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.types.json @@ -0,0 +1,3 @@ +{ + "/": "HotAisleAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.values.json new file mode 100644 index 0000000000..c0126fd4be --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/hotaisle.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "ha_2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d2e", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.input.json new file mode 100644 index 0000000000..1b8e915724 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "jl_7b2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.types.json new file mode 100644 index 0000000000..8ce10aa83c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.types.json @@ -0,0 +1,3 @@ +{ + "/": "JarvisLabsAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.values.json new file mode 100644 index 0000000000..cc76571eea --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/jarvislabs.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "jl_7b2d8a4c6e1f0a3b5c7d9e2f4a6b8c0d", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.input.json new file mode 100644 index 0000000000..a1d9c6588a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "secret_dstack_9c2e8b1d47065f3a9c3e5f7b2d8a4c6e" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.types.json new file mode 100644 index 0000000000..a03b239295 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.types.json @@ -0,0 +1,3 @@ +{ + "/": "LambdaAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.values.json new file mode 100644 index 0000000000..cae5e8d11c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/lambda.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "secret_dstack_9c2e8b1d47065f3a9c3e5f7b2d8a4c6e", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.input.json new file mode 100644 index 0000000000..bef2f4c1e2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.input.json @@ -0,0 +1,6 @@ +{ + "type": "service_account", + "service_account_id": "serviceaccount-e00fake9c2e8b1d4706", + "public_key_id": "publickey-e00fake5f3a9c3e5f7b2d", + "private_key_content": "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIF9mYWtlTmViaXVzS2V5Q29udGVudEZvclRlc3Rz\n-----END PRIVATE KEY-----\n" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.types.json new file mode 100644 index 0000000000..3e4b870291 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.types.json @@ -0,0 +1,3 @@ +{ + "/": "NebiusServiceAccountCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.values.json new file mode 100644 index 0000000000..e389d33c23 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/nebius.values.json @@ -0,0 +1,8 @@ +{ + "filename": null, + "private_key_content": "-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIF9mYWtlTmViaXVzS2V5Q29udGVudEZvclRlc3Rz\n-----END PRIVATE KEY-----\n", + "private_key_file": null, + "public_key_id": "publickey-e00fake5f3a9c3e5f7b2d", + "service_account_id": "serviceaccount-e00fake9c2e8b1d4706", + "type": "service_account" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.input.json new file mode 100644 index 0000000000..4fcaeb8cd1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.input.json @@ -0,0 +1,8 @@ +{ + "type": "client", + "user": "ocid1.user.oc1..aaaaaaaa7k3jvqzhx2mfakeuserocid", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaaq4d9wmfaketenancyocid", + "key_content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC\n-----END PRIVATE KEY-----\n", + "fingerprint": "a1:b2:c3:d4:e5:f6:07:18:29:3a:4b:5c:6d:7e:8f:90", + "region": "eu-frankfurt-1" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.types.json new file mode 100644 index 0000000000..2ea26b6f8b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.types.json @@ -0,0 +1,4 @@ +{ + "/": "OCICreds", + "/__root__": "OCIClientCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.values.json new file mode 100644 index 0000000000..94507718ae --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.client.values.json @@ -0,0 +1,10 @@ +{ + "fingerprint": "a1:b2:c3:d4:e5:f6:07:18:29:3a:4b:5c:6d:7e:8f:90", + "key_content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC\n-----END PRIVATE KEY-----\n", + "key_file": null, + "pass_phrase": null, + "region": "eu-frankfurt-1", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaaq4d9wmfaketenancyocid", + "type": "client", + "user": "ocid1.user.oc1..aaaaaaaa7k3jvqzhx2mfakeuserocid" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.input.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.input.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.types.json new file mode 100644 index 0000000000..6f460a30c0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.types.json @@ -0,0 +1,4 @@ +{ + "/": "OCICreds", + "/__root__": "OCIDefaultCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.values.json new file mode 100644 index 0000000000..9a284ca875 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/oci.default.values.json @@ -0,0 +1,5 @@ +{ + "file": "~/.oci/config", + "profile": "DEFAULT", + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.input.json new file mode 100644 index 0000000000..2a50a6ea90 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "rpa_5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5C7D9E2F" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.types.json new file mode 100644 index 0000000000..eb4abe5938 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.types.json @@ -0,0 +1,3 @@ +{ + "/": "RunpodAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.values.json new file mode 100644 index 0000000000..30d56aa859 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/runpod.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "rpa_5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5C7D9E2F", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.input.json new file mode 100644 index 0000000000..47c3340de5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "1f0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.types.json new file mode 100644 index 0000000000..c6707215ee --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.types.json @@ -0,0 +1,3 @@ +{ + "/": "VastAIAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.values.json new file mode 100644 index 0000000000..95ec3128b6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vastai.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "1f0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.input.json new file mode 100644 index 0000000000..32f1aa18c5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.input.json @@ -0,0 +1,5 @@ +{ + "type": "api_key", + "client_id": "verda-7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "client_secret": "vd_0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.types.json new file mode 100644 index 0000000000..396415f073 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.types.json @@ -0,0 +1,3 @@ +{ + "/": "VerdaAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.values.json new file mode 100644 index 0000000000..f2d37ff9da --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/verda.values.json @@ -0,0 +1,5 @@ +{ + "client_id": "verda-7b2d8a4c-6e1f-4a3b-9c7d-9e2f4a6b8c0d", + "client_secret": "vd_0a3b5c7d9e2f4a6b8c0d2e4f6a8b0c2d", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.input.json new file mode 100644 index 0000000000..94a1f6d836 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.input.json @@ -0,0 +1,4 @@ +{ + "type": "api_key", + "api_key": "5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.types.json new file mode 100644 index 0000000000..0e5f8a82a1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.types.json @@ -0,0 +1,3 @@ +{ + "/": "VultrAPIKeyCreds" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.values.json new file mode 100644 index 0000000000..a40a69dd56 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_creds/vultr.values.json @@ -0,0 +1,4 @@ +{ + "api_key": "5F3A9C2E8B1D4706A9C3E5F7B2D8A4C6E1F0A3B5", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.input.json new file mode 100644 index 0000000000..4f225607e0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.input.json @@ -0,0 +1,5 @@ +{ + "lb_arn": "arn:aws:elasticloadbalancing:eu-west-1:123456789012:loadbalancer/net/gateway/7a8b", + "tg_arn": "arn:aws:elasticloadbalancing:eu-west-1:123456789012:targetgroup/gateway/9c0d", + "listener_arn": "arn:aws:elasticloadbalancing:eu-west-1:123456789012:listener/net/gateway/1e2f" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.types.json new file mode 100644 index 0000000000..18a7f84c32 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.types.json @@ -0,0 +1,3 @@ +{ + "/": "AWSGatewayBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.values.json new file mode 100644 index 0000000000..8fa8d595f6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_gateway.values.json @@ -0,0 +1,6 @@ +{ + "http_listener_arn": null, + "lb_arn": "arn:aws:elasticloadbalancing:eu-west-1:123456789012:loadbalancer/net/gateway/7a8b", + "listener_arn": "arn:aws:elasticloadbalancing:eu-west-1:123456789012:listener/net/gateway/1e2f", + "tg_arn": "arn:aws:elasticloadbalancing:eu-west-1:123456789012:targetgroup/gateway/9c0d" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.input.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.input.json @@ -0,0 +1 @@ +{} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.types.json new file mode 100644 index 0000000000..7fa082ee75 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.types.json @@ -0,0 +1,3 @@ +{ + "/": "AWSInstanceBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.values.json new file mode 100644 index 0000000000..6a909e257a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_instance.values.json @@ -0,0 +1,3 @@ +{ + "eip_allocation_id": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.input.json new file mode 100644 index 0000000000..8f10abeb15 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.input.json @@ -0,0 +1,4 @@ +{ + "volume_type": "io2", + "iops": "12000" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.types.json new file mode 100644 index 0000000000..e43fe754a1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.types.json @@ -0,0 +1,3 @@ +{ + "/": "AWSVolumeBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.values.json new file mode 100644 index 0000000000..5f29a225ef --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/aws_volume.values.json @@ -0,0 +1,4 @@ +{ + "iops": 12000, + "volume_type": "io2" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.input.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.input.json @@ -0,0 +1 @@ +{} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.types.json new file mode 100644 index 0000000000..fc1b04e7ed --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.types.json @@ -0,0 +1,3 @@ +{ + "/": "CrusoeInstanceBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.values.json new file mode 100644 index 0000000000..7fd9cbc5a2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_instance.values.json @@ -0,0 +1,3 @@ +{ + "data_disk_id": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.input.json new file mode 100644 index 0000000000..717d6c1757 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.input.json @@ -0,0 +1,3 @@ +{ + "ib_partition_id": "a1b2c3d4-5678-4abc-9def-000000000010" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.types.json new file mode 100644 index 0000000000..f1f0324c60 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.types.json @@ -0,0 +1,3 @@ +{ + "/": "CrusoePlacementGroupBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.values.json new file mode 100644 index 0000000000..c263f36a65 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/crusoe_placement_group.values.json @@ -0,0 +1,4 @@ +{ + "ib_network_id": null, + "ib_partition_id": "a1b2c3d4-5678-4abc-9def-000000000010" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.input.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.input.json @@ -0,0 +1 @@ +{} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.types.json new file mode 100644 index 0000000000..4063304298 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.types.json @@ -0,0 +1,3 @@ +{ + "/": "GCPOfferBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.values.json new file mode 100644 index 0000000000..150765abea --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_offer.values.json @@ -0,0 +1,3 @@ +{ + "is_dws_calendar_mode": false +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.input.json new file mode 100644 index 0000000000..4c6242d529 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.input.json @@ -0,0 +1,3 @@ +{ + "disk_type": "pd-ssd" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.types.json new file mode 100644 index 0000000000..92c12bc7e6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.types.json @@ -0,0 +1,3 @@ +{ + "/": "GCPVolumeDiskBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.values.json new file mode 100644 index 0000000000..f4f05eb880 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/gcp_volume_disk.values.json @@ -0,0 +1,4 @@ +{ + "disk_type": "pd-ssd", + "type": "disk" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.input.json new file mode 100644 index 0000000000..6694df8107 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.input.json @@ -0,0 +1,4 @@ +{ + "ip_address": "198.51.100.24", + "private_ip_address": "10.0.0.24" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.types.json new file mode 100644 index 0000000000..f9115a9011 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.types.json @@ -0,0 +1,3 @@ +{ + "/": "HotAisleInstanceBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.values.json new file mode 100644 index 0000000000..ce44e385ad --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_instance.values.json @@ -0,0 +1,3 @@ +{ + "ip_address": "198.51.100.24" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.input.json new file mode 100644 index 0000000000..554183da8f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.input.json @@ -0,0 +1,9 @@ +{ + "vm_specs": { + "cpu": {"count": 32, "model": "EPYC"}, + "gpu": {"count": 2, "model": "MI325X"}, + "ram": 512, + "features": ["infiniband", "local-nvme"] + }, + "offer_schema_version": 2 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.types.json new file mode 100644 index 0000000000..69b9dfdae3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.types.json @@ -0,0 +1,3 @@ +{ + "/": "HotAisleOfferBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.values.json new file mode 100644 index 0000000000..e688a67a5a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/hotaisle_offer.values.json @@ -0,0 +1,17 @@ +{ + "vm_specs": { + "cpu": { + "count": 32, + "model": "EPYC" + }, + "features": [ + "infiniband", + "local-nvme" + ], + "gpu": { + "count": 2, + "model": "MI325X" + }, + "ram": 512 + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.input.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.input.json @@ -0,0 +1 @@ +{} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.types.json new file mode 100644 index 0000000000..a5c55f9084 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.types.json @@ -0,0 +1,3 @@ +{ + "/": "JarvisLabsInstanceBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.values.json new file mode 100644 index 0000000000..8a3d14f43e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/jarvislabs_instance.values.json @@ -0,0 +1,3 @@ +{ + "ssh_key_ids": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.input.json new file mode 100644 index 0000000000..c988f6bd6f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.input.json @@ -0,0 +1,6 @@ +{ + "jump_pod_name": "dstack-jump-pod-old", + "jump_pod_service_name": "dstack-jump-pod-old-service", + "user_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeCompatibilityKey dstack@localhost", + "jump_pod_namespace": "dstack-system" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.types.json new file mode 100644 index 0000000000..b408ab367b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.types.json @@ -0,0 +1,3 @@ +{ + "/": "KubernetesBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.values.json new file mode 100644 index 0000000000..a75a278815 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/kubernetes.values.json @@ -0,0 +1,5 @@ +{ + "jump_pod_name": "dstack-jump-pod-old", + "jump_pod_service_name": "dstack-jump-pod-old-service", + "user_ssh_public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFakeCompatibilityKey dstack@localhost" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.input.json new file mode 100644 index 0000000000..35339fb098 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.input.json @@ -0,0 +1,5 @@ +{ + "id": "computecluster-e01compat", + "fabric": "fabric-6", + "created_by": "older-control-plane" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.types.json new file mode 100644 index 0000000000..dac82180cb --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.types.json @@ -0,0 +1,3 @@ +{ + "/": "NebiusClusterBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.values.json new file mode 100644 index 0000000000..f66a169cfe --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_cluster.values.json @@ -0,0 +1,4 @@ +{ + "fabric": "fabric-6", + "id": "computecluster-e01compat" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.input.json new file mode 100644 index 0000000000..cbf0796e2e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.input.json @@ -0,0 +1,4 @@ +{ + "boot_disk_id": "computedisk-e01compat", + "disk_type": "network-ssd" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.types.json new file mode 100644 index 0000000000..59e48fc0bc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.types.json @@ -0,0 +1,3 @@ +{ + "/": "NebiusInstanceBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.values.json new file mode 100644 index 0000000000..8ec67249df --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_instance.values.json @@ -0,0 +1,3 @@ +{ + "boot_disk_id": "computedisk-e01compat" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.input.json new file mode 100644 index 0000000000..e4bc480565 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.input.json @@ -0,0 +1,7 @@ +{ + "cluster": { + "id": "computecluster-e02compat", + "fabric": "fabric-7", + "availability_zone": "eu-north1-c" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.types.json new file mode 100644 index 0000000000..cac59cbd35 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.types.json @@ -0,0 +1,4 @@ +{ + "/": "NebiusPlacementGroupBackendData", + "/cluster": "NebiusClusterBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.values.json new file mode 100644 index 0000000000..aa42429a51 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/nebius_placement_group.values.json @@ -0,0 +1,6 @@ +{ + "cluster": { + "fabric": "fabric-7", + "id": "computecluster-e02compat" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.input.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.input.json @@ -0,0 +1 @@ +{} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.types.json new file mode 100644 index 0000000000..b6c7e0357d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.types.json @@ -0,0 +1,3 @@ +{ + "/": "RunpodOfferBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.values.json new file mode 100644 index 0000000000..d0c070efd9 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/runpod_offer.values.json @@ -0,0 +1,3 @@ +{ + "pod_counts": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.input.json new file mode 100644 index 0000000000..6d312d8f82 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.input.json @@ -0,0 +1,3 @@ +{ + "min_bid": "0.4321" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.types.json new file mode 100644 index 0000000000..9b2e2abccd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.types.json @@ -0,0 +1,3 @@ +{ + "/": "VastAIOfferBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.values.json new file mode 100644 index 0000000000..d7795fb4fe --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/vastai_offer.values.json @@ -0,0 +1,3 @@ +{ + "min_bid": 0.4321 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.input.json new file mode 100644 index 0000000000..b2bdf76c1c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.input.json @@ -0,0 +1,3 @@ +{ + "startup_script_id": "a1b2c3d4-5678-4abc-9def-000000000020" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.types.json new file mode 100644 index 0000000000..81c082b5f5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.types.json @@ -0,0 +1,3 @@ +{ + "/": "VerdaInstanceBackendData" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.values.json new file mode 100644 index 0000000000..1be7aa5215 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/backend_data/verda_instance.values.json @@ -0,0 +1,4 @@ +{ + "ssh_key_ids": null, + "startup_script_id": "a1b2c3d4-5678-4abc-9def-000000000020" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json deleted file mode 100644 index 30e4a69d8f..0000000000 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.input.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": "access_key", - "access_key": "AK", - "secret_key": "SK", - "rotation_hint_from_a_newer_server": "2027-01-01" -} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json deleted file mode 100644 index 2a8ac73adb..0000000000 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/db/aws_creds.values.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "access_key": "AK", - "secret_key": "SK", - "type": "access_key" -} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/amddevcloud.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/amddevcloud.json new file mode 100644 index 0000000000..3d75680fa1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/amddevcloud.json @@ -0,0 +1,7 @@ +{ + "project_name": "dstack", + "regions": [ + "tor1" + ], + "type": "amddevcloud" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/aws.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/aws.json new file mode 100644 index 0000000000..5c73220ac0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/aws.json @@ -0,0 +1,29 @@ +{ + "default_vpcs": false, + "experimental_instance_types": [ + "p5.48xlarge" + ], + "iam_instance_profile": "dstack-instance-profile", + "os_images": { + "cpu": null, + "nvidia": { + "name": "dstack-nvidia-ubuntu-22.04", + "owner": "123456789012", + "user": "ubuntu" + } + }, + "public_ips": true, + "regions": [ + "us-east-1", + "us-west-2" + ], + "tags": { + "env": "prod", + "team": "ml" + }, + "type": "aws", + "vpc_ids": { + "us-east-1": "vpc-0a1b2c3d4e5f67890" + }, + "vpc_name": null +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/azure.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/azure.json new file mode 100644 index 0000000000..a73d3ac9f1 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/azure.json @@ -0,0 +1,21 @@ +{ + "public_ips": true, + "regions": [ + "eastus", + "westeurope" + ], + "resource_group": "dstack-rg", + "subnet_ids": { + "eastus": "dstack-subnet" + }, + "subscription_id": "66666666-7777-8888-9999-000000000000", + "tags": { + "env": "prod" + }, + "tenant_id": "11111111-2222-3333-4444-555555555555", + "type": "azure", + "vm_managed_identity": "dstack-identity", + "vpc_ids": { + "eastus": "dstack-vnet" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/cloudrift.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/cloudrift.json new file mode 100644 index 0000000000..48b73f7a14 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/cloudrift.json @@ -0,0 +1,6 @@ +{ + "regions": [ + "us-east-nc-nrp-1" + ], + "type": "cloudrift" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/crusoe.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/crusoe.json new file mode 100644 index 0000000000..0e1a5965d6 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/crusoe.json @@ -0,0 +1,7 @@ +{ + "project_id": "a1b2c3d4-5678-4abc-9def-000000000000", + "regions": [ + "us-east1" + ], + "type": "crusoe" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/datacrunch.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/datacrunch.json new file mode 100644 index 0000000000..ea91b788dd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/datacrunch.json @@ -0,0 +1,6 @@ +{ + "regions": [ + "FIN-01" + ], + "type": "datacrunch" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/digitalocean.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/digitalocean.json new file mode 100644 index 0000000000..63a880b5c2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/digitalocean.json @@ -0,0 +1,8 @@ +{ + "project_name": "dstack", + "regions": [ + "nyc3", + "ams3" + ], + "type": "digitalocean" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/gcp.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/gcp.json new file mode 100644 index 0000000000..fa8b7052ba --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/gcp.json @@ -0,0 +1,25 @@ +{ + "extra_vpcs": [ + "dstack-extra-vpc" + ], + "nat_check": false, + "preview_features": [ + "g4" + ], + "project_id": "dstack-prod", + "public_ips": true, + "regions": [ + "us-central1", + "europe-west4" + ], + "roce_vpcs": [ + "dstack-roce-vpc" + ], + "tags": { + "env": "prod" + }, + "type": "gcp", + "vm_service_account": "dstack@dstack-prod.iam.gserviceaccount.com", + "vpc_name": "dstack-vpc", + "vpc_project_id": "dstack-network" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/hotaisle.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/hotaisle.json new file mode 100644 index 0000000000..4c37e0d698 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/hotaisle.json @@ -0,0 +1,7 @@ +{ + "regions": [ + "us-michigan-1" + ], + "team_handle": "dstack-team", + "type": "hotaisle" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/jarvislabs.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/jarvislabs.json new file mode 100644 index 0000000000..6fb00aa8b9 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/jarvislabs.json @@ -0,0 +1,6 @@ +{ + "regions": [ + "in-north-1" + ], + "type": "jarvislabs" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/kubernetes.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/kubernetes.json new file mode 100644 index 0000000000..efed0ca1d9 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/kubernetes.json @@ -0,0 +1,22 @@ +{ + "contexts": [ + { + "name": "prod-cluster", + "proxy_jump": { + "hostname": "10.0.0.1", + "port": 22 + } + }, + "staging-cluster" + ], + "kubeconfig": { + "data": "apiVersion: v1\nkind: Config\n", + "filename": "" + }, + "namespace": "dstack", + "proxy_jump": { + "hostname": "bastion.internal", + "port": 2222 + }, + "type": "kubernetes" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/lambda.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/lambda.json new file mode 100644 index 0000000000..2630a4f6f5 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/lambda.json @@ -0,0 +1,7 @@ +{ + "regions": [ + "us-east-1", + "us-west-2" + ], + "type": "lambda" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/nebius.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/nebius.json new file mode 100644 index 0000000000..97ad5eb08c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/nebius.json @@ -0,0 +1,15 @@ +{ + "fabrics": [ + "fabric-3" + ], + "projects": [ + "project-e00dstack" + ], + "regions": [ + "eu-north1" + ], + "tags": { + "env": "prod" + }, + "type": "nebius" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/oci.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/oci.json new file mode 100644 index 0000000000..7340df893b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/oci.json @@ -0,0 +1,10 @@ +{ + "compartment_id": "ocid1.compartment.oc1..aaaaaaaadstack", + "regions": [ + "eu-frankfurt-1" + ], + "subnet_ids_per_region": { + "eu-frankfurt-1": "ocid1.subnet.oc1.eu-frankfurt-1.aaaaaaaadstack" + }, + "type": "oci" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/runpod.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/runpod.json new file mode 100644 index 0000000000..5f877df81b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/runpod.json @@ -0,0 +1,8 @@ +{ + "community_cloud": true, + "regions": [ + "US-KS-2", + "EU-RO-1" + ], + "type": "runpod" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/slurm.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/slurm.json new file mode 100644 index 0000000000..ec648e9fd2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/slurm.json @@ -0,0 +1,27 @@ +{ + "clusters": [ + { + "cpu_partitions": [ + "cpu" + ], + "gpu_partitions": [ + { + "gpu": "H100", + "partitions": [ + "gpu", + "gpu-long" + ] + } + ], + "hostname": "login.hpc.example.com", + "name": "hpc-1", + "port": 22, + "private_key": { + "content": "-----BEGIN PRIVATE KEY-----\nMIIBVgIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----\n", + "path": "" + }, + "user": "dstack" + } + ], + "type": "slurm" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vastai.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vastai.json new file mode 100644 index 0000000000..f83133f12d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vastai.json @@ -0,0 +1,8 @@ +{ + "community_cloud": false, + "regions": [ + "Poland", + "Sweden" + ], + "type": "vastai" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/verda.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/verda.json new file mode 100644 index 0000000000..b84d0c5812 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/verda.json @@ -0,0 +1,6 @@ +{ + "regions": [ + "ICE-01" + ], + "type": "verda" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vultr.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vultr.json new file mode 100644 index 0000000000..bc5f1f806d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_config/vultr.json @@ -0,0 +1,7 @@ +{ + "regions": [ + "ewr", + "ams" + ], + "type": "vultr" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.access_key.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.access_key.json new file mode 100644 index 0000000000..3e245b638e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.access_key.json @@ -0,0 +1,5 @@ +{ + "access_key": "AKIAIOSFODNN7EXAMPLE", + "secret_key": "wJalrXUtnFEMI", + "type": "access_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.default.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.default.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/aws.default.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.client.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.client.json new file mode 100644 index 0000000000..d57569c387 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.client.json @@ -0,0 +1,6 @@ +{ + "client_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "client_secret": "azure-client-secret", + "tenant_id": "11111111-2222-3333-4444-555555555555", + "type": "client" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.default.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.default.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/azure.default.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/cloudrift.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/cloudrift.json new file mode 100644 index 0000000000..f6903573ee --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/cloudrift.json @@ -0,0 +1,4 @@ +{ + "api_key": "rift-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/crusoe.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/crusoe.json new file mode 100644 index 0000000000..084218f3b9 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/crusoe.json @@ -0,0 +1,5 @@ +{ + "access_key": "crusoe-access-key", + "secret_key": "crusoe-secret-key", + "type": "access_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/digitalocean.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/digitalocean.json new file mode 100644 index 0000000000..168d63538d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/digitalocean.json @@ -0,0 +1,4 @@ +{ + "api_key": "dop_v1_digitalocean", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.default.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.default.json new file mode 100644 index 0000000000..e19b934ec2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.default.json @@ -0,0 +1,3 @@ +{ + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.service_account.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.service_account.json new file mode 100644 index 0000000000..a549732a50 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/gcp.service_account.json @@ -0,0 +1,5 @@ +{ + "data": "{\"type\": \"service_account\", \"project_id\": \"dstack-prod\", \"private_key_id\": \"0123456789abcdef0123456789abcdef01234567\"}", + "filename": "", + "type": "service_account" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/hotaisle.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/hotaisle.json new file mode 100644 index 0000000000..a3850dbe43 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/hotaisle.json @@ -0,0 +1,4 @@ +{ + "api_key": "hotaisle-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/jarvislabs.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/jarvislabs.json new file mode 100644 index 0000000000..060faa1962 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/jarvislabs.json @@ -0,0 +1,4 @@ +{ + "api_key": "jarvislabs-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/lambda.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/lambda.json new file mode 100644 index 0000000000..fb1466e550 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/lambda.json @@ -0,0 +1,4 @@ +{ + "api_key": "lambda-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/nebius.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/nebius.json new file mode 100644 index 0000000000..d704ab9327 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/nebius.json @@ -0,0 +1,8 @@ +{ + "filename": null, + "private_key_content": "-----BEGIN PRIVATE KEY-----\nMIIBVgIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----\n", + "private_key_file": null, + "public_key_id": "publickey-e00dstack", + "service_account_id": "serviceaccount-e00dstack", + "type": "service_account" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.client.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.client.json new file mode 100644 index 0000000000..14a8434ff2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.client.json @@ -0,0 +1,10 @@ +{ + "fingerprint": "12:34:56:78:9a:bc:de:f0:12:34:56:78:9a:bc:de:f0", + "key_content": "-----BEGIN PRIVATE KEY-----\nMIIBVgIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----\n", + "key_file": null, + "pass_phrase": null, + "region": "eu-frankfurt-1", + "tenancy": "ocid1.tenancy.oc1..aaaaaaaadstack", + "type": "client", + "user": "ocid1.user.oc1..aaaaaaaadstack" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.default.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.default.json new file mode 100644 index 0000000000..9a284ca875 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/oci.default.json @@ -0,0 +1,5 @@ +{ + "file": "~/.oci/config", + "profile": "DEFAULT", + "type": "default" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/runpod.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/runpod.json new file mode 100644 index 0000000000..d9e61f10e7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/runpod.json @@ -0,0 +1,4 @@ +{ + "api_key": "runpod-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vastai.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vastai.json new file mode 100644 index 0000000000..f836744bb7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vastai.json @@ -0,0 +1,4 @@ +{ + "api_key": "vastai-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/verda.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/verda.json new file mode 100644 index 0000000000..5eee3fff9d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/verda.json @@ -0,0 +1,5 @@ +{ + "client_id": "verda-client-id", + "client_secret": "verda-secret", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vultr.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vultr.json new file mode 100644 index 0000000000..22cac34e59 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_creds/vultr.json @@ -0,0 +1,4 @@ +{ + "api_key": "vultr-api-key", + "type": "api_key" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_gateway.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_gateway.json new file mode 100644 index 0000000000..a6638f8dc7 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_gateway.json @@ -0,0 +1,6 @@ +{ + "http_listener_arn": null, + "lb_arn": "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/net/dstack/1a2b", + "listener_arn": "arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/net/dstack/5e6f", + "tg_arn": "arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/dstack/3c4d" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_instance.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_instance.json new file mode 100644 index 0000000000..5349f10931 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_instance.json @@ -0,0 +1,3 @@ +{ + "eip_allocation_id": "eipalloc-0a1b2c3d4e5f67890" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_volume.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_volume.json new file mode 100644 index 0000000000..04835802f0 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/aws_volume.json @@ -0,0 +1,4 @@ +{ + "iops": 3000, + "volume_type": "gp3" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_instance.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_instance.json new file mode 100644 index 0000000000..8649615e1f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_instance.json @@ -0,0 +1,3 @@ +{ + "data_disk_id": "a1b2c3d4-5678-4abc-9def-000000000001" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_placement_group.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_placement_group.json new file mode 100644 index 0000000000..56bace64f2 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/crusoe_placement_group.json @@ -0,0 +1,4 @@ +{ + "ib_network_id": "a1b2c3d4-5678-4abc-9def-000000000003", + "ib_partition_id": "a1b2c3d4-5678-4abc-9def-000000000002" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_offer.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_offer.json new file mode 100644 index 0000000000..85acfcf84f --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_offer.json @@ -0,0 +1,3 @@ +{ + "is_dws_calendar_mode": true +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_volume_disk.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_volume_disk.json new file mode 100644 index 0000000000..2847af1865 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/gcp_volume_disk.json @@ -0,0 +1,4 @@ +{ + "disk_type": "pd-balanced", + "type": "disk" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_instance.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_instance.json new file mode 100644 index 0000000000..b791946d11 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_instance.json @@ -0,0 +1,3 @@ +{ + "ip_address": "203.0.113.10" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_offer.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_offer.json new file mode 100644 index 0000000000..b2aba088cd --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/hotaisle_offer.json @@ -0,0 +1,12 @@ +{ + "vm_specs": { + "cpu": { + "count": 26 + }, + "gpu": { + "count": 1, + "model": "MI300X" + }, + "ram": 224 + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/jarvislabs_instance.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/jarvislabs_instance.json new file mode 100644 index 0000000000..ba5bf39973 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/jarvislabs_instance.json @@ -0,0 +1,6 @@ +{ + "ssh_key_ids": [ + "1234", + "5678" + ] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/kubernetes.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/kubernetes.json new file mode 100644 index 0000000000..03a16fe38c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/kubernetes.json @@ -0,0 +1,5 @@ +{ + "jump_pod_name": "dstack-jump-pod", + "jump_pod_service_name": "dstack-jump-pod-service", + "user_ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD dstack@localhost" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_cluster.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_cluster.json new file mode 100644 index 0000000000..cda5f63c2e --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_cluster.json @@ -0,0 +1,4 @@ +{ + "fabric": "fabric-3", + "id": "computecluster-e00dstack" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_instance.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_instance.json new file mode 100644 index 0000000000..9b0d446a75 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_instance.json @@ -0,0 +1,3 @@ +{ + "boot_disk_id": "computedisk-e00dstack" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_placement_group.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_placement_group.json new file mode 100644 index 0000000000..7758d3787b --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/nebius_placement_group.json @@ -0,0 +1,6 @@ +{ + "cluster": { + "fabric": "fabric-3", + "id": "computecluster-e00dstack" + } +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/runpod_offer.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/runpod_offer.json new file mode 100644 index 0000000000..7bb65b3e6c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/runpod_offer.json @@ -0,0 +1,8 @@ +{ + "pod_counts": [ + 1, + 2, + 4, + 8 + ] +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/vastai_offer.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/vastai_offer.json new file mode 100644 index 0000000000..320f5137d8 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/vastai_offer.json @@ -0,0 +1,3 @@ +{ + "min_bid": 0.1234 +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/verda_instance.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/verda_instance.json new file mode 100644 index 0000000000..c2fa6d2be3 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/serialization/backend_data/verda_instance.json @@ -0,0 +1,6 @@ +{ + "ssh_key_ids": [ + "a1b2c3d4-5678-4abc-9def-000000000005" + ], + "startup_script_id": "a1b2c3d4-5678-4abc-9def-000000000004" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json b/src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json deleted file mode 100644 index 2a8ac73adb..0000000000 --- a/src/tests/_internal/pydantic_compat/fixtures/serialization/db/aws_creds.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "access_key": "AK", - "secret_key": "SK", - "type": "access_key" -} diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index 9c0ea6349b..13f3dddcc7 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -14,7 +14,8 @@ `*.values.json` and `*.types.json` only. Registries and test classes below follow the same surface order as `test_serialization.py`: -db, api_request, api_response, config, runner, gateway, proxy. +db, backend_config, backend_creds, backend_data, api_request, api_response, config, runner, +gateway, proxy. """ import json @@ -23,6 +24,7 @@ import pytest import yaml +from dstack._internal.core.backends.nebius.models import NebiusOfferBackendData from dstack._internal.core.models.configurations import DstackConfiguration from dstack._internal.core.models.profiles import ProfilesConfig from dstack._internal.proxy.gateway.schemas.registry import ( @@ -43,11 +45,13 @@ MetricsResponse, TaskInfoResponse, ) +from tests._internal.pydantic_compat import backend_factories from tests._internal.pydantic_compat import test_serialization as ser from tests._internal.pydantic_compat.compare import ( FIXTURES_DIR, assert_matches_fixture, canonicalize, + class_name, type_map, ) from tests._internal.pydantic_compat.compat import parse_forbid_extra, parse_ignore_extra @@ -70,6 +74,15 @@ def _derived_registry(surface: str) -> dict[str, Any]: DB_BLOBS = _derived_registry("db") API_REQUESTS = _derived_registry("api_request") API_RESPONSES = _derived_registry("api_response") +BACKEND_DATA = _derived_registry("backend_data") + +# The `config` column is written as `XStoredConfig` but read back as `XConfig`, so this is the one +# surface whose parse target is a different class from the one that produced the bytes. The read is +# a splice of two columns — `XConfig(**json.loads(config), creds=XCreds.parse_raw(auth))` — and the +# inputs mirror that by carrying `creds` inline, which additionally makes the creds union resolve +# here rather than arrive pre-resolved. +BACKEND_CONFIGS = backend_factories.BACKEND_CONFIG_MODELS +BACKEND_CREDS = backend_factories.BACKEND_CREDS_MODELS # Parsed from user-authored YAML with extra="forbid". The only surface whose inputs are `.yml`, # because the sugar pinned here is a YAML phenomenon: `python: 3.10` is a float that survives only @@ -127,6 +140,56 @@ def test_parses_to_expected_values_and_types(self, name, regen): _assert_parses("db", name, model, regen) +class TestBackendConfigParsing: + @pytest.mark.parametrize("name", sorted(BACKEND_CONFIGS)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_ignore_extra(BACKEND_CONFIGS[name], _load_input("backend_config", name)) + _assert_parses("backend_config", name, model, regen) + + +class TestBackendCredsParsing: + @pytest.mark.parametrize("name", sorted(BACKEND_CREDS)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_ignore_extra(BACKEND_CREDS[name], _load_input("backend_creds", name)) + _assert_parses("backend_creds", name, model, regen) + + @pytest.mark.parametrize("name", sorted(backend_factories.CREDS_ARMS)) + def test_resolves_the_intended_union_arm(self, name): + """ + The four custom-root creds unions get this on top of the type map. + + A wrong arm can still produce a byte-identical dump — `AWSDefaultCreds` and a hypothetical + arm carrying only `type` would — so the arm is asserted by name. This is the check that + catches v2 resolving a union in smart mode where v1 went left to right. + """ + creds = parse_ignore_extra(BACKEND_CREDS[name], _load_input("backend_creds", name)) + expected = class_name(backend_factories.CREDS_ARMS[name]) + # Compared by name rather than `isinstance`: the permissive read yields duality's + # `...Response` variant on v1 and the plain class on v2, and `class_name` erases that. + assert class_name(creds.__root__) == expected + + +class TestBackendDataParsing: + @pytest.mark.parametrize("name", sorted(BACKEND_DATA)) + def test_parses_to_expected_values_and_types(self, name, regen): + model = parse_ignore_extra(BACKEND_DATA[name], _load_input("backend_data", name)) + _assert_parses("backend_data", name, model, regen) + + +class TestNebiusOfferBackendDataSetField: + """ + Excluded from `BACKEND_DATA` because `.json()` raises on its `set` field, so there is nothing + for `_assert_parses` to compare. The read path is real regardless, so assert on the value. + """ + + def test_parses_a_list_into_a_set(self): + data = parse_ignore_extra(NebiusOfferBackendData, {"fabrics": ["fabric-3", "fabric-6"]}) + assert data.fabrics == {"fabric-3", "fabric-6"} + + def test_missing_field_defaults_to_an_empty_set(self): + assert parse_ignore_extra(NebiusOfferBackendData, {}).fabrics == set() + + # Every case above earns a second assertion: inject an unknown key into the same input and check the # reader's `extra` policy holds. That covers what happens when the writer is a newer version. # @@ -135,6 +198,9 @@ def test_parses_to_expected_values_and_types(self, name, regen): # parse — they are absent rather than passing vacuously. _REGISTRIES = { "db": DB_BLOBS, + "backend_config": BACKEND_CONFIGS, + "backend_creds": BACKEND_CREDS, + "backend_data": BACKEND_DATA, "api_request": API_REQUESTS, "api_response": API_RESPONSES, "gateway": GATEWAY_PAYLOADS, @@ -143,7 +209,14 @@ def test_parses_to_expected_values_and_types(self, name, regen): # Readers that ignore unknown fields, versus the one that must reject them. _TOLERANCE_CASES = [ (surface, name) - for surface in ("db", "api_response", "gateway") + for surface in ( + "db", + "backend_config", + "backend_creds", + "backend_data", + "api_response", + "gateway", + ) for name in sorted(_REGISTRIES[surface]) ] _FORBID_CASES = [("api_request", name) for name in sorted(API_REQUESTS)] diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index ee8667b33a..5cd7e21f49 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -12,7 +12,7 @@ subset of the `db/` fixtures, which outlive it because stored rows do. Registries and test classes follow the same surface order as `test_parsing.py`: -db, api_request, api_response, runner, gateway, proxy. +db, backend_config, backend_creds, backend_data, api_request, api_response, runner, gateway, proxy. """ import json @@ -24,12 +24,11 @@ from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.fleets import FleetNodesSpec from dstack._internal.server.utils.routers import CustomORJSONResponse -from tests._internal.pydantic_compat import factories +from tests._internal.pydantic_compat import backend_factories, factories from tests._internal.pydantic_compat.compare import assert_matches_fixture # Written to a `Text` column via `.json()`. DB_BLOBS: dict[str, Callable[[], CoreModel]] = { - "aws_creds": factories.aws_creds, "compute_group_provisioning_data": factories.compute_group_provisioning_data, "fleet_spec": factories.fleet_spec, "gateway_compute_configuration": factories.gateway_compute_configuration, @@ -53,6 +52,13 @@ "volume_provisioning_data": factories.volume_provisioning_data, } +# The three backend `Text` columns, all written with `.json()`. Split by column rather than by +# backend so a fixture is the exact bytes of one column: `config` and `auth` are written and read +# separately, and only recombined into an `XConfig` after both have been decoded. +BACKEND_STORED_CONFIGS = backend_factories.BACKEND_STORED_CONFIGS +BACKEND_CREDS = backend_factories.BACKEND_CREDS +BACKEND_DATA = backend_factories.BACKEND_DATA + # Sent by the API client as a request body — `body=X.json()`, 40 call sites in `api/server/`. # This is the new-CLI-against-old-server direction. API_REQUESTS: dict[str, Callable[[], CoreModel]] = { @@ -117,6 +123,9 @@ # `.json()` output and fails for reasons that have nothing to do with parsing. SURFACES: dict[str, tuple[dict[str, Callable[[], Any]], Callable[[Any], Union[bytes, str]]]] = { "db": (DB_BLOBS, lambda model: model.json()), + "backend_config": (BACKEND_STORED_CONFIGS, lambda model: model.json()), + "backend_creds": (BACKEND_CREDS, lambda model: model.json()), + "backend_data": (BACKEND_DATA, lambda model: model.json()), "api_request": (API_REQUESTS, lambda model: model.json()), "api_response": (API_RESPONSES, lambda model: bytes(CustomORJSONResponse(model).body)), "runner": (RUNNER_REQUESTS, lambda model: model.json()), From c1fc6b12a4406ff9886f96cdc96417c37538de97 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 11:08:40 +0500 Subject: [PATCH 12/15] Use production parse_apply_configuration --- .../parsing/config/dev_environment.types.json | 37 +++++++++--------- .../fixtures/parsing/config/fleet.types.json | 33 ++++++++-------- .../fixtures/parsing/config/gateway.input.yml | 18 +++++++++ .../parsing/config/gateway.types.json | 6 +++ .../parsing/config/gateway.values.json | 22 +++++++++++ .../parsing/config/service.types.json | 29 +++++++------- .../fixtures/parsing/config/task.types.json | 31 +++++++-------- .../fixtures/parsing/config/volume.types.json | 8 ++-- .../_internal/pydantic_compat/test_parsing.py | 35 +++++++++-------- .../pydantic_compat/test_rejection.py | 39 +++++++++++++------ 10 files changed, 159 insertions(+), 99 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json index 4de8822cb2..4d2dbc17ff 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/dev_environment.types.json @@ -1,21 +1,20 @@ { - "/": "DstackConfiguration", - "/__root__": "DevEnvironmentConfiguration", - "/__root__/env": "Env", - "/__root__/inactivity_duration": "Duration", - "/__root__/python": "PythonVersion", - "/__root__/resources": "ResourcesSpec", - "/__root__/resources/cpu": "CPUSpec", - "/__root__/resources/cpu/count": "Range[int]", - "/__root__/resources/disk": "DiskSpec", - "/__root__/resources/disk/size": "Range[Memory]", - "/__root__/resources/disk/size/min": "Memory", - "/__root__/resources/gpu": "GPUSpec", - "/__root__/resources/gpu/count": "Range[int]", - "/__root__/resources/gpu/memory": "Range[Memory]", - "/__root__/resources/gpu/memory/max": "Memory", - "/__root__/resources/gpu/memory/min": "Memory", - "/__root__/resources/memory": "Range[Memory]", - "/__root__/resources/memory/min": "Memory", - "/__root__/resources/shm_size": "Memory" + "/": "DevEnvironmentConfiguration", + "/env": "Env", + "/inactivity_duration": "Duration", + "/python": "PythonVersion", + "/resources": "ResourcesSpec", + "/resources/cpu": "CPUSpec", + "/resources/cpu/count": "Range[int]", + "/resources/disk": "DiskSpec", + "/resources/disk/size": "Range[Memory]", + "/resources/disk/size/min": "Memory", + "/resources/gpu": "GPUSpec", + "/resources/gpu/count": "Range[int]", + "/resources/gpu/memory": "Range[Memory]", + "/resources/gpu/memory/max": "Memory", + "/resources/gpu/memory/min": "Memory", + "/resources/memory": "Range[Memory]", + "/resources/memory/min": "Memory", + "/resources/shm_size": "Memory" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.types.json index 57665a1fde..8887ca1483 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/fleet.types.json @@ -1,19 +1,18 @@ { - "/": "DstackConfiguration", - "/__root__": "FleetConfiguration", - "/__root__/env": "Env", - "/__root__/idle_duration": "Duration", - "/__root__/nodes": "FleetNodesSpec", - "/__root__/resources": "ResourcesSpec", - "/__root__/resources/cpu": "CPUSpec", - "/__root__/resources/cpu/count": "Range[int]", - "/__root__/resources/disk": "DiskSpec", - "/__root__/resources/disk/size": "Range[Memory]", - "/__root__/resources/disk/size/min": "Memory", - "/__root__/resources/gpu": "GPUSpec", - "/__root__/resources/gpu/count": "Range[int]", - "/__root__/resources/gpu/memory": "Range[Memory]", - "/__root__/resources/gpu/memory/min": "Memory", - "/__root__/resources/memory": "Range[Memory]", - "/__root__/resources/memory/min": "Memory" + "/": "FleetConfiguration", + "/env": "Env", + "/idle_duration": "Duration", + "/nodes": "FleetNodesSpec", + "/resources": "ResourcesSpec", + "/resources/cpu": "CPUSpec", + "/resources/cpu/count": "Range[int]", + "/resources/disk": "DiskSpec", + "/resources/disk/size": "Range[Memory]", + "/resources/disk/size/min": "Memory", + "/resources/gpu": "GPUSpec", + "/resources/gpu/count": "Range[int]", + "/resources/gpu/memory": "Range[Memory]", + "/resources/gpu/memory/min": "Memory", + "/resources/memory": "Range[Memory]", + "/resources/memory/min": "Memory" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml new file mode 100644 index 0000000000..473703b10d --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.input.yml @@ -0,0 +1,18 @@ +# Exercises the gateway discriminator, its ACM certificate union arm, and the gateway-level router. +type: gateway +name: inference-gateway +default: true +backend: aws +region: us-east-1 +instance_type: t3.small +router: + type: sglang + policy: round_robin +domain: example.com +public_ip: false +certificate: + type: acm + arn: arn:aws:acm:us-east-1:123456789012:certificate/abc123 +replicas: 2 +tags: + env: prod diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json new file mode 100644 index 0000000000..83690c7e9c --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.types.json @@ -0,0 +1,6 @@ +{ + "/": "GatewayConfiguration", + "/backend": "BackendType", + "/certificate": "ACMGatewayCertificate", + "/router": "SGLangGatewayRouterConfig" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json new file mode 100644 index 0000000000..c4a6c91adc --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/gateway.values.json @@ -0,0 +1,22 @@ +{ + "backend": "aws", + "certificate": { + "arn": "arn:aws:acm:us-east-1:123456789012:certificate/abc123", + "type": "acm" + }, + "default": true, + "domain": "example.com", + "instance_type": "t3.small", + "name": "inference-gateway", + "public_ip": false, + "region": "us-east-1", + "replicas": 2, + "router": { + "policy": "round_robin", + "type": "sglang" + }, + "tags": { + "env": "prod" + }, + "type": "gateway" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json index 5bef881a81..b4d29cc655 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/service.types.json @@ -1,17 +1,16 @@ { - "/": "DstackConfiguration", - "/__root__": "ServiceConfiguration", - "/__root__/env": "Env", - "/__root__/model": "OpenAIChatModel", - "/__root__/port": "PortMapping", - "/__root__/resources": "ResourcesSpec", - "/__root__/resources/cpu": "CPUSpec", - "/__root__/resources/cpu/count": "Range[int]", - "/__root__/resources/disk": "DiskSpec", - "/__root__/resources/disk/size": "Range[Memory]", - "/__root__/resources/disk/size/min": "Memory", - "/__root__/resources/gpu": "GPUSpec", - "/__root__/resources/gpu/count": "Range[int]", - "/__root__/resources/memory": "Range[Memory]", - "/__root__/resources/memory/min": "Memory" + "/": "ServiceConfiguration", + "/env": "Env", + "/model": "OpenAIChatModel", + "/port": "PortMapping", + "/resources": "ResourcesSpec", + "/resources/cpu": "CPUSpec", + "/resources/cpu/count": "Range[int]", + "/resources/disk": "DiskSpec", + "/resources/disk/size": "Range[Memory]", + "/resources/disk/size/min": "Memory", + "/resources/gpu": "GPUSpec", + "/resources/gpu/count": "Range[int]", + "/resources/memory": "Range[Memory]", + "/resources/memory/min": "Memory" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.types.json index a3d384fa38..a29b85f05f 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/task.types.json @@ -1,18 +1,17 @@ { - "/": "DstackConfiguration", - "/__root__": "TaskConfiguration", - "/__root__/env": "Env", - "/__root__/env/__root__/B": "EnvSentinel", - "/__root__/max_duration": "Duration", - "/__root__/python": "PythonVersion", - "/__root__/resources": "ResourcesSpec", - "/__root__/resources/cpu": "CPUSpec", - "/__root__/resources/cpu/count": "Range[int]", - "/__root__/resources/disk": "DiskSpec", - "/__root__/resources/disk/size": "Range[Memory]", - "/__root__/resources/disk/size/min": "Memory", - "/__root__/resources/gpu": "GPUSpec", - "/__root__/resources/gpu/count": "Range[int]", - "/__root__/resources/memory": "Range[Memory]", - "/__root__/resources/memory/min": "Memory" + "/": "TaskConfiguration", + "/env": "Env", + "/env/__root__/B": "EnvSentinel", + "/max_duration": "Duration", + "/python": "PythonVersion", + "/resources": "ResourcesSpec", + "/resources/cpu": "CPUSpec", + "/resources/cpu/count": "Range[int]", + "/resources/disk": "DiskSpec", + "/resources/disk/size": "Range[Memory]", + "/resources/disk/size/min": "Memory", + "/resources/gpu": "GPUSpec", + "/resources/gpu/count": "Range[int]", + "/resources/memory": "Range[Memory]", + "/resources/memory/min": "Memory" } diff --git a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json index beb7d52a31..075e76beb0 100644 --- a/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json +++ b/src/tests/_internal/pydantic_compat/fixtures/parsing/config/volume.types.json @@ -1,7 +1,5 @@ { - "/": "DstackConfiguration", - "/__root__": "VolumeConfiguration", - "/__root__/__root__": "AWSVolumeConfiguration", - "/__root__/__root__/backend": "BackendType", - "/__root__/__root__/size": "Memory" + "/": "AWSVolumeConfiguration", + "/backend": "BackendType", + "/size": "Memory" } diff --git a/src/tests/_internal/pydantic_compat/test_parsing.py b/src/tests/_internal/pydantic_compat/test_parsing.py index 13f3dddcc7..f3c7c59093 100644 --- a/src/tests/_internal/pydantic_compat/test_parsing.py +++ b/src/tests/_internal/pydantic_compat/test_parsing.py @@ -19,13 +19,13 @@ """ import json -from typing import Any +from typing import Any, Callable import pytest import yaml from dstack._internal.core.backends.nebius.models import NebiusOfferBackendData -from dstack._internal.core.models.configurations import DstackConfiguration +from dstack._internal.core.models.configurations import parse_apply_configuration from dstack._internal.core.models.profiles import ProfilesConfig from dstack._internal.proxy.gateway.schemas.registry import ( RegisterEntrypointRequest, @@ -84,18 +84,19 @@ def _derived_registry(surface: str) -> dict[str, Any]: BACKEND_CONFIGS = backend_factories.BACKEND_CONFIG_MODELS BACKEND_CREDS = backend_factories.BACKEND_CREDS_MODELS -# Parsed from user-authored YAML with extra="forbid". The only surface whose inputs are `.yml`, -# because the sugar pinned here is a YAML phenomenon: `python: 3.10` is a float that survives only -# via number->str coercion, and `16GB..` / `2..8` / an env list are shorthands people type by hand. -# `DstackConfiguration` dispatches every `.dstack.yml` type through its `__root__` union, so one -# entry point covers task, service, dev-environment, fleet, volume, and gateway. -CONFIGS: dict[str, Any] = { - "dev_environment": DstackConfiguration, - "fleet": DstackConfiguration, - "profiles": ProfilesConfig, - "service": DstackConfiguration, - "task": DstackConfiguration, - "volume": DstackConfiguration, +# Parsed from user-authored YAML through the same entry points as production. Apply configurations +# go through `parse_apply_configuration`, whose two-pass dispatch first identifies a configuration +# permissively, then validates it strictly and reparses volumes by backend. Profiles have their own +# production entry point. The `.yml` inputs preserve YAML-only shapes such as numeric Python +# versions, duration/range shorthand, and environment-variable lists. +CONFIG_PARSERS: dict[str, Callable[[Any], Any]] = { + "dev_environment": parse_apply_configuration, + "fleet": parse_apply_configuration, + "gateway": parse_apply_configuration, + "profiles": ProfilesConfig.parse_obj, + "service": parse_apply_configuration, + "task": parse_apply_configuration, + "volume": parse_apply_configuration, } # Responses the server reads back from the runner (shim), permissively: a newer runner may add @@ -279,9 +280,11 @@ def test_parses_to_expected_values_and_types(self, name, regen): class TestConfigParsing: - @pytest.mark.parametrize("name", sorted(CONFIGS)) + @pytest.mark.parametrize("name", sorted(CONFIG_PARSERS)) def test_parses_to_expected_values_and_types(self, name, regen): - model = parse_forbid_extra(CONFIGS[name], _load_yaml_input("config", name)) + parser = CONFIG_PARSERS[name] + data = _load_yaml_input("config", name) + model = parser(data) _assert_parses("config", name, model, regen) diff --git a/src/tests/_internal/pydantic_compat/test_rejection.py b/src/tests/_internal/pydantic_compat/test_rejection.py index d644ce218a..2099615dd5 100644 --- a/src/tests/_internal/pydantic_compat/test_rejection.py +++ b/src/tests/_internal/pydantic_compat/test_rejection.py @@ -16,7 +16,8 @@ import yaml from pydantic import ValidationError -from dstack._internal.core.models.configurations import DstackConfiguration +from dstack._internal.core.errors import ConfigurationError +from dstack._internal.core.models.configurations import parse_apply_configuration from dstack._internal.server.schemas.volumes import CreateVolumeRequest from tests._internal.pydantic_compat.compat import parse_forbid_extra @@ -27,41 +28,57 @@ def _task(extra_yaml: str) -> Any: return yaml.safe_load(_VALID_TASK + extra_yaml) -# name -> (parser, input). Each input is rejected by v1 today; the test pins that it stays rejected. -REJECTED_INPUTS: dict[str, tuple[Callable, Any]] = { +# name -> (parser, input, exception). Each input is rejected by v1 today; the test pins that it +# stays rejected. The production configuration parser wraps pydantic's error in ConfigurationError. +REJECTED_INPUTS: dict[str, tuple[Callable, Any, type[Exception]]] = { # A mistyped key must not be silently ignored — this is the whole point of `extra="forbid"`, # and the v2 `CoreModel` reaches it through a per-call override that could leak. - "config_unknown_key": (DstackConfiguration.parse_obj, _task("comands: [oops]\n")), + "config_unknown_key": ( + parse_apply_configuration, + _task("comands: [oops]\n"), + ConfigurationError, + ), "request_unknown_key": ( CreateVolumeRequest.parse_obj, { "configuration": {"type": "volume", "name": "v", "backend": "aws", "region": "r"}, "unexpected": 1, }, + ValidationError, ), # Reversed ranges: caught by a validator, not by the type, so a dropped validator during the # `__get_pydantic_core_schema__` port would make these start passing. "resources_cpu_reversed_range": ( - DstackConfiguration.parse_obj, + parse_apply_configuration, _task("resources:\n cpu: 8..2\n"), + ConfigurationError, ), "resources_memory_reversed_range": ( - DstackConfiguration.parse_obj, + parse_apply_configuration, _task("resources:\n memory: 32GB..8GB\n"), + ConfigurationError, ), # Custom-type parsing: `Duration.parse` rejects unknown units and non-numeric input. - "duration_bad_unit": (DstackConfiguration.parse_obj, _task("max_duration: 5 years\n")), - # An unknown discriminator tag. `AnyDstackConfiguration` declares `discriminator="type"`, so + "duration_bad_unit": ( + parse_apply_configuration, + _task("max_duration: 5 years\n"), + ConfigurationError, + ), + # An unknown discriminator tag. `BaseApplyConfiguration` declares `discriminator="type"`, so # the error names the tag instead of accumulating one failure per arm. - "unknown_config_type": (DstackConfiguration.parse_obj, {"type": "not-a-real-type"}), + "unknown_config_type": ( + parse_apply_configuration, + {"type": "not-a-real-type"}, + ConfigurationError, + ), } class TestRejectionParity: @pytest.mark.parametrize("name", sorted(REJECTED_INPUTS)) def test_still_rejected(self, name): - parser, data = REJECTED_INPUTS[name] - with pytest.raises(ValidationError): + parser, data, exception = REJECTED_INPUTS[name] + with pytest.raises(exception): parser(data) From 8bfb6c8c84d91438223b4c446fe871c11f65c4ab Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 11:51:21 +0500 Subject: [PATCH 13/15] Test custom types --- .../pydantic_compat/test_custom_types.py | 411 ++++++++++++++++++ .../pydantic_compat/test_field_parsing.py | 326 ++++++++++++++ .../pydantic_compat/test_serialization.py | 174 +++++++- 3 files changed, 909 insertions(+), 2 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/test_custom_types.py create mode 100644 src/tests/_internal/pydantic_compat/test_field_parsing.py diff --git a/src/tests/_internal/pydantic_compat/test_custom_types.py b/src/tests/_internal/pydantic_compat/test_custom_types.py new file mode 100644 index 0000000000..cdcade6556 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/test_custom_types.py @@ -0,0 +1,411 @@ +""" +Focused parity tests for Pydantic mechanisms that broad boundary fixtures only exercise +incidentally. + +Unlike the wire fixtures, these cases are intentionally small: when a validator or generic +specialization changes during the v2 migration, the failing case should name the mechanism that +drifted. +""" + +import json +from typing import Any, Callable + +import pytest +from pydantic import ValidationError, parse_obj_as + +from dstack._internal.core.models.common import Duration +from dstack._internal.core.models.gateways import GatewaySpec +from dstack._internal.core.models.resources import ( + ComputeCapability, + CPUSpec, + DiskSpec, + GPUSpec, + Memory, + Range, +) +from dstack._internal.core.models.volumes import VolumeSpec +from dstack.plugins.builtin.rest_plugin._models import ( + FleetSpecRequest, + FleetSpecResponse, + GatewaySpecRequest, + GatewaySpecResponse, + RunSpecRequest, + RunSpecResponse, + VolumeSpecRequest, + VolumeSpecResponse, +) +from tests._internal.pydantic_compat import factories +from tests._internal.pydantic_compat.compare import canonicalize, class_name, type_map + + +def _volume_spec() -> VolumeSpec: + return VolumeSpec(configuration=factories.volume_configuration()) + + +def _gateway_spec() -> GatewaySpec: + return GatewaySpec(configuration=factories.gateway_configuration()) + + +_REST_GENERIC_CASES = [ + pytest.param( + RunSpecRequest, + RunSpecResponse, + factories.run_spec, + "RunSpec", + id="run", + ), + pytest.param( + FleetSpecRequest, + FleetSpecResponse, + factories.fleet_spec, + "FleetSpec", + id="fleet", + ), + pytest.param( + VolumeSpecRequest, + VolumeSpecResponse, + _volume_spec, + "VolumeSpec", + id="volume", + ), + pytest.param( + GatewaySpecRequest, + GatewaySpecResponse, + _gateway_spec, + "GatewaySpec", + id="gateway", + ), +] + + +class TestRESTPluginGenericModels: + @pytest.mark.parametrize( + ("request_model", "response_model", "spec_factory", "expected_spec_type"), + _REST_GENERIC_CASES, + ) + def test_specialization_preserves_spec_type_and_production_request_body( + self, + request_model: Any, + response_model: Any, + spec_factory: Callable[[], Any], + expected_spec_type: str, + ): + """ + Pydantic v1 represents these specializations as typing aliases and adds + `__orig_class__` after construction. Production sends `.dict()` to requests, relying on + the request model's override to remove that non-JSON value. + """ + spec = spec_factory() + request = request_model(user="alice", project="main", spec=spec) + + assert class_name(request.spec) == expected_spec_type + body = request.dict() + assert "__orig_class__" not in body + + # Match the exact production handoff: requests receives a plain, stdlib-JSON-safe dict. + encoded_body = json.dumps(body) + assert canonicalize(json.dumps(body["spec"])) == canonicalize(spec.json()) + + # The generic must also choose the same type when its value arrives as an untyped payload. + reparsed_request = request_model(**json.loads(encoded_body)) + assert class_name(reparsed_request.spec) == expected_spec_type + + # This is the production response path in CustomApplyPolicy._on_apply. + response = response_model(spec=body["spec"], error=None) + assert class_name(response.spec) == expected_spec_type + assert response.error is None + + +_RANGE_CASES = [ + pytest.param(Range[int], 2, {"min": 2, "max": 2}, int, id="int-scalar"), + pytest.param(Range[int], "2..8", {"min": 2, "max": 8}, int, id="int-closed"), + pytest.param(Range[int], "2..", {"min": 2, "max": None}, int, id="int-open-max"), + pytest.param( + Range[int], + {"min": None, "max": 8}, + {"min": None, "max": 8}, + int, + id="int-mapping", + ), + pytest.param( + Range[Memory], + "512MB", + {"min": 0.5, "max": 0.5}, + Memory, + id="memory-scalar", + ), + pytest.param( + Range[Memory], + "512MB..1TB", + {"min": 0.5, "max": 1024.0}, + Memory, + id="memory-closed", + ), + pytest.param( + Range[Memory], + {"min": "16GB", "max": None}, + {"min": 16.0, "max": None}, + Memory, + id="memory-mapping", + ), +] + + +class TestRangeGenericModel: + @pytest.mark.parametrize(("range_type", "raw", "expected", "bound_type"), _RANGE_CASES) + def test_specialization_preserves_bound_types_and_json( + self, + range_type: Any, + raw: Any, + expected: dict[str, Any], + bound_type: type, + ): + value = parse_obj_as(range_type, raw) + + assert type(value) is range_type + assert value.dict() == expected + for bound in (value.min, value.max): + if bound is not None: + assert type(bound) is bound_type + assert canonicalize(value.json()) == canonicalize(json.dumps(expected)) + + @pytest.mark.parametrize("raw", ["..", "8..2", "1...3"]) + def test_invalid_int_ranges_stay_rejected(self, raw: str): + with pytest.raises(ValidationError): + parse_obj_as(Range[int], raw) + + @pytest.mark.parametrize("raw", ["...", "2TB..1TB"]) + def test_invalid_memory_ranges_stay_rejected(self, raw: str): + with pytest.raises(ValidationError): + parse_obj_as(Range[Memory], raw) + + +_SCALAR_CASES = [ + pytest.param(Duration, 90, 90, Duration, 90, id="duration-int"), + pytest.param(Duration, 1.9, 1, Duration, 1, id="duration-float-truncates"), + pytest.param(Duration, "90", 90, Duration, 90, id="duration-numeric-string"), + pytest.param(Duration, "2 h", 7200, Duration, 7200, id="duration-unit"), + pytest.param(Duration, "1w", 604800, Duration, 604800, id="duration-week"), + pytest.param(Memory, 1, 1.0, Memory, 1.0, id="memory-int"), + pytest.param(Memory, 1.5, 1.5, Memory, 1.5, id="memory-float"), + pytest.param(Memory, "512MB", 0.5, Memory, 0.5, id="memory-mb"), + pytest.param(Memory, "16 Gb", 16.0, Memory, 16.0, id="memory-gb"), + pytest.param(Memory, "1 TB ", 1024.0, Memory, 1024.0, id="memory-tb"), + pytest.param( + ComputeCapability, + "3.5", + (3, 5), + tuple, + [3, 5], + id="compute-capability-string", + ), + pytest.param( + ComputeCapability, + 8.0, + (8, 0), + tuple, + [8, 0], + id="compute-capability-float", + ), + pytest.param( + ComputeCapability, + [7, 5], + (7, 5), + tuple, + [7, 5], + id="compute-capability-list", + ), + pytest.param( + ComputeCapability, + (9, 0), + (9, 0), + tuple, + [9, 0], + id="compute-capability-tuple", + ), +] + + +class TestCustomScalarTypes: + @pytest.mark.parametrize( + ("scalar_type", "raw", "expected", "expected_type", "expected_json"), + _SCALAR_CASES, + ) + def test_parsed_value_type_and_json_stay_stable( + self, + scalar_type: Any, + raw: Any, + expected: Any, + expected_type: type, + expected_json: Any, + ): + value = parse_obj_as(scalar_type, raw) + + assert value == expected + assert type(value) is expected_type + assert canonicalize(json.dumps(value)) == canonicalize(json.dumps(expected_json)) + + @pytest.mark.parametrize( + ("scalar_type", "raw"), + [ + pytest.param(Duration, "5 years", id="duration-bad-unit"), + pytest.param(Duration, {}, id="duration-bad-type"), + pytest.param(Memory, "1.5xb", id="memory-bad-unit"), + pytest.param(Memory, {}, id="memory-bad-type"), + pytest.param(ComputeCapability, "3.5.1", id="compute-capability-length"), + pytest.param(ComputeCapability, "3.x", id="compute-capability-component"), + ], + ) + def test_invalid_values_stay_rejected(self, scalar_type: Any, raw: Any): + with pytest.raises(ValidationError): + parse_obj_as(scalar_type, raw) + + +_CUSTOM_MODEL_CASES = [ + pytest.param( + CPUSpec, + 1, + {"arch": None, "count": {"min": 1, "max": 1}}, + {"/": "CPUSpec", "/count": "Range[int]"}, + id="cpu-scalar", + ), + pytest.param( + CPUSpec, + "x86:2", + {"arch": "x86", "count": {"min": 2, "max": 2}}, + {"/": "CPUSpec", "/arch": "CPUArchitecture", "/count": "Range[int]"}, + id="cpu-architecture-and-count", + ), + pytest.param( + CPUSpec, + "2..:ARM", + {"arch": "arm", "count": {"min": 2, "max": None}}, + {"/": "CPUSpec", "/arch": "CPUArchitecture", "/count": "Range[int]"}, + id="cpu-open-range-and-architecture", + ), + pytest.param( + CPUSpec, + {"min": 1, "max": 2}, + {"arch": None, "count": {"min": 1, "max": 2}}, + {"/": "CPUSpec", "/count": "Range[int]"}, + id="cpu-legacy-range-mapping", + ), + pytest.param( + GPUSpec, + "1", + { + "vendor": None, + "name": None, + "count": {"min": 1, "max": 1}, + "memory": None, + "total_memory": None, + "compute_capability": None, + }, + {"/": "GPUSpec", "/count": "Range[int]"}, + id="gpu-count", + ), + pytest.param( + GPUSpec, + "A10,A10G:2", + { + "vendor": None, + "name": ["A10", "A10G"], + "count": {"min": 2, "max": 2}, + "memory": None, + "total_memory": None, + "compute_capability": None, + }, + {"/": "GPUSpec", "/count": "Range[int]"}, + id="gpu-names-and-count", + ), + pytest.param( + GPUSpec, + "tpu:v5p-8:2:16GB..32GB", + { + "vendor": "google", + "name": ["v5p-8"], + "count": {"min": 2, "max": 2}, + "memory": {"min": 16.0, "max": 32.0}, + "total_memory": None, + "compute_capability": None, + }, + { + "/": "GPUSpec", + "/vendor": "AcceleratorVendor", + "/count": "Range[int]", + "/memory": "Range[Memory]", + "/memory/min": "Memory", + "/memory/max": "Memory", + }, + id="gpu-tpu-alias-and-memory-range", + ), + pytest.param( + GPUSpec, + "tt:n300:2", + { + "vendor": "tenstorrent", + "name": ["n300"], + "count": {"min": 2, "max": 2}, + "memory": None, + "total_memory": None, + "compute_capability": None, + }, + { + "/": "GPUSpec", + "/vendor": "AcceleratorVendor", + "/count": "Range[int]", + }, + id="gpu-tenstorrent-alias", + ), + pytest.param( + DiskSpec, + "100GB..", + {"size": {"min": 100.0, "max": None}}, + {"/": "DiskSpec", "/size": "Range[Memory]", "/size/min": "Memory"}, + id="disk-scalar", + ), + pytest.param( + DiskSpec, + {"size": {"min": "1TB", "max": "2TB"}}, + {"size": {"min": 1024.0, "max": 2048.0}}, + { + "/": "DiskSpec", + "/size": "Range[Memory]", + "/size/min": "Memory", + "/size/max": "Memory", + }, + id="disk-mapping", + ), +] + + +class TestCustomModelTypes: + @pytest.mark.parametrize( + ("model_type", "raw", "expected_json", "expected_types"), + _CUSTOM_MODEL_CASES, + ) + def test_custom_parser_preserves_values_types_and_json( + self, + model_type: Any, + raw: Any, + expected_json: dict[str, Any], + expected_types: dict[str, str], + ): + value = parse_obj_as(model_type, raw) + + assert canonicalize(value.json()) == canonicalize(json.dumps(expected_json)) + assert type_map(value) == expected_types + + @pytest.mark.parametrize( + ("model_type", "raw"), + [ + pytest.param(CPUSpec, "arm:", id="cpu-empty-token"), + pytest.param(CPUSpec, "2:arm:2", id="cpu-count-conflict"), + pytest.param(GPUSpec, "A100:", id="gpu-empty-token"), + pytest.param(GPUSpec, "Nvidia:A100:2:AMD", id="gpu-vendor-conflict"), + pytest.param(DiskSpec, "...", id="disk-invalid-range"), + ], + ) + def test_invalid_custom_model_values_stay_rejected(self, model_type: Any, raw: Any): + with pytest.raises(ValidationError): + parse_obj_as(model_type, raw) diff --git a/src/tests/_internal/pydantic_compat/test_field_parsing.py b/src/tests/_internal/pydantic_compat/test_field_parsing.py new file mode 100644 index 0000000000..dabf94e81a --- /dev/null +++ b/src/tests/_internal/pydantic_compat/test_field_parsing.py @@ -0,0 +1,326 @@ +"""Focused parity tests for user-facing fields with custom parsing or normalization.""" + +import json +from typing import Any + +import pytest +import yaml + +from dstack._internal.core.errors import ConfigurationError +from dstack._internal.core.models.common import Duration +from dstack._internal.core.models.configurations import ( + PythonVersion, + parse_apply_configuration, +) +from dstack._internal.core.models.envs import EnvSentinel +from dstack._internal.core.models.unix import UnixUser +from tests._internal.pydantic_compat.compare import class_name + + +def _task(**overrides: Any) -> dict[str, Any]: + return {"type": "task", "commands": ["echo hi"], **overrides} + + +def _service(**overrides: Any) -> dict[str, Any]: + return {"type": "service", "commands": ["echo hi"], "port": 8000, **overrides} + + +class TestPythonVersionFieldParsing: + def test_yaml_310_float_is_recovered_as_python_310(self): + data = yaml.safe_load( + """ + type: task + commands: [echo hi] + python: 3.10 + """ + ) + assert data["python"] == 3.1 # This is the lossy value the field validator receives. + + config = parse_apply_configuration(data) + + assert config.python is PythonVersion.PY310 + assert json.loads(config.json())["python"] == "3.10" + + def test_yaml_311_float_stays_python_311(self): + data = yaml.safe_load( + """ + type: task + commands: [echo hi] + python: 3.11 + """ + ) + + config = parse_apply_configuration(data) + + assert config.python is PythonVersion.PY311 + + +class TestEnvironmentFieldParsing: + def test_list_is_normalized_to_mapping_and_missing_value_becomes_sentinel(self): + config = parse_apply_configuration(_task(env=["A=1", "B", "EMPTY="])) + + assert config.env["A"] == "1" + assert config.env["EMPTY"] == "" + assert config.env["B"] == EnvSentinel(key="B") + assert class_name(config.env["B"]) == "EnvSentinel" + assert json.loads(config.json())["env"] == { + "A": "1", + "B": {"key": "B"}, + "EMPTY": "", + } + + @pytest.mark.parametrize( + "env", + [ + pytest.param(["A=1", "A=2"], id="duplicate"), + pytest.param([None], id="non-string-item"), + ], + ) + def test_invalid_list_entries_stay_rejected(self, env: list[Any]): + with pytest.raises(ConfigurationError): + parse_apply_configuration(_task(env=env)) + + +class TestPortFieldParsing: + def test_task_port_variants_are_normalized_to_port_mappings(self): + config = parse_apply_configuration(_task(ports=[8080, "8081:81", "*:82"])) + + assert all(class_name(port) == "PortMapping" for port in config.ports) + assert [port.dict() for port in config.ports] == [ + {"local_port": 8080, "container_port": 8080}, + {"local_port": 8081, "container_port": 81}, + {"local_port": None, "container_port": 82}, + ] + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + pytest.param( + 8000, + {"local_port": 80, "container_port": 8000}, + id="service-container-port", + ), + pytest.param( + "8080:8000", + {"local_port": 8080, "container_port": 8000}, + id="service-mapping", + ), + ], + ) + def test_service_port_variants_are_normalized(self, raw: Any, expected: dict[str, int]): + config = parse_apply_configuration(_service(port=raw)) + + assert class_name(config.port) == "PortMapping" + assert config.port.dict() == expected + + +class TestMountPointFieldParsing: + def test_string_arms_select_volume_and_instance_mount_models(self): + config = parse_apply_configuration( + _task(volumes=["my-volume:/mnt/data", "/host/cache:/cache"]) + ) + + assert class_name(config.volumes[0]) == "VolumeMountPoint" + assert config.volumes[0].dict() == {"name": "my-volume", "path": "/mnt/data"} + assert class_name(config.volumes[1]) == "InstanceMountPoint" + assert config.volumes[1].dict() == { + "instance_path": "/host/cache", + "path": "/cache", + "optional": False, + } + + +class TestFileMappingFieldParsing: + def test_unix_and_windows_sources_keep_colons_in_the_source_path(self): + config = parse_apply_configuration( + _task( + files=[ + "data:/workspace/data", + r"C:\data:/workspace/windows", + ] + ) + ) + + assert all(class_name(mapping) == "FilePathMapping" for mapping in config.files) + assert [mapping.dict() for mapping in config.files] == [ + {"local_path": "data", "path": "/workspace/data"}, + {"local_path": r"C:\data", "path": "/workspace/windows"}, + ] + + +_REPO_CASES = [ + pytest.param( + ".", + {"local_path": ".", "url": None, "path": "."}, + id="local-default-target", + ), + pytest.param( + "src:/workspace/src", + {"local_path": "src", "url": None, "path": "/workspace/src"}, + id="local-explicit-target", + ), + pytest.param( + "https://github.com/dstackai/dstack.git:/workspace/repo", + { + "local_path": None, + "url": "https://github.com/dstackai/dstack.git", + "path": "/workspace/repo", + }, + id="https-url-with-target", + ), + pytest.param( + "git@github.com:dstackai/dstack.git:/workspace/repo", + { + "local_path": None, + "url": "git@github.com:dstackai/dstack.git", + "path": "/workspace/repo", + }, + id="ssh-url-with-colon-and-target", + ), + pytest.param( + r"C:\src:/workspace/src", + {"local_path": r"C:\src", "url": None, "path": "/workspace/src"}, + id="windows-source-with-target", + ), +] + + +class TestRepoFieldParsing: + @pytest.mark.parametrize(("raw", "expected"), _REPO_CASES) + def test_repo_shorthand_is_normalized_without_splitting_url_or_drive_colons( + self, raw: str, expected: dict[str, Any] + ): + config = parse_apply_configuration(_task(repos=[raw])) + repo = config.repos[0] + + assert repo.local_path == expected["local_path"] + assert repo.url == expected["url"] + assert repo.path == expected["path"] + + +_UNIX_USER_CASES = [ + pytest.param( + "ubuntu", + {"uid": None, "gid": None, "username": "ubuntu", "groupname": None}, + id="username", + ), + pytest.param( + "1000", + {"uid": 1000, "gid": None, "username": None, "groupname": None}, + id="uid", + ), + pytest.param( + "ubuntu:staff", + {"uid": None, "gid": None, "username": "ubuntu", "groupname": "staff"}, + id="username-groupname", + ), + pytest.param( + "1000:100", + {"uid": 1000, "gid": 100, "username": None, "groupname": None}, + id="uid-gid", + ), +] + + +class TestUnixUserFieldParsing: + @pytest.mark.parametrize(("raw", "parsed"), _UNIX_USER_CASES) + def test_user_field_runs_unix_user_validation_without_changing_wire_value( + self, raw: str, parsed: dict[str, Any] + ): + config = parse_apply_configuration(_task(user=raw)) + + assert config.user == raw + assert UnixUser.parse(config.user).dict() == parsed + + @pytest.mark.parametrize( + "raw", + [ + pytest.param(":staff", id="empty-user"), + pytest.param("ubuntu:", id="empty-group"), + pytest.param("-1", id="negative-uid"), + pytest.param("user:group:extra", id="too-many-parts"), + ], + ) + def test_invalid_user_stays_rejected(self, raw: str): + with pytest.raises(ConfigurationError): + parse_apply_configuration(_task(user=raw)) + + +_DURATION_SENTINEL_CASES = [ + pytest.param("max_duration", "off", "off", str, id="max-off"), + pytest.param("max_duration", False, "off", str, id="max-false"), + pytest.param("stop_duration", True, None, type(None), id="stop-true"), + pytest.param("idle_duration", "off", -1, int, id="idle-off"), + pytest.param("idle_duration", False, -1, int, id="idle-false"), + pytest.param("idle_duration", -1, -1, int, id="idle-legacy-minus-one"), + pytest.param("max_duration", "2h", 7200, Duration, id="max-duration"), +] + + +class TestDurationSentinelFieldParsing: + @pytest.mark.parametrize( + ("field", "raw", "expected", "expected_type"), + _DURATION_SENTINEL_CASES, + ) + def test_duration_sentinel_normalization( + self, + field: str, + raw: Any, + expected: Any, + expected_type: type, + ): + config = parse_apply_configuration(_task(**{field: raw})) + value = getattr(config, field) + + assert value == expected + assert type(value) is expected_type + + +class TestFleetNodesFieldParsing: + def test_range_shorthand_sets_min_target_and_max(self): + config = parse_apply_configuration({"type": "fleet", "nodes": "1..3"}) + + assert class_name(config.nodes) == "FleetNodesSpec" + assert config.nodes.min == 1 + assert config.nodes.target == 1 + assert config.nodes.max == 3 + + +class TestServiceModelFieldParsing: + def test_string_shorthand_becomes_openai_model(self): + config = parse_apply_configuration(_service(model="llama")) + + assert class_name(config.model) == "OpenAIChatModel" + assert config.model.dict() == { + "type": "chat", + "name": "llama", + "format": "openai", + "prefix": "/v1", + } + + def test_tagged_tgi_mapping_stays_tgi_model(self): + config = parse_apply_configuration( + _service( + model={ + "type": "chat", + "name": "llama", + "format": "tgi", + "chat_template": "{{ prompt }}", + "eos_token": "", + } + ) + ) + + assert class_name(config.model) == "TGIChatModel" + assert config.model.format == "tgi" + + +class TestGatewayReferenceFieldParsing: + def test_project_qualified_string_becomes_entity_reference(self): + config = parse_apply_configuration(_service(gateway="other-project/shared-gateway")) + + assert class_name(config.gateway) == "EntityReference" + assert config.gateway.dict() == { + "project": "other-project", + "name": "shared-gateway", + } diff --git a/src/tests/_internal/pydantic_compat/test_serialization.py b/src/tests/_internal/pydantic_compat/test_serialization.py index 5cd7e21f49..2669ad1c51 100644 --- a/src/tests/_internal/pydantic_compat/test_serialization.py +++ b/src/tests/_internal/pydantic_compat/test_serialization.py @@ -18,14 +18,19 @@ import json from typing import Any, Callable, Union +import gpuhunt import pytest from pydantic import BaseModel from dstack._internal.core.models.common import CoreModel -from dstack._internal.core.models.fleets import FleetNodesSpec +from dstack._internal.core.models.configurations import TaskConfiguration +from dstack._internal.core.models.fleets import FleetConfiguration, FleetNodesSpec +from dstack._internal.core.models.repos.remote import RemoteRunRepoData +from dstack._internal.core.models.resources import CPUSpec, Range, ResourcesSpec +from dstack._internal.core.models.volumes import RunpodVolumeConfiguration, VolumeSpec from dstack._internal.server.utils.routers import CustomORJSONResponse from tests._internal.pydantic_compat import backend_factories, factories -from tests._internal.pydantic_compat.compare import assert_matches_fixture +from tests._internal.pydantic_compat.compare import assert_matches_fixture, canonicalize # Written to a `Text` column via `.json()`. DB_BLOBS: dict[str, Callable[[], CoreModel]] = { @@ -170,7 +175,172 @@ def test_target_is_omitted_when_it_equals_min(self): def test_target_is_kept_when_it_differs_from_min(self): assert FleetNodesSpec(min=1, target=5, max=5).dict()["target"] == 5 + @pytest.mark.parametrize( + ("nodes", "expected"), + [ + pytest.param( + FleetNodesSpec(min=1, target=1, max=1), + {"min": 1, "max": 1}, + id="target-equals-min", + ), + pytest.param( + FleetNodesSpec(min=1, target=5, max=5), + {"min": 1, "target": 5, "max": 5}, + id="target-differs-from-min", + ), + ], + ) + def test_dict_and_json_apply_the_same_override(self, nodes, expected): + assert nodes.dict() == expected + assert json.loads(nodes.json()) == expected + + @pytest.mark.parametrize( + ("nodes", "expected"), + [ + pytest.param( + FleetNodesSpec(min=1, target=1, max=1), + {"min": 1, "max": 1}, + id="target-equals-min", + ), + pytest.param( + FleetNodesSpec(min=1, target=5, max=5), + {"min": 1, "target": 5, "max": 5}, + id="target-differs-from-min", + ), + ], + ) + def test_override_is_applied_when_nested(self, nodes, expected): + configuration = FleetConfiguration(nodes=nodes) + + assert configuration.dict()["nodes"] == expected + assert json.loads(configuration.json())["nodes"] == expected + def test_the_api_fixture_exercises_the_hack(self): nodes = factories.fleet().spec.configuration.nodes assert nodes is not None assert nodes.min == nodes.target, "fixture must hit the target == min branch" + + +class TestResourcesSpecCPUCompatHack: + @pytest.mark.parametrize( + ("arch", "expected"), + [ + pytest.param(None, {"min": 2, "max": 8}, id="unspecified-architecture"), + pytest.param( + gpuhunt.CPUArchitecture.X86, + {"min": 2, "max": 8}, + id="x86-architecture", + ), + pytest.param( + gpuhunt.CPUArchitecture.ARM, + {"arch": "arm", "count": {"min": 2, "max": 8}}, + id="arm-architecture", + ), + ], + ) + def test_dict_and_json_apply_the_same_override(self, arch, expected): + resources = ResourcesSpec( + cpu=CPUSpec(arch=arch, count=Range[int](min=2, max=8)), + ) + + assert json.loads(resources.json())["cpu"] == expected + # CoreModel.json() must call the overridden dict(); this assertion would expose a drift + # even if only one of the two methods retained the compatibility rewrite. + assert canonicalize(json.dumps(resources.dict()["cpu"])) == canonicalize( + json.dumps(expected) + ) + + @pytest.mark.parametrize( + ("arch", "expected"), + [ + pytest.param(None, {"min": 2, "max": 8}, id="unspecified-architecture"), + pytest.param( + gpuhunt.CPUArchitecture.ARM, + {"arch": "arm", "count": {"min": 2, "max": 8}}, + id="arm-architecture", + ), + ], + ) + def test_override_is_applied_when_nested(self, arch, expected): + configuration = TaskConfiguration( + commands=["echo hi"], + resources=ResourcesSpec( + cpu=CPUSpec(arch=arch, count=Range[int](min=2, max=8)), + ), + ) + + assert json.loads(configuration.json())["resources"]["cpu"] == expected + assert canonicalize(json.dumps(configuration.dict()["resources"]["cpu"])) == canonicalize( + json.dumps(expected) + ) + + +class TestFieldSerializationFilters: + def test_submit_body_nested_field_includes_are_preserved(self): + body = factories.submit_body().dict() + + assert set(body["run"]) == {"id", "run_spec"} + assert set(body["run"]["run_spec"]) == { + "run_name", + "repo_id", + "repo_data", + "configuration", + "configuration_path", + } + assert set(body["job_spec"]) == { + "replica_num", + "job_num", + "jobs_per_replica", + "user", + "commands", + "env", + "single_branch", + "max_duration", + "ssh_key", + "working_dir", + "repo_data", + "repo_dir", + "repo_exists_action", + "file_archives", + } + assert set(body["job_submission"]) == {"id"} + + def test_derived_merged_profile_is_excluded_from_dict_and_json(self): + spec = factories.run_spec() + assert spec.merged_profile is not None + + assert "merged_profile" not in spec.dict() + assert "merged_profile" not in json.loads(spec.json()) + + def test_remote_repo_diff_bytes_are_excluded_from_dict_and_json(self): + repo = RemoteRunRepoData(repo_name="dstack", repo_diff=b"secret diff") + assert repo.repo_diff == b"secret diff" + + assert "repo_diff" not in repo.dict() + assert "repo_diff" not in json.loads(repo.json()) + + def test_runpod_compatibility_availability_zone_is_excluded_directly_and_nested(self): + configuration = RunpodVolumeConfiguration( + name="cache", + region="US-KS-2", + size="100GB", + availability_zone="legacy-zone", + ) + assert configuration.availability_zone == "legacy-zone" + + assert "availability_zone" not in configuration.dict() + assert "availability_zone" not in json.loads(configuration.json()) + + spec = VolumeSpec(configuration=configuration) + assert "availability_zone" not in spec.dict()["configuration"] + assert "availability_zone" not in json.loads(spec.json())["configuration"] + + +class TestCustomORJSONResponseCompat: + def test_response_uses_the_same_nested_serializer_overrides_as_model_json(self): + configuration = FleetConfiguration(nodes=FleetNodesSpec(min=1, target=1, max=1)) + + response_body = bytes(CustomORJSONResponse(configuration).body) + + assert canonicalize(response_body) == canonicalize(configuration.json()) + assert "target" not in json.loads(response_body)["nodes"] From dd6ceaeffa449f46eafd286dffaa1e817ac3fb05 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 12:02:10 +0500 Subject: [PATCH 14/15] Test json schema --- .../_internal/pydantic_compat/compare.py | 10 +- .../fixtures/schema/configuration.json | 3787 +++++++++++++++++ .../fixtures/schema/profiles.json | 538 +++ .../_internal/pydantic_compat/test_schema.py | 83 + 4 files changed, 4415 insertions(+), 3 deletions(-) create mode 100644 src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json create mode 100644 src/tests/_internal/pydantic_compat/fixtures/schema/profiles.json create mode 100644 src/tests/_internal/pydantic_compat/test_schema.py diff --git a/src/tests/_internal/pydantic_compat/compare.py b/src/tests/_internal/pydantic_compat/compare.py index f8b46ce5ce..6096f6655d 100644 --- a/src/tests/_internal/pydantic_compat/compare.py +++ b/src/tests/_internal/pydantic_compat/compare.py @@ -8,12 +8,16 @@ Layout is `fixtures///[.].`: - direction: `serialization` or `parsing` -- surface: `db`, `api_request`, `api_response`, `config` +- surface: the boundary, e.g. `db`, `backend_config`, `api_request`, `config`. The registries in + `test_serialization.py` are the list - variant: omitted while a model has only one case; added to *every* case for that model as soon as a second one exists, so `volume.input.yml` becomes `volume.size.input.yml` and `volume.kubernetes.input.yml` together - role: `input` for hand-written parse inputs, `values` / `types` for generated expectations, and a bare `.json` for serialization fixtures, which need no input + +`fixtures/schema/` is the exception: the published JSON Schemas have no surface or role level, so +it is `fixtures/schema/.json`. See `test_schema.py`. """ import difflib @@ -53,7 +57,7 @@ def canonicalize(payload: Union[bytes, str]) -> str: def assert_matches_fixture(kind: str, name: str, payload: Union[bytes, str], regen: bool) -> None: - """Compare a serialized payload against `fixtures//.json`.""" + """Compare a JSON payload against `fixtures//.json`.""" path = FIXTURES_DIR / kind / f"{name}.json" actual = canonicalize(payload) @@ -75,7 +79,7 @@ def assert_matches_fixture(kind: str, name: str, payload: Union[bytes, str], reg tofile=f"{kind}/{name}.json (actual)", ) ) - pytest.fail(f"{kind}/{name} serialization changed:\n{diff}\n{_REGEN_HINT}") + pytest.fail(f"{kind}/{name} changed:\n{diff}\n{_REGEN_HINT}") def type_map(value: Any, path: str = "", out: Union[dict, None] = None) -> dict: diff --git a/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json b/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json new file mode 100644 index 0000000000..8d1ac83a94 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/schema/configuration.json @@ -0,0 +1,3787 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "definitions": { + "ACMGatewayCertificateRequest": { + "additionalProperties": false, + "properties": { + "arn": { + "description": "The ARN of the wildcard ACM certificate for the domain", + "title": "Arn", + "type": "string" + }, + "type": { + "default": "acm", + "description": "Certificates by AWS Certificate Manager (ACM)", + "enum": [ + "acm" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "arn" + ], + "title": "ACMGatewayCertificateRequest", + "type": "object" + }, + "AWSVolumeConfigurationRequest": { + "additionalProperties": false, + "properties": { + "auto_cleanup_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait after volume is no longer used by any job before deleting it. Defaults to keep the volume indefinitely. Use the value `off` or `-1` to disable auto-cleanup", + "title": "Auto Cleanup Duration" + }, + "availability_zone": { + "description": "The volume availability zone", + "title": "Availability Zone", + "type": "string" + }, + "backend": { + "default": "aws", + "description": "The volume backend", + "enum": [ + "aws" + ], + "title": "Backend", + "type": "string" + }, + "name": { + "description": "The volume name", + "title": "Name", + "type": "string" + }, + "region": { + "description": "The volume region", + "title": "Region", + "type": "string" + }, + "size": { + "description": "The volume size. Must be specified when creating new volumes", + "title": "Size", + "type": "number" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the volume. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "type": { + "default": "volume", + "enum": [ + "volume" + ], + "title": "Type", + "type": "string" + }, + "volume_id": { + "description": "The volume ID. Must be specified when registering external volumes", + "title": "Volume Id", + "type": "string" + } + }, + "required": [ + "region" + ], + "title": "AWSVolumeConfigurationRequest", + "type": "object" + }, + "AcceleratorVendor": { + "description": "An enumeration.", + "enum": [ + "nvidia", + "amd", + "google", + "intel", + "tenstorrent" + ], + "title": "AcceleratorVendor", + "type": "string" + }, + "BackendType": { + "description": "Attributes:\n AMDDEVCLOUD (BackendType): AMD Developer Cloud\n AWS (BackendType): Amazon Web Services\n AZURE (BackendType): Microsoft Azure\n CLOUDRIFT (BackendType): CloudRift\n CRUSOE (BackendType): Crusoe\n CUDO (BackendType): Cudo\n DATACRUNCH (BackendType): DataCrunch (for backward compatibility)\n DIGITALOCEAN (BackendType): DigitalOcean\n DSTACK (BackendType): dstack Sky\n GCP (BackendType): Google Cloud Platform\n HOTAISLE (BackendType): Hot Aisle\n JARVISLABS (BackendType): JarvisLabs\n KUBERNETES (BackendType): Kubernetes\n LAMBDA (BackendType): Lambda Cloud\n NEBIUS (BackendType): Nebius AI Cloud\n OCI (BackendType): Oracle Cloud Infrastructure\n RUNPOD (BackendType): Runpod Cloud\n TENSORDOCK (BackendType): TensorDock Marketplace\n VASTAI (BackendType): Vast.ai Marketplace\n VERDA (BackendType): Verda Cloud\n VULTR (BackendType): Vultr\n SLURM (BackendType): Slurm", + "enum": [ + "amddevcloud", + "aws", + "azure", + "cloudrift", + "crusoe", + "cudo", + "datacrunch", + "digitalocean", + "dstack", + "gcp", + "hotaisle", + "jarvislabs", + "kubernetes", + "lambda", + "remote", + "nebius", + "oci", + "runpod", + "tensordock", + "vastai", + "verda", + "vultr", + "slurm" + ], + "title": "BackendType", + "type": "string" + }, + "CPUArchitecture": { + "description": "An enumeration.", + "enum": [ + "x86", + "arm" + ], + "title": "CPUArchitecture", + "type": "string" + }, + "CPUSpecRequest": { + "additionalProperties": false, + "properties": { + "arch": { + "allOf": [ + { + "$ref": "#/definitions/CPUArchitecture" + } + ], + "description": "The CPU architecture, one of: `x86`, `arm`" + }, + "count": { + "anyOf": [ + { + "$ref": "#/definitions/Range_int_" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": { + "max": null, + "min": 2 + }, + "description": "The number of CPU cores", + "title": "Count" + } + }, + "title": "CPUSpecRequest", + "type": "object" + }, + "CreationPolicy": { + "description": "An enumeration.", + "enum": [ + "reuse", + "reuse-or-create" + ], + "title": "CreationPolicy", + "type": "string" + }, + "DevEnvironmentConfigurationRequest": { + "additionalProperties": false, + "properties": { + "availability_zones": { + "description": "The availability zones to consider for provisioning (e.g., `[eu-west-1a, us-west4-a]`)", + "items": { + "type": "string" + }, + "title": "Availability Zones", + "type": "array" + }, + "backend_options": { + "description": "Backend-specific options, applied only to offers from that backend", + "items": { + "$ref": "#/definitions/VastAIProfileOptions" + }, + "title": "Backend Options", + "type": "array" + }, + "backends": { + "description": "The backends to consider for provisioning (e.g., `[aws, gcp]`)", + "items": { + "$ref": "#/definitions/BackendType" + }, + "type": "array" + }, + "creation_policy": { + "allOf": [ + { + "$ref": "#/definitions/CreationPolicy" + } + ], + "description": "The policy for using instances from fleets: `reuse`, `reuse-or-create`. Defaults to `reuse-or-create`" + }, + "docker": { + "description": "Use Docker inside the container. Mutually exclusive with `image`, `python`, and `nvcc`. Overrides `privileged`", + "title": "Docker", + "type": "boolean" + }, + "dstack": { + "default": false, + "description": "Make the dstack server accessible inside the run. No authentication credentials are provided", + "title": "Dstack", + "type": "boolean" + }, + "entrypoint": { + "description": "The Docker entrypoint", + "title": "Entrypoint", + "type": "string" + }, + "env": { + "allOf": [ + { + "$ref": "#/definitions/Env" + } + ], + "default": { + "__root__": {} + }, + "description": "The mapping or the list of environment variables", + "title": "Env" + }, + "files": { + "default": [], + "description": "The local to container file path mappings", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/FilePathMappingRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Files", + "type": "array" + }, + "fleets": { + "description": "The fleets considered for reuse. For fleets owned by the current project, specify fleet names. For imported fleets, specify `/`", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Fleets", + "type": "array" + }, + "home_dir": { + "default": "/root", + "title": "Home Dir", + "type": "string" + }, + "ide": { + "anyOf": [ + { + "enum": [ + "vscode" + ], + "type": "string" + }, + { + "enum": [ + "cursor" + ], + "type": "string" + }, + { + "enum": [ + "windsurf" + ], + "type": "string" + }, + { + "enum": [ + "zed" + ], + "type": "string" + } + ], + "description": "The IDE to pre-install. Supported values include `vscode`, `cursor`, `windsurf`, and `zed`. Defaults to no IDE (SSH only)", + "title": "Ide" + }, + "idle_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait before terminating idle instances. When the run reuses an existing fleet instance, the fleet's `idle_duration` applies. When the run provisions a new instance, the shorter of the fleet's and run's values is used. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration. Only applied for VM-based backends", + "title": "Idle Duration" + }, + "image": { + "description": "The name of the Docker image to run. If no `image` is specified, `dstack` uses an Ubuntu 24.04-based Docker image that comes pre-configured with `uv`, `python`, `pip`, the CUDA 13.0 runtime, InfiniBand, NCCL, and NCCL tests. It may also include provider-specific components such as EFA support on AWS. For non-Nvidia accelerators or NVidia GPUs unsupported by CUDA 13.0 (e.g. V100, P100), specify a custom Docker image.", + "title": "Image", + "type": "string" + }, + "inactivity_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum amount of time the dev environment can be inactive (e.g., `2h`, `1d`, etc). After it elapses, the dev environment is automatically stopped. Inactivity is defined as the absence of SSH connections to the dev environment, including VS Code connections, `ssh ` shells, and attached `dstack apply` or `dstack attach` commands. Use `off` for unlimited duration. Can be updated in-place. Defaults to `off`", + "title": "Inactivity Duration" + }, + "init": { + "default": [], + "description": "The shell commands to run on startup", + "items": { + "type": "string" + }, + "title": "Init", + "type": "array" + }, + "instance_types": { + "description": "The cloud-specific instance types to consider for provisioning (e.g., `[g6e.24xlarge, n1-standard-4]`)", + "items": { + "type": "string" + }, + "title": "Instance Types", + "type": "array" + }, + "instances": { + "description": "The specific fleet instances to consider for reuse. Each value can be an instance name string, or an object with `name`, `hostname`, or `fleet` and `instance`. When set, the run is only placed on matching existing instances.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/InstanceNameSelectorRequest" + }, + { + "$ref": "#/definitions/InstanceHostnameSelectorRequest" + }, + { + "$ref": "#/definitions/FleetInstanceSelectorRequest" + }, + { + "minLength": 1, + "type": "string" + } + ] + }, + "minItems": 1, + "title": "Instances", + "type": "array" + }, + "max_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run (e.g., `2h`, `1d`, etc) in a running state, excluding provisioning and pulling. After it elapses, the run is automatically stopped. Use `off` for unlimited duration. Defaults to `off`", + "title": "Max Duration" + }, + "max_price": { + "description": "The maximum instance price per hour, in dollars", + "exclusiveMinimum": 0.0, + "title": "Max Price", + "type": "number" + }, + "name": { + "description": "The run name. If not specified, a random name is generated", + "title": "Name", + "type": "string" + }, + "nvcc": { + "description": "Use image with NVIDIA CUDA Compiler (NVCC) included. Mutually exclusive with `image` and `docker`", + "title": "Nvcc", + "type": "boolean" + }, + "ports": { + "default": [], + "description": "Port numbers/mapping to expose", + "items": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "maximum": 65536, + "type": "integer" + }, + { + "pattern": "^(?:[0-9]+|\\*):[0-9]+$", + "type": "string" + }, + { + "$ref": "#/definitions/PortMappingRequest" + } + ] + }, + "title": "Ports", + "type": "array" + }, + "priority": { + "description": "The priority of the run, an integer between `0` and `100`. `dstack` tries to provision runs with higher priority first. Defaults to `0`", + "maximum": 100, + "minimum": 0, + "title": "Priority", + "type": "integer" + }, + "privileged": { + "default": false, + "description": "Run the container in privileged mode", + "title": "Privileged", + "type": "boolean" + }, + "python": { + "allOf": [ + { + "$ref": "#/definitions/PythonVersion" + } + ], + "description": "The major version of Python. Mutually exclusive with `image` and `docker`" + }, + "regions": { + "description": "The regions to consider for provisioning (e.g., `[eu-west-1, us-west4, westeurope]`)", + "items": { + "type": "string" + }, + "title": "Regions", + "type": "array" + }, + "registry_auth": { + "allOf": [ + { + "$ref": "#/definitions/RegistryAuthRequest" + } + ], + "description": "Credentials for pulling a private Docker image", + "title": "Registry Auth" + }, + "repos": { + "default": [], + "description": "The list of Git repos", + "items": { + "$ref": "#/definitions/RepoSpecRequest" + }, + "title": "Repos", + "type": "array" + }, + "reservation": { + "description": "The existing reservation to use for instance provisioning. Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations", + "title": "Reservation", + "type": "string" + }, + "resources": { + "allOf": [ + { + "$ref": "#/definitions/ResourcesSpecRequest" + } + ], + "default": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "description": "The resources requirements to run the configuration", + "title": "Resources" + }, + "retry": { + "anyOf": [ + { + "$ref": "#/definitions/ProfileRetryRequest" + }, + { + "type": "boolean" + } + ], + "description": "The policy for resubmitting the run. Defaults to `false`", + "title": "Retry" + }, + "schedule": { + "allOf": [ + { + "$ref": "#/definitions/ScheduleRequest" + } + ], + "description": "The schedule for starting the run at specified time", + "title": "Schedule" + }, + "setup": { + "default": [], + "items": { + "type": "string" + }, + "title": "Setup", + "type": "array" + }, + "shell": { + "description": "The shell used to run commands. Allowed values are `sh`, `bash`, or an absolute path, e.g., `/usr/bin/zsh`. Defaults to `/bin/sh` if the `image` is specified, `/bin/bash` otherwise", + "title": "Shell", + "type": "string" + }, + "single_branch": { + "description": "Whether to clone and track only the current branch or all remote branches. Relevant only when using remote Git repos. Defaults to `false` for dev environments and to `true` for tasks and services", + "title": "Single Branch", + "type": "boolean" + }, + "spot_policy": { + "allOf": [ + { + "$ref": "#/definitions/SpotPolicy" + } + ], + "description": "The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, `auto`. Defaults to `on-demand`" + }, + "startup_order": { + "allOf": [ + { + "$ref": "#/definitions/StartupOrder" + } + ], + "description": "The order in which master and workers jobs are started: `any`, `master-first`, `workers-first`. Defaults to `any`" + }, + "stop_criteria": { + "allOf": [ + { + "$ref": "#/definitions/StopCriteria" + } + ], + "description": "The criteria determining when a multi-node run should be considered finished: `all-done`, `master-done`. Defaults to `all-done`" + }, + "stop_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run graceful stopping. After it elapses, the run is automatically forced stopped. This includes force detaching volumes used by the run. Use `off` for unlimited duration. Defaults to `5m`", + "title": "Stop Duration" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the resource. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "type": { + "default": "dev-environment", + "enum": [ + "dev-environment" + ], + "title": "Type", + "type": "string" + }, + "user": { + "description": "The user inside the container, `user_name_or_id[:group_name_or_id]` (e.g., `ubuntu`, `1000:1000`). Defaults to the default user from the `image`", + "title": "User", + "type": "string" + }, + "utilization_policy": { + "allOf": [ + { + "$ref": "#/definitions/UtilizationPolicyRequest" + } + ], + "description": "Run termination policy based on utilization", + "title": "Utilization Policy" + }, + "version": { + "description": "The version of the IDE. For `windsurf`, the version is in the format `version@commit`", + "title": "Version", + "type": "string" + }, + "volumes": { + "default": [], + "description": "The volumes mount points", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/VolumeMountPointRequest" + }, + { + "$ref": "#/definitions/InstanceMountPointRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Volumes", + "type": "array" + }, + "working_dir": { + "description": "The absolute path to the working directory inside the container. Defaults to the `image`'s default working directory", + "title": "Working Dir", + "type": "string" + } + }, + "title": "DevEnvironmentConfigurationRequest", + "type": "object" + }, + "DiskSpecRequest": { + "additionalProperties": false, + "properties": { + "size": { + "anyOf": [ + { + "$ref": "#/definitions/Range_Memory_" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Disk size", + "title": "Size" + } + }, + "required": [ + "size" + ], + "title": "DiskSpecRequest", + "type": "object" + }, + "EntityReferenceRequest": { + "additionalProperties": false, + "description": "Cross-project entity reference.", + "properties": { + "name": { + "description": "The entity name", + "title": "Name", + "type": "string" + }, + "project": { + "description": "The project name. If unspecified, refers to the current project", + "title": "Project", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "EntityReferenceRequest", + "type": "object" + }, + "Env": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/EnvSentinelRequest" + } + ] + }, + "type": "object" + } + ], + "default": {}, + "description": "Env represents a mapping of process environment variables, as in environ(7).\nEnvironment values may be omitted, in that case the :class:`EnvSentinel`\nobject is used as a placeholder.\n\nTo create an instance from a `dict[str, str]` or a `list[str]` use pydantic's\n:meth:`BaseModel.parse_obj(dict | list)` method.\n\nNB: this is *NOT* a CoreModel, pydantic-duality, which is used as a base\nfor the CoreModel, doesn't play well with custom root models.", + "title": "Env" + }, + "EnvSentinelRequest": { + "additionalProperties": false, + "properties": { + "key": { + "title": "Key", + "type": "string" + } + }, + "required": [ + "key" + ], + "title": "EnvSentinelRequest", + "type": "object" + }, + "FilePathMappingRequest": { + "additionalProperties": false, + "properties": { + "local_path": { + "description": "The path on the user's machine. Relative paths are resolved relative to the parent directory of the the configuration file", + "title": "Local Path", + "type": "string" + }, + "path": { + "description": "The path in the container. Relative paths are resolved relative to the working directory", + "title": "Path", + "type": "string" + } + }, + "required": [ + "local_path", + "path" + ], + "title": "FilePathMappingRequest", + "type": "object" + }, + "FleetConfigurationRequest": { + "additionalProperties": false, + "properties": { + "availability_zones": { + "description": "The availability zones to consider for provisioning (e.g., `[eu-west-1a, us-west4-a]`)", + "items": { + "type": "string" + }, + "title": "Availability Zones", + "type": "array" + }, + "backend_options": { + "description": "Backend-specific options, applied only to offers from that backend", + "items": { + "$ref": "#/definitions/VastAIProfileOptions" + }, + "title": "Backend Options", + "type": "array" + }, + "backends": { + "description": "The backends to consider for provisioning (e.g., `[aws, gcp]`)", + "items": { + "$ref": "#/definitions/BackendType" + }, + "type": "array" + }, + "blocks": { + "anyOf": [ + { + "enum": [ + "auto" + ], + "type": "string" + }, + { + "minimum": 1, + "type": "integer" + } + ], + "default": 1, + "description": "The amount of blocks to split the instance into, a number or `auto`. `auto` means as many as possible. The number of GPUs and CPUs must be divisible by the number of blocks. Defaults to `1`, i.e. do not split", + "title": "Blocks" + }, + "env": { + "allOf": [ + { + "$ref": "#/definitions/Env" + } + ], + "default": { + "__root__": {} + }, + "description": "The mapping or the list of environment variables", + "title": "Env" + }, + "idle_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait before terminating idle instances. Instances are not terminated if the fleet is already at `nodes.min`. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration", + "title": "Idle Duration" + }, + "instance_types": { + "description": "The cloud-specific instance types to consider for provisioning (e.g., `[g6e.24xlarge, n1-standard-4]`)", + "items": { + "type": "string" + }, + "title": "Instance Types", + "type": "array" + }, + "max_price": { + "description": "The maximum instance price per hour, in dollars", + "exclusiveMinimum": 0.0, + "title": "Max Price", + "type": "number" + }, + "name": { + "description": "The fleet name", + "title": "Name", + "type": "string" + }, + "nodes": { + "anyOf": [ + { + "$ref": "#/definitions/FleetNodesSpecRequest" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The number of instances", + "title": "Nodes" + }, + "placement": { + "allOf": [ + { + "$ref": "#/definitions/InstanceGroupPlacement" + } + ], + "description": "The placement of instances: `any` or `cluster`" + }, + "regions": { + "description": "The regions to consider for provisioning (e.g., `[eu-west-1, us-west4, westeurope]`)", + "items": { + "type": "string" + }, + "title": "Regions", + "type": "array" + }, + "reservation": { + "description": "The existing reservation to use for instance provisioning. Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations", + "title": "Reservation", + "type": "string" + }, + "resources": { + "allOf": [ + { + "$ref": "#/definitions/ResourcesSpecRequest" + } + ], + "description": "The resources requirements", + "title": "Resources" + }, + "retry": { + "anyOf": [ + { + "$ref": "#/definitions/ProfileRetryRequest" + }, + { + "type": "boolean" + } + ], + "description": "The policy for provisioning retry. Defaults to `false`", + "title": "Retry" + }, + "spot_policy": { + "allOf": [ + { + "$ref": "#/definitions/SpotPolicy" + } + ], + "description": "The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, `auto`. Defaults to `on-demand`" + }, + "ssh_config": { + "allOf": [ + { + "$ref": "#/definitions/SSHParamsRequest" + } + ], + "description": "The parameters for adding instances via SSH", + "title": "Ssh Config" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the resource. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "type": { + "default": "fleet", + "enum": [ + "fleet" + ], + "title": "Type", + "type": "string" + } + }, + "title": "FleetConfigurationRequest", + "type": "object" + }, + "FleetInstanceSelectorRequest": { + "additionalProperties": false, + "properties": { + "fleet": { + "anyOf": [ + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The fleet reference. For fleets owned by the current project, specify the fleet name. For a fleet from another project, specify `/` or an object with `project` and `name`.", + "title": "Fleet" + }, + "instance": { + "description": "The fleet instance number", + "minimum": 0, + "title": "Instance", + "type": "integer" + } + }, + "required": [ + "fleet", + "instance" + ], + "title": "FleetInstanceSelectorRequest", + "type": "object" + }, + "FleetNodesSpecRequest": { + "additionalProperties": false, + "properties": { + "max": { + "description": "The maximum number of instances allowed in the fleet. Unlimited if not specified", + "title": "Max", + "type": "integer" + }, + "min": { + "description": "The minimum number of instances to maintain in the fleet", + "title": "Min", + "type": "integer" + }, + "target": { + "description": "The number of instances to provision on fleet apply. `min` <= `target` <= `max` Defaults to `min`", + "title": "Target", + "type": "integer" + } + }, + "required": [ + "min", + "target" + ], + "title": "FleetNodesSpecRequest", + "type": "object" + }, + "GCPVolumeConfigurationRequest": { + "additionalProperties": false, + "properties": { + "auto_cleanup_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait after volume is no longer used by any job before deleting it. Defaults to keep the volume indefinitely. Use the value `off` or `-1` to disable auto-cleanup", + "title": "Auto Cleanup Duration" + }, + "availability_zone": { + "description": "The volume availability zone", + "title": "Availability Zone", + "type": "string" + }, + "backend": { + "default": "gcp", + "description": "The volume backend", + "enum": [ + "gcp" + ], + "title": "Backend", + "type": "string" + }, + "name": { + "description": "The volume name", + "title": "Name", + "type": "string" + }, + "region": { + "description": "The volume region", + "title": "Region", + "type": "string" + }, + "size": { + "description": "The volume size. Must be specified when creating new volumes", + "title": "Size", + "type": "number" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the volume. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "type": { + "default": "volume", + "enum": [ + "volume" + ], + "title": "Type", + "type": "string" + }, + "volume_id": { + "description": "The volume ID. Must be specified when registering external volumes", + "title": "Volume Id", + "type": "string" + } + }, + "required": [ + "region" + ], + "title": "GCPVolumeConfigurationRequest", + "type": "object" + }, + "GPUSpecRequest": { + "additionalProperties": false, + "properties": { + "compute_capability": { + "description": "The minimum compute capability of the GPU (e.g., `7.5`)", + "items": {}, + "title": "Compute Capability", + "type": "array" + }, + "count": { + "anyOf": [ + { + "$ref": "#/definitions/Range_int_" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "default": { + "max": null, + "min": 1 + }, + "description": "The number of GPUs", + "title": "Count" + }, + "memory": { + "anyOf": [ + { + "$ref": "#/definitions/Range_Memory_" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The RAM size (e.g., `16GB`). Can be set to a range (e.g. `16GB..`, or `16GB..80GB`)", + "title": "Memory" + }, + "name": { + "anyOf": [ + { + "type": "array" + }, + { + "type": "string" + } + ], + "description": "The name of the GPU (e.g., `A100` or `H100`)", + "items": { + "type": "string" + }, + "title": "Name" + }, + "total_memory": { + "anyOf": [ + { + "$ref": "#/definitions/Range_Memory_" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The total RAM size (e.g., `32GB`). Can be set to a range (e.g. `16GB..`, or `16GB..80GB`)", + "title": "Total Memory" + }, + "vendor": { + "allOf": [ + { + "$ref": "#/definitions/AcceleratorVendor" + } + ], + "description": "The vendor of the GPU/accelerator, one of: `nvidia`, `amd`, `google` (alias: `tpu`), `intel`" + } + }, + "title": "GPUSpecRequest", + "type": "object" + }, + "GatewayConfigurationRequest": { + "additionalProperties": false, + "properties": { + "backend": { + "allOf": [ + { + "$ref": "#/definitions/BackendType" + } + ], + "description": "The gateway backend" + }, + "certificate": { + "default": { + "type": "lets-encrypt" + }, + "description": "The SSL certificate configuration. Set to `null` to disable. Defaults to `type: lets-encrypt`", + "discriminator": { + "mapping": { + "acm": "#/definitions/ACMGatewayCertificateRequest", + "lets-encrypt": "#/definitions/LetsEncryptGatewayCertificateRequest" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/definitions/LetsEncryptGatewayCertificateRequest" + }, + { + "$ref": "#/definitions/ACMGatewayCertificateRequest" + } + ], + "title": "Certificate" + }, + "default": { + "default": false, + "description": "Make the gateway default", + "title": "Default", + "type": "boolean" + }, + "domain": { + "description": "The gateway wildcard domain name, e.g. `example.com`. Service domain names are constructed as `./`. `dstack` will use IP addresses from this network for communication between hosts. If not specified, `dstack` will use IPs from the first found internal network.", + "title": "Network", + "type": "string" + }, + "port": { + "description": "The SSH port to connect to", + "title": "Port", + "type": "integer" + }, + "proxy_jump": { + "allOf": [ + { + "$ref": "#/definitions/SSHProxyParamsRequest" + } + ], + "description": "The SSH proxy configuration for all hosts", + "title": "Proxy Jump" + }, + "ssh_key": { + "$ref": "#/definitions/SSHKeyRequest" + }, + "user": { + "description": "The user to log in with on all hosts", + "title": "User", + "type": "string" + } + }, + "required": [ + "hosts" + ], + "title": "SSHParamsRequest", + "type": "object" + }, + "SSHProxyParamsRequest": { + "additionalProperties": false, + "properties": { + "hostname": { + "description": "The IP address or domain of proxy host", + "title": "Hostname", + "type": "string" + }, + "identity_file": { + "description": "The private key to use for proxy host", + "title": "Identity File", + "type": "string" + }, + "port": { + "description": "The SSH port of proxy host", + "title": "Port", + "type": "integer" + }, + "ssh_key": { + "$ref": "#/definitions/SSHKeyRequest" + }, + "user": { + "description": "The user to log in with for proxy host", + "title": "User", + "type": "string" + } + }, + "required": [ + "hostname", + "user", + "identity_file" + ], + "title": "SSHProxyParamsRequest", + "type": "object" + }, + "ScalingSpecRequest": { + "additionalProperties": false, + "properties": { + "metric": { + "description": "The target metric to track. Currently, the only supported value is `rps` (meaning requests per second)", + "enum": [ + "rps" + ], + "title": "Metric", + "type": "string" + }, + "scale_down_delay": { + "default": 600, + "description": "The minimum time, in seconds, between a scaling event and the next scale-down decision. Used to prevent overly frequent scaling", + "title": "Scale Down Delay", + "type": "integer" + }, + "scale_up_delay": { + "default": 300, + "description": "The minimum time, in seconds, between a scaling event and the next scale-up decision. Used to prevent overly frequent scaling", + "title": "Scale Up Delay", + "type": "integer" + }, + "target": { + "description": "The target value of the metric. The number of replicas is calculated based on this number and automatically adjusts (scales up or down) as this metric changes", + "exclusiveMinimum": 0, + "title": "Target", + "type": "number" + }, + "window": { + "description": "The time window used to calculate requests per second. Allowed values: `30s`, `60s`, `300s`. Defaults to `60s`", + "title": "Window", + "type": "integer" + } + }, + "required": [ + "metric", + "target" + ], + "title": "ScalingSpecRequest", + "type": "object" + }, + "ScheduleRequest": { + "additionalProperties": false, + "properties": { + "cron": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "A cron expression or a list of cron expressions specifying the UTC time when the run needs to be started", + "title": "Cron" + } + }, + "required": [ + "cron" + ], + "title": "ScheduleRequest", + "type": "object" + }, + "ServiceConfigurationRequest": { + "additionalProperties": false, + "properties": { + "auth": { + "default": true, + "description": "Enable the authorization", + "title": "Auth", + "type": "boolean" + }, + "availability_zones": { + "description": "The availability zones to consider for provisioning (e.g., `[eu-west-1a, us-west4-a]`)", + "items": { + "type": "string" + }, + "title": "Availability Zones", + "type": "array" + }, + "backend_options": { + "description": "Backend-specific options, applied only to offers from that backend", + "items": { + "$ref": "#/definitions/VastAIProfileOptions" + }, + "title": "Backend Options", + "type": "array" + }, + "backends": { + "description": "The backends to consider for provisioning (e.g., `[aws, gcp]`)", + "items": { + "$ref": "#/definitions/BackendType" + }, + "type": "array" + }, + "commands": { + "default": [], + "description": "The shell commands to run", + "items": { + "type": "string" + }, + "title": "Commands", + "type": "array" + }, + "creation_policy": { + "allOf": [ + { + "$ref": "#/definitions/CreationPolicy" + } + ], + "description": "The policy for using instances from fleets: `reuse`, `reuse-or-create`. Defaults to `reuse-or-create`" + }, + "docker": { + "description": "Use Docker inside the container. Mutually exclusive with `image`, `python`, and `nvcc`. Overrides `privileged`", + "title": "Docker", + "type": "boolean" + }, + "dstack": { + "default": false, + "description": "Make the dstack server accessible inside the run. No authentication credentials are provided", + "title": "Dstack", + "type": "boolean" + }, + "entrypoint": { + "description": "The Docker entrypoint", + "title": "Entrypoint", + "type": "string" + }, + "env": { + "allOf": [ + { + "$ref": "#/definitions/Env" + } + ], + "default": { + "__root__": {} + }, + "description": "The mapping or the list of environment variables", + "title": "Env" + }, + "files": { + "default": [], + "description": "The local to container file path mappings", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/FilePathMappingRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Files", + "type": "array" + }, + "fleets": { + "description": "The fleets considered for reuse. For fleets owned by the current project, specify fleet names. For imported fleets, specify `/`", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Fleets", + "type": "array" + }, + "gateway": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "type": "string" + } + ], + "description": "The name of the gateway. Specify boolean `false` to run without a gateway. Specify boolean `true` to run with the default gateway. Omit to run with the default gateway if there is one, or without a gateway otherwise", + "title": "Gateway" + }, + "home_dir": { + "default": "/root", + "title": "Home Dir", + "type": "string" + }, + "https": { + "anyOf": [ + { + "type": "boolean" + }, + { + "enum": [ + "auto" + ], + "type": "string" + } + ], + "description": "Enable HTTPS if running with a gateway. Set to `auto` to determine automatically based on gateway configuration. Defaults to `true`", + "title": "Https" + }, + "idle_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait before terminating idle instances. When the run reuses an existing fleet instance, the fleet's `idle_duration` applies. When the run provisions a new instance, the shorter of the fleet's and run's values is used. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration. Only applied for VM-based backends", + "title": "Idle Duration" + }, + "image": { + "description": "The name of the Docker image to run. If no `image` is specified, `dstack` uses an Ubuntu 24.04-based Docker image that comes pre-configured with `uv`, `python`, `pip`, the CUDA 13.0 runtime, InfiniBand, NCCL, and NCCL tests. It may also include provider-specific components such as EFA support on AWS. For non-Nvidia accelerators or NVidia GPUs unsupported by CUDA 13.0 (e.g. V100, P100), specify a custom Docker image.", + "title": "Image", + "type": "string" + }, + "instance_types": { + "description": "The cloud-specific instance types to consider for provisioning (e.g., `[g6e.24xlarge, n1-standard-4]`)", + "items": { + "type": "string" + }, + "title": "Instance Types", + "type": "array" + }, + "instances": { + "description": "The specific fleet instances to consider for reuse. Each value can be an instance name string, or an object with `name`, `hostname`, or `fleet` and `instance`. When set, the run is only placed on matching existing instances.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/InstanceNameSelectorRequest" + }, + { + "$ref": "#/definitions/InstanceHostnameSelectorRequest" + }, + { + "$ref": "#/definitions/FleetInstanceSelectorRequest" + }, + { + "minLength": 1, + "type": "string" + } + ] + }, + "minItems": 1, + "title": "Instances", + "type": "array" + }, + "max_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run (e.g., `2h`, `1d`, etc) in a running state, excluding provisioning and pulling. After it elapses, the run is automatically stopped. Use `off` for unlimited duration. Defaults to `off`", + "title": "Max Duration" + }, + "max_price": { + "description": "The maximum instance price per hour, in dollars", + "exclusiveMinimum": 0.0, + "title": "Max Price", + "type": "number" + }, + "model": { + "anyOf": [ + { + "discriminator": { + "mapping": { + "openai": "#/definitions/OpenAIChatModelRequest", + "tgi": "#/definitions/TGIChatModelRequest" + }, + "propertyName": "format" + }, + "oneOf": [ + { + "$ref": "#/definitions/TGIChatModelRequest" + }, + { + "$ref": "#/definitions/OpenAIChatModelRequest" + } + ] + }, + { + "type": "string" + } + ], + "description": "Mapping of the model for the OpenAI-compatible endpoint provided by `dstack`. Can be a full model format definition or just a model name. If it's a name, the service is expected to expose an OpenAI-compatible API at the `/v1` path", + "title": "Model" + }, + "name": { + "description": "The run name. If not specified, a random name is generated", + "title": "Name", + "type": "string" + }, + "nvcc": { + "description": "Use image with NVIDIA CUDA Compiler (NVCC) included. Mutually exclusive with `image` and `docker`", + "title": "Nvcc", + "type": "boolean" + }, + "port": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "maximum": 65536, + "type": "integer" + }, + { + "pattern": "^[0-9]+:[0-9]+$", + "type": "string" + }, + { + "$ref": "#/definitions/PortMappingRequest" + } + ], + "description": "The port the application listens on", + "title": "Port" + }, + "priority": { + "description": "The priority of the run, an integer between `0` and `100`. `dstack` tries to provision runs with higher priority first. Defaults to `0`", + "maximum": 100, + "minimum": 0, + "title": "Priority", + "type": "integer" + }, + "privileged": { + "default": false, + "description": "Run the container in privileged mode", + "title": "Privileged", + "type": "boolean" + }, + "probes": { + "description": "The list of probes to determine service health. If `model` is set, defaults to a `/v1/chat/completions` probe. Set explicitly to override", + "items": { + "$ref": "#/definitions/ProbeConfigRequest" + }, + "title": "Probes", + "type": "array" + }, + "python": { + "allOf": [ + { + "$ref": "#/definitions/PythonVersion" + } + ], + "description": "The major version of Python. Mutually exclusive with `image` and `docker`" + }, + "rate_limits": { + "default": [], + "description": "Rate limiting rules", + "items": { + "$ref": "#/definitions/RateLimitRequest" + }, + "title": "Rate Limits", + "type": "array" + }, + "regions": { + "description": "The regions to consider for provisioning (e.g., `[eu-west-1, us-west4, westeurope]`)", + "items": { + "type": "string" + }, + "title": "Regions", + "type": "array" + }, + "registry_auth": { + "allOf": [ + { + "$ref": "#/definitions/RegistryAuthRequest" + } + ], + "description": "Credentials for pulling a private Docker image", + "title": "Registry Auth" + }, + "replicas": { + "anyOf": [ + { + "items": { + "$ref": "#/definitions/ReplicaGroupRequest" + }, + "type": "array" + }, + { + "$ref": "#/definitions/Range_int_" + }, + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The number of replicas or a list of replica groups. Can be an integer (e.g., `2`), a range (e.g., `0..4`), or a list of replica groups. Each replica group defines replicas with shared configuration (commands, resources, scaling). When `replicas` is a list of replica groups, top-level `scaling`, `commands`, and `resources` are not allowed and must be specified in each replica group instead. ", + "title": "Replicas" + }, + "repos": { + "default": [], + "description": "The list of Git repos", + "items": { + "$ref": "#/definitions/RepoSpecRequest" + }, + "title": "Repos", + "type": "array" + }, + "reservation": { + "description": "The existing reservation to use for instance provisioning. Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations", + "title": "Reservation", + "type": "string" + }, + "resources": { + "allOf": [ + { + "$ref": "#/definitions/ResourcesSpecRequest" + } + ], + "default": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "description": "The resources requirements to run the configuration", + "title": "Resources" + }, + "retry": { + "anyOf": [ + { + "$ref": "#/definitions/ProfileRetryRequest" + }, + { + "type": "boolean" + } + ], + "description": "The policy for resubmitting the run. Defaults to `false`", + "title": "Retry" + }, + "router": { + "allOf": [ + { + "$ref": "#/definitions/SGLangServiceRouterConfigRequest" + } + ], + "description": "Router configuration for the service. Requires a gateway with matching router enabled. ", + "title": "Router" + }, + "scaling": { + "allOf": [ + { + "$ref": "#/definitions/ScalingSpecRequest" + } + ], + "description": "The auto-scaling rules. Required if `replicas` is set to a range", + "title": "Scaling" + }, + "schedule": { + "allOf": [ + { + "$ref": "#/definitions/ScheduleRequest" + } + ], + "description": "The schedule for starting the run at specified time", + "title": "Schedule" + }, + "setup": { + "default": [], + "items": { + "type": "string" + }, + "title": "Setup", + "type": "array" + }, + "shell": { + "description": "The shell used to run commands. Allowed values are `sh`, `bash`, or an absolute path, e.g., `/usr/bin/zsh`. Defaults to `/bin/sh` if the `image` is specified, `/bin/bash` otherwise", + "title": "Shell", + "type": "string" + }, + "single_branch": { + "description": "Whether to clone and track only the current branch or all remote branches. Relevant only when using remote Git repos. Defaults to `false` for dev environments and to `true` for tasks and services", + "title": "Single Branch", + "type": "boolean" + }, + "spot_policy": { + "allOf": [ + { + "$ref": "#/definitions/SpotPolicy" + } + ], + "description": "The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, `auto`. Defaults to `on-demand`" + }, + "startup_order": { + "allOf": [ + { + "$ref": "#/definitions/StartupOrder" + } + ], + "description": "The order in which master and workers jobs are started: `any`, `master-first`, `workers-first`. Defaults to `any`" + }, + "stop_criteria": { + "allOf": [ + { + "$ref": "#/definitions/StopCriteria" + } + ], + "description": "The criteria determining when a multi-node run should be considered finished: `all-done`, `master-done`. Defaults to `all-done`" + }, + "stop_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run graceful stopping. After it elapses, the run is automatically forced stopped. This includes force detaching volumes used by the run. Use `off` for unlimited duration. Defaults to `5m`", + "title": "Stop Duration" + }, + "strip_prefix": { + "default": true, + "description": "Strip the `/proxy/services///` path prefix when forwarding requests to the service. Only takes effect when running the service without a gateway", + "title": "Strip Prefix", + "type": "boolean" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the resource. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "type": { + "default": "service", + "enum": [ + "service" + ], + "title": "Type", + "type": "string" + }, + "user": { + "description": "The user inside the container, `user_name_or_id[:group_name_or_id]` (e.g., `ubuntu`, `1000:1000`). Defaults to the default user from the `image`", + "title": "User", + "type": "string" + }, + "utilization_policy": { + "allOf": [ + { + "$ref": "#/definitions/UtilizationPolicyRequest" + } + ], + "description": "Run termination policy based on utilization", + "title": "Utilization Policy" + }, + "volumes": { + "default": [], + "description": "The volumes mount points", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/VolumeMountPointRequest" + }, + { + "$ref": "#/definitions/InstanceMountPointRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Volumes", + "type": "array" + }, + "working_dir": { + "description": "The absolute path to the working directory inside the container. Defaults to the `image`'s default working directory", + "title": "Working Dir", + "type": "string" + } + }, + "required": [ + "port" + ], + "title": "ServiceConfigurationRequest", + "type": "object" + }, + "SpotPolicy": { + "description": "An enumeration.", + "enum": [ + "spot", + "on-demand", + "auto" + ], + "title": "SpotPolicy", + "type": "string" + }, + "StartupOrder": { + "description": "An enumeration.", + "enum": [ + "any", + "master-first", + "workers-first" + ], + "title": "StartupOrder", + "type": "string" + }, + "StopCriteria": { + "description": "An enumeration.", + "enum": [ + "all-done", + "master-done" + ], + "title": "StopCriteria", + "type": "string" + }, + "TGIChatModelRequest": { + "additionalProperties": false, + "description": "Mapping of the model for the OpenAI-compatible endpoint.\n\nAttributes:\n type (str): The type of the model, e.g. \"chat\"\n name (str): The name of the model. This name will be used both to load model configuration from the HuggingFace Hub and in the OpenAI-compatible endpoint.\n format (str): The format of the model, e.g. \"tgi\" if the model is served with HuggingFace's Text Generation Inference.\n chat_template (Optional[str]): The custom prompt template for the model. If not specified, the default prompt template from the HuggingFace Hub configuration will be used.\n eos_token (Optional[str]): The custom end of sentence token. If not specified, the default end of sentence token from the HuggingFace Hub configuration will be used.", + "properties": { + "chat_template": { + "description": "The custom prompt template for the model. If not specified, the default prompt template from the HuggingFace Hub configuration will be used", + "title": "Chat Template", + "type": "string" + }, + "eos_token": { + "description": "The custom end of sentence token. If not specified, the default end of sentence token from the HuggingFace Hub configuration will be used", + "title": "Eos Token", + "type": "string" + }, + "format": { + "description": "The serving format. Must be set to `tgi`", + "enum": [ + "tgi" + ], + "title": "Format", + "type": "string" + }, + "name": { + "description": "The name of the model", + "title": "Name", + "type": "string" + }, + "type": { + "default": "chat", + "description": "The type of the model", + "enum": [ + "chat" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "format" + ], + "title": "TGIChatModelRequest", + "type": "object" + }, + "TaskConfigurationRequest": { + "additionalProperties": false, + "properties": { + "availability_zones": { + "description": "The availability zones to consider for provisioning (e.g., `[eu-west-1a, us-west4-a]`)", + "items": { + "type": "string" + }, + "title": "Availability Zones", + "type": "array" + }, + "backend_options": { + "description": "Backend-specific options, applied only to offers from that backend", + "items": { + "$ref": "#/definitions/VastAIProfileOptions" + }, + "title": "Backend Options", + "type": "array" + }, + "backends": { + "description": "The backends to consider for provisioning (e.g., `[aws, gcp]`)", + "items": { + "$ref": "#/definitions/BackendType" + }, + "type": "array" + }, + "commands": { + "default": [], + "description": "The shell commands to run", + "items": { + "type": "string" + }, + "title": "Commands", + "type": "array" + }, + "creation_policy": { + "allOf": [ + { + "$ref": "#/definitions/CreationPolicy" + } + ], + "description": "The policy for using instances from fleets: `reuse`, `reuse-or-create`. Defaults to `reuse-or-create`" + }, + "docker": { + "description": "Use Docker inside the container. Mutually exclusive with `image`, `python`, and `nvcc`. Overrides `privileged`", + "title": "Docker", + "type": "boolean" + }, + "dstack": { + "default": false, + "description": "Make the dstack server accessible inside the run. No authentication credentials are provided", + "title": "Dstack", + "type": "boolean" + }, + "entrypoint": { + "description": "The Docker entrypoint", + "title": "Entrypoint", + "type": "string" + }, + "env": { + "allOf": [ + { + "$ref": "#/definitions/Env" + } + ], + "default": { + "__root__": {} + }, + "description": "The mapping or the list of environment variables", + "title": "Env" + }, + "files": { + "default": [], + "description": "The local to container file path mappings", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/FilePathMappingRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Files", + "type": "array" + }, + "fleets": { + "description": "The fleets considered for reuse. For fleets owned by the current project, specify fleet names. For imported fleets, specify `/`", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Fleets", + "type": "array" + }, + "home_dir": { + "default": "/root", + "title": "Home Dir", + "type": "string" + }, + "idle_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait before terminating idle instances. When the run reuses an existing fleet instance, the fleet's `idle_duration` applies. When the run provisions a new instance, the shorter of the fleet's and run's values is used. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration. Only applied for VM-based backends", + "title": "Idle Duration" + }, + "image": { + "description": "The name of the Docker image to run. If no `image` is specified, `dstack` uses an Ubuntu 24.04-based Docker image that comes pre-configured with `uv`, `python`, `pip`, the CUDA 13.0 runtime, InfiniBand, NCCL, and NCCL tests. It may also include provider-specific components such as EFA support on AWS. For non-Nvidia accelerators or NVidia GPUs unsupported by CUDA 13.0 (e.g. V100, P100), specify a custom Docker image.", + "title": "Image", + "type": "string" + }, + "instance_types": { + "description": "The cloud-specific instance types to consider for provisioning (e.g., `[g6e.24xlarge, n1-standard-4]`)", + "items": { + "type": "string" + }, + "title": "Instance Types", + "type": "array" + }, + "instances": { + "description": "The specific fleet instances to consider for reuse. Each value can be an instance name string, or an object with `name`, `hostname`, or `fleet` and `instance`. When set, the run is only placed on matching existing instances.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/InstanceNameSelectorRequest" + }, + { + "$ref": "#/definitions/InstanceHostnameSelectorRequest" + }, + { + "$ref": "#/definitions/FleetInstanceSelectorRequest" + }, + { + "minLength": 1, + "type": "string" + } + ] + }, + "minItems": 1, + "title": "Instances", + "type": "array" + }, + "max_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run (e.g., `2h`, `1d`, etc) in a running state, excluding provisioning and pulling. After it elapses, the run is automatically stopped. Use `off` for unlimited duration. Defaults to `off`", + "title": "Max Duration" + }, + "max_price": { + "description": "The maximum instance price per hour, in dollars", + "exclusiveMinimum": 0.0, + "title": "Max Price", + "type": "number" + }, + "name": { + "description": "The run name. If not specified, a random name is generated", + "title": "Name", + "type": "string" + }, + "nodes": { + "default": 1, + "description": "Number of nodes", + "minimum": 1, + "title": "Nodes", + "type": "integer" + }, + "nvcc": { + "description": "Use image with NVIDIA CUDA Compiler (NVCC) included. Mutually exclusive with `image` and `docker`", + "title": "Nvcc", + "type": "boolean" + }, + "ports": { + "default": [], + "description": "Port numbers/mapping to expose", + "items": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "maximum": 65536, + "type": "integer" + }, + { + "pattern": "^(?:[0-9]+|\\*):[0-9]+$", + "type": "string" + }, + { + "$ref": "#/definitions/PortMappingRequest" + } + ] + }, + "title": "Ports", + "type": "array" + }, + "priority": { + "description": "The priority of the run, an integer between `0` and `100`. `dstack` tries to provision runs with higher priority first. Defaults to `0`", + "maximum": 100, + "minimum": 0, + "title": "Priority", + "type": "integer" + }, + "privileged": { + "default": false, + "description": "Run the container in privileged mode", + "title": "Privileged", + "type": "boolean" + }, + "python": { + "allOf": [ + { + "$ref": "#/definitions/PythonVersion" + } + ], + "description": "The major version of Python. Mutually exclusive with `image` and `docker`" + }, + "regions": { + "description": "The regions to consider for provisioning (e.g., `[eu-west-1, us-west4, westeurope]`)", + "items": { + "type": "string" + }, + "title": "Regions", + "type": "array" + }, + "registry_auth": { + "allOf": [ + { + "$ref": "#/definitions/RegistryAuthRequest" + } + ], + "description": "Credentials for pulling a private Docker image", + "title": "Registry Auth" + }, + "repos": { + "default": [], + "description": "The list of Git repos", + "items": { + "$ref": "#/definitions/RepoSpecRequest" + }, + "title": "Repos", + "type": "array" + }, + "reservation": { + "description": "The existing reservation to use for instance provisioning. Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations", + "title": "Reservation", + "type": "string" + }, + "resources": { + "allOf": [ + { + "$ref": "#/definitions/ResourcesSpecRequest" + } + ], + "default": { + "cpu": { + "max": null, + "min": 2 + }, + "disk": { + "size": { + "max": null, + "min": 100.0 + } + }, + "gpu": { + "compute_capability": null, + "count": { + "max": null, + "min": 0 + }, + "memory": null, + "name": null, + "total_memory": null, + "vendor": null + }, + "memory": { + "max": null, + "min": 8.0 + }, + "shm_size": null + }, + "description": "The resources requirements to run the configuration", + "title": "Resources" + }, + "retry": { + "anyOf": [ + { + "$ref": "#/definitions/ProfileRetryRequest" + }, + { + "type": "boolean" + } + ], + "description": "The policy for resubmitting the run. Defaults to `false`", + "title": "Retry" + }, + "schedule": { + "allOf": [ + { + "$ref": "#/definitions/ScheduleRequest" + } + ], + "description": "The schedule for starting the run at specified time", + "title": "Schedule" + }, + "setup": { + "default": [], + "items": { + "type": "string" + }, + "title": "Setup", + "type": "array" + }, + "shell": { + "description": "The shell used to run commands. Allowed values are `sh`, `bash`, or an absolute path, e.g., `/usr/bin/zsh`. Defaults to `/bin/sh` if the `image` is specified, `/bin/bash` otherwise", + "title": "Shell", + "type": "string" + }, + "single_branch": { + "description": "Whether to clone and track only the current branch or all remote branches. Relevant only when using remote Git repos. Defaults to `false` for dev environments and to `true` for tasks and services", + "title": "Single Branch", + "type": "boolean" + }, + "spot_policy": { + "allOf": [ + { + "$ref": "#/definitions/SpotPolicy" + } + ], + "description": "The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, `auto`. Defaults to `on-demand`" + }, + "startup_order": { + "allOf": [ + { + "$ref": "#/definitions/StartupOrder" + } + ], + "description": "The order in which master and workers jobs are started: `any`, `master-first`, `workers-first`. Defaults to `any`" + }, + "stop_criteria": { + "allOf": [ + { + "$ref": "#/definitions/StopCriteria" + } + ], + "description": "The criteria determining when a multi-node run should be considered finished: `all-done`, `master-done`. Defaults to `all-done`" + }, + "stop_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run graceful stopping. After it elapses, the run is automatically forced stopped. This includes force detaching volumes used by the run. Use `off` for unlimited duration. Defaults to `5m`", + "title": "Stop Duration" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the resource. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "type": { + "default": "task", + "enum": [ + "task" + ], + "title": "Type", + "type": "string" + }, + "user": { + "description": "The user inside the container, `user_name_or_id[:group_name_or_id]` (e.g., `ubuntu`, `1000:1000`). Defaults to the default user from the `image`", + "title": "User", + "type": "string" + }, + "utilization_policy": { + "allOf": [ + { + "$ref": "#/definitions/UtilizationPolicyRequest" + } + ], + "description": "Run termination policy based on utilization", + "title": "Utilization Policy" + }, + "volumes": { + "default": [], + "description": "The volumes mount points", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/VolumeMountPointRequest" + }, + { + "$ref": "#/definitions/InstanceMountPointRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Volumes", + "type": "array" + }, + "working_dir": { + "description": "The absolute path to the working directory inside the container. Defaults to the `image`'s default working directory", + "title": "Working Dir", + "type": "string" + } + }, + "title": "TaskConfigurationRequest", + "type": "object" + }, + "UtilizationPolicyRequest": { + "additionalProperties": false, + "properties": { + "min_gpu_utilization": { + "description": "Minimum required GPU utilization, percent. If any GPU has utilization below specified value during the whole time window, the run is terminated", + "maximum": 100, + "minimum": 0, + "title": "Min Gpu Utilization", + "type": "integer" + }, + "time_window": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The time window of metric samples taking into account to measure utilization (e.g., `30m`, `1h`). Minimum is `5m`", + "title": "Time Window" + } + }, + "required": [ + "min_gpu_utilization", + "time_window" + ], + "title": "UtilizationPolicyRequest", + "type": "object" + }, + "VastAIOfferOrder": { + "description": "An enumeration.", + "enum": [ + "score", + "price" + ], + "title": "VastAIOfferOrder", + "type": "string" + }, + "VastAIProfileOptions": { + "additionalProperties": false, + "properties": { + "min_reliability": { + "description": "The minimum reliability threshold for offers, on a scale from `0` to `1`. Defaults to `0.9`", + "maximum": 1, + "minimum": 0, + "title": "Min Reliability", + "type": "number" + }, + "min_score": { + "description": "The minimum overall score required for offers to be considered. The scoring scale varies and may require experimentation. Starting with a value in the low hundreds is generally recommended", + "minimum": 0, + "title": "Min Score", + "type": "integer" + }, + "offer_order": { + "allOf": [ + { + "$ref": "#/definitions/VastAIOfferOrder" + } + ], + "description": "Controls the order in which offers are considered for provisioning. Use `score` to prioritize the highest overall score first (the default order in the Vast.ai console), or `price` to prioritize the lowest-cost offers first. Lower-cost offers are often less reliable, so consider applying stricter filters when using `price`. Defaults to `score`" + }, + "type": { + "default": "vastai", + "enum": [ + "vastai" + ], + "title": "Type", + "type": "string" + } + }, + "title": "VastAIProfileOptions", + "type": "object" + }, + "VolumeConfigurationRequest": { + "additionalProperties": false, + "discriminator": { + "mapping": { + "aws": "#/definitions/AWSVolumeConfigurationRequest", + "gcp": "#/definitions/GCPVolumeConfigurationRequest", + "kubernetes": "#/definitions/KubernetesVolumeConfigurationRequest", + "runpod": "#/definitions/RunpodVolumeConfigurationRequest" + }, + "propertyName": "backend" + }, + "oneOf": [ + { + "$ref": "#/definitions/AWSVolumeConfigurationRequest" + }, + { + "$ref": "#/definitions/GCPVolumeConfigurationRequest" + }, + { + "$ref": "#/definitions/RunpodVolumeConfigurationRequest" + }, + { + "$ref": "#/definitions/KubernetesVolumeConfigurationRequest" + } + ], + "title": "VolumeConfigurationRequest" + }, + "VolumeMountPointRequest": { + "additionalProperties": false, + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "description": "The network volume name or the list of network volume names to mount. If a list is specified, one of the volumes in the list will be mounted. Specify volumes from different backends/regions to increase availability", + "title": "Name" + }, + "path": { + "description": "The absolute container path to mount the volume at", + "title": "Path", + "type": "string" + } + }, + "required": [ + "name", + "path" + ], + "title": "VolumeMountPointRequest", + "type": "object" + } + }, + "discriminator": { + "mapping": { + "dev-environment": "#/definitions/DevEnvironmentConfigurationRequest", + "fleet": "#/definitions/FleetConfigurationRequest", + "gateway": "#/definitions/GatewayConfigurationRequest", + "service": "#/definitions/ServiceConfigurationRequest", + "task": "#/definitions/TaskConfigurationRequest", + "volume": "#/definitions/VolumeConfigurationRequest" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/definitions/DevEnvironmentConfigurationRequest" + }, + { + "$ref": "#/definitions/TaskConfigurationRequest" + }, + { + "$ref": "#/definitions/ServiceConfigurationRequest" + }, + { + "$ref": "#/definitions/FleetConfigurationRequest" + }, + { + "$ref": "#/definitions/GatewayConfigurationRequest" + }, + { + "$ref": "#/definitions/VolumeConfigurationRequest" + } + ], + "title": "DstackConfigurationRequest" +} diff --git a/src/tests/_internal/pydantic_compat/fixtures/schema/profiles.json b/src/tests/_internal/pydantic_compat/fixtures/schema/profiles.json new file mode 100644 index 0000000000..9a71d9e472 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/fixtures/schema/profiles.json @@ -0,0 +1,538 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "definitions": { + "BackendType": { + "description": "Attributes:\n AMDDEVCLOUD (BackendType): AMD Developer Cloud\n AWS (BackendType): Amazon Web Services\n AZURE (BackendType): Microsoft Azure\n CLOUDRIFT (BackendType): CloudRift\n CRUSOE (BackendType): Crusoe\n CUDO (BackendType): Cudo\n DATACRUNCH (BackendType): DataCrunch (for backward compatibility)\n DIGITALOCEAN (BackendType): DigitalOcean\n DSTACK (BackendType): dstack Sky\n GCP (BackendType): Google Cloud Platform\n HOTAISLE (BackendType): Hot Aisle\n JARVISLABS (BackendType): JarvisLabs\n KUBERNETES (BackendType): Kubernetes\n LAMBDA (BackendType): Lambda Cloud\n NEBIUS (BackendType): Nebius AI Cloud\n OCI (BackendType): Oracle Cloud Infrastructure\n RUNPOD (BackendType): Runpod Cloud\n TENSORDOCK (BackendType): TensorDock Marketplace\n VASTAI (BackendType): Vast.ai Marketplace\n VERDA (BackendType): Verda Cloud\n VULTR (BackendType): Vultr\n SLURM (BackendType): Slurm", + "enum": [ + "amddevcloud", + "aws", + "azure", + "cloudrift", + "crusoe", + "cudo", + "datacrunch", + "digitalocean", + "dstack", + "gcp", + "hotaisle", + "jarvislabs", + "kubernetes", + "lambda", + "remote", + "nebius", + "oci", + "runpod", + "tensordock", + "vastai", + "verda", + "vultr", + "slurm" + ], + "title": "BackendType", + "type": "string" + }, + "CreationPolicy": { + "description": "An enumeration.", + "enum": [ + "reuse", + "reuse-or-create" + ], + "title": "CreationPolicy", + "type": "string" + }, + "EntityReferenceRequest": { + "additionalProperties": false, + "description": "Cross-project entity reference.", + "properties": { + "name": { + "description": "The entity name", + "title": "Name", + "type": "string" + }, + "project": { + "description": "The project name. If unspecified, refers to the current project", + "title": "Project", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "EntityReferenceRequest", + "type": "object" + }, + "FleetInstanceSelectorRequest": { + "additionalProperties": false, + "properties": { + "fleet": { + "anyOf": [ + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "minLength": 1, + "type": "string" + } + ], + "description": "The fleet reference. For fleets owned by the current project, specify the fleet name. For a fleet from another project, specify `/` or an object with `project` and `name`.", + "title": "Fleet" + }, + "instance": { + "description": "The fleet instance number", + "minimum": 0, + "title": "Instance", + "type": "integer" + } + }, + "required": [ + "fleet", + "instance" + ], + "title": "FleetInstanceSelectorRequest", + "type": "object" + }, + "InstanceHostnameSelectorRequest": { + "additionalProperties": false, + "properties": { + "hostname": { + "description": "The fleet instance hostname or IP address", + "minLength": 1, + "title": "Hostname", + "type": "string" + } + }, + "required": [ + "hostname" + ], + "title": "InstanceHostnameSelectorRequest", + "type": "object" + }, + "InstanceNameSelectorRequest": { + "additionalProperties": false, + "properties": { + "name": { + "description": "The fleet instance name", + "minLength": 1, + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "InstanceNameSelectorRequest", + "type": "object" + }, + "ProfileRequest": { + "additionalProperties": false, + "properties": { + "availability_zones": { + "description": "The availability zones to consider for provisioning (e.g., `[eu-west-1a, us-west4-a]`)", + "items": { + "type": "string" + }, + "title": "Availability Zones", + "type": "array" + }, + "backend_options": { + "description": "Backend-specific options, applied only to offers from that backend", + "items": { + "$ref": "#/definitions/VastAIProfileOptions" + }, + "title": "Backend Options", + "type": "array" + }, + "backends": { + "description": "The backends to consider for provisioning (e.g., `[aws, gcp]`)", + "items": { + "$ref": "#/definitions/BackendType" + }, + "type": "array" + }, + "creation_policy": { + "allOf": [ + { + "$ref": "#/definitions/CreationPolicy" + } + ], + "description": "The policy for using instances from fleets: `reuse`, `reuse-or-create`. Defaults to `reuse-or-create`" + }, + "default": { + "default": false, + "description": "If set to true, `dstack apply` will use this profile by default.", + "title": "Default", + "type": "boolean" + }, + "fleets": { + "description": "The fleets considered for reuse. For fleets owned by the current project, specify fleet names. For imported fleets, specify `/`", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/EntityReferenceRequest" + }, + { + "type": "string" + } + ] + }, + "title": "Fleets", + "type": "array" + }, + "idle_duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "Time to wait before terminating idle instances. When the run reuses an existing fleet instance, the fleet's `idle_duration` applies. When the run provisions a new instance, the shorter of the fleet's and run's values is used. Defaults to `5m` for runs and `3d` for fleets. Use `off` for unlimited duration. Only applied for VM-based backends", + "title": "Idle Duration" + }, + "instance_types": { + "description": "The cloud-specific instance types to consider for provisioning (e.g., `[g6e.24xlarge, n1-standard-4]`)", + "items": { + "type": "string" + }, + "title": "Instance Types", + "type": "array" + }, + "instances": { + "description": "The specific fleet instances to consider for reuse. Each value can be an instance name string, or an object with `name`, `hostname`, or `fleet` and `instance`. When set, the run is only placed on matching existing instances.", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/InstanceNameSelectorRequest" + }, + { + "$ref": "#/definitions/InstanceHostnameSelectorRequest" + }, + { + "$ref": "#/definitions/FleetInstanceSelectorRequest" + }, + { + "minLength": 1, + "type": "string" + } + ] + }, + "minItems": 1, + "title": "Instances", + "type": "array" + }, + "max_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run (e.g., `2h`, `1d`, etc) in a running state, excluding provisioning and pulling. After it elapses, the run is automatically stopped. Use `off` for unlimited duration. Defaults to `off`", + "title": "Max Duration" + }, + "max_price": { + "description": "The maximum instance price per hour, in dollars", + "exclusiveMinimum": 0.0, + "title": "Max Price", + "type": "number" + }, + "name": { + "default": "", + "description": "The name of the profile that can be passed as `--profile` to `dstack apply`", + "title": "Name", + "type": "string" + }, + "regions": { + "description": "The regions to consider for provisioning (e.g., `[eu-west-1, us-west4, westeurope]`)", + "items": { + "type": "string" + }, + "title": "Regions", + "type": "array" + }, + "reservation": { + "description": "The existing reservation to use for instance provisioning. Supports AWS Capacity Reservations, AWS Capacity Blocks, and GCP reservations", + "title": "Reservation", + "type": "string" + }, + "retry": { + "anyOf": [ + { + "$ref": "#/definitions/ProfileRetryRequest" + }, + { + "type": "boolean" + } + ], + "description": "The policy for resubmitting the run. Defaults to `false`", + "title": "Retry" + }, + "schedule": { + "allOf": [ + { + "$ref": "#/definitions/ScheduleRequest" + } + ], + "description": "The schedule for starting the run at specified time", + "title": "Schedule" + }, + "spot_policy": { + "allOf": [ + { + "$ref": "#/definitions/SpotPolicy" + } + ], + "description": "The policy for provisioning spot or on-demand instances: `spot`, `on-demand`, `auto`. Defaults to `on-demand`" + }, + "startup_order": { + "allOf": [ + { + "$ref": "#/definitions/StartupOrder" + } + ], + "description": "The order in which master and workers jobs are started: `any`, `master-first`, `workers-first`. Defaults to `any`" + }, + "stop_criteria": { + "allOf": [ + { + "$ref": "#/definitions/StopCriteria" + } + ], + "description": "The criteria determining when a multi-node run should be considered finished: `all-done`, `master-done`. Defaults to `all-done`" + }, + "stop_duration": { + "anyOf": [ + { + "enum": [ + "off" + ], + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "The maximum duration of a run graceful stopping. After it elapses, the run is automatically forced stopped. This includes force detaching volumes used by the run. Use `off` for unlimited duration. Defaults to `5m`", + "title": "Stop Duration" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "The custom tags to associate with the resource. The tags are also propagated to the underlying backend resources. If there is a conflict with backend-level tags, does not override them", + "title": "Tags", + "type": "object" + }, + "utilization_policy": { + "allOf": [ + { + "$ref": "#/definitions/UtilizationPolicyRequest" + } + ], + "description": "Run termination policy based on utilization", + "title": "Utilization Policy" + } + }, + "title": "ProfileRequest", + "type": "object" + }, + "ProfileRetryRequest": { + "additionalProperties": false, + "properties": { + "duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The maximum period of retrying the run, e.g., `4h` or `1d`. The period is calculated as a run age for `no-capacity` event and as a time passed since the last `interruption` and `error` for `interruption` and `error` events.", + "title": "Duration" + }, + "on_events": { + "description": "The list of events that should be handled with retry. Supported events are `no-capacity`, `interruption`, `error`. Omit to retry on all events", + "items": { + "$ref": "#/definitions/RetryEvent" + }, + "type": "array" + } + }, + "title": "ProfileRetryRequest", + "type": "object" + }, + "RetryEvent": { + "description": "An enumeration.", + "enum": [ + "no-capacity", + "interruption", + "error" + ], + "title": "RetryEvent", + "type": "string" + }, + "ScheduleRequest": { + "additionalProperties": false, + "properties": { + "cron": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "A cron expression or a list of cron expressions specifying the UTC time when the run needs to be started", + "title": "Cron" + } + }, + "required": [ + "cron" + ], + "title": "ScheduleRequest", + "type": "object" + }, + "SpotPolicy": { + "description": "An enumeration.", + "enum": [ + "spot", + "on-demand", + "auto" + ], + "title": "SpotPolicy", + "type": "string" + }, + "StartupOrder": { + "description": "An enumeration.", + "enum": [ + "any", + "master-first", + "workers-first" + ], + "title": "StartupOrder", + "type": "string" + }, + "StopCriteria": { + "description": "An enumeration.", + "enum": [ + "all-done", + "master-done" + ], + "title": "StopCriteria", + "type": "string" + }, + "UtilizationPolicyRequest": { + "additionalProperties": false, + "properties": { + "min_gpu_utilization": { + "description": "Minimum required GPU utilization, percent. If any GPU has utilization below specified value during the whole time window, the run is terminated", + "maximum": 100, + "minimum": 0, + "title": "Min Gpu Utilization", + "type": "integer" + }, + "time_window": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "The time window of metric samples taking into account to measure utilization (e.g., `30m`, `1h`). Minimum is `5m`", + "title": "Time Window" + } + }, + "required": [ + "min_gpu_utilization", + "time_window" + ], + "title": "UtilizationPolicyRequest", + "type": "object" + }, + "VastAIOfferOrder": { + "description": "An enumeration.", + "enum": [ + "score", + "price" + ], + "title": "VastAIOfferOrder", + "type": "string" + }, + "VastAIProfileOptions": { + "additionalProperties": false, + "properties": { + "min_reliability": { + "description": "The minimum reliability threshold for offers, on a scale from `0` to `1`. Defaults to `0.9`", + "maximum": 1, + "minimum": 0, + "title": "Min Reliability", + "type": "number" + }, + "min_score": { + "description": "The minimum overall score required for offers to be considered. The scoring scale varies and may require experimentation. Starting with a value in the low hundreds is generally recommended", + "minimum": 0, + "title": "Min Score", + "type": "integer" + }, + "offer_order": { + "allOf": [ + { + "$ref": "#/definitions/VastAIOfferOrder" + } + ], + "description": "Controls the order in which offers are considered for provisioning. Use `score` to prioritize the highest overall score first (the default order in the Vast.ai console), or `price` to prioritize the lowest-cost offers first. Lower-cost offers are often less reliable, so consider applying stricter filters when using `price`. Defaults to `score`" + }, + "type": { + "default": "vastai", + "enum": [ + "vastai" + ], + "title": "Type", + "type": "string" + } + }, + "title": "VastAIProfileOptions", + "type": "object" + } + }, + "properties": { + "profiles": { + "items": { + "$ref": "#/definitions/ProfileRequest" + }, + "title": "Profiles", + "type": "array" + } + }, + "required": [ + "profiles" + ], + "title": "ProfilesConfigRequest", + "type": "object" +} diff --git a/src/tests/_internal/pydantic_compat/test_schema.py b/src/tests/_internal/pydantic_compat/test_schema.py new file mode 100644 index 0000000000..4853558377 --- /dev/null +++ b/src/tests/_internal/pydantic_compat/test_schema.py @@ -0,0 +1,83 @@ +""" +The two JSON Schemas dstack publishes. + +`generate-json-schema` in `.github/workflows/build-artifacts.yml` renders exactly these two, and +`upload-post-pypi-artifacts.yml` copies them to `s3://dstack-runner-downloads//schemas/` +and `latest/schemas/`. Editor YAML plugins fetch them from there to validate `.dstack.yml` and +`profiles.yml`, which makes the schema a published format with consumers outside this repo rather +than an internal detail — and unlike the other surfaces here, nothing in the codebase reads it +back, so a regression has no failing caller to reveal it. + +pydantic v2 changes it on purpose: JSON Schema moves to draft 2020-12, so `definitions` becomes +`$defs`, `Optional[X]` renders as `anyOf: [..., {"type": "null"}]`, and a `$ref` with siblings +gets wrapped. The snapshot exists to make that diff explicit and reviewable in one place instead +of leaving it to a user whose editor quietly stopped validating. + +The fixtures are canonicalized rather than byte-identical to the published files: sorted keys and +two-space indent, where CI emits one long line. Key order carries no meaning in JSON Schema, and a +3800-line reflow is not a diff anyone can review. +""" + +import json +from typing import Any, Union + +import pytest + +from dstack._internal.core.models.configurations import DstackConfiguration +from dstack._internal.core.models.profiles import ProfilesConfig +from tests._internal.pydantic_compat.compare import assert_matches_fixture + +# Keyed by the filename CI publishes, so the mapping to the S3 object is unambiguous. +PUBLISHED_SCHEMAS: dict[str, Any] = { + "configuration": DstackConfiguration, + "profiles": ProfilesConfig, +} + + +class TestPublishedSchemas: + @pytest.mark.parametrize("name", sorted(PUBLISHED_SCHEMAS)) + def test_matches_fixture(self, name, regen): + # The exact call the CI job makes. + schema_json = PUBLISHED_SCHEMAS[name].schema_json() + assert_matches_fixture("schema", name, schema_json, regen=regen) + + @pytest.mark.parametrize("name", sorted(PUBLISHED_SCHEMAS)) + def test_every_ref_resolves(self, name): + """ + A dangling `$ref` makes the whole schema unusable to a validator, and the snapshot alone + would not flag it — a broken ref is just another line in a 3800-line diff that gets + accepted along with the intended draft change. + + This is not hypothetical: `add_extra_schema_types` in `utils/json_schema.py` rewrites + properties in place and has already produced a `KeyError: '$ref'` in this job once. + """ + schema = json.loads(PUBLISHED_SCHEMAS[name].schema_json()) + definitions = _definitions(schema) + assert definitions, "expected the schema to define types to reference" + unresolved = sorted( + ref for ref in _collect_refs(schema) if _ref_target(ref) not in definitions + ) + assert unresolved == [] + + +def _definitions(schema: dict) -> dict: + """v1 emits `definitions`, draft 2020-12 emits `$defs`. Accept whichever is present.""" + return schema.get("definitions") or schema.get("$defs") or {} + + +def _collect_refs(node: Any, out: Union[list, None] = None) -> list: + out = [] if out is None else out + if isinstance(node, dict): + if isinstance(node.get("$ref"), str): + out.append(node["$ref"]) + for value in node.values(): + _collect_refs(value, out) + elif isinstance(node, list): + for value in node: + _collect_refs(value, out) + return out + + +def _ref_target(ref: str) -> str: + """`#/definitions/Foo` or `#/$defs/Foo` -> `Foo`.""" + return ref.rsplit("/", 1)[-1] From 0ce01a64c276668518b51fcec1dbc2f14c3d2a01 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 30 Jul 2026 12:11:25 +0500 Subject: [PATCH 15/15] Add --pydantic-compat marker --- src/tests/_internal/pydantic_compat/conftest.py | 13 +++++++++++++ src/tests/conftest.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/tests/_internal/pydantic_compat/conftest.py b/src/tests/_internal/pydantic_compat/conftest.py index b4421ecb10..8c7f21bdfc 100644 --- a/src/tests/_internal/pydantic_compat/conftest.py +++ b/src/tests/_internal/pydantic_compat/conftest.py @@ -10,6 +10,19 @@ def pytest_addoption(parser): ) +def pytest_collection_modifyitems(config, items): + """ + Mark everything in this package `pydantic_compat` so `--pydantic-compat` gates it. + + Applied here rather than as a `pytestmark` in each module so a module added later cannot + silently escape the gate and start failing regular CI. + """ + package_dir = __file__.rsplit("/", 1)[0] + for item in items: + if str(item.path).startswith(package_dir): + item.add_marker(pytest.mark.pydantic_compat) + + @pytest.fixture def regen(request) -> bool: return request.config.getoption("--regen-fixtures") diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 8106d67a16..cef0711dee 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -20,6 +20,11 @@ def pytest_configure(config): "markers", "windows: mark test to be run on Windows in addition to POSIX" ) config.addinivalue_line("markers", "windows_only: mark test to be run on Windows only") + config.addinivalue_line( + "markers", + "pydantic_compat: mark test as a pydantic v1/v2 compat check to run only with" + " --pydantic-compat", + ) def pytest_addoption(parser): @@ -27,11 +32,18 @@ def pytest_addoption(parser): parser.addoption( "--runpostgres", action="store_true", default=False, help="Run tests with PostgreSQL" ) + parser.addoption( + "--pydantic-compat", + action="store_true", + default=False, + help="Run the pydantic v1/v2 compat fixtures (src/tests/_internal/pydantic_compat)", + ) def pytest_collection_modifyitems(config, items): skip_ui = pytest.mark.skip(reason="need --runui option to run") skip_postgres = pytest.mark.skip(reason="need --runpostgres option to run") + skip_pydantic_compat = pytest.mark.skip(reason="need --pydantic-compat option to run") is_windows = os.name == "nt" skip_posix = pytest.mark.skip(reason="requires POSIX") skip_windows = pytest.mark.skip(reason="requires Windows") @@ -40,6 +52,8 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_ui) if not config.getoption("--runpostgres") and "postgres" in item.keywords: item.add_marker(skip_postgres) + if not config.getoption("--pydantic-compat") and "pydantic_compat" in item.keywords: + item.add_marker(skip_pydantic_compat) for_windows_only = "windows_only" in item.keywords for_windows = for_windows_only or "windows" in item.keywords if for_windows_only and not is_windows: