From 1377db83ad4378c43a1e01ef40f4ea03178ab9ec Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Thu, 16 Jul 2026 10:48:58 +0800 Subject: [PATCH 1/8] ci: preserve test order across duration shards Use duration-based contiguous chunks so neighboring tests and their expensive class-scoped setup are less likely to be dispersed across all CI shards.\n\nCoding-Agent: Codex\nCodex-Version: codex-cli 0.144.4\nModel: gpt-5.6-sol\nReasoning-Effort: xhigh --- .github/workflows/test_python.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index c735cf1135..e1f2fb77bc 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -60,7 +60,9 @@ jobs: restore-keys: | test2-durations-combined-${{ matrix.python }}-${{ github.sha }} test2-durations-combined-${{ matrix.python }} - - run: pytest --cov=deepmd source/tests --ignore=source/tests/tf2 --splits 12 --group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations --splitting-algorithm least_duration + # Keep neighboring tests together so expensive class-scoped setup is not + # dispersed across every shard by the per-test least-duration algorithm. + - run: pytest --cov=deepmd source/tests --ignore=source/tests/tf2 --splits 12 --group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations --splitting-algorithm duration_based_chunks env: NUM_WORKERS: 0 DP_CI_IMPORT_PADDLE_BEFORE_TF: 1 From dd232a6d80e2a71c9af82cbac4cefe12f8e8c512 Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Thu, 16 Jul 2026 11:14:36 +0800 Subject: [PATCH 2/8] ci: reset test duration cache namespace Start the contiguous splitting algorithm with a clean duration cache so setup costs recorded under least-duration sharding do not distort the new chunk boundaries.\n\nCoding-Agent: Codex\nCodex-Version: codex-cli 0.144.4\nModel: gpt-5.6-sol\nReasoning-Effort: xhigh --- .github/workflows/test_python.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index e1f2fb77bc..329e77b1ad 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -56,10 +56,10 @@ jobs: # the key must never match, even when restarting workflows, as that # will cause durations to get out of sync between groups, the # combined durations will be loaded if available - key: test2-durations-split-${{ github.run_id }}-${{ github.run_number}}-${{ matrix.python }}-${{ matrix.group }} + key: test3-durations-split-${{ github.run_id }}-${{ github.run_number}}-${{ matrix.python }}-${{ matrix.group }} restore-keys: | - test2-durations-combined-${{ matrix.python }}-${{ github.sha }} - test2-durations-combined-${{ matrix.python }} + test3-durations-combined-${{ matrix.python }}-${{ github.sha }} + test3-durations-combined-${{ matrix.python }} # Keep neighboring tests together so expensive class-scoped setup is not # dispersed across every shard by the per-test least-duration algorithm. - run: pytest --cov=deepmd source/tests --ignore=source/tests/tf2 --splits 12 --group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations --splitting-algorithm duration_based_chunks @@ -134,8 +134,8 @@ jobs: # key won't match during the first run for the given commit, but # restore-key will if there's a previous stored durations file, # so cache will both be loaded and stored - key: test2-durations-combined-${{ matrix.python }}-${{ github.sha }} - restore-keys: test2-durations-combined-${{ matrix.python }} + key: test3-durations-combined-${{ matrix.python }}-${{ github.sha }} + restore-keys: test3-durations-combined-${{ matrix.python }} - name: Download artifacts uses: actions/download-artifact@v8 with: From b8b5d0fbc65e128ddee5b70f1b121ba663a1e173 Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Thu, 16 Jul 2026 22:24:34 +0800 Subject: [PATCH 3/8] ci: isolate duration caches across reruns Include github.run_attempt in each per-shard cache key so workflow reruns cannot restore stale shard-local duration data from an earlier attempt. Coding-Agent: Codex Codex-Version: codex-cli 0.144.4 Model: gpt-5.6-sol Reasoning-Effort: xhigh --- .github/workflows/test_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 329e77b1ad..01408df43c 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -56,7 +56,7 @@ jobs: # the key must never match, even when restarting workflows, as that # will cause durations to get out of sync between groups, the # combined durations will be loaded if available - key: test3-durations-split-${{ github.run_id }}-${{ github.run_number}}-${{ matrix.python }}-${{ matrix.group }} + key: test3-durations-split-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.python }}-${{ matrix.group }} restore-keys: | test3-durations-combined-${{ matrix.python }}-${{ github.sha }} test3-durations-combined-${{ matrix.python }} From f36adb73ea864a75783bc6227c9faafd6fd5441d Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Sat, 18 Jul 2026 15:31:43 +0800 Subject: [PATCH 4/8] ci: balance fixture-aware test groups Keep test classes and module-level tests together, then assign those units with deterministic duration-based LPT bin packing. Warm-start the new cache namespace from existing per-test timings so the first CI run remains balanced. Coding-Agent: Codex Codex-Version: codex-cli 0.144.4 Model: gpt-5.6-sol Reasoning-Effort: xhigh --- .github/workflows/test_python.yml | 22 ++- source/tests/ci_split.py | 238 ++++++++++++++++++++++++++++++ source/tests/test_ci_split.py | 140 ++++++++++++++++++ 3 files changed, 392 insertions(+), 8 deletions(-) create mode 100644 source/tests/ci_split.py create mode 100644 source/tests/test_ci_split.py diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 01408df43c..b91f536be4 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -56,13 +56,17 @@ jobs: # the key must never match, even when restarting workflows, as that # will cause durations to get out of sync between groups, the # combined durations will be loaded if available - key: test3-durations-split-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.python }}-${{ matrix.group }} + key: test4-durations-split-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.python }}-${{ matrix.group }} + # Warm-start the first grouped run from the existing per-test + # timings instead of treating every test as equally expensive. restore-keys: | - test3-durations-combined-${{ matrix.python }}-${{ github.sha }} - test3-durations-combined-${{ matrix.python }} - # Keep neighboring tests together so expensive class-scoped setup is not - # dispersed across every shard by the per-test least-duration algorithm. - - run: pytest --cov=deepmd source/tests --ignore=source/tests/tf2 --splits 12 --group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations --splitting-algorithm duration_based_chunks + test4-durations-combined-${{ matrix.python }}-${{ github.sha }} + test4-durations-combined-${{ matrix.python }} + test2-durations-combined-${{ matrix.python }}-${{ github.sha }} + test2-durations-combined-${{ matrix.python }} + # Keep fixture-sharing classes/modules together, then balance those + # indivisible units by their recorded durations. + - run: python -m pytest -p source.tests.ci_split --cov=deepmd source/tests --ignore=source/tests/tf2 --ci-splits 12 --ci-group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations env: NUM_WORKERS: 0 DP_CI_IMPORT_PADDLE_BEFORE_TF: 1 @@ -134,8 +138,10 @@ jobs: # key won't match during the first run for the given commit, but # restore-key will if there's a previous stored durations file, # so cache will both be loaded and stored - key: test3-durations-combined-${{ matrix.python }}-${{ github.sha }} - restore-keys: test3-durations-combined-${{ matrix.python }} + key: test4-durations-combined-${{ matrix.python }}-${{ github.sha }} + restore-keys: | + test4-durations-combined-${{ matrix.python }} + test2-durations-combined-${{ matrix.python }} - name: Download artifacts uses: actions/download-artifact@v8 with: diff --git a/source/tests/ci_split.py b/source/tests/ci_split.py new file mode 100644 index 0000000000..d54df7a1e5 --- /dev/null +++ b/source/tests/ci_split.py @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Keep fixture-sharing test units together while balancing CI shards. + +``pytest-split`` balances individual test items well, but assigning individual +methods independently repeats class- and module-scoped setup in multiple CI +processes. This plugin first groups items into fixture-sharing units, then +uses longest-processing-time (LPT) bin packing to balance those units by their +recorded durations. + +Classes are kept together by default. Module-level tests from the same file +form one unit. Tests that need to share a module-scoped fixture across class +boundaries can use ``@pytest.mark.ci_split_group("name")`` to opt into the same +explicit unit. +""" + +from __future__ import annotations + +import heapq +import json +import math +from collections import OrderedDict +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Protocol + +import pytest + +if TYPE_CHECKING: + from _pytest import nodes + from _pytest.config import Config + from _pytest.config.argparsing import Parser + + +class _SplitItem(Protocol): + """The subset of a pytest item used by the grouping algorithm.""" + + nodeid: str + cls: type[object] | None + + def get_closest_marker(self, name: str) -> pytest.Mark | None: ... + + +@dataclass +class _TestUnit: + """A set of test items that must execute in the same CI shard.""" + + key: str + items: list[_SplitItem] = field(default_factory=list) + estimated_duration: float = 0.0 + + +@dataclass +class _TestGroup: + """One CI shard selected by the grouped splitting algorithm.""" + + items: list[_SplitItem] + estimated_duration: float + unit_count: int + + +def _explicit_group_name(item: _SplitItem) -> str | None: + """Return and validate an explicitly requested cross-class group name.""" + marker = item.get_closest_marker("ci_split_group") + if marker is None: + return None + if len(marker.args) != 1 or marker.kwargs or not isinstance(marker.args[0], str): + raise pytest.UsageError( + "ci_split_group requires exactly one non-empty string argument" + ) + name = marker.args[0].strip() + if not name: + raise pytest.UsageError( + "ci_split_group requires exactly one non-empty string argument" + ) + return name + + +def _unit_key(item: _SplitItem) -> str: + """Return a deterministic fixture-sharing unit key for a pytest item.""" + explicit_name = _explicit_group_name(item) + if explicit_name is not None: + return f"explicit:{explicit_name}" + + # nodeid contains a repository-relative path, unlike item.path which may + # contain runner-specific checkout prefixes. + path = item.nodeid.split("::", maxsplit=1)[0] + if item.cls is not None: + return f"class:{path}::{item.cls.__qualname__}" + return f"module:{path}" + + +def _relevant_durations( + items: list[_SplitItem], durations: dict[str, float] +) -> dict[str, float]: + """Discard stale cache entries before deriving the unknown-test fallback.""" + nodeids = {item.nodeid for item in items} + return { + nodeid: float(duration) + for nodeid, duration in durations.items() + if nodeid in nodeids and duration >= 0 and math.isfinite(duration) + } + + +def _build_units( + items: list[_SplitItem], durations: dict[str, float] +) -> list[_TestUnit]: + """Aggregate test items and their estimated durations into stable units.""" + relevant = _relevant_durations(items, durations) + fallback_duration = sum(relevant.values()) / len(relevant) if relevant else 1.0 + units: OrderedDict[str, _TestUnit] = OrderedDict() + for item in items: + key = _unit_key(item) + unit = units.setdefault(key, _TestUnit(key=key)) + unit.items.append(item) + unit.estimated_duration += relevant.get(item.nodeid, fallback_duration) + return list(units.values()) + + +def _split_items( + items: list[_SplitItem], durations: dict[str, float], splits: int +) -> list[_TestGroup]: + """Assign indivisible units using deterministic LPT bin packing.""" + units = _build_units(items, durations) + item_index = {id(item): index for index, item in enumerate(items)} + + # The group index breaks equal-duration ties deterministically. Sorting + # units by key gives all CI runners the same assignment even when several + # units have identical estimates, including a completely empty cache. + heap: list[tuple[float, int, list[_TestUnit]]] = [ + (0.0, group_index, []) for group_index in range(splits) + ] + heapq.heapify(heap) + for unit in sorted(units, key=lambda value: (-value.estimated_duration, value.key)): + duration, group_index, assigned = heapq.heappop(heap) + assigned.append(unit) + heapq.heappush( + heap, + (duration + unit.estimated_duration, group_index, assigned), + ) + + groups: list[_TestGroup | None] = [None] * splits + for duration, group_index, assigned in heap: + selected = [item for unit in assigned for item in unit.items] + selected.sort(key=lambda item: item_index[id(item)]) + groups[group_index] = _TestGroup( + items=selected, + estimated_duration=duration, + unit_count=len(assigned), + ) + return [group for group in groups if group is not None] + + +def _load_durations(path: Path) -> dict[str, float]: + """Load pytest-split's duration cache, including its legacy list format.""" + try: + data = json.loads(path.read_text()) + except FileNotFoundError: + return {} + except (OSError, json.JSONDecodeError) as exc: + raise pytest.UsageError(f"cannot read duration cache {path}: {exc}") from exc + + if isinstance(data, list): + data = dict(data) + if not isinstance(data, dict): + raise pytest.UsageError(f"duration cache {path} must contain a JSON object") + try: + return {str(nodeid): float(duration) for nodeid, duration in data.items()} + except (TypeError, ValueError) as exc: + raise pytest.UsageError( + f"duration cache {path} contains a non-numeric duration" + ) from exc + + +def pytest_addoption(parser: Parser) -> None: + """Declare CI-specific split options without activating pytest-split.""" + group = parser.getgroup("DeePMD grouped CI splitting") + group.addoption( + "--ci-splits", + type=int, + help="Number of grouped CI test shards.", + ) + group.addoption( + "--ci-group", + type=int, + help="One-based grouped CI shard to execute.", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_cmdline_main(config: Config) -> int | None: + """Validate grouped split options before pytest starts collection.""" + splits = config.getoption("ci_splits") + group = config.getoption("ci_group") + if splits is None and group is None: + return None + if splits is None or group is None: + raise pytest.UsageError("--ci-splits and --ci-group must be used together") + if splits < 1: + raise pytest.UsageError("--ci-splits must be at least 1") + if group < 1 or group > splits: + raise pytest.UsageError("--ci-group must be between 1 and --ci-splits") + return None + + +def pytest_configure(config: Config) -> None: + """Register the marker used to join classes sharing expensive fixtures.""" + config.addinivalue_line( + "markers", + "ci_split_group(name): keep marked tests in one duration-balanced CI shard", + ) + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: Config, items: list[nodes.Item]) -> None: + """Select one grouped shard after the complete test suite is collected.""" + splits = config.getoption("ci_splits") + group_index = config.getoption("ci_group") + if splits is None or group_index is None: + return + + durations_path = Path(config.getoption("durations_path")) + durations = _load_durations(durations_path) + groups = _split_items(items, durations, splits) + selected = groups[group_index - 1] + selected_ids = {id(item) for item in selected.items} + deselected = [item for item in items if id(item) not in selected_ids] + items[:] = selected.items + config.hook.pytest_deselected(items=deselected) + + reporter = config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + cache_state = "warm" if durations else "empty" + reporter.write_line( + "[deepmd-ci-split] " + f"group {group_index}/{splits}: {len(selected.items)} tests in " + f"{selected.unit_count} units, estimated " + f"{selected.estimated_duration:.2f}s ({cache_state} cache)" + ) diff --git a/source/tests/test_ci_split.py b/source/tests/test_ci_split.py new file mode 100644 index 0000000000..b64981c152 --- /dev/null +++ b/source/tests/test_ci_split.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Unit tests for fixture-aware CI test splitting.""" + +from dataclasses import dataclass + +import pytest + +from .ci_split import ( + _TestGroup, + _split_items, + _unit_key, +) + + +class FirstClass: + pass + + +class SecondClass: + pass + + +@dataclass(frozen=True) +class FakeMarker: + args: tuple[object, ...] + kwargs: dict[str, object] + + +@dataclass +class FakeItem: + nodeid: str + cls: type[object] | None = None + explicit_group: str | None = None + + def get_closest_marker(self, name: str) -> FakeMarker | None: + if name == "ci_split_group" and self.explicit_group is not None: + return FakeMarker((self.explicit_group,), {}) + return None + + +def _item( + nodeid: str, + *, + cls: type[object] | None = None, + explicit_group: str | None = None, +) -> FakeItem: + return FakeItem( + nodeid=nodeid, + cls=cls, + explicit_group=explicit_group, + ) + + +def _owner(groups: list[_TestGroup], item: FakeItem) -> int: + return next(index for index, group in enumerate(groups) if item in group.items) + + +class TestUnitConstruction: + def test_class_and_module_items_are_indivisible(self) -> None: + class_items = [ + _item(f"tests/test_a.py::FirstClass::test_{index}", cls=FirstClass) + for index in range(3) + ] + module_items = [ + _item(f"tests/test_a.py::test_module_{index}") for index in range(2) + ] + groups = _split_items(class_items + module_items, {}, splits=2) + + assert len({_owner(groups, item) for item in class_items}) == 1 + assert len({_owner(groups, item) for item in module_items}) == 1 + + def test_explicit_group_joins_classes_across_modules(self) -> None: + first = _item( + "tests/test_a.py::FirstClass::test_one", + cls=FirstClass, + explicit_group="shared-model", + ) + second = _item( + "tests/test_b.py::SecondClass::test_two", + cls=SecondClass, + explicit_group="shared-model", + ) + groups = _split_items([first, second], {}, splits=2) + + assert _owner(groups, first) == _owner(groups, second) + + def test_unit_key_rejects_empty_explicit_group(self) -> None: + item = _item( + "tests/test_a.py::FirstClass::test_one", + cls=FirstClass, + explicit_group=" ", + ) + + with pytest.raises(pytest.UsageError, match="non-empty string"): + _unit_key(item) + + +class TestUnitBalancing: + def test_lpt_balances_recorded_unit_durations(self) -> None: + items = [_item(f"tests/test_{index}.py::test_value") for index in range(4)] + durations = dict( + zip( + (item.nodeid for item in items), + [9.0, 8.0, 7.0, 6.0], + strict=True, + ) + ) + groups = _split_items(items, durations, splits=2) + + assert [group.estimated_duration for group in groups] == [15.0, 15.0] + + def test_empty_cache_balances_by_number_of_tests(self) -> None: + items = [ + *[ + _item(f"tests/test_a.py::FirstClass::test_{index}", cls=FirstClass) + for index in range(5) + ], + *[ + _item(f"tests/test_b.py::SecondClass::test_{index}", cls=SecondClass) + for index in range(4) + ], + *[_item(f"tests/test_c.py::test_{index}") for index in range(3)], + *[_item(f"tests/test_d.py::test_{index}") for index in range(2)], + ] + groups = _split_items(items, {}, splits=2) + + assert [len(group.items) for group in groups] == [7, 7] + + def test_assignment_is_deterministic_and_preserves_collection_order(self) -> None: + items = [_item(f"tests/test_{index}.py::test_value") for index in range(8)] + + first = _split_items(items, {}, splits=3) + second = _split_items(items, {}, splits=3) + + assert [[item.nodeid for item in group.items] for group in first] == [ + [item.nodeid for item in group.items] for group in second + ] + for group in first: + indices = [items.index(item) for item in group.items] + assert indices == sorted(indices) From 2f8f911b76c808c16f0f23c7f35d136240659e7c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:34:00 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- source/tests/ci_split.py | 34 ++++++++++++++++++++++++++-------- source/tests/test_ci_split.py | 6 ++++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/source/tests/ci_split.py b/source/tests/ci_split.py index d54df7a1e5..9b45e914da 100644 --- a/source/tests/ci_split.py +++ b/source/tests/ci_split.py @@ -13,22 +13,40 @@ explicit unit. """ -from __future__ import annotations +from __future__ import ( + annotations, +) import heapq import json import math -from collections import OrderedDict -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING, Protocol +from collections import ( + OrderedDict, +) +from dataclasses import ( + dataclass, + field, +) +from pathlib import ( + Path, +) +from typing import ( + TYPE_CHECKING, + Protocol, +) import pytest if TYPE_CHECKING: - from _pytest import nodes - from _pytest.config import Config - from _pytest.config.argparsing import Parser + from _pytest import ( + nodes, + ) + from _pytest.config import ( + Config, + ) + from _pytest.config.argparsing import ( + Parser, + ) class _SplitItem(Protocol): diff --git a/source/tests/test_ci_split.py b/source/tests/test_ci_split.py index b64981c152..d7d0dffa53 100644 --- a/source/tests/test_ci_split.py +++ b/source/tests/test_ci_split.py @@ -1,13 +1,15 @@ # SPDX-License-Identifier: LGPL-3.0-or-later """Unit tests for fixture-aware CI test splitting.""" -from dataclasses import dataclass +from dataclasses import ( + dataclass, +) import pytest from .ci_split import ( - _TestGroup, _split_items, + _TestGroup, _unit_key, ) From 07187530811c19934b953869d84c0559e5b31295 Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Sat, 18 Jul 2026 16:23:34 +0800 Subject: [PATCH 6/8] fix(ci): seed first grouped test run Use version-controlled Python 3.10 and 3.13 estimates for expensive fixture units instead of relying on a possibly missing test2 cache. Preserve complete class/module units for explicit markers, retain TF2 timings through independent duration files, and report malformed legacy caches as usage errors. Coding-Agent: Codex Codex-Version: codex-cli 0.144.4 Model: gpt-5.6-sol Reasoning-Effort: xhigh --- .github/workflows/test_python.yml | 39 +++++--- source/tests/ci_split.py | 127 ++++++++++++++++++++++----- source/tests/ci_split_seed_3.10.json | 53 +++++++++++ source/tests/ci_split_seed_3.13.json | 55 ++++++++++++ source/tests/test_ci_split.py | 77 ++++++++++++---- 5 files changed, 300 insertions(+), 51 deletions(-) create mode 100644 source/tests/ci_split_seed_3.10.json create mode 100644 source/tests/ci_split_seed_3.13.json diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index b91f536be4..8b73635fcf 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -57,16 +57,19 @@ jobs: # will cause durations to get out of sync between groups, the # combined durations will be loaded if available key: test4-durations-split-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.python }}-${{ matrix.group }} - # Warm-start the first grouped run from the existing per-test - # timings instead of treating every test as equally expensive. restore-keys: | test4-durations-combined-${{ matrix.python }}-${{ github.sha }} test4-durations-combined-${{ matrix.python }} - test2-durations-combined-${{ matrix.python }}-${{ github.sha }} - test2-durations-combined-${{ matrix.python }} + - name: Prepare independent duration inputs + run: | + for output in main tf2 jax2tf consistent_tf2; do + if [ -f .test_durations ]; then + cp .test_durations ".test_durations_${output}" + fi + done # Keep fixture-sharing classes/modules together, then balance those # indivisible units by their recorded durations. - - run: python -m pytest -p source.tests.ci_split --cov=deepmd source/tests --ignore=source/tests/tf2 --ci-splits 12 --ci-group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations + - run: python -m pytest -p source.tests.ci_split --cov=deepmd source/tests --ignore=source/tests/tf2 --ci-splits 12 --ci-group ${{ matrix.group }} --store-durations --clean-durations --durations-path=.test_durations_main env: NUM_WORKERS: 0 DP_CI_IMPORT_PADDLE_BEFORE_TF: 1 @@ -74,29 +77,38 @@ jobs: - name: Test TF2 eager mode run: | run_pytest_allow_no_tests() { + local durations_path=$1 + shift set +e - pytest "$@" + pytest "$@" \ + --store-durations \ + --clean-durations \ + --durations-path="$durations_path" local status=$? set -e if [ "$status" -eq 5 ]; then # pytest-split may leave an individual shard with no selected # tests after path/-k filtering. Other shards still cover the # selected tests, so do not fail the whole matrix for exit 5. + printf '{}\n' > "$durations_path" return 0 fi return "$status" } - run_pytest_allow_no_tests --cov=deepmd --cov-append \ + run_pytest_allow_no_tests .test_durations_tf2 \ + --cov=deepmd --cov-append \ source/tests/tf2 \ --splits 12 \ --group ${{ matrix.group }} - run_pytest_allow_no_tests --cov=deepmd --cov-append \ + run_pytest_allow_no_tests .test_durations_jax2tf \ + --cov=deepmd --cov-append \ source/tests/consistent/io/test_io.py \ source/jax2tf_tests \ --splits 12 \ --group ${{ matrix.group }} - run_pytest_allow_no_tests --cov=deepmd --cov-append \ + run_pytest_allow_no_tests .test_durations_consistent_tf2 \ + --cov=deepmd --cov-append \ source/tests/consistent \ -k tf2 \ --splits 12 \ @@ -106,7 +118,8 @@ jobs: DP_TEST_TF2_ONLY: 1 DP_DTYPE_PROMOTION_STRICT: 1 DP_CI_IMPORT_PADDLE_BEFORE_TF: 1 - - run: mv .test_durations .test_durations_${{ matrix.group }} + - name: Combine shard durations + run: jq -s 'reduce .[] as $durations ({}; . * $durations)' .test_durations_main .test_durations_tf2 .test_durations_jax2tf .test_durations_consistent_tf2 > .test_durations_${{ matrix.group }} - name: Upload partial durations uses: actions/upload-artifact@v7 with: @@ -139,16 +152,14 @@ jobs: # restore-key will if there's a previous stored durations file, # so cache will both be loaded and stored key: test4-durations-combined-${{ matrix.python }}-${{ github.sha }} - restore-keys: | - test4-durations-combined-${{ matrix.python }} - test2-durations-combined-${{ matrix.python }} + restore-keys: test4-durations-combined-${{ matrix.python }} - name: Download artifacts uses: actions/download-artifact@v8 with: pattern: split-${{ matrix.python }}-* merge-multiple: true - name: Combine test durations - run: jq -s add .test_durations_* > .test_durations + run: jq -s 'reduce .[] as $durations ({}; . * $durations)' .test_durations_* > .test_durations pass: name: Pass testing Python needs: [testpython, update_durations] diff --git a/source/tests/ci_split.py b/source/tests/ci_split.py index 9b45e914da..bf27c6a1d8 100644 --- a/source/tests/ci_split.py +++ b/source/tests/ci_split.py @@ -20,6 +20,7 @@ import heapq import json import math +import sys from collections import ( OrderedDict, ) @@ -55,7 +56,17 @@ class _SplitItem(Protocol): nodeid: str cls: type[object] | None - def get_closest_marker(self, name: str) -> pytest.Mark | None: ... + def get_closest_marker(self, name: str) -> pytest.Mark | None: + """Return the closest marker with the requested name.""" + raise NotImplementedError + + +@dataclass(frozen=True) +class _DurationSeed: + """Version-controlled estimates used before a runtime cache exists.""" + + default_test_duration: float = 1.0 + units: dict[str, float] = field(default_factory=dict) @dataclass @@ -93,12 +104,8 @@ def _explicit_group_name(item: _SplitItem) -> str | None: return name -def _unit_key(item: _SplitItem) -> str: - """Return a deterministic fixture-sharing unit key for a pytest item.""" - explicit_name = _explicit_group_name(item) - if explicit_name is not None: - return f"explicit:{explicit_name}" - +def _base_unit_key(item: _SplitItem) -> str: + """Return the class/module unit that must never be split.""" # nodeid contains a repository-relative path, unlike item.path which may # contain runner-specific checkout prefixes. path = item.nodeid.split("::", maxsplit=1)[0] @@ -120,25 +127,68 @@ def _relevant_durations( def _build_units( - items: list[_SplitItem], durations: dict[str, float] + items: list[_SplitItem], + durations: dict[str, float], + seed: _DurationSeed, ) -> list[_TestUnit]: - """Aggregate test items and their estimated durations into stable units.""" + """Build complete class/module units, then merge explicit unit groups.""" relevant = _relevant_durations(items, durations) - fallback_duration = sum(relevant.values()) / len(relevant) if relevant else 1.0 - units: OrderedDict[str, _TestUnit] = OrderedDict() + fallback_duration = ( + sum(relevant.values()) / len(relevant) + if relevant + else seed.default_test_duration + ) + base_units: OrderedDict[str, _TestUnit] = OrderedDict() for item in items: - key = _unit_key(item) - unit = units.setdefault(key, _TestUnit(key=key)) + key = _base_unit_key(item) + unit = base_units.setdefault(key, _TestUnit(key=key)) unit.items.append(item) - unit.estimated_duration += relevant.get(item.nodeid, fallback_duration) - return list(units.values()) + + for unit in base_units.values(): + known_durations = [ + relevant[item.nodeid] for item in unit.items if item.nodeid in relevant + ] + if known_durations: + missing_count = len(unit.items) - len(known_durations) + unit.estimated_duration = sum(known_durations) + ( + missing_count * fallback_duration + ) + else: + unit.estimated_duration = seed.units.get( + unit.key, + len(unit.items) * seed.default_test_duration, + ) + + merged_units: OrderedDict[str, _TestUnit] = OrderedDict() + for unit in base_units.values(): + explicit_names = [ + name + for item in unit.items + if (name := _explicit_group_name(item)) is not None + ] + if explicit_names and len(explicit_names) != len(unit.items): + raise pytest.UsageError( + "ci_split_group must mark every test in its class/module unit" + ) + if len(set(explicit_names)) > 1: + raise pytest.UsageError( + "a class/module unit cannot use multiple ci_split_group names" + ) + key = f"explicit:{explicit_names[0]}" if explicit_names else unit.key + merged = merged_units.setdefault(key, _TestUnit(key=key)) + merged.items.extend(unit.items) + merged.estimated_duration += unit.estimated_duration + return list(merged_units.values()) def _split_items( - items: list[_SplitItem], durations: dict[str, float], splits: int + items: list[_SplitItem], + durations: dict[str, float], + splits: int, + seed: _DurationSeed | None = None, ) -> list[_TestGroup]: """Assign indivisible units using deterministic LPT bin packing.""" - units = _build_units(items, durations) + units = _build_units(items, durations, seed or _DurationSeed()) item_index = {id(item): index for index, item in enumerate(items)} # The group index breaks equal-duration ties deterministically. Sorting @@ -171,14 +221,19 @@ def _split_items( def _load_durations(path: Path) -> dict[str, float]: """Load pytest-split's duration cache, including its legacy list format.""" try: - data = json.loads(path.read_text()) + data = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: return {} except (OSError, json.JSONDecodeError) as exc: raise pytest.UsageError(f"cannot read duration cache {path}: {exc}") from exc if isinstance(data, list): - data = dict(data) + try: + data = dict(data) + except (TypeError, ValueError) as exc: + raise pytest.UsageError( + f"duration cache {path} contains an invalid legacy list" + ) from exc if not isinstance(data, dict): raise pytest.UsageError(f"duration cache {path} must contain a JSON object") try: @@ -189,6 +244,30 @@ def _load_durations(path: Path) -> dict[str, float]: ) from exc +def _load_duration_seed(path: Path) -> _DurationSeed: + """Load the version-controlled fallback for the first grouped CI run.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise pytest.UsageError(f"cannot read duration seed {path}: {exc}") from exc + if not isinstance(data, dict) or not isinstance(data.get("units"), dict): + raise pytest.UsageError(f"duration seed {path} has an invalid schema") + try: + default = float(data["default_test_duration"]) + units = {str(key): float(value) for key, value in data["units"].items()} + except (KeyError, TypeError, ValueError) as exc: + raise pytest.UsageError( + f"duration seed {path} contains a non-numeric duration" + ) from exc + if default <= 0 or not math.isfinite(default): + raise pytest.UsageError( + f"duration seed {path} has an invalid default_test_duration" + ) + if any(value <= 0 or not math.isfinite(value) for value in units.values()): + raise pytest.UsageError(f"duration seed {path} has an invalid unit duration") + return _DurationSeed(default_test_duration=default, units=units) + + def pytest_addoption(parser: Parser) -> None: """Declare CI-specific split options without activating pytest-split.""" group = parser.getgroup("DeePMD grouped CI splitting") @@ -238,7 +317,11 @@ def pytest_collection_modifyitems(config: Config, items: list[nodes.Item]) -> No durations_path = Path(config.getoption("durations_path")) durations = _load_durations(durations_path) - groups = _split_items(items, durations, splits) + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + seed = _load_duration_seed( + Path(__file__).with_name(f"ci_split_seed_{python_version}.json") + ) + groups = _split_items(items, durations, splits, seed) selected = groups[group_index - 1] selected_ids = {id(item) for item in selected.items} deselected = [item for item in items if id(item) not in selected_ids] @@ -247,10 +330,10 @@ def pytest_collection_modifyitems(config: Config, items: list[nodes.Item]) -> No reporter = config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: - cache_state = "warm" if durations else "empty" + estimate_source = "runtime cache" if durations else "committed seed" reporter.write_line( "[deepmd-ci-split] " f"group {group_index}/{splits}: {len(selected.items)} tests in " f"{selected.unit_count} units, estimated " - f"{selected.estimated_duration:.2f}s ({cache_state} cache)" + f"{selected.estimated_duration:.2f}s ({estimate_source})" ) diff --git a/source/tests/ci_split_seed_3.10.json b/source/tests/ci_split_seed_3.10.json new file mode 100644 index 0000000000..c931dedbdc --- /dev/null +++ b/source/tests/ci_split_seed_3.10.json @@ -0,0 +1,53 @@ +{ + "default_test_duration": 0.592, + "provenance": "Python 3.10 unit durations from Test Python run 29456447617; units below 120 seconds use the default estimate.", + "units": { + "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pt2": 865.503, + "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pte": 293.043, + "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pt2": 1197.644, + "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pte": 439.224, + "class:source/tests/jax/test_make_hessian_model.py::TestDPModel": 259.984, + "class:source/tests/pd/model/test_dpa2.py::TestDescrptDPA2": 235.375, + "class:source/tests/pd/test_training.py::TestFparam": 133.835, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipeline": 226.308, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipelineTritonInfer": 195.538, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMFreezeGuards": 305.304, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMWithCommExportPipeline": 220.017, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelCompile": 1989.799, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelLoRACompile": 330.974, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelProperty": 138.229, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMNativeSpinModel": 428.637, + "class:source/tests/pt/model/test_sezm_spin_model.py::TestSeZMSpinModel": 133.554, + "class:source/tests/pt_expt/descriptor/test_dpa2.py::TestDescrptDPA2": 128.333, + "class:source/tests/pt_expt/descriptor/test_dpa3.py::TestDescrptDPA3": 583.549, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEner": 582.428, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparam": 179.801, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparamPt2": 893.994, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerChgSpinPt2": 669.117, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerDefaultFparamPt2": 423.123, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerPt2": 2306.716, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalNlistBackend": 194.501, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalDescriptor": 495.034, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalFittingLastLayer": 474.863, + "class:source/tests/pt_expt/infer/test_deep_eval_metadata_only.py::TestDeepEvalMetadataOnlyPte": 229.435, + "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinAparam": 867.838, + "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinDefaultFparam": 429.227, + "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinInference": 1855.625, + "class:source/tests/pt_expt/model/test_export_pipeline.py::TestExportPipeline": 396.682, + "class:source/tests/pt_expt/model/test_model_compression.py::TestModelCompression": 213.553, + "class:source/tests/pt_expt/test_change_bias.py::TestChangeBias": 1174.856, + "class:source/tests/pt_expt/test_dp_freeze.py::TestDPFreezePtExpt": 916.732, + "class:source/tests/pt_expt/test_finetune.py::TestFinetuneCLI": 235.451, + "class:source/tests/pt_expt/test_multitask.py::TestCompileCaseEmbdVaryingNframes": 199.148, + "class:source/tests/pt_expt/test_multitask.py::TestMultiTaskCompile": 293.683, + "class:source/tests/pt_expt/test_training.py::TestCompiledConsistency": 254.354, + "class:source/tests/pt_expt/test_training.py::TestCompiledSharedFittingDifferentDescriptor": 146.813, + "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNatoms": 1244.152, + "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNframesWithParams": 196.837, + "class:source/tests/pt_expt/test_training.py::TestTraining": 184.091, + "class:source/tests/tf/test_finetune_se_atten.py::TestFinetuneSeAtten": 150.59, + "class:source/tests/tf/test_model_compression_se_atten.py::TestDeepPotAPBC": 216.183, + "module:source/tests/pt_expt/infer/test_graph_deepeval.py": 727.49, + "module:source/tests/pt_expt/utils/test_graph_pt2_metadata.py": 264.404 + } +} diff --git a/source/tests/ci_split_seed_3.13.json b/source/tests/ci_split_seed_3.13.json new file mode 100644 index 0000000000..75ec9f11fa --- /dev/null +++ b/source/tests/ci_split_seed_3.13.json @@ -0,0 +1,55 @@ +{ + "default_test_duration": 0.599, + "provenance": "Python 3.13 unit durations from Test Python run 29456447617; units below 120 seconds use the default estimate.", + "units": { + "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pt2": 584.863, + "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pte": 254.494, + "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pt2": 1105.263, + "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pte": 405.028, + "class:source/tests/jax/test_make_hessian_model.py::TestDPModel": 273.14, + "class:source/tests/pd/model/test_dpa2.py::TestDescrptDPA2": 193.951, + "class:source/tests/pd/test_training.py::TestFparam": 128.816, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipeline": 211.531, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipelineTritonInfer": 300.066, + "class:source/tests/pt/model/test_sezm_export.py::TestSeZMFreezeGuards": 278.717, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelCompile": 1860.224, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelLoRACompile": 218.127, + "class:source/tests/pt/model/test_sezm_model.py::TestSeZMNativeSpinModel": 588.729, + "class:source/tests/pt/model/test_sezm_spin_model.py::TestSeZMSpinModel": 148.399, + "class:source/tests/pt_expt/descriptor/test_dpa2.py::TestDescrptDPA2": 137.791, + "class:source/tests/pt_expt/descriptor/test_dpa3.py::TestDescrptDPA3": 565.619, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEner": 608.076, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparam": 172.449, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparamPt2": 661.455, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerChgSpinPt2": 1043.66, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerDefaultFparamPt2": 480.499, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerPt2": 2417.58, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalNlistBackend": 252.715, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalDescriptor": 495.286, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalFittingLastLayer": 498.551, + "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalTypeEbd": 122.649, + "class:source/tests/pt_expt/infer/test_deep_eval_metadata_only.py::TestDeepEvalMetadataOnlyPte": 207.916, + "class:source/tests/pt_expt/infer/test_deep_eval_pt_checkpoint.py::TestPtExptLoadPtSpin": 123.741, + "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinAparam": 841.667, + "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinDefaultFparam": 443.65, + "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinInference": 1681.299, + "class:source/tests/pt_expt/model/test_export_pipeline.py::TestExportPipeline": 424.391, + "class:source/tests/pt_expt/model/test_model_compression.py::TestModelCompression": 262.056, + "class:source/tests/pt_expt/test_change_bias.py::TestChangeBias": 1218.921, + "class:source/tests/pt_expt/test_dp_freeze.py::TestDPFreezePtExpt": 1109.204, + "class:source/tests/pt_expt/test_finetune.py::TestFinetuneCLI": 213.089, + "class:source/tests/pt_expt/test_multitask.py::TestCompileCaseEmbdVaryingNframes": 234.837, + "class:source/tests/pt_expt/test_multitask.py::TestMultiTaskCompile": 385.028, + "class:source/tests/pt_expt/test_multitask.py::TestMultiTaskDPA3": 126.497, + "class:source/tests/pt_expt/test_training.py::TestCompiledConsistency": 186.814, + "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNatoms": 1370.061, + "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNframesWithParams": 137.311, + "class:source/tests/pt_expt/test_training.py::TestRestart": 134.763, + "class:source/tests/pt_expt/test_training.py::TestTraining": 197.415, + "class:source/tests/tf/test_finetune_se_atten.py::TestFinetuneSeAtten": 136.493, + "class:source/tests/tf/test_model_compression_se_atten.py::TestDeepPotAPBC": 335.841, + "class:source/tests/tf/test_model_compression_se_atten.py::TestDeepPotAPBCExcludeTypes": 168.339, + "module:source/tests/pt_expt/infer/test_graph_deepeval.py": 648.668, + "module:source/tests/pt_expt/utils/test_graph_pt2_metadata.py": 281.378 + } +} diff --git a/source/tests/test_ci_split.py b/source/tests/test_ci_split.py index d7d0dffa53..66e3b57406 100644 --- a/source/tests/test_ci_split.py +++ b/source/tests/test_ci_split.py @@ -4,13 +4,17 @@ from dataclasses import ( dataclass, ) +from pathlib import ( + Path, +) import pytest from .ci_split import ( + _DurationSeed, + _load_durations, _split_items, _TestGroup, - _unit_key, ) @@ -72,21 +76,40 @@ def test_class_and_module_items_are_indivisible(self) -> None: assert len({_owner(groups, item) for item in module_items}) == 1 def test_explicit_group_joins_classes_across_modules(self) -> None: - first = _item( - "tests/test_a.py::FirstClass::test_one", - cls=FirstClass, - explicit_group="shared-model", - ) - second = _item( - "tests/test_b.py::SecondClass::test_two", - cls=SecondClass, - explicit_group="shared-model", - ) - groups = _split_items([first, second], {}, splits=2) + first_class = [ + _item( + f"tests/test_a.py::FirstClass::test_{index}", + cls=FirstClass, + explicit_group="shared-model", + ) + for index in range(2) + ] + second_class = [ + _item( + f"tests/test_b.py::SecondClass::test_{index}", + cls=SecondClass, + explicit_group="shared-model", + ) + for index in range(2) + ] + groups = _split_items(first_class + second_class, {}, splits=2) + + assert len({_owner(groups, item) for item in first_class + second_class}) == 1 - assert _owner(groups, first) == _owner(groups, second) + def test_partial_method_marker_is_rejected(self) -> None: + items = [ + _item( + "tests/test_a.py::FirstClass::test_marked", + cls=FirstClass, + explicit_group="shared-model", + ), + _item("tests/test_a.py::FirstClass::test_unmarked", cls=FirstClass), + ] + + with pytest.raises(pytest.UsageError, match="must mark every test"): + _split_items(items, {}, splits=2) - def test_unit_key_rejects_empty_explicit_group(self) -> None: + def test_empty_explicit_group_is_rejected(self) -> None: item = _item( "tests/test_a.py::FirstClass::test_one", cls=FirstClass, @@ -94,7 +117,7 @@ def test_unit_key_rejects_empty_explicit_group(self) -> None: ) with pytest.raises(pytest.UsageError, match="non-empty string"): - _unit_key(item) + _split_items([item], {}, splits=1) class TestUnitBalancing: @@ -128,6 +151,21 @@ def test_empty_cache_balances_by_number_of_tests(self) -> None: assert [len(group.items) for group in groups] == [7, 7] + def test_committed_seed_weights_expensive_units_without_cache(self) -> None: + slow = [ + _item(f"tests/test_slow.py::FirstClass::test_{index}", cls=FirstClass) + for index in range(2) + ] + fast = [_item("tests/test_fast.py::test_value")] + seed = _DurationSeed( + default_test_duration=1.0, + units={"class:tests/test_slow.py::FirstClass": 20.0}, + ) + + groups = _split_items(slow + fast, {}, splits=2, seed=seed) + + assert sorted(group.estimated_duration for group in groups) == [1.0, 20.0] + def test_assignment_is_deterministic_and_preserves_collection_order(self) -> None: items = [_item(f"tests/test_{index}.py::test_value") for index in range(8)] @@ -140,3 +178,12 @@ def test_assignment_is_deterministic_and_preserves_collection_order(self) -> Non for group in first: indices = [items.index(item) for item in group.items] assert indices == sorted(indices) + + def test_malformed_legacy_duration_list_is_usage_error( + self, tmp_path: Path + ) -> None: + path = tmp_path / "durations.json" + path.write_text('[["valid", 1.0], ["broken"]]', encoding="utf-8") + + with pytest.raises(pytest.UsageError, match="invalid legacy list"): + _load_durations(path) From 1417f3f748711443d42a7abebb4848cc6e5346ed Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Sun, 19 Jul 2026 20:53:32 +0800 Subject: [PATCH 7/8] refactor(ci): drop static duration seeds Use test-count LPT when no grouped runtime cache exists and keep test4 as the only persisted duration source. Retain regression coverage for the custom splitter while removing generated historical timing data. Coding-Agent: Codex Codex-Version: codex-cli 0.144.4 Model: gpt-5.6-sol Reasoning-Effort: xhigh --- source/tests/ci_split.py | 56 +++------------------------- source/tests/ci_split_seed_3.10.json | 53 -------------------------- source/tests/ci_split_seed_3.13.json | 55 --------------------------- source/tests/test_ci_split.py | 16 -------- 4 files changed, 5 insertions(+), 175 deletions(-) delete mode 100644 source/tests/ci_split_seed_3.10.json delete mode 100644 source/tests/ci_split_seed_3.13.json diff --git a/source/tests/ci_split.py b/source/tests/ci_split.py index bf27c6a1d8..dce082f2ce 100644 --- a/source/tests/ci_split.py +++ b/source/tests/ci_split.py @@ -20,7 +20,6 @@ import heapq import json import math -import sys from collections import ( OrderedDict, ) @@ -61,14 +60,6 @@ def get_closest_marker(self, name: str) -> pytest.Mark | None: raise NotImplementedError -@dataclass(frozen=True) -class _DurationSeed: - """Version-controlled estimates used before a runtime cache exists.""" - - default_test_duration: float = 1.0 - units: dict[str, float] = field(default_factory=dict) - - @dataclass class _TestUnit: """A set of test items that must execute in the same CI shard.""" @@ -129,15 +120,10 @@ def _relevant_durations( def _build_units( items: list[_SplitItem], durations: dict[str, float], - seed: _DurationSeed, ) -> list[_TestUnit]: """Build complete class/module units, then merge explicit unit groups.""" relevant = _relevant_durations(items, durations) - fallback_duration = ( - sum(relevant.values()) / len(relevant) - if relevant - else seed.default_test_duration - ) + fallback_duration = sum(relevant.values()) / len(relevant) if relevant else 1.0 base_units: OrderedDict[str, _TestUnit] = OrderedDict() for item in items: key = _base_unit_key(item) @@ -154,10 +140,7 @@ def _build_units( missing_count * fallback_duration ) else: - unit.estimated_duration = seed.units.get( - unit.key, - len(unit.items) * seed.default_test_duration, - ) + unit.estimated_duration = len(unit.items) * fallback_duration merged_units: OrderedDict[str, _TestUnit] = OrderedDict() for unit in base_units.values(): @@ -185,10 +168,9 @@ def _split_items( items: list[_SplitItem], durations: dict[str, float], splits: int, - seed: _DurationSeed | None = None, ) -> list[_TestGroup]: """Assign indivisible units using deterministic LPT bin packing.""" - units = _build_units(items, durations, seed or _DurationSeed()) + units = _build_units(items, durations) item_index = {id(item): index for index, item in enumerate(items)} # The group index breaks equal-duration ties deterministically. Sorting @@ -244,30 +226,6 @@ def _load_durations(path: Path) -> dict[str, float]: ) from exc -def _load_duration_seed(path: Path) -> _DurationSeed: - """Load the version-controlled fallback for the first grouped CI run.""" - try: - data = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise pytest.UsageError(f"cannot read duration seed {path}: {exc}") from exc - if not isinstance(data, dict) or not isinstance(data.get("units"), dict): - raise pytest.UsageError(f"duration seed {path} has an invalid schema") - try: - default = float(data["default_test_duration"]) - units = {str(key): float(value) for key, value in data["units"].items()} - except (KeyError, TypeError, ValueError) as exc: - raise pytest.UsageError( - f"duration seed {path} contains a non-numeric duration" - ) from exc - if default <= 0 or not math.isfinite(default): - raise pytest.UsageError( - f"duration seed {path} has an invalid default_test_duration" - ) - if any(value <= 0 or not math.isfinite(value) for value in units.values()): - raise pytest.UsageError(f"duration seed {path} has an invalid unit duration") - return _DurationSeed(default_test_duration=default, units=units) - - def pytest_addoption(parser: Parser) -> None: """Declare CI-specific split options without activating pytest-split.""" group = parser.getgroup("DeePMD grouped CI splitting") @@ -317,11 +275,7 @@ def pytest_collection_modifyitems(config: Config, items: list[nodes.Item]) -> No durations_path = Path(config.getoption("durations_path")) durations = _load_durations(durations_path) - python_version = f"{sys.version_info.major}.{sys.version_info.minor}" - seed = _load_duration_seed( - Path(__file__).with_name(f"ci_split_seed_{python_version}.json") - ) - groups = _split_items(items, durations, splits, seed) + groups = _split_items(items, durations, splits) selected = groups[group_index - 1] selected_ids = {id(item) for item in selected.items} deselected = [item for item in items if id(item) not in selected_ids] @@ -330,7 +284,7 @@ def pytest_collection_modifyitems(config: Config, items: list[nodes.Item]) -> No reporter = config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: - estimate_source = "runtime cache" if durations else "committed seed" + estimate_source = "runtime cache" if durations else "test-count fallback" reporter.write_line( "[deepmd-ci-split] " f"group {group_index}/{splits}: {len(selected.items)} tests in " diff --git a/source/tests/ci_split_seed_3.10.json b/source/tests/ci_split_seed_3.10.json deleted file mode 100644 index c931dedbdc..0000000000 --- a/source/tests/ci_split_seed_3.10.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "default_test_duration": 0.592, - "provenance": "Python 3.10 unit durations from Test Python run 29456447617; units below 120 seconds use the default estimate.", - "units": { - "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pt2": 865.503, - "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pte": 293.043, - "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pt2": 1197.644, - "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pte": 439.224, - "class:source/tests/jax/test_make_hessian_model.py::TestDPModel": 259.984, - "class:source/tests/pd/model/test_dpa2.py::TestDescrptDPA2": 235.375, - "class:source/tests/pd/test_training.py::TestFparam": 133.835, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipeline": 226.308, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipelineTritonInfer": 195.538, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMFreezeGuards": 305.304, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMWithCommExportPipeline": 220.017, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelCompile": 1989.799, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelLoRACompile": 330.974, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelProperty": 138.229, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMNativeSpinModel": 428.637, - "class:source/tests/pt/model/test_sezm_spin_model.py::TestSeZMSpinModel": 133.554, - "class:source/tests/pt_expt/descriptor/test_dpa2.py::TestDescrptDPA2": 128.333, - "class:source/tests/pt_expt/descriptor/test_dpa3.py::TestDescrptDPA3": 583.549, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEner": 582.428, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparam": 179.801, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparamPt2": 893.994, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerChgSpinPt2": 669.117, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerDefaultFparamPt2": 423.123, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerPt2": 2306.716, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalNlistBackend": 194.501, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalDescriptor": 495.034, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalFittingLastLayer": 474.863, - "class:source/tests/pt_expt/infer/test_deep_eval_metadata_only.py::TestDeepEvalMetadataOnlyPte": 229.435, - "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinAparam": 867.838, - "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinDefaultFparam": 429.227, - "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinInference": 1855.625, - "class:source/tests/pt_expt/model/test_export_pipeline.py::TestExportPipeline": 396.682, - "class:source/tests/pt_expt/model/test_model_compression.py::TestModelCompression": 213.553, - "class:source/tests/pt_expt/test_change_bias.py::TestChangeBias": 1174.856, - "class:source/tests/pt_expt/test_dp_freeze.py::TestDPFreezePtExpt": 916.732, - "class:source/tests/pt_expt/test_finetune.py::TestFinetuneCLI": 235.451, - "class:source/tests/pt_expt/test_multitask.py::TestCompileCaseEmbdVaryingNframes": 199.148, - "class:source/tests/pt_expt/test_multitask.py::TestMultiTaskCompile": 293.683, - "class:source/tests/pt_expt/test_training.py::TestCompiledConsistency": 254.354, - "class:source/tests/pt_expt/test_training.py::TestCompiledSharedFittingDifferentDescriptor": 146.813, - "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNatoms": 1244.152, - "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNframesWithParams": 196.837, - "class:source/tests/pt_expt/test_training.py::TestTraining": 184.091, - "class:source/tests/tf/test_finetune_se_atten.py::TestFinetuneSeAtten": 150.59, - "class:source/tests/tf/test_model_compression_se_atten.py::TestDeepPotAPBC": 216.183, - "module:source/tests/pt_expt/infer/test_graph_deepeval.py": 727.49, - "module:source/tests/pt_expt/utils/test_graph_pt2_metadata.py": 264.404 - } -} diff --git a/source/tests/ci_split_seed_3.13.json b/source/tests/ci_split_seed_3.13.json deleted file mode 100644 index 75ec9f11fa..0000000000 --- a/source/tests/ci_split_seed_3.13.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "default_test_duration": 0.599, - "provenance": "Python 3.13 unit durations from Test Python run 29456447617; units below 120 seconds use the default estimate.", - "units": { - "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pt2": 584.863, - "class:source/tests/infer/test_models.py::TestDeepPot_fparam_aparam_.pte": 254.494, - "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pt2": 1105.263, - "class:source/tests/infer/test_models.py::TestDeepPot_se_e2_a_.pte": 405.028, - "class:source/tests/jax/test_make_hessian_model.py::TestDPModel": 273.14, - "class:source/tests/pd/model/test_dpa2.py::TestDescrptDPA2": 193.951, - "class:source/tests/pd/test_training.py::TestFparam": 128.816, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipeline": 211.531, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMExportPipelineTritonInfer": 300.066, - "class:source/tests/pt/model/test_sezm_export.py::TestSeZMFreezeGuards": 278.717, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelCompile": 1860.224, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMModelLoRACompile": 218.127, - "class:source/tests/pt/model/test_sezm_model.py::TestSeZMNativeSpinModel": 588.729, - "class:source/tests/pt/model/test_sezm_spin_model.py::TestSeZMSpinModel": 148.399, - "class:source/tests/pt_expt/descriptor/test_dpa2.py::TestDescrptDPA2": 137.791, - "class:source/tests/pt_expt/descriptor/test_dpa3.py::TestDescrptDPA3": 565.619, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEner": 608.076, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparam": 172.449, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerAparamPt2": 661.455, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerChgSpinPt2": 1043.66, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerDefaultFparamPt2": 480.499, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalEnerPt2": 2417.58, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestDeepEvalNlistBackend": 252.715, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalDescriptor": 495.286, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalFittingLastLayer": 498.551, - "class:source/tests/pt_expt/infer/test_deep_eval.py::TestEvalTypeEbd": 122.649, - "class:source/tests/pt_expt/infer/test_deep_eval_metadata_only.py::TestDeepEvalMetadataOnlyPte": 207.916, - "class:source/tests/pt_expt/infer/test_deep_eval_pt_checkpoint.py::TestPtExptLoadPtSpin": 123.741, - "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinAparam": 841.667, - "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinDefaultFparam": 443.65, - "class:source/tests/pt_expt/infer/test_deep_eval_spin.py::TestSpinInference": 1681.299, - "class:source/tests/pt_expt/model/test_export_pipeline.py::TestExportPipeline": 424.391, - "class:source/tests/pt_expt/model/test_model_compression.py::TestModelCompression": 262.056, - "class:source/tests/pt_expt/test_change_bias.py::TestChangeBias": 1218.921, - "class:source/tests/pt_expt/test_dp_freeze.py::TestDPFreezePtExpt": 1109.204, - "class:source/tests/pt_expt/test_finetune.py::TestFinetuneCLI": 213.089, - "class:source/tests/pt_expt/test_multitask.py::TestCompileCaseEmbdVaryingNframes": 234.837, - "class:source/tests/pt_expt/test_multitask.py::TestMultiTaskCompile": 385.028, - "class:source/tests/pt_expt/test_multitask.py::TestMultiTaskDPA3": 126.497, - "class:source/tests/pt_expt/test_training.py::TestCompiledConsistency": 186.814, - "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNatoms": 1370.061, - "class:source/tests/pt_expt/test_training.py::TestCompiledVaryingNframesWithParams": 137.311, - "class:source/tests/pt_expt/test_training.py::TestRestart": 134.763, - "class:source/tests/pt_expt/test_training.py::TestTraining": 197.415, - "class:source/tests/tf/test_finetune_se_atten.py::TestFinetuneSeAtten": 136.493, - "class:source/tests/tf/test_model_compression_se_atten.py::TestDeepPotAPBC": 335.841, - "class:source/tests/tf/test_model_compression_se_atten.py::TestDeepPotAPBCExcludeTypes": 168.339, - "module:source/tests/pt_expt/infer/test_graph_deepeval.py": 648.668, - "module:source/tests/pt_expt/utils/test_graph_pt2_metadata.py": 281.378 - } -} diff --git a/source/tests/test_ci_split.py b/source/tests/test_ci_split.py index 66e3b57406..ef97ce127a 100644 --- a/source/tests/test_ci_split.py +++ b/source/tests/test_ci_split.py @@ -11,7 +11,6 @@ import pytest from .ci_split import ( - _DurationSeed, _load_durations, _split_items, _TestGroup, @@ -151,21 +150,6 @@ def test_empty_cache_balances_by_number_of_tests(self) -> None: assert [len(group.items) for group in groups] == [7, 7] - def test_committed_seed_weights_expensive_units_without_cache(self) -> None: - slow = [ - _item(f"tests/test_slow.py::FirstClass::test_{index}", cls=FirstClass) - for index in range(2) - ] - fast = [_item("tests/test_fast.py::test_value")] - seed = _DurationSeed( - default_test_duration=1.0, - units={"class:tests/test_slow.py::FirstClass": 20.0}, - ) - - groups = _split_items(slow + fast, {}, splits=2, seed=seed) - - assert sorted(group.estimated_duration for group in groups) == [1.0, 20.0] - def test_assignment_is_deterministic_and_preserves_collection_order(self) -> None: items = [_item(f"tests/test_{index}.py::test_value") for index in range(8)] From 49dc0282101c6bf80a89f1a2660cbf41bd1a050d Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Sun, 19 Jul 2026 23:14:36 +0800 Subject: [PATCH 8/8] ci: validate warm duration cache Coding-Agent: Codex Codex-Version: codex-cli 0.144.4 Model: gpt-5.6-sol Reasoning-Effort: xhigh