[CI] Update test fetcher script

.github/workflows/pr_test_fetcher.yml

@@ -30,7 +30,6 @@ jobs:
         shell: bash
     outputs:
       matrix: ${{ steps.set_matrix.outputs.matrix }}
-      test_map: ${{ steps.set_matrix.outputs.test_map }}
     steps:
     - name: Checkout diffusers
       uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
@@ -44,42 +43,45 @@ jobs:
         diffusers-cli env
         echo $(git --version)
     - name: Fetch Tests
+      # The checkout step only marks the workspace safe in a temporary git config, so git processes the
+      # fetcher spawns in the container would otherwise refuse the repo as dubiously owned.
       run: |
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
         python utils/tests_fetcher.py | tee test_preparation.txt
+    - name: Report selection in job summary
+      run: |
+        if [ -f test_fetcher_summary.md ]; then
+            cat test_fetcher_summary.md >> $GITHUB_STEP_SUMMARY
+        fi
     - name: Report fetched tests
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
       with:
         name: test_fetched
         path: test_preparation.txt
     - id: set_matrix
       name: Create Test Matrix
-      # The `keys` is used as GitHub actions matrix for jobs, i.e. `models`, `pipelines`, etc.
-      # The `test_map` is used to get the actual identified test files under each key.
-      # If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail)
+      # `test_map.json` is a list of matrix entries `{"name", "paths", "markers"}`, one job each.
+      # If no test to run (so no `test_map.json` file), emit an empty list and skip the test job.
       run: |
         if [ -f test_map.json ]; then
-            keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(json.dumps(d))')
-            test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(json.dumps(test_map))')
+            matrix=$(python3 -c 'import json; print(json.dumps(json.load(open("test_map.json"))))')
         else
-            keys=$(python3 -c 'keys = ["dummy"]; print(keys)')
-            test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)')
+            matrix='[]'
         fi
-        echo $keys
-        echo $test_map
-        echo "matrix=$keys" >> $GITHUB_OUTPUT
-        echo "test_map=$test_map" >> $GITHUB_OUTPUT
+        echo "matrix=$matrix" >> $GITHUB_OUTPUT
+        echo "Matrix: $matrix"
 
   run_pr_tests:
     permissions:
       contents: read
-    name: Run PR Tests
+    name: ${{ matrix.name }}
     needs: setup_pr_tests
-    if: contains(fromJson(needs.setup_pr_tests.outputs.matrix), 'dummy') != true
+    if: needs.setup_pr_tests.outputs.matrix != '[]'
     strategy:
       fail-fast: false
       max-parallel: 2
       matrix:
-        modules: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }}
+        include: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }}
     runs-on:
       group: aws-general-8-plus
     container:
@@ -105,20 +107,20 @@ jobs:
 
     - name: Run all selected tests on CPU
       run: |
-        pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}
+        pytest -n 2 --dist=loadfile -m "${{ matrix.markers }}" --make-reports=${{ matrix.name }}_tests_cpu ${{ matrix.paths }}
 
     - name: Failure short reports
       if: ${{ failure() }}
       continue-on-error: true
       run: |
-        cat reports/${{ matrix.modules }}_tests_cpu_stats.txt
-        cat reports/${{ matrix.modules }}_tests_cpu_failures_short.txt
+        cat reports/${{ matrix.name }}_tests_cpu_stats.txt
+        cat reports/${{ matrix.name }}_tests_cpu_failures_short.txt
 
     - name: Test suite reports artifacts
       if: ${{ always() }}
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
       with:
-          name: ${{ matrix.modules }}_test_reports
+          name: ${{ matrix.name }}_test_reports
           path: reports
 
   run_staging_tests:

tests/conftest.py

@@ -15,10 +15,13 @@
 # tests directory-specific settings - this file is run automatically
 # by pytest before any tests are run
 
+import ast
 import sys
 import warnings
 from os.path import abspath, dirname, join
 
+import pytest
+
 
 # allow having multiple repository checkouts and not needing to remember to rerun
 # 'pip install -e .[dev]' when switching between checkouts and running tests.
@@ -30,7 +33,37 @@
 warnings.simplefilter(action="ignore", category=FutureWarning)
 
 
+def marker_decorators():
+    """Map each `is_*` decorator in testing_utils.py to the pytest marker it applies.
+
+    Read from the source with `ast` rather than imported, so `utils/tests_fetcher.py` can call this without
+    importing torch. Matches `def is_x(test_case): return pytest.mark.<marker>(test_case)`.
+    """
+    tree = ast.parse(open(join(dirname(__file__), "testing_utils.py"), encoding="utf-8").read())
+    decorators = {}
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.FunctionDef) or not node.name.startswith("is_"):
+            continue
+        for ret in ast.walk(node):
+            if isinstance(ret, ast.Return) and isinstance(ret.value, ast.Call):
+                func = ret.value.func
+                if (
+                    isinstance(func, ast.Attribute)
+                    and isinstance(func.value, ast.Attribute)
+                    and func.value.attr == "mark"
+                ):
+                    decorators[node.name] = func.attr
+    return decorators
+
+
+# Every marker an `is_*` decorator applies takes a test out of the `core` set. CI runs each feature in its
+# own job (`-m lora`, ...) and everything else as `-m core`, so a test carrying none of these is marked
+# `core` at collection time. `utils/tests_fetcher.py` uses the same set to decide which jobs to schedule.
+NON_CORE_MARKERS = set(marker_decorators().values())
+
+
 def pytest_configure(config):
+    config.addinivalue_line("markers", "core: marks tests carrying none of the feature markers (added automatically)")
     config.addinivalue_line("markers", "big_accelerator: marks tests as requiring big accelerator resources")
     config.addinivalue_line("markers", "lora: marks tests for LoRA/PEFT functionality")
     config.addinivalue_line("markers", "ip_adapter: marks tests for IP Adapter functionality")
@@ -53,6 +86,14 @@ def pytest_configure(config):
     config.addinivalue_line("markers", "nightly: mark test as nightly")
 
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_collection_modifyitems(items):
+    """Mark every test with no feature marker as `core`, before `-m` deselection runs."""
+    for item in items:
+        if not any(m.name in NON_CORE_MARKERS for m in item.iter_markers()):
+            item.add_marker("core")
+
+
 def pytest_addoption(parser):
     from .testing_utils import pytest_addoption_shared
 

tests/models/test_modeling_common.py

@@ -26,7 +26,7 @@
 from huggingface_hub import ModelCard, delete_repo, snapshot_download, try_to_load_from_cache
 from huggingface_hub.utils import HfHubHTTPError, is_jinja_available
 
-from diffusers.models import FluxTransformer2DModel, SD3Transformer2DModel, UNet2DConditionModel
+from diffusers.models import UNet2DConditionModel
 
 from ..others.test_utils import TOKEN, USER, is_staging_test
 from ..testing_utils import (
@@ -118,6 +118,8 @@ def test_cached_files_are_used_when_no_internet(self):
                 assert False, "Parameters not the same!"
 
     def test_local_files_only_with_sharded_checkpoint(self):
+        from diffusers.models import FluxTransformer2DModel
+
         repo_id = "hf-internal-testing/tiny-flux-sharded"
         error_response = mock.Mock(status_code=500, headers={}, json=mock.Mock(return_value={}))
         # `resolve_revision` inspects `error.response.status_code` to tell a Hub outage from a definitive answer,
@@ -230,6 +232,8 @@ def test_keep_modules_in_fp32(self):
         A simple tests to check if the modules under `_keep_in_fp32_modules` are kept in fp32 when we load the model in fp16/bf16
         Also ensures if inference works.
         """
+        from diffusers.models import SD3Transformer2DModel
+
         fp32_modules = SD3Transformer2DModel._keep_in_fp32_modules
 
         for torch_dtype in [torch.bfloat16, torch.float16]:

tests/others/test_flashpack.py

@@ -15,7 +15,6 @@
 
 import pytest
 
-from diffusers import AutoPipelineForText2Image
 from diffusers.models.auto_model import AutoModel
 
 from ..testing_utils import is_torch_available, require_flashpack, require_torch_gpu
@@ -40,6 +39,8 @@ def test_save_load_model(self, tmp_path):
 
     @require_flashpack
     def test_save_load_pipeline(self, tmp_path):
+        from diffusers import AutoPipelineForText2Image
+
         pipeline = AutoPipelineForText2Image.from_pretrained(self.model_id)
         pipeline.save_pretrained(tmp_path, use_flashpack=True)
         assert (tmp_path / "transformer" / "model.flashpack").exists()

tests/pipelines/kandinsky3/test_kandinsky3.py

@@ -22,8 +22,6 @@
 from transformers import AutoConfig, AutoTokenizer, T5EncoderModel
 
 from diffusers import (
-    AutoPipelineForImage2Image,
-    AutoPipelineForText2Image,
     Kandinsky3Pipeline,
     Kandinsky3UNet,
     VQModel,
@@ -177,6 +175,8 @@ def cleanup(self):
         backend_empty_cache(torch_device)
 
     def test_kandinskyV3(self):
+        from diffusers import AutoPipelineForText2Image
+
         pipe = AutoPipelineForText2Image.from_pretrained(
             "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16
         )
@@ -203,6 +203,8 @@ def test_kandinskyV3(self):
         assert np.allclose(image_np, expected_image_np, atol=5e-2)
 
     def test_kandinskyV3_img2img(self):
+        from diffusers import AutoPipelineForImage2Image
+
         pipe = AutoPipelineForImage2Image.from_pretrained(
             "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16
         )

tests/pipelines/kandinsky3/test_kandinsky3_img2img.py

@@ -23,7 +23,6 @@
 from transformers import AutoConfig, AutoTokenizer, T5EncoderModel
 
 from diffusers import (
-    AutoPipelineForImage2Image,
     Kandinsky3Img2ImgPipeline,
     Kandinsky3UNet,
     VQModel,
@@ -192,6 +191,8 @@ def cleanup(self):
         backend_empty_cache(torch_device)
 
     def test_kandinskyV3_img2img(self):
+        from diffusers import AutoPipelineForImage2Image
+
         pipe = AutoPipelineForImage2Image.from_pretrained(
             "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16
         )

tests/pipelines/pag/test_pag_sd.py

@@ -22,7 +22,6 @@
 
 from diffusers import (
     AutoencoderKL,
-    AutoPipelineForText2Image,
     DDIMScheduler,
     StableDiffusionPAGPipeline,
     StableDiffusionPipeline,
@@ -253,6 +252,8 @@ def get_inputs(self, device, generator_device="cpu", seed=1, guidance_scale=7.0)
         return inputs
 
     def test_pag_cfg(self):
+        from diffusers import AutoPipelineForText2Image
+
         pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)
@@ -271,6 +272,8 @@ def test_pag_cfg(self):
         )
 
     def test_pag_uncond(self):
+        from diffusers import AutoPipelineForText2Image
+
         pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)

tests/pipelines/pag/test_pag_sd3_img2img.py

@@ -15,7 +15,6 @@
 
 from diffusers import (
     AutoencoderKL,
-    AutoPipelineForImage2Image,
     FlowMatchEulerDiscreteScheduler,
     SD3Transformer2DModel,
     StableDiffusion3Img2ImgPipeline,
@@ -196,6 +195,8 @@ def get_inputs(
         return inputs
 
     def test_pag_cfg(self):
+        from diffusers import AutoPipelineForImage2Image
+
         pipeline = AutoPipelineForImage2Image.from_pretrained(
             self.repo_id, enable_pag=True, torch_dtype=torch.float16, pag_applied_layers=["blocks.17"]
         )
@@ -224,6 +225,8 @@ def test_pag_cfg(self):
         )
 
     def test_pag_uncond(self):
+        from diffusers import AutoPipelineForImage2Image
+
         pipeline = AutoPipelineForImage2Image.from_pretrained(
             self.repo_id, enable_pag=True, torch_dtype=torch.float16, pag_applied_layers=["blocks.(4|17)"]
         )

tests/pipelines/pag/test_pag_sd_img2img.py

@@ -23,7 +23,6 @@
 
 from diffusers import (
     AutoencoderKL,
-    AutoPipelineForImage2Image,
     EulerDiscreteScheduler,
     StableDiffusionImg2ImgPipeline,
     StableDiffusionPAGImg2ImgPipeline,
@@ -197,6 +196,8 @@ def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0
         return inputs
 
     def test_pag_cfg(self):
+        from diffusers import AutoPipelineForImage2Image
+
         pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)
@@ -215,6 +216,8 @@ def test_pag_cfg(self):
         )
 
     def test_pag_uncond(self):
+        from diffusers import AutoPipelineForImage2Image
+
         pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)

tests/pipelines/pag/test_pag_sd_inpaint.py

@@ -24,7 +24,6 @@
 
 from diffusers import (
     AutoencoderKL,
-    AutoPipelineForInpainting,
     PNDMScheduler,
     StableDiffusionInpaintPipeline,
     StableDiffusionPAGInpaintPipeline,
@@ -275,6 +274,8 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0)
         return inputs
 
     def test_pag_cfg(self):
+        from diffusers import AutoPipelineForInpainting
+
         pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)
@@ -293,6 +294,8 @@ def test_pag_cfg(self):
         )
 
     def test_pag_uncond(self):
+        from diffusers import AutoPipelineForInpainting
+
         pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)

tests/pipelines/pag/test_pag_sdxl.py

@@ -22,7 +22,6 @@
 
 from diffusers import (
     AutoencoderKL,
-    AutoPipelineForText2Image,
     EulerDiscreteScheduler,
     StableDiffusionXLPAGPipeline,
     StableDiffusionXLPipeline,
@@ -260,6 +259,8 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0)
         return inputs
 
     def test_pag_cfg(self):
+        from diffusers import AutoPipelineForText2Image
+
         pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)
@@ -282,6 +283,8 @@ def test_pag_cfg(self):
         )
 
     def test_pag_uncond(self):
+        from diffusers import AutoPipelineForText2Image
+
         pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16)
         pipeline.enable_model_cpu_offload(device=torch_device)
         pipeline.set_progress_bar_config(disable=None)