From 878306dc2f660f77d7fb4b2b51e0167e846f81e0 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 29 Sep 2026 08:59:33 +0530 Subject: [PATCH 1/2] feat: allow the diffusers bot to run tests that need multiple GPUs. --- .github/workflows/bot_pytest.yml | 76 +++++++++++++++++++-- .github/workflows/diffusers_bot.yml | 18 ++++- utils/bot_pytest_hf_job.py | 100 ++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 utils/bot_pytest_hf_job.py diff --git a/.github/workflows/bot_pytest.yml b/.github/workflows/bot_pytest.yml index 68ed8c5ca1de..cb861f77741f 100644 --- a/.github/workflows/bot_pytest.yml +++ b/.github/workflows/bot_pytest.yml @@ -1,7 +1,6 @@ name: "Bot: Pytest" # Reusable workflow called by diffusers_bot.yml for `/diffusers-bot pytest `. -# Runs pytest on a GPU runner and reports the result back on the PR. on: workflow_call: @@ -9,6 +8,13 @@ on: pytest_args: required: true type: string + pytest_flavor: + required: false + type: string + default: '' + secrets: + DIFFUSERS_BOT_TOKEN: + required: false jobs: gate: @@ -35,9 +41,15 @@ jobs: PR: ${{ github.event.issue.number }} COMMENTER: ${{ github.event.comment.user.login }} PYTEST_ARGS: ${{ inputs.pytest_args }} + PYTEST_FLAVOR: ${{ inputs.pytest_flavor }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | - BODY="⏳ Running \`pytest ${PYTEST_ARGS}\` on a GPU runner — [view logs](${RUN_URL}). + if [[ -n "$PYTEST_FLAVOR" ]]; then + LOCATION="HF Jobs (${PYTEST_FLAVOR})" + else + LOCATION="a GPU runner" + fi + BODY="⏳ Running \`pytest ${PYTEST_ARGS}\` on ${LOCATION} — [view logs](${RUN_URL}). Triggered by @${COMMENTER}." CID=$(gh api -X POST "repos/${REPO}/issues/${PR}/comments" -f body="$BODY" --jq '.id') @@ -46,6 +58,7 @@ jobs: gpu: name: Run pytest on GPU needs: gate + if: ${{ inputs.pytest_flavor == '' }} # A newer command on the same PR supersedes an in-flight one. Scoped to this job # only so the superseded run's `report` still updates its comment. concurrency: @@ -121,9 +134,52 @@ jobs: name: bot_gpu_test_reports path: reports + hf_job: + name: Run pytest on HF Jobs + needs: gate + if: ${{ inputs.pytest_flavor != '' }} + concurrency: + group: diffusers-bot-pytest-${{ github.event.issue.number }} + cancel-in-progress: true + runs-on: ubuntu-22.04 + timeout-minutes: 150 + permissions: + contents: read + pull-requests: read + outputs: + job_url: ${{ steps.run.outputs.job_url }} + steps: + - name: Resolve PR head + id: pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + run: | + echo "sha=$(gh api "repos/${REPO}/pulls/${PR}" --jq '.head.sha')" >> "$GITHUB_OUTPUT" + + - name: Checkout workflow code + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Install Hub client + run: python -m pip install 'huggingface_hub>=1.32,<2' + + - name: Run HF Job + id: run + env: + HF_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + PR_SHA: ${{ steps.pr.outputs.sha }} + PYTEST_ARGS: ${{ inputs.pytest_args }} + PYTEST_FLAVOR: ${{ inputs.pytest_flavor }} + run: python utils/bot_pytest_hf_job.py + report: name: Report status - needs: [gate, gpu] + needs: [gate, gpu, hf_job] # Always run so the comment is updated on success, failure, or cancellation — # but only if `gate` actually posted a comment to update. if: ${{ always() && needs.gate.outputs.comment_id != '' }} @@ -136,8 +192,10 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} CID: ${{ needs.gate.outputs.comment_id }} - RESULT: ${{ needs.gpu.result }} + RESULT: ${{ inputs.pytest_flavor == '' && needs.gpu.result || needs.hf_job.result }} PYTEST_ARGS: ${{ inputs.pytest_args }} + PYTEST_FLAVOR: ${{ inputs.pytest_flavor }} + JOB_URL: ${{ needs.hf_job.outputs.job_url }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | case "$RESULT" in @@ -146,5 +204,13 @@ jobs: cancelled) EMOJI="⚠️"; MSG="was cancelled";; *) EMOJI="⚠️"; MSG="did not run (${RESULT})";; esac - BODY="${EMOJI} \`pytest ${PYTEST_ARGS}\` ${MSG} on GPU — [view logs](${RUN_URL})." + if [[ -n "$PYTEST_FLAVOR" ]]; then + LOCATION="HF Jobs (${PYTEST_FLAVOR})" + if [[ -n "$JOB_URL" ]]; then + RUN_URL="$JOB_URL" + fi + else + LOCATION="GPU" + fi + BODY="${EMOJI} \`pytest ${PYTEST_ARGS}\` ${MSG} on ${LOCATION} — [view logs](${RUN_URL})." gh api -X PATCH "repos/${REPO}/issues/comments/${CID}" -f body="$BODY" diff --git a/.github/workflows/diffusers_bot.yml b/.github/workflows/diffusers_bot.yml index 2d85be33a836..48f0098eb9a2 100644 --- a/.github/workflows/diffusers_bot.yml +++ b/.github/workflows/diffusers_bot.yml @@ -6,6 +6,7 @@ name: Diffusers Bot # /diffusers-bot review request a Serge AI review (also works from an inline review comment) # /diffusers-bot pytest run `pytest ` on a GPU runner, e.g. # `/diffusers-bot pytest tests/models/test_modeling_common.py -k "some_test"` +# /diffusers-bot pytest --flavor a100x4 run pytest on a multi-GPU HF Job # # Auth is checked once in the `authorize` job. Each subcommand lives in its # own reusable workflow file under .github/workflows/bot_*.yml. @@ -43,6 +44,7 @@ jobs: is_authorized: ${{ steps.check.outputs.is_authorized }} subcommand: ${{ steps.check.outputs.subcommand }} pytest_args: ${{ steps.check.outputs.pytest_args }} + pytest_flavor: ${{ steps.check.outputs.pytest_flavor }} steps: - name: Authorize and parse command id: check @@ -64,10 +66,18 @@ jobs: let subcommand = 'unknown'; let pytestArgs = ''; + let pytestFlavor = ''; - if (firstLine.startsWith('/diffusers-bot pytest')) { + if (/^\/diffusers-bot pytest(?:\s|$)/.test(firstLine)) { subcommand = 'pytest'; - pytestArgs = firstLine.replace('/diffusers-bot pytest', '').trim(); + pytestArgs = firstLine.slice('/diffusers-bot pytest'.length).trim(); + const flavor = pytestArgs.match(/^--flavor(?:=|\s+)(\S+)(?:\s+|$)/); + if (flavor) { + pytestFlavor = flavor[1]; + pytestArgs = pytestArgs.slice(flavor[0].length).trim(); + } else if (pytestArgs.startsWith('--flavor')) { + pytestFlavor = 'invalid'; + } } else if (body.includes('/diffusers-bot style')) { subcommand = 'style'; } else if (body.includes('/diffusers-bot review')) { @@ -77,6 +87,7 @@ jobs: console.log(`Subcommand: ${subcommand}, pytestArgs: ${pytestArgs}`); core.setOutput('subcommand', subcommand); core.setOutput('pytest_args', pytestArgs); + core.setOutput('pytest_flavor', pytestFlavor); # ── Dispatch to sub-workflows ──────────────────────────────────────────────── style: @@ -117,3 +128,6 @@ jobs: uses: ./.github/workflows/bot_pytest.yml with: pytest_args: ${{ needs.authorize.outputs.pytest_args }} + pytest_flavor: ${{ needs.authorize.outputs.pytest_flavor }} + secrets: + DIFFUSERS_BOT_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }} diff --git a/utils/bot_pytest_hf_job.py b/utils/bot_pytest_hf_job.py new file mode 100644 index 000000000000..b250610e4a6f --- /dev/null +++ b/utils/bot_pytest_hf_job.py @@ -0,0 +1,100 @@ +import os +import signal +import sys + +from huggingface_hub import HfApi + + +COMMAND = r""" +set -euo pipefail +git init -q /tmp/diffusers +cd /tmp/diffusers +git remote add origin "https://github.com/${REPOSITORY}.git" +git fetch -q --depth=2 origin "refs/pull/${PR_NUMBER}/head" +git checkout -q --detach FETCH_HEAD +test "$(git rev-parse HEAD)" = "$PR_SHA" +nvidia-smi +printf 'torch==2.10.0\ntorchvision==0.25.0\ntorchaudio==2.10.0\n' > "$UV_OVERRIDE" +uv pip install -e ".[quality,training,test]" +uv pip install peft@git+https://github.com/huggingface/peft.git +uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git +uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git +diffusers-cli env +status=0 +python - <<'PY' || status=$? +import os +import shlex +import subprocess + +args = shlex.split(os.environ["PYTEST_ARGS"]) +raise SystemExit(subprocess.call(["pytest", "--make-reports=tests_bot_gpu", *args])) +PY +if [[ "$status" -ne 0 ]]; then + cat reports/tests_bot_gpu_stats.txt || true + cat reports/tests_bot_gpu_failures_short.txt || true +fi +exit "$status" +""" + + +def main(): + flavor = os.environ["PYTEST_FLAVOR"] + api = HfApi(token=os.environ["HF_TOKEN"]) + flavors = { + hardware.name + for hardware in api.list_jobs_hardware() + if hardware.accelerator is not None + and hardware.accelerator.type.lower() == "gpu" + and int(hardware.accelerator.quantity) > 1 + } + if flavor not in flavors: + raise SystemExit(f"Unsupported multi-GPU flavor: {flavor}. Choose from: {', '.join(sorted(flavors))}") + + job = api.run_job( + image="diffusers/diffusers-pytorch-cuda", + command=["bash", "-lc", COMMAND], + flavor=flavor, + timeout="2h", + name=f"diffusers-pr-{os.environ['PR_NUMBER']}-pytest", + env={ + "REPOSITORY": os.environ["REPOSITORY"], + "PR_NUMBER": os.environ["PR_NUMBER"], + "PR_SHA": os.environ["PR_SHA"], + "PYTEST_ARGS": os.environ["PYTEST_ARGS"], + "DIFFUSERS_IS_CI": "yes", + "OMP_NUM_THREADS": "8", + "MKL_NUM_THREADS": "8", + "HF_XET_HIGH_PERFORMANCE": "1", + "PYTEST_TIMEOUT": "600", + "UV_OVERRIDE": "/tmp/uv-overrides.txt", + "CUBLAS_WORKSPACE_CONFIG": ":16:8", + }, + ) + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"job_url={job.url}\n") + print(f"HF Job: {job.url}", flush=True) + + def cancel_job(signum, frame): + api.cancel_job(job_id=job.id) + raise SystemExit(128 + signum) + + signal.signal(signal.SIGINT, cancel_job) + signal.signal(signal.SIGTERM, cancel_job) + + try: + finished = api.wait_for_job(job_id=job.id, poll_interval=20) + except Exception: + api.cancel_job(job_id=job.id) + raise + try: + for line in api.fetch_job_logs(job_id=job.id, tail=200): + print(line, flush=True) + except Exception as error: + print(f"Could not fetch HF Job logs: {error}", flush=True) + print(f"HF Job status: {finished.status.stage}", flush=True) + if finished.status.stage != "COMPLETED": + sys.exit(1) + + +if __name__ == "__main__": + main() From 4a28492b370a3e1d20b0b10884272c111edb7a88 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 29 Sep 2026 09:22:00 +0530 Subject: [PATCH 2/2] token shenanigans --- .github/workflows/bot_pytest.yml | 5 +++-- .github/workflows/diffusers_bot.yml | 2 +- utils/bot_pytest_hf_job.py | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bot_pytest.yml b/.github/workflows/bot_pytest.yml index cb861f77741f..4ff62def3e2e 100644 --- a/.github/workflows/bot_pytest.yml +++ b/.github/workflows/bot_pytest.yml @@ -13,7 +13,7 @@ on: type: string default: '' secrets: - DIFFUSERS_BOT_TOKEN: + HF_JOBS_TOKEN: required: false jobs: @@ -169,7 +169,8 @@ jobs: - name: Run HF Job id: run env: - HF_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }} + HF_TOKEN: ${{ secrets.HF_JOBS_TOKEN }} + HF_JOBS_NAMESPACE: ${{ vars.DIFFUSERS_HF_JOBS_NAMESPACE || 'diffusers-internal-dev' }} REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ github.event.issue.number }} PR_SHA: ${{ steps.pr.outputs.sha }} diff --git a/.github/workflows/diffusers_bot.yml b/.github/workflows/diffusers_bot.yml index 48f0098eb9a2..00192082538b 100644 --- a/.github/workflows/diffusers_bot.yml +++ b/.github/workflows/diffusers_bot.yml @@ -130,4 +130,4 @@ jobs: pytest_args: ${{ needs.authorize.outputs.pytest_args }} pytest_flavor: ${{ needs.authorize.outputs.pytest_flavor }} secrets: - DIFFUSERS_BOT_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }} + HF_JOBS_TOKEN: ${{ secrets.DIFFUSERS_HF_JOBS_TOKEN }} diff --git a/utils/bot_pytest_hf_job.py b/utils/bot_pytest_hf_job.py index b250610e4a6f..c7befc8cd784 100644 --- a/utils/bot_pytest_hf_job.py +++ b/utils/bot_pytest_hf_job.py @@ -39,7 +39,14 @@ def main(): flavor = os.environ["PYTEST_FLAVOR"] - api = HfApi(token=os.environ["HF_TOKEN"]) + namespace = os.environ["HF_JOBS_NAMESPACE"] + token = os.environ["HF_TOKEN"] + if not namespace: + raise SystemExit("HF_JOBS_NAMESPACE must be set before running HF Jobs") + if not token: + raise SystemExit("Set the DIFFUSERS_HF_JOBS_TOKEN repository secret before running HF Jobs") + + api = HfApi(token=token) flavors = { hardware.name for hardware in api.list_jobs_hardware() @@ -54,6 +61,7 @@ def main(): image="diffusers/diffusers-pytorch-cuda", command=["bash", "-lc", COMMAND], flavor=flavor, + namespace=namespace, timeout="2h", name=f"diffusers-pr-{os.environ['PR_NUMBER']}-pytest", env={ @@ -75,19 +83,19 @@ def main(): print(f"HF Job: {job.url}", flush=True) def cancel_job(signum, frame): - api.cancel_job(job_id=job.id) + api.cancel_job(job_id=job.id, namespace=namespace) raise SystemExit(128 + signum) signal.signal(signal.SIGINT, cancel_job) signal.signal(signal.SIGTERM, cancel_job) try: - finished = api.wait_for_job(job_id=job.id, poll_interval=20) + finished = api.wait_for_job(job_id=job.id, namespace=namespace, poll_interval=20) except Exception: - api.cancel_job(job_id=job.id) + api.cancel_job(job_id=job.id, namespace=namespace) raise try: - for line in api.fetch_job_logs(job_id=job.id, tail=200): + for line in api.fetch_job_logs(job_id=job.id, namespace=namespace, tail=200): print(line, flush=True) except Exception as error: print(f"Could not fetch HF Job logs: {error}", flush=True)