diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 748ff652..841ea806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,54 @@ permissions: pull-requests: read jobs: + # Skipped on a Release PR: it only bumps the version and rewrites the changelog. Not + # keyed on github.actor, which is the pusher: clicking Update branch reattributes the + # merge commit to whoever clicked, and the skip would stop firing on the normal release + # path. A human commit pushed onto a Release PR is therefore untested until release.yml + # runs the lane on the merge commit. unit: + if: >- + ${{ !(github.event.pull_request.user.login == 'github-actions[bot]' + && github.event.pull_request.head.repo.full_name == github.repository + && startsWith(github.head_ref, 'release-please--')) }} uses: ./.github/workflows/run_tests.yml - with: - marker: 'not integration' - secrets: inherit + + # The one required status check on main, and it carries no matrix on purpose: a matrix job + # skipped by `if:` publishes a single check run with the template unexpanded, so per-leg + # contexts could never be satisfied on a Release PR. `skipped` is accepted only when the + # condition above holds, repeated here because Actions cannot share an expression; if the + # two drift this fails, which is the safe direction. + tests-passed: + name: ๐Ÿงช Tests + needs: unit + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: Check the unit lane + env: + RESULT: ${{ needs.unit.result }} + RELEASE_PR: >- + ${{ github.event.pull_request.user.login == 'github-actions[bot]' + && github.event.pull_request.head.repo.full_name == github.repository + && startsWith(github.head_ref, 'release-please--') }} + run: | + case "$RESULT" in + success) + echo "unit lane passed" + ;; + skipped) + if [ "$RELEASE_PR" = "true" ]; then + echo "release pr: unit lane skipped by design" + else + echo "::error::the unit lane was skipped on a PR that is not a Release PR" + exit 1 + fi + ;; + *) + echo "::error::unit lane reported $RESULT" + exit 1 + ;; + esac # Cancel in-flight runs for the same branch/PR when new commits arrive concurrency: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a1e65af3..f7654e96 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -164,23 +164,21 @@ jobs: needs: detect if: needs.detect.outputs.ready == 'true' uses: ./.github/workflows/run_tests.yml - with: - marker: 'not integration' - secrets: inherit + # Deliberately absent from `release`'s `needs`. A failure here would skip tagging while + # the Release PR is already merged carrying `autorelease: pending`, so every later push + # would stand down and releases would stay wedged until someone cleared it by hand. test-integration: name: Test (integration) needs: detect if: needs.detect.outputs.ready == 'true' - uses: ./.github/workflows/run_tests.yml - with: - marker: 'integration' + uses: ./.github/workflows/run_integration.yml secrets: inherit # Irreversible half. release: name: ๐Ÿš€ Tag and release - needs: [detect, test-unit, test-integration] + needs: [detect, test-unit] if: needs.detect.outputs.ready == 'true' runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.github/workflows/run_integration.yml b/.github/workflows/run_integration.yml new file mode 100644 index 00000000..a6ffcfa3 --- /dev/null +++ b/.github/workflows/run_integration.yml @@ -0,0 +1,68 @@ +name: _run-integration +on: + workflow_call: + secrets: { } + +# A constant, not `github.workflow`, which inside a reusable workflow resolves to the +# caller's name: the daily run and the pre-tag run would land in separate lanes and hit the +# shared app at once. Queue rather than cancel, so the schedule cannot kill a release gate. +concurrency: + group: run-integration-${{ github.ref }} + cancel-in-progress: false + +env: + UV_FROZEN: "1" + +permissions: + contents: read + +# The only place `-m integration` runs: daily and before a tag, never as a status check on a +# pull request. See DEVELOPMENT.md for why it gates nothing. +jobs: + integration-non-video: + name: ๐Ÿงช Non-video integration (${{ matrix.python-version }}) + environment: + name: ci + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Install dependencies + uses: ./.github/actions/python-uv-setup + with: + python-version: ${{ matrix.python-version }} + - name: Run integration tests + env: + STREAM_API_KEY: ${{ vars.STREAM_CHAT_API_KEY }} + STREAM_API_SECRET: ${{ secrets.STREAM_CHAT_API_SECRET }} + STREAM_BASE_URL: ${{ vars.STREAM_CHAT_BASE_URL }} + run: make test MARKER="integration" + + integration-video: + name: ๐Ÿงช Video integration (${{ matrix.python-version }}) + environment: + name: ci + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v5 + - name: Install dependencies + uses: ./.github/actions/python-uv-setup + with: + python-version: ${{ matrix.python-version }} + - name: Run integration tests + env: + STREAM_API_KEY: ${{ vars.STREAM_API_KEY }} + STREAM_API_SECRET: ${{ secrets.STREAM_API_SECRET }} + STREAM_BASE_URL: ${{ vars.STREAM_BASE_URL }} + run: make test-video MARKER="integration" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index c1f9400a..a10cb469 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -1,14 +1,9 @@ name: _run-tests on: workflow_call: - inputs: - marker: - description: 'pytest -m expression (e.g., `not integration` or `integration`)' - required: true - type: string secrets: { } concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.marker }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: @@ -18,24 +13,12 @@ env: permissions: contents: read -# Every job below skips on a Release PR. release-please only bumps the version and rewrites -# the changelog, and every source commit in one already passed this suite on the PR it came -# from. The guard sits on each job rather than on the calling job in ci.yml, because a job -# skipped by `if:` reports success and satisfies a required status check, while a reusable -# workflow that is never called produces no check at all. It tests the author as well as the -# branch name: on its own, the name would let any PR, a fork's included, call its branch -# release-please--x and skip every required check, which branch protection counts as met. -# github.actor is the third clause, and it is the pusher rather than the PR author, so a -# human commit pushed onto the Release PR to fix a conflict or a changelog entry is tested -# like any other commit instead of riding the skip into main untested. +# The unit lane runs `-m "not integration"`, so it declares no environment and receives no +# credentials. Keep it that way: a fork PR gets no secrets and still goes green, and a live +# test added without the marker fails here instead of passing on someone else's. jobs: ruff: - if: >- - ${{ !(github.actor == 'github-actions[bot]' - && github.event.pull_request.user.login == 'github-actions[bot]' - && github.event.pull_request.head.repo.full_name == github.repository - && startsWith(github.head_ref, 'release-please--')) }} - name: Ruff + name: ๐Ÿงช Ruff runs-on: ubuntu-latest steps: - name: Checkout @@ -46,12 +29,7 @@ jobs: run: make lint typecheck: - if: >- - ${{ !(github.actor == 'github-actions[bot]' - && github.event.pull_request.user.login == 'github-actions[bot]' - && github.event.pull_request.head.repo.full_name == github.repository - && startsWith(github.head_ref, 'release-please--')) }} - name: Type Check (ty) + name: ๐Ÿงช Type Check (ty) runs-on: ubuntu-latest steps: - name: Checkout @@ -65,14 +43,7 @@ jobs: # Uses STREAM_CHAT_* credentials which do NOT have video enabled. # Video paths and manual test paths are defined in the Makefile. test-non-video: - if: >- - ${{ !(github.actor == 'github-actions[bot]' - && github.event.pull_request.user.login == 'github-actions[bot]' - && github.event.pull_request.head.repo.full_name == github.repository - && startsWith(github.head_ref, 'release-please--')) }} - name: Non-video tests (${{ matrix.python-version }}) - environment: - name: ci + name: ๐Ÿงช Non-video tests (${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -87,23 +58,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Run tests - env: - STREAM_API_KEY: ${{ vars.STREAM_CHAT_API_KEY }} - STREAM_API_SECRET: ${{ secrets.STREAM_CHAT_API_SECRET }} - STREAM_BASE_URL: ${{ vars.STREAM_CHAT_BASE_URL }} - run: make test MARKER="${{ inputs.marker }}" + run: make test # โ”€โ”€ Video tests (video-enabled credentials) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # Uses STREAM_* credentials which have video enabled. test-video: - if: >- - ${{ !(github.actor == 'github-actions[bot]' - && github.event.pull_request.user.login == 'github-actions[bot]' - && github.event.pull_request.head.repo.full_name == github.repository - && startsWith(github.head_ref, 'release-please--')) }} - name: Video tests (${{ matrix.python-version }}) - environment: - name: ci + name: ๐Ÿงช Video tests (${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -118,8 +78,4 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Run tests - env: - STREAM_API_KEY: ${{ vars.STREAM_API_KEY }} - STREAM_API_SECRET: ${{ secrets.STREAM_API_SECRET }} - STREAM_BASE_URL: ${{ vars.STREAM_BASE_URL }} - run: make test-video MARKER="${{ inputs.marker }}" + run: make test-video diff --git a/.github/workflows/scheduled_test.yml b/.github/workflows/scheduled_test.yml new file mode 100644 index 00000000..5f7ba0e3 --- /dev/null +++ b/.github/workflows/scheduled_test.yml @@ -0,0 +1,42 @@ +name: Scheduled tests +# The integration lane's home. Advisory: see DEVELOPMENT.md. +on: + schedule: + - cron: "0 9 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + integration: + uses: ./.github/workflows/run_integration.yml + secrets: inherit + + # A failure here lands on no PR, and GitHub emails it only to whoever last edited the cron. + report: + name: Report a red run + needs: integration + if: failure() + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open or update the tracking issue + env: + GH_TOKEN: ${{ github.token }} + TITLE: Daily integration run is red + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + num="$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" \ + --json number,title --jq "map(select(.title == \"$TITLE\")) | .[0].number // empty")" + if [ -n "$num" ]; then + gh issue comment "$num" --repo "$GITHUB_REPOSITORY" --body "Still red: $RUN_URL" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body \ + "The daily \`-m integration\` run failed: $RUN_URL + + Integration is advisory, so nothing is blocked by this. It still has to be read: the suite + runs against a live Stream app shared by five SDK repos, so decide whether this is the app, + the backend, or this SDK, and close the issue once a daily run is green again." + fi diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d41ddd09..8544dcd0 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -22,7 +22,9 @@ make test-all # both of the above ``` Non-video and video tests are split because they require different Stream credentials. -The `MARKER` variable defaults to `"not integration"`. Override it for integration tests: +The `MARKER` variable defaults to `"not integration"`, which is every test that does not +touch a live Stream app, so the default needs no credentials. Override it to run the ones +that do: ``` make test-integration # runs both groups with -m "integration" @@ -36,6 +38,27 @@ make test-jaeger # requires local Jaeger (docker run ... jaegertracing/all make test-prometheus # requires getstream[telemetry] deps ``` +### What CI runs + +| Trigger | What runs | Gates anything? | +| --- | --- | --- | +| Pull request | `run_tests.yml`: ruff, ty, and `-m "not integration"` on five Python versions | yes, `๐Ÿงช Tests` is the required check | +| Daily at 09:00 UTC | `run_integration.yml`: `-m integration`, both credential sets | no | +| Push to `main` with a release pending | both; only the unit lane gates the tag | unit yes, integration no | + +`@pytest.mark.integration` means one thing: the test talks to a live Stream app. The unit +lane therefore runs with no credentials, no `environment:` and no `STREAM_*`. Keep it that +way. A fork PR gets no secrets and still goes green, and a live test added without the +marker fails in CI instead of quietly passing on someone else's credentials. + +Integration gates nothing, anywhere. It runs against an app five SDK repos share, so +another repo's run or a backend regression can redden it with nothing wrong here, and a red +pre-tag run used to wedge every later release behind `autorelease: pending`. A red daily run +opens an issue titled "Daily integration run is red". Fix it, do not route around it. + +A Release PR skips the lane and `๐Ÿงช Tests` passes in seconds on a `skipped` result. The +merge commit still runs it before the tag. + ### Linting and type checking ``` diff --git a/tests/rtc/coordinator/test_connect.py b/tests/rtc/coordinator/test_connect.py index 3d2d156a..461f1110 100644 --- a/tests/rtc/coordinator/test_connect.py +++ b/tests/rtc/coordinator/test_connect.py @@ -17,6 +17,9 @@ from tests.conftest import skip_on_rate_limit +pytestmark = pytest.mark.integration + + @pytest.mark.asyncio async def test_simple_connection_debug(): """Simple test to debug websocket server handler signature.""" diff --git a/tests/rtc/coordinator/test_heartbeat.py b/tests/rtc/coordinator/test_heartbeat.py index 4c30aef5..0d6ec481 100644 --- a/tests/rtc/coordinator/test_heartbeat.py +++ b/tests/rtc/coordinator/test_heartbeat.py @@ -17,6 +17,9 @@ from getstream import Stream +pytestmark = pytest.mark.integration + + @pytest.mark.asyncio async def test_heartbeat_sent_periodically(client: Stream): """Test that heartbeat messages are sent at regular intervals.""" diff --git a/tests/rtc/test_join.py b/tests/rtc/test_join.py index cac66adc..2ba03c60 100644 --- a/tests/rtc/test_join.py +++ b/tests/rtc/test_join.py @@ -20,6 +20,10 @@ # Shared function for process setup and error handling + +pytestmark = pytest.mark.integration + + def run_process_with_stream_client( process_type: str, call_id: str, diff --git a/tests/rtc/test_video_properties.py b/tests/rtc/test_video_properties.py index 481785b1..acdbcfdf 100644 --- a/tests/rtc/test_video_properties.py +++ b/tests/rtc/test_video_properties.py @@ -11,6 +11,9 @@ from getstream.video.rtc.pb.stream.video.sfu.models.models_pb2 import TRACK_TYPE_VIDEO +pytestmark = pytest.mark.integration + + @pytest.mark.asyncio async def test_detect_video_properties(): """Test that video properties are correctly detected from a video file.""" diff --git a/tests/test_chat_channel.py b/tests/test_chat_channel.py index 973f67ab..7b555e95 100644 --- a/tests/test_chat_channel.py +++ b/tests/test_chat_channel.py @@ -17,10 +17,14 @@ UserRequest, ) from tests.base import wait_for_task +import pytest ASSETS_DIR = Path(__file__).parent / "assets" +pytestmark = pytest.mark.integration + + class TestChannelCRUD: def test_create_channel(self, client: Stream, random_users): """Create a channel without specifying an ID (distinct channel).""" diff --git a/tests/test_chat_draft.py b/tests/test_chat_draft.py index c09f0ce3..24689f00 100644 --- a/tests/test_chat_draft.py +++ b/tests/test_chat_draft.py @@ -14,6 +14,9 @@ ) +pytestmark = pytest.mark.integration + + def _create_draft(channel, text, user_id, parent_id=None): """Create a draft via raw HTTP (endpoint is client-side-only, not in generated SDK).""" message = {"text": text, "user_id": user_id} diff --git a/tests/test_chat_integration.py b/tests/test_chat_integration.py index 1c1410f8..5ba24ee3 100644 --- a/tests/test_chat_integration.py +++ b/tests/test_chat_integration.py @@ -12,6 +12,7 @@ import warnings +@pytest.mark.integration def test_upsert_users(client: Stream): users = {} user_id = str(uuid.uuid4()) @@ -22,11 +23,13 @@ def test_upsert_users(client: Stream): client.update_users(users=users) +@pytest.mark.integration def test_query_users(client: Stream): response = client.query_users(QueryUsersPayload(filter_conditions={})) assert response.data.users is not None +@pytest.mark.integration def test_update_users_partial(client: Stream): user_id = str(uuid.uuid4()) users = { @@ -50,6 +53,7 @@ def test_update_users_partial(client: Stream): assert response.data.users[user_id].custom["color"] == "blue" +@pytest.mark.integration def test_deactivate_and_reactivate_users(client: Stream): user_id = str(uuid.uuid4()) users = { @@ -66,6 +70,7 @@ def test_deactivate_and_reactivate_users(client: Stream): assert response.data.task_id is not None +@pytest.mark.integration def test_delete_user(client: Stream): user_id = str(uuid.uuid4()) users = { @@ -81,6 +86,7 @@ def test_delete_user(client: Stream): assert user_id not in user_ids +@pytest.mark.integration @pytest.mark.asyncio async def test_send_message(async_client: AsyncStream): channel = async_client.chat.channel("messaging", str(uuid.uuid4())) @@ -92,8 +98,14 @@ async def test_send_message(async_client: AsyncStream): ) -def test_from_env(): +def test_from_env(monkeypatch): + # Set them rather than reading the ambient environment: CI runs this lane with none. + monkeypatch.setenv("STREAM_API_KEY", "key-from-env") + monkeypatch.setenv("STREAM_API_SECRET", "secret-from-env") + # Suppress the deprecation warning for this explicit compatibility check with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) - Stream.from_env() + client = Stream.from_env() + + assert client.api_key == "key-from-env" diff --git a/tests/test_chat_message.py b/tests/test_chat_message.py index 7ebc80d1..134f6076 100644 --- a/tests/test_chat_message.py +++ b/tests/test_chat_message.py @@ -19,6 +19,9 @@ from tests.base import retry_on_transient_error +pytestmark = pytest.mark.integration + + def test_send_message(channel: Channel, random_user): """Send a message with skip_push option.""" response = channel.send_message( diff --git a/tests/test_chat_misc.py b/tests/test_chat_misc.py index bd0c93cf..ac01e4d3 100644 --- a/tests/test_chat_misc.py +++ b/tests/test_chat_misc.py @@ -19,6 +19,9 @@ ) +pytestmark = pytest.mark.integration + + def test_get_app_settings(client: Stream): """Get application settings.""" response = client.get_app() diff --git a/tests/test_chat_moderation.py b/tests/test_chat_moderation.py index 87c31de7..2227f30c 100644 --- a/tests/test_chat_moderation.py +++ b/tests/test_chat_moderation.py @@ -10,6 +10,10 @@ QueryBannedUsersPayload, QueryMessageFlagsPayload, ) +import pytest + + +pytestmark = pytest.mark.integration def test_ban_user(client: Stream, random_user, server_user): diff --git a/tests/test_chat_polls.py b/tests/test_chat_polls.py index c2333b8c..fa0b4f1b 100644 --- a/tests/test_chat_polls.py +++ b/tests/test_chat_polls.py @@ -8,6 +8,10 @@ PollOptionInput, VoteData, ) +import pytest + + +pytestmark = pytest.mark.integration def test_create_get_update_delete_poll(client: Stream, random_user): diff --git a/tests/test_chat_reminders_locations.py b/tests/test_chat_reminders_locations.py index 33b0817c..626877d2 100644 --- a/tests/test_chat_reminders_locations.py +++ b/tests/test_chat_reminders_locations.py @@ -10,6 +10,9 @@ from tests.base import retry_on_transient_error +pytestmark = pytest.mark.integration + + class TestReminders: @pytest.fixture(autouse=True) def setup_channel_for_reminders(self, channel: Channel): diff --git a/tests/test_chat_team_usage_stats.py b/tests/test_chat_team_usage_stats.py index 8fd13f3a..439c014b 100644 --- a/tests/test_chat_team_usage_stats.py +++ b/tests/test_chat_team_usage_stats.py @@ -1,6 +1,10 @@ from datetime import date, timedelta from getstream import Stream +import pytest + + +pytestmark = pytest.mark.integration def test_query_team_usage_stats_default(client: Stream): diff --git a/tests/test_chat_user.py b/tests/test_chat_user.py index e47b4a31..22b1f517 100644 --- a/tests/test_chat_user.py +++ b/tests/test_chat_user.py @@ -14,6 +14,10 @@ UpdateUserPartialRequest, UserRequest, ) +import pytest + + +pytestmark = pytest.mark.integration def test_upsert_users(client: Stream): diff --git a/tests/test_client.py b/tests/test_client.py index bcbfca31..5cd2c262 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -40,6 +40,7 @@ def test_incorrect_client_throws_exception(monkeypatch): Stream(api_key="xxx", api_secret="xxx", base_url="ftp://example.com") +@pytest.mark.integration def test_client_does_not_raise_exception_without_tracer(client: Stream, monkeypatch): # Monkey patch _get_tracer to always return None from getstream.common import telemetry @@ -50,6 +51,7 @@ def test_client_does_not_raise_exception_without_tracer(client: Stream, monkeypa assert response.data is not None +@pytest.mark.integration def test_client_works_with_no_otel(client: Stream, monkeypatch): # Monkey patch _get_tracer to always return None from getstream.common import telemetry diff --git a/tests/test_feed_integration.py b/tests/test_feed_integration.py index 9b95bbea..f43efaf8 100644 --- a/tests/test_feed_integration.py +++ b/tests/test_feed_integration.py @@ -36,6 +36,9 @@ from getstream.stream_response import StreamResponse +pytestmark = pytest.mark.integration + + class TestFeedIntegration: """ Systematic Integration tests for Feed operations diff --git a/tests/test_video_examples.py b/tests/test_video_examples.py index 6f7509e7..feb8b58c 100644 --- a/tests/test_video_examples.py +++ b/tests/test_video_examples.py @@ -20,6 +20,9 @@ from tests.test_video_integration import get_openai_api_key_or_skip +pytestmark = pytest.mark.integration + + def test_setup_client(): from getstream import Stream diff --git a/tests/test_video_integration.py b/tests/test_video_integration.py index fbfc37d1..b32a2a83 100644 --- a/tests/test_video_integration.py +++ b/tests/test_video_integration.py @@ -33,6 +33,9 @@ EXTERNAL_STORAGE_NAME = f"storage{uuid.uuid4()}" +pytestmark = pytest.mark.integration + + def get_openai_api_key_or_skip(): """ Get the OpenAI API key from environment variables or skip the test. diff --git a/tests/test_video_openai.py b/tests/test_video_openai.py index e273e4d2..4eb1674b 100644 --- a/tests/test_video_openai.py +++ b/tests/test_video_openai.py @@ -29,6 +29,9 @@ ) +pytestmark = pytest.mark.integration + + class TestOpenAIPatching: """Tests for the OpenAI patching functionality."""