diff --git a/.github/workflows/console-route-probe.yml b/.github/workflows/console-route-probe.yml new file mode 100644 index 0000000..7ceea74 --- /dev/null +++ b/.github/workflows/console-route-probe.yml @@ -0,0 +1,55 @@ +name: Public web probe + +on: + schedule: + - cron: "*/5 * * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: console-route-probe + cancel-in-progress: false + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 4 + outputs: + alert_label: ${{ steps.probe.outputs.alert_label }} + steps: + - name: Check out the route contract and probe + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + # Preserve the script's concise failure output before the next step marks + # the job red. The terminal notification job includes that output in its + # pipeline label. + - name: Probe public web routes + id: probe + continue-on-error: true + env: + PROBE_RETRY_DELAY_SECONDS: "15" + run: python3 scripts/probe_console_routes.py + + - name: Mark a persistent route failure + if: steps.probe.outcome == 'failure' + env: + ALERT_LABEL: ${{ steps.probe.outputs.alert_label }} + run: | + printf '%s\n' "${ALERT_LABEL:-public web probe failed}" >&2 + exit 1 + + notify: + needs: [probe] + if: ${{ !cancelled() && !contains(needs.*.result, 'cancelled') }} + permissions: + contents: read + actions: read + uses: mindsdb/github-actions/.github/workflows/notify-main-failure.yml@main + with: + env-name: ${{ needs.probe.outputs.alert_label || 'public web probe' }} + status: ${{ contains(needs.*.result, 'failure') && 'failed' || 'recovered' }} + secrets: inherit diff --git a/README.md b/README.md index 499626e..24479c6 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,79 @@ Use an action from this repo in your workflow like this: **NOTE: This needs to go AFTER any `actions/checkout` step for the current repo** +## Public web probe + +The `Public web probe` workflow checks public Console routes, Cowork, and the MindsHub website every +five minutes. It runs from a GitHub-hosted runner, so it tests the same edge routes a user reaches +without depending on either Kubernetes cluster. Every response must be exactly `200`, redirects are +not followed, and each endpoint must contain its own stable body marker. + +The monitoring tiers use the shortest cadence that each runner supports: + +| Tier | Cadence | Runner | Contract | +| --- | --- | --- | --- | +| Public uptime | 60 seconds | Cloudflare Health Checks | Production and staging Console, production Cowork, and the website public paths return 200 with their body markers; direct ALB and Pages checks remain non-paging diagnostics | +| Public route matrix | 5 minutes | This GitHub Actions workflow | Every endpoint below serves its expected body through the edge | +| Staging integration | Nightly | Each service repository | Existing deployed suites run with writes allowed | +| Production smoke | Nightly | `cowork-server` | A bounded authenticated GET-only selection runs without writes or model turns | +| Certificates | Existing | Prometheus | `CertificatesExpiringIn7Days` retains certificate coverage | + +An authenticated 15-minute browser journey is not scheduled by this work. It needs a separate +interaction contract and test plan before it can become an actionable signal. + +The first attempt covers 21 public endpoints: all 18 Console environment and route pairs, the +production and staging Cowork roots, and the MindsHub website root. If any endpoint fails, the +script waits 15 seconds and retries only those failures. A recovered retry keeps the run green and +sends no failure alert. An endpoint that fails twice makes the run red and passes every persistent +failure to `notify-main-failure.yml`, which posts to the engineering Slack channel. That message +names each failing environment, route, and observed result. A `200` response without the configured +marker is reported as either `status 200 missing SPA marker` or +`status 200 missing website marker`. The next successful run uses that reusable workflow's +existing recovery lookup, so routine green runs stay silent. + +The reviewable source of truth is [`config/console-route-probe.json`](config/console-route-probe.json). +The Console matrix crosses these environments: + + +- `production`: `https://console.mindshub.ai` +- `staging`: `https://console.staging.mindshub.ai` + + +with these routes: + + +- `/` +- `/home` +- `/cowork` +- `/cowork-web` +- `/login` +- `/settings` +- `/billing` +- `/projects` +- `/assets/` + + +The same configuration checks three roots separately because they do not serve the nine Console +routes: + + +- `production Cowork`: `https://cowork.mindshub.ai/` requires `
` +- `staging Cowork`: `https://cowork.staging.mindshub.ai/` requires `
` +- `production website`: `https://mindshub.ai/` requires `` + + +Run the same probe locally without changing its production configuration: + +```bash +python3 scripts/probe_console_routes.py --retry-delay-seconds 0 +``` + +ENG-2317 is the activation gate. Do not merge or promote this workflow to the `github-actions` +`main` branch, and do not manually dispatch it, until ENG-2317 is deployed to production and the +local command above passes all 18 Console environment and route pairs and all 21 public endpoints. A +merge to `main` enables the cron; a merge or dispatch before that gate can send an immediate +failure or recovery message through the real Slack integration. + ## Release-train reusable workflows Four reusable workflows automate the weekly `staging → main` release cycle. diff --git a/config/console-route-probe.json b/config/console-route-probe.json new file mode 100644 index 0000000..1a36cfc --- /dev/null +++ b/config/console-route-probe.json @@ -0,0 +1,47 @@ +{ + "spa_marker": "
", + "environments": [ + { + "name": "production", + "base_url": "https://console.mindshub.ai" + }, + { + "name": "staging", + "base_url": "https://console.staging.mindshub.ai" + } + ], + "routes": [ + "/", + "/home", + "/cowork", + "/cowork-web", + "/login", + "/settings", + "/billing", + "/projects", + "/assets/" + ], + "standalone_endpoints": [ + { + "name": "production Cowork", + "base_url": "https://cowork.mindshub.ai", + "route": "/", + "body_marker": "
", + "marker_label": "SPA" + }, + { + "name": "staging Cowork", + "base_url": "https://cowork.staging.mindshub.ai", + "route": "/", + "body_marker": "
", + "marker_label": "SPA" + }, + { + "name": "production website", + "base_url": "https://mindshub.ai", + "route": "/", + "body_marker": "", + "marker_label": "website" + } + ] +} diff --git a/scripts/probe_console_routes.py b/scripts/probe_console_routes.py new file mode 100644 index 0000000..e115fab --- /dev/null +++ b/scripts/probe_console_routes.py @@ -0,0 +1,457 @@ +"""Probe public Console routes, Cowork, and the website with one-retry debounce. + +The endpoints and body markers live in ``config/console-route-probe.json`` so +operators can review the whole coverage contract without reading this module. +Only endpoints that fail the first attempt are retried. A transient failure is +therefore visible in the run log but does not fail the workflow or alert Slack. +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Iterable, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + + +DEFAULT_CONFIG_PATH = ( + Path(__file__).resolve().parents[1] / "config" / "console-route-probe.json" +) +DEFAULT_ALERT_LABEL = "public web probe" +MAX_BODY_BYTES = 64 * 1024 +MAX_NETWORK_ERROR_CHARS = 64 +MAX_ALERT_LABEL_CHARS = 2_000 + + +@dataclass(frozen=True) +class Environment: + """One named Console deployment.""" + + name: str + base_url: str + + +@dataclass(frozen=True) +class Endpoint: + """One public URL and the body marker it must return.""" + + environment: str + base_url: str + route: str + body_marker: str + marker_label: str + + @property + def url(self) -> str: + return f"{self.base_url.rstrip('/')}{self.route}" + + +@dataclass(frozen=True) +class ProbeConfig: + """Validated route-probe configuration.""" + + spa_marker: str + environments: tuple[Environment, ...] + routes: tuple[str, ...] + standalone_endpoints: tuple[Endpoint, ...] + + @property + def console_endpoints(self) -> tuple[Endpoint, ...]: + return tuple( + Endpoint( + environment.name, + environment.base_url, + route, + self.spa_marker, + "SPA", + ) + for environment in self.environments + for route in self.routes + ) + + @property + def endpoints(self) -> tuple[Endpoint, ...]: + return (*self.console_endpoints, *self.standalone_endpoints) + + +@dataclass(frozen=True) +class FetchResult: + """The response fields needed to decide whether an endpoint is healthy.""" + + status: int | None + body: str = "" + error: str | None = None + + +@dataclass(frozen=True) +class Failure: + """One endpoint that did not return its expected body marker.""" + + endpoint: Endpoint + reason: str + + @property + def summary(self) -> str: + return f"{self.endpoint.environment} {self.endpoint.route} {self.reason}" + + +@dataclass(frozen=True) +class ProbeOutcome: + """Both attempts, retained so the CLI can explain transient recovery.""" + + endpoint_count: int + first_failures: tuple[Failure, ...] + final_failures: tuple[Failure, ...] + + @property + def passed(self) -> bool: + return not self.final_failures + + +Fetch = Callable[[Endpoint, float], FetchResult] +Sleeper = Callable[[float], None] + + +class NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Return redirects to the caller instead of following them.""" + + def redirect_request(self, *args: Any, **kwargs: Any) -> None: + return None + + +def _require_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field} must be a non-empty string") + return value + + +def _require_list(value: object, field: str) -> list[object]: + if not isinstance(value, list) or not value: + raise ValueError(f"{field} must be a non-empty list") + return value + + +def _require_object(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError(f"{field} must be an object") + return value + + +def _require_https_origin(value: object, field: str) -> str: + origin = _require_string(value, field) + parsed = urlsplit(origin) + if parsed.scheme != "https" or not parsed.netloc or parsed.path not in ("", "/"): + raise ValueError(f"{field} must be an HTTPS origin") + if parsed.query or parsed.fragment or parsed.username or parsed.password: + raise ValueError(f"{field} must be an HTTPS origin") + return origin.rstrip("/") + + +def _require_route(value: object, field: str) -> str: + route = _require_string(value, field) + if not route.startswith("/") or "?" in route or "#" in route: + raise ValueError( + f"{field} must be an absolute path without a query or fragment" + ) + return route + + +def load_config(path: Path = DEFAULT_CONFIG_PATH) -> ProbeConfig: + """Load and validate the operator-owned route matrix.""" + + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError("probe config must be a JSON object") + + spa_marker = _require_string(raw.get("spa_marker"), "spa_marker") + raw_environments = _require_list(raw.get("environments"), "environments") + raw_routes = _require_list(raw.get("routes"), "routes") + raw_standalone_endpoints = _require_list( + raw.get("standalone_endpoints"), "standalone_endpoints" + ) + + environments: list[Environment] = [] + for index, value in enumerate(raw_environments): + if not isinstance(value, dict): + raise ValueError(f"environments[{index}] must be an object") + name = _require_string(value.get("name"), f"environments[{index}].name") + base_url = _require_https_origin( + value.get("base_url"), f"environments[{index}].base_url" + ) + environments.append(Environment(name=name, base_url=base_url)) + + if len({environment.name for environment in environments}) != len(environments): + raise ValueError("environment names must be unique") + if len({environment.base_url for environment in environments}) != len(environments): + raise ValueError("environment base URLs must be unique") + + routes: list[str] = [] + for index, value in enumerate(raw_routes): + routes.append(_require_route(value, f"routes[{index}]")) + if len(set(routes)) != len(routes): + raise ValueError("routes must be unique") + + standalone_endpoints: list[Endpoint] = [] + for index, value in enumerate(raw_standalone_endpoints): + field = f"standalone_endpoints[{index}]" + endpoint = _require_object(value, field) + standalone_endpoints.append( + Endpoint( + environment=_require_string(endpoint.get("name"), f"{field}.name"), + base_url=_require_https_origin( + endpoint.get("base_url"), f"{field}.base_url" + ), + route=_require_route(endpoint.get("route"), f"{field}.route"), + body_marker=_require_string( + endpoint.get("body_marker"), f"{field}.body_marker" + ), + marker_label=_require_string( + endpoint.get("marker_label"), f"{field}.marker_label" + ), + ) + ) + + endpoint_names = [environment.name for environment in environments] + endpoint_names.extend(endpoint.environment for endpoint in standalone_endpoints) + if len(set(endpoint_names)) != len(endpoint_names): + raise ValueError("endpoint names must be unique") + + config = ProbeConfig( + spa_marker=spa_marker, + environments=tuple(environments), + routes=tuple(routes), + standalone_endpoints=tuple(standalone_endpoints), + ) + if len({endpoint.url for endpoint in config.endpoints}) != len(config.endpoints): + raise ValueError("configured endpoint URLs must be unique") + return config + + +def _decode_body(payload: bytes) -> str: + return payload.decode("utf-8", errors="replace") + + +def fetch_endpoint(endpoint: Endpoint, timeout_seconds: float) -> FetchResult: + """GET one endpoint with TLS validation and redirects disabled.""" + + opener = urllib.request.build_opener(NoRedirectHandler()) + request = urllib.request.Request( + endpoint.url, + headers={ + "Accept": "text/html", + "User-Agent": "MindsHub-public-web-probe/1.0", + }, + method="GET", + ) + try: + with opener.open(request, timeout=timeout_seconds) as response: + return FetchResult( + status=int(response.getcode()), + body=_decode_body(response.read(MAX_BODY_BYTES)), + ) + except urllib.error.HTTPError as error: + return FetchResult( + status=int(error.code), + body=_decode_body(error.read(MAX_BODY_BYTES)), + ) + except (urllib.error.URLError, TimeoutError, OSError) as error: + return FetchResult(status=None, error=_compact(str(error))) + + +def _compact(value: str, limit: int = 160) -> str: + compacted = " ".join(value.split()) or "unknown error" + compacted = compacted.replace("\\", "/").replace('"', "'") + return compacted[:limit] + + +def evaluate(endpoint: Endpoint, result: FetchResult) -> Failure | None: + """Apply the endpoint's exact-status and body-marker contract.""" + + if result.error is not None: + return Failure( + endpoint, + f"network error: {_compact(result.error, limit=MAX_NETWORK_ERROR_CHARS)}", + ) + if result.status != 200: + status = "unknown" if result.status is None else str(result.status) + return Failure(endpoint, f"status {status}") + if endpoint.body_marker not in result.body: + return Failure(endpoint, f"status 200 missing {endpoint.marker_label} marker") + return None + + +def check_endpoints( + endpoints: Sequence[Endpoint], + *, + timeout_seconds: float, + max_workers: int, + fetcher: Fetch, +) -> tuple[Failure, ...]: + """Probe endpoints concurrently while preserving configuration order.""" + + if not endpoints: + return () + worker_count = min(max_workers, len(endpoints)) + with ThreadPoolExecutor(max_workers=worker_count) as executor: + futures = [ + executor.submit(fetcher, endpoint, timeout_seconds) + for endpoint in endpoints + ] + results = [future.result() for future in futures] + + failures = ( + failure + for endpoint, result in zip(endpoints, results, strict=True) + if (failure := evaluate(endpoint, result)) is not None + ) + return tuple(failures) + + +def run_probe( + config: ProbeConfig, + *, + retry_delay_seconds: float, + timeout_seconds: float, + max_workers: int, + fetcher: Fetch = fetch_endpoint, + sleeper: Sleeper = time.sleep, +) -> ProbeOutcome: + """Run the full matrix once, then retry only the failed endpoints.""" + + endpoints = config.endpoints + first_failures = check_endpoints( + endpoints, + timeout_seconds=timeout_seconds, + max_workers=max_workers, + fetcher=fetcher, + ) + if not first_failures: + return ProbeOutcome(len(endpoints), (), ()) + + sleeper(retry_delay_seconds) + retry_endpoints = tuple(failure.endpoint for failure in first_failures) + final_failures = check_endpoints( + retry_endpoints, + timeout_seconds=timeout_seconds, + max_workers=max_workers, + fetcher=fetcher, + ) + return ProbeOutcome(len(endpoints), first_failures, final_failures) + + +def format_alert_label(failures: Sequence[Failure]) -> str: + """Fit every actionable failure into the notifier's bounded label.""" + + if not failures: + return DEFAULT_ALERT_LABEL + + label_prefix = "public routes: " + separator = "; " + full_label = label_prefix + separator.join(failure.summary for failure in failures) + if len(full_label) <= MAX_ALERT_LABEL_CHARS: + return full_label + + # A broad DNS or TLS outage can put the maximum-length network detail on all + # 21 endpoints. The full details remain in the workflow log; the Slack label + # keeps every endpoint identity and its actionable result category. + compact_summaries = separator.join( + f"{failure.endpoint.environment} {failure.endpoint.route} " + f"{_compact_alert_reason(failure.reason)}" + for failure in failures + ) + compact_label = label_prefix + compact_summaries + if len(compact_label) > MAX_ALERT_LABEL_CHARS: + raise ValueError("failure identities exceed the alert-label size limit") + return compact_label + + +def _compact_alert_reason(reason: str) -> str: + network_prefix = "network error" + if reason.startswith(f"{network_prefix}: "): + return network_prefix + return reason + + +def write_github_output(path: str | None, *, alert_label: str) -> None: + """Expose a single-line label to the terminal notification job.""" + + if not path: + return + with Path(path).open("a", encoding="utf-8") as output: + output.write( + f"alert_label={_compact(alert_label, limit=MAX_ALERT_LABEL_CHARS)}\n" + ) + + +def _print_failures(prefix: str, failures: Iterable[Failure]) -> None: + print(prefix) + for failure in failures: + print(f" - {failure.summary}") + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH) + parser.add_argument( + "--retry-delay-seconds", + type=float, + default=float(os.environ.get("PROBE_RETRY_DELAY_SECONDS", "15")), + ) + parser.add_argument("--timeout-seconds", type=float, default=10.0) + parser.add_argument("--max-workers", type=int, default=8) + parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT")) + args = parser.parse_args(argv) + if args.retry_delay_seconds < 0: + parser.error("--retry-delay-seconds must be non-negative") + if args.timeout_seconds <= 0: + parser.error("--timeout-seconds must be positive") + if args.max_workers <= 0: + parser.error("--max-workers must be positive") + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + config = load_config(args.config) + except (OSError, json.JSONDecodeError, ValueError) as error: + label = _compact(f"public routes: configuration error: {error}") + write_github_output(args.github_output, alert_label=label) + print(label) + return 2 + + outcome = run_probe( + config, + retry_delay_seconds=args.retry_delay_seconds, + timeout_seconds=args.timeout_seconds, + max_workers=args.max_workers, + ) + if outcome.passed: + write_github_output(args.github_output, alert_label=DEFAULT_ALERT_LABEL) + if outcome.first_failures: + _print_failures( + "First attempt failed; every failed endpoint recovered on retry:", + outcome.first_failures, + ) + print( + f"PASS: {outcome.endpoint_count} public endpoints returned status 200 " + "and their expected body marker." + ) + return 0 + + label = format_alert_label(outcome.final_failures) + write_github_output(args.github_output, alert_label=label) + _print_failures("FAIL: endpoints failed both attempts:", outcome.final_failures) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_probe_console_routes.py b/tests/test_probe_console_routes.py new file mode 100644 index 0000000..2d7b6cf --- /dev/null +++ b/tests/test_probe_console_routes.py @@ -0,0 +1,597 @@ +"""Tests for the public Console, Cowork, and website probe contract.""" + +import importlib.util +import json +import re +import sys +from collections import Counter +from io import BytesIO +from pathlib import Path +from urllib.error import HTTPError + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = ROOT / "scripts" / "probe_console_routes.py" +CONFIG_PATH = ROOT / "config" / "console-route-probe.json" +README_PATH = ROOT / "README.md" +WORKFLOW_PATH = ROOT / ".github" / "workflows" / "console-route-probe.yml" + +_spec = importlib.util.spec_from_file_location("probe_console_routes", SCRIPT_PATH) +probe = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = probe +_spec.loader.exec_module(probe) + + +EXPECTED_ENVIRONMENTS = ( + ("production", "https://console.mindshub.ai"), + ("staging", "https://console.staging.mindshub.ai"), +) +EXPECTED_ROUTES = ( + "/", + "/home", + "/cowork", + "/cowork-web", + "/login", + "/settings", + "/billing", + "/projects", + "/assets/", +) +CONSOLE_MARKER = '
' +EXPECTED_STANDALONE_ENDPOINTS = ( + ( + "production Cowork", + "https://cowork.mindshub.ai", + "/", + CONSOLE_MARKER, + "SPA", + ), + ( + "staging Cowork", + "https://cowork.staging.mindshub.ai", + "/", + CONSOLE_MARKER, + "SPA", + ), + ( + "production website", + "https://mindshub.ai", + "/", + '', + "website", + ), +) +EXPECTED_CONSOLE_ENDPOINT_COUNT = len(EXPECTED_ENVIRONMENTS) * len(EXPECTED_ROUTES) +EXPECTED_ENDPOINT_COUNT = EXPECTED_CONSOLE_ENDPOINT_COUNT + len( + EXPECTED_STANDALONE_ENDPOINTS +) +SPA_SHELL = """
""" +WEBSITE_PAGE = ( + "" + '' + "" +) +NGINX_PAGE = """

404 Not Found

nginx
""" + + +def endpoint(route: str = "/home"): + return probe.Endpoint( + "staging", + "https://console.staging.mindshub.ai", + route, + CONSOLE_MARKER, + "SPA", + ) + + +def standalone_endpoint(base_url): + return next( + target + for target in probe.load_config().standalone_endpoints + if target.base_url == base_url + ) + + +def cowork_endpoint(): + return standalone_endpoint("https://cowork.mindshub.ai") + + +def staging_cowork_endpoint(): + return standalone_endpoint("https://cowork.staging.mindshub.ai") + + +def website_endpoint(): + return standalone_endpoint("https://mindshub.ai") + + +def response(status: int = 200, body: str = SPA_SHELL): + return probe.FetchResult(status=status, body=body) + + +class RecordedFetcher: + """Return recorded responses by URL and retain every attempted endpoint.""" + + def __init__(self, recordings): + self.recordings = {url: list(values) for url, values in recordings.items()} + self.calls = [] + + def __call__(self, target, timeout_seconds): + self.calls.append((target.url, timeout_seconds)) + values = self.recordings.get(target.url) + if values: + return values.pop(0) + return response(body=target.body_marker) + + +class FakeHTTPResponse: + def __init__(self, status, body): + self.status = status + self.body = body.encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def getcode(self): + return self.status + + def read(self, limit): + return self.body[:limit] + + +class TestResponseContract: + def test_200_spa_shell_passes(self): + assert probe.evaluate(endpoint(), response()) is None + + def test_200_cowork_spa_shell_passes(self): + assert probe.evaluate(cowork_endpoint(), response()) is None + + def test_200_staging_cowork_spa_shell_passes(self): + assert probe.evaluate(staging_cowork_endpoint(), response()) is None + + def test_200_website_marker_passes(self): + assert probe.evaluate(website_endpoint(), response(body=WEBSITE_PAGE)) is None + + def test_200_spa_shell_fails_for_the_website(self): + failure = probe.evaluate(website_endpoint(), response()) + assert ( + failure.summary == "production website / status 200 missing website marker" + ) + + def test_403_fails_with_the_status(self): + failure = probe.evaluate(endpoint(), response(403, "Forbidden")) + assert failure.summary == "staging /home status 403" + + def test_301_fails_instead_of_counting_as_the_destination(self): + failure = probe.evaluate(endpoint(), response(301, "Moved")) + assert failure.summary == "staging /home status 301" + + def test_200_nginx_page_fails_without_the_spa_marker(self): + failure = probe.evaluate(endpoint(), response(200, NGINX_PAGE)) + assert failure.summary == "staging /home status 200 missing SPA marker" + + def test_network_error_is_actionable(self): + failure = probe.evaluate( + endpoint(), + probe.FetchResult(status=None, error="timed out"), + ) + assert failure.summary == "staging /home network error: timed out" + + def test_fetch_installs_the_no_redirect_handler(self, monkeypatch): + captured = [] + + class Opener: + def open(self, request, timeout): + return FakeHTTPResponse(200, SPA_SHELL) + + def build_opener(*handlers): + captured.extend(handlers) + return Opener() + + monkeypatch.setattr(probe.urllib.request, "build_opener", build_opener) + assert probe.fetch_endpoint(endpoint(), 4).status == 200 + assert len(captured) == 1 + assert isinstance(captured[0], probe.NoRedirectHandler) + assert captured[0].redirect_request(None) is None + + def test_fetch_returns_a_redirect_as_a_301_response(self, monkeypatch): + class RedirectingOpener: + def open(self, request, timeout): + raise HTTPError(request.full_url, 301, "Moved", {}, BytesIO(b"Moved")) + + monkeypatch.setattr( + probe.urllib.request, "build_opener", lambda *handlers: RedirectingOpener() + ) + result = probe.fetch_endpoint(endpoint(), 4) + assert (result.status, result.body) == (301, "Moved") + + +class TestRetryDebounce: + def test_a_clean_first_attempt_does_not_sleep_or_retry(self): + sleeper_calls = [] + fetcher = RecordedFetcher({}) + outcome = probe.run_probe( + probe.load_config(), + retry_delay_seconds=7, + timeout_seconds=4, + max_workers=1, + fetcher=fetcher, + sleeper=sleeper_calls.append, + ) + assert outcome.passed + assert outcome.first_failures == () + assert len(fetcher.calls) == EXPECTED_ENDPOINT_COUNT + assert sleeper_calls == [] + + def test_one_failed_attempt_then_success_is_green(self): + target = endpoint().url + fetcher = RecordedFetcher({target: [response(403, "Forbidden"), response()]}) + sleeper_calls = [] + outcome = probe.run_probe( + probe.load_config(), + retry_delay_seconds=7, + timeout_seconds=4, + max_workers=1, + fetcher=fetcher, + sleeper=sleeper_calls.append, + ) + assert outcome.passed + assert [failure.summary for failure in outcome.first_failures] == [ + "staging /home status 403" + ] + assert outcome.final_failures == () + assert sleeper_calls == [7] + assert Counter(url for url, _ in fetcher.calls)[target] == 2 + assert len(fetcher.calls) == EXPECTED_ENDPOINT_COUNT + 1 + + def test_website_failure_then_success_is_green(self): + target = website_endpoint().url + fetcher = RecordedFetcher( + {target: [response(200, SPA_SHELL), response(200, WEBSITE_PAGE)]} + ) + outcome = probe.run_probe( + probe.load_config(), + retry_delay_seconds=0, + timeout_seconds=4, + max_workers=1, + fetcher=fetcher, + sleeper=lambda _: None, + ) + assert outcome.passed + assert [failure.summary for failure in outcome.first_failures] == [ + "production website / status 200 missing website marker" + ] + assert Counter(url for url, _ in fetcher.calls)[target] == 2 + + def test_the_retry_contains_only_failed_endpoints(self): + first = endpoint("/home").url + second = endpoint("/assets/").url + fetcher = RecordedFetcher( + { + first: [response(403, "Forbidden"), response()], + second: [response(301, "Moved"), response()], + } + ) + outcome = probe.run_probe( + probe.load_config(), + retry_delay_seconds=0, + timeout_seconds=4, + max_workers=1, + fetcher=fetcher, + sleeper=lambda _: None, + ) + counts = Counter(url for url, _ in fetcher.calls) + assert outcome.passed + assert counts[first] == counts[second] == 2 + assert all( + count == 1 for url, count in counts.items() if url not in {first, second} + ) + + def test_two_failed_attempts_exit_with_the_last_concise_reason(self): + target = endpoint().url + fetcher = RecordedFetcher( + {target: [response(403, "Forbidden"), response(200, NGINX_PAGE)]} + ) + outcome = probe.run_probe( + probe.load_config(), + retry_delay_seconds=0, + timeout_seconds=4, + max_workers=1, + fetcher=fetcher, + sleeper=lambda _: None, + ) + assert not outcome.passed + assert [failure.summary for failure in outcome.final_failures] == [ + "staging /home status 200 missing SPA marker" + ] + assert probe.format_alert_label(outcome.final_failures) == ( + "public routes: staging /home status 200 missing SPA marker" + ) + + def test_one_alert_label_names_every_persistent_failure(self, tmp_path): + failures = tuple( + probe.Failure(target, "status 503") + for target in probe.load_config().endpoints + ) + label = probe.format_alert_label(failures) + output = tmp_path / "github-output" + probe.write_github_output(str(output), alert_label=label) + + assert len(label) <= probe.MAX_ALERT_LABEL_CHARS + for failure in failures: + assert failure.summary in label + assert label.count("status 503") == EXPECTED_ENDPOINT_COUNT + assert output.read_text(encoding="utf-8") == f"alert_label={label}\n" + + def test_network_outage_label_keeps_all_21_endpoints_and_results(self): + failures = tuple( + probe.evaluate( + target, + probe.FetchResult( + status=None, + error="x" * (probe.MAX_NETWORK_ERROR_CHARS + 10), + ), + ) + for target in probe.load_config().endpoints + ) + assert all(failures) + + label = probe.format_alert_label(failures) + + assert len(label) <= probe.MAX_ALERT_LABEL_CHARS + assert label.count("network error") == EXPECTED_ENDPOINT_COUNT + for failure in failures: + endpoint_result = ( + f"{failure.endpoint.environment} {failure.endpoint.route} network error" + ) + assert endpoint_result in label + + def test_transient_outcome_makes_main_exit_green_and_write_a_quiet_label( + self, tmp_path, monkeypatch + ): + output = tmp_path / "github-output" + failure = probe.Failure(endpoint(), "status 403") + monkeypatch.setattr( + probe, + "run_probe", + lambda *args, **kwargs: probe.ProbeOutcome( + EXPECTED_ENDPOINT_COUNT, (failure,), () + ), + ) + assert ( + probe.main(["--github-output", str(output), "--retry-delay-seconds", "0"]) + == 0 + ) + assert output.read_text(encoding="utf-8") == "alert_label=public web probe\n" + + def test_persistent_outcome_makes_main_exit_red_and_write_failure_details( + self, tmp_path, monkeypatch + ): + output = tmp_path / "github-output" + failure = probe.Failure(endpoint(), "status 403") + monkeypatch.setattr( + probe, + "run_probe", + lambda *args, **kwargs: probe.ProbeOutcome( + EXPECTED_ENDPOINT_COUNT, (failure,), (failure,) + ), + ) + assert ( + probe.main(["--github-output", str(output), "--retry-delay-seconds", "0"]) + == 1 + ) + assert output.read_text(encoding="utf-8") == ( + "alert_label=public routes: staging /home status 403\n" + ) + + def test_output_is_one_json_safe_line_for_the_slack_payload(self, tmp_path): + output = tmp_path / "github-output" + probe.write_github_output( + str(output), alert_label='bad "host"\ncertificate \\ mismatch' + ) + assert output.read_text(encoding="utf-8") == ( + "alert_label=bad 'host' certificate / mismatch\n" + ) + + +class TestConfigurationAndDocumentation: + def test_config_is_the_exact_console_matrix_plus_standalone_roots(self): + config = probe.load_config() + environments = tuple((item.name, item.base_url) for item in config.environments) + assert environments == EXPECTED_ENVIRONMENTS + assert config.routes == EXPECTED_ROUTES + assert len(config.console_endpoints) == EXPECTED_CONSOLE_ENDPOINT_COUNT + assert ( + len({item.url for item in config.console_endpoints}) + == EXPECTED_CONSOLE_ENDPOINT_COUNT + ) + standalone = tuple( + ( + item.environment, + item.base_url, + item.route, + item.body_marker, + item.marker_label, + ) + for item in config.standalone_endpoints + ) + assert standalone == EXPECTED_STANDALONE_ENDPOINTS + assert len(config.endpoints) == EXPECTED_ENDPOINT_COUNT + assert len({item.url for item in config.endpoints}) == EXPECTED_ENDPOINT_COUNT + + @pytest.mark.parametrize( + ("field", "value", "message"), + ( + ( + "name", + "", + "standalone_endpoints[1].name must be a non-empty string", + ), + ("name", "production", "endpoint names must be unique"), + ( + "base_url", + "http://mindshub.ai", + "standalone_endpoints[1].base_url must be an HTTPS origin", + ), + ( + "base_url", + "https://console.mindshub.ai", + "configured endpoint URLs must be unique", + ), + ( + "route", + "/?preview=true", + "standalone_endpoints[1].route must be an absolute path without a query or fragment", + ), + ( + "body_marker", + "", + "standalone_endpoints[1].body_marker must be a non-empty string", + ), + ( + "marker_label", + "", + "standalone_endpoints[1].marker_label must be a non-empty string", + ), + ), + ) + def test_invalid_standalone_contract_is_rejected( + self, tmp_path, field, value, message + ): + raw = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + raw["standalone_endpoints"][1][field] = value + config_path = tmp_path / "invalid-standalone.json" + config_path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises(ValueError, match=re.escape(message)): + probe.load_config(config_path) + + def test_each_standalone_endpoint_must_be_an_object(self, tmp_path): + raw = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + raw["standalone_endpoints"][1] = [] + config_path = tmp_path / "invalid-standalone.json" + config_path.write_text(json.dumps(raw), encoding="utf-8") + + with pytest.raises( + ValueError, match=r"standalone_endpoints\[1\] must be an object" + ): + probe.load_config(config_path) + + def test_readme_environment_and_route_lists_match_the_config(self): + readme = README_PATH.read_text(encoding="utf-8") + environment_block = re.search( + r"(.*?)" + r"", + readme, + re.DOTALL, + ).group(1) + route_block = re.search( + r"(.*?)" + r"", + readme, + re.DOTALL, + ).group(1) + documented_environments = tuple( + re.findall(r"^- `([^`]+)`: `([^`]+)`$", environment_block, re.MULTILINE) + ) + documented_routes = tuple( + re.findall(r"^- `([^`]+)`$", route_block, re.MULTILINE) + ) + assert documented_environments == EXPECTED_ENVIRONMENTS + assert documented_routes == EXPECTED_ROUTES + standalone_block = re.search( + r"(.*?)" + r"", + readme, + re.DOTALL, + ).group(1) + for name, base_url, route, body_marker, _ in EXPECTED_STANDALONE_ENDPOINTS: + assert f"- `{name}`: `{base_url}{route}` requires `{body_marker}`" in ( + standalone_block + ) + + def test_readme_records_the_tier_and_cadence_decisions(self): + readme = README_PATH.read_text(encoding="utf-8") + for contract in ( + "| Public uptime | 60 seconds | Cloudflare Health Checks |", + "| Public route matrix | 5 minutes | This GitHub Actions workflow |", + "| Staging integration | Nightly | Each service repository |", + "| Production smoke | Nightly | `cowork-server` |", + "| Certificates | Existing | Prometheus |", + "An authenticated 15-minute browser journey is not scheduled by this work.", + ): + assert contract in readme + + def test_readme_blocks_activation_until_eng_2317_is_live(self): + readme = README_PATH.read_text(encoding="utf-8") + + assert "ENG-2317 is the activation gate." in readme + assert "Do not merge or promote this workflow" in readme + assert "passes all 18 Console environment and route pairs" in readme + assert f"all {EXPECTED_ENDPOINT_COUNT} public endpoints" in readme + + def test_workflow_runs_every_five_minutes_and_keeps_the_notify_terminal(self): + workflow = yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + assert workflow["name"] == "Public web probe" + triggers = workflow.get("on", workflow.get(True)) + assert triggers == { + "schedule": [{"cron": "*/5 * * * *"}], + "workflow_dispatch": None, + } + assert set(workflow["jobs"]) == {"probe", "notify"} + notify = workflow["jobs"]["notify"] + probe_job = workflow["jobs"]["probe"] + probe_step = next( + step for step in probe_job["steps"] if step.get("id") == "probe" + ) + failure_step = next( + step + for step in probe_job["steps"] + if step["name"] == "Mark a persistent route failure" + ) + assert probe_step["continue-on-error"] is True + assert probe_step["env"]["PROBE_RETRY_DELAY_SECONDS"] == "15" + assert failure_step["if"] == "steps.probe.outcome == 'failure'" + assert ( + probe_job["outputs"]["alert_label"] + == "${{ steps.probe.outputs.alert_label }}" + ) + assert notify["needs"] == ["probe"] + assert notify["permissions"]["actions"] == "read" + assert notify["uses"] == ( + "mindsdb/github-actions/.github/workflows/notify-main-failure.yml@main" + ) + assert "needs.probe.outputs.alert_label" in notify["with"]["env-name"] + assert "needs.*.result" in notify["with"]["status"] + assert notify["secrets"] == "inherit" + + def test_json_file_contains_no_hidden_route_expansion(self): + raw = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + assert set(raw) == { + "spa_marker", + "environments", + "routes", + "standalone_endpoints", + } + assert ( + len(raw["environments"]) * len(raw["routes"]) + == EXPECTED_CONSOLE_ENDPOINT_COUNT + ) + assert ( + tuple( + ( + item["name"], + item["base_url"], + item["route"], + item["body_marker"], + item["marker_label"], + ) + for item in raw["standalone_endpoints"] + ) + == EXPECTED_STANDALONE_ENDPOINTS + )