diff --git a/docs/mcp.md b/docs/mcp.md index 933b6ab..eed0e3e 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -17,8 +17,9 @@ MCP support is an optional extra: pip install kci-dev[mcp] ``` -Read-only dashboard query tools (trees, builds, boots, tests, hardware, -known issues) are always available and need no configuration. Maestro +Read-only dashboard query tools (trees, builds, boots, tests, logs, +hardware, known issues) are always available and need no configuration. +Maestro node lookup tools are enabled when the configured instance has an `api` URL, and job retry/checkout trigger tools when it also has a `pipeline` URL and a `token`. See the [config file](../config_file.md) documentation. diff --git a/docs/results.md b/docs/results.md index f399348..caf5404 100644 --- a/docs/results.md +++ b/docs/results.md @@ -7,6 +7,7 @@ description = 'Fetch results from the KernelCI ecosystem.' ## Regression comparison and CI gates ```shell +kci-dev results compare --giturl URL --branch BRANCH --format json kci-dev results compare --giturl URL --branch BRANCH --format json BASE HEAD kci-dev results gate --giturl URL --branch BRANCH --base BASE --head HEAD \ --fail-on regression --format json @@ -138,7 +139,12 @@ Compare test results between commits with summary statistics and regressions. This command compares test results between commits showing summary statistics for both commits and identifying tests that transitioned from PASS to FAIL status. This helps identify genuine regressions while distinguishing them from boot-related infrastructure issues. -By default, it compares the latest two commits from history. You can also specify two specific commit hashes to compare. +With no positional commits, it fetches checkout history and compares the +previous checkout (BASE, history index 1) with the latest checkout (HEAD, +history index 0). You can instead specify exactly two commit hashes as +`BASE HEAD`; one commit or more than two commits is invalid. The existing +`--latest` option remains accepted for compatibility and selects the same +latest-two behavior when no commits are given. Example: diff --git a/kcidev/api.py b/kcidev/api.py index d7d6192..7b1f620 100644 --- a/kcidev/api.py +++ b/kcidev/api.py @@ -8,11 +8,18 @@ without invoking Click commands or shelling out to ``kci-dev``. """ +import ipaddress +import socket +import zlib from datetime import datetime, timezone +from time import monotonic +from urllib.parse import urljoin, urlparse import click +import requests from click.testing import CliRunner +from kcidev.libs.common import kcidev_session from kcidev.libs.dashboard import ( dashboard_api_url, dashboard_fetch_boot_issues, @@ -102,6 +109,108 @@ def _as_library_error(action, func, *args, **kwargs): raise KciDevError(action) from exc +NOTHING_RELATED_MARKERS = ( + "No issues found", + "No issues were found", + "No tests found", + "No builds found", +) + +MAX_LOG_BYTES = 1 << 20 +LOG_SCAN_LIMIT = 64 << 20 +LOG_DEADLINE_SECONDS = 60 +_LOG_CHUNK = 1 << 16 +_MAX_LOG_REDIRECTS = 5 + + +def _require_public_url(url): + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + raise KciDevError(f"Refusing to fetch non-http(s) log URL: {url}") + host = parsed.hostname + if not host: + raise KciDevError(f"Log URL has no host: {url}") + default_port = 443 if parsed.scheme == "https" else 80 + try: + port = parsed.port or default_port + except ValueError as exc: + raise KciDevError(f"Invalid port in log URL {url}: {exc}") from exc + try: + infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) + except (OSError, UnicodeError) as exc: + raise KciDevError(f"Could not resolve log host {host}: {exc}") from exc + for info in infos: + ip = ipaddress.ip_address(info[4][0]) + if not ip.is_global: + raise KciDevError( + f"Refusing to fetch log from non-public address {ip} ({host})" + ) + + +def _stream_public_get(url): + for _ in range(_MAX_LOG_REDIRECTS + 1): + _require_public_url(url) + response = kcidev_session.get( + url, stream=True, timeout=30, allow_redirects=False + ) + if 300 <= response.status_code < 400: + location = response.headers.get("Location") + response.close() + if not location: + raise KciDevError(f"Redirect without Location header: {url}") + url = urljoin(url, location) + continue + return response + raise KciDevError("Too many redirects while fetching log") + + +def _gunzip_iter(raw_iter): + decomp = zlib.decompressobj(zlib.MAX_WBITS | 16) + carry = b"" + trailing_garbage = False + for chunk in raw_iter: + to_feed = carry + chunk + carry = b"" + while to_feed: + if decomp.eof: + if to_feed[:2] == b"\x1f\x8b": + decomp = zlib.decompressobj(zlib.MAX_WBITS | 16) + elif len(to_feed) < 2: + carry = to_feed + break + else: + trailing_garbage = True + break + yield decomp.decompress(to_feed, _LOG_CHUNK) + to_feed = decomp.unused_data if decomp.eof else decomp.unconsumed_tail + if trailing_garbage: + break + if trailing_garbage: + return + rest = decomp.flush() + if rest: + yield rest + if not decomp.eof: + raise KciDevError("Log gzip stream is incomplete or malformed") + + +def _pick_log_url(test): + if not isinstance(test, dict): + return None + if test.get("log_url"): + return test["log_url"] + logs = [ + f + for f in (test.get("output_files") or []) + if isinstance(f, dict) + and f.get("url") + and isinstance(f.get("name"), str) + and "log" in f["name"].lower() + ] + logs.sort(key=lambda f: ("stderr" in f["name"].lower(), f["name"].lower())) + return logs[0]["url"] if logs else None + + class KernelCIClient: """Client for interacting with KernelCI services from Python code. @@ -367,6 +476,116 @@ def get_test(self, test_id): "Dashboard test request failed", dashboard_fetch_test, test_id, True ) + def get_log(self, test_id, max_bytes=16384, tail=True): + """Fetch the raw log for a test, decompressing gzip, size-bounded. + + Resolves the test's log URL (``log_url`` or, when that is empty, a + log entry from ``output_files``), downloads it with a bounded head or + tail buffer, decompressing gzip (including multi-member streams) + incrementally, and returns the text with the decompressed + ``total_bytes`` and a ``truncated`` flag. At most ``max_bytes`` bytes + are returned (capped at ``MAX_LOG_BYTES``), taken from the end when + ``tail`` is true (where failures usually are) or the start otherwise. + Reading stops once ``LOG_SCAN_LIMIT`` compressed or decompressed + bytes are seen, to bound memory and download against oversized or + malicious logs; ``scan_limited`` is then set and ``total_bytes`` is a + floor rather than the exact size. Reading also stops after + ``LOG_DEADLINE_SECONDS``, since the request timeout is per read + rather than total and a slow server would otherwise hold the + caller indefinitely; ``deadline_exceeded`` is then set and + ``total_bytes`` is likewise a floor. The log URL is validated (scheme + and resolved address) before fetching, though a DNS rebind between + that check and the request remains a residual gap. + """ + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int): + raise KciDevError("max_bytes must be a positive integer") + if max_bytes <= 0: + raise KciDevError("max_bytes must be a positive integer") + max_bytes = min(max_bytes, MAX_LOG_BYTES) + + test = self.get_test(test_id) + log_url = _pick_log_url(test) + if not log_url: + raise KciDevError(f"No log available for test {test_id}") + + buf = bytearray() + total = 0 + raw_total = 0 + scan_limited = False + deadline_exceeded = False + deadline = monotonic() + LOG_DEADLINE_SECONDS + response = None + try: + response = _stream_public_get(log_url) + response.raise_for_status() + chunks = response.iter_content(_LOG_CHUNK) + + prefix = b"" + for chunk in chunks: + if not chunk: + continue + prefix += chunk + if len(prefix) >= 2: + break + + def raw_iter(): + nonlocal raw_total, scan_limited, deadline_exceeded + if prefix: + raw_total += len(prefix) + yield prefix + for chunk in chunks: + if not chunk: + continue + if monotonic() > deadline: + deadline_exceeded = True + return + raw_total += len(chunk) + if raw_total > LOG_SCAN_LIMIT: + scan_limited = True + return + yield chunk + + source = ( + _gunzip_iter(raw_iter()) if prefix[:2] == b"\x1f\x8b" else raw_iter() + ) + try: + for out in source: + if not out: + continue + total += len(out) + if tail: + buf += out + if len(buf) > max_bytes: + del buf[:-max_bytes] + elif len(buf) < max_bytes: + buf += out[: max_bytes - len(buf)] + if total >= LOG_SCAN_LIMIT: + scan_limited = True + break + except KciDevError: + if not (scan_limited or deadline_exceeded): + raise + except KciDevError: + raise + except (requests.exceptions.RequestException, zlib.error, OSError) as exc: + raise KciDevError(f"Log download failed for test {test_id}: {exc}") from exc + finally: + if response is not None: + response.close() + + returned = bytes(buf) + return { + "test_id": test_id, + "log_url": log_url, + "total_bytes": total, + "returned_bytes": len(returned), + "truncated": scan_limited or deadline_exceeded or total > len(returned), + "scan_limited": scan_limited, + "deadline_exceeded": deadline_exceeded, + "tail": tail, + "text": returned.decode("utf-8", errors="replace"), + } + def get_tree_list(self, origin, days=7): return self._dashboard_request( "Dashboard tree list request failed", @@ -420,8 +639,23 @@ def get_hardware_tests(self, name, origin): True, ) + def _related_or_empty(self, action, func, *args): + """Fetch one artifact's related list, treating "none" as empty. + + The dashboard reports an artifact with nothing related to it as + an error rather than an empty list. A caller asking what an + issue affects, or whether a failure is already known, should + read that as a clean answer rather than a failed call. + """ + try: + return self._dashboard_request(action, func, *args) + except KciDevError as exc: + if any(marker in str(exc) for marker in NOTHING_RELATED_MARKERS): + return [] + raise + def get_build_issues(self, build_id, error_verbose=True): - return self._dashboard_request( + return self._related_or_empty( "Dashboard build issues request failed", dashboard_fetch_build_issues, build_id, @@ -430,7 +664,7 @@ def get_build_issues(self, build_id, error_verbose=True): ) def get_boot_issues(self, test_id, error_verbose=True): - return self._dashboard_request( + return self._related_or_empty( "Dashboard boot issues request failed", dashboard_fetch_boot_issues, test_id, @@ -453,21 +687,23 @@ def get_issue(self, issue_id): ) def get_issue_builds(self, issue_id, origin=None): - return self._dashboard_request( + return self._related_or_empty( "Dashboard issue builds request failed", dashboard_fetch_issue_builds, origin, issue_id, True, + False, ) def get_issue_tests(self, issue_id, origin=None): - return self._dashboard_request( + return self._related_or_empty( "Dashboard issue tests request failed", dashboard_fetch_issue_tests, origin, issue_id, True, + False, ) def get_issues_extra(self, issues): diff --git a/kcidev/libs/dashboard.py b/kcidev/libs/dashboard.py index c264c3a..4859590 100644 --- a/kcidev/libs/dashboard.py +++ b/kcidev/libs/dashboard.py @@ -112,13 +112,14 @@ def wrapper( logging.debug(f"Response data size: {len(json.dumps(data))} bytes") if "error" in data: + explained = _explain_dashboard_error(str(data.get("error"))) if error_verbose: logging.error(f"API returned error: {data.get('error')}") if use_json: kci_msg(data) else: - kci_msg("json error: " + str(data["error"])) - raise click.ClickException(data.get("error")) + kci_msg("API error: " + explained) + raise click.ClickException(explained) logging.info(f"Successfully completed {func.__name__} request") return data @@ -140,6 +141,30 @@ def dashboard_api_post(endpoint, params, use_json, body, max_retries=3): return kcidev_session.post(endpoint, json=body, timeout=HTTP_TIMEOUT) +_NOT_FOUND_HINTS = ( + ( + "No results available for this tree/branch/commit", + "The dashboard has no checkout recorded for that tree, branch and " + "commit together. List the ones it knows with 'kci-dev results " + "trees' (or the list_trees tool) and use a commit from there.", + ), + ( + "Tree checkout not found", + "The dashboard has no checkout recorded for that tree, branch and " + "commit together. List the ones it knows with 'kci-dev results " + "trees' (or the list_trees tool) and use a commit from there.", + ), +) + + +def _explain_dashboard_error(message): + """Append guidance to the dashboard errors that mean "look elsewhere".""" + for marker, hint in _NOT_FOUND_HINTS: + if marker in message: + return f"{message}. {hint}" + return message + + @_dashboard_request def dashboard_api_fetch(endpoint, params, use_json, max_retries=3, error_verbose=True): return kcidev_session.get(endpoint, timeout=HTTP_TIMEOUT) diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py index f55e0b2..ca8f5a7 100644 --- a/kcidev/libs/maestro_common.py +++ b/kcidev/libs/maestro_common.py @@ -34,21 +34,32 @@ def maestro_print_api_call(host, data=None): kci_info(json.dumps(data, indent=4)) -def maestro_api_error(response): +class MaestroApiError(click.ClickException): + """A Maestro API call failed, carrying the detail the API reported.""" + + exit_code = errno.ENOENT + + +def maestro_api_error_message(response): logging.error( f"Maestro API error - Status: {response.status_code}, URL: {response.url}" ) - kci_err(f"API response error code: {response.status_code}") + detail = f"API response error code: {response.status_code}" try: error_data = response.json() logging.error(f"API error response: {json.dumps(error_data, indent=2)}") - kci_err(error_data) + return f"{detail}: {error_data}" except (json.decoder.JSONDecodeError, requests.exceptions.JSONDecodeError): logging.warning(f"No JSON in error response: {response.text}") - kci_warning(f"No JSON response. Plain text: {response.text}") + return f"{detail}. Plain text: {response.text}" except Exception as e: logging.error(f"Error parsing API response: {e}") - kci_err(f"API response error: {e}: {response.text}") + return f"{detail}: {e}: {response.text}" + + +def maestro_api_error(response): + message = maestro_api_error_message(response) + kci_err(message) return @@ -82,12 +93,10 @@ def maestro_get_node(url, nodeid): response.raise_for_status() except requests.exceptions.HTTPError as ex: logging.error(f"HTTP error fetching node {nodeid}: {ex}") - maestro_api_error(ex.response) - sys.exit(errno.ENOENT) + raise MaestroApiError(maestro_api_error_message(ex.response)) from ex except Exception as ex: logging.error(f"Unexpected error fetching node {nodeid}: {ex}") - kci_err(ex) - sys.exit(errno.ENOENT) + raise MaestroApiError(str(ex)) from ex node_data = response.json() if node_data is None: @@ -122,12 +131,10 @@ def maestro_get_nodes(url, limit, offset, filter, paginate): response.raise_for_status() except requests.exceptions.HTTPError as ex: logging.error(f"HTTP error fetching nodes: {ex}") - maestro_api_error(ex.response) - sys.exit(errno.ENOENT) + raise MaestroApiError(maestro_api_error_message(ex.response)) from ex except Exception as ex: logging.error(f"Unexpected error fetching nodes: {ex}") - kci_err(ex) - sys.exit(errno.ENOENT) + raise MaestroApiError(str(ex)) from ex nodes_data = response.json() logging.info(f"Retrieved {len(nodes_data)} nodes") diff --git a/kcidev/mcp/tools_dashboard.py b/kcidev/mcp/tools_dashboard.py index 44fa05f..07a3259 100644 --- a/kcidev/mcp/tools_dashboard.py +++ b/kcidev/mcp/tools_dashboard.py @@ -236,6 +236,46 @@ def get_test(test_id: str): return _current_client().get_test(test_id) +@tool_errors +def get_log(test_id: str, max_bytes: int = 16384, tail: bool = True): + """Fetch the raw log for a test or job by dashboard test id. + + Downloads and decompresses the log, resolving it from the test's + log_url or, when that is empty (common for failures), a log entry in + output_files. Returns it size-bounded: by default the last max_bytes, + where failures usually are (set tail=false for the start). The + response reports total_bytes and truncated so you can widen max_bytes + if needed, up to a 1 MiB ceiling: asking for more returns that + ceiling rather than the whole log, so compare returned_bytes with + total_bytes rather than retrying the same call. A download that runs + past about a minute stops early and sets deadline_exceeded. Use + get_test first for the shorter log_excerpt. + """ + return _current_client().get_log(test_id, max_bytes=max_bytes, tail=tail) + + +@tool_errors +def get_test_issues(test_id: str): + """List known issues detected on a specific test or boot. + + Use this to check a failing test against issues KernelCI already + tracks before treating the failure as new. Test ids look like + 'maestro:'. + """ + return _current_client().get_boot_issues(test_id) + + +@tool_errors +def get_build_issues(build_id: str): + """List known issues detected on a specific build. + + Use this to check a failing build against issues KernelCI already + tracks before treating the failure as new. Build ids look like + 'maestro:'. + """ + return _current_client().get_build_issues(build_id) + + @tool_errors def list_hardware(origin: str = "maestro"): """List hardware platforms with results over the last 7 days. @@ -284,6 +324,9 @@ def get_issue_builds( ): """List builds affected by a known issue. + An empty list means the issue has no builds recorded against it, and + also what an unknown issue id returns, since the dashboard reports + both the same way; confirm the id with get_issue if it matters. Optional status filter ('pass', 'fail' or 'inconclusive') and limit/offset pagination; the response carries 'total' and 'matched' counts; fields projects each entry to only those keys. @@ -303,6 +346,9 @@ def get_issue_tests( ): """List tests affected by a known issue. + An empty list means the issue has no tests recorded against it, and + also what an unknown issue id returns, since the dashboard reports + both the same way; confirm the id with get_issue if it matters. Optional status filter ('pass', 'fail' or 'inconclusive') and limit/offset pagination; the response carries 'total' and 'matched' counts; fields projects each entry to only those keys. @@ -321,6 +367,9 @@ def get_issue_tests( list_tests, get_build, get_test, + get_log, + get_test_issues, + get_build_issues, list_hardware, get_hardware_summary, list_issues, diff --git a/kcidev/subcommands/results/__init__.py b/kcidev/subcommands/results/__init__.py index 70fdb4f..3ed96d0 100644 --- a/kcidev/subcommands/results/__init__.py +++ b/kcidev/subcommands/results/__init__.py @@ -50,6 +50,25 @@ ) +def _resolve_latest_two(client, origin, giturl, branch): + """Return the previous and latest checkout from Dashboard history.""" + from kcidev.api import KciDevError + + giturl, branch, latest = set_giturl_branch_commit( + origin, giturl, branch, None, True, None + ) + history = client.get_commits_history(origin, giturl, branch, latest) + commits = history if isinstance(history, list) else history.get("commits", []) + if len(commits) < 2: + raise KciDevError("fewer than two checkouts are available") + return ( + giturl, + branch, + commits[1]["git_commit_hash"], + commits[0]["git_commit_hash"], + ) + + @click.group( help="""Query and display test results from the KernelCI dashboard. @@ -405,32 +424,42 @@ def compare( This helps identify genuine regressions while distinguishing them from boot-related infrastructure issues. - By default, compares the latest two commits from history. You can also - specify two specific commit hashes to compare. + With no COMMITS, compares history index 1 (BASE) with index 0 (HEAD). + Alternatively, provide exactly two commit hashes as BASE HEAD. + --latest remains accepted for compatibility with latest-two comparisons. \b Examples: # Compare latest two commits - kci-dev results compare --giturl https://git.kernel.org/... + kci-dev results compare --giturl https://git.kernel.org/... --branch main # Compare specific commits - kci-dev results compare --giturl https://git.kernel.org/... abc123 def456 + kci-dev results compare --giturl https://git.kernel.org/... --branch main abc123 def456 """ from kcidev.api import KciDevError, KernelCIClient json_output = use_json or output_format == "json" - if len(commits) != 2: - raise click.UsageError("exactly BASE and HEAD commits are required") + if len(commits) not in (0, 2): + raise click.UsageError( + "provide either zero commits or exactly BASE and HEAD commits" + ) try: - report = KernelCIClient().compare_results( - commits[0], - commits[1], + client = KernelCIClient() + if commits: + base, head = commits + else: + giturl, branch, base, head = _resolve_latest_two( + client, origin, giturl, branch + ) + report = client.compare_results( + base, + head, giturl, branch, origin, include_issues=include_issues, ) - except KciDevError as exc: + except (KciDevError, click.Abort) as exc: if json_output: click.echo(json.dumps({"error": str(exc), "incomplete": True})) raise click.exceptions.Exit(2) from exc @@ -439,7 +468,7 @@ def compare( click.echo(json.dumps(report, sort_keys=True)) else: - click.echo(f"Compared {commits[0]} -> {commits[1]}") + click.echo(f"Compared {base} -> {head}") for category, count in report["counts"].items(): click.echo(f" {category}: {count}") if report["incomplete"]: @@ -478,16 +507,9 @@ def gate(origin, giturl, branch, base, head, fail_on, output_format): if bool(base) != bool(head): raise click.UsageError("provide both --base and --head, or neither") if not base: - giturl, branch, latest = set_giturl_branch_commit( - origin, giturl, branch, None, True, None - ) - history = client.get_commits_history(origin, giturl, branch, latest) - commits = ( - history if isinstance(history, list) else history.get("commits", []) + giturl, branch, base, head = _resolve_latest_two( + client, origin, giturl, branch ) - if len(commits) < 2: - raise KciDevError("fewer than two checkouts are available") - head, base = commits[0]["git_commit_hash"], commits[1]["git_commit_hash"] report = client.compare_results(base, head, giturl, branch, origin) except (KciDevError, click.Abort) as exc: if output_format == "json": diff --git a/tests/test_api.py b/tests/test_api.py index 000451e..87940cb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,7 +4,7 @@ import pytest import requests -from kcidev import KciDevError, KernelCIClient +from kcidev import KciDevError, KernelCIClient, api from kcidev.libs import maestro_common CFG = { @@ -301,3 +301,443 @@ def test_compare_results_only_fetches_issues_when_requested(monkeypatch): assert report["items"][0]["known_issues"] == ["issue-1"] get_issues.assert_called_once_with("head-test", error_verbose=False) + + +class _FakeStream: + def __init__(self, chunks, status_code=200, headers=None, raise_after=None): + self._chunks = list(chunks) + self.status_code = status_code + self.headers = headers or {} + self.closed = False + self._raise_after = raise_after + + def raise_for_status(self): + pass + + def iter_content(self, size): + for i, chunk in enumerate(self._chunks): + if self._raise_after is not None and i == self._raise_after: + raise requests.exceptions.ChunkedEncodingError("conn reset") + yield chunk + + def close(self): + self.closed = True + + +def _log_test(url="https://files.kernelci.org/x.log", output_files=None): + return {"log_url": url, "output_files": output_files or []} + + +def _mock_stream(monkeypatch, chunks): + monkeypatch.setattr(api, "_stream_public_get", lambda url: _FakeStream(chunks)) + + +def _addrinfo(ip, port=443): + return [(2, 1, 6, "", (ip, port))] + + +def test_get_log_decompresses_gzip(monkeypatch): + import gzip + + monkeypatch.setattr( + KernelCIClient, "get_test", lambda self, tid: _log_test("https://f/log.gz") + ) + _mock_stream(monkeypatch, [gzip.compress(b"boot ok\nTEST FAIL: oops\n")]) + out = _client().get_log("maestro:abc") + assert out["truncated"] is False + assert "TEST FAIL: oops" in out["text"] + assert out["total_bytes"] == len(b"boot ok\nTEST FAIL: oops\n") + + +def test_get_log_tail_truncates(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"A" * 100 + b"TAILEND"]) + out = _client().get_log("t", max_bytes=7, tail=True) + assert out["truncated"] is True + assert out["text"] == "TAILEND" + assert out["returned_bytes"] == 7 + + +def test_get_log_head_truncates(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"HEADSTART" + b"Z" * 100]) + out = _client().get_log("t", max_bytes=9, tail=False) + assert out["text"] == "HEADSTART" + assert out["truncated"] is True + + +def test_get_log_tail_spans_chunk_boundaries(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"1234567890", b"abcde", b"XYZ"]) + out = _client().get_log("t", max_bytes=5, tail=True) + assert out["text"] == "deXYZ" + assert out["total_bytes"] == 18 + + +def test_get_log_no_url_raises(monkeypatch): + monkeypatch.setattr( + KernelCIClient, + "get_test", + lambda self, tid: {"log_url": None, "output_files": []}, + ) + with pytest.raises(KciDevError, match="No log available"): + _client().get_log("t") + + +def test_get_log_falls_back_to_output_files(monkeypatch): + test = { + "log_url": None, + "output_files": [ + {"name": "build_kselftest_stderr_log", "url": "https://f/stderr.log.gz"}, + {"name": "test_log", "url": "https://f/test.log.gz"}, + {"name": "job_definition", "url": "https://f/def.json"}, + ], + } + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: test) + _mock_stream(monkeypatch, [b"from output_files"]) + out = _client().get_log("t") + assert out["log_url"] == "https://f/test.log.gz" + assert out["text"] == "from output_files" + + +@pytest.mark.parametrize("bad", [0, -1, -1000, 1.5, True, "100"]) +def test_get_log_rejects_bad_max_bytes(bad): + with pytest.raises(KciDevError, match="positive integer"): + _client().get_log("t", max_bytes=bad) + + +def test_get_log_clamps_oversized_max_bytes(monkeypatch): + monkeypatch.setattr(api, "MAX_LOG_BYTES", 10) + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"B" * 50]) + out = _client().get_log("t", max_bytes=10_000, tail=False) + assert out["returned_bytes"] == 10 + assert out["truncated"] is True + + +def test_get_log_scan_limit_bounds_download(monkeypatch): + monkeypatch.setattr(api, "LOG_SCAN_LIMIT", 20) + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"C" * 15, b"D" * 15, b"E" * 15]) + out = _client().get_log("t", max_bytes=100, tail=False) + assert out["scan_limited"] is True + assert out["truncated"] is True + assert out["total_bytes"] == 15 + + +def test_get_log_truncated_gzip_raises(monkeypatch): + import gzip + + good = gzip.compress(b"hello world" * 50) + monkeypatch.setattr( + KernelCIClient, "get_test", lambda self, tid: _log_test("https://f/x.gz") + ) + _mock_stream(monkeypatch, [good[:20]]) + with pytest.raises(KciDevError, match="incomplete or malformed"): + _client().get_log("t") + + +def test_get_log_corrupt_gzip_raises(monkeypatch): + import gzip + + good = gzip.compress(b"hello world" * 50) + monkeypatch.setattr( + KernelCIClient, "get_test", lambda self, tid: _log_test("https://f/x.gz") + ) + _mock_stream(monkeypatch, [good[:10] + b"\x00" * 40]) + with pytest.raises(KciDevError, match="Log download failed"): + _client().get_log("t") + + +def test_get_log_download_failure_raises(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + + def boom(url): + raise requests.exceptions.ConnectionError("no route") + + monkeypatch.setattr(api, "_stream_public_get", boom) + with pytest.raises(KciDevError, match="Log download failed"): + _client().get_log("t") + + +def test_require_public_url_rejects_non_http(): + with pytest.raises(KciDevError, match="non-http"): + api._require_public_url("ftp://files.kernelci.org/x") + with pytest.raises(KciDevError, match="non-http"): + api._require_public_url("file:///etc/passwd") + + +@pytest.mark.parametrize( + "ip", ["127.0.0.1", "10.0.0.5", "192.168.1.1", "169.254.169.254", "::1"] +) +def test_require_public_url_rejects_private(monkeypatch, ip): + monkeypatch.setattr(api.socket, "getaddrinfo", lambda *a, **k: _addrinfo(ip)) + with pytest.raises(KciDevError, match="non-public"): + api._require_public_url("https://evil.example/x") + + +def test_require_public_url_allows_public(monkeypatch): + monkeypatch.setattr( + api.socket, "getaddrinfo", lambda *a, **k: _addrinfo("93.184.216.34") + ) + api._require_public_url("https://files.kernelci.org/x") + + +def test_require_public_url_unresolvable(monkeypatch): + def boom(*a, **k): + raise api.socket.gaierror("nope") + + monkeypatch.setattr(api.socket, "getaddrinfo", boom) + with pytest.raises(KciDevError, match="resolve"): + api._require_public_url("https://nope.invalid/x") + + +def test_stream_public_get_follows_validated_redirect(monkeypatch): + monkeypatch.setattr( + api.socket, "getaddrinfo", lambda *a, **k: _addrinfo("93.184.216.34") + ) + r1 = Mock(status_code=302, headers={"Location": "https://cdn.example/final"}) + r1.close = Mock() + r2 = _FakeStream([b"ok"]) + calls = [] + + def fake_get(url, **k): + calls.append(url) + return r1 if len(calls) == 1 else r2 + + monkeypatch.setattr(api.kcidev_session, "get", fake_get) + assert api._stream_public_get("https://files.kernelci.org/x") is r2 + assert calls == ["https://files.kernelci.org/x", "https://cdn.example/final"] + + +def test_stream_public_get_rejects_redirect_to_private(monkeypatch): + def ai(host, *a, **k): + good = host == "files.kernelci.org" + return _addrinfo("93.184.216.34" if good else "169.254.169.254") + + monkeypatch.setattr(api.socket, "getaddrinfo", ai) + r1 = Mock( + status_code=302, headers={"Location": "http://169.254.169.254/latest/meta"} + ) + r1.close = Mock() + monkeypatch.setattr(api.kcidev_session, "get", Mock(return_value=r1)) + with pytest.raises(KciDevError, match="non-public"): + api._stream_public_get("https://files.kernelci.org/x") + + +def test_stream_public_get_too_many_redirects(monkeypatch): + monkeypatch.setattr( + api.socket, "getaddrinfo", lambda *a, **k: _addrinfo("93.184.216.34") + ) + rr = Mock(status_code=302, headers={"Location": "https://a.example/loop"}) + rr.close = Mock() + monkeypatch.setattr(api.kcidev_session, "get", Mock(return_value=rr)) + with pytest.raises(KciDevError, match="Too many redirects"): + api._stream_public_get("https://a.example/loop") + + +def test_gunzip_iter_roundtrip(): + import gzip + + assert b"".join(api._gunzip_iter([gzip.compress(b"hello")])) == b"hello" + + +def test_gunzip_iter_truncated_raises(): + import gzip + + good = gzip.compress(b"data" * 100) + with pytest.raises(KciDevError, match="incomplete or malformed"): + list(api._gunzip_iter([good[:15]])) + + +def test_gunzip_iter_multi_member(): + import gzip + + stream = gzip.compress(b"first\n") + gzip.compress(b"SECOND\n") + assert b"".join(api._gunzip_iter([stream])) == b"first\nSECOND\n" + + +def test_gunzip_iter_multi_member_split_and_boundary(): + import gzip + + stream = gzip.compress(b"AAA") + gzip.compress(b"BBB") + split = [stream[:4], stream[4:]] + assert b"".join(api._gunzip_iter(split)) == b"AAABBB" + boundary = [gzip.compress(b"AAA"), gzip.compress(b"BBB")] + assert b"".join(api._gunzip_iter(boundary)) == b"AAABBB" + + +def test_gunzip_iter_tolerates_trailing_garbage(): + import gzip + + assert b"".join(api._gunzip_iter([gzip.compress(b"log") + b"junk"])) == b"log" + + +def test_get_log_reads_multi_member_gzip(monkeypatch): + import gzip + + stream = gzip.compress(b"member one\n") + gzip.compress(b"MEMBER TWO FAIL\n") + monkeypatch.setattr( + KernelCIClient, "get_test", lambda self, tid: _log_test("https://f/x.gz") + ) + _mock_stream(monkeypatch, [stream]) + out = _client().get_log("t") + assert "MEMBER TWO FAIL" in out["text"] + assert out["truncated"] is False + assert out["total_bytes"] == len(b"member one\nMEMBER TWO FAIL\n") + + +def test_get_log_closes_response_on_success(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + stream = _FakeStream([b"hello"]) + monkeypatch.setattr(api, "_stream_public_get", lambda url: stream) + _client().get_log("t") + assert stream.closed is True + + +def test_get_log_closes_response_on_gzip_error(monkeypatch): + import gzip + + monkeypatch.setattr( + KernelCIClient, "get_test", lambda self, tid: _log_test("https://f/x.gz") + ) + stream = _FakeStream([gzip.compress(b"data" * 50)[:20]]) + monkeypatch.setattr(api, "_stream_public_get", lambda url: stream) + with pytest.raises(KciDevError): + _client().get_log("t") + assert stream.closed is True + + +def test_get_log_closes_response_on_stream_error(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + stream = _FakeStream([b"a", b"b"], raise_after=1) + monkeypatch.setattr(api, "_stream_public_get", lambda url: stream) + with pytest.raises(KciDevError, match="Log download failed"): + _client().get_log("t") + assert stream.closed is True + + +def test_get_log_raw_scan_limit_bounds_compressed(monkeypatch): + import gzip + + monkeypatch.setattr(api, "LOG_SCAN_LIMIT", 15) + monkeypatch.setattr( + KernelCIClient, "get_test", lambda self, tid: _log_test("https://f/x.gz") + ) + gz = gzip.compress(b"hi there friend") + _mock_stream(monkeypatch, [gz[:2], gz[2:10], gz[10:20], gz[20:]]) + out = _client().get_log("t") + assert out["scan_limited"] is True + assert out["truncated"] is True + + +def test_get_log_output_files_url_goes_through_guard(monkeypatch): + test = { + "log_url": None, + "output_files": [{"name": "test_log", "url": "https://internal.evil/x.log"}], + } + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: test) + monkeypatch.setattr( + api.socket, "getaddrinfo", lambda *a, **k: _addrinfo("169.254.169.254") + ) + with pytest.raises(KciDevError, match="non-public"): + _client().get_log("t") + + +def test_get_log_ignores_non_string_output_file_name(monkeypatch): + test = { + "log_url": None, + "output_files": [ + {"name": 7, "url": "https://f/weird"}, + {"name": "test_log", "url": "https://f/ok.log"}, + ], + } + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: test) + _mock_stream(monkeypatch, [b"ok"]) + out = _client().get_log("t") + assert out["log_url"] == "https://f/ok.log" + + +def test_require_public_url_rejects_bad_port(): + with pytest.raises(KciDevError, match="[Pp]ort"): + api._require_public_url("https://files.kernelci.org:99999/x") + + +@pytest.mark.parametrize( + "ip", ["::ffff:127.0.0.1", "::ffff:169.254.169.254", "0.0.0.0"] +) +def test_require_public_url_rejects_mapped_and_unspecified(monkeypatch, ip): + monkeypatch.setattr(api.socket, "getaddrinfo", lambda *a, **k: _addrinfo(ip)) + with pytest.raises(KciDevError, match="non-public"): + api._require_public_url("https://evil.example/x") + + +def test_require_public_url_idna_error_is_clean(monkeypatch): + def boom(*a, **k): + raise UnicodeError("label too long") + + monkeypatch.setattr(api.socket, "getaddrinfo", boom) + with pytest.raises(KciDevError, match="resolve"): + api._require_public_url("https://" + "a" * 70 + ".example/x") + + +def test_stream_public_get_redirect_without_location(monkeypatch): + monkeypatch.setattr( + api.socket, "getaddrinfo", lambda *a, **k: _addrinfo("93.184.216.34") + ) + r = Mock(status_code=302, headers={}) + r.close = Mock() + monkeypatch.setattr(api.kcidev_session, "get", Mock(return_value=r)) + with pytest.raises(KciDevError, match="without Location"): + api._stream_public_get("https://files.kernelci.org/x") + + +def test_stream_public_get_rejects_redirect_to_file_scheme(monkeypatch): + monkeypatch.setattr( + api.socket, "getaddrinfo", lambda *a, **k: _addrinfo("93.184.216.34") + ) + r = Mock(status_code=302, headers={"Location": "file:///etc/passwd"}) + r.close = Mock() + monkeypatch.setattr(api.kcidev_session, "get", Mock(return_value=r)) + with pytest.raises(KciDevError, match="non-http"): + api._stream_public_get("https://files.kernelci.org/x") + + +def test_get_log_stops_at_the_total_deadline(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"X" * 1000 for _ in range(50)]) + ticks = iter([0.0] + [api.LOG_DEADLINE_SECONDS + 1] * 200) + monkeypatch.setattr(api, "monotonic", lambda: next(ticks)) + + out = _client().get_log("t", max_bytes=100000) + + assert out["deadline_exceeded"] is True + assert out["truncated"] is True + assert out["total_bytes"] < 50 * 1000 + + +def test_get_log_normal_download_is_not_deadline_limited(monkeypatch): + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, [b"Y" * 100]) + + out = _client().get_log("t", max_bytes=100000) + + assert out["deadline_exceeded"] is False + assert out["truncated"] is False + + +def test_get_log_deadline_returns_partial_gzip(monkeypatch): + import gzip + + raw = bytes(range(256)) * 40 + payload = gzip.compress(raw) + chunks = [payload[i : i + 64] for i in range(0, len(payload), 64)] + monkeypatch.setattr(KernelCIClient, "get_test", lambda self, tid: _log_test()) + _mock_stream(monkeypatch, chunks) + ticks = iter([0.0] + [api.LOG_DEADLINE_SECONDS + 1] * 500) + monkeypatch.setattr(api, "monotonic", lambda: next(ticks)) + + out = _client().get_log("t", max_bytes=100000) + + assert out["deadline_exceeded"] is True + assert out["truncated"] is True diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 5ecd24e..e842813 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -1,5 +1,7 @@ from unittest.mock import Mock +import click +import pytest from click.testing import CliRunner from kcidev.libs import dashboard @@ -170,3 +172,29 @@ def test_kernelci_clients_keep_independent_dashboard_endpoints(monkeypatch): assert urls[1].startswith(dashboard.DASHBOARD_API_DEFAULT) assert urls[2].startswith("https://three.example/api/") assert urls[3].startswith("https://one.example/api/") + + +def _error_response(monkeypatch, message): + response = Mock(status_code=200) + response.json.return_value = {"error": message} + monkeypatch.setattr(dashboard.kcidev_session, "get", Mock(return_value=response)) + + +def test_unknown_checkout_error_says_how_to_find_a_valid_one(monkeypatch): + _error_response(monkeypatch, "No results available for this tree/branch/commit") + + with pytest.raises(click.ClickException) as excinfo: + dashboard.dashboard_api_fetch("tree/deadbeef/tests", {}, False) + + message = excinfo.value.message + assert "No results available" in message + assert "results trees" in message + + +def test_other_dashboard_errors_are_passed_through_unchanged(monkeypatch): + _error_response(monkeypatch, "Build not found") + + with pytest.raises(click.ClickException) as excinfo: + dashboard.dashboard_api_fetch("build/x", {}, False) + + assert excinfo.value.message == "Build not found" diff --git a/tests/test_maestro_common.py b/tests/test_maestro_common.py index 88d2758..718fec6 100644 --- a/tests/test_maestro_common.py +++ b/tests/test_maestro_common.py @@ -124,10 +124,11 @@ def test_maestro_get_node_plain_text_http_error_is_clean(monkeypatch): maestro_common.kcidev_session, "get", Mock(return_value=response) ) - with pytest.raises(SystemExit) as exc_info: + with pytest.raises(maestro_common.MaestroApiError) as exc_info: maestro_common.maestro_get_node("https://api.example.org", "n1") - assert exc_info.value.code == 2 + assert exc_info.value.exit_code == 2 + assert "upstream unavailable" in exc_info.value.message def test_maestro_print_nodes_emits_one_json_document(capsys): diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 0eb5034..0ed785b 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -207,3 +207,31 @@ def test_list_nodes_projects_fields(monkeypatch): assert result.isError is False assert '"commit_message"' not in result.content[0].text assert '"n1"' in result.content[0].text + + +def _http_error_response(monkeypatch, status, payload): + from kcidev.libs import maestro_common + + response = Mock(status_code=status, url="https://api.example.org/latest/node/x") + response.json.return_value = payload + response.raise_for_status.side_effect = requests.exceptions.HTTPError( + response=response + ) + monkeypatch.setattr( + maestro_common.kcidev_session, "get", Mock(return_value=response) + ) + return response + + +def test_get_node_http_error_keeps_api_detail(monkeypatch): + _http_error_response(monkeypatch, 404, {"detail": "Node not found"}) + result = _call_tool(create_server(CFG, "test"), "get_node", {"node_id": "0" * 24}) + assert result.isError is True + assert "404" in result.content[0].text + + +def test_list_nodes_http_error_keeps_api_detail(monkeypatch): + _http_error_response(monkeypatch, 422, {"detail": "bad filter"}) + result = _call_tool(create_server(CFG, "test"), "list_nodes", {}) + assert result.isError is True + assert "422" in result.content[0].text diff --git a/tests/test_mcp_tools_dashboard.py b/tests/test_mcp_tools_dashboard.py index c24d189..2b2e131 100644 --- a/tests/test_mcp_tools_dashboard.py +++ b/tests/test_mcp_tools_dashboard.py @@ -214,3 +214,71 @@ def test_get_summary_detail_returns_full_payload(monkeypatch): detail=True, ) assert result == SUMMARY_PAYLOAD + + +def test_get_test_issues_fetches_dashboard(monkeypatch): + get = _mock_get(monkeypatch, [{"id": "issue1"}]) + result = tools_dashboard.get_test_issues("maestro:t1") + assert result == [{"id": "issue1"}] + assert "test/maestro:t1/issues" in get.call_args[0][0] + + +def test_get_build_issues_fetches_dashboard(monkeypatch): + get = _mock_get(monkeypatch, [{"id": "issue2"}]) + result = tools_dashboard.get_build_issues("maestro:b1") + assert result == [{"id": "issue2"}] + assert "build/maestro:b1/issues" in get.call_args[0][0] + + +def test_get_log_returns_client_payload(monkeypatch): + from kcidev.api import KernelCIClient + + monkeypatch.setattr( + KernelCIClient, + "get_log", + lambda self, tid, max_bytes=16384, tail=True: { + "test_id": tid, + "truncated": False, + "text": "log body", + }, + ) + result = tools_dashboard.get_log("maestro:t1") + assert result["text"] == "log body" + assert result["test_id"] == "maestro:t1" + + +def test_get_test_issues_returns_empty_when_none_are_tracked(monkeypatch): + _mock_get(monkeypatch, {"error": "No issues were found for this test"}) + assert tools_dashboard.get_test_issues("maestro:t1") == [] + + +def test_get_build_issues_returns_empty_when_none_are_tracked(monkeypatch): + _mock_get(monkeypatch, {"error": "No issues found for this build"}) + assert tools_dashboard.get_build_issues("maestro:b1") == [] + + +def test_get_test_issues_still_reports_other_errors(monkeypatch): + _mock_get(monkeypatch, {"error": "Test not found"}) + with pytest.raises(ToolExecutionError): + tools_dashboard.get_test_issues("maestro:nope") + + +def test_get_issue_tests_returns_empty_when_none_are_tracked(monkeypatch): + get = _mock_get(monkeypatch, {"error": "No tests found for this issue"}) + result = tools_dashboard.get_issue_tests("maestro:i1") + assert result["tests"] == [] + assert result["matched"] == 0 + assert get.called + + +def test_get_issue_builds_returns_empty_when_none_are_tracked(monkeypatch): + _mock_get(monkeypatch, {"error": "No builds found for this issue"}) + result = tools_dashboard.get_issue_builds("maestro:i1") + assert result["builds"] == [] + assert result["matched"] == 0 + + +def test_get_issue_tests_still_reports_other_errors(monkeypatch): + _mock_get(monkeypatch, {"error": "Issue not found"}) + with pytest.raises(ToolExecutionError): + tools_dashboard.get_issue_tests("maestro:nope") diff --git a/tests/test_regression.py b/tests/test_regression.py index c67cfe2..78c4733 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,6 +1,7 @@ import json import click +import pytest from click.testing import CliRunner from kcidev.libs.regression import RegressionReport @@ -277,6 +278,149 @@ def test_compare_json_is_one_document_and_regressions_exit_one(monkeypatch): assert json.loads(result.stdout)["counts"]["regression"] == 1 +@pytest.mark.parametrize( + "history", + [ + [ + {"git_commit_hash": "head-from-history"}, + {"git_commit_hash": "base-from-history"}, + ], + { + "commits": [ + {"git_commit_hash": "head-from-history"}, + {"git_commit_hash": "base-from-history"}, + ] + }, + ], +) +def test_compare_without_commits_resolves_latest_pair(monkeypatch, history): + monkeypatch.setattr( + "kcidev.subcommands.results.set_giturl_branch_commit", + lambda *args, **kwargs: ("resolved-url", "resolved-branch", "latest"), + ) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.get_commits_history", + lambda *args, **kwargs: history, + ) + compared = {} + + def compare(_client, base, head, giturl, branch, origin, **kwargs): + compared["args"] = (base, head, giturl, branch, origin) + return _report() + + monkeypatch.setattr("kcidev.api.KernelCIClient.compare_results", compare) + + result = CliRunner().invoke( + get_cli(), + ["results", "compare", "--giturl", "url", "--branch", "main"], + ) + + assert result.exit_code == 0 + assert compared["args"] == ( + "base-from-history", + "head-from-history", + "resolved-url", + "resolved-branch", + "maestro", + ) + + +@pytest.mark.parametrize("commits", [["only"], ["one", "two", "three"]]) +def test_compare_rejects_invalid_positional_commit_counts(commits): + result = CliRunner().invoke( + get_cli(), + [ + "results", + "compare", + "--giturl", + "url", + "--branch", + "main", + *commits, + ], + ) + + assert result.exit_code == 2 + assert "provide either zero commits or exactly BASE and HEAD" in result.output + + +def test_compare_preserves_two_explicit_commits_without_history(monkeypatch): + def unexpected_history(*args, **kwargs): + raise AssertionError("explicit commits should not fetch history") + + monkeypatch.setattr( + "kcidev.api.KernelCIClient.get_commits_history", unexpected_history + ) + compared = {} + + def compare(_client, base, head, *args, **kwargs): + compared["commits"] = (base, head) + return _report() + + monkeypatch.setattr("kcidev.api.KernelCIClient.compare_results", compare) + + result = CliRunner().invoke( + get_cli(), + [ + "results", + "compare", + "--giturl", + "url", + "--branch", + "main", + "base", + "head", + ], + ) + + assert result.exit_code == 0 + assert compared["commits"] == ("base", "head") + + +@pytest.mark.parametrize( + ("history", "message"), + [ + ( + {"commits": [{"git_commit_hash": "only"}]}, + "fewer than two checkouts are available", + ), + (None, "history unavailable"), + ], +) +def test_compare_json_history_failure_is_one_document(monkeypatch, history, message): + monkeypatch.setattr( + "kcidev.subcommands.results.set_giturl_branch_commit", + lambda *args, **kwargs: ("url", "main", "latest"), + ) + + def get_history(*args, **kwargs): + if history is None: + from kcidev.api import KciDevError + + raise KciDevError(message) + return history + + monkeypatch.setattr("kcidev.api.KernelCIClient.get_commits_history", get_history) + + result = CliRunner().invoke( + get_cli(), + [ + "results", + "compare", + "--giturl", + "url", + "--branch", + "main", + "--format", + "json", + ], + ) + + assert result.exit_code == 2 + assert json.loads(result.stdout) == {"error": message, "incomplete": True} + assert result.stdout.count("\n") == 1 + + def test_gate_rejects_unknown_fail_on_categories_before_comparison(monkeypatch): def unexpected_comparison(*args, **kwargs): raise AssertionError("comparison should not run for an invalid policy")