From 159e0cfc924aceb66d3a4f3d4c90c1c98db08f4a Mon Sep 17 00:00:00 2001 From: Georgis Andonis Date: Wed, 16 Sep 2026 12:22:07 +0300 Subject: [PATCH 1/3] fix(mcp): decide tool ownership per request, not from the last listing Capture intent for tools the host owns; skip it only for PostHog's own. The code asked that question two different ways and they disagreed. Interception asked a per-request probe, which was right. Intent and the conversation exemption asked `virtual_tool_collisions`, which only a served tools/list writes. On a process that had served none -- the ordinary multi-pod case the probe exists for -- a host tool named `get_more_tools` dispatched correctly but its $mcp_tool_call carried `$mcp_intent=None`. Permanent on FastMCP and v2 MCPServer; on raw low-level it self-healed after one call, because the MCP SDK's own `req is None` cache pass happens to refresh the state during dispatch. The intent guard turns out to be dead code for its stated purpose: PostHog's virtual tools are intercepted before dispatch and captured by record_missing_capability / record_feedback, so they never reach record_tool_call. Verified by spying on it. Everything that gets there is host-dispatched by construction, so the guard could only ever fire on a host tool sharing the name. Removed. The same applies to the conversation exemption: the virtual tools' capture paths discard the resolved handle, so exempting by name only ever cost the host's tool its handle. start_tool_call_lifecycle now reads `enabled_virtual_tool_names` instead of `injectable_`, so the call path stops reading listing state altogether. That also closes the cross-client report: the collision set is per-server and rewritten by whichever listing ran last, so reading it on the call path let one caller's catalogue decide another caller's call. Note on the reports: the missing conversation id they also cite is not name-specific -- an ordinary tool shows the same in that harness, so it is not a regression here. The new tests assert parity with an ordinary tool instead, which is the rule being fixed. The changeset claimed a real `get_more_tools` "keeps its $mcp_intent", which was only true after a listing on the same instance. Corrected. Reported by QA Swarm and veria-ai on #962. Generated-By: PostHog Desktop Task-Id: 989fd424-fd24-4ae4-8682-548b87f761f6 --- posthog/mcp/_conversation_id.py | 31 ++++++++--------- posthog/mcp/_instrumentation.py | 15 ++++---- posthog/mcp/_intent.py | 27 +++++---------- posthog/test/mcp/test_units.py | 46 ++++++++++++------------ posthog/test/mcp/test_virtual_tools.py | 48 ++++++++++++++++++++++++-- 5 files changed, 100 insertions(+), 67 deletions(-) diff --git a/posthog/mcp/_conversation_id.py b/posthog/mcp/_conversation_id.py index 9d214f6c..2b9ed5ae 100644 --- a/posthog/mcp/_conversation_id.py +++ b/posthog/mcp/_conversation_id.py @@ -77,30 +77,27 @@ def extract_conversation_id(args: Any) -> Optional[str]: def resolve_conversation_id( enabled: bool, args: Any, - tool_name: Optional[str], - missing_capability_tool_name: Optional[str], + tool_name: Optional[str] = None, + missing_capability_tool_name: Optional[str] = None, feedback_tool_name: Optional[str] = None, ) -> Tuple[Optional[str], bool]: - """Return ``(conversation_id, minted)``. Disabled, get_more_tools, or - send_feedback → ``(None, False)``; agent echoed a handle we could have minted - → ``(value, False)``; anything else (omitted, or a value the agent made up) - → ``(new uuid, True)``. + """Return ``(conversation_id, minted)``. Disabled → ``(None, False)``; agent + echoed a handle we could have minted → ``(value, False)``; anything else + (omitted, or a value the agent made up) → ``(new uuid, True)``. - Either virtual tool's name arrives as ``None`` when it is disabled or when a - real application tool owns it. A shadowed name belongs to that real tool, so - it mints and echoes a handle like any other tool's. + No tool is exempt by name. A name-based exemption cannot tell PostHog's + virtual tool from a *host* tool that shares the name, and got it wrong in the + direction that matters: the host's tool lost its handle. PostHog's own + virtual tools discard the resolved handle on their own capture paths, so + resolving one for them costs nothing. + + The three tool-name parameters are accepted and ignored for backwards + compatibility with callers that still pass them positionally. Lowercased on the way in: the shape test is case-insensitive but the hash behind ``$session_id`` is not, so an uppercased echo (some hosts normalise uuids) would land in a different session than the call that minted it.""" - if ( - not enabled - or ( - missing_capability_tool_name is not None - and tool_name == missing_capability_tool_name - ) - or (feedback_tool_name is not None and tool_name == feedback_tool_name) - ): + if not enabled: return None, False supplied = extract_conversation_id(args) if supplied and _MINTED_CONVERSATION_ID.match(supplied): diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index 6c8dcfd7..f199d8e7 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -564,13 +564,14 @@ def start_tool_call_lifecycle( extra: Dict[str, Any], ) -> ToolCallLifecycle: """Resolve adapter-independent policy for a tool call without dispatching it.""" - # A name a real application tool is known to own resolves to None here, so - # every downstream decision treats calls to it like any other tool's: - # interception is skipped, and conversation-id resolution stops exempting it - # as if it were the (shadowed) virtual tool. - injectable = injectable_virtual_tool_names(data) - missing_name = injectable.get(VIRTUAL_TOOL_MISSING_CAPABILITY) - feedback_name = injectable.get(VIRTUAL_TOOL_FEEDBACK) + # Options only -- deliberately NOT the listing-derived collision state. + # That state is per-server and rewritten by whichever listing ran last, so a + # server with caller-specific catalogues would answer one caller from + # another's listing. Ownership on this path is settled per request by the + # adapters' probes, which run right after this and are never stale. + enabled = enabled_virtual_tool_names(data) + missing_name = enabled.get(VIRTUAL_TOOL_MISSING_CAPABILITY) + feedback_name = enabled.get(VIRTUAL_TOOL_FEEDBACK) # Still needed whatever the name resolution says: parsing the report and # running the host's `on_feedback` handler read the configured options. feedback_options = resolve_collect_feedback_options(data.options.collect_feedback) diff --git a/posthog/mcp/_intent.py b/posthog/mcp/_intent.py index fc8875c9..e7d88687 100644 --- a/posthog/mcp/_intent.py +++ b/posthog/mcp/_intent.py @@ -56,25 +56,16 @@ async def resolve_tool_call_intent( request: Dict[str, Any], extra: Optional[Dict[str, Any]] = None, ) -> Optional[ResolvedIntent]: - from ._instrumentation import ( - VIRTUAL_TOOL_MISSING_CAPABILITY, - injectable_virtual_tool_names, - ) - + # Only tools the host actually dispatched get here. PostHog's own virtual + # tools are intercepted before dispatch and captured by + # `record_missing_capability` / `record_feedback`, which set the event's + # intent from their own arguments -- they never reach `record_tool_call`. + # + # So there is no virtual tool to exempt here, and a name-based exemption + # could only ever fire on a *host* tool that happens to share the name, + # silently dropping its `$mcp_intent`. context_argument = _get_context_argument(request) - name = (request.get("params") or {}).get("name") - # The virtual tool carries its intent in its own `context` argument, which - # is captured as the event's own field rather than as `$mcp_intent`. Resolved - # through the injectable map, so a *real* application tool that owns the name - # keeps its `context` captured as intent like any other tool's. - missing_name = injectable_virtual_tool_names(data).get( - VIRTUAL_TOOL_MISSING_CAPABILITY - ) - if ( - is_context_enabled(data.options.context) - and (missing_name is None or name != missing_name) - and context_argument - ): + if is_context_enabled(data.options.context) and context_argument: return (context_argument, "context_parameter") return await _run_intent_fallback(data, request, extra) diff --git a/posthog/test/mcp/test_units.py b/posthog/test/mcp/test_units.py index cce94689..35718218 100644 --- a/posthog/test/mcp/test_units.py +++ b/posthog/test/mcp/test_units.py @@ -2,6 +2,8 @@ schema/loop-back, session-id rollover, and the identity cache. These complement the end-to-end adapter tests by exercising edge branches directly.""" +import pytest + from datetime import datetime, timedelta, timezone from types import SimpleNamespace @@ -58,29 +60,27 @@ async def test_intent_from_context_argument(): assert out == ("do the thing", "context_parameter") -async def test_intent_skips_context_for_missing_capability_tool(): - # a get_more_tools call's context is a capability report, not a tool-call - # intent — but only while the SDK actually owns that name +@pytest.mark.parametrize("report_missing", [False, True]) +async def test_intent_captures_context_whatever_the_tool_is_called(report_missing): + # Only host-dispatched tools reach intent resolution: PostHog's own virtual + # tools are intercepted first and set their event's intent from their own + # arguments. So a call named `get_more_tools` arriving here is the *host's* + # tool, and its context is an ordinary intent. + # + # A name-based exemption used to live here and could only fire on that host + # tool, silently dropping its `$mcp_intent` -- and only on an instance that + # had not yet served a tools/list, which made it look intermittent. out = await resolve_tool_call_intent( - _data(report_missing=True), + _data(report_missing=report_missing), _call(name="get_more_tools", args={"context": "need csv export"}), ) - assert out is None - - -async def test_intent_keeps_context_for_a_real_tool_named_get_more_tools(): - # With report_missing off the SDK advertises no such tool, so one by that - # name is the host's own and its context is an ordinary intent. Interception - # has always been gated on report_missing, so dropping the intent here left - # a real tool dispatched but unattributed. - out = await resolve_tool_call_intent( - _data(), _call(name="get_more_tools", args={"context": "need csv export"}) - ) assert out == ("need csv export", "context_parameter") -async def test_intent_keeps_context_when_a_real_tool_owns_the_name(): - # Same, via the listing-derived collision state rather than the switch. +async def test_intent_is_independent_of_listing_derived_collision_state(): + # Per-request by construction: nothing here reads state a tools/list wrote, + # so a server with caller-specific catalogues cannot attribute one caller's + # call from another caller's listing. data = _data(report_missing=True) data.virtual_tool_collisions.add(VIRTUAL_TOOL_MISSING_CAPABILITY) out = await resolve_tool_call_intent( @@ -311,11 +311,13 @@ def test_resolve_conversation_id_disabled(): assert resolve_conversation_id(False, {}, "t", "get_more_tools") == (None, False) -def test_resolve_conversation_id_skips_missing_capability_tool(): - assert resolve_conversation_id(True, {}, "get_more_tools", "get_more_tools") == ( - None, - False, - ) +def test_resolve_conversation_id_exempts_no_tool_by_name(): + # A name-based exemption cannot tell PostHog's virtual tool from a host tool + # sharing the name, and got it wrong in the direction that matters: the + # host's tool lost its handle. PostHog's virtual tools discard the resolved + # handle on their own capture paths, so resolving one for them costs nothing. + cid, minted = resolve_conversation_id(True, {}, "get_more_tools", "get_more_tools") + assert minted is True and cid def test_resolve_conversation_id_mints_for_a_shadowed_virtual_tool_name(): diff --git a/posthog/test/mcp/test_virtual_tools.py b/posthog/test/mcp/test_virtual_tools.py index 1ef1e925..ee981f2f 100644 --- a/posthog/test/mcp/test_virtual_tools.py +++ b/posthog/test/mcp/test_virtual_tools.py @@ -201,28 +201,45 @@ async def test_collision_un_shadows_once_the_real_tool_is_dropped(): async def test_registry_probe_blocks_interception_before_any_listing(): # The multi-pod case: a call reaching a process that never served a - # tools/list has empty collision state, so the registry is the only signal. + # tools/list has no listing-derived state, so the registry is the only + # signal. The host's tool must run AND be attributed exactly like any other + # tool of theirs -- attribution used to be decided from listing state + # instead, so $mcp_intent silently went missing on these instances only. server = FastMCP("virtual-tools-fastmcp") @server.tool() def get_more_tools(context: str) -> str: return "real tool ran" + @server.tool() + def ordinary(context: str) -> str: + return "ordinary ran" + client = FakeClient() instrument(server, client, MCPAnalyticsOptions(report_missing=True)) out = await server._tool_manager.call_tool( "get_more_tools", {"context": "need csv export"} ) + await server._tool_manager.call_tool("ordinary", {"context": "need csv export"}) await _flush() assert "real tool ran" in str(out) assert _events(client, "$mcp_missing_capability") == [] + colliding, sibling = _events(client, "$mcp_tool_call") + assert colliding["properties"]["$mcp_intent"] == "need csv export" + # Same treatment as an ordinary tool, which is the whole rule: intent is + # captured for tools the host owns, skipped only for PostHog's own. + assert ( + colliding["properties"]["$mcp_intent_source"] + == sibling["properties"]["$mcp_intent_source"] + ) + async def test_raw_list_probe_blocks_interception_before_any_listing(): - # A raw low-level server has no tool registry, so ownership is settled by - # asking the host's own tools/list handler. + # Same, on a raw low-level server: no tool registry, so ownership is settled + # by asking the host's own tools/list handler. server = _make_paged_lowlevel([[_REAL_GET_MORE_TOOLS]]) client = FakeClient() instrument(server, client, MCPAnalyticsOptions(report_missing=True)) @@ -232,6 +249,31 @@ async def test_raw_list_probe_blocks_interception_before_any_listing(): assert out.root.content[0].text == "real tool ran" assert _events(client, "$mcp_missing_capability") == [] + call = _events(client, "$mcp_tool_call")[0] + assert call["properties"]["$mcp_intent"] == "need csv export" + + +async def test_another_clients_listing_cannot_suppress_interception(): + # The collision set is per-server and rewritten by whichever listing ran + # last, so reading it on the call path would let one caller's catalogue + # decide another caller's call. The call path reads the per-request probe + # instead, so a stale collision cannot suppress PostHog's virtual tool. + server = _make_paged_lowlevel([[_ECHO_TOOL]]) + client = FakeClient() + instrument(server, client, MCPAnalyticsOptions(report_missing=True)) + + from posthog.mcp._instrumentation import VIRTUAL_TOOL_MISSING_CAPABILITY + from posthog.mcp._internal import get_server_tracking_data + + get_server_tracking_data(server).virtual_tool_collisions.add( + VIRTUAL_TOOL_MISSING_CAPABILITY + ) + + out = await _call(server, "get_more_tools", {"context": "need csv export"}) + await _flush() + + assert out.root.content[0].text == get_more_tools_result_text() + assert _events(client, "$mcp_missing_capability") async def test_both_virtual_tools_configured_with_the_same_name(): From 15442a56181f6e8f93a52602ef06e010c7039fc6 Mon Sep 17 00:00:00 2001 From: Georgis Andonis Date: Tue, 15 Sep 2026 16:06:33 +0300 Subject: [PATCH 2/3] feat(mcp): anchor the virtual tools to the conversation With enable_conversation_id on, get_more_tools and send_feedback now advertise conversation_id, echo a minted handle back over prompt-back, and stamp $mcp_conversation_id -- under any configured name. They were exempt, in three places that had to agree: the schema pass skipped the argument, resolve_conversation_id short-circuited on their names, and the lifecycle passed None when preparing their session. So a $mcp_feedback or $mcp_missing_capability event was filed under a $session_id of its own. An agent's complaint about a tool landed in a different session than the call it was complaining about, which is most of the value of the report, and the fallback session emitted a second spurious $mcp_initialize. resolve_conversation_id loses its two tool-name parameters: with no tool exempt there is nothing to compare against, so the exemption cannot come back by accident. The virtual tools still never get the injected `context` argument -- they state their intent through their own -- so `is_sdk_virtual_tool` now gates only that. Two tests are inverted on purpose, with the reasoning in their docstrings: test_feedback_never_mints_conversation_id becomes test_feedback_joins_the_conversation, and the renamed-tool schema case now expects the argument to track the option. Stacked on #962. Generated-By: PostHog Desktop Task-Id: 989fd424-fd24-4ae4-8682-548b87f761f6 --- .../mcp-virtual-tool-conversation-id.md | 9 +++ posthog/mcp/README.md | 8 +++ posthog/mcp/_conversation_id.py | 23 +++----- posthog/mcp/_instrument_fastmcp.py | 36 ++++++++++-- posthog/mcp/_instrument_lowlevel.py | 48 +++++++++------ posthog/mcp/_instrument_v2.py | 58 ++++++++++++------- posthog/mcp/_instrumentation.py | 42 +++++++++----- posthog/test/mcp/test_conversation_session.py | 8 +-- posthog/test/mcp/test_feedback.py | 46 +++++++++++++-- posthog/test/mcp/test_units.py | 35 ++++------- posthog/test/mcp/test_virtual_tools.py | 40 +++++++++++-- 11 files changed, 243 insertions(+), 110 deletions(-) create mode 100644 .sampo/changesets/mcp-virtual-tool-conversation-id.md diff --git a/.sampo/changesets/mcp-virtual-tool-conversation-id.md b/.sampo/changesets/mcp-virtual-tool-conversation-id.md new file mode 100644 index 00000000..6dd0a7e7 --- /dev/null +++ b/.sampo/changesets/mcp-virtual-tool-conversation-id.md @@ -0,0 +1,9 @@ +--- +pypi/posthog: patch +--- + +Anchor MCP analytics virtual tools to the conversation. With `enable_conversation_id=True`, `get_more_tools` and `send_feedback` now advertise `conversation_id`, echo a minted handle back, and stamp `$mcp_conversation_id` — whatever you rename them to. + +They were exempt before, so a `$mcp_feedback` or `$mcp_missing_capability` event was filed under a `$session_id` of its own: an agent's complaint about a tool landed in a different session than the call it was complaining about, along with a spurious second `$mcp_initialize`. Reports now share the session they are about. + +The virtual tools still never get the injected `context` argument — they state their intent through their own. diff --git a/posthog/mcp/README.md b/posthog/mcp/README.md index f6e826b7..74413fcd 100644 --- a/posthog/mcp/README.md +++ b/posthog/mcp/README.md @@ -298,6 +298,14 @@ needs no middleware and no ordering discipline, and it is the only thing that correlates a session under the 2026-07-28 revision's per-request server instances. Prefer it if you're on a recent client. +The SDK's own virtual tools take part like any other tool, whatever you rename +them to: `get_more_tools` and `send_feedback` advertise `conversation_id`, echo a +minted handle back, and stamp `$mcp_conversation_id`. That is what keeps a report +in the session it is about — an agent's complaint about a tool should be reachable +from the calls that prompted it, not filed under a session of its own. They still +never get the injected `context` argument, since they state their intent through +their own. + ### How the SDK tells you it's misconfigured The failure used to be silent. It now surfaces two ways: diff --git a/posthog/mcp/_conversation_id.py b/posthog/mcp/_conversation_id.py index 2b9ed5ae..eaa19b1b 100644 --- a/posthog/mcp/_conversation_id.py +++ b/posthog/mcp/_conversation_id.py @@ -74,25 +74,18 @@ def extract_conversation_id(args: Any) -> Optional[str]: return trimmed or None -def resolve_conversation_id( - enabled: bool, - args: Any, - tool_name: Optional[str] = None, - missing_capability_tool_name: Optional[str] = None, - feedback_tool_name: Optional[str] = None, -) -> Tuple[Optional[str], bool]: +def resolve_conversation_id(enabled: bool, args: Any) -> Tuple[Optional[str], bool]: """Return ``(conversation_id, minted)``. Disabled → ``(None, False)``; agent echoed a handle we could have minted → ``(value, False)``; anything else (omitted, or a value the agent made up) → ``(new uuid, True)``. - No tool is exempt by name. A name-based exemption cannot tell PostHog's - virtual tool from a *host* tool that shares the name, and got it wrong in the - direction that matters: the host's tool lost its handle. PostHog's own - virtual tools discard the resolved handle on their own capture paths, so - resolving one for them costs nothing. - - The three tool-name parameters are accepted and ignored for backwards - compatibility with callers that still pass them positionally. + No tool is exempt by name, the SDK's own virtual tools included. A name-based + exemption was wrong in both directions: it could not tell PostHog's virtual + tool from a *host* tool sharing the name, so the host's tool lost its handle; + and for PostHog's own tools it filed a ``$mcp_feedback`` or + ``$mcp_missing_capability`` event under a ``$session_id`` of its own, so an + agent's complaint about a tool landed nowhere near the call it was + complaining about. Anchoring the report to that call is the point of it. Lowercased on the way in: the shape test is case-insensitive but the hash behind ``$session_id`` is not, so an uppercased echo (some hosts normalise diff --git a/posthog/mcp/_instrument_fastmcp.py b/posthog/mcp/_instrument_fastmcp.py index 422d87cf..9548b065 100644 --- a/posthog/mcp/_instrument_fastmcp.py +++ b/posthog/mcp/_instrument_fastmcp.py @@ -121,14 +121,16 @@ async def wrapped( if lifecycle.is_missing_capability and not _name_owned_by_real_tool( server, name ): - await lifecycle.record_missing_capability() - return [ - mcp_types.TextContent(type="text", text=get_more_tools_result_text()) - ] + prompt_back, delivered = _conversation_prompt_back(lifecycle) + await lifecycle.record_missing_capability( + conversation_id_delivered=delivered + ) + return _with_prompt_back(get_more_tools_result_text(), prompt_back) if lifecycle.is_feedback and not _name_owned_by_real_tool(server, name): - reply = await lifecycle.record_feedback() - return [mcp_types.TextContent(type="text", text=reply)] + prompt_back, delivered = _conversation_prompt_back(lifecycle) + reply = await lifecycle.record_feedback(conversation_id_delivered=delivered) + return _with_prompt_back(reply, prompt_back) # Strip each injected key independently. A tool can declare its own # `context` (kept) while `conversation_id` is still SDK-injected (stripped), @@ -332,6 +334,28 @@ def _inject_prompt_back(result: Any, conversation_id: str) -> Any: return result +def _conversation_prompt_back(lifecycle: Any) -> Tuple[Any, bool]: + """``(prompt_back_block_or_None, delivered)`` for a virtual tool's reply. + + A minted handle rides a prompt-back block so the agent can echo it on its + next call and keep the whole exchange in one session; an echoed one needs no + delivery. ``delivered`` gates stamping the handle on the event, so a handle + the agent never received is never recorded.""" + if lifecycle.conversation_id and lifecycle.minted_conversation_id: + block = mcp_types.TextContent( + type="text", text=build_prompt_back(lifecycle.conversation_id)["text"] + ) + return block, True + return None, bool(lifecycle.conversation_id) + + +def _with_prompt_back(text: str, prompt_back: Any) -> list: + blocks = [mcp_types.TextContent(type="text", text=text)] + if prompt_back is not None: + blocks.append(prompt_back) + return blocks + + def _name_owned_by_real_tool(server: Any, name: str) -> bool: """Live registry probe so a real tool by a virtual tool's name is never shadowed, even before the first listing refreshes the collision state. diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index bb016e6b..fe431ca4 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -211,28 +211,18 @@ async def handler(req: Any) -> Any: if lifecycle.is_missing_capability and ( await _name_owned_by_real_tool(high_level, data, name, server) is False ): - await lifecycle.record_missing_capability() - return mcp_types.ServerResult( - mcp_types.CallToolResult( - content=[ - mcp_types.TextContent( - type="text", text=get_more_tools_result_text() - ) - ], - isError=False, - ) + prompt_back, delivered = _conversation_prompt_back(lifecycle) + await lifecycle.record_missing_capability( + conversation_id_delivered=delivered ) + return _virtual_tool_result(get_more_tools_result_text(), prompt_back) if lifecycle.is_feedback and ( await _name_owned_by_real_tool(high_level, data, name, server) is False ): - reply = await lifecycle.record_feedback() - return mcp_types.ServerResult( - mcp_types.CallToolResult( - content=[mcp_types.TextContent(type="text", text=reply)], - isError=False, - ) - ) + prompt_back, delivered = _conversation_prompt_back(lifecycle) + reply = await lifecycle.record_feedback(conversation_id_delivered=delivered) + return _virtual_tool_result(reply, prompt_back) # On raw low-level servers `context`/`conversation_id` are injected as # *optional* schema properties and left in place (a (name, arguments) @@ -454,6 +444,30 @@ async def handler(req: Any) -> Any: handlers[mcp_types.ListToolsRequest] = handler +def _conversation_prompt_back(lifecycle: Any) -> Tuple[Any, bool]: + """``(prompt_back_block_or_None, delivered)`` for a virtual tool's reply. + + A minted handle rides a prompt-back block so the agent can echo it on its + next call and keep the whole exchange in one session; an echoed one needs no + delivery. ``delivered`` gates stamping the handle on the event, so a handle + the agent never received is never recorded.""" + if lifecycle.conversation_id and lifecycle.minted_conversation_id: + block = mcp_types.TextContent( + type="text", text=build_prompt_back(lifecycle.conversation_id)["text"] + ) + return block, True + return None, bool(lifecycle.conversation_id) + + +def _virtual_tool_result(text: str, prompt_back: Any) -> Any: + content = [mcp_types.TextContent(type="text", text=text)] + if prompt_back is not None: + content.append(prompt_back) + return mcp_types.ServerResult( + mcp_types.CallToolResult(content=content, isError=False) + ) + + async def _name_owned_by_real_tool( high_level: Any, data: MCPAnalyticsData, name: str, server: Any ) -> Optional[bool]: diff --git a/posthog/mcp/_instrument_v2.py b/posthog/mcp/_instrument_v2.py index 224f3e4d..e978f38d 100644 --- a/posthog/mcp/_instrument_v2.py +++ b/posthog/mcp/_instrument_v2.py @@ -304,20 +304,16 @@ async def wrapped( if lifecycle.is_missing_capability and not _name_owned_by_real_tool_v2( server, name ): - await lifecycle.record_missing_capability() - return mcp_types.CallToolResult( - content=[ - mcp_types.TextContent( - type="text", text=get_more_tools_result_text() - ) - ] + prompt_back, delivered = _conversation_prompt_back_v2(lifecycle) + await lifecycle.record_missing_capability( + conversation_id_delivered=delivered ) + return _virtual_tool_result_v2(get_more_tools_result_text(), prompt_back) if lifecycle.is_feedback and not _name_owned_by_real_tool_v2(server, name): - reply = await lifecycle.record_feedback() - return mcp_types.CallToolResult( - content=[mcp_types.TextContent(type="text", text=reply)] - ) + prompt_back, delivered = _conversation_prompt_back_v2(lifecycle) + reply = await lifecycle.record_feedback(conversation_id_delivered=delivered) + return _virtual_tool_result_v2(reply, prompt_back) # v2 validates against the function signature and rejects unexpected # keys, so injected parameters are stripped before dispatch — but never @@ -405,6 +401,28 @@ def _append_prompt_back(result: Any, conversation_id: str) -> Tuple[Any, bool]: return result, True +def _conversation_prompt_back_v2(lifecycle: Any) -> Tuple[Any, bool]: + """``(prompt_back_block_or_None, delivered)`` for a virtual tool's reply. + + A minted handle rides a prompt-back block so the agent can echo it on its + next call and keep the whole exchange in one session; an echoed one needs no + delivery. ``delivered`` gates stamping the handle on the event, so a handle + the agent never received is never recorded.""" + if lifecycle.conversation_id and lifecycle.minted_conversation_id: + block = mcp_types.TextContent( + type="text", text=build_prompt_back(lifecycle.conversation_id)["text"] + ) + return block, True + return None, bool(lifecycle.conversation_id) + + +def _virtual_tool_result_v2(text: str, prompt_back: Any) -> Any: + content = [mcp_types.TextContent(type="text", text=text)] + if prompt_back is not None: + content.append(prompt_back) + return mcp_types.CallToolResult(content=content) + + def _deliver_conversation_id( data: MCPAnalyticsData, result: Any, name: str, conversation_id: str, minted: bool ) -> Tuple[Any, bool]: @@ -526,22 +544,18 @@ async def handler(ctx: Any, params: Any) -> Any: if lifecycle.is_missing_capability and ( await raw_listing_owns_tool_name(data, name, ctx) is False ): - await lifecycle.record_missing_capability() - return mcp_types.CallToolResult( - content=[ - mcp_types.TextContent( - type="text", text=get_more_tools_result_text() - ) - ] + prompt_back, delivered = _conversation_prompt_back_v2(lifecycle) + await lifecycle.record_missing_capability( + conversation_id_delivered=delivered ) + return _virtual_tool_result_v2(get_more_tools_result_text(), prompt_back) if lifecycle.is_feedback and ( await raw_listing_owns_tool_name(data, name, ctx) is False ): - reply = await lifecycle.record_feedback() - return mcp_types.CallToolResult( - content=[mcp_types.TextContent(type="text", text=reply)] - ) + prompt_back, delivered = _conversation_prompt_back_v2(lifecycle) + reply = await lifecycle.record_feedback(conversation_id_delivered=delivered) + return _virtual_tool_result_v2(reply, prompt_back) # Settle the shared session before the tool body runs, so an in-tool # `analytics.capture()` is attributed to this caller and not the last one. diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index f199d8e7..d6640d25 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -468,11 +468,15 @@ async def prime_session(self) -> None: self.data, mcp_session_id=self.mcp_session_id, token=self.token ) - async def record_missing_capability(self) -> None: - session_id = await self.prepare_session(None) + async def record_missing_capability( + self, *, conversation_id_delivered: bool = False + ) -> None: + conversation_id = self._anchored_conversation_id(conversation_id_delivered) + session_id = await self.prepare_session(conversation_id) await record_missing_capability( self.data, session_id, + conversation_id=conversation_id, tool_name=self.missing_name or self.name, context=(self.arguments or {}).get("context"), arguments=self.arguments, @@ -484,15 +488,25 @@ async def record_missing_capability(self) -> None: extra=self.extra, ) - async def record_feedback(self) -> str: + def _anchored_conversation_id(self, delivered: bool) -> Optional[str]: + """The handle to anchor this call's event to. A freshly minted handle the + agent never received must not be stamped, or the event anchors to a + session nothing else can join.""" + if self.minted_conversation_id and not delivered: + return None + return self.conversation_id + + async def record_feedback(self, *, conversation_id_delivered: bool = False) -> str: """Capture the ``$mcp_feedback`` event, then run the host's ``on_feedback`` handler and return the reply text for the agent. The event is captured whether or not the handler raises.""" report = parse_feedback_report(self.arguments, self.feedback_options) - session_id = await self.prepare_session(None) + conversation_id = self._anchored_conversation_id(conversation_id_delivered) + session_id = await self.prepare_session(conversation_id) await record_feedback( self.data, session_id, + conversation_id=conversation_id, report=report, tool_name=self.feedback_name or self.name, arguments=self.arguments, @@ -576,11 +590,7 @@ def start_tool_call_lifecycle( # running the host's `on_feedback` handler read the configured options. feedback_options = resolve_collect_feedback_options(data.options.collect_feedback) conversation_id, minted = resolve_conversation_id( - data.options.enable_conversation_id, - arguments, - name, - missing_name, - feedback_name, + data.options.enable_conversation_id, arguments ) return ToolCallLifecycle( data=data, @@ -1053,10 +1063,12 @@ def mutate_tool_schema( data.tool_model_parameter_injected[tool.name] = ( not app_owns_model and schema_has_param(schema, "llm_model") ) - if ( - not is_sdk_virtual_tool - and data.options.enable_conversation_id - and not schema_has_param(schema, "conversation_id") + # Not gated on `is_sdk_virtual_tool`, unlike `context` above: the virtual + # tools state their intent in their own arguments, but they belong to the + # same conversation as the calls around them and must be able to receive and + # echo its handle. + if data.options.enable_conversation_id and not schema_has_param( + schema, "conversation_id" ): schema = add_conversation_id_to_schema(schema, tool.name) if schema is not original_schema: @@ -1181,6 +1193,7 @@ async def record_missing_capability( data: MCPAnalyticsData, session_id: str, *, + conversation_id: Optional[str] = None, tool_name: str, context: Optional[str], arguments: Optional[Dict[str, Any]], @@ -1198,6 +1211,7 @@ async def record_missing_capability( event: Dict[str, Any] = { "event_type": MCPAnalyticsEventType.MCP_MISSING_CAPABILITY, "session_id": session_id, + "conversation_id": conversation_id, "resource_name": tool_name, "parameters": build_captured_mcp_parameters( request, strip_llm_model=allow_self_reported_model @@ -1229,6 +1243,7 @@ async def record_feedback( data: MCPAnalyticsData, session_id: str, *, + conversation_id: Optional[str] = None, report: FeedbackReport, tool_name: str, arguments: Optional[Dict[str, Any]], @@ -1249,6 +1264,7 @@ async def record_feedback( event: Dict[str, Any] = { "event_type": MCPAnalyticsEventType.MCP_FEEDBACK, "session_id": session_id, + "conversation_id": conversation_id, "resource_name": tool_name, "client_name": client_name, "client_version": client_version, diff --git a/posthog/test/mcp/test_conversation_session.py b/posthog/test/mcp/test_conversation_session.py index 037888a6..0e1c04da 100644 --- a/posthog/test/mcp/test_conversation_session.py +++ b/posthog/test/mcp/test_conversation_session.py @@ -62,7 +62,7 @@ def test_derivation_is_deterministic_and_distinct(): def test_echo_of_a_mintable_handle_is_accepted(): cid, minted = resolve_conversation_id( - True, {"conversation_id": MINTED_SHAPE_HANDLE}, "t", "get_more_tools" + True, {"conversation_id": MINTED_SHAPE_HANDLE} ) assert minted is False assert cid == MINTED_SHAPE_HANDLE @@ -73,7 +73,7 @@ def test_uppercased_echo_is_lowercased_before_hashing(): # case-sensitive, so the echo must be folded back or it lands in a # different session than the call that minted it. cid, minted = resolve_conversation_id( - True, {"conversation_id": MINTED_SHAPE_HANDLE.upper()}, "t", "get_more_tools" + True, {"conversation_id": MINTED_SHAPE_HANDLE.upper()} ) assert minted is False assert cid == MINTED_SHAPE_HANDLE @@ -82,9 +82,7 @@ def test_uppercased_echo_is_lowercased_before_hashing(): def test_invented_handle_is_not_anchored(): # Two unrelated users both sending "conv-1" must NOT share a session, so a # value we could not have minted is replaced with a fresh handle. - cid, minted = resolve_conversation_id( - True, {"conversation_id": "conv-1"}, "t", "get_more_tools" - ) + cid, minted = resolve_conversation_id(True, {"conversation_id": "conv-1"}) assert minted is True assert cid != "conv-1" diff --git a/posthog/test/mcp/test_feedback.py b/posthog/test/mcp/test_feedback.py index c0b68ab0..ff479ee0 100644 --- a/posthog/test/mcp/test_feedback.py +++ b/posthog/test/mcp/test_feedback.py @@ -857,7 +857,11 @@ async def test_collision_warning_is_logged_once_across_repeated_listings(): assert len([m for m in messages if "Cannot inject PostHog's" in m]) == 1 -async def test_feedback_never_mints_conversation_id(): +async def test_feedback_joins_the_conversation(): + # Inverted deliberately: send_feedback used to be exempt from conversation + # anchoring, which filed the event under a session of its own — so an + # agent's complaint about a tool landed nowhere near the call it was + # complaining about. server = make_lowlevel() client = FakeClient() instrument( @@ -868,16 +872,48 @@ async def test_feedback_never_mints_conversation_id(): result = await _list_tools_lowlevel(server) virtual = [t for t in result.root.tools if t.name == "send_feedback"][0] - assert "conversation_id" not in virtual.inputSchema["properties"] + assert "conversation_id" in virtual.inputSchema["properties"] call_handler = server.request_handlers[mcp_types.CallToolRequest] out = await call_handler(_call_request("send_feedback", dict(_REPORT_ARGS))) await _flush() feedback = _events(client, "$mcp_feedback") - assert "$mcp_conversation_id" not in feedback[0]["properties"] - # No prompt-back block appended to the acknowledgement. - assert len(out.root.content) == 1 + handle = feedback[0]["properties"]["$mcp_conversation_id"] + assert handle + # The minted handle rides a prompt-back block, so the agent can echo it and + # keep the rest of the exchange in the same session. + assert len(out.root.content) == 2 + assert handle in out.root.content[1].text + + +async def test_feedback_anchors_to_the_session_it_is_about(): + # The point of the change: the complaint and the call it is about share a + # $session_id, so the report is reachable from the session that produced it. + server = make_lowlevel() + client = FakeClient() + instrument( + server, + client, + MCPAnalyticsOptions(collect_feedback=True, enable_conversation_id=True), + ) + await _list_tools_lowlevel(server) + call_handler = server.request_handlers[mcp_types.CallToolRequest] + + handle = "0198d3a7-1111-7222-8333-444455556666" + await call_handler(_call_request("echo", {"msg": "hi", "conversation_id": handle})) + await call_handler( + _call_request( + "send_feedback", {**dict(_REPORT_ARGS), "conversation_id": handle} + ) + ) + await _flush() + + tool_call = _events(client, "$mcp_tool_call")[0] + feedback = _events(client, "$mcp_feedback")[0] + assert ( + tool_call["properties"]["$session_id"] == feedback["properties"]["$session_id"] + ) # --- PostHogMCP custom dispatcher --------------------------------------------------- diff --git a/posthog/test/mcp/test_units.py b/posthog/test/mcp/test_units.py index 35718218..88ffbdd9 100644 --- a/posthog/test/mcp/test_units.py +++ b/posthog/test/mcp/test_units.py @@ -308,24 +308,17 @@ def test_extract_conversation_id(): def test_resolve_conversation_id_disabled(): - assert resolve_conversation_id(False, {}, "t", "get_more_tools") == (None, False) + assert resolve_conversation_id(False, {}) == (None, False) -def test_resolve_conversation_id_exempts_no_tool_by_name(): - # A name-based exemption cannot tell PostHog's virtual tool from a host tool - # sharing the name, and got it wrong in the direction that matters: the - # host's tool lost its handle. PostHog's virtual tools discard the resolved - # handle on their own capture paths, so resolving one for them costs nothing. - cid, minted = resolve_conversation_id(True, {}, "get_more_tools", "get_more_tools") - assert minted is True and cid - - -def test_resolve_conversation_id_mints_for_a_shadowed_virtual_tool_name(): - # `None` means the virtual tool is disabled or a real application tool owns - # the name. Either way the call belongs to that real tool, so it mints and - # echoes a handle like any other tool's. - cid, minted = resolve_conversation_id(True, {}, "get_more_tools", None) - assert minted is True and cid +def test_resolve_conversation_id_applies_to_every_tool(): + # No tool is exempt by name. A name-based exemption could not tell PostHog's + # virtual tool from a host tool sharing the name -- the host's tool lost its + # handle -- and for PostHog's own tools it orphaned the agent's complaint + # from the calls it was complaining about. + for name in ("get_more_tools", "send_feedback", "echo"): + cid, minted = resolve_conversation_id(True, {}) + assert minted is True and cid, name def test_resolve_conversation_id_uses_supplied_when_mintable_shape(): @@ -333,20 +326,16 @@ def test_resolve_conversation_id_uses_supplied_when_mintable_shape(): # the handle becomes $session_id, so an invented value ("conv-1") must not # anchor two unrelated callers to one session (parity with posthog-js). handle = "0198d3a7-1111-7222-8333-444455556666" - assert resolve_conversation_id( - True, {"conversation_id": handle}, "t", "get_more_tools" - ) == (handle, False) + assert resolve_conversation_id(True, {"conversation_id": handle}) == (handle, False) def test_resolve_conversation_id_replaces_invented_values(): - cid, minted = resolve_conversation_id( - True, {"conversation_id": "conv-1"}, "t", "get_more_tools" - ) + cid, minted = resolve_conversation_id(True, {"conversation_id": "conv-1"}) assert minted is True and cid != "conv-1" def test_resolve_conversation_id_mints_when_absent(): - cid, minted = resolve_conversation_id(True, {}, "t", "get_more_tools") + cid, minted = resolve_conversation_id(True, {}) assert minted is True and isinstance(cid, str) and cid diff --git a/posthog/test/mcp/test_virtual_tools.py b/posthog/test/mcp/test_virtual_tools.py index ee981f2f..e58fdc3b 100644 --- a/posthog/test/mcp/test_virtual_tools.py +++ b/posthog/test/mcp/test_virtual_tools.py @@ -302,9 +302,10 @@ async def test_both_virtual_tools_configured_with_the_same_name(): @pytest.mark.parametrize("enable_conversation_id", [False, True]) async def test_renamed_tool_carries_its_own_intent(enable_conversation_id): # The virtual tool states its intent in its own `context` argument, so it - # gets neither an injected `context` nor a `conversation_id` -- and that has - # to hold for a renamed tool too. The name used to be hardcoded here, so a - # renamed tool picked up a `conversation_id` the default-named one never got. + # never gets an injected one -- and that has to hold for a renamed tool too, + # since the name used to be matched literally here. `conversation_id` is a + # different matter: it tracks the option, for the default and renamed tool + # alike, because the tool belongs to the same conversation as its neighbours. server = _make_paged_lowlevel([[_ECHO_TOOL]]) instrument( server, @@ -318,10 +319,41 @@ async def test_renamed_tool_carries_its_own_intent(enable_conversation_id): page = await _list_page(server) virtual = [tool for tool in page.root.tools if tool.name == "find_tools"][0] - assert list(virtual.inputSchema["properties"]) == ["context"] + expected = ["context", "conversation_id"] if enable_conversation_id else ["context"] + assert sorted(virtual.inputSchema["properties"]) == sorted(expected) assert virtual.inputSchema["required"] == ["context"] +async def test_renamed_tool_anchors_to_the_conversation(): + # Whatever it is called, the report joins the session it is about. + server = _make_paged_lowlevel([[_ECHO_TOOL]]) + client = FakeClient() + instrument( + server, + client, + MCPAnalyticsOptions( + report_missing=True, + missing_capability_tool_name="find_tools", + enable_conversation_id=True, + ), + ) + await _list_page(server) + + handle = "0198d3a7-1111-7222-8333-444455556666" + await _call(server, "echo", {"msg": "hi", "conversation_id": handle}) + await _call( + server, "find_tools", {"context": "need csv", "conversation_id": handle} + ) + await _flush() + + tool_call = _events(client, "$mcp_tool_call")[0] + missing = _events(client, "$mcp_missing_capability")[0] + assert missing["properties"]["$mcp_conversation_id"] == handle + assert ( + tool_call["properties"]["$session_id"] == missing["properties"]["$session_id"] + ) + + async def test_renamed_tool_is_intercepted_and_the_default_name_is_not(): server = _make_paged_lowlevel([[_ECHO_TOOL]]) client = FakeClient() From dc9873c4c51703e2906c523fa89e154740a51656 Mon Sep 17 00:00:00 2001 From: Georgis Andonis Date: Wed, 16 Sep 2026 12:29:36 +0300 Subject: [PATCH 3/3] chore(mcp): move the attribution claim to this PR's changeset The per-request ownership fix landed here rather than in #962, so its user-facing effects belong in this changeset, not that one. Generated-By: PostHog Desktop Task-Id: 989fd424-fd24-4ae4-8682-548b87f761f6 --- .sampo/changesets/mcp-virtual-tool-conversation-id.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.sampo/changesets/mcp-virtual-tool-conversation-id.md b/.sampo/changesets/mcp-virtual-tool-conversation-id.md index 6dd0a7e7..f16617e2 100644 --- a/.sampo/changesets/mcp-virtual-tool-conversation-id.md +++ b/.sampo/changesets/mcp-virtual-tool-conversation-id.md @@ -7,3 +7,5 @@ Anchor MCP analytics virtual tools to the conversation. With `enable_conversatio They were exempt before, so a `$mcp_feedback` or `$mcp_missing_capability` event was filed under a `$session_id` of its own: an agent's complaint about a tool landed in a different session than the call it was complaining about, along with a spurious second `$mcp_initialize`. Reports now share the session they are about. The virtual tools still never get the injected `context` argument — they state their intent through their own. + +Ownership is now decided per request rather than from the last served `tools/list`, which also fixes attribution for a real tool of yours sharing a virtual tool's name: it is captured like any other tool of yours, `$mcp_intent` included. Previously that depended on the process having served a listing, so on a multi-pod deployment the intent was silently dropped. And a server that serves different tool sets to different callers can no longer have one caller's listing suppress another caller's virtual-tool handling.