Skip to content

fix(ai): let Prompts.get_all() batch without a label #964

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
posthog wants to merge 1 commit into
base: main
Choose a base branch
from
+122 −40
Draft

fix(ai): let Prompts.get_all() batch without a label #964

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/prompts-get-all-without-label.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: minor
---

`Prompts.get_all()` now works without a label: it fetches the latest version of every prompt in one request and caches each one under the key `get(name)` reads. Apps that do not use labels no longer need one request per prompt per cache cycle.
99 changes: 60 additions & 39 deletions posthog/ai/prompts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,13 @@ def _prompt_reference(
return reference


def _prompt_list_reference(label: Optional[str]) -> str:
"""Format a prompt list reference for logs and errors."""
if label is None:
return "all prompts"
return f'prompts with label "{label}"'


def _extract_config(data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Read config from an API response, tolerating servers that don't send it."""
config = data.get("config")
Expand Down Expand Up @@ -201,6 +208,9 @@ class Prompts:
# Fetch all prompts at a label in one request and warm the cache
prod_prompts = prompts.get_all(label='production')

# Or fetch the latest version of every prompt in one request
all_prompts = prompts.get_all()

# Compile with variables
system_prompt = prompts.compile(template, {
'company': 'Acme Corp',
Expand Down Expand Up @@ -361,29 +371,30 @@ def get(
return fallback
raise

def get_all(self, *, label: str) -> Dict[str, PromptResult]:
def get_all(self, *, label: Optional[str] = None) -> Dict[str, PromptResult]:
"""
Fetch every prompt that carries a label, in one batch.
Fetch every prompt in one batch.

Returns a dict mapping prompt name to :class:`PromptResult`, with each
prompt at the version the label points to. Prompts without the label
are not included.
With ``label``, each prompt comes back at the version the label points
to, and prompts without the label are not included. Without ``label``,
every prompt comes back at its latest version.

Each fetched prompt is stored in the cache, so later
``get(name, label=...)`` calls are served from cache within the TTL.
Each fetched prompt is stored in the cache under the key the matching
``get()`` call reads, so that call is served from cache within the TTL.
An app with many prompts can call this once per cache cycle instead of
making one ``get()`` request per prompt.

Args:
label: The label to resolve, e.g. 'production'.
label: The label to resolve, e.g. 'production'. Omit it to fetch
the latest version of every prompt.

Returns:
Dict of prompt name to PromptResult.

Raises:
Exception: If the request fails, or the server does not support
fetching prompts by label on the list endpoint (PostHog
releases from before September 2026).
Exception: If the request fails, or a label was requested and the
server does not support fetching prompts by label on the list
endpoint (PostHog releases from before September 2026).
"""
try:
rows = self._fetch_prompt_list_from_api(label)
Expand All @@ -398,34 +409,39 @@ def get_all(self, *, label: str) -> Dict[str, PromptResult]:
for row in rows:
if not _is_prompt_api_response(row):
invalid_error = Exception(
f'[PostHog Prompts] Invalid response format for prompts with label "{label}"'
"[PostHog Prompts] Invalid response format for "
f"{_prompt_list_reference(label)}"
)
self._maybe_capture_error(
invalid_error, name="*", version=None, label=label
)
raise invalid_error

label_state = _row_label_state(row, label)
if label_state == "absent":
# Even one unlabeled row proves the server did not filter, and
# then rows that look resolved are only labels that happen to
# point at the latest version. A partial result here would hide
# the rest, so fail loudly instead.
compat_error = Exception(
f'[PostHog Prompts] The server returned a prompt that does not carry label "{label}". '
"It may not support fetching prompts by label on the list endpoint yet. "
"Upgrade PostHog, or fetch prompts one by one with get()."
)
self._maybe_capture_error(
compat_error, name="*", version=None, label=label
)
raise compat_error
if label_state == "moved":
skipped.append(row["name"])
continue
# Without a label the endpoint returns latest versions, so there
# is no label to resolve and no old-server behavior to detect.
if label is not None:
label_state = _row_label_state(row, label)
if label_state == "absent":
# Even one unlabeled row proves the server did not filter,
# and then rows that look resolved are only labels that
# happen to point at the latest version. A partial result
# here would hide the rest, so fail loudly instead.
compat_error = Exception(
f'[PostHog Prompts] The server returned a prompt that does not carry label "{label}". '
"It may not support fetching prompts by label on the list endpoint yet. "
"Upgrade PostHog, or fetch prompts one by one with get()."
)
self._maybe_capture_error(
compat_error, name="*", version=None, label=label
)
raise compat_error
if label_state == "moved":
skipped.append(row["name"])
continue

resolved_rows.append(row)

if rows and not resolved_rows:
if label is not None and rows and not resolved_rows:
# Every returned row was skipped as moved. One moved label is a
# mid-request race, but all of them means the server most likely
# ignored the label param and served latest versions.
Expand Down Expand Up @@ -655,26 +671,31 @@ def _require_credentials(self) -> None:
"Please provide it when initializing the Prompts instance."
)

def _fetch_prompt_list_from_api(self, label: str) -> List[Dict[str, Any]]:
def _fetch_prompt_list_from_api(
self, label: Optional[str] = None
) -> List[Dict[str, Any]]:
"""
Fetch all prompts at a label from the paginated list endpoint.
Fetch prompts from the paginated list endpoint.

Endpoint:
{host}/api/environments/@current/llm_prompts/
?token={encoded_project_api_key}&label={label}&content=full
?token={encoded_project_api_key}[&label={label}]&content=full
Auth: Bearer {personal_api_key}

Follows pagination links until the last page. Returns the raw rows.
Without a label the endpoint returns the latest version of every
prompt. Follows pagination links until the last page. Returns the raw
rows.
"""
self._require_credentials()

query = urllib.parse.urlencode(
{"token": self._project_api_key, "label": label, "content": "full"}
)
params = {"token": self._project_api_key, "content": "full"}
if label is not None:
params["label"] = label
query = urllib.parse.urlencode(params)
Comment on lines +691 to +694

@posthog posthog Bot Sep 16, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unlabeled batch fetches lose prompt-fetch usage events

should_fix compatibility

Issue description

The new unlabeled path calls the list endpoint without label. The deployed server emits $llm_prompt_fetched for labeled list calls only. Repeated get(name) calls emit one event per prompt. An app that switches to get_all() therefore stops reporting these fetches. Prompt-only teams can also disappear from AI observability usage reports. The required server fix remains an open draft.

Why we think it's a valid issue
  • Checked: the new unlabeled request build at posthog/ai/prompts.py:691-694, the whole posthog-python tree for any client-side fetch event, the server list() handler on PostHog/posthog master, the ingestion billable-event list, and the AI observability usage report task.
  • Found: the server tracks list fetches only under a label. On master, list() calls self._track_labeled_list_fetches(prompts, label) inside an if label: guard (posthog/api/llm_prompt.py:685-689), while the single-prompt action calls self._track_prompt_fetch(prompt) unconditionally (posthog/api/llm_prompt.py:358). The premise holds: the path this PR adds produces no $llm_prompt_fetched event, and the path it replaces produces one per prompt per cache cycle.
  • Found: the gap is still open. The most recent commit on posthog/api/llm_prompt.py is 7d776ae0 feat(aio): report prompt fetches on labeled list calls (#98183). feat(aio): report prompt fetches on unlabeled list calls posthog#100690 reports state: OPEN, isDraft: true, mergedAt: null, so the unlabeled tracking is not deployed.
  • Found: the SDK sends nothing of its own to compensate. A search for llm_prompt_fetched across posthog-python returns no match, and _maybe_capture_error reports exceptions only.
  • Found: the loss reaches team selection, not only a count. AI_OBSERVABILITY_REPORT_TRIGGER_EVENTS includes $llm_prompt_fetched (posthog/tasks/ai_observability_usage_report.py:47) and feeds get_teams_with_ai_events(...) at line 911. A team that uses prompt management but emits no $ai_* events drops out of the report after it moves to the unlabeled batch call.
  • Found: this file already records server-version dependencies, so a rollout note fits the existing convention. The get_all docstring names "PostHog releases from before September 2026" for the labeled path (posthog/ai/prompts.py:395-397).
  • Impact: an app that adopts the new path stops reporting prompt fetches until the server change deploys. That telemetry is what shows rate-limit headroom, which is the exact signal this PR's own motivation rests on.
  • Priority: lowered to should_fix. $llm_prompt_fetched sits in NON_BILLABLE_EVENTS in nodejs/src/ingestion/common/usage-records/billable-events.ts:20, so no billing and no customer-facing result changes. No code in this repository is wrong, the correction is server-side and takes effect for every client as soon as it deploys, and the requested server contract test belongs in PostHog/posthog. The action for this PR is release coordination plus a note in the changeset, not a merge blocker.
Suggested fix

Deploy PostHog/posthog#100690 before this SDK release. Record this rollout dependency in the changeset. Add a server contract test that proves an API-key unlabeled list request emits one event for each returned prompt.

Prompt to fix with AI (copy-paste) 
## Context
@posthog/ai/prompts.py#L691-694

<issue_description>
The new unlabeled path calls the list endpoint without `label`. The deployed server emits `$llm_prompt_fetched` for labeled list calls only. Repeated `get(name)` calls emit one event per prompt. An app that switches to `get_all()` therefore stops reporting these fetches. Prompt-only teams can also disappear from AI observability usage reports. The required server fix remains an open draft.
</issue_description>

<issue_validation>
- **Checked:** the new unlabeled request build at posthog/ai/prompts.py:691-694, the whole posthog-python tree for any client-side fetch event, the server `list()` handler on PostHog/posthog master, the ingestion billable-event list, and the AI observability usage report task.
- **Found:** the server tracks list fetches only under a label. On master, `list()` calls `self._track_labeled_list_fetches(prompts, label)` inside an `if label:` guard (posthog/api/llm_prompt.py:685-689), while the single-prompt action calls `self._track_prompt_fetch(prompt)` unconditionally (posthog/api/llm_prompt.py:358). The premise holds: the path this PR adds produces no `$llm_prompt_fetched` event, and the path it replaces produces one per prompt per cache cycle.
- **Found:** the gap is still open. The most recent commit on posthog/api/llm_prompt.py is `7d776ae0 feat(aio): report prompt fetches on labeled list calls (#98183)`. PostHog/posthog#100690 reports `state: OPEN`, `isDraft: true`, `mergedAt: null`, so the unlabeled tracking is not deployed.
- **Found:** the SDK sends nothing of its own to compensate. A search for `llm_prompt_fetched` across posthog-python returns no match, and `_maybe_capture_error` reports exceptions only.
- **Found:** the loss reaches team selection, not only a count. `AI_OBSERVABILITY_REPORT_TRIGGER_EVENTS` includes `$llm_prompt_fetched` (posthog/tasks/ai_observability_usage_report.py:47) and feeds `get_teams_with_ai_events(...)` at line 911. A team that uses prompt management but emits no `$ai_*` events drops out of the report after it moves to the unlabeled batch call.
- **Found:** this file already records server-version dependencies, so a rollout note fits the existing convention. The `get_all` docstring names "PostHog releases from before September 2026" for the labeled path (posthog/ai/prompts.py:395-397).
- **Impact:** an app that adopts the new path stops reporting prompt fetches until the server change deploys. That telemetry is what shows rate-limit headroom, which is the exact signal this PR's own motivation rests on.
- **Priority:** lowered to `should_fix`. `$llm_prompt_fetched` sits in `NON_BILLABLE_EVENTS` in nodejs/src/ingestion/common/usage-records/billable-events.ts:20, so no billing and no customer-facing result changes. No code in this repository is wrong, the correction is server-side and takes effect for every client as soon as it deploys, and the requested server contract test belongs in PostHog/posthog. The action for this PR is release coordination plus a note in the changeset, not a merge blocker.
</issue_validation>

## Task
Investigate the issue and solve it

<potential_solution>
Deploy https://github.com/PostHog/posthog/pull/100690 before this SDK release. Record this rollout dependency in the changeset. Add a server contract test that proves an API-key unlabeled list request emits one event for each returned prompt.
</potential_solution>

@posthog posthog Bot Sep 16, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Escalating: the gap is real, but it closes in the server repository and depends on a release-sequencing call a human must make.

  • I confirmed this SDK sends no prompt-fetch event of its own, so the unlabeled batch path reports nothing until the server side changes.
  • The server change is still an open draft, so the dependency holds today.
  • A human must decide whether to hold this SDK release until that server change deploys, or ship now and accept the gap.
  • I did not add the note to the changeset. That file becomes public release notes, and its wording depends on the decision above.
  • The requested contract test belongs in the server repository, which I must not change from this pull request.
How this was verified

No code changed, so no lint or tests were run. Checks made instead: read the unlabeled request build and the error-capture helper in the prompts module, searched the whole repository for any client-side prompt-fetch event (no match), and queried the state of the referenced server pull request (still open and draft). Working tree left clean.

url: Optional[str] = (
f"{self._host}/api/environments/@current/llm_prompts/?{query}"
)
reference = f'prompts with label "{label}"'
reference = _prompt_list_reference(label)
headers = {
"Authorization": f"Bearer {self._personal_api_key}",
"User-Agent": USER_AGENT,
Expand Down
56 changes: 56 additions & 0 deletions posthog/test/ai/test_prompts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1426,6 +1426,62 @@ def test_fetches_all_pages_and_seeds_the_cache(self, mock_get_session):
self.assertEqual(cached.source, "cache")
self.assertEqual(cached.version, 3)

@patch("posthog.ai.prompts._get_session")
def test_without_a_label_returns_latest_versions_and_seeds_the_cache(
self, mock_get_session
):
# No label means no label to resolve: a prompt that carries no label,
# and one whose label points at an earlier version, both belong in the
# result at their latest version.
mock_get = mock_get_session.return_value.get
unlabeled = {**self.labeled_row("prompt-a", version=2), "all_labels": []}
labeled_elsewhere = {
**self.labeled_row("prompt-b", version=3),
"all_labels": [{"name": "production", "version": 1}],
}
mock_get.return_value = self.list_response([unlabeled, labeled_elsewhere])

prompts = Prompts(self.create_mock_posthog())
results = prompts.get_all()

self.assertNotIn("label", mock_get.call_args.args[0])
self.assertEqual(
results,
{
"prompt-a": PromptResult(
source="api",
prompt="Prompt for prompt-a",
name="prompt-a",
version=2,
),
"prompt-b": PromptResult(
source="api",
prompt="Prompt for prompt-b",
name="prompt-b",
version=3,
),
},
)

# Later unlabeled get() calls are cache hits, not new requests.
cached = prompts.get("prompt-a", with_metadata=True)
self.assertEqual(mock_get.call_count, 1)
self.assertEqual(cached.source, "cache")
self.assertEqual(cached.version, 2)
self.assertIsNone(cached.label)

@patch("posthog.ai.prompts._get_session")
def test_without_a_label_names_all_prompts_in_an_error(self, mock_get_session):
mock_get = mock_get_session.return_value.get
malformed = {**self.labeled_row("prompt-a"), "version": "1"}
mock_get.return_value = self.list_response([malformed])

prompts = Prompts(self.create_mock_posthog())

with self.assertRaises(Exception) as ctx:
prompts.get_all()
self.assertIn("Invalid response format for all prompts", str(ctx.exception))

@patch("posthog.ai.prompts._get_session")
def test_raises_when_the_server_ignores_the_label(self, mock_get_session):
# An old server ignores ?label= and returns latest versions of every
Expand Down
2 changes: 1 addition & 1 deletion references/public_api_snapshot.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1348,7 +1348,7 @@ method posthog.ai.otel.processor.PostHogSpanProcessor.shutdown() -> None
method posthog.ai.prompts.Prompts.clear_cache(name: Optional[str] = None, *, version: Optional[int] = None) -> None
method posthog.ai.prompts.Prompts.compile(prompt: str, variables: PromptVariables) -> str
method posthog.ai.prompts.Prompts.get(name: str, *, with_metadata: Optional[bool] = None, cache_ttl_seconds: Optional[int] = None, fallback: Optional[str] = None, version: Optional[int] = None, label: Optional[str] = None) -> Union[str, PromptResult]
method posthog.ai.prompts.Prompts.get_all(*, label: str) -> Dict[str, PromptResult]
method posthog.ai.prompts.Prompts.get_all(*, label: Optional[str] = None) -> Dict[str, PromptResult]
method posthog.ai.stream.AsyncStreamWrapper.aclose() -> None
method posthog.ai.stream.AsyncStreamWrapper.close() -> None
method posthog.async_client.AsyncClient.alias(previous_id: ID_TYPES, distinct_id: Optional[str], timestamp: Optional[Union[datetime, str]] = None, uuid: Optional[str] = None, disable_geoip: Optional[bool] = None) -> Optional[str]
Expand Down