From 2d7d110b5ad63d4976d2e813e0a1c2ad340e2c43 Mon Sep 17 00:00:00 2001 From: Hugo Richard Date: Thu, 24 Sep 2026 10:19:27 +0100 Subject: [PATCH] docs: add jev guide for auto presets and approval --- apps/docs/content/docs/4.guide/1.presets.md | 2 +- .../docs/4.guide/2.approval-control.md | 2 +- .../content/docs/4.guide/8.jev-typesafe-ai.md | 210 ++++++++++++++++++ apps/docs/nuxt.config.ts | 1 + 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 apps/docs/content/docs/4.guide/8.jev-typesafe-ai.md diff --git a/apps/docs/content/docs/4.guide/1.presets.md b/apps/docs/content/docs/4.guide/1.presets.md index 73e0f25..9b48fe8 100644 --- a/apps/docs/content/docs/4.guide/1.presets.md +++ b/apps/docs/content/docs/4.guide/1.presets.md @@ -113,7 +113,7 @@ On each call, a fast evaluation model reads the latest user message and rates ho 2. The read-only `repo-explorer` preset is only used when no other preset qualifies, since every other preset already has the read tools its task needs. 3. When none reaches `0.7`, the single most likely preset is used. When nothing stands out, as with "hello", that is `repo-explorer`. -`preset: 'auto'` is available on `createGithubAgent` and needs `ai` 7.0.105 or later. To tune it, use `evaluation: { minPresetProbability, maxPresets, model }`. The default model is TypeSafe's Jev. If the evaluation call fails (model not enabled on AI Gateway, no credits, outage), the call uses read-only `repo-explorer` and logs a `github_tools.EVALUATION_FAILED` warning through `console.warn`. +`preset: 'auto'` is available on `createGithubAgent` and needs `ai` 7.0.105 or later. To tune it, use `evaluation: { minPresetProbability, maxPresets, model }`. The default model is [Jev from TypeSafe AI](/guide/jev-typesafe-ai). If the evaluation call fails (model not enabled on AI Gateway, no credits, outage), the call uses read-only `repo-explorer` and logs a `github_tools.EVALUATION_FAILED` warning through `console.warn`. ## Pick the right preset diff --git a/apps/docs/content/docs/4.guide/2.approval-control.md b/apps/docs/content/docs/4.guide/2.approval-control.md index 4021d7b..9144a24 100644 --- a/apps/docs/content/docs/4.guide/2.approval-control.md +++ b/apps/docs/content/docs/4.guide/2.approval-control.md @@ -88,7 +88,7 @@ createGithubAgent({ }) ``` -The values shown are the defaults. The default model is TypeSafe's [Jev](https://vercel.com/i/what-is-jev), called through [AI Gateway](https://vercel.com/docs/ai-gateway/modalities/evaluation). +The values shown are the defaults. The default model is TypeSafe's [Jev](https://vercel.com/i/what-is-jev), called through [AI Gateway](https://vercel.com/docs/ai-gateway/modalities/evaluation). See [Auto presets and approval with Jev](/guide/jev-typesafe-ai) for both `'auto'` modes together. ### When the evaluation model fails diff --git a/apps/docs/content/docs/4.guide/8.jev-typesafe-ai.md b/apps/docs/content/docs/4.guide/8.jev-typesafe-ai.md new file mode 100644 index 0000000..3dbc264 --- /dev/null +++ b/apps/docs/content/docs/4.guide/8.jev-typesafe-ai.md @@ -0,0 +1,210 @@ +--- +title: Auto Presets and Approval with Jev +description: Use Jev from TypeSafe AI to pick GitHub tool presets per request and auto-approve routine writes in AI SDK and eve agents. +navigation: + title: Jev from TypeSafe AI +path: /guide/jev-typesafe-ai +links: + - label: Scope with presets + icon: i-lucide-layers + to: /guide/presets + color: neutral + variant: subtle + - label: Control write safety + icon: i-lucide-shield-check + to: /guide/approval-control + color: neutral + variant: subtle +--- + +[Jev](https://vercel.com/i/what-is-jev) is TypeSafe AI's System One model. It does not write prose: it reads the context you give it and returns typed choices, scores, and probabilities. That makes it a good fit for the small decisions a GitHub agent takes on every turn, like which tools to load or whether a write needs a human. + +GitHub tools uses Jev as the default evaluation model for its two `'auto'` modes: + +- `preset: 'auto'` picks the presets each request needs, so the model only sees the tools for that task. +- `requireApproval: 'auto'` lets routine writes the user asked for run without a prompt, and keeps everything else behind approval. + +Both are opt-in. Without them, presets stay static and every write asks for approval. + +::prompt +--- +description: Enable auto presets and approval with Jev +icon: i-lucide-sparkles +actions: + - copy + - cursor + - windsurf +--- + +Enable the 'auto' modes of @github-tools/sdk in this project. + +- Use createGithubAgent with preset: 'auto' and requireApproval: 'auto' (or the eve extension with the same options), and make sure `ai` is 7.0.105 or later +- Keep merges, file writes, deletes, and releases behind approval; only low-risk writes are auto-approved +- The default evaluation model is typesafe-ai/jev through AI Gateway; confirm it is enabled for the project +- Follow https://github-tools.com/guide/jev-typesafe-ai + +:: + +## Quick start + +With the AI SDK: + +```ts [agent.ts] +import { createGithubAgent } from '@github-tools/sdk' + +const agent = createGithubAgent({ + model: 'anthropic/claude-opus-5.5', + preset: 'auto', + requireApproval: 'auto', +}) + +await agent.generate({ prompt: 'Label #42 as a bug and ask for a repro.' }) +// likely routed to issue-triage; the label and comment were asked for, so they can skip approval +``` + +With the [eve extension](/frameworks/eve-extension): + +```ts [agent/extensions/github.ts] +import githubExtension from '@github-tools/eve-extension' + +export default githubExtension({ + preset: 'auto', + requireApproval: 'auto', +}) +``` + +Both modes need `ai` 7.0.105 or later. Jev is called through [AI Gateway](https://vercel.com/docs/ai-gateway/modalities/evaluation) as `typesafe-ai/jev`, so the project needs AI Gateway access with the model enabled. + +## Pick presets per request + +A single agent that takes open-ended requests would otherwise need the full catalog. With `preset: 'auto'`, Jev reads the latest user message and rates how likely it is to need each [preset](/guide/presets). Only the tools of the chosen presets are exposed for that call: + +1. Presets rated at or above `0.7` are used, most likely first, up to two per call. +2. The read-only `repo-explorer` preset is only used when no other preset qualifies. +3. When none reaches `0.7`, the single most likely preset is used. For a message like "hello", that is `repo-explorer`. + +When exactly one preset is chosen, the agent also uses that preset's system prompt. The full catalog is never exposed. + +On eve, routing runs on each user message. Tools the agent already called in the session stay registered, so a parked approval still resumes if the next message routes elsewhere. + +## Auto-approve routine writes + +Before a low-risk write tool runs, Jev answers two questions: + +- **Risk**: how costly would this call be if it were wrong? `0` negligible, `1` visible but reversible, `2` hard to undo. +- **Intent**: did the latest user message ask for this exact action? + +The call runs only when risk is at most `1` and intent is at least `0.6`. Otherwise the user is asked as usual. A comment the agent decided to post because an issue body told it to still waits for a human, since the user never asked for it. + +`'auto'` only applies to `AUTO_APPROVAL_TOOLS`: labels, assignees, reactions, comments, review-thread replies, reviewer requests, notification reads, and workflow re-runs. Merges, file writes, deletes, releases, and other write tools keep requiring approval. You can opt single tools in or out: + +```ts [approval-per-tool.ts] +import { createGithubTools } from '@github-tools/sdk' + +const tools = createGithubTools({ + requireApproval: { + addLabels: 'auto', + addIssueComment: 'auto', + updateIssue: 'auto', + mergePullRequest: true, + }, +}) +``` + +More detail, including eve approval policies: [Control write safety](/guide/approval-control#auto-approval). + +## Tune the thresholds + +The defaults work without configuration. The `evaluation` option tunes both modes, on `createGithubTools`, `createGithubAgent`, and the eve extension: + +```ts [evaluation.ts] +import { createGithubAgent } from '@github-tools/sdk' + +const agent = createGithubAgent({ + model: 'anthropic/claude-opus-5.5', + preset: 'auto', + requireApproval: 'auto', + evaluation: { + model: 'typesafe-ai/jev', + maxRisk: 1, + minIntent: 0.6, + minPresetProbability: 0.7, + maxPresets: 2, + }, +}) +``` + +| Option | Default | Used by | Effect | +|---|---|---|---| +| `model` | `'typesafe-ai/jev'` | both | Any AI SDK evaluation model | +| `maxRisk` | `1` | approval | Highest risk score that can skip approval | +| `minIntent` | `0.6` | approval | Lowest probability that the user asked for the call | +| `minPresetProbability` | `0.7` | presets | Probability a preset needs to be selected | +| `maxPresets` | `2` | presets | Presets combined per call | + +Lower `maxRisk` or raise `minIntent` to ask more often. Set the thresholds against the cost of a wrong call in your repositories, not against how often the agent prompts. + +## When Jev is unavailable + +If the evaluation call fails, for example because the model is not enabled for your AI Gateway project, the team is out of credits, or the gateway is rate limiting, both modes fall back to the safe side: + +- `requireApproval: 'auto'` asks for approval, as if it were `true`. +- `preset: 'auto'` uses the read-only `repo-explorer` preset. + +The agent keeps running, and a `github_tools.EVALUATION_FAILED` warning with the gateway error as `cause` is logged through `console.warn`. + +::note +`createDurableGithubAgent` does not accept `'auto'`. Use `createGithubAgent` or the [eve extension](/frameworks/eve-extension) for durable agents with auto approval. +:: + +## Learn more about Jev + +These guides from the Vercel Knowledge Base cover Jev outside GitHub tools. + +::card-group + :::card + --- + icon: i-lucide-shield-check + title: Auto-approve tool calls in eve + to: https://vercel.com/kb/guide/auto-approve-tool-calls-eve-jev + target: _blank + --- + Review tool calls in eve with Jev, allow routine actions, and request human approval when needed. + ::: + + :::card + --- + icon: i-lucide-split + title: Classify, route, and score with the AI SDK + to: https://vercel.com/kb/guide/typesafe-jev-and-ai-sdk + target: _blank + --- + Use Jev with the AI SDK's evaluate API to get typed choices, scores, and probabilities. + ::: + + :::card + --- + icon: i-lucide-inbox + title: Route form submissions + to: https://vercel.com/kb/guide/jev-ai-sdk-form-router + target: _blank + --- + Route clear cases with Jev and hand uncertain ones to a fallback model. + ::: + + :::card + --- + icon: i-lucide-message-square-warning + title: Moderate product reviews with TanStack AI + to: https://vercel.com/kb/guide/moderate-product-reviews-jev-tanstack-ai + target: _blank + --- + Publish clear reviews and hold flagged ones with Jev and TanStack AI's `decide()`. + ::: +:: + +## External references + +- [Jev from TypeSafe AI (Vercel Knowledge Base)](https://vercel.com/kb/jev-from-typesafe-ai) +- [What is Jev?](https://vercel.com/i/what-is-jev) +- [AI Gateway evaluation models](https://vercel.com/docs/ai-gateway/modalities/evaluation) diff --git a/apps/docs/nuxt.config.ts b/apps/docs/nuxt.config.ts index 7228bff..312751e 100644 --- a/apps/docs/nuxt.config.ts +++ b/apps/docs/nuxt.config.ts @@ -51,6 +51,7 @@ export default defineNuxtConfig({ '/guide/durable-workflows': { redirect: { to: '/frameworks/vercel-workflow', statusCode: 301 } }, '/guide/token-permissions': { redirect: { to: '/guide/tokens-and-auth', statusCode: 301 } }, '/guide/examples': { redirect: { to: '/examples/overview', statusCode: 301 } }, + '/guide/jev-tyesafe-ai': { redirect: { to: '/guide/jev-typesafe-ai', statusCode: 301 } }, '/frameworks/eve': { redirect: { to: '/deprecated/eve', statusCode: 301 } }, }, content: {