Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/scripts/snippets/check-provider-schemas.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,28 @@ describe("compare: reporting a per-variant provider default", () => {
]);
});
});

describe("compare: numeric bounds", () => {
const ours = (bounds: Record<string, number>) => ({ type: "object", properties: { safety_tolerance: { type: "integer", default: 2, ...bounds } } });
const theirs = { type: "object", properties: { safety_tolerance: { type: "integer", minimum: 0, maximum: 7, default: 2 } } };

test("a documented bound narrower than the provider's only warns: Router may enforce a tighter range", () => {
const { errors, warnings } = run(ours({ minimum: 1, maximum: 4 }), theirs);
expect(errors).toEqual([]);
expect(warnings.filter((m) => /safety_tolerance/.test(m))).toHaveLength(2);
});

test("a documented bound wider than the provider's is an error: it advertises values the provider rejects", () => {
const { errors } = run(ours({ minimum: -1, maximum: 9 }), theirs);
expect(errors).toEqual([
"model input: `safety_tolerance` minimum -1 vs provider 0",
"model input: `safety_tolerance` maximum 9 vs provider 7",
]);
});

test("matching bounds report nothing", () => {
const { errors, warnings } = run(ours({ minimum: 0, maximum: 7 }), theirs);
expect(errors).toEqual([]);
expect(warnings.filter((m) => /safety_tolerance/.test(m))).toEqual([]);
});
});
11 changes: 9 additions & 2 deletions .github/scripts/snippets/check-provider-schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
* omit: [webhook_url, webhook_secret] # provider fields we deliberately do not document
*
* Errors (exit 1): a documented field the provider does not have; a type, default,
* enum, bound or required-ness that disagrees with the provider; a single documented
* enum or required-ness that disagrees with the provider; a bound WIDER than the
* provider's (a narrower one only warns, since Router may enforce a tighter range); a single documented
* `default` for a field whose provider default is PER-VARIANT. Warnings: provider
* fields we do not document (errors under --strict), provider shapes that are
* opaque (`{}`) where we document structure, and a per-variant provider default the
Expand Down Expand Up @@ -211,7 +212,13 @@ export function compare(ours: any, theirs: any, where: string, omit: Set<string>
}
if (o.enum && t.enum) { const extra = o.enum.filter((v: unknown) => !t.enum.includes(v)); if (extra.length) rep.errors.push(`${where}: \`${path}\` enum values ${JSON.stringify(extra)} are not in the provider's ${JSON.stringify(t.enum)}`); const missing = t.enum.filter((v: unknown) => !o.enum.includes(v) && !String(v).endsWith("UNSPECIFIED")); if (missing.length) rep.warnings.push(`${where}: \`${path}\` provider also allows ${JSON.stringify(missing)}`); }
if (t.enum && !o.enum && t.enum.length <= 12) rep.warnings.push(`${where}: \`${path}\` provider enumerates ${JSON.stringify(t.enum)} but we document a free ${o.type}`);
for (const b of ["minimum", "maximum"] as const) if (o[b] !== undefined && t[b] !== undefined && Number(o[b]) !== Number(t[b])) rep.errors.push(`${where}: \`${path}\` ${b} ${o[b]} vs provider ${t[b]}`);
// Router may enforce a tighter range than the provider allows (BFL raised `safety_tolerance` to 7; Router still validates 4),
// so only a documented bound WIDER than the provider's is an error: that would advertise values the provider rejects.
for (const b of ["minimum", "maximum"] as const) {
if (o[b] === undefined || t[b] === undefined || Number(o[b]) === Number(t[b])) continue;
const wider = b === "maximum" ? Number(o[b]) > Number(t[b]) : Number(o[b]) < Number(t[b]);
(wider ? rep.errors : rep.warnings).push(`${where}: \`${path}\` ${b} ${o[b]} vs provider ${t[b]}${wider ? "" : " (narrower than the provider; fine if Router enforces it)"}`);
}
if (o.properties) compare(o, t, where, omit, rep, path, noRequired);
if (o.items?.properties) compare(o.items, t.items, where, omit, rep, `${path}[]`, noRequired);
}
Expand Down
Loading