Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 110 additions & 3 deletions .github/scripts/snippets/gen-router-pricing.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { compactOptions, displayProvider, groupImageTiers, groupResolutionTiers, groupRates, loadCatalog, loadMetronomeData, render, resolveKeyedRates, tokenPriceCell, validateCreditConversion } from "./gen-router-pricing.ts";
import { checkPricingCoverage, compactOptions, displayProvider, findUnpricedRoutes, groupImageTiers, groupResolutionTiers, groupRates, loadCatalog, loadMetronomeData, loadUnpricedWaivers, render, resolveKeyedRates, tokenPriceCell, validateCreditConversion } from "./gen-router-pricing.ts";
import type { UnpricedWaiver } from "./gen-router-pricing.ts";
import { formatAmount, formatOption, formatUnit } from "./router-pricing-display.ts";

const catalog = loadCatalog();
Expand Down Expand Up @@ -196,8 +197,8 @@ describe("public Router pricing", () => {

test("preserves every route's credit amounts and shows a unit in the cell or token header", () => {
expect(snapshot.credits_per_usd).toBe(211);
expect(snapshot.rates).toHaveLength(622);
expect(snapshot.rates.filter((rate) => rate.kind !== "usage")).toHaveLength(591);
expect(snapshot.rates).toHaveLength(649);
expect(snapshot.rates.filter((rate) => rate.kind !== "usage")).toHaveLength(618);
for (const rate of snapshot.rates.filter((candidate) => candidate.kind !== "usage")) {
const model = catalog.find((candidate) => candidate.id === rate.model_id)!;
const provider = displayProvider(model.id, model.providers, rate.serving_provider);
Expand Down Expand Up @@ -690,3 +691,109 @@ describe("public Router pricing", () => {
expect(description.length).toBeLessThanOrEqual(160);
});
});

describe("Router pricing coverage guard", () => {
const route = { model_id: "acme/video-1", provider: "Pika" };
const reason = "Router refuses dispatch: no classification row.";
const status = "catalog_only_not_dispatchable";
const evidence = "Comfy-Org/cloud@abc1234 routerresult/classification.go (no row)";

test("fails on an unpriced route that has no waiver, naming the model, provider, and fix", () => {
const errors = checkPricingCoverage([route], []);
expect(errors).toHaveLength(1);
expect(errors[0]).toContain("acme/video-1");
expect(errors[0]).toContain("provider Pika");
expect(errors[0]).toContain("Not published");
expect(errors[0]).toContain("router-pricing/metronome-rates.json");
expect(errors[0]).toContain("add a waiver with a status, reason and evidence");
});

test("passes when every unpriced route is waived with a reason", () => {
expect(checkPricingCoverage([route], [{ ...route, reason, status, evidence }])).toEqual([]);
});

test("a waiver covers only its own provider leg", () => {
const errors = checkPricingCoverage([route, { ...route, provider: "Comfy" }], [{ ...route, reason, status, evidence }]);
expect(errors).toHaveLength(1);
expect(errors[0]).toContain("provider Comfy");
});

test("fails on a waiver with an empty or missing reason", () => {
for (const badReason of ["", " ", undefined]) {
const errors = checkPricingCoverage([route], [{ ...route, status, evidence, reason: badReason as unknown as string }]);
expect(errors).toHaveLength(1);
expect(errors[0]).toContain("has no reason");
expect(errors[0]).toContain("acme/video-1");
}
});

test("fails on a waiver with empty or missing evidence", () => {
for (const badEvidence of ["", " ", undefined]) {
const errors = checkPricingCoverage([route], [{ ...route, reason, status, evidence: badEvidence as unknown as string }]);
expect(errors).toHaveLength(1);
expect(errors[0]).toContain("has no evidence");
expect(errors[0]).toContain("acme/video-1");
}
});

test("rejects a waiver for a live route or with an unknown status", () => {
const live = checkPricingCoverage([route], [{ ...route, reason, evidence, status: "live" }]);
expect(live).toHaveLength(1);
expect(live[0]).toContain("live route cannot be waived");
const unknown = checkPricingCoverage([route], [{ ...route, reason, evidence, status: "" }]);
expect(unknown).toHaveLength(1);
expect(unknown[0]).toContain("needs a status");
expect(checkPricingCoverage([route], [{ ...route, reason, evidence, status: "embargoed" }])).toEqual([]);
});

test("fails on a stale waiver whose route now has a price, so the list can only shrink", () => {
const errors = checkPricingCoverage([], [{ ...route, reason, status, evidence }]);
expect(errors).toHaveLength(1);
expect(errors[0]).toContain("stale waiver");
expect(errors[0]).toContain("acme/video-1");
});

test("reports a null, primitive or array waiver entry instead of throwing", () => {
for (const bad of [null, "acme/video-1", 42, [route]]) {
const errors = checkPricingCoverage([route], [bad as unknown as UnpricedWaiver]);
expect(errors.some((error) => error.includes("must be an object with a model_id and a provider"))).toBe(true);
expect(errors.some((error) => error.includes("acme/video-1 (provider Pika) would render"))).toBe(true);
}
});

test("fails on a duplicate waiver", () => {
const errors = checkPricingCoverage([route], [{ ...route, reason, status, evidence }, { ...route, reason, status, evidence }]);
expect(errors.some((error) => error.includes("duplicate waiver"))).toBe(true);
});

test("finds every route the page renders as unavailable, per provider leg", () => {
const models = [
{ id: "openai/gpt-image-2", title: "GPT Image 2", page: "p", providers: ["Comfy", "Pika"], category: "images" as const },
{ id: "pika/pika-2.5-text-to-video", title: "Pika", page: "p", providers: ["Comfy"], category: "video" as const },
];
const rate = { kind: "metered" as const, unit: "per second", price_usd: "0.04", credits: "8.44", effective_from: "2026-10-10" };
expect(findUnpricedRoutes(models, [])).toEqual([
{ model_id: "openai/gpt-image-2", provider: "Comfy" },
{ model_id: "openai/gpt-image-2", provider: "Pika" },
{ model_id: "pika/pika-2.5-text-to-video", provider: "Comfy" },
]);
expect(findUnpricedRoutes(models, [
{ ...rate, model_id: "openai/gpt-image-2", serving_provider: "pika" },
{ ...rate, model_id: "pika/pika-2.5-text-to-video", serving_provider: "Pika" },
])).toEqual([{ model_id: "openai/gpt-image-2", provider: "Comfy" }]);
});

test("every checked-in waiver is current, reasoned, and for a route Router does not dispatch", () => {
const unpriced = findUnpricedRoutes(catalog, snapshot.rates);
// Unwaived live routes are reported by router-pricing:check, not here.
expect(checkPricingCoverage(unpriced, loadUnpricedWaivers()).filter((error) => error.startsWith("router-pricing/unpriced-waivers.json:"))).toEqual([]);
const unavailableRows = (creditsPage.match(/\| Not published \|/g) ?? []).length;
expect(unavailableRows).toBe(unpriced.length);
});

test("publishes every Pika route", () => {
const unpriced = findUnpricedRoutes(catalog, snapshot.rates);
expect(unpriced.filter((candidate) => candidate.provider === "Pika" || candidate.model_id.startsWith("pika/"))).toEqual([]);
expect(loadUnpricedWaivers().filter((waiver) => waiver.provider === "Pika" || waiver.model_id.startsWith("pika/"))).toEqual([]);
});
});
98 changes: 95 additions & 3 deletions .github/scripts/snippets/gen-router-pricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import { formatAmount, formatDate, formatOption, formatUnit, pricingCopy, type P

const ROOT = join(import.meta.dir, "../../..");
const METRONOME_FILE = join(ROOT, "router-pricing/metronome-rates.json");
const WAIVER_PATH = "router-pricing/unpriced-waivers.json";
const WAIVER_FILE = join(ROOT, WAIVER_PATH);
const OUTPUT_FILE = join(ROOT, "development/comfy-router/pricing.mdx");
const MODEL_GLOB = "development/comfy-router/models/**/code.mdx";
const MAX_RESOLUTION_COLUMNS = 5;
Expand Down Expand Up @@ -71,6 +73,7 @@ const PROVIDER_LABEL: Record<string, string> = {
moonvalley: "Moonvalley",
openai: "OpenAI",
openrouter: "OpenRouter",
pika: "Pika",
pruna: "Pruna",
qwen: "Qwen",
recraft: "Recraft",
Expand Down Expand Up @@ -519,7 +522,8 @@ function renderCurrencyView(locale: PricingLocale, currency: "credits" | "usd"):
const oneImagePerGeneration = item.unit === "per image"
&& ["wan/wan2.5-i2i-preview", "wan/wan2.5-t2i-preview"].includes(item.modelId.slice(1, -1));
const unitCanBeShortened = headerUnit === item.unit
|| (headerUnit === "per generation" && (item.unit === "per request" || oneImagePerGeneration));
|| (headerUnit === "per generation" && (item.unit === "per request" || oneImagePerGeneration
|| (category === "images" && ["per image", "per output image"].includes(item.unit))));
const credits = headerUnit && unitCanBeShortened && item.kind !== "usage"
? priceWithoutUnit(item.credits, item.unit, locale)
: item.credits;
Expand Down Expand Up @@ -555,7 +559,10 @@ function renderCurrencyView(locale: PricingLocale, currency: "credits" | "usd"):
? `| ${copy.model} | ${copy.modelId} | ${columns.map((unit) => rateHeader("", ` / ${formatUnit(unit, locale)}`)).join(" | ")} |\n| --- | --- | ${columns.map(() => "---:").join(" | ")} |\n${matrixModels.map(([modelId, model]) => `| ${model.name} | ${modelId} | ${columns.map((unit) => model.values.get(unit) ?? "-").join(" | ")} |`).join("\n")}`
: "";
const fallbackRows = items.filter((row) => fallbackIds.has(row.modelId));
return [table, fallbackRows.length ? renderOrdinaryRows(fallbackRows, true) : ""].filter(Boolean).join("\n\n");
// A model with several output-image prices keeps one row per price; its
// amounts are still generations, like the matrix above.
const fallbackUnit = fallbackRows.every((row) => row.kind !== "usage" && unitForRow(row) === "per generation") ? "per generation" : undefined;
return [table, fallbackRows.length ? renderOrdinaryRows(fallbackRows, true, fallbackUnit) : ""].filter(Boolean).join("\n\n");
};
const ordinaryGroups = new Map<string, typeof ordinaryRows>();
for (const row of ordinaryRows) {
Expand Down Expand Up @@ -919,6 +926,7 @@ function renderCurrencyView(locale: PricingLocale, currency: "credits" | "usd"):
const { group } = row;
const unit = group.tiers.values().next().value!.rate.unit;
const optionHeader = row.option === "-" ? copy.option
: /^\d+s$/.test(row.option) ? copy.duration
: group.conditions.includes("draft=") ? copy.mode
: /generate_?Audio=|generate_audio=/.test(group.conditions) ? copy.audio
: group.conditions.includes("video_type=") ? copy.inputType
Expand Down Expand Up @@ -1118,9 +1126,93 @@ ${copy.updated}: ${formatDate(data.snapshot_at, locale)}.
`;
}

export type UnpricedRoute = { model_id: string; provider: string };
export const WAIVABLE_STATUSES = ["embargoed", "catalog_only_not_dispatchable"] as const;
export type UnpricedWaiver = UnpricedRoute & { reason: string; status: string; evidence: string };

const routeKey = (route: UnpricedRoute) => `${route.model_id} @ ${route.provider}`;

/**
* Every model/provider route the page lists that has no rate, i.e. every row
* that would render the "Not published" value. Mirrors renderCurrencyView: one
* row per model and per serving provider on its page, priced by the rates that
* map to that provider.
*/
function findUnpricedRoutes(models: CatalogModel[], rates: MetronomeRate[]): UnpricedRoute[] {
const priced = new Set<string>();
const modelsById = new Map(models.map((model) => [model.id, model]));
for (const rate of rates) {
const model = modelsById.get(rate.model_id);
if (!model) continue;
priced.add(routeKey({ model_id: model.id, provider: displayProvider(model.id, model.providers, rate.serving_provider) }));
}
return models.flatMap((model) => model.providers
.map((provider) => ({ model_id: model.id, provider }))
.filter((route) => !priced.has(routeKey(route))));
}

function loadUnpricedWaivers(): UnpricedWaiver[] {
if (!existsSync(WAIVER_FILE)) return [];
const data = JSON.parse(readFileSync(WAIVER_FILE, "utf8")) as { waivers?: UnpricedWaiver[] };
if (!Array.isArray(data.waivers)) throw new Error(`${WAIVER_FILE}: expected a "waivers" array`);
return data.waivers;
}
Comment thread
mattmillerai marked this conversation as resolved.

/**
* A published, billable Router route must show a price. Returns one message per
* problem: an unwaived unpriced route, a waiver without a reason or evidence, a duplicate
* waiver, or a stale waiver (its route is now priced or no longer listed), so
* the waiver list can only shrink.
*/
function checkPricingCoverage(unpriced: UnpricedRoute[], waivers: UnpricedWaiver[]): string[] {
const errors: string[] = [];
const unpricedKeys = new Set(unpriced.map(routeKey));
const waived = new Set<string>();
for (const waiver of waivers) {
if (waiver === null || typeof waiver !== "object" || Array.isArray(waiver) || !waiver.model_id || !waiver.provider) {
errors.push(`${WAIVER_PATH}: every waiver must be an object with a model_id and a provider (got ${JSON.stringify(waiver)})`);
continue;
}
const key = routeKey(waiver);
if (typeof waiver.reason !== "string" || !waiver.reason.trim()) {
errors.push(`${WAIVER_PATH}: waiver for ${waiver.model_id} (provider ${waiver.provider}) has no reason; say why this route has no published price`);
}
if (typeof waiver.evidence !== "string" || !waiver.evidence.trim()) {
errors.push(`${WAIVER_PATH}: waiver for ${waiver.model_id} (provider ${waiver.provider}) has no evidence; cite the Comfy-Org/cloud file and line that shows its status`);
}
if (!(WAIVABLE_STATUSES as readonly string[]).includes(waiver.status)) {
Comment thread
mattmillerai marked this conversation as resolved.
errors.push(waiver.status === "live"
? `${WAIVER_PATH}: ${waiver.model_id} (provider ${waiver.provider}) is live on Router, and a live route cannot be waived; publish its price in router-pricing/metronome-rates.json`
: `${WAIVER_PATH}: waiver for ${waiver.model_id} (provider ${waiver.provider}) needs a status of ${WAIVABLE_STATUSES.join(" or ")} (got ${JSON.stringify(waiver.status)})`);
}
if (waived.has(key)) errors.push(`${WAIVER_PATH}: duplicate waiver for ${waiver.model_id} (provider ${waiver.provider})`);
waived.add(key);
if (!unpricedKeys.has(key)) {
errors.push(`${WAIVER_PATH}: stale waiver for ${waiver.model_id} (provider ${waiver.provider}); the page no longer shows it as unpriced, so remove the waiver`);
}
}
for (const route of unpriced) {
if (waived.has(routeKey(route))) continue;
errors.push(`${route.model_id} (provider ${route.provider}) would render "${pricingCopy.en.unavailable}" on the Router pricing page. Add its rates to router-pricing/metronome-rates.json, or, only if Router does not dispatch it (embargoed or catalog-only), add a waiver with a status, reason and evidence to ${WAIVER_PATH}`);
}
return errors;
}

function pricingCoverageErrors(): string[] {
const data = loadMetronomeData();
return checkPricingCoverage(findUnpricedRoutes(loadCatalog(), data.rates), loadUnpricedWaivers());
}

if (import.meta.main) {
const check = process.argv.includes("--check");
const output = render();
const coverageErrors = pricingCoverageErrors();
if (coverageErrors.length) {
for (const error of coverageErrors) console.error(error);
// Generation still writes the page so a contributor can see the gap;
// --check (CI) fails until every listed route is priced or waived.
if (check) process.exit(1);
}
if (check) {
if (!existsSync(OUTPUT_FILE) || readFileSync(OUTPUT_FILE, "utf8") !== output) {
console.error(`${OUTPUT_FILE}: stale or missing, run pnpm router-pricing:gen`);
Expand All @@ -1133,4 +1225,4 @@ if (import.meta.main) {
}
}

export { compactOptions, groupImageTiers, groupResolutionTiers, groupRates, loadCatalog, loadMetronomeData, render, resolveKeyedRates, tokenPriceCell, validateCreditConversion };
export { checkPricingCoverage, compactOptions, findUnpricedRoutes, loadUnpricedWaivers, groupImageTiers, groupResolutionTiers, groupRates, loadCatalog, loadMetronomeData, render, resolveKeyedRates, tokenPriceCell, validateCreditConversion };
2 changes: 1 addition & 1 deletion .github/scripts/snippets/router-pricing-display.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ export function formatOption(conditions: string | undefined, locale: PricingLoca
? ["Audio", "音声", "音频", "오디오"][index]
: ["No audio", "音声なし", "无音频", "오디오 없음"][index];
if (key === "draft") return value === "true" ? ["Draft", "ドラフト", "草稿", "초안"][index] : ["Standard", "標準", "标准", "표준"][index];
if (key === "quality") return ({ low: ["Low", "低", "低", "낮음"], medium: ["Medium", "中", "中", "중간"], high: ["High", "高", "高", "높음"], xhigh: ["Extra high", "最高", "超高", "매우 높음"], max: ["Max", "最大", "最高", "최대"], DEFAULT: ["Standard", "標準", "标准", "표준"], TURBO: ["Turbo", "Turbo", "Turbo", "Turbo"], QUALITY: ["Quality", "高品質", "高质量", "고품질"] } as Record<string, string[]>)[value]?.[index] ?? value;
if (key === "quality") return ({ very_low: ["Very low", "最低", "极低", "매우 낮음"], low: ["Low", "低", "低", "낮음"], medium: ["Medium", "中", "中", "중간"], high: ["High", "高", "高", "높음"], xhigh: ["Extra high", "最高", "超高", "매우 높음"], max: ["Max", "最大", "最高", "최대"], DEFAULT: ["Standard", "標準", "标准", "표준"], TURBO: ["Turbo", "Turbo", "Turbo", "Turbo"], QUALITY: ["Quality", "高品質", "高质量", "고품질"] } as Record<string, string[]>)[value]?.[index] ?? value;
if (key === "video_type" || key === "feature") return translate(value);
if (key === "type") return value === "image_edit" ? translate("Edit") : ["Image generation", "画像生成", "图像生成", "이미지 생성"][index];
if (key === "endpoint") return value.includes("dialogue") ? ["Dialogue", "対話", "对话", "대화"][index] : ["Text to speech", "音声合成", "语音合成", "음성 합성"][index];
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/code-pages-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
- 'snippets/comfy-router/**'
- 'router-schemas/**'
- 'tutorials/partner-nodes/pricing.mdx'
- 'router-pricing/metronome-rates.json'
- 'router-pricing/**'
- 'development/comfy-router/pricing.mdx'
- 'ja/development/comfy-router/pricing.mdx'
- 'zh/development/comfy-router/pricing.mdx'
Expand Down
Loading
Loading