diff --git a/package.json b/package.json index aae9f30..0a16b4d 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "@profullstack/player": "0.3.1", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", - "@profullstack/x402-gateway": "0.1.0", + "@profullstack/x402-gateway": "0.2.2", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-dropdown-menu": "^2.1.24", "@radix-ui/react-icons": "^1.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dc96ade..da6af6d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,8 +38,8 @@ importers: specifier: ^0.1.3 version: 0.1.3(next@16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) '@profullstack/x402-gateway': - specifier: 0.1.0 - version: 0.1.0 + specifier: 0.2.2 + version: 0.2.2 '@radix-ui/react-dialog': specifier: ^1.1.23 version: 1.1.23(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -1317,8 +1317,8 @@ packages: react: optional: true - '@profullstack/x402-gateway@0.1.0': - resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + '@profullstack/x402-gateway@0.2.2': + resolution: {integrity: sha512-oAbe4AuJzNEoLcb+8fhR5u9ce7i9h3XuyoaFjuXlgTxTSXkfdAWOfYFZq1aFplVWvZNRW3da7fYCv7nkQdujXA==} engines: {node: '>=20.11'} '@puppeteer/browsers@3.2.1': @@ -6211,7 +6211,7 @@ snapshots: next: 16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) react: 19.2.8 - '@profullstack/x402-gateway@0.1.0': {} + '@profullstack/x402-gateway@0.2.2': {} '@puppeteer/browsers@3.2.1(yauzl@2.10.0)': dependencies: diff --git a/src/lib/crawl-gateway.ts b/src/lib/crawl-gateway.ts index dec579f..66d501d 100644 --- a/src/lib/crawl-gateway.ts +++ b/src/lib/crawl-gateway.ts @@ -6,6 +6,18 @@ * HTML sales page at /crawl. A paid pass in the `x-crawl-pass` header lets them * through. People, Googlebot and retrieval crawlers are untouched. * + * Two edge controls catch crawlers that do not say who they are: + * - `denyCidrs`: hosting ranges that serve no readers get a tiny 403 first. + * - `chargeSpoofedBrowsers`: a "Chrome/..." user agent with no Sec-Fetch-Mode + * header is an HTTP client wearing a copied string (every Chromium since 76 + * sends it and nothing can strip it), so it is charged like GPTBot. Anything + * that declares itself (Googlebot's evergreen string, Bingbot, any "bot") + * is judged by the lists instead, and Firefox/Safari are never judged. + * + * This site is mostly real people, so `exempt` matters most: a request that + * carries a valid-looking Supabase session or a valid-looking `btr_` API + * bearer token is never charged, whatever else it looks like. + * * Used by src/proxy.ts (the gate) and src/app/robots.txt/route.ts (the lists), * so robots.txt and the gate never disagree about who is who. * @@ -14,9 +26,76 @@ import { createGateway } from '@profullstack/x402-gateway'; +/** + * OVH VPS fleet ranges, measured 2026-08-28 on rssamplifier: vps-*.vps.ovh.net + * hosts spoofing "Chrome/148" across these /16s. Hosting ranges serve no + * readers, so they are refused outright rather than offered a pass. + */ +const OVH_VPS_FLEET_CIDRS = [ + '51.38.0.0/16', + '54.38.0.0/16', + '141.94.0.0/16', + '145.239.0.0/16', + '149.202.0.0/16', + '151.80.0.0/16', + '57.129.0.0/16', + '213.32.0.0/16', +]; + +/** The Supabase session cookie src/proxy.ts refreshes and src/lib/auth reads. */ +const SESSION_COOKIE_NAME = 'sb-auth-token'; + +/** A v1 API token from src/lib/api-tokens: `btr_` + 32 random bytes as hex. */ +const API_BEARER_RE = /^Bearer\s+btr_[0-9a-f]{64}$/i; + +/** Three base64url segments: the shape of the Supabase access token. */ +const JWT_RE = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/; + +function cookieValue(request: Request, name: string): string | null { + const header = request.headers.get('cookie'); + if (!header) return null; + for (const part of header.split(';')) { + const eq = part.indexOf('='); + if (eq === -1) continue; + if (part.slice(0, eq).trim() === name) return part.slice(eq + 1).trim(); + } + return null; +} + +/** + * Whether the request carries a valid-looking Supabase session: the cookie + * src/proxy.ts refreshes, holding JSON with a JWT-shaped access_token and a + * refresh_token. Shape only, no verification -- the gate runs before anything + * that could ask Supabase, and the point is that a junk cookie named + * sb-auth-token does not buy a spoofing fleet a way past the toll. + */ +export function hasSessionCookie(request: Request): boolean { + const raw = cookieValue(request, SESSION_COOKIE_NAME); + if (!raw) return false; + try { + const session = JSON.parse(decodeURIComponent(raw)) as { access_token?: unknown; refresh_token?: unknown }; + return ( + typeof session.access_token === 'string' && + JWT_RE.test(session.access_token) && + typeof session.refresh_token === 'string' && + session.refresh_token.length > 0 + ); + } catch { + return false; + } +} + +/** Whether the request carries a valid-looking v1 API bearer token. */ +export function hasApiBearer(request: Request): boolean { + return API_BEARER_RE.test(request.headers.get('authorization') ?? ''); +} + export const gateway = createGateway({ siteUrl: process.env.NEXT_PUBLIC_APP_URL ?? 'https://bittorrented.com', siteName: 'bittorrented', coinpay: { apiKey: process.env.COINPAY_X402_KEY }, payTo: process.env.CRAWL_PAY_TO, + denyCidrs: OVH_VPS_FLEET_CIDRS, + chargeSpoofedBrowsers: true, + exempt: (request) => hasSessionCookie(request) || hasApiBearer(request), }); diff --git a/src/proxy.test.ts b/src/proxy.test.ts index c7a0dba..9017c7f 100644 --- a/src/proxy.test.ts +++ b/src/proxy.test.ts @@ -29,6 +29,7 @@ describe('Bot Handling Middleware', () => { const req = new NextRequest(url, { headers: { ...(userAgent ? { 'user-agent': userAgent } : {}), + 'sec-fetch-mode': 'navigate', // what every real Chromium sends; see the edge-control tests for its absence 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, // unique IP per call to avoid rate limit state }, }); @@ -112,6 +113,7 @@ describe('Crawl Gateway (x402)', () => { const req = new NextRequest(url, { headers: { ...(userAgent ? { 'user-agent': userAgent } : {}), + 'sec-fetch-mode': 'navigate', 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, ...headers, }, @@ -189,6 +191,7 @@ describe('Supabase session refresh and referral cookie', () => { const req = new NextRequest(new URL(`http://localhost${pathname}`), { headers: { 'user-agent': ua, + 'sec-fetch-mode': 'navigate', 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, ...(cookie ? { cookie } : {}), ...headers, @@ -281,13 +284,23 @@ describe('Supabase session refresh and referral cookie', () => { }); it('answers a training crawler with 402 without touching Supabase', async () => { - const res = await call('/browse', { ua: META_UA, cookies: { 'sb-auth-token': authCookie(30) } }); + const res = await call('/browse', { ua: META_UA }); expect(res).toBeDefined(); expect(res!.status).toBe(402); expect(fetchMock).not.toHaveBeenCalled(); expect(res!.headers.get('set-cookie')).toBeNull(); }); + it('a signed-in session is never charged, whatever user agent carries it', async () => { + // The gateway's `exempt` runs before the agent lists: this site is mostly + // people, and a request that presents a real session is treated as one of + // them. It then goes through the ordinary session refresh like any browser. + const res = await call('/browse', { ua: META_UA, cookies: { 'sb-auth-token': authCookie(30), 'x-profile-id': 'p1' } }); + expectPassThrough(res); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(JSON.parse(decodeURIComponent(res.cookies.get('sb-auth-token')!.value)).refresh_token).toBe('new-refresh'); + }); + it('stores a valid ?ref= code in the referral_code cookie', async () => { const res = await call('/browse?ref=ABC-123_x'); expectPassThrough(res); @@ -309,3 +322,136 @@ describe('Supabase session refresh and referral cookie', () => { expect(JSON.parse(decodeURIComponent(res.cookies.get('sb-auth-token')!.value)).refresh_token).toBe('new-refresh'); }); }); + +describe('Edge controls: hosting ranges and spoofed browsers', () => { + const CHROME_UA = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'; + const FIREFOX_UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0'; + const GOOGLEBOT_EVERGREEN_UA = + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/148.0.0.0 Safari/537.36'; + const BINGBOT_UA = + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/148.0.0.0 Safari/537.36'; + + /** + * A Supabase session cookie of the shape src/proxy.ts refreshes. Fixture, + * not a credential: the access token is an unsigned JWT (`alg: none`) that + * expired in 1970 and the refresh token is one letter. The gate checks shape + * only and never verifies either, which is exactly what these tests pin. + */ + // threatcrush-disable-next-line secret-jwt + const FIXTURE_UNSIGNED_EXPIRED_JWT = 'eyJhbGciOiJub25lIn0.eyJleHAiOjB9.sig'; + // threatcrush-disable-next-line secret-generic-credential + const FIXTURE_REFRESH = 'r'; + const SESSION_COOKIE = `sb-auth-token=${encodeURIComponent( + JSON.stringify({ access_token: FIXTURE_UNSIGNED_EXPIRED_JWT, refresh_token: FIXTURE_REFRESH }) + )}`; + const API_BEARER = `Bearer btr_${'ab'.repeat(32)}`; + + /** Exactly the headers given, nothing implied: these tests are about what is missing. */ + function raw(pathname: string, headers: Record) { + const h: Record = { 'x-forwarded-for': `${Math.random().toString(36).slice(2)}.1.1.1`, ...headers }; + return middleware(new NextRequest(new URL(`http://localhost${pathname}`), { headers: h })); + } + + describe('denyCidrs (OVH VPS fleet)', () => { + it('refuses a request whose last x-forwarded-for hop is in an OVH range, even a well-formed browser', async () => { + const res = await raw('/browse', { + 'user-agent': CHROME_UA, + 'sec-fetch-mode': 'navigate', + 'x-forwarded-for': '203.0.113.9, 51.38.12.34', + }); + expect(res).toBeDefined(); + expect(res!.status).toBe(403); + expect(await res!.text()).toContain('Not available from this network'); + }); + + it('refuses by x-real-ip too', async () => { + const res = await raw('/browse', { 'user-agent': CHROME_UA, 'sec-fetch-mode': 'navigate', 'x-real-ip': '145.239.200.1' }); + expect(res!.status).toBe(403); + }); + + it('judges the LAST hop only: a client-seeded first hop cannot get anyone refused', async () => { + const res = await raw('/browse', { + 'user-agent': CHROME_UA, + 'sec-fetch-mode': 'navigate', + 'x-forwarded-for': '51.38.12.34, 203.0.113.9', + }); + expectPassThrough(res); + }); + + it('covers every listed range', async () => { + for (const ip of ['51.38.1.1', '54.38.1.1', '141.94.1.1', '145.239.1.1', '149.202.1.1', '151.80.1.1', '57.129.1.1', '213.32.1.1']) { + const res = await raw('/', { 'user-agent': CHROME_UA, 'sec-fetch-mode': 'navigate', 'x-forwarded-for': ip }); + expect(res!.status, ip).toBe(403); + } + }); + }); + + describe('chargeSpoofedBrowsers', () => { + it('charges a Chrome user agent that sends no Sec-Fetch-Mode', async () => { + const res = await raw('/browse', { 'user-agent': CHROME_UA }); + expect(res).toBeDefined(); + expect(res!.status).toBe(402); + expect(res!.headers.get('content-type')).toContain('application/json'); + }); + + it('passes the same Chrome user agent with Sec-Fetch-Mode to the existing behaviour', async () => { + const res = await raw('/browse', { 'user-agent': CHROME_UA, 'sec-fetch-mode': 'navigate' }); + expectPassThrough(res); + }); + + it('never judges Googlebot\'s evergreen Chrome string', async () => { + const res = await raw('/browse', { 'user-agent': GOOGLEBOT_EVERGREEN_UA }); + expectPassThrough(res); + }); + + it('never judges Bingbot\'s evergreen Chrome string', async () => { + const res = await raw('/browse', { 'user-agent': BINGBOT_UA }); + expectPassThrough(res); + }); + + it('never judges Firefox, which older builds send without Sec-Fetch', async () => { + const res = await raw('/browse', { 'user-agent': FIREFOX_UA }); + expectPassThrough(res); + }); + + it('still lets a spoofed browser read robots.txt and the sales page', async () => { + expectPassThrough(await raw('/robots.txt', { 'user-agent': CHROME_UA })); + const sales = await raw('/crawl', { 'user-agent': CHROME_UA, accept: 'text/html' }); + expect(sales!.status).toBe(402); + expect(sales!.headers.get('content-type')).toContain('text/html'); + }); + }); + + describe('exempt: signed-in people and API clients are never charged', () => { + it('passes a Chrome request without Sec-Fetch when it carries a Supabase session cookie', async () => { + const res = await raw('/browse', { 'user-agent': CHROME_UA, cookie: `${SESSION_COOKIE}; x-profile-id=p1` }); + expectPassThrough(res); + }); + + it('does not accept a junk cookie merely named sb-auth-token', async () => { + const res = await raw('/browse', { 'user-agent': CHROME_UA, cookie: 'sb-auth-token=not-a-session' }); + expect(res!.status).toBe(402); + }); + + it('passes a Chrome request without Sec-Fetch when it carries a valid-looking btr_ API bearer token', async () => { + const res = await raw('/api/v1/me', { 'user-agent': CHROME_UA, authorization: API_BEARER }); + expectPassThrough(res); + }); + + it('does not accept a bearer token of the wrong shape', async () => { + const res = await raw('/api/v1/me', { 'user-agent': CHROME_UA, authorization: 'Bearer btr_short' }); + expect(res!.status).toBe(402); + }); + + it('a session does not get a hosting range past the 403', async () => { + const res = await raw('/browse', { + 'user-agent': CHROME_UA, + 'sec-fetch-mode': 'navigate', + cookie: SESSION_COOKIE, + 'x-forwarded-for': '149.202.3.4', + }); + expect(res!.status).toBe(403); + }); + }); +});