diff --git a/.gitignore b/.gitignore index a08548b..827a83b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ logs .env.* !.env.example *.local +.dev.vars* # Editors .idea diff --git a/.oxlintrc.json b/.oxlintrc.json index 6c35564..14f52b6 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -457,10 +457,16 @@ // React Three Fiber renders three.js objects as JSX, so its props // (position, args, intensity, attach, ...) are not DOM attributes. Scoped to // the renderer package and the app's Canvas only. + // A module Worker is its default export ({ fetch }); Cloudflare finds no + // handler otherwise. Scoped to the one entry file wrangler.jsonc names. "overrides": [ { "files": ["packages/renderer/src/**/*.tsx", "apps/web/src/app.tsx"], "rules": { "react/no-unknown-property": "off" } + }, + { + "files": ["apps/web/src/api/worker.ts"], + "rules": { "import/no-default-export": "off" } } ] } diff --git a/AGENTS.md b/AGENTS.md index dce361d..e794661 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,9 +10,9 @@ Guidelines for AI agents and contributors working in this Turborepo monorepo. ### Apps (`apps/`) -| Name | Filter | Description | -| ---- | ------ | ------------------------------------------------------------------------------------------------ | -| web | `web` | Code Backrooms: Vite 8 + React 19 + React Three Fiber first-person app, deployable to Cloudflare | +| Name | Filter | Description | +| ---- | ------ | -------------------------------------------------------------------------------------------------------------------------- | +| web | `web` | Code Backrooms: Vite 8 + React 19 + React Three Fiber first-person app, and its Cloudflare Worker that fetches GitHub zips | ### Packages (`packages/`) diff --git a/README.md b/README.md index 1b20929..438b8cc 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,8 @@ push to `main` and on PRs, plus CodeQL code scanning. `secrets:scan` needs eithe - JavaScript and TypeScript only. - Up to 400 source files and 8 MB per repository. - Calls through objects (`obj.method()`) and callbacks show as closed doors. -- Loading straight from a GitHub URL is not supported yet. +- GitHub URLs load public repositories only, and branch names containing `/` are not + supported. ## License diff --git a/apps/web/AGENTS.md b/apps/web/AGENTS.md index 2a63ea9..d49a1e8 100644 --- a/apps/web/AGENTS.md +++ b/apps/web/AGENTS.md @@ -169,10 +169,8 @@ not be laid out`. The HUD reads `2/7 modules · last 14 ms` on the way; local files never writes `?seed=`, since no URL can name those files. The filter keeps a superset of the `?code=self` glob (config modules such as `vite.config.ts`), so a zip of this repository has a few more modules - than Demo. Fetching a GitHub URL is not supported; it would need a - Cloudflare Worker proxy (`codeload` sends no CORS headers) feeding - `zip-files.ts`. -- Milestone 21 (current): a moving landing page. Behind the panel the + than Demo. A public GitHub repository loads by URL (Milestone 22). +- Milestone 21: a moving landing page. Behind the panel the bundled `demo` example's entrance is drawn with a camera that drifts toward its first doorway and back while its gaze sways (`landing-backdrop.tsx`, `driftPose` in `backdrop.ts`). It is skipped @@ -180,23 +178,67 @@ not be laid out`. The HUD reads `2/7 modules · last 14 ms` on the way; boundary if the canvas fails, and dropped as soon as a read starts. The world view (`world-app.tsx`) and the backdrop are lazy chunks, so the landing page paints before three.js loads. +- Milestone 22 (current): a public GitHub repository by URL. The landing + page takes `github.com/owner/repo`, `owner/repo` or a + `/tree//` URL (`github-url.ts`; refs with `/` are not + supported). One load is one request to GitHub: the page calls this site's + Worker (`api/`, `GET /api/github/zip`), which fetches the repository's + zip from codeload.github.com, since codeload sends no CORS headers, and + streams it back. The page then reads it with `filesFromZip`, keeping only + the folder's files when one is named. The Worker builds the codeload URL + from checked names and is not an open proxy. Before it fetches, the + visitor must pass a Cloudflare Turnstile check (`turnstile.tsx`, loaded + only once a URL is typed). Each IP is rate-limited to 5 loads a minute. + Answers are edge-cached for 10 minutes, and zips over 64 MB are refused or + cut off. A loaded repository writes `?repo=` so a reload or a shared link + fills the field in again. The visitor still presses open, because each + Turnstile token is single-use. Without a site key at build time the field + is hidden, so production never falls back to the test key. --- ## Workspace Commands -| Task | Command | -| --------- | ----------------------------- | -| Dev | `pnpm --filter web dev` | -| Build | `pnpm --filter web build` | -| Preview | `pnpm --filter web preview` | -| Typecheck | `pnpm --filter web typecheck` | -| Test | `pnpm --filter web test` | -| Clean | `pnpm --filter web clean` | -| Deploy | `pnpm --filter web deploy` | +| Task | Command | +| ---------- | ------------------------------ | +| Task | Command | +| ---------- | ------------------------------ | +| Dev | `pnpm --filter web dev` | +| Dev Worker | `pnpm --filter web dev:worker` | +| Build | `pnpm --filter web build` | +| Preview | `pnpm --filter web preview` | +| Typecheck | `pnpm --filter web typecheck` | +| Test | `pnpm --filter web test` | +| Clean | `pnpm --filter web clean` | +| Deploy | `pnpm --filter web deploy` | Or run repo-wide via root `pnpm typecheck` / `pnpm test` / `pnpm build`. +### GitHub loading in development + +- `dev` alone serves everything except `/api/*`. To load from GitHub, also run + `dev:worker` (`wrangler dev` on port 8787), which Vite proxies `/api` to. +- Dev builds use Turnstile's public always-pass test site key. `wrangler dev` + needs the matching test secret in `apps/web/.dev.vars`, which is gitignored: + `TURNSTILE_SECRET=1x0000000000000000000000000000000AA`. + +### Deploying (Cloudflare Workers Builds, repository connected in the dashboard) + +- **Build settings:** set the root directory to the repository root. The build + command is `pnpm --filter web build`, and the deploy command is + `pnpm --filter web exec wrangler deploy`. +- **Build variables:** `PNPM_VERSION=12.4.2` and `NODE_VERSION=24.21.0`, since the + build image's pnpm default is older. Also set `VITE_TURNSTILE_SITE_KEY` to the + widget's site key. Vite inlines it at build time, so it is a _build_ variable. +- **Runtime secret:** `TURNSTILE_SECRET` under the Worker's Variables and Secrets. + `wrangler.jsonc` declares no value for it, so deploys keep it. +- **Turnstile widget:** create it in the Cloudflare dashboard. Its hostnames must + include the production domain, plus the `workers.dev` domain if preview builds + are on. A preview without its hostname refuses GitHub loads and everything else + still works. +- **Committed values:** the site key, the rate-limit `namespace_id` and the Worker + code are public by design. The secret never goes in the repository. + There are no per-workspace `lint` or `format` scripts: root `pnpm lint` (oxlint) and `pnpm format` (Prettier) each cover the whole repo. @@ -210,7 +252,14 @@ src/ app.tsx landing page, loading screen, or the world as a lazy chunk world-app.tsx the world: full-screen , seed and current-room state, pointer lock, HUD - landing.tsx the bare URL's page: zip drop and pickers, Demo, examples + landing.tsx the bare URL's page: zip drop and pickers, GitHub URL, Demo, examples + github-url.ts a GitHub URL -> owner, repo, ref, folder; checks shared with the Worker, pure + github-files.ts a GitHub repository -> sources: one call to the Worker, then zip-files + github-form.tsx the landing page's GitHub URL field, check and open button, markup only + turnstile.tsx the Turnstile check before a GitHub load, script loaded on demand + api/ the Cloudflare Worker, run only for /api/* (wrangler.jsonc `main`) + worker.ts the entry: routes /api/github/zip, else the static assets + github-zip.ts the route: checks, rate limit, Turnstile, one codeload fetch landing-backdrop.tsx the demo world drifting behind the landing page, lazy backdrop.ts when the backdrop runs and where its camera is, pure backdrop-world.ts the demo world for the backdrop, every area built diff --git a/apps/web/README.md b/apps/web/README.md index 0f434c4..1392a80 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -26,6 +26,14 @@ A bare URL opens the landing page. Drop or pick a `.zip` (GitHub's tab and never uploaded. Only JavaScript and TypeScript sources and manifests are kept, and dependency, build and test directories are skipped. +You can also paste a public GitHub repository URL, such as +`github.com/owner/repo` or `github.com/owner/repo/tree/main/src` for one +folder. After a Cloudflare Turnstile check, this site's Worker downloads the +repository's zip from GitHub in one request and the page reads it like a +dropped zip. `?repo=owner/repo` fills the field in. Running this locally +needs `pnpm --filter web dev:worker` beside `dev`. See `AGENTS.md` for that +and for the deploy settings. + | Limit | Value | | ------------------- | ------ | | Zip size | 64 MB | diff --git a/apps/web/package.json b/apps/web/package.json index 99ab001..e3f4403 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "dev": "vite --port 3000", + "dev:worker": "mkdir -p dist && wrangler dev --port 8787", "build": "vite build", "preview": "vite preview", "deploy": "pnpm build && wrangler deploy", diff --git a/apps/web/public/_headers b/apps/web/public/_headers index 9f4a5ff..2c6d152 100644 --- a/apps/web/public/_headers +++ b/apps/web/public/_headers @@ -2,10 +2,12 @@ # The CSP allows Cloudflare Web Analytics / Browser RUM: the injected beacon script comes from # static.cloudflareinsights.com and reports to /cdn-cgi/rum on this origin or cloudflareinsights.com. # style-src 'self' is enough: React applies `style` props through the CSSOM, which CSP doesn't block. +# Cloudflare Turnstile (the check before a GitHub load) runs a script and an iframe from +# challenges.cloudflare.com. The GitHub zip itself comes from this origin's Worker (/api/*). # Loading anything else from another origin (scripts, fonts, APIs) needs an entry here. /* - Content-Security-Policy: default-src 'self'; script-src 'self' https://static.cloudflareinsights.com; connect-src 'self' https://cloudflareinsights.com; img-src 'self' data:; style-src 'self'; font-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; upgrade-insecure-requests + Content-Security-Policy: default-src 'self'; script-src 'self' https://static.cloudflareinsights.com https://challenges.cloudflare.com; frame-src https://challenges.cloudflare.com; connect-src 'self' https://cloudflareinsights.com; img-src 'self' data:; style-src 'self'; font-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; upgrade-insecure-requests X-Content-Type-Options: nosniff Referrer-Policy: strict-origin-when-cross-origin Permissions-Policy: camera=(), microphone=(), geolocation=() diff --git a/apps/web/src/api/github-zip.test.ts b/apps/web/src/api/github-zip.test.ts new file mode 100644 index 0000000..271ba3d --- /dev/null +++ b/apps/web/src/api/github-zip.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it, vi } from "vitest"; + +import { handleZip as handle } from "./github-zip"; +import type { Fetch, ZipEnv } from "./github-zip"; + +const SITEVERIFY = "https://challenges.cloudflare.com/turnstile/v0/siteverify"; + +const env = (overrides: Partial = {}): ZipEnv => ({ + GITHUB_LIMITER: { limit: vi.fn(() => Promise.resolve({ success: true })) }, + TURNSTILE_SECRET: "secret", + ...overrides, +}); + +/** Turnstile answers `verified`; codeload answers `zip`. */ +const upstream = ( + zip: () => Response, + verified = true +): ReturnType> => + vi.fn((url) => + Promise.resolve( + url === SITEVERIFY ? Response.json({ success: verified }) : zip() + ) + ); + +const zipRequest = ( + query: string, + headers: Record = { "CF-Turnstile-Token": "token" } +): Request => + new Request(`https://backrooms.test/api/github/zip?${query}`, { + headers: { "CF-Connecting-IP": "203.0.113.7", ...headers }, + }); + +const codeloadCalls = (fetcher: ReturnType>): string[] => + fetcher.mock.calls + .map(([url]) => url) + .filter((url) => url.startsWith("https://codeload.github.com/")); + +const errorOf = async (response: Response): Promise => + ((await response.json()) as { error: unknown }).error; + +describe("the GitHub zip route", () => { + it("fetches one zip from codeload, cached, and streams it back", async () => { + const fetcher = upstream(() => new Response(new Uint8Array([1, 2, 3]))); + const response = await handle( + zipRequest("owner=pmndrs&repo=zustand&ref=v4.0.0"), + env(), + fetcher + ); + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("application/zip"); + expect(new Uint8Array(await response.arrayBuffer())).toEqual( + new Uint8Array([1, 2, 3]) + ); + expect(codeloadCalls(fetcher)).toEqual([ + "https://codeload.github.com/pmndrs/zustand/zip/v4.0.0", + ]); + const codeloadInit = fetcher.mock.calls.find(([url]) => + url.startsWith("https://codeload") + )?.[1]; + expect(codeloadInit?.cf).toEqual({ cacheEverything: true, cacheTtl: 600 }); + }); + + it("uses HEAD without a ref", async () => { + const fetcher = upstream(() => new Response("zip")); + await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(codeloadCalls(fetcher)).toEqual([ + "https://codeload.github.com/o/r/zip/HEAD", + ]); + }); + + it("sends the secret, token and visitor IP to Turnstile", async () => { + const fetcher = upstream(() => new Response("zip")); + await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + const body = fetcher.mock.calls.find(([url]) => url === SITEVERIFY)?.[1] + ?.body; + expect(body instanceof FormData ? Object.fromEntries(body) : body).toEqual({ + secret: "secret", + response: "token", + remoteip: "203.0.113.7", + }); + }); + + it.each([ + "owner=o", + "owner=o&repo=r&ref=..", + "owner=o&repo=r&ref=a%2Fb", + "owner=o%2F..&repo=r", + "owner=o&repo=..", + "owner=o&repo=r%3Fx%3D1", + "owner=evil.com&repo=r", + ])("refuses %s before contacting anyone", async (query) => { + const fetcher = upstream(() => new Response("zip")); + const limiter = env(); + const response = await handle(zipRequest(query), limiter, fetcher); + expect(response.status).toBe(400); + expect(fetcher).not.toHaveBeenCalled(); + expect(limiter.GITHUB_LIMITER.limit).not.toHaveBeenCalled(); + }); + + it("refuses a request without a token, never reaching GitHub", async () => { + const fetcher = upstream(() => new Response("zip")); + const response = await handle( + zipRequest("owner=o&repo=r", {}), + env(), + fetcher + ); + expect(response.status).toBe(403); + expect(fetcher).not.toHaveBeenCalled(); + }); + + it("refuses a token Turnstile rejects, never reaching GitHub", async () => { + const fetcher = upstream(() => new Response("zip"), false); + const response = await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(response.status).toBe(403); + expect(codeloadCalls(fetcher)).toEqual([]); + }); + + it("fails closed when Turnstile cannot be reached", async () => { + const fetcher = vi.fn(() => Promise.reject(new Error("offline"))); + const response = await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(response.status).toBe(403); + expect(fetcher).toHaveBeenCalledTimes(1); + }); + + it("refuses when no secret is configured", async () => { + const fetcher = upstream(() => new Response("zip")); + const response = await handle( + zipRequest("owner=o&repo=r"), + env({ TURNSTILE_SECRET: undefined }), + fetcher + ); + expect(response.status).toBe(503); + expect(fetcher).not.toHaveBeenCalled(); + }); + + it("rate-limits by visitor IP before verifying anything", async () => { + const fetcher = upstream(() => new Response("zip")); + const limited = env({ + GITHUB_LIMITER: { + limit: vi.fn(() => Promise.resolve({ success: false })), + }, + }); + const response = await handle( + zipRequest("owner=o&repo=r"), + limited, + fetcher + ); + expect(response.status).toBe(429); + expect(limited.GITHUB_LIMITER.limit).toHaveBeenCalledWith({ + key: "203.0.113.7", + }); + expect(fetcher).not.toHaveBeenCalled(); + }); + + it("says a missing repository is not found or private", async () => { + const fetcher = upstream(() => new Response("", { status: 404 })); + const response = await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(response.status).toBe(404); + expect(await errorOf(response)).toBe("o/r not found, or it is private"); + }); + + it("passes other GitHub failures on as 502", async () => { + const fetcher = upstream(() => new Response("", { status: 503 })); + const response = await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(response.status).toBe(502); + }); + + it("refuses a zip whose length is over 64 MB", async () => { + const fetcher = upstream( + () => + new Response("zip", { + headers: { "Content-Length": String(64 * 1024 * 1024 + 1) }, + }) + ); + const response = await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(response.status).toBe(413); + }); + + it("breaks off a zip without a length once it passes 64 MB", async () => { + const chunk = new Uint8Array(16 * 1024 * 1024); + let sent = 0; + const endless = new ReadableStream({ + pull(controller) { + sent += 1; + controller.enqueue(chunk); + }, + }); + const fetcher = upstream(() => new Response(endless)); + const response = await handle(zipRequest("owner=o&repo=r"), env(), fetcher); + expect(response.status).toBe(200); + await expect(response.arrayBuffer()).rejects.toThrow(); + expect(sent).toBeLessThanOrEqual(6); + }); + + it("allows GET only", async () => { + const fetcher = upstream(() => new Response("zip")); + const response = await handle( + new Request("https://backrooms.test/api/github/zip?owner=o&repo=r", { + method: "POST", + }), + env(), + fetcher + ); + expect(response.status).toBe(405); + expect(fetcher).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/api/github-zip.ts b/apps/web/src/api/github-zip.ts new file mode 100644 index 0000000..02e5135 --- /dev/null +++ b/apps/web/src/api/github-zip.ts @@ -0,0 +1,158 @@ +import { isOwner, isRef, isRepo, TOKEN_HEADER } from "~/github-url"; + +// `GET /api/github/zip`: fetches a public repository's zip from +// codeload.github.com (which sends no CORS headers, so the page cannot) and +// streams it back unchanged. +// +// One load is one request to GitHub, and only after the visitor passed a +// Turnstile challenge and a per-IP rate limit. Responses are cached at the +// edge for ten minutes, so repeat loads of a repository never reach GitHub. +// The route builds the codeload URL itself from checked names: it is not an +// open proxy. + +/** The bindings this route uses; the secret is set in Cloudflare. */ +type ZipEnv = { + readonly GITHUB_LIMITER: { + readonly limit: (options: { + readonly key: string; + }) => Promise<{ readonly success: boolean }>; + }; + readonly TURNSTILE_SECRET?: string; +}; + +/** Workers' `fetch` options that the DOM's `RequestInit` does not declare. */ +type CloudflareInit = RequestInit & { + readonly cf?: { + readonly cacheEverything: boolean; + readonly cacheTtl: number; + }; +}; + +type Fetch = (url: string, init?: CloudflareInit) => Promise; + +const SITEVERIFY = "https://challenges.cloudflare.com/turnstile/v0/siteverify"; +/** The page refuses bigger zips too (`MAX_ZIP_BYTES` in `zip-files.ts`). */ +const MAX_ZIP_BYTES = 64 * 1024 * 1024; +const CACHE_SECONDS = 600; + +const failure = (status: number, error: string): Response => + Response.json( + { error }, + { status, headers: { "Cache-Control": "no-store" } } + ); + +/** Whether Turnstile accepts the visitor's token. Fails closed. */ +const verifyToken = async ( + fetcher: Fetch, + secret: string, + token: string, + ip: string | null +): Promise => { + const body = new FormData(); + body.set("secret", secret); + body.set("response", token); + if (ip !== null) { + body.set("remoteip", ip); + } + try { + const response = await fetcher(SITEVERIFY, { method: "POST", body }); + if (!response.ok) { + return false; + } + const outcome: unknown = await response.json(); + return ( + typeof outcome === "object" && + outcome !== null && + "success" in outcome && + outcome.success === true + ); + } catch { + return false; + } +}; + +/** + * Passes `body` through, erroring the stream once it carries more than + * `limit` bytes. The page's read then fails instead of parsing a cut zip. + */ +const capped = ( + body: ReadableStream, + limit: number +): ReadableStream => { + let seen = 0; + return body.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + seen += chunk.byteLength; + if (seen > limit) { + controller.error(new Error("zip over the size limit")); + return; + } + controller.enqueue(chunk); + }, + }) + ); +}; + +/** `GET /api/github/zip?owner=&repo=[&ref=]`, the Turnstile token in a header. */ +const handleZip = async ( + request: Request, + env: ZipEnv, + fetcher: Fetch +): Promise => { + if (request.method !== "GET") { + return failure(405, "only GET is supported"); + } + const query = new URL(request.url).searchParams; + const owner = query.get("owner") ?? ""; + const repo = query.get("repo") ?? ""; + const ref = query.get("ref") ?? "HEAD"; + if (!isOwner(owner) || !isRepo(repo) || !isRef(ref)) { + return failure(400, "not a GitHub repository name"); + } + const ip = request.headers.get("CF-Connecting-IP"); + const { success } = await env.GITHUB_LIMITER.limit({ key: ip ?? "unknown" }); + if (!success) { + return failure(429, "too many repositories loaded: wait a minute"); + } + const secret = env.TURNSTILE_SECRET; + if (secret === undefined || secret === "") { + return failure(503, "loading from GitHub is not set up on this server"); + } + const token = request.headers.get(TOKEN_HEADER); + if (token === null || token === "") { + return failure(403, "the verification check is missing: try again"); + } + if (!(await verifyToken(fetcher, secret, token, ip))) { + return failure(403, "the verification check failed: try again"); + } + const url = `https://codeload.github.com/${owner}/${repo}/zip/${ref}`; + let upstream: Response; + try { + upstream = await fetcher(url, { + cf: { cacheEverything: true, cacheTtl: CACHE_SECONDS }, + }); + } catch { + return failure(502, "GitHub could not be reached"); + } + if (upstream.status === 404) { + return failure(404, `${owner}/${repo} not found, or it is private`); + } + if (!upstream.ok || upstream.body === null) { + return failure(502, `GitHub answered ${upstream.status}`); + } + const length = Number(upstream.headers.get("Content-Length") ?? "0"); + if (length > MAX_ZIP_BYTES) { + return failure(413, "the repository's zip is over 64 MB"); + } + return new Response(capped(upstream.body, MAX_ZIP_BYTES), { + headers: { + "Content-Type": "application/zip", + "Cache-Control": "no-store", + "X-Content-Type-Options": "nosniff", + }, + }); +}; + +export { handleZip }; +export type { Fetch, ZipEnv }; diff --git a/apps/web/src/api/worker.test.ts b/apps/web/src/api/worker.test.ts new file mode 100644 index 0000000..017b2ef --- /dev/null +++ b/apps/web/src/api/worker.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { Fetch } from "./github-zip"; +import { handle } from "./worker"; +import type { Env } from "./worker"; + +const env = (): Env => ({ + ASSETS: { fetch: vi.fn(() => Promise.resolve(new Response("asset"))) }, + GITHUB_LIMITER: { limit: vi.fn(() => Promise.resolve({ success: true })) }, + TURNSTILE_SECRET: "secret", +}); + +describe("the Worker", () => { + it("sends the zip route to its handler, not the assets", async () => { + const bindings = env(); + const fetcher = vi.fn(() => Promise.resolve(new Response("zip"))); + // No token: the route answers 403 itself without any upstream call. + const response = await handle( + new Request("https://backrooms.test/api/github/zip?owner=o&repo=r"), + bindings, + fetcher + ); + expect(response.status).toBe(403); + expect(bindings.ASSETS.fetch).not.toHaveBeenCalled(); + }); + + it("sends every other path to the static assets", async () => { + const bindings = env(); + const fetcher = vi.fn(() => Promise.resolve(new Response("zip"))); + for (const path of ["/index.html", "/api/other", "/api/github/zip/x"]) { + const response = await handle( + new Request(`https://backrooms.test${path}`), + bindings, + fetcher + ); + expect(await response.text()).toBe("asset"); + } + expect(bindings.ASSETS.fetch).toHaveBeenCalledTimes(3); + expect(fetcher).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/api/worker.ts b/apps/web/src/api/worker.ts new file mode 100644 index 0000000..82a86d1 --- /dev/null +++ b/apps/web/src/api/worker.ts @@ -0,0 +1,31 @@ +import { handleZip } from "~/api/github-zip"; +import type { Fetch, ZipEnv } from "~/api/github-zip"; +import { ZIP_ROUTE } from "~/github-url"; + +// The Cloudflare Worker in front of the static app, named by `main` in +// `wrangler.jsonc`. `run_worker_first` sends it only `/api/*`; everything +// else is served from the static assets without running it. Routes live +// beside this file, one per module. + +/** The bindings `wrangler.jsonc` declares. */ +type Env = ZipEnv & { + readonly ASSETS: { readonly fetch: (request: Request) => Promise }; +}; + +/** The API's routes, else the static assets. */ +const handle = ( + request: Request, + env: Env, + fetcher: Fetch +): Promise => + new URL(request.url).pathname === ZIP_ROUTE + ? handleZip(request, env, fetcher) + : env.ASSETS.fetch(request); + +export default { + fetch: (request: Request, env: Env): Promise => + handle(request, env, (url, init) => fetch(url, init)), +}; + +export { handle }; +export type { Env }; diff --git a/apps/web/src/app.tsx b/apps/web/src/app.tsx index 8f785a2..7efbcd4 100644 --- a/apps/web/src/app.tsx +++ b/apps/web/src/app.tsx @@ -3,6 +3,7 @@ import { lazy, Suspense, useState } from "react"; import { Landing } from "~/landing"; import { labelOfRequest, + repoOfSearch, requestFromParams, showsLanding, } from "~/source-request"; @@ -23,7 +24,12 @@ const App = () => { const [request, setRequest] = useState(() => requestFromParams(initial)); const setup = useWorldSetup(request); if (showsLanding(request, globalThis.location.search)) { - return ; + return ( + + ); } const loading = ( Promise; + +const zipOf = (files: Readonly>): Uint8Array => + zipSync( + Object.fromEntries( + Object.entries(files).map(([name, text]) => [name, strToU8(text)]) + ) + ); + +const zipResponse = (bytes: Uint8Array): Response => + new Response(new Uint8Array(bytes), { + headers: { "Content-Type": "application/zip" }, + }); + +const REPO: GitHubTarget = { + owner: "o", + repo: "r", + ref: null, + subpath: null, +}; + +const GITHUB_ZIP = zipOf({ + "r-0123abc/package.json": '{ "name": "r" }', + "r-0123abc/src/a.ts": "export const a = 1;", + "r-0123abc/packages/p/src/b.ts": "export const b = 2;", +}); + +const failureMessage = async (promise: Promise): Promise => { + try { + await promise; + } catch (error) { + if (error instanceof LocalLoadError) { + return error.message; + } + throw error; + } + throw new Error("Expected a LocalLoadError"); +}; + +describe("readGitHubRepo", () => { + it("makes one request to the Worker with the token, then reads the zip", async () => { + const fetch = vi.fn(() => Promise.resolve(zipResponse(GITHUB_ZIP))); + const result = await readGitHubRepo({ ...REPO, ref: "main" }, "token", { + fetch, + }); + expect(fetch).toHaveBeenCalledTimes(1); + const [url, init] = fetch.mock.calls[0] ?? []; + expect(url).toBe("/api/github/zip?owner=o&repo=r&ref=main"); + expect(init?.headers).toEqual({ "CF-Turnstile-Token": "token" }); + expect(result.files.map((file) => file.path)).toEqual([ + "package.json", + "packages/p/src/b.ts", + "src/a.ts", + ]); + }); + + it("leaves the ref out for the default branch", async () => { + const fetch = vi.fn(() => Promise.resolve(zipResponse(GITHUB_ZIP))); + await readGitHubRepo(REPO, "token", { fetch }); + expect(fetch.mock.calls[0]?.[0]).toBe("/api/github/zip?owner=o&repo=r"); + }); + + it("reads only the files under a subpath, relative to it", async () => { + const fetch = vi.fn(() => Promise.resolve(zipResponse(GITHUB_ZIP))); + const { files } = await readGitHubRepo( + { ...REPO, subpath: "packages/p" }, + "token", + { fetch } + ); + expect(files).toEqual([{ path: "b.ts", source: "export const b = 2;" }]); + }); + + it("names a subpath the repository does not have", async () => { + const fetch = vi.fn(() => Promise.resolve(zipResponse(GITHUB_ZIP))); + expect( + await failureMessage( + readGitHubRepo({ ...REPO, subpath: "nope" }, "token", { fetch }) + ) + ).toBe("no folder nope in the repository"); + }); + + it("shows the Worker's error message", async () => { + const fetch = vi.fn(() => + Promise.resolve( + Response.json( + { error: "o/r not found, or it is private" }, + { status: 404 } + ) + ) + ); + expect(await failureMessage(readGitHubRepo(REPO, "t", { fetch }))).toBe( + "o/r not found, or it is private" + ); + }); + + it("explains a failure without the Worker's JSON", async () => { + const fetch = vi.fn(() => + Promise.resolve(new Response("Bad Gateway", { status: 502 })) + ); + expect(await failureMessage(readGitHubRepo(REPO, "t", { fetch }))).toBe( + "loading from GitHub failed (502): is the Worker running?" + ); + }); + + it("refuses a page that is not a zip, as Vite's dev server sends", async () => { + const fetch = vi.fn(() => + Promise.resolve( + new Response("", { + headers: { "Content-Type": "text/html" }, + }) + ) + ); + expect(await failureMessage(readGitHubRepo(REPO, "t", { fetch }))).toMatch( + /^this server cannot load from GitHub/ + ); + }); + + it("reports a download that broke off", async () => { + const broken = new ReadableStream({ + start(controller) { + controller.error(new Error("cut")); + }, + }); + const fetch = vi.fn(() => + Promise.resolve( + new Response(broken, { headers: { "Content-Type": "application/zip" } }) + ) + ); + expect(await failureMessage(readGitHubRepo(REPO, "t", { fetch }))).toBe( + "the download from GitHub broke off: the zip may be over 64 MB" + ); + }); + + it("reports an unreachable site as a load error", async () => { + const fetch = vi.fn(() => Promise.reject(new TypeError("offline"))); + expect(await failureMessage(readGitHubRepo(REPO, "t", { fetch }))).toBe( + "could not reach this site's GitHub loader" + ); + }); + + it("passes an abort through as is", async () => { + const controller = new AbortController(); + const fetch = vi.fn(() => { + controller.abort(); + return Promise.reject(new DOMException("aborted", "AbortError")); + }); + await expect( + readGitHubRepo(REPO, "t", { fetch, signal: controller.signal }) + ).rejects.not.toBeInstanceOf(LocalLoadError); + }); +}); diff --git a/apps/web/src/github-files.ts b/apps/web/src/github-files.ts new file mode 100644 index 0000000..fe3d7af --- /dev/null +++ b/apps/web/src/github-files.ts @@ -0,0 +1,90 @@ +import { TOKEN_HEADER, ZIP_ROUTE } from "~/github-url"; +import type { GitHubTarget } from "~/github-url"; +import { LocalLoadError } from "~/source-filter"; +import { filesFromZip, MAX_ZIP_BYTES } from "~/zip-files"; +import type { LocalFiles } from "~/zip-files"; + +// A public GitHub repository to the files of a code world: one request to +// this site's Worker (`api/github-zip.ts`), which fetches the repository's zip from +// GitHub, then the same zip reader as a dropped file. + +type Fetch = (url: string, init: RequestInit) => Promise; + +/** The Worker's route for `target`, relative to this site. */ +const zipUrl = ({ owner, repo, ref }: GitHubTarget): string => { + const query = new URLSearchParams({ owner, repo }); + if (ref !== null) { + query.set("ref", ref); + } + return `${ZIP_ROUTE}?${query.toString()}`; +}; + +/** The Worker's `{ error }` message, else one for the status alone. */ +const failureOf = async (response: Response): Promise => { + try { + const body: unknown = await response.json(); + if ( + typeof body === "object" && + body !== null && + "error" in body && + typeof body.error === "string" + ) { + return new LocalLoadError(body.error); + } + } catch { + // Not the Worker's JSON: a dev server without it, or a proxy's page. + } + return new LocalLoadError( + `loading from GitHub failed (${response.status}): is the Worker running?` + ); +}; + +/** + * The repository's sources, under `target.subpath` when one is given. + * `token` is the visitor's Turnstile token, used once. + */ +const readGitHubRepo = async ( + target: GitHubTarget, + token: string, + { fetch, signal }: { readonly fetch: Fetch; readonly signal?: AbortSignal } +): Promise => { + let response: Response; + try { + response = await fetch(zipUrl(target), { + headers: { [TOKEN_HEADER]: token }, + signal, + }); + } catch (error) { + if (signal?.aborted === true) { + throw error; + } + throw new LocalLoadError("could not reach this site's GitHub loader"); + } + if (!response.ok) { + throw await failureOf(response); + } + if (response.headers.get("Content-Type") !== "application/zip") { + // Vite's dev server answers unknown paths with the app's own page. + throw new LocalLoadError( + "this server cannot load from GitHub: run the Worker (see the README)" + ); + } + let bytes: ArrayBuffer; + try { + bytes = await response.arrayBuffer(); + } catch (error) { + if (signal?.aborted === true) { + throw error; + } + // The Worker cuts a download off past 64 MB. + throw new LocalLoadError( + "the download from GitHub broke off: the zip may be over 64 MB" + ); + } + if (bytes.byteLength > MAX_ZIP_BYTES) { + throw new LocalLoadError("the repository's zip is over 64 MB"); + } + return filesFromZip(new Uint8Array(bytes), target.subpath); +}; + +export { readGitHubRepo }; diff --git a/apps/web/src/github-form.tsx b/apps/web/src/github-form.tsx new file mode 100644 index 0000000..5927a96 --- /dev/null +++ b/apps/web/src/github-form.tsx @@ -0,0 +1,73 @@ +import type { ReactNode, SubmitEvent } from "react"; + +// The landing page's GitHub row: a public repository's URL, the Turnstile +// check before it may load, and an open button. Markup only; `landing.tsx` +// holds the state. + +/** The GitHub URL field's state; absent when this build has no Turnstile key. */ +type GitHubField = { + readonly url: string; + readonly onUrl: (url: string) => void; + readonly onOpen: () => void; + /** The URL names a GitHub repository. */ + readonly valid: boolean; + /** The URL is valid and the visitor passed the check. */ + readonly ready: boolean; + /** The Turnstile widget, shown once the URL is valid. */ + readonly check: ReactNode; +}; + +/** A public repository's URL, its verification check and an open button. */ +const GitHubForm = ({ + field, + busy, +}: { + readonly field: GitHubField; + readonly busy: boolean; +}) => { + const onSubmit = (event: SubmitEvent) => { + event.preventDefault(); + if (!busy && field.ready) { + field.onOpen(); + } + }; + return ( +
+
+ field.onUrl(event.currentTarget.value)} + /> + +
+ {field.valid && !busy && field.check} + {!field.valid && field.url.trim() !== "" && ( +

+ a public repository: github.com/owner/repo, or …/tree/branch/folder + for one folder +

+ )} +
+ ); +}; + +export { GitHubForm }; +export type { GitHubField }; diff --git a/apps/web/src/github-url.test.ts b/apps/web/src/github-url.test.ts new file mode 100644 index 0000000..660485d --- /dev/null +++ b/apps/web/src/github-url.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; + +import { gitHubPath, isRef, parseGitHubUrl } from "./github-url"; + +describe("parseGitHubUrl", () => { + it.each([ + "https://github.com/pmndrs/zustand", + "http://github.com/pmndrs/zustand/", + "github.com/pmndrs/zustand", + "https://www.github.com/pmndrs/zustand.git", + " pmndrs/zustand ", + "https://github.com/pmndrs/zustand?tab=readme#install", + ])("reads %s as the whole repository", (input) => { + expect(parseGitHubUrl(input)).toEqual({ + owner: "pmndrs", + repo: "zustand", + ref: null, + subpath: null, + }); + }); + + it("reads a branch and a folder from a /tree/ URL", () => { + expect( + parseGitHubUrl( + "https://github.com/vitejs/vite/tree/main/packages/vite/src" + ) + ).toEqual({ + owner: "vitejs", + repo: "vite", + ref: "main", + subpath: "packages/vite/src", + }); + expect(parseGitHubUrl("o/r/tree/v1.2.3")).toEqual({ + owner: "o", + repo: "r", + ref: "v1.2.3", + subpath: null, + }); + }); + + it.each([ + "", + "pmndrs", + "https://gitlab.com/pmndrs/zustand", + "https://github.com.evil.com/pmndrs/zustand", + "javascript:alert(1)", + "pmndrs/zustand/blob/main/src/index.ts", + "pmndrs/zustand/tree", + "pmndrs/zustand/tree/../x", + "pmndrs/zustand/tree/main/../../secrets", + "pmndrs/zustand/tree/main/a%2Fb", + "-pmndrs/zustand", + "pmn--drs/zustand", + "pmndrs/..", + "pmndrs/zust and", + ])("refuses %j", (input) => { + expect(parseGitHubUrl(input)).toBeNull(); + }); +}); + +describe("isRef", () => { + it("takes branch, tag and commit names and HEAD", () => { + for (const ref of ["main", "HEAD", "v1.0.0", "release_2", "a1b2c3d"]) { + expect(isRef(ref)).toBe(true); + } + }); + + it("refuses names that could leave the path or are not refs", () => { + for (const ref of ["", "..", "a..b", "-x", "x.lock", "a/b", "a?b", "a b"]) { + expect(isRef(ref)).toBe(false); + } + }); +}); + +describe("gitHubPath", () => { + it.each([ + "pmndrs/zustand", + "pmndrs/zustand/tree/main", + "vitejs/vite/tree/main/packages/vite/src", + ])("parses back to the same target: %s", (path) => { + const target = parseGitHubUrl(path); + expect(target === null ? null : gitHubPath(target)).toBe(path); + }); +}); diff --git a/apps/web/src/github-url.ts b/apps/web/src/github-url.ts new file mode 100644 index 0000000..3d24f64 --- /dev/null +++ b/apps/web/src/github-url.ts @@ -0,0 +1,99 @@ +// A GitHub repository named by the user, as pasted on the landing page or +// given as `?repo=`. Shared by the page and the Worker (`api/github-zip.ts`), which +// checks the same names again before it builds the one codeload URL it +// fetches: nothing the client sends is ever used as a URL. + +/** The Worker's route that streams a repository's zip. */ +const ZIP_ROUTE = "/api/github/zip"; +/** The request header carrying the visitor's Turnstile token. */ +const TOKEN_HEADER = "CF-Turnstile-Token"; + +type GitHubTarget = { + readonly owner: string; + readonly repo: string; + /** A branch, tag or commit, or null for the default branch. */ + readonly ref: string | null; + /** A folder inside the repository, `/`-separated, or null for all of it. */ + readonly subpath: string | null; +}; + +// GitHub's own rules: a user or organisation is up to 39 letters, digits or +// inner single hyphens; a repository up to 100 of letters, digits, `.`, `-`, `_`. +const OWNER = /^[A-Za-z0-9](?:-?[A-Za-z0-9]){0,38}$/; +const REPO = /^[A-Za-z0-9._-]{1,100}$/; +// One path segment of a ref: refs with `/` cannot be told apart from the +// folder after them in a `/tree/` URL, so they are not supported. +const REF = /^[A-Za-z0-9._-]{1,100}$/; +// Folder names with spaces or other characters a URL must escape are left +// out, so the canonical form stays readable in `?repo=`. +const SEGMENT = /^[A-Za-z0-9._@+-]{1,255}$/; + +const isDots = (name: string): boolean => /^\.+$/.test(name); + +const isOwner = (name: string): boolean => OWNER.test(name); + +const isRepo = (name: string): boolean => REPO.test(name) && !isDots(name); + +const isRef = (name: string): boolean => + REF.test(name) && + !isDots(name) && + !name.startsWith("-") && + !name.includes("..") && + !name.endsWith(".lock"); + +const isSubpath = (path: string): boolean => + path.split("/").every((segment) => SEGMENT.test(segment) && !isDots(segment)); + +/** `https://github.com/o/r`, `github.com/o/r.git`, `o/r/tree/main/src` and the like. */ +const parseGitHubUrl = (input: string): GitHubTarget | null => { + let rest = input.trim().replace(/[?#].*$/, ""); + rest = rest.replace(/^https?:\/\//i, "").replace(/^www\./i, ""); + if (/^github\.com\//i.test(rest)) { + rest = rest.slice("github.com/".length); + } else if (rest.includes(":") || /^[^/]*\.[^/]*\//.test(rest)) { + // Another host, or a scheme other than http(s). + return null; + } + const segments = rest.split("/"); + while (segments.at(-1) === "") { + segments.pop(); + } + const [owner = "", rawRepo = "", kind, ref, ...folder] = segments; + const repo = rawRepo.replace(/\.git$/i, ""); + if (!isOwner(owner) || !isRepo(repo)) { + return null; + } + if (kind === undefined) { + return { owner, repo, ref: null, subpath: null }; + } + if (kind !== "tree" || ref === undefined || !isRef(ref)) { + return null; + } + const subpath = folder.length === 0 ? null : folder.join("/"); + if (subpath !== null && !isSubpath(subpath)) { + return null; + } + return { owner, repo, ref, subpath }; +}; + +/** The shortest form that parses back to the same target, for `?repo=` and the HUD. */ +const gitHubPath = ({ owner, repo, ref, subpath }: GitHubTarget): string => { + const base = `${owner}/${repo}`; + if (ref === null) { + return base; + } + return subpath === null + ? `${base}/tree/${ref}` + : `${base}/tree/${ref}/${subpath}`; +}; + +export { + gitHubPath, + isOwner, + isRef, + isRepo, + parseGitHubUrl, + TOKEN_HEADER, + ZIP_ROUTE, +}; +export type { GitHubTarget }; diff --git a/apps/web/src/landing.test.tsx b/apps/web/src/landing.test.tsx index 89466ba..c30251a 100644 --- a/apps/web/src/landing.test.tsx +++ b/apps/web/src/landing.test.tsx @@ -11,6 +11,7 @@ const render = ( reading?: string | null; error?: string | null; backdrop?: ReactNode; + github?: { url: string; valid: boolean; ready: boolean }; } = {} ): string => renderToStaticMarkup( @@ -21,6 +22,16 @@ const render = ( onZip={ignore} onFolder={ignore} onDragging={ignore} + github={ + state.github === undefined + ? undefined + : { + ...state.github, + onUrl: ignore, + onOpen: ignore, + check:
, + } + } backdrop={state.backdrop} /> ); @@ -62,6 +73,41 @@ describe("LandingView", () => { expect(render()).toContain("nothing is uploaded"); }); + it("has no GitHub field when the build has no Turnstile key", () => { + expect(render()).not.toContain("GitHub repository URL"); + }); + + it("offers a GitHub URL field whose open waits for the check", () => { + const empty = render({ github: { url: "", valid: false, ready: false } }); + expect(empty).toContain('aria-label="GitHub repository URL"'); + expect(empty).toMatch(/
+ {github !== undefined && } {error !== null && (

{error} @@ -239,20 +251,38 @@ const LandingView = ({ ); }; -/** The landing page: reads a zip or folder, then hands its files to `onLoad`. */ +/** + * The landing page: reads a zip, folder or GitHub repository, then hands its + * files to `onLoad`. `initialRepo` fills the GitHub field (`?repo=`); the + * visitor still presses open, since the fetch needs their Turnstile token. + */ const Landing = ({ onLoad, + initialRepo, }: { readonly onLoad: (request: SourceRequest) => void; + readonly initialRepo: string; }) => { const [reading, setReading] = useState(null); const [failure, setFailure] = useState(null); const [dragging, setDragging] = useState(false); + const [repoUrl, setRepoUrl] = useState(initialRepo); + const [token, setToken] = useState(null); + // Bumped to remount the check after each use: a token is accepted once. + const [checkRound, setCheckRound] = useState(0); + const download = useRef(null); const guard = useMemo(() => createReadGuard(), []); + const siteKey = useMemo(() => turnstileSiteKey(), []); const backdrop = useBackdrop(); // A read that ends after the page is gone never loads a world. - useEffect(() => () => guard.cancel(), [guard]); + useEffect( + () => () => { + guard.cancel(); + download.current?.abort(); + }, + [guard] + ); const read = useCallback( (name: string, label: string, load: () => Promise) => { @@ -301,6 +331,50 @@ const Landing = ({ [read] ); + const target = useMemo(() => parseGitHubUrl(repoUrl), [repoUrl]); + + const onRepo = useCallback(() => { + // The button is disabled until both hold; Enter is checked the same way. + if (target === null || token === null) { + return; + } + const path = gitHubPath(target); + const controller = new AbortController(); + download.current = controller; + setToken(null); + setCheckRound((round) => round + 1); + read(path, path, async () => { + const files = await readGitHubRepo(target, token, { + fetch: (url, init) => globalThis.fetch(url, init), + signal: controller.signal, + }); + // A reload comes back to this repository's field, ready to open. + globalThis.history.replaceState(null, "", searchOfRepo(path)); + return files; + }); + }, [target, token, read]); + + const github: GitHubField | undefined = + siteKey === null + ? undefined + : { + url: repoUrl, + onUrl: (url) => { + setRepoUrl(url); + setFailure(null); + }, + onOpen: onRepo, + valid: target !== null, + ready: target !== null && token !== null, + check: ( + + ), + }; + return ( { const selection = selectSources(entries(...many)); expect(selection.kept).toEqual([]); expect(selection.fatal).toBe( - "401 files to load, more than the 400 allowed: zip or pick a subfolder" + "401 files to load, more than the 400 allowed: load a subfolder" ); expect(selectSources(entries(...many.slice(1))).fatal).toBeNull(); }); @@ -191,7 +191,7 @@ describe("selectSources", () => { size: MAX_FILE_BYTES, })); expect(selectSources(heavy).fatal).toBe( - "8.3 MB of sources, more than the 8.0 MB allowed: zip or pick a subfolder" + "8.3 MB of sources, more than the 8.0 MB allowed: load a subfolder" ); }); diff --git a/apps/web/src/source-filter.ts b/apps/web/src/source-filter.ts index 098f9e2..1bbd52a 100644 --- a/apps/web/src/source-filter.ts +++ b/apps/web/src/source-filter.ts @@ -111,11 +111,11 @@ const megabytes = (bytes: number): string => (bytes / (1024 * 1024)).toFixed(1); /** The fatal reason for what survived the rules, or null when it loads. */ const fatalOf = (kept: readonly KeptEntry[]): string | null => { if (kept.length > MAX_KEPT_FILES) { - return `${kept.length} files to load, more than the ${MAX_KEPT_FILES} allowed: zip or pick a subfolder`; + return `${kept.length} files to load, more than the ${MAX_KEPT_FILES} allowed: load a subfolder`; } const bytes = kept.reduce((sum, entry) => sum + entry.size, 0); if (bytes > MAX_KEPT_BYTES) { - return `${megabytes(bytes)} MB of sources, more than the ${megabytes(MAX_KEPT_BYTES)} MB allowed: zip or pick a subfolder`; + return `${megabytes(bytes)} MB of sources, more than the ${megabytes(MAX_KEPT_BYTES)} MB allowed: load a subfolder`; } if (kept.every((entry) => isManifest(entry.path))) { return "no JavaScript or TypeScript sources found"; diff --git a/apps/web/src/source-request.test.ts b/apps/web/src/source-request.test.ts index 9dac87d..f7f5028 100644 --- a/apps/web/src/source-request.test.ts +++ b/apps/web/src/source-request.test.ts @@ -4,7 +4,9 @@ import { examples } from "./examples"; import { parseWorldParams } from "./params"; import { labelOfRequest, + repoOfSearch, requestFromParams, + searchOfRepo, showsLanding, } from "./source-request"; import type { SourceRequest } from "./source-request"; @@ -49,12 +51,39 @@ describe("showsLanding", () => { expect(showsLanding(NONE, search)).toBe(false); }); + it("still shows for ?repo=, which waits for the visitor's check", () => { + expect(showsLanding(NONE, "?repo=o/r")).toBe(true); + }); + it("never shows once there is something to build", () => { expect(showsLanding({ kind: "example", name: "demo" }, "")).toBe(false); expect(showsLanding(LOCAL, "")).toBe(false); }); }); +describe("repoOfSearch", () => { + it("reads ?repo=, slashes escaped or not", () => { + expect(repoOfSearch("?repo=o/r/tree/main/src")).toBe("o/r/tree/main/src"); + expect(repoOfSearch("?repo=o%2Fr")).toBe("o/r"); + expect(repoOfSearch("")).toBe(""); + }); +}); + +describe("searchOfRepo", () => { + it("keeps slashes readable", () => { + expect(searchOfRepo("o/r/tree/main/src")).toBe("?repo=o/r/tree/main/src"); + }); + + it.each([ + "o/r", + "o/r/tree/main/a+b", + "o/r/tree/v1.0/c++/@scope", + "o/r/tree/main/x&y=1#z", + ])("reads back as the same path: %s", (path) => { + expect(repoOfSearch(searchOfRepo(path))).toBe(path); + }); +}); + describe("labelOfRequest", () => { it("names examples and local files, not random worlds", () => { expect(labelOfRequest(NONE)).toBeNull(); diff --git a/apps/web/src/source-request.ts b/apps/web/src/source-request.ts index e42d9d4..efd45d7 100644 --- a/apps/web/src/source-request.ts +++ b/apps/web/src/source-request.ts @@ -4,9 +4,10 @@ import type { ExampleName } from "~/example-files"; import type { WorldParams } from "~/params"; // What the 3D app builds its world from: nothing yet (the landing page, or a -// random or preset world), a bundled example named by the URL, or a zip or -// folder the user brought. Local files live in memory only, so a reload -// returns to the landing page. +// random or preset world), a bundled example named by the URL, or a zip, +// folder or GitHub repository the user brought. Their files live in memory +// only, so a reload returns to the landing page (with `?repo=` filled in for +// a GitHub repository). type SourceRequest = | { readonly kind: "none" } @@ -48,6 +49,20 @@ const showsLanding = (request: SourceRequest, search: string): boolean => { return !WORLD_KEYS.some((key) => params.has(key)); }; +/** + * `?repo=`, a GitHub repository to fill in on the landing page, or "". It + * never opens a world by itself: the fetch waits for the visitor's check. + */ +const repoOfSearch = (search: string): string => + new URLSearchParams(search).get("repo") ?? ""; + +/** + * The `?repo=` query that `repoOfSearch` reads back as `path`. Escaped, since + * a query reads `+` as a space; `/` stays readable, a query never splits on it. + */ +const searchOfRepo = (path: string): string => + `?repo=${encodeURIComponent(path).replaceAll("%2F", "/")}`; + /** The name the HUD and loading screen show, null for a random world. */ const labelOfRequest = (request: SourceRequest): string | null => { if (request.kind === "none") { @@ -56,5 +71,11 @@ const labelOfRequest = (request: SourceRequest): string | null => { return request.kind === "example" ? request.name : request.label; }; -export { labelOfRequest, requestFromParams, showsLanding }; +export { + labelOfRequest, + repoOfSearch, + requestFromParams, + searchOfRepo, + showsLanding, +}; export type { SourceRequest }; diff --git a/apps/web/src/turnstile.tsx b/apps/web/src/turnstile.tsx new file mode 100644 index 0000000..f7d0eb0 --- /dev/null +++ b/apps/web/src/turnstile.tsx @@ -0,0 +1,119 @@ +import { useEffect, useRef, useState } from "react"; + +// Cloudflare Turnstile, the check a visitor passes before this site fetches a +// repository from GitHub for them (`api/github-zip.ts` verifies the token). The +// script loads only once the GitHub field is used, so zip and folder visits +// never contact Cloudflare's challenge servers. + +type TurnstileApi = { + readonly render: ( + container: HTMLElement, + options: { + readonly sitekey: string; + readonly theme: "dark"; + readonly callback: (token: string) => void; + readonly "expired-callback": () => void; + readonly "error-callback": () => void; + } + ) => string; + readonly remove: (widgetId: string) => void; +}; + +/** The global the script installs, if it has run. */ +const installedApi = (): TurnstileApi | undefined => + (globalThis as { readonly turnstile?: TurnstileApi }).turnstile; + +const SCRIPT = + "https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit"; + +/** + * Cloudflare's public always-pass test key, used by `pnpm dev` when no key is + * configured. Production builds never fall back to it. + */ +const TEST_SITE_KEY = "1x00000000000000000000AA"; + +/** The site key the build was given, the test key in dev, else null (GitHub loading off). */ +const turnstileSiteKey = (): string | null => { + const configured: unknown = import.meta.env.VITE_TURNSTILE_SITE_KEY; + if (typeof configured === "string" && configured !== "") { + return configured; + } + return import.meta.env.DEV ? TEST_SITE_KEY : null; +}; + +let loading: Promise | null = null; + +/** Adds the script once; a failed load may be retried. */ +const loadTurnstile = (): Promise => { + loading ??= new Promise((resolve, reject) => { + const script = document.createElement("script"); + script.src = SCRIPT; + script.onload = () => { + const api = installedApi(); + if (api === undefined) { + reject(new Error("Turnstile did not start")); + } else { + resolve(api); + } + }; + script.onerror = () => { + script.remove(); + loading = null; + reject(new Error("Turnstile could not load")); + }; + document.head.append(script); + }); + return loading; +}; + +/** + * One Turnstile widget. `onToken` gets each passed check's token, and null + * when it expires or fails. Remount (change `key`) for a fresh token: each + * token is accepted once. + */ +const TurnstileCheck = ({ + siteKey, + onToken, +}: { + readonly siteKey: string; + readonly onToken: (token: string | null) => void; +}) => { + const box = useRef(null); + const [failed, setFailed] = useState(false); + useEffect(() => { + let live = true; + let widget: string | null = null; + loadTurnstile().then( + (api) => { + if (live && box.current !== null) { + widget = api.render(box.current, { + sitekey: siteKey, + theme: "dark", + callback: (token) => onToken(token), + "expired-callback": () => onToken(null), + "error-callback": () => onToken(null), + }); + } + }, + () => { + if (live) { + setFailed(true); + } + } + ); + return () => { + live = false; + if (widget !== null) { + installedApi()?.remove(widget); + } + onToken(null); + }; + }, [siteKey, onToken]); + return failed ? ( +

the verification check could not load

+ ) : ( +
+ ); +}; + +export { TurnstileCheck, turnstileSiteKey }; diff --git a/apps/web/src/zip-files.test.ts b/apps/web/src/zip-files.test.ts index 6ef246b..afdef97 100644 --- a/apps/web/src/zip-files.test.ts +++ b/apps/web/src/zip-files.test.ts @@ -89,6 +89,20 @@ describe("filesFromZip", () => { ); }); + it("reads only a subpath's files, relative to it, when one is given", () => { + const bytes = zipOf({ + "repo-main/src/a.ts": "a;", + "repo-main/packages/web/src/b.ts": "b;", + "repo-main/packages/web-extra/c.ts": "c;", + }); + expect(filesFromZip(bytes, "packages/web").files).toEqual([ + { path: "b.ts", source: "b;" }, + ]); + expect(loadError(() => filesFromZip(bytes, "lib")).message).toBe( + "no folder lib in the repository" + ); + }); + it("reports an entry that fails to inflate as a damaged zip", () => { const bytes = zipOf({ "r/a.ts": "export const a = 1;\n".repeat(50) }); // The entry's data starts after the 30-byte local header and its name; diff --git a/apps/web/src/zip-files.ts b/apps/web/src/zip-files.ts index f9a9512..0a9d88d 100644 --- a/apps/web/src/zip-files.ts +++ b/apps/web/src/zip-files.ts @@ -18,7 +18,7 @@ type LocalFiles = { readonly skipped: number; }; -/** Larger zips are refused before they are read. */ +/** Larger zips are refused before they are read; `api/github-zip.ts` caps GitHub's too. */ const MAX_ZIP_BYTES = 64 * 1024 * 1024; const messageOf = (error: unknown): string => @@ -50,16 +50,53 @@ const listEntries = (bytes: Uint8Array): readonly FileEntry[] => { }; /** - * The sources and manifests of a zip, root stripped and sorted. Two passes: - * fflate inflates an accepted entry before it sees the next, so the entries - * are listed first and only the selected ones inflated after. + * The entries under `subpath` inside the zip's one top folder (a GitHub + * zip's `repo-sha/`), renamed relative to it, each mapped to its real name. */ -const filesFromZip = (bytes: Uint8Array): LocalFiles => { - const { kept, skipped, fatal } = selectSources(listEntries(bytes)); +const underSubpath = ( + entries: readonly FileEntry[], + subpath: string +): { readonly entries: FileEntry[]; readonly names: Map } => { + const top = entries[0]?.path.split("/")[0] ?? ""; + const prefix = `${top}/${subpath}/`; + const names = new Map(); + const inside: FileEntry[] = []; + for (const entry of entries) { + if (entry.path.startsWith(prefix) && entry.path.length > prefix.length) { + const path = entry.path.slice(prefix.length); + names.set(path, entry.path); + inside.push({ path, size: entry.size }); + } + } + if (inside.length === 0) { + throw new LocalLoadError(`no folder ${subpath} in the repository`); + } + return { entries: inside, names }; +}; + +/** + * The sources and manifests of a zip, root stripped and sorted, only those + * under `subpath` when one is given. Two passes: fflate inflates an accepted + * entry before it sees the next, so the entries are listed first and only + * the selected ones inflated after. + */ +const filesFromZip = ( + bytes: Uint8Array, + subpath: string | null = null +): LocalFiles => { + const listed = listEntries(bytes); + const scoped = + subpath === null + ? { entries: listed, names: null } + : underSubpath(listed, subpath); + const { kept, skipped, fatal } = selectSources(scoped.entries); if (fatal !== null) { throw new LocalLoadError(fatal); } - const wanted = new Set(kept.map((entry) => entry.original)); + // The name each kept entry has inside the zip. + const nameOf = (original: string): string => + scoped.names?.get(original) ?? original; + const wanted = new Set(kept.map((entry) => nameOf(entry.original))); let inflated: Readonly>; try { inflated = unzipSync(bytes, { filter: (file) => wanted.has(file.name) }); @@ -68,11 +105,10 @@ const filesFromZip = (bytes: Uint8Array): LocalFiles => { } const decoder = new TextDecoder("utf-8"); const files = kept.map((entry): SourceFile => { - const data = inflated[entry.original]; + const name = nameOf(entry.original); + const data = inflated[name]; if (data === undefined) { - throw new LocalLoadError( - `the zip is damaged: ${entry.original} is missing` - ); + throw new LocalLoadError(`the zip is damaged: ${name} is missing`); } return { path: entry.path, source: decoder.decode(data) }; }); @@ -92,5 +128,5 @@ const readZipFile = async ( return filesFromZip(new Uint8Array(await file.arrayBuffer())); }; -export { filesFromZip, readZipFile }; +export { filesFromZip, MAX_ZIP_BYTES, readZipFile }; export type { LocalFiles }; diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index e07c63c..9725dcf 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -5,4 +5,7 @@ import { defineConfig } from "vite"; export default defineConfig({ plugins: [react()], resolve: { tsconfigPaths: true }, + // The GitHub zip route lives in the Worker (`src/api/`): run + // `pnpm --filter web dev:worker` beside `dev` to load repositories from GitHub. + server: { proxy: { "/api": "http://localhost:8787" } }, }); diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index e9555cb..12a9edf 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -5,7 +5,24 @@ // fail in `wrangler dev`. Bump it together with wrangler. "compatibility_date": "2026-09-11", "observability": { "enabled": true }, - // Static assets only: Vite's build output, including public/_headers. No + // The Worker serves only /api/* (the GitHub zip route); everything else is Vite's + // build output, including public/_headers, served without running it. No // not_found_handling: the app is a single page without routes, so unknown paths 404. - "assets": { "directory": "./dist" }, + "main": "src/api/worker.ts", + "assets": { + "directory": "./dist", + "binding": "ASSETS", + "run_worker_first": ["/api/*"], + }, + // Per visitor IP, on top of the Turnstile check. namespace_id is any number unique + // to this account's rate limiters; it is not a secret. + "ratelimits": [ + { + "name": "GITHUB_LIMITER", + "namespace_id": "1001", + "simple": { "limit": 5, "period": 60 }, + }, + ], + // TURNSTILE_SECRET is a secret set in the Cloudflare dashboard (or with + // `wrangler secret put`), and in an uncommitted .dev.vars for `wrangler dev`. }