Keep the static mirror out of search results

scripts/build-pages.js

@@ -152,6 +152,13 @@ function renderPrompt(command) {
   );
 }
 
+// Every mirror page is emitted noindex. root.vc should be the only result
+// Google shows: the terminal is the front door, and sitelinks to /about/ and
+// /team/ give the trick away. noindex is a search-INDEXING directive, not a
+// fetch permission — crawlers may still read the page, so GPTBot, ClaudeBot and
+// PerplexityBot keep getting the full content. That is the whole reason this is
+// not a robots.txt Disallow, which would hide the mirror from them too.
+// "follow" keeps link equity flowing back to the homepage.
 function layout({ title, description, pathname, command, body, graph }) {
   const canonical = absUrl(pathname);
   const jsonLd = jsonLdScript({ "@context": "https://schema.org", "@graph": graph });
@@ -163,6 +170,7 @@ function layout({ title, description, pathname, command, body, graph }) {
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>${esc(title)}</title>
     <meta name="description" content="${esc(description)}" />
+    <meta name="robots" content="noindex, follow" />
     <link rel="canonical" href="${esc(canonical)}" />
     <link rel="shortcut icon" href="/favicon.png" />
     <meta property="og:site_name" content="Root Ventures" />
@@ -947,14 +955,14 @@ function buildPages(config = loadConfig()) {
     files.push(renderPerson(config, slug));
   });
 
-  // Sitemap covers the terminal, the GeoCities page, and every generated page.
+  // The sitemap lists ONLY the homepage. Every mirror page is noindex, and
+  // listing a noindex URL in a sitemap asks Google to index something the page
+  // itself forbids — that contradiction is what produced sitelinks to /about/,
+  // /team/, and the rest under the root.vc result. Crawlers still reach the
+  // mirror via the <noscript> links on the homepage and via llms.txt.
   // No <lastmod>: every deploy rebuilds every file, so a build timestamp would
   // claim all 70-odd pages changed each time and teach crawlers to ignore it.
-  const urls = [
-    `${ORIGIN}/`,
-    `${ORIGIN}/welcome.htm`,
-    ...files.map((file) => absUrl(`/${file.path.replace(/index\.html$/, "")}`)),
-  ];
+  const urls = [`${ORIGIN}/`];
 
   files.push({ path: "sitemap.xml", content: renderSitemap(urls) });
   files.push({ path: "robots.txt", content: renderRobots() });

tests/build-pages.test.js

@@ -384,6 +384,55 @@ describe("generated output stays out of the repo", () => {
   });
 });
 
+describe("search invisibility", () => {
+  // root.vc is meant to be the only result Google shows — the terminal is the
+  // front door, and sitelinks to /about/ and /team/ give the trick away. This
+  // is done with `noindex` (a search-indexing directive that crawlers may still
+  // read past) rather than a robots.txt Disallow, which would also hide the
+  // mirror from the AI crawlers it exists to serve.
+
+  it.each(mirrorPages.map((file) => file.path))("%s is noindex", (filePath) => {
+    const page = mirrorPages.find((file) => file.path === filePath);
+    const robots = parse(page.content).querySelector('meta[name="robots"]');
+    expect(robots).not.toBeNull();
+    expect(robots.content).toMatch(/\bnoindex\b/);
+    // `follow` keeps link equity flowing back to the homepage.
+    expect(robots.content).toMatch(/\bfollow\b/);
+  });
+
+  it("leaves the terminal homepage indexable", () => {
+    // The whole point is that root.vc itself still ranks.
+    const robots = parse(fileNamed("index.html")).querySelector(
+      'meta[name="robots"]'
+    );
+    expect(robots?.content ?? "").not.toMatch(/\bnoindex\b/);
+  });
+
+  it("keeps welcome.htm out of search results too", () => {
+    const source = fs.readFileSync(path.join(REPO_ROOT, "welcome.htm"), "utf8");
+    expect(source).toMatch(/<meta name="robots" content="[^"]*noindex/);
+  });
+
+  it("does not use robots.txt to hide the mirror from AI crawlers", () => {
+    // A Disallow here would stop GPTBot/ClaudeBot/PerplexityBot from ever
+    // fetching the content, defeating the reason the mirror exists.
+    const robots = fileNamed("robots.txt");
+    for (const dir of ["/portfolio/", "/team/", "/about/", "/jobs/"]) {
+      expect(robots).not.toContain(`Disallow: ${dir}`);
+    }
+    expect(robots).not.toContain("Disallow: /llms");
+  });
+
+  it("still exposes the full mirror to LLM crawlers", () => {
+    // llms.txt and the homepage <noscript> remain the discovery paths.
+    const llms = fileNamed("llms.txt");
+    for (const [slug, company] of Object.entries(config.portfolio)) {
+      expect(llms).toContain(`${ORIGIN}/portfolio/${slug}/`);
+      expect(llms).toContain(company.description);
+    }
+  });
+});
+
 describe("sitemap.xml", () => {
   const sitemap = fileNamed("sitemap.xml");
   const locs = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
@@ -394,16 +443,17 @@ describe("sitemap.xml", () => {
     );
   });
 
-  it("lists every generated page exactly once", () => {
-    const expected = mirrorPages.map((file) => `${ORIGIN}${pathnameFor(file.path)}`);
-    for (const url of expected) {
-      expect(locs).toContain(url);
-    }
-    expect(new Set(locs).size).toBe(locs.length);
+  it("lists the homepage and nothing else", () => {
+    // root.vc should be the only Google result. Listing a noindex page in the
+    // sitemap asks Google to index something the page itself forbids, and that
+    // contradiction is what generated sitelinks to /about/, /team/, etc.
+    expect(locs).toEqual([`${ORIGIN}/`]);
   });
 
-  it("includes the terminal homepage", () => {
-    expect(locs).toContain(`${ORIGIN}/`);
+  it("excludes every mirror page", () => {
+    for (const file of mirrorPages) {
+      expect(locs).not.toContain(`${ORIGIN}${pathnameFor(file.path)}`);
+    }
   });
 
   it("only lists apex https URLs", () => {

welcome.htm

@@ -7,6 +7,10 @@
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <link rel="shortcut icon" href="/favicon.png" />
         <title>Root Ventures — Our Home Page on the World Wide Web</title>
+        <!-- Kept out of search results so root.vc is the only result Google
+             shows. Crawlers may still read this page; it is only hidden from
+             the index, not from fetching. -->
+        <meta name="robots" content="noindex, follow" />
         <!-- Self-referencing. This previously pointed at the homepage, which
              told search engines to drop this page entirely. -->
         <link rel="canonical" href="https://root.vc/welcome.htm" />