From 3ae81530bc64093f598039d8ff21b287f9fb0894 Mon Sep 17 00:00:00 2001
From: Kane Hsieh <kane.hsieh@gmail.com>
Date: Tue, 11 Aug 2026 12:09:13 -0700
Subject: [PATCH] Keep the static mirror out of search results
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Google was generating sitelinks to /about/, /team/, and portfolio pages under
the root.vc result, which undercuts the terminal being the front door.

Emits `noindex, follow` on every mirror page and on welcome.htm, and trims
sitemap.xml to the homepage alone — listing a noindex URL in a sitemap asks
Google to index something the page itself forbids, and that contradiction is
what surfaced the sitelinks.

Deliberately not a robots.txt Disallow: noindex is a search-indexing
directive, so crawlers may still fetch and read these pages. GPTBot,
ClaudeBot, PerplexityBot and llms.txt consumers keep getting the full
content, while root.vc stays the only result Google shows.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 scripts/build-pages.js    | 20 ++++++++----
 tests/build-pages.test.js | 66 ++++++++++++++++++++++++++++++++++-----
 welcome.htm               |  4 +++
 3 files changed, 76 insertions(+), 14 deletions(-)

diff --git a/scripts/build-pages.js b/scripts/build-pages.js
index 7679fe12..d0c16bb9 100644
--- a/scripts/build-pages.js
+++ b/scripts/build-pages.js
@@ -152,6 +152,13 @@ function renderPrompt(command) {
   );
 }
 
+// Every mirror page is emitted noindex. root.vc should be the only result
+// Google shows: the terminal is the front door, and sitelinks to /about/ and
+// /team/ give the trick away. noindex is a search-INDEXING directive, not a
+// fetch permission — crawlers may still read the page, so GPTBot, ClaudeBot and
+// PerplexityBot keep getting the full content. That is the whole reason this is
+// not a robots.txt Disallow, which would hide the mirror from them too.
+// "follow" keeps link equity flowing back to the homepage.
 function layout({ title, description, pathname, command, body, graph }) {
   const canonical = absUrl(pathname);
   const jsonLd = jsonLdScript({ "@context": "https://schema.org", "@graph": graph });
@@ -163,6 +170,7 @@ function layout({ title, description, pathname, command, body, graph }) {
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>${esc(title)}</title>
     <meta name="description" content="${esc(description)}" />
+    <meta name="robots" content="noindex, follow" />
     <link rel="canonical" href="${esc(canonical)}" />
     <link rel="shortcut icon" href="/favicon.png" />
     <meta property="og:site_name" content="Root Ventures" />
@@ -947,14 +955,14 @@ function buildPages(config = loadConfig()) {
     files.push(renderPerson(config, slug));
   });
 
-  // Sitemap covers the terminal, the GeoCities page, and every generated page.
+  // The sitemap lists ONLY the homepage. Every mirror page is noindex, and
+  // listing a noindex URL in a sitemap asks Google to index something the page
+  // itself forbids — that contradiction is what produced sitelinks to /about/,
+  // /team/, and the rest under the root.vc result. Crawlers still reach the
+  // mirror via the <noscript> links on the homepage and via llms.txt.
   // No <lastmod>: every deploy rebuilds every file, so a build timestamp would
   // claim all 70-odd pages changed each time and teach crawlers to ignore it.
-  const urls = [
-    `${ORIGIN}/`,
-    `${ORIGIN}/welcome.htm`,
-    ...files.map((file) => absUrl(`/${file.path.replace(/index\.html$/, "")}`)),
-  ];
+  const urls = [`${ORIGIN}/`];
 
   files.push({ path: "sitemap.xml", content: renderSitemap(urls) });
   files.push({ path: "robots.txt", content: renderRobots() });
diff --git a/tests/build-pages.test.js b/tests/build-pages.test.js
index 8097489e..f38e42f7 100644
--- a/tests/build-pages.test.js
+++ b/tests/build-pages.test.js
@@ -384,6 +384,55 @@ describe("generated output stays out of the repo", () => {
   });
 });
 
+describe("search invisibility", () => {
+  // root.vc is meant to be the only result Google shows — the terminal is the
+  // front door, and sitelinks to /about/ and /team/ give the trick away. This
+  // is done with `noindex` (a search-indexing directive that crawlers may still
+  // read past) rather than a robots.txt Disallow, which would also hide the
+  // mirror from the AI crawlers it exists to serve.
+
+  it.each(mirrorPages.map((file) => file.path))("%s is noindex", (filePath) => {
+    const page = mirrorPages.find((file) => file.path === filePath);
+    const robots = parse(page.content).querySelector('meta[name="robots"]');
+    expect(robots).not.toBeNull();
+    expect(robots.content).toMatch(/\bnoindex\b/);
+    // `follow` keeps link equity flowing back to the homepage.
+    expect(robots.content).toMatch(/\bfollow\b/);
+  });
+
+  it("leaves the terminal homepage indexable", () => {
+    // The whole point is that root.vc itself still ranks.
+    const robots = parse(fileNamed("index.html")).querySelector(
+      'meta[name="robots"]'
+    );
+    expect(robots?.content ?? "").not.toMatch(/\bnoindex\b/);
+  });
+
+  it("keeps welcome.htm out of search results too", () => {
+    const source = fs.readFileSync(path.join(REPO_ROOT, "welcome.htm"), "utf8");
+    expect(source).toMatch(/<meta name="robots" content="[^"]*noindex/);
+  });
+
+  it("does not use robots.txt to hide the mirror from AI crawlers", () => {
+    // A Disallow here would stop GPTBot/ClaudeBot/PerplexityBot from ever
+    // fetching the content, defeating the reason the mirror exists.
+    const robots = fileNamed("robots.txt");
+    for (const dir of ["/portfolio/", "/team/", "/about/", "/jobs/"]) {
+      expect(robots).not.toContain(`Disallow: ${dir}`);
+    }
+    expect(robots).not.toContain("Disallow: /llms");
+  });
+
+  it("still exposes the full mirror to LLM crawlers", () => {
+    // llms.txt and the homepage <noscript> remain the discovery paths.
+    const llms = fileNamed("llms.txt");
+    for (const [slug, company] of Object.entries(config.portfolio)) {
+      expect(llms).toContain(`${ORIGIN}/portfolio/${slug}/`);
+      expect(llms).toContain(company.description);
+    }
+  });
+});
+
 describe("sitemap.xml", () => {
   const sitemap = fileNamed("sitemap.xml");
   const locs = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
@@ -394,16 +443,17 @@ describe("sitemap.xml", () => {
     );
   });
 
-  it("lists every generated page exactly once", () => {
-    const expected = mirrorPages.map((file) => `${ORIGIN}${pathnameFor(file.path)}`);
-    for (const url of expected) {
-      expect(locs).toContain(url);
-    }
-    expect(new Set(locs).size).toBe(locs.length);
+  it("lists the homepage and nothing else", () => {
+    // root.vc should be the only Google result. Listing a noindex page in the
+    // sitemap asks Google to index something the page itself forbids, and that
+    // contradiction is what generated sitelinks to /about/, /team/, etc.
+    expect(locs).toEqual([`${ORIGIN}/`]);
   });
 
-  it("includes the terminal homepage", () => {
-    expect(locs).toContain(`${ORIGIN}/`);
+  it("excludes every mirror page", () => {
+    for (const file of mirrorPages) {
+      expect(locs).not.toContain(`${ORIGIN}${pathnameFor(file.path)}`);
+    }
   });
 
   it("only lists apex https URLs", () => {
diff --git a/welcome.htm b/welcome.htm
index 85345be0..52e58aff 100644
--- a/welcome.htm
+++ b/welcome.htm
@@ -7,6 +7,10 @@
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <link rel="shortcut icon" href="/favicon.png" />
         <title>Root Ventures — Our Home Page on the World Wide Web</title>
+        <!-- Kept out of search results so root.vc is the only result Google
+             shows. Crawlers may still read this page; it is only hidden from
+             the index, not from fetching. -->
+        <meta name="robots" content="noindex, follow" />
         <!-- Self-referencing. This previously pointed at the homepage, which
              told search engines to drop this page entirely. -->
         <link rel="canonical" href="https://root.vc/welcome.htm" />