From 513247fb6b870ca3e391baaab63d41e07f1ecf6b Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 17:51:11 +0000 Subject: [PATCH] Read the page an address points at, and know when it points at none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gravatar answers for about one person in a hundred: 18 identities from the first 1,800 of a real list. That is not enough to call an address enriched, and the reason is that Gravatar is opt-in twice over — the person must have an account and must have attached their other accounts to it. An address usually points somewhere anyway. Measured against the 16,268-row list now in production: 36% a publication handle — 5,934 substack.com addresses, where the local part *is* the subdomain, so 0xshah@substack.com is the account behind 0xshah.substack.com 31% a company domain — 4,235 distinct sites 33% nothing at all — gmail, yahoo, icloud and friends Those first two are pages the crawler already knows how to read: it pulls a name, a title and published social links out of a site, which is exactly what is missing from an imported row. This is derivation, not guessing, and the distinction is load-bearing. Every rule turns an address into a page that address demonstrably belongs to. None invents a handle on a network the person never mentioned — the codebase already refuses to do that for email addresses it could at least verify by sending to, and an unverifiable social identity is worse. Knowing when to do nothing is the more valuable half. Sending every Gmail address to google.com would be five thousand fetches of a page about nobody, and it would look like the feature working. `qq.com` was missing from the freemail list entirely — 127 addresses that would have been read as company domains and sent the crawler to Tencent — which is what the test caught and why the international consumer providers are now listed. Crawls are enqueued under the same dedupe key as everything else, so four hundred people at one company read that company's site once. That lesson cost 226 identical crawls of accenture.com. Co-Authored-By: Claude Opus 5 (1M context) --- apps/server/src/index.ts | 1 + packages/domain/src/contact-import.ts | 34 ++++++ packages/domain/src/index.ts | 1 + packages/domain/src/web-presence.test.ts | 102 ++++++++++++++++ packages/domain/src/web-presence.ts | 148 +++++++++++++++++++++++ packages/pipeline/src/enrich-contact.ts | 51 +++++++- 6 files changed, 332 insertions(+), 5 deletions(-) create mode 100644 packages/domain/src/web-presence.test.ts create mode 100644 packages/domain/src/web-presence.ts diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index aff0df0..c4ad5fd 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -669,6 +669,7 @@ async function tick(): Promise { if (swept.looked > 0) { console.log( `enrichment: looked up ${swept.looked} in ${workspaceId}, ` + + `${swept.pagesQueued} page(s) queued, ${swept.noPresence} point nowhere, ` + `${swept.found} had a profile, ${swept.identities} identities, ` + `${swept.remaining} left`, ); diff --git a/packages/domain/src/contact-import.ts b/packages/domain/src/contact-import.ts index 69cbc91..f20623d 100644 --- a/packages/domain/src/contact-import.ts +++ b/packages/domain/src/contact-import.ts @@ -212,6 +212,40 @@ const FREEMAIL_DOMAINS = new Set([ 'fastmail.com', 'hey.com', 'tutanota.com', + // Consumer mailboxes outside the English-speaking web. Their absence was a + // real gap rather than an oversight of scale: a 16,268-row list carried 127 + // `qq.com` addresses, every one of which would have been treated as a + // company domain and sent the crawler to Tencent's homepage. + 'qq.com', + '163.com', + '126.com', + 'sina.com', + 'sina.cn', + 'foxmail.com', + 'naver.com', + 'daum.net', + 'hanmail.net', + 'rediffmail.com', + 'seznam.cz', + 'libero.it', + 'orange.fr', + 'free.fr', + 'laposte.net', + 't-online.de', + 'web.de', + 'bluewin.ch', + 'telenet.be', + 'bigpond.com', + 'ymail.com', + 'rocketmail.com', + 'live.co.uk', + 'btinternet.com', + 'comcast.net', + 'verizon.net', + 'sbcglobal.net', + 'att.net', + 'cox.net', + 'charter.net', ]); /** diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 470c084..da8c3a8 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -23,6 +23,7 @@ export * from './playbooks'; export * from './plans'; export * from './credits'; export * from './contact-import'; +export * from './web-presence'; export * from './rules'; export * from './pipeline'; export * from './outreach'; diff --git a/packages/domain/src/web-presence.test.ts b/packages/domain/src/web-presence.test.ts new file mode 100644 index 0000000..a081ba8 --- /dev/null +++ b/packages/domain/src/web-presence.test.ts @@ -0,0 +1,102 @@ +/** + * Turning an address into the page it points at. + * + * The tests that matter are the ones that must resolve to *nothing*. A wrong + * publication URL costs one wasted fetch; a rule that sends every Gmail + * address to google.com costs sixteen thousand fetches of a page about nobody, + * and looks like the feature working. + */ + +import { describe, expect, test } from 'bun:test'; +import { presenceBreakdown, webPresenceFor } from './web-presence'; + +describe('publications', () => { + test('a substack handle is that publication', () => { + // Measured: 5,934 of one real 16,268-row list. The local part is the + // subdomain, so the page is certain rather than guessed. + expect(webPresenceFor('0xshah@substack.com')).toMatchObject({ + url: 'https://0xshah.substack.com', + kind: 'publication', + }); + }); + + test('the other newsletter and blog hosts', () => { + expect(webPresenceFor('nils@ghost.io')?.url).toBe('https://nils.ghost.io'); + expect(webPresenceFor('dave@medium.com')?.url).toBe('https://medium.com/@dave'); + expect(webPresenceFor('show@anchor.fm')?.url).toBe('https://anchor.fm/show'); + expect(webPresenceFor('news@beehiiv.com')?.url).toBe('https://news.beehiiv.com'); + }); + + test('a local part that cannot be a subdomain resolves to nothing', () => { + // `first.last@substack.com` is not `first.last.substack.com`; dots and + // plus tags would build a hostname that does not exist. + expect(webPresenceFor('first.last@substack.com')).toBeUndefined(); + expect(webPresenceFor('dave+news@substack.com')).toBeUndefined(); + expect(webPresenceFor('under_score@substack.com')).toBeUndefined(); + }); + + test('the basis says where the page came from', () => { + expect(webPresenceFor('pasta@substack.com')?.basis).toContain('publication handle'); + }); +}); + +describe('company domains', () => { + test('a company mailbox points at that company', () => { + expect(webPresenceFor('dave@acme.io')).toMatchObject({ + url: 'https://acme.io', + kind: 'company', + }); + }); + + test('a subdomain is kept, because that is where the mail lands', () => { + expect(webPresenceFor('dave@eng.acme.io')?.url).toBe('https://eng.acme.io'); + }); +}); + +describe('addresses that must point nowhere', () => { + test('consumer mailboxes', () => { + // The expensive mistake: 33% of the list. Crawling these would read + // Google's homepage five thousand times. + for (const email of [ + 'dave@gmail.com', + 'dave@yahoo.com', + 'dave@hotmail.com', + 'dave@proton.me', + 'dave@icloud.com', + 'dave@outlook.com', + 'dave@qq.com', + ]) { + expect(webPresenceFor(email)).toBeUndefined(); + } + }); + + test('forwarders and mail hosts that are not the person', () => { + // These look like company domains and are not: crawling them reads a + // hosting company's marketing page. + expect(webPresenceFor('x@agentmail.to')).toBeUndefined(); + expect(webPresenceFor('x@duck.com')).toBeUndefined(); + expect(webPresenceFor('x@privaterelay.appleid.com')).toBeUndefined(); + expect(webPresenceFor('x@fastmail.com')).toBeUndefined(); + }); + + test('malformed input', () => { + expect(webPresenceFor('not-an-address')).toBeUndefined(); + expect(webPresenceFor('@nolocal.com')).toBeUndefined(); + expect(webPresenceFor('dave@nodot')).toBeUndefined(); + }); +}); + +describe('presenceBreakdown', () => { + test('says what a list will actually reach before anyone waits for it', () => { + const breakdown = presenceBreakdown([ + '0xshah@substack.com', + 'nils@substack.com', + 'dave@acme.io', + 'someone@gmail.com', + 'other@gmail.com', + 'third@yahoo.com', + ]); + + expect(breakdown).toEqual({ publication: 2, company: 1, none: 3 }); + }); +}); diff --git a/packages/domain/src/web-presence.ts b/packages/domain/src/web-presence.ts new file mode 100644 index 0000000..7828bc9 --- /dev/null +++ b/packages/domain/src/web-presence.ts @@ -0,0 +1,148 @@ +/** + * The page an address implies. + * + * An email is not just a mailbox; for a large share of a real list it is a + * pointer to somewhere on the web that says who the person is. Reading that + * page is how a row with nothing but an address acquires a bio, a company, and + * the social accounts the owner chose to publish. + * + * The mapping is deliberately **derivation, not guesswork**. Every rule here + * turns an address into the page that address provably belongs to: + * `0xshah@substack.com` is the account behind `0xshah.substack.com`, and + * `dave@acme.com` is a mailbox at the company whose site is `acme.com`. No + * rule invents a handle on a network the person never mentioned — this + * codebase already refuses to do that for email addresses, and the same + * argument applies harder to social identities, which cannot be verified by + * sending to them. + * + * Measured against a real 16,268-row list: 36% resolved to a publication, 31% + * to a company site, and 33% were mailbox providers that imply nothing at all. + */ + +import { isFreemailDomain } from './contact-import'; + +export type PresenceKind = + /** A page about this person, published by them. */ + | 'publication' + /** The site of the company whose domain they receive mail at. */ + | 'company'; + +export interface WebPresence { + readonly url: string; + readonly kind: PresenceKind; + /** Why we think this page is theirs, shown next to anything it produces. */ + readonly basis: string; +} + +/** + * Hosts where the local part names a page rather than a person's mailbox. + * + * These are the high-yield cases: the address *is* the identifier of a public + * profile, so the page is certain rather than inferred. A list assembled from + * newsletters is mostly this, which is why it is worth special-casing at all. + */ +const PUBLICATION_HOSTS: Readonly string>> = { + 'substack.com': (local) => `https://${local}.substack.com`, + 'medium.com': (local) => `https://medium.com/@${local}`, + 'ghost.io': (local) => `https://${local}.ghost.io`, + 'wordpress.com': (local) => `https://${local}.wordpress.com`, + 'blogspot.com': (local) => `https://${local}.blogspot.com`, + 'tumblr.com': (local) => `https://${local}.tumblr.com`, + 'anchor.fm': (local) => `https://anchor.fm/${local}`, + 'beehiiv.com': (local) => `https://${local}.beehiiv.com`, +}; + +/** + * Domains that host mail for other people's sites, so the domain says nothing + * about the person and crawling it would read a hosting company's homepage. + * + * Distinct from the freemail list, which is about consumer mailboxes. These + * are the ones that look like a company domain and are not. + */ +const NON_COMPANY_DOMAINS = new Set([ + 'agentmail.to', + 'sharklasers.com', + 'simplelogin.com', + 'simplelogin.io', + 'anonaddy.com', + 'anonaddy.me', + 'duck.com', + 'relay.firefox.com', + 'privaterelay.appleid.com', + 'mozmail.com', + 'hey.com', + 'fastmail.com', + 'migadu.com', + 'zoho.com', + 'yandex.com', + 'mail.com', + 'gmx.net', + 'email.com', +]); + +/** + * The page this address points at, if any. + * + * `undefined` for a consumer mailbox, which is the honest answer: `gmail.com` + * is not a website about anybody, and crawling it would read Google's homepage + * sixteen thousand times. + */ +export function webPresenceFor(email: string): WebPresence | undefined { + const at = email.lastIndexOf('@'); + if (at <= 0) return undefined; + + const local = email.slice(0, at).toLowerCase(); + const domain = email.slice(at + 1).toLowerCase(); + + const publication = PUBLICATION_HOSTS[domain]; + + if (publication) { + // A local part with punctuation a subdomain cannot carry is not a handle. + if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(local)) return undefined; + + return { + url: publication(local), + kind: 'publication', + basis: `${domain} publication handle "${local}"`, + }; + } + + if (isFreemailDomain(domain) || NON_COMPANY_DOMAINS.has(domain)) return undefined; + + // A bare domain with no dot cannot resolve, and a public-suffix-only domain + // would send us to a registry. + if (!domain.includes('.')) return undefined; + + return { + url: `https://${domain}`, + kind: 'company', + basis: `receives mail at ${domain}`, + }; +} + +/** + * How many of a list we can find a page for, by kind. + * + * Exported so the import screen can say what enrichment will actually reach + * before anyone waits an hour for it. "We looked everywhere and found nothing" + * is a much worse answer than "a third of this list is Gmail addresses, which + * point nowhere." + */ +export function presenceBreakdown(emails: readonly string[]): { + readonly publication: number; + readonly company: number; + readonly none: number; +} { + let publication = 0; + let company = 0; + let none = 0; + + for (const email of emails) { + const presence = webPresenceFor(email); + if (!presence) none += 1; + else if (presence.kind === 'publication') publication += 1; + else company += 1; + } + + return { publication, company, none }; +} diff --git a/packages/pipeline/src/enrich-contact.ts b/packages/pipeline/src/enrich-contact.ts index f4eacd8..c53899a 100644 --- a/packages/pipeline/src/enrich-contact.ts +++ b/packages/pipeline/src/enrich-contact.ts @@ -21,9 +21,11 @@ * people who never published them. */ -import { newId, isFreemailDomain } from '@outreachgraph/domain'; +import { newId, isFreemailDomain, webPresenceFor } from '@outreachgraph/domain'; import { GRAVATAR_NETWORKS, lookupGravatar, type GravatarOptions } from '@outreachgraph/providers'; import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { enqueue } from './queue'; +import { crawlDedupeKey } from './auto-approve'; /** * How much to believe a Gravatar-published account. @@ -52,6 +54,10 @@ export interface SweepResult { readonly found: number; readonly identities: number; readonly remaining: number; + /** Pages queued to read, deduplicated by host. */ + readonly pagesQueued: number; + /** Addresses that point at no page at all — mailbox providers. */ + readonly noPresence: number; } /** @@ -72,20 +78,22 @@ export async function sweepContactEnrichment( ): Promise { const limit = input.limit ?? SWEEP_SIZE; - const rows = await queryAll<{ person_id: string }>( + const rows = await queryAll<{ person_id: string; address: string }>( db, - `SELECT pe.person_id + `SELECT pe.person_id, min(pe.address) AS address FROM person_emails pe JOIN people p ON p.id = pe.person_id WHERE pe.workspace_id = ? AND p.contact_enriched_at IS NULL AND p.status = 'active' GROUP BY pe.person_id - ORDER BY pe.created_at + ORDER BY min(pe.created_at) LIMIT ?`, [input.workspaceId, limit], ); let found = 0; let identities = 0; + let pagesQueued = 0; + let noPresence = 0; // Fixed-size waves rather than one promise per person: seventeen thousand // concurrent fetches would be a denial of service on somebody else's free @@ -114,6 +122,32 @@ export async function sweepContactEnrichment( identities += result?.identities ?? 0; } + // Reading the page the address points at is where the rest comes from. + // Gravatar answers for about one person in a hundred; a Substack handle or + // a company domain is a page that exists by construction, and the crawler + // already knows how to pull a bio, a title and published social links out + // of one. + for (const row of wave) { + const presence = webPresenceFor(row.address); + + if (!presence) { + noPresence += 1; + continue; + } + + const queued = await enqueue(db, { + workspaceId: input.workspaceId, + kind: 'crawl_site', + payload: { url: presence.url }, + // The same key everything else uses, so four hundred people at one + // company read that company's site once — the lesson from #63, which + // cost 226 identical crawls of accenture.com to learn. + dedupeKey: crawlDedupeKey(presence.url), + }); + + if (queued.queued) pagesQueued += 1; + } + await db.batch( wave.map((row) => ({ sql: 'UPDATE people SET contact_enriched_at = ? WHERE id = ?', @@ -131,7 +165,14 @@ export async function sweepContactEnrichment( [input.workspaceId], ); - return { looked: rows.length, found, identities, remaining: Number(left?.n ?? 0) }; + return { + looked: rows.length, + found, + identities, + pagesQueued, + noPresence, + remaining: Number(left?.n ?? 0), + }; } /** Workspaces with imported people still waiting to be looked up. */