diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index aff0df0..c4ad5fd 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -669,6 +669,7 @@ async function tick(): Promise { if (swept.looked > 0) { console.log( `enrichment: looked up ${swept.looked} in ${workspaceId}, ` + + `${swept.pagesQueued} page(s) queued, ${swept.noPresence} point nowhere, ` + `${swept.found} had a profile, ${swept.identities} identities, ` + `${swept.remaining} left`, ); diff --git a/packages/domain/src/contact-import.ts b/packages/domain/src/contact-import.ts index 69cbc91..f20623d 100644 --- a/packages/domain/src/contact-import.ts +++ b/packages/domain/src/contact-import.ts @@ -212,6 +212,40 @@ const FREEMAIL_DOMAINS = new Set([ 'fastmail.com', 'hey.com', 'tutanota.com', + // Consumer mailboxes outside the English-speaking web. Their absence was a + // real gap rather than an oversight of scale: a 16,268-row list carried 127 + // `qq.com` addresses, every one of which would have been treated as a + // company domain and sent the crawler to Tencent's homepage. + 'qq.com', + '163.com', + '126.com', + 'sina.com', + 'sina.cn', + 'foxmail.com', + 'naver.com', + 'daum.net', + 'hanmail.net', + 'rediffmail.com', + 'seznam.cz', + 'libero.it', + 'orange.fr', + 'free.fr', + 'laposte.net', + 't-online.de', + 'web.de', + 'bluewin.ch', + 'telenet.be', + 'bigpond.com', + 'ymail.com', + 'rocketmail.com', + 'live.co.uk', + 'btinternet.com', + 'comcast.net', + 'verizon.net', + 'sbcglobal.net', + 'att.net', + 'cox.net', + 'charter.net', ]); /** diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 470c084..da8c3a8 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -23,6 +23,7 @@ export * from './playbooks'; export * from './plans'; export * from './credits'; export * from './contact-import'; +export * from './web-presence'; export * from './rules'; export * from './pipeline'; export * from './outreach'; diff --git a/packages/domain/src/web-presence.test.ts b/packages/domain/src/web-presence.test.ts new file mode 100644 index 0000000..a081ba8 --- /dev/null +++ b/packages/domain/src/web-presence.test.ts @@ -0,0 +1,102 @@ +/** + * Turning an address into the page it points at. + * + * The tests that matter are the ones that must resolve to *nothing*. A wrong + * publication URL costs one wasted fetch; a rule that sends every Gmail + * address to google.com costs sixteen thousand fetches of a page about nobody, + * and looks like the feature working. + */ + +import { describe, expect, test } from 'bun:test'; +import { presenceBreakdown, webPresenceFor } from './web-presence'; + +describe('publications', () => { + test('a substack handle is that publication', () => { + // Measured: 5,934 of one real 16,268-row list. The local part is the + // subdomain, so the page is certain rather than guessed. + expect(webPresenceFor('0xshah@substack.com')).toMatchObject({ + url: 'https://0xshah.substack.com', + kind: 'publication', + }); + }); + + test('the other newsletter and blog hosts', () => { + expect(webPresenceFor('nils@ghost.io')?.url).toBe('https://nils.ghost.io'); + expect(webPresenceFor('dave@medium.com')?.url).toBe('https://medium.com/@dave'); + expect(webPresenceFor('show@anchor.fm')?.url).toBe('https://anchor.fm/show'); + expect(webPresenceFor('news@beehiiv.com')?.url).toBe('https://news.beehiiv.com'); + }); + + test('a local part that cannot be a subdomain resolves to nothing', () => { + // `first.last@substack.com` is not `first.last.substack.com`; dots and + // plus tags would build a hostname that does not exist. + expect(webPresenceFor('first.last@substack.com')).toBeUndefined(); + expect(webPresenceFor('dave+news@substack.com')).toBeUndefined(); + expect(webPresenceFor('under_score@substack.com')).toBeUndefined(); + }); + + test('the basis says where the page came from', () => { + expect(webPresenceFor('pasta@substack.com')?.basis).toContain('publication handle'); + }); +}); + +describe('company domains', () => { + test('a company mailbox points at that company', () => { + expect(webPresenceFor('dave@acme.io')).toMatchObject({ + url: 'https://acme.io', + kind: 'company', + }); + }); + + test('a subdomain is kept, because that is where the mail lands', () => { + expect(webPresenceFor('dave@eng.acme.io')?.url).toBe('https://eng.acme.io'); + }); +}); + +describe('addresses that must point nowhere', () => { + test('consumer mailboxes', () => { + // The expensive mistake: 33% of the list. Crawling these would read + // Google's homepage five thousand times. + for (const email of [ + 'dave@gmail.com', + 'dave@yahoo.com', + 'dave@hotmail.com', + 'dave@proton.me', + 'dave@icloud.com', + 'dave@outlook.com', + 'dave@qq.com', + ]) { + expect(webPresenceFor(email)).toBeUndefined(); + } + }); + + test('forwarders and mail hosts that are not the person', () => { + // These look like company domains and are not: crawling them reads a + // hosting company's marketing page. + expect(webPresenceFor('x@agentmail.to')).toBeUndefined(); + expect(webPresenceFor('x@duck.com')).toBeUndefined(); + expect(webPresenceFor('x@privaterelay.appleid.com')).toBeUndefined(); + expect(webPresenceFor('x@fastmail.com')).toBeUndefined(); + }); + + test('malformed input', () => { + expect(webPresenceFor('not-an-address')).toBeUndefined(); + expect(webPresenceFor('@nolocal.com')).toBeUndefined(); + expect(webPresenceFor('dave@nodot')).toBeUndefined(); + }); +}); + +describe('presenceBreakdown', () => { + test('says what a list will actually reach before anyone waits for it', () => { + const breakdown = presenceBreakdown([ + '0xshah@substack.com', + 'nils@substack.com', + 'dave@acme.io', + 'someone@gmail.com', + 'other@gmail.com', + 'third@yahoo.com', + ]); + + expect(breakdown).toEqual({ publication: 2, company: 1, none: 3 }); + }); +}); diff --git a/packages/domain/src/web-presence.ts b/packages/domain/src/web-presence.ts new file mode 100644 index 0000000..7828bc9 --- /dev/null +++ b/packages/domain/src/web-presence.ts @@ -0,0 +1,148 @@ +/** + * The page an address implies. + * + * An email is not just a mailbox; for a large share of a real list it is a + * pointer to somewhere on the web that says who the person is. Reading that + * page is how a row with nothing but an address acquires a bio, a company, and + * the social accounts the owner chose to publish. + * + * The mapping is deliberately **derivation, not guesswork**. Every rule here + * turns an address into the page that address provably belongs to: + * `0xshah@substack.com` is the account behind `0xshah.substack.com`, and + * `dave@acme.com` is a mailbox at the company whose site is `acme.com`. No + * rule invents a handle on a network the person never mentioned — this + * codebase already refuses to do that for email addresses, and the same + * argument applies harder to social identities, which cannot be verified by + * sending to them. + * + * Measured against a real 16,268-row list: 36% resolved to a publication, 31% + * to a company site, and 33% were mailbox providers that imply nothing at all. + */ + +import { isFreemailDomain } from './contact-import'; + +export type PresenceKind = + /** A page about this person, published by them. */ + | 'publication' + /** The site of the company whose domain they receive mail at. */ + | 'company'; + +export interface WebPresence { + readonly url: string; + readonly kind: PresenceKind; + /** Why we think this page is theirs, shown next to anything it produces. */ + readonly basis: string; +} + +/** + * Hosts where the local part names a page rather than a person's mailbox. + * + * These are the high-yield cases: the address *is* the identifier of a public + * profile, so the page is certain rather than inferred. A list assembled from + * newsletters is mostly this, which is why it is worth special-casing at all. + */ +const PUBLICATION_HOSTS: Readonly string>> = { + 'substack.com': (local) => `https://${local}.substack.com`, + 'medium.com': (local) => `https://medium.com/@${local}`, + 'ghost.io': (local) => `https://${local}.ghost.io`, + 'wordpress.com': (local) => `https://${local}.wordpress.com`, + 'blogspot.com': (local) => `https://${local}.blogspot.com`, + 'tumblr.com': (local) => `https://${local}.tumblr.com`, + 'anchor.fm': (local) => `https://anchor.fm/${local}`, + 'beehiiv.com': (local) => `https://${local}.beehiiv.com`, +}; + +/** + * Domains that host mail for other people's sites, so the domain says nothing + * about the person and crawling it would read a hosting company's homepage. + * + * Distinct from the freemail list, which is about consumer mailboxes. These + * are the ones that look like a company domain and are not. + */ +const NON_COMPANY_DOMAINS = new Set([ + 'agentmail.to', + 'sharklasers.com', + 'simplelogin.com', + 'simplelogin.io', + 'anonaddy.com', + 'anonaddy.me', + 'duck.com', + 'relay.firefox.com', + 'privaterelay.appleid.com', + 'mozmail.com', + 'hey.com', + 'fastmail.com', + 'migadu.com', + 'zoho.com', + 'yandex.com', + 'mail.com', + 'gmx.net', + 'email.com', +]); + +/** + * The page this address points at, if any. + * + * `undefined` for a consumer mailbox, which is the honest answer: `gmail.com` + * is not a website about anybody, and crawling it would read Google's homepage + * sixteen thousand times. + */ +export function webPresenceFor(email: string): WebPresence | undefined { + const at = email.lastIndexOf('@'); + if (at <= 0) return undefined; + + const local = email.slice(0, at).toLowerCase(); + const domain = email.slice(at + 1).toLowerCase(); + + const publication = PUBLICATION_HOSTS[domain]; + + if (publication) { + // A local part with punctuation a subdomain cannot carry is not a handle. + if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(local)) return undefined; + + return { + url: publication(local), + kind: 'publication', + basis: `${domain} publication handle "${local}"`, + }; + } + + if (isFreemailDomain(domain) || NON_COMPANY_DOMAINS.has(domain)) return undefined; + + // A bare domain with no dot cannot resolve, and a public-suffix-only domain + // would send us to a registry. + if (!domain.includes('.')) return undefined; + + return { + url: `https://${domain}`, + kind: 'company', + basis: `receives mail at ${domain}`, + }; +} + +/** + * How many of a list we can find a page for, by kind. + * + * Exported so the import screen can say what enrichment will actually reach + * before anyone waits an hour for it. "We looked everywhere and found nothing" + * is a much worse answer than "a third of this list is Gmail addresses, which + * point nowhere." + */ +export function presenceBreakdown(emails: readonly string[]): { + readonly publication: number; + readonly company: number; + readonly none: number; +} { + let publication = 0; + let company = 0; + let none = 0; + + for (const email of emails) { + const presence = webPresenceFor(email); + if (!presence) none += 1; + else if (presence.kind === 'publication') publication += 1; + else company += 1; + } + + return { publication, company, none }; +} diff --git a/packages/pipeline/src/enrich-contact.ts b/packages/pipeline/src/enrich-contact.ts index f4eacd8..c53899a 100644 --- a/packages/pipeline/src/enrich-contact.ts +++ b/packages/pipeline/src/enrich-contact.ts @@ -21,9 +21,11 @@ * people who never published them. */ -import { newId, isFreemailDomain } from '@outreachgraph/domain'; +import { newId, isFreemailDomain, webPresenceFor } from '@outreachgraph/domain'; import { GRAVATAR_NETWORKS, lookupGravatar, type GravatarOptions } from '@outreachgraph/providers'; import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { enqueue } from './queue'; +import { crawlDedupeKey } from './auto-approve'; /** * How much to believe a Gravatar-published account. @@ -52,6 +54,10 @@ export interface SweepResult { readonly found: number; readonly identities: number; readonly remaining: number; + /** Pages queued to read, deduplicated by host. */ + readonly pagesQueued: number; + /** Addresses that point at no page at all — mailbox providers. */ + readonly noPresence: number; } /** @@ -72,20 +78,22 @@ export async function sweepContactEnrichment( ): Promise { const limit = input.limit ?? SWEEP_SIZE; - const rows = await queryAll<{ person_id: string }>( + const rows = await queryAll<{ person_id: string; address: string }>( db, - `SELECT pe.person_id + `SELECT pe.person_id, min(pe.address) AS address FROM person_emails pe JOIN people p ON p.id = pe.person_id WHERE pe.workspace_id = ? AND p.contact_enriched_at IS NULL AND p.status = 'active' GROUP BY pe.person_id - ORDER BY pe.created_at + ORDER BY min(pe.created_at) LIMIT ?`, [input.workspaceId, limit], ); let found = 0; let identities = 0; + let pagesQueued = 0; + let noPresence = 0; // Fixed-size waves rather than one promise per person: seventeen thousand // concurrent fetches would be a denial of service on somebody else's free @@ -114,6 +122,32 @@ export async function sweepContactEnrichment( identities += result?.identities ?? 0; } + // Reading the page the address points at is where the rest comes from. + // Gravatar answers for about one person in a hundred; a Substack handle or + // a company domain is a page that exists by construction, and the crawler + // already knows how to pull a bio, a title and published social links out + // of one. + for (const row of wave) { + const presence = webPresenceFor(row.address); + + if (!presence) { + noPresence += 1; + continue; + } + + const queued = await enqueue(db, { + workspaceId: input.workspaceId, + kind: 'crawl_site', + payload: { url: presence.url }, + // The same key everything else uses, so four hundred people at one + // company read that company's site once — the lesson from #63, which + // cost 226 identical crawls of accenture.com to learn. + dedupeKey: crawlDedupeKey(presence.url), + }); + + if (queued.queued) pagesQueued += 1; + } + await db.batch( wave.map((row) => ({ sql: 'UPDATE people SET contact_enriched_at = ? WHERE id = ?', @@ -131,7 +165,14 @@ export async function sweepContactEnrichment( [input.workspaceId], ); - return { looked: rows.length, found, identities, remaining: Number(left?.n ?? 0) }; + return { + looked: rows.length, + found, + identities, + pagesQueued, + noPresence, + remaining: Number(left?.n ?? 0), + }; } /** Workspaces with imported people still waiting to be looked up. */