diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a6e72..33ae143 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,11 @@ jobs: node-version: 22 cache: npm - run: npm ci - - name: Fetch two small countries and every station from Wikidata - run: npm run fetch -- LU MC + - name: Fetch three countries from Wikidata + # Great Britain and the United States are the heaviest countries for the time limit. + run: npm run fetch -- MC GB US + env: + FETCH_TRIES: "1" - run: npm run build - name: Sizes run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 46b5fa6..18c19f2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,7 +21,8 @@ concurrency: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 300 + # A full fetch takes a few hours; GitHub stops a job at six. + timeout-minutes: 355 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 diff --git a/docs/format.md b/docs/format.md index e626a17..ce696d6 100644 --- a/docs/format.md +++ b/docs/format.md @@ -93,7 +93,7 @@ have an article on it, to keep the best known when a client needs fewer: } ``` -Open mainline railway stations with coordinates and a UIC code or a Wikipedia article. +Open mainline railway stations with coordinates and a UIC code or a Wikipedia (or other Wikimedia) page. `id` is the station's Wikidata item. ## `places/pack.json` diff --git a/scripts/fetch.ts b/scripts/fetch.ts index fc847d8..305c7ff 100644 --- a/scripts/fetch.ts +++ b/scripts/fetch.ts @@ -1,16 +1,15 @@ /** * Fetch towns, cities, their sights and railway stations from Wikidata to - * data/raw/wikidata.json (git-ignored). One query per country, then the sights in batches - * of cities, then the stations page by page, slowly enough to respect the query service's - * limits. A country, batch or page that keeps failing is left out and named. + * data/raw/wikidata.json (git-ignored). One query per country for towns and for stations, + * then the sights in batches of cities, slowly enough to respect the query service's + * limits. A batch that fails is retried in smaller ones; whatever keeps failing is left + * out and named. * Run: npm run fetch [-- FR DE] (default: every country) */ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { COUNTRIES_QUERY, - STATION_PAGE, - bindings, citiesQuery, parseCities, parseCountries, @@ -25,17 +24,24 @@ const ENDPOINT = "https://query.wikidata.org/sparql"; // Wikidata asks every client to name itself and give a contact. const USER_AGENT = "offware-data/0.1 (https://github.com/offware-apps/data)"; const OUT = resolve("data/raw/wikidata.json"); -const BATCH = 50; -const TRIES = 3; +const BATCH = 20; +/** A failed batch of sights is retried this many cities at a time. */ +const SMALL_BATCH = 4; +/** FETCH_TRIES=1 makes a test run fail fast. */ +const TRIES = Number(process.env.FETCH_TRIES) || 3; const PAUSE_MS = 1_000; /** Tiny territories time out now and then; more than this many missing countries fails the run. */ const MAX_MISSING_COUNTRIES = 2; +/** Share of towns whose sights may be missing before the run fails. */ +const MAX_MISSING_SIGHTS = 0.01; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const started = Date.now(); +const log = (msg: string): void => console.log(`[fetch +${Math.round((Date.now() - started) / 60_000)} min] ${msg}`); -async function sparql(query: string): Promise { +async function sparql(query: string, tries = TRIES): Promise { let last: unknown; - for (let i = 1; i <= TRIES; i++) { + for (let i = 1; i <= tries; i++) { try { const res = await fetch(ENDPOINT, { method: "POST", @@ -55,9 +61,9 @@ async function sparql(query: string): Promise { return await res.json(); } catch (err) { last = err; - console.error(`[fetch] attempt ${i}/${TRIES} failed: ${String(err)}`); + console.error(`[fetch] attempt ${i}/${tries} failed: ${String(err)}`); const wait = typeof (err as Error).cause === "number" ? ((err as Error).cause as number) * 1000 : 5_000 * i; - if (i < TRIES) await sleep(wait); + if (i < tries) await sleep(wait); } } throw last; @@ -65,68 +71,81 @@ async function sparql(query: string): Promise { const countries = process.argv.slice(2).filter((c) => /^[A-Z]{2}$/.test(c)); const raw: Raw = { fetchedAt: new Date().toISOString(), cities: [], sights: {}, stations: [] }; -const missing: string[] = []; const all = countries.length ? countries : parseCountries(await sparql(COUNTRIES_QUERY)); -console.log(`[fetch] ${all.length} countries`); +log(`${all.length} countries`); +const missing: string[] = []; for (const cc of all) { try { - const cities = parseCities(await sparql(citiesQuery(cc)), cc); + // The quick way first; a country too large for it is read the slow, sure way. + const result = await sparql(citiesQuery(cc), 1).catch(() => sparql(citiesQuery(cc, true))); + const cities = parseCities(result, cc); raw.cities.push(...cities); - console.log(`[fetch] ${cc}: ${cities.length} towns and cities`); + log(`${cc}: ${cities.length} towns and cities`); } catch (err) { missing.push(cc); - console.error(`[fetch] ${cc}: left out (${String(err)})`); + console.error(`[fetch] ${cc}: towns left out (${String(err)})`); } await sleep(PAUSE_MS); } if (raw.cities.length === 0) throw new Error("no town or city could be fetched from Wikidata"); -const ids = raw.cities.map((c) => c.wikidata); -let failedBatches = 0; -for (let i = 0; i < ids.length; i += BATCH) { +const missingStations: string[] = []; +const seen = new Set(); +for (const cc of all) { try { - Object.assign(raw.sights, parseSights(await sparql(sightsQuery(ids.slice(i, i + BATCH))))); + const stations = parseStations(await sparql(stationsQuery(cc))).filter((s) => !seen.has(s.wikidata)); + for (const s of stations) seen.add(s.wikidata); + raw.stations.push(...stations); + if (stations.length) log(`${cc}: ${stations.length} railway stations`); } catch (err) { - failedBatches++; - console.error(`[fetch] sights for cities ${i}–${i + BATCH} left out (${String(err)})`); + missingStations.push(cc); + console.error(`[fetch] ${cc}: stations left out (${String(err)})`); } await sleep(PAUSE_MS); } +log(`${raw.stations.length} railway stations`); -// Stations worldwide (or only in the countries asked for), a page at a time. -const wanted = new Set(all); -const seen = new Set(); -let failedPages = 0; -for (let offset = 0; ; offset += STATION_PAGE) { - let result: unknown; +/** Sights of a batch of towns; a batch that fails is tried again a few towns at a time. */ +async function sights(ids: string[]): Promise { try { - result = await sparql(stationsQuery(STATION_PAGE, offset)); + Object.assign(raw.sights, parseSights(await sparql(sightsQuery(ids)))); + return []; } catch (err) { - failedPages++; - console.error(`[fetch] stations from ${offset} left out (${String(err)})`); - if (failedPages >= 3) break; - continue; - } - for (const s of parseStations(result)) { - if (!wanted.has(s.countryIso2) || seen.has(s.wikidata)) continue; - seen.add(s.wikidata); - raw.stations.push(s); + if (ids.length <= SMALL_BATCH) { + console.error(`[fetch] sights of ${ids.join(" ")} left out (${String(err)})`); + return ids; + } + const failed: string[] = []; + for (let i = 0; i < ids.length; i += SMALL_BATCH) { + await sleep(PAUSE_MS); + failed.push(...(await sights(ids.slice(i, i + SMALL_BATCH)))); + } + return failed; } - if (bindings(result).length < STATION_PAGE) break; +} + +const ids = raw.cities.map((c) => c.wikidata); +const noSights: string[] = []; +for (let i = 0; i < ids.length; i += BATCH) { + noSights.push(...(await sights(ids.slice(i, i + BATCH)))); + if ((i / BATCH) % 50 === 0) log(`sights: ${Math.min(i + BATCH, ids.length)} of ${ids.length} towns`); await sleep(PAUSE_MS); } -console.log(`[fetch] ${raw.stations.length} railway stations`); mkdirSync(dirname(OUT), { recursive: true }); writeFileSync(OUT, JSON.stringify(raw), "utf-8"); const sightCount = Object.values(raw.sights).reduce((n, s) => n + s.length, 0); -console.log(`[fetch] ${raw.cities.length} towns and cities, ${sightCount} sights → ${OUT}`); -if (missing.length) console.log(`[fetch] countries left out: ${missing.join(" ")}`); -if (failedBatches) console.log(`[fetch] sight batches left out: ${failedBatches}`); -if (failedPages) console.log(`[fetch] station pages left out: ${failedPages}`); -// A partial fetch would publish a dataset with holes: stop instead, and let the next run retry. -if (missing.length > MAX_MISSING_COUNTRIES || failedBatches > 0 || failedPages > 0) { +log(`${raw.cities.length} towns and cities, ${sightCount} sights, ${raw.stations.length} stations → ${OUT}`); +if (missing.length) console.log(`[fetch] countries without towns: ${missing.join(" ")}`); +if (missingStations.length) console.log(`[fetch] countries without stations: ${missingStations.join(" ")}`); +if (noSights.length) console.log(`[fetch] towns without sights: ${noSights.length}`); +// A fetch with real holes would publish them: stop instead, and let the next run retry. +if ( + missing.length > MAX_MISSING_COUNTRIES || + missingStations.length > MAX_MISSING_COUNTRIES || + noSights.length > ids.length * MAX_MISSING_SIGHTS +) { console.error("[fetch] too much was left out to publish; nothing is published this run."); process.exit(1); } diff --git a/src/catalog.ts b/src/catalog.ts index 7f5b122..c738356 100644 --- a/src/catalog.ts +++ b/src/catalog.ts @@ -41,7 +41,7 @@ export const DATASETS: Dataset[] = [ { id: "stations", name: "Railway stations worldwide", - description: "Open mainline railway stations with coordinates and a UIC code or a Wikipedia article.", + description: "Open mainline railway stations with coordinates and a UIC code or a Wikipedia (or other Wikimedia) page.", license: "CC0-1.0", sources: ["wikidata"], schedule: "weekly", diff --git a/src/places.ts b/src/places.ts index 670ab1a..b32e063 100644 --- a/src/places.ts +++ b/src/places.ts @@ -147,7 +147,7 @@ export function stationsFile(stations: Station[], retrieved: string): unknown { url: "https://www.wikidata.org", license: LICENSE, retrieved: retrieved.slice(0, 10), - note: "Open mainline stations with coordinates and a UIC code or a Wikipedia article; most tram and metro stops are left out. Each station's id is its Wikidata item, where it can be corrected.", + note: "Open mainline stations with coordinates and a UIC code or a Wikipedia (or other Wikimedia) page; most tram and metro stops are left out. Each station's id is its Wikidata item, where it can be corrected.", coverage: "worldwide", }, stations: [...stations] diff --git a/src/wikidata.ts b/src/wikidata.ts index d88faf3..e5f8287 100644 --- a/src/wikidata.ts +++ b/src/wikidata.ts @@ -84,20 +84,44 @@ export function parseCountries(result: unknown): string[] { return [...new Set(codes)].sort(); } -/** Towns and cities of one country with at least minPopulation() people. */ -export function citiesQuery(countryIso2: string): string { - return `SELECT ?city ?cityLabel ?pop ?coord ?voy ?wp ?site (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names) WHERE { - ?country wdt:P297 "${countryIso2}" . - ?city wdt:P17 ?country ; wdt:P1082 ?pop ; wdt:P625 ?coord ; wdt:P31 ?cls . - FILTER(?pop >= ${minPopulation(countryIso2)}) - ?cls wdt:P279* wd:Q486972 . +/** + * Towns and cities of one country with at least minPopulation() people. The towns are + * found first, in a named subquery the query service always runs before the rest, then + * their details read once each: a town with several populations, coordinates and labels + * would otherwise multiply into hundreds of rows, which made large countries time out. + */ +export function citiesQuery(countryIso2: string, byPopulation = false): string { + // byPopulation: for countries with millions of items (Great Britain, the United States), + // read every item with a population and keep the country's, in that order, rather than + // read every item of the country. Slower for small countries, but never times out. + const towns = byPopulation + ? `hint:SubQuery hint:optimizer "None" . + ?country wdt:P297 "${countryIso2}" . + ?city wdt:P1082 ?p0 . + FILTER(?p0 >= ${minPopulation(countryIso2)}) + ?city wdt:P17 ?country ; wdt:P31 ?cls . + ?cls wdt:P279* wd:Q486972 .` + : `?country wdt:P297 "${countryIso2}" . + ?city wdt:P17 ?country ; wdt:P1082 ?p0 ; wdt:P31 ?cls . + FILTER(?p0 >= ${minPopulation(countryIso2)}) + ?cls wdt:P279* wd:Q486972 .`; + return `SELECT ?city ?cityLabel (MAX(?p) AS ?pop) (SAMPLE(?c) AS ?coord) (SAMPLE(?v) AS ?voy) (SAMPLE(?w) AS ?wp) (SAMPLE(?s) AS ?site) + (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names) +WITH { + SELECT DISTINCT ?city WHERE { + ${towns} + } +} AS %towns +WHERE { + INCLUDE %towns . + ?city wdt:P1082 ?p ; wdt:P625 ?c . OPTIONAL { ?city rdfs:label ?alt . FILTER(LANG(?alt) IN (${LANGS.map((l) => `"${l}"`).join(", ")})) } - OPTIONAL { ?voy schema:about ?city ; schema:isPartOf . } - OPTIONAL { ?wp schema:about ?city ; schema:isPartOf . } - OPTIONAL { ?city wdt:P856 ?site . } + OPTIONAL { ?v schema:about ?city ; schema:isPartOf . } + OPTIONAL { ?w schema:about ?city ; schema:isPartOf . } + OPTIONAL { ?city wdt:P856 ?s . } ${labelService} } -GROUP BY ?city ?cityLabel ?pop ?coord ?voy ?wp ?site`; +GROUP BY ?city ?cityLabel`; } /** Kinds of sight, by Wikidata class. UNESCO World Heritage sites are kept whatever their class. */ @@ -116,11 +140,18 @@ export const SIGHT_CLASSES: [string, string][] = [ ["Q12280", "bridge"], ]; -/** Sights in a batch of cities (Wikidata ids), up to two administrative levels down. */ +/** + * Sights in a batch of cities (Wikidata ids), up to two administrative levels down. The + * query runs in the order written (optimizer off): from the cities down to the items in + * them, then the well-known ones, then their class. Left to itself the query service + * started from every church or museum on Wikidata and timed out. + */ export function sightsQuery(cityIds: string[]): string { return `SELECT ?city ?item ?itemLabel ?category ?coord ?links ?unesco ?wp WHERE { + hint:Query hint:optimizer "None" . VALUES ?city { ${cityIds.map((id) => `wd:${id}`).join(" ")} } - ?item wdt:P131/wdt:P131? ?city ; wdt:P625 ?coord ; wikibase:sitelinks ?links . + { ?item wdt:P131 ?city . } UNION { ?part wdt:P131 ?city . ?item wdt:P131 ?part . } + ?item wikibase:sitelinks ?links . FILTER(?links >= 3) { VALUES (?cls ?category) { ${SIGHT_CLASSES.map(([c, t]) => `(wd:${c} "${t}")`).join(" ")} } @@ -128,6 +159,7 @@ export function sightsQuery(cityIds: string[]): string { } UNION { ?item wdt:P757 ?whs . BIND("heritage site" AS ?category) } + ?item wdt:P625 ?coord . OPTIONAL { ?item wdt:P757 ?unesco . } OPTIONAL { ?wp schema:about ?item ; schema:isPartOf . } ${labelService} @@ -242,29 +274,36 @@ export function parseSights(result: unknown): Record { // ---------------------------------------------------------------- railway stations -/** Rows per page of the stations query. */ -export const STATION_PAGE = 5000; - /** - * Mainline railway stations worldwide that are still open, with coordinates and either - * a UIC code or a Wikipedia article (which leaves out most tram and metro stops). + * Mainline railway stations of one country that are still open, with coordinates and + * either a UIC code or a page on a Wikimedia project (which leaves out most tram and metro + * stops). One query per country, run in the written order: every railway station on + * Wikidata, then the country's, then their labels. Reading a large country's items first + * timed out. */ -export function stationsQuery(limit: number, offset: number): string { - return `SELECT ?station ?stationLabel ?iso2 ?coord WHERE { - ?station wdt:P31 wd:Q55488 ; wdt:P625 ?coord ; wdt:P17 ?country . - ?country wdt:P297 ?iso2 . - FILTER NOT EXISTS { ?station wdt:P576 ?closed . } - FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915 . } - FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883 . } - FILTER( EXISTS { ?station wdt:P954 ?uic . } - || EXISTS { ?article schema:about ?station ; schema:isPartOf ?wiki . FILTER(CONTAINS(STR(?wiki), "wikipedia.org")) } ) +export function stationsQuery(countryIso2: string): string { + return `SELECT ?station ?stationLabel ?iso2 ?coord +WITH { + SELECT ?station (SAMPLE(?c) AS ?coord) WHERE { + hint:SubQuery hint:optimizer "None" . + ?country wdt:P297 "${countryIso2}" . + ?station wdt:P31 wd:Q55488 . + ?station wdt:P17 ?country ; wdt:P625 ?c ; wikibase:sitelinks ?links . + FILTER NOT EXISTS { ?station wdt:P576 ?closed . } + FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915 . } + FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883 . } + FILTER(?links > 0 || EXISTS { ?station wdt:P954 ?uic . }) + } + GROUP BY ?station +} AS %stations +WHERE { + INCLUDE %stations . + BIND("${countryIso2}" AS ?iso2) ${labelService} -} -ORDER BY ?station -LIMIT ${limit} OFFSET ${offset}`; +}`; } -/** Stations from one page of stationsQuery; a station in two countries keeps the first. */ +/** Stations from stationsQuery; a station listed twice keeps the first. */ export function parseStations(result: unknown): Station[] { const out: Station[] = []; const seen = new Set(); diff --git a/tests/places.test.ts b/tests/places.test.ts index 783f3a5..27c42e7 100644 --- a/tests/places.test.ts +++ b/tests/places.test.ts @@ -50,7 +50,7 @@ describe("Wikidata results", () => { it("keeps smaller towns in Europe than elsewhere", () => { expect(minPopulation("FR")).toBe(5000); expect(minPopulation("JP")).toBe(15000); - expect(citiesQuery("JP")).toContain("?pop >= 15000"); + expect(citiesQuery("JP")).toContain("?p0 >= 15000"); }); it("builds queries for a country and a batch of cities", () => {