Make the weekly Wikidata fetch finish reliably

.github/workflows/ci.yml

@@ -31,8 +31,11 @@ jobs:
           node-version: 22
           cache: npm
       - run: npm ci
-      - name: Fetch two small countries and every station from Wikidata
-        run: npm run fetch -- LU MC
+      - name: Fetch three countries from Wikidata
+        # Great Britain and the United States are the heaviest countries for the time limit.
+        run: npm run fetch -- MC GB US
+        env:
+          FETCH_TRIES: "1"
       - run: npm run build
       - name: Sizes
         run: |

.github/workflows/publish.yml

@@ -21,7 +21,8 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
-    timeout-minutes: 300
+    # A full fetch takes a few hours; GitHub stops a job at six.
+    timeout-minutes: 355
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4

docs/format.md

@@ -93,7 +93,7 @@ have an article on it, to keep the best known when a client needs fewer:
 }
 ```
 
-Open mainline railway stations with coordinates and a UIC code or a Wikipedia article.
+Open mainline railway stations with coordinates and a UIC code or a Wikipedia (or other Wikimedia) page.
 `id` is the station's Wikidata item.
 
 ## `places/pack.json`

scripts/fetch.ts

@@ -1,16 +1,15 @@
 /**
  * Fetch towns, cities, their sights and railway stations from Wikidata to
- * data/raw/wikidata.json (git-ignored). One query per country, then the sights in batches
- * of cities, then the stations page by page, slowly enough to respect the query service's
- * limits. A country, batch or page that keeps failing is left out and named.
+ * data/raw/wikidata.json (git-ignored). One query per country for towns and for stations,
+ * then the sights in batches of cities, slowly enough to respect the query service's
+ * limits. A batch that fails is retried in smaller ones; whatever keeps failing is left
+ * out and named.
  * Run: npm run fetch [-- FR DE]   (default: every country)
  */
 import { mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import {
   COUNTRIES_QUERY,
-  STATION_PAGE,
-  bindings,
   citiesQuery,
   parseCities,
   parseCountries,
@@ -25,17 +24,24 @@ const ENDPOINT = "https://query.wikidata.org/sparql";
 // Wikidata asks every client to name itself and give a contact.
 const USER_AGENT = "offware-data/0.1 (https://github.com/offware-apps/data)";
 const OUT = resolve("data/raw/wikidata.json");
-const BATCH = 50;
-const TRIES = 3;
+const BATCH = 20;
+/** A failed batch of sights is retried this many cities at a time. */
+const SMALL_BATCH = 4;
+/** FETCH_TRIES=1 makes a test run fail fast. */
+const TRIES = Number(process.env.FETCH_TRIES) || 3;
 const PAUSE_MS = 1_000;
 /** Tiny territories time out now and then; more than this many missing countries fails the run. */
 const MAX_MISSING_COUNTRIES = 2;
+/** Share of towns whose sights may be missing before the run fails. */
+const MAX_MISSING_SIGHTS = 0.01;
 
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const started = Date.now();
+const log = (msg: string): void => console.log(`[fetch +${Math.round((Date.now() - started) / 60_000)} min] ${msg}`);
 
-async function sparql(query: string): Promise<unknown> {
+async function sparql(query: string, tries = TRIES): Promise<unknown> {
   let last: unknown;
-  for (let i = 1; i <= TRIES; i++) {
+  for (let i = 1; i <= tries; i++) {
     try {
       const res = await fetch(ENDPOINT, {
         method: "POST",
@@ -55,78 +61,91 @@ async function sparql(query: string): Promise<unknown> {
       return await res.json();
     } catch (err) {
       last = err;
-      console.error(`[fetch] attempt ${i}/${TRIES} failed: ${String(err)}`);
+      console.error(`[fetch] attempt ${i}/${tries} failed: ${String(err)}`);
       const wait = typeof (err as Error).cause === "number" ? ((err as Error).cause as number) * 1000 : 5_000 * i;
-      if (i < TRIES) await sleep(wait);
+      if (i < tries) await sleep(wait);
     }
   }
   throw last;
 }
 
 const countries = process.argv.slice(2).filter((c) => /^[A-Z]{2}$/.test(c));
 const raw: Raw = { fetchedAt: new Date().toISOString(), cities: [], sights: {}, stations: [] };
-const missing: string[] = [];
 const all = countries.length ? countries : parseCountries(await sparql(COUNTRIES_QUERY));
-console.log(`[fetch] ${all.length} countries`);
+log(`${all.length} countries`);
 
+const missing: string[] = [];
 for (const cc of all) {
   try {
-    const cities = parseCities(await sparql(citiesQuery(cc)), cc);
+    // The quick way first; a country too large for it is read the slow, sure way.
+    const result = await sparql(citiesQuery(cc), 1).catch(() => sparql(citiesQuery(cc, true)));
+    const cities = parseCities(result, cc);
     raw.cities.push(...cities);
-    console.log(`[fetch] ${cc}: ${cities.length} towns and cities`);
+    log(`${cc}: ${cities.length} towns and cities`);
   } catch (err) {
     missing.push(cc);
-    console.error(`[fetch] ${cc}: left out (${String(err)})`);
+    console.error(`[fetch] ${cc}: towns left out (${String(err)})`);
   }
   await sleep(PAUSE_MS);
 }
 if (raw.cities.length === 0) throw new Error("no town or city could be fetched from Wikidata");
 
-const ids = raw.cities.map((c) => c.wikidata);
-let failedBatches = 0;
-for (let i = 0; i < ids.length; i += BATCH) {
+const missingStations: string[] = [];
+const seen = new Set<string>();
+for (const cc of all) {
   try {
-    Object.assign(raw.sights, parseSights(await sparql(sightsQuery(ids.slice(i, i + BATCH)))));
+    const stations = parseStations(await sparql(stationsQuery(cc))).filter((s) => !seen.has(s.wikidata));
+    for (const s of stations) seen.add(s.wikidata);
+    raw.stations.push(...stations);
+    if (stations.length) log(`${cc}: ${stations.length} railway stations`);
   } catch (err) {
-    failedBatches++;
-    console.error(`[fetch] sights for cities ${i}–${i + BATCH} left out (${String(err)})`);
+    missingStations.push(cc);
+    console.error(`[fetch] ${cc}: stations left out (${String(err)})`);
   }
   await sleep(PAUSE_MS);
 }
+log(`${raw.stations.length} railway stations`);
 
-// Stations worldwide (or only in the countries asked for), a page at a time.
-const wanted = new Set(all);
-const seen = new Set<string>();
-let failedPages = 0;
-for (let offset = 0; ; offset += STATION_PAGE) {
-  let result: unknown;
+/** Sights of a batch of towns; a batch that fails is tried again a few towns at a time. */
+async function sights(ids: string[]): Promise<string[]> {
   try {
-    result = await sparql(stationsQuery(STATION_PAGE, offset));
+    Object.assign(raw.sights, parseSights(await sparql(sightsQuery(ids))));
+    return [];
   } catch (err) {
-    failedPages++;
-    console.error(`[fetch] stations from ${offset} left out (${String(err)})`);
-    if (failedPages >= 3) break;
-    continue;
-  }
-  for (const s of parseStations(result)) {
-    if (!wanted.has(s.countryIso2) || seen.has(s.wikidata)) continue;
-    seen.add(s.wikidata);
-    raw.stations.push(s);
+    if (ids.length <= SMALL_BATCH) {
+      console.error(`[fetch] sights of ${ids.join(" ")} left out (${String(err)})`);
+      return ids;
+    }
+    const failed: string[] = [];
+    for (let i = 0; i < ids.length; i += SMALL_BATCH) {
+      await sleep(PAUSE_MS);
+      failed.push(...(await sights(ids.slice(i, i + SMALL_BATCH))));
+    }
+    return failed;
   }
-  if (bindings(result).length < STATION_PAGE) break;
+}
+
+const ids = raw.cities.map((c) => c.wikidata);
+const noSights: string[] = [];
+for (let i = 0; i < ids.length; i += BATCH) {
+  noSights.push(...(await sights(ids.slice(i, i + BATCH))));
+  if ((i / BATCH) % 50 === 0) log(`sights: ${Math.min(i + BATCH, ids.length)} of ${ids.length} towns`);
   await sleep(PAUSE_MS);
 }
-console.log(`[fetch] ${raw.stations.length} railway stations`);
 
 mkdirSync(dirname(OUT), { recursive: true });
 writeFileSync(OUT, JSON.stringify(raw), "utf-8");
 const sightCount = Object.values(raw.sights).reduce((n, s) => n + s.length, 0);
-console.log(`[fetch] ${raw.cities.length} towns and cities, ${sightCount} sights → ${OUT}`);
-if (missing.length) console.log(`[fetch] countries left out: ${missing.join(" ")}`);
-if (failedBatches) console.log(`[fetch] sight batches left out: ${failedBatches}`);
-if (failedPages) console.log(`[fetch] station pages left out: ${failedPages}`);
-// A partial fetch would publish a dataset with holes: stop instead, and let the next run retry.
-if (missing.length > MAX_MISSING_COUNTRIES || failedBatches > 0 || failedPages > 0) {
+log(`${raw.cities.length} towns and cities, ${sightCount} sights, ${raw.stations.length} stations → ${OUT}`);
+if (missing.length) console.log(`[fetch] countries without towns: ${missing.join(" ")}`);
+if (missingStations.length) console.log(`[fetch] countries without stations: ${missingStations.join(" ")}`);
+if (noSights.length) console.log(`[fetch] towns without sights: ${noSights.length}`);
+// A fetch with real holes would publish them: stop instead, and let the next run retry.
+if (
+  missing.length > MAX_MISSING_COUNTRIES ||
+  missingStations.length > MAX_MISSING_COUNTRIES ||
+  noSights.length > ids.length * MAX_MISSING_SIGHTS
+) {
   console.error("[fetch] too much was left out to publish; nothing is published this run.");
   process.exit(1);
 }

src/catalog.ts

@@ -41,7 +41,7 @@ export const DATASETS: Dataset[] = [
   {
     id: "stations",
     name: "Railway stations worldwide",
-    description: "Open mainline railway stations with coordinates and a UIC code or a Wikipedia article.",
+    description: "Open mainline railway stations with coordinates and a UIC code or a Wikipedia (or other Wikimedia) page.",
     license: "CC0-1.0",
     sources: ["wikidata"],
     schedule: "weekly",

src/places.ts

@@ -147,7 +147,7 @@ export function stationsFile(stations: Station[], retrieved: string): unknown {
       url: "https://www.wikidata.org",
       license: LICENSE,
       retrieved: retrieved.slice(0, 10),
-      note: "Open mainline stations with coordinates and a UIC code or a Wikipedia article; most tram and metro stops are left out. Each station's id is its Wikidata item, where it can be corrected.",
+      note: "Open mainline stations with coordinates and a UIC code or a Wikipedia (or other Wikimedia) page; most tram and metro stops are left out. Each station's id is its Wikidata item, where it can be corrected.",
       coverage: "worldwide",
     },
     stations: [...stations]

src/wikidata.ts

@@ -84,20 +84,44 @@ export function parseCountries(result: unknown): string[] {
   return [...new Set(codes)].sort();
 }
 
-/** Towns and cities of one country with at least minPopulation() people. */
-export function citiesQuery(countryIso2: string): string {
-  return `SELECT ?city ?cityLabel ?pop ?coord ?voy ?wp ?site (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names) WHERE {
-  ?country wdt:P297 "${countryIso2}" .
-  ?city wdt:P17 ?country ; wdt:P1082 ?pop ; wdt:P625 ?coord ; wdt:P31 ?cls .
-  FILTER(?pop >= ${minPopulation(countryIso2)})
-  ?cls wdt:P279* wd:Q486972 .
+/**
+ * Towns and cities of one country with at least minPopulation() people. The towns are
+ * found first, in a named subquery the query service always runs before the rest, then
+ * their details read once each: a town with several populations, coordinates and labels
+ * would otherwise multiply into hundreds of rows, which made large countries time out.
+ */
+export function citiesQuery(countryIso2: string, byPopulation = false): string {
+  // byPopulation: for countries with millions of items (Great Britain, the United States),
+  // read every item with a population and keep the country's, in that order, rather than
+  // read every item of the country. Slower for small countries, but never times out.
+  const towns = byPopulation
+    ? `hint:SubQuery hint:optimizer "None" .
+    ?country wdt:P297 "${countryIso2}" .
+    ?city wdt:P1082 ?p0 .
+    FILTER(?p0 >= ${minPopulation(countryIso2)})
+    ?city wdt:P17 ?country ; wdt:P31 ?cls .
+    ?cls wdt:P279* wd:Q486972 .`
+    : `?country wdt:P297 "${countryIso2}" .
+    ?city wdt:P17 ?country ; wdt:P1082 ?p0 ; wdt:P31 ?cls .
+    FILTER(?p0 >= ${minPopulation(countryIso2)})
+    ?cls wdt:P279* wd:Q486972 .`;
+  return `SELECT ?city ?cityLabel (MAX(?p) AS ?pop) (SAMPLE(?c) AS ?coord) (SAMPLE(?v) AS ?voy) (SAMPLE(?w) AS ?wp) (SAMPLE(?s) AS ?site)
+       (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names)
+WITH {
+  SELECT DISTINCT ?city WHERE {
+    ${towns}
+  }
+} AS %towns
+WHERE {
+  INCLUDE %towns .
+  ?city wdt:P1082 ?p ; wdt:P625 ?c .
   OPTIONAL { ?city rdfs:label ?alt . FILTER(LANG(?alt) IN (${LANGS.map((l) => `"${l}"`).join(", ")})) }
-  OPTIONAL { ?voy schema:about ?city ; schema:isPartOf <https://en.wikivoyage.org/> . }
-  OPTIONAL { ?wp schema:about ?city ; schema:isPartOf <https://en.wikipedia.org/> . }
-  OPTIONAL { ?city wdt:P856 ?site . }
+  OPTIONAL { ?v schema:about ?city ; schema:isPartOf <https://en.wikivoyage.org/> . }
+  OPTIONAL { ?w schema:about ?city ; schema:isPartOf <https://en.wikipedia.org/> . }
+  OPTIONAL { ?city wdt:P856 ?s . }
   ${labelService}
 }
-GROUP BY ?city ?cityLabel ?pop ?coord ?voy ?wp ?site`;
+GROUP BY ?city ?cityLabel`;
 }
 
 /** Kinds of sight, by Wikidata class. UNESCO World Heritage sites are kept whatever their class. */
@@ -116,18 +140,26 @@ export const SIGHT_CLASSES: [string, string][] = [
   ["Q12280", "bridge"],
 ];
 
-/** Sights in a batch of cities (Wikidata ids), up to two administrative levels down. */
+/**
+ * Sights in a batch of cities (Wikidata ids), up to two administrative levels down. The
+ * query runs in the order written (optimizer off): from the cities down to the items in
+ * them, then the well-known ones, then their class. Left to itself the query service
+ * started from every church or museum on Wikidata and timed out.
+ */
 export function sightsQuery(cityIds: string[]): string {
   return `SELECT ?city ?item ?itemLabel ?category ?coord ?links ?unesco ?wp WHERE {
+  hint:Query hint:optimizer "None" .
   VALUES ?city { ${cityIds.map((id) => `wd:${id}`).join(" ")} }
-  ?item wdt:P131/wdt:P131? ?city ; wdt:P625 ?coord ; wikibase:sitelinks ?links .
+  { ?item wdt:P131 ?city . } UNION { ?part wdt:P131 ?city . ?item wdt:P131 ?part . }
+  ?item wikibase:sitelinks ?links .
   FILTER(?links >= 3)
   {
     VALUES (?cls ?category) { ${SIGHT_CLASSES.map(([c, t]) => `(wd:${c} "${t}")`).join(" ")} }
     ?item wdt:P31 ?cls .
   } UNION {
     ?item wdt:P757 ?whs . BIND("heritage site" AS ?category)
   }
+  ?item wdt:P625 ?coord .
   OPTIONAL { ?item wdt:P757 ?unesco . }
   OPTIONAL { ?wp schema:about ?item ; schema:isPartOf <https://en.wikipedia.org/> . }
   ${labelService}
@@ -242,29 +274,36 @@ export function parseSights(result: unknown): Record<string, Sight[]> {
 
 // ---------------------------------------------------------------- railway stations
 
-/** Rows per page of the stations query. */
-export const STATION_PAGE = 5000;
-
 /**
- * Mainline railway stations worldwide that are still open, with coordinates and either
- * a UIC code or a Wikipedia article (which leaves out most tram and metro stops).
+ * Mainline railway stations of one country that are still open, with coordinates and
+ * either a UIC code or a page on a Wikimedia project (which leaves out most tram and metro
+ * stops). One query per country, run in the written order: every railway station on
+ * Wikidata, then the country's, then their labels. Reading a large country's items first
+ * timed out.
  */
-export function stationsQuery(limit: number, offset: number): string {
-  return `SELECT ?station ?stationLabel ?iso2 ?coord WHERE {
-  ?station wdt:P31 wd:Q55488 ; wdt:P625 ?coord ; wdt:P17 ?country .
-  ?country wdt:P297 ?iso2 .
-  FILTER NOT EXISTS { ?station wdt:P576 ?closed . }
-  FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915 . }
-  FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883 . }
-  FILTER( EXISTS { ?station wdt:P954 ?uic . }
-          || EXISTS { ?article schema:about ?station ; schema:isPartOf ?wiki . FILTER(CONTAINS(STR(?wiki), "wikipedia.org")) } )
+export function stationsQuery(countryIso2: string): string {
+  return `SELECT ?station ?stationLabel ?iso2 ?coord
+WITH {
+  SELECT ?station (SAMPLE(?c) AS ?coord) WHERE {
+    hint:SubQuery hint:optimizer "None" .
+    ?country wdt:P297 "${countryIso2}" .
+    ?station wdt:P31 wd:Q55488 .
+    ?station wdt:P17 ?country ; wdt:P625 ?c ; wikibase:sitelinks ?links .
+    FILTER NOT EXISTS { ?station wdt:P576 ?closed . }
+    FILTER NOT EXISTS { ?station wdt:P5817 wd:Q56556915 . }
+    FILTER NOT EXISTS { ?station wdt:P5817 wd:Q45382883 . }
+    FILTER(?links > 0 || EXISTS { ?station wdt:P954 ?uic . })
+  }
+  GROUP BY ?station
+} AS %stations
+WHERE {
+  INCLUDE %stations .
+  BIND("${countryIso2}" AS ?iso2)
   ${labelService}
-}
-ORDER BY ?station
-LIMIT ${limit} OFFSET ${offset}`;
+}`;
 }
 
-/** Stations from one page of stationsQuery; a station in two countries keeps the first. */
+/** Stations from stationsQuery; a station listed twice keeps the first. */
 export function parseStations(result: unknown): Station[] {
   const out: Station[] = [];
   const seen = new Set<string>();

tests/places.test.ts

@@ -50,7 +50,7 @@ describe("Wikidata results", () => {
   it("keeps smaller towns in Europe than elsewhere", () => {
     expect(minPopulation("FR")).toBe(5000);
     expect(minPopulation("JP")).toBe(15000);
-    expect(citiesQuery("JP")).toContain("?pop >= 15000");
+    expect(citiesQuery("JP")).toContain("?p0 >= 15000");
   });
 
   it("builds queries for a country and a batch of cities", () => {