From ddfbc69b5ab07477afc49e6e7eb50cd733a3c7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 12 Jun 2026 10:32:56 -0400 Subject: [PATCH] build: abort download task on abnormal status code --- data/resources.js | 6 +++--- package.json | 3 +-- scripts/download.js | 32 ++++++++++++++++++++++++++++---- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/data/resources.js b/data/resources.js index 1b9ac8b..acad835 100644 --- a/data/resources.js +++ b/data/resources.js @@ -870,12 +870,12 @@ const resources = [ emoji: 'https://unicode.org/Public/17.0.0/ucd/emoji/emoji-data.txt', // Emoji_Keycap_Sequence, Emoji_Flag_Sequence, Emoji_Modifier_Sequence 'emoji-sequences': - 'https://www.unicode.org/Public/17.0.0/emoji/emoji-sequences.txt', + 'https://unicode.org/Public/17.0.0/emoji/emoji-sequences.txt', // Emoji_ZWJ_Sequence 'emoji-zwj-sequences': - 'https://www.unicode.org/Public/17.0.0/emoji/emoji-zwj-sequences.txt', + 'https://unicode.org/Public/17.0.0/emoji/emoji-zwj-sequences.txt', // Emoji_Test (not an official property) - 'emoji-test': 'https://www.unicode.org/Public/17.0.0/emoji/emoji-test.txt', + 'emoji-test': 'https://unicode.org/Public/17.0.0/emoji/emoji-test.txt', }, ]; diff --git a/package.json b/package.json index 6a2b0a7..1a255e8 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "regenerate": "^1.4.2", "unicode-loose-match": "^2.8.0", "unicode-property-aliases": "^2.1.0", - "unicode-property-value-aliases": "^3.9.0", - "when": "^3.7.8" + "unicode-property-value-aliases": "^3.9.0" } } diff --git a/scripts/download.js b/scripts/download.js index a79a1cc..9f49f75 100644 --- a/scripts/download.js +++ b/scripts/download.js @@ -1,7 +1,6 @@ 'use strict'; const fs = require('fs'); -const guard = require('when/guard'); const path = require('path'); const { Readable } = require('stream'); const { finished } = require('stream/promises'); @@ -17,12 +16,35 @@ const download = async function(url, version, type) { ); console.log(' ', url, '→', path.basename(file)); //console.log(`curl ${url} > data/${path.basename(file)};`); + if (!res.ok) { + throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`); + } return finished( Readable.fromWeb(res.body).pipe(fs.createWriteStream(file)) ); }; + +async function parallelLimit(tasks, limit) { + const results = new Array(tasks.length); + const iterator = tasks.entries(); + + async function worker() { + for (const [index, task] of iterator) { + results[index] = await task(); + } + } + + const workers = new Array(limit).fill(null).map(() => worker()); + await Promise.all(workers); + return results; +} + +const tasks = []; +const addDownloadTask = (url, version, type) => tasks.push(() => download(url, version, type)); + // Limit maximum parallelism to something reasonable -const guardedDownload = guard(guard.n(PARALLEL_REQUEST_LIMIT), download); +parallelLimit(tasks, PARALLEL_REQUEST_LIMIT); +const guardedDownload = () => parallelLimit(tasks, PARALLEL_REQUEST_LIMIT); console.log('Downloading resources…'); @@ -53,10 +75,12 @@ const TYPES = [ for (const resource of resources) { const version = resource.version; - guardedDownload(resource.main, version, 'database'); + addDownloadTask(resource.main, version, 'database'); for (const type of TYPES) { if (resource[type]) { - guardedDownload(resource[type], version, type); + addDownloadTask(resource[type], version, type); } } } + +guardedDownload();