feat: regression test

.vscode/launch.json

@@ -4,10 +4,18 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Program",
+      "name": "Launch Basic Usage",
       "skipFiles": ["<node_internals>/**"],
       "program": "${workspaceFolder}/dist/basic-usage.js",
       "outFiles": ["${workspaceFolder}/**/*.js"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Run Regression Test",
+      "skipFiles": ["<node_internals>/**"],
+      "program": "${workspaceFolder}/dist/regression-test/main.js",
+      "outFiles": ["${workspaceFolder}/**/*.js"]
     }
   ]
 }

.vscode/settings.json

@@ -9,5 +9,8 @@
   "liveServer.settings.port": 5501,
   "[json]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[plaintext]": {
+    "editor.renderControlCharacters": false
   }
 }

package.json

@@ -74,4 +74,4 @@
     "dist",
     "usage-examples"
   ]
-}
+}

src/regression-test/main.ts

@@ -0,0 +1,114 @@
+/*
+    Regression tests. Run this file and check the git diff of the output files to see what changed.
+*/
+
+import { readFileSync, writeFileSync } from 'fs';
+import { Config } from '../config.js';
+import { EntityMatch } from '../interfaces/entity-match.js';
+import { EntityResult } from '../interfaces/entity-result.js';
+import { Meta } from '../interfaces/meta.js';
+import { Query } from '../interfaces/query.js';
+import { SearcherFactory } from '../searcher-factory.js';
+
+const outputPath = './src/regression-test/output';
+const shortColumnWidth = 8;
+const wideColumnWidth = 40;
+const RLI = '\u2067';
+const LRI = '\u2066';
+const PDI = '\u2069';
+
+const text = readFileSync('./data/world-ctvs.txt', 'utf8');
+const lines = text.split('\n').slice(1);
+const entities = lines.map((l, index) => ({ id: index, name: l }));
+
+interface GeoEntity {
+  id: number;
+  name: string;
+}
+
+const config = Config.createDefaultConfig();
+config.normalizerConfig.allowCharacter = (_) => true;
+const searcher = SearcherFactory.createSearcher<GeoEntity, number>(config);
+
+console.log(`Indexing ${entities.length} entities...`);
+const indexingMeta: Meta = searcher.indexEntities(
+  entities,
+  (e) => e.id,
+  (e) => e.name.split(';')
+);
+writeFileSync(`${outputPath}/_indexing-meta.txt`, metaToJson(indexingMeta), { encoding: 'utf8' });
+
+console.log('Running queries...');
+
+runQuery('carcassonne-prefix', 'carcasso');
+runQuery('carcassonne-infix', 'cassonn');
+runQuery('carcassonne-suffix', 'sonne');
+runQuery('munich-insertion', 'muniich');
+runQuery('boston-deletion', 'bostn');
+runQuery('boulder-creek-substitution', 'boulder creak');
+runQuery('tübingen-transposition', 'tübignen');
+runQuery('tokyo', '東京都');
+runQuery('tokyo-prefix', '東京');
+runQuery('tbilisi', 'თბილისი');
+runQuery('tbilisi-deletion', 'თბიისი');
+runQuery('kuwait-city', 'مدينة الكويت', true);
+runQuery('kuwait-city-prefix', 'مدينة الك', true);
+
+console.log('Finished.');
+
+function runQuery(queryName: string, queryString: string, rtl: boolean = false) {
+  const query: Query = new Query(queryString);
+  const result: EntityResult<GeoEntity> = searcher.getMatches(query);
+  console.log(`'${queryString}' (${queryName}): ${result.matches.length} matches.`);
+  const queryJson = JSON.stringify(query, null, 2);
+  const metaJson = metaToJson(result.meta);
+  const matchesString = matchesToString(result.matches, rtl);
+  const output = `${queryJson}\n\n${metaJson}\n\n${matchesString}`;
+  writeFileSync(`${outputPath}/${queryName}.txt`, output);
+}
+
+function metaToJson(meta: Meta): string {
+  return JSON.stringify(Object.fromEntries(meta.allEntries), null, 2);
+}
+
+function matchesToString(matches: EntityMatch<GeoEntity>[], rtl: boolean): string {
+  const header =
+    padRight('Rank', shortColumnWidth) +
+    padRight('Entity', wideColumnWidth) +
+    padRight('Matched String', wideColumnWidth) +
+    padRight('Quality', shortColumnWidth) +
+    '\n\n';
+  const matchesString = matches.map((m, i) => matchToString(m, i + 1, rtl)).join('\n');
+  return header + matchesString;
+}
+
+function matchToString(match: EntityMatch<GeoEntity>, rank: number, rtl: boolean): string {
+  if (!rtl) {
+    return (
+      padRight(rank.toString(), shortColumnWidth) +
+      padRight(match.entity.name, wideColumnWidth) +
+      padRight(match.matchedString, wideColumnWidth) +
+      padRight(match.quality.toFixed(2), shortColumnWidth)
+    );
+  }
+  return (
+    padAndMark(rank.toString(), shortColumnWidth, false) +
+    padAndMark(match.entity.name, wideColumnWidth, true) +
+    padAndMark(match.matchedString, wideColumnWidth, true) +
+    padAndMark(match.quality.toFixed(2), shortColumnWidth, false)
+  );
+}
+
+function padAndMark(s: string, targetWidth: number, rtl: boolean): string {
+  const padded = rtl ? padLeft(s, targetWidth) : padRight(s, targetWidth);
+  const mark = rtl ? RLI : LRI;
+  return mark + padded + PDI;
+}
+
+function padRight(s: string, targetWidth: number): string {
+  return s + ' '.repeat(Math.max(0, targetWidth - s.length));
+}
+
+function padLeft(s: string, targetWidth: number): string {
+  return ' '.repeat(Math.max(0, targetWidth - s.length)) + s;
+}

src/regression-test/output/_indexing-meta.txt

@@ -0,0 +1,9 @@
+{
+  "numberOfInvalidTerms": 1,
+  "numberOfDistinctTerms": 1190185,
+  "normalizationDuration": 1342,
+  "numberOfSurrogateCharacters": 46,
+  "numberOfEntities": 1237154,
+  "numberOfTerms": 1237486,
+  "indexingDuration": 6741
+}

src/regression-test/output/boston-deletion.txt

@@ -0,0 +1,22 @@
+{
+  "string": "bostn",
+  "topN": 10,
+  "minQuality": 0.3
+}
+
+{
+  "queryDuration": 2
+}
+
+Rank    Entity                                  Matched String                          Quality 
+
+1       Bosti                                   Bosti                                   0.67    
+2       Bosto                                   Bosto                                   0.67    
+3       Bosta                                   Bosta                                   0.67    
+4       Bost                                    Bost                                    0.63    
+5       Boston                                  Boston                                  0.54    
+6       Boston                                  Boston                                  0.54    
+7       Boston                                  Boston                                  0.54    
+8       Boston                                  Boston                                  0.54    
+9       Bostel                                  Bostel                                  0.54    
+10      Bostan                                  Bostan                                  0.54    

src/regression-test/output/boulder-creek-substitution.txt

@@ -0,0 +1,22 @@
+{
+  "string": "boulder creak",
+  "topN": 10,
+  "minQuality": 0.3
+}
+
+{
+  "queryDuration": 2
+}
+
+Rank    Entity                                  Matched String                          Quality 
+
+1       Boulder Creek                           Boulder Creek                           0.75    
+2       Boulder City                            Boulder City                            0.61    
+3       Boulder                                 Boulder                                 0.54    
+4       Bouldercombe                            Bouldercombe                            0.54    
+5       South Boulder                           South Boulder                           0.54    
+6       Boulder                                 Boulder                                 0.54    
+7       Boulder Hill                            Boulder Hill                            0.54    
+8       The Boulders                            The Boulders                            0.47    
+9       Boulders                                Boulders                                0.47    
+10      Boulder Junction                        Boulder Junction                        0.45    

src/regression-test/output/carcassonne-infix.txt

@@ -0,0 +1,22 @@
+{
+  "string": "cassonn",
+  "topN": 10,
+  "minQuality": 0.3
+}
+
+{
+  "queryDuration": 4
+}
+
+Rank    Entity                                  Matched String                          Quality 
+
+1       Cassone                                 Cassone                                 0.75    
+2       Casson                                  Casson                                  0.71    
+3       Cassou                                  Cassou                                  0.59    
+4       Cassola                                 Cassola                                 0.59    
+5       Cassop                                  Cassop                                  0.59    
+6       Canossa                                 Canossa                                 0.59    
+7       Cassoneca                               Cassoneca                               0.57    
+8       Cassongue                               Cassongue                               0.57    
+9       Carcassonne                             Carcassonne                             0.55    
+10      Cassoday                                Cassoday                                0.53    

src/regression-test/output/carcassonne-prefix.txt

@@ -0,0 +1,22 @@
+{
+  "string": "carcasso",
+  "topN": 10,
+  "minQuality": 0.3
+}
+
+{
+  "queryDuration": 9
+}
+
+Rank    Entity                                  Matched String                          Quality 
+
+1       Carcasse                                Carcasse                                0.74    
+2       Carcassonne                             Carcassonne                             0.63    
+3       Carasso                                 Carasso                                 0.63    
+4       Carcas                                  Carcas                                  0.63    
+5       Carcasí                                 Carcasí                                 0.63    
+6       Carcasi                                 Carcasi                                 0.63    
+7       Carcaboso                               Carcaboso                               0.57    
+8       Casaracra                               Casaracra                               0.57    
+9       Cassou                                  Cassou                                  0.53    
+10      Caracase                                Caracase                                0.53    

src/regression-test/output/carcassonne-suffix.txt

@@ -0,0 +1,22 @@
+{
+  "string": "sonne",
+  "topN": 10,
+  "minQuality": 0.3
+}
+
+{
+  "queryDuration": 2
+}
+
+Rank    Entity                                  Matched String                          Quality 
+
+1       Sonnen                                  Sonnen                                  0.81    
+2       Sonene                                  Sonene                                  0.68    
+3       Sonna                                   Sonna                                   0.67    
+4       Sonna                                   Sonna                                   0.67    
+5       Sone                                    Sone                                    0.63    
+6       Sonon                                   Sonon                                   0.63    
+7       Sone                                    Sone                                    0.63    
+8       Sone                                    Sone                                    0.63    
+9       Sonnega                                 Sonnega                                 0.59    
+10      Sonnar                                  Sonnar                                  0.54    

src/regression-test/output/kuwait-city-prefix.txt

@@ -0,0 +1,22 @@
+{
+  "string": "مدينة الك",
+  "topN": 10,
+  "minQuality": 0.3
+}
+
+{
+  "queryDuration": 2
+}
+
+Rank    Entity                                  Matched String                          Quality 
+
+⁦1       ⁩⁧                              مدينة الرد⁩⁧                              مدينة الرد⁩⁦0.69    ⁩
+⁦2       ⁩⁧                              مدينة الحق⁩⁧                              مدينة الحق⁩⁦0.69    ⁩
+⁦3       ⁩⁧                            مدينة الكويت⁩⁧                            مدينة الكويت⁩⁦0.66    ⁩
+⁦4       ⁩⁧                             مدينة الشرق⁩⁧                             مدينة الشرق⁩⁦0.63    ⁩
+⁦5       ⁩⁧                             مدينة الشعب⁩⁧                             مدينة الشعب⁩⁦0.63    ⁩
+⁦6       ⁩⁧                             مدينة الصدر⁩⁧                             مدينة الصدر⁩⁦0.63    ⁩
+⁦7       ⁩⁧                             مدينة البعث⁩⁧                             مدينة البعث⁩⁦0.63    ⁩
+⁦8       ⁩⁧                             مدينة المجد⁩⁧                             مدينة المجد⁩⁦0.63    ⁩
+⁦9       ⁩⁧                             مدينة العرب⁩⁧                             مدينة العرب⁩⁦0.63    ⁩
+⁦10      ⁩⁧                             مدينة الطيب⁩⁧                             مدينة الطيب⁩⁦0.63    ⁩