fix(db): open a database whose keyword index was built by an older version

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to MeMesh are documented here.
 
 ### Fixed
 
+- A small database last opened by an older release (before 4.2.11, or with Thai, Lao, Khmer, half-width katakana or rare CJK characters indexed by an older build) opens again after upgrading. Before, every open failed with "database disk image is malformed" although the database was not damaged, because an upgrade step removed keyword-index rows using the new text splitting before the index was rebuilt (#522).
 - A message sent through the MCP `message` tool keeps every `null` in its JSON payload. Before, the MCP boundary removed each null-valued key at any depth of the payload while `send` still reported success, so the recipient got different data than the CLI or HTTP would have stored (#517). A null-valued top-level tool parameter still means "left blank".
 
 ## [4.10.11] — 2026-09-29

dist/mcp/THIRD_PARTY_NOTICES.txt

@@ -1,7 +1,7 @@
 Third-party software bundled in dist/mcp/server.js
 
-Bundle SHA-256: e38a65e55d9a8be9760a92d20e69a2f71d0a5bd6c26cddb004fa3c26f84364e5
-Source map SHA-256: 6f92117584bd8615b1b57545acd20bf6e77467e1c395342242f6dbe8df3e3615
+Bundle SHA-256: e59e6480958c3587992901515c5ca01bbad351df6ffa6f639b18b4dc985c5604
+Source map SHA-256: 4b47a732a51578419c678f69096085b4fe7005d0ed2691032b7b14309e0129b2
 
 Generated by scripts/build-mcp-bundle.mjs from the esbuild input closure.
 

dist/mcp/server.js

@@ -27091,6 +27091,7 @@ function migrateToCurrentSchema(db2, resolvedPath) {
   migrateEntitiesSchema(db2);
   runAutoDecay(db2);
   backfillSignalScores(db2);
+  ensureFtsSegmentation(db2);
   backfillTitles(db2);
   backfillAcceptedProposalTrust(db2);
   dedupeObservations(db2);
@@ -27100,7 +27101,6 @@ function migrateToCurrentSchema(db2, resolvedPath) {
   splitFusedLessons(db2, { deriveTitle: deriveHeuristicTitle });
   repairFusedLessonShellHistory(db2);
   ensureDreamProposalsTable(db2);
-  ensureFtsSegmentation(db2);
   dropArchivedIndexRows(db2);
 }
 function ensureDreamProposalsTable(db2) {

dist/transports/cli/cli.js

@@ -5894,6 +5894,7 @@ function migrateToCurrentSchema(db2, resolvedPath) {
   migrateEntitiesSchema(db2);
   runAutoDecay(db2);
   backfillSignalScores(db2);
+  ensureFtsSegmentation(db2);
   backfillTitles(db2);
   backfillAcceptedProposalTrust(db2);
   dedupeObservations(db2);
@@ -5903,7 +5904,6 @@ function migrateToCurrentSchema(db2, resolvedPath) {
   splitFusedLessons(db2, { deriveTitle: deriveHeuristicTitle });
   repairFusedLessonShellHistory(db2);
   ensureDreamProposalsTable(db2);
-  ensureFtsSegmentation(db2);
   dropArchivedIndexRows(db2);
 }
 function reindexFts() {

src/db.ts

@@ -163,6 +163,18 @@ function migrateToCurrentSchema(db: MemeshDatabase, resolvedPath: string): void
   // marker key.
   backfillSignalScores(db);
 
+  // Rebuild entities_fts once when the way text is segmented changes.
+  // Databases written before CJK segmentation hold whole-run tokens that no
+  // segmented query can match, so without this the change would take Chinese
+  // recall from bad to zero while English kept working — a silent regression.
+  //
+  // It runs BEFORE every pass that removes FTS rows one by one (the title
+  // backfill below is the first): a contentless delete must repeat the exact
+  // tokens that were indexed, and on an old index today's segmentation
+  // produces different ones. On a small database that mismatch makes the
+  // delete fail with "database disk image is malformed" on every open (#522).
+  ensureFtsSegmentation(db);
+
   // UX-1: give pre-title rows a human-readable heuristic title. Same
   // marker + fill-only discipline as backfillSignalScores above.
   backfillTitles(db);
@@ -201,12 +213,6 @@ function migrateToCurrentSchema(db: MemeshDatabase, resolvedPath: string): void
   // Agent work packages stage proposals here for explicit human review.
   ensureDreamProposalsTable(db);
 
-  // Rebuild entities_fts once when the way text is segmented changes.
-  // Databases written before CJK segmentation hold whole-run tokens that no
-  // segmented query can match, so without this the change would take Chinese
-  // recall from bad to zero while English kept working — a silent regression.
-  ensureFtsSegmentation(db);
-
   // One-shot repair for FTS rows written before archived entities were removed
   // from the keyword index. It rebuilds from active entities only and has no
   // dependency on an optional secondary index.

tests/db-old-index-opens.test.ts

@@ -0,0 +1,58 @@
+/**
+ * #522: a small database last opened by an older build holds keyword-index
+ * rows split the old way (whole runs of CJK / Thai text). Opening it must
+ * rebuild that index before any step removes rows from it one by one with
+ * today's splitting, or the removal fails and every open reports
+ * "database disk image is malformed" for a database that is not corrupt.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { closeDatabase, openDatabase } from '../src/db.js';
+import { KnowledgeGraph } from '../src/knowledge-graph.js';
+import { MemeshDatabase } from '../src/storage/sqlite.js';
+import { FTS_SQL, SCHEMA_SQL, migrateEntitiesSchema } from '../src/storage/schema.js';
+
+function writeLegacyDatabase(dbPath: string, name: string, observation: string, segmentationVersion?: string): void {
+  const db = new MemeshDatabase(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.exec(SCHEMA_SQL);
+  db.exec(FTS_SQL);
+  migrateEntitiesSchema(db);
+  db.prepare("INSERT INTO entities (name, type) VALUES (?, 'decision')").run(name);
+  db.prepare('INSERT INTO observations (entity_id, content) VALUES (1, ?)').run(observation);
+  // Indexed the way an older build did: the whole unspaced run as one token.
+  db.prepare('INSERT INTO entities_fts (rowid, name, observations) VALUES (1, ?, ?)').run(name, observation);
+  if (segmentationVersion) {
+    db.prepare("INSERT INTO memesh_metadata (key, value) VALUES ('fts_segmentation_version', ?)").run(segmentationVersion);
+  }
+  db.close();
+}
+
+describe('Feature: #522 an old small database opens after upgrading', () => {
+  let dir: string;
+  let dbPath: string;
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'memesh-old-index-'));
+    dbPath = path.join(dir, 'knowledge-graph.db');
+  });
+
+  afterEach(() => {
+    try { closeDatabase(); } catch { /* not open */ }
+    fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  });
+
+  it('a pre-segmentation database with Chinese text opens and finds it', () => {
+    writeLegacyDatabase(dbPath, '資料庫遷移', '資料庫遷移前一定要先備份');
+    const db = openDatabase(dbPath);
+    expect(new KnowledgeGraph(db).search('資料庫').map((e) => e.name)).toContain('資料庫遷移');
+  });
+
+  it('a database indexed before Thai was segmented opens and finds it', () => {
+    writeLegacyDatabase(dbPath, 'ฐานข้อมูล', 'สำรองข้อมูลก่อนย้ายฐานข้อมูลเสมอ', '2');
+    const db = openDatabase(dbPath);
+    expect(new KnowledgeGraph(db).search('ฐานข้อมูล').map((e) => e.name)).toContain('ฐานข้อมูล');
+  });
+});