fix(wallet-toolbox): run SQLite migrations inside knex's per-file transaction

packages/wallet/wallet-toolbox/CHANGELOG.md

@@ -6,6 +6,12 @@ attention to changes that materially alter behavior or extend functionality.
 
 ## wallet-toolbox (unreleased)
 
+- Restore transactional SQLite migrations in the unpublished 2.13.2 candidate.
+  DDL, migration journal and lock changes roll back after an interrupted attempt;
+  foreign-key enforcement is restored after success or failure. Existing stores
+  with unjournaled partial schema need operator-reviewed recovery; this change
+  does not delete or automatically reconcile historical wallet data.
+
 - Implement `BHServiceClient.findChainTipHash()` by delegating to its existing
   `findChainTipHeader()` call against `/api/v1/chain/tip/longest`, instead of
   throwing `Not implemented`. `ChaintracksChainTracker.getVerificationContextToken()`

packages/wallet/wallet-toolbox/README.md

@@ -27,6 +27,20 @@ Timing compares successive candidates, not a controlled comparison against upstr
 `main`. Byte verification was sampled, not database-wide. See
 [test methods and limits](#sync-performance-and-recovery) for details.
 
+### SQLite migration recovery
+
+The unpublished 2.13.2 candidate runs SQLite migration DDL and the migration
+journal update transactionally. Foreign-key enforcement is disabled before the
+migration transaction for table rebuilds and restored after success or failure.
+Failed migrations can be retried after reopening the database without partial
+schema objects from that attempt. MySQL's existing transaction configuration
+is unchanged.
+
+This prevents future partial migrations. It does not automatically repair a
+store already left with unjournaled schema objects by an older version. Preserve
+the database and verified backups and reconcile the exact schema and migration
+journal before recovery; do not delete journal rows or wallet data blindly.
+
 ## Overview
 
 The Wallet Toolbox is the reference implementation of the BRC-100 wallet interface. It connects the BSV SDK's cryptographic primitives to real storage backends, network services, and signing flows so that application developers don't have to wire these layers together themselves.

packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts

@@ -1598,17 +1598,22 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide
     const clientName = (this.knex.client as { config?: { client?: string } }).config?.client ?? ''
     const isSQLite = clientName.includes('sqlite')
 
-    // For SQLite, disable transactions during migrations and turn off foreign keys.
-    // PRAGMA foreign_keys is silently ignored inside transactions, so we must
-    // disable transactions for the migration to allow the PRAGMA to take effect.
-    // See: https://github.com/knex/knex/issues/4155
+    // For SQLite, turn foreign keys off for the duration of the migration.
+    // PRAGMA foreign_keys is silently ignored *when executed inside* a
+    // transaction (https://github.com/knex/knex/issues/4155), so it is issued
+    // here, outside migrate.latest(). SQLite's single-connection pool means
+    // knex's per-migration transaction runs on this same connection and
+    // inherits the setting, and knex's own SQLite alter-table rebuild leaves an
+    // ambient pragma alone while transacting (sqlite3/schema/ddl.js: alter()
+    // uses `enforceForeignCheck = this.client.transacting ? null : false`).
     if (isSQLite) {
       await this.knex.raw('PRAGMA foreign_keys = OFF;')
     }
     try {
       const config = {
         migrationSource: new KnexMigrations(this.chain, storageName, storageIdentityKey, 1024),
-        disableTransactions: isSQLite
+        // Keep DDL and its migration journal entry in the same transaction.
+        disableTransactions: false
       }
       await this.knex.migrate.latest(config)
       return await this.knex.migrate.currentVersion(config)

packages/wallet/wallet-toolbox/src/storage/__test/StorageKnexMigrationAtomicity.test.ts

@@ -0,0 +1,215 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { spawn } from 'node:child_process'
+import { once } from 'node:events'
+import { knex as makeKnex, type Knex } from 'knex'
+import { StorageKnex } from '../StorageKnex'
+import { KnexMigrations } from '../schema/KnexMigrations'
+
+const initial = '2026-09-23-001 atomicity fixture'
+const altered = '2026-09-23-002 populated alter fixture'
+
+function open(filename: string): StorageKnex {
+  return new StorageKnex({
+    ...StorageKnex.defaultOptions(),
+    chain: 'test',
+    knex: makeKnex({
+      client: 'better-sqlite3',
+      connection: { filename },
+      useNullAsDefault: true,
+      pool: { min: 1, max: 1 }
+    })
+  })
+}
+
+async function createParent(db: Knex): Promise<void> {
+  await db.schema.createTable('migration_parent', table => {
+    table.integer('id').primary()
+    table.string('label').nullable()
+  })
+}
+
+async function createChildAndRows(db: Knex): Promise<void> {
+  await db.schema.createTable('migration_child', table => {
+    table.integer('id').primary()
+    table.integer('parent').references('id').inTable('migration_parent')
+  })
+  await db('migration_parent').insert({ id: 1, label: 'retained' })
+  await db('migration_child').insert({ id: 2, parent: 1 })
+}
+
+afterEach(() => jest.restoreAllMocks())
+
+describe('SQLite migration atomicity through StorageKnex', () => {
+  test('an abruptly killed migration leaves neither partial DDL nor a stuck migration lock', async () => {
+    const directory = await mkdtemp(join(tmpdir(), 'wallet-migration-crash-'))
+    const filename = join(directory, 'wallet.sqlite')
+    // Exercise the built artifact in a separate process so SIGKILL cannot run
+    // Knex catch/finally cleanup. The parent reopens the same durable database.
+    const child = spawn(
+      process.execPath,
+      [
+        '-e',
+        `
+      const { knex } = require('knex')
+      const { StorageKnex } = require('./out/src/storage/StorageKnex.js')
+      const { KnexMigrations } = require('./out/src/storage/schema/KnexMigrations.js')
+      KnexMigrations.prototype.getMigrations = async () => ['2026-09-23-001 atomicity fixture']
+      KnexMigrations.prototype.getMigration = async () => ({
+        down: async db => { await db.schema.dropTableIfExists('migration_parent') },
+        up: async db => {
+          await db.schema.createTable('migration_parent', table => { table.integer('id').primary() })
+          process.send('ddl-written')
+          await new Promise(() => {})
+        }
+      })
+      const storage = new StorageKnex({ ...StorageKnex.defaultOptions(), chain: 'test',
+        knex: knex({ client: 'better-sqlite3', connection: { filename: process.argv[1] },
+          useNullAsDefault: true, pool: { min: 1, max: 1 } }) })
+      storage.migrate('fixture', '1'.repeat(64)).catch(() => process.exit(1))
+    `,
+        filename
+      ],
+      { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }
+    )
+    let storage: StorageKnex | undefined
+    try {
+      await new Promise<void>((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error('Child migration did not reach DDL')), 10000)
+        child.once('message', message => {
+          clearTimeout(timer)
+          if (message === 'ddl-written') resolve()
+          else reject(new Error('Unexpected migration checkpoint'))
+        })
+        child.once('exit', () => {
+          clearTimeout(timer)
+          reject(new Error('Child exited before checkpoint'))
+        })
+        child.once('error', error => {
+          clearTimeout(timer)
+          reject(error)
+        })
+      })
+      const exited = once(child, 'exit')
+      child.kill('SIGKILL')
+      await exited
+      storage = open(filename)
+      expect(await storage.knex.schema.hasTable('migration_parent')).toBe(false)
+      expect(await storage.knex('knex_migrations').select()).toEqual([])
+      expect(await storage.knex('knex_migrations_lock').pluck('is_locked')).toEqual([0])
+      jest.spyOn(KnexMigrations.prototype, 'getMigrations').mockResolvedValue([initial])
+      jest.spyOn(KnexMigrations.prototype, 'getMigration').mockResolvedValue({
+        up: createParent,
+        down: async db => {
+          await db.schema.dropTableIfExists('migration_parent')
+        }
+      })
+      await storage.migrate('fixture', '1'.repeat(64))
+      expect(await storage.knex('knex_migrations').pluck('name')).toEqual([initial])
+    } finally {
+      if (child.exitCode === null && child.signalCode === null) {
+        const exited = once(child, 'exit')
+        child.kill('SIGKILL')
+        await exited
+      }
+      await storage?.destroy()
+      await rm(directory, { recursive: true, force: true })
+    }
+  })
+
+  test.each(['mid-file', 'journal-write'] as const)(
+    'rolls back a %s failure and migrates successfully after reopening',
+    async boundary => {
+      const directory = await mkdtemp(join(tmpdir(), 'wallet-migration-'))
+      const filename = join(directory, 'wallet.sqlite')
+      let storage = open(filename)
+      let interrupted = true
+      jest.spyOn(KnexMigrations.prototype, 'getMigrations').mockResolvedValue([initial])
+      jest.spyOn(KnexMigrations.prototype, 'getMigration').mockResolvedValue({
+        down: async db => {
+          await db.schema.dropTableIfExists('migration_child')
+          await db.schema.dropTableIfExists('migration_parent')
+        },
+        up: async db => {
+          expect((await db.raw('PRAGMA foreign_keys'))[0].foreign_keys).toBe(0)
+          await createParent(db)
+          if (interrupted && boundary === 'mid-file') throw new Error('injected interruption')
+          await createChildAndRows(db)
+        }
+      })
+      try {
+        if (boundary === 'journal-write') {
+          await storage.knex.migrate.list({
+            migrationSource: new KnexMigrations('test', 'fixture', '1'.repeat(64), 1024)
+          })
+          await storage.knex.raw(
+            "CREATE TRIGGER reject_journal BEFORE INSERT ON knex_migrations BEGIN SELECT RAISE(ABORT, 'injected journal interruption'); END"
+          )
+        }
+        await expect(storage.migrate('fixture', '1'.repeat(64))).rejects.toThrow(/interruption/)
+        expect(await storage.knex.schema.hasTable('migration_parent')).toBe(false)
+        expect(await storage.knex.schema.hasTable('migration_child')).toBe(false)
+        expect(await storage.knex('knex_migrations').select()).toEqual([])
+        expect((await storage.knex.raw('PRAGMA foreign_keys'))[0].foreign_keys).toBe(1)
+        await storage.destroy()
+        storage = open(filename)
+        interrupted = false
+        await storage.knex.raw('DROP TRIGGER IF EXISTS reject_journal')
+        await storage.migrate('fixture', '1'.repeat(64))
+        expect(await storage.knex('migration_parent').select()).toEqual([{ id: 1, label: 'retained' }])
+        expect(await storage.knex('migration_child').select()).toEqual([{ id: 2, parent: 1 }])
+        expect(await storage.knex('knex_migrations').pluck('name')).toEqual([initial])
+        expect(await storage.knex.raw('PRAGMA foreign_key_check')).toEqual([])
+        expect((await storage.knex.raw('PRAGMA integrity_check'))[0].integrity_check).toBe('ok')
+        await storage.migrate('fixture', '1'.repeat(64))
+        expect(await storage.knex('knex_migrations').pluck('name')).toEqual([initial])
+      } finally {
+        await storage.destroy()
+        await rm(directory, { recursive: true, force: true })
+      }
+    }
+  )
+
+  test('preserves populated referenced rows during an alter-table rebuild and restores enforcement', async () => {
+    const directory = await mkdtemp(join(tmpdir(), 'wallet-migration-'))
+    const storage = open(join(directory, 'wallet.sqlite'))
+    const migrations = jest.spyOn(KnexMigrations.prototype, 'getMigrations').mockResolvedValue([initial])
+    jest.spyOn(KnexMigrations.prototype, 'getMigration').mockImplementation(async name => ({
+      down: async db => {
+        if (name === initial) {
+          await db.schema.dropTableIfExists('migration_child')
+          await db.schema.dropTableIfExists('migration_parent')
+        } else {
+          await db.schema.alterTable('migration_parent', table => {
+            table.string('label').nullable().alter()
+          })
+        }
+      },
+      up: async db => {
+        if (name === initial) {
+          await createParent(db)
+          await createChildAndRows(db)
+        } else {
+          await db.schema.alterTable('migration_parent', table => {
+            table.string('label', 128).notNullable().alter()
+          })
+        }
+      }
+    }))
+    try {
+      await storage.migrate('fixture', '1'.repeat(64))
+      migrations.mockResolvedValue([initial, altered])
+      await storage.migrate('fixture', '1'.repeat(64))
+      expect(await storage.knex('migration_parent').select()).toEqual([{ id: 1, label: 'retained' }])
+      expect(await storage.knex('migration_child').select()).toEqual([{ id: 2, parent: 1 }])
+      expect(await storage.knex('knex_migrations').pluck('name')).toEqual([initial, altered])
+      expect(await storage.knex.raw('PRAGMA foreign_key_check')).toEqual([])
+      await expect(storage.knex('migration_child').insert({ id: 3, parent: 99 })).rejects.toThrow(/FOREIGN KEY/)
+      await expect(storage.knex('migration_parent').insert({ id: 4, label: null })).rejects.toThrow(/NOT NULL/)
+    } finally {
+      await storage.destroy()
+      await rm(directory, { recursive: true, force: true })
+    }
+  })
+})

packages/wallet/wallet-toolbox/src/storage/__test/StorageKnexMigrationFailure.security.test.ts

@@ -5,6 +5,22 @@ function storageWithKnex(knex: object): StorageKnex {
 }
 
 describe('StorageKnex migration failure boundaries', () => {
+  test('retains MySQL transaction settings without SQLite pragmas', async () => {
+    const knex = {
+      client: { config: { client: 'mysql2' } },
+      raw: jest.fn(),
+      migrate: {
+        latest: jest.fn().mockResolvedValue([1, ['fixture']]),
+        currentVersion: jest.fn().mockResolvedValue('fixture')
+      }
+    }
+    await expect(StorageKnex.prototype.migrate.call(storageWithKnex(knex), 'wallet', '1'.repeat(64))).resolves.toBe(
+      'fixture'
+    )
+    expect(knex.migrate.latest).toHaveBeenCalledWith(expect.objectContaining({ disableTransactions: false }))
+    expect(knex.raw).not.toHaveBeenCalled()
+  })
+
   test('dropAllData stops only at the explicit empty-schema state', async () => {
     const knex = {
       client: { config: { client: 'better-sqlite3' } },
@@ -51,9 +67,9 @@ describe('StorageKnex migration failure boundaries', () => {
       }
     }
 
-    await expect(
-      StorageKnex.prototype.migrate.call(storageWithKnex(knex), 'wallet', '1'.repeat(64))
-    ).rejects.toThrow('migration failed')
+    await expect(StorageKnex.prototype.migrate.call(storageWithKnex(knex), 'wallet', '1'.repeat(64))).rejects.toThrow(
+      'migration failed'
+    )
     expect(knex.raw).toHaveBeenLastCalledWith('PRAGMA foreign_keys = ON;')
   })
 })