From c1267f0a04f470e3e3c44d3e5f49f6ef4fefbe71 Mon Sep 17 00:00:00 2001 From: waleed Date: Tue, 9 Jun 2026 21:16:35 -0700 Subject: [PATCH 1/3] fix(db): serialize concurrent migrations with a Postgres advisory lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployments start N app replicas at once, each with a migration sidecar. drizzle migrate() has no cross-process lock, so all N read __drizzle_migrations, all see the same migration pending, and all apply it concurrently — one wins, the losers run the same DDL against already-mutated state and exit 1 (e.g. DROP TABLE "form" -> table does not exist / TaskFailedToStart). Wrap migrate() in a session-level pg_advisory_lock so runners serialize: the winner migrates, the losers block, then re-read and find nothing pending. Session locks auto-release on disconnect, so a crashed runner never wedges the lock. --- packages/db/scripts/migrate.ts | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts index 9d967a9db7c..4c2ab74f3c5 100644 --- a/packages/db/scripts/migrate.ts +++ b/packages/db/scripts/migrate.ts @@ -38,12 +38,34 @@ if (!url) { const client = postgres(url, { max: 1, connect_timeout: 10 }) +/** + * Cross-process migration lock key (a stable, app-wide 64-bit constant). + * + * drizzle's `migrate()` has no built-in lock, so when a deployment starts N app + * replicas at once — each with a migration sidecar — all N read + * `__drizzle_migrations`, all see the same migration pending, and all try to apply + * it concurrently. One wins; the losers run the same DDL against already-mutated + * state and die (e.g. `DROP TABLE "form"` → `table "form" does not exist`, + * exit 1 / TaskFailedToStart). + * + * A session-level `pg_advisory_lock` serializes runners: the first to acquire it + * migrates while the rest block, then each loser acquires the lock, re-reads + * `__drizzle_migrations`, finds nothing pending, and exits cleanly. Session locks + * auto-release if the connection drops, so a crashed runner never wedges the lock. + */ +const MIGRATION_LOCK_KEY = 4_961_002_270n + try { // statement_timeout=0: index builds (esp. CONCURRENTLY on large tables) can run // far longer than the app default; a migration must never be killed mid-build. await client`SET statement_timeout = 0` - await migrate(drizzle(client), { migrationsFolder: './migrations' }) - console.log('Migrations applied successfully.') + await client`SELECT pg_advisory_lock(${MIGRATION_LOCK_KEY})` + try { + await migrate(drizzle(client), { migrationsFolder: './migrations' }) + console.log('Migrations applied successfully.') + } finally { + await client`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})` + } } catch (error) { console.error('ERROR: Migration failed.') printMigrationError(error) From f591b967a068d482d29365dd2cb1b76135d6b7e1 Mon Sep 17 00:00:00 2001 From: waleed Date: Tue, 9 Jun 2026 21:33:51 -0700 Subject: [PATCH 2/3] fix(db): guard pg_advisory_unlock so it cannot mask a successful migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the explicit unlock throws (e.g. connection drops in the window after migrate() commits), the exception bubbled to the outer catch and exited 1 — falsely reporting a failed migration to the deploy orchestrator. The session lock auto-releases on disconnect anyway, so swallow and log instead. --- packages/db/scripts/migrate.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts index 4c2ab74f3c5..ca3e6c3236c 100644 --- a/packages/db/scripts/migrate.ts +++ b/packages/db/scripts/migrate.ts @@ -64,7 +64,17 @@ try { await migrate(drizzle(client), { migrationsFolder: './migrations' }) console.log('Migrations applied successfully.') } finally { - await client`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})` + try { + await client`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})` + } catch (unlockError) { + // A failed explicit unlock must never mask a successful migration with a + // non-zero exit — the session lock auto-releases on disconnect anyway, so + // log and move on rather than letting this reach the outer catch. + console.error( + 'WARN: pg_advisory_unlock failed; the session lock will auto-release on disconnect.', + unlockError + ) + } } } catch (error) { console.error('ERROR: Migration failed.') From 34980b007b4b4dfffbd708a64432596b31aba02c Mon Sep 17 00:00:00 2001 From: waleed Date: Tue, 9 Jun 2026 21:35:48 -0700 Subject: [PATCH 3/3] refactor(db): move unlock-guard rationale to TSDoc helper --- packages/db/scripts/migrate.ts | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts index ca3e6c3236c..ed0af3b1c4d 100644 --- a/packages/db/scripts/migrate.ts +++ b/packages/db/scripts/migrate.ts @@ -64,17 +64,7 @@ try { await migrate(drizzle(client), { migrationsFolder: './migrations' }) console.log('Migrations applied successfully.') } finally { - try { - await client`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})` - } catch (unlockError) { - // A failed explicit unlock must never mask a successful migration with a - // non-zero exit — the session lock auto-releases on disconnect anyway, so - // log and move on rather than letting this reach the outer catch. - console.error( - 'WARN: pg_advisory_unlock failed; the session lock will auto-release on disconnect.', - unlockError - ) - } + await releaseMigrationLock() } } catch (error) { console.error('ERROR: Migration failed.') @@ -84,6 +74,24 @@ try { await client.end() } +/** + * Release the advisory lock without ever failing the process. The session-level + * lock auto-releases when the connection closes, so a thrown unlock — e.g. the + * connection dropped right after `migrate()` committed — must be swallowed. + * Letting it reach the outer `catch` would exit 1 and falsely report a + * successful migration as failed to the deploy orchestrator. + */ +async function releaseMigrationLock(): Promise { + try { + await client`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})` + } catch (unlockError) { + console.error( + 'WARN: pg_advisory_unlock failed; the session lock will auto-release on disconnect.', + unlockError + ) + } +} + /** * Print every diagnostic field a Postgres driver puts on a thrown error. The default * `error.message` loses the constraint name, affected table/column, PG code, and hint —