From 37d1a76051b03ad2b6b8422a38e62d624cc62700 Mon Sep 17 00:00:00 2001 From: Luc Vidal Date: Fri, 24 Jul 2026 09:45:37 +0200 Subject: [PATCH 1/6] fix: count correctly rollbacked documents --- src/MongoBulkDataMigration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MongoBulkDataMigration.ts b/src/MongoBulkDataMigration.ts index a4d8787..4697b8b 100644 --- a/src/MongoBulkDataMigration.ts +++ b/src/MongoBulkDataMigration.ts @@ -384,9 +384,9 @@ export default class MongoBulkDataMigration< rollbackDocument = (await cursor.next()) as unknown as RollbackDocument | null; if (!rollbackDocument || bulkRollback.size >= this.options.maxBulkSize) { + treatedDocumentsCount += bulkRollback.size; await bulkRollback.execute(); - treatedDocumentsCount += bulkRollback.size; this.logger.info( { treatedDocumentsCount, From 73e7ef0792e42bbf086139075d9a8f963853dd2c Mon Sep 17 00:00:00 2001 From: Luc Vidal Date: Fri, 24 Jul 2026 11:27:10 +0200 Subject: [PATCH 2/6] refactor: extract log.info calls assertion --- .../MongoBulkDataMigration.update.test.ts | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/__tests__/MongoBulkDataMigration.update.test.ts b/__tests__/MongoBulkDataMigration.update.test.ts index b81ff6b..85e0dfe 100644 --- a/__tests__/MongoBulkDataMigration.update.test.ts +++ b/__tests__/MongoBulkDataMigration.update.test.ts @@ -8,6 +8,7 @@ import { NO_UPDATE } from '../src/MongoBulkDataMigration'; const COLLECTION = 'testCollection'; const SCRIPT_ID = 'scriptId'; +const END_OF_BULK_LOG = 'Documents migration is successful'; describe('MongoBulkDataMigration', () => { let db: Db; @@ -226,7 +227,6 @@ describe('MongoBulkDataMigration', () => { }); describe('Bulk splitting', () => { - const END_OF_BULK_LOG = 'Documents migration is successful'; let update: (arg: { value: number }) => UpdateFilter<{ value: number }>; beforeEach(async () => { await collection.insertMany( @@ -250,10 +250,12 @@ describe('MongoBulkDataMigration', () => { await dataMigration.update(); - const batchSizes = loggerMock.info.mock.calls - .filter(([_, msg]) => msg === END_OF_BULK_LOG) - .map(([{ nModified }]) => nModified); - expect(batchSizes).toEqual([30, 30, 30, 10]); + expect(extractLogsPayload(END_OF_BULK_LOG)).toEqual([ + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 10, nModified: 10, ok: 1 }, + ]); }); it('should perform updates in batches, event when not rollbackable', async () => { @@ -269,10 +271,12 @@ describe('MongoBulkDataMigration', () => { await dataMigration.update(); - const batchSizes = loggerMock.info.mock.calls - .filter(([_, msg]) => msg === END_OF_BULK_LOG) - .map(([{ nModified }]) => nModified); - expect(batchSizes).toEqual([30, 30, 30, 10]); + expect(extractLogsPayload(END_OF_BULK_LOG)).toEqual([ + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 10, nModified: 10, ok: 1 }, + ]); }); }); @@ -503,10 +507,9 @@ describe('MongoBulkDataMigration', () => { await dataMigration.update(); - const batchSizes = loggerMock.info.mock.calls - .filter(([_, msg]) => msg === 'Documents migration is successful') - .map(([{ nMatched, nModified }]) => ({ nMatched, nModified })); - expect(batchSizes).toEqual([{ nMatched: 2, nModified: 2 }]); + expect(extractLogsPayload(END_OF_BULK_LOG)).toEqual([ + { nMatched: 2, nModified: 2, ok: 1 }, + ]); }); it('should not send any batch update if all documents are ignored', async () => { @@ -518,10 +521,7 @@ describe('MongoBulkDataMigration', () => { await dataMigration.update(); - const batchSizes = loggerMock.info.mock.calls.filter( - ([_, msg]) => msg === 'Documents migration is successful', - ); - expect(batchSizes).toEqual([]); + expect(extractLogsPayload(END_OF_BULK_LOG)).toEqual([]); }); }); }); @@ -693,4 +693,10 @@ describe('MongoBulkDataMigration', () => { expect(updatedDocuments).toEqual([{ key: 1 }, { key: 3 }]); }); }); + + function extractLogsPayload(logText: string) { + return loggerMock.info.mock.calls + .filter(([_, msg]) => msg === logText) + .map(([payload]) => payload); + } }); From d33091fdafb8eea4b2f76188ba2d3184316b4b53 Mon Sep 17 00:00:00 2001 From: Luc Vidal Date: Fri, 24 Jul 2026 11:33:09 +0200 Subject: [PATCH 3/6] feat: add assertions on the rollback bulk logs --- .../MongoBulkDataMigration.update.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/__tests__/MongoBulkDataMigration.update.test.ts b/__tests__/MongoBulkDataMigration.update.test.ts index 85e0dfe..4b60e89 100644 --- a/__tests__/MongoBulkDataMigration.update.test.ts +++ b/__tests__/MongoBulkDataMigration.update.test.ts @@ -9,6 +9,7 @@ import { NO_UPDATE } from '../src/MongoBulkDataMigration'; const COLLECTION = 'testCollection'; const SCRIPT_ID = 'scriptId'; const END_OF_BULK_LOG = 'Documents migration is successful'; +const END_OF_ROLLBACK_BULK_LOG = 'Documents backup is successful'; describe('MongoBulkDataMigration', () => { let db: Db; @@ -243,7 +244,7 @@ describe('MongoBulkDataMigration', () => { options: { maxBulkSize: 30, maxConcurrentUpdateCalls: 1000, - rollbackable: false, + rollbackable: true, }, update, }); @@ -256,15 +257,21 @@ describe('MongoBulkDataMigration', () => { { nMatched: 30, nModified: 30, ok: 1 }, { nMatched: 10, nModified: 10, ok: 1 }, ]); + expect(extractLogsPayload(END_OF_ROLLBACK_BULK_LOG)).toEqual([ + { nUpserted: 30, ok: 1 }, + { nUpserted: 30, ok: 1 }, + { nUpserted: 30, ok: 1 }, + { nUpserted: 10, ok: 1 }, + ]); }); - it('should perform updates in batches, event when not rollbackable', async () => { + it('should perform updates in batches, even when not rollbackable', async () => { const dataMigration = new MongoBulkDataMigration({ ...DM_DEFAULT_SETUP, options: { maxBulkSize: 30, maxConcurrentUpdateCalls: 1000, - rollbackable: true, + rollbackable: false, }, update, }); @@ -277,6 +284,7 @@ describe('MongoBulkDataMigration', () => { { nMatched: 30, nModified: 30, ok: 1 }, { nMatched: 10, nModified: 10, ok: 1 }, ]); + expect(extractLogsPayload(END_OF_ROLLBACK_BULK_LOG)).toEqual([]); }); }); @@ -510,6 +518,9 @@ describe('MongoBulkDataMigration', () => { expect(extractLogsPayload(END_OF_BULK_LOG)).toEqual([ { nMatched: 2, nModified: 2, ok: 1 }, ]); + expect(extractLogsPayload(END_OF_ROLLBACK_BULK_LOG)).toEqual([ + { nUpserted: 2, ok: 1 }, + ]); }); it('should not send any batch update if all documents are ignored', async () => { @@ -522,6 +533,7 @@ describe('MongoBulkDataMigration', () => { await dataMigration.update(); expect(extractLogsPayload(END_OF_BULK_LOG)).toEqual([]); + expect(extractLogsPayload(END_OF_ROLLBACK_BULK_LOG)).toEqual([]); }); }); }); From 3263e30c5ddb9b84055af04ea5f66f3d136c9505 Mon Sep 17 00:00:00 2001 From: Luc Vidal Date: Fri, 24 Jul 2026 11:38:24 +0200 Subject: [PATCH 4/6] feat: add assertions on the progress logs --- .../MongoBulkDataMigration.update.test.ts | 52 ++++++++++++++++++- src/MongoBulkDataMigration.ts | 7 ++- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/__tests__/MongoBulkDataMigration.update.test.ts b/__tests__/MongoBulkDataMigration.update.test.ts index 4b60e89..92e4a86 100644 --- a/__tests__/MongoBulkDataMigration.update.test.ts +++ b/__tests__/MongoBulkDataMigration.update.test.ts @@ -4,12 +4,13 @@ import { Collection, Db, Document, ObjectId, UpdateFilter } from 'mongodb'; import { MongoBulkDataMigration, DELETE_OPERATION, FETCH_ALL } from '../src'; import { INITIAL_BULK_INFOS } from '../src/lib/AbstractBulkOperationResults'; import { LoggerInterface } from '../src/types'; -import { NO_UPDATE } from '../src/MongoBulkDataMigration'; +import { DONT_COUNT_LOG, NO_UPDATE } from '../src/MongoBulkDataMigration'; const COLLECTION = 'testCollection'; const SCRIPT_ID = 'scriptId'; const END_OF_BULK_LOG = 'Documents migration is successful'; const END_OF_ROLLBACK_BULK_LOG = 'Documents backup is successful'; +const BULK_SUMMARY_LOG = 'Documents migrated'; describe('MongoBulkDataMigration', () => { let db: Db; @@ -263,6 +264,12 @@ describe('MongoBulkDataMigration', () => { { nUpserted: 30, ok: 1 }, { nUpserted: 10, ok: 1 }, ]); + expect(extractLogsPayload(BULK_SUMMARY_LOG)).toEqual([ + { treatedDocumentsCount: 30, totalEntries: 100, progress: '30.00' }, + { treatedDocumentsCount: 60, totalEntries: 100, progress: '60.00' }, + { treatedDocumentsCount: 90, totalEntries: 100, progress: '90.00' }, + { treatedDocumentsCount: 100, totalEntries: 100, progress: '100.00' }, + ]); }); it('should perform updates in batches, even when not rollbackable', async () => { @@ -285,6 +292,49 @@ describe('MongoBulkDataMigration', () => { { nMatched: 10, nModified: 10, ok: 1 }, ]); expect(extractLogsPayload(END_OF_ROLLBACK_BULK_LOG)).toEqual([]); + expect(extractLogsPayload(BULK_SUMMARY_LOG)).toEqual([ + { treatedDocumentsCount: 30, totalEntries: 100, progress: '30.00' }, + { treatedDocumentsCount: 60, totalEntries: 100, progress: '60.00' }, + { treatedDocumentsCount: 90, totalEntries: 100, progress: '90.00' }, + { treatedDocumentsCount: 100, totalEntries: 100, progress: '100.00' }, + ]); + }); + + it('should not log progress when dontCount option is ON', async () => { + const dataMigration = new MongoBulkDataMigration({ + ...DM_DEFAULT_SETUP, + options: { + maxBulkSize: 30, + maxConcurrentUpdateCalls: 1000, + dontCount: true, + }, + update, + }); + + await dataMigration.update(); + + expect(extractLogsPayload(BULK_SUMMARY_LOG)).toEqual([ + { + treatedDocumentsCount: 30, + totalEntries: DONT_COUNT_LOG, + progress: DONT_COUNT_LOG, + }, + { + treatedDocumentsCount: 60, + totalEntries: DONT_COUNT_LOG, + progress: DONT_COUNT_LOG, + }, + { + treatedDocumentsCount: 90, + totalEntries: DONT_COUNT_LOG, + progress: DONT_COUNT_LOG, + }, + { + treatedDocumentsCount: 100, + totalEntries: DONT_COUNT_LOG, + progress: DONT_COUNT_LOG, + }, + ]); }); }); diff --git a/src/MongoBulkDataMigration.ts b/src/MongoBulkDataMigration.ts index 4697b8b..9542dad 100644 --- a/src/MongoBulkDataMigration.ts +++ b/src/MongoBulkDataMigration.ts @@ -31,6 +31,7 @@ export const DELETE_COLLECTION = Symbol(); export const FETCH_ALL = Symbol(); /** Special return value for update function, when we don't need to update a given document */ export const NO_UPDATE = Symbol(); +export const DONT_COUNT_LOG = 'N/A (dontCount option ON)'; const defaultLogger = { info: (...args: unknown[]) => { @@ -140,9 +141,7 @@ export default class MongoBulkDataMigration< rollbackCollection, ); const formattedTotalEntries = - totalEntries === NO_COUNT_AVAILABLE - ? 'N/A (dontCount option ON)' - : totalEntries; + totalEntries === NO_COUNT_AVAILABLE ? DONT_COUNT_LOG : totalEntries; this.logger.info( { collectionName: this.collectionName, @@ -190,7 +189,7 @@ export default class MongoBulkDataMigration< totalEntries: formattedTotalEntries, progress: totalEntries === NO_COUNT_AVAILABLE - ? 'N/A (dontCount option ON)' + ? DONT_COUNT_LOG : ((treatedDocumentsCount / totalEntries) * 100).toFixed(2), }, 'Documents migrated', From 1066f8adc1d53f6d50fbbb6f5b850b0b92fab9be Mon Sep 17 00:00:00 2001 From: Luc Vidal Date: Fri, 24 Jul 2026 12:08:00 +0200 Subject: [PATCH 5/6] feat: add tests on rollback progress logs --- .../MongoBulkDataMigration.rollback.test.ts | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/__tests__/MongoBulkDataMigration.rollback.test.ts b/__tests__/MongoBulkDataMigration.rollback.test.ts index 855a56a..07f6405 100644 --- a/__tests__/MongoBulkDataMigration.rollback.test.ts +++ b/__tests__/MongoBulkDataMigration.rollback.test.ts @@ -13,6 +13,8 @@ import { LoggerInterface } from '../src/types'; const COLLECTION = 'testCollection'; const ROLLBACK_COLLECTION = `_rollback_${COLLECTION}_scriptId`; const SCRIPT_ID = 'scriptId'; +const ROLLBACK_BULK_LOG = 'Documents rollback is successful'; +const ROLLBACK_SUMMARY_LOG = 'Documents rollbacked'; type DmDemoCollection = { _id: ObjectId; @@ -34,7 +36,7 @@ describe('MongoBulkDataMigration', () => { projection: any; logger: LoggerInterface; }; - let loggerMock: LoggerInterface; + let loggerMock: jest.Mocked; beforeEach(async () => { db = global.db; @@ -1104,5 +1106,43 @@ describe('MongoBulkDataMigration', () => { expect(restoredDocuments).toEqual([document]); }); }); + + describe('bulk splitting', () => { + it('should perform updates in batches', async () => { + await collection.insertMany( + Array.from({ length: 100 }, (_, i) => ({ value: i + 1 })), + ); + const dataMigration = new MongoBulkDataMigration({ + ...DM_DEFAULT_SETUP, + update: { $set: { value: 1000 } }, + options: { + maxBulkSize: 30, + maxConcurrentUpdateCalls: 1000, + }, + }); + + await dataMigration.update(); + await dataMigration.rollback(); + + expect(extractLogsPayload(ROLLBACK_BULK_LOG)).toEqual([ + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 30, nModified: 30, ok: 1 }, + { nMatched: 10, nModified: 10, ok: 1 }, + ]); + expect(extractLogsPayload(ROLLBACK_SUMMARY_LOG)).toEqual([ + { treatedDocumentsCount: 30, totalEntries: 100, progress: '30.00' }, + { treatedDocumentsCount: 60, totalEntries: 100, progress: '60.00' }, + { treatedDocumentsCount: 90, totalEntries: 100, progress: '90.00' }, + { treatedDocumentsCount: 100, totalEntries: 100, progress: '100.00' }, + ]); + }); + }); }); + + function extractLogsPayload(logText: string) { + return loggerMock.info.mock.calls + .filter(([_, msg]) => msg === logText) + .map(([payload]) => payload); + } }); From 4988db127183e12db1a2039f5e15adf4ea09c6a2 Mon Sep 17 00:00:00 2001 From: Luc Vidal Date: Fri, 24 Jul 2026 12:13:04 +0200 Subject: [PATCH 6/6] chore: update to 1.8.2 --- CHANGLOG.md | 4 ++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGLOG.md b/CHANGLOG.md index b2455a1..092cf93 100644 --- a/CHANGLOG.md +++ b/CHANGLOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.8.2 (2026-07-24) + +- Fix bug: rollback progress correctly logged + ## 1.8.1 (2026-07-21) - Allow update functions to return the NO_UPDATE symbol diff --git a/package-lock.json b/package-lock.json index 575836e..a1d5ce5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@360-l/mongo-bulk-data-migration", - "version": "1.8.1", + "version": "1.8.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@360-l/mongo-bulk-data-migration", - "version": "1.8.1", + "version": "1.8.2", "license": "MIT", "dependencies": { "lodash": "^4.18.1", diff --git a/package.json b/package.json index 396f832..ba7faf9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@360-l/mongo-bulk-data-migration", - "version": "1.8.1", + "version": "1.8.2", "description": "MongoDB bulk data migration for node scripts", "main": "./dist/index.js", "types": "./dist/index.d.ts",