From 2ad462028b629109e7b7c4e97811aefb5d8db39c Mon Sep 17 00:00:00 2001 From: Rajarshee Chatterjee Date: Fri, 14 Nov 2025 21:22:04 +0530 Subject: [PATCH 01/10] chore: Release v2.0.0 [skip ci] From adc8a12c5527cb3c8a143203ad660561b8b98206 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Mon, 21 Sep 2026 13:21:22 +0200 Subject: [PATCH 02/10] Optimize backup restore transactions Generated with OpenAI Codex. --- src/database/manager/manager.d.ts | 4 +- src/database/queries/NovelQueries.ts | 161 +++++++++++++----- .../queries/__tests__/NovelQueries.test.ts | 120 +++++++++++++ src/services/backup/__tests__/utils.test.ts | 2 + src/services/backup/utils.ts | 4 +- 5 files changed, 246 insertions(+), 45 deletions(-) diff --git a/src/database/manager/manager.d.ts b/src/database/manager/manager.d.ts index 195bab40bb..95276d7ea5 100644 --- a/src/database/manager/manager.d.ts +++ b/src/database/manager/manager.d.ts @@ -21,7 +21,9 @@ export type TransactionParameter = SQLiteTransaction< export interface IDbManager { /** * Efficiently executes a Drizzle query for multiple data rows using - * op-sqlite executeBatch under the hood. + * op-sqlite executeBatch under the hood. The batch owns its transaction; + * it cannot be nested inside write(). The callback argument is only used + * to build the prepared query and is not the surrounding write transaction. */ batch>( data: T[], diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index 3d66c4d298..dbdde905e2 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -17,6 +17,7 @@ import { NOVEL_STORAGE } from '@utils/Storages'; import { downloadFile } from '@plugins/helpers/fetch'; import { getPlugin } from '@plugins/pluginManager'; import { dbManager } from '@database/db'; +import { createNovelTriggerQueryDelete } from '@database/queryStrings/triggers'; import { novelSchema, novelCategorySchema, @@ -475,66 +476,139 @@ export const updateNovelCategories = async ( }); }; +const restoreNovelRecord = async ( + tx: TransactionParameter, + novel: Omit, +) => { + const restoredNovel = await tx + .insert(novelSchema) + .values({ + ...novel, + totalChapters: 0, + chaptersDownloaded: 0, + chaptersUnread: 0, + lastReadAt: null, + lastUpdatedAt: null, + }) + .onConflictDoUpdate({ + target: [novelSchema.path, novelSchema.pluginId], + set: { + ...novel, + totalChapters: 0, + chaptersDownloaded: 0, + chaptersUnread: 0, + lastReadAt: null, + lastUpdatedAt: null, + }, + }) + .returning({ id: novelSchema.id }) + .get(); + + if (novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) { + const cacheSuffix = novel.cover.match(/[?#].*$/)?.[0] ?? ''; + await tx + .update(novelSchema) + .set({ + cover: `file://${NOVEL_STORAGE}/${novel.pluginId}/${restoredNovel.id}/cover.png${cacheSuffix}`, + }) + .where(eq(novelSchema.id, restoredNovel.id)) + .run(); + } + + // Recomputing aggregates for every deleted chapter is quadratic for restores. + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); + await tx + .delete(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovel.id)) + .run(); + await tx.run(sql.raw(createNovelTriggerQueryDelete)); + + return restoredNovel; +}; +const restoreChapterValues = ( + chapters: BackupNovel['chapters'], + novelId: number, +) => + chapters.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ + ...chapter, + novelId, + })); + +const refreshRestoredNovelStats = async ( + tx: TransactionParameter, + novelId: number, +) => { + await tx + .update(novelSchema) + .set({ + totalChapters: sql`(SELECT COUNT(*) FROM Chapter WHERE novelId = ${novelId})`, + chaptersDownloaded: sql`(SELECT COUNT(*) FROM Chapter WHERE novelId = ${novelId} AND isDownloaded = 1)`, + chaptersUnread: sql`(SELECT COUNT(*) FROM Chapter WHERE novelId = ${novelId} AND unread = 1)`, + lastReadAt: sql`(SELECT MAX(readTime) FROM Chapter WHERE novelId = ${novelId})`, + lastUpdatedAt: sql`( + SELECT updatedTime + FROM Chapter + WHERE novelId = ${novelId} AND updatedTime IS NOT NULL + ORDER BY julianday(updatedTime) DESC + LIMIT 1 + )`, + }) + .where(eq(novelSchema.id, novelId)) + .run(); +}; + +const RESTORE_CHAPTER_BATCH_SIZE = 100; + +type RestoreNovelOptions = { + includeChapterMappings?: boolean; +}; + /** * Restores novel and chapters from a backup object. */ export const _restoreNovelAndChapters = async ( backupNovel: BackupNovel, + options: RestoreNovelOptions = {}, ): Promise => { const { chapters, id: backupNovelId, ...novel } = backupNovel; - return dbManager.write(async tx => { - // Match novels by their stable source identity, not the database-local ID. - const restoredNovel = await tx - .insert(novelSchema) - .values({ - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }, - }) - .returning({ id: novelSchema.id }) - .get(); + const includeChapterMappings = options.includeChapterMappings ?? true; - if (novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) { - const cacheSuffix = novel.cover.match(/[?#].*$/)?.[0] ?? ''; - await tx - .update(novelSchema) - .set({ - cover: `file://${NOVEL_STORAGE}/${novel.pluginId}/${restoredNovel.id}/cover.png${cacheSuffix}`, - }) - .where(eq(novelSchema.id, restoredNovel.id)) - .run(); - } + if (!includeChapterMappings) { + return dbManager.write(async tx => { + const restoredNovel = await restoreNovelRecord(tx, novel); - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); + for (let i = 0; i < chapters.length; i += RESTORE_CHAPTER_BATCH_SIZE) { + const batch = chapters.slice(i, i + RESTORE_CHAPTER_BATCH_SIZE); + if (batch.length > 0) { + await tx + .insert(chapterSchema) + .values(restoreChapterValues(batch, restoredNovel.id)) + .run(); + } + } + await refreshRestoredNovelStats(tx, restoredNovel.id); + return { + pluginId: novel.pluginId, + backupNovelId, + restoredNovelId: restoredNovel.id, + chapters: [], + }; + }); + } + + return dbManager.write(async tx => { + const restoredNovel = await restoreNovelRecord(tx, novel); const chapterMappings: RestoredNovelMapping['chapters'] = []; - // Restore chapters in batches + // Restore chapters in batches. if (chapters.length > 0) { - const BATCH_SIZE = 100; + const BATCH_SIZE = RESTORE_CHAPTER_BATCH_SIZE; for (let i = 0; i < chapters.length; i += BATCH_SIZE) { const batch = chapters.slice(i, i + BATCH_SIZE); const restoredChapters = await tx .insert(chapterSchema) - .values( - batch.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ - ...chapter, - novelId: restoredNovel.id, - })), - ) + .values(restoreChapterValues(batch, restoredNovel.id)) .returning({ id: chapterSchema.id, path: chapterSchema.path }) .all(); const restoredIdsByPath = new Map( @@ -551,6 +625,7 @@ export const _restoreNovelAndChapters = async ( } } } + await refreshRestoredNovelStats(tx, restoredNovel.id); return { pluginId: novel.pluginId, diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index 916daad39b..05e738ac3c 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -8,6 +8,7 @@ import './mockDb'; import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; import { insertTestNovel, + insertTestChapter, insertTestNovelCategory, insertTestCategory, clearAllTables, @@ -525,6 +526,125 @@ describe('NovelQueries', () => { }, ]); }); + it('restores chapters without allocating ID mappings when requested', async () => { + const mapping = await _restoreNovelAndChapters( + { + id: 2, + path: '/restored/without-mappings', + pluginId: 'restored-plugin', + name: 'Restored Without Mappings', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 20, + novelId: 2, + path: '/restored/chapter-2', + name: 'Chapter 2', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 2, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + expect(mapping.chapters).toEqual([]); + const restoredChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-2'); + }); + it('rebuilds aggregate timestamps after replacing existing chapters', async () => { + const testDb = getTestDb(); + const novelId = await insertTestNovel(testDb, { + path: '/restore/stats', + pluginId: 'stats-plugin', + }); + await insertTestChapter(testDb, novelId, { + path: '/restore/old-chapter', + readTime: '2020-01-01T00:00:00.000Z', + updatedTime: '2020-01-01T00:00:00.000Z', + }); + await testDb.drizzleDb + .update(novelSchema) + .set({ + lastReadAt: 'stale-read-time', + lastUpdatedAt: 'stale-update-time', + }) + .where(eq(novelSchema.id, novelId)) + .run(); + + await _restoreNovelAndChapters( + { + id: 3, + path: '/restore/stats', + pluginId: 'stats-plugin', + name: 'Restored Stats', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 30, + novelId: 3, + path: '/restore/new-chapter', + name: 'New Chapter', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: true, + updatedTime: '2025-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + const restored = await getNovelByPath('/restore/stats', 'stats-plugin'); + expect(restored).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 0, + lastReadAt: '2024-01-01T00:00:00.000Z', + lastUpdatedAt: '2025-01-01T00:00:00.000Z', + }), + ); + }); }); describe('updateNovelInfo', () => { diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 7c018e283a..4f46a302a0 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -331,10 +331,12 @@ describe('selective backup data', () => { id: 1, cover: 'file:///storage/Novels/source/1/cover.png?123', }), + { includeChapterMappings: false }, ); expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( 2, expect.objectContaining({ id: 2, cover: null }), + { includeChapterMappings: false }, ); }); diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index d7b186ef80..7acf9e8543 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -312,7 +312,9 @@ export const restoreData = async ( backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; } - const novelMapping = await _restoreNovelAndChapters(backupNovel); + const novelMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: manifest.sections.downloadedFiles, + }); novelMappings.push(novelMapping); novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); From 91995980874f7a00147d8b42e1212e915c578aaa Mon Sep 17 00:00:00 2001 From: CD-Z Date: Mon, 21 Sep 2026 20:22:57 +0200 Subject: [PATCH 03/10] perf(backup): optimize restore batching and archives Generated with OpenAI Codex. --- .../modules/nativefile/NativeFileModule.kt | 153 ++++++--- .../NativeZipArchiveModule.kt | 79 ++++- .../ios/NativeZipArchiveModule.swift | 4 + .../src/NativeZipArchiveModule.ts | 4 + src/database/queries/NovelQueries.ts | 248 +++++++++----- .../queries/__tests__/NovelQueries.test.ts | 140 ++++++++ .../backup/__tests__/fileSections.test.ts | 33 +- src/services/backup/__tests__/local.test.ts | 236 +++++++++++++- src/services/backup/__tests__/utils.test.ts | 278 +++++++++++++++- src/services/backup/drive/index.ts | 3 +- src/services/backup/fileSections.ts | 3 +- src/services/backup/local/index.ts | 72 ++++- src/services/backup/selfhost/index.ts | 3 +- src/services/backup/types.ts | 3 +- src/services/backup/utils.ts | 303 +++++++++++++----- test/mocks/nativeModules.js | 1 + 16 files changed, 1320 insertions(+), 243 deletions(-) diff --git a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt index 3a957a4d08..83db1e6d94 100644 --- a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt +++ b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt @@ -28,19 +28,22 @@ import okhttp3.JavaNetCookieJar import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response +import java.io.BufferedInputStream +import java.io.BufferedOutputStream import java.io.File +import java.io.FileInputStream import java.io.FileOutputStream import java.io.FileWriter import java.io.IOException import java.io.InputStream import java.io.OutputStream -import java.util.UUID import java.io.PushbackInputStream +import java.util.UUID import java.util.zip.GZIPInputStream import kotlin.coroutines.coroutineContext class NativeFileModule : Module() { - private val BUFFER_SIZE = 4096 + private val BUFFER_SIZE = 64 * 1024 private val okHttpClient = OkHttpClientProvider.createClient() private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private var pendingDocumentPromise: Promise? = null @@ -108,60 +111,67 @@ class NativeFileModule : Module() { ?: throw Exception("ENOENT: could not open an input stream for '$filepath'") } - private val writeAccessByAPILevel: String - get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private fun getOutputStream(filepath: String): OutputStream { val uri = getFileUri(filepath) return reactContext?.contentResolver?.openOutputStream(uri, writeAccessByAPILevel) ?: throw Exception("ENOENT: could not open an output stream for '$filepath'") } + private val writeAccessByAPILevel: String + get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private suspend fun copyFileContent( - filepath: String, - destPath: String, - onDone: (() -> Unit)? = null, - ): Long { - try { - val inputStream = getInputStream(filepath) - var copiedBytes = 0L - try { - val outputStream = getOutputStream(destPath) - try { - val buffer = ByteArray(BUFFER_SIZE) - var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { - coroutineContext.ensureActive() - outputStream.write(buffer, 0, length) - copiedBytes += length - } - outputStream.flush() - } finally { - outputStream.close() - } - } finally { - inputStream.close() - } - if (onDone != null) { - onDone() - } - return copiedBytes - } catch (e: IOException) { - throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + + private fun localFileOrNull(filepath: String): File? { + val uri = Uri.parse(filepath) + return when { + uri.scheme == null -> File(filepath) + uri.scheme.equals(ContentResolver.SCHEME_FILE, ignoreCase = true) -> + uri.path?.let(::File) + else -> null } } - private fun contentResolver(): ContentResolver = reactContext?.contentResolver ?: throw IOException("React context is unavailable") - private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long { + + private suspend fun copyLocalFile(sourceFile: File, destinationFile: File): Long { + if (sourceFile.isDirectory) { + throw IOException("Invalid file, folder found!") + } + + val sourceSize = sourceFile.length() var copiedBytes = 0L - getInputStream(sourcePath).use { inputStream -> - outputStream.use { output -> + FileInputStream(sourceFile).channel.use { sourceChannel -> + FileOutputStream(destinationFile).channel.use { destinationChannel -> + while (copiedBytes < sourceSize) { + coroutineContext.ensureActive() + val bytesToTransfer = minOf(BUFFER_SIZE.toLong(), sourceSize - copiedBytes) + val transferredBytes = + sourceChannel.transferTo(copiedBytes, bytesToTransfer, destinationChannel) + if (transferredBytes <= 0) { + throw IOException("Could not copy the complete local file") + } + copiedBytes += transferredBytes + } + } + } + + if (destinationFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + return copiedBytes + } + + private suspend fun copyStreams( + inputStream: InputStream, + outputStream: OutputStream, + ): Long { + var copiedBytes = 0L + BufferedInputStream(inputStream, BUFFER_SIZE).use { input -> + BufferedOutputStream(outputStream, BUFFER_SIZE).use { output -> val buffer = ByteArray(BUFFER_SIZE) var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { + while (input.read(buffer).also { length = it } > 0) { coroutineContext.ensureActive() output.write(buffer, 0, length) copiedBytes += length @@ -172,6 +182,44 @@ class NativeFileModule : Module() { return copiedBytes } + private suspend fun copyFileContent( + filepath: String, + destPath: String, + onDone: (() -> Unit)? = null, + ): Long { + try { + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val copiedBytes = if (sourceFile != null && destinationFile != null) { + try { + copyLocalFile(sourceFile, destinationFile) + } catch (_: IOException) { + FileInputStream(sourceFile).use { inputStream -> + FileOutputStream(destinationFile).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + } else { + getInputStream(filepath).use { inputStream -> + getOutputStream(destPath).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + if (sourceFile != null && sourceFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + onDone?.invoke() + return copiedBytes + } catch (e: IOException) { + throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + } + } + + private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long = + copyStreams(getInputStream(sourcePath), outputStream) + private fun resolveDirectoryFile(directoryUri: String): File { val uri = Uri.parse(directoryUri) val directory = if (uri.scheme == null) { @@ -479,7 +527,28 @@ class NativeFileModule : Module() { AsyncFunction("moveFile") { filepath: String, destPath: String, promise: Promise -> coroutineScope.launch { try { - val inFile = File(filepath) + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val renamed = if ( + sourceFile != null && + destinationFile != null && + sourceFile.isFile && + !destinationFile.exists() + ) { + try { + sourceFile.renameTo(destinationFile) + } catch (_: SecurityException) { + false + } + } else { + false + } + if (renamed) { + promise.resolve(null) + return@launch + } + + val inFile = sourceFile ?: File(filepath) copyFileContent(filepath, destPath) { if (!inFile.delete()) { throw IOException("Failed to delete source file '$filepath'") diff --git a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt index 75222b47a1..e000269efd 100644 --- a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt +++ b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt @@ -13,11 +13,20 @@ import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream class NativeZipArchiveModule : Module() { - private fun zipProcess(sourceDirPath: String, zos: ZipOutputStream) { + private fun zipProcess( + sourceDirPath: String, + zos: ZipOutputStream, + archivePrefix: String = "", + archiveEntries: MutableSet? = null, + ) { val sourceDir = File(sourceDirPath) sourceDir.walkBottomUp().filter { it.isFile }.forEach { file -> - val zipFileName = + val relativePath = file.absolutePath.removePrefix(sourceDir.absolutePath).removePrefix("/") + val zipFileName = joinArchivePath(archivePrefix, relativePath) + if (archiveEntries != null && !archiveEntries.add(zipFileName)) { + throw IllegalArgumentException("Duplicate ZIP entry: $zipFileName") + } val entry = ZipEntry("$zipFileName${(if (file.isDirectory) "/" else "")}") zos.putNextEntry(entry) file.inputStream().use { fis -> @@ -26,16 +35,45 @@ class NativeZipArchiveModule : Module() { } } + private fun joinArchivePath(prefix: String, relativePath: String): String { + val normalizedRelativePath = relativePath.replace('\\', '/').trimStart('/') + return if (prefix.isEmpty()) { + normalizedRelativePath + } else { + "$prefix/$normalizedRelativePath" + } + } + + private fun normalizeArchivePrefix(prefix: String): String { + require(prefix.isEmpty() || prefix.isNotBlank()) { "ZIP prefix must not be blank" } + val normalized = prefix.replace('\\', '/') + require(!normalized.startsWith("/") && !(normalized.length >= 2 && normalized[1] == ':')) { + "ZIP prefix must be relative" + } + val components = normalized.split('/').filter { it.isNotEmpty() && it != "." } + require(components.none { it == ".." }) { "ZIP prefix must not contain .." } + return components.joinToString("/") + } + + private fun ensureParentDirectory(file: File, createdDirectories: MutableSet) { + val parent = file.parentFile ?: return + val parentPath = parent.absolutePath + if (parentPath !in createdDirectories && (parent.exists() || parent.mkdirs())) { + createdDirectories.add(parentPath) + } + } + override fun definition() = ModuleDefinition { Name("NativeZipArchive") AsyncFunction("unzip") { sourceFilePath: String, distDirPath: String, promise: Promise -> Thread { try { + val createdDirectories = mutableSetOf() ZipFile(sourceFilePath).use { zis -> zis.entries().asSequence().filterNot { it.isDirectory }.forEach { zipEntry -> val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + ensureParentDirectory(newFile, createdDirectories) zis.getInputStream(zipEntry).use { inputStream -> FileOutputStream(newFile).use { fos -> inputStream.copyTo(fos, COPY_BUFFER_SIZE) } } @@ -61,6 +99,38 @@ class NativeZipArchiveModule : Module() { }.start() } + AsyncFunction("zipDirectories") { sources: List>, zipFilePath: String, promise: Promise -> + Thread { + try { + require(sources.isNotEmpty()) { "ZIP sources must not be empty" } + require(zipFilePath.isNotBlank()) { "ZIP file path must not be blank" } + val normalizedSources = sources.map { source -> + val sourceDirPath = + source["path"]?.takeIf { it.isNotBlank() } + ?: throw IllegalArgumentException("ZIP source path must not be blank") + require(File(sourceDirPath).isDirectory) { + "ZIP source path is not a directory: $sourceDirPath" + } + val archivePrefix = + source["prefix"]?.let(::normalizeArchivePrefix) + ?: throw IllegalArgumentException("ZIP source prefix is missing") + sourceDirPath to archivePrefix + } + val archiveEntries = mutableSetOf() + FileOutputStream(zipFilePath).use { fos -> + ZipOutputStream(fos).use { zos -> + normalizedSources.forEach { (sourceDirPath, archivePrefix) -> + zipProcess(sourceDirPath, zos, archivePrefix, archiveEntries) + } + } + } + promise.resolve(null) + } catch (e: Exception) { + promise.reject("ZIP_FAILED", e.message ?: "Zip failed", e) + } + }.start() + } + AsyncFunction("remoteUnzip") { distDirPath: String, urlString: String, headers: Map, promise: Promise -> Thread { val connection = URL(urlString).openConnection() as HttpURLConnection @@ -69,12 +139,13 @@ class NativeZipArchiveModule : Module() { headers.forEach { (key, value) -> connection.setRequestProperty(key, value) } + val createdDirectories = mutableSetOf() ZipInputStream(connection.inputStream).use { zis -> generateSequence { zis.nextEntry } .filterNot { it.isDirectory } .forEach { zipEntry -> val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + ensureParentDirectory(newFile, createdDirectories) FileOutputStream(newFile).use { fos -> zis.copyTo(fos, COPY_BUFFER_SIZE) } } } diff --git a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift index 7060acd82a..c36bbd2b6b 100644 --- a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift +++ b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift @@ -13,6 +13,10 @@ public class NativeZipArchiveModule: Module { promise.reject("NOT_IMPLEMENTED", "zip is not implemented on iOS") } + AsyncFunction("zipDirectories") { (sources: [[String: String]], zipFilePath: String, promise: Promise) in + promise.reject("NOT_IMPLEMENTED", "zipDirectories is not implemented on iOS") + } + AsyncFunction("remoteUnzip") { (distDirPath: String, url: String, headers: [String: String], promise: Promise) in promise.reject("NOT_IMPLEMENTED", "remoteUnzip is not implemented on iOS") } diff --git a/modules/native-zip-archive/src/NativeZipArchiveModule.ts b/modules/native-zip-archive/src/NativeZipArchiveModule.ts index cd62ad6fbd..f912e2175c 100644 --- a/modules/native-zip-archive/src/NativeZipArchiveModule.ts +++ b/modules/native-zip-archive/src/NativeZipArchiveModule.ts @@ -3,6 +3,10 @@ import { requireNativeModule } from 'expo-modules-core'; type NativeZipArchiveModule = { unzip(sourceFilePath: string, distDirPath: string): Promise; zip(sourceDirPath: string, zipFilePath: string): Promise; + zipDirectories( + sources: Array<{ path: string; prefix: string }>, + zipFilePath: string, + ): Promise; remoteUnzip( distDirPath: string, urlString: string, diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index dbdde905e2..c87f1fa7ab 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -17,7 +17,10 @@ import { NOVEL_STORAGE } from '@utils/Storages'; import { downloadFile } from '@plugins/helpers/fetch'; import { getPlugin } from '@plugins/pluginManager'; import { dbManager } from '@database/db'; -import { createNovelTriggerQueryDelete } from '@database/queryStrings/triggers'; +import { + createNovelTriggerQueryDelete, + createNovelTriggerQueryInsert, +} from '@database/queryStrings/triggers'; import { novelSchema, novelCategorySchema, @@ -476,6 +479,16 @@ export const updateNovelCategories = async ( }); }; +const disableNovelStatsTriggers = async (tx: TransactionParameter) => { + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats')); + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); +}; + +const restoreNovelStatsTriggers = async (tx: TransactionParameter) => { + await tx.run(sql.raw(createNovelTriggerQueryInsert)); + await tx.run(sql.raw(createNovelTriggerQueryDelete)); +}; + const restoreNovelRecord = async ( tx: TransactionParameter, novel: Omit, @@ -515,13 +528,10 @@ const restoreNovelRecord = async ( .run(); } - // Recomputing aggregates for every deleted chapter is quadratic for restores. - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); await tx .delete(chapterSchema) .where(eq(chapterSchema.novelId, restoredNovel.id)) .run(); - await tx.run(sql.raw(createNovelTriggerQueryDelete)); return restoredNovel; }; @@ -536,102 +546,178 @@ const restoreChapterValues = ( const refreshRestoredNovelStats = async ( tx: TransactionParameter, - novelId: number, + novelIds: number[], ) => { - await tx - .update(novelSchema) - .set({ - totalChapters: sql`(SELECT COUNT(*) FROM Chapter WHERE novelId = ${novelId})`, - chaptersDownloaded: sql`(SELECT COUNT(*) FROM Chapter WHERE novelId = ${novelId} AND isDownloaded = 1)`, - chaptersUnread: sql`(SELECT COUNT(*) FROM Chapter WHERE novelId = ${novelId} AND unread = 1)`, - lastReadAt: sql`(SELECT MAX(readTime) FROM Chapter WHERE novelId = ${novelId})`, - lastUpdatedAt: sql`( - SELECT updatedTime - FROM Chapter - WHERE novelId = ${novelId} AND updatedTime IS NOT NULL - ORDER BY julianday(updatedTime) DESC + if (novelIds.length === 0) { + return; + } + + const stats = await tx + .select({ + novelId: chapterSchema.novelId, + totalChapters: sql`COUNT(*)`, + chaptersDownloaded: sql`SUM(CASE WHEN ${chapterSchema.isDownloaded} = 1 THEN 1 ELSE 0 END)`, + chaptersUnread: sql`SUM(CASE WHEN ${chapterSchema.unread} = 1 THEN 1 ELSE 0 END)`, + lastReadAt: sql`MAX(${chapterSchema.readTime})`, + lastUpdatedAt: sql`( + SELECT updatedChapter.updatedTime + FROM Chapter AS updatedChapter + WHERE updatedChapter.novelId = Chapter.novelId + AND updatedChapter.updatedTime IS NOT NULL + ORDER BY julianday(updatedChapter.updatedTime) DESC LIMIT 1 )`, }) - .where(eq(novelSchema.id, novelId)) - .run(); + .from(chapterSchema) + .where(inArray(chapterSchema.novelId, novelIds)) + .groupBy(chapterSchema.novelId) + .all(); + + if (stats.length === 0) { + return; + } + + const ids = sql.join( + novelIds.map(id => sql`${id}`), + sql`, `, + ); + const cases = ( + field: T, + fallback: unknown, + ) => + sql`CASE ${novelSchema.id} ${sql.join( + stats.map( + (stat: (typeof stats)[number]) => + sql`WHEN ${stat.novelId} THEN ${stat[field]}`, + ), + sql` `, + )} ELSE ${fallback} END`; + + await tx.run(sql` + UPDATE Novel + SET totalChapters = ${cases('totalChapters', novelSchema.totalChapters)}, + chaptersDownloaded = ${cases( + 'chaptersDownloaded', + novelSchema.chaptersDownloaded, + )}, + chaptersUnread = ${cases('chaptersUnread', novelSchema.chaptersUnread)}, + lastReadAt = ${cases('lastReadAt', novelSchema.lastReadAt)}, + lastUpdatedAt = ${cases('lastUpdatedAt', novelSchema.lastUpdatedAt)} + WHERE id IN (${ids}) + `); }; -const RESTORE_CHAPTER_BATCH_SIZE = 100; +const RESTORE_CHAPTER_BATCH_SIZE = 500; type RestoreNovelOptions = { includeChapterMappings?: boolean; }; -/** - * Restores novel and chapters from a backup object. - */ -export const _restoreNovelAndChapters = async ( - backupNovel: BackupNovel, - options: RestoreNovelOptions = {}, -): Promise => { - const { chapters, id: backupNovelId, ...novel } = backupNovel; - const includeChapterMappings = options.includeChapterMappings ?? true; - - if (!includeChapterMappings) { - return dbManager.write(async tx => { +const restoreNovelsAndChaptersInTransaction = async ( + tx: TransactionParameter, + backupNovels: BackupNovel[], + includeChapterMappings: boolean, +): Promise => { + await disableNovelStatsTriggers(tx); + + try { + const restoredNovels: { + backupNovelId: number; + chapters: BackupNovel['chapters']; + novel: Omit; + restoredNovelId: number; + }[] = []; + for (const backupNovel of backupNovels) { + const { chapters, id: backupNovelId, ...novel } = backupNovel; const restoredNovel = await restoreNovelRecord(tx, novel); + restoredNovels.push({ + backupNovelId, + chapters, + novel, + restoredNovelId: restoredNovel.id, + }); + } - for (let i = 0; i < chapters.length; i += RESTORE_CHAPTER_BATCH_SIZE) { - const batch = chapters.slice(i, i + RESTORE_CHAPTER_BATCH_SIZE); - if (batch.length > 0) { + const mappings: RestoredNovelMapping[] = []; + for (const restoredNovel of restoredNovels) { + const chapterMappings: RestoredNovelMapping['chapters'] = []; + for ( + let i = 0; + i < restoredNovel.chapters.length; + i += RESTORE_CHAPTER_BATCH_SIZE + ) { + const batch = restoredNovel.chapters.slice( + i, + i + RESTORE_CHAPTER_BATCH_SIZE, + ); + if (includeChapterMappings) { + const restoredChapters = (await tx + .insert(chapterSchema) + .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) + .returning({ id: chapterSchema.id, path: chapterSchema.path }) + .all()) as { id: number; path: string }[]; + const restoredIdsByPath = new Map( + restoredChapters.map(chapter => [chapter.path, chapter.id]), + ); + for (const chapter of batch) { + const restoredChapterId = restoredIdsByPath.get(chapter.path); + if (restoredChapterId !== undefined) { + chapterMappings.push({ + backupChapterId: chapter.id, + restoredChapterId, + }); + } + } + } else { await tx .insert(chapterSchema) - .values(restoreChapterValues(batch, restoredNovel.id)) + .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) .run(); } } - await refreshRestoredNovelStats(tx, restoredNovel.id); + mappings.push({ + pluginId: restoredNovel.novel.pluginId, + backupNovelId: restoredNovel.backupNovelId, + restoredNovelId: restoredNovel.restoredNovelId, + chapters: chapterMappings, + }); + } + await refreshRestoredNovelStats( + tx, + restoredNovels.map(restoredNovel => restoredNovel.restoredNovelId), + ); + return mappings; + } finally { + await restoreNovelStatsTriggers(tx); + } +}; - return { - pluginId: novel.pluginId, - backupNovelId, - restoredNovelId: restoredNovel.id, - chapters: [], - }; - }); +export const _restoreNovelsAndChapters = async ( + backupNovels: BackupNovel[], + options: RestoreNovelOptions = {}, +): Promise => { + if (backupNovels.length === 0) { + return []; } + return dbManager.write(tx => + restoreNovelsAndChaptersInTransaction( + tx, + backupNovels, + options.includeChapterMappings ?? true, + ), + ); +}; - return dbManager.write(async tx => { - const restoredNovel = await restoreNovelRecord(tx, novel); - const chapterMappings: RestoredNovelMapping['chapters'] = []; - - // Restore chapters in batches. - if (chapters.length > 0) { - const BATCH_SIZE = RESTORE_CHAPTER_BATCH_SIZE; - for (let i = 0; i < chapters.length; i += BATCH_SIZE) { - const batch = chapters.slice(i, i + BATCH_SIZE); - const restoredChapters = await tx - .insert(chapterSchema) - .values(restoreChapterValues(batch, restoredNovel.id)) - .returning({ id: chapterSchema.id, path: chapterSchema.path }) - .all(); - const restoredIdsByPath = new Map( - restoredChapters.map(chapter => [chapter.path, chapter.id]), - ); - for (const chapter of batch) { - const restoredChapterId = restoredIdsByPath.get(chapter.path); - if (restoredChapterId !== undefined) { - chapterMappings.push({ - backupChapterId: chapter.id, - restoredChapterId, - }); - } - } - } - } - await refreshRestoredNovelStats(tx, restoredNovel.id); - - return { - pluginId: novel.pluginId, - backupNovelId, - restoredNovelId: restoredNovel.id, - chapters: chapterMappings, - }; - }); +/** + * Restores novel and chapters from a backup object. + */ +export const _restoreNovelAndChapters = async ( + backupNovel: BackupNovel, + options: RestoreNovelOptions = {}, +): Promise => { + const [mapping] = await _restoreNovelsAndChapters([backupNovel], options); + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; }; diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index 05e738ac3c..cf42dfd92c 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -37,6 +37,7 @@ import { updateNovelCategoryById, updateNovelCategories, _restoreNovelAndChapters, + _restoreNovelsAndChapters, } from '../NovelQueries'; const mockGetLibraryDefaultCategoryId = jest.fn(); @@ -526,6 +527,145 @@ describe('NovelQueries', () => { }, ]); }); + it('restores multiple novels in one batch', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 100, + path: '/bulk/one', + pluginId: 'bulk-plugin', + name: 'Bulk One', + chapters: [], + }, + { + id: 101, + path: '/bulk/two', + pluginId: 'bulk-plugin', + name: 'Bulk Two', + chapters: [], + }, + ], + { includeChapterMappings: false }, + ); + + expect(mappings).toHaveLength(2); + expect(mappings.map(mapping => mapping.pluginId)).toEqual([ + 'bulk-plugin', + 'bulk-plugin', + ]); + expect(await getNovelByPath('/bulk/one', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk One' }), + ); + expect(await getNovelByPath('/bulk/two', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk Two' }), + ); + }); + it('preserves mappings and aggregate stats for multiple novels', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 200, + path: '/bulk/mapped-one', + pluginId: 'bulk-plugin', + name: 'Mapped One', + chapters: [ + { + id: 2001, + novelId: 200, + path: '/bulk/chapter-one', + name: 'Chapter One', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + { + id: 2002, + novelId: 200, + path: '/bulk/chapter-two', + name: 'Chapter Two', + releaseTime: null, + readTime: '2025-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: false, + updatedTime: '2023-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + progress: null, + position: 1, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 201, + path: '/bulk/mapped-two', + pluginId: 'bulk-plugin', + name: 'Mapped Two', + chapters: [ + { + id: 2011, + novelId: 201, + path: '/bulk/chapter-three', + name: 'Chapter Three', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ], + { includeChapterMappings: true }, + ); + + expect(mappings).toHaveLength(2); + expect(mappings[0].chapters).toHaveLength(2); + expect(mappings[1].chapters).toHaveLength(1); + expect( + new Set( + mappings + .flatMap(mapping => mapping.chapters) + .map(chapter => chapter.restoredChapterId), + ).size, + ).toBe(3); + expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-01-01T00:00:00.000Z', + }), + ); + expect(await getNovelByPath('/bulk/mapped-two', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: null, + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + }); it('restores chapters without allocating ID mappings when requested', async () => { const mapping = await _restoreNovelAndChapters( { diff --git a/src/services/backup/__tests__/fileSections.test.ts b/src/services/backup/__tests__/fileSections.test.ts index f7d5bac972..d12400459c 100644 --- a/src/services/backup/__tests__/fileSections.test.ts +++ b/src/services/backup/__tests__/fileSections.test.ts @@ -1,10 +1,41 @@ import NativeFile from '@modules/native-file'; -import { restoreNovelFiles } from '../fileSections'; +import { + getSelectedBackupFileSections, + restoreNovelFiles, +} from '../fileSections'; +import { ZipBackupName } from '../types'; jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', })); +describe('selected backup file sections', () => { + const options = { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }; + + it('keeps novel files in the legacy v2 section only', () => { + expect(getSelectedBackupFileSections(options)).toEqual([ + { + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + ]); + expect(getSelectedBackupFileSections(options, 2)).toEqual([ + { + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + { + archiveName: ZipBackupName.NOVEL_FILES, + storagePath: '/storage/Novels', + }, + ]); + }); +}); describe('restoreNovelFiles', () => { it('moves downloaded files to remapped novel and chapter IDs', async () => { diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index 5601615128..c68ed5419d 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -35,12 +35,35 @@ jest.mock('@i18n/translations', () => ({ })); describe('local selective backup', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + beforeEach(() => { + jest.mocked(prepareBackupData).mockReset(); + jest.mocked(restoreData).mockReset(); + jest.mocked(NativeZipArchive.zip).mockReset().mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.zipDirectories) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.unzip) + .mockReset() + .mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.exists).mockReset().mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockReset().mockResolvedValue([]); + jest.mocked(finalizeRestoredPlugins).mockReset().mockResolvedValue([]); + }); + it('creates archives only for selected file sections', async () => { jest.mocked(prepareBackupData).mockResolvedValue({ failedNovelCount: 0, failedSectionCount: 0, }); - jest.mocked(NativeZipArchive.zip).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.zipDirectories).mockResolvedValue(undefined); jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); await createBackup({ @@ -53,12 +76,16 @@ describe('local selective backup', () => { }, }); - expect(prepareBackupData).toHaveBeenCalledWith('/cache/BackupData', { - library: true, - settings: true, - plugins: true, - downloadedFiles: false, - }); + expect(prepareBackupData).toHaveBeenCalledWith( + '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }, + 3, + ); expect(NativeZipArchive.zip).toHaveBeenCalledWith( '/storage/Plugins', '/cache/BackupData/plugins.zip', @@ -67,8 +94,54 @@ describe('local selective backup', () => { '/storage/Novels', expect.any(String), ); - expect(NativeZipArchive.zip).toHaveBeenCalledWith( + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [{ path: '/cache/BackupData', prefix: '' }], + '/cache/BackupData.zip', + ); + }); + + it('adds novel files to the v3 outer archive without a nested archive', async () => { + jest.mocked(prepareBackupData).mockResolvedValue({ + failedNovelCount: 0, + failedSectionCount: 0, + }); + jest.mocked(NativeZipArchive.zip).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.zipDirectories).mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + + await createBackup({ + destinationUri: 'content://backup.zip', + options: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }); + + expect(prepareBackupData).toHaveBeenCalledWith( '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + 3, + ); + expect(NativeZipArchive.zip).toHaveBeenCalledWith( + '/storage/Plugins', + '/cache/BackupData/plugins.zip', + ); + expect(NativeZipArchive.zip).not.toHaveBeenCalledWith( + '/storage/Novels', + expect.any(String), + ); + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [ + { path: '/cache/BackupData', prefix: '' }, + { path: '/storage/Novels', prefix: 'NovelFiles' }, + ], '/cache/BackupData.zip', ); }); @@ -113,4 +186,151 @@ describe('local selective backup', () => { jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], ); }); + it('extracts the v1 downloaded archive into the legacy staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + manifest: { + appVersion: '1.0.0', + formatVersion: 1 as const, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/download.zip', + '/cache/BackupData/RestoredLegacyFiles', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + }); + + it('extracts the v2 novel-files archive into the novel staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + manifest: { + appVersion: '2.0.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + '/cache/BackupData/RestoredNovelFiles', + ); + }); + + it('restores v3 novel files from the outer archive without nested extraction', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/plugins.zip', + '/storage/Plugins', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + }); + it('rejects a v3 downloaded-file restore without NovelFiles', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + }); }); diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 4f46a302a0..4d4736972c 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -1,5 +1,6 @@ import { _restoreNovelAndChapters, + _restoreNovelsAndChapters, getAllNovels, } from '@database/queries/NovelQueries'; import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; @@ -11,10 +12,12 @@ import { import NativeFile from '@modules/native-file'; import { MMKVStorage } from '@utils/mmkv/mmkv'; import { prepareBackupData, restoreData } from '../utils'; +import type { BackupNovel, RestoredNovelMapping } from '@database/types'; import type { BackupOptions } from '../options'; jest.mock('@database/queries/NovelQueries', () => ({ _restoreNovelAndChapters: jest.fn(), + _restoreNovelsAndChapters: jest.fn(), getAllNovels: jest.fn(), })); @@ -71,6 +74,7 @@ const pluginOnlyOptions: BackupOptions = { describe('selective backup data', () => { beforeEach(() => { + jest.clearAllMocks(); jest.mocked(NativeFile.exists).mockResolvedValue(false); jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); @@ -79,7 +83,9 @@ describe('selective backup data', () => { jest.mocked(getAllNovelChaptersForBackup).mockResolvedValue([]); jest.mocked(getCategoriesFromDb).mockResolvedValue([]); jest.mocked(getAllNovelCategories).mockResolvedValue([]); - jest.mocked(_restoreNovelAndChapters).mockImplementation(async novel => ({ + const restoreNovel = async ( + novel: BackupNovel, + ): Promise => ({ pluginId: novel.pluginId, backupNovelId: novel.id, restoredNovelId: novel.id, @@ -87,7 +93,13 @@ describe('selective backup data', () => { backupChapterId: chapter.id, restoredChapterId: chapter.id, })), - })); + }); + jest.mocked(_restoreNovelAndChapters).mockImplementation(restoreNovel); + jest + .mocked(_restoreNovelsAndChapters) + .mockImplementation(async (novels: BackupNovel[]) => + Promise.all(novels.map(restoreNovel)), + ); }); it('writes the selected sections to the v2 manifest', async () => { @@ -108,6 +120,30 @@ describe('selective backup data', () => { ); }); + it('accepts v3 section manifests', async () => { + const options: BackupOptions = { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }; + jest.mocked(NativeFile.readFile).mockResolvedValueOnce( + JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }), + ); + + const result = await restoreData('/cache'); + + expect(result.manifest).toEqual({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }); + }); + it('does not warn about sections intentionally omitted by the manifest', async () => { jest .mocked(NativeFile.readFile) @@ -260,6 +296,40 @@ describe('selective backup data', () => { ); }); + it('does not duplicate covers when downloaded files are included', async () => { + jest.mocked(getAllNovels).mockResolvedValueOnce([ + { + id: 1, + name: 'Example', + path: '/example', + pluginId: 'source', + cover: 'file:///storage/Novels/source/1/cover.png?123', + }, + ]); + + jest.mocked(NativeFile.copyFile).mockClear(); + jest.mocked(NativeFile.mkdir).mockClear(); + + await prepareBackupData('/cache', { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }); + + const novelWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => path.endsWith('/1.json')); + expect(JSON.parse(novelWrite?.[1] ?? '{}')).toMatchObject({ + cover: '/Novels/source/1/cover.png?123', + }); + expect(NativeFile.copyFile).not.toHaveBeenCalledWith( + 'file:///storage/Novels/source/1/cover.png', + '/cache/Covers/1', + ); + expect(NativeFile.mkdir).not.toHaveBeenCalledWith('/cache/Covers'); + }); + it('restores stored covers from library data and preserves missing covers', async () => { const options: BackupOptions = { library: true, @@ -325,19 +395,17 @@ describe('selective backup data', () => { '/cache/Covers/1', '/storage/Novels/source/1/cover.png', ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - id: 1, - cover: 'file:///storage/Novels/source/1/cover.png?123', - }), - { includeChapterMappings: false }, - ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ id: 2, cover: null }), + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 1, + cover: 'file:///storage/Novels/source/1/cover.png?123', + }), + expect.objectContaining({ id: 2, cover: null }), + ], { includeChapterMappings: false }, ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); }); it('omits the installed-plugin registry when plugin files are excluded', async () => { @@ -369,6 +437,190 @@ describe('selective backup data', () => { }); }); + it('restores downloaded-file novels in one mapping-aware batch', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 11, + name: 'First', + path: '/first', + pluginId: 'source', + chapters: [ + { + id: 101, + novelId: 11, + name: 'First chapter', + path: '/first/1', + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + page: null, + progress: null, + timeSpent: 0, + }, + ], + }, + { + id: 22, + name: 'Second', + path: '/second', + pluginId: 'source', + chapters: [ + { + id: 202, + novelId: 22, + name: 'Second chapter', + path: '/second/1', + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + page: null, + progress: null, + timeSpent: 0, + }, + ], + }, + ]; + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'source', + backupNovelId: 11, + restoredNovelId: 111, + chapters: [{ backupChapterId: 101, restoredChapterId: 1001 }], + }, + { + pluginId: 'source', + backupNovelId: 22, + restoredNovelId: 222, + chapters: [{ backupChapterId: 202, restoredChapterId: 2002 }], + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + + it('retries each downloaded-file novel when its batch restore fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 31, + name: 'First', + path: '/first', + pluginId: 'source', + chapters: [], + }, + { + id: 32, + name: 'Second', + path: '/second', + pluginId: 'source', + chapters: [], + }, + ]; + const mappings: RestoredNovelMapping[] = novels.map((novel, index) => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + chapters: [ + { backupChapterId: index + 1, restoredChapterId: index + 101 }, + ], + })); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest + .mocked(_restoreNovelsAndChapters) + .mockRejectedValueOnce(new Error('batch failed')); + jest + .mocked(_restoreNovelAndChapters) + .mockImplementation( + async novel => + mappings.find(mapping => mapping.backupNovelId === novel.id)!, + ); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(1, novels[0], { + includeChapterMappings: true, + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(2, novels[1], { + includeChapterMappings: true, + }); + expect(result).toMatchObject({ + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + it('treats backups without a section manifest as legacy full backups', async () => { jest .mocked(NativeFile.readFile) diff --git a/src/services/backup/drive/index.ts b/src/services/backup/drive/index.ts index d00e13041c..e65e4cca4c 100644 --- a/src/services/backup/drive/index.ts +++ b/src/services/backup/drive/index.ts @@ -77,7 +77,7 @@ export const createDriveBackup = async ( progressText: getString('backupScreen.uploadingSelectedFiles'), })); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await uploadBackupSection( section.storagePath, section.archiveName, @@ -149,6 +149,7 @@ export const driveRestore = async ( const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); for (const section of getSelectedBackupFileSections( restoreResult.manifest.sections, + 2, )) { const file = await exists(section.archiveName, false, backupFolder.id); if (!file) { diff --git a/src/services/backup/fileSections.ts b/src/services/backup/fileSections.ts index ece55687ad..5ff536db40 100644 --- a/src/services/backup/fileSections.ts +++ b/src/services/backup/fileSections.ts @@ -11,6 +11,7 @@ export type BackupFileSection = { export const getSelectedBackupFileSections = ( options: BackupOptions, + formatVersion: 2 | 3 = 3, ): BackupFileSection[] => { const sections: BackupFileSection[] = []; @@ -20,7 +21,7 @@ export const getSelectedBackupFileSections = ( storagePath: PLUGIN_STORAGE, }); } - if (options.downloadedFiles) { + if (formatVersion === 2 && options.downloadedFiles) { sections.push({ archiveName: ZipBackupName.NOVEL_FILES, storagePath: NOVEL_STORAGE, diff --git a/src/services/backup/local/index.ts b/src/services/backup/local/index.ts index 85e7822c32..f4eea43231 100644 --- a/src/services/backup/local/index.ts +++ b/src/services/backup/local/index.ts @@ -10,11 +10,12 @@ import { } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import NativeZipArchive from '@modules/native-zip-archive'; -import { ZipBackupName } from '../types'; +import { BackupEntryName, ZipBackupName } from '../types'; import NativeFile from '@modules/native-file'; import { getString } from '@i18n/translations'; import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; import { sleep } from '@utils/sleep'; +import { NOVEL_STORAGE } from '@utils/Storages'; import { getLegacyFilesRestorePath, getNovelFilesRestorePath, @@ -24,6 +25,19 @@ import { } from '../fileSections'; import { resolveBackupOptions, type BackupOptions } from '../options'; +const logRestoreBenchmark = (message: string) => { + if (!__DEV__) { + return; + } + // Benchmark output is consumed from the Metro client log. + // eslint-disable-next-line no-console + console.log( + `[restore-benchmark] ${new Date().toISOString()} ${performance + .now() + .toFixed(3)} ${message}`, + ); +}; + export const createBackup = async ( { destinationUri, @@ -40,7 +54,7 @@ export const createBackup = async ( progressText: getString('backupScreen.preparingData'), })); - const backupResult = await prepareBackupData(CACHE_DIR_PATH, options); + const backupResult = await prepareBackupData(CACHE_DIR_PATH, options, 3); setMeta?.(meta => ({ ...meta, @@ -65,7 +79,15 @@ export const createBackup = async ( await sleep(200); - await NativeZipArchive.zip(CACHE_DIR_PATH, CACHE_DIR_PATH + '.zip'); + await NativeZipArchive.zipDirectories( + [ + { path: CACHE_DIR_PATH, prefix: '' }, + ...(options.downloadedFiles + ? [{ path: NOVEL_STORAGE, prefix: BackupEntryName.NOVEL_FILES }] + : []), + ], + CACHE_DIR_PATH + '.zip', + ); setMeta?.(meta => ({ ...meta, @@ -96,6 +118,7 @@ export const restoreBackup = async ( { sourceUri }: { sourceUri: string }, setMeta?: TaskProgressUpdater, ) => { + logRestoreBenchmark('local:start'); try { setMeta?.(meta => ({ ...meta, @@ -107,6 +130,7 @@ export const restoreBackup = async ( await clearBackupCache(); const localPath = CACHE_DIR_PATH + '-source.zip'; await NativeFile.copyFile(sourceUri, localPath); + logRestoreBenchmark('local:copy:done'); setMeta?.(meta => ({ ...meta, @@ -117,6 +141,7 @@ export const restoreBackup = async ( await sleep(200); await NativeZipArchive.unzip(localPath, CACHE_DIR_PATH); + logRestoreBenchmark('local:outer-unzip:done'); setMeta?.(meta => ({ ...meta, @@ -126,16 +151,12 @@ export const restoreBackup = async ( await sleep(200); - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta?.(meta => ({ - ...meta, - progress: 3 / 4, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - + const restoreResult = await restoreData( + CACHE_DIR_PATH, + setMeta, + logRestoreBenchmark, + ); + logRestoreBenchmark('local:restore-data:done'); if (restoreResult.manifest.formatVersion === 1) { const legacyArchive = CACHE_DIR_PATH + '/' + ZipBackupName.DOWNLOAD; if (!(await NativeFile.exists(legacyArchive))) { @@ -143,15 +164,28 @@ export const restoreBackup = async ( } const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); await NativeZipArchive.unzip(legacyArchive, legacyFilesRestorePath); + logRestoreBenchmark('local:selected-archives:done'); await restoreLegacyFiles( legacyFilesRestorePath, restoreResult.novelMappings, ); + logRestoreBenchmark('local:downloaded-files:done'); } else { const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( + const sections = getSelectedBackupFileSections( restoreResult.manifest.sections, - )) { + restoreResult.manifest.formatVersion, + ); + if ( + restoreResult.manifest.formatVersion === 3 && + restoreResult.manifest.sections.downloadedFiles && + !(await NativeFile.exists( + `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}`, + )) + ) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + for (const section of sections) { const archivePath = `${CACHE_DIR_PATH}/${section.archiveName}`; if (!(await NativeFile.exists(archivePath))) { throw new Error(getString('backupScreen.invalidBackupFolder')); @@ -163,18 +197,24 @@ export const restoreBackup = async ( : section.storagePath, ); } + logRestoreBenchmark('local:selected-archives:done'); if (restoreResult.manifest.sections.downloadedFiles) { await restoreNovelFiles( - novelFilesRestorePath, + restoreResult.manifest.formatVersion === 3 + ? `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}` + : novelFilesRestorePath, restoreResult.novelMappings, ); } + logRestoreBenchmark('local:downloaded-files:done'); } + logRestoreBenchmark('local:selected-files:done'); const missingPluginIds = await finalizeRestoredPlugins(restoreResult); const completionText = getRestoreCompletionText( restoreResult, missingPluginIds, ); + logRestoreBenchmark('local:finalize:done'); setMeta?.(meta => ({ ...meta, diff --git a/src/services/backup/selfhost/index.ts b/src/services/backup/selfhost/index.ts index b25a8194ec..e3f81203b5 100644 --- a/src/services/backup/selfhost/index.ts +++ b/src/services/backup/selfhost/index.ts @@ -58,7 +58,7 @@ export const createSelfHostBackup = async ( await sleep(200); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await upload(host, backupFolder, section.archiveName, section.storagePath); } @@ -120,6 +120,7 @@ export const selfHostRestore = async ( const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); for (const section of getSelectedBackupFileSections( restoreResult.manifest.sections, + 2, )) { await download( host, diff --git a/src/services/backup/types.ts b/src/services/backup/types.ts index aefd5cf65f..4825052742 100644 --- a/src/services/backup/types.ts +++ b/src/services/backup/types.ts @@ -14,11 +14,12 @@ export enum BackupEntryName { PLUGIN_METADATA = 'Plugins.json', COVERS = 'Covers', NOVEL_AND_CHAPTERS = 'NovelAndChapters', + NOVEL_FILES = 'NovelFiles', } export type BackupManifest = { appVersion: string; - formatVersion: 2; + formatVersion: 2 | 3; sections: BackupOptions; }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index 7acf9e8543..a0995332e1 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -5,6 +5,7 @@ import { MMKVStorage } from '@utils/mmkv/mmkv'; import { version } from '../../../package.json'; import { _restoreNovelAndChapters, + _restoreNovelsAndChapters, getAllNovels, } from '@database/queries/NovelQueries'; import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; @@ -40,6 +41,12 @@ import type { PluginItem } from '@plugins/types'; const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; +const BACKUP_NOVEL_BATCH_SIZE = 100; + +const RESTORE_NOVEL_BATCH_SIZE = 100; + +const BACKUP_FILE_CONCURRENCY = 8; + const stripUriSuffix = (uri: string) => uri.split(/[?#]/, 1)[0]; const parentDirectory = (path: string) => @@ -96,6 +103,7 @@ const restoreMMKVData = (data: any) => { export const prepareBackupData = async ( cacheDirPath: string, requestedOptions?: BackupOptions, + formatVersion: BackupManifest['formatVersion'] = 2, ): Promise => { const options = resolveBackupOptions(requestedOptions); const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; @@ -109,7 +117,7 @@ export const prepareBackupData = async ( // version const manifest: BackupManifest = { appVersion: version, - formatVersion: 2, + formatVersion, sections: options, }; await NativeFile.writeFile( @@ -120,60 +128,109 @@ export const prepareBackupData = async ( // novels if (options.library) { await NativeFile.mkdir(novelDirPath); - await NativeFile.mkdir(coversDirPath); - await getAllNovels().then(async novels => { - for (const novel of novels) { - try { - const chapters = await getAllNovelChaptersForBackup(novel.id); - const backedUpChapters = options.downloadedFiles - ? chapters - : chapters.map(chapter => ({ - ...chapter, - isDownloaded: false, - })); - let cover = novel.cover; - if (cover?.startsWith(APP_STORAGE_URI)) { + if (!options.downloadedFiles) { + await NativeFile.mkdir(coversDirPath); + } + const novels = await getAllNovels(); + for ( + let start = 0; + start < novels.length; + start += BACKUP_NOVEL_BATCH_SIZE + ) { + const novelBatch = novels.slice(start, start + BACKUP_NOVEL_BATCH_SIZE); + let chapters; + try { + chapters = await getAllNovelChaptersForBackup( + novelBatch.map(novel => novel.id), + ); + } catch { + failedNovelCount += novelBatch.length; + continue; + } + + const chaptersByNovel = new Map(); + for (const chapter of chapters) { + const novelChapters = chaptersByNovel.get(chapter.novelId); + if (novelChapters) { + novelChapters.push(chapter); + } else { + chaptersByNovel.set(chapter.novelId, [chapter]); + } + } + + for ( + let fileStart = 0; + fileStart < novelBatch.length; + fileStart += BACKUP_FILE_CONCURRENCY + ) { + const fileBatch = novelBatch.slice( + fileStart, + fileStart + BACKUP_FILE_CONCURRENCY, + ); + await Promise.all( + fileBatch.map(async novel => { try { - await NativeFile.copyFile( - stripUriSuffix(cover), - coversDirPath + '/' + novel.id, + const novelChapters = chaptersByNovel.get(novel.id) ?? []; + const backedUpChapters = options.downloadedFiles + ? novelChapters + : novelChapters.map(chapter => ({ + ...chapter, + isDownloaded: false, + })); + let cover = novel.cover; + if (cover?.startsWith(APP_STORAGE_URI)) { + if (options.downloadedFiles) { + cover = cover.replace(APP_STORAGE_URI, ''); + } else { + try { + await NativeFile.copyFile( + stripUriSuffix(cover), + coversDirPath + '/' + novel.id, + ); + cover = cover.replace(APP_STORAGE_URI, ''); + } catch { + cover = null; + } + } + } + await NativeFile.writeFile( + novelDirPath + '/' + novel.id + '.json', + JSON.stringify({ + chapters: backedUpChapters, + ...novel, + cover, + }), ); - cover = cover.replace(APP_STORAGE_URI, ''); } catch { - cover = options.downloadedFiles - ? cover.replace(APP_STORAGE_URI, '') - : null; + failedNovelCount++; } - } - await NativeFile.writeFile( - novelDirPath + '/' + novel.id + '.json', - JSON.stringify({ - chapters: backedUpChapters, - ...novel, - cover, - }), - ); - } catch { - failedNovelCount++; - } + }), + ); } - }); + } // categories try { const categories = await getCategoriesFromDb(); const novelCategories = await getAllNovelCategories(); + const novelIdsByCategory = new Map(); + for (const novelCategory of novelCategories) { + const novelIds = novelIdsByCategory.get(novelCategory.categoryId); + if (novelIds) { + novelIds.push(novelCategory.novelId); + } else { + novelIdsByCategory.set(novelCategory.categoryId, [ + novelCategory.novelId, + ]); + } + } await NativeFile.writeFile( cacheDirPath + '/' + BackupEntryName.CATEGORY, JSON.stringify( - categories.map(category => { - return { - ...category, - novelIds: novelCategories - .filter(nc => nc.categoryId === category.id) - .map(nc => nc.novelId), - }; - }), + categories.map(category => ({ + ...category, + novelIds: novelIdsByCategory.get(category.id) ?? [], + })), ), ); } catch { @@ -221,10 +278,13 @@ const getBackupManifest = async ( const data = JSON.parse(fileContent) as Partial & { version?: string; }; - if (data.formatVersion === 2 && data.sections) { + if ( + (data.formatVersion === 2 || data.formatVersion === 3) && + data.sections + ) { return { appVersion: data.appVersion ?? data.version ?? '', - formatVersion: 2, + formatVersion: data.formatVersion, sections: resolveBackupOptions(data.sections), }; } @@ -252,11 +312,15 @@ const updateRestoreProgress = ( })); }; +type RestoreBenchmarkLogger = (message: string) => void; + export const restoreData = async ( cacheDirPath: string, setMeta?: TaskProgressUpdater, + benchmarkLog?: RestoreBenchmarkLogger, ): Promise => { const manifest = await getBackupManifest(cacheDirPath); + benchmarkLog?.('restoreData:manifest:loaded'); const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; const pluginIds = new Set(); @@ -273,16 +337,17 @@ export const restoreData = async ( })(); let pluginsFromSettings: PluginItem[] = []; - // version - // nothing to do - - // novels + benchmarkLog?.('restoreData:novels:start'); if (manifest.sections.library) { updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); } let novelCount = 0; let failedCount = 0; let failedSectionCount = 0; + let readMs = 0; + let parseMs = 0; + let databaseMs = 0; + let coverMs = 0; if (!manifest.sections.library) { // Intentionally omitted from this backup. @@ -293,6 +358,82 @@ export const restoreData = async ( const items = (await NativeFile.readDir(novelDirPath)).filter( item => !item.isDirectory, ); + const pendingNovels: BackupNovel[] = []; + const restoreNovelBatchSize = RESTORE_NOVEL_BATCH_SIZE; + const restoreNovelBatch = async () => { + if (pendingNovels.length === 0) { + return; + } + const batch = pendingNovels.splice(0, pendingNovels.length); + let restoredNovels: { + backupNovel: BackupNovel; + mapping: RestoredNovelMapping; + }[]; + const databaseStartedAt = performance.now(); + try { + const mappings = await _restoreNovelsAndChapters(batch, { + includeChapterMappings: manifest.sections.downloadedFiles, + }); + if (mappings.length !== batch.length) { + throw new Error('Restore returned incomplete novel mappings'); + } + restoredNovels = batch.map((backupNovel, index) => ({ + backupNovel, + mapping: mappings[index], + })); + } catch { + restoredNovels = []; + for (const backupNovel of batch) { + try { + restoredNovels.push({ + backupNovel, + mapping: await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: manifest.sections.downloadedFiles, + }), + }); + } catch { + failedCount++; + } + } + } finally { + databaseMs += performance.now() - databaseStartedAt; + } + + const coverStartedAt = performance.now(); + for ( + let start = 0; + start < restoredNovels.length; + start += BACKUP_FILE_CONCURRENCY + ) { + const coverBatch = restoredNovels.slice( + start, + start + BACKUP_FILE_CONCURRENCY, + ); + await Promise.all( + coverBatch.map(async ({ backupNovel, mapping: novelMapping }) => { + if ( + !manifest.sections.downloadedFiles && + backupNovel.cover?.startsWith(APP_STORAGE_URI) + ) { + const coverBackupPath = coversDirPath + '/' + backupNovel.id; + if (await NativeFile.exists(coverBackupPath)) { + const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; + await NativeFile.mkdir(parentDirectory(coverPath)); + await NativeFile.copyFile(coverBackupPath, coverPath); + } + } + }), + ); + } + coverMs += performance.now() - coverStartedAt; + + for (const { backupNovel, mapping: novelMapping } of restoredNovels) { + novelMappings.push(novelMapping); + novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); + novelCount++; + } + }; + for (const [index, item] of items.entries()) { updateRestoreProgress( setMeta, @@ -301,42 +442,50 @@ export const restoreData = async ( total: items.length, }), ); + let fileContent: string; + const readStartedAt = performance.now(); try { - const fileContent = await NativeFile.readFile(item.path); - const backupNovel = JSON.parse(fileContent) as BackupNovel; + fileContent = await NativeFile.readFile(item.path); + } catch { + failedCount++; + continue; + } finally { + readMs += performance.now() - readStartedAt; + } + + const parseStartedAt = performance.now(); + try { + const backupNovel = JSON.parse(fileContent!) as BackupNovel; pluginIds.add(backupNovel.pluginId); - const hasStoredCover = - backupNovel.cover && !backupNovel.cover.startsWith('http'); - if (hasStoredCover) { + if (backupNovel.cover && !backupNovel.cover.startsWith('http')) { backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; } - - const novelMapping = await _restoreNovelAndChapters(backupNovel, { - includeChapterMappings: manifest.sections.downloadedFiles, - }); - novelMappings.push(novelMapping); - novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); - - if (hasStoredCover) { - const coverBackupPath = coversDirPath + '/' + backupNovel.id; - if (await NativeFile.exists(coverBackupPath)) { - const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; - await NativeFile.mkdir(parentDirectory(coverPath)); - await NativeFile.copyFile(coverBackupPath, coverPath); - } - } - novelCount++; + pendingNovels.push(backupNovel); } catch { failedCount++; + } finally { + parseMs += performance.now() - parseStartedAt; + } + + if (pendingNovels.length >= restoreNovelBatchSize) { + await restoreNovelBatch(); } } + await restoreNovelBatch(); + benchmarkLog?.( + `restoreData:novels:done count=${novelCount} failed=${failedCount} readMs=${readMs.toFixed( + 1, + )} parseMs=${parseMs.toFixed(1)} databaseMs=${databaseMs.toFixed( + 1, + )} coverMs=${coverMs.toFixed(1)}`, + ); } catch { failedSectionCount++; } } - // categories + benchmarkLog?.('restoreData:categories:start'); if (manifest.sections.library) { updateRestoreProgress( setMeta, @@ -383,8 +532,11 @@ export const restoreData = async ( failedSectionCount++; } } + benchmarkLog?.( + `restoreData:categories:done count=${categoryCount} failed=${failedCategoryCount}`, + ); - // settings + benchmarkLog?.('restoreData:settings:start'); if (manifest.sections.settings) { updateRestoreProgress(setMeta, getString('backupScreen.restoringSettings')); } @@ -411,11 +563,12 @@ export const restoreData = async ( // Included in the completion warning below. } } + benchmarkLog?.(`restoreData:settings:done restored=${settingsRestored}`); - // installed plugin registry + benchmarkLog?.('restoreData:plugins:start'); + let restoredPlugins = pluginsFromSettings; if (manifest.sections.plugins) { - let restoredPlugins = pluginsFromSettings; - if (manifest.formatVersion === 2) { + if (manifest.formatVersion === 2 || manifest.formatVersion === 3) { const pluginMetadataPath = cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA; if (!(await NativeFile.exists(pluginMetadataPath))) { @@ -440,6 +593,8 @@ export const restoreData = async ( ]; MMKVStorage.set(INSTALLED_PLUGINS_KEY, JSON.stringify(mergedPlugins)); } + benchmarkLog?.(`restoreData:plugins:done count=${restoredPlugins.length}`); + benchmarkLog?.('restoreData:done'); return { novelCount, diff --git a/test/mocks/nativeModules.js b/test/mocks/nativeModules.js index e5f48124bf..a228365d9d 100644 --- a/test/mocks/nativeModules.js +++ b/test/mocks/nativeModules.js @@ -81,6 +81,7 @@ jest.mock('@modules/native-zip-archive', () => ({ __esModule: true, default: { zip: jest.fn().mockResolvedValue(), + zipDirectories: jest.fn().mockResolvedValue(), unzip: jest.fn().mockResolvedValue(), remoteUnzip: jest.fn().mockResolvedValue(), remoteZip: jest.fn().mockResolvedValue(''), From dd81bbfa6fccb6ebb399da61c9d58ee899278c79 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Tue, 22 Sep 2026 13:07:39 +0200 Subject: [PATCH 04/10] Optimize backup restore pipeline --- src/database/db.ts | 1 + src/database/manager/manager.d.ts | 13 +- src/database/queries/ChapterQueries.ts | 20 +- src/database/queries/NovelQueries.ts | 68 ++- .../queries/__tests__/ChapterQueries.test.ts | 22 + .../queries/__tests__/NovelQueries.test.ts | 5 + src/services/backup/__tests__/local.test.ts | 1 + src/services/backup/__tests__/utils.test.ts | 529 +++++++++++++++++- src/services/backup/novelPayload.ts | 266 +++++++++ src/services/backup/types.ts | 2 + src/services/backup/utils.ts | 74 ++- 11 files changed, 931 insertions(+), 70 deletions(-) create mode 100644 src/services/backup/novelPayload.ts diff --git a/src/database/db.ts b/src/database/db.ts index acdd5efcc2..c20d424a96 100644 --- a/src/database/db.ts +++ b/src/database/db.ts @@ -66,6 +66,7 @@ export const drizzleDb = drizzle(_db, { schema, logger: __DEV__ ? new MyLogger() : false, }); +export type DrizzleDb = typeof drizzleDb export const dbManager = createDbManager(drizzleDb); diff --git a/src/database/manager/manager.d.ts b/src/database/manager/manager.d.ts index 95276d7ea5..30997a11c7 100644 --- a/src/database/manager/manager.d.ts +++ b/src/database/manager/manager.d.ts @@ -1,19 +1,14 @@ // db-manager.types.ts +import type { DrizzleDb } from '@database/db'; import type { - SQLiteTransaction, - TablesRelationalConfig, Placeholder, } from 'drizzle-orm'; import { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; // Define the TransactionParameter type based on your DrizzleDb -export type TransactionParameter = SQLiteTransaction< - 'async', - { changes: number; lastInsertRowid: number } | void, - Record, - TablesRelationalConfig ->; - +export type TransactionParameter = Parameters< + Parameters[0] +>[0]; /** * Interface defining the public API and documentation for the Drizzle database manager. * This contract ensures consistent documentation and type safety across the application. diff --git a/src/database/queries/ChapterQueries.ts b/src/database/queries/ChapterQueries.ts index 9b7ca28cf4..3bcddc945b 100644 --- a/src/database/queries/ChapterQueries.ts +++ b/src/database/queries/ChapterQueries.ts @@ -468,14 +468,24 @@ export const getNovelChapters = async ( * a paginated UI query because doing so silently produces incomplete backups. */ export const getAllNovelChaptersForBackup = async ( - novelId: number, -): Promise => - dbManager + novelIds: number | number[], +): Promise => { + const ids = Array.isArray(novelIds) ? novelIds : [novelIds]; + if (ids.length === 0) { + return []; + } + + return dbManager .select() .from(chapterSchema) - .where(eq(chapterSchema.novelId, novelId)) - .orderBy(asc(chapterSchema.id)) + .where( + Array.isArray(novelIds) + ? inArray(chapterSchema.novelId, ids) + : eq(chapterSchema.novelId, ids[0]), + ) + .orderBy(asc(chapterSchema.novelId), asc(chapterSchema.id)) .all(); +}; export const getNovelChaptersSync = ( novelId: number, diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index c87f1fa7ab..c3f5a03a90 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -20,6 +20,7 @@ import { dbManager } from '@database/db'; import { createNovelTriggerQueryDelete, createNovelTriggerQueryInsert, + createNovelTriggerQueryUpdate, } from '@database/queryStrings/triggers'; import { novelSchema, @@ -481,60 +482,71 @@ export const updateNovelCategories = async ( const disableNovelStatsTriggers = async (tx: TransactionParameter) => { await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats')); + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_update')); await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); }; const restoreNovelStatsTriggers = async (tx: TransactionParameter) => { await tx.run(sql.raw(createNovelTriggerQueryInsert)); await tx.run(sql.raw(createNovelTriggerQueryDelete)); + await tx.run(sql.raw(createNovelTriggerQueryUpdate)); }; const restoreNovelRecord = async ( tx: TransactionParameter, novel: Omit, ) => { + const resetState = { + totalChapters: 0, + chaptersDownloaded: 0, + chaptersUnread: 0, + lastReadAt: null, + lastUpdatedAt: null, + }; + + // The order here makes resetState authoritative. + const values = { + ...resetState, + ...novel, + }; + + const localCover = values.cover?.startsWith(`file://${NOVEL_STORAGE}/`) + ? values.cover + : undefined; + + const cacheSuffix = localCover?.match(/[?#].*$/)?.[0] ?? ''; + const restoredNovel = await tx .insert(novelSchema) - .values({ - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - lastReadAt: null, - lastUpdatedAt: null, - }) + .values(values) .onConflictDoUpdate({ target: [novelSchema.path, novelSchema.pluginId], - set: { - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - lastReadAt: null, - lastUpdatedAt: null, - }, + set: values, }) .returning({ id: novelSchema.id }) .get(); - if (novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) { - const cacheSuffix = novel.cover.match(/[?#].*$/)?.[0] ?? ''; + await tx + .delete(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovel.id)) + .run(); + + if (localCover !== undefined) { await tx .update(novelSchema) .set({ - cover: `file://${NOVEL_STORAGE}/${novel.pluginId}/${restoredNovel.id}/cover.png${cacheSuffix}`, + cover: + `file://${NOVEL_STORAGE}/` + + `${values.pluginId}/${restoredNovel.id}/cover.png` + + cacheSuffix, }) .where(eq(novelSchema.id, restoredNovel.id)) .run(); } - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); - return restoredNovel; }; + const restoreChapterValues = ( chapters: BackupNovel['chapters'], novelId: number, @@ -682,10 +694,10 @@ const restoreNovelsAndChaptersInTransaction = async ( chapters: chapterMappings, }); } - await refreshRestoredNovelStats( - tx, - restoredNovels.map(restoredNovel => restoredNovel.restoredNovelId), - ); + // await refreshRestoredNovelStats( + // tx, + // restoredNovels.map(restoredNovel => restoredNovel.restoredNovelId), + // ); return mappings; } finally { await restoreNovelStatsTriggers(tx); diff --git a/src/database/queries/__tests__/ChapterQueries.test.ts b/src/database/queries/__tests__/ChapterQueries.test.ts index 687f833db3..12c9a95305 100644 --- a/src/database/queries/__tests__/ChapterQueries.test.ts +++ b/src/database/queries/__tests__/ChapterQueries.test.ts @@ -185,6 +185,28 @@ describe('ChapterQueries', () => { expect(backupChapters).toHaveLength(1001); expect(backupChapters.at(-1)?.name).toBe('Chapter 1001'); }); + + it('returns chapters for multiple novels in novel and chapter order', async () => { + const testDb = getTestDb(); + testDb.sqlite.executeSync(` + INSERT INTO Chapter + (novelId, path, name, chapterNumber, page, position) + VALUES + (10, '/chapter/1', 'Chapter 1', 1, '1', 1), + (10, '/chapter/2', 'Chapter 2', 2, '1', 2), + (20, '/chapter/1', 'Chapter 1', 1, '1', 1), + (20, '/chapter/2', 'Chapter 2', 2, '1', 2) + `); + + const chapters = await getAllNovelChaptersForBackup([20, 10]); + + expect(chapters.map(chapter => [chapter.novelId, chapter.name])).toEqual([ + [10, 'Chapter 1'], + [10, 'Chapter 2'], + [20, 'Chapter 1'], + [20, 'Chapter 2'], + ]); + }); }); describe('markChaptersRead', () => { diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index cf42dfd92c..a6e6b375ed 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -638,6 +638,11 @@ describe('NovelQueries', () => { ); expect(mappings).toHaveLength(2); + expect( + mappings.map(mapping => + mapping.chapters.map(chapter => chapter.backupChapterId), + ), + ).toEqual([[2001, 2002], [2011]]); expect(mappings[0].chapters).toHaveLength(2); expect(mappings[1].chapters).toHaveLength(1); expect( diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index c68ed5419d..846037cb8d 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -273,6 +273,7 @@ describe('local selective backup', () => { manifest: { appVersion: '2.1.3', formatVersion: 3 as const, + novelDataFormat: 2 as const, sections: { library: true, settings: false, diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 4d4736972c..83a0ad5c65 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -12,7 +12,12 @@ import { import NativeFile from '@modules/native-file'; import { MMKVStorage } from '@utils/mmkv/mmkv'; import { prepareBackupData, restoreData } from '../utils'; -import type { BackupNovel, RestoredNovelMapping } from '@database/types'; +import { decodeNovelBatch, encodeNovelBatch } from '../novelPayload'; +import type { + BackupNovel, + ChapterInfo, + RestoredNovelMapping, +} from '@database/types'; import type { BackupOptions } from '../options'; jest.mock('@database/queries/NovelQueries', () => ({ @@ -71,6 +76,45 @@ const pluginOnlyOptions: BackupOptions = { plugins: true, downloadedFiles: false, }; +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: null, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); describe('selective backup data', () => { beforeEach(() => { @@ -285,11 +329,18 @@ describe('selective backup data', () => { const novelWrite = jest .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/1.json')); - expect(JSON.parse(novelWrite?.[1] ?? '{}')).toMatchObject({ - cover: '/Novels/source/1/cover.png?123', - chapters: [{ id: 10, isDownloaded: false }], + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', }); + expect(compactNovel.c).toHaveLength(1); + expect(compactNovel.c[0]).toHaveLength(15); + expect(compactNovel.c[0][0]).toBe(10); + expect(compactNovel.c[0][7]).toBe(false); expect(NativeFile.copyFile).toHaveBeenCalledWith( 'file:///storage/Novels/source/1/cover.png', '/cache/Covers/1', @@ -319,10 +370,15 @@ describe('selective backup data', () => { const novelWrite = jest .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/1.json')); - expect(JSON.parse(novelWrite?.[1] ?? '{}')).toMatchObject({ - cover: '/Novels/source/1/cover.png?123', + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', }); + expect(NativeFile.copyFile).not.toHaveBeenCalledWith( 'file:///storage/Novels/source/1/cover.png', '/cache/Covers/1', @@ -330,6 +386,212 @@ describe('selective backup data', () => { expect(NativeFile.mkdir).not.toHaveBeenCalledWith('/cache/Covers'); }); + it('round-trips a fully populated novel through the compact codec', () => { + const novel: BackupNovel & { + chaptersDownloaded: number; + chaptersUnread: number; + totalChapters: number; + lastReadAt: string; + lastUpdatedAt: string; + } = { + id: 7, + name: 'The Compact Novel', + path: '/novels/compact', + pluginId: 'source', + cover: '/covers/compact.png?cache=1', + summary: null, + author: 'Author', + artist: null, + status: 'ongoing', + genres: null, + inLibrary: null, + isLocal: false, + totalPages: 2048, + chaptersDownloaded: 1, + chaptersUnread: 1, + totalChapters: 2, + lastReadAt: '2024-03-01T10:20:30.000Z', + lastUpdatedAt: '2024-03-02T10:20:30.000Z', + chapters: [ + { + id: 701, + novelId: 7, + path: '/novels/compact/1', + name: 'First chapter', + releaseTime: '2024-01-01T00:00:00.000Z', + readTime: '2024-03-01T10:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 1, + page: '4', + position: 2, + progress: 0.75, + scanlator: 'Team A', + timeSpent: 90, + }, + { + id: 702, + novelId: 7, + path: '/novels/compact/2', + name: 'Second chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: null, + }, + ], + }; + + const [compactNovel] = encodeNovelBatch([novel]); + + expect(Object.keys(compactNovel).sort()).toEqual( + [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', + ].sort(), + ); + expect(compactNovel).toMatchObject({ + id: 7, + p: '/novels/compact', + pi: 'source', + n: 'The Compact Novel', + co: '/covers/compact.png?cache=1', + d: 1, + u: 1, + tc: 2, + lr: '2024-03-01T10:20:30.000Z', + lu: '2024-03-02T10:20:30.000Z', + }); + expect(compactNovel.c).toEqual([ + [ + 701, + '/novels/compact/1', + 'First chapter', + '2024-01-01T00:00:00.000Z', + true, + false, + '2024-03-01T10:00:00.000Z', + true, + '2024-02-01T00:00:00.000Z', + 1, + '4', + 2, + 0.75, + 'Team A', + 90, + ], + [ + 702, + '/novels/compact/2', + 'Second chapter', + null, + null, + true, + null, + false, + null, + null, + null, + null, + null, + null, + null, + ], + ]); + expect(Object.keys(compactNovel.c[0])).not.toContain('novelId'); + expect(compactNovel).not.toHaveProperty('chapters'); + + expect(decodeNovelBatch([compactNovel])).toEqual([novel]); + }); + + it('writes 101 novels as ordered compact batches with a format marker', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels = Array.from({ length: 101 }, (_, index) => { + const id = index + 1; + return makeTestNovel(id, 'source', [ + makeTestChapter(id, 1), + makeTestChapter(id, 2), + ]); + }); + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + jest.mocked(getAllNovelChaptersForBackup).mockImplementation(async ids => { + const requestedIds = Array.isArray(ids) ? ids : [ids]; + return novels + .flatMap(novel => novel.chapters) + .filter(chapter => requestedIds.includes(chapter.novelId)); + }); + + await prepareBackupData('/cache', options); + + const batchWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.includes('/NovelAndChapters/batch-')); + expect(batchWrites.map(([path]) => path)).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + ]); + expect(batchWrites.some(([path]) => /\/\d+\.json$/.test(path))).toBe(false); + + const batches = batchWrites.map(([, content]) => JSON.parse(content)); + expect(batches.map(batch => batch.length)).toEqual([100, 1]); + const records = batches.flat(); + expect(records.map(record => record.id)).toEqual( + novels.map(novel => novel.id), + ); + expect( + records.map(record => + record.c.map((chapter: [number, ...unknown[]]) => chapter[0]), + ), + ).toEqual(novels.map(novel => novel.chapters.map(chapter => chapter.id))); + expect( + records.every( + record => + Array.isArray(record.c) && + record.c.every((chapter: unknown[]) => chapter.length === 15), + ), + ).toBe(true); + + const manifestWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => path.endsWith('/Version.json')); + expect(JSON.parse(manifestWrite?.[1] ?? '{}')).toMatchObject({ + formatVersion: 2, + novelDataFormat: 2, + }); + }); + it('restores stored covers from library data and preserves missing covers', async () => { const options: BackupOptions = { library: true, @@ -541,6 +803,98 @@ describe('selective backup data', () => { }); }); + it('restores scrambled compact batches in order with downloaded-file mappings', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const firstNovel = makeTestNovel(11, 'plugin-a', [makeTestChapter(11, 1)]); + const secondNovel = makeTestNovel(22, 'plugin-b', [makeTestChapter(22, 1)]); + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'plugin-a', + backupNovelId: 11, + restoredNovelId: 111, + chapters: [{ backupChapterId: 1101, restoredChapterId: 1001 }], + }, + { + pluginId: 'plugin-b', + backupNovelId: 22, + restoredNovelId: 222, + chapters: [{ backupChapterId: 2201, restoredChapterId: 2001 }], + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'batch-000002.json', + path: '/cache/NovelAndChapters/batch-000002.json', + isDirectory: false, + }, + { + name: 'ignored-directory', + path: '/cache/NovelAndChapters/ignored-directory', + isDirectory: true, + }, + { + name: 'batch-000001.json', + path: '/cache/NovelAndChapters/batch-000001.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + if (path.endsWith('/batch-000001.json')) { + return JSON.stringify(encodeNovelBatch([firstNovel])); + } + if (path.endsWith('/batch-000002.json')) { + return JSON.stringify(encodeNovelBatch([secondNovel])); + } + throw new Error(`Unexpected read: ${path}`); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + const restoredNovels = jest.mocked(_restoreNovelsAndChapters).mock + .calls[0][0] as BackupNovel[]; + expect(restoredNovels.map(novel => novel.id)).toEqual([11, 22]); + expect( + restoredNovels.map(novel => + novel.chapters.map(chapter => ({ + id: chapter.id, + novelId: chapter.novelId, + isDownloaded: chapter.isDownloaded, + })), + ), + ).toEqual([ + [{ id: 1101, novelId: 11, isDownloaded: true }], + [{ id: 2201, novelId: 22, isDownloaded: true }], + ]); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(expect.any(Array), { + includeChapterMappings: true, + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + novelCount: 2, + failedNovelCount: 0, + pluginIds: ['plugin-a', 'plugin-b'], + novelMappings: mappings, + }); + }); + it('retries each downloaded-file novel when its batch restore fails', async () => { const options: BackupOptions = { library: true, @@ -621,6 +975,165 @@ describe('selective backup data', () => { }); }); + it.each([ + { + label: 'v1', + manifest: { version: '1.0.0' }, + formatVersion: 1, + }, + { + label: 'v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 2, + }, + { + label: 'v3', + manifest: { + appVersion: '3.0.0', + formatVersion: 3, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 3, + }, + ])( + '$label object payloads remain restorable', + async ({ manifest, formatVersion }) => { + const novel = makeTestNovel(301, 'legacy-source'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify(manifest); + } + return JSON.stringify(novel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 301, + pluginId: 'legacy-source', + chapters: [ + expect.objectContaining({ + id: 30101, + novelId: 301, + }), + ], + }), + ], + { includeChapterMappings: true }, + ); + expect(result.manifest).toMatchObject({ formatVersion }); + expect(result).toMatchObject({ + novelCount: 1, + failedNovelCount: 0, + novelMappings: [ + expect.objectContaining({ + backupNovelId: 301, + pluginId: 'legacy-source', + }), + ], + }); + }, + ); + + it('skips malformed compact files without partially restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(901, 'valid-source'); + const [validRecord] = encodeNovelBatch([validNovel]); + const malformedFiles: Record = { + 'batch-000001.json': '{', + 'batch-000002.json': JSON.stringify({}), + 'batch-000003.json': JSON.stringify([{ ...validRecord, n: undefined }]), + 'batch-000004.json': JSON.stringify([ + validRecord, + { + ...validRecord, + c: [[...validRecord.c[0]].slice(0, 14)], + }, + ]), + 'batch-000005.json': JSON.stringify([{ ...validRecord, id: 'bad' }]), + 'batch-000006.json': JSON.stringify([{ ...validRecord, p: 123 }]), + 'batch-000007.json': JSON.stringify([validRecord]), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(malformedFiles) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return malformedFiles[path.split('/').pop() ?? '']; + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 901, + pluginId: 'valid-source', + chapters: [ + expect.objectContaining({ + id: 90101, + novelId: 901, + }), + ], + }), + ], + { includeChapterMappings: true }, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + novelCount: 1, + failedNovelCount: 6, + }); + }); + it('treats backups without a section manifest as legacy full backups', async () => { jest .mocked(NativeFile.readFile) diff --git a/src/services/backup/novelPayload.ts b/src/services/backup/novelPayload.ts new file mode 100644 index 0000000000..ff261466dd --- /dev/null +++ b/src/services/backup/novelPayload.ts @@ -0,0 +1,266 @@ +import type { BackupNovel, ChapterInfo } from '@database/types'; + +type NullableString = string | null; +type NullableNumber = number | null; +type NullableBoolean = boolean | null; + +type CompactChapter = [ + number, + string, + string, + NullableString, + NullableBoolean, + NullableBoolean, + NullableString, + NullableBoolean, + NullableString, + NullableNumber, + NullableString, + NullableNumber, + NullableNumber, + NullableString, + NullableNumber, +]; + +type CompactNovel = { + c: CompactChapter[]; + id: number; + p: string; + pi: string; + n: string; + co: NullableString; + s: NullableString; + a: NullableString; + ar: NullableString; + st: NullableString; + g: NullableString; + l: NullableBoolean; + lo: NullableBoolean; + t: NullableNumber; + d: NullableNumber; + u: NullableNumber; + tc: NullableNumber; + lr: NullableString; + lu: NullableString; +}; + +type BackupNovelWithAggregates = BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; +}; + +const NOVEL_KEYS = [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', +] as const; + +const isNullable = ( + value: unknown, + predicate: (value: unknown) => value is T, +): value is T | null => value === null || predicate(value); + +const isString = (value: unknown): value is string => typeof value === 'string'; +const isNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); +const isId = (value: unknown): value is number => + isNumber(value) && Number.isInteger(value); +const isBoolean = (value: unknown): value is boolean => + typeof value === 'boolean'; + +const nullableString = (value: unknown): value is NullableString => + isNullable(value, isString); +const nullableNumber = (value: unknown): value is NullableNumber => + isNullable(value, isNumber); +const nullableBoolean = (value: unknown): value is NullableBoolean => + isNullable(value, isBoolean); + +const assertNovelKeys = (novel: Record) => { + const keys = Object.keys(novel).sort(); + const expected = [...NOVEL_KEYS].sort(); + if ( + keys.length !== expected.length || + keys.some((key, index) => key !== expected[index]) + ) { + throw new Error('Invalid compact novel keys'); + } +}; + +const isCompactChapter = (value: unknown): value is CompactChapter => { + if (!Array.isArray(value) || value.length !== 15) { + return false; + } + + return ( + isId(value[0]) && + isString(value[1]) && + isString(value[2]) && + nullableString(value[3]) && + nullableBoolean(value[4]) && + nullableBoolean(value[5]) && + nullableString(value[6]) && + nullableBoolean(value[7]) && + nullableString(value[8]) && + nullableNumber(value[9]) && + nullableString(value[10]) && + nullableNumber(value[11]) && + nullableNumber(value[12]) && + nullableString(value[13]) && + nullableNumber(value[14]) + ); +}; + +const isCompactNovel = (value: unknown): value is CompactNovel => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const novel = value as Record; + assertNovelKeys(novel); + return ( + Array.isArray(novel.c) && + novel.c.every(isCompactChapter) && + isId(novel.id) && + isString(novel.p) && + isString(novel.pi) && + isString(novel.n) && + nullableString(novel.co) && + nullableString(novel.s) && + nullableString(novel.a) && + nullableString(novel.ar) && + nullableString(novel.st) && + nullableString(novel.g) && + nullableBoolean(novel.l) && + nullableBoolean(novel.lo) && + nullableNumber(novel.t) && + nullableNumber(novel.d) && + nullableNumber(novel.u) && + nullableNumber(novel.tc) && + nullableString(novel.lr) && + nullableString(novel.lu) + ); +}; + +const nullable = (value: T | undefined | null): T | null => value ?? null; + +const encodeChapter = (chapter: ChapterInfo): CompactChapter => [ + chapter.id, + chapter.path, + chapter.name, + nullable(chapter.releaseTime), + nullable(chapter.bookmark), + nullable(chapter.unread), + nullable(chapter.readTime), + nullable(chapter.isDownloaded), + nullable(chapter.updatedTime), + nullable(chapter.chapterNumber), + nullable(chapter.page), + nullable(chapter.position), + nullable(chapter.progress), + nullable(chapter.scanlator), + nullable(chapter.timeSpent), +]; + +const encodeNovel = (novel: BackupNovel): CompactNovel => { + const novelWithAggregates = novel as BackupNovelWithAggregates; + return { + c: novel.chapters.map(encodeChapter), + id: novel.id, + p: novel.path, + pi: novel.pluginId, + n: novel.name, + co: nullable(novel.cover), + s: nullable(novel.summary), + a: nullable(novel.author), + ar: nullable(novel.artist), + st: nullable(novel.status), + g: nullable(novel.genres), + l: nullable(novel.inLibrary), + lo: nullable(novel.isLocal), + t: nullable(novel.totalPages), + d: nullable(novelWithAggregates.chaptersDownloaded), + u: nullable(novelWithAggregates.chaptersUnread), + tc: nullable(novelWithAggregates.totalChapters), + lr: nullable(novelWithAggregates.lastReadAt), + lu: nullable(novelWithAggregates.lastUpdatedAt), + }; +}; + +const decodeNovel = (novel: CompactNovel): BackupNovel => { + const decoded: BackupNovelWithAggregates = { + id: novel.id, + path: novel.p, + pluginId: novel.pi, + name: novel.n, + cover: novel.co, + summary: novel.s, + author: novel.a, + artist: novel.ar, + status: novel.st, + genres: novel.g, + inLibrary: novel.l, + isLocal: novel.lo, + totalPages: novel.t, + chaptersDownloaded: novel.d, + chaptersUnread: novel.u, + totalChapters: novel.tc, + lastReadAt: novel.lr, + lastUpdatedAt: novel.lu, + chapters: novel.c.map(chapter => ({ + id: chapter[0], + novelId: novel.id, + path: chapter[1], + name: chapter[2], + releaseTime: chapter[3], + bookmark: chapter[4], + unread: chapter[5], + readTime: chapter[6], + isDownloaded: chapter[7], + updatedTime: chapter[8], + chapterNumber: chapter[9], + page: chapter[10], + position: chapter[11], + progress: chapter[12], + scanlator: chapter[13], + timeSpent: chapter[14], + })), + }; + return decoded; +}; + +export const encodeNovelBatch = (novels: BackupNovel[]): CompactNovel[] => { + if (novels.length > 100) { + throw new Error('Compact novel batch exceeds 100 novels'); + } + return novels.map(encodeNovel); +}; + +export const decodeNovelBatch = (payload: unknown): BackupNovel[] => { + if (!Array.isArray(payload) || payload.length > 100) { + throw new Error('Invalid compact novel batch'); + } + return payload.map(value => { + if (!isCompactNovel(value)) { + throw new Error('Invalid compact novel record'); + } + return decodeNovel(value); + }); +}; diff --git a/src/services/backup/types.ts b/src/services/backup/types.ts index 4825052742..9cc5ac5670 100644 --- a/src/services/backup/types.ts +++ b/src/services/backup/types.ts @@ -20,6 +20,7 @@ export enum BackupEntryName { export type BackupManifest = { appVersion: string; formatVersion: 2 | 3; + novelDataFormat?: 1 | 2; sections: BackupOptions; }; @@ -28,5 +29,6 @@ export type ResolvedBackupManifest = | { appVersion?: string; formatVersion: 1; + novelDataFormat?: 1 | 2; sections: BackupOptions; }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index a0995332e1..4acf7e658e 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -19,6 +19,7 @@ import { BackupNovel, type RestoredNovelMapping, } from '@database/types'; +import { decodeNovelBatch, encodeNovelBatch } from './novelPayload'; import { BackupEntryName, type BackupManifest, @@ -114,10 +115,10 @@ export const prepareBackupData = async ( await clearBackupCache(cacheDirPath); await NativeFile.mkdir(cacheDirPath); - // version const manifest: BackupManifest = { appVersion: version, formatVersion, + novelDataFormat: 2, sections: options, }; await NativeFile.writeFile( @@ -158,6 +159,7 @@ export const prepareBackupData = async ( } } + const preparedNovels: BackupNovel[] = []; for ( let fileStart = 0; fileStart < novelBatch.length; @@ -167,8 +169,8 @@ export const prepareBackupData = async ( fileStart, fileStart + BACKUP_FILE_CONCURRENCY, ); - await Promise.all( - fileBatch.map(async novel => { + const prepared = await Promise.all( + fileBatch.map(async (novel): Promise => { try { const novelChapters = chaptersByNovel.get(novel.id) ?? []; const backedUpChapters = options.downloadedFiles @@ -193,19 +195,40 @@ export const prepareBackupData = async ( } } } - await NativeFile.writeFile( - novelDirPath + '/' + novel.id + '.json', - JSON.stringify({ - chapters: backedUpChapters, - ...novel, - cover, - }), - ); + return { + ...novel, + chapters: backedUpChapters, + cover, + }; } catch { failedNovelCount++; + return null; } }), ); + preparedNovels.push( + ...prepared.filter((novel): novel is BackupNovel => novel !== null), + ); + } + + if (preparedNovels.length > 0) { + const batchName = `batch-${String( + Math.floor(start / BACKUP_NOVEL_BATCH_SIZE) + 1, + ).padStart(6, '0')}.json`; + const batchPath = novelDirPath + '/' + batchName; + try { + await NativeFile.writeFile( + batchPath, + JSON.stringify(encodeNovelBatch(preparedNovels)), + ); + } catch { + failedNovelCount += preparedNovels.length; + try { + await NativeFile.unlink(batchPath); + } catch { + // Best effort cleanup prevents a failed write from restoring partial data. + } + } } } @@ -282,9 +305,14 @@ const getBackupManifest = async ( (data.formatVersion === 2 || data.formatVersion === 3) && data.sections ) { + const novelDataFormat = + data.novelDataFormat === 1 || data.novelDataFormat === 2 + ? data.novelDataFormat + : undefined; return { appVersion: data.appVersion ?? data.version ?? '', formatVersion: data.formatVersion, + ...(novelDataFormat === undefined ? {} : { novelDataFormat }), sections: resolveBackupOptions(data.sections), }; } @@ -355,9 +383,11 @@ export const restoreData = async ( failedSectionCount++; } else { try { - const items = (await NativeFile.readDir(novelDirPath)).filter( - item => !item.isDirectory, - ); + const items = (await NativeFile.readDir(novelDirPath)) + .filter(item => !item.isDirectory) + .sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, + ); const pendingNovels: BackupNovel[] = []; const restoreNovelBatchSize = RESTORE_NOVEL_BATCH_SIZE; const restoreNovelBatch = async () => { @@ -455,13 +485,17 @@ export const restoreData = async ( const parseStartedAt = performance.now(); try { - const backupNovel = JSON.parse(fileContent!) as BackupNovel; - pluginIds.add(backupNovel.pluginId); - - if (backupNovel.cover && !backupNovel.cover.startsWith('http')) { - backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; + const decodedNovels = + manifest.novelDataFormat === 2 + ? decodeNovelBatch(JSON.parse(fileContent!)) + : [JSON.parse(fileContent!) as BackupNovel]; + for (const backupNovel of decodedNovels) { + pluginIds.add(backupNovel.pluginId); + if (backupNovel.cover && !backupNovel.cover.startsWith('http')) { + backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; + } + pendingNovels.push(backupNovel); } - pendingNovels.push(backupNovel); } catch { failedCount++; } finally { From 2ba2fd15564f8dcc36abe6b539fcf677a66e3537 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Tue, 22 Sep 2026 13:12:15 +0200 Subject: [PATCH 05/10] Extract novel restore queries --- src/database/queries/NovelQueries.ts | 332 +-------------- src/database/queries/NovelRestoreQueries.ts | 337 +++++++++++++++ .../queries/__tests__/NovelQueries.test.ts | 374 ---------------- .../__tests__/NovelRestoreQueries.test.ts | 398 ++++++++++++++++++ src/database/queries/__tests__/index.ts | 8 +- src/services/backup/__tests__/utils.test.ts | 9 +- src/services/backup/utils.ts | 4 +- 7 files changed, 752 insertions(+), 710 deletions(-) create mode 100644 src/database/queries/NovelRestoreQueries.ts create mode 100644 src/database/queries/__tests__/NovelRestoreQueries.test.ts diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index c3f5a03a90..59c6cfc212 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -6,27 +6,16 @@ import { insertChapters } from './ChapterQueries'; import { showToast } from '@utils/showToast'; import { getString } from '@i18n/translations'; -import { - BackupNovel, - DBNovelInfo, - NovelInfo, - type RestoredNovelMapping, -} from '../types'; +import { DBNovelInfo, NovelInfo } from '../types'; import { SourceNovel } from '@plugins/types'; import { NOVEL_STORAGE } from '@utils/Storages'; import { downloadFile } from '@plugins/helpers/fetch'; import { getPlugin } from '@plugins/pluginManager'; import { dbManager } from '@database/db'; -import { - createNovelTriggerQueryDelete, - createNovelTriggerQueryInsert, - createNovelTriggerQueryUpdate, -} from '@database/queryStrings/triggers'; import { novelSchema, novelCategorySchema, categorySchema, - chapterSchema, } from '@database/schema'; import type { TransactionParameter } from '@database/manager/manager.d'; import { getLibraryDefaultCategoryId } from '@hooks/persisted/useSettings'; @@ -80,7 +69,7 @@ const getCategoriesForNewNovel = async ( return getBuiltInDefaultCategory(tx); }; -const getCategoryForNewNovel = async (tx: TransactionParameter) => +export const getCategoryForNewNovel = async (tx: TransactionParameter) => (await getCategoriesForNewNovel(tx))[0]; /** @@ -292,69 +281,6 @@ export const deleteCachedNovels = async () => { showToast(getString('advancedSettingsScreen.cachedNovelsDeletedToast')); }; -/** - * Restore a novel from backup using Drizzle ORM. - */ -export const restoreLibrary = async (novel: NovelInfo) => { - const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { - throw e; - }); - - const novelId = await dbManager.write(async tx => { - const row = await tx - .insert(novelSchema) - .values({ - path: sourceNovel.path, - name: novel.name, - pluginId: novel.pluginId, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - name: novel.name, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }, - }) - .returning() - .get(); - - if (row) { - const defaultCategory = await getCategoryForNewNovel(tx); - - if (defaultCategory) { - await tx - .insert(novelCategorySchema) - .values({ - novelId: row.id, - categoryId: defaultCategory.id, - }) - .onConflictDoNothing() - .run(); - } - } - return row?.id; - }); - - if (novelId && sourceNovel.chapters) { - await insertChapters(novelId, sourceNovel.chapters); - } -}; - export const updateNovelInfo = async (info: NovelInfo) => { await dbManager.write(async tx => { await tx @@ -479,257 +405,3 @@ export const updateNovelCategories = async ( } }); }; - -const disableNovelStatsTriggers = async (tx: TransactionParameter) => { - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats')); - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_update')); - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); -}; - -const restoreNovelStatsTriggers = async (tx: TransactionParameter) => { - await tx.run(sql.raw(createNovelTriggerQueryInsert)); - await tx.run(sql.raw(createNovelTriggerQueryDelete)); - await tx.run(sql.raw(createNovelTriggerQueryUpdate)); -}; - -const restoreNovelRecord = async ( - tx: TransactionParameter, - novel: Omit, -) => { - const resetState = { - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - lastReadAt: null, - lastUpdatedAt: null, - }; - - // The order here makes resetState authoritative. - const values = { - ...resetState, - ...novel, - }; - - const localCover = values.cover?.startsWith(`file://${NOVEL_STORAGE}/`) - ? values.cover - : undefined; - - const cacheSuffix = localCover?.match(/[?#].*$/)?.[0] ?? ''; - - const restoredNovel = await tx - .insert(novelSchema) - .values(values) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: values, - }) - .returning({ id: novelSchema.id }) - .get(); - - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); - - if (localCover !== undefined) { - await tx - .update(novelSchema) - .set({ - cover: - `file://${NOVEL_STORAGE}/` + - `${values.pluginId}/${restoredNovel.id}/cover.png` + - cacheSuffix, - }) - .where(eq(novelSchema.id, restoredNovel.id)) - .run(); - } - - return restoredNovel; -}; - -const restoreChapterValues = ( - chapters: BackupNovel['chapters'], - novelId: number, -) => - chapters.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ - ...chapter, - novelId, - })); - -const refreshRestoredNovelStats = async ( - tx: TransactionParameter, - novelIds: number[], -) => { - if (novelIds.length === 0) { - return; - } - - const stats = await tx - .select({ - novelId: chapterSchema.novelId, - totalChapters: sql`COUNT(*)`, - chaptersDownloaded: sql`SUM(CASE WHEN ${chapterSchema.isDownloaded} = 1 THEN 1 ELSE 0 END)`, - chaptersUnread: sql`SUM(CASE WHEN ${chapterSchema.unread} = 1 THEN 1 ELSE 0 END)`, - lastReadAt: sql`MAX(${chapterSchema.readTime})`, - lastUpdatedAt: sql`( - SELECT updatedChapter.updatedTime - FROM Chapter AS updatedChapter - WHERE updatedChapter.novelId = Chapter.novelId - AND updatedChapter.updatedTime IS NOT NULL - ORDER BY julianday(updatedChapter.updatedTime) DESC - LIMIT 1 - )`, - }) - .from(chapterSchema) - .where(inArray(chapterSchema.novelId, novelIds)) - .groupBy(chapterSchema.novelId) - .all(); - - if (stats.length === 0) { - return; - } - - const ids = sql.join( - novelIds.map(id => sql`${id}`), - sql`, `, - ); - const cases = ( - field: T, - fallback: unknown, - ) => - sql`CASE ${novelSchema.id} ${sql.join( - stats.map( - (stat: (typeof stats)[number]) => - sql`WHEN ${stat.novelId} THEN ${stat[field]}`, - ), - sql` `, - )} ELSE ${fallback} END`; - - await tx.run(sql` - UPDATE Novel - SET totalChapters = ${cases('totalChapters', novelSchema.totalChapters)}, - chaptersDownloaded = ${cases( - 'chaptersDownloaded', - novelSchema.chaptersDownloaded, - )}, - chaptersUnread = ${cases('chaptersUnread', novelSchema.chaptersUnread)}, - lastReadAt = ${cases('lastReadAt', novelSchema.lastReadAt)}, - lastUpdatedAt = ${cases('lastUpdatedAt', novelSchema.lastUpdatedAt)} - WHERE id IN (${ids}) - `); -}; - -const RESTORE_CHAPTER_BATCH_SIZE = 500; - -type RestoreNovelOptions = { - includeChapterMappings?: boolean; -}; - -const restoreNovelsAndChaptersInTransaction = async ( - tx: TransactionParameter, - backupNovels: BackupNovel[], - includeChapterMappings: boolean, -): Promise => { - await disableNovelStatsTriggers(tx); - - try { - const restoredNovels: { - backupNovelId: number; - chapters: BackupNovel['chapters']; - novel: Omit; - restoredNovelId: number; - }[] = []; - for (const backupNovel of backupNovels) { - const { chapters, id: backupNovelId, ...novel } = backupNovel; - const restoredNovel = await restoreNovelRecord(tx, novel); - restoredNovels.push({ - backupNovelId, - chapters, - novel, - restoredNovelId: restoredNovel.id, - }); - } - - const mappings: RestoredNovelMapping[] = []; - for (const restoredNovel of restoredNovels) { - const chapterMappings: RestoredNovelMapping['chapters'] = []; - for ( - let i = 0; - i < restoredNovel.chapters.length; - i += RESTORE_CHAPTER_BATCH_SIZE - ) { - const batch = restoredNovel.chapters.slice( - i, - i + RESTORE_CHAPTER_BATCH_SIZE, - ); - if (includeChapterMappings) { - const restoredChapters = (await tx - .insert(chapterSchema) - .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) - .returning({ id: chapterSchema.id, path: chapterSchema.path }) - .all()) as { id: number; path: string }[]; - const restoredIdsByPath = new Map( - restoredChapters.map(chapter => [chapter.path, chapter.id]), - ); - for (const chapter of batch) { - const restoredChapterId = restoredIdsByPath.get(chapter.path); - if (restoredChapterId !== undefined) { - chapterMappings.push({ - backupChapterId: chapter.id, - restoredChapterId, - }); - } - } - } else { - await tx - .insert(chapterSchema) - .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) - .run(); - } - } - mappings.push({ - pluginId: restoredNovel.novel.pluginId, - backupNovelId: restoredNovel.backupNovelId, - restoredNovelId: restoredNovel.restoredNovelId, - chapters: chapterMappings, - }); - } - // await refreshRestoredNovelStats( - // tx, - // restoredNovels.map(restoredNovel => restoredNovel.restoredNovelId), - // ); - return mappings; - } finally { - await restoreNovelStatsTriggers(tx); - } -}; - -export const _restoreNovelsAndChapters = async ( - backupNovels: BackupNovel[], - options: RestoreNovelOptions = {}, -): Promise => { - if (backupNovels.length === 0) { - return []; - } - return dbManager.write(tx => - restoreNovelsAndChaptersInTransaction( - tx, - backupNovels, - options.includeChapterMappings ?? true, - ), - ); -}; - -/** - * Restores novel and chapters from a backup object. - */ -export const _restoreNovelAndChapters = async ( - backupNovel: BackupNovel, - options: RestoreNovelOptions = {}, -): Promise => { - const [mapping] = await _restoreNovelsAndChapters([backupNovel], options); - if (!mapping) { - throw new Error('Failed to restore novel'); - } - return mapping; -}; diff --git a/src/database/queries/NovelRestoreQueries.ts b/src/database/queries/NovelRestoreQueries.ts new file mode 100644 index 0000000000..ff4aeb0b7f --- /dev/null +++ b/src/database/queries/NovelRestoreQueries.ts @@ -0,0 +1,337 @@ +import { eq, inArray, sql } from 'drizzle-orm'; + +import { fetchNovel } from '@services/plugin/fetch'; +import { insertChapters } from './ChapterQueries'; +import { getCategoryForNewNovel } from './NovelQueries'; + +import { dbManager } from '@database/db'; +import { + createNovelTriggerQueryDelete, + createNovelTriggerQueryInsert, + createNovelTriggerQueryUpdate, +} from '@database/queryStrings/triggers'; +import { + chapterSchema, + novelCategorySchema, + novelSchema, +} from '@database/schema'; +import type { TransactionParameter } from '@database/manager/manager.d'; +import { NOVEL_STORAGE } from '@utils/Storages'; +import type { BackupNovel, NovelInfo, RestoredNovelMapping } from '../types'; + +/** + * Restore a novel from backup using Drizzle ORM. + */ +export const restoreLibrary = async (novel: NovelInfo) => { + const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { + throw e; + }); + + const novelId = await dbManager.write(async tx => { + const row = await tx + .insert(novelSchema) + .values({ + path: sourceNovel.path, + name: novel.name, + pluginId: novel.pluginId, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }) + .onConflictDoUpdate({ + target: [novelSchema.path, novelSchema.pluginId], + set: { + name: novel.name, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }, + }) + .returning() + .get(); + + if (row) { + const defaultCategory = await getCategoryForNewNovel(tx); + + if (defaultCategory) { + await tx + .insert(novelCategorySchema) + .values({ + novelId: row.id, + categoryId: defaultCategory.id, + }) + .onConflictDoNothing() + .run(); + } + } + return row?.id; + }); + + if (novelId && sourceNovel.chapters) { + await insertChapters(novelId, sourceNovel.chapters); + } +}; + +const disableNovelStatsTriggers = async (tx: TransactionParameter) => { + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats')); + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_update')); + await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); +}; + +const restoreNovelStatsTriggers = async (tx: TransactionParameter) => { + await tx.run(sql.raw(createNovelTriggerQueryInsert)); + await tx.run(sql.raw(createNovelTriggerQueryDelete)); + await tx.run(sql.raw(createNovelTriggerQueryUpdate)); +}; + +const restoreNovelRecord = async ( + tx: TransactionParameter, + novel: Omit, +) => { + const resetState = { + totalChapters: 0, + chaptersDownloaded: 0, + chaptersUnread: 0, + lastReadAt: null, + lastUpdatedAt: null, + }; + + // The order here makes resetState authoritative. + const values = { + ...resetState, + ...novel, + }; + + const localCover = values.cover?.startsWith(`file://${NOVEL_STORAGE}/`) + ? values.cover + : undefined; + + const cacheSuffix = localCover?.match(/[?#].*$/)?.[0] ?? ''; + + const restoredNovel = await tx + .insert(novelSchema) + .values(values) + .onConflictDoUpdate({ + target: [novelSchema.path, novelSchema.pluginId], + set: values, + }) + .returning({ id: novelSchema.id }) + .get(); + + await tx + .delete(chapterSchema) + .where(eq(chapterSchema.novelId, restoredNovel.id)) + .run(); + + if (localCover !== undefined) { + await tx + .update(novelSchema) + .set({ + cover: + `file://${NOVEL_STORAGE}/` + + `${values.pluginId}/${restoredNovel.id}/cover.png` + + cacheSuffix, + }) + .where(eq(novelSchema.id, restoredNovel.id)) + .run(); + } + + return restoredNovel; +}; + +const restoreChapterValues = ( + chapters: BackupNovel['chapters'], + novelId: number, +) => + chapters.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ + ...chapter, + novelId, + })); + +const refreshRestoredNovelStats = async ( + tx: TransactionParameter, + novelIds: number[], +) => { + if (novelIds.length === 0) { + return; + } + + const stats = await tx + .select({ + novelId: chapterSchema.novelId, + totalChapters: sql`COUNT(*)`, + chaptersDownloaded: sql`SUM(CASE WHEN ${chapterSchema.isDownloaded} = 1 THEN 1 ELSE 0 END)`, + chaptersUnread: sql`SUM(CASE WHEN ${chapterSchema.unread} = 1 THEN 1 ELSE 0 END)`, + lastReadAt: sql`MAX(${chapterSchema.readTime})`, + lastUpdatedAt: sql`( + SELECT updatedChapter.updatedTime + FROM Chapter AS updatedChapter + WHERE updatedChapter.novelId = Chapter.novelId + AND updatedChapter.updatedTime IS NOT NULL + ORDER BY julianday(updatedChapter.updatedTime) DESC + LIMIT 1 + )`, + }) + .from(chapterSchema) + .where(inArray(chapterSchema.novelId, novelIds)) + .groupBy(chapterSchema.novelId) + .all(); + + if (stats.length === 0) { + return; + } + + const ids = sql.join( + novelIds.map(id => sql`${id}`), + sql`, `, + ); + const cases = ( + field: T, + fallback: unknown, + ) => + sql`CASE ${novelSchema.id} ${sql.join( + stats.map( + (stat: (typeof stats)[number]) => + sql`WHEN ${stat.novelId} THEN ${stat[field]}`, + ), + sql` `, + )} ELSE ${fallback} END`; + + await tx.run(sql` + UPDATE Novel + SET totalChapters = ${cases('totalChapters', novelSchema.totalChapters)}, + chaptersDownloaded = ${cases( + 'chaptersDownloaded', + novelSchema.chaptersDownloaded, + )}, + chaptersUnread = ${cases('chaptersUnread', novelSchema.chaptersUnread)}, + lastReadAt = ${cases('lastReadAt', novelSchema.lastReadAt)}, + lastUpdatedAt = ${cases('lastUpdatedAt', novelSchema.lastUpdatedAt)} + WHERE id IN (${ids}) + `); +}; + +const RESTORE_CHAPTER_BATCH_SIZE = 500; + +type RestoreNovelOptions = { + includeChapterMappings?: boolean; +}; + +const restoreNovelsAndChaptersInTransaction = async ( + tx: TransactionParameter, + backupNovels: BackupNovel[], + includeChapterMappings: boolean, +): Promise => { + await disableNovelStatsTriggers(tx); + + try { + const restoredNovels: { + backupNovelId: number; + chapters: BackupNovel['chapters']; + novel: Omit; + restoredNovelId: number; + }[] = []; + for (const backupNovel of backupNovels) { + const { chapters, id: backupNovelId, ...novel } = backupNovel; + const restoredNovel = await restoreNovelRecord(tx, novel); + restoredNovels.push({ + backupNovelId, + chapters, + novel, + restoredNovelId: restoredNovel.id, + }); + } + + const mappings: RestoredNovelMapping[] = []; + for (const restoredNovel of restoredNovels) { + const chapterMappings: RestoredNovelMapping['chapters'] = []; + for ( + let i = 0; + i < restoredNovel.chapters.length; + i += RESTORE_CHAPTER_BATCH_SIZE + ) { + const batch = restoredNovel.chapters.slice( + i, + i + RESTORE_CHAPTER_BATCH_SIZE, + ); + if (includeChapterMappings) { + const restoredChapters = (await tx + .insert(chapterSchema) + .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) + .returning({ id: chapterSchema.id, path: chapterSchema.path }) + .all()) as { id: number; path: string }[]; + const restoredIdsByPath = new Map( + restoredChapters.map(chapter => [chapter.path, chapter.id]), + ); + for (const chapter of batch) { + const restoredChapterId = restoredIdsByPath.get(chapter.path); + if (restoredChapterId !== undefined) { + chapterMappings.push({ + backupChapterId: chapter.id, + restoredChapterId, + }); + } + } + } else { + await tx + .insert(chapterSchema) + .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) + .run(); + } + } + mappings.push({ + pluginId: restoredNovel.novel.pluginId, + backupNovelId: restoredNovel.backupNovelId, + restoredNovelId: restoredNovel.restoredNovelId, + chapters: chapterMappings, + }); + } + await refreshRestoredNovelStats( + tx, + restoredNovels.map(restoredNovel => restoredNovel.restoredNovelId), + ); + return mappings; + } finally { + await restoreNovelStatsTriggers(tx); + } +}; + +export const _restoreNovelsAndChapters = async ( + backupNovels: BackupNovel[], + options: RestoreNovelOptions = {}, +): Promise => { + if (backupNovels.length === 0) { + return []; + } + return dbManager.write(tx => + restoreNovelsAndChaptersInTransaction( + tx, + backupNovels, + options.includeChapterMappings ?? true, + ), + ); +}; + +/** + * Restores novel and chapters from a backup object. + */ +export const _restoreNovelAndChapters = async ( + backupNovel: BackupNovel, + options: RestoreNovelOptions = {}, +): Promise => { + const [mapping] = await _restoreNovelsAndChapters([backupNovel], options); + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; +}; diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index a6e6b375ed..7185332849 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -8,14 +8,12 @@ import './mockDb'; import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; import { insertTestNovel, - insertTestChapter, insertTestNovelCategory, insertTestCategory, clearAllTables, } from './testData'; import { categorySchema, - chapterSchema, novelCategorySchema, novelSchema, } from '@database/schema'; @@ -31,13 +29,10 @@ import { removeNovelsFromLibrary, getCachedNovels, deleteCachedNovels, - restoreLibrary, updateNovelInfo, pickCustomNovelCover, updateNovelCategoryById, updateNovelCategories, - _restoreNovelAndChapters, - _restoreNovelsAndChapters, } from '../NovelQueries'; const mockGetLibraryDefaultCategoryId = jest.fn(); @@ -423,375 +418,6 @@ describe('NovelQueries', () => { }); }); - describe('restoreLibrary', () => { - it('should restore novel from backup', async () => { - const novel = { - id: 999, - path: '/test/novel', - pluginId: 'test-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 1, - chaptersDownloaded: 0, - chaptersUnread: 0, - totalChapters: 0, - lastReadAt: null, - lastUpdatedAt: null, - }; - - // Mock fetchNovel to return a valid SourceNovel - const { fetchNovel } = require('@services/plugin/fetch'); - jest.mocked(fetchNovel).mockResolvedValueOnce({ - id: undefined, - path: '/test/novel', - name: 'Restored Novel', - chapters: [], - }); - - await restoreLibrary(novel); - - const restored = await getNovelByPath('/test/novel', 'test-plugin'); - expect(restored?.name).toBe('Restored Novel'); - }); - }); - - describe('_restoreNovelAndChapters', () => { - it('does not replace an unrelated novel when backup IDs collide', async () => { - const testDb = getTestDb(); - await insertTestNovel(testDb, { - path: '/existing/novel', - pluginId: 'existing-plugin', - name: 'Existing Novel', - inLibrary: true, - }); - - const mapping = await _restoreNovelAndChapters({ - id: 1, - path: '/restored/novel', - pluginId: 'restored-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 10, - novelId: 1, - path: '/restored/chapter-1', - name: 'Chapter 1', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: null, - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }); - - expect(mapping.restoredNovelId).not.toBe(1); - expect( - (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, - ).toBe('Existing Novel'); - expect( - (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, - ).toBe(mapping.restoredNovelId); - const restoredChapters = await testDb.drizzleDb - .select() - .from(chapterSchema) - .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) - .all(); - expect(mapping.chapters).toEqual([ - { - backupChapterId: 10, - restoredChapterId: restoredChapters[0].id, - }, - ]); - }); - it('restores multiple novels in one batch', async () => { - const mappings = await _restoreNovelsAndChapters( - [ - { - id: 100, - path: '/bulk/one', - pluginId: 'bulk-plugin', - name: 'Bulk One', - chapters: [], - }, - { - id: 101, - path: '/bulk/two', - pluginId: 'bulk-plugin', - name: 'Bulk Two', - chapters: [], - }, - ], - { includeChapterMappings: false }, - ); - - expect(mappings).toHaveLength(2); - expect(mappings.map(mapping => mapping.pluginId)).toEqual([ - 'bulk-plugin', - 'bulk-plugin', - ]); - expect(await getNovelByPath('/bulk/one', 'bulk-plugin')).toEqual( - expect.objectContaining({ name: 'Bulk One' }), - ); - expect(await getNovelByPath('/bulk/two', 'bulk-plugin')).toEqual( - expect.objectContaining({ name: 'Bulk Two' }), - ); - }); - it('preserves mappings and aggregate stats for multiple novels', async () => { - const mappings = await _restoreNovelsAndChapters( - [ - { - id: 200, - path: '/bulk/mapped-one', - pluginId: 'bulk-plugin', - name: 'Mapped One', - chapters: [ - { - id: 2001, - novelId: 200, - path: '/bulk/chapter-one', - name: 'Chapter One', - releaseTime: null, - readTime: '2024-01-01T00:00:00.000Z', - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: '2024-01-01T00:00:00.000Z', - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - { - id: 2002, - novelId: 200, - path: '/bulk/chapter-two', - name: 'Chapter Two', - releaseTime: null, - readTime: '2025-01-01T00:00:00.000Z', - bookmark: false, - unread: false, - isDownloaded: false, - updatedTime: '2023-01-01T00:00:00.000Z', - chapterNumber: 2, - page: '1', - progress: null, - position: 1, - scanlator: null, - timeSpent: 0, - }, - ], - }, - { - id: 201, - path: '/bulk/mapped-two', - pluginId: 'bulk-plugin', - name: 'Mapped Two', - chapters: [ - { - id: 2011, - novelId: 201, - path: '/bulk/chapter-three', - name: 'Chapter Three', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: '2026-01-01T00:00:00.000Z', - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }, - ], - { includeChapterMappings: true }, - ); - - expect(mappings).toHaveLength(2); - expect( - mappings.map(mapping => - mapping.chapters.map(chapter => chapter.backupChapterId), - ), - ).toEqual([[2001, 2002], [2011]]); - expect(mappings[0].chapters).toHaveLength(2); - expect(mappings[1].chapters).toHaveLength(1); - expect( - new Set( - mappings - .flatMap(mapping => mapping.chapters) - .map(chapter => chapter.restoredChapterId), - ).size, - ).toBe(3); - expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( - expect.objectContaining({ - totalChapters: 2, - chaptersDownloaded: 1, - chaptersUnread: 1, - lastReadAt: '2025-01-01T00:00:00.000Z', - lastUpdatedAt: '2024-01-01T00:00:00.000Z', - }), - ); - expect(await getNovelByPath('/bulk/mapped-two', 'bulk-plugin')).toEqual( - expect.objectContaining({ - totalChapters: 1, - chaptersDownloaded: 1, - chaptersUnread: 1, - lastReadAt: null, - lastUpdatedAt: '2026-01-01T00:00:00.000Z', - }), - ); - }); - it('restores chapters without allocating ID mappings when requested', async () => { - const mapping = await _restoreNovelAndChapters( - { - id: 2, - path: '/restored/without-mappings', - pluginId: 'restored-plugin', - name: 'Restored Without Mappings', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 20, - novelId: 2, - path: '/restored/chapter-2', - name: 'Chapter 2', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: false, - updatedTime: null, - chapterNumber: 2, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }, - { includeChapterMappings: false }, - ); - - expect(mapping.chapters).toEqual([]); - const restoredChapters = await getTestDb() - .drizzleDb.select() - .from(chapterSchema) - .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) - .all(); - expect(restoredChapters).toHaveLength(1); - expect(restoredChapters[0].path).toBe('/restored/chapter-2'); - }); - it('rebuilds aggregate timestamps after replacing existing chapters', async () => { - const testDb = getTestDb(); - const novelId = await insertTestNovel(testDb, { - path: '/restore/stats', - pluginId: 'stats-plugin', - }); - await insertTestChapter(testDb, novelId, { - path: '/restore/old-chapter', - readTime: '2020-01-01T00:00:00.000Z', - updatedTime: '2020-01-01T00:00:00.000Z', - }); - await testDb.drizzleDb - .update(novelSchema) - .set({ - lastReadAt: 'stale-read-time', - lastUpdatedAt: 'stale-update-time', - }) - .where(eq(novelSchema.id, novelId)) - .run(); - - await _restoreNovelAndChapters( - { - id: 3, - path: '/restore/stats', - pluginId: 'stats-plugin', - name: 'Restored Stats', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 30, - novelId: 3, - path: '/restore/new-chapter', - name: 'New Chapter', - releaseTime: null, - readTime: '2024-01-01T00:00:00.000Z', - bookmark: false, - unread: false, - isDownloaded: true, - updatedTime: '2025-01-01T00:00:00.000Z', - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }, - { includeChapterMappings: false }, - ); - - const restored = await getNovelByPath('/restore/stats', 'stats-plugin'); - expect(restored).toEqual( - expect.objectContaining({ - totalChapters: 1, - chaptersDownloaded: 1, - chaptersUnread: 0, - lastReadAt: '2024-01-01T00:00:00.000Z', - lastUpdatedAt: '2025-01-01T00:00:00.000Z', - }), - ); - }); - }); - describe('updateNovelInfo', () => { it('should update novel information', async () => { const testDb = getTestDb(); diff --git a/src/database/queries/__tests__/NovelRestoreQueries.test.ts b/src/database/queries/__tests__/NovelRestoreQueries.test.ts new file mode 100644 index 0000000000..7630e21123 --- /dev/null +++ b/src/database/queries/__tests__/NovelRestoreQueries.test.ts @@ -0,0 +1,398 @@ +import './mockDb'; +import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; +import { insertTestNovel, insertTestChapter, clearAllTables } from './testData'; +import { chapterSchema, novelSchema } from '@database/schema'; +import { eq } from 'drizzle-orm'; + +import { getNovelByPath } from '../NovelQueries'; +import { + restoreLibrary, + _restoreNovelAndChapters, + _restoreNovelsAndChapters, +} from '../NovelRestoreQueries'; + +const mockGetLibraryDefaultCategoryId = jest.fn(); + +jest.mock('@hooks/persisted/useSettings', () => ({ + getLibraryDefaultCategoryId: () => mockGetLibraryDefaultCategoryId(), +})); + +describe('NovelRestoreQueries', () => { + beforeEach(() => { + const testDb = setupTestDatabase(); + clearAllTables(testDb); + mockGetLibraryDefaultCategoryId.mockReturnValue(undefined); + }); + + afterAll(() => { + teardownTestDatabase(); + }); + describe('restoreLibrary', () => { + it('should restore novel from backup', async () => { + const novel = { + id: 999, + path: '/test/novel', + pluginId: 'test-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 1, + chaptersDownloaded: 0, + chaptersUnread: 0, + totalChapters: 0, + lastReadAt: null, + lastUpdatedAt: null, + }; + + // Mock fetchNovel to return a valid SourceNovel + const { fetchNovel } = require('@services/plugin/fetch'); + jest.mocked(fetchNovel).mockResolvedValueOnce({ + id: undefined, + path: '/test/novel', + name: 'Restored Novel', + chapters: [], + }); + + await restoreLibrary(novel); + + const restored = await getNovelByPath('/test/novel', 'test-plugin'); + expect(restored?.name).toBe('Restored Novel'); + }); + }); + + describe('_restoreNovelAndChapters', () => { + it('does not replace an unrelated novel when backup IDs collide', async () => { + const testDb = getTestDb(); + await insertTestNovel(testDb, { + path: '/existing/novel', + pluginId: 'existing-plugin', + name: 'Existing Novel', + inLibrary: true, + }); + + const mapping = await _restoreNovelAndChapters({ + id: 1, + path: '/restored/novel', + pluginId: 'restored-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 10, + novelId: 1, + path: '/restored/chapter-1', + name: 'Chapter 1', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }); + + expect(mapping.restoredNovelId).not.toBe(1); + expect( + (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, + ).toBe('Existing Novel'); + expect( + (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, + ).toBe(mapping.restoredNovelId); + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(mapping.chapters).toEqual([ + { + backupChapterId: 10, + restoredChapterId: restoredChapters[0].id, + }, + ]); + }); + it('restores multiple novels in one batch', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 100, + path: '/bulk/one', + pluginId: 'bulk-plugin', + name: 'Bulk One', + chapters: [], + }, + { + id: 101, + path: '/bulk/two', + pluginId: 'bulk-plugin', + name: 'Bulk Two', + chapters: [], + }, + ], + { includeChapterMappings: false }, + ); + + expect(mappings).toHaveLength(2); + expect(mappings.map(mapping => mapping.pluginId)).toEqual([ + 'bulk-plugin', + 'bulk-plugin', + ]); + expect(await getNovelByPath('/bulk/one', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk One' }), + ); + expect(await getNovelByPath('/bulk/two', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk Two' }), + ); + }); + it('preserves mappings and aggregate stats for multiple novels', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 200, + path: '/bulk/mapped-one', + pluginId: 'bulk-plugin', + name: 'Mapped One', + chapters: [ + { + id: 2001, + novelId: 200, + path: '/bulk/chapter-one', + name: 'Chapter One', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + { + id: 2002, + novelId: 200, + path: '/bulk/chapter-two', + name: 'Chapter Two', + releaseTime: null, + readTime: '2025-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: false, + updatedTime: '2023-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + progress: null, + position: 1, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 201, + path: '/bulk/mapped-two', + pluginId: 'bulk-plugin', + name: 'Mapped Two', + chapters: [ + { + id: 2011, + novelId: 201, + path: '/bulk/chapter-three', + name: 'Chapter Three', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ], + { includeChapterMappings: true }, + ); + + expect(mappings).toHaveLength(2); + expect( + mappings.map(mapping => + mapping.chapters.map(chapter => chapter.backupChapterId), + ), + ).toEqual([[2001, 2002], [2011]]); + expect(mappings[0].chapters).toHaveLength(2); + expect(mappings[1].chapters).toHaveLength(1); + expect( + new Set( + mappings + .flatMap(mapping => mapping.chapters) + .map(chapter => chapter.restoredChapterId), + ).size, + ).toBe(3); + expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-01-01T00:00:00.000Z', + }), + ); + expect(await getNovelByPath('/bulk/mapped-two', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: null, + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + }); + it('restores chapters without allocating ID mappings when requested', async () => { + const mapping = await _restoreNovelAndChapters( + { + id: 2, + path: '/restored/without-mappings', + pluginId: 'restored-plugin', + name: 'Restored Without Mappings', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 20, + novelId: 2, + path: '/restored/chapter-2', + name: 'Chapter 2', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 2, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + expect(mapping.chapters).toEqual([]); + const restoredChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-2'); + }); + it('rebuilds aggregate timestamps after replacing existing chapters', async () => { + const testDb = getTestDb(); + const novelId = await insertTestNovel(testDb, { + path: '/restore/stats', + pluginId: 'stats-plugin', + }); + await insertTestChapter(testDb, novelId, { + path: '/restore/old-chapter', + readTime: '2020-01-01T00:00:00.000Z', + updatedTime: '2020-01-01T00:00:00.000Z', + }); + await testDb.drizzleDb + .update(novelSchema) + .set({ + lastReadAt: 'stale-read-time', + lastUpdatedAt: 'stale-update-time', + }) + .where(eq(novelSchema.id, novelId)) + .run(); + + await _restoreNovelAndChapters( + { + id: 3, + path: '/restore/stats', + pluginId: 'stats-plugin', + name: 'Restored Stats', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 30, + novelId: 3, + path: '/restore/new-chapter', + name: 'New Chapter', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: true, + updatedTime: '2025-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + const restored = await getNovelByPath('/restore/stats', 'stats-plugin'); + expect(restored).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 0, + lastReadAt: '2024-01-01T00:00:00.000Z', + lastUpdatedAt: '2025-01-01T00:00:00.000Z', + }), + ); + }); + }); +}); diff --git a/src/database/queries/__tests__/index.ts b/src/database/queries/__tests__/index.ts index fe7903cab7..2b9fec59f9 100644 --- a/src/database/queries/__tests__/index.ts +++ b/src/database/queries/__tests__/index.ts @@ -122,12 +122,18 @@ export const testModules = { 'removeNovelsFromLibrary', 'getCachedNovels', 'deleteCachedNovels', - 'restoreLibrary', 'updateNovelInfo', 'pickCustomNovelCover', 'updateNovelCategoryById', 'updateNovelCategories', + ], + }, + NovelRestoreQueries: { + description: 'Novel restore operations', + functions: [ + 'restoreLibrary', '_restoreNovelAndChapters', + '_restoreNovelsAndChapters', ], }, RepositoryQueries: { diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 83a0ad5c65..38ba10e4ca 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -1,8 +1,8 @@ import { _restoreNovelAndChapters, _restoreNovelsAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; +} from '@database/queries/NovelRestoreQueries'; +import { getAllNovels } from '@database/queries/NovelQueries'; import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; import { _restoreCategory, @@ -21,9 +21,12 @@ import type { import type { BackupOptions } from '../options'; jest.mock('@database/queries/NovelQueries', () => ({ + getAllNovels: jest.fn(), +})); + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ _restoreNovelAndChapters: jest.fn(), _restoreNovelsAndChapters: jest.fn(), - getAllNovels: jest.fn(), })); jest.mock('@database/queries/ChapterQueries', () => ({ diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index 4acf7e658e..b7885cc578 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -3,11 +3,11 @@ import { OLD_TRACKED_NOVEL_PREFIX } from '@hooks/persisted/migrations/trackerMig import { LAST_UPDATE_TIME } from '@hooks/persisted/useUpdates'; import { MMKVStorage } from '@utils/mmkv/mmkv'; import { version } from '../../../package.json'; +import { getAllNovels } from '@database/queries/NovelQueries'; import { _restoreNovelAndChapters, _restoreNovelsAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; +} from '@database/queries/NovelRestoreQueries'; import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; import { _restoreCategory, From 9ea934f74e335499400f179997e0c7cb38dd964c Mon Sep 17 00:00:00 2001 From: CD-Z Date: Wed, 23 Sep 2026 12:09:16 +0200 Subject: [PATCH 06/10] Improve backup restore handling --- .../migration.sql | 39 + .../snapshot.json | 793 ++++++++++++++++++ drizzle/migrations.js | 2 + src/database/__tests__/db.test.ts | 1 + .../manager/__tests__/manager.test.ts | 45 + src/database/manager/manager.d.ts | 16 +- src/database/manager/manager.ts | 31 +- src/database/queries/NovelRestoreQueries.ts | 582 ++++++++----- .../__tests__/NovelRestoreQueries.test.ts | 379 ++++++--- src/database/queries/__tests__/testData.ts | 2 +- src/database/queries/__tests__/testDb.ts | 13 +- src/database/schema/chapter.ts | 5 +- src/database/schema/index.ts | 7 + src/database/schema/restoreChapterMapping.ts | 40 + src/database/types/index.ts | 4 - src/i18n/languages/en/strings.json | 2 + src/i18n/types/index.ts | 2 + .../backup/__tests__/fileSections.test.ts | 126 ++- src/services/backup/__tests__/local.test.ts | 63 ++ .../backup/__tests__/restoreResult.test.ts | 1 + src/services/backup/__tests__/utils.test.ts | 413 ++++++++- src/services/backup/drive/index.ts | 159 ++-- src/services/backup/fileSections.ts | 58 +- src/services/backup/local/index.ts | 14 +- src/services/backup/novelPayload.ts | 181 +++- src/services/backup/restoreResult.ts | 1 + src/services/backup/selfhost/index.ts | 145 ++-- src/services/backup/utils.ts | 274 +++++- 28 files changed, 2814 insertions(+), 584 deletions(-) create mode 100644 drizzle/20260922120323_demonic_rattler/migration.sql create mode 100644 drizzle/20260922120323_demonic_rattler/snapshot.json create mode 100644 src/database/manager/__tests__/manager.test.ts create mode 100644 src/database/schema/restoreChapterMapping.ts diff --git a/drizzle/20260922120323_demonic_rattler/migration.sql b/drizzle/20260922120323_demonic_rattler/migration.sql new file mode 100644 index 0000000000..92aa90da11 --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/migration.sql @@ -0,0 +1,39 @@ +CREATE TABLE `RestoreChapterMapping` ( + `restoreRunId` text NOT NULL, + `backupNovelId` integer NOT NULL, + `backupChapterId` integer NOT NULL, + `restoredNovelId` integer NOT NULL, + `restoredChapterId` integer NOT NULL, + CONSTRAINT `fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk` FOREIGN KEY (`restoredNovelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk` FOREIGN KEY (`restoredChapterId`) REFERENCES `Chapter`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_Chapter` ( + `id` integer PRIMARY KEY AUTOINCREMENT, + `novelId` integer NOT NULL, + `path` text NOT NULL, + `name` text NOT NULL, + `releaseTime` text, + `bookmark` integer DEFAULT false, + `unread` integer DEFAULT true, + `readTime` text, + `isDownloaded` integer DEFAULT false, + `updatedTime` text, + `chapterNumber` real, + `page` text DEFAULT '1', + `position` integer DEFAULT 0, + `progress` integer, + `scanlator` text, + `timeSpent` integer DEFAULT 0, + CONSTRAINT `fk_Chapter_novelId_Novel_id_fk` FOREIGN KEY (`novelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +INSERT INTO `__new_Chapter`(`id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent`) SELECT `id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent` FROM `Chapter`;--> statement-breakpoint +DROP TABLE `Chapter`;--> statement-breakpoint +ALTER TABLE `__new_Chapter` RENAME TO `Chapter`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE UNIQUE INDEX `chapter_novel_path_unique` ON `Chapter` (`novelId`,`path`);--> statement-breakpoint +CREATE INDEX `chapterNovelIdIndex` ON `Chapter` (`novelId`,`position`,`page`,`id`);--> statement-breakpoint +CREATE UNIQUE INDEX `restore_chapter_mapping_unique` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`,`backupChapterId`);--> statement-breakpoint +CREATE INDEX `restore_chapter_mapping_novel_index` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`); \ No newline at end of file diff --git a/drizzle/20260922120323_demonic_rattler/snapshot.json b/drizzle/20260922120323_demonic_rattler/snapshot.json new file mode 100644 index 0000000000..fc040891a4 --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/snapshot.json @@ -0,0 +1,793 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "bcdf1b04-31ef-45ca-8a9a-8ecd417c3512", + "prevIds": [ + "2fada83f-d142-46d9-ba03-5ae1e232e3ce" + ], + "ddl": [ + { + "name": "Category", + "entityType": "tables" + }, + { + "name": "Chapter", + "entityType": "tables" + }, + { + "name": "NovelCategory", + "entityType": "tables" + }, + { + "name": "Novel", + "entityType": "tables" + }, + { + "name": "Repository", + "entityType": "tables" + }, + { + "name": "RestoreChapterMapping", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Category" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "sort", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "releaseTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "bookmark", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "unread", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "readTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isDownloaded", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "real", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "chapterNumber", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'1'", + "generated": null, + "name": "page", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "position", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "progress", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "scanlator", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "timeSpent", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "categoryId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "pluginId", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cover", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "author", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "artist", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'Unknown'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "genres", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "inLibrary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isLocal", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalPages", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersDownloaded", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersUnread", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalChapters", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastReadAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastUpdatedAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoreRunId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "novelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_Chapter_novelId_Novel_id_fk", + "entityType": "fks", + "table": "Chapter" + }, + { + "columns": [ + "restoredNovelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "restoredChapterId" + ], + "tableTo": "Chapter", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Category_pk", + "table": "Category", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Chapter_pk", + "table": "Chapter", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "NovelCategory_pk", + "table": "NovelCategory", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Novel_pk", + "table": "Novel", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Repository_pk", + "table": "Repository", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "name", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "category_name_unique", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "sort", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "category_sort_idx", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "chapter_novel_path_unique", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "position", + "isExpression": false + }, + { + "value": "page", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "chapterNovelIdIndex", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "categoryId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_category_unique", + "entityType": "indexes", + "table": "NovelCategory" + }, + { + "columns": [ + { + "value": "path", + "isExpression": false + }, + { + "value": "pluginId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_path_plugin_unique", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "pluginId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + }, + { + "value": "inLibrary", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "NovelIndex", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "url", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "repository_url_unique", + "entityType": "indexes", + "table": "Repository" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + }, + { + "value": "backupChapterId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_unique", + "entityType": "indexes", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_novel_index", + "entityType": "indexes", + "table": "RestoreChapterMapping" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/drizzle/migrations.js b/drizzle/migrations.js index beee4c381c..74735a0528 100644 --- a/drizzle/migrations.js +++ b/drizzle/migrations.js @@ -5,6 +5,7 @@ import m0001 from './20260612232322_normal_saracen/migration.sql'; import m0002 from './20260719143427_long_moondragon/migration.sql'; import m0003 from './20260727081855_calm_chimera/migration.sql'; import m0004 from './20260811071655_parched_human_torch/migration.sql'; +import m0005 from './20260922120323_demonic_rattler/migration.sql'; export default { migrations: { @@ -13,5 +14,6 @@ export default { '20260719143427_long_moondragon': m0002, '20260727081855_calm_chimera': m0003, '20260811071655_parched_human_torch': m0004, + '20260922120323_demonic_rattler': m0005, }, }; diff --git a/src/database/__tests__/db.test.ts b/src/database/__tests__/db.test.ts index 155f67f84d..71a99e61b5 100644 --- a/src/database/__tests__/db.test.ts +++ b/src/database/__tests__/db.test.ts @@ -809,6 +809,7 @@ describe('production migrations', () => { '20260719143427_long_moondragon', '20260727081855_calm_chimera', '20260811071655_parched_human_torch', + '20260922120323_demonic_rattler', ]); } finally { sqlite.close(); diff --git a/src/database/manager/__tests__/manager.test.ts b/src/database/manager/__tests__/manager.test.ts new file mode 100644 index 0000000000..97b34fb2a5 --- /dev/null +++ b/src/database/manager/__tests__/manager.test.ts @@ -0,0 +1,45 @@ +import { + createTestDb, + cleanupTestDb, +} from '@database/queries/__tests__/testDb'; + +describe('DbManager.executeBatch', () => { + const testDb = createTestDb(); + + afterAll(() => { + cleanupTestDb(testDb); + }); + + beforeEach(() => { + testDb.sqlite.executeSync('DROP TABLE IF EXISTS BatchTest'); + testDb.sqlite.executeSync( + 'CREATE TABLE BatchTest (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL)', + ); + }); + + it('executes heterogeneous and repeated-parameter commands atomically', async () => { + const result = await testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['first'], ['second']]], + ['UPDATE BatchTest SET value = ? WHERE value = ?', ['updated', 'first']], + ['DELETE FROM BatchTest WHERE value = ?', [['second']]], + ]); + + expect(result.rowsAffected).toBe(4); + expect( + testDb.sqlite.executeSync('SELECT value FROM BatchTest').rows, + ).toEqual([{ value: 'updated' }]); + }); + + it('rolls back every command when a later command fails', async () => { + await expect( + testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['rolled back']]], + ['INSERT INTO MissingBatchTable (value) VALUES (?)', [['failure']]], + ]), + ).rejects.toThrow(); + + expect(testDb.sqlite.executeSync('SELECT * FROM BatchTest').rows).toEqual( + [], + ); + }); +}); diff --git a/src/database/manager/manager.d.ts b/src/database/manager/manager.d.ts index 30997a11c7..95584c8c0d 100644 --- a/src/database/manager/manager.d.ts +++ b/src/database/manager/manager.d.ts @@ -1,9 +1,11 @@ // db-manager.types.ts -import type { DrizzleDb } from '@database/db'; import type { - Placeholder, -} from 'drizzle-orm'; -import { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; + BatchQueryResult, + SQLBatchTuple, +} from '@op-engineering/op-sqlite'; +import type { DrizzleDb } from '@database/db'; +import type { Placeholder } from 'drizzle-orm'; +import type { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; // Define the TransactionParameter type based on your DrizzleDb export type TransactionParameter = Parameters< @@ -14,6 +16,10 @@ export type TransactionParameter = Parameters< * This contract ensures consistent documentation and type safety across the application. */ export interface IDbManager { + /** + * Executes raw op-sqlite commands as one atomic transaction. + */ + executeBatch(commands: SQLBatchTuple[]): Promise; /** * Efficiently executes a Drizzle query for multiple data rows using * op-sqlite executeBatch under the hood. The batch owns its transaction; @@ -26,7 +32,7 @@ export interface IDbManager { tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ): Promise; + ): Promise; /** * Creates a subquery that defines a temporary named result set as a CTE. diff --git a/src/database/manager/manager.ts b/src/database/manager/manager.ts index cc5690d755..1354ceae7f 100644 --- a/src/database/manager/manager.ts +++ b/src/database/manager/manager.ts @@ -1,5 +1,9 @@ +import type { + BatchQueryResult, + SQLBatchTuple, + Scalar, +} from '@op-engineering/op-sqlite'; import type { drizzleDb } from '@database/db'; -import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; import { IDbManager } from './manager.d'; import { DbTaskQueue } from './queue'; import { Schema } from '../schema'; @@ -88,15 +92,28 @@ class DbManager implements IDbManager { .rows as Awaited>; } + public async executeBatch( + commands: SQLBatchTuple[], + ): Promise { + if (!commands.length) { + return { rowsAffected: 0 }; + } + + return await this.queue.enqueue({ + id: 'write', + run: async () => await this.db.$client.executeBatch(commands), + }); + } + public async batch>( data: T[], fn: ( tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ) { + ): Promise { if (!data.length) { - return; + return { rowsAffected: 0 }; } const ph = (arg: Extract) => sql.placeholder(arg); @@ -110,13 +127,7 @@ class DbManager implements IDbManager { }); const commands: SQLBatchTuple[] = [[query.sql, params]]; - await this.queue.enqueue({ - id: 'write', - run: async () => { - await this.db.$client.executeBatch(commands); - this.db.$client?.flushPendingReactiveQueries(); - }, - }); + return await this.executeBatch(commands); } public async write( diff --git a/src/database/queries/NovelRestoreQueries.ts b/src/database/queries/NovelRestoreQueries.ts index ff4aeb0b7f..e22523e423 100644 --- a/src/database/queries/NovelRestoreQueries.ts +++ b/src/database/queries/NovelRestoreQueries.ts @@ -1,4 +1,5 @@ -import { eq, inArray, sql } from 'drizzle-orm'; +import { and, eq, inArray, or } from 'drizzle-orm'; +import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; import { fetchNovel } from '@services/plugin/fetch'; import { insertChapters } from './ChapterQueries'; @@ -11,11 +12,10 @@ import { createNovelTriggerQueryUpdate, } from '@database/queryStrings/triggers'; import { - chapterSchema, novelCategorySchema, novelSchema, + restoreChapterMappingSchema, } from '@database/schema'; -import type { TransactionParameter } from '@database/manager/manager.d'; import { NOVEL_STORAGE } from '@utils/Storages'; import type { BackupNovel, NovelInfo, RestoredNovelMapping } from '../types'; @@ -81,229 +81,322 @@ export const restoreLibrary = async (novel: NovelInfo) => { await insertChapters(novelId, sourceNovel.chapters); } }; +const sqliteBoolean = (value: boolean | null | undefined): Scalar => + value == null ? null : value ? 1 : 0; +const RESTORE_NOVEL_BATCH_SIZE = 100; +const RESTORE_CHAPTER_BATCH_SIZE = 10_000; -const disableNovelStatsTriggers = async (tx: TransactionParameter) => { - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats')); - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_update')); - await tx.run(sql.raw('DROP TRIGGER IF EXISTS update_novel_stats_on_delete')); -}; - -const restoreNovelStatsTriggers = async (tx: TransactionParameter) => { - await tx.run(sql.raw(createNovelTriggerQueryInsert)); - await tx.run(sql.raw(createNovelTriggerQueryDelete)); - await tx.run(sql.raw(createNovelTriggerQueryUpdate)); +export type RestoreNovelOptions = { + includeChapterMappings?: boolean; + restoreRunId?: string; }; -const restoreNovelRecord = async ( - tx: TransactionParameter, - novel: Omit, -) => { - const resetState = { - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - lastReadAt: null, - lastUpdatedAt: null, - }; +const NOVEL_UPSERT_SQL = ` + INSERT INTO Novel ( + path, pluginId, name, cover, summary, author, artist, status, genres, + inLibrary, isLocal, totalPages + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path, pluginId) DO UPDATE SET + path = excluded.path, + pluginId = excluded.pluginId, + name = excluded.name, + cover = excluded.cover, + summary = excluded.summary, + author = excluded.author, + artist = excluded.artist, + status = excluded.status, + genres = excluded.genres, + inLibrary = excluded.inLibrary, + isLocal = excluded.isLocal, + totalPages = excluded.totalPages +`; - // The order here makes resetState authoritative. - const values = { - ...resetState, - ...novel, - }; +const NOVEL_COVER_UPDATE_SQL = ` + UPDATE Novel + SET cover = ? || pluginId || '/' || id || '/cover.png' || ? + WHERE pluginId = ? AND path = ? +`; - const localCover = values.cover?.startsWith(`file://${NOVEL_STORAGE}/`) - ? values.cover - : undefined; +const restoreNovelValues = (novel: BackupNovel): Scalar[] => [ + novel.path, + novel.pluginId, + novel.name, + novel.cover ?? null, + novel.summary ?? null, + novel.author ?? null, + novel.artist ?? null, + novel.status ?? null, + novel.genres ?? null, + sqliteBoolean(novel.inLibrary), + sqliteBoolean(novel.isLocal), + novel.totalPages ?? null, +]; - const cacheSuffix = localCover?.match(/[?#].*$/)?.[0] ?? ''; +const restoreNovelChunk = async ( + backupNovels: BackupNovel[], +): Promise => { + const commands: SQLBatchTuple[] = [ + [NOVEL_UPSERT_SQL, backupNovels.map(restoreNovelValues)], + ]; + const storedCovers = backupNovels + .filter(novel => novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) + .map( + novel => + [ + `file://${NOVEL_STORAGE}/`, + novel.cover?.match(/[?#].*$/)?.[0] ?? '', + novel.pluginId, + novel.path, + ] as Scalar[], + ); + if (storedCovers.length > 0) { + commands.push([NOVEL_COVER_UPDATE_SQL, storedCovers]); + } + await dbManager.executeBatch(commands); - const restoredNovel = await tx - .insert(novelSchema) - .values(values) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: values, + const rows = await dbManager + .select({ + id: novelSchema.id, + path: novelSchema.path, + pluginId: novelSchema.pluginId, }) - .returning({ id: novelSchema.id }) - .get(); - - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); + .from(novelSchema) + .where( + or( + ...backupNovels.map(novel => + and( + eq(novelSchema.pluginId, novel.pluginId), + eq(novelSchema.path, novel.path), + ), + ), + ), + ) + .all(); + const rowsByIdentity = new Map( + rows.map(row => [`${row.pluginId}\u0000${row.path}`, row]), + ); + return backupNovels.map(backupNovel => { + const row = rowsByIdentity.get( + `${backupNovel.pluginId}\u0000${backupNovel.path}`, + ); + if (!row) { + throw new Error('Restore returned incomplete novel mapping'); + } + return { + pluginId: backupNovel.pluginId, + backupNovelId: backupNovel.id, + restoredNovelId: row.id, + }; + }); +}; - if (localCover !== undefined) { - await tx - .update(novelSchema) - .set({ - cover: - `file://${NOVEL_STORAGE}/` + - `${values.pluginId}/${restoredNovel.id}/cover.png` + - cacheSuffix, - }) - .where(eq(novelSchema.id, restoredNovel.id)) - .run(); +const restoreNovelChunkWithRetry = async ( + backupNovels: BackupNovel[], +): Promise => { + try { + return await restoreNovelChunk(backupNovels); + } catch { + const mappings: RestoredNovelMapping[] = []; + let failed = false; + for (const backupNovel of backupNovels) { + try { + mappings.push( + await restoreNovelChunk([backupNovel]).then(([mapping]) => { + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; + }), + ); + } catch { + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more novels'); + } + return mappings; } - - return restoredNovel; }; const restoreChapterValues = ( - chapters: BackupNovel['chapters'], - novelId: number, -) => - chapters.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ - ...chapter, - novelId, - })); + chapter: BackupNovel['chapters'][number], + restoredNovelId: number, +): Scalar[] => [ + restoredNovelId, + chapter.path, + chapter.name, + chapter.releaseTime ?? null, + sqliteBoolean(chapter.bookmark), + sqliteBoolean(chapter.unread), + chapter.readTime ?? null, + sqliteBoolean(chapter.isDownloaded), + chapter.updatedTime ?? null, + chapter.chapterNumber ?? null, + chapter.page ?? null, + chapter.position ?? null, + chapter.progress ?? null, + chapter.scanlator ?? null, + chapter.timeSpent ?? null, +]; -const refreshRestoredNovelStats = async ( - tx: TransactionParameter, - novelIds: number[], -) => { - if (novelIds.length === 0) { - return; - } +type ChapterRestoreRecord = { + backupNovelId: number; + backupChapterId: number; + restoredNovelId: number; + chapter: BackupNovel['chapters'][number]; +}; - const stats = await tx - .select({ - novelId: chapterSchema.novelId, - totalChapters: sql`COUNT(*)`, - chaptersDownloaded: sql`SUM(CASE WHEN ${chapterSchema.isDownloaded} = 1 THEN 1 ELSE 0 END)`, - chaptersUnread: sql`SUM(CASE WHEN ${chapterSchema.unread} = 1 THEN 1 ELSE 0 END)`, - lastReadAt: sql`MAX(${chapterSchema.readTime})`, - lastUpdatedAt: sql`( - SELECT updatedChapter.updatedTime - FROM Chapter AS updatedChapter - WHERE updatedChapter.novelId = Chapter.novelId - AND updatedChapter.updatedTime IS NOT NULL - ORDER BY julianday(updatedChapter.updatedTime) DESC - LIMIT 1 - )`, - }) - .from(chapterSchema) - .where(inArray(chapterSchema.novelId, novelIds)) - .groupBy(chapterSchema.novelId) - .all(); +const CHAPTER_UPSERT_SQL = ` + INSERT INTO Chapter ( + novelId, path, name, releaseTime, bookmark, unread, readTime, + isDownloaded, updatedTime, chapterNumber, page, position, progress, + scanlator, timeSpent + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(novelId, path) DO UPDATE SET + novelId = excluded.novelId, + path = excluded.path, + name = excluded.name, + releaseTime = excluded.releaseTime, + bookmark = excluded.bookmark, + unread = excluded.unread, + readTime = excluded.readTime, + isDownloaded = excluded.isDownloaded, + updatedTime = excluded.updatedTime, + chapterNumber = excluded.chapterNumber, + page = excluded.page, + position = excluded.position, + progress = excluded.progress, + scanlator = excluded.scanlator, + timeSpent = excluded.timeSpent +`; - if (stats.length === 0) { +const RESTORE_CHAPTER_MAPPING_INSERT_SQL = ` + INSERT INTO RestoreChapterMapping ( + restoreRunId, backupNovelId, backupChapterId, + restoredNovelId, restoredChapterId + ) + SELECT ?, ?, ?, ?, id + FROM Chapter + WHERE novelId = ? AND path = ? + ON CONFLICT(restoreRunId, backupNovelId, backupChapterId) DO UPDATE SET + restoredNovelId = excluded.restoredNovelId, + restoredChapterId = excluded.restoredChapterId +`; + +const restoreChapterChunk = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, +) => { + if (records.length === 0) { return; } + if (includeChapterMappings && !restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } - const ids = sql.join( - novelIds.map(id => sql`${id}`), - sql`, `, - ); - const cases = ( - field: T, - fallback: unknown, - ) => - sql`CASE ${novelSchema.id} ${sql.join( - stats.map( - (stat: (typeof stats)[number]) => - sql`WHEN ${stat.novelId} THEN ${stat[field]}`, + const commands: SQLBatchTuple[] = [ + ['DROP TRIGGER IF EXISTS update_novel_stats'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_update'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_delete'], + [ + CHAPTER_UPSERT_SQL, + records.map(record => + restoreChapterValues(record.chapter, record.restoredNovelId), ), - sql` `, - )} ELSE ${fallback} END`; - - await tx.run(sql` - UPDATE Novel - SET totalChapters = ${cases('totalChapters', novelSchema.totalChapters)}, - chaptersDownloaded = ${cases( - 'chaptersDownloaded', - novelSchema.chaptersDownloaded, - )}, - chaptersUnread = ${cases('chaptersUnread', novelSchema.chaptersUnread)}, - lastReadAt = ${cases('lastReadAt', novelSchema.lastReadAt)}, - lastUpdatedAt = ${cases('lastUpdatedAt', novelSchema.lastUpdatedAt)} - WHERE id IN (${ids}) - `); -}; - -const RESTORE_CHAPTER_BATCH_SIZE = 500; - -type RestoreNovelOptions = { - includeChapterMappings?: boolean; + ], + ]; + if (includeChapterMappings) { + commands.push([ + RESTORE_CHAPTER_MAPPING_INSERT_SQL, + records.map( + record => + [ + restoreRunId, + record.backupNovelId, + record.backupChapterId, + record.restoredNovelId, + record.restoredNovelId, + record.chapter.path, + ] as Scalar[], + ), + ]); + } + commands.push( + [createNovelTriggerQueryInsert], + [createNovelTriggerQueryDelete], + [createNovelTriggerQueryUpdate], + ); + await dbManager.executeBatch(commands); }; -const restoreNovelsAndChaptersInTransaction = async ( - tx: TransactionParameter, - backupNovels: BackupNovel[], +const restoreChapterChunkWithRetry = async ( + records: ChapterRestoreRecord[], includeChapterMappings: boolean, -): Promise => { - await disableNovelStatsTriggers(tx); - + restoreRunId: string | undefined, +) => { try { - const restoredNovels: { - backupNovelId: number; - chapters: BackupNovel['chapters']; - novel: Omit; - restoredNovelId: number; - }[] = []; - for (const backupNovel of backupNovels) { - const { chapters, id: backupNovelId, ...novel } = backupNovel; - const restoredNovel = await restoreNovelRecord(tx, novel); - restoredNovels.push({ - backupNovelId, - chapters, - novel, - restoredNovelId: restoredNovel.id, - }); - } - - const mappings: RestoredNovelMapping[] = []; - for (const restoredNovel of restoredNovels) { - const chapterMappings: RestoredNovelMapping['chapters'] = []; - for ( - let i = 0; - i < restoredNovel.chapters.length; - i += RESTORE_CHAPTER_BATCH_SIZE - ) { - const batch = restoredNovel.chapters.slice( - i, - i + RESTORE_CHAPTER_BATCH_SIZE, + await restoreChapterChunk(records, includeChapterMappings, restoreRunId); + } catch { + let failed = false; + for (const record of records) { + try { + await restoreChapterChunk( + [record], + includeChapterMappings, + restoreRunId, ); - if (includeChapterMappings) { - const restoredChapters = (await tx - .insert(chapterSchema) - .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) - .returning({ id: chapterSchema.id, path: chapterSchema.path }) - .all()) as { id: number; path: string }[]; - const restoredIdsByPath = new Map( - restoredChapters.map(chapter => [chapter.path, chapter.id]), - ); - for (const chapter of batch) { - const restoredChapterId = restoredIdsByPath.get(chapter.path); - if (restoredChapterId !== undefined) { - chapterMappings.push({ - backupChapterId: chapter.id, - restoredChapterId, - }); - } - } - } else { - await tx - .insert(chapterSchema) - .values(restoreChapterValues(batch, restoredNovel.restoredNovelId)) - .run(); - } + } catch { + failed = true; } - mappings.push({ - pluginId: restoredNovel.novel.pluginId, - backupNovelId: restoredNovel.backupNovelId, - restoredNovelId: restoredNovel.restoredNovelId, - chapters: chapterMappings, - }); } - await refreshRestoredNovelStats( - tx, - restoredNovels.map(restoredNovel => restoredNovel.restoredNovelId), - ); - return mappings; - } finally { - await restoreNovelStatsTriggers(tx); + if (failed) { + throw new Error('Failed to restore one or more chapters'); + } + } +}; + +const NOVEL_STATS_UPDATE_SQL = ` + UPDATE Novel + SET totalChapters = ( + SELECT COUNT(*) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + chaptersDownloaded = COALESCE(( + SELECT SUM(CASE WHEN Chapter.isDownloaded = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + chaptersUnread = COALESCE(( + SELECT SUM(CASE WHEN Chapter.unread = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + lastReadAt = ( + SELECT MAX(Chapter.readTime) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + lastUpdatedAt = ( + SELECT updatedChapter.updatedTime + FROM Chapter AS updatedChapter + WHERE updatedChapter.novelId = Novel.id + AND updatedChapter.updatedTime IS NOT NULL + ORDER BY julianday(updatedChapter.updatedTime) DESC + LIMIT 1 + ) + WHERE Novel.id = ? +`; + +const refreshRestoredNovelStats = async (novelIds: number[]) => { + if (novelIds.length === 0) { + return; } + await dbManager.executeBatch([ + [NOVEL_STATS_UPDATE_SQL, novelIds.map(id => [id] as Scalar[])], + ]); }; export const _restoreNovelsAndChapters = async ( @@ -313,17 +406,63 @@ export const _restoreNovelsAndChapters = async ( if (backupNovels.length === 0) { return []; } - return dbManager.write(tx => - restoreNovelsAndChaptersInTransaction( - tx, - backupNovels, - options.includeChapterMappings ?? true, - ), + const includeChapterMappings = options.includeChapterMappings ?? true; + if (includeChapterMappings && !options.restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const mappings: RestoredNovelMapping[] = []; + for ( + let start = 0; + start < backupNovels.length; + start += RESTORE_NOVEL_BATCH_SIZE + ) { + mappings.push( + ...(await restoreNovelChunkWithRetry( + backupNovels.slice(start, start + RESTORE_NOVEL_BATCH_SIZE), + )), + ); + } + + const restoredNovelIds = new Map( + mappings.map(mapping => [mapping.backupNovelId, mapping.restoredNovelId]), ); + const chapterChunk: ChapterRestoreRecord[] = []; + for (const backupNovel of backupNovels) { + const restoredNovelId = restoredNovelIds.get(backupNovel.id); + if (restoredNovelId === undefined) { + throw new Error('Missing restored novel mapping'); + } + for (const chapter of backupNovel.chapters) { + chapterChunk.push({ + backupNovelId: backupNovel.id, + backupChapterId: chapter.id, + restoredNovelId, + chapter, + }); + if (chapterChunk.length === RESTORE_CHAPTER_BATCH_SIZE) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + ); + chapterChunk.length = 0; + } + } + } + if (chapterChunk.length > 0) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + ); + } + await refreshRestoredNovelStats([...restoredNovelIds.values()]); + return mappings; }; /** - * Restores novel and chapters from a backup object. + * Restores a novel and its chapters from a backup object. */ export const _restoreNovelAndChapters = async ( backupNovel: BackupNovel, @@ -335,3 +474,36 @@ export const _restoreNovelAndChapters = async ( } return mapping; }; + +export const getRestoreChapterMappings = async ( + restoreRunId: string, + backupNovelId: number, + backupChapterIds: number[], +) => { + if (backupChapterIds.length === 0) { + return []; + } + return dbManager + .select({ + backupChapterId: restoreChapterMappingSchema.backupChapterId, + restoredChapterId: restoreChapterMappingSchema.restoredChapterId, + }) + .from(restoreChapterMappingSchema) + .where( + and( + eq(restoreChapterMappingSchema.restoreRunId, restoreRunId), + eq(restoreChapterMappingSchema.backupNovelId, backupNovelId), + inArray(restoreChapterMappingSchema.backupChapterId, backupChapterIds), + ), + ) + .all(); +}; + +export const clearRestoreChapterMappings = async (restoreRunId: string) => { + await dbManager.write(tx => + tx + .delete(restoreChapterMappingSchema) + .where(eq(restoreChapterMappingSchema.restoreRunId, restoreRunId)) + .run(), + ); +}; diff --git a/src/database/queries/__tests__/NovelRestoreQueries.test.ts b/src/database/queries/__tests__/NovelRestoreQueries.test.ts index 7630e21123..1d24c99513 100644 --- a/src/database/queries/__tests__/NovelRestoreQueries.test.ts +++ b/src/database/queries/__tests__/NovelRestoreQueries.test.ts @@ -2,10 +2,12 @@ import './mockDb'; import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; import { insertTestNovel, insertTestChapter, clearAllTables } from './testData'; import { chapterSchema, novelSchema } from '@database/schema'; -import { eq } from 'drizzle-orm'; +import { and, eq } from 'drizzle-orm'; import { getNovelByPath } from '../NovelQueries'; import { + clearRestoreChapterMappings, + getRestoreChapterMappings, restoreLibrary, _restoreNovelAndChapters, _restoreNovelsAndChapters, @@ -76,41 +78,44 @@ describe('NovelRestoreQueries', () => { inLibrary: true, }); - const mapping = await _restoreNovelAndChapters({ - id: 1, - path: '/restored/novel', - pluginId: 'restored-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 10, - novelId: 1, - path: '/restored/chapter-1', - name: 'Chapter 1', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: null, - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }); + const mapping = await _restoreNovelAndChapters( + { + id: 1, + path: '/restored/novel', + pluginId: 'restored-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 10, + novelId: 1, + path: '/restored/chapter-1', + name: 'Chapter 1', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { restoreRunId: 'restore-collision' }, + ); expect(mapping.restoredNovelId).not.toBe(1); expect( @@ -124,12 +129,8 @@ describe('NovelRestoreQueries', () => { .from(chapterSchema) .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) .all(); - expect(mapping.chapters).toEqual([ - { - backupChapterId: 10, - restoredChapterId: restoredChapters[0].id, - }, - ]); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-1'); }); it('restores multiple novels in one batch', async () => { const mappings = await _restoreNovelsAndChapters( @@ -165,7 +166,7 @@ describe('NovelRestoreQueries', () => { ); }); it('preserves mappings and aggregate stats for multiple novels', async () => { - const mappings = await _restoreNovelsAndChapters( + await _restoreNovelsAndChapters( [ { id: 200, @@ -238,22 +239,28 @@ describe('NovelRestoreQueries', () => { ], }, ], - { includeChapterMappings: true }, + { includeChapterMappings: true, restoreRunId: 'restore-mappings' }, ); - expect(mappings).toHaveLength(2); + const stagedMappingsOne = await getRestoreChapterMappings( + 'restore-mappings', + 200, + [2001, 2002], + ); + const stagedMappingsTwo = await getRestoreChapterMappings( + 'restore-mappings', + 201, + [2011], + ); expect( - mappings.map(mapping => - mapping.chapters.map(chapter => chapter.backupChapterId), - ), - ).toEqual([[2001, 2002], [2011]]); - expect(mappings[0].chapters).toHaveLength(2); - expect(mappings[1].chapters).toHaveLength(1); + stagedMappingsOne.map(row => row.backupChapterId).sort((a, b) => a - b), + ).toEqual([2001, 2002]); + expect(stagedMappingsTwo.map(row => row.backupChapterId)).toEqual([2011]); expect( new Set( - mappings - .flatMap(mapping => mapping.chapters) - .map(chapter => chapter.restoredChapterId), + [...stagedMappingsOne, ...stagedMappingsTwo].map( + row => row.restoredChapterId, + ), ).size, ).toBe(3); expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( @@ -315,7 +322,6 @@ describe('NovelRestoreQueries', () => { { includeChapterMappings: false }, ); - expect(mapping.chapters).toEqual([]); const restoredChapters = await getTestDb() .drizzleDb.select() .from(chapterSchema) @@ -323,76 +329,221 @@ describe('NovelRestoreQueries', () => { .all(); expect(restoredChapters).toHaveLength(1); expect(restoredChapters[0].path).toBe('/restored/chapter-2'); + expect( + await getRestoreChapterMappings('without-mappings', 2, [20]), + ).toEqual([]); }); - it('rebuilds aggregate timestamps after replacing existing chapters', async () => { + it('merges restored identities without replacing existing rows', async () => { const testDb = getTestDb(); - const novelId = await insertTestNovel(testDb, { - path: '/restore/stats', - pluginId: 'stats-plugin', - }); - await insertTestChapter(testDb, novelId, { - path: '/restore/old-chapter', - readTime: '2020-01-01T00:00:00.000Z', - updatedTime: '2020-01-01T00:00:00.000Z', + const existingNovelId = await insertTestNovel(testDb, { + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Before Restore', + cover: 'before-cover', + summary: 'before-summary', + author: 'Before Author', + artist: 'Before Artist', + status: 'Paused', + genres: 'before', + inLibrary: false, + isLocal: false, + totalPages: 1, }); - await testDb.drizzleDb - .update(novelSchema) - .set({ - lastReadAt: 'stale-read-time', - lastUpdatedAt: 'stale-update-time', - }) - .where(eq(novelSchema.id, novelId)) - .run(); - - await _restoreNovelAndChapters( + const conflictingChapterId = await insertTestChapter( + testDb, + existingNovelId, { - id: 3, - path: '/restore/stats', - pluginId: 'stats-plugin', - name: 'Restored Stats', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 30, - novelId: 3, - path: '/restore/new-chapter', - name: 'New Chapter', - releaseTime: null, - readTime: '2024-01-01T00:00:00.000Z', - bookmark: false, - unread: false, - isDownloaded: true, - updatedTime: '2025-01-01T00:00:00.000Z', - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], + path: '/restore/merge/chapter', + name: 'Before Chapter', + bookmark: false, + unread: true, + readTime: '2023-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2022-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + position: 0, + progress: 1, + timeSpent: 2, + }, + ); + const unrelatedChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/unrelated', + name: 'Unrelated Chapter', + bookmark: false, + unread: true, + readTime: '2024-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + position: 1, + progress: 0, + timeSpent: 3, }, - { includeChapterMappings: false }, ); + expect(existingNovelId).not.toBe(900); + expect(conflictingChapterId).not.toBe(9001); - const restored = await getNovelByPath('/restore/stats', 'stats-plugin'); - expect(restored).toEqual( + const restoreRunId = 'restore-merge'; + const backupNovel = { + id: 900, + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Restored Novel', + cover: 'file:///mock/novel/storage/cache-key?size=large', + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: true, + isLocal: true, + totalPages: 123, + chapters: [ + { + id: 9001, + novelId: 900, + path: '/restore/merge/chapter', + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }, + ], + }; + + const firstMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(firstMapping).toEqual({ + pluginId: 'merge-plugin', + backupNovelId: 900, + restoredNovelId: existingNovelId, + }); + + const restoredNovel = await getNovelByPath( + '/restore/merge', + 'merge-plugin', + ); + expect(restoredNovel).toEqual( expect.objectContaining({ - totalChapters: 1, + id: existingNovelId, + name: 'Restored Novel', + cover: `file:///mock/novel/storage/merge-plugin/${existingNovelId}/cover.png?size=large`, + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: 1, + isLocal: 1, + totalPages: 123, + totalChapters: 2, chaptersDownloaded: 1, - chaptersUnread: 0, - lastReadAt: '2024-01-01T00:00:00.000Z', - lastUpdatedAt: '2025-01-01T00:00:00.000Z', + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2026-01-01T00:00:00.000Z', }), ); + + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(restoredChapters).toHaveLength(2); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/chapter', + ), + ).toEqual( + expect.objectContaining({ + id: conflictingChapterId, + novelId: existingNovelId, + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }), + ); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/unrelated', + ), + ).toEqual( + expect.objectContaining({ + id: unrelatedChapterId, + name: 'Unrelated Chapter', + }), + ); + + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([ + { + backupChapterId: 9001, + restoredChapterId: conflictingChapterId, + }, + ]); + + const secondMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(secondMapping.restoredNovelId).toBe(existingNovelId); + const novelsAfterReplay = await testDb.drizzleDb + .select() + .from(novelSchema) + .where( + and( + eq(novelSchema.path, '/restore/merge'), + eq(novelSchema.pluginId, 'merge-plugin'), + ), + ) + .all(); + const chaptersAfterReplay = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(novelsAfterReplay).toHaveLength(1); + expect(chaptersAfterReplay).toHaveLength(2); + expect( + chaptersAfterReplay.map(chapter => chapter.id).sort((a, b) => a - b), + ).toEqual( + [conflictingChapterId, unrelatedChapterId].sort((a, b) => a - b), + ); + + await clearRestoreChapterMappings(restoreRunId); + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([]); + expect( + testDb.sqlite.executeSync('PRAGMA foreign_key_check').rows, + ).toEqual([]); }); }); }); diff --git a/src/database/queries/__tests__/testData.ts b/src/database/queries/__tests__/testData.ts index 834b2f5d7e..c4a3dc9df3 100644 --- a/src/database/queries/__tests__/testData.ts +++ b/src/database/queries/__tests__/testData.ts @@ -21,7 +21,7 @@ import { */ export function clearAllTables(testDb: TestDb) { const { sqlite } = testDb; - sqlite.executeSync('DELETE FROM NovelCategory'); + sqlite.executeSync('DELETE FROM RestoreChapterMapping'); sqlite.executeSync('DELETE FROM Chapter'); sqlite.executeSync('DELETE FROM Novel'); sqlite.executeSync('DELETE FROM Repository'); diff --git a/src/database/queries/__tests__/testDb.ts b/src/database/queries/__tests__/testDb.ts index 0232c41bdd..dcc43981fd 100644 --- a/src/database/queries/__tests__/testDb.ts +++ b/src/database/queries/__tests__/testDb.ts @@ -30,7 +30,7 @@ const MIGRATION_STATEMENTS = [ `CREATE INDEX IF NOT EXISTS category_sort_idx ON Category (sort)`, `CREATE TABLE IF NOT EXISTS Chapter ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, - novelId integer NOT NULL, + novelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, path text NOT NULL, name text NOT NULL, releaseTime text, @@ -70,6 +70,17 @@ const MIGRATION_STATEMENTS = [ )`, `CREATE UNIQUE INDEX IF NOT EXISTS novel_path_plugin_unique ON Novel (path, pluginId)`, `CREATE INDEX IF NOT EXISTS NovelIndex ON Novel (pluginId, path, id, inLibrary)`, + `CREATE TABLE IF NOT EXISTS RestoreChapterMapping ( + restoreRunId text NOT NULL, + backupNovelId integer NOT NULL, + backupChapterId integer NOT NULL, + restoredNovelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, + restoredChapterId integer NOT NULL REFERENCES Chapter(id) ON DELETE CASCADE +)`, + `CREATE UNIQUE INDEX IF NOT EXISTS restore_chapter_mapping_unique + ON RestoreChapterMapping (restoreRunId, backupNovelId, backupChapterId)`, + `CREATE INDEX IF NOT EXISTS restore_chapter_mapping_novel_index + ON RestoreChapterMapping (restoreRunId, backupNovelId)`, `CREATE TABLE IF NOT EXISTS NovelCategory ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, novelId integer NOT NULL, diff --git a/src/database/schema/chapter.ts b/src/database/schema/chapter.ts index 08354a1282..c4720deb9f 100644 --- a/src/database/schema/chapter.ts +++ b/src/database/schema/chapter.ts @@ -6,12 +6,15 @@ import { text, uniqueIndex, } from 'drizzle-orm/sqlite-core'; +import { novel } from './novel'; export const chapter = sqliteTable( 'Chapter', { id: integer('id').primaryKey({ autoIncrement: true }), - novelId: integer('novelId').notNull(), + novelId: integer('novelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), path: text('path').notNull(), name: text('name').notNull(), releaseTime: text('releaseTime'), diff --git a/src/database/schema/index.ts b/src/database/schema/index.ts index 744c84cd73..2fa0c4edea 100644 --- a/src/database/schema/index.ts +++ b/src/database/schema/index.ts @@ -3,6 +3,7 @@ import { novel } from './novel'; import { chapter } from './chapter'; import { novelCategory } from './novelCategory'; import { repository } from './repository'; +import { restoreChapterMapping } from './restoreChapterMapping'; export { category as categorySchema, @@ -25,6 +26,11 @@ export { type RepositoryRow, type RepositoryInsert, } from './repository'; +export { + restoreChapterMapping as restoreChapterMappingSchema, + type RestoreChapterMappingRow, + type RestoreChapterMappingInsert, +} from './restoreChapterMapping'; /** * Unified schema object containing all database tables @@ -36,6 +42,7 @@ export const schema = { chapter, novelCategory, repository, + restoreChapterMapping, } as const; export type Schema = typeof schema; diff --git a/src/database/schema/restoreChapterMapping.ts b/src/database/schema/restoreChapterMapping.ts new file mode 100644 index 0000000000..c47f1f68f0 --- /dev/null +++ b/src/database/schema/restoreChapterMapping.ts @@ -0,0 +1,40 @@ +import { + index, + integer, + sqliteTable, + text, + uniqueIndex, +} from 'drizzle-orm/sqlite-core'; +import { chapter } from './chapter'; +import { novel } from './novel'; + +export const restoreChapterMapping = sqliteTable( + 'RestoreChapterMapping', + { + restoreRunId: text('restoreRunId').notNull(), + backupNovelId: integer('backupNovelId').notNull(), + backupChapterId: integer('backupChapterId').notNull(), + restoredNovelId: integer('restoredNovelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), + restoredChapterId: integer('restoredChapterId') + .notNull() + .references(() => chapter.id, { onDelete: 'cascade' }), + }, + table => [ + uniqueIndex('restore_chapter_mapping_unique').on( + table.restoreRunId, + table.backupNovelId, + table.backupChapterId, + ), + index('restore_chapter_mapping_novel_index').on( + table.restoreRunId, + table.backupNovelId, + ), + ], +); + +export type RestoreChapterMappingRow = + typeof restoreChapterMapping.$inferSelect; +export type RestoreChapterMappingInsert = + typeof restoreChapterMapping.$inferInsert; diff --git a/src/database/types/index.ts b/src/database/types/index.ts index 349bfaf0bb..5c260c7601 100644 --- a/src/database/types/index.ts +++ b/src/database/types/index.ts @@ -125,10 +125,6 @@ export interface RestoredNovelMapping { pluginId: string; backupNovelId: number; restoredNovelId: number; - chapters: { - backupChapterId: number; - restoredChapterId: number; - }[]; } export interface BackupCategory extends Category { diff --git a/src/i18n/languages/en/strings.json b/src/i18n/languages/en/strings.json index f2c1401d18..c06ecad714 100644 --- a/src/i18n/languages/en/strings.json +++ b/src/i18n/languages/en/strings.json @@ -141,6 +141,8 @@ "novelDirectoryReadFailed": "Failed to read novel directory: %{error}", "restoringCategories": "Restoring Categories", "restoringCategoriesProgress": "Restoring categories (%{current}/%{total})", + "validatingNovels": "Validating Novels", + "validatingNovelsProgress": "Validating novels (%{current}/%{total})", "restoringNovels": "Restoring Novels", "restoringNovelsProgress": "Restoring novels (%{current}/%{total})", "restoringSettings": "Restoring Settings", diff --git a/src/i18n/types/index.ts b/src/i18n/types/index.ts index 924f4b15b0..36cbe80929 100644 --- a/src/i18n/types/index.ts +++ b/src/i18n/types/index.ts @@ -124,6 +124,8 @@ export interface StringMap { 'backupScreen.novelDirectoryReadFailed': 'string'; 'backupScreen.restoringCategories': 'string'; 'backupScreen.restoringCategoriesProgress': 'string'; + 'backupScreen.validatingNovels': 'string'; + 'backupScreen.validatingNovelsProgress': 'string'; 'backupScreen.restoringNovels': 'string'; 'backupScreen.restoringNovelsProgress': 'string'; 'backupScreen.restoringSettings': 'string'; diff --git a/src/services/backup/__tests__/fileSections.test.ts b/src/services/backup/__tests__/fileSections.test.ts index d12400459c..a3426bef79 100644 --- a/src/services/backup/__tests__/fileSections.test.ts +++ b/src/services/backup/__tests__/fileSections.test.ts @@ -1,14 +1,21 @@ import NativeFile from '@modules/native-file'; import { getSelectedBackupFileSections, + restoreLegacyFiles, restoreNovelFiles, } from '../fileSections'; +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; import { ZipBackupName } from '../types'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', })); + describe('selected backup file sections', () => { const options = { library: true, @@ -37,12 +44,21 @@ describe('selected backup file sections', () => { }); }); -describe('restoreNovelFiles', () => { - it('moves downloaded files to remapped novel and chapter IDs', async () => { +describe('restore file sections', () => { + beforeEach(() => { + jest.mocked(getRestoreChapterMappings).mockReset(); + }); + + it('moves downloaded files with bounded staged mapping lookups', async () => { jest.mocked(NativeFile.exists).mockResolvedValue(true); jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + const chapterItems = Array.from({ length: 101 }, (_, index) => ({ + name: String(index + 1), + path: `/staging/source/1/${index + 1}`, + isDirectory: true, + })); jest.mocked(NativeFile.readDir).mockImplementation(async path => { if (path === '/staging/source/1') { return [ @@ -51,16 +67,7 @@ describe('restoreNovelFiles', () => { path: '/staging/source/1/cover.png', isDirectory: false, }, - { - name: '10', - path: '/staging/source/1/10', - isDirectory: true, - }, - { - name: '20', - path: '/staging/source/1/20', - isDirectory: true, - }, + ...chapterItems, ]; } if (path === '/staging/source/1/10') { @@ -72,18 +79,54 @@ describe('restoreNovelFiles', () => { }, ]; } + if (path === '/staging/source/1/101') { + return [ + { + name: 'index.html', + path: '/staging/source/1/101/index.html', + isDirectory: false, + }, + ]; + } return []; }); + jest + .mocked(getRestoreChapterMappings) + .mockImplementation( + async (_restoreRunId, _backupNovelId, backupChapterIds) => + backupChapterIds + .filter(id => id === 10 || id === 101) + .map(backupChapterId => ({ + backupChapterId, + restoredChapterId: backupChapterId === 10 ? 99 : 199, + })), + ); - await restoreNovelFiles('/staging', [ - { - pluginId: 'source', - backupNovelId: 1, - restoredNovelId: 7, - chapters: [{ backupChapterId: 10, restoredChapterId: 99 }], - }, - ]); + await restoreNovelFiles( + '/staging', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'restore-run-1', + ); + expect(getRestoreChapterMappings).toHaveBeenCalledTimes(2); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 1, + 'restore-run-1', + 1, + Array.from({ length: 100 }, (_, index) => index + 1), + ); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 2, + 'restore-run-1', + 1, + [101], + ); expect(NativeFile.moveFile).toHaveBeenCalledWith( '/staging/source/1/cover.png', '/storage/Novels/source/7/cover.png', @@ -92,10 +135,53 @@ describe('restoreNovelFiles', () => { '/staging/source/1/10/index.html', '/storage/Novels/source/7/99/index.html', ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/101/index.html', + '/storage/Novels/source/7/199/index.html', + ); expect(NativeFile.moveFile).not.toHaveBeenCalledWith( expect.stringContaining('/20/'), expect.any(String), ); expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); }); + + it('passes the restore run ID through legacy file restoration', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockImplementation(async path => { + if (path === '/legacy/Novels/source/1') { + return [ + { + name: '10', + path: '/legacy/Novels/source/1/10', + isDirectory: true, + }, + ]; + } + return []; + }); + jest + .mocked(getRestoreChapterMappings) + .mockResolvedValue([{ backupChapterId: 10, restoredChapterId: 99 }]); + + await restoreLegacyFiles( + '/legacy', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'legacy-run', + ); + + expect(getRestoreChapterMappings).toHaveBeenCalledWith('legacy-run', 1, [ + 10, + ]); + expect(NativeFile.unlink).toHaveBeenCalledWith('/legacy'); + }); }); diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index 846037cb8d..b5a371d5a9 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -1,9 +1,25 @@ import NativeFile from '@modules/native-file'; import NativeZipArchive from '@modules/native-zip-archive'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; import { createBackup, restoreBackup } from '../local'; +import * as fileSections from '../fileSections'; import { finalizeRestoredPlugins } from '../restoreResult'; import { prepareBackupData, restoreData } from '../utils'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(), + getRestoreChapterMappings: jest.fn(), +})); + +jest.mock('../fileSections', () => { + const actual = jest.requireActual('../fileSections'); + return { + ...actual, + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), + }; +}); + jest.mock('../utils', () => ({ CACHE_DIR_PATH: '/cache/BackupData', clearBackupCache: jest.fn(), @@ -56,6 +72,18 @@ describe('local selective backup', () => { jest.mocked(NativeFile.unlink).mockReset().mockResolvedValue(undefined); jest.mocked(NativeFile.readDir).mockReset().mockResolvedValue([]); jest.mocked(finalizeRestoredPlugins).mockReset().mockResolvedValue([]); + jest + .mocked(clearRestoreChapterMappings) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreLegacyFiles) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreNovelFiles) + .mockReset() + .mockResolvedValue(undefined); }); it('creates archives only for selected file sections', async () => { @@ -156,6 +184,7 @@ describe('local selective backup', () => { failedSectionCount: 0, pluginIds: ['restored'], novelMappings: [], + restoreRunId: 'restore-run-plugins', manifest: { appVersion: '2.1.0', formatVersion: 2 as const, @@ -180,6 +209,9 @@ describe('local selective backup', () => { '/storage/Plugins', ); expect(finalizeRestoredPlugins).toHaveBeenCalledWith(restoreResult); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); expect( jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], ).toBeGreaterThan( @@ -196,6 +228,7 @@ describe('local selective backup', () => { failedSectionCount: 0, pluginIds: [], novelMappings: [], + restoreRunId: 'restore-run-v1', manifest: { appVersion: '1.0.0', formatVersion: 1 as const, @@ -223,6 +256,14 @@ describe('local selective backup', () => { '/cache/BackupData/novel-files.zip', expect.any(String), ); + expect(fileSections.restoreLegacyFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredLegacyFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); }); it('extracts the v2 novel-files archive into the novel staging path', async () => { @@ -235,6 +276,7 @@ describe('local selective backup', () => { failedSectionCount: 0, pluginIds: [], novelMappings: [], + restoreRunId: 'restore-run-v2', manifest: { appVersion: '2.0.0', formatVersion: 2 as const, @@ -258,6 +300,14 @@ describe('local selective backup', () => { '/cache/BackupData/novel-files.zip', '/cache/BackupData/RestoredNovelFiles', ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredNovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); }); it('restores v3 novel files from the outer archive without nested extraction', async () => { @@ -270,6 +320,7 @@ describe('local selective backup', () => { failedSectionCount: 0, pluginIds: [], novelMappings: [], + restoreRunId: 'restore-run-v3', manifest: { appVersion: '2.1.3', formatVersion: 3 as const, @@ -298,6 +349,14 @@ describe('local selective backup', () => { '/cache/BackupData/novel-files.zip', expect.any(String), ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/NovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); }); it('rejects a v3 downloaded-file restore without NovelFiles', async () => { const restoreResult = { @@ -309,6 +368,7 @@ describe('local selective backup', () => { failedSectionCount: 0, pluginIds: [], novelMappings: [], + restoreRunId: 'restore-run-v3-missing', manifest: { appVersion: '2.1.3', formatVersion: 3 as const, @@ -333,5 +393,8 @@ describe('local selective backup', () => { restoreBackup({ sourceUri: 'content://backup.zip' }), ).rejects.toThrow('backupScreen.invalidBackupFolder'); expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); }); }); diff --git a/src/services/backup/__tests__/restoreResult.test.ts b/src/services/backup/__tests__/restoreResult.test.ts index 74932e74cc..5797e64ce6 100644 --- a/src/services/backup/__tests__/restoreResult.test.ts +++ b/src/services/backup/__tests__/restoreResult.test.ts @@ -65,6 +65,7 @@ const successfulResult: RestoreResult = { failedSectionCount: 0, pluginIds: ['installed'], novelMappings: [], + restoreRunId: 'restore-run-1', manifest: { appVersion: '2.1.0', formatVersion: 2, diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 38ba10e4ca..d84f5e2082 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -19,12 +19,14 @@ import type { RestoredNovelMapping, } from '@database/types'; import type { BackupOptions } from '../options'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; jest.mock('@database/queries/NovelQueries', () => ({ getAllNovels: jest.fn(), })); jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(async () => undefined), _restoreNovelAndChapters: jest.fn(), _restoreNovelsAndChapters: jest.fn(), })); @@ -132,21 +134,22 @@ describe('selective backup data', () => { jest.mocked(getAllNovelCategories).mockResolvedValue([]); const restoreNovel = async ( novel: BackupNovel, + _options?: { includeChapterMappings?: boolean; restoreRunId?: string }, ): Promise => ({ pluginId: novel.pluginId, backupNovelId: novel.id, restoredNovelId: novel.id, - chapters: novel.chapters.map(chapter => ({ - backupChapterId: chapter.id, - restoredChapterId: chapter.id, - })), }); jest.mocked(_restoreNovelAndChapters).mockImplementation(restoreNovel); - jest - .mocked(_restoreNovelsAndChapters) - .mockImplementation(async (novels: BackupNovel[]) => - Promise.all(novels.map(restoreNovel)), - ); + jest.mocked(_restoreNovelsAndChapters).mockImplementation( + async ( + novels: BackupNovel[], + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => Promise.all(novels.map(novel => restoreNovel(novel, _options))), + ); }); it('writes the selected sections to the v2 manifest', async () => { @@ -189,6 +192,7 @@ describe('selective backup data', () => { formatVersion: 3, sections: options, }); + expect(result.restoreRunId).toEqual(expect.any(String)); }); it('does not warn about sections intentionally omitted by the manifest', async () => { @@ -209,6 +213,7 @@ describe('selective backup data', () => { const result = await restoreData('/cache'); expect(result).toMatchObject({ + restoreRunId: expect.any(String), failedNovelCount: 0, failedCategoryCount: 0, failedSectionCount: 0, @@ -715,19 +720,32 @@ describe('selective backup data', () => { name: 'First', path: '/first', pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, chapters: [ { id: 101, novelId: 11, name: 'First chapter', path: '/first/1', + releaseTime: null, readTime: null, bookmark: null, unread: null, isDownloaded: true, updatedTime: null, + chapterNumber: null, page: null, + position: null, progress: null, + scanlator: null, timeSpent: 0, }, ], @@ -737,19 +755,32 @@ describe('selective backup data', () => { name: 'Second', path: '/second', pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, chapters: [ { id: 202, novelId: 22, name: 'Second chapter', path: '/second/1', + releaseTime: null, readTime: null, bookmark: null, unread: null, isDownloaded: true, updatedTime: null, + chapterNumber: null, page: null, + position: null, progress: null, + scanlator: null, timeSpent: 0, }, ], @@ -760,13 +791,11 @@ describe('selective backup data', () => { pluginId: 'source', backupNovelId: 11, restoredNovelId: 111, - chapters: [{ backupChapterId: 101, restoredChapterId: 1001 }], }, { pluginId: 'source', backupNovelId: 22, restoredNovelId: 222, - chapters: [{ backupChapterId: 202, restoredChapterId: 2002 }], }, ]; jest @@ -797,14 +826,89 @@ describe('selective backup data', () => { expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { includeChapterMappings: true, + restoreRunId: expect.any(String), }); + expect(result.restoreRunId).toBe( + jest.mocked(_restoreNovelsAndChapters).mock.calls[0][1]?.restoreRunId, + ); expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); expect(result).toMatchObject({ + restoreRunId: expect.any(String), novelCount: 2, failedNovelCount: 0, novelMappings: mappings, }); }); + it('reports validation and restore progress separately', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const novel = makeTestNovel(11); + const mapping: RestoredNovelMapping = { + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: 111, + }; + const progressTexts: string[] = []; + const setMeta: TaskProgressUpdater = transform => { + const next = transform({ + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }); + if (next.progressText) { + progressTexts.push(next.progressText); + } + }; + const benchmarkLog = jest.fn(); + + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: '11.json', + path: '/cache/NovelAndChapters/11.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(novel); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce([mapping]); + + await restoreData('/cache', setMeta, benchmarkLog); + + expect(progressTexts.slice(0, 4)).toEqual([ + 'backupScreen.validatingNovels', + 'backupScreen.validatingNovelsProgress', + 'backupScreen.restoringNovels', + 'backupScreen.restoringNovelsProgress', + ]); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:validation:start', + ); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:validation:done total=1', + ); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:restore:start total=1', + ); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:restore:progress current=1 total=1', + ); + }); it('restores scrambled compact batches in order with downloaded-file mappings', async () => { const options: BackupOptions = { @@ -820,13 +924,11 @@ describe('selective backup data', () => { pluginId: 'plugin-a', backupNovelId: 11, restoredNovelId: 111, - chapters: [{ backupChapterId: 1101, restoredChapterId: 1001 }], }, { pluginId: 'plugin-b', backupNovelId: 22, restoredNovelId: 222, - chapters: [{ backupChapterId: 2201, restoredChapterId: 2001 }], }, ]; jest @@ -888,9 +990,11 @@ describe('selective backup data', () => { ]); expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(expect.any(Array), { includeChapterMappings: true, + restoreRunId: expect.any(String), }); expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); expect(result).toMatchObject({ + restoreRunId: expect.any(String), novelCount: 2, failedNovelCount: 0, pluginIds: ['plugin-a', 'plugin-b'], @@ -911,6 +1015,15 @@ describe('selective backup data', () => { name: 'First', path: '/first', pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, chapters: [], }, { @@ -918,16 +1031,22 @@ describe('selective backup data', () => { name: 'Second', path: '/second', pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, chapters: [], }, ]; - const mappings: RestoredNovelMapping[] = novels.map((novel, index) => ({ + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ pluginId: novel.pluginId, backupNovelId: novel.id, restoredNovelId: novel.id + 100, - chapters: [ - { backupChapterId: index + 1, restoredChapterId: index + 101 }, - ], })); jest .mocked(NativeFile.exists) @@ -953,25 +1072,32 @@ describe('selective backup data', () => { jest .mocked(_restoreNovelsAndChapters) .mockRejectedValueOnce(new Error('batch failed')); - jest - .mocked(_restoreNovelAndChapters) - .mockImplementation( - async novel => - mappings.find(mapping => mapping.backupNovelId === novel.id)!, - ); + jest.mocked(_restoreNovelAndChapters).mockImplementation( + async ( + novel, + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => mappings.find(mapping => mapping.backupNovelId === novel.id)!, + ); const result = await restoreData('/cache'); expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { includeChapterMappings: true, + restoreRunId: expect.any(String), }); expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(1, novels[0], { includeChapterMappings: true, + restoreRunId: expect.any(String), }); expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(2, novels[1], { includeChapterMappings: true, + restoreRunId: expect.any(String), }); expect(result).toMatchObject({ + restoreRunId: expect.any(String), novelCount: 2, failedNovelCount: 0, novelMappings: mappings, @@ -1048,10 +1174,14 @@ describe('selective backup data', () => { ], }), ], - { includeChapterMappings: true }, + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, ); expect(result.manifest).toMatchObject({ formatVersion }); expect(result).toMatchObject({ + restoreRunId: expect.any(String), novelCount: 1, failedNovelCount: 0, novelMappings: [ @@ -1064,6 +1194,221 @@ describe('selective backup data', () => { }, ); + it('normalizes legacy object payloads before restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + chapters: [ + { + id: 50101, + novelId: 999, + path: '/legacy/1', + name: 'Legacy chapter', + }, + ], + ignoredLegacyField: 'ignored', + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 50101, + novelId: 501, + path: '/legacy/1', + name: 'Legacy chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], + }, + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + }); + }); + it('restores legacy novels whose stored name is empty', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + chapters: [], + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'empty-name.json', + path: '/cache/NovelAndChapters/empty-name.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + }), + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + novelCount: 1, + failedNovelCount: 0, + }); + }); + + it('rejects duplicate source IDs, novel identities, and chapter identities before restoring invalid files', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(601, 'source'); + const duplicateIdNovel = makeTestNovel(601, 'other-source'); + const duplicateIdentityNovel = { + ...makeTestNovel(602, 'source'), + path: validNovel.path, + }; + const duplicateChapterIdentityNovel = { + ...makeTestNovel(603, 'source'), + chapters: [ + makeTestChapter(603, 1), + { ...makeTestChapter(603, 1), id: 60302 }, + ], + }; + const payloads = { + 'batch-000001.json': JSON.stringify(encodeNovelBatch([validNovel])), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([duplicateIdNovel])), + 'batch-000003.json': JSON.stringify( + encodeNovelBatch([duplicateIdentityNovel]), + ), + 'batch-000004.json': JSON.stringify( + encodeNovelBatch([duplicateChapterIdentityNovel]), + ), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(payloads) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return payloads[path.split('/').pop() as keyof typeof payloads]; + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith([validNovel], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 3, + pluginIds: ['source'], + }); + }); + it('skips malformed compact files without partially restoring them', async () => { const options: BackupOptions = { library: true, @@ -1113,6 +1458,17 @@ describe('selective backup data', () => { }); const result = await restoreData('/cache'); + expect( + jest + .mocked(NativeFile.readFile) + .mock.calls.map(([path]) => path) + .filter(path => path.includes('/NovelAndChapters/')), + ).toEqual([ + ...Object.keys(malformedFiles) + .sort() + .map(name => `/cache/NovelAndChapters/${name}`), + '/cache/NovelAndChapters/batch-000007.json', + ]); expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( @@ -1128,12 +1484,16 @@ describe('selective backup data', () => { ], }), ], - { includeChapterMappings: true }, + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, ); expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); expect(result).toMatchObject({ + restoreRunId: expect.any(String), novelCount: 1, - failedNovelCount: 6, + failedNovelCount: 7, }); }); @@ -1154,5 +1514,6 @@ describe('selective backup data', () => { downloadedFiles: true, }, }); + expect(result.restoreRunId).toEqual(expect.any(String)); }); }); diff --git a/src/services/backup/drive/index.ts b/src/services/backup/drive/index.ts index e65e4cca4c..40da78bff7 100644 --- a/src/services/backup/drive/index.ts +++ b/src/services/backup/drive/index.ts @@ -11,6 +11,7 @@ import { import { finalizeRestoredPlugins, getRestoreCompletionText, + type RestoreResult, } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import { download, updateMetadata, uploadMedia } from '@api/drive/request'; @@ -27,6 +28,7 @@ import { restoreNovelFiles, } from '../fileSections'; import { resolveBackupOptions } from '../options'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; const uploadBackupSection = async ( sourcePath: string, @@ -99,87 +101,100 @@ export const driveRestore = async ( backupFolder: DriveFile, setMeta: TaskProgressUpdater, ) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - const zipDataFile = await exists(ZipBackupName.DATA, false, backupFolder.id); - if (!zipDataFile) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - - await clearBackupCache(); - await download(zipDataFile, CACHE_DIR_PATH); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFile = await exists( - ZipBackupName.DOWNLOAD, + let restoreResult: RestoreResult | undefined; + try { + setMeta(meta => ({ + ...meta, + isRunning: true, + progress: 0 / 3, + progressText: getString('backupScreen.downloadingData'), + })); + + const zipDataFile = await exists( + ZipBackupName.DATA, false, backupFolder.id, ); - if (!legacyFile) { + if (!zipDataFile) { throw new Error(getString('backupScreen.invalidBackupFolder')); } - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download(legacyFile, legacyFilesRestorePath); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - 2, - )) { - const file = await exists(section.archiveName, false, backupFolder.id); - if (!file) { + + await clearBackupCache(); + await download(zipDataFile, CACHE_DIR_PATH); + await sleep(500); + + setMeta(meta => ({ + ...meta, + progress: 1 / 3, + progressText: getString('backupScreen.restoringData'), + })); + + restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); + await sleep(500); + + setMeta(meta => ({ + ...meta, + progress: 2 / 3, + progressText: getString('backupScreen.restoringSelectedFiles'), + })); + + if (restoreResult.manifest.formatVersion === 1) { + const legacyFile = await exists( + ZipBackupName.DOWNLOAD, + false, + backupFolder.id, + ); + if (!legacyFile) { throw new Error(getString('backupScreen.invalidBackupFolder')); } - await download( - file, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, + const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); + await download(legacyFile, legacyFilesRestorePath); + await restoreLegacyFiles( + legacyFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); + } else { + const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); + for (const section of getSelectedBackupFileSections( + restoreResult.manifest.sections, + 2, + )) { + const file = await exists(section.archiveName, false, backupFolder.id); + if (!file) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + await download( + file, + section.archiveName === ZipBackupName.NOVEL_FILES + ? novelFilesRestorePath + : section.storagePath, + ); + } + if (restoreResult.manifest.sections.downloadedFiles) { + await restoreNovelFiles( + novelFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + } } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); + const missingPluginIds = await finalizeRestoredPlugins(restoreResult); + const completionText = getRestoreCompletionText( + restoreResult, + missingPluginIds, + ); - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); + setMeta(meta => ({ + ...meta, + progress: 3 / 3, + isRunning: false, + progressText: completionText, + completionText, + })); + } finally { + if (restoreResult) { + await clearRestoreChapterMappings(restoreResult.restoreRunId); + } + } }; diff --git a/src/services/backup/fileSections.ts b/src/services/backup/fileSections.ts index 5ff536db40..71bde584cc 100644 --- a/src/services/backup/fileSections.ts +++ b/src/services/backup/fileSections.ts @@ -3,7 +3,7 @@ import type { BackupOptions } from './options'; import { ZipBackupName } from './types'; import NativeFile from '@modules/native-file'; import type { RestoredNovelMapping } from '@database/types'; - +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; export type BackupFileSection = { archiveName: ZipBackupName; storagePath: string; @@ -52,9 +52,12 @@ const moveDirectoryContents = async (source: string, destination: string) => { } }; +const RESTORE_CHAPTER_LOOKUP_BATCH_SIZE = 100; + export const restoreNovelFiles = async ( stagingPath: string, novelMappings: RestoredNovelMapping[], + restoreRunId: string, ) => { for (const mapping of novelMappings) { const sourceNovelPath = `${stagingPath}/${mapping.pluginId}/${mapping.backupNovelId}`; @@ -64,28 +67,48 @@ export const restoreNovelFiles = async ( const destinationNovelPath = `${NOVEL_STORAGE}/${mapping.pluginId}/${mapping.restoredNovelId}`; await NativeFile.mkdir(destinationNovelPath); - const chapterIds = new Map( - mapping.chapters.map(chapter => [ - String(chapter.backupChapterId), - chapter.restoredChapterId, - ]), - ); - - for (const item of await NativeFile.readDir(sourceNovelPath)) { + const items = await NativeFile.readDir(sourceNovelPath); + for (const item of items) { if (!item.isDirectory) { await NativeFile.moveFile( item.path, `${destinationNovelPath}/${item.name}`, ); - continue; } + } - const restoredChapterId = chapterIds.get(item.name); - if (restoredChapterId !== undefined) { - await moveDirectoryContents( - item.path, - `${destinationNovelPath}/${restoredChapterId}`, - ); + const chapterItems = items.filter(item => item.isDirectory); + for ( + let start = 0; + start < chapterItems.length; + start += RESTORE_CHAPTER_LOOKUP_BATCH_SIZE + ) { + const chapterBatch = chapterItems.slice( + start, + start + RESTORE_CHAPTER_LOOKUP_BATCH_SIZE, + ); + const backupChapterIds = chapterBatch + .map(item => Number(item.name)) + .filter(id => Number.isInteger(id) && id > 0); + const chapterMappings = await getRestoreChapterMappings( + restoreRunId, + mapping.backupNovelId, + backupChapterIds, + ); + const restoredChapterIds = new Map( + chapterMappings.map(chapter => [ + String(chapter.backupChapterId), + chapter.restoredChapterId, + ]), + ); + for (const item of chapterBatch) { + const restoredChapterId = restoredChapterIds.get(item.name); + if (restoredChapterId !== undefined) { + await moveDirectoryContents( + item.path, + `${destinationNovelPath}/${restoredChapterId}`, + ); + } } } } @@ -98,9 +121,10 @@ export const restoreNovelFiles = async ( export const restoreLegacyFiles = async ( stagingPath: string, novelMappings: RestoredNovelMapping[], + restoreRunId: string, ) => { await moveDirectoryContents(`${stagingPath}/Plugins`, PLUGIN_STORAGE); - await restoreNovelFiles(`${stagingPath}/Novels`, novelMappings); + await restoreNovelFiles(`${stagingPath}/Novels`, novelMappings, restoreRunId); if (await NativeFile.exists(stagingPath)) { await NativeFile.unlink(stagingPath); } diff --git a/src/services/backup/local/index.ts b/src/services/backup/local/index.ts index f4eea43231..ef596c8e97 100644 --- a/src/services/backup/local/index.ts +++ b/src/services/backup/local/index.ts @@ -7,6 +7,7 @@ import { import { finalizeRestoredPlugins, getRestoreCompletionText, + type RestoreResult, } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import NativeZipArchive from '@modules/native-zip-archive'; @@ -24,6 +25,7 @@ import { restoreNovelFiles, } from '../fileSections'; import { resolveBackupOptions, type BackupOptions } from '../options'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; const logRestoreBenchmark = (message: string) => { if (!__DEV__) { @@ -119,6 +121,7 @@ export const restoreBackup = async ( setMeta?: TaskProgressUpdater, ) => { logRestoreBenchmark('local:start'); + let restoreResult: RestoreResult | undefined; try { setMeta?.(meta => ({ ...meta, @@ -151,7 +154,7 @@ export const restoreBackup = async ( await sleep(200); - const restoreResult = await restoreData( + restoreResult = await restoreData( CACHE_DIR_PATH, setMeta, logRestoreBenchmark, @@ -164,10 +167,10 @@ export const restoreBackup = async ( } const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); await NativeZipArchive.unzip(legacyArchive, legacyFilesRestorePath); - logRestoreBenchmark('local:selected-archives:done'); await restoreLegacyFiles( legacyFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); logRestoreBenchmark('local:downloaded-files:done'); } else { @@ -204,10 +207,11 @@ export const restoreBackup = async ( ? `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}` : novelFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); } - logRestoreBenchmark('local:downloaded-files:done'); } + logRestoreBenchmark('local:downloaded-files:done'); logRestoreBenchmark('local:selected-files:done'); const missingPluginIds = await finalizeRestoredPlugins(restoreResult); const completionText = getRestoreCompletionText( @@ -229,5 +233,9 @@ export const restoreBackup = async ( isRunning: false, })); throw error; + } finally { + if (restoreResult) { + await clearRestoreChapterMappings(restoreResult.restoreRunId); + } } }; diff --git a/src/services/backup/novelPayload.ts b/src/services/backup/novelPayload.ts index ff261466dd..27a7a80198 100644 --- a/src/services/backup/novelPayload.ts +++ b/src/services/backup/novelPayload.ts @@ -83,7 +83,7 @@ const isString = (value: unknown): value is string => typeof value === 'string'; const isNumber = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value); const isId = (value: unknown): value is number => - isNumber(value) && Number.isInteger(value); + isNumber(value) && Number.isInteger(value) && value > 0; const isBoolean = (value: unknown): value is boolean => typeof value === 'boolean'; @@ -93,6 +93,177 @@ const nullableNumber = (value: unknown): value is NullableNumber => isNullable(value, isNumber); const nullableBoolean = (value: unknown): value is NullableBoolean => isNullable(value, isBoolean); +const nonEmptyString = (value: unknown): value is string => + isString(value) && value.trim().length > 0; + +const nullableLegacyString = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isString(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyNumber = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isNumber(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyBoolean = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isBoolean(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const positiveId = (value: unknown): value is number => + isId(value) && value > 0; + +const normalizeLegacyChapter = ( + value: unknown, + containingNovelId: number, +): ChapterInfo => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy chapter'); + } + const chapter = value as Record; + if ( + !positiveId(chapter.id) || + !nonEmptyString(chapter.path) || + !nonEmptyString(chapter.name) || + (chapter.novelId !== undefined && + chapter.novelId !== null && + !positiveId(chapter.novelId)) + ) { + throw new Error('Invalid legacy chapter fields'); + } + return { + id: chapter.id, + novelId: containingNovelId, + path: chapter.path, + name: chapter.name, + releaseTime: nullableLegacyString(chapter, 'releaseTime'), + readTime: nullableLegacyString(chapter, 'readTime'), + bookmark: nullableLegacyBoolean(chapter, 'bookmark'), + unread: nullableLegacyBoolean(chapter, 'unread'), + isDownloaded: nullableLegacyBoolean(chapter, 'isDownloaded'), + updatedTime: nullableLegacyString(chapter, 'updatedTime'), + chapterNumber: nullableLegacyNumber(chapter, 'chapterNumber'), + page: nullableLegacyString(chapter, 'page'), + progress: nullableLegacyNumber(chapter, 'progress'), + position: nullableLegacyNumber(chapter, 'position'), + scanlator: nullableLegacyString(chapter, 'scanlator'), + timeSpent: nullableLegacyNumber(chapter, 'timeSpent'), + }; +}; + +export const validateBackupNovel = (novel: BackupNovel): BackupNovel => { + if ( + !positiveId(novel.id) || + !nonEmptyString(novel.path) || + !nonEmptyString(novel.pluginId) || + !isString(novel.name) || + !Array.isArray(novel.chapters) + ) { + throw new Error('Invalid novel fields'); + } + const stringFields = [ + novel.cover, + novel.summary, + novel.author, + novel.artist, + novel.status, + novel.genres, + ]; + if (stringFields.some(value => value !== null && !isString(value))) { + throw new Error('Invalid novel string field'); + } + const booleanFields = [novel.inLibrary, novel.isLocal]; + if (booleanFields.some(value => value !== null && !isBoolean(value))) { + throw new Error('Invalid novel boolean field'); + } + if (novel.totalPages !== null && !isNumber(novel.totalPages)) { + throw new Error('Invalid novel number field'); + } + for (const chapter of novel.chapters) { + if ( + !positiveId(chapter.id) || + !positiveId(chapter.novelId) || + !nonEmptyString(chapter.path) || + !isStringOrNull(chapter.releaseTime) || + !isStringOrNull(chapter.readTime) || + !isBooleanOrNull(chapter.bookmark) || + !isBooleanOrNull(chapter.unread) || + !isBooleanOrNull(chapter.isDownloaded) || + !isStringOrNull(chapter.updatedTime) || + !isNumberOrNull(chapter.chapterNumber) || + !isStringOrNull(chapter.page) || + !isNumberOrNull(chapter.progress) || + !isNumberOrNull(chapter.position) || + !isStringOrNull(chapter.scanlator) || + !isNumberOrNull(chapter.timeSpent) + ) { + throw new Error('Invalid chapter fields'); + } + } + return novel; +}; + +const isStringOrNull = (value: unknown): value is string | null => + value === null || isString(value); +const isNumberOrNull = (value: unknown): value is number | null => + value === null || isNumber(value); +const isBooleanOrNull = (value: unknown): value is boolean | null => + value === null || isBoolean(value); + +export const normalizeLegacyNovel = (value: unknown): BackupNovel => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy novel'); + } + const source = value as Record; + if ( + !positiveId(source.id) || + !nonEmptyString(source.path) || + !nonEmptyString(source.pluginId) || + !isString(source.name) || + !Array.isArray(source.chapters) + ) { + throw new Error('Invalid legacy novel fields'); + } + const novelId = source.id; + const novel: BackupNovel = { + id: novelId, + path: source.path, + pluginId: source.pluginId, + name: source.name, + cover: nullableLegacyString(source, 'cover'), + summary: nullableLegacyString(source, 'summary'), + author: nullableLegacyString(source, 'author'), + artist: nullableLegacyString(source, 'artist'), + status: nullableLegacyString(source, 'status'), + genres: nullableLegacyString(source, 'genres'), + inLibrary: nullableLegacyBoolean(source, 'inLibrary'), + isLocal: nullableLegacyBoolean(source, 'isLocal'), + totalPages: nullableLegacyNumber(source, 'totalPages'), + chapters: source.chapters.map(chapter => + normalizeLegacyChapter(chapter, novelId), + ), + }; + return validateBackupNovel(novel); +}; const assertNovelKeys = (novel: Record) => { const keys = Object.keys(novel).sort(); @@ -112,8 +283,8 @@ const isCompactChapter = (value: unknown): value is CompactChapter => { return ( isId(value[0]) && - isString(value[1]) && - isString(value[2]) && + nonEmptyString(value[1]) && + nonEmptyString(value[2]) && nullableString(value[3]) && nullableBoolean(value[4]) && nullableBoolean(value[5]) && @@ -139,8 +310,8 @@ const isCompactNovel = (value: unknown): value is CompactNovel => { Array.isArray(novel.c) && novel.c.every(isCompactChapter) && isId(novel.id) && - isString(novel.p) && - isString(novel.pi) && + nonEmptyString(novel.p) && + nonEmptyString(novel.pi) && isString(novel.n) && nullableString(novel.co) && nullableString(novel.s) && diff --git a/src/services/backup/restoreResult.ts b/src/services/backup/restoreResult.ts index dc11bd9880..694e65c757 100644 --- a/src/services/backup/restoreResult.ts +++ b/src/services/backup/restoreResult.ts @@ -17,6 +17,7 @@ export type RestoreResult = { failedSectionCount: number; pluginIds: string[]; novelMappings: RestoredNovelMapping[]; + restoreRunId: string; manifest: ResolvedBackupManifest; }; diff --git a/src/services/backup/selfhost/index.ts b/src/services/backup/selfhost/index.ts index e3f81203b5..a5a6be8fc2 100644 --- a/src/services/backup/selfhost/index.ts +++ b/src/services/backup/selfhost/index.ts @@ -10,6 +10,7 @@ import { import { finalizeRestoredPlugins, getRestoreCompletionText, + type RestoreResult, } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import { ZipBackupName } from '../types'; @@ -25,6 +26,7 @@ import { restoreNovelFiles, } from '../fileSections'; import { resolveBackupOptions } from '../options'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; export const createSelfHostBackup = async ( { host, backupFolder, options: requestedOptions }: SelfHostData, @@ -76,79 +78,88 @@ export const selfHostRestore = async ( { host, backupFolder }: SelfHostData, setMeta: TaskProgressUpdater, ) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - await clearBackupCache(); - await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download( - host, - backupFolder, - ZipBackupName.DOWNLOAD, - legacyFilesRestorePath, - ); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - 2, - )) { + let restoreResult: RestoreResult | undefined; + try { + setMeta(meta => ({ + ...meta, + isRunning: true, + progress: 0 / 3, + progressText: getString('backupScreen.downloadingData'), + })); + + await clearBackupCache(); + await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); + + setMeta(meta => ({ + ...meta, + progress: 1 / 3, + progressText: getString('backupScreen.restoringData'), + })); + + await sleep(200); + + restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); + + setMeta(meta => ({ + ...meta, + progress: 2 / 3, + progressText: getString('backupScreen.restoringSelectedFiles'), + })); + + await sleep(200); + + if (restoreResult.manifest.formatVersion === 1) { + const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); await download( host, backupFolder, - section.archiveName, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, + ZipBackupName.DOWNLOAD, + legacyFilesRestorePath, ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, + await restoreLegacyFiles( + legacyFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); + } else { + const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); + for (const section of getSelectedBackupFileSections( + restoreResult.manifest.sections, + 2, + )) { + await download( + host, + backupFolder, + section.archiveName, + section.archiveName === ZipBackupName.NOVEL_FILES + ? novelFilesRestorePath + : section.storagePath, + ); + } + if (restoreResult.manifest.sections.downloadedFiles) { + await restoreNovelFiles( + novelFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + } } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); + const missingPluginIds = await finalizeRestoredPlugins(restoreResult); + const completionText = getRestoreCompletionText( + restoreResult, + missingPluginIds, + ); - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); + setMeta(meta => ({ + ...meta, + progress: 3 / 3, + isRunning: false, + progressText: completionText, + completionText, + })); + } finally { + if (restoreResult) { + await clearRestoreChapterMappings(restoreResult.restoreRunId); + } + } }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index b7885cc578..3eeb5fcd8b 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -5,6 +5,7 @@ import { MMKVStorage } from '@utils/mmkv/mmkv'; import { version } from '../../../package.json'; import { getAllNovels } from '@database/queries/NovelQueries'; import { + clearRestoreChapterMappings, _restoreNovelAndChapters, _restoreNovelsAndChapters, } from '@database/queries/NovelRestoreQueries'; @@ -19,7 +20,12 @@ import { BackupNovel, type RestoredNovelMapping, } from '@database/types'; -import { decodeNovelBatch, encodeNovelBatch } from './novelPayload'; +import { + decodeNovelBatch, + encodeNovelBatch, + normalizeLegacyNovel, + validateBackupNovel, +} from './novelPayload'; import { BackupEntryName, type BackupManifest, @@ -340,12 +346,126 @@ const updateRestoreProgress = ( })); }; +type BackupNovelFileDescriptor = { + name: string; + path: string; +}; + +const createRestoreRunId = () => + `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + +const decodeRestoreNovelFile = ( + fileContent: string, + manifest: ResolvedBackupManifest, +) => { + const payload: unknown = JSON.parse(fileContent); + const novels = + manifest.novelDataFormat === 2 + ? decodeNovelBatch(payload) + : [normalizeLegacyNovel(payload)]; + return novels.map(novel => { + const { + chaptersDownloaded: _chaptersDownloaded, + chaptersUnread: _chaptersUnread, + totalChapters: _totalChapters, + lastReadAt: _lastReadAt, + lastUpdatedAt: _lastUpdatedAt, + ...withoutAggregates + } = novel as BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; + }; + const normalized = validateBackupNovel(withoutAggregates); + return normalized.cover && !normalized.cover.startsWith('http') + ? { ...normalized, cover: APP_STORAGE_URI + normalized.cover } + : normalized; + }); +}; +const getNovelFileRecordCount = ( + fileContent: string, + manifest: ResolvedBackupManifest, +) => { + if (manifest.novelDataFormat !== 2) { + return 1; + } + try { + const payload: unknown = JSON.parse(fileContent); + return Array.isArray(payload) ? Math.max(1, payload.length) : 1; + } catch { + return 1; + } +}; + +const validateNovelFileRecords = ( + novels: BackupNovel[], + seenNovelIds: Set, + seenNovelIdentities: Set, + seenChapterIdentities: Set, +) => { + const fileNovelIds = new Set(); + const fileNovelIdentities = new Set(); + const fileChapterIdentities = new Set(); + for (const novel of novels) { + const novelIdentity = `${novel.pluginId}\u0000${novel.path}`; + if ( + fileNovelIds.has(novel.id) || + seenNovelIds.has(novel.id) || + fileNovelIdentities.has(novelIdentity) || + seenNovelIdentities.has(novelIdentity) + ) { + throw new Error('Duplicate backup novel identity'); + } + fileNovelIds.add(novel.id); + fileNovelIdentities.add(novelIdentity); + for (const chapter of novel.chapters) { + const chapterIdentity = `${novel.id}\u0000${chapter.path}`; + if ( + fileChapterIdentities.has(chapterIdentity) || + seenChapterIdentities.has(chapterIdentity) + ) { + throw new Error('Duplicate backup chapter identity'); + } + fileChapterIdentities.add(chapterIdentity); + } + } + for (const novelId of fileNovelIds) { + seenNovelIds.add(novelId); + } + for (const novelIdentity of fileNovelIdentities) { + seenNovelIdentities.add(novelIdentity); + } + for (const chapterIdentity of fileChapterIdentities) { + seenChapterIdentities.add(chapterIdentity); + } +}; + +const decodeAndValidateNovelFile = ( + fileContent: string, + manifest: ResolvedBackupManifest, + seenNovelIds?: Set, + seenNovelIdentities?: Set, + seenChapterIdentities?: Set, +) => { + const novels = decodeRestoreNovelFile(fileContent, manifest); + validateNovelFileRecords( + novels, + seenNovelIds ?? new Set(), + seenNovelIdentities ?? new Set(), + seenChapterIdentities ?? new Set(), + ); + return novels; +}; + type RestoreBenchmarkLogger = (message: string) => void; -export const restoreData = async ( +const restoreDataInternal = async ( cacheDirPath: string, - setMeta?: TaskProgressUpdater, - benchmarkLog?: RestoreBenchmarkLogger, + setMeta: TaskProgressUpdater | undefined, + benchmarkLog: RestoreBenchmarkLogger | undefined, + restoreRunId: string, ): Promise => { const manifest = await getBackupManifest(cacheDirPath); benchmarkLog?.('restoreData:manifest:loaded'); @@ -365,11 +485,12 @@ export const restoreData = async ( })(); let pluginsFromSettings: PluginItem[] = []; - benchmarkLog?.('restoreData:novels:start'); if (manifest.sections.library) { - updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); + benchmarkLog?.('restoreData:novels:validation:start'); + updateRestoreProgress(setMeta, getString('backupScreen.validatingNovels')); } let novelCount = 0; + let totalNovelCount = 0; let failedCount = 0; let failedSectionCount = 0; let readMs = 0; @@ -388,22 +509,75 @@ export const restoreData = async ( .sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0, ); + const validItems: BackupNovelFileDescriptor[] = []; + const seenNovelIds = new Set(); + const seenNovelIdentities = new Set(); + const seenChapterIdentities = new Set(); + + for (const [index, item] of items.entries()) { + if (index % 100 === 0) { + setTimeout(async () => { + updateRestoreProgress( + setMeta, + getString('backupScreen.validatingNovelsProgress', { + current: index + 1, + total: items.length, + }), + ); + }, 0) + } + const readStartedAt = performance.now(); + let fileContent: string; + try { + fileContent = await NativeFile.readFile(item.path); + } catch { + failedCount++; + continue; + } finally { + readMs += performance.now() - readStartedAt; + } + + const parseStartedAt = performance.now(); + try { + const validatedNovels = decodeAndValidateNovelFile( + fileContent, + manifest, + seenNovelIds, + seenNovelIdentities, + seenChapterIdentities, + ); + totalNovelCount += validatedNovels.length; + validItems.push({ name: item.name, path: item.path }); + } catch { + failedCount += getNovelFileRecordCount(fileContent, manifest); + } finally { + parseMs += performance.now() - parseStartedAt; + } + } + benchmarkLog?.( + `restoreData:novels:validation:done total=${totalNovelCount}`, + ); + const pendingNovels: BackupNovel[] = []; - const restoreNovelBatchSize = RESTORE_NOVEL_BATCH_SIZE; const restoreNovelBatch = async () => { if (pendingNovels.length === 0) { return; } const batch = pendingNovels.splice(0, pendingNovels.length); + const restoreOptions = { + includeChapterMappings: manifest.sections.downloadedFiles, + ...(manifest.sections.downloadedFiles ? { restoreRunId } : {}), + }; let restoredNovels: { backupNovel: BackupNovel; mapping: RestoredNovelMapping; - }[]; + }[] = []; const databaseStartedAt = performance.now(); try { - const mappings = await _restoreNovelsAndChapters(batch, { - includeChapterMappings: manifest.sections.downloadedFiles, - }); + const mappings = await _restoreNovelsAndChapters( + batch, + restoreOptions, + ); if (mappings.length !== batch.length) { throw new Error('Restore returned incomplete novel mappings'); } @@ -412,14 +586,14 @@ export const restoreData = async ( mapping: mappings[index], })); } catch { - restoredNovels = []; for (const backupNovel of batch) { try { restoredNovels.push({ backupNovel, - mapping: await _restoreNovelAndChapters(backupNovel, { - includeChapterMappings: manifest.sections.downloadedFiles, - }), + mapping: await _restoreNovelAndChapters( + backupNovel, + restoreOptions, + ), }); } catch { failedCount++; @@ -464,16 +638,13 @@ export const restoreData = async ( } }; - for (const [index, item] of items.entries()) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringNovelsProgress', { - current: index + 1, - total: items.length, - }), - ); - let fileContent: string; + benchmarkLog?.( + `restoreData:novels:restore:start total=${totalNovelCount}`, + ); + updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); + for (const item of validItems) { const readStartedAt = performance.now(); + let fileContent: string; try { fileContent = await NativeFile.readFile(item.path); } catch { @@ -485,28 +656,45 @@ export const restoreData = async ( const parseStartedAt = performance.now(); try { - const decodedNovels = - manifest.novelDataFormat === 2 - ? decodeNovelBatch(JSON.parse(fileContent!)) - : [JSON.parse(fileContent!) as BackupNovel]; + const decodedNovels = decodeAndValidateNovelFile( + fileContent, + manifest, + ); for (const backupNovel of decodedNovels) { pluginIds.add(backupNovel.pluginId); - if (backupNovel.cover && !backupNovel.cover.startsWith('http')) { - backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; - } pendingNovels.push(backupNovel); } } catch { - failedCount++; + failedCount += getNovelFileRecordCount(fileContent, manifest); } finally { parseMs += performance.now() - parseStartedAt; } - if (pendingNovels.length >= restoreNovelBatchSize) { + if (pendingNovels.length >= RESTORE_NOVEL_BATCH_SIZE) { await restoreNovelBatch(); + updateRestoreProgress( + setMeta, + getString('backupScreen.restoringNovelsProgress', { + current: novelCount, + total: totalNovelCount, + }), + ); + benchmarkLog?.( + `restoreData:novels:restore:progress current=${novelCount} total=${totalNovelCount}`, + ); } } await restoreNovelBatch(); + updateRestoreProgress( + setMeta, + getString('backupScreen.restoringNovelsProgress', { + current: novelCount, + total: totalNovelCount, + }), + ); + benchmarkLog?.( + `restoreData:novels:restore:progress current=${novelCount} total=${totalNovelCount}`, + ); benchmarkLog?.( `restoreData:novels:done count=${novelCount} failed=${failedCount} readMs=${readMs.toFixed( 1, @@ -639,6 +827,26 @@ export const restoreData = async ( failedSectionCount, pluginIds: [...pluginIds], novelMappings, + restoreRunId, manifest, }; }; + +export const restoreData = async ( + cacheDirPath: string, + setMeta?: TaskProgressUpdater, + benchmarkLog?: RestoreBenchmarkLogger, +): Promise => { + const restoreRunId = createRestoreRunId(); + try { + return await restoreDataInternal( + cacheDirPath, + setMeta, + benchmarkLog, + restoreRunId, + ); + } catch (error) { + await clearRestoreChapterMappings(restoreRunId).catch(() => undefined); + throw error; + } +}; From 2a30d79b75f312752576a7750c3b9b10dab421e7 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Wed, 23 Sep 2026 13:12:19 +0200 Subject: [PATCH 07/10] Fix backup restore and ZIP edge cases Generated with OpenAI Codex --- .../NativeZipArchiveModule.kt | 10 +- .../manager/__tests__/manager.test.ts | 18 ++- src/database/manager/manager.ts | 6 +- .../queries/__tests__/ChapterQueries.test.ts | 6 +- src/database/queries/__tests__/testData.ts | 1 + src/services/backup/__tests__/local.test.ts | 90 ++++++++++++++ src/services/backup/__tests__/utils.test.ts | 110 +++++++++++++++++- src/services/backup/drive/index.ts | 4 +- src/services/backup/local/index.ts | 4 +- src/services/backup/selfhost/index.ts | 4 +- src/services/backup/utils.ts | 52 +++++---- 11 files changed, 270 insertions(+), 35 deletions(-) diff --git a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt index e000269efd..044696763e 100644 --- a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt +++ b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt @@ -18,9 +18,13 @@ class NativeZipArchiveModule : Module() { zos: ZipOutputStream, archivePrefix: String = "", archiveEntries: MutableSet? = null, + excludedFile: File? = null, ) { val sourceDir = File(sourceDirPath) - sourceDir.walkBottomUp().filter { it.isFile }.forEach { file -> + val excludedCanonicalPath = excludedFile?.canonicalPath + sourceDir.walkBottomUp().filter { file -> + file.isFile && file.canonicalPath != excludedCanonicalPath + }.forEach { file -> val relativePath = file.absolutePath.removePrefix(sourceDir.absolutePath).removePrefix("/") val zipFileName = joinArchivePath(archivePrefix, relativePath) @@ -90,7 +94,7 @@ class NativeZipArchiveModule : Module() { Thread { try { FileOutputStream(zipFilePath).use { fos -> - ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos) } + ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos, excludedFile = File(zipFilePath)) } } promise.resolve(null) } catch (e: Exception) { @@ -120,7 +124,7 @@ class NativeZipArchiveModule : Module() { FileOutputStream(zipFilePath).use { fos -> ZipOutputStream(fos).use { zos -> normalizedSources.forEach { (sourceDirPath, archivePrefix) -> - zipProcess(sourceDirPath, zos, archivePrefix, archiveEntries) + zipProcess(sourceDirPath, zos, archivePrefix, archiveEntries, File(zipFilePath)) } } } diff --git a/src/database/manager/__tests__/manager.test.ts b/src/database/manager/__tests__/manager.test.ts index 97b34fb2a5..41f76efd31 100644 --- a/src/database/manager/__tests__/manager.test.ts +++ b/src/database/manager/__tests__/manager.test.ts @@ -5,13 +5,20 @@ import { describe('DbManager.executeBatch', () => { const testDb = createTestDb(); + const flushPendingReactiveQueries = jest.fn(); afterAll(() => { cleanupTestDb(testDb); }); beforeEach(() => { + flushPendingReactiveQueries.mockClear(); + Object.defineProperty(testDb.sqlite, 'flushPendingReactiveQueries', { + configurable: true, + value: flushPendingReactiveQueries, + }); testDb.sqlite.executeSync('DROP TABLE IF EXISTS BatchTest'); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS MissingBatchTable'); testDb.sqlite.executeSync( 'CREATE TABLE BatchTest (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL)', ); @@ -25,6 +32,7 @@ describe('DbManager.executeBatch', () => { ]); expect(result.rowsAffected).toBe(4); + expect(flushPendingReactiveQueries).toHaveBeenCalledTimes(1); expect( testDb.sqlite.executeSync('SELECT value FROM BatchTest').rows, ).toEqual([{ value: 'updated' }]); @@ -36,10 +44,18 @@ describe('DbManager.executeBatch', () => { ['INSERT INTO BatchTest (value) VALUES (?)', [['rolled back']]], ['INSERT INTO MissingBatchTable (value) VALUES (?)', [['failure']]], ]), - ).rejects.toThrow(); + ).rejects.toThrow('MissingBatchTable'); + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); expect(testDb.sqlite.executeSync('SELECT * FROM BatchTest').rows).toEqual( [], ); }); + + it('does not flush for an empty batch', async () => { + const result = await testDb.dbManager.executeBatch([]); + + expect(result).toEqual({ rowsAffected: 0 }); + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + }); }); diff --git a/src/database/manager/manager.ts b/src/database/manager/manager.ts index 1354ceae7f..6ce98c6ef2 100644 --- a/src/database/manager/manager.ts +++ b/src/database/manager/manager.ts @@ -101,7 +101,11 @@ class DbManager implements IDbManager { return await this.queue.enqueue({ id: 'write', - run: async () => await this.db.$client.executeBatch(commands), + run: async () => { + const result = await this.db.$client.executeBatch(commands); + this.db.$client.flushPendingReactiveQueries(); + return result; + }, }); } diff --git a/src/database/queries/__tests__/ChapterQueries.test.ts b/src/database/queries/__tests__/ChapterQueries.test.ts index 12c9a95305..cf277170ee 100644 --- a/src/database/queries/__tests__/ChapterQueries.test.ts +++ b/src/database/queries/__tests__/ChapterQueries.test.ts @@ -153,9 +153,7 @@ describe('ChapterQueries', () => { describe('getAllNovelChaptersForBackup', () => { it('returns chapters beyond the 1000-row UI query limit', async () => { const testDb = getTestDb(); - // No Novel row is needed for this selector test. Keeping the id orphaned - // also avoids running aggregate-stat triggers 1001 times during setup. - const novelId = 123456; + const novelId = await insertTestNovel(testDb, { id: 123456 }); const values = Array.from( { length: 1001 }, @@ -188,6 +186,8 @@ describe('ChapterQueries', () => { it('returns chapters for multiple novels in novel and chapter order', async () => { const testDb = getTestDb(); + await insertTestNovel(testDb, { id: 10 }); + await insertTestNovel(testDb, { id: 20 }); testDb.sqlite.executeSync(` INSERT INTO Chapter (novelId, path, name, chapterNumber, page, position) diff --git a/src/database/queries/__tests__/testData.ts b/src/database/queries/__tests__/testData.ts index c4a3dc9df3..cd2761495d 100644 --- a/src/database/queries/__tests__/testData.ts +++ b/src/database/queries/__tests__/testData.ts @@ -21,6 +21,7 @@ import { */ export function clearAllTables(testDb: TestDb) { const { sqlite } = testDb; + sqlite.executeSync('DELETE FROM NovelCategory'); sqlite.executeSync('DELETE FROM RestoreChapterMapping'); sqlite.executeSync('DELETE FROM Chapter'); sqlite.executeSync('DELETE FROM Novel'); diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index b5a371d5a9..19ef368dd1 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -23,6 +23,15 @@ jest.mock('../fileSections', () => { jest.mock('../utils', () => ({ CACHE_DIR_PATH: '/cache/BackupData', clearBackupCache: jest.fn(), + clearRestoreChapterMappingsSafely: jest.fn(async (restoreRunId: string) => { + try { + await jest + .requireMock('@database/queries/NovelRestoreQueries') + .clearRestoreChapterMappings(restoreRunId); + } catch { + // Match the production helper's best-effort cleanup. + } + }), prepareBackupData: jest.fn(), restoreData: jest.fn(), })); @@ -218,6 +227,87 @@ describe('local selective backup', () => { jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], ); }); + it('keeps a successful restore successful when mapping cleanup fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-cleanup-failure', + manifest: { + appVersion: '2.1.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('preserves the restore error when mapping cleanup also fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-restore-failure', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + it('extracts the v1 downloaded archive into the legacy staging path', async () => { const restoreResult = { novelCount: 1, diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index d84f5e2082..3b77d86f6c 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -1,4 +1,5 @@ import { + clearRestoreChapterMappings, _restoreNovelAndChapters, _restoreNovelsAndChapters, } from '@database/queries/NovelRestoreQueries'; @@ -11,7 +12,11 @@ import { } from '@database/queries/CategoryQueries'; import NativeFile from '@modules/native-file'; import { MMKVStorage } from '@utils/mmkv/mmkv'; -import { prepareBackupData, restoreData } from '../utils'; +import { + clearRestoreChapterMappingsSafely, + prepareBackupData, + restoreData, +} from '../utils'; import { decodeNovelBatch, encodeNovelBatch } from '../novelPayload'; import type { BackupNovel, @@ -678,6 +683,99 @@ describe('selective backup data', () => { expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); }); + it('keeps restored mappings when a stored cover copy fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const novels = [1, 2, 3].map(id => ({ + ...makeTestNovel(id), + cover: `/Novels/source/${id}/cover.png`, + })); + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + const category = { id: 1, name: 'Category', novelIds: [1, 2, 3] }; + + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/NovelAndChapters', + '/cache/Covers/1', + '/cache/Covers/2', + '/cache/Covers/3', + '/cache/Category.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return JSON.stringify([category]); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + if (!novel) { + throw new Error(`Unexpected read: ${path}`); + } + return JSON.stringify(novel); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + jest.mocked(NativeFile.copyFile).mockImplementation(async source => { + if (source === '/cache/Covers/1') { + throw new Error('Cover copy failed'); + } + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + novels.map(novel => ({ + ...novel, + cover: `file:///storage${novel.cover}`, + })), + { includeChapterMappings: false }, + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/2', + '/storage/Novels/source/102/cover.png', + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/3', + '/storage/Novels/source/103/cover.png', + ); + expect(_restoreCategory).toHaveBeenCalledWith( + expect.objectContaining({ novelIds: [1, 2, 3] }), + new Map([ + [1, 101], + [2, 102], + [3, 103], + ]), + ); + expect(result).toMatchObject({ + novelCount: 3, + failedNovelCount: 1, + failedSectionCount: 0, + novelMappings: mappings, + }); + }); + it('omits the installed-plugin registry when plugin files are excluded', async () => { jest .mocked(MMKVStorage.getAllKeys) @@ -1516,4 +1614,14 @@ describe('selective backup data', () => { }); expect(result.restoreRunId).toEqual(expect.any(String)); }); + it('does not reject when restore mapping cleanup fails', async () => { + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + clearRestoreChapterMappingsSafely('restore-run'), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith('restore-run'); + }); }); diff --git a/src/services/backup/drive/index.ts b/src/services/backup/drive/index.ts index 40da78bff7..8068387416 100644 --- a/src/services/backup/drive/index.ts +++ b/src/services/backup/drive/index.ts @@ -7,6 +7,7 @@ import { clearBackupCache, prepareBackupData, restoreData, + clearRestoreChapterMappingsSafely, } from '../utils'; import { finalizeRestoredPlugins, @@ -28,7 +29,6 @@ import { restoreNovelFiles, } from '../fileSections'; import { resolveBackupOptions } from '../options'; -import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; const uploadBackupSection = async ( sourcePath: string, @@ -194,7 +194,7 @@ export const driveRestore = async ( })); } finally { if (restoreResult) { - await clearRestoreChapterMappings(restoreResult.restoreRunId); + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); } } }; diff --git a/src/services/backup/local/index.ts b/src/services/backup/local/index.ts index ef596c8e97..51959cc455 100644 --- a/src/services/backup/local/index.ts +++ b/src/services/backup/local/index.ts @@ -1,5 +1,6 @@ import { CACHE_DIR_PATH, + clearRestoreChapterMappingsSafely, clearBackupCache, prepareBackupData, restoreData, @@ -25,7 +26,6 @@ import { restoreNovelFiles, } from '../fileSections'; import { resolveBackupOptions, type BackupOptions } from '../options'; -import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; const logRestoreBenchmark = (message: string) => { if (!__DEV__) { @@ -235,7 +235,7 @@ export const restoreBackup = async ( throw error; } finally { if (restoreResult) { - await clearRestoreChapterMappings(restoreResult.restoreRunId); + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); } } }; diff --git a/src/services/backup/selfhost/index.ts b/src/services/backup/selfhost/index.ts index a5a6be8fc2..07919eacd2 100644 --- a/src/services/backup/selfhost/index.ts +++ b/src/services/backup/selfhost/index.ts @@ -6,6 +6,7 @@ import { clearBackupCache, prepareBackupData, restoreData, + clearRestoreChapterMappingsSafely, } from '../utils'; import { finalizeRestoredPlugins, @@ -26,7 +27,6 @@ import { restoreNovelFiles, } from '../fileSections'; import { resolveBackupOptions } from '../options'; -import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; export const createSelfHostBackup = async ( { host, backupFolder, options: requestedOptions }: SelfHostData, @@ -159,7 +159,7 @@ export const selfHostRestore = async ( })); } finally { if (restoreResult) { - await clearRestoreChapterMappings(restoreResult.restoreRunId); + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); } } }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index 3eeb5fcd8b..fbb2bcca65 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -516,16 +516,14 @@ const restoreDataInternal = async ( for (const [index, item] of items.entries()) { if (index % 100 === 0) { - setTimeout(async () => { - updateRestoreProgress( - setMeta, - getString('backupScreen.validatingNovelsProgress', { - current: index + 1, - total: items.length, - }), - ); - }, 0) - } + updateRestoreProgress( + setMeta, + getString('backupScreen.validatingNovelsProgress', { + current: index + 1, + total: items.length, + }), + ); + } const readStartedAt = performance.now(); let fileContent: string; try { @@ -615,16 +613,20 @@ const restoreDataInternal = async ( ); await Promise.all( coverBatch.map(async ({ backupNovel, mapping: novelMapping }) => { - if ( - !manifest.sections.downloadedFiles && - backupNovel.cover?.startsWith(APP_STORAGE_URI) - ) { - const coverBackupPath = coversDirPath + '/' + backupNovel.id; - if (await NativeFile.exists(coverBackupPath)) { - const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; - await NativeFile.mkdir(parentDirectory(coverPath)); - await NativeFile.copyFile(coverBackupPath, coverPath); + try { + if ( + !manifest.sections.downloadedFiles && + backupNovel.cover?.startsWith(APP_STORAGE_URI) + ) { + const coverBackupPath = coversDirPath + '/' + backupNovel.id; + if (await NativeFile.exists(coverBackupPath)) { + const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; + await NativeFile.mkdir(parentDirectory(coverPath)); + await NativeFile.copyFile(coverBackupPath, coverPath); + } } + } catch { + failedCount++; } }), ); @@ -832,6 +834,16 @@ const restoreDataInternal = async ( }; }; +export const clearRestoreChapterMappingsSafely = async ( + restoreRunId: string, +) => { + try { + await clearRestoreChapterMappings(restoreRunId); + } catch { + // Restore mappings are run-scoped and do not affect later restores. + } +}; + export const restoreData = async ( cacheDirPath: string, setMeta?: TaskProgressUpdater, @@ -846,7 +858,7 @@ export const restoreData = async ( restoreRunId, ); } catch (error) { - await clearRestoreChapterMappings(restoreRunId).catch(() => undefined); + await clearRestoreChapterMappingsSafely(restoreRunId); throw error; } }; From 1ebebe3de68a782e0c50684bc46e5dd46038c4f9 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Wed, 23 Sep 2026 13:44:28 +0200 Subject: [PATCH 08/10] fix: harden backup restore and archive handling Generated with OpenAI Codex Co-authored-by: OpenAI Codex --- .../NativeZipArchiveModule.kt | 15 +++++++++-- .../queries/__tests__/ChapterQueries.test.ts | 27 ++++++++++--------- src/services/backup/__tests__/utils.test.ts | 18 +++---------- src/services/backup/novelPayload.ts | 4 +-- src/services/backup/utils.ts | 3 ++- 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt index 044696763e..88c2ec6dee 100644 --- a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt +++ b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt @@ -66,6 +66,17 @@ class NativeZipArchiveModule : Module() { createdDirectories.add(parentPath) } } + private fun resolveZipEntry(destination: File, entryName: String): File { + val canonicalDestination = destination.canonicalFile + val outputFile = File(destination, entryName).canonicalFile + val destinationPath = + canonicalDestination.path.let { if (it.endsWith(File.separator)) it else it + File.separator } + require(outputFile.path.startsWith(destinationPath)) { + "ZIP entry is outside the destination directory: $entryName" + } + return outputFile + } + override fun definition() = ModuleDefinition { Name("NativeZipArchive") @@ -76,7 +87,7 @@ class NativeZipArchiveModule : Module() { val createdDirectories = mutableSetOf() ZipFile(sourceFilePath).use { zis -> zis.entries().asSequence().filterNot { it.isDirectory }.forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) ensureParentDirectory(newFile, createdDirectories) zis.getInputStream(zipEntry).use { inputStream -> FileOutputStream(newFile).use { fos -> inputStream.copyTo(fos, COPY_BUFFER_SIZE) } @@ -148,7 +159,7 @@ class NativeZipArchiveModule : Module() { generateSequence { zis.nextEntry } .filterNot { it.isDirectory } .forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) ensureParentDirectory(newFile, createdDirectories) FileOutputStream(newFile).use { fos -> zis.copyTo(fos, COPY_BUFFER_SIZE) } } diff --git a/src/database/queries/__tests__/ChapterQueries.test.ts b/src/database/queries/__tests__/ChapterQueries.test.ts index cf277170ee..263d790888 100644 --- a/src/database/queries/__tests__/ChapterQueries.test.ts +++ b/src/database/queries/__tests__/ChapterQueries.test.ts @@ -153,7 +153,7 @@ describe('ChapterQueries', () => { describe('getAllNovelChaptersForBackup', () => { it('returns chapters beyond the 1000-row UI query limit', async () => { const testDb = getTestDb(); - const novelId = await insertTestNovel(testDb, { id: 123456 }); + const novelId = await insertTestNovel(testDb); const values = Array.from( { length: 1001 }, @@ -186,25 +186,28 @@ describe('ChapterQueries', () => { it('returns chapters for multiple novels in novel and chapter order', async () => { const testDb = getTestDb(); - await insertTestNovel(testDb, { id: 10 }); - await insertTestNovel(testDb, { id: 20 }); + const firstNovelId = await insertTestNovel(testDb); + const secondNovelId = await insertTestNovel(testDb); testDb.sqlite.executeSync(` INSERT INTO Chapter (novelId, path, name, chapterNumber, page, position) VALUES - (10, '/chapter/1', 'Chapter 1', 1, '1', 1), - (10, '/chapter/2', 'Chapter 2', 2, '1', 2), - (20, '/chapter/1', 'Chapter 1', 1, '1', 1), - (20, '/chapter/2', 'Chapter 2', 2, '1', 2) + (${firstNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${firstNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2), + (${secondNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${secondNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2) `); - const chapters = await getAllNovelChaptersForBackup([20, 10]); + const chapters = await getAllNovelChaptersForBackup([ + secondNovelId, + firstNovelId, + ]); expect(chapters.map(chapter => [chapter.novelId, chapter.name])).toEqual([ - [10, 'Chapter 1'], - [10, 'Chapter 2'], - [20, 'Chapter 1'], - [20, 'Chapter 2'], + [firstNovelId, 'Chapter 1'], + [firstNovelId, 'Chapter 2'], + [secondNovelId, 'Chapter 1'], + [secondNovelId, 'Chapter 2'], ]); }); }); diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 3b77d86f6c..1df406abb4 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -316,23 +316,16 @@ describe('selective backup data', () => { it('includes stored covers with library data when downloads are omitted', async () => { jest.mocked(getAllNovels).mockResolvedValueOnce([ { - id: 1, - name: 'Example', - path: '/example', - pluginId: 'source', + ...makeTestNovel(1), cover: 'file:///storage/Novels/source/1/cover.png?123', }, ]); jest.mocked(getAllNovelChaptersForBackup).mockResolvedValueOnce([ { + ...makeTestChapter(1), id: 10, - novelId: 1, - path: '/chapter-1', - name: 'Chapter 1', - isDownloaded: true, }, - ] as Awaited>); - + ]); await prepareBackupData('/cache', { library: true, settings: false, @@ -363,10 +356,7 @@ describe('selective backup data', () => { it('does not duplicate covers when downloaded files are included', async () => { jest.mocked(getAllNovels).mockResolvedValueOnce([ { - id: 1, - name: 'Example', - path: '/example', - pluginId: 'source', + ...makeTestNovel(1), cover: 'file:///storage/Novels/source/1/cover.png?123', }, ]); diff --git a/src/services/backup/novelPayload.ts b/src/services/backup/novelPayload.ts index 27a7a80198..6791fde47b 100644 --- a/src/services/backup/novelPayload.ts +++ b/src/services/backup/novelPayload.ts @@ -143,7 +143,7 @@ const normalizeLegacyChapter = ( if ( !positiveId(chapter.id) || !nonEmptyString(chapter.path) || - !nonEmptyString(chapter.name) || + !isString(chapter.name) || (chapter.novelId !== undefined && chapter.novelId !== null && !positiveId(chapter.novelId)) @@ -284,7 +284,7 @@ const isCompactChapter = (value: unknown): value is CompactChapter => { return ( isId(value[0]) && nonEmptyString(value[1]) && - nonEmptyString(value[2]) && + isString(value[2]) && nullableString(value[3]) && nullableBoolean(value[4]) && nullableBoolean(value[5]) && diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index fbb2bcca65..e54cecc629 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -201,11 +201,12 @@ export const prepareBackupData = async ( } } } - return { + const preparedNovel = { ...novel, chapters: backedUpChapters, cover, }; + return validateBackupNovel(preparedNovel); } catch { failedNovelCount++; return null; From db21e498be672e190acfafde3f01135518834f01 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Wed, 23 Sep 2026 14:37:55 +0200 Subject: [PATCH 09/10] fix: discard orphan chapters safely --- .../migration.sql | 21 ++++++- src/database/__tests__/db.test.ts | 63 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/drizzle/20260922120323_demonic_rattler/migration.sql b/drizzle/20260922120323_demonic_rattler/migration.sql index 92aa90da11..c0b084a3da 100644 --- a/drizzle/20260922120323_demonic_rattler/migration.sql +++ b/drizzle/20260922120323_demonic_rattler/migration.sql @@ -29,9 +29,28 @@ CREATE TABLE `__new_Chapter` ( CONSTRAINT `fk_Chapter_novelId_Novel_id_fk` FOREIGN KEY (`novelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE ); --> statement-breakpoint -INSERT INTO `__new_Chapter`(`id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent`) SELECT `id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent` FROM `Chapter`;--> statement-breakpoint +-- Preserve the previous AUTOINCREMENT high-water mark across the rebuild. +CREATE TEMP TABLE `__chapter_sequence` AS +SELECT COALESCE(MAX(`seq`), 0) AS `seq` +FROM `sqlite_sequence` +WHERE `name` = 'Chapter'; +--> statement-breakpoint +-- Chapters without a novel are invalid legacy data and are intentionally discarded. +INSERT INTO `__new_Chapter`(`id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent`) SELECT `id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent` FROM `Chapter` WHERE EXISTS (SELECT 1 FROM `Novel` WHERE `Novel`.`id` = `Chapter`.`novelId`);--> statement-breakpoint DROP TABLE `Chapter`;--> statement-breakpoint ALTER TABLE `__new_Chapter` RENAME TO `Chapter`;--> statement-breakpoint +INSERT INTO `sqlite_sequence` (`name`, `seq`) +SELECT 'Chapter', `seq` FROM `__chapter_sequence` +WHERE NOT EXISTS ( + SELECT 1 FROM `sqlite_sequence` WHERE `name` = 'Chapter' +); +--> statement-breakpoint +UPDATE `sqlite_sequence` +SET `seq` = MAX(`seq`, (SELECT `seq` FROM `__chapter_sequence`)) +WHERE `name` = 'Chapter'; +--> statement-breakpoint +DROP TABLE `__chapter_sequence`; +--> statement-breakpoint PRAGMA foreign_keys=ON;--> statement-breakpoint CREATE UNIQUE INDEX `chapter_novel_path_unique` ON `Chapter` (`novelId`,`path`);--> statement-breakpoint CREATE INDEX `chapterNovelIdIndex` ON `Chapter` (`novelId`,`position`,`page`,`id`);--> statement-breakpoint diff --git a/src/database/__tests__/db.test.ts b/src/database/__tests__/db.test.ts index 71a99e61b5..75a1857e56 100644 --- a/src/database/__tests__/db.test.ts +++ b/src/database/__tests__/db.test.ts @@ -1211,4 +1211,67 @@ describe('production migrations', () => { sqlite.close(); } }); + it('discards orphan chapters and preserves the AUTOINCREMENT sequence', async () => { + const sqlite = open({ name: ':memory:' }); + sqlite.executeSync('PRAGMA foreign_keys = ON'); + // Drizzle expects async aliases that op-sqlite's DB type does not declare. + const asyncSqlite = sqlite as unknown as { + executeAsync?: typeof sqlite.execute; + executeRawAsync?: typeof sqlite.executeRaw; + }; + asyncSqlite.executeAsync ??= sqlite.execute; + asyncSqlite.executeRawAsync ??= sqlite.executeRaw; + try { + createSchema(sqlite); + sqlite.executeSync('ALTER TABLE Chapter ADD scanlator text'); + sqlite.executeSync('ALTER TABLE Chapter ADD timeSpent integer DEFAULT 0'); + sqlite.executeSync( + `INSERT INTO Novel (id, path, pluginId, name) + VALUES (1, '/novel', 'plugin', 'Novel')`, + ); + sqlite.executeSync(` + INSERT INTO Chapter (id, novelId, path, name) + VALUES + (1, 1, '/valid', 'Valid'), + (100, 1, '/deleted', 'Deleted') + `); + sqlite.executeSync('DELETE FROM Chapter WHERE id = 100'); + sqlite.executeSync( + `INSERT INTO Chapter (id, novelId, path, name) + VALUES (3, 999, '/orphan', 'Orphan')`, + ); + recordAppliedMigrations(sqlite); + + const drizzleDb = drizzle(sqlite, { schema }); + await migrate(drizzleDb, { + migrations: { + '20260922120323_demonic_rattler': + migrations.migrations['20260922120323_demonic_rattler'], + }, + }); + + expect( + sqlite.executeSync('SELECT id, novelId FROM Chapter').rows, + ).toEqual([{ id: 1, novelId: 1 }]); + expect( + sqlite.executeSync( + "SELECT seq FROM sqlite_sequence WHERE name = 'Chapter'", + ).rows, + ).toEqual([{ seq: 100 }]); + + sqlite.executeSync( + `INSERT INTO Chapter (novelId, path, name) + VALUES (1, '/next', 'Next')`, + ); + expect( + sqlite.executeSync('SELECT last_insert_rowid() AS id').rows, + ).toEqual([{ id: 101 }]); + expect(sqlite.executeRawSync('PRAGMA foreign_key_check;')).toEqual([]); + expect(sqlite.executeRawSync('PRAGMA integrity_check;')).toEqual([ + ['ok'], + ]); + } finally { + sqlite.close(); + } + }); }); From 3844d0061afa17152b47d8a922d4fad5083c82f0 Mon Sep 17 00:00:00 2001 From: CD-Z Date: Wed, 23 Sep 2026 16:03:07 +0200 Subject: [PATCH 10/10] fix failing test --- src/services/backup/__tests__/options.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/services/backup/__tests__/options.test.ts b/src/services/backup/__tests__/options.test.ts index 33fff6b8bf..edf8fe6cb5 100644 --- a/src/services/backup/__tests__/options.test.ts +++ b/src/services/backup/__tests__/options.test.ts @@ -7,6 +7,10 @@ import { import { getSelectedBackupFileSections } from '../fileSections'; import { ZipBackupName } from '../types'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins',