diff --git a/drizzle/20260922120323_demonic_rattler/migration.sql b/drizzle/20260922120323_demonic_rattler/migration.sql new file mode 100644 index 0000000000..c0b084a3da --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/migration.sql @@ -0,0 +1,58 @@ +CREATE TABLE `RestoreChapterMapping` ( + `restoreRunId` text NOT NULL, + `backupNovelId` integer NOT NULL, + `backupChapterId` integer NOT NULL, + `restoredNovelId` integer NOT NULL, + `restoredChapterId` integer NOT NULL, + CONSTRAINT `fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk` FOREIGN KEY (`restoredNovelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk` FOREIGN KEY (`restoredChapterId`) REFERENCES `Chapter`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_Chapter` ( + `id` integer PRIMARY KEY AUTOINCREMENT, + `novelId` integer NOT NULL, + `path` text NOT NULL, + `name` text NOT NULL, + `releaseTime` text, + `bookmark` integer DEFAULT false, + `unread` integer DEFAULT true, + `readTime` text, + `isDownloaded` integer DEFAULT false, + `updatedTime` text, + `chapterNumber` real, + `page` text DEFAULT '1', + `position` integer DEFAULT 0, + `progress` integer, + `scanlator` text, + `timeSpent` integer DEFAULT 0, + CONSTRAINT `fk_Chapter_novelId_Novel_id_fk` FOREIGN KEY (`novelId`) REFERENCES `Novel`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +-- Preserve the previous AUTOINCREMENT high-water mark across the rebuild. +CREATE TEMP TABLE `__chapter_sequence` AS +SELECT COALESCE(MAX(`seq`), 0) AS `seq` +FROM `sqlite_sequence` +WHERE `name` = 'Chapter'; +--> statement-breakpoint +-- Chapters without a novel are invalid legacy data and are intentionally discarded. +INSERT INTO `__new_Chapter`(`id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent`) SELECT `id`, `novelId`, `path`, `name`, `releaseTime`, `bookmark`, `unread`, `readTime`, `isDownloaded`, `updatedTime`, `chapterNumber`, `page`, `position`, `progress`, `scanlator`, `timeSpent` FROM `Chapter` WHERE EXISTS (SELECT 1 FROM `Novel` WHERE `Novel`.`id` = `Chapter`.`novelId`);--> statement-breakpoint +DROP TABLE `Chapter`;--> statement-breakpoint +ALTER TABLE `__new_Chapter` RENAME TO `Chapter`;--> statement-breakpoint +INSERT INTO `sqlite_sequence` (`name`, `seq`) +SELECT 'Chapter', `seq` FROM `__chapter_sequence` +WHERE NOT EXISTS ( + SELECT 1 FROM `sqlite_sequence` WHERE `name` = 'Chapter' +); +--> statement-breakpoint +UPDATE `sqlite_sequence` +SET `seq` = MAX(`seq`, (SELECT `seq` FROM `__chapter_sequence`)) +WHERE `name` = 'Chapter'; +--> statement-breakpoint +DROP TABLE `__chapter_sequence`; +--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE UNIQUE INDEX `chapter_novel_path_unique` ON `Chapter` (`novelId`,`path`);--> statement-breakpoint +CREATE INDEX `chapterNovelIdIndex` ON `Chapter` (`novelId`,`position`,`page`,`id`);--> statement-breakpoint +CREATE UNIQUE INDEX `restore_chapter_mapping_unique` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`,`backupChapterId`);--> statement-breakpoint +CREATE INDEX `restore_chapter_mapping_novel_index` ON `RestoreChapterMapping` (`restoreRunId`,`backupNovelId`); \ No newline at end of file diff --git a/drizzle/20260922120323_demonic_rattler/snapshot.json b/drizzle/20260922120323_demonic_rattler/snapshot.json new file mode 100644 index 0000000000..fc040891a4 --- /dev/null +++ b/drizzle/20260922120323_demonic_rattler/snapshot.json @@ -0,0 +1,793 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "bcdf1b04-31ef-45ca-8a9a-8ecd417c3512", + "prevIds": [ + "2fada83f-d142-46d9-ba03-5ae1e232e3ce" + ], + "ddl": [ + { + "name": "Category", + "entityType": "tables" + }, + { + "name": "Chapter", + "entityType": "tables" + }, + { + "name": "NovelCategory", + "entityType": "tables" + }, + { + "name": "Novel", + "entityType": "tables" + }, + { + "name": "Repository", + "entityType": "tables" + }, + { + "name": "RestoreChapterMapping", + "entityType": "tables" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Category" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "sort", + "entityType": "columns", + "table": "Category" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "releaseTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "bookmark", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "unread", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "readTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isDownloaded", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updatedTime", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "real", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "chapterNumber", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'1'", + "generated": null, + "name": "page", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "position", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "progress", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "scanlator", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "timeSpent", + "entityType": "columns", + "table": "Chapter" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "novelId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "categoryId", + "entityType": "columns", + "table": "NovelCategory" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "path", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "pluginId", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "cover", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "summary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "author", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "artist", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": "'Unknown'", + "generated": null, + "name": "status", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "genres", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "inLibrary", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "false", + "generated": null, + "name": "isLocal", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalPages", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersDownloaded", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "chaptersUnread", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "totalChapters", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastReadAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "lastUpdatedAt", + "entityType": "columns", + "table": "Novel" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": true, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "url", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "true", + "generated": null, + "name": "enabled", + "entityType": "columns", + "table": "Repository" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoreRunId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "backupChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredNovelId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "restoredChapterId", + "entityType": "columns", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "novelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_Chapter_novelId_Novel_id_fk", + "entityType": "fks", + "table": "Chapter" + }, + { + "columns": [ + "restoredNovelId" + ], + "tableTo": "Novel", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredNovelId_Novel_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "restoredChapterId" + ], + "tableTo": "Chapter", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_RestoreChapterMapping_restoredChapterId_Chapter_id_fk", + "entityType": "fks", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Category_pk", + "table": "Category", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Chapter_pk", + "table": "Chapter", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "NovelCategory_pk", + "table": "NovelCategory", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Novel_pk", + "table": "Novel", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "Repository_pk", + "table": "Repository", + "entityType": "pks" + }, + { + "columns": [ + { + "value": "name", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "category_name_unique", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "sort", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "category_sort_idx", + "entityType": "indexes", + "table": "Category" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "chapter_novel_path_unique", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "position", + "isExpression": false + }, + { + "value": "page", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "chapterNovelIdIndex", + "entityType": "indexes", + "table": "Chapter" + }, + { + "columns": [ + { + "value": "novelId", + "isExpression": false + }, + { + "value": "categoryId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_category_unique", + "entityType": "indexes", + "table": "NovelCategory" + }, + { + "columns": [ + { + "value": "path", + "isExpression": false + }, + { + "value": "pluginId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "novel_path_plugin_unique", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "pluginId", + "isExpression": false + }, + { + "value": "path", + "isExpression": false + }, + { + "value": "id", + "isExpression": false + }, + { + "value": "inLibrary", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "NovelIndex", + "entityType": "indexes", + "table": "Novel" + }, + { + "columns": [ + { + "value": "url", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "repository_url_unique", + "entityType": "indexes", + "table": "Repository" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + }, + { + "value": "backupChapterId", + "isExpression": false + } + ], + "isUnique": true, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_unique", + "entityType": "indexes", + "table": "RestoreChapterMapping" + }, + { + "columns": [ + { + "value": "restoreRunId", + "isExpression": false + }, + { + "value": "backupNovelId", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "restore_chapter_mapping_novel_index", + "entityType": "indexes", + "table": "RestoreChapterMapping" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/drizzle/migrations.js b/drizzle/migrations.js index beee4c381c..74735a0528 100644 --- a/drizzle/migrations.js +++ b/drizzle/migrations.js @@ -5,6 +5,7 @@ import m0001 from './20260612232322_normal_saracen/migration.sql'; import m0002 from './20260719143427_long_moondragon/migration.sql'; import m0003 from './20260727081855_calm_chimera/migration.sql'; import m0004 from './20260811071655_parched_human_torch/migration.sql'; +import m0005 from './20260922120323_demonic_rattler/migration.sql'; export default { migrations: { @@ -13,5 +14,6 @@ export default { '20260719143427_long_moondragon': m0002, '20260727081855_calm_chimera': m0003, '20260811071655_parched_human_torch': m0004, + '20260922120323_demonic_rattler': m0005, }, }; diff --git a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt index 3a957a4d08..83db1e6d94 100644 --- a/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt +++ b/modules/native-file/android/src/main/java/expo/modules/nativefile/NativeFileModule.kt @@ -28,19 +28,22 @@ import okhttp3.JavaNetCookieJar import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response +import java.io.BufferedInputStream +import java.io.BufferedOutputStream import java.io.File +import java.io.FileInputStream import java.io.FileOutputStream import java.io.FileWriter import java.io.IOException import java.io.InputStream import java.io.OutputStream -import java.util.UUID import java.io.PushbackInputStream +import java.util.UUID import java.util.zip.GZIPInputStream import kotlin.coroutines.coroutineContext class NativeFileModule : Module() { - private val BUFFER_SIZE = 4096 + private val BUFFER_SIZE = 64 * 1024 private val okHttpClient = OkHttpClientProvider.createClient() private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private var pendingDocumentPromise: Promise? = null @@ -108,60 +111,67 @@ class NativeFileModule : Module() { ?: throw Exception("ENOENT: could not open an input stream for '$filepath'") } - private val writeAccessByAPILevel: String - get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private fun getOutputStream(filepath: String): OutputStream { val uri = getFileUri(filepath) return reactContext?.contentResolver?.openOutputStream(uri, writeAccessByAPILevel) ?: throw Exception("ENOENT: could not open an output stream for '$filepath'") } + private val writeAccessByAPILevel: String + get() = if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) "w" else "rwt" - private suspend fun copyFileContent( - filepath: String, - destPath: String, - onDone: (() -> Unit)? = null, - ): Long { - try { - val inputStream = getInputStream(filepath) - var copiedBytes = 0L - try { - val outputStream = getOutputStream(destPath) - try { - val buffer = ByteArray(BUFFER_SIZE) - var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { - coroutineContext.ensureActive() - outputStream.write(buffer, 0, length) - copiedBytes += length - } - outputStream.flush() - } finally { - outputStream.close() - } - } finally { - inputStream.close() - } - if (onDone != null) { - onDone() - } - return copiedBytes - } catch (e: IOException) { - throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + + private fun localFileOrNull(filepath: String): File? { + val uri = Uri.parse(filepath) + return when { + uri.scheme == null -> File(filepath) + uri.scheme.equals(ContentResolver.SCHEME_FILE, ignoreCase = true) -> + uri.path?.let(::File) + else -> null } } - private fun contentResolver(): ContentResolver = reactContext?.contentResolver ?: throw IOException("React context is unavailable") - private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long { + + private suspend fun copyLocalFile(sourceFile: File, destinationFile: File): Long { + if (sourceFile.isDirectory) { + throw IOException("Invalid file, folder found!") + } + + val sourceSize = sourceFile.length() var copiedBytes = 0L - getInputStream(sourcePath).use { inputStream -> - outputStream.use { output -> + FileInputStream(sourceFile).channel.use { sourceChannel -> + FileOutputStream(destinationFile).channel.use { destinationChannel -> + while (copiedBytes < sourceSize) { + coroutineContext.ensureActive() + val bytesToTransfer = minOf(BUFFER_SIZE.toLong(), sourceSize - copiedBytes) + val transferredBytes = + sourceChannel.transferTo(copiedBytes, bytesToTransfer, destinationChannel) + if (transferredBytes <= 0) { + throw IOException("Could not copy the complete local file") + } + copiedBytes += transferredBytes + } + } + } + + if (destinationFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + return copiedBytes + } + + private suspend fun copyStreams( + inputStream: InputStream, + outputStream: OutputStream, + ): Long { + var copiedBytes = 0L + BufferedInputStream(inputStream, BUFFER_SIZE).use { input -> + BufferedOutputStream(outputStream, BUFFER_SIZE).use { output -> val buffer = ByteArray(BUFFER_SIZE) var length: Int - while (inputStream.read(buffer).also { length = it } > 0) { + while (input.read(buffer).also { length = it } > 0) { coroutineContext.ensureActive() output.write(buffer, 0, length) copiedBytes += length @@ -172,6 +182,44 @@ class NativeFileModule : Module() { return copiedBytes } + private suspend fun copyFileContent( + filepath: String, + destPath: String, + onDone: (() -> Unit)? = null, + ): Long { + try { + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val copiedBytes = if (sourceFile != null && destinationFile != null) { + try { + copyLocalFile(sourceFile, destinationFile) + } catch (_: IOException) { + FileInputStream(sourceFile).use { inputStream -> + FileOutputStream(destinationFile).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + } else { + getInputStream(filepath).use { inputStream -> + getOutputStream(destPath).use { outputStream -> + copyStreams(inputStream, outputStream) + } + } + } + if (sourceFile != null && sourceFile.length() != copiedBytes) { + throw IOException("Copied file size does not match the source file") + } + onDone?.invoke() + return copiedBytes + } catch (e: IOException) { + throw Exception("Failed to copy file from '$filepath' to '$destPath': ${e.message}") + } + } + + private suspend fun copyToOutputStream(sourcePath: String, outputStream: OutputStream): Long = + copyStreams(getInputStream(sourcePath), outputStream) + private fun resolveDirectoryFile(directoryUri: String): File { val uri = Uri.parse(directoryUri) val directory = if (uri.scheme == null) { @@ -479,7 +527,28 @@ class NativeFileModule : Module() { AsyncFunction("moveFile") { filepath: String, destPath: String, promise: Promise -> coroutineScope.launch { try { - val inFile = File(filepath) + val sourceFile = localFileOrNull(filepath) + val destinationFile = localFileOrNull(destPath) + val renamed = if ( + sourceFile != null && + destinationFile != null && + sourceFile.isFile && + !destinationFile.exists() + ) { + try { + sourceFile.renameTo(destinationFile) + } catch (_: SecurityException) { + false + } + } else { + false + } + if (renamed) { + promise.resolve(null) + return@launch + } + + val inFile = sourceFile ?: File(filepath) copyFileContent(filepath, destPath) { if (!inFile.delete()) { throw IOException("Failed to delete source file '$filepath'") diff --git a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt index 75222b47a1..88c2ec6dee 100644 --- a/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt +++ b/modules/native-zip-archive/android/src/main/java/expo/modules/nativeziparchive/NativeZipArchiveModule.kt @@ -13,11 +13,24 @@ import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream class NativeZipArchiveModule : Module() { - private fun zipProcess(sourceDirPath: String, zos: ZipOutputStream) { + private fun zipProcess( + sourceDirPath: String, + zos: ZipOutputStream, + archivePrefix: String = "", + archiveEntries: MutableSet? = null, + excludedFile: File? = null, + ) { val sourceDir = File(sourceDirPath) - sourceDir.walkBottomUp().filter { it.isFile }.forEach { file -> - val zipFileName = + val excludedCanonicalPath = excludedFile?.canonicalPath + sourceDir.walkBottomUp().filter { file -> + file.isFile && file.canonicalPath != excludedCanonicalPath + }.forEach { file -> + val relativePath = file.absolutePath.removePrefix(sourceDir.absolutePath).removePrefix("/") + val zipFileName = joinArchivePath(archivePrefix, relativePath) + if (archiveEntries != null && !archiveEntries.add(zipFileName)) { + throw IllegalArgumentException("Duplicate ZIP entry: $zipFileName") + } val entry = ZipEntry("$zipFileName${(if (file.isDirectory) "/" else "")}") zos.putNextEntry(entry) file.inputStream().use { fis -> @@ -26,16 +39,56 @@ class NativeZipArchiveModule : Module() { } } + private fun joinArchivePath(prefix: String, relativePath: String): String { + val normalizedRelativePath = relativePath.replace('\\', '/').trimStart('/') + return if (prefix.isEmpty()) { + normalizedRelativePath + } else { + "$prefix/$normalizedRelativePath" + } + } + + private fun normalizeArchivePrefix(prefix: String): String { + require(prefix.isEmpty() || prefix.isNotBlank()) { "ZIP prefix must not be blank" } + val normalized = prefix.replace('\\', '/') + require(!normalized.startsWith("/") && !(normalized.length >= 2 && normalized[1] == ':')) { + "ZIP prefix must be relative" + } + val components = normalized.split('/').filter { it.isNotEmpty() && it != "." } + require(components.none { it == ".." }) { "ZIP prefix must not contain .." } + return components.joinToString("/") + } + + private fun ensureParentDirectory(file: File, createdDirectories: MutableSet) { + val parent = file.parentFile ?: return + val parentPath = parent.absolutePath + if (parentPath !in createdDirectories && (parent.exists() || parent.mkdirs())) { + createdDirectories.add(parentPath) + } + } + private fun resolveZipEntry(destination: File, entryName: String): File { + val canonicalDestination = destination.canonicalFile + val outputFile = File(destination, entryName).canonicalFile + val destinationPath = + canonicalDestination.path.let { if (it.endsWith(File.separator)) it else it + File.separator } + require(outputFile.path.startsWith(destinationPath)) { + "ZIP entry is outside the destination directory: $entryName" + } + return outputFile + } + + override fun definition() = ModuleDefinition { Name("NativeZipArchive") AsyncFunction("unzip") { sourceFilePath: String, distDirPath: String, promise: Promise -> Thread { try { + val createdDirectories = mutableSetOf() ZipFile(sourceFilePath).use { zis -> zis.entries().asSequence().filterNot { it.isDirectory }.forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) + ensureParentDirectory(newFile, createdDirectories) zis.getInputStream(zipEntry).use { inputStream -> FileOutputStream(newFile).use { fos -> inputStream.copyTo(fos, COPY_BUFFER_SIZE) } } @@ -52,7 +105,39 @@ class NativeZipArchiveModule : Module() { Thread { try { FileOutputStream(zipFilePath).use { fos -> - ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos) } + ZipOutputStream(fos).use { zos -> zipProcess(sourceDirPath, zos, excludedFile = File(zipFilePath)) } + } + promise.resolve(null) + } catch (e: Exception) { + promise.reject("ZIP_FAILED", e.message ?: "Zip failed", e) + } + }.start() + } + + AsyncFunction("zipDirectories") { sources: List>, zipFilePath: String, promise: Promise -> + Thread { + try { + require(sources.isNotEmpty()) { "ZIP sources must not be empty" } + require(zipFilePath.isNotBlank()) { "ZIP file path must not be blank" } + val normalizedSources = sources.map { source -> + val sourceDirPath = + source["path"]?.takeIf { it.isNotBlank() } + ?: throw IllegalArgumentException("ZIP source path must not be blank") + require(File(sourceDirPath).isDirectory) { + "ZIP source path is not a directory: $sourceDirPath" + } + val archivePrefix = + source["prefix"]?.let(::normalizeArchivePrefix) + ?: throw IllegalArgumentException("ZIP source prefix is missing") + sourceDirPath to archivePrefix + } + val archiveEntries = mutableSetOf() + FileOutputStream(zipFilePath).use { fos -> + ZipOutputStream(fos).use { zos -> + normalizedSources.forEach { (sourceDirPath, archivePrefix) -> + zipProcess(sourceDirPath, zos, archivePrefix, archiveEntries, File(zipFilePath)) + } + } } promise.resolve(null) } catch (e: Exception) { @@ -69,12 +154,13 @@ class NativeZipArchiveModule : Module() { headers.forEach { (key, value) -> connection.setRequestProperty(key, value) } + val createdDirectories = mutableSetOf() ZipInputStream(connection.inputStream).use { zis -> generateSequence { zis.nextEntry } .filterNot { it.isDirectory } .forEach { zipEntry -> - val newFile = File(distDirPath, zipEntry.name) - newFile.parentFile?.mkdirs() + val newFile = resolveZipEntry(File(distDirPath), zipEntry.name) + ensureParentDirectory(newFile, createdDirectories) FileOutputStream(newFile).use { fos -> zis.copyTo(fos, COPY_BUFFER_SIZE) } } } diff --git a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift index 7060acd82a..c36bbd2b6b 100644 --- a/modules/native-zip-archive/ios/NativeZipArchiveModule.swift +++ b/modules/native-zip-archive/ios/NativeZipArchiveModule.swift @@ -13,6 +13,10 @@ public class NativeZipArchiveModule: Module { promise.reject("NOT_IMPLEMENTED", "zip is not implemented on iOS") } + AsyncFunction("zipDirectories") { (sources: [[String: String]], zipFilePath: String, promise: Promise) in + promise.reject("NOT_IMPLEMENTED", "zipDirectories is not implemented on iOS") + } + AsyncFunction("remoteUnzip") { (distDirPath: String, url: String, headers: [String: String], promise: Promise) in promise.reject("NOT_IMPLEMENTED", "remoteUnzip is not implemented on iOS") } diff --git a/modules/native-zip-archive/src/NativeZipArchiveModule.ts b/modules/native-zip-archive/src/NativeZipArchiveModule.ts index cd62ad6fbd..f912e2175c 100644 --- a/modules/native-zip-archive/src/NativeZipArchiveModule.ts +++ b/modules/native-zip-archive/src/NativeZipArchiveModule.ts @@ -3,6 +3,10 @@ import { requireNativeModule } from 'expo-modules-core'; type NativeZipArchiveModule = { unzip(sourceFilePath: string, distDirPath: string): Promise; zip(sourceDirPath: string, zipFilePath: string): Promise; + zipDirectories( + sources: Array<{ path: string; prefix: string }>, + zipFilePath: string, + ): Promise; remoteUnzip( distDirPath: string, urlString: string, diff --git a/src/database/__tests__/db.test.ts b/src/database/__tests__/db.test.ts index 155f67f84d..75a1857e56 100644 --- a/src/database/__tests__/db.test.ts +++ b/src/database/__tests__/db.test.ts @@ -809,6 +809,7 @@ describe('production migrations', () => { '20260719143427_long_moondragon', '20260727081855_calm_chimera', '20260811071655_parched_human_torch', + '20260922120323_demonic_rattler', ]); } finally { sqlite.close(); @@ -1210,4 +1211,67 @@ describe('production migrations', () => { sqlite.close(); } }); + it('discards orphan chapters and preserves the AUTOINCREMENT sequence', async () => { + const sqlite = open({ name: ':memory:' }); + sqlite.executeSync('PRAGMA foreign_keys = ON'); + // Drizzle expects async aliases that op-sqlite's DB type does not declare. + const asyncSqlite = sqlite as unknown as { + executeAsync?: typeof sqlite.execute; + executeRawAsync?: typeof sqlite.executeRaw; + }; + asyncSqlite.executeAsync ??= sqlite.execute; + asyncSqlite.executeRawAsync ??= sqlite.executeRaw; + try { + createSchema(sqlite); + sqlite.executeSync('ALTER TABLE Chapter ADD scanlator text'); + sqlite.executeSync('ALTER TABLE Chapter ADD timeSpent integer DEFAULT 0'); + sqlite.executeSync( + `INSERT INTO Novel (id, path, pluginId, name) + VALUES (1, '/novel', 'plugin', 'Novel')`, + ); + sqlite.executeSync(` + INSERT INTO Chapter (id, novelId, path, name) + VALUES + (1, 1, '/valid', 'Valid'), + (100, 1, '/deleted', 'Deleted') + `); + sqlite.executeSync('DELETE FROM Chapter WHERE id = 100'); + sqlite.executeSync( + `INSERT INTO Chapter (id, novelId, path, name) + VALUES (3, 999, '/orphan', 'Orphan')`, + ); + recordAppliedMigrations(sqlite); + + const drizzleDb = drizzle(sqlite, { schema }); + await migrate(drizzleDb, { + migrations: { + '20260922120323_demonic_rattler': + migrations.migrations['20260922120323_demonic_rattler'], + }, + }); + + expect( + sqlite.executeSync('SELECT id, novelId FROM Chapter').rows, + ).toEqual([{ id: 1, novelId: 1 }]); + expect( + sqlite.executeSync( + "SELECT seq FROM sqlite_sequence WHERE name = 'Chapter'", + ).rows, + ).toEqual([{ seq: 100 }]); + + sqlite.executeSync( + `INSERT INTO Chapter (novelId, path, name) + VALUES (1, '/next', 'Next')`, + ); + expect( + sqlite.executeSync('SELECT last_insert_rowid() AS id').rows, + ).toEqual([{ id: 101 }]); + expect(sqlite.executeRawSync('PRAGMA foreign_key_check;')).toEqual([]); + expect(sqlite.executeRawSync('PRAGMA integrity_check;')).toEqual([ + ['ok'], + ]); + } finally { + sqlite.close(); + } + }); }); diff --git a/src/database/db.ts b/src/database/db.ts index acdd5efcc2..c20d424a96 100644 --- a/src/database/db.ts +++ b/src/database/db.ts @@ -66,6 +66,7 @@ export const drizzleDb = drizzle(_db, { schema, logger: __DEV__ ? new MyLogger() : false, }); +export type DrizzleDb = typeof drizzleDb export const dbManager = createDbManager(drizzleDb); diff --git a/src/database/manager/__tests__/manager.test.ts b/src/database/manager/__tests__/manager.test.ts new file mode 100644 index 0000000000..41f76efd31 --- /dev/null +++ b/src/database/manager/__tests__/manager.test.ts @@ -0,0 +1,61 @@ +import { + createTestDb, + cleanupTestDb, +} from '@database/queries/__tests__/testDb'; + +describe('DbManager.executeBatch', () => { + const testDb = createTestDb(); + const flushPendingReactiveQueries = jest.fn(); + + afterAll(() => { + cleanupTestDb(testDb); + }); + + beforeEach(() => { + flushPendingReactiveQueries.mockClear(); + Object.defineProperty(testDb.sqlite, 'flushPendingReactiveQueries', { + configurable: true, + value: flushPendingReactiveQueries, + }); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS BatchTest'); + testDb.sqlite.executeSync('DROP TABLE IF EXISTS MissingBatchTable'); + testDb.sqlite.executeSync( + 'CREATE TABLE BatchTest (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL)', + ); + }); + + it('executes heterogeneous and repeated-parameter commands atomically', async () => { + const result = await testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['first'], ['second']]], + ['UPDATE BatchTest SET value = ? WHERE value = ?', ['updated', 'first']], + ['DELETE FROM BatchTest WHERE value = ?', [['second']]], + ]); + + expect(result.rowsAffected).toBe(4); + expect(flushPendingReactiveQueries).toHaveBeenCalledTimes(1); + expect( + testDb.sqlite.executeSync('SELECT value FROM BatchTest').rows, + ).toEqual([{ value: 'updated' }]); + }); + + it('rolls back every command when a later command fails', async () => { + await expect( + testDb.dbManager.executeBatch([ + ['INSERT INTO BatchTest (value) VALUES (?)', [['rolled back']]], + ['INSERT INTO MissingBatchTable (value) VALUES (?)', [['failure']]], + ]), + ).rejects.toThrow('MissingBatchTable'); + + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + expect(testDb.sqlite.executeSync('SELECT * FROM BatchTest').rows).toEqual( + [], + ); + }); + + it('does not flush for an empty batch', async () => { + const result = await testDb.dbManager.executeBatch([]); + + expect(result).toEqual({ rowsAffected: 0 }); + expect(flushPendingReactiveQueries).not.toHaveBeenCalled(); + }); +}); diff --git a/src/database/manager/manager.d.ts b/src/database/manager/manager.d.ts index 195bab40bb..95584c8c0d 100644 --- a/src/database/manager/manager.d.ts +++ b/src/database/manager/manager.d.ts @@ -1,27 +1,30 @@ // db-manager.types.ts import type { - SQLiteTransaction, - TablesRelationalConfig, - Placeholder, -} from 'drizzle-orm'; -import { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; + BatchQueryResult, + SQLBatchTuple, +} from '@op-engineering/op-sqlite'; +import type { DrizzleDb } from '@database/db'; +import type { Placeholder } from 'drizzle-orm'; +import type { SQLitePreparedQuery } from 'drizzle-orm/sqlite-core'; // Define the TransactionParameter type based on your DrizzleDb -export type TransactionParameter = SQLiteTransaction< - 'async', - { changes: number; lastInsertRowid: number } | void, - Record, - TablesRelationalConfig ->; - +export type TransactionParameter = Parameters< + Parameters[0] +>[0]; /** * Interface defining the public API and documentation for the Drizzle database manager. * This contract ensures consistent documentation and type safety across the application. */ export interface IDbManager { + /** + * Executes raw op-sqlite commands as one atomic transaction. + */ + executeBatch(commands: SQLBatchTuple[]): Promise; /** * Efficiently executes a Drizzle query for multiple data rows using - * op-sqlite executeBatch under the hood. + * op-sqlite executeBatch under the hood. The batch owns its transaction; + * it cannot be nested inside write(). The callback argument is only used + * to build the prepared query and is not the surrounding write transaction. */ batch>( data: T[], @@ -29,7 +32,7 @@ export interface IDbManager { tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ): Promise; + ): Promise; /** * Creates a subquery that defines a temporary named result set as a CTE. diff --git a/src/database/manager/manager.ts b/src/database/manager/manager.ts index cc5690d755..6ce98c6ef2 100644 --- a/src/database/manager/manager.ts +++ b/src/database/manager/manager.ts @@ -1,5 +1,9 @@ +import type { + BatchQueryResult, + SQLBatchTuple, + Scalar, +} from '@op-engineering/op-sqlite'; import type { drizzleDb } from '@database/db'; -import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; import { IDbManager } from './manager.d'; import { DbTaskQueue } from './queue'; import { Schema } from '../schema'; @@ -88,15 +92,32 @@ class DbManager implements IDbManager { .rows as Awaited>; } + public async executeBatch( + commands: SQLBatchTuple[], + ): Promise { + if (!commands.length) { + return { rowsAffected: 0 }; + } + + return await this.queue.enqueue({ + id: 'write', + run: async () => { + const result = await this.db.$client.executeBatch(commands); + this.db.$client.flushPendingReactiveQueries(); + return result; + }, + }); + } + public async batch>( data: T[], fn: ( tx: TransactionParameter, ph: (arg: Extract) => Placeholder, ) => SQLitePreparedQuery, - ) { + ): Promise { if (!data.length) { - return; + return { rowsAffected: 0 }; } const ph = (arg: Extract) => sql.placeholder(arg); @@ -110,13 +131,7 @@ class DbManager implements IDbManager { }); const commands: SQLBatchTuple[] = [[query.sql, params]]; - await this.queue.enqueue({ - id: 'write', - run: async () => { - await this.db.$client.executeBatch(commands); - this.db.$client?.flushPendingReactiveQueries(); - }, - }); + return await this.executeBatch(commands); } public async write( diff --git a/src/database/queries/ChapterQueries.ts b/src/database/queries/ChapterQueries.ts index 9b7ca28cf4..3bcddc945b 100644 --- a/src/database/queries/ChapterQueries.ts +++ b/src/database/queries/ChapterQueries.ts @@ -468,14 +468,24 @@ export const getNovelChapters = async ( * a paginated UI query because doing so silently produces incomplete backups. */ export const getAllNovelChaptersForBackup = async ( - novelId: number, -): Promise => - dbManager + novelIds: number | number[], +): Promise => { + const ids = Array.isArray(novelIds) ? novelIds : [novelIds]; + if (ids.length === 0) { + return []; + } + + return dbManager .select() .from(chapterSchema) - .where(eq(chapterSchema.novelId, novelId)) - .orderBy(asc(chapterSchema.id)) + .where( + Array.isArray(novelIds) + ? inArray(chapterSchema.novelId, ids) + : eq(chapterSchema.novelId, ids[0]), + ) + .orderBy(asc(chapterSchema.novelId), asc(chapterSchema.id)) .all(); +}; export const getNovelChaptersSync = ( novelId: number, diff --git a/src/database/queries/NovelQueries.ts b/src/database/queries/NovelQueries.ts index 3d66c4d298..59c6cfc212 100644 --- a/src/database/queries/NovelQueries.ts +++ b/src/database/queries/NovelQueries.ts @@ -6,12 +6,7 @@ import { insertChapters } from './ChapterQueries'; import { showToast } from '@utils/showToast'; import { getString } from '@i18n/translations'; -import { - BackupNovel, - DBNovelInfo, - NovelInfo, - type RestoredNovelMapping, -} from '../types'; +import { DBNovelInfo, NovelInfo } from '../types'; import { SourceNovel } from '@plugins/types'; import { NOVEL_STORAGE } from '@utils/Storages'; import { downloadFile } from '@plugins/helpers/fetch'; @@ -21,7 +16,6 @@ import { novelSchema, novelCategorySchema, categorySchema, - chapterSchema, } from '@database/schema'; import type { TransactionParameter } from '@database/manager/manager.d'; import { getLibraryDefaultCategoryId } from '@hooks/persisted/useSettings'; @@ -75,7 +69,7 @@ const getCategoriesForNewNovel = async ( return getBuiltInDefaultCategory(tx); }; -const getCategoryForNewNovel = async (tx: TransactionParameter) => +export const getCategoryForNewNovel = async (tx: TransactionParameter) => (await getCategoriesForNewNovel(tx))[0]; /** @@ -287,69 +281,6 @@ export const deleteCachedNovels = async () => { showToast(getString('advancedSettingsScreen.cachedNovelsDeletedToast')); }; -/** - * Restore a novel from backup using Drizzle ORM. - */ -export const restoreLibrary = async (novel: NovelInfo) => { - const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { - throw e; - }); - - const novelId = await dbManager.write(async tx => { - const row = await tx - .insert(novelSchema) - .values({ - path: sourceNovel.path, - name: novel.name, - pluginId: novel.pluginId, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - name: novel.name, - cover: novel.cover || '', - summary: novel.summary || '', - author: novel.author || '', - artist: novel.artist || '', - status: novel.status || '', - genres: novel.genres || '', - totalPages: sourceNovel.totalPages || 0, - inLibrary: true, - }, - }) - .returning() - .get(); - - if (row) { - const defaultCategory = await getCategoryForNewNovel(tx); - - if (defaultCategory) { - await tx - .insert(novelCategorySchema) - .values({ - novelId: row.id, - categoryId: defaultCategory.id, - }) - .onConflictDoNothing() - .run(); - } - } - return row?.id; - }); - - if (novelId && sourceNovel.chapters) { - await insertChapters(novelId, sourceNovel.chapters); - } -}; - export const updateNovelInfo = async (info: NovelInfo) => { await dbManager.write(async tx => { await tx @@ -474,89 +405,3 @@ export const updateNovelCategories = async ( } }); }; - -/** - * Restores novel and chapters from a backup object. - */ -export const _restoreNovelAndChapters = async ( - backupNovel: BackupNovel, -): Promise => { - const { chapters, id: backupNovelId, ...novel } = backupNovel; - return dbManager.write(async tx => { - // Match novels by their stable source identity, not the database-local ID. - const restoredNovel = await tx - .insert(novelSchema) - .values({ - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }) - .onConflictDoUpdate({ - target: [novelSchema.path, novelSchema.pluginId], - set: { - ...novel, - totalChapters: 0, - chaptersDownloaded: 0, - chaptersUnread: 0, - }, - }) - .returning({ id: novelSchema.id }) - .get(); - - if (novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) { - const cacheSuffix = novel.cover.match(/[?#].*$/)?.[0] ?? ''; - await tx - .update(novelSchema) - .set({ - cover: `file://${NOVEL_STORAGE}/${novel.pluginId}/${restoredNovel.id}/cover.png${cacheSuffix}`, - }) - .where(eq(novelSchema.id, restoredNovel.id)) - .run(); - } - - await tx - .delete(chapterSchema) - .where(eq(chapterSchema.novelId, restoredNovel.id)) - .run(); - - const chapterMappings: RestoredNovelMapping['chapters'] = []; - - // Restore chapters in batches - if (chapters.length > 0) { - const BATCH_SIZE = 100; - for (let i = 0; i < chapters.length; i += BATCH_SIZE) { - const batch = chapters.slice(i, i + BATCH_SIZE); - const restoredChapters = await tx - .insert(chapterSchema) - .values( - batch.map(({ id: _chapterId, novelId: _novelId, ...chapter }) => ({ - ...chapter, - novelId: restoredNovel.id, - })), - ) - .returning({ id: chapterSchema.id, path: chapterSchema.path }) - .all(); - const restoredIdsByPath = new Map( - restoredChapters.map(chapter => [chapter.path, chapter.id]), - ); - for (const chapter of batch) { - const restoredChapterId = restoredIdsByPath.get(chapter.path); - if (restoredChapterId !== undefined) { - chapterMappings.push({ - backupChapterId: chapter.id, - restoredChapterId, - }); - } - } - } - } - - return { - pluginId: novel.pluginId, - backupNovelId, - restoredNovelId: restoredNovel.id, - chapters: chapterMappings, - }; - }); -}; diff --git a/src/database/queries/NovelRestoreQueries.ts b/src/database/queries/NovelRestoreQueries.ts new file mode 100644 index 0000000000..e22523e423 --- /dev/null +++ b/src/database/queries/NovelRestoreQueries.ts @@ -0,0 +1,509 @@ +import { and, eq, inArray, or } from 'drizzle-orm'; +import type { SQLBatchTuple, Scalar } from '@op-engineering/op-sqlite'; + +import { fetchNovel } from '@services/plugin/fetch'; +import { insertChapters } from './ChapterQueries'; +import { getCategoryForNewNovel } from './NovelQueries'; + +import { dbManager } from '@database/db'; +import { + createNovelTriggerQueryDelete, + createNovelTriggerQueryInsert, + createNovelTriggerQueryUpdate, +} from '@database/queryStrings/triggers'; +import { + novelCategorySchema, + novelSchema, + restoreChapterMappingSchema, +} from '@database/schema'; +import { NOVEL_STORAGE } from '@utils/Storages'; +import type { BackupNovel, NovelInfo, RestoredNovelMapping } from '../types'; + +/** + * Restore a novel from backup using Drizzle ORM. + */ +export const restoreLibrary = async (novel: NovelInfo) => { + const sourceNovel = await fetchNovel(novel.pluginId, novel.path).catch(e => { + throw e; + }); + + const novelId = await dbManager.write(async tx => { + const row = await tx + .insert(novelSchema) + .values({ + path: sourceNovel.path, + name: novel.name, + pluginId: novel.pluginId, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }) + .onConflictDoUpdate({ + target: [novelSchema.path, novelSchema.pluginId], + set: { + name: novel.name, + cover: novel.cover || '', + summary: novel.summary || '', + author: novel.author || '', + artist: novel.artist || '', + status: novel.status || '', + genres: novel.genres || '', + totalPages: sourceNovel.totalPages || 0, + inLibrary: true, + }, + }) + .returning() + .get(); + + if (row) { + const defaultCategory = await getCategoryForNewNovel(tx); + + if (defaultCategory) { + await tx + .insert(novelCategorySchema) + .values({ + novelId: row.id, + categoryId: defaultCategory.id, + }) + .onConflictDoNothing() + .run(); + } + } + return row?.id; + }); + + if (novelId && sourceNovel.chapters) { + await insertChapters(novelId, sourceNovel.chapters); + } +}; +const sqliteBoolean = (value: boolean | null | undefined): Scalar => + value == null ? null : value ? 1 : 0; +const RESTORE_NOVEL_BATCH_SIZE = 100; +const RESTORE_CHAPTER_BATCH_SIZE = 10_000; + +export type RestoreNovelOptions = { + includeChapterMappings?: boolean; + restoreRunId?: string; +}; + +const NOVEL_UPSERT_SQL = ` + INSERT INTO Novel ( + path, pluginId, name, cover, summary, author, artist, status, genres, + inLibrary, isLocal, totalPages + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(path, pluginId) DO UPDATE SET + path = excluded.path, + pluginId = excluded.pluginId, + name = excluded.name, + cover = excluded.cover, + summary = excluded.summary, + author = excluded.author, + artist = excluded.artist, + status = excluded.status, + genres = excluded.genres, + inLibrary = excluded.inLibrary, + isLocal = excluded.isLocal, + totalPages = excluded.totalPages +`; + +const NOVEL_COVER_UPDATE_SQL = ` + UPDATE Novel + SET cover = ? || pluginId || '/' || id || '/cover.png' || ? + WHERE pluginId = ? AND path = ? +`; + +const restoreNovelValues = (novel: BackupNovel): Scalar[] => [ + novel.path, + novel.pluginId, + novel.name, + novel.cover ?? null, + novel.summary ?? null, + novel.author ?? null, + novel.artist ?? null, + novel.status ?? null, + novel.genres ?? null, + sqliteBoolean(novel.inLibrary), + sqliteBoolean(novel.isLocal), + novel.totalPages ?? null, +]; + +const restoreNovelChunk = async ( + backupNovels: BackupNovel[], +): Promise => { + const commands: SQLBatchTuple[] = [ + [NOVEL_UPSERT_SQL, backupNovels.map(restoreNovelValues)], + ]; + const storedCovers = backupNovels + .filter(novel => novel.cover?.startsWith(`file://${NOVEL_STORAGE}/`)) + .map( + novel => + [ + `file://${NOVEL_STORAGE}/`, + novel.cover?.match(/[?#].*$/)?.[0] ?? '', + novel.pluginId, + novel.path, + ] as Scalar[], + ); + if (storedCovers.length > 0) { + commands.push([NOVEL_COVER_UPDATE_SQL, storedCovers]); + } + await dbManager.executeBatch(commands); + + const rows = await dbManager + .select({ + id: novelSchema.id, + path: novelSchema.path, + pluginId: novelSchema.pluginId, + }) + .from(novelSchema) + .where( + or( + ...backupNovels.map(novel => + and( + eq(novelSchema.pluginId, novel.pluginId), + eq(novelSchema.path, novel.path), + ), + ), + ), + ) + .all(); + const rowsByIdentity = new Map( + rows.map(row => [`${row.pluginId}\u0000${row.path}`, row]), + ); + return backupNovels.map(backupNovel => { + const row = rowsByIdentity.get( + `${backupNovel.pluginId}\u0000${backupNovel.path}`, + ); + if (!row) { + throw new Error('Restore returned incomplete novel mapping'); + } + return { + pluginId: backupNovel.pluginId, + backupNovelId: backupNovel.id, + restoredNovelId: row.id, + }; + }); +}; + +const restoreNovelChunkWithRetry = async ( + backupNovels: BackupNovel[], +): Promise => { + try { + return await restoreNovelChunk(backupNovels); + } catch { + const mappings: RestoredNovelMapping[] = []; + let failed = false; + for (const backupNovel of backupNovels) { + try { + mappings.push( + await restoreNovelChunk([backupNovel]).then(([mapping]) => { + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; + }), + ); + } catch { + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more novels'); + } + return mappings; + } +}; + +const restoreChapterValues = ( + chapter: BackupNovel['chapters'][number], + restoredNovelId: number, +): Scalar[] => [ + restoredNovelId, + chapter.path, + chapter.name, + chapter.releaseTime ?? null, + sqliteBoolean(chapter.bookmark), + sqliteBoolean(chapter.unread), + chapter.readTime ?? null, + sqliteBoolean(chapter.isDownloaded), + chapter.updatedTime ?? null, + chapter.chapterNumber ?? null, + chapter.page ?? null, + chapter.position ?? null, + chapter.progress ?? null, + chapter.scanlator ?? null, + chapter.timeSpent ?? null, +]; + +type ChapterRestoreRecord = { + backupNovelId: number; + backupChapterId: number; + restoredNovelId: number; + chapter: BackupNovel['chapters'][number]; +}; + +const CHAPTER_UPSERT_SQL = ` + INSERT INTO Chapter ( + novelId, path, name, releaseTime, bookmark, unread, readTime, + isDownloaded, updatedTime, chapterNumber, page, position, progress, + scanlator, timeSpent + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(novelId, path) DO UPDATE SET + novelId = excluded.novelId, + path = excluded.path, + name = excluded.name, + releaseTime = excluded.releaseTime, + bookmark = excluded.bookmark, + unread = excluded.unread, + readTime = excluded.readTime, + isDownloaded = excluded.isDownloaded, + updatedTime = excluded.updatedTime, + chapterNumber = excluded.chapterNumber, + page = excluded.page, + position = excluded.position, + progress = excluded.progress, + scanlator = excluded.scanlator, + timeSpent = excluded.timeSpent +`; + +const RESTORE_CHAPTER_MAPPING_INSERT_SQL = ` + INSERT INTO RestoreChapterMapping ( + restoreRunId, backupNovelId, backupChapterId, + restoredNovelId, restoredChapterId + ) + SELECT ?, ?, ?, ?, id + FROM Chapter + WHERE novelId = ? AND path = ? + ON CONFLICT(restoreRunId, backupNovelId, backupChapterId) DO UPDATE SET + restoredNovelId = excluded.restoredNovelId, + restoredChapterId = excluded.restoredChapterId +`; + +const restoreChapterChunk = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, +) => { + if (records.length === 0) { + return; + } + if (includeChapterMappings && !restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const commands: SQLBatchTuple[] = [ + ['DROP TRIGGER IF EXISTS update_novel_stats'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_update'], + ['DROP TRIGGER IF EXISTS update_novel_stats_on_delete'], + [ + CHAPTER_UPSERT_SQL, + records.map(record => + restoreChapterValues(record.chapter, record.restoredNovelId), + ), + ], + ]; + if (includeChapterMappings) { + commands.push([ + RESTORE_CHAPTER_MAPPING_INSERT_SQL, + records.map( + record => + [ + restoreRunId, + record.backupNovelId, + record.backupChapterId, + record.restoredNovelId, + record.restoredNovelId, + record.chapter.path, + ] as Scalar[], + ), + ]); + } + commands.push( + [createNovelTriggerQueryInsert], + [createNovelTriggerQueryDelete], + [createNovelTriggerQueryUpdate], + ); + await dbManager.executeBatch(commands); +}; + +const restoreChapterChunkWithRetry = async ( + records: ChapterRestoreRecord[], + includeChapterMappings: boolean, + restoreRunId: string | undefined, +) => { + try { + await restoreChapterChunk(records, includeChapterMappings, restoreRunId); + } catch { + let failed = false; + for (const record of records) { + try { + await restoreChapterChunk( + [record], + includeChapterMappings, + restoreRunId, + ); + } catch { + failed = true; + } + } + if (failed) { + throw new Error('Failed to restore one or more chapters'); + } + } +}; + +const NOVEL_STATS_UPDATE_SQL = ` + UPDATE Novel + SET totalChapters = ( + SELECT COUNT(*) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + chaptersDownloaded = COALESCE(( + SELECT SUM(CASE WHEN Chapter.isDownloaded = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + chaptersUnread = COALESCE(( + SELECT SUM(CASE WHEN Chapter.unread = 1 THEN 1 ELSE 0 END) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), 0), + lastReadAt = ( + SELECT MAX(Chapter.readTime) + FROM Chapter + WHERE Chapter.novelId = Novel.id + ), + lastUpdatedAt = ( + SELECT updatedChapter.updatedTime + FROM Chapter AS updatedChapter + WHERE updatedChapter.novelId = Novel.id + AND updatedChapter.updatedTime IS NOT NULL + ORDER BY julianday(updatedChapter.updatedTime) DESC + LIMIT 1 + ) + WHERE Novel.id = ? +`; + +const refreshRestoredNovelStats = async (novelIds: number[]) => { + if (novelIds.length === 0) { + return; + } + await dbManager.executeBatch([ + [NOVEL_STATS_UPDATE_SQL, novelIds.map(id => [id] as Scalar[])], + ]); +}; + +export const _restoreNovelsAndChapters = async ( + backupNovels: BackupNovel[], + options: RestoreNovelOptions = {}, +): Promise => { + if (backupNovels.length === 0) { + return []; + } + const includeChapterMappings = options.includeChapterMappings ?? true; + if (includeChapterMappings && !options.restoreRunId) { + throw new Error('Restore run ID is required for chapter mappings'); + } + + const mappings: RestoredNovelMapping[] = []; + for ( + let start = 0; + start < backupNovels.length; + start += RESTORE_NOVEL_BATCH_SIZE + ) { + mappings.push( + ...(await restoreNovelChunkWithRetry( + backupNovels.slice(start, start + RESTORE_NOVEL_BATCH_SIZE), + )), + ); + } + + const restoredNovelIds = new Map( + mappings.map(mapping => [mapping.backupNovelId, mapping.restoredNovelId]), + ); + const chapterChunk: ChapterRestoreRecord[] = []; + for (const backupNovel of backupNovels) { + const restoredNovelId = restoredNovelIds.get(backupNovel.id); + if (restoredNovelId === undefined) { + throw new Error('Missing restored novel mapping'); + } + for (const chapter of backupNovel.chapters) { + chapterChunk.push({ + backupNovelId: backupNovel.id, + backupChapterId: chapter.id, + restoredNovelId, + chapter, + }); + if (chapterChunk.length === RESTORE_CHAPTER_BATCH_SIZE) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + ); + chapterChunk.length = 0; + } + } + } + if (chapterChunk.length > 0) { + await restoreChapterChunkWithRetry( + chapterChunk, + includeChapterMappings, + options.restoreRunId, + ); + } + await refreshRestoredNovelStats([...restoredNovelIds.values()]); + return mappings; +}; + +/** + * Restores a novel and its chapters from a backup object. + */ +export const _restoreNovelAndChapters = async ( + backupNovel: BackupNovel, + options: RestoreNovelOptions = {}, +): Promise => { + const [mapping] = await _restoreNovelsAndChapters([backupNovel], options); + if (!mapping) { + throw new Error('Failed to restore novel'); + } + return mapping; +}; + +export const getRestoreChapterMappings = async ( + restoreRunId: string, + backupNovelId: number, + backupChapterIds: number[], +) => { + if (backupChapterIds.length === 0) { + return []; + } + return dbManager + .select({ + backupChapterId: restoreChapterMappingSchema.backupChapterId, + restoredChapterId: restoreChapterMappingSchema.restoredChapterId, + }) + .from(restoreChapterMappingSchema) + .where( + and( + eq(restoreChapterMappingSchema.restoreRunId, restoreRunId), + eq(restoreChapterMappingSchema.backupNovelId, backupNovelId), + inArray(restoreChapterMappingSchema.backupChapterId, backupChapterIds), + ), + ) + .all(); +}; + +export const clearRestoreChapterMappings = async (restoreRunId: string) => { + await dbManager.write(tx => + tx + .delete(restoreChapterMappingSchema) + .where(eq(restoreChapterMappingSchema.restoreRunId, restoreRunId)) + .run(), + ); +}; diff --git a/src/database/queries/__tests__/ChapterQueries.test.ts b/src/database/queries/__tests__/ChapterQueries.test.ts index 687f833db3..263d790888 100644 --- a/src/database/queries/__tests__/ChapterQueries.test.ts +++ b/src/database/queries/__tests__/ChapterQueries.test.ts @@ -153,9 +153,7 @@ describe('ChapterQueries', () => { describe('getAllNovelChaptersForBackup', () => { it('returns chapters beyond the 1000-row UI query limit', async () => { const testDb = getTestDb(); - // No Novel row is needed for this selector test. Keeping the id orphaned - // also avoids running aggregate-stat triggers 1001 times during setup. - const novelId = 123456; + const novelId = await insertTestNovel(testDb); const values = Array.from( { length: 1001 }, @@ -185,6 +183,33 @@ describe('ChapterQueries', () => { expect(backupChapters).toHaveLength(1001); expect(backupChapters.at(-1)?.name).toBe('Chapter 1001'); }); + + it('returns chapters for multiple novels in novel and chapter order', async () => { + const testDb = getTestDb(); + const firstNovelId = await insertTestNovel(testDb); + const secondNovelId = await insertTestNovel(testDb); + testDb.sqlite.executeSync(` + INSERT INTO Chapter + (novelId, path, name, chapterNumber, page, position) + VALUES + (${firstNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${firstNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2), + (${secondNovelId}, '/chapter/1', 'Chapter 1', 1, '1', 1), + (${secondNovelId}, '/chapter/2', 'Chapter 2', 2, '1', 2) + `); + + const chapters = await getAllNovelChaptersForBackup([ + secondNovelId, + firstNovelId, + ]); + + expect(chapters.map(chapter => [chapter.novelId, chapter.name])).toEqual([ + [firstNovelId, 'Chapter 1'], + [firstNovelId, 'Chapter 2'], + [secondNovelId, 'Chapter 1'], + [secondNovelId, 'Chapter 2'], + ]); + }); }); describe('markChaptersRead', () => { diff --git a/src/database/queries/__tests__/NovelQueries.test.ts b/src/database/queries/__tests__/NovelQueries.test.ts index 916daad39b..7185332849 100644 --- a/src/database/queries/__tests__/NovelQueries.test.ts +++ b/src/database/queries/__tests__/NovelQueries.test.ts @@ -14,7 +14,6 @@ import { } from './testData'; import { categorySchema, - chapterSchema, novelCategorySchema, novelSchema, } from '@database/schema'; @@ -30,12 +29,10 @@ import { removeNovelsFromLibrary, getCachedNovels, deleteCachedNovels, - restoreLibrary, updateNovelInfo, pickCustomNovelCover, updateNovelCategoryById, updateNovelCategories, - _restoreNovelAndChapters, } from '../NovelQueries'; const mockGetLibraryDefaultCategoryId = jest.fn(); @@ -421,112 +418,6 @@ describe('NovelQueries', () => { }); }); - describe('restoreLibrary', () => { - it('should restore novel from backup', async () => { - const novel = { - id: 999, - path: '/test/novel', - pluginId: 'test-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 1, - chaptersDownloaded: 0, - chaptersUnread: 0, - totalChapters: 0, - lastReadAt: null, - lastUpdatedAt: null, - }; - - // Mock fetchNovel to return a valid SourceNovel - const { fetchNovel } = require('@services/plugin/fetch'); - jest.mocked(fetchNovel).mockResolvedValueOnce({ - id: undefined, - path: '/test/novel', - name: 'Restored Novel', - chapters: [], - }); - - await restoreLibrary(novel); - - const restored = await getNovelByPath('/test/novel', 'test-plugin'); - expect(restored?.name).toBe('Restored Novel'); - }); - }); - - describe('_restoreNovelAndChapters', () => { - it('does not replace an unrelated novel when backup IDs collide', async () => { - const testDb = getTestDb(); - await insertTestNovel(testDb, { - path: '/existing/novel', - pluginId: 'existing-plugin', - name: 'Existing Novel', - inLibrary: true, - }); - - const mapping = await _restoreNovelAndChapters({ - id: 1, - path: '/restored/novel', - pluginId: 'restored-plugin', - name: 'Restored Novel', - cover: null, - summary: null, - author: null, - artist: null, - status: 'Ongoing', - genres: null, - inLibrary: true, - isLocal: false, - totalPages: 0, - chapters: [ - { - id: 10, - novelId: 1, - path: '/restored/chapter-1', - name: 'Chapter 1', - releaseTime: null, - readTime: null, - bookmark: false, - unread: true, - isDownloaded: true, - updatedTime: null, - chapterNumber: 1, - page: '1', - progress: null, - position: 0, - scanlator: null, - timeSpent: 0, - }, - ], - }); - - expect(mapping.restoredNovelId).not.toBe(1); - expect( - (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, - ).toBe('Existing Novel'); - expect( - (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, - ).toBe(mapping.restoredNovelId); - const restoredChapters = await testDb.drizzleDb - .select() - .from(chapterSchema) - .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) - .all(); - expect(mapping.chapters).toEqual([ - { - backupChapterId: 10, - restoredChapterId: restoredChapters[0].id, - }, - ]); - }); - }); - describe('updateNovelInfo', () => { it('should update novel information', async () => { const testDb = getTestDb(); diff --git a/src/database/queries/__tests__/NovelRestoreQueries.test.ts b/src/database/queries/__tests__/NovelRestoreQueries.test.ts new file mode 100644 index 0000000000..1d24c99513 --- /dev/null +++ b/src/database/queries/__tests__/NovelRestoreQueries.test.ts @@ -0,0 +1,549 @@ +import './mockDb'; +import { setupTestDatabase, getTestDb, teardownTestDatabase } from './setup'; +import { insertTestNovel, insertTestChapter, clearAllTables } from './testData'; +import { chapterSchema, novelSchema } from '@database/schema'; +import { and, eq } from 'drizzle-orm'; + +import { getNovelByPath } from '../NovelQueries'; +import { + clearRestoreChapterMappings, + getRestoreChapterMappings, + restoreLibrary, + _restoreNovelAndChapters, + _restoreNovelsAndChapters, +} from '../NovelRestoreQueries'; + +const mockGetLibraryDefaultCategoryId = jest.fn(); + +jest.mock('@hooks/persisted/useSettings', () => ({ + getLibraryDefaultCategoryId: () => mockGetLibraryDefaultCategoryId(), +})); + +describe('NovelRestoreQueries', () => { + beforeEach(() => { + const testDb = setupTestDatabase(); + clearAllTables(testDb); + mockGetLibraryDefaultCategoryId.mockReturnValue(undefined); + }); + + afterAll(() => { + teardownTestDatabase(); + }); + describe('restoreLibrary', () => { + it('should restore novel from backup', async () => { + const novel = { + id: 999, + path: '/test/novel', + pluginId: 'test-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 1, + chaptersDownloaded: 0, + chaptersUnread: 0, + totalChapters: 0, + lastReadAt: null, + lastUpdatedAt: null, + }; + + // Mock fetchNovel to return a valid SourceNovel + const { fetchNovel } = require('@services/plugin/fetch'); + jest.mocked(fetchNovel).mockResolvedValueOnce({ + id: undefined, + path: '/test/novel', + name: 'Restored Novel', + chapters: [], + }); + + await restoreLibrary(novel); + + const restored = await getNovelByPath('/test/novel', 'test-plugin'); + expect(restored?.name).toBe('Restored Novel'); + }); + }); + + describe('_restoreNovelAndChapters', () => { + it('does not replace an unrelated novel when backup IDs collide', async () => { + const testDb = getTestDb(); + await insertTestNovel(testDb, { + path: '/existing/novel', + pluginId: 'existing-plugin', + name: 'Existing Novel', + inLibrary: true, + }); + + const mapping = await _restoreNovelAndChapters( + { + id: 1, + path: '/restored/novel', + pluginId: 'restored-plugin', + name: 'Restored Novel', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 10, + novelId: 1, + path: '/restored/chapter-1', + name: 'Chapter 1', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: null, + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { restoreRunId: 'restore-collision' }, + ); + + expect(mapping.restoredNovelId).not.toBe(1); + expect( + (await getNovelByPath('/existing/novel', 'existing-plugin'))?.name, + ).toBe('Existing Novel'); + expect( + (await getNovelByPath('/restored/novel', 'restored-plugin'))?.id, + ).toBe(mapping.restoredNovelId); + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-1'); + }); + it('restores multiple novels in one batch', async () => { + const mappings = await _restoreNovelsAndChapters( + [ + { + id: 100, + path: '/bulk/one', + pluginId: 'bulk-plugin', + name: 'Bulk One', + chapters: [], + }, + { + id: 101, + path: '/bulk/two', + pluginId: 'bulk-plugin', + name: 'Bulk Two', + chapters: [], + }, + ], + { includeChapterMappings: false }, + ); + + expect(mappings).toHaveLength(2); + expect(mappings.map(mapping => mapping.pluginId)).toEqual([ + 'bulk-plugin', + 'bulk-plugin', + ]); + expect(await getNovelByPath('/bulk/one', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk One' }), + ); + expect(await getNovelByPath('/bulk/two', 'bulk-plugin')).toEqual( + expect.objectContaining({ name: 'Bulk Two' }), + ); + }); + it('preserves mappings and aggregate stats for multiple novels', async () => { + await _restoreNovelsAndChapters( + [ + { + id: 200, + path: '/bulk/mapped-one', + pluginId: 'bulk-plugin', + name: 'Mapped One', + chapters: [ + { + id: 2001, + novelId: 200, + path: '/bulk/chapter-one', + name: 'Chapter One', + releaseTime: null, + readTime: '2024-01-01T00:00:00.000Z', + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + { + id: 2002, + novelId: 200, + path: '/bulk/chapter-two', + name: 'Chapter Two', + releaseTime: null, + readTime: '2025-01-01T00:00:00.000Z', + bookmark: false, + unread: false, + isDownloaded: false, + updatedTime: '2023-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + progress: null, + position: 1, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 201, + path: '/bulk/mapped-two', + pluginId: 'bulk-plugin', + name: 'Mapped Two', + chapters: [ + { + id: 2011, + novelId: 201, + path: '/bulk/chapter-three', + name: 'Chapter Three', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ], + { includeChapterMappings: true, restoreRunId: 'restore-mappings' }, + ); + + const stagedMappingsOne = await getRestoreChapterMappings( + 'restore-mappings', + 200, + [2001, 2002], + ); + const stagedMappingsTwo = await getRestoreChapterMappings( + 'restore-mappings', + 201, + [2011], + ); + expect( + stagedMappingsOne.map(row => row.backupChapterId).sort((a, b) => a - b), + ).toEqual([2001, 2002]); + expect(stagedMappingsTwo.map(row => row.backupChapterId)).toEqual([2011]); + expect( + new Set( + [...stagedMappingsOne, ...stagedMappingsTwo].map( + row => row.restoredChapterId, + ), + ).size, + ).toBe(3); + expect(await getNovelByPath('/bulk/mapped-one', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2024-01-01T00:00:00.000Z', + }), + ); + expect(await getNovelByPath('/bulk/mapped-two', 'bulk-plugin')).toEqual( + expect.objectContaining({ + totalChapters: 1, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: null, + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + }); + it('restores chapters without allocating ID mappings when requested', async () => { + const mapping = await _restoreNovelAndChapters( + { + id: 2, + path: '/restored/without-mappings', + pluginId: 'restored-plugin', + name: 'Restored Without Mappings', + cover: null, + summary: null, + author: null, + artist: null, + status: 'Ongoing', + genres: null, + inLibrary: true, + isLocal: false, + totalPages: 0, + chapters: [ + { + id: 20, + novelId: 2, + path: '/restored/chapter-2', + name: 'Chapter 2', + releaseTime: null, + readTime: null, + bookmark: false, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: 2, + page: '1', + progress: null, + position: 0, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { includeChapterMappings: false }, + ); + + const restoredChapters = await getTestDb() + .drizzleDb.select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, mapping.restoredNovelId)) + .all(); + expect(restoredChapters).toHaveLength(1); + expect(restoredChapters[0].path).toBe('/restored/chapter-2'); + expect( + await getRestoreChapterMappings('without-mappings', 2, [20]), + ).toEqual([]); + }); + it('merges restored identities without replacing existing rows', async () => { + const testDb = getTestDb(); + const existingNovelId = await insertTestNovel(testDb, { + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Before Restore', + cover: 'before-cover', + summary: 'before-summary', + author: 'Before Author', + artist: 'Before Artist', + status: 'Paused', + genres: 'before', + inLibrary: false, + isLocal: false, + totalPages: 1, + }); + const conflictingChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/chapter', + name: 'Before Chapter', + bookmark: false, + unread: true, + readTime: '2023-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2022-01-01T00:00:00.000Z', + chapterNumber: 1, + page: '1', + position: 0, + progress: 1, + timeSpent: 2, + }, + ); + const unrelatedChapterId = await insertTestChapter( + testDb, + existingNovelId, + { + path: '/restore/merge/unrelated', + name: 'Unrelated Chapter', + bookmark: false, + unread: true, + readTime: '2024-01-01T00:00:00.000Z', + isDownloaded: false, + updatedTime: '2024-01-01T00:00:00.000Z', + chapterNumber: 2, + page: '1', + position: 1, + progress: 0, + timeSpent: 3, + }, + ); + expect(existingNovelId).not.toBe(900); + expect(conflictingChapterId).not.toBe(9001); + + const restoreRunId = 'restore-merge'; + const backupNovel = { + id: 900, + path: '/restore/merge', + pluginId: 'merge-plugin', + name: 'Restored Novel', + cover: 'file:///mock/novel/storage/cache-key?size=large', + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: true, + isLocal: true, + totalPages: 123, + chapters: [ + { + id: 9001, + novelId: 900, + path: '/restore/merge/chapter', + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }, + ], + }; + + const firstMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(firstMapping).toEqual({ + pluginId: 'merge-plugin', + backupNovelId: 900, + restoredNovelId: existingNovelId, + }); + + const restoredNovel = await getNovelByPath( + '/restore/merge', + 'merge-plugin', + ); + expect(restoredNovel).toEqual( + expect.objectContaining({ + id: existingNovelId, + name: 'Restored Novel', + cover: `file:///mock/novel/storage/merge-plugin/${existingNovelId}/cover.png?size=large`, + summary: 'backup-summary', + author: 'Backup Author', + artist: 'Backup Artist', + status: 'Completed', + genres: 'backup', + inLibrary: 1, + isLocal: 1, + totalPages: 123, + totalChapters: 2, + chaptersDownloaded: 1, + chaptersUnread: 1, + lastReadAt: '2025-01-01T00:00:00.000Z', + lastUpdatedAt: '2026-01-01T00:00:00.000Z', + }), + ); + + const restoredChapters = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(restoredChapters).toHaveLength(2); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/chapter', + ), + ).toEqual( + expect.objectContaining({ + id: conflictingChapterId, + novelId: existingNovelId, + name: 'Restored Chapter', + releaseTime: '2024-02-01T00:00:00.000Z', + readTime: '2025-01-01T00:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2026-01-01T00:00:00.000Z', + chapterNumber: 10, + page: '7', + progress: 42, + position: 4, + scanlator: 'Backup Scanlator', + timeSpent: 9, + }), + ); + expect( + restoredChapters.find( + chapter => chapter.path === '/restore/merge/unrelated', + ), + ).toEqual( + expect.objectContaining({ + id: unrelatedChapterId, + name: 'Unrelated Chapter', + }), + ); + + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([ + { + backupChapterId: 9001, + restoredChapterId: conflictingChapterId, + }, + ]); + + const secondMapping = await _restoreNovelAndChapters(backupNovel, { + includeChapterMappings: true, + restoreRunId, + }); + expect(secondMapping.restoredNovelId).toBe(existingNovelId); + const novelsAfterReplay = await testDb.drizzleDb + .select() + .from(novelSchema) + .where( + and( + eq(novelSchema.path, '/restore/merge'), + eq(novelSchema.pluginId, 'merge-plugin'), + ), + ) + .all(); + const chaptersAfterReplay = await testDb.drizzleDb + .select() + .from(chapterSchema) + .where(eq(chapterSchema.novelId, existingNovelId)) + .all(); + expect(novelsAfterReplay).toHaveLength(1); + expect(chaptersAfterReplay).toHaveLength(2); + expect( + chaptersAfterReplay.map(chapter => chapter.id).sort((a, b) => a - b), + ).toEqual( + [conflictingChapterId, unrelatedChapterId].sort((a, b) => a - b), + ); + + await clearRestoreChapterMappings(restoreRunId); + expect( + await getRestoreChapterMappings(restoreRunId, 900, [9001]), + ).toEqual([]); + expect( + testDb.sqlite.executeSync('PRAGMA foreign_key_check').rows, + ).toEqual([]); + }); + }); +}); diff --git a/src/database/queries/__tests__/index.ts b/src/database/queries/__tests__/index.ts index fe7903cab7..2b9fec59f9 100644 --- a/src/database/queries/__tests__/index.ts +++ b/src/database/queries/__tests__/index.ts @@ -122,12 +122,18 @@ export const testModules = { 'removeNovelsFromLibrary', 'getCachedNovels', 'deleteCachedNovels', - 'restoreLibrary', 'updateNovelInfo', 'pickCustomNovelCover', 'updateNovelCategoryById', 'updateNovelCategories', + ], + }, + NovelRestoreQueries: { + description: 'Novel restore operations', + functions: [ + 'restoreLibrary', '_restoreNovelAndChapters', + '_restoreNovelsAndChapters', ], }, RepositoryQueries: { diff --git a/src/database/queries/__tests__/testData.ts b/src/database/queries/__tests__/testData.ts index 834b2f5d7e..cd2761495d 100644 --- a/src/database/queries/__tests__/testData.ts +++ b/src/database/queries/__tests__/testData.ts @@ -22,6 +22,7 @@ import { export function clearAllTables(testDb: TestDb) { const { sqlite } = testDb; sqlite.executeSync('DELETE FROM NovelCategory'); + sqlite.executeSync('DELETE FROM RestoreChapterMapping'); sqlite.executeSync('DELETE FROM Chapter'); sqlite.executeSync('DELETE FROM Novel'); sqlite.executeSync('DELETE FROM Repository'); diff --git a/src/database/queries/__tests__/testDb.ts b/src/database/queries/__tests__/testDb.ts index 0232c41bdd..dcc43981fd 100644 --- a/src/database/queries/__tests__/testDb.ts +++ b/src/database/queries/__tests__/testDb.ts @@ -30,7 +30,7 @@ const MIGRATION_STATEMENTS = [ `CREATE INDEX IF NOT EXISTS category_sort_idx ON Category (sort)`, `CREATE TABLE IF NOT EXISTS Chapter ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, - novelId integer NOT NULL, + novelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, path text NOT NULL, name text NOT NULL, releaseTime text, @@ -70,6 +70,17 @@ const MIGRATION_STATEMENTS = [ )`, `CREATE UNIQUE INDEX IF NOT EXISTS novel_path_plugin_unique ON Novel (path, pluginId)`, `CREATE INDEX IF NOT EXISTS NovelIndex ON Novel (pluginId, path, id, inLibrary)`, + `CREATE TABLE IF NOT EXISTS RestoreChapterMapping ( + restoreRunId text NOT NULL, + backupNovelId integer NOT NULL, + backupChapterId integer NOT NULL, + restoredNovelId integer NOT NULL REFERENCES Novel(id) ON DELETE CASCADE, + restoredChapterId integer NOT NULL REFERENCES Chapter(id) ON DELETE CASCADE +)`, + `CREATE UNIQUE INDEX IF NOT EXISTS restore_chapter_mapping_unique + ON RestoreChapterMapping (restoreRunId, backupNovelId, backupChapterId)`, + `CREATE INDEX IF NOT EXISTS restore_chapter_mapping_novel_index + ON RestoreChapterMapping (restoreRunId, backupNovelId)`, `CREATE TABLE IF NOT EXISTS NovelCategory ( id integer PRIMARY KEY AUTOINCREMENT NOT NULL, novelId integer NOT NULL, diff --git a/src/database/schema/chapter.ts b/src/database/schema/chapter.ts index 08354a1282..c4720deb9f 100644 --- a/src/database/schema/chapter.ts +++ b/src/database/schema/chapter.ts @@ -6,12 +6,15 @@ import { text, uniqueIndex, } from 'drizzle-orm/sqlite-core'; +import { novel } from './novel'; export const chapter = sqliteTable( 'Chapter', { id: integer('id').primaryKey({ autoIncrement: true }), - novelId: integer('novelId').notNull(), + novelId: integer('novelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), path: text('path').notNull(), name: text('name').notNull(), releaseTime: text('releaseTime'), diff --git a/src/database/schema/index.ts b/src/database/schema/index.ts index 744c84cd73..2fa0c4edea 100644 --- a/src/database/schema/index.ts +++ b/src/database/schema/index.ts @@ -3,6 +3,7 @@ import { novel } from './novel'; import { chapter } from './chapter'; import { novelCategory } from './novelCategory'; import { repository } from './repository'; +import { restoreChapterMapping } from './restoreChapterMapping'; export { category as categorySchema, @@ -25,6 +26,11 @@ export { type RepositoryRow, type RepositoryInsert, } from './repository'; +export { + restoreChapterMapping as restoreChapterMappingSchema, + type RestoreChapterMappingRow, + type RestoreChapterMappingInsert, +} from './restoreChapterMapping'; /** * Unified schema object containing all database tables @@ -36,6 +42,7 @@ export const schema = { chapter, novelCategory, repository, + restoreChapterMapping, } as const; export type Schema = typeof schema; diff --git a/src/database/schema/restoreChapterMapping.ts b/src/database/schema/restoreChapterMapping.ts new file mode 100644 index 0000000000..c47f1f68f0 --- /dev/null +++ b/src/database/schema/restoreChapterMapping.ts @@ -0,0 +1,40 @@ +import { + index, + integer, + sqliteTable, + text, + uniqueIndex, +} from 'drizzle-orm/sqlite-core'; +import { chapter } from './chapter'; +import { novel } from './novel'; + +export const restoreChapterMapping = sqliteTable( + 'RestoreChapterMapping', + { + restoreRunId: text('restoreRunId').notNull(), + backupNovelId: integer('backupNovelId').notNull(), + backupChapterId: integer('backupChapterId').notNull(), + restoredNovelId: integer('restoredNovelId') + .notNull() + .references(() => novel.id, { onDelete: 'cascade' }), + restoredChapterId: integer('restoredChapterId') + .notNull() + .references(() => chapter.id, { onDelete: 'cascade' }), + }, + table => [ + uniqueIndex('restore_chapter_mapping_unique').on( + table.restoreRunId, + table.backupNovelId, + table.backupChapterId, + ), + index('restore_chapter_mapping_novel_index').on( + table.restoreRunId, + table.backupNovelId, + ), + ], +); + +export type RestoreChapterMappingRow = + typeof restoreChapterMapping.$inferSelect; +export type RestoreChapterMappingInsert = + typeof restoreChapterMapping.$inferInsert; diff --git a/src/database/types/index.ts b/src/database/types/index.ts index 349bfaf0bb..5c260c7601 100644 --- a/src/database/types/index.ts +++ b/src/database/types/index.ts @@ -125,10 +125,6 @@ export interface RestoredNovelMapping { pluginId: string; backupNovelId: number; restoredNovelId: number; - chapters: { - backupChapterId: number; - restoredChapterId: number; - }[]; } export interface BackupCategory extends Category { diff --git a/src/i18n/languages/en/strings.json b/src/i18n/languages/en/strings.json index f2c1401d18..c06ecad714 100644 --- a/src/i18n/languages/en/strings.json +++ b/src/i18n/languages/en/strings.json @@ -141,6 +141,8 @@ "novelDirectoryReadFailed": "Failed to read novel directory: %{error}", "restoringCategories": "Restoring Categories", "restoringCategoriesProgress": "Restoring categories (%{current}/%{total})", + "validatingNovels": "Validating Novels", + "validatingNovelsProgress": "Validating novels (%{current}/%{total})", "restoringNovels": "Restoring Novels", "restoringNovelsProgress": "Restoring novels (%{current}/%{total})", "restoringSettings": "Restoring Settings", diff --git a/src/i18n/types/index.ts b/src/i18n/types/index.ts index 924f4b15b0..36cbe80929 100644 --- a/src/i18n/types/index.ts +++ b/src/i18n/types/index.ts @@ -124,6 +124,8 @@ export interface StringMap { 'backupScreen.novelDirectoryReadFailed': 'string'; 'backupScreen.restoringCategories': 'string'; 'backupScreen.restoringCategoriesProgress': 'string'; + 'backupScreen.validatingNovels': 'string'; + 'backupScreen.validatingNovelsProgress': 'string'; 'backupScreen.restoringNovels': 'string'; 'backupScreen.restoringNovelsProgress': 'string'; 'backupScreen.restoringSettings': 'string'; diff --git a/src/services/backup/__tests__/fileSections.test.ts b/src/services/backup/__tests__/fileSections.test.ts index f7d5bac972..a3426bef79 100644 --- a/src/services/backup/__tests__/fileSections.test.ts +++ b/src/services/backup/__tests__/fileSections.test.ts @@ -1,17 +1,64 @@ import NativeFile from '@modules/native-file'; -import { restoreNovelFiles } from '../fileSections'; +import { + getSelectedBackupFileSections, + restoreLegacyFiles, + restoreNovelFiles, +} from '../fileSections'; +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; +import { ZipBackupName } from '../types'; + +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', })); -describe('restoreNovelFiles', () => { - it('moves downloaded files to remapped novel and chapter IDs', async () => { +describe('selected backup file sections', () => { + const options = { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }; + + it('keeps novel files in the legacy v2 section only', () => { + expect(getSelectedBackupFileSections(options)).toEqual([ + { + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + ]); + expect(getSelectedBackupFileSections(options, 2)).toEqual([ + { + archiveName: ZipBackupName.PLUGINS, + storagePath: '/storage/Plugins', + }, + { + archiveName: ZipBackupName.NOVEL_FILES, + storagePath: '/storage/Novels', + }, + ]); + }); +}); + +describe('restore file sections', () => { + beforeEach(() => { + jest.mocked(getRestoreChapterMappings).mockReset(); + }); + + it('moves downloaded files with bounded staged mapping lookups', async () => { jest.mocked(NativeFile.exists).mockResolvedValue(true); jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + const chapterItems = Array.from({ length: 101 }, (_, index) => ({ + name: String(index + 1), + path: `/staging/source/1/${index + 1}`, + isDirectory: true, + })); jest.mocked(NativeFile.readDir).mockImplementation(async path => { if (path === '/staging/source/1') { return [ @@ -20,16 +67,7 @@ describe('restoreNovelFiles', () => { path: '/staging/source/1/cover.png', isDirectory: false, }, - { - name: '10', - path: '/staging/source/1/10', - isDirectory: true, - }, - { - name: '20', - path: '/staging/source/1/20', - isDirectory: true, - }, + ...chapterItems, ]; } if (path === '/staging/source/1/10') { @@ -41,18 +79,54 @@ describe('restoreNovelFiles', () => { }, ]; } + if (path === '/staging/source/1/101') { + return [ + { + name: 'index.html', + path: '/staging/source/1/101/index.html', + isDirectory: false, + }, + ]; + } return []; }); + jest + .mocked(getRestoreChapterMappings) + .mockImplementation( + async (_restoreRunId, _backupNovelId, backupChapterIds) => + backupChapterIds + .filter(id => id === 10 || id === 101) + .map(backupChapterId => ({ + backupChapterId, + restoredChapterId: backupChapterId === 10 ? 99 : 199, + })), + ); - await restoreNovelFiles('/staging', [ - { - pluginId: 'source', - backupNovelId: 1, - restoredNovelId: 7, - chapters: [{ backupChapterId: 10, restoredChapterId: 99 }], - }, - ]); + await restoreNovelFiles( + '/staging', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'restore-run-1', + ); + expect(getRestoreChapterMappings).toHaveBeenCalledTimes(2); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 1, + 'restore-run-1', + 1, + Array.from({ length: 100 }, (_, index) => index + 1), + ); + expect(getRestoreChapterMappings).toHaveBeenNthCalledWith( + 2, + 'restore-run-1', + 1, + [101], + ); expect(NativeFile.moveFile).toHaveBeenCalledWith( '/staging/source/1/cover.png', '/storage/Novels/source/7/cover.png', @@ -61,10 +135,53 @@ describe('restoreNovelFiles', () => { '/staging/source/1/10/index.html', '/storage/Novels/source/7/99/index.html', ); + expect(NativeFile.moveFile).toHaveBeenCalledWith( + '/staging/source/1/101/index.html', + '/storage/Novels/source/7/199/index.html', + ); expect(NativeFile.moveFile).not.toHaveBeenCalledWith( expect.stringContaining('/20/'), expect.any(String), ); expect(NativeFile.unlink).toHaveBeenCalledWith('/staging'); }); + + it('passes the restore run ID through legacy file restoration', async () => { + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); + jest.mocked(NativeFile.moveFile).mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockImplementation(async path => { + if (path === '/legacy/Novels/source/1') { + return [ + { + name: '10', + path: '/legacy/Novels/source/1/10', + isDirectory: true, + }, + ]; + } + return []; + }); + jest + .mocked(getRestoreChapterMappings) + .mockResolvedValue([{ backupChapterId: 10, restoredChapterId: 99 }]); + + await restoreLegacyFiles( + '/legacy', + [ + { + pluginId: 'source', + backupNovelId: 1, + restoredNovelId: 7, + }, + ], + 'legacy-run', + ); + + expect(getRestoreChapterMappings).toHaveBeenCalledWith('legacy-run', 1, [ + 10, + ]); + expect(NativeFile.unlink).toHaveBeenCalledWith('/legacy'); + }); }); diff --git a/src/services/backup/__tests__/local.test.ts b/src/services/backup/__tests__/local.test.ts index 5601615128..19ef368dd1 100644 --- a/src/services/backup/__tests__/local.test.ts +++ b/src/services/backup/__tests__/local.test.ts @@ -1,12 +1,37 @@ import NativeFile from '@modules/native-file'; import NativeZipArchive from '@modules/native-zip-archive'; +import { clearRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; import { createBackup, restoreBackup } from '../local'; +import * as fileSections from '../fileSections'; import { finalizeRestoredPlugins } from '../restoreResult'; import { prepareBackupData, restoreData } from '../utils'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(), + getRestoreChapterMappings: jest.fn(), +})); + +jest.mock('../fileSections', () => { + const actual = jest.requireActual('../fileSections'); + return { + ...actual, + restoreLegacyFiles: jest.fn(), + restoreNovelFiles: jest.fn(), + }; +}); + jest.mock('../utils', () => ({ CACHE_DIR_PATH: '/cache/BackupData', clearBackupCache: jest.fn(), + clearRestoreChapterMappingsSafely: jest.fn(async (restoreRunId: string) => { + try { + await jest + .requireMock('@database/queries/NovelRestoreQueries') + .clearRestoreChapterMappings(restoreRunId); + } catch { + // Match the production helper's best-effort cleanup. + } + }), prepareBackupData: jest.fn(), restoreData: jest.fn(), })); @@ -35,12 +60,47 @@ jest.mock('@i18n/translations', () => ({ })); describe('local selective backup', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + beforeEach(() => { + jest.mocked(prepareBackupData).mockReset(); + jest.mocked(restoreData).mockReset(); + jest.mocked(NativeZipArchive.zip).mockReset().mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.zipDirectories) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(NativeZipArchive.unzip) + .mockReset() + .mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.exists).mockReset().mockResolvedValue(false); + jest.mocked(NativeFile.mkdir).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.unlink).mockReset().mockResolvedValue(undefined); + jest.mocked(NativeFile.readDir).mockReset().mockResolvedValue([]); + jest.mocked(finalizeRestoredPlugins).mockReset().mockResolvedValue([]); + jest + .mocked(clearRestoreChapterMappings) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreLegacyFiles) + .mockReset() + .mockResolvedValue(undefined); + jest + .mocked(fileSections.restoreNovelFiles) + .mockReset() + .mockResolvedValue(undefined); + }); + it('creates archives only for selected file sections', async () => { jest.mocked(prepareBackupData).mockResolvedValue({ failedNovelCount: 0, failedSectionCount: 0, }); - jest.mocked(NativeZipArchive.zip).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.zipDirectories).mockResolvedValue(undefined); jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); await createBackup({ @@ -53,12 +113,16 @@ describe('local selective backup', () => { }, }); - expect(prepareBackupData).toHaveBeenCalledWith('/cache/BackupData', { - library: true, - settings: true, - plugins: true, - downloadedFiles: false, - }); + expect(prepareBackupData).toHaveBeenCalledWith( + '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: false, + }, + 3, + ); expect(NativeZipArchive.zip).toHaveBeenCalledWith( '/storage/Plugins', '/cache/BackupData/plugins.zip', @@ -67,8 +131,54 @@ describe('local selective backup', () => { '/storage/Novels', expect.any(String), ); - expect(NativeZipArchive.zip).toHaveBeenCalledWith( + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [{ path: '/cache/BackupData', prefix: '' }], + '/cache/BackupData.zip', + ); + }); + + it('adds novel files to the v3 outer archive without a nested archive', async () => { + jest.mocked(prepareBackupData).mockResolvedValue({ + failedNovelCount: 0, + failedSectionCount: 0, + }); + jest.mocked(NativeZipArchive.zip).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.zipDirectories).mockResolvedValue(undefined); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + + await createBackup({ + destinationUri: 'content://backup.zip', + options: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }); + + expect(prepareBackupData).toHaveBeenCalledWith( '/cache/BackupData', + { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + 3, + ); + expect(NativeZipArchive.zip).toHaveBeenCalledWith( + '/storage/Plugins', + '/cache/BackupData/plugins.zip', + ); + expect(NativeZipArchive.zip).not.toHaveBeenCalledWith( + '/storage/Novels', + expect.any(String), + ); + expect(NativeZipArchive.zipDirectories).toHaveBeenCalledWith( + [ + { path: '/cache/BackupData', prefix: '' }, + { path: '/storage/Novels', prefix: 'NovelFiles' }, + ], '/cache/BackupData.zip', ); }); @@ -83,6 +193,7 @@ describe('local selective backup', () => { failedSectionCount: 0, pluginIds: ['restored'], novelMappings: [], + restoreRunId: 'restore-run-plugins', manifest: { appVersion: '2.1.0', formatVersion: 2 as const, @@ -107,10 +218,273 @@ describe('local selective backup', () => { '/storage/Plugins', ); expect(finalizeRestoredPlugins).toHaveBeenCalledWith(restoreResult); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); expect( jest.mocked(finalizeRestoredPlugins).mock.invocationCallOrder[0], ).toBeGreaterThan( jest.mocked(NativeZipArchive.unzip).mock.invocationCallOrder[1], ); }); + it('keeps a successful restore successful when mapping cleanup fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-cleanup-failure', + manifest: { + appVersion: '2.1.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('preserves the restore error when mapping cleanup also fails', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-restore-failure', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('extracts the v1 downloaded archive into the legacy staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v1', + manifest: { + appVersion: '1.0.0', + formatVersion: 1 as const, + sections: { + library: true, + settings: true, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/download.zip', + '/cache/BackupData/RestoredLegacyFiles', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + expect(fileSections.restoreLegacyFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredLegacyFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('extracts the v2 novel-files archive into the novel staging path', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + '/cache/BackupData/RestoredNovelFiles', + ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/RestoredNovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + + it('restores v3 novel files from the outer archive without nested extraction', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v3', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + novelDataFormat: 2 as const, + sections: { + library: true, + settings: false, + plugins: true, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest.mocked(NativeFile.exists).mockResolvedValue(true); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + jest.mocked(finalizeRestoredPlugins).mockResolvedValueOnce([]); + + await restoreBackup({ sourceUri: 'content://backup.zip' }); + + expect(NativeZipArchive.unzip).toHaveBeenCalledWith( + '/cache/BackupData/plugins.zip', + '/storage/Plugins', + ); + expect(NativeZipArchive.unzip).not.toHaveBeenCalledWith( + '/cache/BackupData/novel-files.zip', + expect.any(String), + ); + expect(fileSections.restoreNovelFiles).toHaveBeenCalledWith( + '/cache/BackupData/NovelFiles', + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); + it('rejects a v3 downloaded-file restore without NovelFiles', async () => { + const restoreResult = { + novelCount: 1, + failedNovelCount: 0, + categoryCount: 0, + failedCategoryCount: 0, + settingsRestored: true, + failedSectionCount: 0, + pluginIds: [], + novelMappings: [], + restoreRunId: 'restore-run-v3-missing', + manifest: { + appVersion: '2.1.3', + formatVersion: 3 as const, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + }; + jest.mocked(restoreData).mockResolvedValueOnce(restoreResult); + jest + .mocked(NativeFile.exists) + .mockImplementation( + async path => path !== '/cache/BackupData/NovelFiles', + ); + jest.mocked(NativeFile.copyFile).mockResolvedValue(undefined); + jest.mocked(NativeZipArchive.unzip).mockResolvedValue(undefined); + + await expect( + restoreBackup({ sourceUri: 'content://backup.zip' }), + ).rejects.toThrow('backupScreen.invalidBackupFolder'); + expect(finalizeRestoredPlugins).not.toHaveBeenCalled(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith( + restoreResult.restoreRunId, + ); + }); }); diff --git a/src/services/backup/__tests__/options.test.ts b/src/services/backup/__tests__/options.test.ts index 33fff6b8bf..edf8fe6cb5 100644 --- a/src/services/backup/__tests__/options.test.ts +++ b/src/services/backup/__tests__/options.test.ts @@ -7,6 +7,10 @@ import { import { getSelectedBackupFileSections } from '../fileSections'; import { ZipBackupName } from '../types'; +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + getRestoreChapterMappings: jest.fn(), +})); + jest.mock('@utils/Storages', () => ({ NOVEL_STORAGE: '/storage/Novels', PLUGIN_STORAGE: '/storage/Plugins', diff --git a/src/services/backup/__tests__/restoreResult.test.ts b/src/services/backup/__tests__/restoreResult.test.ts index 74932e74cc..5797e64ce6 100644 --- a/src/services/backup/__tests__/restoreResult.test.ts +++ b/src/services/backup/__tests__/restoreResult.test.ts @@ -65,6 +65,7 @@ const successfulResult: RestoreResult = { failedSectionCount: 0, pluginIds: ['installed'], novelMappings: [], + restoreRunId: 'restore-run-1', manifest: { appVersion: '2.1.0', formatVersion: 2, diff --git a/src/services/backup/__tests__/utils.test.ts b/src/services/backup/__tests__/utils.test.ts index 7c018e283a..1df406abb4 100644 --- a/src/services/backup/__tests__/utils.test.ts +++ b/src/services/backup/__tests__/utils.test.ts @@ -1,7 +1,9 @@ import { + clearRestoreChapterMappings, _restoreNovelAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; + _restoreNovelsAndChapters, +} from '@database/queries/NovelRestoreQueries'; +import { getAllNovels } from '@database/queries/NovelQueries'; import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; import { _restoreCategory, @@ -10,14 +12,30 @@ import { } from '@database/queries/CategoryQueries'; import NativeFile from '@modules/native-file'; import { MMKVStorage } from '@utils/mmkv/mmkv'; -import { prepareBackupData, restoreData } from '../utils'; +import { + clearRestoreChapterMappingsSafely, + prepareBackupData, + restoreData, +} from '../utils'; +import { decodeNovelBatch, encodeNovelBatch } from '../novelPayload'; +import type { + BackupNovel, + ChapterInfo, + RestoredNovelMapping, +} from '@database/types'; import type { BackupOptions } from '../options'; +import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; jest.mock('@database/queries/NovelQueries', () => ({ - _restoreNovelAndChapters: jest.fn(), getAllNovels: jest.fn(), })); +jest.mock('@database/queries/NovelRestoreQueries', () => ({ + clearRestoreChapterMappings: jest.fn(async () => undefined), + _restoreNovelAndChapters: jest.fn(), + _restoreNovelsAndChapters: jest.fn(), +})); + jest.mock('@database/queries/ChapterQueries', () => ({ getAllNovelChaptersForBackup: jest.fn(), })); @@ -68,9 +86,49 @@ const pluginOnlyOptions: BackupOptions = { plugins: true, downloadedFiles: false, }; +const makeTestChapter = (novelId: number, chapterNumber = 1): ChapterInfo => ({ + id: novelId * 100 + chapterNumber, + novelId, + path: `/novel/${novelId}/chapter/${chapterNumber}`, + name: `Chapter ${chapterNumber}`, + releaseTime: `2024-01-${String(chapterNumber).padStart(2, '0')}`, + readTime: null, + bookmark: chapterNumber % 2 === 0, + unread: chapterNumber % 2 !== 0, + isDownloaded: true, + updatedTime: `2024-02-${String(chapterNumber).padStart(2, '0')}`, + chapterNumber, + page: String(chapterNumber), + position: chapterNumber - 1, + progress: chapterNumber / 10, + scanlator: `scanlator-${chapterNumber}`, + timeSpent: chapterNumber * 60, +}); + +const makeTestNovel = ( + id: number, + pluginId = 'source', + chapters = [makeTestChapter(id)], +): BackupNovel => ({ + id, + name: `Novel ${id}`, + path: `/novel/${id}`, + pluginId, + cover: null, + summary: `Summary ${id}`, + author: `Author ${id}`, + artist: `Artist ${id}`, + status: 'ongoing', + genres: 'fantasy', + inLibrary: true, + isLocal: false, + totalPages: 100, + chapters, +}); describe('selective backup data', () => { beforeEach(() => { + jest.clearAllMocks(); jest.mocked(NativeFile.exists).mockResolvedValue(false); jest.mocked(NativeFile.mkdir).mockResolvedValue(undefined); jest.mocked(NativeFile.writeFile).mockResolvedValue(undefined); @@ -79,15 +137,24 @@ describe('selective backup data', () => { jest.mocked(getAllNovelChaptersForBackup).mockResolvedValue([]); jest.mocked(getCategoriesFromDb).mockResolvedValue([]); jest.mocked(getAllNovelCategories).mockResolvedValue([]); - jest.mocked(_restoreNovelAndChapters).mockImplementation(async novel => ({ + const restoreNovel = async ( + novel: BackupNovel, + _options?: { includeChapterMappings?: boolean; restoreRunId?: string }, + ): Promise => ({ pluginId: novel.pluginId, backupNovelId: novel.id, restoredNovelId: novel.id, - chapters: novel.chapters.map(chapter => ({ - backupChapterId: chapter.id, - restoredChapterId: chapter.id, - })), - })); + }); + jest.mocked(_restoreNovelAndChapters).mockImplementation(restoreNovel); + jest.mocked(_restoreNovelsAndChapters).mockImplementation( + async ( + novels: BackupNovel[], + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => Promise.all(novels.map(novel => restoreNovel(novel, _options))), + ); }); it('writes the selected sections to the v2 manifest', async () => { @@ -108,6 +175,31 @@ describe('selective backup data', () => { ); }); + it('accepts v3 section manifests', async () => { + const options: BackupOptions = { + library: false, + settings: false, + plugins: false, + downloadedFiles: false, + }; + jest.mocked(NativeFile.readFile).mockResolvedValueOnce( + JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }), + ); + + const result = await restoreData('/cache'); + + expect(result.manifest).toEqual({ + appVersion: '2.1.3', + formatVersion: 3, + sections: options, + }); + expect(result.restoreRunId).toEqual(expect.any(String)); + }); + it('does not warn about sections intentionally omitted by the manifest', async () => { jest .mocked(NativeFile.readFile) @@ -126,6 +218,7 @@ describe('selective backup data', () => { const result = await restoreData('/cache'); expect(result).toMatchObject({ + restoreRunId: expect.any(String), failedNovelCount: 0, failedCategoryCount: 0, failedSectionCount: 0, @@ -223,23 +316,16 @@ describe('selective backup data', () => { it('includes stored covers with library data when downloads are omitted', async () => { jest.mocked(getAllNovels).mockResolvedValueOnce([ { - id: 1, - name: 'Example', - path: '/example', - pluginId: 'source', + ...makeTestNovel(1), cover: 'file:///storage/Novels/source/1/cover.png?123', }, ]); jest.mocked(getAllNovelChaptersForBackup).mockResolvedValueOnce([ { + ...makeTestChapter(1), id: 10, - novelId: 1, - path: '/chapter-1', - name: 'Chapter 1', - isDownloaded: true, }, - ] as Awaited>); - + ]); await prepareBackupData('/cache', { library: true, settings: false, @@ -249,17 +335,266 @@ describe('selective backup data', () => { const novelWrite = jest .mocked(NativeFile.writeFile) - .mock.calls.find(([path]) => path.endsWith('/1.json')); - expect(JSON.parse(novelWrite?.[1] ?? '{}')).toMatchObject({ - cover: '/Novels/source/1/cover.png?123', - chapters: [{ id: 10, isDownloaded: false }], + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', }); + expect(compactNovel.c).toHaveLength(1); + expect(compactNovel.c[0]).toHaveLength(15); + expect(compactNovel.c[0][0]).toBe(10); + expect(compactNovel.c[0][7]).toBe(false); expect(NativeFile.copyFile).toHaveBeenCalledWith( 'file:///storage/Novels/source/1/cover.png', '/cache/Covers/1', ); }); + it('does not duplicate covers when downloaded files are included', async () => { + jest.mocked(getAllNovels).mockResolvedValueOnce([ + { + ...makeTestNovel(1), + cover: 'file:///storage/Novels/source/1/cover.png?123', + }, + ]); + + jest.mocked(NativeFile.copyFile).mockClear(); + jest.mocked(NativeFile.mkdir).mockClear(); + + await prepareBackupData('/cache', { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }); + + const novelWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => + path.endsWith('/NovelAndChapters/batch-000001.json'), + ); + const compactNovel = JSON.parse(novelWrite?.[1] ?? '[]')[0]; + expect(compactNovel).toMatchObject({ + id: 1, + co: '/Novels/source/1/cover.png?123', + }); + + expect(NativeFile.copyFile).not.toHaveBeenCalledWith( + 'file:///storage/Novels/source/1/cover.png', + '/cache/Covers/1', + ); + expect(NativeFile.mkdir).not.toHaveBeenCalledWith('/cache/Covers'); + }); + + it('round-trips a fully populated novel through the compact codec', () => { + const novel: BackupNovel & { + chaptersDownloaded: number; + chaptersUnread: number; + totalChapters: number; + lastReadAt: string; + lastUpdatedAt: string; + } = { + id: 7, + name: 'The Compact Novel', + path: '/novels/compact', + pluginId: 'source', + cover: '/covers/compact.png?cache=1', + summary: null, + author: 'Author', + artist: null, + status: 'ongoing', + genres: null, + inLibrary: null, + isLocal: false, + totalPages: 2048, + chaptersDownloaded: 1, + chaptersUnread: 1, + totalChapters: 2, + lastReadAt: '2024-03-01T10:20:30.000Z', + lastUpdatedAt: '2024-03-02T10:20:30.000Z', + chapters: [ + { + id: 701, + novelId: 7, + path: '/novels/compact/1', + name: 'First chapter', + releaseTime: '2024-01-01T00:00:00.000Z', + readTime: '2024-03-01T10:00:00.000Z', + bookmark: true, + unread: false, + isDownloaded: true, + updatedTime: '2024-02-01T00:00:00.000Z', + chapterNumber: 1, + page: '4', + position: 2, + progress: 0.75, + scanlator: 'Team A', + timeSpent: 90, + }, + { + id: 702, + novelId: 7, + path: '/novels/compact/2', + name: 'Second chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: true, + isDownloaded: false, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: null, + }, + ], + }; + + const [compactNovel] = encodeNovelBatch([novel]); + + expect(Object.keys(compactNovel).sort()).toEqual( + [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', + ].sort(), + ); + expect(compactNovel).toMatchObject({ + id: 7, + p: '/novels/compact', + pi: 'source', + n: 'The Compact Novel', + co: '/covers/compact.png?cache=1', + d: 1, + u: 1, + tc: 2, + lr: '2024-03-01T10:20:30.000Z', + lu: '2024-03-02T10:20:30.000Z', + }); + expect(compactNovel.c).toEqual([ + [ + 701, + '/novels/compact/1', + 'First chapter', + '2024-01-01T00:00:00.000Z', + true, + false, + '2024-03-01T10:00:00.000Z', + true, + '2024-02-01T00:00:00.000Z', + 1, + '4', + 2, + 0.75, + 'Team A', + 90, + ], + [ + 702, + '/novels/compact/2', + 'Second chapter', + null, + null, + true, + null, + false, + null, + null, + null, + null, + null, + null, + null, + ], + ]); + expect(Object.keys(compactNovel.c[0])).not.toContain('novelId'); + expect(compactNovel).not.toHaveProperty('chapters'); + + expect(decodeNovelBatch([compactNovel])).toEqual([novel]); + }); + + it('writes 101 novels as ordered compact batches with a format marker', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels = Array.from({ length: 101 }, (_, index) => { + const id = index + 1; + return makeTestNovel(id, 'source', [ + makeTestChapter(id, 1), + makeTestChapter(id, 2), + ]); + }); + jest.mocked(getAllNovels).mockResolvedValueOnce(novels); + jest.mocked(getAllNovelChaptersForBackup).mockImplementation(async ids => { + const requestedIds = Array.isArray(ids) ? ids : [ids]; + return novels + .flatMap(novel => novel.chapters) + .filter(chapter => requestedIds.includes(chapter.novelId)); + }); + + await prepareBackupData('/cache', options); + + const batchWrites = jest + .mocked(NativeFile.writeFile) + .mock.calls.filter(([path]) => path.includes('/NovelAndChapters/batch-')); + expect(batchWrites.map(([path]) => path)).toEqual([ + '/cache/NovelAndChapters/batch-000001.json', + '/cache/NovelAndChapters/batch-000002.json', + ]); + expect(batchWrites.some(([path]) => /\/\d+\.json$/.test(path))).toBe(false); + + const batches = batchWrites.map(([, content]) => JSON.parse(content)); + expect(batches.map(batch => batch.length)).toEqual([100, 1]); + const records = batches.flat(); + expect(records.map(record => record.id)).toEqual( + novels.map(novel => novel.id), + ); + expect( + records.map(record => + record.c.map((chapter: [number, ...unknown[]]) => chapter[0]), + ), + ).toEqual(novels.map(novel => novel.chapters.map(chapter => chapter.id))); + expect( + records.every( + record => + Array.isArray(record.c) && + record.c.every((chapter: unknown[]) => chapter.length === 15), + ), + ).toBe(true); + + const manifestWrite = jest + .mocked(NativeFile.writeFile) + .mock.calls.find(([path]) => path.endsWith('/Version.json')); + expect(JSON.parse(manifestWrite?.[1] ?? '{}')).toMatchObject({ + formatVersion: 2, + novelDataFormat: 2, + }); + }); + it('restores stored covers from library data and preserves missing covers', async () => { const options: BackupOptions = { library: true, @@ -325,17 +660,110 @@ describe('selective backup data', () => { '/cache/Covers/1', '/storage/Novels/source/1/cover.png', ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - id: 1, - cover: 'file:///storage/Novels/source/1/cover.png?123', - }), + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 1, + cover: 'file:///storage/Novels/source/1/cover.png?123', + }), + expect.objectContaining({ id: 2, cover: null }), + ], + { includeChapterMappings: false }, ); - expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ id: 2, cover: null }), + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + }); + + it('keeps restored mappings when a stored cover copy fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const novels = [1, 2, 3].map(id => ({ + ...makeTestNovel(id), + cover: `/Novels/source/${id}/cover.png`, + })); + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + const category = { id: 1, name: 'Category', novelIds: [1, 2, 3] }; + + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => + [ + '/cache/NovelAndChapters', + '/cache/Covers/1', + '/cache/Covers/2', + '/cache/Covers/3', + '/cache/Category.json', + ].includes(path), + ); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + if (path.endsWith('/Category.json')) { + return JSON.stringify([category]); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + if (!novel) { + throw new Error(`Unexpected read: ${path}`); + } + return JSON.stringify(novel); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + jest.mocked(NativeFile.copyFile).mockImplementation(async source => { + if (source === '/cache/Covers/1') { + throw new Error('Cover copy failed'); + } + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + novels.map(novel => ({ + ...novel, + cover: `file:///storage${novel.cover}`, + })), + { includeChapterMappings: false }, + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/2', + '/storage/Novels/source/102/cover.png', + ); + expect(NativeFile.copyFile).toHaveBeenCalledWith( + '/cache/Covers/3', + '/storage/Novels/source/103/cover.png', + ); + expect(_restoreCategory).toHaveBeenCalledWith( + expect.objectContaining({ novelIds: [1, 2, 3] }), + new Map([ + [1, 101], + [2, 102], + [3, 103], + ]), ); + expect(result).toMatchObject({ + novelCount: 3, + failedNovelCount: 1, + failedSectionCount: 0, + novelMappings: mappings, + }); }); it('omits the installed-plugin registry when plugin files are excluded', async () => { @@ -367,6 +795,796 @@ describe('selective backup data', () => { }); }); + it('restores downloaded-file novels in one mapping-aware batch', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 11, + name: 'First', + path: '/first', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 101, + novelId: 11, + name: 'First chapter', + path: '/first/1', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, + }, + ], + }, + { + id: 22, + name: 'Second', + path: '/second', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 202, + novelId: 22, + name: 'Second chapter', + path: '/second/1', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: true, + updatedTime: null, + chapterNumber: null, + page: null, + position: null, + progress: null, + scanlator: null, + timeSpent: 0, + }, + ], + }, + ]; + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'source', + backupNovelId: 11, + restoredNovelId: 111, + }, + { + pluginId: 'source', + backupNovelId: 22, + restoredNovelId: 222, + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(result.restoreRunId).toBe( + jest.mocked(_restoreNovelsAndChapters).mock.calls[0][1]?.restoreRunId, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + it('reports validation and restore progress separately', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const novel = makeTestNovel(11); + const mapping: RestoredNovelMapping = { + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: 111, + }; + const progressTexts: string[] = []; + const setMeta: TaskProgressUpdater = transform => { + const next = transform({ + name: 'LOCAL_RESTORE', + isRunning: true, + progress: undefined, + progressText: undefined, + }); + if (next.progressText) { + progressTexts.push(next.progressText); + } + }; + const benchmarkLog = jest.fn(); + + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: '11.json', + path: '/cache/NovelAndChapters/11.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(novel); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce([mapping]); + + await restoreData('/cache', setMeta, benchmarkLog); + + expect(progressTexts.slice(0, 4)).toEqual([ + 'backupScreen.validatingNovels', + 'backupScreen.validatingNovelsProgress', + 'backupScreen.restoringNovels', + 'backupScreen.restoringNovelsProgress', + ]); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:validation:start', + ); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:validation:done total=1', + ); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:restore:start total=1', + ); + expect(benchmarkLog).toHaveBeenCalledWith( + 'restoreData:novels:restore:progress current=1 total=1', + ); + }); + + it('restores scrambled compact batches in order with downloaded-file mappings', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const firstNovel = makeTestNovel(11, 'plugin-a', [makeTestChapter(11, 1)]); + const secondNovel = makeTestNovel(22, 'plugin-b', [makeTestChapter(22, 1)]); + const mappings: RestoredNovelMapping[] = [ + { + pluginId: 'plugin-a', + backupNovelId: 11, + restoredNovelId: 111, + }, + { + pluginId: 'plugin-b', + backupNovelId: 22, + restoredNovelId: 222, + }, + ]; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'batch-000002.json', + path: '/cache/NovelAndChapters/batch-000002.json', + isDirectory: false, + }, + { + name: 'ignored-directory', + path: '/cache/NovelAndChapters/ignored-directory', + isDirectory: true, + }, + { + name: 'batch-000001.json', + path: '/cache/NovelAndChapters/batch-000001.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + if (path.endsWith('/batch-000001.json')) { + return JSON.stringify(encodeNovelBatch([firstNovel])); + } + if (path.endsWith('/batch-000002.json')) { + return JSON.stringify(encodeNovelBatch([secondNovel])); + } + throw new Error(`Unexpected read: ${path}`); + }); + jest.mocked(_restoreNovelsAndChapters).mockResolvedValueOnce(mappings); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + const restoredNovels = jest.mocked(_restoreNovelsAndChapters).mock + .calls[0][0] as BackupNovel[]; + expect(restoredNovels.map(novel => novel.id)).toEqual([11, 22]); + expect( + restoredNovels.map(novel => + novel.chapters.map(chapter => ({ + id: chapter.id, + novelId: chapter.novelId, + isDownloaded: chapter.isDownloaded, + })), + ), + ).toEqual([ + [{ id: 1101, novelId: 11, isDownloaded: true }], + [{ id: 2201, novelId: 22, isDownloaded: true }], + ]); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(expect.any(Array), { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + pluginIds: ['plugin-a', 'plugin-b'], + novelMappings: mappings, + }); + }); + + it('retries each downloaded-file novel when its batch restore fails', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const novels: BackupNovel[] = [ + { + id: 31, + name: 'First', + path: '/first', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [], + }, + { + id: 32, + name: 'Second', + path: '/second', + pluginId: 'source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [], + }, + ]; + const mappings: RestoredNovelMapping[] = novels.map(novel => ({ + pluginId: novel.pluginId, + backupNovelId: novel.id, + restoredNovelId: novel.id + 100, + })); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + novels.map(novel => ({ + name: `${novel.id}.json`, + path: `/cache/NovelAndChapters/${novel.id}.json`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + const novel = novels.find(item => path.endsWith(`/${item.id}.json`)); + return JSON.stringify(novel ?? {}); + }); + jest + .mocked(_restoreNovelsAndChapters) + .mockRejectedValueOnce(new Error('batch failed')); + jest.mocked(_restoreNovelAndChapters).mockImplementation( + async ( + novel, + _options?: { + includeChapterMappings?: boolean; + restoreRunId?: string; + }, + ) => mappings.find(mapping => mapping.backupNovelId === novel.id)!, + ); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith(novels, { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(1, novels[0], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).toHaveBeenNthCalledWith(2, novels[1], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 2, + failedNovelCount: 0, + novelMappings: mappings, + }); + }); + + it.each([ + { + label: 'v1', + manifest: { version: '1.0.0' }, + formatVersion: 1, + }, + { + label: 'v2', + manifest: { + appVersion: '2.0.0', + formatVersion: 2, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 2, + }, + { + label: 'v3', + manifest: { + appVersion: '3.0.0', + formatVersion: 3, + sections: { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }, + }, + formatVersion: 3, + }, + ])( + '$label object payloads remain restorable', + async ({ manifest, formatVersion }) => { + const novel = makeTestNovel(301, 'legacy-source'); + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify(manifest); + } + return JSON.stringify(novel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 301, + pluginId: 'legacy-source', + chapters: [ + expect.objectContaining({ + id: 30101, + novelId: 301, + }), + ], + }), + ], + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, + ); + expect(result.manifest).toMatchObject({ formatVersion }); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + novelMappings: [ + expect.objectContaining({ + backupNovelId: 301, + pluginId: 'legacy-source', + }), + ], + }); + }, + ); + + it('normalizes legacy object payloads before restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + chapters: [ + { + id: 50101, + novelId: 999, + path: '/legacy/1', + name: 'Legacy chapter', + }, + ], + ignoredLegacyField: 'ignored', + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'legacy.json', + path: '/cache/NovelAndChapters/legacy.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + { + id: 501, + name: 'Legacy novel', + path: '/legacy', + pluginId: 'legacy-source', + cover: null, + summary: null, + author: null, + artist: null, + status: null, + genres: null, + inLibrary: null, + isLocal: null, + totalPages: null, + chapters: [ + { + id: 50101, + novelId: 501, + path: '/legacy/1', + name: 'Legacy chapter', + releaseTime: null, + readTime: null, + bookmark: null, + unread: null, + isDownloaded: null, + updatedTime: null, + chapterNumber: null, + page: null, + progress: null, + position: null, + scanlator: null, + timeSpent: null, + }, + ], + }, + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 0, + }); + }); + it('restores legacy novels whose stored name is empty', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: false, + }; + const legacyNovel = { + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + chapters: [], + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue([ + { + name: 'empty-name.json', + path: '/cache/NovelAndChapters/empty-name.json', + isDirectory: false, + }, + ]); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + sections: options, + }); + } + return JSON.stringify(legacyNovel); + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 701, + name: '', + path: '/legacy/empty-name', + pluginId: 'legacy-source', + }), + ], + { includeChapterMappings: false }, + ); + expect(result).toMatchObject({ + novelCount: 1, + failedNovelCount: 0, + }); + }); + + it('rejects duplicate source IDs, novel identities, and chapter identities before restoring invalid files', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(601, 'source'); + const duplicateIdNovel = makeTestNovel(601, 'other-source'); + const duplicateIdentityNovel = { + ...makeTestNovel(602, 'source'), + path: validNovel.path, + }; + const duplicateChapterIdentityNovel = { + ...makeTestNovel(603, 'source'), + chapters: [ + makeTestChapter(603, 1), + { ...makeTestChapter(603, 1), id: 60302 }, + ], + }; + const payloads = { + 'batch-000001.json': JSON.stringify(encodeNovelBatch([validNovel])), + 'batch-000002.json': JSON.stringify(encodeNovelBatch([duplicateIdNovel])), + 'batch-000003.json': JSON.stringify( + encodeNovelBatch([duplicateIdentityNovel]), + ), + 'batch-000004.json': JSON.stringify( + encodeNovelBatch([duplicateChapterIdentityNovel]), + ), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(payloads) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return payloads[path.split('/').pop() as keyof typeof payloads]; + }); + + const result = await restoreData('/cache'); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith([validNovel], { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 3, + pluginIds: ['source'], + }); + }); + + it('skips malformed compact files without partially restoring them', async () => { + const options: BackupOptions = { + library: true, + settings: false, + plugins: false, + downloadedFiles: true, + }; + const validNovel = makeTestNovel(901, 'valid-source'); + const [validRecord] = encodeNovelBatch([validNovel]); + const malformedFiles: Record = { + 'batch-000001.json': '{', + 'batch-000002.json': JSON.stringify({}), + 'batch-000003.json': JSON.stringify([{ ...validRecord, n: undefined }]), + 'batch-000004.json': JSON.stringify([ + validRecord, + { + ...validRecord, + c: [[...validRecord.c[0]].slice(0, 14)], + }, + ]), + 'batch-000005.json': JSON.stringify([{ ...validRecord, id: 'bad' }]), + 'batch-000006.json': JSON.stringify([{ ...validRecord, p: 123 }]), + 'batch-000007.json': JSON.stringify([validRecord]), + }; + jest + .mocked(NativeFile.exists) + .mockImplementation(async path => path === '/cache/NovelAndChapters'); + jest.mocked(NativeFile.readDir).mockResolvedValue( + Object.keys(malformedFiles) + .reverse() + .map(name => ({ + name, + path: `/cache/NovelAndChapters/${name}`, + isDirectory: false, + })), + ); + jest.mocked(NativeFile.readFile).mockImplementation(async path => { + if (path.endsWith('/Version.json')) { + return JSON.stringify({ + appVersion: '2.1.3', + formatVersion: 2, + novelDataFormat: 2, + sections: options, + }); + } + return malformedFiles[path.split('/').pop() ?? '']; + }); + + const result = await restoreData('/cache'); + expect( + jest + .mocked(NativeFile.readFile) + .mock.calls.map(([path]) => path) + .filter(path => path.includes('/NovelAndChapters/')), + ).toEqual([ + ...Object.keys(malformedFiles) + .sort() + .map(name => `/cache/NovelAndChapters/${name}`), + '/cache/NovelAndChapters/batch-000007.json', + ]); + + expect(_restoreNovelsAndChapters).toHaveBeenCalledTimes(1); + expect(_restoreNovelsAndChapters).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: 901, + pluginId: 'valid-source', + chapters: [ + expect.objectContaining({ + id: 90101, + novelId: 901, + }), + ], + }), + ], + { + includeChapterMappings: true, + restoreRunId: expect.any(String), + }, + ); + expect(_restoreNovelAndChapters).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + restoreRunId: expect.any(String), + novelCount: 1, + failedNovelCount: 7, + }); + }); + it('treats backups without a section manifest as legacy full backups', async () => { jest .mocked(NativeFile.readFile) @@ -384,5 +1602,16 @@ describe('selective backup data', () => { downloadedFiles: true, }, }); + expect(result.restoreRunId).toEqual(expect.any(String)); + }); + it('does not reject when restore mapping cleanup fails', async () => { + jest + .mocked(clearRestoreChapterMappings) + .mockRejectedValueOnce(new Error('mapping cleanup failed')); + + await expect( + clearRestoreChapterMappingsSafely('restore-run'), + ).resolves.toBeUndefined(); + expect(clearRestoreChapterMappings).toHaveBeenCalledWith('restore-run'); }); }); diff --git a/src/services/backup/drive/index.ts b/src/services/backup/drive/index.ts index d00e13041c..8068387416 100644 --- a/src/services/backup/drive/index.ts +++ b/src/services/backup/drive/index.ts @@ -7,10 +7,12 @@ import { clearBackupCache, prepareBackupData, restoreData, + clearRestoreChapterMappingsSafely, } from '../utils'; import { finalizeRestoredPlugins, getRestoreCompletionText, + type RestoreResult, } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import { download, updateMetadata, uploadMedia } from '@api/drive/request'; @@ -77,7 +79,7 @@ export const createDriveBackup = async ( progressText: getString('backupScreen.uploadingSelectedFiles'), })); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await uploadBackupSection( section.storagePath, section.archiveName, @@ -99,86 +101,100 @@ export const driveRestore = async ( backupFolder: DriveFile, setMeta: TaskProgressUpdater, ) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - const zipDataFile = await exists(ZipBackupName.DATA, false, backupFolder.id); - if (!zipDataFile) { - throw new Error(getString('backupScreen.invalidBackupFolder')); - } - - await clearBackupCache(); - await download(zipDataFile, CACHE_DIR_PATH); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - await sleep(500); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFile = await exists( - ZipBackupName.DOWNLOAD, + let restoreResult: RestoreResult | undefined; + try { + setMeta(meta => ({ + ...meta, + isRunning: true, + progress: 0 / 3, + progressText: getString('backupScreen.downloadingData'), + })); + + const zipDataFile = await exists( + ZipBackupName.DATA, false, backupFolder.id, ); - if (!legacyFile) { + if (!zipDataFile) { throw new Error(getString('backupScreen.invalidBackupFolder')); } - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download(legacyFile, legacyFilesRestorePath); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { - const file = await exists(section.archiveName, false, backupFolder.id); - if (!file) { + + await clearBackupCache(); + await download(zipDataFile, CACHE_DIR_PATH); + await sleep(500); + + setMeta(meta => ({ + ...meta, + progress: 1 / 3, + progressText: getString('backupScreen.restoringData'), + })); + + restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); + await sleep(500); + + setMeta(meta => ({ + ...meta, + progress: 2 / 3, + progressText: getString('backupScreen.restoringSelectedFiles'), + })); + + if (restoreResult.manifest.formatVersion === 1) { + const legacyFile = await exists( + ZipBackupName.DOWNLOAD, + false, + backupFolder.id, + ); + if (!legacyFile) { throw new Error(getString('backupScreen.invalidBackupFolder')); } - await download( - file, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, - ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, + const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); + await download(legacyFile, legacyFilesRestorePath); + await restoreLegacyFiles( + legacyFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); + } else { + const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); + for (const section of getSelectedBackupFileSections( + restoreResult.manifest.sections, + 2, + )) { + const file = await exists(section.archiveName, false, backupFolder.id); + if (!file) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + await download( + file, + section.archiveName === ZipBackupName.NOVEL_FILES + ? novelFilesRestorePath + : section.storagePath, + ); + } + if (restoreResult.manifest.sections.downloadedFiles) { + await restoreNovelFiles( + novelFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + } } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); + const missingPluginIds = await finalizeRestoredPlugins(restoreResult); + const completionText = getRestoreCompletionText( + restoreResult, + missingPluginIds, + ); - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); + setMeta(meta => ({ + ...meta, + progress: 3 / 3, + isRunning: false, + progressText: completionText, + completionText, + })); + } finally { + if (restoreResult) { + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); + } + } }; diff --git a/src/services/backup/fileSections.ts b/src/services/backup/fileSections.ts index ece55687ad..71bde584cc 100644 --- a/src/services/backup/fileSections.ts +++ b/src/services/backup/fileSections.ts @@ -3,7 +3,7 @@ import type { BackupOptions } from './options'; import { ZipBackupName } from './types'; import NativeFile from '@modules/native-file'; import type { RestoredNovelMapping } from '@database/types'; - +import { getRestoreChapterMappings } from '@database/queries/NovelRestoreQueries'; export type BackupFileSection = { archiveName: ZipBackupName; storagePath: string; @@ -11,6 +11,7 @@ export type BackupFileSection = { export const getSelectedBackupFileSections = ( options: BackupOptions, + formatVersion: 2 | 3 = 3, ): BackupFileSection[] => { const sections: BackupFileSection[] = []; @@ -20,7 +21,7 @@ export const getSelectedBackupFileSections = ( storagePath: PLUGIN_STORAGE, }); } - if (options.downloadedFiles) { + if (formatVersion === 2 && options.downloadedFiles) { sections.push({ archiveName: ZipBackupName.NOVEL_FILES, storagePath: NOVEL_STORAGE, @@ -51,9 +52,12 @@ const moveDirectoryContents = async (source: string, destination: string) => { } }; +const RESTORE_CHAPTER_LOOKUP_BATCH_SIZE = 100; + export const restoreNovelFiles = async ( stagingPath: string, novelMappings: RestoredNovelMapping[], + restoreRunId: string, ) => { for (const mapping of novelMappings) { const sourceNovelPath = `${stagingPath}/${mapping.pluginId}/${mapping.backupNovelId}`; @@ -63,28 +67,48 @@ export const restoreNovelFiles = async ( const destinationNovelPath = `${NOVEL_STORAGE}/${mapping.pluginId}/${mapping.restoredNovelId}`; await NativeFile.mkdir(destinationNovelPath); - const chapterIds = new Map( - mapping.chapters.map(chapter => [ - String(chapter.backupChapterId), - chapter.restoredChapterId, - ]), - ); - - for (const item of await NativeFile.readDir(sourceNovelPath)) { + const items = await NativeFile.readDir(sourceNovelPath); + for (const item of items) { if (!item.isDirectory) { await NativeFile.moveFile( item.path, `${destinationNovelPath}/${item.name}`, ); - continue; } + } - const restoredChapterId = chapterIds.get(item.name); - if (restoredChapterId !== undefined) { - await moveDirectoryContents( - item.path, - `${destinationNovelPath}/${restoredChapterId}`, - ); + const chapterItems = items.filter(item => item.isDirectory); + for ( + let start = 0; + start < chapterItems.length; + start += RESTORE_CHAPTER_LOOKUP_BATCH_SIZE + ) { + const chapterBatch = chapterItems.slice( + start, + start + RESTORE_CHAPTER_LOOKUP_BATCH_SIZE, + ); + const backupChapterIds = chapterBatch + .map(item => Number(item.name)) + .filter(id => Number.isInteger(id) && id > 0); + const chapterMappings = await getRestoreChapterMappings( + restoreRunId, + mapping.backupNovelId, + backupChapterIds, + ); + const restoredChapterIds = new Map( + chapterMappings.map(chapter => [ + String(chapter.backupChapterId), + chapter.restoredChapterId, + ]), + ); + for (const item of chapterBatch) { + const restoredChapterId = restoredChapterIds.get(item.name); + if (restoredChapterId !== undefined) { + await moveDirectoryContents( + item.path, + `${destinationNovelPath}/${restoredChapterId}`, + ); + } } } } @@ -97,9 +121,10 @@ export const restoreNovelFiles = async ( export const restoreLegacyFiles = async ( stagingPath: string, novelMappings: RestoredNovelMapping[], + restoreRunId: string, ) => { await moveDirectoryContents(`${stagingPath}/Plugins`, PLUGIN_STORAGE); - await restoreNovelFiles(`${stagingPath}/Novels`, novelMappings); + await restoreNovelFiles(`${stagingPath}/Novels`, novelMappings, restoreRunId); if (await NativeFile.exists(stagingPath)) { await NativeFile.unlink(stagingPath); } diff --git a/src/services/backup/local/index.ts b/src/services/backup/local/index.ts index 85e7822c32..51959cc455 100644 --- a/src/services/backup/local/index.ts +++ b/src/services/backup/local/index.ts @@ -1,5 +1,6 @@ import { CACHE_DIR_PATH, + clearRestoreChapterMappingsSafely, clearBackupCache, prepareBackupData, restoreData, @@ -7,14 +8,16 @@ import { import { finalizeRestoredPlugins, getRestoreCompletionText, + type RestoreResult, } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import NativeZipArchive from '@modules/native-zip-archive'; -import { ZipBackupName } from '../types'; +import { BackupEntryName, ZipBackupName } from '../types'; import NativeFile from '@modules/native-file'; import { getString } from '@i18n/translations'; import type { TaskProgressUpdater } from '@services/backgroundTasks/contracts'; import { sleep } from '@utils/sleep'; +import { NOVEL_STORAGE } from '@utils/Storages'; import { getLegacyFilesRestorePath, getNovelFilesRestorePath, @@ -24,6 +27,19 @@ import { } from '../fileSections'; import { resolveBackupOptions, type BackupOptions } from '../options'; +const logRestoreBenchmark = (message: string) => { + if (!__DEV__) { + return; + } + // Benchmark output is consumed from the Metro client log. + // eslint-disable-next-line no-console + console.log( + `[restore-benchmark] ${new Date().toISOString()} ${performance + .now() + .toFixed(3)} ${message}`, + ); +}; + export const createBackup = async ( { destinationUri, @@ -40,7 +56,7 @@ export const createBackup = async ( progressText: getString('backupScreen.preparingData'), })); - const backupResult = await prepareBackupData(CACHE_DIR_PATH, options); + const backupResult = await prepareBackupData(CACHE_DIR_PATH, options, 3); setMeta?.(meta => ({ ...meta, @@ -65,7 +81,15 @@ export const createBackup = async ( await sleep(200); - await NativeZipArchive.zip(CACHE_DIR_PATH, CACHE_DIR_PATH + '.zip'); + await NativeZipArchive.zipDirectories( + [ + { path: CACHE_DIR_PATH, prefix: '' }, + ...(options.downloadedFiles + ? [{ path: NOVEL_STORAGE, prefix: BackupEntryName.NOVEL_FILES }] + : []), + ], + CACHE_DIR_PATH + '.zip', + ); setMeta?.(meta => ({ ...meta, @@ -96,6 +120,8 @@ export const restoreBackup = async ( { sourceUri }: { sourceUri: string }, setMeta?: TaskProgressUpdater, ) => { + logRestoreBenchmark('local:start'); + let restoreResult: RestoreResult | undefined; try { setMeta?.(meta => ({ ...meta, @@ -107,6 +133,7 @@ export const restoreBackup = async ( await clearBackupCache(); const localPath = CACHE_DIR_PATH + '-source.zip'; await NativeFile.copyFile(sourceUri, localPath); + logRestoreBenchmark('local:copy:done'); setMeta?.(meta => ({ ...meta, @@ -117,6 +144,7 @@ export const restoreBackup = async ( await sleep(200); await NativeZipArchive.unzip(localPath, CACHE_DIR_PATH); + logRestoreBenchmark('local:outer-unzip:done'); setMeta?.(meta => ({ ...meta, @@ -126,16 +154,12 @@ export const restoreBackup = async ( await sleep(200); - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta?.(meta => ({ - ...meta, - progress: 3 / 4, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - + restoreResult = await restoreData( + CACHE_DIR_PATH, + setMeta, + logRestoreBenchmark, + ); + logRestoreBenchmark('local:restore-data:done'); if (restoreResult.manifest.formatVersion === 1) { const legacyArchive = CACHE_DIR_PATH + '/' + ZipBackupName.DOWNLOAD; if (!(await NativeFile.exists(legacyArchive))) { @@ -146,12 +170,25 @@ export const restoreBackup = async ( await restoreLegacyFiles( legacyFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); + logRestoreBenchmark('local:downloaded-files:done'); } else { const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( + const sections = getSelectedBackupFileSections( restoreResult.manifest.sections, - )) { + restoreResult.manifest.formatVersion, + ); + if ( + restoreResult.manifest.formatVersion === 3 && + restoreResult.manifest.sections.downloadedFiles && + !(await NativeFile.exists( + `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}`, + )) + ) { + throw new Error(getString('backupScreen.invalidBackupFolder')); + } + for (const section of sections) { const archivePath = `${CACHE_DIR_PATH}/${section.archiveName}`; if (!(await NativeFile.exists(archivePath))) { throw new Error(getString('backupScreen.invalidBackupFolder')); @@ -163,18 +200,25 @@ export const restoreBackup = async ( : section.storagePath, ); } + logRestoreBenchmark('local:selected-archives:done'); if (restoreResult.manifest.sections.downloadedFiles) { await restoreNovelFiles( - novelFilesRestorePath, + restoreResult.manifest.formatVersion === 3 + ? `${CACHE_DIR_PATH}/${BackupEntryName.NOVEL_FILES}` + : novelFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); } } + logRestoreBenchmark('local:downloaded-files:done'); + logRestoreBenchmark('local:selected-files:done'); const missingPluginIds = await finalizeRestoredPlugins(restoreResult); const completionText = getRestoreCompletionText( restoreResult, missingPluginIds, ); + logRestoreBenchmark('local:finalize:done'); setMeta?.(meta => ({ ...meta, @@ -189,5 +233,9 @@ export const restoreBackup = async ( isRunning: false, })); throw error; + } finally { + if (restoreResult) { + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); + } } }; diff --git a/src/services/backup/novelPayload.ts b/src/services/backup/novelPayload.ts new file mode 100644 index 0000000000..6791fde47b --- /dev/null +++ b/src/services/backup/novelPayload.ts @@ -0,0 +1,437 @@ +import type { BackupNovel, ChapterInfo } from '@database/types'; + +type NullableString = string | null; +type NullableNumber = number | null; +type NullableBoolean = boolean | null; + +type CompactChapter = [ + number, + string, + string, + NullableString, + NullableBoolean, + NullableBoolean, + NullableString, + NullableBoolean, + NullableString, + NullableNumber, + NullableString, + NullableNumber, + NullableNumber, + NullableString, + NullableNumber, +]; + +type CompactNovel = { + c: CompactChapter[]; + id: number; + p: string; + pi: string; + n: string; + co: NullableString; + s: NullableString; + a: NullableString; + ar: NullableString; + st: NullableString; + g: NullableString; + l: NullableBoolean; + lo: NullableBoolean; + t: NullableNumber; + d: NullableNumber; + u: NullableNumber; + tc: NullableNumber; + lr: NullableString; + lu: NullableString; +}; + +type BackupNovelWithAggregates = BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; +}; + +const NOVEL_KEYS = [ + 'c', + 'id', + 'p', + 'pi', + 'n', + 'co', + 's', + 'a', + 'ar', + 'st', + 'g', + 'l', + 'lo', + 't', + 'd', + 'u', + 'tc', + 'lr', + 'lu', +] as const; + +const isNullable = ( + value: unknown, + predicate: (value: unknown) => value is T, +): value is T | null => value === null || predicate(value); + +const isString = (value: unknown): value is string => typeof value === 'string'; +const isNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); +const isId = (value: unknown): value is number => + isNumber(value) && Number.isInteger(value) && value > 0; +const isBoolean = (value: unknown): value is boolean => + typeof value === 'boolean'; + +const nullableString = (value: unknown): value is NullableString => + isNullable(value, isString); +const nullableNumber = (value: unknown): value is NullableNumber => + isNullable(value, isNumber); +const nullableBoolean = (value: unknown): value is NullableBoolean => + isNullable(value, isBoolean); +const nonEmptyString = (value: unknown): value is string => + isString(value) && value.trim().length > 0; + +const nullableLegacyString = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isString(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyNumber = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isNumber(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const nullableLegacyBoolean = (value: Record, key: string) => { + const field = value[key]; + if (field === undefined || field === null) { + return null; + } + if (!isBoolean(field)) { + throw new Error(`Invalid legacy ${key}`); + } + return field; +}; + +const positiveId = (value: unknown): value is number => + isId(value) && value > 0; + +const normalizeLegacyChapter = ( + value: unknown, + containingNovelId: number, +): ChapterInfo => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy chapter'); + } + const chapter = value as Record; + if ( + !positiveId(chapter.id) || + !nonEmptyString(chapter.path) || + !isString(chapter.name) || + (chapter.novelId !== undefined && + chapter.novelId !== null && + !positiveId(chapter.novelId)) + ) { + throw new Error('Invalid legacy chapter fields'); + } + return { + id: chapter.id, + novelId: containingNovelId, + path: chapter.path, + name: chapter.name, + releaseTime: nullableLegacyString(chapter, 'releaseTime'), + readTime: nullableLegacyString(chapter, 'readTime'), + bookmark: nullableLegacyBoolean(chapter, 'bookmark'), + unread: nullableLegacyBoolean(chapter, 'unread'), + isDownloaded: nullableLegacyBoolean(chapter, 'isDownloaded'), + updatedTime: nullableLegacyString(chapter, 'updatedTime'), + chapterNumber: nullableLegacyNumber(chapter, 'chapterNumber'), + page: nullableLegacyString(chapter, 'page'), + progress: nullableLegacyNumber(chapter, 'progress'), + position: nullableLegacyNumber(chapter, 'position'), + scanlator: nullableLegacyString(chapter, 'scanlator'), + timeSpent: nullableLegacyNumber(chapter, 'timeSpent'), + }; +}; + +export const validateBackupNovel = (novel: BackupNovel): BackupNovel => { + if ( + !positiveId(novel.id) || + !nonEmptyString(novel.path) || + !nonEmptyString(novel.pluginId) || + !isString(novel.name) || + !Array.isArray(novel.chapters) + ) { + throw new Error('Invalid novel fields'); + } + const stringFields = [ + novel.cover, + novel.summary, + novel.author, + novel.artist, + novel.status, + novel.genres, + ]; + if (stringFields.some(value => value !== null && !isString(value))) { + throw new Error('Invalid novel string field'); + } + const booleanFields = [novel.inLibrary, novel.isLocal]; + if (booleanFields.some(value => value !== null && !isBoolean(value))) { + throw new Error('Invalid novel boolean field'); + } + if (novel.totalPages !== null && !isNumber(novel.totalPages)) { + throw new Error('Invalid novel number field'); + } + for (const chapter of novel.chapters) { + if ( + !positiveId(chapter.id) || + !positiveId(chapter.novelId) || + !nonEmptyString(chapter.path) || + !isStringOrNull(chapter.releaseTime) || + !isStringOrNull(chapter.readTime) || + !isBooleanOrNull(chapter.bookmark) || + !isBooleanOrNull(chapter.unread) || + !isBooleanOrNull(chapter.isDownloaded) || + !isStringOrNull(chapter.updatedTime) || + !isNumberOrNull(chapter.chapterNumber) || + !isStringOrNull(chapter.page) || + !isNumberOrNull(chapter.progress) || + !isNumberOrNull(chapter.position) || + !isStringOrNull(chapter.scanlator) || + !isNumberOrNull(chapter.timeSpent) + ) { + throw new Error('Invalid chapter fields'); + } + } + return novel; +}; + +const isStringOrNull = (value: unknown): value is string | null => + value === null || isString(value); +const isNumberOrNull = (value: unknown): value is number | null => + value === null || isNumber(value); +const isBooleanOrNull = (value: unknown): value is boolean | null => + value === null || isBoolean(value); + +export const normalizeLegacyNovel = (value: unknown): BackupNovel => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid legacy novel'); + } + const source = value as Record; + if ( + !positiveId(source.id) || + !nonEmptyString(source.path) || + !nonEmptyString(source.pluginId) || + !isString(source.name) || + !Array.isArray(source.chapters) + ) { + throw new Error('Invalid legacy novel fields'); + } + const novelId = source.id; + const novel: BackupNovel = { + id: novelId, + path: source.path, + pluginId: source.pluginId, + name: source.name, + cover: nullableLegacyString(source, 'cover'), + summary: nullableLegacyString(source, 'summary'), + author: nullableLegacyString(source, 'author'), + artist: nullableLegacyString(source, 'artist'), + status: nullableLegacyString(source, 'status'), + genres: nullableLegacyString(source, 'genres'), + inLibrary: nullableLegacyBoolean(source, 'inLibrary'), + isLocal: nullableLegacyBoolean(source, 'isLocal'), + totalPages: nullableLegacyNumber(source, 'totalPages'), + chapters: source.chapters.map(chapter => + normalizeLegacyChapter(chapter, novelId), + ), + }; + return validateBackupNovel(novel); +}; + +const assertNovelKeys = (novel: Record) => { + const keys = Object.keys(novel).sort(); + const expected = [...NOVEL_KEYS].sort(); + if ( + keys.length !== expected.length || + keys.some((key, index) => key !== expected[index]) + ) { + throw new Error('Invalid compact novel keys'); + } +}; + +const isCompactChapter = (value: unknown): value is CompactChapter => { + if (!Array.isArray(value) || value.length !== 15) { + return false; + } + + return ( + isId(value[0]) && + nonEmptyString(value[1]) && + isString(value[2]) && + nullableString(value[3]) && + nullableBoolean(value[4]) && + nullableBoolean(value[5]) && + nullableString(value[6]) && + nullableBoolean(value[7]) && + nullableString(value[8]) && + nullableNumber(value[9]) && + nullableString(value[10]) && + nullableNumber(value[11]) && + nullableNumber(value[12]) && + nullableString(value[13]) && + nullableNumber(value[14]) + ); +}; + +const isCompactNovel = (value: unknown): value is CompactNovel => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const novel = value as Record; + assertNovelKeys(novel); + return ( + Array.isArray(novel.c) && + novel.c.every(isCompactChapter) && + isId(novel.id) && + nonEmptyString(novel.p) && + nonEmptyString(novel.pi) && + isString(novel.n) && + nullableString(novel.co) && + nullableString(novel.s) && + nullableString(novel.a) && + nullableString(novel.ar) && + nullableString(novel.st) && + nullableString(novel.g) && + nullableBoolean(novel.l) && + nullableBoolean(novel.lo) && + nullableNumber(novel.t) && + nullableNumber(novel.d) && + nullableNumber(novel.u) && + nullableNumber(novel.tc) && + nullableString(novel.lr) && + nullableString(novel.lu) + ); +}; + +const nullable = (value: T | undefined | null): T | null => value ?? null; + +const encodeChapter = (chapter: ChapterInfo): CompactChapter => [ + chapter.id, + chapter.path, + chapter.name, + nullable(chapter.releaseTime), + nullable(chapter.bookmark), + nullable(chapter.unread), + nullable(chapter.readTime), + nullable(chapter.isDownloaded), + nullable(chapter.updatedTime), + nullable(chapter.chapterNumber), + nullable(chapter.page), + nullable(chapter.position), + nullable(chapter.progress), + nullable(chapter.scanlator), + nullable(chapter.timeSpent), +]; + +const encodeNovel = (novel: BackupNovel): CompactNovel => { + const novelWithAggregates = novel as BackupNovelWithAggregates; + return { + c: novel.chapters.map(encodeChapter), + id: novel.id, + p: novel.path, + pi: novel.pluginId, + n: novel.name, + co: nullable(novel.cover), + s: nullable(novel.summary), + a: nullable(novel.author), + ar: nullable(novel.artist), + st: nullable(novel.status), + g: nullable(novel.genres), + l: nullable(novel.inLibrary), + lo: nullable(novel.isLocal), + t: nullable(novel.totalPages), + d: nullable(novelWithAggregates.chaptersDownloaded), + u: nullable(novelWithAggregates.chaptersUnread), + tc: nullable(novelWithAggregates.totalChapters), + lr: nullable(novelWithAggregates.lastReadAt), + lu: nullable(novelWithAggregates.lastUpdatedAt), + }; +}; + +const decodeNovel = (novel: CompactNovel): BackupNovel => { + const decoded: BackupNovelWithAggregates = { + id: novel.id, + path: novel.p, + pluginId: novel.pi, + name: novel.n, + cover: novel.co, + summary: novel.s, + author: novel.a, + artist: novel.ar, + status: novel.st, + genres: novel.g, + inLibrary: novel.l, + isLocal: novel.lo, + totalPages: novel.t, + chaptersDownloaded: novel.d, + chaptersUnread: novel.u, + totalChapters: novel.tc, + lastReadAt: novel.lr, + lastUpdatedAt: novel.lu, + chapters: novel.c.map(chapter => ({ + id: chapter[0], + novelId: novel.id, + path: chapter[1], + name: chapter[2], + releaseTime: chapter[3], + bookmark: chapter[4], + unread: chapter[5], + readTime: chapter[6], + isDownloaded: chapter[7], + updatedTime: chapter[8], + chapterNumber: chapter[9], + page: chapter[10], + position: chapter[11], + progress: chapter[12], + scanlator: chapter[13], + timeSpent: chapter[14], + })), + }; + return decoded; +}; + +export const encodeNovelBatch = (novels: BackupNovel[]): CompactNovel[] => { + if (novels.length > 100) { + throw new Error('Compact novel batch exceeds 100 novels'); + } + return novels.map(encodeNovel); +}; + +export const decodeNovelBatch = (payload: unknown): BackupNovel[] => { + if (!Array.isArray(payload) || payload.length > 100) { + throw new Error('Invalid compact novel batch'); + } + return payload.map(value => { + if (!isCompactNovel(value)) { + throw new Error('Invalid compact novel record'); + } + return decodeNovel(value); + }); +}; diff --git a/src/services/backup/restoreResult.ts b/src/services/backup/restoreResult.ts index dc11bd9880..694e65c757 100644 --- a/src/services/backup/restoreResult.ts +++ b/src/services/backup/restoreResult.ts @@ -17,6 +17,7 @@ export type RestoreResult = { failedSectionCount: number; pluginIds: string[]; novelMappings: RestoredNovelMapping[]; + restoreRunId: string; manifest: ResolvedBackupManifest; }; diff --git a/src/services/backup/selfhost/index.ts b/src/services/backup/selfhost/index.ts index b25a8194ec..07919eacd2 100644 --- a/src/services/backup/selfhost/index.ts +++ b/src/services/backup/selfhost/index.ts @@ -6,10 +6,12 @@ import { clearBackupCache, prepareBackupData, restoreData, + clearRestoreChapterMappingsSafely, } from '../utils'; import { finalizeRestoredPlugins, getRestoreCompletionText, + type RestoreResult, } from '../restoreResult'; import { getBackupCompletionText } from '../backupResult'; import { ZipBackupName } from '../types'; @@ -58,7 +60,7 @@ export const createSelfHostBackup = async ( await sleep(200); - for (const section of getSelectedBackupFileSections(options)) { + for (const section of getSelectedBackupFileSections(options, 2)) { await upload(host, backupFolder, section.archiveName, section.storagePath); } @@ -76,78 +78,88 @@ export const selfHostRestore = async ( { host, backupFolder }: SelfHostData, setMeta: TaskProgressUpdater, ) => { - setMeta(meta => ({ - ...meta, - isRunning: true, - progress: 0 / 3, - progressText: getString('backupScreen.downloadingData'), - })); - - await clearBackupCache(); - await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); - - setMeta(meta => ({ - ...meta, - progress: 1 / 3, - progressText: getString('backupScreen.restoringData'), - })); - - await sleep(200); - - const restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); - - setMeta(meta => ({ - ...meta, - progress: 2 / 3, - progressText: getString('backupScreen.restoringSelectedFiles'), - })); - - await sleep(200); - - if (restoreResult.manifest.formatVersion === 1) { - const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); - await download( - host, - backupFolder, - ZipBackupName.DOWNLOAD, - legacyFilesRestorePath, - ); - await restoreLegacyFiles( - legacyFilesRestorePath, - restoreResult.novelMappings, - ); - } else { - const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); - for (const section of getSelectedBackupFileSections( - restoreResult.manifest.sections, - )) { + let restoreResult: RestoreResult | undefined; + try { + setMeta(meta => ({ + ...meta, + isRunning: true, + progress: 0 / 3, + progressText: getString('backupScreen.downloadingData'), + })); + + await clearBackupCache(); + await download(host, backupFolder, ZipBackupName.DATA, CACHE_DIR_PATH); + + setMeta(meta => ({ + ...meta, + progress: 1 / 3, + progressText: getString('backupScreen.restoringData'), + })); + + await sleep(200); + + restoreResult = await restoreData(CACHE_DIR_PATH, setMeta); + + setMeta(meta => ({ + ...meta, + progress: 2 / 3, + progressText: getString('backupScreen.restoringSelectedFiles'), + })); + + await sleep(200); + + if (restoreResult.manifest.formatVersion === 1) { + const legacyFilesRestorePath = getLegacyFilesRestorePath(CACHE_DIR_PATH); await download( host, backupFolder, - section.archiveName, - section.archiveName === ZipBackupName.NOVEL_FILES - ? novelFilesRestorePath - : section.storagePath, + ZipBackupName.DOWNLOAD, + legacyFilesRestorePath, ); - } - if (restoreResult.manifest.sections.downloadedFiles) { - await restoreNovelFiles( - novelFilesRestorePath, + await restoreLegacyFiles( + legacyFilesRestorePath, restoreResult.novelMappings, + restoreResult.restoreRunId, ); + } else { + const novelFilesRestorePath = getNovelFilesRestorePath(CACHE_DIR_PATH); + for (const section of getSelectedBackupFileSections( + restoreResult.manifest.sections, + 2, + )) { + await download( + host, + backupFolder, + section.archiveName, + section.archiveName === ZipBackupName.NOVEL_FILES + ? novelFilesRestorePath + : section.storagePath, + ); + } + if (restoreResult.manifest.sections.downloadedFiles) { + await restoreNovelFiles( + novelFilesRestorePath, + restoreResult.novelMappings, + restoreResult.restoreRunId, + ); + } } - } - const missingPluginIds = await finalizeRestoredPlugins(restoreResult); - const completionText = getRestoreCompletionText( - restoreResult, - missingPluginIds, - ); + const missingPluginIds = await finalizeRestoredPlugins(restoreResult); + const completionText = getRestoreCompletionText( + restoreResult, + missingPluginIds, + ); - setMeta(meta => ({ - ...meta, - progress: 3 / 3, - isRunning: false, - progressText: completionText, - completionText, - })); + setMeta(meta => ({ + ...meta, + progress: 3 / 3, + isRunning: false, + progressText: completionText, + completionText, + })); + } finally { + if (restoreResult) { + await clearRestoreChapterMappingsSafely(restoreResult.restoreRunId); + } + } }; diff --git a/src/services/backup/types.ts b/src/services/backup/types.ts index aefd5cf65f..9cc5ac5670 100644 --- a/src/services/backup/types.ts +++ b/src/services/backup/types.ts @@ -14,11 +14,13 @@ export enum BackupEntryName { PLUGIN_METADATA = 'Plugins.json', COVERS = 'Covers', NOVEL_AND_CHAPTERS = 'NovelAndChapters', + NOVEL_FILES = 'NovelFiles', } export type BackupManifest = { appVersion: string; - formatVersion: 2; + formatVersion: 2 | 3; + novelDataFormat?: 1 | 2; sections: BackupOptions; }; @@ -27,5 +29,6 @@ export type ResolvedBackupManifest = | { appVersion?: string; formatVersion: 1; + novelDataFormat?: 1 | 2; sections: BackupOptions; }; diff --git a/src/services/backup/utils.ts b/src/services/backup/utils.ts index d7b186ef80..e54cecc629 100644 --- a/src/services/backup/utils.ts +++ b/src/services/backup/utils.ts @@ -3,10 +3,12 @@ import { OLD_TRACKED_NOVEL_PREFIX } from '@hooks/persisted/migrations/trackerMig import { LAST_UPDATE_TIME } from '@hooks/persisted/useUpdates'; import { MMKVStorage } from '@utils/mmkv/mmkv'; import { version } from '../../../package.json'; +import { getAllNovels } from '@database/queries/NovelQueries'; import { + clearRestoreChapterMappings, _restoreNovelAndChapters, - getAllNovels, -} from '@database/queries/NovelQueries'; + _restoreNovelsAndChapters, +} from '@database/queries/NovelRestoreQueries'; import { getAllNovelChaptersForBackup } from '@database/queries/ChapterQueries'; import { _restoreCategory, @@ -18,6 +20,12 @@ import { BackupNovel, type RestoredNovelMapping, } from '@database/types'; +import { + decodeNovelBatch, + encodeNovelBatch, + normalizeLegacyNovel, + validateBackupNovel, +} from './novelPayload'; import { BackupEntryName, type BackupManifest, @@ -40,6 +48,12 @@ import type { PluginItem } from '@plugins/types'; const APP_STORAGE_URI = 'file://' + ROOT_STORAGE; +const BACKUP_NOVEL_BATCH_SIZE = 100; + +const RESTORE_NOVEL_BATCH_SIZE = 100; + +const BACKUP_FILE_CONCURRENCY = 8; + const stripUriSuffix = (uri: string) => uri.split(/[?#]/, 1)[0]; const parentDirectory = (path: string) => @@ -96,6 +110,7 @@ const restoreMMKVData = (data: any) => { export const prepareBackupData = async ( cacheDirPath: string, requestedOptions?: BackupOptions, + formatVersion: BackupManifest['formatVersion'] = 2, ): Promise => { const options = resolveBackupOptions(requestedOptions); const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; @@ -106,10 +121,10 @@ export const prepareBackupData = async ( await clearBackupCache(cacheDirPath); await NativeFile.mkdir(cacheDirPath); - // version const manifest: BackupManifest = { appVersion: version, - formatVersion: 2, + formatVersion, + novelDataFormat: 2, sections: options, }; await NativeFile.writeFile( @@ -120,60 +135,132 @@ export const prepareBackupData = async ( // novels if (options.library) { await NativeFile.mkdir(novelDirPath); - await NativeFile.mkdir(coversDirPath); - await getAllNovels().then(async novels => { - for (const novel of novels) { - try { - const chapters = await getAllNovelChaptersForBackup(novel.id); - const backedUpChapters = options.downloadedFiles - ? chapters - : chapters.map(chapter => ({ - ...chapter, - isDownloaded: false, - })); - let cover = novel.cover; - if (cover?.startsWith(APP_STORAGE_URI)) { + if (!options.downloadedFiles) { + await NativeFile.mkdir(coversDirPath); + } + const novels = await getAllNovels(); + for ( + let start = 0; + start < novels.length; + start += BACKUP_NOVEL_BATCH_SIZE + ) { + const novelBatch = novels.slice(start, start + BACKUP_NOVEL_BATCH_SIZE); + let chapters; + try { + chapters = await getAllNovelChaptersForBackup( + novelBatch.map(novel => novel.id), + ); + } catch { + failedNovelCount += novelBatch.length; + continue; + } + + const chaptersByNovel = new Map(); + for (const chapter of chapters) { + const novelChapters = chaptersByNovel.get(chapter.novelId); + if (novelChapters) { + novelChapters.push(chapter); + } else { + chaptersByNovel.set(chapter.novelId, [chapter]); + } + } + + const preparedNovels: BackupNovel[] = []; + for ( + let fileStart = 0; + fileStart < novelBatch.length; + fileStart += BACKUP_FILE_CONCURRENCY + ) { + const fileBatch = novelBatch.slice( + fileStart, + fileStart + BACKUP_FILE_CONCURRENCY, + ); + const prepared = await Promise.all( + fileBatch.map(async (novel): Promise => { try { - await NativeFile.copyFile( - stripUriSuffix(cover), - coversDirPath + '/' + novel.id, - ); - cover = cover.replace(APP_STORAGE_URI, ''); + const novelChapters = chaptersByNovel.get(novel.id) ?? []; + const backedUpChapters = options.downloadedFiles + ? novelChapters + : novelChapters.map(chapter => ({ + ...chapter, + isDownloaded: false, + })); + let cover = novel.cover; + if (cover?.startsWith(APP_STORAGE_URI)) { + if (options.downloadedFiles) { + cover = cover.replace(APP_STORAGE_URI, ''); + } else { + try { + await NativeFile.copyFile( + stripUriSuffix(cover), + coversDirPath + '/' + novel.id, + ); + cover = cover.replace(APP_STORAGE_URI, ''); + } catch { + cover = null; + } + } + } + const preparedNovel = { + ...novel, + chapters: backedUpChapters, + cover, + }; + return validateBackupNovel(preparedNovel); } catch { - cover = options.downloadedFiles - ? cover.replace(APP_STORAGE_URI, '') - : null; + failedNovelCount++; + return null; } - } + }), + ); + preparedNovels.push( + ...prepared.filter((novel): novel is BackupNovel => novel !== null), + ); + } + + if (preparedNovels.length > 0) { + const batchName = `batch-${String( + Math.floor(start / BACKUP_NOVEL_BATCH_SIZE) + 1, + ).padStart(6, '0')}.json`; + const batchPath = novelDirPath + '/' + batchName; + try { await NativeFile.writeFile( - novelDirPath + '/' + novel.id + '.json', - JSON.stringify({ - chapters: backedUpChapters, - ...novel, - cover, - }), + batchPath, + JSON.stringify(encodeNovelBatch(preparedNovels)), ); } catch { - failedNovelCount++; + failedNovelCount += preparedNovels.length; + try { + await NativeFile.unlink(batchPath); + } catch { + // Best effort cleanup prevents a failed write from restoring partial data. + } } } - }); + } // categories try { const categories = await getCategoriesFromDb(); const novelCategories = await getAllNovelCategories(); + const novelIdsByCategory = new Map(); + for (const novelCategory of novelCategories) { + const novelIds = novelIdsByCategory.get(novelCategory.categoryId); + if (novelIds) { + novelIds.push(novelCategory.novelId); + } else { + novelIdsByCategory.set(novelCategory.categoryId, [ + novelCategory.novelId, + ]); + } + } await NativeFile.writeFile( cacheDirPath + '/' + BackupEntryName.CATEGORY, JSON.stringify( - categories.map(category => { - return { - ...category, - novelIds: novelCategories - .filter(nc => nc.categoryId === category.id) - .map(nc => nc.novelId), - }; - }), + categories.map(category => ({ + ...category, + novelIds: novelIdsByCategory.get(category.id) ?? [], + })), ), ); } catch { @@ -221,10 +308,18 @@ const getBackupManifest = async ( const data = JSON.parse(fileContent) as Partial & { version?: string; }; - if (data.formatVersion === 2 && data.sections) { + if ( + (data.formatVersion === 2 || data.formatVersion === 3) && + data.sections + ) { + const novelDataFormat = + data.novelDataFormat === 1 || data.novelDataFormat === 2 + ? data.novelDataFormat + : undefined; return { appVersion: data.appVersion ?? data.version ?? '', - formatVersion: 2, + formatVersion: data.formatVersion, + ...(novelDataFormat === undefined ? {} : { novelDataFormat }), sections: resolveBackupOptions(data.sections), }; } @@ -252,11 +347,129 @@ const updateRestoreProgress = ( })); }; -export const restoreData = async ( +type BackupNovelFileDescriptor = { + name: string; + path: string; +}; + +const createRestoreRunId = () => + `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + +const decodeRestoreNovelFile = ( + fileContent: string, + manifest: ResolvedBackupManifest, +) => { + const payload: unknown = JSON.parse(fileContent); + const novels = + manifest.novelDataFormat === 2 + ? decodeNovelBatch(payload) + : [normalizeLegacyNovel(payload)]; + return novels.map(novel => { + const { + chaptersDownloaded: _chaptersDownloaded, + chaptersUnread: _chaptersUnread, + totalChapters: _totalChapters, + lastReadAt: _lastReadAt, + lastUpdatedAt: _lastUpdatedAt, + ...withoutAggregates + } = novel as BackupNovel & { + chaptersDownloaded?: number | null; + chaptersUnread?: number | null; + totalChapters?: number | null; + lastReadAt?: string | null; + lastUpdatedAt?: string | null; + }; + const normalized = validateBackupNovel(withoutAggregates); + return normalized.cover && !normalized.cover.startsWith('http') + ? { ...normalized, cover: APP_STORAGE_URI + normalized.cover } + : normalized; + }); +}; +const getNovelFileRecordCount = ( + fileContent: string, + manifest: ResolvedBackupManifest, +) => { + if (manifest.novelDataFormat !== 2) { + return 1; + } + try { + const payload: unknown = JSON.parse(fileContent); + return Array.isArray(payload) ? Math.max(1, payload.length) : 1; + } catch { + return 1; + } +}; + +const validateNovelFileRecords = ( + novels: BackupNovel[], + seenNovelIds: Set, + seenNovelIdentities: Set, + seenChapterIdentities: Set, +) => { + const fileNovelIds = new Set(); + const fileNovelIdentities = new Set(); + const fileChapterIdentities = new Set(); + for (const novel of novels) { + const novelIdentity = `${novel.pluginId}\u0000${novel.path}`; + if ( + fileNovelIds.has(novel.id) || + seenNovelIds.has(novel.id) || + fileNovelIdentities.has(novelIdentity) || + seenNovelIdentities.has(novelIdentity) + ) { + throw new Error('Duplicate backup novel identity'); + } + fileNovelIds.add(novel.id); + fileNovelIdentities.add(novelIdentity); + for (const chapter of novel.chapters) { + const chapterIdentity = `${novel.id}\u0000${chapter.path}`; + if ( + fileChapterIdentities.has(chapterIdentity) || + seenChapterIdentities.has(chapterIdentity) + ) { + throw new Error('Duplicate backup chapter identity'); + } + fileChapterIdentities.add(chapterIdentity); + } + } + for (const novelId of fileNovelIds) { + seenNovelIds.add(novelId); + } + for (const novelIdentity of fileNovelIdentities) { + seenNovelIdentities.add(novelIdentity); + } + for (const chapterIdentity of fileChapterIdentities) { + seenChapterIdentities.add(chapterIdentity); + } +}; + +const decodeAndValidateNovelFile = ( + fileContent: string, + manifest: ResolvedBackupManifest, + seenNovelIds?: Set, + seenNovelIdentities?: Set, + seenChapterIdentities?: Set, +) => { + const novels = decodeRestoreNovelFile(fileContent, manifest); + validateNovelFileRecords( + novels, + seenNovelIds ?? new Set(), + seenNovelIdentities ?? new Set(), + seenChapterIdentities ?? new Set(), + ); + return novels; +}; + +type RestoreBenchmarkLogger = (message: string) => void; + +const restoreDataInternal = async ( cacheDirPath: string, - setMeta?: TaskProgressUpdater, + setMeta: TaskProgressUpdater | undefined, + benchmarkLog: RestoreBenchmarkLogger | undefined, + restoreRunId: string, ): Promise => { const manifest = await getBackupManifest(cacheDirPath); + benchmarkLog?.('restoreData:manifest:loaded'); const novelDirPath = cacheDirPath + '/' + BackupEntryName.NOVEL_AND_CHAPTERS; const coversDirPath = cacheDirPath + '/' + BackupEntryName.COVERS; const pluginIds = new Set(); @@ -273,16 +486,18 @@ export const restoreData = async ( })(); let pluginsFromSettings: PluginItem[] = []; - // version - // nothing to do - - // novels if (manifest.sections.library) { - updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); + benchmarkLog?.('restoreData:novels:validation:start'); + updateRestoreProgress(setMeta, getString('backupScreen.validatingNovels')); } let novelCount = 0; + let totalNovelCount = 0; let failedCount = 0; let failedSectionCount = 0; + let readMs = 0; + let parseMs = 0; + let databaseMs = 0; + let coverMs = 0; if (!manifest.sections.library) { // Intentionally omitted from this backup. @@ -290,51 +505,212 @@ export const restoreData = async ( failedSectionCount++; } else { try { - const items = (await NativeFile.readDir(novelDirPath)).filter( - item => !item.isDirectory, - ); - for (const [index, item] of items.entries()) { - updateRestoreProgress( - setMeta, - getString('backupScreen.restoringNovelsProgress', { - current: index + 1, - total: items.length, - }), + const items = (await NativeFile.readDir(novelDirPath)) + .filter(item => !item.isDirectory) + .sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, ); + const validItems: BackupNovelFileDescriptor[] = []; + const seenNovelIds = new Set(); + const seenNovelIdentities = new Set(); + const seenChapterIdentities = new Set(); + + for (const [index, item] of items.entries()) { + if (index % 100 === 0) { + updateRestoreProgress( + setMeta, + getString('backupScreen.validatingNovelsProgress', { + current: index + 1, + total: items.length, + }), + ); + } + const readStartedAt = performance.now(); + let fileContent: string; try { - const fileContent = await NativeFile.readFile(item.path); - const backupNovel = JSON.parse(fileContent) as BackupNovel; - pluginIds.add(backupNovel.pluginId); - - const hasStoredCover = - backupNovel.cover && !backupNovel.cover.startsWith('http'); - if (hasStoredCover) { - backupNovel.cover = APP_STORAGE_URI + backupNovel.cover; - } + fileContent = await NativeFile.readFile(item.path); + } catch { + failedCount++; + continue; + } finally { + readMs += performance.now() - readStartedAt; + } - const novelMapping = await _restoreNovelAndChapters(backupNovel); - novelMappings.push(novelMapping); - novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); + const parseStartedAt = performance.now(); + try { + const validatedNovels = decodeAndValidateNovelFile( + fileContent, + manifest, + seenNovelIds, + seenNovelIdentities, + seenChapterIdentities, + ); + totalNovelCount += validatedNovels.length; + validItems.push({ name: item.name, path: item.path }); + } catch { + failedCount += getNovelFileRecordCount(fileContent, manifest); + } finally { + parseMs += performance.now() - parseStartedAt; + } + } + benchmarkLog?.( + `restoreData:novels:validation:done total=${totalNovelCount}`, + ); - if (hasStoredCover) { - const coverBackupPath = coversDirPath + '/' + backupNovel.id; - if (await NativeFile.exists(coverBackupPath)) { - const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; - await NativeFile.mkdir(parentDirectory(coverPath)); - await NativeFile.copyFile(coverBackupPath, coverPath); + const pendingNovels: BackupNovel[] = []; + const restoreNovelBatch = async () => { + if (pendingNovels.length === 0) { + return; + } + const batch = pendingNovels.splice(0, pendingNovels.length); + const restoreOptions = { + includeChapterMappings: manifest.sections.downloadedFiles, + ...(manifest.sections.downloadedFiles ? { restoreRunId } : {}), + }; + let restoredNovels: { + backupNovel: BackupNovel; + mapping: RestoredNovelMapping; + }[] = []; + const databaseStartedAt = performance.now(); + try { + const mappings = await _restoreNovelsAndChapters( + batch, + restoreOptions, + ); + if (mappings.length !== batch.length) { + throw new Error('Restore returned incomplete novel mappings'); + } + restoredNovels = batch.map((backupNovel, index) => ({ + backupNovel, + mapping: mappings[index], + })); + } catch { + for (const backupNovel of batch) { + try { + restoredNovels.push({ + backupNovel, + mapping: await _restoreNovelAndChapters( + backupNovel, + restoreOptions, + ), + }); + } catch { + failedCount++; } } + } finally { + databaseMs += performance.now() - databaseStartedAt; + } + + const coverStartedAt = performance.now(); + for ( + let start = 0; + start < restoredNovels.length; + start += BACKUP_FILE_CONCURRENCY + ) { + const coverBatch = restoredNovels.slice( + start, + start + BACKUP_FILE_CONCURRENCY, + ); + await Promise.all( + coverBatch.map(async ({ backupNovel, mapping: novelMapping }) => { + try { + if ( + !manifest.sections.downloadedFiles && + backupNovel.cover?.startsWith(APP_STORAGE_URI) + ) { + const coverBackupPath = coversDirPath + '/' + backupNovel.id; + if (await NativeFile.exists(coverBackupPath)) { + const coverPath = `${NOVEL_STORAGE}/${backupNovel.pluginId}/${novelMapping.restoredNovelId}/cover.png`; + await NativeFile.mkdir(parentDirectory(coverPath)); + await NativeFile.copyFile(coverBackupPath, coverPath); + } + } + } catch { + failedCount++; + } + }), + ); + } + coverMs += performance.now() - coverStartedAt; + + for (const { backupNovel, mapping: novelMapping } of restoredNovels) { + novelMappings.push(novelMapping); + novelIdMap.set(backupNovel.id, novelMapping.restoredNovelId); novelCount++; + } + }; + + benchmarkLog?.( + `restoreData:novels:restore:start total=${totalNovelCount}`, + ); + updateRestoreProgress(setMeta, getString('backupScreen.restoringNovels')); + for (const item of validItems) { + const readStartedAt = performance.now(); + let fileContent: string; + try { + fileContent = await NativeFile.readFile(item.path); } catch { failedCount++; + continue; + } finally { + readMs += performance.now() - readStartedAt; + } + + const parseStartedAt = performance.now(); + try { + const decodedNovels = decodeAndValidateNovelFile( + fileContent, + manifest, + ); + for (const backupNovel of decodedNovels) { + pluginIds.add(backupNovel.pluginId); + pendingNovels.push(backupNovel); + } + } catch { + failedCount += getNovelFileRecordCount(fileContent, manifest); + } finally { + parseMs += performance.now() - parseStartedAt; + } + + if (pendingNovels.length >= RESTORE_NOVEL_BATCH_SIZE) { + await restoreNovelBatch(); + updateRestoreProgress( + setMeta, + getString('backupScreen.restoringNovelsProgress', { + current: novelCount, + total: totalNovelCount, + }), + ); + benchmarkLog?.( + `restoreData:novels:restore:progress current=${novelCount} total=${totalNovelCount}`, + ); } } + await restoreNovelBatch(); + updateRestoreProgress( + setMeta, + getString('backupScreen.restoringNovelsProgress', { + current: novelCount, + total: totalNovelCount, + }), + ); + benchmarkLog?.( + `restoreData:novels:restore:progress current=${novelCount} total=${totalNovelCount}`, + ); + benchmarkLog?.( + `restoreData:novels:done count=${novelCount} failed=${failedCount} readMs=${readMs.toFixed( + 1, + )} parseMs=${parseMs.toFixed(1)} databaseMs=${databaseMs.toFixed( + 1, + )} coverMs=${coverMs.toFixed(1)}`, + ); } catch { failedSectionCount++; } } - // categories + benchmarkLog?.('restoreData:categories:start'); if (manifest.sections.library) { updateRestoreProgress( setMeta, @@ -381,8 +757,11 @@ export const restoreData = async ( failedSectionCount++; } } + benchmarkLog?.( + `restoreData:categories:done count=${categoryCount} failed=${failedCategoryCount}`, + ); - // settings + benchmarkLog?.('restoreData:settings:start'); if (manifest.sections.settings) { updateRestoreProgress(setMeta, getString('backupScreen.restoringSettings')); } @@ -409,11 +788,12 @@ export const restoreData = async ( // Included in the completion warning below. } } + benchmarkLog?.(`restoreData:settings:done restored=${settingsRestored}`); - // installed plugin registry + benchmarkLog?.('restoreData:plugins:start'); + let restoredPlugins = pluginsFromSettings; if (manifest.sections.plugins) { - let restoredPlugins = pluginsFromSettings; - if (manifest.formatVersion === 2) { + if (manifest.formatVersion === 2 || manifest.formatVersion === 3) { const pluginMetadataPath = cacheDirPath + '/' + BackupEntryName.PLUGIN_METADATA; if (!(await NativeFile.exists(pluginMetadataPath))) { @@ -438,6 +818,8 @@ export const restoreData = async ( ]; MMKVStorage.set(INSTALLED_PLUGINS_KEY, JSON.stringify(mergedPlugins)); } + benchmarkLog?.(`restoreData:plugins:done count=${restoredPlugins.length}`); + benchmarkLog?.('restoreData:done'); return { novelCount, @@ -448,6 +830,36 @@ export const restoreData = async ( failedSectionCount, pluginIds: [...pluginIds], novelMappings, + restoreRunId, manifest, }; }; + +export const clearRestoreChapterMappingsSafely = async ( + restoreRunId: string, +) => { + try { + await clearRestoreChapterMappings(restoreRunId); + } catch { + // Restore mappings are run-scoped and do not affect later restores. + } +}; + +export const restoreData = async ( + cacheDirPath: string, + setMeta?: TaskProgressUpdater, + benchmarkLog?: RestoreBenchmarkLogger, +): Promise => { + const restoreRunId = createRestoreRunId(); + try { + return await restoreDataInternal( + cacheDirPath, + setMeta, + benchmarkLog, + restoreRunId, + ); + } catch (error) { + await clearRestoreChapterMappingsSafely(restoreRunId); + throw error; + } +}; diff --git a/test/mocks/nativeModules.js b/test/mocks/nativeModules.js index e5f48124bf..a228365d9d 100644 --- a/test/mocks/nativeModules.js +++ b/test/mocks/nativeModules.js @@ -81,6 +81,7 @@ jest.mock('@modules/native-zip-archive', () => ({ __esModule: true, default: { zip: jest.fn().mockResolvedValue(), + zipDirectories: jest.fn().mockResolvedValue(), unzip: jest.fn().mockResolvedValue(), remoteUnzip: jest.fn().mockResolvedValue(), remoteZip: jest.fn().mockResolvedValue(''),