From 22754e81a9347c90f384e2794a5a34aec69c912b Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 09:22:06 +0900 Subject: [PATCH 01/10] Update LibvirtAblestackNasBackupHelper.java --- .../LibvirtAblestackNasBackupHelper.java | 126 +++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java index 5bfa880feb05..4c4a38ce6952 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java @@ -217,6 +217,7 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma Path dest = null; String dummyVmName = String.format("DUMMY-VM-%s", command.getCheckpointName().replace('.', '-')); Connect conn = null; + long startedAt = System.currentTimeMillis(); try { LOGGER.info("Starting stopped VM NAS backup for vm=[{}], dummyVm=[{}], backupType=[{}]", command.getVmName(), dummyVmName, command.getBackupType()); @@ -228,10 +229,16 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma dest = mountPoint.resolve(command.getBackupPath()); Files.createDirectories(dest.resolve("checkpoints")); markBackupInProgress(dest, command); + logStoppedBackupTrace(command, dummyVmName, "T0_BEFORE_DUMMY_CREATE", startedAt, + "source qcow2 state before dummy VM creation"); + logSourceQcow2State(command, dummyVmName, "T0_BEFORE_DUMMY_CREATE", diskPaths, startedAt); conn = LibvirtConnection.getConnection(); String dummyVmXml = buildDummyVmXml(dummyVmName, diskPaths, conn); resource.startVM(conn, dummyVmName, dummyVmXml, Domain.CreateFlags.PAUSED); + logStoppedBackupTrace(command, dummyVmName, "T1_AFTER_DUMMY_PAUSED_CREATE", startedAt, + "dummy VM was created with PAUSED flag"); + logDummyVmRuntimeState(command, dummyVmName, "T1_AFTER_DUMMY_PAUSED_CREATE", startedAt); if (isIncremental(command) && command.getParentCheckpointPath() != null && !command.getParentCheckpointPath().isEmpty()) { redefineCheckpointIfNeeded(dummyVmName, mountPoint.resolve(command.getParentCheckpointPath())); @@ -252,10 +259,16 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma LOGGER.error(failureDetails); return new Pair<>(backupBeginResult.first(), failureDetails); } + logStoppedBackupTrace(command, dummyVmName, "T2_AFTER_BACKUP_BEGIN", startedAt, + "backup-begin accepted by libvirt"); + logDummyVmRuntimeState(command, dummyVmName, "T2_AFTER_BACKUP_BEGIN", startedAt); try { final long effectiveTimeoutMillis = command.getWait() > 0 ? TimeUnit.SECONDS.toMillis(command.getWait()) : resource.getCmdsTimeout(); - waitForBackup(dummyVmName, effectiveTimeoutMillis); + waitForBackup(command, dummyVmName, effectiveTimeoutMillis, startedAt); + logStoppedBackupTrace(command, dummyVmName, "T3_AFTER_DOMJOB_COMPLETED", startedAt, + "domjobinfo reported completed backup job"); + logDummyVmRuntimeState(command, dummyVmName, "T3_AFTER_DOMJOB_COMPLETED", startedAt); } catch (IOException e) { cancelBackupJob(dummyVmName); throw e; @@ -266,11 +279,18 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma } dumpCheckpointXml(dummyVmName, command.getCheckpointName(), dest); + logStoppedBackupTrace(command, dummyVmName, "CHECKPOINT_DUMPED", startedAt, + "checkpoint XML was dumped before dummy cleanup"); + logCheckpointState(command, dummyVmName, command.getCheckpointName(), startedAt); Files.deleteIfExists(backupXml); Files.deleteIfExists(checkpointXml); runCommand(String.format("sync")); + logStoppedBackupTrace(command, dummyVmName, "BACKUP_MARKER_CREATE_BEGIN", startedAt, + "creating backup complete marker before finally cleanup"); markBackupComplete(dest, command); + logStoppedBackupTrace(command, dummyVmName, "BACKUP_MARKER_CREATED", startedAt, + "backup complete marker created"); String output = listTopLevelFileSizes(dest); LOGGER.info("Completed stopped VM NAS backup for vm=[{}], dummyVm=[{}]", command.getVmName(), dummyVmName); return new Pair<>(0, output); @@ -284,7 +304,12 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma mountPoint = null; return new Pair<>(1, e.getMessage()); } finally { + logStoppedBackupTrace(command, dummyVmName, "DUMMY_DESTROY_BEGIN", startedAt, + "starting dummy VM cleanup"); cleanupDummyVm(dummyVmName); + logStoppedBackupTrace(command, dummyVmName, "T4_AFTER_DUMMY_DESTROY", startedAt, + "dummy VM cleanup finished; collecting offline source qcow2 state"); + logSourceQcow2State(command, dummyVmName, "T4_AFTER_DUMMY_DESTROY", diskPaths, startedAt); unmountRepository(command, mountPoint); } } @@ -533,10 +558,12 @@ private Path writeCheckpointXml(Path dest, AblestackNasTakeBackupCommand command return checkpointXml; } - private void waitForBackup(String vmName, long timeoutMillis) throws IOException { + private void waitForBackup(AblestackNasTakeBackupCommand command, String vmName, long timeoutMillis, long startedAt) throws IOException { long remainingMillis = timeoutMillis; while (remainingMillis > 0) { String result = checkBackupJob(vmName); + LOGGER.info("[STOPPED_BACKUP_TRACE] phase=[DOMJOBINFO_POLL] vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], output=[{}]", + command.getVmName(), vmName, command.getBackupType(), command.getCheckpointName(), System.currentTimeMillis() - startedAt, result); if (result != null && result.contains("Completed") && result.contains("Backup")) { return; } @@ -586,6 +613,101 @@ private void dumpCheckpointXml(String vmName, String checkpointName, Path dest) shellQuote(vmName), shellQuote(checkpointName), shellQuote(checkpointDest.toString()))); } + private void logStoppedBackupTrace(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, long startedAt, String message) { + LOGGER.info("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], message=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), + System.currentTimeMillis() - startedAt, message); + } + + private void logDummyVmRuntimeState(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, long startedAt) { + logTraceCommandOutput(command, dummyVmName, phase, startedAt, "query-status", String.format( + "virsh -c qemu:///system qemu-monitor-command %s --pretty %s", + shellQuote(dummyVmName), shellQuote("{\"execute\":\"query-status\"}"))); + logTraceCommandOutput(command, dummyVmName, phase, startedAt, "query-block", String.format( + "virsh -c qemu:///system qemu-monitor-command %s --pretty %s", + shellQuote(dummyVmName), shellQuote("{\"execute\":\"query-block\"}"))); + logTraceCommandOutput(command, dummyVmName, phase, startedAt, "query-named-block-nodes", String.format( + "virsh -c qemu:///system qemu-monitor-command %s --pretty %s", + shellQuote(dummyVmName), shellQuote("{\"execute\":\"query-named-block-nodes\"}"))); + logTraceCommandOutput(command, dummyVmName, phase, startedAt, "domjobinfo", String.format( + "virsh -c qemu:///system domjobinfo %s --completed --keep-completed", + shellQuote(dummyVmName))); + } + + private void logCheckpointState(AblestackNasTakeBackupCommand command, String dummyVmName, String checkpointName, long startedAt) { + logTraceCommandOutput(command, dummyVmName, "CHECKPOINT_DUMPED", startedAt, "checkpoint-list", String.format( + "virsh -c qemu:///system checkpoint-list --domain %s --tree", + shellQuote(dummyVmName))); + logTraceCommandOutput(command, dummyVmName, "CHECKPOINT_DUMPED", startedAt, "checkpoint-info", String.format( + "virsh -c qemu:///system checkpoint-info --domain %s --checkpointname %s", + shellQuote(dummyVmName), shellQuote(checkpointName))); + } + + private void logSourceQcow2State(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, List diskPaths, long startedAt) { + for (int i = 0; i < diskPaths.size(); i++) { + String diskPath = diskPaths.get(i); + if (diskPath == null || diskPath.startsWith("rbd:")) { + continue; + } + logTraceDiskCommandOutput(command, dummyVmName, phase, startedAt, i, diskPath, "qemu-img-info", String.format( + "if command -v jq >/dev/null 2>&1; then qemu-img info -U --output=json %s | jq -c '{filename, virtual_size: .\"virtual-size\", actual_size: .\"actual-size\", dirty_flag: .\"dirty-flag\", bitmaps: .\"format-specific\".data.bitmaps}'; else qemu-img info -U --output=json %s; fi", + shellQuote(diskPath), shellQuote(diskPath))); + logTraceDiskCommandOutput(command, dummyVmName, phase, startedAt, i, diskPath, "qemu-img-check", String.format( + "qemu-img check %s", shellQuote(diskPath))); + } + } + + private void logTraceCommandOutput(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, long startedAt, String label, String traceCommand) { + Pair result = runCommandWithOutput(traceCommand); + if (result.first() == 0) { + LOGGER.info("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], command=[{}], output=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), + System.currentTimeMillis() - startedAt, label, result.second()); + warnIfRuntimeBitmapLooksUnsafe(command, dummyVmName, phase, label, result.second()); + } else { + LOGGER.warn("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], command=[{}], exitCode=[{}], output=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), + System.currentTimeMillis() - startedAt, label, result.first(), sanitizeCommandOutput(result.second())); + } + } + + private void logTraceDiskCommandOutput(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, long startedAt, int diskIndex, String diskPath, String label, String traceCommand) { + Pair result = runCommandWithOutput(traceCommand); + if (result.first() == 0) { + LOGGER.info("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], diskIndex=[{}], sourceDisk=[{}], command=[{}], output=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), + System.currentTimeMillis() - startedAt, diskIndex, diskPath, label, result.second()); + warnIfOfflineBitmapLooksUnsafe(command, dummyVmName, phase, diskIndex, diskPath, label, result.second()); + } else { + LOGGER.warn("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], diskIndex=[{}], sourceDisk=[{}], command=[{}], exitCode=[{}], output=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), + System.currentTimeMillis() - startedAt, diskIndex, diskPath, label, result.first(), sanitizeCommandOutput(result.second())); + } + } + + private void warnIfRuntimeBitmapLooksUnsafe(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, String label, String output) { + if (!"query-block".equals(label) && !"query-named-block-nodes".equals(label)) { + return; + } + if (output == null || output.isBlank()) { + return; + } + if (output.contains("\"busy\": true") || output.contains("\"inconsistent\": true") || output.contains("\"persistent\": false")) { + LOGGER.warn("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], command=[{}], bitmapWarning=[runtime bitmap may not be finalized], output=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), label, output); + } + } + + private void warnIfOfflineBitmapLooksUnsafe(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, int diskIndex, String diskPath, String label, String output) { + if (!"qemu-img-info".equals(label) || output == null || output.isBlank()) { + return; + } + if (output.contains("\"in-use\"") || output.contains("in-use")) { + LOGGER.warn("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], diskIndex=[{}], sourceDisk=[{}], bitmapWarning=[offline qcow2 bitmap still has in-use flag], output=[{}]", + phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), diskIndex, diskPath, output); + } + } + private String listTopLevelFileSizes(Path dest) throws IOException { try (var stream = Files.list(dest)) { return stream.filter(Files::isRegularFile) From 8e41ca4c86ea1a27404bde0f15beea2448a2e65a Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 14:18:42 +0900 Subject: [PATCH 02/10] =?UTF-8?q?NAS=20file-volume=20restore=20=EC=8B=9C?= =?UTF-8?q?=20target-is-zero=20=EC=B5=9C=EC=A0=81=ED=99=94=20=EB=81=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../LibvirtAblestackNasBackupHelper.java | 20 +++++++++++++++---- ...lestackNasRestoreBackupCommandWrapper.java | 3 ++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java index 4c4a38ce6952..a419e0a66004 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java @@ -659,33 +659,38 @@ private void logSourceQcow2State(AblestackNasTakeBackupCommand command, String d private void logTraceCommandOutput(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, long startedAt, String label, String traceCommand) { Pair result = runCommandWithOutput(traceCommand); + String output = formatTraceOutput(result.second()); if (result.first() == 0) { LOGGER.info("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], command=[{}], output=[{}]", phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), - System.currentTimeMillis() - startedAt, label, result.second()); + System.currentTimeMillis() - startedAt, label, output); warnIfRuntimeBitmapLooksUnsafe(command, dummyVmName, phase, label, result.second()); } else { LOGGER.warn("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], command=[{}], exitCode=[{}], output=[{}]", phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), - System.currentTimeMillis() - startedAt, label, result.first(), sanitizeCommandOutput(result.second())); + System.currentTimeMillis() - startedAt, label, result.first(), output); } } private void logTraceDiskCommandOutput(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, long startedAt, int diskIndex, String diskPath, String label, String traceCommand) { Pair result = runCommandWithOutput(traceCommand); + String output = formatTraceOutput(result.second()); if (result.first() == 0) { LOGGER.info("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], diskIndex=[{}], sourceDisk=[{}], command=[{}], output=[{}]", phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), - System.currentTimeMillis() - startedAt, diskIndex, diskPath, label, result.second()); + System.currentTimeMillis() - startedAt, diskIndex, diskPath, label, output); warnIfOfflineBitmapLooksUnsafe(command, dummyVmName, phase, diskIndex, diskPath, label, result.second()); } else { LOGGER.warn("[STOPPED_BACKUP_TRACE] phase=[{}], vm=[{}], dummyVm=[{}], backupType=[{}], checkpoint=[{}], elapsedMillis=[{}], diskIndex=[{}], sourceDisk=[{}], command=[{}], exitCode=[{}], output=[{}]", phase, command.getVmName(), dummyVmName, command.getBackupType(), command.getCheckpointName(), - System.currentTimeMillis() - startedAt, diskIndex, diskPath, label, result.first(), sanitizeCommandOutput(result.second())); + System.currentTimeMillis() - startedAt, diskIndex, diskPath, label, result.first(), output); } } private void warnIfRuntimeBitmapLooksUnsafe(AblestackNasTakeBackupCommand command, String dummyVmName, String phase, String label, String output) { + if (!"T3_AFTER_DOMJOB_COMPLETED".equals(phase)) { + return; + } if (!"query-block".equals(label) && !"query-named-block-nodes".equals(label)) { return; } @@ -708,6 +713,13 @@ private void warnIfOfflineBitmapLooksUnsafe(AblestackNasTakeBackupCommand comman } } + private String formatTraceOutput(String output) { + if (output == null || output.isBlank()) { + return ""; + } + return output.replace("\r", "\\r").replace("\n", "\\n").trim(); + } + private String listTopLevelFileSizes(Path dest) throws IOException { try (var stream = Files.list(dest)) { return stream.filter(Files::isRegularFile) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index fd2108638ccf..a1c35359e12d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -335,9 +335,10 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath QemuImgFile srcBackupFile = null; QemuImgFile destVolumeFile = null; try { - QemuImg qemu = new QemuImg(timeout * 1000, true, false); + QemuImg qemu = new QemuImg(timeout * 1000, false, false); srcBackupFile = new QemuImgFile(backupPath, getBackupFileFormat(backupPath)); destVolumeFile = new QemuImgFile(volumePath, getFileVolumeFormat(volumePath)); + logger.info("Restoring NAS backup file [{}] to file volume [{}] without target-is-zero optimization.", backupPath, volumePath); qemu.convert(srcBackupFile, destVolumeFile); return true; } catch (QemuImgException | LibvirtException e) { From 1165f8e617987a3d3b53045c414aeece784c8a41 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 14:28:47 +0900 Subject: [PATCH 03/10] =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8=EB=A1=9C?= =?UTF-8?q?=EA=B7=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index a1c35359e12d..15499e86c76d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -85,6 +85,8 @@ public Answer execute(AblestackNasRestoreBackupCommand command, LibvirtComputing List volumeChainStates = command.getVolumeChainStates(); BackupRestorePlan restorePlan = command.getRestorePlan(); + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[ENTER], vm=[{}], backupPath=[{}], repoType=[{}], repoAddress=[{}], vmExists=[{}], restorePlan=[{}], volumePaths=[{}], restoreVolumePaths=[{}], backupFiles=[{}], backupFileChains=[{}]", + vmName, backupPath, backupRepoType, backupRepoAddress, vmExists, restorePlan, volumePaths, restoreVolumePaths, backupFiles, backupFileChains); String newVolumeId = null; try { validateChainStatePlan(volumeChainStates, restorePlan); @@ -113,6 +115,8 @@ public Answer execute(AblestackNasRestoreBackupCommand command, LibvirtComputing return new BackupAnswer(command, false, errorMessage); } + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DONE], vm=[{}], backupPath=[{}], vmExists=[{}], newVolumeId=[{}]", + vmName, backupPath, vmExists, newVolumeId); return new BackupAnswer(command, true, newVolumeId); } @@ -296,6 +300,8 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr if (backupPaths == null || backupPaths.isEmpty()) { return false; } + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[RESTORE_VOLUME_BEGIN], poolType=[{}], targetVolume=[{}], backupPaths=[{}], backupIndex=[{}], createTargetVolume=[{}]", + volumePool.getPoolType(), volumePath, backupPaths, backupIndex, createTargetVolume); if (volumePool.getPoolType() != Storage.StoragePoolType.RBD) { if (backupPaths.stream().anyMatch(path -> path.endsWith(".rbdiff"))) { return restoreIncrementalRbdBackupChainToFileVolume(volumePath, backupPaths, timeout, backupRootPath, backupIndex); @@ -339,7 +345,12 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath srcBackupFile = new QemuImgFile(backupPath, getBackupFileFormat(backupPath)); destVolumeFile = new QemuImgFile(volumePath, getFileVolumeFormat(volumePath)); logger.info("Restoring NAS backup file [{}] to file volume [{}] without target-is-zero optimization.", backupPath, volumePath); + logFileRestoreQcow2State("BEFORE_SOURCE", backupPath, timeout); + logFileRestoreQcow2State("BEFORE_DEST", volumePath, timeout); qemu.convert(srcBackupFile, destVolumeFile); + logFileRestoreQcow2State("AFTER_SOURCE", backupPath, timeout); + logFileRestoreQcow2State("AFTER_DEST", volumePath, timeout); + logFileRestoreCompare(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout); return true; } catch (QemuImgException | LibvirtException e) { String srcFilename = srcBackupFile != null ? srcBackupFile.getFileName() : null; @@ -434,6 +445,67 @@ private boolean replaceRbdVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, return true; } + private void logFileRestoreQcow2State(String phase, String imagePath, int timeout) { + logFileRestoreCommand(phase, imagePath, "qemu-img-info", String.format( + "qemu-img info -U --output=json %s", quote(imagePath)), timeout); + logFileRestoreCommand(phase, imagePath, "qemu-img-check", String.format( + "qemu-img check -U %s", quote(imagePath)), timeout); + } + + private void logFileRestoreCompare(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, + QemuImg.PhysicalDiskFormat volumeFormat, int timeout) { + String compareCommand = String.format("qemu-img compare -f %s -F %s %s %s", + backupFormat.toString().toLowerCase(Locale.ROOT), volumeFormat.toString().toLowerCase(Locale.ROOT), + quote(backupPath), quote(volumePath)); + Pair result = runCommandWithOutput(compareCommand, timeout * 1000); + String output = formatTraceOutput(result.second()); + if (result.first() == 0) { + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[AFTER_COMPARE], source=[{}], target=[{}], command=[qemu-img-compare], output=[{}]", + backupPath, volumePath, output); + } else { + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[AFTER_COMPARE], source=[{}], target=[{}], command=[qemu-img-compare], exitCode=[{}], output=[{}]", + backupPath, volumePath, result.first(), output); + } + } + + private void logFileRestoreCommand(String phase, String imagePath, String label, String traceCommand, int timeout) { + Pair result = runCommandWithOutput(traceCommand, timeout * 1000); + String output = formatTraceOutput(result.second()); + if (result.first() == 0) { + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[{}], image=[{}], command=[{}], output=[{}]", + phase, imagePath, label, output); + } else { + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[{}], image=[{}], command=[{}], exitCode=[{}], output=[{}]", + phase, imagePath, label, result.first(), output); + } + } + + private Pair runCommandWithOutput(String command, int timeout) { + String wrappedCommand = String.format("set +e; %s 2>&1; rc=$?; echo __CMD_EXIT__=$rc", command); + String output = Script.runSimpleBashScriptWithFullResult(wrappedCommand, timeout); + if (output == null) { + return new Pair<>(-1, ""); + } + + List lines = new ArrayList<>(Arrays.asList(output.split("\n"))); + int exitCode = -1; + if (!lines.isEmpty()) { + String lastLine = lines.get(lines.size() - 1).trim(); + if (lastLine.startsWith("__CMD_EXIT__=")) { + exitCode = Integer.parseInt(lastLine.substring("__CMD_EXIT__=".length())); + lines.remove(lines.size() - 1); + } + } + return new Pair<>(exitCode, String.join("\n", lines).trim()); + } + + private String formatTraceOutput(String output) { + if (output == null || output.isBlank()) { + return ""; + } + return output.replace("\r", "\\r").replace("\n", "\\n").trim(); + } + private boolean importRawBackupToRbd(KVMStoragePool volumeStoragePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume) { if (!createTargetVolume && !deleteExistingRbdVolumeIfPresent(volumeStoragePool, volumePath)) { logger.error("Failed to delete existing RBD volume {} before raw import", volumePath); From e9e2676460169e979e2d9698d2c87b9405aaa921 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 15:42:51 +0900 Subject: [PATCH 04/10] =?UTF-8?q?=EB=B3=B5=EC=9B=90=EC=8B=9C=20=ED=8C=8C?= =?UTF-8?q?=EC=9D=BC=20=EB=8D=AE=EC=96=B4=EC=93=B0=EC=A7=80=EC=95=8A?= =?UTF-8?q?=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index 15499e86c76d..6680902f1914 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -340,6 +341,7 @@ private String getFirstExistingBackupPath(List backupPaths) { private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath, int timeout) { QemuImgFile srcBackupFile = null; QemuImgFile destVolumeFile = null; + Path movedAsideTarget = null; try { QemuImg qemu = new QemuImg(timeout * 1000, false, false); srcBackupFile = new QemuImgFile(backupPath, getBackupFileFormat(backupPath)); @@ -347,15 +349,20 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath logger.info("Restoring NAS backup file [{}] to file volume [{}] without target-is-zero optimization.", backupPath, volumePath); logFileRestoreQcow2State("BEFORE_SOURCE", backupPath, timeout); logFileRestoreQcow2State("BEFORE_DEST", volumePath, timeout); + movedAsideTarget = moveExistingFileVolumeAside(volumePath); qemu.convert(srcBackupFile, destVolumeFile); logFileRestoreQcow2State("AFTER_SOURCE", backupPath, timeout); logFileRestoreQcow2State("AFTER_DEST", volumePath, timeout); - logFileRestoreCompare(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout); + if (!logFileRestoreCompare(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout)) { + throw new QemuImgException(String.format("Restored file volume [%s] differs from NAS backup file [%s]", volumePath, backupPath)); + } + deleteMovedAsideFileVolume(movedAsideTarget); return true; - } catch (QemuImgException | LibvirtException e) { + } catch (QemuImgException | LibvirtException | IOException e) { String srcFilename = srcBackupFile != null ? srcBackupFile.getFileName() : null; String destFilename = destVolumeFile != null ? destVolumeFile.getFileName() : null; logger.error("Failed to convert backup {} to volume {}, the error was: {}", srcFilename, destFilename, e.getMessage()); + restoreMovedAsideFileVolume(volumePath, movedAsideTarget); return false; } } @@ -452,7 +459,7 @@ private void logFileRestoreQcow2State(String phase, String imagePath, int timeou "qemu-img check -U %s", quote(imagePath)), timeout); } - private void logFileRestoreCompare(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, + private boolean logFileRestoreCompare(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, QemuImg.PhysicalDiskFormat volumeFormat, int timeout) { String compareCommand = String.format("qemu-img compare -f %s -F %s %s %s", backupFormat.toString().toLowerCase(Locale.ROOT), volumeFormat.toString().toLowerCase(Locale.ROOT), @@ -462,9 +469,53 @@ private void logFileRestoreCompare(String backupPath, String volumePath, QemuImg if (result.first() == 0) { logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[AFTER_COMPARE], source=[{}], target=[{}], command=[qemu-img-compare], output=[{}]", backupPath, volumePath, output); + return true; } else { logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[AFTER_COMPARE], source=[{}], target=[{}], command=[qemu-img-compare], exitCode=[{}], output=[{}]", backupPath, volumePath, result.first(), output); + return false; + } + } + + private Path moveExistingFileVolumeAside(String volumePath) throws IOException { + Path targetPath = Paths.get(volumePath); + if (!Files.exists(targetPath)) { + return null; + } + + Path movedAsidePath = targetPath.resolveSibling(targetPath.getFileName() + ".csrestore." + System.currentTimeMillis() + ".bak"); + Files.move(targetPath, movedAsidePath); + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[TARGET_MOVED_ASIDE], target=[{}], movedAside=[{}]", + volumePath, movedAsidePath); + return movedAsidePath; + } + + private void deleteMovedAsideFileVolume(Path movedAsideTarget) { + if (movedAsideTarget == null) { + return; + } + try { + Files.deleteIfExists(movedAsideTarget); + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[TARGET_MOVED_ASIDE_DELETED], movedAside=[{}]", movedAsideTarget); + } catch (IOException e) { + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[TARGET_MOVED_ASIDE_DELETE_FAILED], movedAside=[{}], error=[{}]", + movedAsideTarget, e.getMessage()); + } + } + + private void restoreMovedAsideFileVolume(String volumePath, Path movedAsideTarget) { + if (movedAsideTarget == null || !Files.exists(movedAsideTarget)) { + return; + } + Path targetPath = Paths.get(volumePath); + try { + Files.deleteIfExists(targetPath); + Files.move(movedAsideTarget, targetPath); + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[TARGET_MOVED_ASIDE_RESTORED], target=[{}], movedAside=[{}]", + volumePath, movedAsideTarget); + } catch (IOException e) { + logger.error("[ABLESTACK_NAS_RESTORE_TRACE] phase=[TARGET_MOVED_ASIDE_RESTORE_FAILED], target=[{}], movedAside=[{}], error=[{}]", + volumePath, movedAsideTarget, e.getMessage()); } } From ced3b77489095a436cd05bde8cdb07c51bec9c16 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 16:20:45 +0900 Subject: [PATCH 05/10] =?UTF-8?q?=ED=8C=A8=EC=B9=98=20=EC=8B=A4=ED=8C=A8?= =?UTF-8?q?=EC=8B=9C=20bak=20=EC=9B=90=EB=B3=B5=20=ED=99=95=EC=9D=B8,=20?= =?UTF-8?q?=EC=9D=B4=ED=9B=84=EC=97=90=EB=8F=84=20=EC=9B=90=EC=9D=B8=20?= =?UTF-8?q?=EB=8F=99=EC=9D=BC=ED=95=98=EC=97=AC=20=EB=AA=85=EC=8B=9C?= =?UTF-8?q?=EC=A0=81=EC=9C=BC=EB=A1=9C=20shell=20=EC=8B=A4=ED=96=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index 6680902f1914..cbd1c95734ff 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -343,14 +343,13 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath QemuImgFile destVolumeFile = null; Path movedAsideTarget = null; try { - QemuImg qemu = new QemuImg(timeout * 1000, false, false); srcBackupFile = new QemuImgFile(backupPath, getBackupFileFormat(backupPath)); destVolumeFile = new QemuImgFile(volumePath, getFileVolumeFormat(volumePath)); logger.info("Restoring NAS backup file [{}] to file volume [{}] without target-is-zero optimization.", backupPath, volumePath); logFileRestoreQcow2State("BEFORE_SOURCE", backupPath, timeout); logFileRestoreQcow2State("BEFORE_DEST", volumePath, timeout); movedAsideTarget = moveExistingFileVolumeAside(volumePath); - qemu.convert(srcBackupFile, destVolumeFile); + convertFileVolumeWithQemuImg(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout); logFileRestoreQcow2State("AFTER_SOURCE", backupPath, timeout); logFileRestoreQcow2State("AFTER_DEST", volumePath, timeout); if (!logFileRestoreCompare(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout)) { @@ -358,7 +357,7 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath } deleteMovedAsideFileVolume(movedAsideTarget); return true; - } catch (QemuImgException | LibvirtException | IOException e) { + } catch (QemuImgException | IOException e) { String srcFilename = srcBackupFile != null ? srcBackupFile.getFileName() : null; String destFilename = destVolumeFile != null ? destVolumeFile.getFileName() : null; logger.error("Failed to convert backup {} to volume {}, the error was: {}", srcFilename, destFilename, e.getMessage()); @@ -519,6 +518,23 @@ private void restoreMovedAsideFileVolume(String volumePath, Path movedAsideTarge } } + private void convertFileVolumeWithQemuImg(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, + QemuImg.PhysicalDiskFormat volumeFormat, int timeout) throws QemuImgException { + String convertCommand = String.format("qemu-img convert -p -f %s -O %s %s %s", + backupFormat.toString().toLowerCase(Locale.ROOT), volumeFormat.toString().toLowerCase(Locale.ROOT), + quote(backupPath), quote(volumePath)); + Pair result = runCommandWithOutput(convertCommand, timeout * 1000); + String output = formatTraceOutput(result.second()); + if (result.first() == 0) { + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[CONVERT], source=[{}], target=[{}], command=[qemu-img-convert], output=[{}]", + backupPath, volumePath, output); + return; + } + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[CONVERT], source=[{}], target=[{}], command=[qemu-img-convert], exitCode=[{}], output=[{}]", + backupPath, volumePath, result.first(), output); + throw new QemuImgException(String.format("qemu-img convert failed with exitCode [%s], output [%s]", result.first(), output)); + } + private void logFileRestoreCommand(String phase, String imagePath, String label, String traceCommand, int timeout) { Pair result = runCommandWithOutput(traceCommand, timeout * 1000); String output = formatTraceOutput(result.second()); From d5600241c7bae4f35db2f3d7e61e25988e35173a Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 16:43:46 +0900 Subject: [PATCH 06/10] =?UTF-8?q?=EA=B2=B0=EA=B3=BC=20=EB=8F=99=EC=9D=BC?= =?UTF-8?q?=ED=95=98=EC=97=AC,=20convert=20=EC=A0=9C=EC=99=B8=ED=95=98?= =?UTF-8?q?=EA=B3=A0=20=EB=8F=99=EC=9D=BC=ED=95=9C=20=ED=8F=AC=EB=A7=B7?= =?UTF-8?q?=EC=9D=BC=EB=95=8C=20exact=20copy=EC=88=98=ED=96=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index cbd1c95734ff..9f9c1c606823 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -349,7 +349,7 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath logFileRestoreQcow2State("BEFORE_SOURCE", backupPath, timeout); logFileRestoreQcow2State("BEFORE_DEST", volumePath, timeout); movedAsideTarget = moveExistingFileVolumeAside(volumePath); - convertFileVolumeWithQemuImg(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout); + restoreFileVolumeData(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout); logFileRestoreQcow2State("AFTER_SOURCE", backupPath, timeout); logFileRestoreQcow2State("AFTER_DEST", volumePath, timeout); if (!logFileRestoreCompare(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout)) { @@ -518,6 +518,29 @@ private void restoreMovedAsideFileVolume(String volumePath, Path movedAsideTarge } } + private void restoreFileVolumeData(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, + QemuImg.PhysicalDiskFormat volumeFormat, int timeout) throws QemuImgException { + if (backupFormat == QemuImg.PhysicalDiskFormat.QCOW2 && volumeFormat == QemuImg.PhysicalDiskFormat.QCOW2) { + copyQcow2BackupFile(backupPath, volumePath, timeout); + return; + } + convertFileVolumeWithQemuImg(backupPath, volumePath, backupFormat, volumeFormat, timeout); + } + + private void copyQcow2BackupFile(String backupPath, String volumePath, int timeout) throws QemuImgException { + String copyCommand = String.format("cp --sparse=always %s %s", quote(backupPath), quote(volumePath)); + Pair result = runCommandWithOutput(copyCommand, timeout * 1000); + String output = formatTraceOutput(result.second()); + if (result.first() == 0) { + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[COPY], source=[{}], target=[{}], command=[cp-qcow2], output=[{}]", + backupPath, volumePath, output); + return; + } + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[COPY], source=[{}], target=[{}], command=[cp-qcow2], exitCode=[{}], output=[{}]", + backupPath, volumePath, result.first(), output); + throw new QemuImgException(String.format("cp qcow2 backup failed with exitCode [%s], output [%s]", result.first(), output)); + } + private void convertFileVolumeWithQemuImg(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, QemuImg.PhysicalDiskFormat volumeFormat, int timeout) throws QemuImgException { String convertCommand = String.format("qemu-img convert -p -f %s -O %s %s %s", From 4d8ee5a826562c077158e0fed295481ef4342358 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 12 Aug 2026 16:53:01 +0900 Subject: [PATCH 07/10] =?UTF-8?q?=EB=B3=B5=EC=9B=90=20=ED=9D=90=EB=A6=84?= =?UTF-8?q?=EC=9D=80=20cp=EB=A1=9C=20=EB=B3=80=EA=B2=BD=ED=95=98=EB=A9=B0,?= =?UTF-8?q?=20=EC=A7=84=EB=8B=A8=20convert=20=ED=98=B8=EC=B6=9C=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index 9f9c1c606823..d035fbd9b274 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -522,6 +522,7 @@ private void restoreFileVolumeData(String backupPath, String volumePath, QemuImg QemuImg.PhysicalDiskFormat volumeFormat, int timeout) throws QemuImgException { if (backupFormat == QemuImg.PhysicalDiskFormat.QCOW2 && volumeFormat == QemuImg.PhysicalDiskFormat.QCOW2) { copyQcow2BackupFile(backupPath, volumePath, timeout); + diagnoseQcow2ConvertToTemporaryFile(backupPath, volumePath, timeout); return; } convertFileVolumeWithQemuImg(backupPath, volumePath, backupFormat, volumeFormat, timeout); @@ -541,6 +542,44 @@ private void copyQcow2BackupFile(String backupPath, String volumePath, int timeo throw new QemuImgException(String.format("cp qcow2 backup failed with exitCode [%s], output [%s]", result.first(), output)); } + private void diagnoseQcow2ConvertToTemporaryFile(String backupPath, String volumePath, int timeout) { + Path diagnosticTarget = Paths.get(volumePath).resolveSibling(Paths.get(volumePath).getFileName() + ".convertdiag." + System.currentTimeMillis() + ".qcow2"); + String diagnosticTargetPath = diagnosticTarget.toString(); + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_BEGIN], source=[{}], restoreTarget=[{}], diagnosticTarget=[{}]", + backupPath, volumePath, diagnosticTargetPath); + logFileLifecycleState("DIAG_BEFORE_SOURCE", backupPath, timeout); + logFileLifecycleState("DIAG_BEFORE_RESTORE_TARGET", volumePath, timeout); + try { + convertFileVolumeWithQemuImg(backupPath, diagnosticTargetPath, QemuImg.PhysicalDiskFormat.QCOW2, QemuImg.PhysicalDiskFormat.QCOW2, timeout); + logFileRestoreQcow2State("DIAG_AFTER_CONVERT_TARGET", diagnosticTargetPath, timeout); + logFileLifecycleState("DIAG_AFTER_CONVERT_TARGET", diagnosticTargetPath, timeout); + logFileRestoreCompare(backupPath, diagnosticTargetPath, QemuImg.PhysicalDiskFormat.QCOW2, QemuImg.PhysicalDiskFormat.QCOW2, timeout); + } catch (QemuImgException e) { + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_FAILED], source=[{}], diagnosticTarget=[{}], error=[{}]", + backupPath, diagnosticTargetPath, e.getMessage()); + } finally { + deleteDiagnosticConvertFile(diagnosticTarget); + } + } + + private void logFileLifecycleState(String phase, String imagePath, int timeout) { + logFileRestoreCommand(phase, imagePath, "readlink", String.format("readlink -f %s", quote(imagePath)), timeout); + logFileRestoreCommand(phase, imagePath, "stat", String.format( + "stat -c 'dev=%%D inode=%%i size=%%s blocks=%%b mtime=%%y ctime=%%z' %s", quote(imagePath)), timeout); + logFileRestoreCommand(phase, imagePath, "lsof", String.format("lsof -- %s", quote(imagePath)), timeout); + logFileRestoreCommand(phase, imagePath, "fuser", String.format("fuser -v %s", quote(imagePath)), timeout); + } + + private void deleteDiagnosticConvertFile(Path diagnosticTarget) { + try { + Files.deleteIfExists(diagnosticTarget); + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_DELETED], diagnosticTarget=[{}]", diagnosticTarget); + } catch (IOException e) { + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_DELETE_FAILED], diagnosticTarget=[{}], error=[{}]", + diagnosticTarget, e.getMessage()); + } + } + private void convertFileVolumeWithQemuImg(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, QemuImg.PhysicalDiskFormat volumeFormat, int timeout) throws QemuImgException { String convertCommand = String.format("qemu-img convert -p -f %s -O %s %s %s", From ebe7c40da0672a22aa33a982f659750c4de21769 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Thu, 13 Aug 2026 09:16:31 +0900 Subject: [PATCH 08/10] =?UTF-8?q?cp=20=EC=97=90=EC=84=9C=20rsync=20?= =?UTF-8?q?=EB=B0=A9=EC=8B=9D=EC=9C=BC=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index d035fbd9b274..87c3ed0cc797 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -66,6 +66,7 @@ public class LibvirtAblestackNasRestoreBackupCommandWrapper extends CommandWrapp private static final String ATTACH_QCOW2_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --subdriver qcow2 --cache none"; private static final String ATTACH_RBD_DISK_XML_COMMAND = " virsh attach-device %s /dev/stdin < result = runCommandWithOutput(copyCommand, timeout * 1000); + private boolean hasBackingChain(String qcow2Path) { + return runCommandWithOutput(String.format(QEMU_IMG_HAS_BACKING_COMMAND, quote(qcow2Path)), 0).first() == 0; + } + + private void rsyncQcow2BackupFile(String backupPath, String volumePath, int timeout) throws QemuImgException { + String rsyncCommand = String.format("rsync -az %s %s", quote(backupPath), quote(volumePath)); + Pair result = runCommandWithOutput(rsyncCommand, timeout * 1000); String output = formatTraceOutput(result.second()); if (result.first() == 0) { - logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[COPY], source=[{}], target=[{}], command=[cp-qcow2], output=[{}]", + logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[RSYNC], source=[{}], target=[{}], command=[rsync-qcow2], output=[{}]", backupPath, volumePath, output); return; } - logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[COPY], source=[{}], target=[{}], command=[cp-qcow2], exitCode=[{}], output=[{}]", + logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[RSYNC], source=[{}], target=[{}], command=[rsync-qcow2], exitCode=[{}], output=[{}]", backupPath, volumePath, result.first(), output); - throw new QemuImgException(String.format("cp qcow2 backup failed with exitCode [%s], output [%s]", result.first(), output)); + throw new QemuImgException(String.format("rsync qcow2 backup failed with exitCode [%s], output [%s]", result.first(), output)); } private void diagnoseQcow2ConvertToTemporaryFile(String backupPath, String volumePath, int timeout) { From dc3248f2772c84b213f1e8305d7c5807baece69a Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Thu, 13 Aug 2026 12:03:43 +0900 Subject: [PATCH 09/10] =?UTF-8?q?convert=20=EC=A7=84=EB=8B=A8=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index 87c3ed0cc797..e4d45ca3c5f1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -527,7 +527,6 @@ private void restoreFileVolumeData(String backupPath, String volumePath, QemuImg } else { rsyncQcow2BackupFile(backupPath, volumePath, timeout); } - diagnoseQcow2ConvertToTemporaryFile(backupPath, volumePath, timeout); return; } convertFileVolumeWithQemuImg(backupPath, volumePath, backupFormat, volumeFormat, timeout); @@ -551,44 +550,6 @@ private void rsyncQcow2BackupFile(String backupPath, String volumePath, int time throw new QemuImgException(String.format("rsync qcow2 backup failed with exitCode [%s], output [%s]", result.first(), output)); } - private void diagnoseQcow2ConvertToTemporaryFile(String backupPath, String volumePath, int timeout) { - Path diagnosticTarget = Paths.get(volumePath).resolveSibling(Paths.get(volumePath).getFileName() + ".convertdiag." + System.currentTimeMillis() + ".qcow2"); - String diagnosticTargetPath = diagnosticTarget.toString(); - logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_BEGIN], source=[{}], restoreTarget=[{}], diagnosticTarget=[{}]", - backupPath, volumePath, diagnosticTargetPath); - logFileLifecycleState("DIAG_BEFORE_SOURCE", backupPath, timeout); - logFileLifecycleState("DIAG_BEFORE_RESTORE_TARGET", volumePath, timeout); - try { - convertFileVolumeWithQemuImg(backupPath, diagnosticTargetPath, QemuImg.PhysicalDiskFormat.QCOW2, QemuImg.PhysicalDiskFormat.QCOW2, timeout); - logFileRestoreQcow2State("DIAG_AFTER_CONVERT_TARGET", diagnosticTargetPath, timeout); - logFileLifecycleState("DIAG_AFTER_CONVERT_TARGET", diagnosticTargetPath, timeout); - logFileRestoreCompare(backupPath, diagnosticTargetPath, QemuImg.PhysicalDiskFormat.QCOW2, QemuImg.PhysicalDiskFormat.QCOW2, timeout); - } catch (QemuImgException e) { - logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_FAILED], source=[{}], diagnosticTarget=[{}], error=[{}]", - backupPath, diagnosticTargetPath, e.getMessage()); - } finally { - deleteDiagnosticConvertFile(diagnosticTarget); - } - } - - private void logFileLifecycleState(String phase, String imagePath, int timeout) { - logFileRestoreCommand(phase, imagePath, "readlink", String.format("readlink -f %s", quote(imagePath)), timeout); - logFileRestoreCommand(phase, imagePath, "stat", String.format( - "stat -c 'dev=%%D inode=%%i size=%%s blocks=%%b mtime=%%y ctime=%%z' %s", quote(imagePath)), timeout); - logFileRestoreCommand(phase, imagePath, "lsof", String.format("lsof -- %s", quote(imagePath)), timeout); - logFileRestoreCommand(phase, imagePath, "fuser", String.format("fuser -v %s", quote(imagePath)), timeout); - } - - private void deleteDiagnosticConvertFile(Path diagnosticTarget) { - try { - Files.deleteIfExists(diagnosticTarget); - logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_DELETED], diagnosticTarget=[{}]", diagnosticTarget); - } catch (IOException e) { - logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[DIAG_CONVERT_DELETE_FAILED], diagnosticTarget=[{}], error=[{}]", - diagnosticTarget, e.getMessage()); - } - } - private void convertFileVolumeWithQemuImg(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, QemuImg.PhysicalDiskFormat volumeFormat, int timeout) throws QemuImgException { String convertCommand = String.format("qemu-img convert -p -f %s -O %s %s %s", From ca5beb08dad77cc548d8d6b6a5d20b682e7638ff Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Thu, 13 Aug 2026 14:13:54 +0900 Subject: [PATCH 10/10] =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8=20compare=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lestackNasRestoreBackupCommandWrapper.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index e4d45ca3c5f1..92cdeb856aa2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -353,9 +353,6 @@ private boolean replaceFileVolumeWithBackup(String volumePath, String backupPath restoreFileVolumeData(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout); logFileRestoreQcow2State("AFTER_SOURCE", backupPath, timeout); logFileRestoreQcow2State("AFTER_DEST", volumePath, timeout); - if (!logFileRestoreCompare(backupPath, volumePath, srcBackupFile.getFormat(), destVolumeFile.getFormat(), timeout)) { - throw new QemuImgException(String.format("Restored file volume [%s] differs from NAS backup file [%s]", volumePath, backupPath)); - } deleteMovedAsideFileVolume(movedAsideTarget); return true; } catch (QemuImgException | IOException e) { @@ -459,24 +456,6 @@ private void logFileRestoreQcow2State(String phase, String imagePath, int timeou "qemu-img check -U %s", quote(imagePath)), timeout); } - private boolean logFileRestoreCompare(String backupPath, String volumePath, QemuImg.PhysicalDiskFormat backupFormat, - QemuImg.PhysicalDiskFormat volumeFormat, int timeout) { - String compareCommand = String.format("qemu-img compare -f %s -F %s %s %s", - backupFormat.toString().toLowerCase(Locale.ROOT), volumeFormat.toString().toLowerCase(Locale.ROOT), - quote(backupPath), quote(volumePath)); - Pair result = runCommandWithOutput(compareCommand, timeout * 1000); - String output = formatTraceOutput(result.second()); - if (result.first() == 0) { - logger.info("[ABLESTACK_NAS_RESTORE_TRACE] phase=[AFTER_COMPARE], source=[{}], target=[{}], command=[qemu-img-compare], output=[{}]", - backupPath, volumePath, output); - return true; - } else { - logger.warn("[ABLESTACK_NAS_RESTORE_TRACE] phase=[AFTER_COMPARE], source=[{}], target=[{}], command=[qemu-img-compare], exitCode=[{}], output=[{}]", - backupPath, volumePath, result.first(), output); - return false; - } - } - private Path moveExistingFileVolumeAside(String volumePath) throws IOException { Path targetPath = Paths.get(volumePath); if (!Files.exists(targetPath)) {