From 17684aa7f7ed81dabaeb2b18a30a31cd86e87e44 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Wed, 27 May 2026 14:00:27 +0900 Subject: [PATCH 01/12] veeam api --- .../apache/cloudstack/api/ApiConstants.java | 4 + .../backup/CreateAblestackVeeamBackupCmd.java | 120 ++++++ .../ImportAblestackVeeamBackupSeedCmd.java | 146 +++++++ .../backup/ListAblestackVeeamBackupsCmd.java | 89 +++++ .../backup/ListVeeamRestorePointsCmd.java | 84 ++++ .../RestoreAblestackVeeamBackupCmd.java | 94 +++++ .../cloudstack/backup/BackupManager.java | 20 + .../cloudstack/backup/BackupProvider.java | 8 + .../backup/BackupProviderNameUtils.java | 11 + client/pom.xml | 5 + .../AblestackNasImportVeeamSeedCommand.java | 158 ++++++++ .../backup/AblestackNasBackupProvider.java | 80 +++- plugins/backup/ablestack-veeam/pom.xml | 48 +++ .../backup/AblestackVeeamBackupOffering.java | 81 ++++ .../backup/AblestackVeeamBackupProvider.java | 367 ++++++++++++++++++ .../ablestackveeam/AblestackVeeamClient.java | 126 ++++++ plugins/backup/veeam/pom.xml | 50 ++- .../LibvirtAblestackNasBackupHelper.java | 30 ++ ...stackNasImportVeeamSeedCommandWrapper.java | 56 +++ plugins/pom.xml | 3 +- .../cloudstack/backup/BackupManagerImpl.java | 176 ++++++++- 21 files changed, 1747 insertions(+), 9 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java create mode 100644 plugins/backup/ablestack-veeam/pom.xml create mode 100644 plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java create mode 100644 plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java create mode 100644 plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ac239e254da4..e80f12d837a4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -533,6 +533,10 @@ public class ApiConstants { public static final String QUALIFIERS = "qualifiers"; public static final String QUERY_FILTER = "queryfilter"; public static final String QUIESCE_VM = "quiescevm"; + public static final String STAGING_DISK_PATHS = "stagingdiskpaths"; + public static final String VEEAM_RESTORE_POINT_ID = "veeamrestorepointid"; + public static final String SOURCE_DISK_FORMAT = "sourcediskformat"; + public static final String BOOTSTRAP_CHECKPOINT = "bootstrapcheckpoint"; public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java new file mode 100644 index 000000000000..b7110025b174 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "createAblestackVeeamBackup", + description = "Create an incremental NAS backup for a VM assigned to the ablestack-veeam offering " + + "(typically after a Veeam job completes)", + responseObject = SuccessResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class CreateAblestackVeeamBackupCmd extends BaseAsyncCreateCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance ID") + private Long vmId; + + @Parameter(name = ApiConstants.QUIESCE_VM, + type = CommandType.BOOLEAN, + required = false, + description = "Quiesce VM via QEMU guest agent before backup") + private Boolean quiesceVM; + + public Long getVmId() { + return vmId; + } + + public Boolean getQuiesceVM() { + return quiesceVM; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + boolean result = backupManager.createAblestackVeeamBackup(this, getJob()); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new CloudRuntimeException("Failed to create Ablestack Veeam backup"); + } + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Backup; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Creating Ablestack Veeam NAS backup for Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); + } + + @Override + public void create() throws ResourceAllocationException { + } + + @Override + public Long getEntityId() { + return vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java new file mode 100644 index 000000000000..f26debbf72aa --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; + +@APICommand(name = "importAblestackVeeamBackupSeed", + description = "Import a Veeam restore point as a NAS backup seed for Ablestack Veeam incremental KVM backups", + responseObject = BackupResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class ImportAblestackVeeamBackupSeedCmd extends BaseAsyncCreateCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance to register the seed backup for") + private Long vmId; + + @Parameter(name = ApiConstants.VEEAM_RESTORE_POINT_ID, + type = CommandType.STRING, + required = false, + description = "Veeam restore point ID. If omitted, stagingdiskpaths must be provided.") + private String veeamRestorePointId; + + @Parameter(name = ApiConstants.STAGING_DISK_PATHS, + type = CommandType.STRING, + required = false, + description = "Comma-separated disk file paths on the KVM host (from shared staging). " + + "If omitted, disks are exported from Veeam using veeamrestorepointid.") + private String stagingDiskPaths; + + @Parameter(name = ApiConstants.SOURCE_DISK_FORMAT, + type = CommandType.STRING, + required = false, + description = "Staging disk format: vmdk, flat, qcow2, or raw. Default: vmdk") + private String sourceDiskFormat; + + @Parameter(name = ApiConstants.BOOTSTRAP_CHECKPOINT, + type = CommandType.BOOLEAN, + required = false, + description = "Create libvirt checkpoint on the KVM VM after import. Default: true") + private Boolean bootstrapCheckpoint; + + public Long getVmId() { + return vmId; + } + + public String getVeeamRestorePointId() { + return veeamRestorePointId; + } + + public String getStagingDiskPaths() { + return stagingDiskPaths; + } + + public String getSourceDiskFormat() { + return sourceDiskFormat; + } + + public Boolean getBootstrapCheckpoint() { + return bootstrapCheckpoint; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + Backup backup = backupManager.importAblestackVeeamBackupSeed(this); + BackupResponse response = backupManager.createBackupResponse(backup, false); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Backup; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Importing Ablestack Veeam backup seed for Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); + } + + @Override + public void create() throws ResourceAllocationException { + } + + @Override + public Long getEntityId() { + return vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java new file mode 100644 index 000000000000..2f31289adb1d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.Pair; + +@APICommand(name = "listAblestackVeeamBackups", + description = "List NAS backups for a VM using the ablestack-veeam backup offering", + responseObject = BackupResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListAblestackVeeamBackupsCmd extends BaseListCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance ID") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + Pair, Integer> result = backupManager.listAblestackVeeamBackups(this); + ListResponse response = new ListResponse<>(); + List backupResponses = new ArrayList<>(); + for (Backup backup : result.first()) { + backupResponses.add(backupManager.createBackupResponse(backup, false)); + } + response.setResponses(backupResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java new file mode 100644 index 000000000000..b7b0cc6e686b --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRestorePointResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; + +@APICommand(name = "listVeeamRestorePoints", + description = "List Veeam B&R restore points for a KVM instance using the ablestack-veeam backup offering", + responseObject = BackupRestorePointResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class ListVeeamRestorePointsCmd extends BaseCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance UUID") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + List points = backupManager.listVeeamRestorePoints(this); + ListResponse response = new ListResponse<>(); + List list = backupManager.createVeeamRestorePointResponses(points); + response.setResponses(list); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java new file mode 100644 index 000000000000..4e58344583b6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "restoreAblestackVeeamBackup", + description = "Restore a VM from an ablestack-veeam NAS backup entry", + responseObject = SuccessResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class RestoreAblestackVeeamBackupCmd extends BaseAsyncCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "Backup ID") + private Long backupId; + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + boolean result = backupManager.restoreAblestackVeeamBackup(backupId); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new CloudRuntimeException("Failed to restore Ablestack Veeam backup"); + } + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_RESTORE; + } + + @Override + public String getEventDescription() { + return "Restoring Instance from Ablestack Veeam backup ID: " + getResourceUuid(ApiConstants.ID); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index b3d2ab09a958..d00aac87d1dd 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -25,7 +25,12 @@ import org.apache.cloudstack.api.command.admin.backup.CloneBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateAblestackVeeamBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; +import org.apache.cloudstack.api.command.user.backup.ImportAblestackVeeamBackupSeedCmd; +import org.apache.cloudstack.api.command.user.backup.ListAblestackVeeamBackupsCmd; +import org.apache.cloudstack.api.command.user.backup.ListVeeamRestorePointsCmd; +import org.apache.cloudstack.api.response.BackupRestorePointResponse; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; @@ -219,6 +224,21 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllocationException; + /** + * Import a Veeam restore point as NAS seed for Ablestack Veeam incremental backups. + */ + Backup importAblestackVeeamBackupSeed(ImportAblestackVeeamBackupSeedCmd cmd) throws ResourceAllocationException; + + List listVeeamRestorePoints(ListVeeamRestorePointsCmd cmd); + + List createVeeamRestorePointResponses(List points); + + boolean createAblestackVeeamBackup(CreateAblestackVeeamBackupCmd cmd, Object job) throws ResourceAllocationException; + + boolean restoreAblestackVeeamBackup(Long backupId); + + Pair, Integer> listAblestackVeeamBackups(ListAblestackVeeamBackupsCmd cmd); + /** * List existing backups for a VM */ diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 1cdbe75b169e..900a8bf1d3b1 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -83,6 +83,14 @@ default Pair takeBackup(VirtualMachine vm, Boolean quiesceVM, L return takeBackup(vm, quiesceVM); } + /** + * Import a Veeam restore point as a NAS backup seed (Ablestack Veeam provider only). + */ + default Pair importAblestackVeeamBackupSeed(VirtualMachine vm, String veeamRestorePointId, + List stagingDiskPaths, String sourceDiskFormat, Boolean bootstrapCheckpoint) { + throw new UnsupportedOperationException("Provider " + getName() + " does not support Veeam seed import"); + } + /** * Delete an existing backup * @param backup The backup to exclude diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java index 9788398cc3d4..0adc85620e37 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java @@ -23,6 +23,7 @@ public final class BackupProviderNameUtils { public static final String COMMVAULT = "commvault"; public static final String ABLESTACK_NAS = "ablestack-nas"; public static final String ABLESTACK_COMMVAULT = "ablestack-commvault"; + public static final String ABLESTACK_VEEAM = "ablestack-veeam"; private BackupProviderNameUtils() { } @@ -37,6 +38,9 @@ public static String canonicalize(final String providerName) { if (COMMVAULT.equalsIgnoreCase(providerName) || ABLESTACK_COMMVAULT.equalsIgnoreCase(providerName)) { return ABLESTACK_COMMVAULT; } + if ("veeam".equalsIgnoreCase(providerName) || ABLESTACK_VEEAM.equalsIgnoreCase(providerName)) { + return ABLESTACK_VEEAM; + } return providerName; } @@ -50,6 +54,9 @@ public static String toDisplayName(final String providerName) { if (ABLESTACK_COMMVAULT.equalsIgnoreCase(providerName) || COMMVAULT.equalsIgnoreCase(providerName)) { return COMMVAULT; } + if (ABLESTACK_VEEAM.equalsIgnoreCase(providerName) || "veeam".equalsIgnoreCase(providerName)) { + return "veeam"; + } return providerName; } @@ -60,4 +67,8 @@ public static boolean isNasFamily(final String providerName) { public static boolean isCommvaultFamily(final String providerName) { return ABLESTACK_COMMVAULT.equalsIgnoreCase(canonicalize(providerName)); } + + public static boolean isVeeamFamily(final String providerName) { + return ABLESTACK_VEEAM.equalsIgnoreCase(canonicalize(providerName)); + } } diff --git a/client/pom.xml b/client/pom.xml index 3b0b75659b30..00ec06722022 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -632,6 +632,11 @@ cloud-plugin-backup-ablestack-commvault ${project.version} + + org.apache.cloudstack + cloud-plugin-backup-ablestack-veeam + ${project.version} + org.apache.cloudstack cloud-plugin-backup-bx diff --git a/core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java b/core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java new file mode 100644 index 000000000000..cdfff2f475ab --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java @@ -0,0 +1,158 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + +import java.util.List; + +public class AblestackNasImportVeeamSeedCommand extends Command { + private String vmName; + private String backupPath; + private String checkpointName; + private String backupRepoType; + private String backupRepoAddress; + private List volumePools; + private List volumePaths; + private List backupFiles; + private List stagingDiskPaths; + private String sourceFormat; + private String veeamRestorePointId; + private Boolean bootstrapCheckpoint; + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + + public AblestackNasImportVeeamSeedCommand(String vmName, String backupPath) { + super(); + this.vmName = vmName; + this.backupPath = backupPath; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public String getCheckpointName() { + return checkpointName; + } + + public void setCheckpointName(String checkpointName) { + this.checkpointName = checkpointName; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public List getVolumePools() { + return volumePools; + } + + public void setVolumePools(List volumePools) { + this.volumePools = volumePools; + } + + public List getVolumePaths() { + return volumePaths; + } + + public void setVolumePaths(List volumePaths) { + this.volumePaths = volumePaths; + } + + public List getBackupFiles() { + return backupFiles; + } + + public void setBackupFiles(List backupFiles) { + this.backupFiles = backupFiles; + } + + public List getStagingDiskPaths() { + return stagingDiskPaths; + } + + public void setStagingDiskPaths(List stagingDiskPaths) { + this.stagingDiskPaths = stagingDiskPaths; + } + + public String getSourceFormat() { + return sourceFormat; + } + + public void setSourceFormat(String sourceFormat) { + this.sourceFormat = sourceFormat; + } + + public String getVeeamRestorePointId() { + return veeamRestorePointId; + } + + public void setVeeamRestorePointId(String veeamRestorePointId) { + this.veeamRestorePointId = veeamRestorePointId; + } + + public Boolean getBootstrapCheckpoint() { + return bootstrapCheckpoint; + } + + public void setBootstrapCheckpoint(Boolean bootstrapCheckpoint) { + this.bootstrapCheckpoint = bootstrapCheckpoint; + } + + public String getMountOptions() { + return mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java index 0fd82a32e2ca..a14b3c35915a 100644 --- a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java +++ b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java @@ -106,6 +106,9 @@ public class AblestackNasBackupProvider extends AdapterBase implements BackupPro private static final String DETAIL_CHAIN_SEALED = "nas.chain.sealed"; private static final String DETAIL_CHAIN_SEAL_REASON = "nas.chain.seal.reason"; private static final String DETAIL_FALLBACK_VOLUME_UUIDS = "nas.fallback.volume.uuids"; + public static final String DETAIL_BACKUP_SOURCE = "nas.backup.source"; + public static final String DETAIL_VEEAM_RESTORE_POINT_ID = "nas.veeam.restore.point.id"; + public static final String DETAIL_VEEAM_IMPORTED = "nas.veeam.imported"; private static final String MISSING_PARENT_RBD_SNAPSHOT_ERROR = "Parent RBD snapshot"; private static final long STALE_BACKUP_THRESHOLD_MS = TimeUnit.DAYS.toMillis(1); @@ -282,6 +285,81 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce return new Pair<>(result.success, result.backup); } + /** + * Import Veeam-exported disks as a FULL NAS backup seed for subsequent incremental backups. + */ + public Pair importVeeamBackupSeed(final VirtualMachine vm, final List stagingDiskPaths, + final String veeamRestorePointId, final String sourceFormat, final Boolean bootstrapCheckpoint) { + if (CollectionUtils.isEmpty(stagingDiskPaths)) { + throw new CloudRuntimeException("Staging disk paths are required to import a Veeam backup seed"); + } + final Host host = getVMHypervisorHostForBackup(vm); + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException("No valid backup repository found for the VM"); + } + + validateNoKvmFileBasedVmSnapshots(vm); + List vmVolumes = volumeDao.findByInstance(vm.getId()); + vmVolumes.sort(Comparator.comparing(Volume::getDeviceId)); + Pair, List> volumePoolsAndPaths = getVolumePoolsAndPaths(vmVolumes); + validateVolumePoolTypes(volumePoolsAndPaths.first()); + + final String backupPath = buildBackupPath(vm); + final String checkpointName = backupPath.substring(backupPath.lastIndexOf("/") + 1); + final String backupEngine = areAllVolumesOnRbdPool(volumePoolsAndPaths.first()) ? BACKUP_ENGINE_RBD_DIFF : BACKUP_ENGINE_QCOW2; + final List backupFiles = buildBackupFileNames(vmVolumes, backupEngine, false); + + BackupVO backupVO = createBackupObject(vm, backupPath, BACKUP_TYPE_FULL, checkpointName, backupEngine, null, volumePoolsAndPaths.second()); + updateBackupDetail(backupVO, DETAIL_BACKUP_SOURCE, "ablestack-veeam"); + updateBackupDetail(backupVO, DETAIL_VEEAM_IMPORTED, "true"); + if (StringUtils.isNotBlank(veeamRestorePointId)) { + updateBackupDetail(backupVO, DETAIL_VEEAM_RESTORE_POINT_ID, veeamRestorePointId); + } + + AblestackNasImportVeeamSeedCommand command = new AblestackNasImportVeeamSeedCommand(vm.getInstanceName(), backupPath); + command.setCheckpointName(checkpointName); + command.setBackupFiles(backupFiles); + command.setVolumePools(volumePoolsAndPaths.first()); + command.setVolumePaths(volumePoolsAndPaths.second()); + command.setStagingDiskPaths(stagingDiskPaths); + command.setSourceFormat(StringUtils.defaultIfBlank(sourceFormat, "vmdk")); + command.setVeeamRestorePointId(veeamRestorePointId); + command.setBootstrapCheckpoint(bootstrapCheckpoint == null || bootstrapCheckpoint); + command.setBackupRepoType(backupRepository.getType()); + command.setBackupRepoAddress(backupRepository.getAddress()); + command.setMountOptions(backupRepository.getMountOptions()); + + BackupAnswer answer; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), command); + } catch (AgentUnavailableException e) { + removeBackupWithDetails(backupVO.getId()); + throw new CloudRuntimeException("Unable to contact hypervisor host to import Veeam backup seed"); + } catch (OperationTimedoutException e) { + removeBackupWithDetails(backupVO.getId()); + throw new CloudRuntimeException("Timed out importing Veeam backup seed, please try again"); + } + + if (answer != null && answer.getResult()) { + backupVO.setDate(new Date()); + backupVO.setSize(answer.getSize()); + backupVO.setStatus(Backup.Status.BackedUp); + backupVO.setBackedUpVolumes(createVolumeInfoFromVolumes(vmVolumes, backupFiles)); + backupDao.update(backupVO.getId(), backupVO); + return new Pair<>(true, backupVO); + } + + final String details = answer != null ? answer.getDetails() : "No answer received"; + logger.error("Failed to import Veeam backup seed for VM {}: {}", vm.getInstanceName(), details); + removeBackupWithDetails(backupVO.getId()); + throw new CloudRuntimeException("Failed to import Veeam backup seed: " + details); + } + + public boolean hasBackedUpSeed(final VirtualMachine vm) { + return getLatestBackedUpBackup(vm) != null; + } + private BackupExecutionResult executeBackup(VirtualMachine vm, Boolean quiesceVM, Host host, BackupRepository backupRepository, List vmVolumes, Pair, List> volumePoolsAndPaths, Backup parentBackup, boolean incrementalBackup, boolean retryAsFullOnFailure) { @@ -537,7 +615,7 @@ private void sealBackupChain(Backup backup, String reason) { updateBackupDetail(backup, DETAIL_CHAIN_SEAL_REASON, reason); } - private void updateBackupDetail(Backup backup, String key, String value) { + public void updateBackupDetail(Backup backup, String key, String value) { if (backup == null || StringUtils.isBlank(key)) { return; } diff --git a/plugins/backup/ablestack-veeam/pom.xml b/plugins/backup/ablestack-veeam/pom.xml new file mode 100644 index 000000000000..06f19313539f --- /dev/null +++ b/plugins/backup/ablestack-veeam/pom.xml @@ -0,0 +1,48 @@ + + + 4.0.0 + cloud-plugin-backup-ablestack-veeam + Ablestack Plugin - Veeam + NAS KVM Backup Plugin + + cloudstack-plugins + org.apache.cloudstack + 4.22.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-plugin-backup-ablestack-nas + ${project.version} + + + org.apache.cloudstack + cloud-plugin-backup-veeam + ${project.version} + + + org.apache.commons + commons-lang3 + ${cs.commons-lang3.version} + + + diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java new file mode 100644 index 000000000000..a9f7a7d62645 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; + +public class AblestackVeeamBackupOffering implements BackupOffering { + + private final String name; + private final String uid; + + public AblestackVeeamBackupOffering(String name, String uid) { + this.name = name; + this.uid = uid; + } + + @Override + public String getExternalId() { + return uid; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "Ablestack Veeam + NAS KVM Backup Offering (Job)"; + } + + @Override + public long getZoneId() { + return -1; + } + + @Override + public boolean isUserDrivenBackupAllowed() { + return true; + } + + @Override + public String getProvider() { + return "ablestack-veeam"; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public String getUuid() { + return uid; + } + + @Override + public long getId() { + return -1; + } + + @Override + public String getRetentionPeriod() { + return null; + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java new file mode 100644 index 000000000000..5fbffae451cd --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamClient; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; + +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; + +/** + * Ablestack Veeam backup provider for KVM: seeds from Veeam restore points, then incremental NAS backups. + */ +public class AblestackVeeamBackupProvider extends AdapterBase implements BackupProvider, Configurable { + + public static final String PROVIDER_NAME = "ablestack-veeam"; + public static final String DETAIL_VEEAM_RESTORE_POINT_ID = "ablestack.veeam.restore.point.id"; + public static final String DETAIL_VEEAM_VM_NAME = "ablestack.veeam.vm.name"; + + public ConfigKey AblestackVeeamUrl = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.url", "https://localhost:9398/api/", + "The Ablestack Veeam B&R REST API URL.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamVersion = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.version", "0", + "Veeam server major version (0 = auto-detect via PowerShell).", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamUsername = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.username", "administrator", + "Veeam B&R username for Ablestack Veeam plugin.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamPassword = new ConfigKey<>("Secure", String.class, + "backup.plugin.ablestack-veeam.password", "", + "Veeam B&R password for Ablestack Veeam plugin.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamValidateSsl = new ConfigKey<>("Advanced", Boolean.class, + "backup.plugin.ablestack-veeam.validate.ssl", "false", + "Validate SSL when connecting to Veeam REST API.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamApiTimeout = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.request.timeout", "300", + "Veeam API request timeout in seconds.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.restore.timeout", "3600", + "Veeam export/restore operation timeout in seconds.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamTaskPollInterval = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.task.poll.interval", "5", + "Veeam task poll interval in seconds.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamTaskPollMaxRetry = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.task.poll.max.retry", "240", + "Max retries when polling Veeam tasks.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamStagingPath = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.staging.path", "/var/ablestack-veeam-staging", + "Shared staging directory on the Veeam server (must be visible to KVM hosts) for disk export before NAS seed import.", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + @Inject + private AblestackNasBackupProvider nasBackupProvider; + + @Inject + private BackupManager backupManager; + + protected AblestackVeeamClient getClient(final Long zoneId) { + try { + return new AblestackVeeamClient( + AblestackVeeamUrl.valueIn(zoneId), + AblestackVeeamVersion.valueIn(zoneId), + AblestackVeeamUsername.valueIn(zoneId), + AblestackVeeamPassword.valueIn(zoneId), + AblestackVeeamValidateSsl.valueIn(zoneId), + AblestackVeeamApiTimeout.valueIn(zoneId), + AblestackVeeamRestoreTimeout.valueIn(zoneId), + AblestackVeeamTaskPollInterval.valueIn(zoneId), + AblestackVeeamTaskPollMaxRetry.valueIn(zoneId)); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Ablestack Veeam API URL: " + e.getMessage()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new CloudRuntimeException("Failed to build Ablestack Veeam client: " + e.getMessage()); + } + } + + @Override + public String getName() { + return PROVIDER_NAME; + } + + @Override + public String getDescription() { + return "Ablestack Veeam + NAS KVM Backup Plugin"; + } + + @Override + public List listBackupOfferings(Long zoneId) { + return nasBackupProvider.listBackupOfferings(zoneId); + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + return nasBackupProvider.isValidProviderOffering(zoneId, uuid); + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { + if (!Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType())) { + throw new CloudRuntimeException("Ablestack Veeam backup provider supports KVM instances only"); + } + return true; + } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + return true; + } + + @Override + public boolean willDeleteBackupsOnOfferingRemoval() { + return false; + } + + @Override + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { + if (!nasBackupProvider.hasBackedUpSeed(vm)) { + final String restorePointId = resolveRestorePointId(vm); + final String stagingSubDir = String.format("%s/%s", vm.getInstanceName(), System.currentTimeMillis()); + final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); + logger.info("No NAS seed found for VM [{}], exporting Veeam restore point [{}] to [{}]", + vm.getInstanceName(), restorePointId, stagingPath); + final List stagingDisks = getClient(vm.getDataCenterId()) + .exportRestorePointDisksToStaging(restorePointId, stagingPath); + final Pair seedResult = nasBackupProvider.importVeeamBackupSeed( + vm, stagingDisks, restorePointId, "vmdk", true); + if (!seedResult.first()) { + return seedResult; + } + tagBackupAsVeeamSourced(seedResult.second(), restorePointId, vm); + return seedResult; + } + final Pair result = nasBackupProvider.takeBackup(vm, quiesceVM); + if (result.second() != null) { + tagBackupAsVeeamSourced(result.second(), getVmRestorePointDetail(vm), vm); + } + return result; + } + + @Override + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM, Long backupScheduleId) { + return takeBackup(vm, quiesceVM); + } + + @Override + public Pair importAblestackVeeamBackupSeed(VirtualMachine vm, String veeamRestorePointId, + List stagingDiskPaths, String sourceFormat, Boolean bootstrapCheckpoint) { + List staging = stagingDiskPaths; + if (CollectionUtils.isEmpty(staging)) { + if (StringUtils.isBlank(veeamRestorePointId)) { + throw new CloudRuntimeException("Either veeam restore point id or staging disk paths are required"); + } + final String stagingSubDir = String.format("%s/%s-import", vm.getInstanceName(), System.currentTimeMillis()); + final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); + staging = getClient(vm.getDataCenterId()).exportRestorePointDisksToStaging(veeamRestorePointId, stagingPath); + } + final Pair result = nasBackupProvider.importVeeamBackupSeed( + vm, staging, veeamRestorePointId, sourceFormat, bootstrapCheckpoint); + if (result.second() != null) { + tagBackupAsVeeamSourced(result.second(), veeamRestorePointId, vm); + } + return result; + } + + private void tagBackupAsVeeamSourced(Backup backup, String restorePointId, VirtualMachine vm) { + if (backup == null) { + return; + } + nasBackupProvider.updateBackupDetail(backup, AblestackNasBackupProvider.DETAIL_BACKUP_SOURCE, PROVIDER_NAME); + if (StringUtils.isNotBlank(restorePointId)) { + nasBackupProvider.updateBackupDetail(backup, DETAIL_VEEAM_RESTORE_POINT_ID, restorePointId); + } + nasBackupProvider.updateBackupDetail(backup, DETAIL_VEEAM_VM_NAME, vm.getInstanceName()); + } + + private String resolveRestorePointId(VirtualMachine vm) { + final String fromVm = getVmRestorePointDetail(vm); + if (StringUtils.isNotBlank(fromVm)) { + return fromVm; + } + final List points = listRestorePoints(vm); + if (CollectionUtils.isEmpty(points)) { + throw new CloudRuntimeException(String.format( + "No Veeam restore point found for VM [%s]. Set detail [%s] or import a seed via API.", + vm.getInstanceName(), DETAIL_VEEAM_RESTORE_POINT_ID)); + } + return points.get(0).getId(); + } + + private String getVmRestorePointDetail(VirtualMachine vm) { + Map details = backupManager.getBackupDetailsFromVM(vm); + return details != null ? details.get(DETAIL_VEEAM_RESTORE_POINT_ID) : null; + } + + @Override + public boolean deleteBackup(Backup backup, boolean forced) { + return nasBackupProvider.deleteBackup(backup, forced); + } + + @Override + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return nasBackupProvider.restoreBackupToVM(vm, backup, hostIp, dataStoreUuid); + } + + @Override + public Pair restoreBackupToVM(Long backupId, String vmName) { + return nasBackupProvider.restoreBackupToVM(backupId, vmName); + } + + @Override + public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + return nasBackupProvider.restoreVMFromBackup(vm, backup); + } + + @Override + public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, + String dataStoreUuid, Pair vmNameAndState) { + return nasBackupProvider.restoreBackedUpVolume(backup, backupVolumeInfo, hostIp, dataStoreUuid, vmNameAndState); + } + + @Override + public void syncBackupMetrics(Long zoneId) { + nasBackupProvider.syncBackupMetrics(zoneId); + } + + @Override + public List listRestorePoints(VirtualMachine vm) { + final String veeamVmName = getVeeamSourceVmName(vm); + final AblestackVeeamClient client = getClient(vm.getDataCenterId()); + client.syncBackupRepository(); + return client.listRestorePointsForVmDisplayName(veeamVmName); + } + + private String getVeeamSourceVmName(VirtualMachine vm) { + Map details = backupManager.getBackupDetailsFromVM(vm); + if (details != null && StringUtils.isNotBlank(details.get(DETAIL_VEEAM_VM_NAME))) { + return details.get(DETAIL_VEEAM_VM_NAME); + } + return vm.getInstanceName(); + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { + throw new CloudRuntimeException("Use importVeeamNasBackupSeed API to register a Veeam restore point as NAS seed"); + } + + @Override + public boolean supportsInstanceFromBackup() { + return nasBackupProvider.supportsInstanceFromBackup(); + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + return nasBackupProvider.getBackupStorageStats(zoneId); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + nasBackupProvider.syncBackupStorageStats(zoneId); + } + + @Override + public void syncBackups(VirtualMachine vm) { + nasBackupProvider.syncBackups(vm); + } + + @Override + public boolean checkBackupAgent(Long zoneId) { + return true; + } + + @Override + public boolean installBackupAgent(Long zoneId) { + return true; + } + + @Override + public boolean importBackupPlan(Long zoneId, String retentionPeriod, String externalId) { + return true; + } + + @Override + public boolean updateBackupPlan(Long zoneId, String retentionPeriod, String externalId) { + return true; + } + + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return nasBackupProvider.crossZoneInstanceCreationEnabled(backupOffering); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + AblestackVeeamUrl, + AblestackVeeamVersion, + AblestackVeeamUsername, + AblestackVeeamPassword, + AblestackVeeamValidateSsl, + AblestackVeeamApiTimeout, + AblestackVeeamRestoreTimeout, + AblestackVeeamTaskPollInterval, + AblestackVeeamTaskPollMaxRetry, + AblestackVeeamStagingPath + }; + } + + @Override + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } + + @Override + public boolean supportsVolumeLevelChainState() { + return nasBackupProvider.supportsVolumeLevelChainState(); + } + + @Override + public boolean supportsRestorePlan() { + return nasBackupProvider.supportsRestorePlan(); + } + + @Override + public boolean supportsRestoreChainValidation() { + return nasBackupProvider.supportsRestoreChainValidation(); + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java new file mode 100644 index 000000000000..133de7134147 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.ablestackveeam; + +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.veeam.VeeamClient; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; + +/** + * Ablestack Veeam client: extends the stock Veeam REST/SSH client with NAS seed export helpers. + */ +public class AblestackVeeamClient extends VeeamClient { + + public AblestackVeeamClient(final String url, final Integer version, final String username, final String password, + final boolean validateCertificate, final int timeout, final int restoreTimeout, final int taskPollInterval, + final int taskPollMaxRetry) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + super(url, version, username, password, validateCertificate, timeout, restoreTimeout, taskPollInterval, taskPollMaxRetry); + } + + /** + * Export all hard disks from a Veeam restore point to a directory on the Veeam server. + * The directory must be reachable from the KVM hypervisor (shared NFS recommended). + * + * @return absolute paths of exported disk files on the Veeam server + */ + public List exportRestorePointDisksToStaging(final String restorePointId, final String stagingPath) { + logger.debug(String.format("Exporting Veeam restore point [%s] to staging [%s]", restorePointId, stagingPath)); + final String escapedStaging = stagingPath.replace("'", "''"); + final String escapedId = restorePointId.replace("'", "''"); + final List cmds = Arrays.asList( + String.format("$staging = '%s'", escapedStaging), + "New-Item -ItemType Directory -Force -Path $staging | Out-Null", + String.format("$restorePoint = Get-VBRRestorePoint | Where-Object { $_.Id -eq '%s' -or $_.Id.Guid -eq '%s' }", escapedId, escapedId), + "if (-not $restorePoint) { Write-Output 'Failed: restore point not found'; Exit 1 }", + "$session = Start-VBRFLRSession -RestorePoint $restorePoint", + "$items = Get-VBRFLRItem -Session $session", + "$exported = @()", + "foreach ($item in $items) {", + " if ($item.Type -eq 'HardDisk') {", + " $target = Join-Path $staging ($item.Name + '.vmdk')", + " Copy-VBRFLRItem -FLRSession $session -Item $item -Destination $target", + " $exported += $target", + " }", + "}", + "Stop-VBRFLRSession -Session $session", + "if ($exported.Count -eq 0) { Write-Output 'Failed: no disks exported'; Exit 1 }", + "$exported -join ','" + ); + Pair result = executePowerShellCommands(cmds); + if (result == null || !result.first() || StringUtils.isBlank(result.second())) { + throw new CloudRuntimeException(String.format("Failed to export Veeam restore point [%s] to [%s]", restorePointId, stagingPath)); + } + if (result.second().contains("Failed:")) { + throw new CloudRuntimeException(result.second().trim()); + } + return Arrays.stream(result.second().trim().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + /** + * List Veeam restore points for a VM display name (KVM migration; no vCenter hierarchy required). + */ + public List listRestorePointsForVmDisplayName(final String vmDisplayName) { + final String escapedName = vmDisplayName.replace("'", "''"); + final List cmds = Arrays.asList( + String.format("$points = Get-VBRRestorePoint | Where-Object { $_.VmName -eq '%s' -or $_.Name -like '*%s*' }", escapedName, escapedName), + "if (-not $points) { Exit 0 }", + "$points | Sort-Object CreationTime -Descending | ForEach-Object {", + " Write-Output $_.Id.Guid", + " Write-Output $_.CreationTime.ToString('yyyy-MM-ddTHH:mm:ss')", + " Write-Output $_.Type", + " Write-Output '-----'", + "}" + ); + Pair response = executePowerShellCommands(cmds); + if (response == null || !response.first() || StringUtils.isBlank(response.second())) { + return new ArrayList<>(); + } + final List restorePoints = new ArrayList<>(); + for (final String block : response.second().split("-----\r\n")) { + final String[] parts = block.trim().split("\r\n"); + if (parts.length < 3) { + continue; + } + try { + final SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + final Date created = fmt.parse(parts[1].trim()); + restorePoints.add(new Backup.RestorePoint(parts[0].trim(), created, parts[2].trim(), null, null)); + } catch (ParseException e) { + logger.warn("Skipping unparseable Veeam restore point block: {}", block); + } + } + return restorePoints; + } +} diff --git a/plugins/backup/veeam/pom.xml b/plugins/backup/veeam/pom.xml index 0626c5f8e842..92a0011f6870 100644 --- a/plugins/backup/veeam/pom.xml +++ b/plugins/backup/veeam/pom.xml @@ -43,11 +43,6 @@ cloud-engine-components-api ${project.version} - - org.apache.cloudstack - cloud-plugin-hypervisor-vmware - ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-xml @@ -66,4 +61,49 @@ + + + + veeam-vmware-provider + + + noredist + + + + + org.apache.cloudstack + cloud-plugin-hypervisor-vmware + ${project.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org/apache/cloudstack/backup/VeeamBackupProvider.java + + + org/apache/cloudstack/backup/VeeamBackupProviderTest.java + + + + + + diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java index 073aa248a8ba..f13db4556f8a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java @@ -28,6 +28,7 @@ import com.cloud.storage.Storage; import com.cloud.utils.Pair; import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.AblestackNasImportVeeamSeedCommand; import org.apache.cloudstack.backup.AblestackNasTakeBackupCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.utils.security.ParserUtils; @@ -87,6 +88,16 @@ String getScriptOperation() { this.resource = resource; } + Pair executeImportVeeamSeed(AblestackNasImportVeeamSeedCommand command, List diskPaths) { + if (CollectionUtils.isNullOrEmpty(command.getStagingDiskPaths())) { + return new Pair<>(1, "Staging disk paths are required for Veeam seed import"); + } + String[] scriptCommand = buildImportVeeamSeedScriptCommand(command, diskPaths); + List commands = new ArrayList<>(); + commands.add(scriptCommand); + return Script.executePipedCommands(commands, resource.getCmdsTimeout()); + } + Pair executeBackup(AblestackNasTakeBackupCommand command) { LOGGER.info("LibvirtNasBackupHelper executeBackup entered for vm=[{}], backupPath=[{}], backupType=[{}]", command.getVmName(), command.getBackupPath(), command.getBackupType()); @@ -153,6 +164,25 @@ private boolean isWholeNumber(String value) { return value != null && !value.isEmpty() && value.chars().allMatch(Character::isDigit); } + private String[] buildImportVeeamSeedScriptCommand(AblestackNasImportVeeamSeedCommand command, List diskPaths) { + return new String[] { + resource.getAbleNasBackupPath(), + "-o", "import-veeam-seed", + "-v", command.getVmName(), + "-t", command.getBackupRepoType(), + "-s", command.getBackupRepoAddress(), + "-m", Objects.nonNull(command.getMountOptions()) ? command.getMountOptions() : "", + "-p", command.getBackupPath(), + "-c", Objects.nonNull(command.getCheckpointName()) ? command.getCheckpointName() : "", + "-f", CollectionUtils.isNullOrEmpty(command.getBackupFiles()) ? "" : String.join(",", command.getBackupFiles()), + "-d", diskPaths.isEmpty() ? "" : String.join(",", diskPaths), + "--staging-disks", String.join(",", command.getStagingDiskPaths()), + "--source-format", Objects.nonNull(command.getSourceFormat()) ? command.getSourceFormat() : "vmdk", + "--veeam-restore-point", Objects.nonNull(command.getVeeamRestorePointId()) ? command.getVeeamRestorePointId() : "", + "--bootstrap-checkpoint", command.getBootstrapCheckpoint() != null && command.getBootstrapCheckpoint() ? "true" : "false" + }; + } + private String[] buildBackupScriptCommand(AblestackNasTakeBackupCommand command, List diskPaths, BackupExecutionMode executionMode) { return new String[] { resource.getAbleNasBackupPath(), diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java new file mode 100644 index 000000000000..83eec305929a --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java @@ -0,0 +1,56 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import org.apache.cloudstack.backup.AblestackNasImportVeeamSeedCommand; +import org.apache.cloudstack.backup.BackupAnswer; + +import java.util.List; + +@ResourceWrapper(handles = AblestackNasImportVeeamSeedCommand.class) +public class LibvirtAblestackNasImportVeeamSeedCommandWrapper extends CommandWrapper { + @Override + public Answer execute(AblestackNasImportVeeamSeedCommand command, LibvirtComputingResource libvirtComputingResource) { + LibvirtAblestackNasBackupHelper backupHelper = new LibvirtAblestackNasBackupHelper(libvirtComputingResource); + List diskPaths = backupHelper.resolveDiskPaths(command.getVolumePools(), command.getVolumePaths()); + Pair result = backupHelper.executeImportVeeamSeed(command, diskPaths); + + if (result.first() != 0) { + BackupAnswer answer = new BackupAnswer(command, false, result.second().trim()); + if (result.first() == LibvirtAblestackNasBackupHelper.EXIT_CLEANUP_FAILED) { + answer.setNeedsCleanup(true); + } + return answer; + } + + BackupAnswer answer = new BackupAnswer(command, true, result.second().trim()); + try { + answer.setSize(backupHelper.parseBackupSize(result.second(), diskPaths)); + } catch (RuntimeException e) { + logger.warn("Failed to parse Veeam seed import size for vm=[{}]: {}", command.getVmName(), e.getMessage()); + } + return answer; + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index 3e84762551ef..284d5a67f7f3 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -65,6 +65,8 @@ backup/nas backup/ablestack-nas backup/ablestack-commvault + backup/veeam + backup/ablestack-veeam backup/bx ca/root-ca @@ -238,7 +240,6 @@ api/vmware-sioc - backup/veeam hypervisors/vmware network-elements/cisco-vnmc network-elements/nsx diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 3fd26bd210a8..232fc33e9e5f 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -52,7 +52,13 @@ import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.vm.CreateVMFromBackupCmdByAdmin; import org.apache.cloudstack.api.command.user.backup.AssignVirtualMachineToBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateAblestackVeeamBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; +import org.apache.cloudstack.api.command.user.backup.ImportAblestackVeeamBackupSeedCmd; +import org.apache.cloudstack.api.command.user.backup.ListAblestackVeeamBackupsCmd; +import org.apache.cloudstack.api.command.user.backup.ListVeeamRestorePointsCmd; +import org.apache.cloudstack.api.command.user.backup.RestoreAblestackVeeamBackupCmd; +import org.apache.cloudstack.api.response.BackupRestorePointResponse; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; @@ -780,8 +786,9 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { if (!BackupProviderNameUtils.isNasFamily(offering.getProvider()) && !BackupProviderNameUtils.isCommvaultFamily(offering.getProvider()) && + !BackupProviderNameUtils.isVeeamFamily(offering.getProvider()) && cmd.getQuiesceVM() != null) { - throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault backup provider"); + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault, and Ablestack Veeam backup providers"); } final String timezoneId = timeZone.getID(); @@ -984,8 +991,9 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo if (!BackupProviderNameUtils.isNasFamily(offering.getProvider()) && !BackupProviderNameUtils.isCommvaultFamily(offering.getProvider()) && + !BackupProviderNameUtils.isVeeamFamily(offering.getProvider()) && cmd.getQuiesceVM() != null) { - throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault backup provider"); + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault, and Ablestack Veeam backup providers"); } Long backupScheduleId = getBackupScheduleId(job); @@ -1009,6 +1017,165 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo return true; } + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "importing Ablestack Veeam backup seed", async = true) + public Backup importAblestackVeeamBackupSeed(ImportAblestackVeeamBackupSeedCmd cmd) throws ResourceAllocationException { + final VMInstanceVO vm = findVmById(cmd.getVmId()); + validateBackupForZone(vm.getDataCenterId()); + final Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, null, true, vm); + + if (vm.getBackupOfferingId() == null) { + throw new CloudRuntimeException("VM must be assigned to an Ablestack Veeam backup offering before importing a seed"); + } + + final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); + if (offering == null || !BackupProviderNameUtils.isVeeamFamily(offering.getProvider())) { + throw new CloudRuntimeException("VM backup offering must use the ablestack-veeam provider"); + } + + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + + List stagingPaths = null; + if (org.apache.commons.lang3.StringUtils.isNotBlank(cmd.getStagingDiskPaths())) { + stagingPaths = Arrays.stream(cmd.getStagingDiskPaths().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + final Pair result = backupProvider.importAblestackVeeamBackupSeed( + vm, cmd.getVeeamRestorePointId(), stagingPaths, cmd.getSourceDiskFormat(), cmd.getBootstrapCheckpoint()); + + if (!result.first() || result.second() == null) { + throw new CloudRuntimeException("Failed to import Ablestack Veeam backup seed"); + } + + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + if (result.second().getSize() != null) { + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, result.second().getSize()); + } + return result.second(); + } + + private BackupOffering validateVmAblestackVeeamOffering(final VMInstanceVO vm) { + if (vm.getBackupOfferingId() == null) { + throw new CloudRuntimeException("VM must be assigned to an Ablestack Veeam backup offering"); + } + final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); + if (offering == null || !BackupProviderNameUtils.isVeeamFamily(offering.getProvider())) { + throw new CloudRuntimeException("VM backup offering must use the ablestack-veeam provider"); + } + return offering; + } + + @Override + public List listVeeamRestorePoints(final ListVeeamRestorePointsCmd cmd) { + final VMInstanceVO vm = findVmById(cmd.getVmId()); + validateBackupForZone(vm.getDataCenterId()); + final Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, null, true, vm); + validateVmAblestackVeeamOffering(vm); + final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + return backupProvider.listRestorePoints(vm); + } + + @Override + public List createVeeamRestorePointResponses(final List points) { + final List responses = new ArrayList<>(); + if (points == null) { + return responses; + } + for (final Backup.RestorePoint point : points) { + final BackupRestorePointResponse response = new BackupRestorePointResponse(); + response.setId(point.getId()); + response.setCreated(point.getCreated()); + response.setType(point.getType()); + response.setObjectName(point.getId()); + responses.add(response); + } + return responses; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating Ablestack Veeam NAS backup", async = true) + public boolean createAblestackVeeamBackup(final CreateAblestackVeeamBackupCmd cmd, final Object job) + throws ResourceAllocationException { + final Long vmId = cmd.getVmId(); + final Account caller = CallContext.current().getCallingAccount(); + final VMInstanceVO vm = findVmById(vmId); + validateBackupForZone(vm.getDataCenterId()); + accountManager.checkAccess(caller, null, true, vm); + final BackupOffering offering = validateVmAblestackVeeamOffering(vm); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + final Account owner = accountManager.getAccount(vm.getAccountId()); + Long backupSize = 0L; + for (final Volume volume : volumeDao.findByInstance(vmId)) { + if (Volume.State.Ready.equals(volume.getState())) { + Long volumeSize = volumeApiService.getVolumePhysicalSize(volume.getFormat(), volume.getPath(), volume.getChainInfo()); + if (volumeSize == null) { + volumeSize = volume.getSize(); + } + backupSize += volumeSize; + } + } + createCheckedBackupVeeam(vm, vmId, backupProvider, cmd.getQuiesceVM(), backupSize, owner, getBackupScheduleId(job)); + return true; + } + + private void createCheckedBackupVeeam(final VMInstanceVO vm, final Long vmId, final BackupProvider backupProvider, + final Boolean quiesceVM, final Long backupSize, final Account owner, final Long backupScheduleId) + throws ResourceAllocationException { + try (CheckedReservation backupReservation = new CheckedReservation(owner, Resource.ResourceType.backup, + 1L, reservationDao, resourceLimitMgr); + CheckedReservation backupStorageReservation = new CheckedReservation(owner, + Resource.ResourceType.backup_storage, backupSize, reservationDao, resourceLimitMgr)) { + + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), + EventTypes.EVENT_VM_BACKUP_CREATE, "creating Ablestack Veeam backup for VM ID:" + vm.getUuid(), + vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); + + final Pair result = backupProvider.takeBackup(vm, quiesceVM); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create Ablestack Veeam VM backup"); + } + final Backup backup = result.second(); + if (backup != null) { + final BackupVO vmBackup = backupDao.findById(backup.getId()); + vmBackup.setBackupScheduleId(backupScheduleId); + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + } + } + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring VM from Ablestack Veeam backup", async = true) + public boolean restoreAblestackVeeamBackup(final Long backupId) { + final BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup " + backupId + " does not exist"); + } + final BackupOffering backupOffering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (backupOffering == null || !BackupProviderNameUtils.isVeeamFamily(backupOffering.getProvider())) { + throw new CloudRuntimeException("Backup is not from an ablestack-veeam offering"); + } + return restoreBackup(backupId); + } + + @Override + public Pair, Integer> listAblestackVeeamBackups(final ListAblestackVeeamBackupsCmd cmd) { + final VMInstanceVO vm = findVmById(cmd.getVmId()); + validateBackupForZone(vm.getDataCenterId()); + final Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, null, true, vm); + final BackupOffering offering = validateVmAblestackVeeamOffering(vm); + final List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), offering.getId()); + return new Pair<>(backups, backups.size()); + } + private void createCheckedBackup(CreateBackupCmd cmd, Account owner, boolean isScheduledBackup, Long backupSize, VMInstanceVO vm, Long vmId, BackupProvider backupProvider, Long backupScheduleId) throws ResourceAllocationException { @@ -2145,6 +2312,11 @@ public List> getCommands() { cmdList.add(DeleteBackupScheduleCmd.class); // Operations cmdList.add(CreateBackupCmd.class); + cmdList.add(ImportAblestackVeeamBackupSeedCmd.class); + cmdList.add(ListVeeamRestorePointsCmd.class); + cmdList.add(CreateAblestackVeeamBackupCmd.class); + cmdList.add(RestoreAblestackVeeamBackupCmd.class); + cmdList.add(ListAblestackVeeamBackupsCmd.class); cmdList.add(ListBackupsCmd.class); cmdList.add(RestoreBackupCmd.class); cmdList.add(DeleteBackupCmd.class); From 86dafb3dfae785c476595c6c3b3cba19370c2bf3 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Wed, 27 May 2026 15:38:56 +0900 Subject: [PATCH 02/12] veeam api --- .../backup/VeeamBackupProvider.java | 28 ++++++++++--------- .../cloudstack/backup/veeam/VeeamClient.java | 15 ++++++---- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 840f336a10ca..c571585a717c 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -56,35 +56,35 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public static final String BACKUP_IDENTIFIER = "-CSBKP-"; - public ConfigKey VeeamUrl = new ConfigKey<>("Advanced", String.class, + public ConfigKey VeeamUrl = new ConfigKey("Advanced", String.class, "backup.plugin.veeam.url", "https://localhost:9398/api/", "The Veeam backup and recovery URL.", true, ConfigKey.Scope.Zone); - public ConfigKey VeeamVersion = new ConfigKey<>("Advanced", Integer.class, + public ConfigKey VeeamVersion = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.version", "0", "The version of Veeam backup and recovery. CloudStack will get Veeam server version via PowerShell commands if it is 0 or not set", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamUsername = new ConfigKey<>("Advanced", String.class, + private ConfigKey VeeamUsername = new ConfigKey("Advanced", String.class, "backup.plugin.veeam.username", "administrator", "The Veeam backup and recovery username.", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamPassword = new ConfigKey<>("Secure", String.class, + private ConfigKey VeeamPassword = new ConfigKey("Secure", String.class, "backup.plugin.veeam.password", "", "The Veeam backup and recovery password.", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamValidateSSLSecurity = new ConfigKey<>("Advanced", Boolean.class, "backup.plugin.veeam.validate.ssl", "false", + private ConfigKey VeeamValidateSSLSecurity = new ConfigKey("Advanced", Boolean.class, "backup.plugin.veeam.validate.ssl", "false", "When set to true, this will validate the SSL certificate when connecting to https/ssl enabled Veeam API service.", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamApiRequestTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.request.timeout", "300", + private ConfigKey VeeamApiRequestTimeout = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.request.timeout", "300", "The Veeam B&R API request timeout in seconds.", true, ConfigKey.Scope.Zone); - private static ConfigKey VeeamRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.restore.timeout", "600", + private static ConfigKey VeeamRestoreTimeout = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.restore.timeout", "600", "The Veeam B&R API restore backup timeout in seconds.", true, ConfigKey.Scope.Zone); - private static ConfigKey VeeamTaskPollInterval = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.task.poll.interval", "5", + private static ConfigKey VeeamTaskPollInterval = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.task.poll.interval", "5", "The time interval in seconds when the management server polls for Veeam task status.", true, ConfigKey.Scope.Zone); - private static ConfigKey VeeamTaskPollMaxRetry = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.task.poll.max.retry", "120", + private static ConfigKey VeeamTaskPollMaxRetry = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.task.poll.max.retry", "120", "The max number of retrying times when the management server polls for Veeam task status.", true, ConfigKey.Scope.Zone); @Inject @@ -104,7 +104,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, @Inject private VolumeDao volumeDao; - private Map backupFilesMetricsMap = new HashMap<>(); + private Map backupFilesMetricsMap = new HashMap(); protected VeeamClient getClient(final Long zoneId) { try { @@ -113,14 +113,16 @@ protected VeeamClient getClient(final Long zoneId) { VeeamTaskPollInterval.valueIn(zoneId), VeeamTaskPollMaxRetry.valueIn(zoneId)); } catch (URISyntaxException e) { throw new CloudRuntimeException("Failed to parse Veeam API URL: " + e.getMessage()); - } catch (NoSuchAlgorithmException | KeyManagementException e) { + } catch (NoSuchAlgorithmException e) { + logger.error("Failed to build Veeam API client due to: ", e); + } catch (KeyManagementException e) { logger.error("Failed to build Veeam API client due to: ", e); } throw new CloudRuntimeException("Failed to build Veeam API client"); } public List listBackupOfferings(final Long zoneId) { - List policies = new ArrayList<>(); + List policies = new ArrayList(); for (final BackupOffering policy : getClient(zoneId).listJobs()) { if (!policy.getName().contains(BACKUP_IDENTIFIER)) { policies.add(policy); @@ -321,7 +323,7 @@ public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoi backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setName(backupManager.getBackupNameFromVM(vm)); - List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + List volumes = new ArrayList(volumeDao.findByInstance(vm.getId())); backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); Map details = backupManager.getBackupDetailsFromVM(vm); backup.setDetails(details); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 8a111f928680..035f4e087f6c 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -463,7 +463,7 @@ public List listJobs() { final HttpResponse response = get("/jobs"); checkResponseOK(response); final EntityReferences entityReferences = OBJECT_MAPPER.readValue(response.getEntity().getContent(), EntityReferences.class); - final List policies = new ArrayList<>(); + final List policies = new ArrayList(); if (entityReferences == null || entityReferences.getRefs() == null) { return policies; } @@ -475,7 +475,7 @@ public List listJobs() { logger.error("Failed to list Veeam jobs due to:", e); checkResponseTimeOut(e); } - return new ArrayList<>(); + return new ArrayList(); } public Job listJob(final String jobId) { @@ -698,11 +698,11 @@ public Map getBackupMetricsViaVeeamAPI() { logger.error("Failed to get backup metrics via Veeam B&R API due to:", e); checkResponseTimeOut(e); } - return new HashMap<>(); + return new HashMap(); } protected Map processHttpResponseForBackupMetrics(final InputStream content) { - Map metrics = new HashMap<>(); + Map metrics = new HashMap(); try { final BackupFiles backupFiles = OBJECT_MAPPER.readValue(content, BackupFiles.class); if (backupFiles == null || CollectionUtils.isEmpty(backupFiles.getBackupFiles())) { @@ -755,7 +755,7 @@ public Map getBackupMetricsLegacy() { protected Map processPowerShellResultForBackupMetrics(final String result) { logger.debug("Processing powershell result: " + result); final String separator = "====="; - Map metrics = new HashMap<>(); + Map metrics = new HashMap(); for (final String block : result.split(separator + "\r\n")) { final String[] parts = block.split("\r\n"); if (parts.length != 3) { @@ -891,7 +891,10 @@ public List processHttpResponseForVmRestorePoints(InputStre } vmRestorePointList.add(new Backup.RestorePoint(vmRestorePointId, created, type, backupSize, dataSize)); } - } catch (final IOException | ParseException e) { + } catch (final IOException e) { + logger.error("Failed to process response to get VM restore points via Veeam B&R API due to:", e); + checkResponseTimeOut(e); + } catch (final ParseException e) { logger.error("Failed to process response to get VM restore points via Veeam B&R API due to:", e); checkResponseTimeOut(e); } From a2793455086dc3d2c33bc20a94d27521626c5744 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Wed, 27 May 2026 15:50:36 +0900 Subject: [PATCH 03/12] test --- .../apache/cloudstack/backup/VeeamBackupProvider.java | 4 ++-- .../apache/cloudstack/backup/veeam/VeeamClient.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c571585a717c..6e692f2aa5ca 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -224,7 +224,7 @@ public boolean willDeleteBackupsOnOfferingRemoval() { public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { final VeeamClient client = getClient(vm.getDataCenterId()); Boolean result = client.startBackupJob(vm.getBackupExternalId()); - return new Pair<>(result, null); + return new Pair(result, null); } @Override @@ -353,7 +353,7 @@ public boolean supportsInstanceFromBackup() { @Override public Pair getBackupStorageStats(Long zoneId) { - return new Pair<>(0L, 0L); + return new Pair(0L, 0L); } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 035f4e087f6c..179072ac13fe 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -395,7 +395,7 @@ protected void checkIfRestoreSessionFinished(String type, String path) throws IO private Pair getRelatedLinkPair(List links) { for (Link link : links) { if (link.getRel().equals("Related")) { - return new Pair<>(link.getHref(), link.getType()); + return new Pair(link.getHref(), link.getType()); } } return null; @@ -811,7 +811,7 @@ public List listRestorePointsLegacy(String backupName, Stri return null; } - final List restorePoints = new ArrayList<>(); + final List restorePoints = new ArrayList(); for (final String block : response.second().split("\r\n\r\n")) { if (block.isEmpty()) { continue; @@ -842,11 +842,11 @@ public List listVmRestorePointsViaVeeamAPI(String vmwareDcN logger.error("Failed to list VM restore points via Veeam B&R API due to:", e); checkResponseTimeOut(e); } - return new ArrayList<>(); + return new ArrayList(); } public List processHttpResponseForVmRestorePoints(InputStream content, String vmwareDcName, String vmInternalName, Map metricsMap) { - List vmRestorePointList = new ArrayList<>(); + List vmRestorePointList = new ArrayList(); try { final VmRestorePoints vmRestorePoints = OBJECT_MAPPER.readValue(content, VmRestorePoints.class); final String hierarchyId = findDCHierarchy(vmwareDcName); @@ -922,7 +922,7 @@ public Pair restoreVMToDifferentLocation(String restorePointId, if (result == null || !result.first()) { throw new CloudRuntimeException("Failed to restore VM to location " + restoreLocation); } - return new Pair<>(result.first(), restoreLocation); + return new Pair(result.first(), restoreLocation); } /** From 7c25cdcbd5ec0e603f1a308ef99ba3c9667e2f52 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Wed, 27 May 2026 15:58:46 +0900 Subject: [PATCH 04/12] test1 --- .../apache/cloudstack/backup/VeeamBackupProviderTest.java | 2 +- .../org/apache/cloudstack/backup/veeam/VeeamClientTest.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java index a82ff551b8a9..c437ba724aa3 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java @@ -98,7 +98,7 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasJustOneBackup() { Mockito.when(vmInstanceDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(vmInstanceVO); Mockito.doReturn(client).when(backupProvider).getClient(2l); Mockito.doReturn(true).when(client).deleteBackup("abc"); - List backups = new ArrayList<>(); + List backups = new ArrayList(); backups.add(backup); Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(backups); Mockito.verify(backupDao, Mockito.never()).remove(Mockito.anyLong()); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 333c3e16053a..bfcd2931c91a 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -173,7 +173,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti @Test public void getRestoreVmErrorDescriptionTestFindErrorDescription() { - Pair response = new Pair<>(true, "Example of error description found in Veeam."); + Pair response = new Pair(true, "Example of error description found in Veeam."); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); @@ -182,7 +182,7 @@ public void getRestoreVmErrorDescriptionTestFindErrorDescription() { @Test public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { - Pair response = new Pair<>(true, "Cannot find restore session with provided uid uuid"); + Pair response = new Pair(true, "Cannot find restore session with provided uid uuid"); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); @@ -199,7 +199,7 @@ public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { @Test public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { - Pair response = new Pair<>(false, null); + Pair response = new Pair(false, null); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); From 151fea3db95ae160ceccadc240b6e4fd6ae3c304 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Thu, 28 May 2026 08:17:48 +0900 Subject: [PATCH 05/12] test3 --- .../backup/VeeamBackupProviderTest.java | 5 ++++- .../backup/veeam/VeeamClientTest.java | 22 +++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java index c437ba724aa3..97a54713d53f 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java @@ -124,7 +124,10 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasMoreThanOneBackup() { Mockito.when(vmInstanceDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(vmInstanceVO); Mockito.doReturn(client).when(backupProvider).getClient(2l); Mockito.doReturn(true).when(client).deleteBackup("abc"); - Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(List.of(backup, backup2)); + List backups = new ArrayList(); + backups.add(backup); + backups.add(backup2); + Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(backups); boolean result = backupProvider.deleteBackup(backup, true); Mockito.verify(backupDao, Mockito.times(1)).remove(2l); assertEquals(true, result); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index bfcd2931c91a..87fa05c16e72 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -107,7 +107,7 @@ public void testVeeamJobs() { public void getRepositoryNameFromJobTestExceptionCmdWithoutResult() throws Exception { String backupName = "TEST-BACKUP"; try { - Mockito.doReturn(null).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(null).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); mockClient.getRepositoryNameFromJob(backupName); fail(); } catch (Exception e) { @@ -120,7 +120,7 @@ public void getRepositoryNameFromJobTestExceptionCmdWithoutResult() throws Excep public void getRepositoryNameFromJobTestExceptionCmdWithFalseResult() { String backupName = "TEST-BACKUP2"; Pair response = new Pair(Boolean.FALSE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); try { mockClient.getRepositoryNameFromJob(backupName); fail(); @@ -134,7 +134,7 @@ public void getRepositoryNameFromJobTestExceptionCmdWithFalseResult() { public void getRepositoryNameFromJobTestExceptionWhenResultIsInWrongFormat() { String backupName = "TEST-BACKUP3"; Pair response = new Pair(Boolean.TRUE, "\nName:\n\nName-test"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); try { mockClient.getRepositoryNameFromJob(backupName); fail(); @@ -148,7 +148,7 @@ public void getRepositoryNameFromJobTestExceptionWhenResultIsInWrongFormat() { public void getRepositoryNameFromJobTestSuccess() throws Exception { String backupName = "TEST-BACKUP3"; Pair response = new Pair(Boolean.TRUE, "\r\nName : test"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); String repositoryNameFromJob = mockClient.getRepositoryNameFromJob(backupName); Assert.assertEquals("test", repositoryNameFromJob); } @@ -175,7 +175,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti public void getRestoreVmErrorDescriptionTestFindErrorDescription() { Pair response = new Pair(true, "Example of error description found in Veeam."); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Example of error description found in Veeam.", result); } @@ -184,7 +184,7 @@ public void getRestoreVmErrorDescriptionTestFindErrorDescription() { public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { Pair response = new Pair(true, "Cannot find restore session with provided uid uuid"); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Cannot find restore session with provided uid uuid", result); } @@ -192,7 +192,7 @@ public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { @Test public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(null); + Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(null); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); } @@ -201,7 +201,7 @@ public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { Pair response = new Pair(false, null); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); } @@ -544,21 +544,21 @@ public void testListVmRestorePointsViaVeeamAPI() { @Test public void testGetVeeamServerVersionAllGood() { Pair response = new Pair(Boolean.TRUE, "12.0.0.1"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); Assert.assertEquals(12, (int) mockClient.getVeeamServerVersion()); } @Test public void testGetVeeamServerVersionWithError() { Pair response = new Pair(Boolean.FALSE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); } @Test public void testGetVeeamServerVersionWithEmptyVersion() { Pair response = new Pair(Boolean.TRUE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); } } From 210f65393df0918d379acd484f2582f707f5cbb8 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Thu, 28 May 2026 08:29:30 +0900 Subject: [PATCH 06/12] test4 --- .../backup/veeam/VeeamClientTest.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 87fa05c16e72..de62dc9d46a3 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -61,6 +61,11 @@ public class VeeamClientTest { @Rule public WireMockRule wireMockRule = new WireMockRule(9399); + @SuppressWarnings("unchecked") + private static List anyCommandList() { + return (List) Mockito.any(List.class); + } + @Before public void setUp() throws Exception { wireMockRule.stubFor(post(urlMatching(".*/sessionMngr/.*")) @@ -107,7 +112,7 @@ public void testVeeamJobs() { public void getRepositoryNameFromJobTestExceptionCmdWithoutResult() throws Exception { String backupName = "TEST-BACKUP"; try { - Mockito.doReturn(null).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(null).when(mockClient).executePowerShellCommands(anyCommandList()); mockClient.getRepositoryNameFromJob(backupName); fail(); } catch (Exception e) { @@ -120,7 +125,7 @@ public void getRepositoryNameFromJobTestExceptionCmdWithoutResult() throws Excep public void getRepositoryNameFromJobTestExceptionCmdWithFalseResult() { String backupName = "TEST-BACKUP2"; Pair response = new Pair(Boolean.FALSE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); try { mockClient.getRepositoryNameFromJob(backupName); fail(); @@ -134,7 +139,7 @@ public void getRepositoryNameFromJobTestExceptionCmdWithFalseResult() { public void getRepositoryNameFromJobTestExceptionWhenResultIsInWrongFormat() { String backupName = "TEST-BACKUP3"; Pair response = new Pair(Boolean.TRUE, "\nName:\n\nName-test"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); try { mockClient.getRepositoryNameFromJob(backupName); fail(); @@ -148,7 +153,7 @@ public void getRepositoryNameFromJobTestExceptionWhenResultIsInWrongFormat() { public void getRepositoryNameFromJobTestSuccess() throws Exception { String backupName = "TEST-BACKUP3"; Pair response = new Pair(Boolean.TRUE, "\r\nName : test"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); String repositoryNameFromJob = mockClient.getRepositoryNameFromJob(backupName); Assert.assertEquals("test", repositoryNameFromJob); } @@ -175,7 +180,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti public void getRestoreVmErrorDescriptionTestFindErrorDescription() { Pair response = new Pair(true, "Example of error description found in Veeam."); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Example of error description found in Veeam.", result); } @@ -184,7 +189,7 @@ public void getRestoreVmErrorDescriptionTestFindErrorDescription() { public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { Pair response = new Pair(true, "Cannot find restore session with provided uid uuid"); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Cannot find restore session with provided uid uuid", result); } @@ -192,7 +197,7 @@ public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { @Test public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(null); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(null); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); } @@ -201,7 +206,7 @@ public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { Pair response = new Pair(false, null); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.anyListOf(String.class))).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); } @@ -544,21 +549,21 @@ public void testListVmRestorePointsViaVeeamAPI() { @Test public void testGetVeeamServerVersionAllGood() { Pair response = new Pair(Boolean.TRUE, "12.0.0.1"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); Assert.assertEquals(12, (int) mockClient.getVeeamServerVersion()); } @Test public void testGetVeeamServerVersionWithError() { Pair response = new Pair(Boolean.FALSE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); } @Test public void testGetVeeamServerVersionWithEmptyVersion() { Pair response = new Pair(Boolean.TRUE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyListOf(String.class)); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); } } From 1784b03e953dc33e3a8967347a6c3858cf9ea227 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Wed, 10 Jun 2026 10:14:59 +0900 Subject: [PATCH 07/12] =?UTF-8?q?=EB=AA=A8=EB=93=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ablestack-veeam/module.properties | 18 +++++++++++++ .../spring-backup-ablestack-veeam-context.xml | 26 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties create mode 100644 plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml diff --git a/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties new file mode 100644 index 000000000000..1700af0e6132 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=ablestack-veeam +parent=ablestack-nas diff --git a/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml new file mode 100644 index 000000000000..cbf9bbfb6f4e --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml @@ -0,0 +1,26 @@ + + + + + + + From 2b22a8bd731672eba204a562a8b85554ca6c623d Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Wed, 10 Jun 2026 10:24:58 +0900 Subject: [PATCH 08/12] Add backup name parameter to Ablestack Veeam API commands --- .../backup/CreateAblestackVeeamBackupCmd.java | 10 ++++++++++ .../ImportAblestackVeeamBackupSeedCmd.java | 10 ++++++++++ .../cloudstack/backup/BackupManagerImpl.java | 17 ++++++++++++++--- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java index b7110025b174..c1632bbb30ea 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java @@ -64,6 +64,12 @@ public class CreateAblestackVeeamBackupCmd extends BaseAsyncCreateCmd { description = "Quiesce VM via QEMU guest agent before backup") private Boolean quiesceVM; + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup (default: VM hostname + ISO-8601 timestamp)", + since = "4.22.0.0") + private String name; + public Long getVmId() { return vmId; } @@ -72,6 +78,10 @@ public Boolean getQuiesceVM() { return quiesceVM; } + public String getName() { + return name; + } + @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java index f26debbf72aa..90eda81f51ee 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java @@ -82,6 +82,12 @@ public class ImportAblestackVeeamBackupSeedCmd extends BaseAsyncCreateCmd { description = "Create libvirt checkpoint on the KVM VM after import. Default: true") private Boolean bootstrapCheckpoint; + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup (default: VM hostname + ISO-8601 timestamp)", + since = "4.22.0.0") + private String name; + public Long getVmId() { return vmId; } @@ -102,6 +108,10 @@ public Boolean getBootstrapCheckpoint() { return bootstrapCheckpoint; } + public String getName() { + return name; + } + @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 232fc33e9e5f..2c0dc5a8623d 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1051,11 +1051,17 @@ public Backup importAblestackVeeamBackupSeed(ImportAblestackVeeamBackupSeedCmd c throw new CloudRuntimeException("Failed to import Ablestack Veeam backup seed"); } + BackupVO backupVO = backupDao.findById(result.second().getId()); + if (cmd.getName() != null) { + backupVO.setName(cmd.getName()); + backupDao.update(backupVO.getId(), backupVO); + } + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); if (result.second().getSize() != null) { resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, result.second().getSize()); } - return result.second(); + return backupVO; } private BackupOffering validateVmAblestackVeeamOffering(final VMInstanceVO vm) { @@ -1120,12 +1126,14 @@ public boolean createAblestackVeeamBackup(final CreateAblestackVeeamBackupCmd cm backupSize += volumeSize; } } - createCheckedBackupVeeam(vm, vmId, backupProvider, cmd.getQuiesceVM(), backupSize, owner, getBackupScheduleId(job)); + createCheckedBackupVeeam(vm, vmId, backupProvider, cmd.getQuiesceVM(), backupSize, owner, getBackupScheduleId(job), + cmd.getName()); return true; } private void createCheckedBackupVeeam(final VMInstanceVO vm, final Long vmId, final BackupProvider backupProvider, - final Boolean quiesceVM, final Long backupSize, final Account owner, final Long backupScheduleId) + final Boolean quiesceVM, final Long backupSize, final Account owner, final Long backupScheduleId, + final String backupName) throws ResourceAllocationException { try (CheckedReservation backupReservation = new CheckedReservation(owner, Resource.ResourceType.backup, 1L, reservationDao, resourceLimitMgr); @@ -1144,6 +1152,9 @@ private void createCheckedBackupVeeam(final VMInstanceVO vm, final Long vmId, fi if (backup != null) { final BackupVO vmBackup = backupDao.findById(backup.getId()); vmBackup.setBackupScheduleId(backupScheduleId); + if (backupName != null) { + vmBackup.setName(backupName); + } backupDao.update(vmBackup.getId(), vmBackup); resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); From b52c533f34892c0430dad517cbd31e9db94f2405 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Tue, 30 Jun 2026 10:21:18 +0900 Subject: [PATCH 09/12] =?UTF-8?q?=ED=94=84=EB=A1=9C=EB=B0=94=EC=9D=B4?= =?UTF-8?q?=EB=8D=94=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backup/AblestackVeeamBackupProvider.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java index 5fbffae451cd..9695287a37d7 100644 --- a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java @@ -304,6 +304,20 @@ public void syncBackups(VirtualMachine vm) { nasBackupProvider.syncBackups(vm); } + /** + * Backups for this provider are NAS-managed (delegated to nasBackupProvider). + * Veeam restore points returned by {@link #listRestorePoints(VirtualMachine)} are + * a seed source only, not the authoritative backup list. The generic out-of-band + * sync reconciles DB backups against listRestorePoints() and DELETES any DB backup + * that has no matching Veeam restore point, which would wrongly wipe NAS/agent backups + * (e.g. when Veeam Enterprise Manager reports no restore points for the VM display name). + * Opt out so those backups are not destroyed; NAS sync is handled via syncBackups(vm). + */ + @Override + public boolean supportsOutOfBandBackupSync() { + return false; + } + @Override public boolean checkBackupAgent(Long zoneId) { return true; From a19727d061ea8168adf2a6368e22a51c0aaa43c2 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Tue, 30 Jun 2026 14:49:28 +0900 Subject: [PATCH 10/12] =?UTF-8?q?=EC=95=94=ED=98=B8=ED=99=94=ED=82=A4=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backup/AblestackNasBackupProvider.java | 22 +- .../ablestackveeam/AblestackVeeamClient.java | 56 ++++ .../AblestackVeeamRestClient.java | 247 ++++++++++++++++++ .../AblestackVeeamSshClient.java | 168 ++++++++++++ 4 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java create mode 100644 plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java diff --git a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java index a14b3c35915a..07369d9f4abe 100644 --- a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java +++ b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java @@ -534,31 +534,49 @@ private BackupVO getLatestBackedUpBackup(VirtualMachine vm) { private boolean shouldUseIncrementalBackup(VirtualMachine vm, Backup latestBackup, List vmVolumes, Long backupScheduleId) { if (latestBackup == null) { + LOG.debug("NAS backup for VM [{}] will be FULL: no previous BackedUp backup.", vm.getInstanceName()); + return false; + } + loadBackupDetailsIfNeeded(latestBackup); + + if (Boolean.parseBoolean(getBackupDetail(latestBackup, DETAIL_CHAIN_SEALED))) { + LOG.info("NAS backup for VM [{}] will be FULL: backup chain [{}] is sealed ({})", + vm.getInstanceName(), latestBackup.getUuid(), getBackupDetail(latestBackup, DETAIL_CHAIN_SEAL_REASON)); return false; } if (backupScheduleId != null && !hasBackedUpBackupForSchedule(backupScheduleId)) { + LOG.debug("NAS backup for VM [{}] will be FULL: no BackedUp backup for schedule [{}].", vm.getInstanceName(), backupScheduleId); return false; } final Long clusterId = getClusterIdFromRootVolume(vm); if (clusterId == null) { - LOG.debug("Unable to resolve cluster for VM [{}], fallback to full backup.", vm); + LOG.info("NAS backup for VM [{}] will be FULL: unable to resolve cluster from root volume.", vm.getInstanceName()); return false; } - if (!KvmIncrementalBackup.valueIn(clusterId)) { + if (!Boolean.TRUE.equals(KvmIncrementalBackup.valueIn(clusterId))) { + LOG.info("NAS backup for VM [{}] will be FULL: kvm.incremental.backup is disabled for cluster [{}] " + + "(set cluster configuration kvm.incremental.backup=true or run veeam_config.sh / pre-notify).", + vm.getInstanceName(), clusterId); return false; } if (!hasHealthyIncrementalSource(latestBackup)) { markVolumeFallbackAndSeal(latestBackup, "unhealthy-chain"); + LOG.info("NAS backup for VM [{}] will be FULL: latest backup [{}] has an unhealthy volume chain.", + vm.getInstanceName(), latestBackup.getUuid()); return false; } if (getBackupChainSize(vm, latestBackup) >= BackupChainSize.value()) { sealBackupChain(latestBackup, "chain-size-limit"); + LOG.info("NAS backup for VM [{}] will be FULL: incremental chain size limit reached for backup [{}].", + vm.getInstanceName(), latestBackup.getUuid()); return false; } + LOG.info("NAS incremental backup selected for VM [{}] using parent backup [{}].", + vm.getInstanceName(), latestBackup.getUuid()); return true; } diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java index 133de7134147..f1294a33ebfc 100644 --- a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java @@ -18,14 +18,17 @@ package org.apache.cloudstack.backup.ablestackveeam; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Date; import java.util.List; +import java.util.StringJoiner; import java.util.stream.Collectors; import org.apache.cloudstack.backup.Backup; @@ -40,12 +43,65 @@ */ public class AblestackVeeamClient extends VeeamClient { + /** + * PowerShell executable used on the Veeam server. Override with + * -Dveeam.powershell.bin=pwsh if the Veeam (v12.1+/v13) module requires + * PowerShell 7 instead of Windows PowerShell 5.1. + */ + private static final String POWERSHELL_BIN = System.getProperty("veeam.powershell.bin", "powershell"); + public AblestackVeeamClient(final String url, final Integer version, final String username, final String password, final boolean validateCertificate, final int timeout, final int restoreTimeout, final int taskPollInterval, final int taskPollMaxRetry) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { super(url, version, username, password, validateCertificate, timeout, restoreTimeout, taskPollInterval, taskPollMaxRetry); } + /** + * Build the single SSH command that runs the given PowerShell statements on the + * Veeam server. + * + *

The stock implementation joins commands with ';' and prefixes a bare + * {@code PowerShell ...} token, relying on the Windows SSH default shell + * (cmd.exe) to leave the rest untouched. That breaks for any statement that + * contains cmd.exe metacharacters - most notably the pipeline operator + * {@code |} (e.g. {@code Get-VBRRestorePoint | Where-Object ...}) and script + * blocks {@code { }}. cmd.exe intercepts those, which is why the logs show + * errors like {@code 'Where-Object' is not recognized as an internal or + * external command}.

+ * + *

Instead we assemble one PowerShell script and pass it via + * {@code -EncodedCommand} (Base64 of UTF-16LE). The encoded payload contains + * only Base64 characters, so cmd.exe cannot mangle it and the whole script + * runs inside a single PowerShell process.

+ */ + @Override + protected String transformPowerShellCommandList(final List cmds) { + // The Ablestack Veeam integration targets modern Veeam (v12+/v13) and uses + // Veeam.Backup.PowerShell cmdlets, so always use the module import (non-legacy + // PSSnapin) path. Keeping this independent of the parent's legacy detection. + final StringJoiner script = new StringJoiner("\n"); + script.add("Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue"); + script.add("$ProgressPreference='SilentlyContinue'"); + for (final String cmd : cmds) { + script.add(normalizeToPowerShell(cmd)); + } + final String encoded = Base64.getEncoder().encodeToString(script.toString().getBytes(StandardCharsets.UTF_16LE)); + return String.format("%s -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand %s", POWERSHELL_BIN, encoded); + } + + /** + * Some inherited command strings are pre-escaped for cmd.exe passthrough + * (e.g. {@code ^|} for the pipeline operator and {@code \"} for quotes). + * When the script is run via -EncodedCommand it is pure PowerShell, so those + * cmd escapes must be reverted to their native PowerShell form. + */ + private String normalizeToPowerShell(final String cmd) { + if (cmd == null) { + return ""; + } + return cmd.replace("^|", "|").replace("\\\"", "\""); + } + /** * Export all hard disks from a Veeam restore point to a directory on the Veeam server. * The directory must be reachable from the KVM hypervisor (shared NFS recommended). diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java new file mode 100644 index 000000000000..2703d95df9d2 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.ablestackveeam; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; + +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.nio.TrustAllManager; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Veeam Backup & Replication (VBR) REST API client (port 9419, v12+/v13). + * + *

Unlike the Enterprise Manager XML API (9398) used by {@link org.apache.cloudstack.backup.veeam.VeeamClient}, + * the VBR REST API is JSON based, uses OAuth 2.0 bearer tokens and is available on every + * VBR server without Enterprise Manager. This client covers the read/query operations that + * can be served over REST (restore point discovery), removing the dependency on + * PowerShell-over-SSH for those calls.

+ * + *

Note: there is no VBR REST endpoint that exports backup disk content to a filesystem + * path. Disk export for NAS seed creation/restore must still be performed via PowerShell + * ({@link AblestackVeeamClient#exportRestorePointDisksToStaging}).

+ */ +public class AblestackVeeamRestClient { + + private static final Logger LOG = LogManager.getLogger(AblestackVeeamRestClient.class); + + private final URI apiURI; + private final String apiVersion; + private final String username; + private final String password; + private final HttpClient httpClient; + + private String accessToken; + + public AblestackVeeamRestClient(final String url, final String apiVersion, final String username, + final String password, final boolean validateCertificate, final int timeoutSeconds) + throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + this.apiURI = new URI(StringUtils.appendIfMissing(url, "/")); + this.apiVersion = StringUtils.defaultIfBlank(apiVersion, "1.3-rev1"); + this.username = username; + this.password = password; + + final RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeoutSeconds * 1000) + .setConnectionRequestTimeout(timeoutSeconds * 1000) + .setSocketTimeout(timeoutSeconds * 1000) + .build(); + + if (!validateCertificate) { + final SSLContext sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, new X509TrustManager[]{new TrustAllManager()}, new SecureRandom()); + final SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + this.httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).setSSLSocketFactory(factory).build(); + } else { + this.httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); + } + + authenticate(); + } + + private String endpoint(final String relativePath) { + return apiURI.toString() + relativePath; + } + + private void authenticate() { + final HttpPost request = new HttpPost(endpoint("oauth2/token")); + request.setHeader("x-api-version", apiVersion); + request.setHeader(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded"); + request.setHeader(HttpHeaders.ACCEPT, "application/json"); + final String body = String.format("grant_type=password&username=%s&password=%s", + urlEncode(username), urlEncode(password)); + request.setEntity(new StringEntity(body, StandardCharsets.UTF_8)); + try { + final HttpResponse response = httpClient.execute(request); + final int code = response.getStatusLine().getStatusCode(); + final String payload = readBody(response); + if (code != HttpStatus.SC_OK) { + throw new CloudRuntimeException(String.format( + "Failed to authenticate to Veeam VBR REST API [%s]: HTTP %d %s", apiURI, code, payload)); + } + final JsonNode node = new ObjectMapper().readTree(payload); + this.accessToken = node.path("access_token").asText(null); + if (StringUtils.isBlank(accessToken)) { + throw new CloudRuntimeException("Veeam VBR REST API returned no access_token"); + } + } catch (IOException e) { + throw new CloudRuntimeException("Error authenticating to Veeam VBR REST API: " + e.getMessage(), e); + } + } + + /** + * List restore points for a VM (object) name, newest first. The returned restore-point + * id is the bare GUID (urn:uuid: prefix stripped) so it stays compatible with the + * PowerShell disk-export path that looks up {@code Get-VBRRestorePoint} by Id. + */ + public List listRestorePointsForVm(final String vmName) { + final String path = String.format("v1/objectRestorePoints?nameFilter=%s&orderColumn=CreationTime&orderAsc=false", + urlEncode(vmName)); + final JsonNode root = getJson(path); + final List points = new ArrayList<>(); + final JsonNode data = root.has("data") ? root.get("data") : root; + if (data == null || !data.isArray()) { + return points; + } + for (final JsonNode rp : data) { + final String rawId = rp.path("id").asText(null); + if (StringUtils.isBlank(rawId)) { + continue; + } + final String name = rp.path("name").asText(""); + // Restore points are returned for all objects matching the name filter; keep + // only those whose name actually corresponds to the requested VM. + if (StringUtils.isNotBlank(vmName) && StringUtils.isNotBlank(name) + && !name.toLowerCase().contains(vmName.toLowerCase())) { + continue; + } + final Date created = parseDate(rp.path("creationTime").asText(null)); + final String type = firstNonBlank(rp.path("type").asText(null), rp.path("pointType").asText(null), ""); + points.add(new Backup.RestorePoint(stripUrn(rawId), created, type, null, null)); + } + return points; + } + + private JsonNode getJson(final String relativePath) { + try { + JsonNode node = executeGet(relativePath); + return node; + } catch (TokenExpiredException e) { + authenticate(); + return executeGet(relativePath); + } + } + + private JsonNode executeGet(final String relativePath) { + final HttpGet request = new HttpGet(endpoint(relativePath)); + request.setHeader("x-api-version", apiVersion); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); + request.setHeader(HttpHeaders.ACCEPT, "application/json"); + try { + final HttpResponse response = httpClient.execute(request); + final int code = response.getStatusLine().getStatusCode(); + final String payload = readBody(response); + if (code == HttpStatus.SC_UNAUTHORIZED) { + throw new TokenExpiredException(); + } + if (code != HttpStatus.SC_OK) { + throw new CloudRuntimeException(String.format( + "Veeam VBR REST API GET [%s] failed: HTTP %d %s", relativePath, code, payload)); + } + return new ObjectMapper().readTree(payload); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Error calling Veeam VBR REST API [%s]: %s", relativePath, e.getMessage()), e); + } + } + + private static String readBody(final HttpResponse response) throws IOException { + final HttpEntity entity = response.getEntity(); + return entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8); + } + + private static String urlEncode(final String value) { + return URLEncoder.encode(value == null ? "" : value, StandardCharsets.UTF_8); + } + + private static String stripUrn(final String id) { + if (id == null) { + return null; + } + final int idx = id.lastIndexOf(':'); + return idx >= 0 ? id.substring(idx + 1) : id; + } + + private static String firstNonBlank(final String... values) { + for (final String v : values) { + if (StringUtils.isNotBlank(v)) { + return v; + } + } + return ""; + } + + private static Date parseDate(final String value) { + if (StringUtils.isBlank(value)) { + return null; + } + try { + return Date.from(Instant.parse(value)); + } catch (DateTimeParseException e) { + LOG.warn("Unable to parse Veeam restore point creationTime [{}]", value); + return null; + } + } + + private static final class TokenExpiredException extends RuntimeException { + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java new file mode 100644 index 000000000000..6e9a1a02b85f --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.ablestackveeam; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; + +/** + * Standalone SSH/PowerShell client for the Veeam server that performs the operations + * which have no Veeam REST API equivalent - notably exporting restore point disks to a + * staging directory for NAS seed creation/restore. + * + *

Unlike {@link org.apache.cloudstack.backup.veeam.VeeamClient} (and its subclass + * {@link AblestackVeeamClient}), this client does NOT authenticate against the Veeam + * Enterprise Manager REST API (port 9398). It only opens an SSH session (port 22) and + * runs PowerShell. That decouples disk export from Enterprise Manager so deployments + * using the VBR REST API (port 9419) for queries do not need EM/9398 at all.

+ * + *

PowerShell is delivered via {@code -EncodedCommand} (Base64 of UTF-16LE) so the + * Windows SSH default shell (cmd.exe) cannot mangle PowerShell metacharacters such as + * {@code |}, {@code { }} and {@code >}.

+ */ +public class AblestackVeeamSshClient { + + protected Logger logger = LogManager.getLogger(getClass()); + + /** + * Override with -Dveeam.powershell.bin=pwsh if the Veeam (v12.1+/v13) module + * requires PowerShell 7 instead of Windows PowerShell 5.1. + */ + private static final String POWERSHELL_BIN = System.getProperty("veeam.powershell.bin", "powershell"); + + private static final int SSH_PORT = 22; + private static final int CONNECT_TIMEOUT_MS = 120000; + private static final int KEX_TIMEOUT_MS = 120000; + private static final int WAIT_TIMEOUT_MS = 3600000; + + private final String host; + private final String username; + private final String password; + private final boolean legacy; + + public AblestackVeeamSshClient(final String host, final String username, final String password) { + this(host, username, password, false); + } + + public AblestackVeeamSshClient(final String host, final String username, final String password, final boolean legacy) { + if (StringUtils.isBlank(host)) { + throw new CloudRuntimeException("Veeam SSH host is required for disk export"); + } + this.host = host; + this.username = username; + this.password = password; + this.legacy = legacy; + } + + /** + * Export all hard disks from a Veeam restore point to a directory on the Veeam server. + * The directory must be reachable from the KVM hypervisor (shared NFS recommended). + * + * @return absolute paths of exported disk files on the Veeam server + */ + public List exportRestorePointDisksToStaging(final String restorePointId, final String stagingPath) { + logger.debug(String.format("Exporting Veeam restore point [%s] to staging [%s] via SSH", restorePointId, stagingPath)); + final String escapedStaging = stagingPath.replace("'", "''"); + final String escapedId = restorePointId.replace("'", "''"); + final List cmds = Arrays.asList( + String.format("$staging = '%s'", escapedStaging), + "New-Item -ItemType Directory -Force -Path $staging | Out-Null", + String.format("$restorePoint = Get-VBRRestorePoint | Where-Object { $_.Id -eq '%s' -or $_.Id.Guid -eq '%s' }", escapedId, escapedId), + "if (-not $restorePoint) { Write-Output 'Failed: restore point not found'; Exit 1 }", + "$session = Start-VBRFLRSession -RestorePoint $restorePoint", + "$items = Get-VBRFLRItem -Session $session", + "$exported = @()", + "foreach ($item in $items) {", + " if ($item.Type -eq 'HardDisk') {", + " $target = Join-Path $staging ($item.Name + '.vmdk')", + " Copy-VBRFLRItem -FLRSession $session -Item $item -Destination $target", + " $exported += $target", + " }", + "}", + "Stop-VBRFLRSession -Session $session", + "if ($exported.Count -eq 0) { Write-Output 'Failed: no disks exported'; Exit 1 }", + "$exported -join ','" + ); + final Pair result = executePowerShellCommands(cmds); + if (result == null || !result.first() || StringUtils.isBlank(result.second())) { + throw new CloudRuntimeException(String.format("Failed to export Veeam restore point [%s] to [%s]", restorePointId, stagingPath)); + } + if (result.second().contains("Failed:")) { + throw new CloudRuntimeException(result.second().trim()); + } + return Arrays.stream(result.second().trim().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + private Pair executePowerShellCommands(final List cmds) { + final String command = transformPowerShellCommandList(cmds); + try { + final Pair response = SshHelper.sshExecute(host, SSH_PORT, username, null, password, + command, CONNECT_TIMEOUT_MS, KEX_TIMEOUT_MS, WAIT_TIMEOUT_MS); + if (response == null || !response.first()) { + logger.error(String.format("Veeam SSH PowerShell command failed: [%s]", + response != null ? response.second() : "no output returned")); + } + return response; + } catch (Exception e) { + throw new CloudRuntimeException("Error while executing Veeam SSH PowerShell command: " + e.getMessage(), e); + } + } + + /** + * Build the single SSH command that runs the given PowerShell statements, passed via + * {@code -EncodedCommand} so cmd.exe cannot intercept PowerShell metacharacters. + */ + private String transformPowerShellCommandList(final List cmds) { + final StringJoiner script = new StringJoiner("\n"); + if (legacy) { + script.add("Add-PSSnapin VeeamPSSnapin"); + } else { + script.add("Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue"); + script.add("$ProgressPreference='SilentlyContinue'"); + } + final List all = new ArrayList<>(cmds); + for (final String cmd : all) { + script.add(normalizeToPowerShell(cmd)); + } + final String encoded = Base64.getEncoder().encodeToString(script.toString().getBytes(StandardCharsets.UTF_16LE)); + return String.format("%s -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand %s", POWERSHELL_BIN, encoded); + } + + private String normalizeToPowerShell(final String cmd) { + if (cmd == null) { + return ""; + } + return cmd.replace("^|", "|").replace("\\\"", "\""); + } +} From e2e07ddd1856564fac68f4ce166b88bd8d202fbb Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Tue, 21 Jul 2026 10:05:09 +0900 Subject: [PATCH 11/12] =?UTF-8?q?=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backup/AblestackVeeamBackupProvider.java | 143 +- .../resource/LibvirtComputingResource.java | 12 +- .../LibvirtAblestackNasBackupHelper.java | 20 +- ...lestackNasRestoreBackupCommandWrapper.java | 13 +- .../LibvirtBackupRepositoryMountHelper.java | 72 + .../LibvirtRestoreBackupCommandWrapper.java | 13 +- .../vm/hypervisor/kvm/ablestack_nasbackup.sh | 392 +- .../hypervisor/kvm/postinstall_mold_backup.sh | 26 + scripts/vm/hypervisor/kvm/setup_agent.sh | 18 + scripts/vm/hypervisor/kvm/veeam/README.ko.md | 80 + .../kvm/veeam/ablestack.key.default | 1 + .../kvm/veeam/ablestack_cvtbackup.sh | 485 ++ .../kvm/veeam/ablestack_veeam_post_notify.sh | 34 + .../kvm/veeam/ablestack_veeam_pre_notify.sh | 35 + .../veeam/ablestack_veeam_restore_event.sh | 68 + .../veeam/ablestack_veeam_restore_notify.sh | 32 + .../kvm/veeam/create-veeam-agent-job.ps1 | 627 +++ .../kvm/veeam/enable-veeam-mold-restore.sh | 124 + .../kvm/veeam/install-ablestack-secret-key.sh | 33 + .../kvm/veeam/install-veeam-job.ps1 | 645 +++ scripts/vm/hypervisor/kvm/veeam/install.sh | 168 + .../kvm/veeam/mold-backup-secret.sh | 151 + .../kvm/veeam/mold-backup.conf.default | 79 + .../kvm/veeam/mold-backup.env.example | 81 + .../hypervisor/kvm/veeam/mold-backup.lib.sh | 4389 +++++++++++++++++ .../vm/hypervisor/kvm/veeam/mold-backup.sh | 345 ++ .../veeam/mold-backup.windows.conf.default | 46 + .../hypervisor/kvm/veeam/mold-guest-common.sh | 417 ++ .../veeam/mold-veeam-restore-agent.service | 33 + .../kvm/veeam/mold-veeam-restore-agent.sh | 34 + .../kvm/veeam/mold-veeam-restore-agent.timer | 28 + .../kvm/veeam/mold-veeam-trigger-hook.sh | 63 + .../vm/hypervisor/kvm/veeam/push-to-kvm.sh | 117 + .../vm/hypervisor/kvm/veeam/push-to-veeam.sh | 271 + .../kvm/veeam/setup-datadisk-veeam-backup.sh | 174 + .../kvm/veeam/setup-veeam-host-repo.ps1 | 91 + .../kvm/veeam/setup-veeam-mold-job.ps1 | 165 + .../kvm/veeam/veeam-job-post-backup.ps1 | 67 + .../kvm/veeam/veeam-job-post-restore.ps1 | 59 + .../kvm/veeam/veeam-job-pre-backup.ps1 | 280 ++ .../vm/hypervisor/kvm/veeam/veeam_config.sh | 620 +++ 41 files changed, 10500 insertions(+), 51 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java create mode 100755 scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/README.ko.md create mode 100644 scripts/vm/hypervisor/kvm/veeam/ablestack.key.default create mode 100644 scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 create mode 100644 scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/install-ablestack-secret-key.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 create mode 100755 scripts/vm/hypervisor/kvm/veeam/install.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example create mode 100755 scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/mold-backup.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer create mode 100755 scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh create mode 100755 scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh create mode 100644 scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 create mode 100644 scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 create mode 100644 scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 create mode 100644 scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 create mode 100644 scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 create mode 100755 scripts/vm/hypervisor/kvm/veeam/veeam_config.sh diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java index 9695287a37d7..f271717d92fc 100644 --- a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.backup; +import java.net.URI; import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; @@ -26,6 +27,8 @@ import javax.inject.Inject; import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamClient; +import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamRestClient; +import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamSshClient; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.commons.collections4.CollectionUtils; @@ -89,6 +92,22 @@ public class AblestackVeeamBackupProvider extends AdapterBase implements BackupP "Shared staging directory on the Veeam server (must be visible to KVM hosts) for disk export before NAS seed import.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + public ConfigKey AblestackVeeamUseRestApi = new ConfigKey<>("Advanced", Boolean.class, + "backup.plugin.ablestack-veeam.use.rest.api", "false", + "Use the Veeam Backup & Replication REST API (port 9419) for restore point discovery instead of " + + "PowerShell-over-SSH. Disk export for seed/restore still uses PowerShell (no REST equivalent).", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamRestUrl = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.rest.url", "https://localhost:9419/api/", + "Veeam Backup & Replication REST API base URL (port 9419), used when use.rest.api is true.", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamRestApiVersion = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.rest.api.version", "1.3-rev1", + "Veeam Backup & Replication REST API version header (e.g. 1.3-rev1 for v13.0.1).", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + @Inject private AblestackNasBackupProvider nasBackupProvider; @@ -114,6 +133,47 @@ protected AblestackVeeamClient getClient(final Long zoneId) { } } + protected AblestackVeeamRestClient getRestClient(final Long zoneId) { + try { + return new AblestackVeeamRestClient( + AblestackVeeamRestUrl.valueIn(zoneId), + AblestackVeeamRestApiVersion.valueIn(zoneId), + AblestackVeeamUsername.valueIn(zoneId), + AblestackVeeamPassword.valueIn(zoneId), + AblestackVeeamValidateSsl.valueIn(zoneId), + AblestackVeeamApiTimeout.valueIn(zoneId)); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Ablestack Veeam REST API URL: " + e.getMessage()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new CloudRuntimeException("Failed to build Ablestack Veeam REST client: " + e.getMessage()); + } + } + + /** + * SSH/PowerShell client for disk export. Intentionally does NOT use Enterprise Manager + * (9398) so that REST-API (9419) deployments do not depend on EM at all. The Veeam SSH + * host is derived from the REST URL (falling back to the EM URL); credentials reuse the + * configured Veeam username/password (same as the EM/SSH credentials). + */ + protected AblestackVeeamSshClient getSshClient(final Long zoneId) { + String host = extractHost(AblestackVeeamRestUrl.valueIn(zoneId)); + if (StringUtils.isBlank(host)) { + host = extractHost(AblestackVeeamUrl.valueIn(zoneId)); + } + return new AblestackVeeamSshClient(host, AblestackVeeamUsername.valueIn(zoneId), AblestackVeeamPassword.valueIn(zoneId)); + } + + private String extractHost(final String url) { + if (StringUtils.isBlank(url)) { + return null; + } + try { + return new URI(url).getHost(); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Veeam URL for SSH host: " + e.getMessage()); + } + } + @Override public String getName() { return PROVIDER_NAME; @@ -160,7 +220,7 @@ public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); logger.info("No NAS seed found for VM [{}], exporting Veeam restore point [{}] to [{}]", vm.getInstanceName(), restorePointId, stagingPath); - final List stagingDisks = getClient(vm.getDataCenterId()) + final List stagingDisks = getSshClient(vm.getDataCenterId()) .exportRestorePointDisksToStaging(restorePointId, stagingPath); final Pair seedResult = nasBackupProvider.importVeeamBackupSeed( vm, stagingDisks, restorePointId, "vmdk", true); @@ -192,7 +252,7 @@ public Pair importAblestackVeeamBackupSeed(VirtualMachine vm, S } final String stagingSubDir = String.format("%s/%s-import", vm.getInstanceName(), System.currentTimeMillis()); final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); - staging = getClient(vm.getDataCenterId()).exportRestorePointDisksToStaging(veeamRestorePointId, stagingPath); + staging = getSshClient(vm.getDataCenterId()).exportRestorePointDisksToStaging(veeamRestorePointId, stagingPath); } final Pair result = nasBackupProvider.importVeeamBackupSeed( vm, staging, veeamRestorePointId, sourceFormat, bootstrapCheckpoint); @@ -239,17 +299,84 @@ public boolean deleteBackup(Backup backup, boolean forced) { @Override public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return nasBackupProvider.restoreBackupToVM(vm, backup, hostIp, dataStoreUuid); + try { + final Pair local = nasBackupProvider.restoreBackupToVM(vm, backup, hostIp, dataStoreUuid); + if (local != null && Boolean.TRUE.equals(local.first())) { + return local; + } + logger.warn("Local NAS restore did not succeed for backup [{}]; attempting Veeam chain fallback.", backup.getUuid()); + } catch (Exception e) { + logger.warn(String.format("Local NAS restore failed for backup [%s]; attempting Veeam chain fallback: %s", + backup.getUuid(), e.getMessage())); + } + final Backup seed = rebuildBackupFromVeeam(vm, backup); + return nasBackupProvider.restoreBackupToVM(vm, seed, hostIp, dataStoreUuid); } @Override public Pair restoreBackupToVM(Long backupId, String vmName) { + // This entry point only carries ids; the Veeam fallback needs the full VM + // and Backup objects (restore-point detail, zone, staging path). The + // object-based overloads above provide the Veeam chain fallback. return nasBackupProvider.restoreBackupToVM(backupId, vmName); } @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - return nasBackupProvider.restoreVMFromBackup(vm, backup); + try { + if (nasBackupProvider.restoreVMFromBackup(vm, backup)) { + return true; + } + logger.warn("Local NAS restore did not succeed for backup [{}]; attempting Veeam chain fallback.", backup.getUuid()); + } catch (Exception e) { + logger.warn(String.format("Local NAS restore failed for backup [%s]; attempting Veeam chain fallback: %s", + backup.getUuid(), e.getMessage())); + } + final Backup seed = rebuildBackupFromVeeam(vm, backup); + return nasBackupProvider.restoreVMFromBackup(vm, seed); + } + + /** + * Veeam chain restore fallback used when the local qcow2 chain cannot satisfy a + * restore (e.g. the chain is incomplete because the authoritative data lives in + * Veeam). We re-export the exact Veeam restore point recorded on the backup, + * re-import it as a fresh NAS seed (a self-contained full point that does not + * depend on the local chain) and let the caller restore from that seed. + * + *

The happy path (local restore succeeds) never reaches here, so this adds no + * regression risk to working local restores.

+ */ + private Backup rebuildBackupFromVeeam(VirtualMachine vm, Backup backup) { + final String restorePointId = getVeeamRestorePointId(backup); + if (StringUtils.isBlank(restorePointId)) { + throw new CloudRuntimeException(String.format( + "Local restore failed for backup [%s] and no Veeam restore point id is recorded; cannot fall back to Veeam.", + backup.getUuid())); + } + logger.info("Rebuilding backup [{}] from Veeam restore point [{}] for restore.", backup.getUuid(), restorePointId); + final String stagingSubDir = String.format("restore-%s/%s", vm.getInstanceName(), System.currentTimeMillis()); + final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); + final List stagingDisks = getSshClient(vm.getDataCenterId()) + .exportRestorePointDisksToStaging(restorePointId, stagingPath); + final Pair seed = nasBackupProvider.importVeeamBackupSeed( + vm, stagingDisks, restorePointId, "vmdk", true); + if (seed == null || !Boolean.TRUE.equals(seed.first()) || seed.second() == null) { + throw new CloudRuntimeException(String.format( + "Failed to rebuild backup [%s] from Veeam restore point [%s].", backup.getUuid(), restorePointId)); + } + return seed.second(); + } + + private String getVeeamRestorePointId(Backup backup) { + final Map details = backup != null ? backup.getDetails() : null; + if (details == null) { + return null; + } + final String fromProvider = details.get(DETAIL_VEEAM_RESTORE_POINT_ID); + if (StringUtils.isNotBlank(fromProvider)) { + return fromProvider; + } + return details.get(AblestackNasBackupProvider.DETAIL_VEEAM_RESTORE_POINT_ID); } @Override @@ -266,6 +393,9 @@ public void syncBackupMetrics(Long zoneId) { @Override public List listRestorePoints(VirtualMachine vm) { final String veeamVmName = getVeeamSourceVmName(vm); + if (Boolean.TRUE.equals(AblestackVeeamUseRestApi.valueIn(vm.getDataCenterId()))) { + return getRestClient(vm.getDataCenterId()).listRestorePointsForVm(veeamVmName); + } final AblestackVeeamClient client = getClient(vm.getDataCenterId()); client.syncBackupRepository(); return client.listRestorePointsForVmDisplayName(veeamVmName); @@ -355,7 +485,10 @@ public ConfigKey[] getConfigKeys() { AblestackVeeamRestoreTimeout, AblestackVeeamTaskPollInterval, AblestackVeeamTaskPollMaxRetry, - AblestackVeeamStagingPath + AblestackVeeamStagingPath, + AblestackVeeamUseRestApi, + AblestackVeeamRestUrl, + AblestackVeeamRestApiVersion }; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 170d0d7d812e..6154382e039e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -390,7 +390,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win"; public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; - public static final String VDDK_AUTODETECT_PATH_CMD = "find / -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; + public static final String VDDK_AUTODETECT_PATH_CMD = + "find /opt /usr /usr/local -maxdepth 5 -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; + private static final int VDDK_AUTODETECT_TIMEOUT_SECONDS = 15; public static final int LIBVIRT_CGROUP_CPU_SHARES_MIN = 2; public static final int LIBVIRT_CGROUP_CPU_SHARES_MAX = 262144; @@ -6854,10 +6856,7 @@ public boolean hostSupportsVddk(String overriddenVddkLibDir) { if (StringUtils.isBlank(effectiveVddkLibDir)) { effectiveVddkLibDir = StringUtils.trimToNull(vddkLibDir); } - if (StringUtils.isBlank(effectiveVddkLibDir) || !isVddkLibDirValid(effectiveVddkLibDir)) { - effectiveVddkLibDir = detectVddkLibDir(); - } - return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(detectVddkVersion()); + return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(vddkVersion); } protected boolean isVddkLibDirValid(String path) { @@ -6873,7 +6872,8 @@ protected boolean isVddkLibDirValid(String path) { } protected String detectVddkLibDir() { - String detectedPath = StringUtils.trimToNull(Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD)); + String detectedPath = StringUtils.trimToNull( + Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD, VDDK_AUTODETECT_TIMEOUT_SECONDS)); if (StringUtils.isNotBlank(detectedPath) && isVddkLibDirValid(detectedPath)) { return detectedPath; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java index f13db4556f8a..97952e5ebc26 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java @@ -27,6 +27,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.storage.Storage; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import org.apache.cloudstack.backup.AblestackNasImportVeeamSeedCommand; import org.apache.cloudstack.backup.AblestackNasTakeBackupCommand; @@ -297,14 +298,17 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma private Path mountRepository(AblestackNasTakeBackupCommand command) throws IOException { Path mountPoint = Files.createTempDirectory("csbackup."); - StringBuilder mount = new StringBuilder() - .append("mount -t ").append(shellQuote(command.getBackupRepoType())) - .append(" ").append(shellQuote(command.getBackupRepoAddress())) - .append(" ").append(shellQuote(mountPoint.toString())); - if (command.getMountOptions() != null && !command.getMountOptions().isEmpty()) { - mount.append(" -o ").append(shellQuote(command.getMountOptions())); - } - if (Script.runSimpleBashScriptForExitValue(mount.toString(), resource.getCmdsTimeout(), false) != 0) { + final String mount; + try { + mount = LibvirtBackupRepositoryMountHelper.buildMountCommand( + command.getBackupRepoAddress(), + command.getBackupRepoType(), + command.getMountOptions(), + mountPoint.toString()); + } catch (CloudRuntimeException e) { + throw new IOException(e.getMessage(), e); + } + if (Script.runSimpleBashScriptForExitValue(mount, resource.getCmdsTimeout(), false) != 0) { throw new IOException("Failed to mount backup repository"); } return mountPoint; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index fd2108638ccf..47e03c60c04d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -227,17 +227,8 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } - String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); - if ("cifs".equals(backupRepoType)) { - if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) { - mountOptions = "nobrl"; - } else { - mountOptions += ",nobrl"; - } - } - if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) { - mount += " -o " + mountOptions; - } + final String mount = LibvirtBackupRepositoryMountHelper.buildMountCommand( + backupRepoAddress, backupRepoType, mountOptions, mountDirectory); int exitValue = Script.runSimpleBashScriptForExitValue(mount, mountTimeout, false); if (exitValue != 0) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java new file mode 100644 index 000000000000..8cebb62221d8 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java @@ -0,0 +1,72 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Locale; +import java.util.Objects; + +/** + * Mount helpers for backup repositories on KVM hosts. + * Local repositories use bind mounts (see ablestack_nasbackup.sh mount_operation). + */ +public final class LibvirtBackupRepositoryMountHelper { + + private static final String MOUNT_COMMAND = "sudo mount -t %s %s %s"; + + private LibvirtBackupRepositoryMountHelper() { + } + + public static boolean isLocalBackupRepositoryType(final String backupRepoType) { + if (StringUtils.isBlank(backupRepoType)) { + return false; + } + final String normalized = backupRepoType.toLowerCase(Locale.ROOT); + return "local".equals(normalized) || "dir".equals(normalized) || "localfs".equals(normalized); + } + + public static String buildMountCommand(final String backupRepoAddress, final String backupRepoType, + String mountOptions, final String mountDirectory) { + if (isLocalBackupRepositoryType(backupRepoType)) { + if (!Files.isDirectory(Paths.get(backupRepoAddress))) { + throw new CloudRuntimeException(String.format( + "Local backup directory does not exist on the KVM host: %s", backupRepoAddress)); + } + return String.format("sudo mount --bind %s %s", backupRepoAddress, mountDirectory); + } + + String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); + if ("cifs".equals(backupRepoType)) { + if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) { + mountOptions = "nobrl"; + } else { + mountOptions += ",nobrl"; + } + } + if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) { + mount += " -o " + mountOptions; + } + return mount; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 78c5346c8335..e0951c375de3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -205,17 +205,8 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } - String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); - if ("cifs".equals(backupRepoType)) { - if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) { - mountOptions = "nobrl"; - } else { - mountOptions += ",nobrl"; - } - } - if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) { - mount += " -o " + mountOptions; - } + final String mount = LibvirtBackupRepositoryMountHelper.buildMountCommand( + backupRepoAddress, backupRepoType, mountOptions, mountDirectory); int exitValue = Script.runSimpleBashScriptForExitValue(mount, mountTimeout, false); if (exitValue != 0) { diff --git a/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh b/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh index 3fa7985c6fb2..863598d012e6 100755 --- a/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh @@ -38,6 +38,10 @@ PARENT_CHECKPOINT_NAME="" PARENT_CHECKPOINT_PATH="" BACKUP_FILES="" DISK_PATHS="" +STAGING_DISK_PATHS="" +SOURCE_FORMAT="vmdk" +VEEAM_RESTORE_POINT_ID="" +BOOTSTRAP_CHECKPOINT="true" QUIESCE="" FORCED="false" logFile="/var/log/cloudstack/agent/agent.log" @@ -100,7 +104,7 @@ backup_running_vm() { local parent_checkpoint_file="" if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_PATH" ]]; then parent_checkpoint_file="$mount_point/$PARENT_CHECKPOINT_PATH" - redefine_checkpoint_if_needed "$VM" "$parent_checkpoint_file" + redefine_checkpoint_if_needed "$VM" "$parent_checkpoint_file" "$mount_point" fi echo "" > "$dest/backup.xml" @@ -129,9 +133,12 @@ backup_running_vm() { fi # Start push backup - local backup_begin=0 - if virsh -c qemu:///system backup-begin --domain "$VM" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" 2>&1 > /dev/null; then - backup_begin=1; + local backup_begin=0 backup_begin_err="" + if backup_begin_err=$(virsh -c qemu:///system backup-begin --domain "$VM" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" 2>&1); then + backup_begin=1 + else + echo "backup-begin failed for VM $VM: $backup_begin_err" >> "$logFile" + echo "backup-begin failed for VM $VM: $backup_begin_err" fi if [[ $thaw -eq 1 ]]; then @@ -373,6 +380,27 @@ get_backup_stats() { mount_operation() { mount_point=$(mktemp -d -t csbackup.XXXXX) dest="$mount_point/${BACKUP_DIR}" + + # Local data disk: NAS_ADDRESS is a directory already mounted on this host + # (e.g. a dedicated data disk). Bind-mount it so the rest of the flow + # (dest/umount/df) keeps working without a network mount. + case "${NAS_TYPE}" in + local|dir|localfs) + if [[ ! -d "${NAS_ADDRESS}" ]]; then + echo "Local backup directory does not exist: ${NAS_ADDRESS}" + exit 1 + fi + mount --bind "${NAS_ADDRESS}" "${mount_point}" 2>&1 | tee -a "$logFile" + if [ ${PIPESTATUS[0]} -eq 0 ]; then + log -ne "Successfully bind-mounted local backup dir ${NAS_ADDRESS}" + else + echo "Failed to bind-mount local backup dir ${NAS_ADDRESS}" + exit 1 + fi + return 0 + ;; + esac + if [ ${NAS_TYPE} == "cifs" ]; then MOUNT_OPTS="${MOUNT_OPTS},nobrl" fi @@ -400,6 +428,7 @@ cleanup() { echo "Backup cleanup failed" exit $EXIT_CLEANUP_FAILED fi + exit 1 } split_csv() { @@ -436,19 +465,89 @@ dump_checkpoint_xml() { fi } +strip_checkpoint_parent_from_xml() { + local xml_file="$1" + [[ -f "$xml_file" ]] || return 0 + python3 - "$xml_file" <<'PY' 2>/dev/null || true +import sys +import xml.etree.ElementTree as ET + +path = sys.argv[1] +tree = ET.parse(path) +root = tree.getroot() +for parent in list(root.findall('parent')): + root.remove(parent) +tree.write(path, encoding='unicode', xml_declaration=True) +PY +} + +get_parent_checkpoint_name_from_xml() { + local xml_file="$1" + [[ -f "$xml_file" ]] || return 0 + python3 - "$xml_file" <<'PY' 2>/dev/null +import sys +import xml.etree.ElementTree as ET + +try: + root = ET.parse(sys.argv[1]).getroot() + parent = root.find('parent') + if parent is None: + sys.exit(0) + name = parent.findtext('name', '').strip() + if name: + print(name) +except Exception: + pass +PY +} + +find_checkpoint_xml_on_nas() { + local search_root="$1" checkpoint_name="$2" + [[ -n "$search_root" && -n "$checkpoint_name" && -d "$search_root" ]] || return 1 + find "$search_root" -maxdepth 5 -type f -name "${checkpoint_name}.xml" 2>/dev/null | head -1 +} + +redefine_checkpoint_chain_if_needed() { + local vm_name="$1" checkpoint_file="$2" search_root="$3" + local checkpoint_name parent_name parent_file visited_key + + [[ -n "$checkpoint_file" && -f "$checkpoint_file" ]] || return 0 + checkpoint_name="$(basename "$checkpoint_file" .xml)" + visited_key="|${checkpoint_name}|" + if [[ "${REDEFINE_VISITED_CHECKPOINTS:-}" == *"$visited_key"* ]]; then + return 0 + fi + REDEFINE_VISITED_CHECKPOINTS="${REDEFINE_VISITED_CHECKPOINTS:-}${visited_key}" + + parent_name="$(get_parent_checkpoint_name_from_xml "$checkpoint_file")" + if [[ -n "$parent_name" ]]; then + parent_file="$(find_checkpoint_xml_on_nas "$search_root" "$parent_name")" + if [[ -n "$parent_file" && -f "$parent_file" ]]; then + redefine_checkpoint_chain_if_needed "$vm_name" "$parent_file" "$search_root" + else + strip_checkpoint_parent_from_xml "$checkpoint_file" + fi + fi + + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$checkpoint_name" > /dev/null 2>&1; then + return 0 + fi + if ! virsh -c qemu:///system checkpoint-create --domain "$vm_name" --xmlfile "$checkpoint_file" --redefine >> "$logFile" 2>&1; then + echo "Failed to redefine checkpoint ${checkpoint_name} on domain ${vm_name}" + cleanup + fi +} + redefine_checkpoint_if_needed() { - local vm_name="$1" - local checkpoint_file="$2" + local vm_name="$1" checkpoint_file="$2" search_root="${3:-}" if [[ -z "$PARENT_CHECKPOINT_NAME" || -z "$checkpoint_file" || ! -f "$checkpoint_file" ]]; then return fi if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$PARENT_CHECKPOINT_NAME" > /dev/null 2>&1; then return fi - if ! virsh -c qemu:///system checkpoint-create --domain "$vm_name" --xmlfile "$checkpoint_file" --redefine > /dev/null 2>&1; then - echo "Failed to redefine checkpoint $PARENT_CHECKPOINT_NAME on domain $vm_name" - cleanup - fi + REDEFINE_VISITED_CHECKPOINTS="" + redefine_checkpoint_chain_if_needed "$vm_name" "$checkpoint_file" "${search_root:-$mount_point}" } parse_rbd_uri() { @@ -523,6 +622,251 @@ EOF log -ne "Wrote RBD backup metadata to [$dest/rbd-backup.meta]" } +write_veeam_seed_metadata() { + local backup_engine="$1" + cat > "$dest/veeam-seed.meta" <> "$logFile" 2>&1; then + echo "Failed to convert staging disk $staging_path to $output" + cleanup + fi + ;; + *) + echo "Unsupported source format: $SOURCE_FORMAT" + cleanup + ;; + esac +} + +import_rbd_seed_disk() { + local staging_path="$1" + local output="$2" + local disk_uri="$3" + + if ! qemu-img convert -p -O raw "$staging_path" "$output" >> "$logFile" 2>&1; then + echo "Failed to convert staging disk $staging_path to $output" + cleanup + fi + + parse_rbd_uri "$disk_uri" + build_rbd_cmd + if [[ -z "$RBD_IMAGE" ]]; then + echo "Unable to parse RBD disk path for seed import: $disk_uri" + cleanup + fi + + if ! timeout 30s "${RBD_CMD[@]}" snap ls "$RBD_IMAGE" 2>>"$logFile" | awk 'NR>1 {print $2}' | grep -Fxq "$CHECKPOINT_NAME"; then + if ! timeout 30s "${RBD_CMD[@]}" snap create "${RBD_IMAGE}@${CHECKPOINT_NAME}" >> "$logFile" 2>&1; then + echo "Failed to create RBD baseline snapshot ${RBD_IMAGE}@${CHECKPOINT_NAME}" + cleanup + fi + fi +} + +bootstrap_qcow2_checkpoint() { + local vm_name="$1" + + if [[ -z "$vm_name" ]]; then + log -ne "Skip checkpoint bootstrap: VM name not set" + return 0 + fi + + if ! virsh -c qemu:///system dominfo "$vm_name" > /dev/null 2>&1; then + log -ne "Skip checkpoint bootstrap: VM [$vm_name] not found in libvirt" + return 0 + fi + + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$CHECKPOINT_NAME" > /dev/null 2>&1; then + dump_checkpoint_xml "$vm_name" + return 0 + fi + + echo "" > "$dest/backup.xml" + echo "" >> "$dest/backup.xml" + echo "$CHECKPOINT_NAME" > "$dest/checkpoint.xml" + local index=0 + while IFS='|' read -r disk target; do + [[ -z "$disk" ]] && continue + local backup_file + backup_file=$(get_backup_file_by_index "$index" "$(basename "$target").qcow2") + echo "" >> "$dest/backup.xml" + echo "" >> "$dest/checkpoint.xml" + index=$((index + 1)) + done < <(virsh -c qemu:///system domblklist "$vm_name" --details 2>/dev/null | awk '/disk/ {print $3 "|" $4}') + echo "" >> "$dest/backup.xml" + echo "" >> "$dest/checkpoint.xml" + + if ! virsh -c qemu:///system backup-begin --domain "$vm_name" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" >> "$logFile" 2>&1; then + echo "Failed to bootstrap checkpoint on VM $vm_name" + cleanup + fi + + while true; do + local status + status=$(virsh -c qemu:///system domjobinfo "$vm_name" --completed --keep-completed 2>/dev/null | awk '/Job type:/ {print $3}') + case "$status" in + Completed) break ;; + Failed) + echo "Virsh checkpoint bootstrap job failed for VM $vm_name" + cleanup ;; + esac + sleep 5 + done + + dump_checkpoint_xml "$vm_name" + rm -f "$dest/backup.xml" "$dest/checkpoint.xml" + log -ne "Bootstrapped libvirt checkpoint [$CHECKPOINT_NAME] on VM [$vm_name]" +} + +# Veeam seed: NAS qcow2 already exists from staging convert — only create live-disk bitmap checkpoint. +bootstrap_qcow2_checkpoint_seed() { + local vm_name="$1" + local -a diskspec_args=() + local disk + + if [[ -z "$vm_name" ]]; then + log -ne "Skip checkpoint bootstrap: VM name not set" + return 0 + fi + + if ! virsh -c qemu:///system dominfo "$vm_name" > /dev/null 2>&1; then + log -ne "Skip checkpoint bootstrap: VM [$vm_name] not found in libvirt" + return 0 + fi + + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$CHECKPOINT_NAME" > /dev/null 2>&1; then + dump_checkpoint_xml "$vm_name" + return 0 + fi + + while IFS='|' read -r disk _target; do + [[ -z "$disk" ]] && continue + diskspec_args+=(--diskspec "${disk},bitmap=${CHECKPOINT_NAME}") + done < <(virsh -c qemu:///system domblklist "$vm_name" --details 2>/dev/null | awk '/disk/ {print $3 "|" $4}') + + if [[ ${#diskspec_args[@]} -eq 0 ]]; then + echo "No disks found for checkpoint bootstrap on VM $vm_name" + cleanup + fi + + if ! virsh -c qemu:///system checkpoint-create-as "$vm_name" "$CHECKPOINT_NAME" \ + "${diskspec_args[@]}" >> "$logFile" 2>&1; then + log -ne "Warn: checkpoint-create-as failed for seed on VM [$vm_name] (glue-gfs/raw may not support bitmap); continuing without checkpoint" + return 0 + fi + + dump_checkpoint_xml "$vm_name" + strip_checkpoint_parent_from_xml "$dest/checkpoints/$CHECKPOINT_NAME.xml" + log -ne "Bootstrapped libvirt checkpoint (seed) [$CHECKPOINT_NAME] on VM [$vm_name]" +} + +import_veeam_seed() { + log -ne "Entered import_veeam_seed staging=[$STAGING_DISK_PATHS] backupDir=[$BACKUP_DIR]" + mount_operation + mkdir -p "$dest" "$dest/checkpoints" || { echo "Failed to create backup directory $dest"; exit 1; } + + if [[ -z "$STAGING_DISK_PATHS" ]]; then + echo "Staging disk paths are required for import-veeam-seed" + cleanup + fi + + local use_rbd=0 + if [[ -n "$DISK_PATHS" ]]; then + while IFS= read -r disk_path; do + [[ -z "$disk_path" ]] && continue + if is_rbd_disk_path "$disk_path"; then + use_rbd=1 + break + fi + done < <(split_csv "$DISK_PATHS") + fi + + local backup_engine="QCOW2" + [[ $use_rbd -eq 1 ]] && backup_engine="RBD_DIFF" + + local index=0 + local staging_index=0 + while IFS= read -r staging_disk; do + [[ -z "$staging_disk" ]] && continue + local backup_file live_disk="" + if [[ -n "$DISK_PATHS" ]]; then + live_disk=$(split_csv "$DISK_PATHS" | sed -n "$((staging_index + 1))p") + fi + if [[ $use_rbd -eq 1 && -n "$live_disk" ]]; then + backup_file=$(get_backup_file_by_index "$index" "${live_disk##*/}.raw") + else + backup_file=$(get_backup_file_by_index "$index" "disk-${index}.qcow2") + fi + local output="$dest/$backup_file" + if [[ $use_rbd -eq 1 ]]; then + import_rbd_seed_disk "$staging_disk" "$output" "$live_disk" + else + convert_staging_disk_to_backup "$staging_disk" "$output" + fi + stat -c %s "$output" + index=$((index + 1)) + staging_index=$((staging_index + 1)) + done < <(split_csv "$STAGING_DISK_PATHS") + + backup_domain_information "$VM" + + if [[ "$backup_engine" == "RBD_DIFF" ]]; then + write_rbd_backup_metadata "FULL" "$CHECKPOINT_NAME" "" + cat > "$dest/checkpoints/${CHECKPOINT_NAME}.meta" < -v|--vm -t -s -m -p -b -c -r -i -j -f -d -q|--quiesce -x|--forced " @@ -607,6 +951,26 @@ while [[ $# -gt 0 ]]; do shift shift ;; + --staging-disks) + STAGING_DISK_PATHS="$2" + shift + shift + ;; + --source-format) + SOURCE_FORMAT="$2" + shift + shift + ;; + --veeam-restore-point) + VEEAM_RESTORE_POINT_ID="$2" + shift + shift + ;; + --bootstrap-checkpoint) + BOOTSTRAP_CHECKPOINT="$2" + shift + shift + ;; -h|--help) usage shift @@ -631,4 +995,12 @@ elif [ "$OP" = "delete" ]; then delete_backup elif [ "$OP" = "stats" ]; then get_backup_stats +elif [ "$OP" = "import-veeam-seed" ]; then + import_veeam_seed +fi + +# Optional Mold->Veeam trigger (bidirectional mode C). Best-effort: never affects backup result. +VEEAM_TRIGGER_HOOK="${VEEAM_TRIGGER_HOOK:-/etc/ablestack/veeam/mold-veeam-trigger-hook.sh}" +if [[ -x "$VEEAM_TRIGGER_HOOK" ]]; then + "$VEEAM_TRIGGER_HOOK" "$OP" "$VM" "$BACKUP_TYPE" >/dev/null 2>&1 || true fi diff --git a/scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh b/scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh new file mode 100755 index 000000000000..c90dbb2b516f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Entry point for mold-agent or cloudstack-common RPM %post. +set -e +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -x "${DIR}/veeam/install.sh" ]]; then + "${DIR}/veeam/install.sh" +fi +# veeam/install.sh deploys /root/.ssh/ablestack.key from ablestack.key.default when missing (chmod 600). +# mold-backup hooks decrypt MOLD_API_SECRET via mold-backup-secret.sh + MOLD_SECRET_KEY_FILE. diff --git a/scripts/vm/hypervisor/kvm/setup_agent.sh b/scripts/vm/hypervisor/kvm/setup_agent.sh index 761f1109830f..a3de9691c821 100755 --- a/scripts/vm/hypervisor/kvm/setup_agent.sh +++ b/scripts/vm/hypervisor/kvm/setup_agent.sh @@ -56,6 +56,7 @@ install_cloud_agent() { then let retry=retry-1 else + install_mold_veeam_backup_hooks break fi done @@ -69,6 +70,7 @@ install_cloud_agent() { then let retry=retry-1 else + install_mold_veeam_backup_hooks break fi @@ -82,6 +84,22 @@ install_cloud_agent() { fi } +# Install Ablestack Veeam backup pre/post hooks (idempotent). +install_mold_veeam_backup_hooks() { + local candidates=" + /usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/veeam/install.sh + $(dirname "$0")/veeam/install.sh + " + local installer + for installer in $candidates; do + if [ -x "$installer" ]; then + "$installer" && return 0 + fi + done + printf "Note: mold Veeam backup installer not found (skip)\n" + return 0 +} + install_cloud_consoleP() { local dev=$1 local retry=10 diff --git a/scripts/vm/hypervisor/kvm/veeam/README.ko.md b/scripts/vm/hypervisor/kvm/veeam/README.ko.md new file mode 100644 index 000000000000..28c9cf65d0b7 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/README.ko.md @@ -0,0 +1,80 @@ +# Ablestack Veeam + Mold 백업 (호스트 / datadisk) + +## 핵심 + +| 항목 | 값 | +|------|-----| +| 모드 | **host** — KVM Agent + `/tmp/mold/veeam` (게스트별 Veeam Job 없음) | +| Mold 백업 오퍼링 | `BACKUP_OFFERING_NAME` (예: `VeeamBackup`) | +| Job conf | Veeam Job 이름과 맞추면 편함 (`--job-name`) | +| FLR→Mold | KVM `mold-veeam-restore-agent.timer` → `restoreBackup` | + +## 최초 설정 (KVM) + +```bash +/etc/ablestack/veeam/veeam_config.sh \ + --job-name "Mold ablecube31-2" \ + --offering-name "VeeamBackup" \ + --mold-url "http://10.10.31.20:8080/client/api" \ + --api-key KEY --api-secret 'SECRET' \ + --zone-id ZONE_UUID \ + --vm-include "*" \ + --kvm-host "10.10.31.2" \ + --backup-mode host \ + --install +``` + +Datadisk 한 번에: + +```bash +bash /etc/ablestack/veeam/setup-datadisk-veeam-backup.sh --env-file /etc/ablestack/veeam/mold-backup.env +``` + +## 백업 / 복원 + +```bash +# 전체(실행 중 VM) 또는 VM_INCLUDE에 맞춘 백업 +/etc/ablestack/veeam/mold-backup.sh backup-full --job "Mold ablecube31-2" + +# 백업 ID 확인 후 개별 복원 (VM 정지 필요) +/etc/ablestack/veeam/mold-backup.sh list-backups --job "Mold ablecube31-2" +virsh -c qemu:///system shutdown i-2-11-VM +/etc/ablestack/veeam/mold-backup.sh restore --job "Mold ablecube31-2" \ + --vm-name i-2-11-VM --backup-id "" +``` + +## Veeam Job (KVM file-level) + +```bash +# mold-backup.env 에 VEEAM_SSH_HOST=10.10.254.246 등 설정 후 +bash /etc/ablestack/veeam/push-to-veeam.sh --env-file /etc/ablestack/veeam/mold-backup.env +``` + +| 단계 | 내용 | +|------|------| +| PS1 배포 | `C:\ProgramData\Mold\backup\veeam\` | +| `setup-veeam-mold-job.ps1` | SelectedFiles `/tmp/mold/veeam` + Pre/Post | +| Pre/Post | KVM `ablestack_veeam_pre/post_notify.sh` → Mold API | + +## Veeam UI Guest files → Mold + +```bash +bash /etc/ablestack/veeam/enable-veeam-mold-restore.sh --vm-include 'i-2-XX-VM' +# timer: mold-veeam-restore-agent.timer → mold-backup.sh restore-watch --trigger-mold +``` + +## 배포 경로 + +| 목적 | 스크립트 | +|------|----------| +| KVM 훅 설치 | `push-to-kvm.sh` → `install.sh` | +| 설정 생성 | `veeam_config.sh` / `setup-datadisk-veeam-backup.sh` | +| Veeam PS1/Job | `push-to-veeam.sh` | +| UI FLR→Mold | `enable-veeam-mold-restore.sh` | + +## 유지 파일 (요약) + +- **KVM 코어**: `mold-backup.sh`, `mold-backup.lib.sh`, `veeam_config.sh`, `install.sh`, pre/post notify, `ablestack_cvtbackup.sh` +- **FLR**: `enable-veeam-mold-restore.sh`, `mold-veeam-restore-agent.*` +- **Veeam PS1**: `setup-veeam-mold-job.ps1`, `create-veeam-agent-job.ps1`, `install-veeam-job.ps1`, `veeam-job-*.ps1` +- **패키지/에이전트**: 상위 `ablestack_nasbackup.sh`, `postinstall_mold_backup.sh`, `setup_agent.sh` diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack.key.default b/scripts/vm/hypervisor/kvm/veeam/ablestack.key.default new file mode 100644 index 000000000000..99ad546ab3f5 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack.key.default @@ -0,0 +1 @@ +QWJsZWNsb3VkMSE= diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh new file mode 100644 index 000000000000..08366ff05aab --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh @@ -0,0 +1,485 @@ +#!/usr/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +# CloudStack B&R Commvault Backup and Recovery Tool for KVM + +# TODO: do libvirt/logging etc checks + +### Declare variables ### + +OP="" +VM="" +BACKUP_DIR="" +DISK_PATHS="" +QUIESCE="" +BACKUP_TYPE="FULL" +CHECKPOINT_NAME="" +PARENT_BACKUP_DIR="" +PARENT_CHECKPOINT_NAME="" +PARENT_CHECKPOINT_PATH="" +BACKUP_FILES="" +FORCED="false" +logFile="/var/log/cloudstack/agent/agent.log" + +EXIT_CLEANUP_FAILED=20 + +log() { + [[ "$verb" -eq 1 ]] && builtin echo "$@" + if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then + builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile" + else + builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile" + fi +} + +vercomp() { + local IFS=. + local i ver1=($1) ver2=($3) + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 0 + elif ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 + fi + done + return 0 +} + +sanity_checks() { + hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}') + libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1) + apiVersion=$(virsh version | grep API | awk '{print $(NF)}') + + vercomp "$hvVersion" ">=" "4.2.0" + hvStatus=$? + vercomp "$libvVersion" ">=" "7.2.0" + libvStatus=$? + + if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then + log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" + else + echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" + exit 1 + fi +} + +cleanup() { + local status=0 + rm -rf "$dest" || { echo "Failed to delete $dest"; status=1; } + if [[ -e "$dest" ]]; then + echo "Backup directory still exists after cleanup: $dest" + status=1 + fi + if [[ $status -ne 0 ]]; then + echo "Backup cleanup failed" + exit $EXIT_CLEANUP_FAILED + fi +} + +split_csv() { + tr ',' '\n' <<< "$1" +} + +is_rbd_disk_path() { + local disk_path="$1" + [[ "$disk_path" == rbd:* || "$disk_path" == rbd/* ]] +} + +get_backup_file_by_index() { + local index="$1" + local fallback="$2" + if [[ -z "$BACKUP_FILES" ]]; then + echo "$fallback" + return + fi + local current=0 + while IFS= read -r value; do + if [[ "$current" -eq "$index" ]]; then + echo "$value" + return + fi + current=$((current + 1)) + done < <(split_csv "$BACKUP_FILES") + echo "$fallback" +} + +dump_checkpoint_xml() { + local vm_name="$1" + if [[ -n "$CHECKPOINT_NAME" ]]; then + virsh -c qemu:///system checkpoint-dumpxml --domain "$vm_name" --checkpointname "$CHECKPOINT_NAME" --no-domain > "$dest/checkpoints/$CHECKPOINT_NAME.xml" 2>/dev/null || true + fi +} + +redefine_checkpoint_if_needed() { + local vm_name="$1" + local checkpoint_file="$2" + if [[ -z "$PARENT_CHECKPOINT_NAME" || -z "$checkpoint_file" || ! -f "$checkpoint_file" ]]; then + return + fi + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$PARENT_CHECKPOINT_NAME" > /dev/null 2>&1; then + return + fi + if ! virsh -c qemu:///system checkpoint-create --domain "$vm_name" --xmlfile "$checkpoint_file" --redefine > /dev/null 2>&1; then + echo "Failed to redefine checkpoint $PARENT_CHECKPOINT_NAME on domain $vm_name" + exit 1 + fi +} + + +parse_rbd_uri() { + local uri="$1" + log -ne "parse_rbd_uri called with uri=[$uri]" + + RBD_IMAGE="" + RBD_MON_HOST="" + RBD_USER="" + RBD_KEY="" + + if [[ "$uri" == rbd:* ]]; then + local payload="${uri#rbd:}" + RBD_IMAGE="${payload%%:*}" + + if [[ "$uri" =~ :mon_host=([^:]*) ]]; then + RBD_MON_HOST="${BASH_REMATCH[1]}" + RBD_MON_HOST="${RBD_MON_HOST//\\;/,}" + RBD_MON_HOST="${RBD_MON_HOST//\\:/:}" + fi + + if [[ "$uri" =~ :id=([^:]*) ]]; then + RBD_USER="${BASH_REMATCH[1]}" + fi + + if [[ "$uri" =~ :key=([^:]*) ]]; then + RBD_KEY="${BASH_REMATCH[1]}" + fi + elif [[ "$uri" == rbd/* ]]; then + RBD_IMAGE="$uri" + else + echo "Invalid RBD disk path: $uri" + cleanup + fi + + if [[ -z "$RBD_IMAGE" ]]; then + echo "Failed to parse RBD image from uri: $uri" + cleanup + fi + + log -ne "Parsed RBD uri -> IMAGE=[$RBD_IMAGE], MON=[$RBD_MON_HOST], USER=[$RBD_USER]" +} + +build_rbd_cmd() { + RBD_CMD=(rbd) + if [[ -n "$RBD_MON_HOST" ]]; then + RBD_CMD+=(-m "$RBD_MON_HOST") + fi + if [[ -n "$RBD_USER" ]]; then + RBD_CMD+=(--id "$RBD_USER") + fi + if [[ -n "$RBD_KEY" ]]; then + RBD_CMD+=(--key "$RBD_KEY") + fi +} + +write_rbd_backup_metadata() { + local backup_type="$1" + local checkpoint_name="$2" + local parent_checkpoint_name="$3" + + cat > "$dest/rbd-backup.meta" < "$dest/checkpoints/$checkpoint_name.meta" < /dev/null 2>&1; then + virsh -c qemu:///system dumpxml "$vm_name" > "$dest/domain-config.xml" 2>/dev/null || true + virsh -c qemu:///system dominfo "$vm_name" > "$dest/dominfo.xml" 2>/dev/null || true + virsh -c qemu:///system domiflist "$vm_name" > "$dest/domiflist.xml" 2>/dev/null || true + virsh -c qemu:///system domblklist "$vm_name" > "$dest/domblklist.xml" 2>/dev/null || true + + if [[ -n "$CHECKPOINT_NAME" ]]; then + cat > "$dest/checkpoints/$CHECKPOINT_NAME.meta" <" > "$dest/backup.xml" + if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_NAME" ]]; then + echo "$PARENT_CHECKPOINT_NAME" >> "$dest/backup.xml" + fi + echo "" >> "$dest/backup.xml" + local index=0 + for disk in $(virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '/disk/{print $3}'); do + local target_file="$dest/$(get_backup_file_by_index "$index")" + echo "" >> "$dest/backup.xml" + index=$((index + 1)) + done + echo "" >> "$dest/backup.xml" + + echo "$CHECKPOINT_NAME" > "$dest/checkpoint.xml" + for disk in $(virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '/disk/{print $3}'); do + echo "" >> "$dest/checkpoint.xml" + done + echo "" >> "$dest/checkpoint.xml" + + local thaw=0 + if [[ ${QUIESCE} == "true" ]]; then + if virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-freeze"}' > /dev/null 2>/dev/null; then + thaw=1 + fi + fi + + local backup_begin=0 + local backup_begin_output="" + if backup_begin_output=$(virsh -c qemu:///system backup-begin --domain "$VM" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" 2>&1); then + backup_begin=1 + fi + + if [[ $thaw -eq 1 ]]; then + virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-thaw"}' > /dev/null 2>&1 || true + fi + + if [[ $backup_begin -ne 1 ]]; then + echo "Failed to start libvirt backup for VM [$VM]: ${backup_begin_output:-Unknown error}" + cleanup + exit 1 + fi + + backup_domain_information "$VM" + + while true; do + status=$(virsh -c qemu:///system domjobinfo "$VM" --completed --keep-completed | awk '/Job type:/ {print $3}') + case "$status" in + Completed) break ;; + Failed) echo "Virsh backup job failed"; cleanup ;; + esac + sleep 5 + done + + dump_checkpoint_xml "$VM" + rm -f "$dest/backup.xml" "$dest/checkpoint.xml" + sync +} + +backup_rbd_volumes() { + mkdir -p "$dest/checkpoints" || { echo "Failed to create backup directory $dest"; exit 1; } + backup_domain_information "$VM" + local index=0 + while IFS= read -r disk_path; do + [[ -z "$disk_path" ]] && continue + local created_snapshot="" + log -ne "Loop disk raw value=[$disk_path]" + parse_rbd_uri "$disk_path" + build_rbd_cmd + log -ne "Built RBD command: ${RBD_CMD[*]}" + + local output_file="$dest/$(get_backup_file_by_index "$index" "${RBD_IMAGE##*/}.raw")" + log -ne "Starting RBD backup for disk path [$disk_path], resolved image [$RBD_IMAGE], output [$output_file]" + + if ! timeout 30s "${RBD_CMD[@]}" info "$RBD_IMAGE" >> "$logFile" 2>&1; then + echo "Failed to access RBD image $RBD_IMAGE" + cleanup + fi + + if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_NAME" ]]; then + if ! timeout 30s "${RBD_CMD[@]}" snap ls "$RBD_IMAGE" 2>>"$logFile" | awk 'NR>1 {print $2}' | grep -Fxq "$PARENT_CHECKPOINT_NAME"; then + echo "Parent RBD snapshot ${RBD_IMAGE}@${PARENT_CHECKPOINT_NAME} not found for incremental backup" + cleanup + fi + fi + + if ! timeout 30s "${RBD_CMD[@]}" snap create "${RBD_IMAGE}@${CHECKPOINT_NAME}" >> "$logFile" 2>&1; then + echo "Failed to create RBD snapshot ${RBD_IMAGE}@${CHECKPOINT_NAME}" + cleanup + fi + created_snapshot="${RBD_IMAGE}@${CHECKPOINT_NAME}" + + if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_NAME" ]]; then + if ! timeout 6h "${RBD_CMD[@]}" export-diff --from-snap "$PARENT_CHECKPOINT_NAME" "${RBD_IMAGE}@${CHECKPOINT_NAME}" "$output_file" >> "$logFile" 2>&1; then + echo "Failed to export incremental RBD diff for ${RBD_IMAGE}@${CHECKPOINT_NAME}" + [[ -n "$created_snapshot" ]] && "${RBD_CMD[@]}" snap rm "$created_snapshot" >> "$logFile" 2>&1 || true + cleanup + fi + else + if ! timeout 6h "${RBD_CMD[@]}" export "${RBD_IMAGE}@${CHECKPOINT_NAME}" "$output_file" >> "$logFile" 2>&1; then + echo "Failed to export full RBD snapshot ${RBD_IMAGE}@${CHECKPOINT_NAME}" + [[ -n "$created_snapshot" ]] && "${RBD_CMD[@]}" snap rm "$created_snapshot" >> "$logFile" 2>&1 || true + cleanup + fi + fi + + log -ne "Finished exporting backup file [$output_file] size=[$(stat -c %s "$output_file" 2>/dev/null)]" + index=$((index + 1)) + done < <(split_csv "$DISK_PATHS") + + write_rbd_backup_metadata "$BACKUP_TYPE" "$CHECKPOINT_NAME" "$PARENT_CHECKPOINT_NAME" + write_rbd_checkpoint_metadata "$CHECKPOINT_NAME" "$PARENT_CHECKPOINT_NAME" +} + +has_child_backup() { + local checkpoint_name="$1" + [[ -z "$checkpoint_name" ]] && return 1 + grep -R -q "^parent_checkpoint_name=$checkpoint_name$" "$(dirname "$dest")"/*/rbd-backup.meta 2>/dev/null +} + +delete_rbd_snapshot_if_unreferenced() { + local disk_paths="$1" + local checkpoint_name="$2" + + [[ -z "$checkpoint_name" ]] && return 0 + + if has_child_backup "$checkpoint_name"; then + log -ne "Skip snapshot delete [$checkpoint_name] (child exists)" + return 0 + fi + + while IFS= read -r disk_path; do + [[ -z "$disk_path" ]] && continue + parse_rbd_uri "$disk_path" + build_rbd_cmd + + if timeout 30s "${RBD_CMD[@]}" snap ls "$RBD_IMAGE" 2>/dev/null | awk 'NR>1 {print $2}' | grep -Fxq "$checkpoint_name"; then + log -ne "Deleting snapshot [${RBD_IMAGE}@${checkpoint_name}]" + "${RBD_CMD[@]}" snap rm "${RBD_IMAGE}@${checkpoint_name}" >> "$logFile" 2>&1 || true + fi + done < <(split_csv "$disk_paths") +} + +delete_backup() { + if [[ -f "$dest/rbd-backup.meta" ]]; then + source "$dest/rbd-backup.meta" + + log -ne "Deleting backup with metadata [$dest]" + + if [[ "$FORCED" != "true" ]] && has_child_backup "$checkpoint_name"; then + echo "Cannot delete backup [$backup_dir]: child backup exists" + exit 1 + fi + + delete_rbd_snapshot_if_unreferenced "$disk_paths" "$checkpoint_name" + elif [[ -n "$CHECKPOINT_NAME" && -n "$DISK_PATHS" ]]; then + log -ne "Deleting backup using command metadata [$dest]" + delete_rbd_snapshot_if_unreferenced "$DISK_PATHS" "$CHECKPOINT_NAME" + fi + + rm -frv "$dest" + sync +} + +usage() { + echo "" + echo "Usage: $0 -o -v|--vm -p -b -c -r -i -j -f -d -q|--quiesce " + echo "" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + -o|--operation) OP="$2"; shift; shift ;; + -v|--vm) VM="$2"; shift; shift ;; + -p|--path) BACKUP_DIR="$2"; shift; shift ;; + -b|--backuptype) BACKUP_TYPE="$2"; shift; shift ;; + -c|--checkpoint) CHECKPOINT_NAME="$2"; shift; shift ;; + -r|--parentbackup) PARENT_BACKUP_DIR="$2"; shift; shift ;; + -i|--parentcheckpoint) PARENT_CHECKPOINT_NAME="$2"; shift; shift ;; + -j|--parentcheckpointpath) PARENT_CHECKPOINT_PATH="$2"; shift; shift ;; + -f|--backupfiles) BACKUP_FILES="$2"; shift; shift ;; + -q|--quiesce) QUIESCE="$2"; shift; shift ;; + -d|--diskpaths) DISK_PATHS="$2"; shift; shift ;; + -x|--forced) FORCED="$2"; shift; shift ;; + -h|--help) usage ;; + *) echo "Invalid option: $1"; usage ;; + esac +done + +if [[ -z "$BACKUP_DIR" ]]; then + echo "Backup path (-p|--path) is required" + exit 1 +fi + +dest="$BACKUP_DIR" +sanity_checks + +log -ne "ablestack_cvtbackup.sh start op=[$OP] vm=[$VM] backupDir=[$BACKUP_DIR] backupType=[$BACKUP_TYPE] checkpoint=[$CHECKPOINT_NAME] parentBackup=[$PARENT_BACKUP_DIR] parentCheckpoint=[$PARENT_CHECKPOINT_NAME] diskPaths=[$DISK_PATHS] backupFiles=[$BACKUP_FILES]" + +if [[ "$OP" == "backup-running" ]]; then + backup_running_vm +elif [[ "$OP" == "backup-rbd" ]]; then + backup_rbd_volumes +elif [[ "$OP" == "delete" ]]; then + delete_backup +else + echo "Unsupported operation: $OP" + exit 1 +fi diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh new file mode 100755 index 000000000000..997649254ff4 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup bpend_notify equivalent for Veeam + Mold. +# Args: CLIENT JOB [SCHEDULE] [TYPE] + +set -euo pipefail + +CLIENT="${1:-$(hostname -s)}" +JOB="${2:-${VEEAM_JOB_NAME:-}}" +SCHEDULE="${3:-${VEEAM_SCHEDULE_NAME:-default}}" +TYPE="${4:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +export VEEAM_JOB_NAME="$JOB" +mold_backup_post_notify "$CLIENT" "$JOB" "$SCHEDULE" "$TYPE" diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh new file mode 100755 index 000000000000..91114ffd6da5 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh @@ -0,0 +1,35 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup bpstart_notify equivalent for Veeam + Mold. +# Args: CLIENT JOB [SCHEDULE] [TYPE] +# Log to file only; do not write to stdout (Veeam/NetBackup policy requirement). + +set -euo pipefail + +CLIENT="${1:-$(hostname -s)}" +JOB="${2:-${VEEAM_JOB_NAME:-}}" +SCHEDULE="${3:-${VEEAM_SCHEDULE_NAME:-default}}" +TYPE="${4:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +export VEEAM_JOB_NAME="$JOB" +mold_backup_pre_notify "$CLIENT" "$JOB" "$SCHEDULE" "$TYPE" diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh new file mode 100644 index 000000000000..909f4f094b2a --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh @@ -0,0 +1,68 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Restore event ingress (Veeam PS1 SSH push, restore-watch agent, or manual). +# +# Events: +# veeam.restore.completed Veeam UI/Agent restore session finished → Mold datadisk restore +# mold.restore.manual Operator/Mold UI path with BACKUP_ID already set +# +# Usage: +# ablestack_veeam_restore_event.sh veeam.restore.completed [detail] +# BACKUP_ID= ablestack_veeam_restore_event.sh mold.restore.manual manual + +set -euo pipefail + +EVENT="${1:-}" +SESSION_ID="${2:-}" +VM_NAME_E="${3:-}" +DETAIL="${4:-}" +JOB="${VEEAM_JOB_NAME:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +mold_backup_load_config || exit 1 +[[ -n "$JOB" ]] || JOB="${VEEAM_JOB_NAME:-}" + +case "$EVENT" in + veeam.restore.completed) + [[ -n "$SESSION_ID" && -n "$VM_NAME_E" ]] || { + mold_backup_notify_log err "restore-event: veeam.restore.completed needs session-id and vm-name" + exit 1 + } + mold_backup_handle_veeam_restore_session "$JOB" "$VM_NAME_E" "$SESSION_ID" "$DETAIL" true + ;; + mold.restore.manual) + [[ -n "$VM_NAME_E" ]] || { + mold_backup_notify_log err "restore-event: mold.restore.manual needs vm-name" + exit 1 + } + export VM_NAME="$VM_NAME_E" + [[ -n "${BACKUP_ID:-}" ]] || mold_backup_die "mold.restore.manual requires BACKUP_ID" + mold_backup_emit_restore_event "mold.restore.requested" "$VM_NAME_E" "backup_id=${BACKUP_ID};source=manual" + mold_backup_trigger_mark "mold-restore-active" "$VM_NAME_E" + mold_backup_restore_notify "$(hostname -s)" "$JOB" + mold_backup_emit_restore_event "mold.restore.completed" "$VM_NAME_E" "backup_id=${BACKUP_ID}" + ;; + *) + mold_backup_notify_log err "restore-event: unknown event '${EVENT}'" + exit 1 + ;; +esac diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh new file mode 100755 index 000000000000..b371629eef79 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh @@ -0,0 +1,32 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup restore_notify equivalent for Veeam + Mold. +# Set BACKUP_ID (and optional VM_NAME / VM_UUID) in job conf or environment. + +set -euo pipefail + +CLIENT="${1:-$(hostname -s)}" +JOB="${2:-${VEEAM_JOB_NAME:-}}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +export VEEAM_JOB_NAME="$JOB" +mold_backup_restore_notify "$CLIENT" "$JOB" diff --git a/scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 b/scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 new file mode 100644 index 000000000000..28b9ac393c04 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 @@ -0,0 +1,627 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Create (or update) a Veeam Linux Agent file-level backup job for Mold integration. +# Idempotent: safe to re-run. Registers Pre/Post bash hooks via install-veeam-job.ps1. +# +# Prerequisite: Veeam Agent for Linux on KVM host, connected to this B&R server. +# +# Example (run on Veeam B&R server as Administrator, PowerShell 7+): +# .\create-veeam-agent-job.ps1 ` +# -JobName "Mold KVM Backup" ` +# -KvmHost "10.10.31.2" ` +# -AgentHostName "ablecube31-2" ` +# -BackupPath "/tmp/mold/veeam" ` +# -RepositoryName "Default Backup Repository" + +param( + [Parameter(Mandatory = $true)] + [string]$JobName, + + [Parameter(Mandatory = $true)] + [string]$KvmHost, + + [string]$AgentHostName = "", + [string]$BackupPath = "/tmp/mold/veeam", + [string]$RepositoryName = "", + + [string]$InstallDir = "C:\ProgramData\Mold\backup\veeam", + # Custom PG for PowerShell job creation (not "Manually Added"). + [string]$ProtectionGroupName = "Mold KVM Agents", + + [string]$AgentPreNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh", + [string]$AgentPostNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh", + + # Linux SSH for Protection Group (required — temporary certificate fails with + # "Cannot find credentials for agent " on managed Linux agents). + [string]$KvmSshUser = "root", + [string]$KvmSshPassword = "", + + [switch]$SkipScriptRegistration, + [switch]$StartJob, + [switch]$WhatIf +) + +$ErrorActionPreference = "Stop" + +if ($PSVersionTable.PSVersion.Major -lt 7) { + throw "Veeam.Backup.PowerShell requires PowerShell 7+. Run: pwsh -File $PSCommandPath" +} + +if (-not $KvmSshPassword -and $env:MOLD_KVM_SSH_PASSWORD) { + $KvmSshPassword = $env:MOLD_KVM_SSH_PASSWORD +} +if (-not $KvmSshUser) { $KvmSshUser = "root" } + +function Write-MoldVeeamInfo([string]$Message) { Write-Host "[mold-veeam] $Message" } +function Write-MoldVeeamWarn([string]$Message) { Write-Warning "[mold-veeam] $Message" } + +function Connect-MoldVeeamSession { + if (-not (Get-Command Connect-VBRServer -ErrorAction SilentlyContinue)) { return } + $sessions = @() + try { $sessions = @(Get-VBRServerSession -ErrorAction SilentlyContinue) } catch { } + if ($sessions.Count -eq 0) { + Write-MoldVeeamInfo "Connecting to local Veeam B&R (localhost)..." + Connect-VBRServer -Server localhost | Out-Null + } +} + +function Invoke-MoldVeeamCmdlet { + param( + [scriptblock]$Command, + [string]$Context = "Veeam cmdlet" + ) + try { + return & $Command + } catch { + if ($_.Exception.Message -match 'Identity service') { + throw @" +$Context failed: Veeam Identity service is unreachable. + +On this B&R server (Administrator pwsh): + Get-Service VeeamBackupSvc, VeeamBrokerSvc | Restart-Service + Connect-VBRServer -Server localhost + Get-VBRProtectionGroup | Select-Object -First 3 Name + +Then re-run setup-veeam-mold-job.ps1 +"@ + } + throw + } +} + +function Get-MoldVeeamProtectionGroupByName { + param([string]$Name) + if (-not $Name) { return $null } + try { + return Get-VBRProtectionGroup -Name $Name -ErrorAction Stop + } catch { + if ($_.Exception.Message -match 'does not exist|not found|Cannot find') { + return $null + } + if ($_.Exception.Message -match 'Identity service') { + throw @" +Get-VBRProtectionGroup -Name '$Name' failed: Veeam Identity service is unreachable. + +On this B&R server (Administrator pwsh): + Get-Service VeeamBackupSvc, VeeamBrokerSvc | Restart-Service + Connect-VBRServer -Server localhost + Get-VBRProtectionGroup | Select-Object -First 3 Name + +Then re-run setup-veeam-mold-job.ps1 +"@ + } + throw + } +} + +function Get-MoldVeeamPropertyValue { + param($Object, [string[]]$Names) + foreach ($name in $Names) { + if ($null -eq $Object) { continue } + if ($Object.PSObject.Properties.Name -contains $name) { + $val = $Object.$name + if ($null -ne $val -and "$val" -ne "") { return "$val" } + } + } + return "" +} + +function Test-MoldVeeamHostMatch { + param($Computer, [string]$HostIp, [string]$HostName) + $candidates = @( + (Get-MoldVeeamPropertyValue $Computer @("Name", "ComputerName", "HostName", "DnsName")) + (Get-MoldVeeamPropertyValue $Computer @("IPAddress", "IpAddress", "Address")) + (Get-MoldVeeamPropertyValue $Computer @("DisplayName", "Description")) + ) | Where-Object { $_ -ne "" } + + foreach ($c in $candidates) { + if ($HostName -and ($c -eq $HostName -or $c -like "*$HostName*")) { return $true } + if ($HostIp -and ($c -eq $HostIp -or $c -like "*$HostIp*")) { return $true } + } + return $false +} + +function Get-MoldVeeamDiscoveredComputers { + param([string]$GroupName = "") + + if ($GroupName) { + $group = Get-MoldVeeamProtectionGroupByName -Name $GroupName + if ($group) { + return @( + Invoke-MoldVeeamCmdlet -Context "Get-VBRDiscoveredComputer (group $GroupName)" { + Get-VBRDiscoveredComputer -ProtectionGroup $group -ErrorAction SilentlyContinue + } + ) + } + Write-MoldVeeamWarn "Protection group not found: $GroupName" + } + + return @( + Invoke-MoldVeeamCmdlet -Context "Get-VBRDiscoveredComputer" { + Get-VBRDiscoveredComputer -ErrorAction SilentlyContinue + } + ) +} + +function Format-MoldVeeamDiscoveredComputerLine { + param($Computer) + $name = Get-MoldVeeamPropertyValue $Computer @("Name", "ComputerName", "HostName") + $ip = Get-MoldVeeamPropertyValue $Computer @("IPAddress", "IpAddress", "Address") + if ($name -and $ip) { return "${name} (${ip})" } + if ($name) { return $name } + if ($ip) { return $ip } + return ($Computer | Out-String).Trim() +} + +function Find-MoldVeeamDiscoveredComputer { + param([string]$HostIp, [string]$HostName, [string]$GroupName) + + $computers = @(Get-MoldVeeamDiscoveredComputers -GroupName $GroupName) + + $match = $computers | Where-Object { Test-MoldVeeamHostMatch -Computer $_ -HostIp $HostIp -HostName $HostName } | Select-Object -First 1 + if ($match) { return $match } + + if ($HostName) { + $match = $computers | Where-Object { + $n = Get-MoldVeeamPropertyValue $_ @("Name", "ComputerName", "HostName") + $n -and ($n -eq $HostName) + } | Select-Object -First 1 + if ($match) { return $match } + } + + return $null +} + +function Test-MoldVeeamHostInContainer { + param($Container, [string[]]$Targets) + $hostNames = @() + if ($Container -and $Container.CustomCredentials) { + foreach ($c in @($Container.CustomCredentials)) { + $h = Get-MoldVeeamPropertyValue $c @("HostName", "Name") + if ($h) { $hostNames += $h } + } + } + foreach ($t in $Targets) { + if ($hostNames -contains $t) { return $true } + if (($hostNames | Where-Object { $_ -like "*$t*" }).Count -gt 0) { return $true } + } + return $false +} + +function Ensure-MoldVeeamLinuxKvmCredentialRecord { + param( + [string]$User, + [string]$Password + ) + if (-not $Password) { + throw "KvmSshPassword is required (KVM_SSH_PASSWORD in mold-backup.windows.conf or -KvmSshPassword)" + } + + $desc = "Mold KVM Linux SSH ($User)" + $records = @(Get-VBRCredentials -ErrorAction SilentlyContinue) + $match = $records | Where-Object { + "$($_.Description)" -like "*Mold KVM Linux SSH*" + } | Select-Object -First 1 + + if (-not $match) { + $match = $records | Where-Object { + (Get-MoldVeeamPropertyValue $_ @("User", "UserName")) -eq $User + } | Select-Object -First 1 + } + + if ($match) { + Write-MoldVeeamInfo "Updating Veeam credentials record for KVM Linux user: $User" + try { + Set-VBRCredentials -Credential $match -User $User -Password $Password -Description $desc -SshPort 22 | Out-Null + } catch { + Set-VBRCredentials -Credential $match -Password $Password | Out-Null + } + return $match + } + + Write-MoldVeeamInfo "Creating Veeam Linux credentials record for KVM user: $User" + return Add-VBRCredentials -Type Linux -User $User -Password $Password -Description $desc -SshPort 22 +} + +function New-MoldVeeamLinuxKvmComputerCredential { + param( + [string]$HostIp, + $VeeamCredentials, + [string]$User + ) + + $last = "" + $credName = Get-MoldVeeamPropertyValue $VeeamCredentials @("User", "UserName", "Name") + if (-not $credName) { $credName = $User } + + $attempts = @( + { New-VBRIndividualComputerCustomCredentials -HostName $HostIp -Credentials $VeeamCredentials }, + { New-VBRIndividualComputerCustomCredentials -HostName $HostIp -Credentials $credName } + ) + + foreach ($attempt in $attempts) { + try { + $cred = & $attempt + Write-MoldVeeamInfo "PG computer OK: ${User}@${HostIp}" + return $cred + } catch { + $last = $_.Exception.Message + } + } + throw "New-VBRIndividualComputerCustomCredentials failed for ${User}@${HostIp}: $last" +} + +function Ensure-MoldVeeamAgentProtectionGroup { + param( + [string]$HostIp, + [string]$HostName, + [string]$GroupName, + [string]$KvmSshUser = "root", + [string]$KvmSshPassword = "" + ) + + $targets = @($HostIp) + if ($HostName -and $HostName -ne $HostIp) { $targets += $HostName } + + $pg = Get-MoldVeeamProtectionGroupByName -Name $GroupName + + if ($KvmSshPassword) { + Write-MoldVeeamInfo "Protection group: $GroupName (Individual computers + Linux SSH for KVM)" + $veeamCredRecord = Ensure-MoldVeeamLinuxKvmCredentialRecord -User $KvmSshUser -Password $KvmSshPassword + + $targetIps = @($HostIp) + if ($pg) { + $existingIps = @( + Get-VBRDiscoveredComputer -ProtectionGroup $pg -ErrorAction SilentlyContinue | ForEach-Object { + $ip = $null + if ($_.IPAddress) { $ip = @($_.IPAddress)[0] } + if (-not $ip) { $ip = $_.Name } + $ip + } | Where-Object { $_ -match '^\d{1,3}(\.\d{1,3}){3}$' } + ) + if ($existingIps.Count -gt 0) { + Write-MoldVeeamInfo "Existing PG computers kept: $($existingIps -join ', ')" + } + $targetIps = @($existingIps + $HostIp | Select-Object -Unique) + } + + $creds = $targetIps | ForEach-Object { + New-MoldVeeamLinuxKvmComputerCredential -HostIp $_ -VeeamCredentials $veeamCredRecord -User $KvmSshUser + } + $container = New-VBRIndividualComputerContainer -CustomCredentials $creds + + if (-not $pg) { + $pg = Add-VBRProtectionGroup -Name $GroupName ` + -Description "Mold KVM Linux agents (Linux SSH credentials)" ` + -Container $container + } else { + Set-VBRProtectionGroup -ProtectionGroup $pg -Container $container | Out-Null + $pg = Get-MoldVeeamProtectionGroupByName -Name $GroupName + } + } else { + Write-MoldVeeamWarn @" +KvmSshPassword not set — using temporary certificate (often fails with: + Cannot find credentials for agent $HostIp + +Set KVM_SSH_PASSWORD in mold-backup.windows.conf and re-run setup-veeam-mold-job.ps1 +"@ + if (-not $pg) { + Write-MoldVeeamInfo "Creating protection group: $GroupName (Linux, certificate auth)" + $creds = @($HostIp) | ForEach-Object { + New-VBRIndividualComputerCustomCredentials -HostName $_ -UseTemporaryCertificate + } + $container = New-VBRIndividualComputerContainer -CustomCredentials $creds + $pg = Add-VBRProtectionGroup -Name $GroupName ` + -Description "Mold KVM Linux agents — used by setup-veeam-mold-job.ps1" ` + -Container $container + } elseif (-not (Test-MoldVeeamHostInContainer -Container $pg.Container -Targets $targets)) { + Write-MoldVeeamInfo "Adding $HostIp to protection group: $GroupName" + $comps = @($pg.Container.CustomCredentials) + $comps += New-VBRIndividualComputerCustomCredentials -HostName $HostIp -UseTemporaryCertificate + $newContainer = Set-VBRIndividualComputerContainer -Container $pg.Container -CustomCredentials $comps + Set-VBRProtectionGroup -ProtectionGroup $pg -Container $newContainer | Out-Null + $pg = Get-MoldVeeamProtectionGroupByName -Name $GroupName + } + } + + Write-MoldVeeamInfo "Rescanning protection group: $GroupName" + Rescan-VBREntity -Entity $pg | Out-Null + return $pg +} + +function Get-MoldVeeamSingleObject { + param($Value) + if ($null -eq $Value) { return $null } + if ($Value -is [System.Array]) { + if ($Value.Count -eq 0) { return $null } + return $Value[0] + } + return $Value +} + +function Get-MoldVeeamBackupRepository { + param([string]$PreferredName) + + $candidates = @() + if ($PreferredName) { + $candidates = @(Get-VBRBackupRepository -Name $PreferredName -ErrorAction SilentlyContinue) + } + if ($candidates.Count -eq 0) { + $candidates = @(Get-VBRBackupRepository -ErrorAction SilentlyContinue) + } + if ($candidates.Count -eq 0) { + throw "No Veeam backup repository found. Run: Get-VBRBackupRepository | Select-Object Name, Id" + } + + $pick = $candidates | Where-Object { $_.Name -like "*Default*" } | Select-Object -First 1 + if (-not $pick) { $pick = $candidates | Select-Object -First 1 } + $pick = Get-MoldVeeamSingleObject $pick + if (-not $pick -or -not $pick.Name) { + throw "Could not resolve a backup repository object." + } + + # Re-fetch by exact name — avoids pipeline/array wrapper issues with Add-VBRComputerBackupJob. + $repo = Get-MoldVeeamSingleObject (Get-VBRBackupRepository -Name $pick.Name -ErrorAction SilentlyContinue) + if (-not $repo) { $repo = $pick } + + Write-MoldVeeamInfo "Using backup repository: $($repo.Name) (type=$($repo.GetType().Name))" + return $repo +} + +function Get-MoldVeeamBackupServerName { + $server = $null + try { $server = Get-MoldVeeamSingleObject (Get-VBRServer -ErrorAction SilentlyContinue) } catch { } + if ($server) { + foreach ($key in @("Name", "DnsHostName", "DisplayName")) { + if ($server.PSObject.Properties.Name -contains $key -and $server.$key) { + return "$($server.$key)" + } + } + } + return $env:COMPUTERNAME +} + +function New-MoldVeeamComputerBackupJob { + param( + [string]$JobName, + [string]$Description, + $BackupObject, + $Scope, + $Repository + ) + + $common = @{ + OSPlatform = "Linux" + Type = "Server" + Mode = "ManagedByBackupServer" + Name = $JobName + Description = $Description + BackupObject = $BackupObject + BackupType = "SelectedFiles" + SelectedFilesOptions = $Scope + } + + try { + Add-VBRComputerBackupJob @common -BackupRepository $Repository | Out-Null + Write-MoldVeeamInfo "Created Agent backup job via -BackupRepository" + return + } catch { + $msg = $_.Exception.Message + if ($msg -notmatch 'BackupRepository|Destination') { throw } + Write-MoldVeeamWarn "Add-VBRComputerBackupJob -BackupRepository failed: $msg" + } + + $serverName = Get-MoldVeeamBackupServerName + Write-MoldVeeamInfo "Retrying with -DestinationOptions (BackupServerName=$serverName)" + $destination = New-VBRComputerDestinationOptions ` + -OSPlatform Linux ` + -BackupRepository $Repository ` + -BackupServerName $serverName + Add-VBRComputerBackupJob @common -DestinationOptions $destination -BackupRepository $Repository | Out-Null + Write-MoldVeeamInfo "Created Agent backup job via -DestinationOptions + -BackupRepository" +} + +function Assert-MoldVeeamLinuxAgentPath { + param([string]$Path) + if ($Path -match '^[A-Za-z]:[\\/]') { + throw "Linux Agent SelectedFiles path must be on the KVM host (e.g. /tmp/mold/veeam), not: $Path" + } + if (-not $Path.StartsWith("/")) { + throw "Linux Agent SelectedFiles path must be absolute (start with /): $Path" + } +} + +function New-MoldVeeamFileLevelScope { + param([string]$Path) + Assert-MoldVeeamLinuxAgentPath -Path $Path + New-VBRSelectedFilesBackupOptions -OSPlatform Linux -BackupSelectedFiles -SelectedFiles $Path +} + +function Update-MoldVeeamAgentJobScope { + param($Job, [string]$Path) + $scope = New-MoldVeeamFileLevelScope -Path $Path + Set-VBRComputerBackupJob -Job $Job -BackupType SelectedFiles -SelectedFilesOptions $scope | Out-Null + Write-MoldVeeamInfo "Updated job scope to file-level path: $Path" +} + +if (-not $AgentHostName) { + try { + $resolved = [System.Net.Dns]::GetHostEntry($KvmHost) + if ($resolved.HostName) { + $AgentHostName = ($resolved.HostName -split '\.')[0] + } + } catch { + $AgentHostName = $KvmHost + } +} + +Write-MoldVeeamInfo "Job=$JobName KVM=$KvmHost agentHost=$AgentHostName path=$BackupPath (script=mold-veeam-v3)" + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue +Connect-MoldVeeamSession + +# PowerShell cannot add individual computers from "Manually Added" — use a dedicated PG. +$protectionGroup = Ensure-MoldVeeamAgentProtectionGroup ` + -HostIp $KvmHost ` + -HostName $AgentHostName ` + -GroupName $ProtectionGroupName ` + -KvmSshUser $KvmSshUser ` + -KvmSshPassword $KvmSshPassword + +$discovered = Find-MoldVeeamDiscoveredComputer -HostIp $KvmHost -HostName $AgentHostName -GroupName $ProtectionGroupName +if ($discovered) { + $discName = Get-MoldVeeamPropertyValue $discovered @("Name", "ComputerName", "HostName") + Write-MoldVeeamInfo "Found discovered computer in '$ProtectionGroupName': $discName" +} else { + $all = @(Get-MoldVeeamDiscoveredComputers) + $knownList = if ($all.Count -gt 0) { + ($all | ForEach-Object { Format-MoldVeeamDiscoveredComputerLine $_ }) -join "`n - " + } else { + "(none)" + } + Write-MoldVeeamWarn @" +KVM not yet visible in '$ProtectionGroupName' after rescan. +Job will still be created with protection group '$ProtectionGroupName'. + +On KVM, ensure agent is connected to B&R: + veeamconfig vbrserver add --name vbr01 --address --port 10006 --login administrator --password '...' + veeamconfig vbrserver list + +Open firewall on KVM from B&R: tcp/6160, tcp/10006 + +Known discovered computers: + - $knownList +"@ +} + +$repository = Get-MoldVeeamBackupRepository -PreferredName $RepositoryName +$scope = New-MoldVeeamFileLevelScope -Path $BackupPath +$backupObject = @($protectionGroup) +Write-MoldVeeamInfo "Backup object: protection group '$ProtectionGroupName' (not Manually Added computer)" +Write-MoldVeeamInfo "Backup repository: $($repository.Name)" + +$existing = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue +if ($existing) { + Write-MoldVeeamInfo "Agent backup job already exists: $JobName" + if (-not $WhatIf) { + Update-MoldVeeamAgentJobScope -Job $existing -Path $BackupPath + } + } else { + Write-MoldVeeamInfo "Creating Agent backup job: $JobName" + if ($WhatIf) { + Write-MoldVeeamInfo "WhatIf: would call Add-VBRComputerBackupJob" + } else { + New-MoldVeeamComputerBackupJob ` + -JobName $JobName ` + -Description "Mold NAS backup + file-level staging ($BackupPath)" ` + -BackupObject $backupObject ` + -Scope $scope ` + -Repository $repository + Write-MoldVeeamInfo "Created Agent backup job: $JobName" + } + } + +if ($SkipScriptRegistration) { + Write-MoldVeeamInfo "SkipScriptRegistration set — done." + exit 0 +} + +$installScript = Join-Path $InstallDir "install-veeam-job.ps1" +if (-not (Test-Path $installScript)) { + $installScript = Join-Path $PSScriptRoot "install-veeam-job.ps1" +} +if (-not (Test-Path $installScript)) { + throw "install-veeam-job.ps1 not found under $InstallDir or $PSScriptRoot" +} + +if ($WhatIf) { + Write-MoldVeeamInfo "WhatIf: would register Pre/Post via $installScript" + exit 0 +} + +function Invoke-MoldVeeamInstallJob { + param( + [string]$ScriptPath, + [string]$JobName, + [string]$InstallDir, + [string]$KvmHost, + [string]$ProtectionGroupName, + [string]$AgentPreNotifyScript, + [string]$AgentPostNotifyScript + ) + + $installParams = @{ + JobName = $JobName + InstallDir = $InstallDir + KvmHost = $KvmHost + ProtectionGroupName = $ProtectionGroupName + AgentPreNotifyScript = $AgentPreNotifyScript + AgentPostNotifyScript = $AgentPostNotifyScript + } + + $cmd = Get-Command $ScriptPath -ErrorAction Stop + if ($cmd.Parameters.ContainsKey("LinuxAgent")) { + $installParams["LinuxAgent"] = $true + } else { + Write-MoldVeeamWarn "install-veeam-job.ps1 is outdated (no -LinuxAgent). Update from repo: curl install-veeam-job.ps1" + } + + & $ScriptPath @installParams +} + +Invoke-MoldVeeamInstallJob ` + -ScriptPath $installScript ` + -JobName $JobName ` + -InstallDir $InstallDir ` + -KvmHost $KvmHost ` + -ProtectionGroupName $ProtectionGroupName ` + -AgentPreNotifyScript $AgentPreNotifyScript ` + -AgentPostNotifyScript $AgentPostNotifyScript + +if ($StartJob) { + $jobToRun = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue + if ($jobToRun) { + Write-MoldVeeamInfo "Starting job: $JobName" + Start-VBRComputerBackupJob -Job $jobToRun | Out-Null + } +} + +Write-MoldVeeamInfo "Done. Scope=SelectedFiles path=$BackupPath (VM export only, not entire host)." +Write-MoldVeeamInfo "KVM: set VM targets — /etc/ablestack/veeam/veeam_config.sh --job-name '$JobName' --vm-include 'i-2-7-VM'" +Write-MoldVeeamInfo "Start: Veeam UI -> Jobs -> $JobName -> Start" +Write-MoldVeeamInfo "KVM log: tail -f /var/log/mold/veeam-hook.log" diff --git a/scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh b/scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh new file mode 100644 index 000000000000..8072f98bd32d --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh @@ -0,0 +1,124 @@ +#!/usr/bin/bash +# Enable Veeam UI restore → Mold datadisk restore chain on this KVM host. +# +# bash enable-veeam-mold-restore.sh +# bash enable-veeam-mold-restore.sh --vm-include 'i-2-62-VM,i-2-64-VM' +# bash enable-veeam-mold-restore.sh --restore-vm i-2-62-VM # FLR 시 복원할 VM (host job, 다중 VM일 때) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +ENV_FILE="${ETC_DIR}/mold-backup.env" +HOST_CONF="${ETC_DIR}/Mold_Host_Backup.conf" +VM_INCLUDE_ARG="" +RESTORE_VM="" + +die() { echo "ERROR: $*" >&2; exit 1; } + +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" + +safe_install() { + local mode="$1" src="$2" dst="$3" + [[ -f "$src" ]] || die "Missing source file: $src" + if [[ -e "$src" && -e "$dst" ]] && [[ "$(stat -c '%d:%i' "$src" 2>/dev/null)" == "$(stat -c '%d:%i' "$dst" 2>/dev/null)" ]]; then + chmod "$mode" "$dst" 2>/dev/null || true + return 0 + fi + install -m "$mode" "$src" "$dst" +} + +resolve_bundle_file() { + local name="$1" d + for d in "${SCRIPT_DIR}" "${ETC_DIR}" "${SHARE_DIR}" "/tmp/veeam-install"; do + [[ -f "${d}/${name}" ]] && { echo "${d}/${name}"; return 0; } + done + if [[ -f "/etc/systemd/system/${name}" ]]; then + echo "/etc/systemd/system/${name}" + return 0 + fi + die "Missing ${name} — run: bash ${ETC_DIR}/install.sh or push-to-kvm.sh from Git repo" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="$2"; shift 2 ;; + --vm-include) VM_INCLUDE_ARG="$2"; shift 2 ;; + --restore-vm) RESTORE_VM="$2"; shift 2 ;; + -h|--help) + sed -n '2,12p' "$0" + exit 0 + ;; + *) die "Unknown: $1" ;; + esac +done + +[[ -f "$HOST_CONF" ]] || die "Missing ${HOST_CONF} — run setup-datadisk-veeam-backup.sh first" + +# shellcheck source=/dev/null +[[ -f "$ENV_FILE" ]] && set -a && source "$ENV_FILE" && set +a + +set_kv() { + local k="$1" v="$2" f="$HOST_CONF" + if grep -q "^${k}=" "$f" 2>/dev/null; then + sed -i "s|^${k}=.*|${k}=\"${v}\"|" "$f" + else + echo "${k}=\"${v}\"" >>"$f" + fi +} + +job="${VEEAM_JOB_NAME:-Mold ablecube31-2}" +[[ -n "$VM_INCLUDE_ARG" ]] && set_kv VM_INCLUDE "$VM_INCLUDE_ARG" +[[ -n "$RESTORE_VM" ]] && set_kv VEEAM_RESTORE_VM "$RESTORE_VM" + +set_kv RESTORE_WATCH_TRIGGER_MOLD true +set_kv VEEAM_UI_RESTORE_SOURCE mold-only +set_kv RESTORE_SOURCE mold-only +set_kv BACKUP_MODE host +set_kv VEEAM_JOB_NAME "$job" +set_kv VEEAM_RESTORE_WATCH_WINDOW_MIN "${VEEAM_RESTORE_WATCH_WINDOW_MIN:-10}" +[[ -n "${VEEAM_SSH_HOST:-}" ]] && set_kv VEEAM_SSH_HOST "$VEEAM_SSH_HOST" + +echo "=== Mold Veeam restore agent (KVM) ===" +grep -E 'VEEAM_JOB|VM_INCLUDE|VEEAM_RESTORE_VM|RESTORE_|VEEAM_SSH|BACKUP_MODE' "$HOST_CONF" || true + +bash "${SCRIPT_DIR}/install.sh" 2>/dev/null || bash "${ETC_DIR}/install.sh" 2>/dev/null || true + +mkdir -p "${ETC_DIR}/events" "${ETC_DIR}/registry" "${ETC_DIR}/state" 2>/dev/null || true +touch "${ETC_DIR}/events/restore.log" 2>/dev/null || true + +agent_sh="$(resolve_bundle_file mold-veeam-restore-agent.sh)" +agent_svc="$(resolve_bundle_file mold-veeam-restore-agent.service)" +agent_timer="$(resolve_bundle_file mold-veeam-restore-agent.timer)" + +safe_install 0755 "$agent_sh" "${ETC_DIR}/mold-veeam-restore-agent.sh" +safe_install 0644 "$agent_svc" "${ETC_DIR}/mold-veeam-restore-agent.service" +safe_install 0644 "$agent_timer" "${ETC_DIR}/mold-veeam-restore-agent.timer" +safe_install 0644 "$agent_svc" /etc/systemd/system/mold-veeam-restore-agent.service +safe_install 0644 "$agent_timer" /etc/systemd/system/mold-veeam-restore-agent.timer + +systemctl daemon-reload +# Legacy timer without --trigger-mold — disable to avoid duplicate polls. +systemctl disable mold-veeam-restore-watch.timer 2>/dev/null || true +systemctl stop mold-veeam-restore-watch.timer 2>/dev/null || true +systemctl enable mold-veeam-restore-agent.timer +systemctl start mold-veeam-restore-agent.timer + +echo "" +echo "Enabled: mold-veeam-restore-agent.timer ($(systemctl is-active mold-veeam-restore-agent.timer 2>/dev/null || echo unknown))" +echo "" +cat <&2 + echo " printf 'QWJsZWNsb3VkMSE=' > ${KEY_FILE} && chmod 600 ${KEY_FILE}" >&2 + exit 1 +fi +install -m 0600 "$src" "$KEY_FILE" +echo "Installed ${KEY_FILE} from ${src} ($(wc -c < "$KEY_FILE") bytes)" diff --git a/scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 b/scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 new file mode 100644 index 000000000000..87a7dcc380d0 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 @@ -0,0 +1,645 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Run on Veeam Backup & Replication server (Administrator PowerShell). +# 1) Copies Mold Veeam scripts + default config to ProgramData +# 2) Registers Pre-job / Post-job on a Veeam Agent backup job (Guest Processing scripts) +# +# Example: +# .\install-veeam-job.ps1 -JobName "Rocky Agent Backup" -SourceDir "\\git\...\veeam" +# .\install-veeam-job.ps1 -JobName "Rocky Agent Backup" -VmName "rocky94" -StagingPath "D:\veeam-staging\rocky94" -KvmHost "10.0.0.10" + +param( + [Parameter(Mandatory = $true)] + [string]$JobName, + + [string]$SourceDir = $PSScriptRoot, + + [string]$InstallDir = "C:\ProgramData\Mold\backup\veeam", + + [string]$VmName = "", + [string]$StagingPath = "", + [string]$KvmHost = "", + [string]$KvmSshUser = "root", + [string]$KvmSshKey = "", + + # Linux Agent jobs run pre/post scripts ON THE AGENT HOST (bash), not on this Windows server. + [switch]$LinuxAgent, + [string]$AgentPreNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh", + [string]$AgentPostNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh", + [string]$ProtectionGroupName = "Mold KVM Agents", + + # Guest VM mode: pre/post SSH to KVM for Mold; scripts run ON the guest agent. + [switch]$GuestVm, + [string]$VmLibvirtName = "", + [string]$VmIp = "", + $DiscoveredComputer = $null, + + # Only generate the pre/post wrapper scripts into $InstallDir on this Veeam + # server. Skips Veeam module load, job lookup, and job registration so you + # can point the Veeam UI Guest Processing scripts at the generated files. + [switch]$ScriptsOnly, + + [switch]$EnableRestoreScript +) + +$ErrorActionPreference = "Stop" + +function Get-MoldVeeamGuestWrapperPath { + param( + [string]$InstallDir, + [string]$Phase, + [string]$VmLibvirtName + ) + $safeVm = ($VmLibvirtName -replace '[^\w\-]', '_') + return (Join-Path $InstallDir "mold-guest-${Phase}-${safeVm}.sh") +} + +function Write-MoldAgentWrapperScript { + param( + [string]$Path, + [string]$AgentScript, + [string]$JobName + ) + $escapedJob = $JobName -replace "'", "'\''" + $escapedScript = $AgentScript -replace "'", "'\''" + $content = @" +#!/bin/bash +exec '$escapedScript' "`$(hostname -s)" '$escapedJob' +"@ + $utf8NoBom = New-Object System.Text.UTF8Encoding $false + [System.IO.File]::WriteAllText($Path, $content.Replace("`r`n", "`n"), $utf8NoBom) +} + +function Write-MoldGuestKvmSshWrapperScript { + param( + [string]$Path, + [string]$KvmHost, + [string]$KvmUser, + [string]$VmLibvirtName, + [string]$JobName, + [string]$RemoteScript, + [switch]$IsPost + ) + $escapedJob = $JobName -replace "'", "'\''" + $escapedVm = $VmLibvirtName -replace "'", "'\''" + $escapedRemote = $RemoteScript -replace "'", "'\''" + $escapedKvm = $KvmHost -replace "'", "'\''" + $escapedUser = $KvmUser -replace "'", "'\''" + $hookPhase = if ($IsPost) { "post" } else { "pre" } + if ($IsPost) { + $remoteBody = "MOLD_BACKUP_CONF=/etc/ablestack/veeam/Mold_Guest_Backup.conf VM_INCLUDE='${escapedVm}' VM_NAME='${escapedVm}' bash '${escapedRemote}' \`"`$CLIENT\`" '${escapedJob}' default" + } else { + $remoteBody = "MOLD_BACKUP_CONF=/etc/ablestack/veeam/Mold_Guest_Backup.conf VM_INCLUDE='${escapedVm}' bash '${escapedRemote}' \`"`$CLIENT\`" '${escapedJob}' default '${escapedVm}'" + } + $content = @" +#!/bin/bash +# Veeam Guest Processing: SSH from guest VM to KVM for Mold pre/post-notify. +GUEST_LOG=/var/log/mold/guest-veeam-hook.log +mkdir -p /var/log/mold 2>/dev/null || true +log() { echo "[`$(date '+%Y-%m-%d %H:%M:%S')] [$hookPhase] `$*" >>"`$GUEST_LOG"; } +SSH=/usr/bin/ssh +[[ -x "`$SSH" ]] || SSH=ssh +CLIENT="`$(hostname -s 2>/dev/null || hostname)" +log "start job=${escapedJob} vm=${escapedVm} kvm=${escapedKvm} client=`$CLIENT" +`$SSH -o BatchMode=yes -o ConnectTimeout=30 -o StrictHostKeyChecking=accept-new '${escapedUser}@${escapedKvm}' \ + "$remoteBody" +rc=`$? +if [[ `$rc -ne 0 ]]; then + log "kvm notify failed rc=`$rc" + exit `$rc +fi +log "kvm notify ok" +exit 0 +"@ + $utf8NoBom = New-Object System.Text.UTF8Encoding $false + [System.IO.File]::WriteAllText($Path, $content.Replace("`r`n", "`n"), $utf8NoBom) +} + +function Clear-MoldVeeamGuestJobIncompatibleOptions { + param( + $AgentJob, + $BackupObject + ) + + if ($BackupObject) { + try { + $indexDisable = New-VBRComputerIndexingOptions ` + -BackupObject $BackupObject ` + -OSPlatform Linux ` + -IndexingMode Disable + Set-VBRComputerBackupJob -Job $AgentJob ` + -EnableIndexing:$false ` + -IndexingOptions @($indexDisable) | Out-Null + Write-Host "Disabled guest file indexing (SelectedFiles incompatible with IndexIncludedOnly)." + } catch { + Write-Warning "Disable indexing: $($_.Exception.Message)" + } + } else { + try { + Set-VBRComputerBackupJob -Job $AgentJob -EnableIndexing:$false | Out-Null + } catch { + Write-Warning "Disable indexing (no BackupObject): $($_.Exception.Message)" + } + } + + try { + Set-VBRComputerBackupJob -Job $AgentJob -EnableApplicationProcessing:$false | Out-Null + } catch { + Write-Warning "Disable ApplicationProcessing: $($_.Exception.Message)" + } +} + +function Register-MoldVeeamGuestSelectedFilesScripts { + param( + $AgentJob, + [string]$VmIp, + $DiscoveredComputer, + [string]$PreWrapper, + [string]$PostWrapper + ) + + $backupObject = $DiscoveredComputer + if (-not $backupObject) { + $backupObject = $AgentJob.BackupObject + } + if ($backupObject -is [System.Array]) { + $backupObject = $backupObject | Select-Object -First 1 + } + if (-not $backupObject) { + throw "BackupObject not found for guest VM ${VmIp}." + } + + try { + Rescan-VBREntity -Entity $backupObject | Out-Null + } catch { + Write-Warning "Rescan backup object failed (continuing): $($_.Exception.Message)" + } + + Clear-MoldVeeamGuestJobIncompatibleOptions -AgentJob $AgentJob -BackupObject $backupObject + + $indexDisable = New-VBRComputerIndexingOptions ` + -BackupObject $backupObject ` + -OSPlatform Linux ` + -IndexingMode Disable + + # SelectedFiles: pre-job/post-job (not pre-freeze/post-thaw). ScriptOptions is for Windows agents. + $scriptProcessing = New-VBRScriptProcessingOptions ` + -ProcessingAction RequireSuccess ` + -ScriptPreJobCommand $PreWrapper ` + -ScriptPostJobCommand $PostWrapper + + $appProcessing = New-VBRApplicationProcessingOptions ` + -Enable ` + -OSPlatform Linux ` + -BackupObject $backupObject ` + -ScriptProcessingOptions $scriptProcessing + + $disabledJobScripts = New-VBRJobScriptOptions -PreScriptEnabled:$false -PostScriptEnabled:$false + + try { + Set-VBRComputerBackupJob -Job $AgentJob ` + -ScriptOptions $disabledJobScripts ` + -EnableIndexing:$false ` + -IndexingOptions @($indexDisable) ` + -EnableApplicationProcessing ` + -ApplicationProcessingOptions @($appProcessing) | Out-Null + } catch { + throw @" +SelectedFiles script registration failed for ${VmIp}: $($_.Exception.Message) + +If indexing was enabled in Veeam UI, open Job -> Guest Processing: + 1) Indexing tab -> disable file indexing + 2) Scripts tab -> Job scripts: + Pre-job : $PreWrapper + Post-job: $PostWrapper +Then re-run push-to-veeam.sh +"@ + } + + Write-Host "Registered pre-job/post-job on SelectedFiles guest job ${VmIp} (runs on guest -> SSH KVM for Mold)." + Write-Host " Pre-job : $PreWrapper" + Write-Host " Post-job : $PostWrapper" + Write-Host " On guest, Veeam uploads to /var/lib/veeam/scripts/ and runs as root." + Write-Host " Guest log: /var/log/mold/guest-veeam-hook.log on ${VmIp}" +} + +function Register-MoldVeeamGuestVmGuestScripts { + param( + $AgentJob, + [string]$JobName, + [string]$InstallDir, + [string]$KvmHost, + [string]$KvmSshUser, + [string]$VmLibvirtName, + [string]$VmIp, + $DiscoveredComputer, + [string]$AgentPreNotifyScript, + [string]$AgentPostNotifyScript + ) + + if (-not $VmLibvirtName -or -not $VmIp) { + throw "GuestVm mode requires -VmLibvirtName and -VmIp" + } + if (-not $KvmHost) { + throw "GuestVm mode requires -KvmHost (Mold trigger on hypervisor)" + } + + $safeVm = ($VmLibvirtName -replace '[^\w\-]', '_') + $preDeployed = Join-Path $InstallDir "mold-guest-pre-${safeVm}.sh" + $postDeployed = Join-Path $InstallDir "mold-guest-post-${safeVm}.sh" + if ((Test-Path $preDeployed) -and (Test-Path $postDeployed)) { + $preWrapper = $preDeployed + $postWrapper = $postDeployed + Write-Host "Using deployed guest wrappers from KVM (libvirt name):" + } else { + $preWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "pre" -VmLibvirtName $VmLibvirtName + $postWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "post" -VmLibvirtName $VmLibvirtName + Write-MoldGuestKvmSshWrapperScript -Path $preWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPreNotifyScript + Write-MoldGuestKvmSshWrapperScript -Path $postWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPostNotifyScript -IsPost + Write-Host "Created guest KVM-SSH wrappers (run on VM ${VmIp}, Mold on ${KvmHost}):" + } + Write-Host " $preWrapper" + Write-Host " $postWrapper" + + $backupType = "" + foreach ($key in @("BackupType", "Type", "BackupJobType")) { + if ($AgentJob.PSObject.Properties.Name -contains $key -and $AgentJob.$key) { + $backupType = "$($AgentJob.$key)" + break + } + } + + # SelectedFiles (file-level) Linux jobs: pre-job/post-job via Guest Processing scripts. + if ($backupType -eq "SelectedFiles") { + Register-MoldVeeamGuestSelectedFilesScripts ` + -AgentJob $AgentJob ` + -VmIp $VmIp ` + -DiscoveredComputer $DiscoveredComputer ` + -PreWrapper $preWrapper ` + -PostWrapper $postWrapper + return + } + + $backupObject = $DiscoveredComputer + if (-not $backupObject) { + $backupObject = $AgentJob.BackupObject + } + if ($backupObject -is [System.Array]) { + $backupObject = $backupObject | Select-Object -First 1 + } + if (-not $backupObject) { + throw "Job BackupObject not found for guest VM ${VmIp}. Re-run create-veeam-guest-vm-jobs.ps1 after agent rescan." + } + + try { + Rescan-VBREntity -Entity $backupObject | Out-Null + } catch { + Write-Warning "Rescan backup object failed (continuing): $($_.Exception.Message)" + } + + $scriptProcessing = New-VBRScriptProcessingOptions ` + -ProcessingAction RequireSuccess ` + -ScriptPrefreezeCommand $preWrapper ` + -ScriptPostthawCommand $postWrapper + + $appProcessing = New-VBRApplicationProcessingOptions ` + -Enable ` + -OSPlatform Linux ` + -BackupObject $backupObject ` + -ScriptProcessingOptions $scriptProcessing + + $disabledJobScripts = New-VBRJobScriptOptions -PreScriptEnabled:$false -PostScriptEnabled:$false + + try { + Set-VBRComputerBackupJob -Job $AgentJob ` + -ScriptOptions $disabledJobScripts ` + -EnableApplicationProcessing ` + -ApplicationProcessingOptions @($appProcessing) | Out-Null + } catch { + throw "Guest Processing registration failed for ${VmIp}: $($_.Exception.Message). For SelectedFiles jobs use pre-job/post-job (re-run push-to-veeam.sh with updated install-veeam-job.ps1)." + } + + Write-Host "Registered Guest Processing pre-freeze/post-thaw on VM ${VmIp} (runs on guest -> SSH KVM for Mold)." + Write-Host " Pre-freeze : $preWrapper" + Write-Host " Post-thaw : $postWrapper" + Write-Host " On guest, Veeam uploads to /var/lib/veeam/scripts/ and runs as root." + Write-Host " Guest log: /var/log/mold/guest-veeam-hook.log on ${VmIp}" +} + +function Register-MoldVeeamLinuxAgentGuestScripts { + param( + $AgentJob, + [string]$JobName, + [string]$InstallDir, + [string]$ProtectionGroupName, + [string]$AgentPreNotifyScript, + [string]$AgentPostNotifyScript + ) + + $safeJob = ($JobName -replace '[^\w\-]', '_') + $preWrapper = Join-Path $InstallDir "mold-agent-pre-$safeJob.sh" + $postWrapper = Join-Path $InstallDir "mold-agent-post-$safeJob.sh" + Write-MoldAgentWrapperScript -Path $preWrapper -AgentScript $AgentPreNotifyScript -JobName $JobName + Write-MoldAgentWrapperScript -Path $postWrapper -AgentScript $AgentPostNotifyScript -JobName $JobName + Write-Host "Created agent wrapper scripts (uploaded to KVM by Veeam on job run):" + Write-Host " $preWrapper" + Write-Host " $postWrapper" + + $backupObject = $null + if ($ProtectionGroupName) { + $backupObject = Get-VBRProtectionGroup -Name $ProtectionGroupName -ErrorAction SilentlyContinue + } + if (-not $backupObject -and $AgentJob.BackupObject) { + $backupObject = @($AgentJob.BackupObject) | Select-Object -First 1 + } + if (-not $backupObject) { + throw "Protection group not found for Guest Processing scripts. Pass -ProtectionGroupName or fix job BackupObject." + } + + if ($backupObject -is [System.Array]) { + $backupObject = $backupObject | Select-Object -First 1 + } + + Write-Host "Rescanning protection group before guest script registration..." + Rescan-VBREntity -Entity $backupObject | Out-Null + + # Pre-freeze/post-thaw run ON the Linux guest (Veeam uploads to /var/lib/veeam/scripts/). + $scriptProcessing = New-VBRScriptProcessingOptions ` + -ProcessingAction RequireSuccess ` + -ScriptPrefreezeCommand $preWrapper ` + -ScriptPostthawCommand $postWrapper + + $appProcessing = New-VBRApplicationProcessingOptions ` + -Enable ` + -OSPlatform Linux ` + -BackupObject $backupObject ` + -ScriptProcessingOptions $scriptProcessing + + # Job-level scripts (Storage -> Advanced -> Scripts) run on Windows B&R — disable them. + $disabledJobScripts = New-VBRJobScriptOptions -PreScriptEnabled:$false -PostScriptEnabled:$false + + Set-VBRComputerBackupJob -Job $AgentJob ` + -ScriptOptions $disabledJobScripts ` + -EnableApplicationProcessing ` + -ApplicationProcessingOptions @($appProcessing) | Out-Null + + Write-Host "Registered Guest Processing pre/post (run on KVM agent, not this Windows server)." +} + +if ($ScriptsOnly) { + # Generate ONLY the pre/post wrapper scripts on this Veeam server. + # No Veeam module, no job lookup, no job registration. Point the Veeam UI + # (Job -> Guest Processing -> Scripts) at the generated files manually. + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + $safeJob = ($JobName -replace '[^\w\-]', '_') + + if ($GuestVm.IsPresent) { + if (-not $VmLibvirtName -or -not $VmIp) { + throw "ScriptsOnly + GuestVm requires -VmLibvirtName and -VmIp" + } + if (-not $KvmHost) { + throw "ScriptsOnly + GuestVm requires -KvmHost" + } + $preWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "pre" -VmLibvirtName $VmLibvirtName + $postWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "post" -VmLibvirtName $VmLibvirtName + Write-MoldGuestKvmSshWrapperScript -Path $preWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPreNotifyScript + Write-MoldGuestKvmSshWrapperScript -Path $postWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPostNotifyScript -IsPost + Write-Host "Generated guest KVM-SSH wrapper scripts (runs on VM ${VmIp}, Mold on ${KvmHost}):" + } else { + $preWrapper = Join-Path $InstallDir "mold-agent-pre-$safeJob.sh" + $postWrapper = Join-Path $InstallDir "mold-agent-post-$safeJob.sh" + Write-MoldAgentWrapperScript -Path $preWrapper -AgentScript $AgentPreNotifyScript -JobName $JobName + Write-MoldAgentWrapperScript -Path $postWrapper -AgentScript $AgentPostNotifyScript -JobName $JobName + Write-Host "Generated agent wrapper scripts (uploaded to the agent host by Veeam on job run):" + } + + Write-Host " Pre-freeze : $preWrapper" + Write-Host " Post-thaw : $postWrapper" + Write-Host "" + Write-Host "Next (Veeam UI): Job -> Guest Processing -> Applications -> (select VM) Edit ->" + Write-Host " Scripts tab -> Pre-freeze script = $preWrapper" + Write-Host " Post-thaw script = $postWrapper" + return +} + +$files = @( + "veeam-job-pre-backup.ps1", + "veeam-job-post-backup.ps1", + "veeam-job-post-restore.ps1", + "mold-backup.windows.conf.default" +) + +New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + +$sourceRoot = [System.IO.Path]::GetFullPath($SourceDir).TrimEnd('\', '/') +$installRoot = [System.IO.Path]::GetFullPath($InstallDir).TrimEnd('\', '/') +$skipCopy = ($sourceRoot -ieq $installRoot) +if ($skipCopy) { + Write-Host "SourceDir equals InstallDir ($installRoot); skipping file copy." +} + +# Linux Agent: pre/post run on KVM (bash). Windows PS1 bundle is optional for guest/Linux agent jobs. +$requireWindowsBundle = (-not $LinuxAgent.IsPresent) -and (-not $GuestVm.IsPresent) + +if ($GuestVm.IsPresent -and [string]::IsNullOrWhiteSpace($SourceDir)) { + $SourceDir = $InstallDir +} +if ($GuestVm.IsPresent -and -not (Test-Path (Join-Path $SourceDir "veeam-job-pre-backup.ps1"))) { + if (Test-Path (Join-Path $InstallDir "veeam-job-pre-backup.ps1")) { + $SourceDir = $InstallDir + } +} + +foreach ($f in $files) { + $src = Join-Path $SourceDir $f + $dest = Join-Path $InstallDir $f + if (-not (Test-Path $src)) { + if ($requireWindowsBundle) { + throw "Missing file: $src" + } + Write-Warning "Skipping optional file (Linux Agent mode): $src" + continue + } + if ($skipCopy) { + continue + } + $srcFull = [System.IO.Path]::GetFullPath($src) + $destFull = [System.IO.Path]::GetFullPath($dest) + if ($srcFull -ieq $destFull) { + continue + } + Copy-Item -Path $src -Destination $dest -Force +} + +$confPath = Join-Path $InstallDir "mold-backup.windows.conf" +$confDefault = Join-Path $InstallDir "mold-backup.windows.conf.default" +if (-not (Test-Path $confPath) -and (Test-Path $confDefault)) { + Copy-Item $confDefault $confPath + Write-Host "Created $confPath — edit VM_NAME, STAGING_PATH, KVM_HOST if needed." +} + +function Set-ConfValue { + param([string]$Key, [string]$Value) + if ([string]::IsNullOrWhiteSpace($Value)) { return } + $lines = Get-Content $confPath + $found = $false + $out = foreach ($line in $lines) { + if ($line -match "^\s*$([regex]::Escape($Key))\s*=") { + $found = $true + "$Key=`"$Value`"" + } else { $line } + } + if (-not $found) { $out += "$Key=`"$Value`"" } + Set-Content -Path $confPath -Value $out -Encoding UTF8 +} + +Set-ConfValue -Key "VEEAM_JOB_NAME" -Value $JobName +Set-ConfValue -Key "VM_NAME" -Value $VmName +Set-ConfValue -Key "STAGING_PATH" -Value $StagingPath +Set-ConfValue -Key "KVM_HOST" -Value $KvmHost +Set-ConfValue -Key "KVM_SSH_USER" -Value $KvmSshUser +Set-ConfValue -Key "KVM_SSH_KEY" -Value $KvmSshKey + +$prePs1 = Join-Path $InstallDir "veeam-job-pre-backup.ps1" +$postPs1 = Join-Path $InstallDir "veeam-job-post-backup.ps1" +$restorePs1 = Join-Path $InstallDir "veeam-job-post-restore.ps1" + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + +$agentJob = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue +$job = $null +if (-not $agentJob) { + $job = Get-VBRJob -Name $JobName -ErrorAction SilentlyContinue +} +if (-not $agentJob -and -not $job) { + Write-Host "Available Agent backup jobs:" + Get-VBRComputerBackupJob | Select-Object -ExpandProperty Name + $createScript = Join-Path $InstallDir "create-veeam-agent-job.ps1" + if (-not (Test-Path $createScript)) { + $createScript = Join-Path $PSScriptRoot "create-veeam-agent-job.ps1" + } + throw @" +Job not found: $JobName + +Create the Agent backup job first (run on this Veeam B&R server): + pwsh -File '$createScript' -JobName '$JobName' -KvmHost '' -AgentHostName '' + +Or from Git repo / ccvm: + bash push-to-veeam.sh --env-file mold-backup.env +"@ +} + +# Linux Agent: pre/post scripts run ON THE AGENT HOST (bash), not on this Windows B&R server. +$useLinuxAgentScripts = $LinuxAgent.IsPresent +if ($agentJob -and -not $useLinuxAgentScripts) { + $osType = $agentJob.OSPlatform + if ($osType -eq "Linux") { $useLinuxAgentScripts = $true } +} +if ($useLinuxAgentScripts -and -not $GuestVm.IsPresent) { + Write-Host "Linux Agent job: Guest Processing scripts -> KVM (Mold VM backup pre/post)." + Write-Host "Ensure scripts exist on KVM:" + Write-Host " $AgentPreNotifyScript" + Write-Host " $AgentPostNotifyScript" + Write-Host "On KVM: /etc/veeam/veeam.ini [scripts] timeoutPrePost = 1800 ; systemctl restart veeamservice" +} else { + $psExe = "pwsh.exe" + if (-not (Get-Command $psExe -ErrorAction SilentlyContinue)) { + Write-Warning "pwsh.exe not found; falling back to powershell.exe (Veeam module may fail on PS 5.1)" + $psExe = "powershell.exe" + } + $preCmd = "$psExe -ExecutionPolicy Bypass -NoProfile -File `"$prePs1`"" + $postCmd = "$psExe -ExecutionPolicy Bypass -NoProfile -File `"$postPs1`"" + $restoreCmd = "$psExe -ExecutionPolicy Bypass -NoProfile -File `"$restorePs1`"" +} + +if ($agentJob) { + if ($GuestVm.IsPresent) { + Register-MoldVeeamGuestVmGuestScripts ` + -AgentJob $agentJob ` + -JobName $JobName ` + -InstallDir $InstallDir ` + -KvmHost $KvmHost ` + -KvmSshUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName ` + -VmIp $VmIp ` + -DiscoveredComputer $DiscoveredComputer ` + -AgentPreNotifyScript $AgentPreNotifyScript ` + -AgentPostNotifyScript $AgentPostNotifyScript + $preCmd = "Guest VM ${VmIp} -> SSH KVM Mold pre" + $postCmd = "Guest VM ${VmIp} -> SSH KVM Mold post" + } elseif ($useLinuxAgentScripts) { + Register-MoldVeeamLinuxAgentGuestScripts ` + -AgentJob $agentJob ` + -JobName $JobName ` + -InstallDir $InstallDir ` + -ProtectionGroupName $ProtectionGroupName ` + -AgentPreNotifyScript $AgentPreNotifyScript ` + -AgentPostNotifyScript $AgentPostNotifyScript + $preCmd = "Guest Processing pre-job -> KVM ($AgentPreNotifyScript)" + $postCmd = "Guest Processing post-job -> KVM ($AgentPostNotifyScript)" + } else { + $scriptOptions = New-VBRJobScriptOptions ` + -PreScriptEnabled ` + -PreCommand $preCmd ` + -PostScriptEnabled ` + -PostCommand $postCmd ` + -Periodicity Cycles ` + -Frequency 1 + + Set-VBRComputerBackupJob -Job $agentJob -ScriptOptions $scriptOptions | Out-Null + Write-Host "Registered Pre/Post scripts on Agent backup job: $JobName" + } +} elseif ($job) { + $opts = Get-VBRJobOptions -Job $job + $base = $opts.JobScriptOptions + if (-not $base) { + $base = New-VBRJobScriptOptions + } + $newScript = Set-VBRJobScriptOptions -JobScriptOptions $base ` + -PreScriptEnabled -PreCommand $preCmd ` + -PostScriptEnabled -PostCommand $postCmd ` + -Periodicity Cycles -Frequency 1 + $opts.JobScriptOptions = $newScript + Set-VBRJobOptions -Job $job -Options $opts | Out-Null + Write-Host "Registered Pre/Post scripts on backup job: $JobName" +} + +Write-Host "" +Write-Host "Installed to: $InstallDir" +Write-Host " Pre-job : $preCmd" +Write-Host " Post-job: $postCmd" +Write-Host "" +Write-Host "Verify in Veeam UI: Job -> Guest Processing -> Processing Settings -> Scripts (Pre-job / Post-job)" +if ($GuestVm.IsPresent) { + Write-Host "Guest VM ${VmIp}: ensure passwordless ssh ${KvmSshUser}@${KvmHost}" + Write-Host "SelectedFiles jobs: Veeam UI -> Job -> Storage -> Advanced -> Scripts (Pre-job / Post-job)" +} elseif ($useLinuxAgentScripts) { + Write-Host "KVM host: run veeam_config.sh then ensure Veeam job backs up /tmp/mold/veeam" +} else { + Write-Host "KVM host: run veeam_config.sh for Mold integration" +} +if ($EnableRestoreScript) { + Write-Host "" + Write-Host "Post-restore script (run manually after Veeam restore completes):" + Write-Host " $restoreCmd" + Write-Host "Set BACKUP_ID in mold-backup.windows.conf before running." +} diff --git a/scripts/vm/hypervisor/kvm/veeam/install.sh b/scripts/vm/hypervisor/kvm/veeam/install.sh new file mode 100755 index 000000000000..a40a4124958e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/install.sh @@ -0,0 +1,168 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Install Ablestack Veeam backup hooks on KVM host (mold-agent post-install). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +LEGACY_ETC="/etc/mold/backup/veeam" +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" +LOG_DIR="/var/log/mold" +ABLESTACK_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + +install -d -m 0755 "${SHARE_DIR}" "${ETC_DIR}" "${ETC_DIR}/secrets" "${ETC_DIR}/state" "${ETC_DIR}/hooks" "${ETC_DIR}/registry" "${LOG_DIR}" + +ABLESTACK_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE}" bash "${SCRIPT_DIR}/install-ablestack-secret-key.sh" +if [[ -f "${SCRIPT_DIR}/ablestack.key.default" ]]; then + install -m 0644 "${SCRIPT_DIR}/ablestack.key.default" "${SHARE_DIR}/ablestack.key.default" + install -m 0644 "${SCRIPT_DIR}/ablestack.key.default" "${ETC_DIR}/ablestack.key.default" +fi + +for f in install.sh install-ablestack-secret-key.sh mold-backup.lib.sh mold-backup-secret.sh mold-backup.sh veeam_config.sh \ + ablestack_veeam_pre_notify.sh ablestack_veeam_post_notify.sh ablestack_veeam_restore_notify.sh ablestack_veeam_restore_event.sh \ + mold-veeam-trigger-hook.sh; do + [[ -f "${SCRIPT_DIR}/${f}" ]] || continue + install -m 0755 "${SCRIPT_DIR}/${f}" "${ETC_DIR}/${f}" + install -m 0755 "${SCRIPT_DIR}/${f}" "${SHARE_DIR}/${f}" 2>/dev/null || true +done + +CVT_SRC="" +for _cvt_candidate in \ + "${ABLESTACK_CVT_BACKUP_SRC:-}" \ + "${SCRIPT_DIR}/ablestack_cvtbackup.sh" \ + "${SCRIPT_DIR}/../ablestack_cvtbackup.sh" \ + "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_cvtbackup.sh"; do + [[ -n "${_cvt_candidate}" && -f "${_cvt_candidate}" ]] || continue + CVT_SRC="${_cvt_candidate}" + break +done +if [[ -n "${CVT_SRC}" ]]; then + install -m 0755 "${CVT_SRC}" "${ETC_DIR}/ablestack_cvtbackup.sh" + install -m 0755 "${CVT_SRC}" "${SHARE_DIR}/ablestack_cvtbackup.sh" + CS_CVT_DIR="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm" + if [[ -d "${CS_CVT_DIR}" ]]; then + install -m 0755 "${CVT_SRC}" "${CS_CVT_DIR}/ablestack_cvtbackup.sh" + fi + echo "Installed host export script: ${ETC_DIR}/ablestack_cvtbackup.sh (from ${CVT_SRC})" +else + echo "ERROR: ablestack_cvtbackup.sh not found." >&2 + echo " Copy scripts/vm/hypervisor/kvm/veeam/ (includes ablestack_cvtbackup.sh) or set ABLESTACK_CVT_BACKUP_SRC=/path/to/ablestack_cvtbackup.sh" >&2 + exit 1 +fi + +NAS_SRC="" +for _nas_candidate in \ + "${ABLESTACK_NAS_BACKUP_SRC:-}" \ + "${SCRIPT_DIR}/ablestack_nasbackup.sh" \ + "${SCRIPT_DIR}/../ablestack_nasbackup.sh" \ + "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh"; do + [[ -n "${_nas_candidate}" && -f "${_nas_candidate}" ]] || continue + NAS_SRC="${_nas_candidate}" + break +done +if [[ -n "${NAS_SRC}" ]]; then + install -m 0755 "${NAS_SRC}" "${ETC_DIR}/ablestack_nasbackup.sh" + install -m 0755 "${NAS_SRC}" "${SHARE_DIR}/ablestack_nasbackup.sh" + CS_KVM_DIR="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm" + install -d -m 0755 "${CS_KVM_DIR}" + install -m 0755 "${NAS_SRC}" "${CS_KVM_DIR}/ablestack_nasbackup.sh" + echo "Installed NAS backup script: ${CS_KVM_DIR}/ablestack_nasbackup.sh (from ${NAS_SRC})" +else + echo "WARN: ablestack_nasbackup.sh not found — import-veeam-seed requires agent + this script" >&2 +fi + +install -m 0644 "${SCRIPT_DIR}/mold-ms-backup-schema-fix.sql" "${SHARE_DIR}/mold-ms-backup-schema-fix.sql" 2>/dev/null || true +install -m 0644 "${SCRIPT_DIR}/mold-ms-backup-schema-fix.sql" "${ETC_DIR}/mold-ms-backup-schema-fix.sql" 2>/dev/null || true + +install -m 0644 "${SCRIPT_DIR}/mold-backup.conf.default" "${SHARE_DIR}/mold-backup.conf.default" +install -m 0644 "${SCRIPT_DIR}/mold-backup.env.example" "${SHARE_DIR}/mold-backup.env.example" +if [[ ! -f "${ETC_DIR}/mold-backup.env" ]]; then + install -m 0600 "${SCRIPT_DIR}/mold-backup.env.example" "${ETC_DIR}/mold-backup.env" + echo "Created ${ETC_DIR}/mold-backup.env — set MOLD_API_KEY / MOLD_API_SECRET, then veeam_config.sh --job-name ..." +else + echo "Keeping existing ${ETC_DIR}/mold-backup.env" +fi +install -m 0644 "${SCRIPT_DIR}/mold-backup.windows.conf.default" "${SHARE_DIR}/mold-backup.windows.conf.default" +install -m 0644 "${SCRIPT_DIR}/veeam-job-pre-backup.ps1" "${SHARE_DIR}/veeam-job-pre-backup.ps1" +install -m 0644 "${SCRIPT_DIR}/veeam-job-post-backup.ps1" "${SHARE_DIR}/veeam-job-post-backup.ps1" +install -m 0644 "${SCRIPT_DIR}/veeam-job-post-restore.ps1" "${SHARE_DIR}/veeam-job-post-restore.ps1" +install -m 0644 "${SCRIPT_DIR}/install-veeam-job.ps1" "${SHARE_DIR}/install-veeam-job.ps1" +install -m 0644 "${SCRIPT_DIR}/create-veeam-agent-job.ps1" "${SHARE_DIR}/create-veeam-agent-job.ps1" 2>/dev/null || true +install -m 0644 "${SCRIPT_DIR}/setup-veeam-mold-job.ps1" "${SHARE_DIR}/setup-veeam-mold-job.ps1" 2>/dev/null || true +install -m 0755 "${SCRIPT_DIR}/setup-datadisk-veeam-backup.sh" "${ETC_DIR}/setup-datadisk-veeam-backup.sh" 2>/dev/null || true +install -m 0644 "${SCRIPT_DIR}/setup-veeam-host-repo.ps1" "${SHARE_DIR}/setup-veeam-host-repo.ps1" 2>/dev/null || true +install -m 0755 "${SCRIPT_DIR}/mold-guest-common.sh" "${ETC_DIR}/mold-guest-common.sh" +install -m 0755 "${SCRIPT_DIR}/push-to-veeam.sh" "${SHARE_DIR}/push-to-veeam.sh" 2>/dev/null || true +install -m 0755 "${SCRIPT_DIR}/push-to-veeam.sh" "${ETC_DIR}/push-to-veeam.sh" 2>/dev/null || true + +if [[ -f "${SCRIPT_DIR}/README.ko.md" ]]; then + install -m 0644 "${SCRIPT_DIR}/README.ko.md" "${SHARE_DIR}/README.ko.md" +fi + +if [[ ! -f "${ETC_DIR}/mold-backup.conf" ]]; then + install -m 0600 "${SCRIPT_DIR}/mold-backup.conf.default" "${ETC_DIR}/mold-backup.conf" + echo "Created ${ETC_DIR}/mold-backup.conf — run veeam_config.sh for job-based setup." +else + echo "Keeping existing ${ETC_DIR}/mold-backup.conf" +fi + +# Backward-compatible symlink for older paths +if [[ ! -e "${LEGACY_ETC}" ]]; then + ln -sfn "${ETC_DIR}" "${LEGACY_ETC}" 2>/dev/null || true +fi + +mkdir -p "${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" 2>/dev/null || true + +install_restore_agent_units() { + local unit_dir="/etc/systemd/system" + [[ -f "${SCRIPT_DIR}/mold-veeam-restore-agent.service" ]] || return 0 + for f in mold-veeam-restore-agent.sh enable-veeam-mold-restore.sh; do + [[ -f "${SCRIPT_DIR}/${f}" ]] || continue + install -m 0755 "${SCRIPT_DIR}/${f}" "${ETC_DIR}/${f}" + install -m 0755 "${SCRIPT_DIR}/${f}" "${SHARE_DIR}/${f}" 2>/dev/null || true + done + for f in mold-veeam-restore-agent.service mold-veeam-restore-agent.timer; do + [[ -f "${SCRIPT_DIR}/${f}" ]] || continue + install -m 0644 "${SCRIPT_DIR}/${f}" "${ETC_DIR}/${f}" + install -m 0644 "${SCRIPT_DIR}/${f}" "${SHARE_DIR}/${f}" 2>/dev/null || true + done + install -m 0644 "${SCRIPT_DIR}/mold-veeam-restore-agent.service" "${unit_dir}/mold-veeam-restore-agent.service" + install -m 0644 "${SCRIPT_DIR}/mold-veeam-restore-agent.timer" "${unit_dir}/mold-veeam-restore-agent.timer" + if command -v systemctl >/dev/null 2>&1; then + systemctl daemon-reload 2>/dev/null || true + systemctl disable mold-veeam-restore-watch.timer 2>/dev/null || true + systemctl stop mold-veeam-restore-watch.timer 2>/dev/null || true + systemctl enable mold-veeam-restore-agent.timer 2>/dev/null || true + systemctl start mold-veeam-restore-agent.timer 2>/dev/null || true + echo "Enabled mold-veeam-restore-agent.timer (3min poll → Mold restoreBackup on Veeam FLR)" + echo " Run: ${ETC_DIR}/enable-veeam-mold-restore.sh --vm-include 'i-2-XX-VM'" + fi +} +install_restore_agent_units + +echo "Installed Ablestack Veeam backup hooks (host/datadisk mode):" +echo " Active: ${ETC_DIR}/" +echo " Reference: ${SHARE_DIR}/" +echo " Secret key: ${ABLESTACK_SECRET_KEY_FILE}" +echo " Host backup path: /tmp/mold/veeam" +echo " Configure: veeam_config.sh --job-name ... --install" +echo " Datadisk setup: ${ETC_DIR}/setup-datadisk-veeam-backup.sh" +echo " FLR→Mold: ${ETC_DIR}/enable-veeam-mold-restore.sh" +echo " Veeam PS1/Job: ${ETC_DIR}/push-to-veeam.sh" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh b/scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh new file mode 100755 index 000000000000..c13c603311d6 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh @@ -0,0 +1,151 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Encrypt/decrypt Mold API secret for mold-backup hooks (AES-256-CBC + PBKDF2). +# Default host key: /root/.ssh/ablestack.key (base64-encoded passphrase, chmod 600). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OPENSSL_CIPHER="aes-256-cbc" +DEFAULT_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + +usage() { + cat < --key-file [--out ] + mold-backup-secret.sh decrypt --enc-file --key-file + mold-backup-secret.sh gen-key --key-file + +Files: + --key-file Passphrase file (chmod 600). Default on hosts: ${DEFAULT_KEY_FILE} + Ablestack installs a base64-encoded passphrase in this file; the script + decodes base64 when valid, otherwise uses the file contents as-is. + --enc-file Encrypted secret blob (base64 OpenSSL output). + --out Output path for encrypt (default: same dir as key-file, api-secret.enc) +EOF +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +require_openssl() { + command -v openssl >/dev/null 2>&1 || die "openssl is required" +} + +# Read passphrase from key file (base64 one-liner or raw text). +mold_secret_read_passphrase() { + local key_file="$1" + local raw decoded + raw="$(tr -d '\n\r' < "$key_file")" + [[ -n "$raw" ]] || die "Key file is empty: $key_file" + if decoded="$(printf '%s' "$raw" | base64 -d 2>/dev/null)" && [[ -n "$decoded" ]]; then + printf '%s' "$decoded" + return 0 + fi + printf '%s' "$raw" +} + +mold_secret_openssl_passfile() { + local pass="$1" + local tmp + tmp="$(mktemp)" + chmod 600 "$tmp" + printf '%s' "$pass" > "$tmp" + echo "$tmp" +} + +mold_secret_encrypt() { + local secret="" key_file="" out_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --secret) secret="$2"; shift 2 ;; + --key-file) key_file="$2"; shift 2 ;; + --out) out_file="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + [[ -n "$secret" ]] || die "--secret is required" + [[ -n "$key_file" ]] || die "--key-file is required" + [[ -f "$key_file" ]] || die "Key file not found: $key_file" + if [[ -z "$out_file" ]]; then + out_file="$(dirname "$key_file")/api-secret.enc" + fi + require_openssl + install -d -m 0700 "$(dirname "$out_file")" + local pass pass_file + pass="$(mold_secret_read_passphrase "$key_file")" + pass_file="$(mold_secret_openssl_passfile "$pass")" + echo -n "$secret" | openssl enc -"${OPENSSL_CIPHER}" -salt -pbkdf2 -pass "file:${pass_file}" -base64 -out "$out_file" + rm -f "$pass_file" + chmod 0600 "$out_file" + echo "$out_file" +} + +mold_secret_decrypt() { + local enc_file="" key_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --enc-file) enc_file="$2"; shift 2 ;; + --key-file) key_file="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + [[ -f "$enc_file" ]] || die "Encrypted file not found: $enc_file" + [[ -f "$key_file" ]] || die "Key file not found: $key_file" + require_openssl + local pass pass_file + pass="$(mold_secret_read_passphrase "$key_file")" + pass_file="$(mold_secret_openssl_passfile "$pass")" + local plain + plain="$(openssl enc -"${OPENSSL_CIPHER}" -d -salt -pbkdf2 -pass "file:${pass_file}" -base64 -in "$enc_file" 2>/dev/null)" \ + || die "Decrypt failed (wrong key file or corrupt enc file): $enc_file" + rm -f "$pass_file" + # Strip trailing newline — OpenSSL may add one; breaks CloudStack API HMAC. + plain="${plain//$'\r'/}" + plain="${plain%"${plain##*[![:space:]]}"}" + printf '%s' "$plain" +} + +mold_secret_gen_key() { + local key_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --key-file) key_file="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + [[ -n "$key_file" ]] || die "--key-file is required" + require_openssl + install -d -m 0700 "$(dirname "$key_file")" + if [[ -f "$key_file" ]]; then + die "Key file already exists: $key_file (refuse to overwrite)" + fi + openssl rand -base64 32 > "$key_file" + chmod 0600 "$key_file" + echo "Created key file: $key_file" +} + +cmd="${1:-}" +shift || true +case "$cmd" in + encrypt) mold_secret_encrypt "$@" ;; + decrypt) mold_secret_decrypt "$@" ;; + gen-key) mold_secret_gen_key "$@" ;; + -h|--help|help|"") usage ;; + *) die "Unknown command: $cmd (use --help)" ;; +esac diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default b/scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default new file mode 100644 index 000000000000..2b7cb8a7398b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default @@ -0,0 +1,79 @@ +# Ablestack Veeam + Mold backup configuration (NetBackup-style) +# Per-job config: /etc/ablestack/veeam/.conf (created by veeam_config.sh) + +# --- API --- +MOLD_API_URL="https://127.0.0.1:8080/client/api" +MOLD_API_KEY="" +MOLD_API_SECRET="" +MOLD_API_SECRET_ENC_FILE="/etc/ablestack/veeam/secrets/secret.enc" +MOLD_SECRET_KEY_FILE="/root/.ssh/ablestack.key" + +# --- Mold backup offering (UI name; separate from Veeam job if needed) --- +BACKUP_OFFERING_NAME="VeeamBackup" + +# --- Job (policy) --- +VEEAM_JOB_NAME="" +VEEAM_SCHEDULE_NAME="default" +VEEAM_MAX_CHAIN="7" +ZONE_ID="" +RETENTION_PERIOD="P7D" + +# --- VM filter (libvirt names) --- +VM_INCLUDE="*" +VM_EXCLUDE="" + +# --- Legacy single-VM --- +VM_UUID="" +VM_NAME="" + +# --- Mold zone / Veeam plugin --- +VEEAM_URL="" +VEEAM_USERNAME="" +VEEAM_PASSWORD="" + +# --- Veeam server SSH (restore-watch polls Get-VBRRestoreSession) --- +VEEAM_SSH_HOST="" +VEEAM_SSH_USER="administrator" +VEEAM_SSH_KEY="/root/.ssh/veeam_id_rsa" + +# --- Storage model: datadisk (no NAS) --- +# GFS: qcow2 (libvirt checkpoint) | HCI: rbd (rbd export-diff on Ceph primary) +# BACKUP_STORAGE_ENGINE: auto (detect from VM disks) | qcow2 | rbd +BACKUP_STORAGE_ENGINE="auto" +BACKUP_STORAGE_MODE="datadisk" +BACKUP_REPO_TYPE="local" +BACKUP_REPO_PROVIDER="localfs" +MOLD_DATADISK_PATH="/data/backup" +BACKUP_REPO_ADDRESS="/data/backup" +BACKUP_REPO_NAME="Ablestack Data Disk" +NAS_REPO_MOUNT="" +VEEAM_HOST_REPO_ROOT="E:/opt1/veeam" +VEEAM_REPO_NAME="" + +VEEAM_HOST_BACKUP_PATH="/tmp/mold/veeam" +STAGING_PATH="/tmp/mold/veeam" +KVM_PRE_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" +KVM_POST_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" + +BACKUP_ID="" +# mold-only = Veeam FLR 후 datadisk(/data/backup)만 VM 볼륨에 반영 — NAS 마운트·Veeam chain export 없음 +RESTORE_SOURCE="mold-only" + +KVM_HOSTNAME="" +RESTORE_WATCH_TRIGGER_MOLD="true" +VEEAM_UI_RESTORE_SOURCE="mold-only" +RESTORE_LOCK_DIR="" +VEEAM_RESTORE_WATCH_WINDOW_MIN="10" +# Host job FLR: explicit libvirt VM to restore when VM_INCLUDE has multiple VMs +VEEAM_RESTORE_VM="" + +# guest = Veeam Agent on VM; pre/post SSH to KVM for Mold datadisk backup +# host = Veeam Linux Agent ON KVM (10.10.31.2) — one job per hypervisor +BACKUP_MODE="host" +IMPORT_MODE="auto" + +CLEANUP_STAGING_AFTER_BACKUP="true" +NAS_BACKUP_SCRIPT="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh" +CVT_BACKUP_SCRIPT="/etc/ablestack/veeam/ablestack_cvtbackup.sh" +LOG_FILE="/var/log/mold/veeam-hook.log" +LOG_TAG="mold-veeam-hook" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example b/scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example new file mode 100644 index 000000000000..30d1a58128bc --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example @@ -0,0 +1,81 @@ +# Mold backup env — installed to /etc/ablestack/veeam/mold-backup.env by install.sh +# veeam_config.sh reads this file automatically (API key/secret, NAS repo, zone-id). +# Do not commit mold-backup.env (contains secrets). + +MS_HOST=root@10.10.31.20 +KVM_HOST=root@10.10.31.2 +MS_PLUGIN_DIR=/usr/share/cloudstack-management/lib + +MOLD_API_URL=http://10.10.31.20:8080/client/api +MOLD_API_KEY= +MOLD_API_SECRET= +ZONE_ID= + +# Veeam Agent Job 이름 (Veeam UI와 동일) +JOB_NAME="Mold ablecube31-2" +VEEAM_JOB_NAME="Mold ablecube31-2" +KVM_HOSTNAME="ablecube31-2" +# host = KVM Linux Agent on 10.10.31.2 (one job per hypervisor) +VEEAM_BACKUP_TARGET="host" +# KVM root SSH — Veeam Protection Group credentials (필수) +KVM_SSH_USER=root +KVM_SSH_PASSWORD= +# Optional: libvirt:ip for FLR session → VM matching (not guest-mode jobs) +# VM_TARGETS="i-2-5-VM:10.10.254.70,i-2-40-VM:10.10.254.61" +# setup 후 Job 자동 Start (false 로 끄기) +VEEAM_START_JOBS=true +BACKUP_OFFERING_NAME="VeeamBackup" + +# 양방향 연동 (모드 C): Mold 백업 완료 → 매칭 Veeam Agent Job 자동 실행 +# - Veeam→Mold (pre-freeze)와 Mold→Veeam (이 트리거)는 마커로 루프 차단 +VEEAM_TRIGGER_ENABLED=false +VEEAM_TRIGGER_TTL=1800 +# 트리거 방식: auto(REST 먼저, 실패 시 SSH) | rest(curl만, SSH 불필요) | ssh +VEEAM_TRIGGER_METHOD=auto +# REST API (포트 9419) — SSH 없이 curl로 Job 시작. user/password 비우면 VEEAM_USER/PASSWORD 사용 +VEEAM_API_URL=https://10.10.254.246:9419 +VEEAM_API_VERSION=1.2-rev0 +VEEAM_API_USER=administrator +VEEAM_API_PASSWORD= + +# Veeam B&R 서버 (push-to-veeam.sh) +# NOTE: use Preferred IP (.246). .245 is Duplicate on this host and breaks SSH. +VEEAM_SSH_HOST=10.10.254.246 +VEEAM_SSH_USER=administrator +VEEAM_SSH_KEY= +VEEAM_URL=https://10.10.254.246:9398/api/ +VEEAM_USER=administrator +VEEAM_PASSWORD= +VEEAM_REPO_NAME= +KVM_HOSTNAME=ablecube31-2 +KVM_IP=10.10.31.2 +VEEAM_HOST_BACKUP_PATH=/tmp/mold/veeam + +# 전체 VM: * 또는 콤마 구분 libvirt 이름 +VM_INCLUDE="i-2-5-VM,i-2-40-VM" +VM_EXCLUDE= + +# KVM data disk path for Mold qcow2 (local directory on hypervisor — not Mold addBackupRepository) +# GFS: BACKUP_STORAGE_ENGINE=auto|qcow2 | HCI (RBD primary): auto|rbd +BACKUP_STORAGE_ENGINE=auto +BACKUP_STORAGE_MODE=datadisk +MOLD_DATADISK_PATH=/data/backup +BACKUP_REPO_ADDRESS=/data/backup +BACKUP_REPO_NAME=Ablestack Data Disk +BACKUP_REPO_TYPE=local +BACKUP_REPO_PROVIDER=localfs +# Veeam B&R local disk — one folder per KVM hypervisor (not per VM) +VEEAM_HOST_REPO_ROOT=E:/opt1/veeam +VEEAM_REPO_NAME=Mold ablecube31-2 + +# Veeam UI guest-file restore → KVM Mold datadisk restore (mold-only = no NAS / no Veeam chain) +VEEAM_UI_RESTORE_SOURCE=mold-only +RESTORE_SOURCE=mold-only +RESTORE_WATCH_TRIGGER_MOLD=true + +RUN_BACKUP=true +RUN_BUILD=false +RESTART_MS=true +SKIP_CONFIGURE=false +SKIP_DEPLOY=false +RUN_OPERATION= diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh b/scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh new file mode 100755 index 000000000000..2d63c34ba2bb --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh @@ -0,0 +1,4389 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Shared library for Ablestack Veeam backup/restore hooks on KVM hosts. +# NetBackup-style layout: /etc/ablestack/veeam/.conf, host staging /tmp/mold/veeam + +ABLESTACK_VEEAM_ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +MOLD_BACKUP_ETC_DIR="${MOLD_BACKUP_ETC_DIR:-${ABLESTACK_VEEAM_ETC_DIR}}" +MOLD_BACKUP_CONF="${MOLD_BACKUP_CONF:-}" +VEEAM_HOST_BACKUP_PATH="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" +CVT_BACKUP_SCRIPT="${CVT_BACKUP_SCRIPT:-/etc/ablestack/veeam/ablestack_cvtbackup.sh}" +VEEAM_PROVIDER_NAME="${VEEAM_PROVIDER_NAME:-ablestack-veeam}" + +# Resolve config: MOLD_BACKUP_CONF, or /etc/ablestack/veeam/.conf, or mold-backup.conf +mold_backup_resolve_conf_path() { + if [[ -n "${MOLD_BACKUP_CONF:-}" && -f "$MOLD_BACKUP_CONF" ]]; then + echo "$MOLD_BACKUP_CONF" + return 0 + fi + local job="${VEEAM_JOB_NAME:-${1:-}}" + if [[ -n "$job" ]]; then + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/${job}.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/${job}.conf" + return 0 + fi + local safe_job + safe_job="$(mold_backup_safe_job_name "$job")" + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/${safe_job}.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/${safe_job}.conf" + return 0 + fi + # Guest Veeam jobs: Mold VM 10-10-254-70 → shared KVM policy conf + if [[ "$job" == Mold\ VM\ * ]]; then + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Guest_Backup.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Guest_Backup.conf" + return 0 + fi + elif [[ "$job" == Mold\ * ]]; then + # Host Veeam job: Mold ablecube31-2 → Mold_Host_Backup.conf + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Host_Backup.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Host_Backup.conf" + return 0 + fi + fi + fi + for candidate in \ + "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Guest_Backup.conf" \ + "${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.conf" \ + "/etc/mold/backup/veeam/mold-backup.conf" \ + "${MOLD_BACKUP_ETC_DIR}/mold-backup.conf"; do + if [[ -f "$candidate" ]]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +# Allow runtime override: BACKUP_OPERATION=backup (Veeam post-notify path) +mold_backup_load_config() { + local resolved + local _preserve_backup_id="${BACKUP_ID:-}" + local _preserve_restore_source="${RESTORE_SOURCE:-}" + local _preserve_vm_name="${VM_NAME:-}" + local _preserve_vm_uuid="${VM_UUID:-}" + local _preserve_vm_include="${VM_INCLUDE:-}" + resolved="$(mold_backup_resolve_conf_path "${VEEAM_JOB_NAME:-}")" || { + echo "Config not found — run veeam_config.sh or install.sh" >&2 + return 1 + } + MOLD_BACKUP_CONF="$resolved" + # shellcheck source=/dev/null + source "$MOLD_BACKUP_CONF" + # Caller-provided selectors must survive sourcing the job conf (which may define them). + [[ -n "${_preserve_backup_id}" ]] && BACKUP_ID="${_preserve_backup_id}" + [[ -n "${_preserve_restore_source}" ]] && RESTORE_SOURCE="${_preserve_restore_source}" + [[ -n "${_preserve_vm_name}" ]] && VM_NAME="${_preserve_vm_name}" + [[ -n "${_preserve_vm_uuid}" ]] && VM_UUID="${_preserve_vm_uuid}" + [[ -n "${_preserve_vm_include}" && "${_preserve_vm_include}" != "*" ]] && VM_INCLUDE="${_preserve_vm_include}" + + LOG_FILE="${LOG_FILE:-/var/log/mold/backup-veeam.log}" + LOG_TAG="${LOG_TAG:-mold-veeam-backup}" + NAS_BACKUP_SCRIPT="${NAS_BACKUP_SCRIPT:-/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh}" + IMPORT_MODE="${IMPORT_MODE:-auto}" + BACKUP_MODE="${BACKUP_MODE:-host}" + VM_INCLUDE="${VM_INCLUDE:-*}" + VM_EXCLUDE="${VM_EXCLUDE:-}" + ZONE_ID="${ZONE_ID:-}" + RETENTION_PERIOD="${RETENTION_PERIOD:-}" + VEEAM_URL="${VEEAM_URL:-}" + VEEAM_USERNAME="${VEEAM_USERNAME:-}" + VEEAM_PASSWORD="${VEEAM_PASSWORD:-}" + VEEAM_HOST_BACKUP_PATH="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + STAGING_PATH="${STAGING_PATH:-${VEEAM_HOST_BACKUP_PATH}}" + NAS_REPO_MOUNT="${NAS_REPO_MOUNT:-}" + SOURCE_DISK_FORMAT="${SOURCE_DISK_FORMAT:-vmdk}" + BOOTSTRAP_CHECKPOINT="${BOOTSTRAP_CHECKPOINT:-true}" + QUIESCE_VM="${QUIESCE_VM:-false}" + BACKUP_OPERATION="${BACKUP_OPERATION:-seed-import}" + CLEANUP_STAGING_AFTER_BACKUP="${CLEANUP_STAGING_AFTER_BACKUP:-true}" + CLEANUP_STAGING_ON_ERROR="${CLEANUP_STAGING_ON_ERROR:-true}" + BACKUP_OFFERING_NAME="${BACKUP_OFFERING_NAME:-VeeamBackup}" + BACKUP_REPO_TYPE="${BACKUP_REPO_TYPE:-local}" + BACKUP_REPO_NAME="${BACKUP_REPO_NAME:-Ablestack Data Disk}" + BACKUP_REPO_PROVIDER="${BACKUP_REPO_PROVIDER:-localfs}" + MOLD_DATADISK_PATH="${MOLD_DATADISK_PATH:-}" + VEEAM_HOST_REPO_ROOT="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + VEEAM_REPO_NAME="${VEEAM_REPO_NAME:-}" + BACKUP_STORAGE_MODE="${BACKUP_STORAGE_MODE:-datadisk}" + BACKUP_STORAGE_ENGINE="${BACKUP_STORAGE_ENGINE:-auto}" + # Mold→Veeam trigger (bidirectional mode C): start the matching Veeam Agent job + # over SSH after a Mold backup completes. Loop is broken by veeam-active/mold-active markers. + VEEAM_TRIGGER_ENABLED="${VEEAM_TRIGGER_ENABLED:-false}" + VEEAM_TRIGGER_TTL="${VEEAM_TRIGGER_TTL:-1800}" + VEEAM_SSH_HOST="${VEEAM_SSH_HOST:-}" + VEEAM_SSH_USER="${VEEAM_SSH_USER:-administrator}" + VEEAM_SSH_KEY="${VEEAM_SSH_KEY:-}" + # Veeam VBR native REST API (port 9419) — used by VEEAM_TRIGGER_METHOD=rest/auto (no SSH). + VEEAM_API_URL="${VEEAM_API_URL:-}" + VEEAM_API_HOST="${VEEAM_API_HOST:-}" + VEEAM_API_PORT="${VEEAM_API_PORT:-9419}" + VEEAM_API_VERSION="${VEEAM_API_VERSION:-1.2-rev0}" + VEEAM_API_USER="${VEEAM_API_USER:-${VEEAM_USERNAME:-}}" + VEEAM_API_PASSWORD="${VEEAM_API_PASSWORD:-${VEEAM_PASSWORD:-}}" + VEEAM_GUEST_JOB_PREFIX="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + VM_TARGETS="${VM_TARGETS:-}" + KVM_HOSTNAME="${KVM_HOSTNAME:-}" + RESTORE_WATCH_TRIGGER_MOLD="${RESTORE_WATCH_TRIGGER_MOLD:-false}" + VEEAM_UI_RESTORE_SOURCE="${VEEAM_UI_RESTORE_SOURCE:-mold-only}" + RESTORE_LOCK_DIR="${RESTORE_LOCK_DIR:-}" + VEEAM_RESTORE_WATCH_WINDOW_MIN="${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}" + + mold_backup_resolve_api_secret + [[ -n "${MOLD_BACKUP_OPERATION:-}" ]] && BACKUP_OPERATION="${MOLD_BACKUP_OPERATION}" + mold_backup_supplement_guest_config + mold_backup_apply_datadisk_profile + # Guest mode is bidirectional: Mold UI backup must start the matching Veeam Agent job. + if [[ "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + VEEAM_TRIGGER_ENABLED=true + case "${VEEAM_TRIGGER_METHOD:-}" in + ''|auto) VEEAM_TRIGGER_METHOD=ssh ;; + esac + else + VEEAM_TRIGGER_METHOD="${VEEAM_TRIGGER_METHOD:-auto}" + fi + return 0 +} + +# Datadisk + Veeam E:\opt1\veeam\: Mold backups on KVM data disk (no NAS mount/restore). +mold_backup_apply_datadisk_profile() { + if [[ "${BACKUP_STORAGE_MODE:-}" != "datadisk" && "${BACKUP_REPO_TYPE:-}" != "local" ]]; then + return 0 + fi + [[ -z "${KVM_HOSTNAME:-}" ]] && KVM_HOSTNAME="$(hostname -s 2>/dev/null || hostname)" + BACKUP_REPO_TYPE=local + BACKUP_REPO_PROVIDER="${BACKUP_REPO_PROVIDER:-localfs}" + local disk="${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS:-/data/backup}}" + if [[ "$disk" == *glue-gfs* && -d /data/backup ]]; then + disk="/data/backup" + fi + MOLD_DATADISK_PATH="$disk" + BACKUP_REPO_ADDRESS="${MOLD_DATADISK_PATH}" + BACKUP_REPO_NAME="${BACKUP_REPO_NAME:-Ablestack Data Disk}" + NAS_REPO_MOUNT="" + VEEAM_HOST_REPO_ROOT="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + [[ -z "${VEEAM_REPO_NAME:-}" ]] && VEEAM_REPO_NAME="Mold ${KVM_HOSTNAME}" + # 복원: datadisk bind-mount만 사용 (NAS/GFS 마운트·Veeam chain export 없음) + RESTORE_SOURCE="mold-only" + VEEAM_UI_RESTORE_SOURCE="mold-only" + RESTORE_WATCH_TRIGGER_MOLD="${RESTORE_WATCH_TRIGGER_MOLD:-true}" +} + +mold_backup_is_datadisk_mode() { + [[ "${BACKUP_STORAGE_MODE:-}" == "datadisk" || "${BACKUP_REPO_TYPE:-}" == "local" ]] +} + +mold_backup_datadisk_root() { + mold_backup_apply_datadisk_profile + echo "${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS:-/data/backup}}" +} + +mold_backup_veeam_host_repo_folder() { + local host="${KVM_HOSTNAME:-$(hostname -s)}" + local root="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + echo "${root%/}/${host}" +} + +# Read one KEY=value from an env file (strips optional quotes). +mold_backup_read_env_var() { + local key="$1" file="$2" line val + [[ -f "$file" ]] || return 1 + line="$(grep -E "^[[:space:]]*${key}=" "$file" 2>/dev/null | head -1)" || return 1 + val="${line#*=}" + val="${val#\"}"; val="${val%\"}" + val="${val#\'}"; val="${val%\'}" + val="${val//$'\r'/}" + [[ -n "$val" ]] || return 1 + printf '%s' "$val" +} + +# Guest hooks use per-job .conf; VM_TARGETS often lives only in mold-backup.env. +mold_backup_vm_targets_merge() { + local combined="" pair name ip out="" k + declare -A _vm_target_map=() + for combined in "$1" "$2"; do + [[ -n "$combined" ]] || continue + IFS=',' read -ra _pairs <<<"${combined// /}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + [[ -n "$name" && -n "$ip" && "$ip" != "$name" ]] || continue + _vm_target_map["$name"]="$ip" + done + done + for k in "${!_vm_target_map[@]}"; do + [[ -n "$out" ]] && out+="," + out+="${k}:${_vm_target_map[$k]}" + done + echo "$out" +} + +# Write or update KEY="value" in a job .conf (best-effort; used to persist VM_TARGETS). +mold_backup_upsert_conf_var() { + local conf="$1" key="$2" val="$3" + [[ -f "$conf" && -n "$key" && -n "$val" ]] || return 0 + val="${val//\"/\\\"}" + if grep -qE "^${key}=" "$conf" 2>/dev/null; then + sed -i "s#^${key}=.*#${key}=\"${val}\"#" "$conf" + else + echo "${key}=\"${val}\"" >> "$conf" + fi +} + +mold_backup_supplement_guest_config() { + local env_file key val env_targets + for env_file in \ + "${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.env" \ + "${MOLD_BACKUP_ETC_DIR}/mold-backup.env" \ + "$(dirname "${BASH_SOURCE[0]}")/mold-backup.env"; do + [[ -f "$env_file" ]] || continue + env_targets="$(mold_backup_read_env_var VM_TARGETS "$env_file" 2>/dev/null || true)" + if [[ -n "$env_targets" ]]; then + if [[ -n "${VM_TARGETS:-}" ]]; then + VM_TARGETS="$(mold_backup_vm_targets_merge "$VM_TARGETS" "$env_targets")" + else + VM_TARGETS="$env_targets" + fi + fi + for key in VEEAM_SSH_HOST VEEAM_SSH_USER VEEAM_SSH_KEY VEEAM_GUEST_JOB_PREFIX \ + VEEAM_TRIGGER_ENABLED VEEAM_TRIGGER_METHOD VEEAM_TRIGGER_TTL \ + VEEAM_USERNAME VEEAM_PASSWORD VEEAM_API_URL; do + [[ -n "${!key:-}" ]] && continue + val="$(mold_backup_read_env_var "$key" "$env_file" 2>/dev/null || true)" + [[ -n "$val" ]] && export "$key=$val" + done + break + done + # Persist merged VM_TARGETS into guest policy conf so hooks do not depend on env alone. + if [[ -n "${VM_TARGETS:-}" && "${MOLD_BACKUP_CONF:-}" == *Mold_Guest_Backup.conf ]]; then + if ! grep -qE '^[[:space:]]*VM_TARGETS=' "$MOLD_BACKUP_CONF" 2>/dev/null; then + mold_backup_upsert_conf_var "$MOLD_BACKUP_CONF" VM_TARGETS "$VM_TARGETS" + fi + fi +} + +mold_backup_resolve_api_secret() { + if [[ -n "${MOLD_API_SECRET:-}" ]]; then + return 0 + fi + if [[ -z "${MOLD_API_SECRET_ENC_FILE:-}" ]]; then + return 0 + fi + if [[ -z "${MOLD_SECRET_KEY_FILE:-}" ]]; then + MOLD_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + fi + local secret_script="${MOLD_BACKUP_ETC_DIR}/mold-backup-secret.sh" + [[ -x "$secret_script" ]] || secret_script="$(dirname "${BASH_SOURCE[0]}")/mold-backup-secret.sh" + if [[ ! -x "$secret_script" ]]; then + mold_backup_log warn "Cannot decrypt API secret: mold-backup-secret.sh not found" + return 0 + fi + MOLD_API_SECRET=$("$secret_script" decrypt --enc-file "${MOLD_API_SECRET_ENC_FILE}" --key-file "${MOLD_SECRET_KEY_FILE}") \ + || mold_backup_die "Failed to decrypt MOLD_API_SECRET from ${MOLD_API_SECRET_ENC_FILE}" + # OpenSSL decrypt may append a newline; breaks CloudStack API HMAC signature. + MOLD_API_SECRET="${MOLD_API_SECRET//$'\r'/}" + MOLD_API_SECRET="${MOLD_API_SECRET%"${MOLD_API_SECRET##*[![:space:]]}"}" +} + +mold_backup_log() { + local level="$1" + shift + local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" + echo "$msg" >&2 + mkdir -p "$(dirname "${LOG_FILE}")" 2>/dev/null || true + echo "$msg" >> "${LOG_FILE}" 2>/dev/null || true + if command -v logger >/dev/null 2>&1; then + case "$level" in + err) logger -t "${LOG_TAG}" -p user.err "$*" ;; + warn) logger -t "${LOG_TAG}" -p user.warning "$*" ;; + *) logger -t "${LOG_TAG}" -p user.info "$*" ;; + esac + fi +} + +mold_backup_die() { + mold_backup_log err "$@" + exit 1 +} + +mold_backup_require_var() { + local name="$1" + local value="${!name:-}" + if [[ -z "$value" ]]; then + mold_backup_die "Required config [$name] is not set in ${MOLD_BACKUP_CONF}" + fi +} + +mold_backup_cmk_bin() { + command -v cmk >/dev/null 2>&1 && echo "cmk" && return 0 + command -v cloudmonkey >/dev/null 2>&1 && echo "cloudmonkey" && return 0 + return 1 +} + +mold_backup_require_cmd() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || mold_backup_die "Required command not found: $cmd" +} + +mold_backup_cloudstack_api_call() { + local cmd="$1" + shift + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + mold_backup_require_cmd curl + mold_backup_require_cmd python3 + + # Sign per ApiServer.verifyRequest: sort param names, URLEncode values (+ -> %20), + # lowercase the full unsigned string, HMAC-SHA256, Base64 signature. + local url + url="$(python3 - "$MOLD_API_URL" "$MOLD_API_KEY" "$MOLD_API_SECRET" "$cmd" "$@" <<'PY' +import base64, hashlib, hmac, sys +from urllib.parse import quote, quote_plus, urlsplit, urlunsplit, parse_qsl + +api_url, apikey, secret, command, *pairs = sys.argv[1:] + +params = { + "apikey": apikey, + "command": command, + "response": "json", +} + +for p in pairs: + if "=" not in p: + continue + k, v = p.split("=", 1) + if k and v: + params[k] = v + +parts = urlsplit(api_url) +base = urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + +for k, v in parse_qsl(parts.query, keep_blank_values=True): + if k and v and k not in params: + params[k] = v + +def enc_value(v: str) -> str: + # Match Java URLEncoder.encode(..., UTF_8).replaceAll("\\+", "%20") + return quote_plus(str(v), safe="").replace("+", "%20") + +# Case-sensitive sort (java.util.Collections.sort on param names) +req_items = sorted(params.items(), key=lambda kv: kv[0]) +req_query = "&".join([f"{k}={enc_value(v)}" for k, v in req_items]) + +unsigned = req_query.lower() +sig = base64.b64encode( + hmac.new(secret.encode("utf-8"), unsigned.encode("utf-8"), hashlib.sha256).digest() +).decode("ascii") + +signed = f"{base}?{req_query}&signature={quote(sig, safe='')}" +print(signed) +PY +)" || exit $? + + mold_backup_log info "API: ${cmd} (curl)" >&2 + # Show response body on errors for debugging. + local tmp rc http_code body api_err + tmp="$(mktemp)" + http_code="$(curl -sS --connect-timeout 10 --max-time 120 -o "$tmp" -w '%{http_code}' "$url")" || { + rc=$? + rm -f "$tmp" + return "$rc" + } + body="$(cat "$tmp")" + rm -f "$tmp" + if [[ "$http_code" -lt 200 || "$http_code" -ge 300 ]]; then + echo "$body" >&2 + return 22 + fi + api_err="$(mold_backup_api_extract_error "$body" 2>/dev/null || true)" + if [[ -n "$api_err" ]]; then + mold_backup_api_log_ms_schema_hint "$api_err" + mold_backup_log err "API ${cmd} failed: ${api_err}" >&2 + echo "$body" + return 1 + fi + echo "$body" +} + +mold_backup_cmk_run() { + local cmd="$1" + shift + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + local cmk + cmk=$(mold_backup_cmk_bin) || { + mold_backup_cloudstack_api_call "$cmd" "$@" + return $? + } + local -a args=(-u "${MOLD_API_URL}" -a "${MOLD_API_KEY}" -s "${MOLD_API_SECRET}" "${cmd}") + local pair key value + for pair in "$@"; do + key="${pair%%=*}" + value="${pair#*=}" + [[ -n "$key" && -n "$value" ]] && args+=("${key}=${value}") + done + mold_backup_log info "Executing: ${cmk} ${cmd} $*" >&2 + local out rc api_err + out="$("${cmk}" "${args[@]}" 2>/dev/null)" || rc=$? + api_err="$(mold_backup_api_extract_error "$out" 2>/dev/null || true)" + if [[ -n "$api_err" ]]; then + mold_backup_api_log_ms_schema_hint "$api_err" + mold_backup_log err "API ${cmd} failed: ${api_err}" >&2 + echo "$out" + return 1 + fi + echo "$out" + return "${rc:-0}" +} + +mold_backup_resolve_vm_name() { + if [[ -n "${VM_NAME:-}" ]]; then + return 0 + fi + mold_backup_require_var VM_UUID + local json name + json=$(mold_backup_cmk_run listVirtualMachines "id=${VM_UUID}" 2>/dev/null) || true + name=$(echo "$json" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + vms = d.get('listvirtualmachinesresponse', {}).get('virtualmachine', []) + if isinstance(vms, dict): vms = [vms] + print(vms[0].get('instancename','') if vms else '') +except Exception: + print('') +" 2>/dev/null) + if [[ -n "$name" ]]; then + VM_NAME="$name" + mold_backup_log info "Resolved VM_NAME=${VM_NAME} from API" + return 0 + fi + mold_backup_die "VM_NAME is empty and could not be resolved from VM_UUID via API" +} + +mold_backup_check_libvirt_vm() { + mold_backup_resolve_vm_name + if ! virsh -c qemu:///system dominfo "${VM_NAME}" >/dev/null 2>&1; then + mold_backup_die "Libvirt domain [${VM_NAME}] not found on this host" + fi + mold_backup_log info "Libvirt domain [${VM_NAME}] is ready" +} + +mold_backup_get_live_disk_paths() { + mold_backup_log info "Resolving live libvirt disk paths (file + RBD)" + mold_backup_get_all_disk_paths +} + +mold_backup_get_all_disk_paths() { + mold_backup_resolve_vm_name + local paths=() + local target + while IFS= read -r target; do + [[ -z "$target" ]] && continue + paths+=("$target") + done < <(virsh -c qemu:///system domblklist "${VM_NAME}" --details 2>/dev/null | awk '/disk/ {print $4}') + if [[ ${#paths[@]} -eq 0 ]]; then + mold_backup_die "No disks found for VM ${VM_NAME}" + fi + (IFS=,; echo "${paths[*]}") +} + +mold_backup_has_rbd_disk() { + local csv="$1" + [[ "$csv" == rbd:* ]] && return 0 + [[ "$csv" == *",rbd:"* ]] && return 0 + return 1 +} + +# auto | qcow2 (GFS/file) | rbd (HCI/Ceph primary) +mold_backup_detect_storage_engine() { + local disk_paths_csv="${1:-}" + case "${BACKUP_STORAGE_ENGINE:-auto}" in + qcow2|rbd) echo "${BACKUP_STORAGE_ENGINE}"; return 0 ;; + esac + if mold_backup_has_rbd_disk "$disk_paths_csv"; then + echo "rbd" + else + echo "qcow2" + fi +} + +mold_backup_parse_rbd_volume_id() { + local uri="$1" image="" + [[ -n "$uri" ]] || return 0 + if [[ "$uri" == rbd:* ]]; then + image="${uri#rbd:}" + image="${image%%:*}" + elif [[ "$uri" == rbd/* ]]; then + image="${uri##*/}" + fi + echo "$image" +} + +mold_backup_disk_target_kind() { + local target + target="$(echo "${1:-}" | tr '[:upper:]' '[:lower:]')" + case "$target" in + vda|sda|hda) echo "root" ;; + *) echo "datadisk" ;; + esac +} + +# Lines: target|source_path (libvirt domblklist) +mold_backup_list_disk_specs() { + mold_backup_resolve_vm_name + virsh -c qemu:///system domblklist "${VM_NAME}" --details 2>/dev/null \ + | awk '/disk/ {print $3 "|" $4}' +} + +mold_backup_qcow2_volume_id_from_path() { + local path="$1" base uuid + base="$(basename "$path")" + uuid="${base%.qcow2}" + uuid="${uuid%.raw}" + if [[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]]; then + echo "$uuid" + return 0 + fi + echo "" +} + +mold_backup_is_vm_running() { + mold_backup_resolve_vm_name + local state + state=$(virsh -c qemu:///system dominfo "${VM_NAME}" 2>/dev/null | awk -F: '/^State:/ {gsub(/^[ \t]+/, "", $2); print $2; exit}') + [[ "$state" == "running" ]] +} + +mold_backup_clean_repo_address() { + local addr="${BACKUP_REPO_ADDRESS}" + addr="${addr#nfs://}" + addr="${addr#cifs://}" + echo "$addr" +} + +mold_backup_meta_field() { + local file="$1" key="$2" + [[ -f "$file" ]] || return 1 + grep -E "^${key}=" "$file" 2>/dev/null | head -1 | cut -d= -f2- +} + +mold_backup_with_repo_mount() { + local callback="$1" + if mold_backup_is_datadisk_mode; then + case "${BACKUP_REPO_TYPE:-local}" in + nfs|cifs|glusterfs) + mold_backup_die "datadisk mode: NAS/network restore disabled — use BACKUP_REPO_TYPE=local and MOLD_DATADISK_PATH=${MOLD_DATADISK_PATH:-/data/backup}" + ;; + esac + fi + if [[ -n "${NAS_REPO_MOUNT:-}" && -d "${NAS_REPO_MOUNT}" ]]; then + "$callback" "${NAS_REPO_MOUNT}" + return $? + fi + + # Local data disk repo: BACKUP_REPO_ADDRESS is a directory on this host. + case "${BACKUP_REPO_TYPE:-nfs}" in + local|dir|localfs) + local local_dir + local_dir="$(mold_backup_clean_repo_address)" + [[ -d "$local_dir" ]] || mold_backup_die "Local backup directory not found: ${local_dir}" + "$callback" "$local_dir" + return $? + ;; + esac + + mold_backup_require_var BACKUP_REPO_TYPE + mold_backup_require_var BACKUP_REPO_ADDRESS + [[ -x "${NAS_BACKUP_SCRIPT}" ]] || mold_backup_die "NAS backup script not found: ${NAS_BACKUP_SCRIPT}" + + local mount_point repo_addr nas_type mount_opts mopts=() + mount_point=$(mktemp -d -t moldbackup.XXXXX) + repo_addr=$(mold_backup_clean_repo_address) + nas_type="${BACKUP_REPO_TYPE}" + mount_opts="${BACKUP_REPO_MOUNT_OPTS:-}" + if [[ "$nas_type" == "cifs" && -n "$mount_opts" ]]; then + mount_opts="${mount_opts},nobrl" + elif [[ "$nas_type" == "cifs" ]]; then + mount_opts="nobrl" + fi + [[ -n "$mount_opts" ]] && mopts=(-o "$mount_opts") + + if ! mount -t "${nas_type}" "${repo_addr}" "${mount_point}" "${mopts[@]}" 2>/dev/null; then + rmdir "${mount_point}" 2>/dev/null || true + mold_backup_die "Failed to mount NAS repository ${repo_addr} for parent lookup" + fi + + local rc=0 + "$callback" "${mount_point}" || rc=$? + umount "${mount_point}" 2>/dev/null || true + rmdir "${mount_point}" 2>/dev/null || true + return "$rc" +} + +# Sets PARENT_BACKUP_DIR_REL, PARENT_CHECKPOINT_NAME, PARENT_CHECKPOINT_PATH_REL, PARENT_BACKUP_FILES. +mold_backup_find_latest_nas_parent() { + local mount_point="$1" + mold_backup_resolve_vm_name + + local vm_dir="${mount_point}/${VM_NAME}" + [[ -d "$vm_dir" ]] || { + mold_backup_log err "No backup directory for VM on NAS: ${vm_dir}" + return 1 + } + + local latest_name="" latest_dir="" d base + for d in "${vm_dir}"/*; do + [[ -d "$d" ]] || continue + base=$(basename "$d") + if [[ -f "${d}/veeam-seed.meta" || -d "${d}/checkpoints" ]]; then + if [[ -z "$latest_name" || "$base" > "$latest_name" ]]; then + latest_name="$base" + latest_dir="$d" + fi + fi + done + + [[ -n "$latest_dir" ]] || { + mold_backup_log err "No seed or checkpoint backup found under ${vm_dir}" + return 1 + } + + PARENT_BACKUP_DIR_REL="${VM_NAME}/${latest_name}" + PARENT_CHECKPOINT_NAME="" + PARENT_CHECKPOINT_PATH_REL="" + PARENT_BACKUP_FILES="" + + if [[ -f "${latest_dir}/veeam-seed.meta" ]]; then + PARENT_CHECKPOINT_NAME=$(mold_backup_meta_field "${latest_dir}/veeam-seed.meta" checkpoint_name || true) + PARENT_BACKUP_FILES=$(mold_backup_meta_field "${latest_dir}/veeam-seed.meta" backup_files || true) + elif [[ -f "${latest_dir}/rbd-backup.meta" ]]; then + PARENT_CHECKPOINT_NAME=$(mold_backup_meta_field "${latest_dir}/rbd-backup.meta" checkpoint_name || true) + PARENT_BACKUP_FILES=$(mold_backup_meta_field "${latest_dir}/rbd-backup.meta" backup_files || true) + fi + [[ -z "$PARENT_CHECKPOINT_NAME" ]] && PARENT_CHECKPOINT_NAME="$latest_name" + + if [[ -f "${latest_dir}/checkpoints/${PARENT_CHECKPOINT_NAME}.xml" ]]; then + PARENT_CHECKPOINT_PATH_REL="${PARENT_BACKUP_DIR_REL}/checkpoints/${PARENT_CHECKPOINT_NAME}.xml" + elif [[ -f "${latest_dir}/checkpoints/${PARENT_CHECKPOINT_NAME}.meta" ]]; then + PARENT_CHECKPOINT_PATH_REL="${PARENT_BACKUP_DIR_REL}/checkpoints/${PARENT_CHECKPOINT_NAME}.meta" + else + mold_backup_log warn "Parent checkpoint file not found under ${latest_dir}/checkpoints; incremental may fail" + PARENT_CHECKPOINT_PATH_REL="${PARENT_BACKUP_DIR_REL}/checkpoints/${PARENT_CHECKPOINT_NAME}.xml" + fi + + mold_backup_log info "Local backup repo parent backup=${PARENT_BACKUP_DIR_REL} checkpoint=${PARENT_CHECKPOINT_NAME}" + return 0 +} + +mold_backup_run_local_incremental_on_mount() { + local mount_point="$1" + mold_backup_find_latest_nas_parent "$mount_point" || mold_backup_die "Cannot resolve NAS parent for incremental backup" + + mold_backup_require_var BACKUP_REPO_TYPE + mold_backup_require_var BACKUP_REPO_ADDRESS + [[ -x "${NAS_BACKUP_SCRIPT}" ]] || mold_backup_die "NAS backup script not found: ${NAS_BACKUP_SCRIPT}" + + local disk_paths backup_path checkpoint backup_files repo_addr op quiesce btype + disk_paths=$(mold_backup_get_all_disk_paths) + backup_path=$(mold_backup_generate_backup_path) + checkpoint="${backup_path##*/}" + btype="FULL" + [[ -n "${PARENT_BACKUP_DIR_REL:-}" ]] && btype="INCREMENTAL" + if [[ -n "${PARENT_BACKUP_FILES:-}" ]]; then + backup_files="${PARENT_BACKUP_FILES}" + if mold_backup_has_rbd_disk "$disk_paths" && [[ "$btype" == "INCREMENTAL" ]]; then + backup_files="${backup_files//.raw/.rbdiff}" + fi + else + backup_files=$(mold_backup_build_backup_files "$disk_paths" "$btype") + fi + repo_addr=$(mold_backup_clean_repo_address) + quiesce="${QUIESCE_VM:-false}" + + if mold_backup_has_rbd_disk "$disk_paths"; then + op="backup-rbd" + mold_backup_log info "Storage engine=rbd (HCI/Ceph primary)" + elif mold_backup_is_vm_running; then + op="backup-running" + else + mold_backup_die "VM ${VM_NAME} is not running; local incremental needs backup-running (start VM or use BACKUP_MODE=api)" + fi + + mold_backup_log info "Local datadisk incremental op=${op} path=${backup_path} parent=${PARENT_BACKUP_DIR_REL}" + "${NAS_BACKUP_SCRIPT}" \ + -o "${op}" \ + -v "${VM_NAME}" \ + -t "${BACKUP_REPO_TYPE}" \ + -s "${repo_addr}" \ + -m "${BACKUP_REPO_MOUNT_OPTS:-}" \ + -p "${backup_path}" \ + -b "INCREMENTAL" \ + -c "${checkpoint}" \ + -r "${PARENT_BACKUP_DIR_REL}" \ + -i "${PARENT_CHECKPOINT_NAME}" \ + -j "${PARENT_CHECKPOINT_PATH_REL}" \ + -q "${quiesce}" \ + -f "${backup_files}" \ + -d "${disk_paths}" \ + || mold_backup_die "ablestack_nasbackup.sh ${op} failed" +} + +mold_backup_run_local_incremental() { + mold_backup_with_repo_mount mold_backup_run_local_incremental_on_mount +} + +mold_backup_run_backup() { + case "${BACKUP_MODE}" in + host) + mold_backup_die "host mode uses pre-notify/post-notify hooks, not mold_backup_run_backup" + ;; + api) + mold_backup_api_create_backup + ;; + local) + mold_backup_run_local_incremental + ;; + auto) + if mold_backup_cmk_bin >/dev/null 2>&1; then + mold_backup_api_create_backup || { + mold_backup_log warn "API incremental backup failed, trying local NAS script" + mold_backup_run_local_incremental + } + else + mold_backup_run_local_incremental + fi + ;; + *) + mold_backup_die "Invalid BACKUP_MODE=${BACKUP_MODE} (use api|local|auto)" + ;; + esac +} + +mold_backup_check_staging() { + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" ]]; then + mold_backup_log info "FileLevel Veeam backup: staging VMDK optional (NAS seed uses live libvirt disks)" + return 0 + fi + if [[ -n "${STAGING_DISK_PATHS:-}" ]]; then + mold_backup_log info "Using STAGING_DISK_PATHS from config" + return 0 + fi + mold_backup_require_var STAGING_PATH + if [[ ! -d "${STAGING_PATH}" ]]; then + mold_backup_die "Staging directory not found: ${STAGING_PATH}" + fi + mold_backup_log info "Staging directory OK: ${STAGING_PATH}" +} + +# Disk paths for NAS seed: staging VMDK files, or live libvirt disks (FileLevel Agent). +mold_backup_resolve_seed_disk_paths() { + local staging + staging=$(mold_backup_list_staging_disks) + if [[ -n "$staging" ]]; then + echo "$staging" + return 0 + fi + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" ]]; then + mold_backup_log info "No staging disks; using live libvirt disk paths for seed import" + mold_backup_get_live_disk_paths + return 0 + fi + return 1 +} + +mold_backup_list_staging_disks() { + if [[ -n "${STAGING_DISK_PATHS:-}" ]]; then + echo "${STAGING_DISK_PATHS}" + return 0 + fi + find "${STAGING_PATH}" -maxdepth 3 -type f \( -name '*.vmdk' -o -name '*.flat' -o -name '*.qcow2' -o -name '*.raw' \) 2>/dev/null \ + | sort | paste -sd, - +} + +mold_backup_generate_backup_path() { + if [[ -n "${BACKUP_PATH:-}" ]]; then + echo "${BACKUP_PATH}" + return 0 + fi + mold_backup_resolve_vm_name + echo "${VM_NAME}/$(date '+%Y.%m.%d.%H.%M.%S.%3N')" +} + +mold_backup_build_backup_files() { + local disk_paths_csv="$1" + local backup_type="${2:-FULL}" + local -a out=() + if [[ -n "${BACKUP_FILES:-}" ]]; then + echo "${BACKUP_FILES}" + return 0 + fi + local engine suffix target path kind vol_id + engine="$(mold_backup_detect_storage_engine "$disk_paths_csv")" + suffix=".qcow2" + [[ "$backup_type" == "INCREMENTAL" ]] && suffix=".rbdiff" || true + if [[ "$engine" == "rbd" ]]; then + [[ "$backup_type" == "INCREMENTAL" ]] && suffix=".rbdiff" || suffix=".raw" + while IFS='|' read -r target path; do + [[ -n "$path" ]] || continue + vol_id="$(mold_backup_parse_rbd_volume_id "$path")" + [[ -n "$vol_id" ]] || vol_id="$(basename "$path")" + kind="$(mold_backup_disk_target_kind "$target")" + out+=("${kind}.${vol_id}${suffix}") + done < <(mold_backup_list_disk_specs) + else + local i=0 + while IFS='|' read -r target path; do + [[ -n "$path" ]] || continue + vol_id="$(mold_backup_qcow2_volume_id_from_path "$path")" + kind="$(mold_backup_disk_target_kind "$target")" + if [[ -n "$vol_id" ]]; then + out+=("${kind}.${vol_id}.qcow2") + else + out+=("disk-${i}.qcow2") + i=$((i + 1)) + fi + done < <(mold_backup_list_disk_specs) + if [[ ${#out[@]} -eq 0 ]]; then + local -a disks + IFS=, read -ra disks <<< "$disk_paths_csv" + local j=0 + for _ in "${disks[@]}"; do + out+=("disk-${j}.qcow2") + j=$((j + 1)) + done + fi + fi + [[ ${#out[@]} -gt 0 ]] || mold_backup_die "Cannot build backup file names for disks: ${disk_paths_csv}" + (IFS=,; echo "${out[*]}") +} + +mold_backup_veeam_export_ssh() { + [[ "${ENABLE_VEEAM_SSH_EXPORT}" == "true" ]] || return 0 + mold_backup_require_var VEEAM_SSH_HOST + mold_backup_require_var VEEAM_RESTORE_POINT_ID + local remote_staging="${VEEAM_STAGING_PATH_ON_SERVER:-${STAGING_PATH}}" + mold_backup_log info "Triggering Veeam FLR export on ${VEEAM_SSH_HOST}" + ssh -i "${VEEAM_SSH_KEY}" -o StrictHostKeyChecking=no "${VEEAM_SSH_USER}@${VEEAM_SSH_HOST}" powershell -Command " + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + \$rp = Get-VBRRestorePoint | Where-Object { \$_.Id -eq '${VEEAM_RESTORE_POINT_ID}' -or \$_.Id.Guid -eq '${VEEAM_RESTORE_POINT_ID}' } + if (-not \$rp) { exit 1 } + New-Item -ItemType Directory -Force -Path '${remote_staging}' | Out-Null + \$session = Start-VBRFLRSession -RestorePoint \$rp + Get-VBRFLRItem -Session \$session | Where-Object { \$_.Type -eq 'HardDisk' } | ForEach-Object { + Copy-VBRFLRItem -FLRSession \$session -Item \$_ -Destination (Join-Path '${remote_staging}' (\$_.Name + '.vmdk')) + } + Stop-VBRFLRSession -Session \$session + " || mold_backup_die "Veeam SSH export failed" +} + +mold_backup_run_local_seed_import() { + mold_backup_require_var BACKUP_REPO_TYPE + mold_backup_require_var BACKUP_REPO_ADDRESS + [[ -x "${NAS_BACKUP_SCRIPT}" ]] || mold_backup_die "NAS backup script not found: ${NAS_BACKUP_SCRIPT}" + + mold_backup_resolve_vm_name + local disk_paths staging checkpoint backup_path backup_files repo_addr source_format btype + disk_paths=$(mold_backup_get_all_disk_paths) + staging=$(mold_backup_resolve_seed_disk_paths) || mold_backup_die "No seed disk paths (staging or live libvirt disks)" + backup_path=$(mold_backup_generate_backup_path) + checkpoint="${backup_path##*/}" + btype="FULL" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "$btype") + + repo_addr="${BACKUP_REPO_ADDRESS}" + repo_addr="${repo_addr#nfs://}" + repo_addr="${repo_addr#cifs://}" + + source_format="${SOURCE_DISK_FORMAT:-vmdk}" + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" ]]; then + local first_seed="${staging%%,*}" + if [[ -f "$first_seed" ]] && command -v qemu-img >/dev/null 2>&1; then + if qemu-img info "$first_seed" 2>/dev/null | grep -q 'file format: qcow2'; then + source_format="qcow2" + elif qemu-img info "$first_seed" 2>/dev/null | grep -q 'file format: raw'; then + source_format="raw" + fi + fi + fi + + mold_backup_log info "Local datadisk seed import path=${backup_path} checkpoint=${checkpoint} source_format=${source_format}" + "${NAS_BACKUP_SCRIPT}" \ + -o import-veeam-seed \ + -v "${VM_NAME}" \ + -t "${BACKUP_REPO_TYPE}" \ + -s "${repo_addr}" \ + -m "${BACKUP_REPO_MOUNT_OPTS:-}" \ + -p "${backup_path}" \ + -c "${checkpoint}" \ + -f "${backup_files}" \ + -d "${disk_paths}" \ + --staging-disks "${staging}" \ + --source-format "${source_format}" \ + --veeam-restore-point "${VEEAM_RESTORE_POINT_ID}" \ + --bootstrap-checkpoint "${BOOTSTRAP_CHECKPOINT}" \ + || mold_backup_die "ablestack_nasbackup.sh import-veeam-seed failed" +} + +mold_backup_api_import_seed() { + mold_backup_require_var VM_UUID + local staging backup_name + staging=$(mold_backup_resolve_seed_disk_paths) || mold_backup_die "No seed disk paths for API import" + backup_name="$(mold_backup_api_build_backup_name_for_vm "${VM_UUID}" "${VM_NAME:-}")" + if mold_backup_cmk_supports importAblestackVeeamBackupSeed 2>/dev/null; then + mold_backup_cmk_run importAblestackVeeamBackupSeed \ + "virtualmachineid=${VM_UUID}" \ + "name=${backup_name}" \ + "veeamrestorepointid=${VEEAM_RESTORE_POINT_ID}" \ + "stagingdiskpaths=${staging}" \ + "sourcediskformat=${SOURCE_DISK_FORMAT}" \ + "bootstrapcheckpoint=${BOOTSTRAP_CHECKPOINT}" + return 0 + fi + mold_backup_die "importAblestackVeeamBackupSeed API not available in cloudmonkey/cmk" +} + +mold_backup_api_create_backup() { + mold_backup_api_create_veeam_backup +} + +mold_backup_api_restore() { + mold_backup_require_var BACKUP_ID + local json job_id + json=$(mold_backup_cmk_run restoreAblestackVeeamBackup "id=${BACKUP_ID}" 2>/dev/null) \ + || json=$(mold_backup_cmk_run restoreBackup "id=${BACKUP_ID}" 2>/dev/null) \ + || return 1 + job_id="$(mold_backup_api_json_field "$json" "restoreablestackveeambackupresponse.jobid")" + [[ -z "$job_id" ]] && job_id="$(mold_backup_api_json_field "$json" "restorebackupresponse.jobid")" + if [[ -n "$job_id" ]]; then + mold_backup_notify_log info "restoreAblestackVeeamBackup job=${job_id}; waiting for MS/agent restore" + mold_backup_api_wait_async_job "$job_id" 3600 || return 1 + mold_backup_notify_log info "Restore async job completed: ${job_id}" + return 0 + fi + return 0 +} + +mold_backup_cmk_supports() { + local cmd="$1" + local cmk + cmk=$(mold_backup_cmk_bin) || return 1 + "${cmk}" -h 2>/dev/null | grep -q "${cmd}" || return 1 +} + +mold_backup_api_create_veeam_backup() { + mold_backup_require_var VM_UUID + local backup_name args + backup_name="$(mold_backup_api_build_backup_name_for_vm "${VM_UUID}" "${VM_NAME:-}")" + args=("virtualmachineid=${VM_UUID}" "name=${backup_name}") + [[ "${QUIESCE_VM}" == "true" ]] && args+=("quiescevm=true") + mold_backup_cmk_run createAblestackVeeamBackup "${args[@]}" \ + || mold_backup_cmk_run createBackup "${args[@]}" +} + +mold_backup_cleanup_staging() { + [[ "${CLEANUP_STAGING_AFTER_BACKUP}" == "true" ]] || return 0 + # Guest VM mode: no Windows FLR staging on KVM hypervisor. + if [[ "${BACKUP_MODE:-}" =~ ^(guest|veeam-guest)$ ]]; then + return 0 + fi + if [[ -n "${STAGING_DISK_PATHS:-}" ]]; then + mold_backup_log info "Skipping staging dir cleanup (STAGING_DISK_PATHS set)" + return 0 + fi + if [[ -z "${STAGING_PATH:-}" ]]; then + mold_backup_log info "Skipping staging cleanup (STAGING_PATH not set)" + return 0 + fi + if [[ ! -d "${STAGING_PATH}" ]]; then + mold_backup_log info "Staging path already absent: ${STAGING_PATH}" + return 0 + fi + mold_backup_log info "Cleaning staging directory: ${STAGING_PATH}" + find "${STAGING_PATH}" -mindepth 1 -maxdepth 3 \( -name '*.vmdk' -o -name '*.flat' -o -name '*.qcow2' -o -name '*.raw' -o -name '*.meta' \) -delete 2>/dev/null || true + find "${STAGING_PATH}" -mindepth 1 -maxdepth 2 -type d -empty -delete 2>/dev/null || true +} + +mold_backup_import_seed() { + mold_backup_check_staging + mold_backup_require_var VEEAM_RESTORE_POINT_ID + case "${IMPORT_MODE}" in + api) + mold_backup_api_import_seed + ;; + local) + mold_backup_run_local_seed_import + ;; + auto) + if mold_backup_cmk_bin >/dev/null 2>&1; then + mold_backup_api_import_seed || { + mold_backup_log warn "API import failed, trying local NAS import" + mold_backup_run_local_seed_import + } + else + mold_backup_run_local_seed_import + fi + ;; + *) + mold_backup_die "Invalid IMPORT_MODE=${IMPORT_MODE} (use api|local|auto)" + ;; + esac +} + +mold_backup_run_operation() { + case "${BACKUP_OPERATION}" in + seed-import) + mold_backup_veeam_export_ssh + mold_backup_import_seed + ;; + backup) + mold_backup_run_backup + ;; + restore) + mold_backup_require_var BACKUP_ID + mold_backup_api_restore + ;; + *) + mold_backup_die "Unknown BACKUP_OPERATION=${BACKUP_OPERATION}" + ;; + esac +} + +# --- NetBackup-style policy/job hooks (bpstart / bpend / restore_notify) --- + +mold_backup_notify_log() { + local level="$1" + shift + LOG_FILE="${LOG_FILE:-/var/log/mold/veeam-hook.log}" + LOG_TAG="${LOG_TAG:-mold-veeam-hook}" + mold_backup_log "$level" "$@" +} + +mold_backup_state_dir() { + echo "${ABLESTACK_VEEAM_ETC_DIR}/state" +} + +mold_backup_state_file_for_job() { + local job="$1" + local run_id="${2:-$(date '+%Y%m%d%H%M%S')}" + echo "$(mold_backup_state_dir)/${job}.${run_id}.state" +} + +mold_backup_latest_state_file() { + local job="$1" + local dir found + dir="$(mold_backup_state_dir)" + [[ -d "$dir" ]] || return 0 + found="$(ls -1t "${dir}/${job}".*.state 2>/dev/null | head -1 || true)" + echo "$found" +} + +mold_backup_list_running_domains() { + virsh -c qemu:///system list --name --state-running 2>/dev/null | awk 'NF' || true +} + +mold_backup_domain_exists() { + local vm_name="$1" + [[ -n "$vm_name" ]] || return 1 + virsh -c qemu:///system dominfo "$vm_name" >/dev/null 2>&1 && return 0 + virsh dominfo "$vm_name" >/dev/null 2>&1 +} + +# Restore-watch target: libvirt domain and/or Mold VM on this hypervisor (shut-off OK). +mold_backup_vm_restorable_on_local_host() { + local vm="$1" + [[ -n "$vm" ]] || return 1 + if mold_backup_domain_exists "$vm" 2>/dev/null; then + return 0 + fi + local vm_id + vm_id="$(mold_backup_api_get_vm_id "$vm" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || return 1 + mold_backup_vm_owned_by_local_host "$vm" 2>/dev/null +} + +# Pre-notify/list-backups targets: running VMs, plus explicit VM_INCLUDE names (even if shut off). +mold_backup_list_target_domains() { + local -a targets=() seen="" vm_name token + while IFS= read -r vm_name; do + [[ -z "$vm_name" ]] && continue + mold_backup_vm_in_filter "$vm_name" || continue + [[ "$seen" == *"|${vm_name}|"* ]] && continue + seen="${seen}|${vm_name}|" + targets+=("$vm_name") + done < <(mold_backup_list_running_domains) + + local include="${VM_INCLUDE:-*}" + if [[ "$include" != "*" ]]; then + IFS=',' read -ra _in <<< "$include" + for token in "${_in[@]}"; do + token="$(echo "$token" | xargs)" + [[ -z "$token" ]] && continue + mold_backup_vm_in_filter "$token" || continue + mold_backup_domain_exists "$token" || continue + [[ "$seen" == *"|${token}|"* ]] && continue + seen="${seen}|${token}|" + targets+=("$token") + done + fi + + if [[ ${#targets[@]} -eq 0 ]]; then + return 0 + fi + printf '%s\n' "${targets[@]}" +} + +mold_backup_vm_in_filter() { + local vm_name="$1" + local include="${VM_INCLUDE:-*}" + local exclude="${VM_EXCLUDE:-}" + local token + + if [[ -n "$exclude" ]]; then + IFS=',' read -ra _ex <<< "$exclude" + for token in "${_ex[@]}"; do + token="$(echo "$token" | xargs)" + [[ -z "$token" ]] && continue + [[ "$vm_name" == "$token" ]] && return 1 + done + fi + + [[ "$include" == "*" ]] && return 0 + IFS=',' read -ra _in <<< "$include" + for token in "${_in[@]}"; do + token="$(echo "$token" | xargs)" + [[ -z "$token" ]] && continue + [[ "$vm_name" == "$token" ]] && return 0 + done + return 1 +} + +mold_backup_api_json_field() { + local json="$1" path="$2" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + parts = sys.argv[1].split('.') + cur = d + for p in parts: + if isinstance(cur, list) and cur: + cur = cur[0] + if not isinstance(cur, dict): + cur = None + break + cur = cur.get(p) + if isinstance(cur, list) and cur: + cur = cur[0] + print('' if cur is None else cur) +except Exception: + print('') +" "$path" 2>/dev/null +} + +mold_backup_api_list_config_value() { + local name="$1" + local json val + json=$(mold_backup_cmk_run listConfigurations "name=${name}" 2>/dev/null) || return 1 + val=$(echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + cfgs = d.get('listconfigurationsresponse', {}).get('configuration', []) + if isinstance(cfgs, dict): cfgs = [cfgs] + print(cfgs[0].get('value','') if cfgs else '') +except Exception: + print('') +" 2>/dev/null) + echo "$val" +} + +mold_backup_api_update_config_if_needed() { + local name="$1" value="$2" + local current + current="$(mold_backup_api_list_config_value "$name" 2>/dev/null || true)" + [[ "$current" == "$value" ]] && return 0 + mold_backup_cmk_run updateConfiguration "name=${name}" "value=${value}" >/dev/null \ + || mold_backup_notify_log warn "updateConfiguration ${name} failed (may need admin API key)" +} + +mold_backup_api_list_cluster_config_value() { + local name="$1" cluster_id="$2" + local json val + json=$(mold_backup_cmk_run listConfigurations "name=${name}" "clusterid=${cluster_id}" 2>/dev/null) || return 1 + val=$(echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + cfgs = d.get('listconfigurationsresponse', {}).get('configuration', []) + if isinstance(cfgs, dict): cfgs = [cfgs] + print(cfgs[0].get('value','') if cfgs else '') +except Exception: + print('') +" 2>/dev/null) + echo "$val" +} + +mold_backup_api_update_cluster_config_if_needed() { + local name="$1" value="$2" cluster_id="$3" + local current + [[ -n "$cluster_id" ]] || return 0 + current="$(mold_backup_api_list_cluster_config_value "$name" "$cluster_id" 2>/dev/null || true)" + [[ "$current" == "$value" ]] && return 0 + mold_backup_cmk_run updateConfiguration "name=${name}" "value=${value}" "clusterid=${cluster_id}" >/dev/null \ + && mold_backup_notify_log info "Enabled ${name}=${value} for cluster ${cluster_id}" \ + || mold_backup_notify_log warn "updateConfiguration ${name} clusterid=${cluster_id} failed (admin API key required)" +} + +# kvm.incremental.backup defaults to false at cluster scope — MS always chooses FULL without this. +mold_backup_api_ensure_cluster_incremental_backup() { + local json cluster_id + [[ -n "${ZONE_ID:-}" ]] || return 0 + json=$(mold_backup_cmk_run listClusters "zoneid=${ZONE_ID}" 2>/dev/null) || return 0 + while IFS= read -r cluster_id; do + [[ -z "$cluster_id" ]] && continue + mold_backup_api_update_cluster_config_if_needed "kvm.incremental.backup" "true" "$cluster_id" + done < <(printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + cs = d.get('listclustersresponse', {}).get('cluster', []) + if isinstance(cs, dict): cs = [cs] + for c in cs: + cid = c.get('id') + if cid: + print(cid) +except Exception: + pass +" 2>/dev/null) +} + +mold_backup_api_ensure_global_settings() { + mold_backup_api_update_config_if_needed "backup.framework.enabled" "true" + mold_backup_api_update_config_if_needed "backup.enable.attach.detach.of.volumes" "true" + mold_backup_api_update_config_if_needed "backup.framework.provider.plugin" "${VEEAM_PROVIDER_NAME}" + [[ -n "${VEEAM_URL:-}" ]] && mold_backup_api_update_config_if_needed "backup.plugin.ablestack-veeam.url" "${VEEAM_URL}" + [[ -n "${VEEAM_USERNAME:-}" ]] && mold_backup_api_update_config_if_needed "backup.plugin.ablestack-veeam.username" "${VEEAM_USERNAME}" + [[ -n "${VEEAM_PASSWORD:-}" ]] && mold_backup_api_update_config_if_needed "backup.plugin.ablestack-veeam.password" "${VEEAM_PASSWORD}" + mold_backup_api_ensure_cluster_incremental_backup +} + +mold_backup_api_extract_error() { + local json="$1" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + err = d.get('errorresponse', {}) + if err: + print(err.get('errortext', err)) + sys.exit(0) + for k, v in d.items(): + if k.endswith('response') and isinstance(v, dict) and v.get('errortext'): + print(v.get('errortext')) + sys.exit(0) +except Exception: + pass +" 2>/dev/null +} + +mold_backup_api_log_ms_schema_hint() { + local msg="$1" + [[ "$msg" == *backup_offering_details* ]] || return 0 + mold_backup_log err "Mold MS DB is missing table cloud.backup_offering_details (schema 4.23+). On MS host run: mysql cloud < mold-ms-backup-schema-fix.sql ; restart management server" >&2 +} + +mold_backup_api_list_backup_offerings() { + local json count err + local -a args=() + [[ -n "${ZONE_ID:-}" ]] && args+=("zoneid=${ZONE_ID}") + json=$(mold_backup_cmk_run listBackupOfferings "${args[@]}" 2>/dev/null) || return 1 + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + if [[ -n "$err" ]]; then + mold_backup_api_log_ms_schema_hint "$err" + mold_backup_notify_log err "listBackupOfferings failed: ${err}" + return 1 + fi + count=$(echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + r = d.get('listbackupofferingsresponse', {}) + c = r.get('count') + if c is None: + offs = r.get('backupoffering', []) + if isinstance(offs, dict): offs = [offs] + c = len(offs) + print(int(c or 0)) +except Exception: + print(0) +" 2>/dev/null) + if [[ "${count:-0}" -eq 0 && -n "${ZONE_ID:-}" ]]; then + json=$(mold_backup_cmk_run listBackupOfferings 2>/dev/null) || return 1 + fi + echo "$json" +} + +mold_backup_api_list_backup_repositories() { + local -a args=() + [[ -n "${ZONE_ID:-}" ]] && args+=("zoneid=${ZONE_ID}") + mold_backup_cmk_run listBackupRepositories "${args[@]}" 2>/dev/null +} + +# First zone UUID (listZones) — used by veeam_config.sh auto-fill. +mold_backup_api_first_zone_id() { + local json + json=$(mold_backup_cmk_run listZones 2>/dev/null) || return 1 + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + zones = d.get('listzonesresponse', {}).get('zone', []) + if isinstance(zones, dict): + zones = [zones] + if zones: + print(zones[0].get('id', '')) +except Exception: + pass +" 2>/dev/null +} + +# First backup repository NFS/CIFS address — used by veeam_config.sh auto-fill. +mold_backup_api_first_repo_address() { + local json name="${1:-}" + json=$(mold_backup_api_list_backup_repositories) || return 1 + echo "$json" | python3 -c " +import json, sys +name = sys.argv[1] if len(sys.argv) > 1 else '' +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): + repos = [repos] + if name: + for r in repos: + if r.get('name') == name: + print(r.get('address', '')) + sys.exit(0) + if repos: + print(repos[0].get('address', '')) +except Exception: + pass +" "$name" 2>/dev/null +} + +# importBackupOffering externalid MUST equal backup repository UUID (see BackupRepositoryDaoImpl.findByBackupOfferingId). +mold_backup_api_find_backup_repository_uuid() { + local json name="${1:-}" + json=$(mold_backup_api_list_backup_repositories) || return 1 + echo "$json" | python3 -c " +import json, sys +name = sys.argv[1] if len(sys.argv) > 1 else '' +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): repos = [repos] + if name: + for r in repos: + if r.get('name') == name: + print(r.get('id', '')) + sys.exit(0) + if repos: + print(repos[0].get('id', '')) +except Exception: + pass +" "$name" 2>/dev/null +} + +mold_backup_api_find_backup_repository_uuid_by_address() { + local address="$1" json + [[ -n "$address" ]] || return 1 + address="${address#nfs://}" + address="${address#cifs://}" + json=$(mold_backup_api_list_backup_repositories) || return 1 + echo "$json" | python3 -c " +import json, sys +want = sys.argv[1] +def norm(a): + if not a: return '' + a = a.strip() + for p in ('nfs://', 'cifs://'): + if a.startswith(p): + a = a[len(p):] + return a +want = norm(want) +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): repos = [repos] + for r in repos: + if norm(r.get('address', '')) == want: + print(r.get('id', '')) + sys.exit(0) +except Exception: + pass +" "$address" 2>/dev/null +} + +# True when id is a backup repository UUID (not a backup offering id). +mold_backup_api_repository_exists() { + local want="$1" json + [[ -n "$want" ]] || return 1 + json=$(mold_backup_api_list_backup_repositories 2>/dev/null) || return 1 + echo "$json" | python3 -c " +import json, sys +want = sys.argv[1] +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): + repos = [repos] + for r in repos: + if r.get('id') == want: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$want" +} + +# Create Mold backup repository when BACKUP_REPO_ADDRESS is set (addBackupRepository). +mold_backup_api_ensure_repository() { + local repo_id name addr repo_type args json err + repo_id="${BACKUP_REPOSITORY_UUID:-}" + if [[ -n "$repo_id" ]]; then + if mold_backup_api_repository_exists "$repo_id"; then + echo "$repo_id" + return 0 + fi + mold_backup_notify_log warn "BACKUP_REPOSITORY_UUID=${repo_id} is not a repository id (maybe an offering id?) — resolving from address/name" + repo_id="" + fi + + name="${BACKUP_REPO_NAME:-Ablestack Veeam NAS}" + if [[ -n "${BACKUP_REPO_ADDRESS:-}" ]]; then + addr="$(mold_backup_clean_repo_address)" + repo_id="$(mold_backup_api_find_backup_repository_uuid_by_address "$addr" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + fi + repo_id="$(mold_backup_api_find_backup_repository_uuid "$name" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + repo_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + + [[ -n "${BACKUP_REPO_ADDRESS:-}" ]] || { + mold_backup_notify_log err "No backup repository — set BACKUP_REPO_ADDRESS in conf or create in Mold UI" + return 1 + } + [[ -n "${ZONE_ID:-}" ]] || { + mold_backup_notify_log err "ZONE_ID required to addBackupRepository" + return 1 + } + + addr="$(mold_backup_clean_repo_address)" + repo_type="${BACKUP_REPO_TYPE:-nfs}" + args=( + "name=${name}" + "address=${addr}" + "type=${repo_type}" + "zoneid=${ZONE_ID}" + ) + [[ -n "${BACKUP_REPO_MOUNT_OPTS:-}" ]] && args+=("mountoptions=${BACKUP_REPO_MOUNT_OPTS}") + [[ -n "${BACKUP_REPO_PROVIDER:-}" ]] && args+=("provider=${BACKUP_REPO_PROVIDER}") + + mold_backup_notify_log info "addBackupRepository name=${name} address=${addr} type=${repo_type}" + if ! json=$(mold_backup_cmk_run addBackupRepository "${args[@]}" 2>&1); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "addBackupRepository failed${err:+: ${err}}" + repo_id="$(mold_backup_api_find_backup_repository_uuid_by_address "$addr" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + return 1 + fi + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + if [[ -n "$err" ]]; then + mold_backup_notify_log err "addBackupRepository failed: ${err}" + repo_id="$(mold_backup_api_find_backup_repository_uuid_by_address "$addr" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + return 1 + fi + repo_id="$(mold_backup_api_json_field "$json" "addbackuprepositoryresponse.backuprepository.id")" + [[ -n "$repo_id" ]] || repo_id="$(mold_backup_api_find_backup_repository_uuid "$name" 2>/dev/null || true)" + [[ -n "$repo_id" ]] || { + mold_backup_notify_log err "addBackupRepository returned no repository id" + return 1 + } + mold_backup_notify_log info "Backup repository ready id=${repo_id}" + echo "$repo_id" +} + +# Ensure backup repository + offering exist (NAS/guest). Datadisk host mode uses KVM +# /data/backup only — assign backup offering in Mold UI; no addBackupRepository. +mold_backup_api_ensure_backup_resources() { + local offering_id repo_id + if mold_backup_is_datadisk_mode; then + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" "$(mold_backup_offering_name)" 2>/dev/null || true)" + [[ -n "$offering_id" ]] || offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + if [[ -n "$offering_id" ]]; then + echo "$offering_id" + return 0 + fi + mold_backup_notify_log warn "datadisk mode: assign backup offering '$(mold_backup_offering_name)' (${VEEAM_PROVIDER_NAME}) in Mold UI — Mold backup repository is not auto-created" + return 1 + fi + repo_id="$(mold_backup_api_ensure_repository 2>/dev/null || true)" + if [[ -n "$repo_id" ]]; then + BACKUP_REPOSITORY_UUID="$repo_id" + OFFERING_EXTERNAL_ID="${OFFERING_EXTERNAL_ID:-$repo_id}" + fi + offering_id="$(mold_backup_api_ensure_offering 2>/dev/null || true)" + [[ -n "$offering_id" ]] && { echo "$offering_id"; return 0; } + return 1 +} + +mold_backup_api_get_offering_external_id() { + local offering_id="$1" json + json=$(mold_backup_api_list_backup_offerings 2>/dev/null) || return 1 + echo "$json" | python3 -c " +import json, sys +oid = sys.argv[1] +try: + d = json.load(sys.stdin) + offs = d.get('listbackupofferingsresponse', {}).get('backupoffering', []) + if isinstance(offs, dict): offs = [offs] + for o in offs: + if o.get('id') == oid: + print(o.get('externalid', '')) + break +except Exception: + pass +" "$offering_id" 2>/dev/null +} + +mold_backup_api_validate_offering_repository() { + local offering_id="$1" + if mold_backup_is_datadisk_mode; then + [[ -n "$offering_id" ]] && return 0 + return 1 + fi + local ext_id repo_id + ext_id="$(mold_backup_api_get_offering_external_id "$offering_id" 2>/dev/null || true)" + repo_id="${BACKUP_REPOSITORY_UUID:-}" + [[ -n "$repo_id" ]] || repo_id="$(mold_backup_api_find_backup_repository_uuid "${BACKUP_REPO_NAME:-}" 2>/dev/null || true)" + [[ -n "$repo_id" ]] || repo_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + [[ -n "$repo_id" ]] || { + mold_backup_notify_log err "No backup repository in zone — create one in Mold UI (Infrastructure → Backup Repositories)" + return 1 + } + [[ "$ext_id" == "$repo_id" ]] && return 0 + mold_backup_notify_log err "Backup offering externalid=${ext_id:-} does not match repository id=${repo_id}. Re-import offering with: externalid=${repo_id} (or set BACKUP_REPOSITORY_UUID in conf)" + return 1 +} + +mold_backup_api_log_repository_hint() { + local msg="$1" + [[ "$msg" == *"backup repository"* ]] || return 0 + local repo_id + repo_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + mold_backup_notify_log err "Fix: importBackupOffering externalid must equal backup repository UUID${repo_id:+ (${repo_id})}" +} + +mold_backup_api_log_agent_import_seed_hint() { + local msg="$1" + [[ "$msg" == *UnsupportedAnswer* ]] || return 0 + mold_backup_notify_log err "KVM mold-agent does not handle AblestackNasImportVeeamSeedCommand (import-veeam-seed). Update mold-agent to a build that includes LibvirtAblestackNasImportVeeamSeedCommandWrapper, ensure ablestack_nasbackup.sh supports -o import-veeam-seed, then: systemctl restart cloudstack-agent" +} + +mold_backup_api_extract_async_job_error() { + local json="$1" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + r = d.get('queryasyncjobresultresponse', {}) + jr = r.get('jobresult') + if isinstance(jr, dict): + if jr.get('errortext'): + print(jr.get('errortext')) + elif jr.get('errorresponse', {}).get('errortext'): + print(jr['errorresponse']['errortext']) + elif isinstance(jr, str) and jr.strip(): + print(jr.strip()) + if r.get('errortext'): + print(r.get('errortext')) +except Exception: + pass +" 2>/dev/null +} + +mold_backup_offering_name() { + echo "${BACKUP_OFFERING_NAME:-VeeamBackup}" +} + +mold_backup_api_find_offering_id() { + local provider="${1:-${VEEAM_PROVIDER_NAME}}" + local offering_name="${2:-$(mold_backup_offering_name)}" + local json + json=$(mold_backup_api_list_backup_offerings) || return 1 + echo "$json" | python3 -c " +import json, sys +provider = (sys.argv[1] or '').lower() +name = sys.argv[2] if len(sys.argv) > 2 else '' +aliases = {provider} +if provider in ('ablestack-veeam', 'veeam'): + aliases.update(['ablestack-veeam', 'veeam']) +if provider == 'ablestack-nas': + aliases.update(['ablestack-nas', 'nas']) +try: + d = json.load(sys.stdin) + offs = d.get('listbackupofferingsresponse', {}).get('backupoffering', []) + if isinstance(offs, dict): offs = [offs] + if name: + for o in offs: + if o.get('name') == name: + print(o.get('id', '')) + sys.exit(0) + for o in offs: + p = (o.get('provider') or '').lower() + if p in aliases: + print(o.get('id', '')) + break +except Exception: + pass +" "$provider" "$offering_name" 2>/dev/null +} + +mold_backup_api_ensure_offering() { + local offering_id json err want_ext ext_id + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + want_ext="${OFFERING_EXTERNAL_ID:-${BACKUP_REPOSITORY_UUID:-}}" + if [[ -n "$offering_id" && -n "$want_ext" ]]; then + ext_id="$(mold_backup_api_get_offering_external_id "$offering_id" 2>/dev/null || true)" + if [[ -n "$ext_id" && "$ext_id" != "$want_ext" ]]; then + mold_backup_notify_log warn "Backup offering id=${offering_id} externalid=${ext_id} != repository ${want_ext}" + mold_backup_notify_log warn "Delete '$(mold_backup_offering_name)' in Mold UI (Infrastructure → Backup Offerings) then re-run: mold_backup_api_ensure_backup_resources" + offering_id="" + fi + fi + [[ -n "$offering_id" ]] && { echo "$offering_id"; return 0; } + [[ -n "${ZONE_ID:-}" ]] || { + mold_backup_notify_log err "ZONE_ID required to importBackupOffering" + return 1 + } + local name + name="$(mold_backup_offering_name)" + local ext_id="${OFFERING_EXTERNAL_ID:-${BACKUP_REPOSITORY_UUID:-}}" + if [[ -z "$ext_id" ]]; then + ext_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + fi + [[ -n "$ext_id" ]] || { + mold_backup_notify_log err "No backup repository UUID for importBackupOffering — create Backup Repository in Mold UI or set BACKUP_REPOSITORY_UUID" + return 1 + } + local retention="${RETENTION_PERIOD:-P7D}" + local args=( + "name=${name}" + "description=Ablestack Veeam backup offering (${name})" + "provider=${VEEAM_PROVIDER_NAME}" + "externalid=${ext_id}" + "zoneid=${ZONE_ID}" + "allowuserdrivenbackups=false" + "retentionperiod=${retention}" + ) + if ! json=$(mold_backup_cmk_run importBackupOffering "${args[@]}" 2>&1); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importBackupOffering failed${err:+: ${err}}" + return 1 + fi + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + if [[ -n "$err" ]]; then + mold_backup_notify_log err "importBackupOffering failed: ${err}" + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + [[ -n "$offering_id" ]] && { echo "$offering_id"; return 0; } + return 1 + fi + offering_id="$(mold_backup_api_json_field "$json" "importbackupofferingresponse.backupoffering.id")" + local job_id + job_id="$(mold_backup_api_json_field "$json" "importbackupofferingresponse.jobid")" + if [[ -z "$offering_id" && -n "$job_id" ]]; then + mold_backup_notify_log info "importBackupOffering async job=${job_id}; waiting" + json=$(mold_backup_api_wait_async_job "$job_id" 300) || return 1 + offering_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backupoffering.id")" + [[ -z "$offering_id" ]] && offering_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.id")" + fi + [[ -n "$offering_id" ]] || offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + [[ -n "$offering_id" ]] || { + mold_backup_notify_log err "importBackupOffering returned no offering id (async job may still be running; check listBackupOfferings)" + return 1 + } + echo "$offering_id" +} + +mold_backup_safe_job_name() { + echo "$1" | tr ' /' '__' +} + +mold_backup_registry_dir() { + echo "${ABLESTACK_VEEAM_ETC_DIR}/registry" +} + +mold_backup_api_pick_vm_record() { + local json="$1" lookup="$2" + echo "$json" | python3 -c " +import json, sys +lookup = sys.argv[1] +try: + d = json.load(sys.stdin) + vms = d.get('listvirtualmachinesresponse', {}).get('virtualmachine', []) + if isinstance(vms, dict): vms = [vms] + for v in vms: + if v.get('instancename') == lookup or v.get('name') == lookup: + print(json.dumps({'listvirtualmachinesresponse': {'count': 1, 'virtualmachine': v}})) + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$lookup" 2>/dev/null +} + +mold_backup_api_get_vm_record() { + local lookup="$1" + local -a args=("listall=true") + local json picked + local try_zone="${2:-}" + + _mold_backup_list_vms() { + local -a call_args=("listall=true") + [[ -n "$1" ]] && call_args+=("zoneid=$1") + mold_backup_cmk_run listVirtualMachines "${call_args[@]}" 2>/dev/null + } + + if [[ -n "$try_zone" ]]; then + json="$(_mold_backup_list_vms "$try_zone")" || json="" + if picked=$(mold_backup_api_pick_vm_record "$json" "$lookup" 2>/dev/null); then + echo "$picked" + return 0 + fi + fi + + json=$(mold_backup_cmk_run listVirtualMachines "listall=true" "name=${lookup}" 2>/dev/null) || true + if picked=$(mold_backup_api_pick_vm_record "$json" "$lookup" 2>/dev/null); then + echo "$picked" + return 0 + fi + + json="$(_mold_backup_list_vms "")" || return 1 + mold_backup_api_pick_vm_record "$json" "$lookup" +} + +mold_backup_api_get_vm_id() { + local instance_name="$1" json + if [[ "${VM_NAME:-}" == "$instance_name" && -n "${VM_UUID:-}" ]]; then + echo "$VM_UUID" + return 0 + fi + json=$(mold_backup_api_get_vm_record "$instance_name" "${ZONE_ID:-}") || return 1 + mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.id" +} + +mold_backup_api_get_vm_offering_id() { + local instance_name="$1" json + json=$(mold_backup_api_get_vm_record "$instance_name") || return 1 + mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid" +} + +# Mold UI backup name: {vm-hostname}-{yyyy-MM-ddTHH:mm:ss+0000} (same as netbackup / getBackupNameFromVM) +mold_backup_api_format_backup_name() { + local vm_label="$1" + echo "${vm_label}-$(date -u +%Y-%m-%dT%H:%M:%S+0000)" +} + +mold_backup_api_get_vm_hostname() { + local vm_id="$1" json name + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null) || return 1 + name="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.name")" + [[ -n "$name" ]] || name="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.instancename")" + [[ -n "$name" ]] || return 1 + echo "$name" +} + +mold_backup_api_build_backup_name_for_vm() { + local vm_id="$1" fallback_label="${2:-}" + local vm_label + # Prefer Mold VM hostname (e.g. backup-test) over libvirt instance name (i-2-7-VM) + vm_label="$(mold_backup_api_get_vm_hostname "$vm_id" 2>/dev/null || true)" + [[ -n "$vm_label" ]] || vm_label="$fallback_label" + [[ -n "$vm_label" ]] || vm_label="$vm_id" + mold_backup_api_format_backup_name "$vm_label" +} + +mold_backup_api_assign_offering_if_needed() { + local vm_id="$1" offering_id="$2" + local json current + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null) || return 1 + current="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + [[ "$current" == "$offering_id" ]] && return 0 + mold_backup_cmk_run assignVirtualMachineToBackupOffering "virtualmachineid=${vm_id}" "backupofferingid=${offering_id}" >/dev/null \ + || mold_backup_notify_log warn "assignVirtualMachineToBackupOffering failed for vm=${vm_id}" +} + +mold_backup_api_wait_async_job() { + local job_id="$1" max_wait="${2:-600}" + local elapsed=0 json status result + [[ -z "$job_id" ]] && return 1 + while [[ "$elapsed" -lt "$max_wait" ]]; do + json=$(mold_backup_cmk_run queryAsyncJobResult "jobid=${job_id}" 2>/dev/null) || return 1 + status="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobstatus")" + if [[ "$status" == "1" ]]; then + result="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult")" + echo "$json" + return 0 + fi + if [[ "$status" == "2" ]]; then + local job_err + job_err="$(mold_backup_api_extract_async_job_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "Async job failed: ${job_id}${job_err:+ — ${job_err}}" + mold_backup_api_log_repository_hint "$job_err" + mold_backup_api_log_agent_import_seed_hint "$job_err" + return 1 + fi + if [[ $((elapsed % 30)) -eq 0 ]]; then + mold_backup_notify_log info "Async job ${job_id} pending (status=${status:-0}, elapsed=${elapsed}s/${max_wait}s)" + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + mold_backup_notify_log err "Async job timeout: ${job_id}" + return 1 +} + +mold_backup_api_create_veeam_and_wait() { + local vm_id="$1" vm_label="${2:-}" + local json job_id backup_id backup_type backup_name + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "createAblestackVeeamBackup vm=${vm_id} name=${backup_name} (MS→agent NAS backup)" + json=$(mold_backup_cmk_run createAblestackVeeamBackup "virtualmachineid=${vm_id}" "name=${backup_name}" 2>/dev/null) \ + || json=$(mold_backup_cmk_run createBackup "virtualmachineid=${vm_id}" "name=${backup_name}" 2>/dev/null) \ + || return 1 + job_id="$(mold_backup_api_json_field "$json" "createablestackveeambackupresponse.jobid")" + [[ -z "$job_id" ]] && job_id="$(mold_backup_api_json_field "$json" "createbackupresponse.jobid")" + if [[ -n "$job_id" ]]; then + mold_backup_notify_log info "createAblestackVeeamBackup job=${job_id}; waiting for MS/NAS backup" + json=$(mold_backup_api_wait_async_job "$job_id" 1200) || return 1 + backup_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.id")" + backup_type="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.type")" + if [[ -z "$backup_id" ]]; then + local latest + latest="$(mold_backup_api_find_latest_backed_up_backup_for_vm "$vm_id" 2>/dev/null || true)" + if [[ -n "$latest" ]]; then + backup_id="${latest%%|*}" + backup_type="${latest#*|}" + mold_backup_notify_log info "Resolved backup_id=${backup_id} type=${backup_type} via listAblestackVeeamBackups (API returns SuccessResponse only)" + fi + fi + [[ -n "$backup_id" ]] && { echo "${backup_id}|${backup_type:-User}"; return 0; } + fi + backup_id="$(mold_backup_api_json_field "$json" "createablestackveeambackupresponse.backup.id")" + backup_type="$(mold_backup_api_json_field "$json" "createablestackveeambackupresponse.backup.type")" + [[ -n "$backup_id" ]] && { echo "${backup_id}|${backup_type:-User}"; return 0; } + return 1 +} + +# Step 4 — environment check: target VM + incremental chain count (설계: 대상머신, Chain 수) +mold_backup_api_count_backed_up_from_json() { + local json="$1" + printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): b = [b] + backed = [x for x in b if str(x.get('status', '')).lower() == 'backedup'] + print(len(backed)) +except Exception: + print(0) +" 2>/dev/null +} + +mold_backup_api_check_vm_environment() { + local vm_id="$1" vm_name="$2" + local json chain_size max_chain offering_id + max_chain="${VEEAM_MAX_CHAIN:-7}" + json=$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null) || { + mold_backup_notify_log info "env vm=${vm_name} id=${vm_id} chain=0 max=${max_chain} (no BackedUp backups)" + return 0 + } + chain_size=$(mold_backup_api_count_backed_up_from_json "$json") + mold_backup_notify_log info "env vm=${vm_name} id=${vm_id} chain=${chain_size} max=${max_chain}" + if [[ "$chain_size" -ge "$max_chain" ]]; then + mold_backup_notify_log warn "Chain size ${chain_size} >= max ${max_chain}; next backup may be full" + fi +} + +mold_backup_api_veeam_backup_count() { + local vm_id="$1" + local json + json=$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null) || { + echo 0 + return 0 + } + mold_backup_api_count_backed_up_from_json "$json" +} + +# createAblestackVeeamBackup async job returns SuccessResponse (no backup id in jobresult). +mold_backup_api_find_latest_backed_up_backup_for_vm() { + local vm_id="$1" json + json=$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null) || return 1 + printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): + b = [b] + backed = [x for x in b if str(x.get('status', '')).lower() == 'backedup'] + if not backed: + sys.exit(1) + latest = max(backed, key=lambda x: x.get('created') or '') + bid = latest.get('id', '') + btype = latest.get('type', 'User') + if not bid: + sys.exit(1) + print(f\"{bid}|{btype}\") +except Exception: + sys.exit(1) +" 2>/dev/null +} + +mold_backup_collect_host_staging_paths() { + local host_dir="$1" + local -a files=() + local f seen="" key + [[ -d "$host_dir" ]] || return 1 + while IFS= read -r -d '' f; do + key=",${f}," + [[ "$seen" == *"$key"* ]] && continue + seen="${seen}${key}" + files+=("$f") + done < <(find "$host_dir" -maxdepth 1 -type f \( -name 'disk-*' -o -name '*.qcow2' -o -name '*.qcow' -o -name '*.vmdk' -o -name '*.raw' \) -print0 2>/dev/null) + [[ ${#files[@]} -gt 0 ]] || return 1 + (IFS=,; echo "${files[*]}") +} + +mold_backup_detect_staging_source_format() { + local staging_paths="$1" + local first="${staging_paths%%,*}" + [[ -n "$first" && -f "$first" ]] || { echo "qcow2"; return 0; } + if command -v qemu-img >/dev/null 2>&1; then + if qemu-img info "$first" 2>/dev/null | grep -q 'file format: raw'; then + echo "raw" + return 0 + fi + if qemu-img info "$first" 2>/dev/null | grep -q 'file format: qcow2'; then + echo "qcow2" + return 0 + fi + fi + case "$first" in + *.raw) echo "raw" ;; + *.vmdk) echo "vmdk" ;; + *) echo "qcow2" ;; + esac +} + +# cmk may prefix stderr noise when captured with 2>&1; keep the JSON object only. +mold_backup_api_sanitize_json() { + local raw="$1" + printf '%s' "$raw" | python3 -c " +import json, sys +raw = sys.stdin.read() +start = raw.find('{') +if start < 0: + print(raw) + sys.exit(0) +blob = raw[start:] +for end in range(len(blob), 0, -1): + try: + d = json.loads(blob[:end]) + print(json.dumps(d)) + sys.exit(0) + except Exception: + pass +print(blob) +" 2>/dev/null +} + +# Parse importAblestackVeeamBackupSeed response; waits on jobid when present. +# Prints backup_id|type on stdout. +mold_backup_api_finish_import_seed_response() { + local json="$1" + local job_id backup_id backup_type err + json="$(mold_backup_api_sanitize_json "$json")" + job_id="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.jobid")" + backup_id="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.backup.id")" + [[ -z "$backup_id" ]] && backup_id="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.id")" + backup_type="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.backup.type")" + [[ -z "$backup_type" ]] && backup_type="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.type")" + if [[ -n "$job_id" ]]; then + mold_backup_notify_log info "importAblestackVeeamBackupSeed job=${job_id}; waiting for NAS seed import" + json=$(mold_backup_api_wait_async_job "$job_id" 1200) || return 1 + json="$(mold_backup_api_sanitize_json "$json")" + backup_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.id")" + backup_type="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.type")" + [[ -z "$backup_id" ]] && backup_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.id")" + [[ -z "$backup_type" ]] && backup_type="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.type")" + fi + [[ -n "$backup_id" ]] && { echo "${backup_id}|${backup_type:-User}"; return 0; } + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importAblestackVeeamBackupSeed: no backup id in response${err:+ — ${err}} (raw=${json:0:240})" + return 1 +} + +mold_backup_api_import_seed_and_wait() { + local vm_id="$1" staging_paths="$2" source_format="${3:-qcow2}" vm_label="${4:-}" + local json backup_name + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "importAblestackVeeamBackupSeed vm=${vm_id} name=${backup_name} (host staging, no MS→Veeam API)" + json=$(mold_backup_cmk_run importAblestackVeeamBackupSeed \ + "virtualmachineid=${vm_id}" \ + "name=${backup_name}" \ + "stagingdiskpaths=${staging_paths}" \ + "sourcediskformat=${source_format}" \ + "bootstrapcheckpoint=true" 2>/dev/null) || return 1 + mold_backup_api_finish_import_seed_response "$json" +} + +# Import NAS seed from KVM host staging; optionally tag the Veeam restore point on the backup. +mold_backup_api_import_staging_rp_seed_and_wait() { + local vm_id="$1" staging_paths="$2" source_format="${3:-qcow2}" rp_id="${4:-}" vm_label="${5:-}" + local json backup_name err + [[ -n "$staging_paths" ]] || return 1 + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "importAblestackVeeamBackupSeed vm=${vm_id} rp=${rp_id:-n/a} name=${backup_name} (host staging)" + local -a api_args=( + "virtualmachineid=${vm_id}" + "name=${backup_name}" + "stagingdiskpaths=${staging_paths}" + "sourcediskformat=${source_format}" + "bootstrapcheckpoint=true" + ) + [[ -n "$rp_id" ]] && api_args+=("veeamrestorepointid=${rp_id}") + if ! json=$(mold_backup_cmk_run importAblestackVeeamBackupSeed "${api_args[@]}" 2>/dev/null); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importAblestackVeeamBackupSeed failed: ${err:-${json:0:200}}" + return 1 + fi + mold_backup_api_finish_import_seed_response "$json" +} + +# Import NAS seed from a Veeam restore point (MS exports disks from Veeam; no KVM staging). +mold_backup_api_import_rp_seed_and_wait() { + local vm_id="$1" rp_id="$2" vm_label="${3:-}" + local json backup_name err + [[ -n "$rp_id" ]] || return 1 + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "importAblestackVeeamBackupSeed vm=${vm_id} rp=${rp_id} name=${backup_name} (MS→Veeam export)" + if ! json=$(mold_backup_cmk_run importAblestackVeeamBackupSeed \ + "virtualmachineid=${vm_id}" \ + "name=${backup_name}" \ + "veeamrestorepointid=${rp_id}" \ + "bootstrapcheckpoint=true" 2>/dev/null); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importAblestackVeeamBackupSeed (MS→Veeam) failed: ${err:-${json:0:200}}" + return 1 + fi + mold_backup_api_finish_import_seed_response "$json" +} + +mold_backup_api_list_backup_details() { + local backup_id="$1" + mold_backup_cmk_run listBackups "id=${backup_id}" "listvmdetails=true" 2>/dev/null +} + +mold_backup_api_backup_detail_field() { + local backup_id="$1" key="$2" + local json + json=$(mold_backup_api_list_backup_details "$backup_id") || return 1 + echo "$json" | python3 -c " +import json, sys +key = sys.argv[1] +try: + d = json.load(sys.stdin) + b = d.get('listbackupsresponse', {}).get('backup', {}) + if isinstance(b, list): + b = b[0] if b else {} + details = b.get('vmdetails') or b.get('vmDetails') or {} + if isinstance(details, str): + details = json.loads(details) if details else {} + val = details.get(key, '') + if not val and isinstance(b.get('details'), dict): + val = b['details'].get(key, '') + print(val or '') +except Exception: + print('') +" "$key" 2>/dev/null +} + +mold_backup_api_get_backup_type() { + local backup_id="$1" json + json=$(mold_backup_api_list_backup_details "$backup_id") || return 1 + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listbackupsresponse', {}).get('backup', {}) + if isinstance(b, list): + b = b[0] if b else {} + print(b.get('type', '') or '') +except Exception: + print('') +" 2>/dev/null +} + +mold_backup_api_get_parent_backup_id() { + local backup_id="$1" + mold_backup_api_backup_detail_field "$backup_id" "nas.parent.backup.uuid" +} + +mold_backup_api_is_full_backup() { + local backup_id="$1" btype parent + btype="$(mold_backup_api_get_backup_type "$backup_id" 2>/dev/null || true)" + case "${btype^^}" in + INCREMENTAL) return 1 ;; + FULL) return 0 ;; + esac + parent="$(mold_backup_api_get_parent_backup_id "$backup_id" 2>/dev/null || true)" + [[ -z "$parent" ]] +} + +# Build restore chain oldest → newest (설계 5: inc 복원 시 필요한 백업본 배열) +mold_backup_api_build_restore_chain() { + local backup_id="$1" + local -a chain=() + local current="$backup_id" parent visited="" + while [[ -n "$current" ]]; do + if [[ ",${visited}," == *",${current},"* ]]; then + mold_backup_notify_log err "Restore chain cycle at backup ${current}" + return 1 + fi + visited="${visited},${current}" + chain=("$current" "${chain[@]}") + parent="$(mold_backup_api_get_parent_backup_id "$current" 2>/dev/null || true)" + current="$parent" + done + (IFS=,; echo "${chain[*]}") +} + +mold_backup_state_write_line() { + local state_file="$1" + shift + echo "$*" >> "$state_file" +} + +mold_backup_state_parse_field() { + local line="$1" key="$2" + if [[ "$line" != *"${key}="* ]]; then + echo "" + return 0 + fi + local val="${line#*${key}=}" + val="${val%% *}" + echo "$val" +} + +# Canonical per-VM latest backup id (used by restore-watch / Mold restore). +mold_backup_vm_backup_id_map_file() { + echo "$(mold_backup_registry_dir)/vm-backup-ids.map" +} + +mold_backup_registry_set_vm_backup_id() { + local vm="$1" backup_id="$2" rp_id="${3:-}" job="${4:-${VEEAM_JOB_NAME:-}}" + local map_file line key + [[ -n "$vm" && -n "$backup_id" ]] || return 1 + map_file="$(mold_backup_vm_backup_id_map_file)" + mkdir -p "$(dirname "$map_file")" + key="vm=${vm} backup_id=${backup_id}" + [[ -n "$rp_id" ]] && key="${key} rp=${rp_id}" + [[ -n "$job" ]] && key="${key} job=${job}" + if [[ -f "$map_file" ]] && grep -q " vm=${vm} " "$map_file" 2>/dev/null; then + sed -i "s|.* vm=${vm} .*|$(date -Iseconds) ${key}|" "$map_file" + else + echo "$(date -Iseconds) ${key}" >> "$map_file" + fi + echo "${backup_id}" > "$(mold_backup_registry_dir)/${vm}.latest-backup-id" + [[ -n "$rp_id" ]] && echo "${rp_id}" > "$(mold_backup_registry_dir)/${vm}.latest-rp-id" + [[ -n "$rp_id" ]] && mold_backup_registry_index_rp_backup "$vm" "$rp_id" "$backup_id" "$job" + mold_backup_sync_vm_backup_ids_conf "$vm" "$backup_id" +} + +mold_backup_normalize_rp_id() { + local rp="${1:-}" + rp="${rp//\{/}" + rp="${rp//\}/}" + rp="$(printf '%s' "$rp" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + echo "$rp" +} + +mold_backup_registry_rp_map_file() { + echo "$(mold_backup_registry_dir)/veeam-rp-backup.map" +} + +# Persist Veeam restore point → Mold backup_id (per VM) for FLR restore-watch. +mold_backup_registry_index_rp_backup() { + local vm="$1" rp_id="$2" backup_id="$3" job="${4:-${VEEAM_JOB_NAME:-}}" + local map_file norm_rp + [[ -n "$vm" && -n "$rp_id" && -n "$backup_id" ]] || return 0 + norm_rp="$(mold_backup_normalize_rp_id "$rp_id")" + [[ -n "$norm_rp" ]] || return 0 + map_file="$(mold_backup_registry_rp_map_file)" + mkdir -p "$(dirname "$map_file")" + if [[ -f "$map_file" ]]; then + grep -viE " rp=${norm_rp} vm=${vm} " "$map_file" > "${map_file}.tmp" 2>/dev/null || : >"${map_file}.tmp" + mv -f "${map_file}.tmp" "$map_file" 2>/dev/null || true + fi + echo "$(date -Iseconds) rp=${norm_rp} vm=${vm} backup_id=${backup_id} job=${job}" >>"$map_file" + echo "${backup_id}" >"$(mold_backup_registry_dir)/${vm}.rp-${norm_rp}.backup-id" +} + +mold_backup_registry_get_backup_id_by_rp() { + local vm="$1" rp_id="$2" + local norm_rp map_file line bid reg_dir + [[ -n "$vm" && -n "$rp_id" ]] || return 1 + norm_rp="$(mold_backup_normalize_rp_id "$rp_id")" + [[ -n "$norm_rp" ]] || return 1 + reg_dir="$(mold_backup_registry_dir)" + if [[ -f "${reg_dir}/${vm}.rp-${norm_rp}.backup-id" ]]; then + bid="$(tr -d '[:space:]' <"${reg_dir}/${vm}.rp-${norm_rp}.backup-id" 2>/dev/null || true)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + fi + map_file="$(mold_backup_registry_rp_map_file)" + if [[ -f "$map_file" ]]; then + line="$(grep -E " rp=${norm_rp} vm=${vm} " "$map_file" 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + fi + if [[ -d "$reg_dir" ]]; then + line="$(grep -hE "vm=${vm}.*backup_id=.*rp=${norm_rp}|vm=${vm}.*rp=${norm_rp}.*backup_id=" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" + fi +} + +mold_backup_registry_get_backup_id_by_checkpoint() { + local vm="$1" ckpt="$2" reg_dir line bid + [[ -n "$vm" && -n "$ckpt" ]] || return 1 + reg_dir="$(mold_backup_registry_dir)" + line="$(grep -hE "vm=${vm}.*backup_id=.*${ckpt}" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + line="$(grep "backup_id=.*${vm}.*${ckpt}\|${vm}/${ckpt}" /var/log/mold/veeam-hook.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" +} + +mold_backup_registry_get_vm_backup_id() { + local vm="$1" map_file line bid + [[ -n "$vm" ]] || return 1 + if [[ -f "$(mold_backup_registry_dir)/${vm}.latest-backup-id" ]]; then + bid="$(tr -d '[:space:]' < "$(mold_backup_registry_dir)/${vm}.latest-backup-id" 2>/dev/null || true)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + fi + map_file="$(mold_backup_vm_backup_id_map_file)" + [[ -f "$map_file" ]] || return 1 + line="$(grep -E "^[^ ]* vm=${vm} " "$map_file" 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" +} + +mold_backup_registry_get_backup_id_by_checkpoint() { + local vm="$1" ckpt="$2" reg_dir line bid + [[ -n "$vm" && -n "$ckpt" ]] || return 1 + reg_dir="$(mold_backup_registry_dir)" + line="$(grep -hE "vm=${vm}.*backup_id=.*${ckpt}|backup_id=.*vm=${vm}.*${ckpt}" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + line="$(grep -rh "path=.*${vm}/${ckpt}\|${vm}/${ckpt}" /var/log/mold/veeam-hook.log 2>/dev/null | grep backup_id | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" +} + +# Keep mold-backup.windows.conf VM_BACKUP_IDS in sync for Veeam restore-watch. +mold_backup_sync_vm_backup_ids_conf() { + local vm="$1" backup_id="$2" + local win_conf="${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.windows.conf" + local existing entry new_val + [[ -n "$vm" && -n "$backup_id" && -f "$win_conf" ]] || return 0 + entry="${vm}:${backup_id}" + existing="$(grep -E '^VM_BACKUP_IDS=' "$win_conf" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + if [[ -z "$existing" ]]; then + echo "VM_BACKUP_IDS=\"${entry}\"" >> "$win_conf" + return 0 + fi + new_val="$existing" + if [[ "$existing" == *"${vm}:"* ]]; then + new_val="$(echo "$existing" | sed -E "s#${vm}:[^,;]*#${vm}:${backup_id}#g")" + else + new_val="${existing},${entry}" + fi + if grep -qE '^VM_BACKUP_IDS=' "$win_conf" 2>/dev/null; then + sed -i "s#^VM_BACKUP_IDS=.*#VM_BACKUP_IDS=\"${new_val}\"#" "$win_conf" + else + echo "VM_BACKUP_IDS=\"${new_val}\"" >> "$win_conf" + fi +} + +mold_backup_registry_save_backup() { + local job="$1" vm_name="$2" backup_id="$3" status="${4:-success}" rp_id="${5:-}" + local reg_dir reg_file + reg_dir="$(mold_backup_registry_dir)" + mkdir -p "$reg_dir" + reg_file="${reg_dir}/$(mold_backup_safe_job_name "$job").log" + echo "$(date -Iseconds) job=${job} vm=${vm_name} backup_id=${backup_id} status=${status}" >> "$reg_file" + mold_backup_registry_set_vm_backup_id "$vm_name" "$backup_id" "$rp_id" "$job" + mold_backup_notify_log info "Saved backup registry: vm=${vm_name} backup_id=${backup_id} status=${status}" +} + +# Record a restore event in the Mold-side registry (separate file per job, suffix .restore.log). +# event: source (veeam|mold), session id, restore point/end time. Used to reflect a restore +# performed directly in the Veeam UI back into Mold's state view. +mold_backup_registry_save_restore() { + local job="$1" vm_name="$2" source="${3:-veeam}" session="${4:-}" detail="${5:-}" status="${6:-restored}" + local reg_dir reg_file + reg_dir="$(mold_backup_registry_dir)" + mkdir -p "$reg_dir" + reg_file="${reg_dir}/$(mold_backup_safe_job_name "$job").restore.log" + echo "$(date -Iseconds) job=${job} vm=${vm_name} source=${source} session=${session} detail=${detail} status=${status}" >> "$reg_file" + mold_backup_notify_log info "Saved restore registry: vm=${vm_name} source=${source} session=${session} status=${status}" +} + +mold_backup_process_vm_pre_notify() { + local vm_name="$1" offering_id="$2" state_file="$3" + local vm_id backup_result backup_id backup_type host_path + + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { + mold_backup_notify_log err "No Mold VM id for ${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} status=fail reason=no-vm-id" + return 1 + } + + mold_backup_api_check_vm_environment "$vm_id" "$vm_name" + + local vm_offering json + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null || true) + vm_offering="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + if [[ -z "$offering_id" && -n "$vm_offering" ]]; then + offering_id="$vm_offering" + mold_backup_notify_log info "Using VM-assigned backup offering id=${offering_id}" + fi + if [[ -n "$offering_id" ]]; then + mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null || true) + vm_offering="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + fi + [[ -n "$vm_offering" ]] || { + mold_backup_notify_log err "VM ${vm_name} has no backup offering (assign '$(mold_backup_offering_name)' / ${VEEAM_PROVIDER_NAME} in Mold UI)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=no-offering" + return 1 + } + + # Loop guard (bidirectional): if this Veeam run was itself triggered by a Mold + # backup, the Mold NAS backup already happened — let Veeam do disk-only and skip + # createBackup to avoid re-triggering Mold. + if mold_backup_trigger_active "mold-active" "$vm_name"; then + mold_backup_trigger_clear "mold-active" "$vm_name" + mold_backup_notify_log info "mold-active marker present for ${vm_name}: Mold already backed up; Veeam disk-only (skip createBackup)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=success reason=mold-triggered" + return 0 + fi + # Mark this VM as Veeam-driven so the Mold→Veeam hook does not start Veeam again. + mold_backup_trigger_mark "veeam-active" "$vm_name" + + local chain_count staging_paths source_format + chain_count="$(mold_backup_api_veeam_backup_count "$vm_id")" + + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" && "${chain_count:-0}" -eq 0 ]]; then + mold_backup_notify_log info "First file-level backup: host export → importAblestackVeeamBackupSeed (skip MS→Veeam on seed)" + if ! host_path=$(mold_backup_run_host_export "$vm_name" "1" 2>/dev/null); then + mold_backup_notify_log err "Host export failed for ${vm_name} (seed bootstrap; check /var/log/mold/veeam-hook.log and cvtbackup output)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=host-export" + return 1 + fi + if [[ ! -d "$host_path" ]]; then + mold_backup_notify_log err "Host export returned invalid path (not a directory): ${host_path}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=host-export" + return 1 + fi + staging_paths="$(mold_backup_collect_host_staging_paths "$host_path" 2>/dev/null || true)" + if [[ -z "$staging_paths" ]]; then + mold_backup_notify_log err "No staging disk files under ${host_path}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=no-staging-disks" + return 1 + fi + source_format="$(mold_backup_detect_staging_source_format "$staging_paths")" + mold_backup_api_validate_offering_repository "$vm_offering" || { + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=offering-repo-mismatch" + return 1 + } + backup_result="$(mold_backup_api_import_seed_and_wait "$vm_id" "$staging_paths" "$source_format" "$vm_name" 2>/dev/null || true)" + if [[ -z "$backup_result" ]]; then + mold_backup_notify_log err "importAblestackVeeamBackupSeed failed for ${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=import-seed" + return 1 + fi + backup_id="${backup_result%%|*}" + backup_type="${backup_result#*|}" + mold_backup_state_write_line "$state_file" \ + "vm=${vm_name} id=${vm_id} backup_id=${backup_id} type=${backup_type} path=${host_path} status=success" + mold_backup_notify_log info "Pre-notify OK vm=${vm_name} backup_id=${backup_id} type=${backup_type} path=${host_path} (seed import)" + return 0 + fi + + backup_result="$(mold_backup_api_create_veeam_and_wait "$vm_id" "$vm_name" || true)" + if [[ -z "$backup_result" ]]; then + mold_backup_notify_log err "Mold API backup request failed for ${vm_name} (see Async job failed above or agent.log)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=create-backup" + return 1 + fi + backup_id="${backup_result%%|*}" + backup_type="${backup_result#*|}" + + mold_backup_notify_log info "Host export starting vm=${vm_name} backup_id=${backup_id}" + if ! host_path=$(mold_backup_run_host_export "$vm_name" 2>/dev/null); then + mold_backup_notify_log err "Host export failed for ${vm_name} (backup_id=${backup_id})" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} backup_id=${backup_id} status=fail reason=host-export" + return 1 + fi + if [[ ! -d "$host_path" ]]; then + mold_backup_notify_log err "Host export failed for ${vm_name} (backup_id=${backup_id})" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} backup_id=${backup_id} status=fail reason=host-export" + return 1 + fi + + mold_backup_state_write_line "$state_file" \ + "vm=${vm_name} id=${vm_id} backup_id=${backup_id} type=${backup_type} path=${host_path} status=success" + mold_backup_notify_log info "Pre-notify OK vm=${vm_name} backup_id=${backup_id} type=${backup_type} path=${host_path}" + return 0 +} + +mold_backup_veeam_restore_chain_to_host() { + local backup_id="$1" + local chain rp_ids rp_id script_dir ps1 + chain="$(mold_backup_api_build_restore_chain "$backup_id")" || return 1 + rp_ids="" + local bid + for bid in ${chain//,/ }; do + [[ -z "$bid" ]] && continue + rp_id="$(mold_backup_api_backup_detail_field "$bid" "ablestack.veeam.restore.point.id" 2>/dev/null || true)" + [[ -n "$rp_id" ]] && rp_ids="${rp_ids},${rp_id}" + done + rp_ids="${rp_ids#,}" + [[ -n "$rp_ids" ]] || { + mold_backup_notify_log warn "No Veeam restore point ids in chain; datadisk restore only" + return 0 + } + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "VEEAM_SSH_HOST not set; skip external Veeam chain restore" + return 0 + } + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ps1="${script_dir}/veeam-restore-chain.ps1" + [[ -f "$ps1" ]] || ps1="/usr/share/mold/backup/veeam/veeam-restore-chain.ps1" + [[ -f "$ps1" ]] || { + mold_backup_notify_log warn "veeam-restore-chain.ps1 not found; datadisk restore only (no Veeam chain export)" + return 0 + } + mold_backup_notify_log info "Veeam chain restore RPs=${rp_ids} → ${VEEAM_HOST_BACKUP_PATH}" + ssh -i "${VEEAM_SSH_KEY:-/root/.ssh/id_rsa}" -o StrictHostKeyChecking=no \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "powershell -ExecutionPolicy Bypass -File '${ps1}' -RestorePointIds '${rp_ids}' -DestinationPath '${VEEAM_HOST_BACKUP_PATH}'" \ + || mold_backup_notify_log warn "Veeam chain restore script failed (continuing Mold restore)" +} + +mold_backup_get_domain_disk_paths() { + local vm_name="$1" + VM_NAME="$vm_name" + mold_backup_get_all_disk_paths +} + +mold_backup_run_host_export() { + local vm_name="$1" + local backup_subdir checkpoint disk_paths backup_files parent_dir parent_ckpt parent_ckpt_path backup_type op + [[ -x "${CVT_BACKUP_SCRIPT}" ]] || { + mold_backup_notify_log err "Host backup script not found: ${CVT_BACKUP_SCRIPT} (run veeam/install.sh on this KVM host)" + return 1 + } + + backup_subdir="${VEEAM_HOST_BACKUP_PATH}/${vm_name}/$(date '+%Y.%m.%d.%H.%M.%S.%3N')" + checkpoint="$(basename "$backup_subdir")" + mkdir -p "${VEEAM_HOST_BACKUP_PATH}/${vm_name}" + disk_paths=$(mold_backup_get_domain_disk_paths "$vm_name") + parent_dir="" + parent_ckpt="" + parent_ckpt_path="" + backup_type="FULL" + op="backup-running" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "$backup_type") + + if mold_backup_has_rbd_disk "$disk_paths"; then + op="backup-rbd" + mold_backup_notify_log info "Host export storage engine=rbd (HCI)" + fi + + local latest_parent="${VEEAM_HOST_BACKUP_PATH}/${vm_name}" + local latest_name="" latest_path="" d base force_full="${2:-0}" + for d in "${latest_parent}"/*; do + [[ -d "$d" ]] || continue + base=$(basename "$d") + [[ "$base" == "$checkpoint" ]] && continue + if [[ -f "${d}/checkpoints/${base}.xml" || -f "${d}/checkpoints/${base}.meta" \ + || -f "${d}/rbd-backup.meta" || -f "${d}/veeam-seed.meta" ]]; then + if [[ -z "$latest_name" || "$base" > "$latest_name" ]]; then + latest_name="$base" + latest_path="$d" + fi + fi + done + if [[ "$force_full" == "1" ]]; then + backup_type="FULL" + parent_dir="" + parent_ckpt="" + parent_ckpt_path="" + elif [[ -n "$latest_path" && -f "${latest_path}/checkpoints/${latest_name}.xml" ]]; then + backup_type="INCREMENTAL" + parent_dir="${vm_name}/${latest_name}" + parent_ckpt="$latest_name" + parent_ckpt_path="${latest_path}/checkpoints/${latest_name}.xml" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "INCREMENTAL") + elif [[ -n "$latest_path" && -f "${latest_path}/checkpoints/${latest_name}.meta" ]]; then + backup_type="INCREMENTAL" + parent_dir="${vm_name}/${latest_name}" + parent_ckpt="$latest_name" + parent_ckpt_path="${latest_path}/checkpoints/${latest_name}.meta" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "INCREMENTAL") + elif [[ -n "$latest_path" && -f "${latest_path}/rbd-backup.meta" ]]; then + backup_type="INCREMENTAL" + parent_dir="${vm_name}/${latest_name}" + parent_ckpt="$(mold_backup_meta_field "${latest_path}/rbd-backup.meta" checkpoint_name || echo "$latest_name")" + parent_ckpt_path="${latest_path}/checkpoints/${parent_ckpt}.meta" + [[ -f "$parent_ckpt_path" ]] || parent_ckpt_path="${latest_path}/rbd-backup.meta" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "INCREMENTAL") + elif [[ -n "$latest_path" ]]; then + mold_backup_notify_log warn "Prior export missing checkpoint xml under ${latest_path}; forcing FULL" + backup_type="FULL" + parent_dir="" + parent_ckpt="" + parent_ckpt_path="" + fi + + mold_backup_notify_log info "Host export vm=${vm_name} path=${backup_subdir} type=${backup_type} op=${op}" + local cvt_log + cvt_log="$(mktemp "${TMPDIR:-/tmp}/mold-cvt.XXXXXX")" + if ! "${CVT_BACKUP_SCRIPT}" \ + -o "${op}" \ + -v "${vm_name}" \ + -p "${backup_subdir}" \ + -b "${backup_type}" \ + -c "${checkpoint}" \ + -r "${parent_dir}" \ + -i "${parent_ckpt}" \ + -j "${parent_ckpt_path}" \ + -f "${backup_files}" \ + -d "${disk_paths}" \ + -q "${QUIESCE_VM:-false}" \ + >"$cvt_log" 2>&1; then + mold_backup_notify_log err "Host export cvtbackup failed: $(tail -5 "$cvt_log" | tr '\n' ' ')" + rm -f "$cvt_log" + return 1 + fi + rm -f "$cvt_log" + echo "${backup_subdir}" +} + +mold_backup_cleanup_host_path() { + [[ -d "${VEEAM_HOST_BACKUP_PATH}" ]] || return 0 + mold_backup_notify_log info "Cleaning host backup path: ${VEEAM_HOST_BACKUP_PATH}" + find "${VEEAM_HOST_BACKUP_PATH}" -mindepth 1 -maxdepth 4 -type f -delete 2>/dev/null || true + find "${VEEAM_HOST_BACKUP_PATH}" -mindepth 1 -maxdepth 3 -type d -empty -delete 2>/dev/null || true +} + +# --- Bidirectional Mold<->Veeam trigger loop guard --- +# Two short-lived markers under state/triggers/ break the trigger loop: +# veeam-active- : set by pre_notify before requesting the Mold backup. +# The Mold->Veeam hook skips Start-VBRJob while present. +# mold-active- : set by the Mold->Veeam hook before Start-VBRJob. +# pre_notify skips createBackup while present (Veeam disk-only). +mold_backup_trigger_dir() { + local d="$(mold_backup_state_dir)/triggers" + mkdir -p "$d" 2>/dev/null || true + echo "$d" +} + +mold_backup_trigger_mark() { + local kind="$1" vm="$2" + date +%s > "$(mold_backup_trigger_dir)/${kind}.$(mold_backup_safe_job_name "$vm")" 2>/dev/null || true +} + +mold_backup_trigger_active() { + local kind="$1" vm="$2" ttl="${3:-${VEEAM_TRIGGER_TTL:-1800}}" + local f ts now + f="$(mold_backup_trigger_dir)/${kind}.$(mold_backup_safe_job_name "$vm")" + [[ -f "$f" ]] || return 1 + ts="$(cat "$f" 2>/dev/null || echo 0)" + now="$(date +%s)" + if (( now - ts > ttl )); then + rm -f "$f" 2>/dev/null || true + return 1 + fi + return 0 +} + +mold_backup_trigger_clear() { + rm -f "$(mold_backup_trigger_dir)/${1}.$(mold_backup_safe_job_name "$2")" 2>/dev/null || true +} + +# libvirt VM name -> guest IP, from VM_TARGETS=i-2-5-VM:10.10.254.70,i-2-40-VM:10.10.254.61 +mold_backup_vm_guest_ip() { + local vm="$1" pair name ip targets env_file list + for list in "${VM_TARGETS:-}"; do + [[ -n "$list" ]] || continue + IFS=',' read -ra _pairs <<<"${list}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + if [[ "$name" == "$vm" && -n "$ip" && "$ip" != "$name" ]]; then + echo "$ip" + return 0 + fi + done + done + for env_file in \ + "${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.env" \ + "${MOLD_BACKUP_ETC_DIR}/mold-backup.env" \ + "$(dirname "${BASH_SOURCE[0]}")/mold-backup.env"; do + [[ -f "$env_file" ]] || continue + targets="$(mold_backup_read_env_var VM_TARGETS "$env_file" 2>/dev/null || true)" + [[ -n "$targets" ]] || continue + IFS=',' read -ra _pairs <<<"${targets}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + if [[ "$name" == "$vm" && -n "$ip" && "$ip" != "$name" ]]; then + echo "$ip" + return 0 + fi + done + done + return 1 +} + +# guest IP -> Veeam job name (legacy): 10.10.254.70 -> "Mold VM 10-10-254-70" +mold_backup_veeam_job_name_for_ip() { + local ip="$1" prefix="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + echo "${prefix} ${ip//./-}" +} + +# libvirt VM name -> Veeam job name: i-2-61-VM -> "Mold VM i-2-61-VM" +mold_backup_veeam_job_name_for_vm() { + local vm="$1" prefix="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + echo "${prefix} ${vm}" +} + +# Veeam job name -> libvirt VM name: "Mold VM i-2-61-VM" -> i-2-61-VM +mold_backup_vm_name_for_job() { + local job="$1" prefix="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + [[ "$job" == "${prefix} "* ]] || return 1 + echo "${job#${prefix} }" +} + +# guest IP -> libvirt VM name (reverse of mold_backup_vm_guest_ip), from VM_TARGETS. +mold_backup_vm_name_for_ip() { + local want="$1" pair name ip + [[ -n "${VM_TARGETS:-}" ]] || return 1 + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + if [[ "$ip" == "$want" && -n "$name" && "$ip" != "$name" ]]; then + echo "$name" + return 0 + fi + done + return 1 +} + +# Resolve the Veeam B&R REST API base (https://host:9419) from explicit or SSH host. +mold_backup_veeam_api_base() { + if [[ -n "${VEEAM_API_URL:-}" ]]; then + echo "${VEEAM_API_URL%/}" + return 0 + fi + local host="${VEEAM_API_HOST:-${VEEAM_SSH_HOST:-}}" + [[ -n "$host" ]] || return 1 + echo "https://${host}:${VEEAM_API_PORT:-9419}" +} + +# Start a Veeam job via the native VBR REST API (port 9419) — no SSH required. +# Returns 0 on start (or already running), 1 on any failure (caller may fall back to SSH). +mold_backup_trigger_veeam_job_rest() { + local vm="$1" job="$2" + local api ver user pass token job_id running + api="$(mold_backup_veeam_api_base)" || { + mold_backup_notify_log warn "Mold→Veeam(REST): no VEEAM_API_URL/VEEAM_API_HOST/VEEAM_SSH_HOST" + return 1 + } + ver="${VEEAM_API_VERSION:-1.2-rev0}" + user="${VEEAM_API_USER:-${VEEAM_USERNAME:-administrator}}" + pass="${VEEAM_API_PASSWORD:-${VEEAM_PASSWORD:-}}" + [[ -n "$pass" ]] || { + mold_backup_notify_log warn "Mold→Veeam(REST): VEEAM_API_PASSWORD/VEEAM_PASSWORD not set" + return 1 + } + + token="$(curl -sk --max-time 30 -X POST "${api}/api/oauth2/token" \ + -H "x-api-version: ${ver}" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=password&username=${user}&password=${pass}" 2>/dev/null \ + | python3 -c "import sys,json +try: print(json.load(sys.stdin).get('access_token','')) +except Exception: print('')" 2>/dev/null)" + [[ -n "$token" ]] || { + mold_backup_notify_log warn "Mold→Veeam(REST): auth failed at ${api} (check x-api-version='${ver}', user, port 9419)" + return 1 + } + + # Find the job id + running state by exact name. + local jobs_json + jobs_json="$(curl -sk --max-time 30 "${api}/api/v1/jobs" \ + -H "x-api-version: ${ver}" -H "Authorization: Bearer ${token}" 2>/dev/null)" + read -r job_id running <<<"$(echo "$jobs_json" | python3 -c " +import sys,json +want='''${job}''' +try: + d=json.load(sys.stdin) +except Exception: + print(''); sys.exit() +items=d.get('data', d if isinstance(d,list) else []) +for j in items: + if j.get('name')==want: + st=str(j.get('status') or j.get('lastResult') or '') + print(j.get('id',''), 'running' if str(j.get('isRunning','')).lower()=='true' or st.lower()=='running' else 'idle'); break +" 2>/dev/null)" + [[ -n "$job_id" ]] || { + mold_backup_notify_log warn "Mold→Veeam(REST): job '${job}' not found in /api/v1/jobs (Agent jobs may need SSH); will fall back" + return 1 + } + if [[ "$running" == "running" ]]; then + mold_backup_notify_log info "Mold→Veeam(REST): job '${job}' already running" + return 0 + fi + + local http_code + http_code="$(curl -sk --max-time 30 -o /dev/null -w '%{http_code}' -X POST \ + "${api}/api/v1/jobs/${job_id}/start" \ + -H "x-api-version: ${ver}" -H "Authorization: Bearer ${token}" 2>/dev/null)" + if [[ "$http_code" =~ ^20[0-9]$ ]]; then + mold_backup_notify_log info "Mold→Veeam(REST): job '${job}' start accepted (HTTP ${http_code})" + return 0 + fi + mold_backup_notify_log warn "Mold→Veeam(REST): start failed for '${job}' (HTTP ${http_code})" + return 1 +} + +# Run a short PowerShell script on Veeam via SSH (UTF-16LE EncodedCommand). +mold_backup_veeam_ssh_ps() { + local ps_script="$1" + local ps_enc + [[ -n "${VEEAM_SSH_HOST:-}" ]] || return 1 + ps_enc="$(printf '%s' "$ps_script" | iconv -f UTF-8 -t UTF-16LE 2>/dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || return 1 + mold_backup_veeam_ssh_encoded "$ps_enc" >/dev/null +} + +# Start a Veeam job over SSH (PowerShell) — fallback when REST cannot manage agent jobs. +mold_backup_trigger_veeam_job_ssh() { + local vm="$1" job="$2" job_esc ps_script + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "Mold→Veeam(SSH): VEEAM_SSH_HOST not set" + return 1 + } + job_esc="${job//\'/\'\'}" + ps_script="$(cat <Veeam direction). +# Method: rest (curl, no SSH), ssh (PowerShell), or auto (REST then SSH fallback). +mold_backup_trigger_veeam_job() { + local vm="$1" ip job method rc=1 + [[ "${VEEAM_TRIGGER_ENABLED:-false}" == "true" ]] || return 0 + ip="$(mold_backup_vm_guest_ip "$vm" 2>/dev/null || true)" + [[ -n "$ip" ]] || { + mold_backup_notify_log warn "No guest IP for ${vm} in VM_TARGETS; skip Veeam job start" + return 0 + } + job="$(mold_backup_veeam_job_name_for_vm "$vm")" + if [[ "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + method="${VEEAM_TRIGGER_METHOD:-ssh}" + else + method="${VEEAM_TRIGGER_METHOD:-auto}" + fi + + mold_backup_trigger_mark "mold-active" "$vm" + mold_backup_notify_log info "Mold→Veeam: starting Veeam job '${job}' for ${vm} (${ip}) method=${method}" + + case "$method" in + rest) mold_backup_trigger_veeam_job_rest "$vm" "$job"; rc=$? ;; + ssh) mold_backup_trigger_veeam_job_ssh "$vm" "$job"; rc=$? ;; + auto|*) + mold_backup_trigger_veeam_job_rest "$vm" "$job"; rc=$? + if [[ $rc -ne 0 ]]; then + mold_backup_notify_log info "Mold→Veeam: REST failed/unavailable, trying SSH fallback" + mold_backup_trigger_veeam_job_ssh "$vm" "$job"; rc=$? + fi + ;; + esac + + if [[ $rc -ne 0 ]]; then + mold_backup_trigger_clear "mold-active" "$vm" + mold_backup_notify_log warn "Mold→Veeam: could not start job '${job}' (cleared mold-active for ${vm})" + fi + return $rc +} + +# --- Veeam UI restore -> Mold reflect (reverse restore sync) --- +# A restore performed directly in the Veeam UI restores guest data in-place (the Mold +# VM's disks are updated by the guest agent), so Mold storage is already current. To make +# Mold "aware" of it, the KVM host polls Veeam over the existing KVM->Veeam SSH channel for +# recently-completed restore sessions, maps the target computer IP back to a libvirt VM via +# VM_TARGETS, and records the event in the Mold restore registry (+ Mold hook log). +# Loop guard: a Mold-initiated restore sets the mold-restore-active marker, so the watcher +# skips reflecting Mold's own restore (avoids double-recording). + +# State dir holding the set of already-reflected Veeam restore session ids. +mold_backup_restore_watch_state() { + local d="$(mold_backup_state_dir)/restore-watch" + mkdir -p "$d" 2>/dev/null || true + echo "$d/processed-sessions" +} + +mold_backup_restore_session_seen() { + local sid="$1" f + f="$(mold_backup_restore_watch_state)" + [[ -f "$f" ]] || return 1 + grep -qxF "$sid" "$f" 2>/dev/null +} + +mold_backup_restore_session_mark_seen() { + local sid="$1" f + f="$(mold_backup_restore_watch_state)" + echo "$sid" >> "$f" 2>/dev/null || true + # keep the file bounded + if [[ -f "$f" ]] && (( $(wc -l <"$f" 2>/dev/null || echo 0) > 2000 )); then + tail -n 1000 "$f" > "${f}.tmp" 2>/dev/null && mv -f "${f}.tmp" "$f" 2>/dev/null || true + fi +} + +# Query Veeam for restore sessions that completed within the last N minutes. +# Emits one line per session: sessionId|targetIp|endTimeUTC|result|name|backupName|restorePointId +# The target IP/computer is parsed out of the session Options XML (FLR/restore specs put +# IpOrDnsName/MachineName/BackupName there even when the session Name is hostname-based). +# Result is NOT filtered (e.g. an FLR session can end as 'Failed' even though files were +# restored), only completion + recency. Uses the existing KVM->Veeam SSH channel. +mold_backup_query_veeam_restores() { + local since_min="${1:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}" + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(restore): VEEAM_SSH_HOST not set" + return 1 + } + local ssh_key_opt=() + if [[ -n "${VEEAM_SSH_KEY:-}" ]]; then + if [[ -f "${VEEAM_SSH_KEY}" ]]; then + ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + else + mold_backup_notify_log warn "Veeam→Mold(restore): VEEAM_SSH_KEY=${VEEAM_SSH_KEY} missing; using default SSH keys" + fi + fi + local -a ssh_host_opts=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) + if [[ "${VEEAM_SSH_STRICT_HOSTKEY:-false}" == "true" ]]; then + ssh_host_opts=(-o StrictHostKeyChecking=accept-new) + fi + # PowerShell run on the Veeam B&R server: connect (warm-up loop), then emit every + # completed restore session as "Id|ip|epoch|result|name|backup|rpId". Falls back to + # Export-VBRAudit when Get-VBRRestoreSession is empty (common for Agent FLR over SSH). + local ps_script since_min_ps + since_min_ps="${since_min}" + ps_script="$(cat </dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || { mold_backup_notify_log warn "Veeam→Mold(restore): failed to encode PS script"; return 1; } + # Veeam.Backup.PowerShell needs PS 7+: full-path pwsh first (SSH often only has + # Windows PowerShell 5.1 "powershell", which cannot Import-Module Veeam). + # Also retry SSH itself: ARP/L2 to the Veeam host intermittently flaps. + local attempt out rc ps_launcher + local -a ps_launchers=( + '"C:\Program Files\PowerShell\7\pwsh.exe" -NoProfile -EncodedCommand' + 'pwsh.exe -NoProfile -EncodedCommand' + 'pwsh -NoProfile -EncodedCommand' + ) + for attempt in 1 2 3; do + for ps_launcher in "${ps_launchers[@]}"; do + out="$(ssh "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 "${ssh_host_opts[@]}" \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "${ps_launcher} ${ps_enc}" 2>/dev/null)" + rc=$? + if [[ $rc -eq 0 ]]; then + # PowerShell emits CRLF; strip CR so it doesn't end up in the last field. + out="$(printf '%s' "$out" | tr -d '\r')" + if [[ -z "${out//[[:space:]]/}" ]]; then + mold_backup_notify_log info "Veeam→Mold(restore): SSH ok (pwsh), 0 sessions from Get-VBRRestoreSession/audit (window=${since_min}min)" + else + local _raw_n + _raw_n="$(printf '%s\n' "$out" | grep -c '|' 2>/dev/null || echo 0)" + mold_backup_notify_log info "Veeam→Mold(restore): raw session lines=${_raw_n}" + fi + # Apply the time window here (3rd field = Unix epoch seconds). Lines without a + # numeric epoch are kept (fail-open) so we never silently drop real sessions. + local now_epoch cutoff line ep + now_epoch=$(date +%s) + cutoff=$(( now_epoch - since_min * 60 )) + while IFS= read -r line; do + [[ -n "$line" ]] || continue + ep="$(printf '%s' "$line" | cut -d'|' -f3)" + if [[ "$ep" =~ ^[0-9]+$ ]]; then + [[ "$ep" -ge "$cutoff" ]] && printf '%s\n' "$line" + else + printf '%s\n' "$line" + fi + done <<< "$out" + return 0 + fi + done + mold_backup_notify_log warn "Veeam→Mold(restore): SSH/pwsh query attempt ${attempt} failed (rc=${rc}); retrying" + sleep 3 + done + mold_backup_notify_log warn "Veeam→Mold(restore): SSH query failed after retries (need pwsh 7+ path)" + return 1 +} + +# Host Agent FLR restores files back under /tmp/mold/veeam// — detect locally when Veeam SSH returns nothing. +# Emits: sessionId|ip|epoch|result|name|backupName|restorePointId +mold_backup_restore_preflight() { + local vm owner bid d + mold_backup_notify_log info "restore preflight: host=$(mold_backup_local_kvm_name) job=${VEEAM_JOB_NAME:-n/a}" + [[ -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]] || { + mold_backup_notify_log warn "restore preflight: VM_INCLUDE not set" + return 0 + } + for vm in ${VM_INCLUDE//,/ }; do + vm="$(echo "$vm" | xargs)" + [[ -n "$vm" ]] || continue + if mold_backup_domain_exists "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore preflight: ${vm} libvirt=running (stop VM in Mold UI before restore)" + elif mold_backup_vm_restorable_on_local_host "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore preflight: ${vm} Mold Stopped on $(mold_backup_local_kvm_name) — no libvirt domain (normal for Mold; OK to restore)" + else + owner="$(mold_backup_api_get_vm_host_name "$vm" 2>/dev/null || echo unknown)" + mold_backup_notify_log warn "restore preflight: ${vm} not on this host (Mold hostname=${owner}); run restore on owner KVM" + fi + d="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}/${vm}" + if [[ -d "$d" ]] && [[ -n "$(find "$d" -mindepth 1 -print -quit 2>/dev/null)" ]]; then + mold_backup_notify_log info "restore preflight: ${vm} staging=${d} has files (FLR or backup staging)" + else + mold_backup_notify_log info "restore preflight: ${vm} staging=${d} empty — Veeam FLR must complete within restore-watch --since-min window" + fi + bid="$(mold_backup_registry_get_vm_backup_id "$vm" 2>/dev/null || true)" + mold_backup_notify_log info "restore preflight: ${vm} registry backup_id=${bid:-none}" + done +} + +mold_backup_query_local_host_flr() { + local since_min="${1:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}" + [[ "${BACKUP_MODE:-host}" == "host" ]] || return 0 + [[ -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]] || return 0 + local base="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + local cutoff now vm d epoch newest last state_f sid kvm_ip ckpt rp_id + now=$(date +%s) + cutoff=$((now - since_min * 60)) + kvm_ip="${KVM_IP:-}" + [[ -z "$kvm_ip" && -n "${KVM_HOST:-}" && "$KVM_HOST" == *@* ]] && kvm_ip="${KVM_HOST#*@}" + for vm in ${VM_INCLUDE//,/ }; do + vm="$(echo "$vm" | xargs)" + [[ -n "$vm" ]] || continue + d="${base}/${vm}" + if [[ ! -d "$d" ]]; then + mold_backup_notify_log info "local FLR: ${vm} no staging dir ${d}" + continue + fi + mold_backup_trigger_active "veeam-active" "$vm" && continue + mold_backup_trigger_active "mold-restore-active" "$vm" && continue + newest="$(find "$d" -mindepth 1 \( -type f -o -type d \) -printf '%T@\n' 2>/dev/null | sort -rn | head -1 || true)" + if [[ -z "$newest" ]]; then + mold_backup_notify_log info "local FLR: ${vm} staging empty ${d}" + continue + fi + epoch="${newest%.*}" + [[ "$epoch" =~ ^[0-9]+$ ]] || continue + if [[ "$epoch" -lt "$cutoff" ]]; then + mold_backup_notify_log info "local FLR: ${vm} files older than ${since_min}min (mtime epoch=${epoch}; widen --since-min or re-FLR)" + continue + fi + state_f="$(mold_backup_state_dir)/restore-watch/flr-${vm}.last-epoch" + mkdir -p "$(dirname "$state_f")" 2>/dev/null || true + last=0 + [[ -f "$state_f" ]] && last="$(tr -d '[:space:]' <"$state_f" 2>/dev/null || echo 0)" + [[ "$epoch" -le "${last:-0}" ]] && continue + ckpt="$(find "$d" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -rn | head -1 || true)" + rp_id="" + if [[ -n "$ckpt" ]]; then + local reg_line + reg_line="$(grep -h "vm=${vm}.*rp=" "$(mold_backup_registry_dir)"/*.log 2>/dev/null | tail -1 || true)" + rp_id="$(sed -n 's/.*rp=\([^ ]*\).*/\1/p' <<<"$reg_line" | tail -1)" + [[ -z "$rp_id" ]] && rp_id="$(tr -d '[:space:]' <"$(mold_backup_registry_dir)/${vm}.latest-rp-id" 2>/dev/null || true)" + fi + [[ -z "$rp_id" ]] && rp_id="$(mold_backup_query_veeam_rp_near_epoch "${VEEAM_JOB_NAME:-}" "$epoch" 2>/dev/null || true)" + sid="local-flr-${vm}-${epoch}" + mold_backup_notify_log info "local FLR detect: vm=${vm} epoch=${epoch} ckpt=${ckpt:-n/a} rp=${rp_id:-n/a}" + printf '%s|%s|%s|Success|local-flr-%s|%s|%s\n' "$sid" "${kvm_ip:-}" "$epoch" "$vm" "${ckpt:-}" "${rp_id:-}" + done +} + +mold_backup_local_flr_mark_epoch() { + local vm="$1" epoch="$2" state_f + [[ -n "$vm" && -n "$epoch" ]] || return 0 + state_f="$(mold_backup_state_dir)/restore-watch/flr-${vm}.last-epoch" + mkdir -p "$(dirname "$state_f")" 2>/dev/null || true + echo "$epoch" >"$state_f" +} + +# Pick Veeam restore point GUID whose CreationTime is closest to a Unix epoch (FLR time). +mold_backup_query_veeam_rp_near_epoch() { + local job="$1" epoch="$2" + [[ -n "${VEEAM_SSH_HOST:-}" && -n "$job" && "$epoch" =~ ^[0-9]+$ ]] || return 1 + local job_esc epoch_ps ps_script ps_enc out + job_esc="${job//\'/\'\'}" + epoch_ps="${epoch}" + ps_script="$(cat </dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || return 1 + local ssh_key_opt=() + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + out="$(ssh "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 -o StrictHostKeyChecking=no \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "pwsh -NoProfile -EncodedCommand ${ps_enc}" 2>/dev/null | tr -d '\r' | head -1)" + [[ -n "$out" ]] && echo "$out" +} + +# Reflect a single Veeam restore session into Mold state for one VM. +mold_backup_reflect_one_restore() { + local job="$1" vm="$2" sid="$3" detail="$4" + # Loop guard: Mold itself initiated this restore -> Veeam restore is part of that flow. + if mold_backup_trigger_active "mold-restore-active" "$vm"; then + mold_backup_trigger_clear "mold-restore-active" "$vm" + mold_backup_notify_log info "mold-restore-active for ${vm}: restore initiated by Mold; skip reflect (session=${sid})" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "mold.restore.skipped.mold-active" "$vm" "session=${sid}" + return 0 + fi + mold_backup_registry_save_restore "$job" "$vm" "veeam" "$sid" "$detail" "veeam-restored" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "veeam.restore.reflected" "$vm" "session=${sid};${detail}" + mold_backup_notify_log info "Veeam→Mold: reflected restore for ${vm} (session=${sid})" +} + +# --- Restore agent: host ownership, cluster lock, Mold API trigger --- + +mold_backup_local_kvm_name() { + if [[ -n "${KVM_HOSTNAME:-}" ]]; then + echo "$KVM_HOSTNAME" + return 0 + fi + local agent_props="/etc/cloudstack/agent/agent.properties" h + if [[ -f "$agent_props" ]]; then + h="$(grep -E '^host\.name=' "$agent_props" 2>/dev/null | tail -1 | cut -d= -f2-)" + h="${h//$'\r'/}" + [[ -n "$h" ]] && { echo "$h"; return 0; } + h="$(grep -E '^resource=' "$agent_props" 2>/dev/null | tail -1 | cut -d= -f2-)" + h="${h//$'\r'/}" + [[ -n "$h" ]] && { echo "$h"; return 0; } + h="$(grep -E '^host=' "$agent_props" 2>/dev/null | tail -1 | cut -d= -f2-)" + h="${h//$'\r'/}" + # host= is often MS resource id (e.g. 10.10.31.20@static), not hypervisor name — skip @ forms + if [[ -n "$h" && "$h" != *@* ]]; then + echo "$h" + return 0 + fi + fi + hostname -s +} + +mold_backup_api_get_vm_host_name() { + local vm_name="$1" json host + json="$(mold_backup_api_get_vm_record "$vm_name" "${ZONE_ID:-}")" || return 1 + host="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.hostname")" + [[ -n "$host" ]] || return 1 + echo "$host" +} + +mold_backup_vm_owned_by_local_host() { + local vm_name="$1" vm_host local_host + vm_host="$(mold_backup_api_get_vm_host_name "$vm_name" 2>/dev/null || true)" + local_host="$(mold_backup_local_kvm_name)" + if [[ -z "$vm_host" ]]; then + mold_backup_notify_log warn "restore-agent: no Mold hostname for ${vm_name}; allow local=${local_host}" + return 0 + fi + [[ "$vm_host" == "$local_host" ]] +} + +mold_backup_restore_lock_dir() { + local d="${RESTORE_LOCK_DIR:-}" + if [[ -z "$d" && -n "${BACKUP_REPO_ADDRESS:-}" ]]; then + d="${BACKUP_REPO_ADDRESS%/}/.mold/restore-locks" + fi + if [[ -z "$d" ]]; then + d="$(mold_backup_state_dir)/restore-locks" + fi + mkdir -p "$d" 2>/dev/null || true + echo "$d" +} + +mold_backup_restore_lock_acquire() { + local vm="$1" lock_file + lock_file="$(mold_backup_restore_lock_dir)/$(mold_backup_safe_job_name "$vm").lock" + exec {MOLD_RESTORE_LOCK_FD}>"$lock_file" || return 1 + if ! flock -n "$MOLD_RESTORE_LOCK_FD"; then + exec {MOLD_RESTORE_LOCK_FD}>&- + unset MOLD_RESTORE_LOCK_FD + return 1 + fi + return 0 +} + +mold_backup_restore_lock_release() { + [[ -n "${MOLD_RESTORE_LOCK_FD:-}" ]] || return 0 + flock -u "$MOLD_RESTORE_LOCK_FD" 2>/dev/null || true + exec {MOLD_RESTORE_LOCK_FD}>&- + unset MOLD_RESTORE_LOCK_FD +} + +mold_backup_events_log_file() { + local d="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}/events" + mkdir -p "$d" 2>/dev/null || true + echo "$d/restore.log" +} + +mold_backup_emit_restore_event() { + local event="$1" vm="$2" detail="${3:-}" host + host="$(mold_backup_local_kvm_name)" + echo "$(date -Iseconds) event=${event} host=${host} vm=${vm} ${detail}" >> "$(mold_backup_events_log_file)" + mold_backup_notify_log info "restore-event ${event} vm=${vm} ${detail}" +} + +mold_backup_api_find_backup_by_veeam_rp() { + local vm_name="$1" rp_id="$2" + local vm_id json norm_rp bid detail_rp + [[ -n "$vm_name" && -n "$rp_id" ]] || return 1 + norm_rp="$(mold_backup_normalize_rp_id "$rp_id")" + [[ -n "$norm_rp" ]] || return 1 + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || return 1 + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + [[ -n "$json" ]] || return 1 + while IFS= read -r bid; do + [[ -n "$bid" ]] || continue + detail_rp="$(mold_backup_api_backup_detail_field "$bid" "ablestack.veeam.restore.point.id" 2>/dev/null || true)" + [[ -n "$detail_rp" ]] || continue + if [[ "$(mold_backup_normalize_rp_id "$detail_rp")" == "$norm_rp" ]]; then + echo "$bid" + return 0 + fi + done < <(printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): + b = [b] + for x in b: + if str(x.get('status', '')).lower() == 'backedup' and x.get('id'): + print(x['id']) +except Exception: + pass +" 2>/dev/null) + return 1 +} + +mold_backup_resolve_backup_id_for_vm() { + local vm_name="$1" job="${2:-${VEEAM_JOB_NAME:-}}" rp_id="${3:-}" ckpt="${4:-}" + local line backup_id vm_id json + if [[ -n "$ckpt" ]]; then + backup_id="$(mold_backup_registry_get_backup_id_by_checkpoint "$vm_name" "$ckpt" 2>/dev/null || true)" + if [[ -n "$backup_id" ]]; then + mold_backup_notify_log info "restore-watch: vm=${vm_name} ckpt=${ckpt} → backup_id=${backup_id} (registry checkpoint)" + echo "$backup_id" + return 0 + fi + fi + if [[ -n "$rp_id" ]]; then + backup_id="$(mold_backup_registry_get_backup_id_by_rp "$vm_name" "$rp_id" 2>/dev/null || true)" + if [[ -n "$backup_id" ]]; then + mold_backup_notify_log info "restore-watch: vm=${vm_name} rp=${rp_id} → backup_id=${backup_id} (registry)" + echo "$backup_id" + return 0 + fi + backup_id="$(mold_backup_api_find_backup_by_veeam_rp "$vm_name" "$rp_id" 2>/dev/null || true)" + if [[ -n "$backup_id" ]]; then + mold_backup_notify_log info "restore-watch: vm=${vm_name} rp=${rp_id} → backup_id=${backup_id} (Mold API)" + mold_backup_registry_index_rp_backup "$vm_name" "$rp_id" "$backup_id" "$job" + echo "$backup_id" + return 0 + fi + mold_backup_notify_log warn "restore-watch: no Mold backup for Veeam restore point ${rp_id} vm=${vm_name}; using latest" + fi + if [[ -n "${BACKUP_ID:-}" ]]; then + echo "$BACKUP_ID" + return 0 + fi + backup_id="$(mold_backup_registry_get_vm_backup_id "$vm_name" 2>/dev/null || true)" + [[ -n "$backup_id" ]] && { echo "$backup_id"; return 0; } + local reg_dir + reg_dir="$(mold_backup_registry_dir)" + if [[ -d "$reg_dir" ]]; then + line="$(grep -h "vm=${vm_name}.*backup_id=" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + backup_id="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$backup_id" ]] && { echo "$backup_id"; return 0; } + fi + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || return 1 + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + backup_id="$(python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): b = [b] + backed = [x for x in b if str(x.get('status','')).lower() == 'backedup'] + backed.sort(key=lambda x: x.get('date',''), reverse=True) + print(backed[0]['id'] if backed else '') +except Exception: + print('') +" <<<"$json" 2>/dev/null)" + [[ -n "$backup_id" ]] && echo "$backup_id" +} + +# Handle one Veeam restore session: dedup, host check, flock, optional Mold restore API. +mold_backup_handle_veeam_restore_session() { + local job="$1" vm="$2" sid="$3" detail="$4" trigger_mold="${5:-false}" rp_id="${6:-}" + if [[ -z "$rp_id" && "$detail" == *"rp="* ]]; then + rp_id="$(sed -n 's/.*rp=\([^;]*\).*/\1/p' <<<"$detail" | tail -1)" + rp_id="${rp_id// /}" + fi + if [[ -z "$rp_id" && "$detail" == *"end="* ]]; then + local _ep + _ep="$(sed -n 's/.*end=\([^;]*\).*/\1/p' <<<"$detail" | tail -1)" + if [[ "$_ep" =~ ^[0-9]+$ ]]; then + rp_id="$(mold_backup_query_veeam_rp_near_epoch "$job" "$_ep" 2>/dev/null || true)" + fi + fi + if mold_backup_restore_session_seen "$sid"; then + mold_backup_emit_restore_event "mold.restore.skipped.duplicate" "$vm" "session=${sid}" + return 0 + fi + if mold_backup_trigger_active "mold-restore-active" "$vm"; then + mold_backup_trigger_clear "mold-restore-active" "$vm" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "mold.restore.skipped.mold-active" "$vm" "session=${sid}" + return 0 + fi + if [[ "$trigger_mold" != "true" ]]; then + mold_backup_reflect_one_restore "$job" "$vm" "$sid" "$detail" + return 0 + fi + if ! mold_backup_vm_owned_by_local_host "$vm"; then + local owner + owner="$(mold_backup_api_get_vm_host_name "$vm" 2>/dev/null || echo unknown)" + mold_backup_emit_restore_event "mold.restore.skipped.not-owner" "$vm" \ + "session=${sid};owner=${owner};local=$(mold_backup_local_kvm_name)" + mold_backup_restore_session_mark_seen "$sid" + return 0 + fi + if ! mold_backup_restore_lock_acquire "$vm"; then + mold_backup_emit_restore_event "mold.restore.skipped.locked" "$vm" "session=${sid}" + return 0 + fi + local backup_id rc=0 ckpt="" + ckpt="$(sed -n 's/.*backup=\([^;]*\).*/\1/p' <<<"$detail" | tail -1)" + ckpt="${ckpt// /}" + backup_id="$(mold_backup_resolve_backup_id_for_vm "$vm" "$job" "$rp_id" "$ckpt" 2>/dev/null || true)" + if [[ -z "$backup_id" ]]; then + mold_backup_emit_restore_event "mold.restore.failed" "$vm" "session=${sid};reason=no-backup-id;rp=${rp_id:-n/a}" + mold_backup_restore_lock_release + return 1 + fi + export BACKUP_ID="$backup_id" VM_NAME="$vm" + [[ -n "$rp_id" ]] && export VEEAM_RESTORE_POINT_ID="$rp_id" + export RESTORE_SOURCE="${RESTORE_SOURCE:-${VEEAM_UI_RESTORE_SOURCE:-mold-only}}" + mold_backup_notify_log info "Veeam UI restore session=${sid} vm=${vm} rp=${rp_id:-n/a} → Mold datadisk restore backup_id=${backup_id} (RESTORE_SOURCE=${RESTORE_SOURCE})" + mold_backup_emit_restore_event "veeam.restore.completed" "$vm" "session=${sid};rp=${rp_id:-n/a};backup_id=${backup_id};${detail}" + mold_backup_emit_restore_event "mold.restore.requested" "$vm" "session=${sid};rp=${rp_id:-n/a};backup_id=${backup_id};source=${RESTORE_SOURCE}" + mold_backup_trigger_mark "mold-restore-active" "$vm" + if mold_backup_restore_notify "$(hostname -s)" "$job"; then + mold_backup_registry_save_restore "$job" "$vm" "veeam" "$sid" "${detail};backup_id=${backup_id}" "mold-restored" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "mold.restore.completed" "$vm" "session=${sid};backup_id=${backup_id}" + else + mold_backup_emit_restore_event "mold.restore.failed" "$vm" "session=${sid};backup_id=${backup_id}" + rc=1 + fi + mold_backup_trigger_clear "mold-restore-active" "$vm" + mold_backup_restore_lock_release + return "$rc" +} + +# Poll Veeam restore sessions and reflect new ones for VMs we manage (VM_TARGETS). +# When trigger_mold=true, the owning KVM host acquires a cluster flock and calls Mold restore API. +mold_backup_watch_veeam_restores() { + local job="${1:-${VEEAM_JOB_NAME:-}}" + local since_min="${2:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}" + local trigger_mold="${3:-${RESTORE_WATCH_TRIGGER_MOLD:-false}}" + if [[ -z "${VM_TARGETS:-}" && "${BACKUP_MODE:-host}" == "host" && -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]]; then + # Host mode: build name:ip pairs from libvirt when VM_TARGETS unset. + local _vm _ip + VM_TARGETS="" + for _vm in ${VM_INCLUDE//,/ }; do + _vm="$(echo "$_vm" | xargs)" + [[ -n "$_vm" ]] || continue + _ip="$(virsh -c qemu:///system domifaddr "$_vm" 2>/dev/null | awk '/ipv4/ {print $4; exit}' | cut -d/ -f1)" + VM_TARGETS+="${VM_TARGETS:+,}${_vm}:${_ip:-${_vm}}" + done + fi + [[ -n "${VM_TARGETS:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(restore): VM_TARGETS empty; set VM_INCLUDE or VM_TARGETS" + return 0 + } + [[ "${trigger_mold}" == "true" ]] && mold_backup_restore_preflight + mold_backup_notify_log info "=== restore-watch job=${job} window=${since_min}min trigger_mold=${trigger_mold} host=$(mold_backup_local_kvm_name) ===" + local sid sip et result nm bn rp_id matched_ip vm + local processed=0 handled=0 + while IFS='|' read -r sid sip et result nm bn rp_id; do + [[ -n "$sid" ]] || continue + processed=$((processed+1)) + if mold_backup_restore_session_seen "$sid"; then + mold_backup_notify_log info "restore-watch: session ${sid} already processed (skip duplicate)" + continue + fi + # Match: prefer the IP parsed from the session Options; fall back to scanning + # VM_TARGETS IPs (dots or dashes form) against the session name / backup name. + matched_ip="" + if [[ -n "$sip" ]] && mold_backup_vm_name_for_ip "$sip" >/dev/null 2>&1; then + matched_ip="$sip" + else + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + local pair ip ipd vmn + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + ip="${pair#*:}" + [[ -n "$ip" && "$ip" != "$vmn" ]] || continue + ipd="${ip//./-}" + # Match by IP (dots/dashes) for legacy "Mold VM " jobs, or by the + # libvirt internal name (e.g. i-2-51-VM) for jobs named by internal name. + if [[ "$sip" == "$ip" || "$nm" == *"$ip"* || "$nm" == *"$ipd"* || "$bn" == *"$ip"* || "$bn" == *"$ipd"* \ + || ( -n "$vmn" && ( "$nm" == *"$vmn"* || "$bn" == *"$vmn"* ) ) ]]; then + matched_ip="$ip" + break + fi + done + fi + # Host backup (KVM agent e.g. 10.10.31.2): FLR session targets the hypervisor. + if [[ -z "$matched_ip" && "${BACKUP_MODE:-host}" == "host" ]]; then + local kvm_ip="${KVM_IP:-}" + [[ -z "$kvm_ip" && -n "${KVM_HOST:-}" && "$KVM_HOST" == *@* ]] && kvm_ip="${KVM_HOST#*@}" + [[ -z "$kvm_ip" && -n "${KVM_HOST:-}" && "$KVM_HOST" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] && kvm_ip="${KVM_HOST}" + [[ -z "$kvm_ip" ]] && kvm_ip="$(hostname -I 2>/dev/null | awk '{print $1}')" + local kvm_hn="${KVM_HOSTNAME:-$(hostname -s)}" + local host_hit=false + if [[ -n "$kvm_ip" && ( "$sip" == "$kvm_ip" || "$nm" == *"$kvm_ip"* || "$bn" == *"$kvm_ip"* ) ]]; then + host_hit=true + elif [[ -n "$kvm_hn" && ( "$nm" == *"$kvm_hn"* || "$bn" == *"$kvm_hn"* || "$bn" == *"$job"* ) ]]; then + host_hit=true + fi + if [[ "$host_hit" == "true" ]]; then + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + local pair vmn ip + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + [[ -n "$vmn" ]] || continue + if [[ "$nm" == *"$vmn"* || "$bn" == *"$vmn"* ]]; then + ip="${pair#*:}" + matched_ip="${ip:-$vmn}" + break + fi + done + if [[ -z "$matched_ip" && -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]]; then + local _one + if [[ -n "${VEEAM_RESTORE_VM:-}" ]]; then + _one="${VEEAM_RESTORE_VM}" + mold_backup_notify_log info "restore-watch: host FLR session ${sid} → VEEAM_RESTORE_VM=${_one}" + else + _one="$(echo "${VM_INCLUDE}" | tr ',' ' ' | awk '{print $1}')" + mold_backup_notify_log info "restore-watch: host FLR session ${sid} → VM_INCLUDE=${_one} (set VEEAM_RESTORE_VM for explicit target)" + fi + if [[ -n "$_one" ]]; then + matched_ip="$_one" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + ip="${pair#*:}" + [[ "$vmn" == "$_one" && -n "$ip" && "$ip" != "$vmn" ]] && matched_ip="$ip" && break + done + fi + fi + fi + fi + # Agent FLR to hypervisor: audit/session metadata often lacks libvirt VM name — use explicit target. + if [[ -z "$matched_ip" && "${BACKUP_MODE:-host}" == "host" && -n "${VEEAM_RESTORE_VM:-}" ]]; then + local _pair _vmn _ip + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + for _pair in "${_pairs[@]}"; do + _pair="${_pair// /}" + _vmn="${_pair%%:*}" + _ip="${_pair#*:}" + [[ "$_vmn" == "${VEEAM_RESTORE_VM}" ]] || continue + matched_ip="${_ip:-$_vmn}" + [[ "$matched_ip" == "$_vmn" ]] && matched_ip="$_vmn" + break + done + [[ -z "$matched_ip" ]] && matched_ip="${VEEAM_RESTORE_VM}" + mold_backup_notify_log info "restore-watch: session ${sid} name='${nm}' → host FLR fallback VEEAM_RESTORE_VM=${VEEAM_RESTORE_VM}" + fi + if [[ -z "$matched_ip" ]]; then + mold_backup_notify_log info "restore-watch: session ${sid} ip='${sip}' name='${nm}' backup='${bn}' rp='${rp_id:-}' — no VM_TARGETS match (skip)" + continue + fi + vm="$(mold_backup_vm_name_for_ip "$matched_ip" 2>/dev/null || true)" + [[ -n "$vm" ]] || vm="$matched_ip" + if ! mold_backup_vm_restorable_on_local_host "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore-watch: session ${sid} vm='${vm}' — not on this Mold host (virsh empty when Stopped is normal)" + continue + fi + if ! mold_backup_domain_exists "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore-watch: vm=${vm} Mold Stopped (no libvirt) — triggering Mold restoreBackup via API" + fi + [[ -n "$rp_id" ]] && mold_backup_notify_log info "restore-watch: session ${sid} vm=${vm} veeam_rp=${rp_id}" + mold_backup_handle_veeam_restore_session "$job" "$vm" "$sid" \ + "name=${nm};end=${et};result=${result};backup=${bn};ip=${matched_ip};rp=${rp_id}" "$trigger_mold" "$rp_id" \ + && { + handled=$((handled+1)) + [[ "$sid" == local-flr-* ]] && mold_backup_local_flr_mark_epoch "$vm" "$et" + } || mold_backup_notify_log warn "restore-watch: session ${sid} vm=${vm} handle failed (see restore.log)" + done < <( + mold_backup_query_veeam_restores "$since_min" 2>/dev/null || true + mold_backup_query_local_host_flr "$since_min" 2>/dev/null || true + ) + mold_backup_notify_log info "=== restore-watch done: scanned=${processed} handled=${handled} trigger_mold=${trigger_mold} ===" + return 0 +} + +# === Veeam UI backup reflection (backup-watch) ============================= +# Mirror of restore-watch for the *backup* direction: poll Veeam backup +# sessions and record new ones into the Mold-side registry, so a backup that +# was started directly from the Veeam console shows up in Mold without any +# per-job pre/post script. Dedup is by Veeam session id (separate state file). + +mold_backup_backup_watch_state() { + local d="$(mold_backup_state_dir)/backup-watch" + mkdir -p "$d" 2>/dev/null || true + echo "$d/processed-sessions" +} + +mold_backup_backup_session_seen() { + local sid="$1" f + f="$(mold_backup_backup_watch_state)" + [[ -f "$f" ]] || return 1 + grep -qxF "$sid" "$f" 2>/dev/null +} + +mold_backup_backup_session_mark_seen() { + local sid="$1" f + f="$(mold_backup_backup_watch_state)" + echo "$sid" >> "$f" 2>/dev/null || true + # keep the file bounded + if [[ -f "$f" ]] && (( $(wc -l <"$f" 2>/dev/null || echo 0) > 2000 )); then + tail -n 1000 "$f" > "${f}.tmp" 2>/dev/null && mv -f "${f}.tmp" "$f" 2>/dev/null || true + fi +} + +# Run PowerShell on Veeam B&R via SSH (UTF-16LE base64 -EncodedCommand). +# Tries pwsh full path, then pwsh.exe, then powershell.exe (Windows OpenSSH PATH quirks). +mold_backup_veeam_ssh_encoded() { + local ps_enc="$1" + local attempt out rc last_err="" ps_launcher + [[ -n "${VEEAM_SSH_HOST:-}" ]] || return 1 + [[ -n "$ps_enc" ]] || return 1 + local ssh_key_opt=() + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + local -a ssh_host_opts=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) + if [[ "${VEEAM_SSH_STRICT_HOSTKEY:-false}" == "true" ]]; then + ssh_host_opts=(-o StrictHostKeyChecking=accept-new) + fi + local -a ps_launchers=( + '"C:\Program Files\PowerShell\7\pwsh.exe" -NoProfile -EncodedCommand' + 'pwsh.exe -NoProfile -EncodedCommand' + 'pwsh -NoProfile -EncodedCommand' + 'powershell.exe -NoProfile -EncodedCommand' + ) + for attempt in 1 2 3; do + for ps_launcher in "${ps_launchers[@]}"; do + out="$(ssh -n "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 "${ssh_host_opts[@]}" \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "${ps_launcher} ${ps_enc}" 2>&1)" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + printf '%s' "$out" + return 0 + fi + last_err="$out" + done + sleep 2 + done + last_err="${last_err//$'\r'/}" + last_err="${last_err//$'\n'/; }" + mold_backup_notify_log warn "Veeam SSH failed (host=${VEEAM_SSH_HOST} rc=${rc}): ${last_err:0:240}" + return 1 +} + +# Query Veeam B&R for the newest restore point GUID for a guest Agent job. +# Guest Agent backups register under computer IP/hostname in Veeam, not libvirt i-2-XX-VM. +# Tries: computer backup job → backup chain → restore points, then name/IP filters. +# Prints restore point GUID on stdout; returns 1 if none found. +mold_backup_query_veeam_latest_restore_point() { + local job="$1" vm_name="$2" guest_ip="${3:-}" retries="${4:-6}" attempt rp_id + for ((attempt=1; attempt<=retries; attempt++)); do + rp_id="$(mold_backup_query_veeam_latest_restore_point_once "$job" "$vm_name" "$guest_ip" 2>/dev/null || true)" + [[ -n "$rp_id" ]] && { echo "$rp_id"; return 0; } + if [[ "$attempt" -lt "$retries" ]]; then + mold_backup_notify_log info "guest post: restore point not ready (attempt ${attempt}/${retries}); retry in 10s job=${job} vm=${vm_name}" + sleep 10 + fi + done + mold_backup_notify_log warn "guest post: no Veeam restore point after ${retries} attempts job=${job} vm=${vm_name}" + return 1 +} + +mold_backup_query_veeam_latest_restore_point_once() { + local job="$1" vm_name="$2" guest_ip="${3:-}" + local job_esc vm_esc ip_esc dash_ip ps_script ps_enc out + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "guest post: VEEAM_SSH_HOST not set; cannot query restore points" + return 1 + } + job_esc="${job//\'/\'\'}" + vm_esc="${vm_name//\'/\'\'}" + ip_esc="${guest_ip//\'/\'\'}" + dash_ip="${guest_ip//./-}" + ps_script="$(cat </dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || return 1 + local out + out="$(mold_backup_veeam_ssh_encoded "$ps_enc" 2>/dev/null || true)" + [[ -n "$out" ]] || return 1 + out="${out//$'\r'/}" + out="$(printf '%s' "$out" | grep -E '^[0-9a-fA-F-]{36}$' | tail -1)" + [[ -n "$out" ]] && { echo "$out"; return 0; } + mold_backup_notify_log warn "Veeam restore point query: SSH ok but no GUID (job=${job})" + return 1 +} + +# Query Veeam for backup sessions that completed within the last N minutes. +# Emits one line per session: sessionId|targetIp|endEpoch|result|name|jobName +# The target IP is parsed from the session/job name (guest VM jobs are named +# "Mold VM 10-10-254-70" — dash form — so the dash IP is recovered to dots). +# Time-window filtering is applied afterwards in bash on the epoch field, same +# as restore-watch (avoids PowerShell/Veeam timezone quirks). +mold_backup_query_veeam_backups() { + local since_min="${1:-${VEEAM_BACKUP_WATCH_WINDOW_MIN:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}}" + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(backup): VEEAM_SSH_HOST not set" + return 1 + } + local ssh_key_opt=() + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + local -a ssh_host_opts=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) + if [[ "${VEEAM_SSH_STRICT_HOSTKEY:-false}" == "true" ]]; then + ssh_host_opts=(-o StrictHostKeyChecking=accept-new) + fi + local ps_script + ps_script="$(cat <<'PS' +$ErrorActionPreference = 'SilentlyContinue' +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue +# Warm-up loop (same quirk as restore sessions). Union regular backup sessions +# with Agent (computer) backup job sessions so guest-VM Agent jobs are included. +$sessions = @() +for ($k = 0; $k -lt 12; $k++) { + try { Connect-VBRServer -Server localhost -ErrorAction Stop } catch {} + $tmp = @() + try { $tmp += @(Get-VBRBackupSession) } catch {} + try { $tmp += @(Get-VBRComputerBackupJobSession) } catch {} + $sessions = @($tmp) + if ($sessions.Count -gt 0) { break } + Start-Sleep -Milliseconds 700 +} +$tmp = @() +try { $tmp += @(Get-VBRBackupSession) } catch {} +try { $tmp += @(Get-VBRComputerBackupJobSession) } catch {} +$sessions = @($tmp) +# Dedup by Id inside the loop via a hashtable (a "| Select-Object -Unique" +# reassignment can make the following foreach emit nothing in pwsh). +$seen = @{} +foreach ($s in $sessions) { + if ($null -eq $s) { continue } + $sidKey = [string]$s.Id + if ($seen.ContainsKey($sidKey)) { continue } + $seen[$sidKey] = $true + # Completion differs by session type: CBackupSession has IsCompleted, while the + # Agent VBRSession (Get-VBRComputerBackupJobSession) only exposes State — treat + # State=Stopped/Completed as done. (Relying on IsCompleted alone skipped every + # agent session because that property does not exist on VBRSession.) + $done = $false + if ($s.IsCompleted -eq $true) { $done = $true } + $st = [string]$s.State + if ($st -eq 'Stopped' -or $st -eq 'Completed') { $done = $true } + if (-not $done) { continue } + # Agent VBRSession has no JobName; its Name holds the job/computer (dash-IP) name. + $nm = ([string]$s.Name) -replace '[\|\r\n]',' ' + $jn = ([string]$s.JobName) -replace '[\|\r\n]',' ' + $rs = [string]$s.Result + $hay = "$nm $jn" + $ip = '' + $m = [regex]::Match($hay, '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') + if ($m.Success) { $ip = $m.Groups[1].Value } + if (-not $ip) { + $md = [regex]::Match($hay, '(\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})') + if ($md.Success) { $ip = ($md.Groups[1].Value -replace '-','.') } + } + $epoch = 0 + $etObj = if ($null -ne $s.EndTime) { $s.EndTime } else { $s.EndTimeUTC } + try { if ($null -ne $etObj) { $epoch = [int64]([DateTimeOffset]$etObj).ToUnixTimeSeconds() } } catch { $epoch = 0 } + "$($s.Id)|$ip|$epoch|$rs|$nm|$jn" +} +PS +)" + local ps_enc + ps_enc="$(printf '%s' "$ps_script" | iconv -f UTF-8 -t UTF-16LE 2>/dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || { mold_backup_notify_log warn "Veeam→Mold(backup): failed to encode PS script"; return 1; } + local attempt out rc + for attempt in 1 2 3; do + out="$(ssh "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 "${ssh_host_opts[@]}" \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "pwsh -NoProfile -EncodedCommand ${ps_enc}" 2>/dev/null)" + rc=$? + if [[ $rc -eq 0 ]]; then + out="${out//$'\r'/}" + local now_epoch cutoff line ep + now_epoch=$(date +%s) + cutoff=$(( now_epoch - since_min * 60 )) + while IFS= read -r line; do + [[ -n "$line" ]] || continue + ep="$(printf '%s' "$line" | cut -d'|' -f3)" + if [[ "$ep" =~ ^[0-9]+$ ]]; then + [[ "$ep" -ge "$cutoff" ]] && printf '%s\n' "$line" + else + printf '%s\n' "$line" + fi + done <<< "$out" + return 0 + fi + mold_backup_notify_log warn "Veeam→Mold(backup): SSH query attempt ${attempt} failed (rc=${rc}); retrying" + sleep 3 + done + mold_backup_notify_log warn "Veeam→Mold(backup): SSH query failed after retries" + return 1 +} + +# Reflect a single Veeam backup session into the Mold-side registry for one VM. +mold_backup_reflect_one_backup() { + local job="$1" vm="$2" sid="$3" detail="$4" + # Loop guard: this backup was initiated by Mold itself (Mold->Veeam trigger); + # the Mold record already exists, so don't double-record. + if mold_backup_trigger_active "mold-active" "$vm"; then + mold_backup_trigger_clear "mold-active" "$vm" + mold_backup_notify_log info "mold-active for ${vm}: backup initiated by Mold; skip reflect (session=${sid})" + mold_backup_backup_session_mark_seen "$sid" + return 0 + fi + mold_backup_registry_save_backup "$job" "$vm" "veeam:${sid}" "veeam-backed-up" + mold_backup_backup_session_mark_seen "$sid" + mold_backup_notify_log info "Veeam→Mold: reflected backup for ${vm} (session=${sid}) detail=${detail}" +} + +# Poll Veeam backup sessions and reflect new ones for VMs we manage (VM_TARGETS). +mold_backup_watch_veeam_backups() { + local job="${1:-${VEEAM_JOB_NAME:-}}" + local since_min="${2:-${VEEAM_BACKUP_WATCH_WINDOW_MIN:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}}" + [[ -n "${VM_TARGETS:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(backup): VM_TARGETS empty; nothing to watch" + return 0 + } + mold_backup_notify_log info "=== backup-watch job=${job} window=${since_min}min ===" + local sid sip et result nm jn matched_ip vm + local processed=0 reflected=0 + while IFS='|' read -r sid sip et result nm jn; do + [[ -n "$sid" ]] || continue + processed=$((processed+1)) + mold_backup_backup_session_seen "$sid" && continue + # Only reflect successful/warning backups (a failed backup is not a restore point). + case "$(printf '%s' "$result" | tr '[:upper:]' '[:lower:]')" in + success|warning) ;; + *) + mold_backup_notify_log info "backup-watch: session ${sid} result='${result}' (skip non-success)" + mold_backup_backup_session_mark_seen "$sid" + continue + ;; + esac + # Match: prefer the IP parsed from the session/job name; fall back to scanning + # VM_TARGETS IPs (dots or dashes form) against the session / job name. + matched_ip="" + if [[ -n "$sip" ]] && mold_backup_vm_name_for_ip "$sip" >/dev/null 2>&1; then + matched_ip="$sip" + else + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + local pair ip ipd vmn + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + ip="${pair#*:}" + [[ -n "$ip" && "$ip" != "$vmn" ]] || continue + ipd="${ip//./-}" + # Match by IP (dots/dashes) for legacy "Mold VM " jobs, or by the + # libvirt internal name (e.g. i-2-51-VM) for jobs named by internal name. + if [[ "$sip" == "$ip" || "$nm" == *"$ip"* || "$nm" == *"$ipd"* || "$jn" == *"$ip"* || "$jn" == *"$ipd"* \ + || ( -n "$vmn" && ( "$nm" == *"$vmn"* || "$jn" == *"$vmn"* ) ) ]]; then + matched_ip="$ip" + break + fi + done + fi + if [[ -z "$matched_ip" ]]; then + mold_backup_notify_log info "backup-watch: session ${sid} ip='${sip}' name='${nm}' job='${jn}' — no VM_TARGETS match (skip)" + continue + fi + vm="$(mold_backup_vm_name_for_ip "$matched_ip" 2>/dev/null || true)" + [[ -n "$vm" ]] || continue + mold_backup_reflect_one_backup "$job" "$vm" "$sid" "name=${nm};job=${jn};end=${et};result=${result};ip=${matched_ip}" + reflected=$((reflected+1)) + done < <(mold_backup_query_veeam_backups "$since_min" || true) + mold_backup_notify_log info "=== backup-watch done: scanned=${processed} reflected=${reflected} ===" + return 0 +} + +# Guest Agent on VM: Veeam pre runs before restore point exists — defer Mold API to post. +mold_backup_guest_pre_notify_vm() { + local vm_name="$1" offering_id="$2" state_file="$3" + local vm_id + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { + mold_backup_notify_log err "No Mold VM id for ${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} status=fail reason=no-vm-id" + return 1 + } + [[ -n "$offering_id" ]] && mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + mold_backup_api_check_vm_environment "$vm_id" "$vm_name" + if mold_backup_trigger_active "mold-active" "$vm_name"; then + mold_backup_trigger_clear "mold-active" "$vm_name" + mold_backup_notify_log info "mold-active for ${vm_name}: skip guest pre (Mold already backed up)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=success reason=mold-triggered" + return 0 + fi + mold_backup_trigger_mark "veeam-active" "$vm_name" + mold_backup_notify_log info "guest pre: defer Mold backup until Veeam post (restore point not ready yet) vm=${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=pending reason=veeam-guest-pre" + return 0 +} + +# After Veeam backup completes, restore point exists — create Mold backup record now. +mold_backup_guest_post_notify_vm() { + local vm_name="$1" vm_id="$2" job="$3" + local backup_result backup_id backup_type offering_id guest_ip rp_id chain_count + local host_path staging_paths source_format vm_offering json + [[ -n "$vm_id" ]] || vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { + mold_backup_notify_log err "guest post: no Mold VM id for ${vm_name}" + return 1 + } + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" "$(mold_backup_offering_name)" 2>/dev/null || true)" + [[ -n "$offering_id" ]] && mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + [[ -n "$job" ]] || job="$(mold_backup_veeam_job_name_for_vm "$vm_name")" + guest_ip="$(mold_backup_vm_guest_ip "$vm_name" 2>/dev/null || true)" + [[ -n "$guest_ip" ]] || mold_backup_notify_log warn "guest post: VM_TARGETS missing ${vm_name}:ip in conf/env (see mold-backup.env)" + rp_id="$(mold_backup_query_veeam_latest_restore_point "$job" "$vm_name" "$guest_ip" 2>/dev/null || true)" + if [[ -z "$rp_id" ]]; then + mold_backup_notify_log err "guest post: no Veeam restore point for ${vm_name} (job=${job} ip=${guest_ip:-n/a} veeam=${VEEAM_SSH_HOST:-unset}; run Veeam backup to Success first)" + return 1 + fi + mold_backup_notify_log info "guest post: Veeam restore point=${rp_id} vm=${vm_name} job=${job}" + chain_count="$(mold_backup_api_veeam_backup_count "$vm_id")" + if [[ "${chain_count:-0}" -eq 0 ]]; then + # Guest SelectedFiles backups have file-level restore points (no exportable disks on Veeam). + # Seed NAS from live KVM disks (same as host file-level first backup), tag Veeam RP id. + mold_backup_notify_log info "guest post: first backup — host export + importSeed (SelectedFiles; skip MS→Veeam disk export)" + if ! host_path=$(mold_backup_run_host_export "$vm_name" "1" 2>/dev/null); then + mold_backup_notify_log err "guest post: host export failed for ${vm_name} (check ${CVT_BACKUP_SCRIPT} and /var/log/mold/veeam-hook.log)" + return 1 + fi + if [[ ! -d "$host_path" ]]; then + mold_backup_notify_log err "guest post: host export invalid path: ${host_path}" + return 1 + fi + staging_paths="$(mold_backup_collect_host_staging_paths "$host_path" 2>/dev/null || true)" + if [[ -z "$staging_paths" ]]; then + mold_backup_notify_log err "guest post: no staging disks under ${host_path}" + return 1 + fi + source_format="$(mold_backup_detect_staging_source_format "$staging_paths")" + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null || true) + vm_offering="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + mold_backup_api_validate_offering_repository "$vm_offering" || return 1 + backup_result="$(mold_backup_api_import_staging_rp_seed_and_wait "$vm_id" "$staging_paths" "$source_format" "$rp_id" "$vm_name" || true)" + else + mold_backup_notify_log info "guest post: incremental — createAblestackVeeamBackup vm=${vm_name} (chain=${chain_count})" + backup_result="$(mold_backup_api_create_veeam_and_wait "$vm_id" "$vm_name" 2>/dev/null || true)" + fi + backup_id="${backup_result%%|*}" + backup_type="${backup_result#*|}" + if [[ -n "$backup_id" ]]; then + mold_backup_registry_save_backup "$job" "$vm_name" "$backup_id" "veeam-backed-up rp=${rp_id}" "$rp_id" + export BACKUP_ID="$backup_id" VM_NAME="$vm_name" + mold_backup_notify_log info "guest post OK vm=${vm_name} backup_id=${backup_id} type=${backup_type} rp=${rp_id}" + return 0 + fi + mold_backup_notify_log err "guest post: Mold API backup failed for ${vm_name} (rp=${rp_id} chain=${chain_count:-0}; check MS/agent.log)" + return 1 +} + +mold_backup_pre_notify() { + local client="${1:-$(hostname -s)}" + local job="${2:-${VEEAM_JOB_NAME:-}}" + local schedule="${3:-${VEEAM_SCHEDULE_NAME:-default}}" + local single_vm="${4:-}" + local saved_include="" + [[ -n "$job" ]] || mold_backup_die "VEEAM_JOB_NAME is required for pre-notify" + VEEAM_JOB_NAME="$job" + mold_backup_load_config || exit 1 + + if [[ -n "$single_vm" ]]; then + saved_include="${VM_INCLUDE:-*}" + VM_INCLUDE="$single_vm" + mold_backup_notify_log info "single-vm scope: ${single_vm}" + fi + + mold_backup_notify_log info "=== pre-notify (설계4: Pre-script + Mold API 백업요청) client=${client} job=${job} schedule=${schedule} ===" + mkdir -p "$(mold_backup_state_dir)" "${VEEAM_HOST_BACKUP_PATH}" + + local offering_id="" vm_name + local success=0 fail=0 + local state_file run_id + run_id="$(date '+%Y%m%d%H%M%S')" + state_file="$(mold_backup_state_file_for_job "$job" "$run_id")" + : > "$state_file" + echo "run_id=${run_id}" >> "$state_file" + + if mold_backup_cmk_bin >/dev/null 2>&1 || command -v curl >/dev/null 2>&1; then + mold_backup_api_ensure_global_settings || true + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" "$(mold_backup_offering_name)" 2>/dev/null || true)" + if [[ -z "$offering_id" ]]; then + if ! mold_backup_api_list_backup_offerings >/dev/null 2>&1; then + mold_backup_notify_log err "Cannot list/import backup offerings (check API key/secret or MS DB schema; see mold-ms-backup-schema-fix.sql)" + else + offering_id="$(mold_backup_api_ensure_backup_resources 2>/dev/null || true)" + fi + fi + [[ -n "$offering_id" ]] || { + if mold_backup_is_datadisk_mode; then + mold_backup_notify_log warn "No backup offering '$(mold_backup_offering_name)' for ${VEEAM_PROVIDER_NAME}; assign in Mold UI (datadisk mode does not auto-create backup repository)" + else + mold_backup_notify_log warn "No backup offering '$(mold_backup_offering_name)' for ${VEEAM_PROVIDER_NAME}; set BACKUP_REPO_ADDRESS + ZONE_ID and re-run veeam_config.sh" + fi + } + fi + + while IFS= read -r vm_name; do + [[ -z "$vm_name" ]] && continue + mold_backup_vm_in_filter "$vm_name" || continue + mold_backup_notify_log info "Target VM ${vm_name}" + + case "${BACKUP_MODE}" in + guest|veeam-guest) + if mold_backup_guest_pre_notify_vm "$vm_name" "$offering_id" "$state_file"; then + success=$((success + 1)) + else + fail=$((fail + 1)) + fi + ;; + host|policy) + if mold_backup_process_vm_pre_notify "$vm_name" "$offering_id" "$state_file"; then + success=$((success + 1)) + else + fail=$((fail + 1)) + fi + ;; + api|local|auto) + local vm_id backup_result backup_id + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { fail=$((fail + 1)); continue; } + [[ -n "$offering_id" ]] && mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + mold_backup_api_check_vm_environment "$vm_id" "$vm_name" + backup_result="$(mold_backup_api_create_veeam_and_wait "$vm_id" "$vm_name" 2>/dev/null || true)" + backup_id="${backup_result%%|*}" + if [[ -n "$backup_id" ]]; then + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} backup_id=${backup_id} status=success" + success=$((success + 1)) + else + fail=$((fail + 1)) + fi + ;; + *) + mold_backup_die "Invalid BACKUP_MODE=${BACKUP_MODE} (use guest|host|policy|api|local|auto)" + ;; + esac + done < <(mold_backup_list_target_domains || true) + + if [[ "$success" -eq 0 && "$fail" -eq 0 ]]; then + mold_backup_notify_log warn "No target VMs for job=${job} (vm_include=${VM_INCLUDE:-*}). Start VM or set VM_INCLUDE to libvirt name(s)." + if [[ "${VM_INCLUDE:-*}" != "*" ]]; then + local _t + for _t in ${VM_INCLUDE//,/ }; do + _t="$(echo "$_t" | xargs)" + [[ -z "$_t" ]] && continue + if mold_backup_domain_exists "$_t"; then + if virsh -c qemu:///system dominfo "$_t" 2>/dev/null | grep -q 'State:.*shut off'; then + mold_backup_notify_log warn "VM ${_t} exists but is shut off — start it for host export: virsh start ${_t}" + fi + else + mold_backup_notify_log warn "VM ${_t} not found in libvirt on $(hostname -s)" + fi + done + fi + fi + + mold_backup_notify_log info "pre-notify done success=${success} fail=${fail} state=${state_file}" + [[ -n "$saved_include" ]] && VM_INCLUDE="$saved_include" + [[ "$success" -gt 0 ]] && return 0 + return 1 +} + +mold_backup_post_notify() { + local client="${1:-$(hostname -s)}" + local job="${2:-${VEEAM_JOB_NAME:-}}" + local schedule="${3:-${VEEAM_SCHEDULE_NAME:-default}}" + [[ -n "$job" ]] || mold_backup_die "VEEAM_JOB_NAME is required for post-notify" + VEEAM_JOB_NAME="$job" + mold_backup_load_config || exit 1 + + mold_backup_notify_log info "=== post-notify (설계4: Post-script + 백업ID 저장) client=${client} job=${job} ===" + local state_file line vm_name backup_id status guest_handled=0 guest_fail=0 vm_id reason rc=0 + local host_rp_id="" + state_file="$(mold_backup_latest_state_file "$job" || true)" + + if [[ "${BACKUP_MODE:-host}" == "host" || "${BACKUP_MODE:-host}" == "policy" ]]; then + host_rp_id="$(mold_backup_query_veeam_latest_restore_point "$job" "" "" 6 2>/dev/null || true)" + [[ -n "$host_rp_id" ]] && mold_backup_notify_log info "post-notify: host job Veeam restore point=${host_rp_id}" + fi + + if [[ -f "$state_file" ]]; then + while IFS= read -r line; do + [[ "$line" =~ ^vm= ]] || continue + vm_name="$(mold_backup_state_parse_field "$line" "vm")" + backup_id="$(mold_backup_state_parse_field "$line" "backup_id")" + status="$(mold_backup_state_parse_field "$line" "status")" + vm_id="$(mold_backup_state_parse_field "$line" "id")" + reason="$(mold_backup_state_parse_field "$line" "reason")" + # Mold schedule/UI backup → Veeam trigger: Mold NAS backup already exists (HOURLY/MANUAL). + if [[ "$status" == "success" && "$reason" == "mold-triggered" ]]; then + mold_backup_notify_log info "guest post: skip for ${vm_name} (Mold backup already done; no duplicate import)" + guest_handled=1 + [[ -n "$vm_name" ]] && mold_backup_trigger_clear "veeam-active" "$vm_name" + continue + fi + if [[ "$status" == "pending" && "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + mold_backup_notify_log info "guest post: pending vm=${vm_name} mode=${BACKUP_MODE} reason=${reason:-veeam-guest-pre}" + if mold_backup_guest_post_notify_vm "$vm_name" "$vm_id" "$job"; then + guest_handled=1 + else + guest_fail=$((guest_fail + 1)) + fi + [[ -n "$vm_name" ]] && mold_backup_trigger_clear "veeam-active" "$vm_name" + continue + fi + # Veeam job for this VM finished — clear the loop-guard marker. + [[ -n "$vm_name" ]] && mold_backup_trigger_clear "veeam-active" "$vm_name" + if [[ "$status" == "pending" ]]; then + mold_backup_notify_log warn "post-notify: pending vm=${vm_name} but BACKUP_MODE=${BACKUP_MODE:-host} (expected guest)" + fi + [[ "$status" == "success" && -n "$backup_id" ]] || continue + mold_backup_registry_save_backup "$job" "$vm_name" "$backup_id" "veeam-backed-up rp=${host_rp_id:-n/a}" "$host_rp_id" + done < "$state_file" + else + mold_backup_notify_log warn "No state file for job ${job}" + if [[ "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + vm_name="$(mold_backup_vm_name_for_job "$job" 2>/dev/null || true)" + if [[ -n "$vm_name" ]]; then + mold_backup_notify_log info "guest post: fallback (no state) vm=${vm_name}" + if mold_backup_guest_post_notify_vm "$vm_name" "" "$job"; then + guest_handled=1 + else + guest_fail=$((guest_fail + 1)) + fi + mold_backup_trigger_clear "veeam-active" "$vm_name" + fi + fi + fi + + if [[ "$guest_handled" -eq 0 && "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + vm_name="$(mold_backup_vm_name_for_job "$job" 2>/dev/null || true)" + if [[ -n "$vm_name" ]]; then + mold_backup_notify_log info "guest post: fallback (no pending state) vm=${vm_name}" + if mold_backup_guest_post_notify_vm "$vm_name" "" "$job"; then + guest_handled=1 + else + guest_fail=$((guest_fail + 1)) + fi + mold_backup_trigger_clear "veeam-active" "$vm_name" + fi + fi + + if [[ "$guest_fail" -gt 0 ]]; then + mold_backup_notify_log err "post-notify: guest Mold backup failed for job=${job} (restore will not work until post succeeds)" + rc=1 + else + rc=0 + fi + + if [[ "$guest_handled" -eq 0 && "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + mold_backup_notify_log warn "post-notify: no guest VM processed for job=${job} (re-run pre-notify before post, or check state dir)" + fi + + mold_backup_cleanup_host_path + mold_backup_cleanup_staging + [[ -f "$state_file" ]] && rm -f "$state_file" + mold_backup_notify_log info "=== post-notify done (guest_fail=${guest_fail}) ===" + return "$rc" +} + +mold_backup_api_backup_vm_id() { + local backup_id="$1" json + json=$(mold_backup_cmk_run listBackups "id=${backup_id}" 2>/dev/null) || return 1 + mold_backup_api_json_field "$json" "listbackupsresponse.backup.virtualmachineid" +} + +mold_backup_api_verify_backup_for_vm() { + local backup_id="$1" vm_id="$2" + local owner + owner="$(mold_backup_api_backup_vm_id "$backup_id" 2>/dev/null || true)" + [[ -n "$owner" ]] || return 0 + [[ "$owner" == "$vm_id" ]] || { + mold_backup_notify_log err "Backup ${backup_id} belongs to VM ${owner}, not ${vm_id}" + return 1 + } + return 0 +} + +mold_backup_restore_notify() { + local client="${1:-$(hostname -s)}" + local job="${2:-${VEEAM_JOB_NAME:-}}" + mold_backup_load_config || exit 1 + mold_backup_apply_datadisk_profile + local restore_source="${RESTORE_SOURCE:-auto}" + if mold_backup_is_datadisk_mode; then + restore_source="mold-only" + RESTORE_SOURCE="mold-only" + fi + mold_backup_notify_log info "=== restore-notify client=${client} job=${job} backup_id=${BACKUP_ID:-} vm=${VM_NAME:-} source=${restore_source} ===" + mold_backup_require_var BACKUP_ID + [[ -n "${VM_UUID:-}" ]] || { + [[ -n "${VM_NAME:-}" ]] && VM_UUID="$(mold_backup_api_get_vm_id "$VM_NAME" 2>/dev/null || true)" + } + [[ -n "${VM_UUID:-}" ]] || mold_backup_die "restore requires VM_NAME or VM_UUID (individual VM restore)" + mold_backup_api_verify_backup_for_vm "$BACKUP_ID" "$VM_UUID" || exit 1 + + # Loop guard for reverse restore-sync: mark this VM so the Veeam->Mold restore + # watcher skips reflecting a restore that Mold itself initiated. + [[ -n "${VM_NAME:-}" ]] && mold_backup_trigger_mark "mold-restore-active" "$VM_NAME" + + if mold_backup_api_is_full_backup "$BACKUP_ID"; then + mold_backup_notify_log info "Restore type=FULL → Mold restoreBackup (datadisk ${BACKUP_REPO_ADDRESS:-/data/backup})" + mold_backup_api_restore + else + mold_backup_notify_log info "Restore type=INCREMENTAL → datadisk only (qcow2/raw/rbdiff on ${BACKUP_REPO_ADDRESS:-/data/backup}, no NAS)" + if [[ "$restore_source" != "mold-only" ]]; then + mold_backup_veeam_restore_chain_to_host "$BACKUP_ID" + fi + mold_backup_api_restore + fi + mold_backup_notify_log info "=== restore-notify done ===" +} diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.sh b/scripts/vm/hypervisor/kvm/veeam/mold-backup.sh new file mode 100755 index 000000000000..0faf4d98513f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.sh @@ -0,0 +1,345 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Mold + Veeam backup operations on KVM. +# +# backup-full 전체 VM 백업 (VM_INCLUDE=* 또는 목록) +# list-backups VM별 BackedUp 백업 목록 +# restore 개별 VM 복원 (--vm-name + --backup-id) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ABLESTACK_VEEAM_ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +CMD="${1:-}" +shift || true + +JOB_NAME="${VEEAM_JOB_NAME:-VeeamBackup}" +BACKUP_ID="${BACKUP_ID:-}" +VM_NAME_ARG="" +VM_UUID_ARG="" +CLIENT="$(hostname -s)" +DRY_RUN="${DRY_RUN:-false}" +RESTORE_SOURCE="${RESTORE_SOURCE:-}" +SINCE_MIN="${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}" +TRIGGER_MOLD="${RESTORE_WATCH_TRIGGER_MOLD:-false}" +RESTORE_EVENT="" +RESTORE_EVENT_ARGS=() + +usage() { + cat <<'EOF' +Usage: mold-backup.sh [options] + +Commands: + backup Pre: Mold API + /tmp/mold/veeam export (대상: VM_INCLUDE) + backup-complete Post: registry + staging 정리 + backup-full backup + backup-complete (전체/다중 VM 한 번에) + list-backups VM별 BackedUp 백업 목록 (개별 복원용 ID 확인) + restore 개별 VM 복원 (--vm-name + --backup-id, VM 정지 권장) + restore-notify restore 와 동일 (ablestack_veeam_restore_notify.sh 호출) + restore-watch Veeam UI 복원 감지 → (옵션) Mold datadisk restoreBackup 호출 + restore-event 복원 이벤트 수동 주입 (veeam.restore.completed | mold.restore.manual) + backup-watch Veeam UI에서 직접 백업한 세션을 감지해 Mold 상태에 반영 + status 한 VM 백업 + registry + +Options: + --job NAME Job conf 이름 (default: VeeamBackup) + --backup-id UUID restore 시 백업 ID + --vm-name NAME libvirt 이름 (restore/status/list) + --vm-uuid UUID Mold VM UUID + --restore-source SRC 복원 소스: auto | veeam | mold-only (default: auto) + auto = FULL은 Mold, INCREMENTAL은 Veeam 체인 회수 후 Mold 복원 + veeam = Veeam 체인을 호스트로 회수 후 Mold 복원 + mold-only= datadisk 백업만으로 복원 (NAS·Veeam chain export 없음) + --since-min N restore-watch/backup-watch 조회 창(분, default: 60) + --trigger-mold restore-watch: VM 소유 호스트만 Mold restoreBackup 호출 + -n, --dry-run + +Mold backup offering: BACKUP_OFFERING_NAME=VeeamBackup (veeam_config --offering-name) + +Examples: + # 전체 실행 중 VM 백업 + mold-backup.sh backup-full --job VeeamBackup + + # VM별 백업 ID 확인 + mold-backup.sh list-backups --job VeeamBackup + + # i-2-11-VM 만 복원 (auto: FULL=Mold, INCREMENTAL=Veeam 회수 후 Mold) + virsh shutdown i-2-11-VM + mold-backup.sh restore --job VeeamBackup --vm-name i-2-11-VM --backup-id + + # Veeam 리포지토리에서 복원 (Veeam 체인 회수 후 Mold 복원) + mold-backup.sh restore --job VeeamBackup --vm-name i-2-11-VM --backup-id --restore-source veeam + + # Mold datadisk만으로 복원 (NAS/Veeam chain 없음) + mold-backup.sh restore --job VeeamBackup --vm-name i-2-11-VM --backup-id --restore-source mold-only + + # Veeam UI 복원 감지 (+ --trigger-mold 시 KVM이 datadisk restoreBackup 실행) + mold-backup.sh restore-watch --job Mold_Guest_Backup --since-min 10 --trigger-mold + + # 이벤트 직접 주입 (Veeam PS1 SSH push) + mold-backup.sh restore-event --job Mold_Guest_Backup veeam.restore.completed i-2-63-VM + + # Veeam UI에서 직접 백업한 내역을 Mold에 반영 (cron/systemd timer로 주기 실행 권장) + mold-backup.sh backup-watch --job VeeamBackup --since-min 60 +EOF +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --job) JOB_NAME="$2"; shift 2 ;; + --backup-id) BACKUP_ID="$2"; shift 2 ;; + --vm-name) VM_NAME_ARG="$2"; shift 2 ;; + --vm-uuid) VM_UUID_ARG="$2"; shift 2 ;; + --restore-source) RESTORE_SOURCE="$2"; shift 2 ;; + --since-min) SINCE_MIN="$2"; shift 2 ;; + --trigger-mold) TRIGGER_MOLD=true; shift ;; + -n|--dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + if [[ "$CMD" == "restore-event" ]]; then + RESTORE_EVENT_ARGS+=("$1") + shift + else + die "Unknown option: $1" + fi + ;; + esac +done + +[[ -n "$CMD" ]] || { usage; exit 1; } + +export VEEAM_JOB_NAME="$JOB_NAME" +mold_backup_load_config || die "Job config not found — run veeam_config.sh first" + +[[ -n "$VM_NAME_ARG" ]] && export VM_NAME="$VM_NAME_ARG" +[[ -n "$VM_UUID_ARG" ]] && export VM_UUID="$VM_UUID_ARG" +[[ -n "$BACKUP_ID" ]] && export BACKUP_ID +[[ -n "${RESTORE_SOURCE:-}" ]] && export RESTORE_SOURCE + +run_pre() { + echo "=== backup (pre-notify) job=${JOB_NAME} offering=$(mold_backup_offering_name) vm_include=${VM_INCLUDE:-*} ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] ablestack_veeam_pre_notify.sh ${CLIENT} ${JOB_NAME}" + return 0 + fi + "${ABLESTACK_VEEAM_ETC_DIR}/ablestack_veeam_pre_notify.sh" "$CLIENT" "$JOB_NAME" +} + +run_post() { + echo "=== backup-complete job=${JOB_NAME} ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] ablestack_veeam_post_notify.sh ${CLIENT} ${JOB_NAME}" + return 0 + fi + "${ABLESTACK_VEEAM_ETC_DIR}/ablestack_veeam_post_notify.sh" "$CLIENT" "$JOB_NAME" +} + +# Mold-initiated bidirectional (mode C): after a Mold backup, start the matching Veeam +# Agent job(s) over SSH. mold_backup_trigger_veeam_job sets the mold-active marker so the +# Veeam-driven pre_notify skips a duplicate createAblestackVeeamBackup (one Mold record). +# Requires VEEAM_TRIGGER_ENABLED/METHOD/SSH_* and VM_TARGETS in the loaded job conf. +run_veeam_trigger() { + if [[ "${VEEAM_TRIGGER_ENABLED:-false}" != "true" ]]; then + return 0 + fi + echo "=== Mold→Veeam trigger (job=${JOB_NAME}) ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] mold_backup_trigger_veeam_job for VM_INCLUDE=${VM_INCLUDE:-*}" + return 0 + fi + local vm + local -a _domains=() + while IFS= read -r vm; do + [[ -z "$vm" ]] && continue + _domains+=("$vm") + done < <(mold_backup_list_target_domains || true) + for vm in "${_domains[@]}"; do + mold_backup_vm_in_filter "$vm" || continue + mold_backup_trigger_veeam_job "$vm" [session-id] [vm-name] [detail]" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] ablestack_veeam_restore_event.sh $*" + return 0 + fi + "${ABLESTACK_VEEAM_ETC_DIR}/ablestack_veeam_restore_event.sh" "$@" +} + +# Veeam UI에서 직접 수행한 백업을 감지해 Mold 상태(registry/로그)에 반영. +# restore-watch와 동일하게 KVM→Veeam SSH로 최근 완료된 백업 세션을 조회하고, +# 대상 게스트 IP를 VM_TARGETS로 libvirt VM에 역매핑해 기록한다. +run_backup_watch() { + if [[ "${VEEAM_TRIGGER_ENABLED:-false}" != "true" && -z "${VEEAM_SSH_HOST:-}" ]]; then + die "backup-watch needs VEEAM_SSH_HOST (KVM→Veeam SSH) in job conf" + fi + echo "=== backup-watch job=${JOB_NAME} window=${SINCE_MIN}min ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] mold_backup_watch_veeam_backups ${JOB_NAME} ${SINCE_MIN}" + return 0 + fi + mold_backup_watch_veeam_backups "$JOB_NAME" "$SINCE_MIN" + local safe_job reg + safe_job="$(mold_backup_safe_job_name "$JOB_NAME")" + reg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.log" + if [[ -f "$reg" ]]; then + echo "=== backup registry (last 10) ===" + tail -10 "$reg" + fi +} + +run_list_backups() { + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + echo "=== BackedUp backups (offering=$(mold_backup_offering_name)) ===" + local vm_name vm_id json + while IFS= read -r vm_name; do + [[ -z "$vm_name" ]] && continue + mold_backup_vm_in_filter "$vm_name" || continue + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { echo "${vm_name}: (no Mold VM id)"; continue; } + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + echo "--- ${vm_name} (${vm_id}) ---" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): b = [b] + backed = [x for x in b if str(x.get('status','')).lower() == 'backedup'] + if not backed: + print(' (no BackedUp backups)') + for x in backed: + print(f\" restore: mold-backup.sh restore --vm-name {sys.argv[1]} --backup-id {x.get('id')} # {x.get('created')} {x.get('type')}\") +except Exception as e: + print(' (parse error)', e) +" "$vm_name" 2>/dev/null || echo "$json" | head -c 200 + done < <(mold_backup_list_target_domains || true) + local safe_job reg + safe_job="$(mold_backup_safe_job_name "$JOB_NAME")" + reg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.log" + if [[ -f "$reg" ]]; then + echo "=== registry (last backup_id per VM) ===" + tail -30 "$reg" + fi +} + +run_status() { + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + local vm_id="${VM_UUID:-}" + if [[ -z "$vm_id" && -n "${VM_NAME:-}" ]]; then + vm_id="$(mold_backup_api_get_vm_id "$VM_NAME" 2>/dev/null || true)" + fi + [[ -n "$vm_id" ]] || die "status needs --vm-name, --vm-uuid, or VM_NAME in conf" + + echo "=== Mold backups VM ${vm_id} ===" + mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" \ + | python3 -m json.tool 2>/dev/null || true + local json cnt + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + cnt="$(mold_backup_api_count_backed_up_from_json "$json" 2>/dev/null || echo "?")" + echo "BackedUp chain count: ${cnt}" + local safe_job reg + safe_job="$(mold_backup_safe_job_name "$JOB_NAME")" + reg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.log" + echo "=== registry ===" + [[ -f "$reg" ]] && grep "vm=${VM_NAME:-}" "$reg" 2>/dev/null | tail -5 || tail -10 "$reg" 2>/dev/null || echo "(none)" + local rreg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.restore.log" + echo "=== restore registry (Veeam UI 복원 반영) ===" + if [[ -f "$rreg" ]]; then + grep "vm=${VM_NAME:-}" "$rreg" 2>/dev/null | tail -5 || tail -10 "$rreg" 2>/dev/null + else + echo "(none)" + fi +} + +case "$CMD" in + backup|pre) run_pre ;; + backup-complete|post) run_post ;; + backup-full|full) + run_pre + run_post + run_veeam_trigger + ;; + restore|restore-notify) run_restore ;; + restore-watch|watch-restore) run_restore_watch ;; + restore-event) run_restore_event "${RESTORE_EVENT_ARGS[@]}" ;; + backup-watch|watch-backup) run_backup_watch ;; + list-backups|list) run_list_backups ;; + status) run_status ;; + -h|--help|help) usage ;; + *) die "Unknown command: ${CMD}" ;; +esac + +echo "=== mold-backup.sh ${CMD} done ===" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default b/scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default new file mode 100644 index 000000000000..09ad538b37f4 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default @@ -0,0 +1,46 @@ +# Veeam Backup & Replication server configuration +# Copy to C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf and edit. + +VEEAM_JOB_NAME="" +VEEAM_SCHEDULE_NAME="default" +VEEAM_MAX_CHAIN="7" +# filelevel = KVM Agent SelectedFiles /tmp/mold/veeam (host mode) +# guest = one Agent job per VM IP (SelectedFiles on guest; pre-job/post-job for Mold hooks) +VEEAM_BACKUP_MODE="filelevel" +VEEAM_BACKUP_TARGET="host" +# Guest mode: libvirt-name:guest-ip pairs (comma-separated) +VM_TARGETS="" +VEEAM_GUEST_JOB_PREFIX="Mold VM" +# Guest VM SSH (Veeam Protection Group — auto Individual computers + rescan) +GUEST_VM_SSH_USER=root +GUEST_VM_SSH_PASSWORD= +# Linux path on KVM — used by setup-veeam-mold-job.ps1 / create-veeam-agent-job.ps1 (host mode only) +VEEAM_HOST_BACKUP_PATH="/tmp/mold/veeam" +# Custom protection group for Agent job (PowerShell). Not "Manually Added". +# guest mode: create in Veeam UI first (Inventory -> Protection Groups) +VEEAM_PROTECTION_GROUP_NAME="Mold KVM Agents" + +VM_NAME="" +# Windows-only: FLR disk export on B&R server (veeam-job-pre-backup.ps1). Do NOT use for Linux Agent scope. +STAGING_PATH="D:\veeam-staging" + +KVM_HOST="" +KVM_SSH_USER="root" +KVM_SSH_KEY="" +KVM_SSH_PASSWORD="" +KVM_PRE_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" +KVM_POST_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" +KVM_RESTORE_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_restore_notify.sh" +KVM_RESTORE_EVENT_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_restore_event.sh" +CLEANUP_STAGING_AFTER_BACKUP="true" + +VEEAM_RESTORE_POINT_ID="" +BACKUP_ID="" +# Per-VM Mold backup UUID for restore-watch / post-restore (libvirt-name:uuid pairs) +# Example: VM_BACKUP_IDS=i-2-63-VM:2d48660f-de6b-42de-ac47-9f1337b9bfce +VM_BACKUP_IDS="" +# Datadisk + Veeam host repo (E:\opt1\veeam\) +VEEAM_HOST_REPO_ROOT="E:/opt1/veeam" +VEEAM_REPO_NAME="" +# After Veeam UI guest-file restore: Mold datadisk qcow2 only (Veeam already restored files) +VEEAM_UI_RESTORE_SOURCE="mold-only" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh b/scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh new file mode 100644 index 000000000000..e44d777ee392 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh @@ -0,0 +1,417 @@ +#!/usr/bin/bash +# Shared helpers for Mold guest VM + Veeam onboarding (sourced by other scripts). +# shellcheck shell=bash + +mold_guest_common_die() { echo "ERROR: $*" >&2; exit 1; } + +mold_guest_resolve_script() { + local name="$1" + local base="${MOLD_GUEST_SCRIPT_DIR:-}" + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local candidate + for candidate in \ + "${base}/${name}" \ + "${etc}/${name}" \ + "/usr/share/mold/backup/veeam/${name}" \ + "/tmp/veeam-install/${name}"; do + if [[ -f "$candidate" ]]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +mold_guest_sync_api_from_env() { + local env_file="$1" conf_file="$2" key val + [[ -f "$env_file" && -f "$conf_file" ]] || return 0 + for key in MOLD_API_URL MOLD_API_KEY MOLD_API_SECRET ZONE_ID BACKUP_REPO_ADDRESS BACKUP_OFFERING_NAME \ + VEEAM_SSH_HOST VM_TARGETS VEEAM_PASSWORD VEEAM_USERNAME VEEAM_TRIGGER_METHOD \ + MOLD_DATADISK_PATH VEEAM_HOST_REPO_ROOT VEEAM_REPO_NAME BACKUP_STORAGE_MODE KVM_HOSTNAME \ + KVM_HOST KVM_SSH_USER KVM_SSH_PASSWORD VEEAM_JOB_NAME VEEAM_PROTECTION_GROUP_NAME; do + val=$(grep -E "^${key}=" "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"') || true + [[ -n "$val" ]] || continue + if grep -qE "^${key}=" "$conf_file" 2>/dev/null; then + sed -i "s#^${key}=.*#${key}=\"${val}\"#" "$conf_file" + else + echo "${key}=\"${val}\"" >> "$conf_file" + fi + done +} + +# Push mold-backup.windows.conf (with KVM_SSH_PASSWORD) to Veeam B&R for setup-veeam-mold-job.ps1. +mold_guest_sync_windows_conf_to_veeam() { + local env_file="$1" + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local win_local="${etc}/mold-backup.windows.conf" + local veeam_host veeam_user install_dir + local -a ssh_opts=() scp_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + [[ -n "$veeam_host" ]] || return 0 + [[ -f "$win_local" ]] || return 0 + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + mold_guest_sync_api_from_env "$env_file" "$win_local" + echo "=== Sync windows conf → ${veeam_user}@${veeam_host}:${install_dir}/mold-backup.windows.conf ===" + scp "${scp_opts[@]}" "${ssh_opts[@]}" "$win_local" \ + "${veeam_user}@${veeam_host}:${install_dir}/mold-backup.windows.conf" +} + +mold_guest_ensure_conf() { + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local guest_conf="${etc}/Mold_Guest_Backup.conf" + local main_conf="" candidate + for candidate in "${etc}/mold-backup.conf" "${etc}/Mold_Guest_Backup.conf" "${etc}/"*.conf; do + [[ -f "$candidate" ]] || continue + main_conf="$candidate" + break + done + [[ -n "$main_conf" ]] || mold_guest_common_die "Missing KVM backup conf under ${etc} — run veeam_config.sh first" + if [[ ! -f "$guest_conf" ]]; then + cp -a "$main_conf" "$guest_conf" + chmod 0600 "$guest_conf" + echo "Created ${guest_conf} from $(basename "$main_conf")" + fi + local env_file="${etc}/mold-backup.env" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$guest_conf" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$main_conf" + sed -i 's/^BACKUP_MODE=.*/BACKUP_MODE="guest"/' "$guest_conf" + for key_val in \ + 'VEEAM_TRIGGER_ENABLED="true"' \ + 'VEEAM_TRIGGER_METHOD="ssh"'; do + key="${key_val%%=*}" + if grep -qE "^${key}=" "$guest_conf" 2>/dev/null; then + sed -i "s#^${key}=.*#${key_val}#" "$guest_conf" + else + echo "$key_val" >> "$guest_conf" + fi + done +} + +# Passwordless SSH guest VM -> KVM (required for Veeam pre/post hooks). +mold_guest_run_ssh_setup() { + local env_file="$1" + local vm_targets="$2" + local setup_script rc=0 + + [[ -n "$vm_targets" ]] || mold_guest_common_die "VM_TARGETS empty for guest SSH setup" + [[ -f "$env_file" ]] || mold_guest_common_die "Missing env file: $env_file" + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + if [[ -z "${GUEST_VM_SSH_PASSWORD:-}" ]]; then + mold_guest_common_die "GUEST_VM_SSH_PASSWORD is not set in ${env_file} (required for automatic guest->KVM SSH)" + fi + + setup_script="$(mold_guest_resolve_script setup-guest-kvm-ssh.sh)" \ + || mold_guest_common_die "setup-guest-kvm-ssh.sh not found" + + echo "=== Auto: guest -> KVM SSH (${vm_targets}) ===" + bash "$setup_script" --env-file "$env_file" --vm-targets "$vm_targets" || rc=$? + if [[ "$rc" -ne 0 ]]; then + mold_guest_common_die "guest->KVM SSH setup failed (check GUEST_VM_SSH_PASSWORD and guest reachability)" + fi +} + +# conf + SSH + optional Veeam .sh upload (called from deploy / push / veeam_config). +mold_guest_prepare_onboarding() { + local env_file="$1" + local vm_targets="$2" + local skip_ssh="${3:-false}" + + mold_guest_ensure_conf + if [[ "$skip_ssh" != "true" ]]; then + mold_guest_run_ssh_setup "$env_file" "$vm_targets" + fi +} + +mold_guest_resolve_kvm_ip_from_env() { + local env_file="$1" + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + if [[ -n "${KVM_IP:-}" ]]; then + echo "$KVM_IP" + return 0 + fi + local host="${KVM_HOST:-}" + if [[ "$host" =~ @(.+)$ ]]; then + echo "${BASH_REMATCH[1]}" + return 0 + fi + [[ -n "$host" ]] && echo "$host" && return 0 + mold_guest_common_die "Set KVM_IP or KVM_HOST in ${env_file}" +} + +# E:\opt1\veeam\ — one Veeam repository per hypervisor (not per VM). +mold_guest_setup_veeam_host_repo() { + local env_file="$1" kvm_hostname="${2:-}" + local veeam_host veeam_user install_dir host_root repo_name + local -a ssh_opts=() scp_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" ]] || mold_guest_common_die "VEEAM_SSH_HOST not set" + [[ -n "$kvm_hostname" ]] || kvm_hostname="${KVM_HOSTNAME:-$(hostname -s)}" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + host_root="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + repo_name="${VEEAM_REPO_NAME:-Mold ${kvm_hostname}}" + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + mold_guest_sync_ps1_to_veeam "$env_file" + local ps1_local + ps1_local="$(mold_guest_resolve_script setup-veeam-host-repo.ps1 2>/dev/null || true)" + if [[ -n "$ps1_local" ]]; then + scp "${scp_opts[@]}" "${ssh_opts[@]}" "$ps1_local" \ + "${veeam_user}@${veeam_host}:${install_dir}/setup-veeam-host-repo.ps1" + fi + + echo "=== Veeam host repo: ${host_root}/${kvm_hostname} (name: ${repo_name}) ===" + local remote_cmd + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -ExecutionPolicy Bypass -File \"${install_dir}/setup-veeam-host-repo.ps1\" " \ + "-KvmHostname \"${kvm_hostname}\" " \ + "-HostRepoRoot \"${host_root}\" " \ + "-RepositoryName \"${repo_name}\"") + ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=120 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" \ + || mold_guest_common_die "setup-veeam-host-repo.ps1 failed — create E:\\opt1\\veeam\\${kvm_hostname} in Veeam UI" +} + +# Copy required PS1 from KVM share to Veeam (not installed under /etc/ablestack/veeam). +mold_guest_sync_ps1_to_veeam() { + local env_file="$1" + local veeam_host veeam_user install_dir ps1_dir f local_path + local -a ssh_opts=() scp_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + [[ -n "$veeam_host" ]] || return 0 + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + ps1_dir="" + for f in install-veeam-job.ps1 create-veeam-agent-job.ps1 setup-veeam-mold-job.ps1 \ + setup-veeam-host-repo.ps1; do + local_path="$(mold_guest_resolve_script "$f" 2>/dev/null || true)" + [[ -n "$local_path" ]] && ps1_dir="$(dirname "$local_path")" && break + done + [[ -n "$ps1_dir" ]] || { + echo "WARN: PS1 not found on KVM — run: bash /tmp/veeam-install/install.sh" + return 0 + } + + echo "=== Sync PS1 → ${veeam_user}@${veeam_host}:${install_dir} (from ${ps1_dir}) ===" + for f in install-veeam-job.ps1 create-veeam-agent-job.ps1 setup-veeam-mold-job.ps1 \ + setup-veeam-host-repo.ps1 veeam-job-pre-backup.ps1 veeam-job-post-backup.ps1 \ + veeam-job-post-restore.ps1; do + [[ -f "${ps1_dir}/${f}" ]] || continue + scp "${scp_opts[@]}" "${ssh_opts[@]}" "${ps1_dir}/${f}" \ + "${veeam_user}@${veeam_host}:${install_dir}/${f}" + done +} + +# KVM hypervisor (10.10.31.2) — one Veeam Agent job + Mold_Host_Backup.conf (not per-guest VM). +mold_host_ensure_conf() { + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local host_conf="${etc}/Mold_Host_Backup.conf" + local main_conf="" candidate + for candidate in "${etc}/mold-backup.conf" "${etc}/Mold_Host_Backup.conf" "${etc}/"*.conf; do + [[ -f "$candidate" ]] || continue + main_conf="$candidate" + break + done + [[ -n "$main_conf" ]] || mold_guest_common_die "Missing KVM backup conf under ${etc} — run veeam_config.sh first" + if [[ ! -f "$host_conf" ]]; then + cp -a "$main_conf" "$host_conf" + chmod 0600 "$host_conf" + echo "Created ${host_conf} from $(basename "$main_conf")" + fi + local env_file="${etc}/mold-backup.env" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$host_conf" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$main_conf" + sed -i 's/^BACKUP_MODE=.*/BACKUP_MODE="host"/' "$host_conf" + local pre="${etc}/ablestack_veeam_pre_notify.sh" + local post="${etc}/ablestack_veeam_post_notify.sh" + for key_val in \ + "KVM_PRE_NOTIFY_SCRIPT=\"${pre}\"" \ + "KVM_POST_NOTIFY_SCRIPT=\"${post}\"" \ + 'VEEAM_TRIGGER_ENABLED="false"' \ + 'VEEAM_TRIGGER_METHOD="auto"'; do + key="${key_val%%=*}" + if grep -qE "^${key}=" "$host_conf" 2>/dev/null; then + sed -i "s#^${key}=.*#${key_val}#" "$host_conf" + else + echo "$key_val" >> "$host_conf" + fi + done +} + +# Veeam Agent job on KVM (e.g. 10.10.31.2) → E:\opt1\veeam\\ repository. +mold_host_ensure_kvm_prepost_scripts() { + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local pre="${KVM_PRE_NOTIFY_SCRIPT:-${etc}/ablestack_veeam_pre_notify.sh}" + local post="${KVM_POST_NOTIFY_SCRIPT:-${etc}/ablestack_veeam_post_notify.sh}" + local staging="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + local ini="/etc/veeam/veeam.ini" + local timeout="${VEEAM_SCRIPT_TIMEOUT:-1800}" + + [[ -x "$pre" ]] || mold_guest_common_die "Missing pre-notify on KVM: ${pre} — run install.sh" + [[ -x "$post" ]] || mold_guest_common_die "Missing post-notify on KVM: ${post} — run install.sh" + mkdir -p "$staging" + chmod 0755 "$staging" 2>/dev/null || true + + if [[ -f "$ini" ]]; then + if grep -q '^\[scripts\]' "$ini" 2>/dev/null; then + if grep -q '^timeoutPrePost' "$ini" 2>/dev/null; then + sed -i "s/^timeoutPrePost.*/timeoutPrePost = ${timeout}/" "$ini" + else + sed -i "/^\[scripts\]/a timeoutPrePost = ${timeout}" "$ini" + fi + else + printf '\n[scripts]\ntimeoutPrePost = %s\n' "$timeout" >>"$ini" + fi + if command -v systemctl >/dev/null 2>&1 && systemctl is-active veeamservice >/dev/null 2>&1; then + systemctl restart veeamservice 2>/dev/null || true + echo "Veeam Agent: ${ini} timeoutPrePost=${timeout}s (veeamservice restarted)" + else + echo "Veeam Agent: set ${ini} [scripts] timeoutPrePost = ${timeout}" + fi + else + echo "WARN: ${ini} not found — install Veeam Agent for Linux on this KVM host" + fi + echo "KVM pre/post ready: ${pre} | ${post} | staging=${staging}" +} + +# Register Guest Processing pre-freeze/post-thaw on the KVM Linux Agent job (runs ON 10.10.31.2). +mold_host_register_veeam_scripts() { + local env_file="$1" kvm_hostname="${2:-}" kvm_ip="${3:-}" + local veeam_host veeam_user install_dir job_name pg_name pre post + local -a ssh_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" ]] || mold_guest_common_die "VEEAM_SSH_HOST not set in ${env_file}" + [[ -n "$kvm_hostname" ]] || kvm_hostname="${KVM_HOSTNAME:-$(hostname -s)}" + [[ -n "$kvm_ip" ]] || kvm_ip="$(mold_guest_resolve_kvm_ip_from_env "$env_file")" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + job_name="${VEEAM_JOB_NAME:-Mold ${kvm_hostname}}" + pg_name="${VEEAM_PROTECTION_GROUP_NAME:-Mold KVM Agents}" + pre="${KVM_PRE_NOTIFY_SCRIPT:-/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh}" + post="${KVM_POST_NOTIFY_SCRIPT:-/etc/ablestack/veeam/ablestack_veeam_post_notify.sh}" + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + mold_guest_sync_ps1_to_veeam "$env_file" + + if ! mold_guest_veeam_job_exists "$env_file" "$job_name"; then + echo "WARN: Veeam job '${job_name}' not found — run mold_host_setup_veeam_job first" + return 1 + fi + + echo "=== Register pre/post on Veeam host job '${job_name}' (Linux Agent ${kvm_ip}) ===" + local remote_ps1="${install_dir}/install-veeam-job.ps1" + local remote_cmd + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -ExecutionPolicy Bypass -File \"${remote_ps1}\" " \ + "-JobName \"${job_name}\" " \ + "-LinuxAgent " \ + "-KvmHost \"${kvm_ip}\" " \ + "-ProtectionGroupName \"${pg_name}\" " \ + "-AgentPreNotifyScript \"${pre}\" " \ + "-AgentPostNotifyScript \"${post}\" " \ + "-InstallDir \"${install_dir}\" " \ + "-SourceDir \"${install_dir}\"") + ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=120 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" \ + || mold_guest_common_die "install-veeam-job.ps1 -LinuxAgent failed for '${job_name}'" + + echo "Registered Guest Processing scripts on '${job_name}':" + echo " Pre-freeze → ${pre} (Mold export → ${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam})" + echo " Post-thaw → ${post} (registry + staging cleanup)" +} + +mold_host_setup_veeam_job() { + local env_file="$1" kvm_hostname="${2:-}" kvm_ip="${3:-}" + local veeam_host veeam_user install_dir job_name repo_name backup_path pg_name + local -a ssh_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" ]] || mold_guest_common_die "VEEAM_SSH_HOST not set in ${env_file}" + [[ -n "$kvm_hostname" ]] || kvm_hostname="${KVM_HOSTNAME:-$(hostname -s)}" + [[ -n "$kvm_ip" ]] || kvm_ip="$(mold_guest_resolve_kvm_ip_from_env "$env_file")" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + job_name="${VEEAM_JOB_NAME:-Mold ${kvm_hostname}}" + repo_name="${VEEAM_REPO_NAME:-Mold ${kvm_hostname}}" + backup_path="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + pg_name="${VEEAM_PROTECTION_GROUP_NAME:-Mold KVM Agents}" + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + [[ -n "${KVM_SSH_PASSWORD:-}" ]] || mold_guest_common_die "KVM_SSH_PASSWORD required in ${env_file} (Veeam: Cannot find credentials for agent)" + + mold_guest_sync_ps1_to_veeam "$env_file" + mold_guest_sync_windows_conf_to_veeam "$env_file" + mold_guest_setup_veeam_host_repo "$env_file" "$kvm_hostname" 2>/dev/null \ + || echo "WARN: Veeam host repo setup skipped" + + echo "=== Veeam KVM host job: ${job_name} (agent ${kvm_ip} / ${kvm_hostname}) ===" + local remote_cmd win_conf="${install_dir}/mold-backup.windows.conf" + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -ExecutionPolicy Bypass -File \"${install_dir}/setup-veeam-mold-job.ps1\" " \ + "-ConfPath \"${win_conf}\" " \ + "-JobName \"${job_name}\" " \ + "-KvmHost \"${kvm_ip}\" " \ + "-AgentHostName \"${kvm_hostname}\" " \ + "-BackupPath \"${backup_path}\" " \ + "-RepositoryName \"${repo_name}\" " \ + "-ProtectionGroupName \"${pg_name}\"") + ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" \ + || mold_guest_common_die "setup-veeam-mold-job.ps1 failed for KVM host ${kvm_ip}" + + mold_host_register_veeam_scripts "$env_file" "$kvm_hostname" "$kvm_ip" \ + || echo "WARN: pre/post registration skipped — re-run: mold_host_register_veeam_scripts" +} + +mold_guest_veeam_job_exists() { + local env_file="$1" job_name="$2" + local veeam_host veeam_user install_dir remote_cmd out + local -a ssh_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" && -n "$job_name" ]] || return 1 + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -Command \"Import-Module Veeam.Backup.PowerShell -ErrorAction SilentlyContinue; " \ + "if (Get-VBRComputerBackupJob -Name '${job_name//\'/\'\'\'}' -ErrorAction SilentlyContinue) { Write-Output 'true' } else { Write-Output 'false' }\"") + + out="$(ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=60 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" 2>/dev/null | tr -d '[:space:]')" + [[ "$out" == "true" ]] +} + +# Create one guest Agent job on Veeam (+ register Guest Processing via install-veeam-job.ps1). +mold_guest_create_veeam_job_for_vm() { + mold_guest_common_die "Guest-mode Veeam jobs were removed. Use mold_host_setup_veeam_job / setup-datadisk-veeam-backup.sh." +} + +# Register Pre-job/Post-job on an EXISTING Veeam Agent job (UI-created job). +mold_guest_register_veeam_job_scripts() { + mold_guest_common_die "Guest-mode Veeam jobs were removed. Use mold_host_register_veeam_scripts / setup-datadisk-veeam-backup.sh." +} diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service new file mode 100644 index 000000000000..66ed6c3045f2 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Polls Veeam for completed restore sessions and triggers Mold datadisk restore on the +# KVM host that owns the VM. Uses the Veeam UI restore point id to select the matching +# Mold backup_id (not always the latest). +[Unit] +Description=Mold Veeam restore agent (detect Veeam restore, trigger Mold restore) +After=network-online.target cloudstack-agent.service +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=-/etc/ablestack/veeam/mold-backup.env +Environment="MOLD_BACKUP_CONF=/etc/ablestack/veeam/Mold_Host_Backup.conf" +ExecStart=/etc/ablestack/veeam/mold-veeam-restore-agent.sh + +[Install] +WantedBy=multi-user.target diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh new file mode 100644 index 000000000000..d40373d35131 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash +# Poll Veeam for completed restore sessions and trigger Mold restoreBackup on this KVM host. +# Matches the Veeam UI restore point (FLR) to the corresponding Mold backup_id. +# Invoked by mold-veeam-restore-agent.timer (every 3 min). +# +# Flow: Veeam UI Guest files restore (FLR) on host job → this agent → Mold datadisk restore +# +# Manual: +# bash /etc/ablestack/veeam/mold-veeam-restore-agent.sh +# bash /etc/ablestack/veeam/mold-backup.sh restore-watch --job 'Mold ablecube31-2' --trigger-mold --since-min 30 + +set -euo pipefail + +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +ENV_FILE="${ETC_DIR}/mold-backup.env" +HOST_CONF="${MOLD_BACKUP_CONF:-${ETC_DIR}/Mold_Host_Backup.conf}" +SINCE_MIN="${VEEAM_RESTORE_WATCH_WINDOW_MIN:-10}" + +[[ -f "$ENV_FILE" ]] && { set -a; # shellcheck source=/dev/null + source "$ENV_FILE"; set +a; } + +export MOLD_BACKUP_CONF="$HOST_CONF" +mkdir -p "${ETC_DIR}/events" "${ETC_DIR}/registry" "${ETC_DIR}/state" 2>/dev/null || true + +job="" +if [[ -f "$HOST_CONF" ]]; then + job="$(grep -E '^VEEAM_JOB_NAME=' "$HOST_CONF" 2>/dev/null | head -1 | cut -d= -f2- | sed -e 's/^"//' -e 's/"$//' -e "s/^'//" -e "s/'$//")" +fi +job="${job:-${VEEAM_JOB_NAME:-Mold ablecube31-2}}" + +exec "${ETC_DIR}/mold-backup.sh" restore-watch \ + --job "$job" \ + --since-min "${SINCE_MIN}" \ + --trigger-mold diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer new file mode 100644 index 000000000000..cbfce699848a --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[Unit] +Description=Poll Veeam restores and trigger Mold datadisk restore every 3 minutes + +[Timer] +OnBootSec=1min +OnUnitActiveSec=3min +AccuracySec=30s +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh new file mode 100755 index 000000000000..4a74d4d99497 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh @@ -0,0 +1,63 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Mold -> Veeam trigger (bidirectional mode C). +# Invoked best-effort by ablestack_nasbackup.sh after a VM backup completes. +# Starts the matching Veeam Agent job over SSH unless the current backup was +# itself triggered by Veeam (veeam-active marker present). +# +# Args: [backup-type] +# Never fail the caller: always exit 0. + +OP="${1:-}" +VM="${2:-}" +BACKUP_TYPE="${3:-}" + +# Only react to the actual VM backup operation. +case "$OP" in + backup-running|backup-rbd) ;; + *) exit 0 ;; +esac +[[ -n "$VM" ]] || exit 0 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIB="${SCRIPT_DIR}/mold-backup.lib.sh" +[[ -f "$LIB" ]] || LIB="/etc/ablestack/veeam/mold-backup.lib.sh" +[[ -f "$LIB" ]] || exit 0 + +# shellcheck source=/dev/null +source "$LIB" +# Guest VMs use Mold_Guest_Backup.conf (VEEAM_TRIGGER_*, VM_TARGETS, VEEAM_SSH_*). +export MOLD_BACKUP_CONF="${MOLD_BACKUP_CONF:-/etc/ablestack/veeam/Mold_Guest_Backup.conf}" +mold_backup_load_config || exit 0 + +[[ "${VEEAM_TRIGGER_ENABLED:-false}" == "true" ]] || { + mold_backup_notify_log info "Mold→Veeam trigger disabled (set VEEAM_TRIGGER_ENABLED=true in Mold_Guest_Backup.conf)" + exit 0 +} + +# If Veeam started this backup (veeam-active marker), do NOT start Veeam again. +if mold_backup_trigger_active "veeam-active" "$VM"; then + mold_backup_notify_log info "veeam-active present for ${VM}: Veeam-driven backup, skip Mold→Veeam trigger" + exit 0 +fi + +mold_backup_notify_log info "Mold→Veeam trigger hook: vm=${VM} op=${OP}" +mold_backup_trigger_veeam_job "$VM" || mold_backup_notify_log warn "Mold→Veeam trigger failed for ${VM} (see log above)" + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh b/scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh new file mode 100755 index 000000000000..4e56035c770e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh @@ -0,0 +1,117 @@ +#!/usr/bin/bash +# Copy veeam scripts to KVM and run install.sh (+ optional veeam_config). +# Run from Git repo on Mac or any host with SSH to KVM. +# +# cp mold-backup.env.example mold-backup.env # edit API keys +# bash scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh --env-file mold-backup.env +# +# Or: +# KVM_HOST=root@10.10.31.2 bash scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh +# bash scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh --host root@10.10.31.2 --no-configure + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" +RUN_CONFIGURE="${RUN_CONFIGURE:-true}" + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) + [[ -f "$2" ]] || die "not found: $2" + # shellcheck source=/dev/null + set -a && source "$2" && set +a + shift 2 + ;; + --host|--kvm-host) + [[ -n "${2:-}" ]] || die "$1 requires a value (e.g. root@10.10.31.2)" + KVM_HOST="$2" + [[ "$KVM_HOST" == *@* ]] || KVM_HOST="root@${KVM_HOST}" + shift 2 + ;; + --jump|--jump-host) + [[ -n "${2:-}" ]] || die "$1 requires a value (e.g. root@10.10.31.20)" + JUMP_HOST="$2" + shift 2 + ;; + --no-configure) RUN_CONFIGURE=false; shift ;; + -h|--help) + echo "Usage: push-to-kvm.sh [--host root@KVM] [--jump root@JUMP] [--env-file mold-backup.env] [--no-configure]" + echo "" + echo "Examples:" + echo " bash push-to-kvm.sh --host root@10.10.31.2 --no-configure" + echo " KVM_HOST=root@10.10.31.2 bash push-to-kvm.sh" + echo " bash push-to-kvm.sh --env-file mold-backup.env" + exit 0 + ;; + *) die "Unknown: $1 (try --help)" ;; + esac +done + +KVM_HOST="${KVM_HOST:-}" +[[ -n "$KVM_HOST" ]] || die "Set KVM_HOST=root@10.10.31.2 (or use --env-file)" + +# SSH via jump host (e.g. Mac → ccvm → KVM): JUMP_HOST=root@10.10.31.20 +SSH_OPTS=() +SCP_OPTS=() +if [[ -n "${JUMP_HOST:-}" ]]; then + SSH_OPTS=(-o "ProxyJump=${JUMP_HOST}") + SCP_OPTS=(-o "ProxyJump=${JUMP_HOST}") + echo "Using jump host: ${JUMP_HOST}" +fi + +kvm_ssh() { + if ((${#SSH_OPTS[@]} > 0)); then + ssh "${SSH_OPTS[@]}" "$@" + else + ssh "$@" + fi +} + +kvm_scp() { + if ((${#SCP_OPTS[@]} > 0)); then + scp "${SCP_OPTS[@]}" "$@" + else + scp "$@" + fi +} + +echo "=== SCP veeam scripts → ${KVM_HOST}:/tmp/veeam-install/ ===" +kvm_ssh "$KVM_HOST" "mkdir -p /tmp/veeam-install" +# Include parent ablestack_nasbackup.sh (agent restore path) +NAS_PARENT="${SCRIPT_DIR}/../ablestack_nasbackup.sh" +kvm_scp -r "${SCRIPT_DIR}/"* "${KVM_HOST}:/tmp/veeam-install/" +[[ -f "$NAS_PARENT" ]] && kvm_scp "$NAS_PARENT" "${KVM_HOST}:/tmp/veeam-install/ablestack_nasbackup.sh" + +echo "=== install.sh on KVM ===" +kvm_ssh "$KVM_HOST" "bash /tmp/veeam-install/install.sh" + +if [[ "$RUN_CONFIGURE" == "true" ]]; then + JOB_NAME="${JOB_NAME:-Mold KVM Backup}" + CONFIG_ARGS=(--job-name "${JOB_NAME}" --install) + [[ -n "${KVM_IP:-}" ]] && CONFIG_ARGS+=(--kvm-host "${KVM_IP}") + if [[ -n "${MOLD_API_URL:-}" && -n "${MOLD_API_KEY:-}" && -n "${MOLD_API_SECRET:-}" ]]; then + CONFIG_ARGS+=( + --mold-url "${MOLD_API_URL}" + --api-key "${MOLD_API_KEY}" + --api-secret "${MOLD_API_SECRET}" + ) + [[ -n "${ZONE_ID:-}" ]] && CONFIG_ARGS+=(--zone-id "${ZONE_ID}") + [[ -n "${VM_INCLUDE:-}" ]] && CONFIG_ARGS+=(--vm-include "${VM_INCLUDE}") + [[ -n "${VM_NAME:-}" ]] && CONFIG_ARGS+=(--vm-name "${VM_NAME}") + [[ -n "${VM_UUID:-}" ]] && CONFIG_ARGS+=(--vm-uuid "${VM_UUID}") + [[ -n "${BACKUP_REPO_ADDRESS:-}" ]] && CONFIG_ARGS+=(--nas-repo "${BACKUP_REPO_ADDRESS}") + [[ -n "${VEEAM_URL:-}" ]] && CONFIG_ARGS+=(--veeam-url "${VEEAM_URL}") + else + echo "=== veeam_config.sh on KVM (reads /etc/ablestack/veeam/mold-backup.env) ===" + fi + echo "=== veeam_config.sh on KVM ===" + kvm_ssh "$KVM_HOST" "cd /etc/ablestack/veeam && ./veeam_config.sh $(printf '%q ' "${CONFIG_ARGS[@]}")" +fi + +echo "" +echo "=== Done. On KVM run: ===" +echo " /etc/ablestack/veeam/mold-backup.sh backup-full --job \"${JOB_NAME:-Mold KVM Backup}\"" +echo " /etc/ablestack/veeam/mold-backup.sh status --job \"${JOB_NAME:-Mold KVM Backup}\"" diff --git a/scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh b/scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh new file mode 100755 index 000000000000..6a86480d9935 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh @@ -0,0 +1,271 @@ +#!/usr/bin/bash +# Deploy Mold Veeam scripts to B&R server and create Agent backup job (idempotent). +# +# cp mold-backup.env.example mold-backup.env # edit VEEAM_SSH_HOST, KVM_IP, JOB_NAME +# bash push-to-veeam.sh --env-file mold-backup.env +# +# Requires: SSH from this host to Veeam Windows server (OpenSSH on Windows). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + +mold_push_resolve_ps1_dir() { + local candidate + for candidate in "$SCRIPT_DIR" "$SHARE_DIR" "$ETC_DIR"; do + [[ -f "${candidate}/setup-veeam-mold-job.ps1" ]] && { echo "$candidate"; return 0; } + done + echo "$SCRIPT_DIR" +} + +mold_push_sync_windows_conf() { + local env_file="${1:-${ETC_DIR}/mold-backup.env}" + local win_conf="${ETC_DIR}/mold-backup.windows.conf" + [[ -f "$env_file" ]] || return 0 + [[ -f "$win_conf" ]] || return 0 + + local pw user targets kvm_host kvm_pw kvm_user + pw="$(grep -E '^GUEST_VM_SSH_PASSWORD=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + user="$(grep -E '^GUEST_VM_SSH_USER=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + kvm_pw="$(grep -E '^KVM_SSH_PASSWORD=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + kvm_user="$(grep -E '^KVM_SSH_USER=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + targets="$(grep -E '^VM_TARGETS=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + kvm_host="$(grep -E '^KVM_HOST=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + [[ -z "$kvm_host" ]] && kvm_host="$(grep -E '^KVM_IP=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + + if [[ -n "$pw" ]]; then + if grep -qE '^GUEST_VM_SSH_PASSWORD=' "$win_conf" 2>/dev/null; then + sed -i "s#^GUEST_VM_SSH_PASSWORD=.*#GUEST_VM_SSH_PASSWORD=\"${pw}\"#" "$win_conf" + else + echo "GUEST_VM_SSH_PASSWORD=\"${pw}\"" >>"$win_conf" + fi + fi + if [[ -n "$user" ]]; then + if grep -qE '^GUEST_VM_SSH_USER=' "$win_conf" 2>/dev/null; then + sed -i "s#^GUEST_VM_SSH_USER=.*#GUEST_VM_SSH_USER=\"${user}\"#" "$win_conf" + else + echo "GUEST_VM_SSH_USER=\"${user}\"" >>"$win_conf" + fi + fi + if [[ -n "$targets" ]]; then + if grep -qE '^VM_TARGETS=' "$win_conf" 2>/dev/null; then + sed -i "s#^VM_TARGETS=.*#VM_TARGETS=\"${targets}\"#" "$win_conf" + else + echo "VM_TARGETS=\"${targets}\"" >>"$win_conf" + fi + fi + if [[ -n "$kvm_host" ]]; then + if grep -qE '^KVM_HOST=' "$win_conf" 2>/dev/null; then + sed -i "s#^KVM_HOST=.*#KVM_HOST=\"${kvm_host}\"#" "$win_conf" + else + echo "KVM_HOST=\"${kvm_host}\"" >>"$win_conf" + fi + fi + if [[ -n "$kvm_pw" ]]; then + if grep -qE '^KVM_SSH_PASSWORD=' "$win_conf" 2>/dev/null; then + sed -i "s#^KVM_SSH_PASSWORD=.*#KVM_SSH_PASSWORD=\"${kvm_pw}\"#" "$win_conf" + else + echo "KVM_SSH_PASSWORD=\"${kvm_pw}\"" >>"$win_conf" + fi + fi + if [[ -n "$kvm_user" ]]; then + if grep -qE '^KVM_SSH_USER=' "$win_conf" 2>/dev/null; then + sed -i "s#^KVM_SSH_USER=.*#KVM_SSH_USER=\"${kvm_user}\"#" "$win_conf" + else + echo "KVM_SSH_USER=\"${kvm_user}\"" >>"$win_conf" + fi + fi +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) + [[ -f "$2" ]] || die "not found: $2" + ENV_FILE="$2" + # shellcheck source=/dev/null + set -a && source "$2" && set +a + shift 2 + ;; + --vm-targets) + die "Guest-mode VM_TARGETS removed. Use host/datadisk mode only." + ;; + --skip-create) SKIP_CREATE=true; shift ;; + --no-start) SKIP_START=true; shift ;; + --skip-guest-onboard) shift ;; # ignored (compat) + -h|--help) + echo "Usage: push-to-veeam.sh [--env-file mold-backup.env] [--skip-create] [--no-start]" + echo " Deploys host-mode PS1 to Veeam and runs setup-veeam-mold-job.ps1" + exit 0 + ;; + *) die "Unknown: $1" ;; + esac +done + +SKIP_CREATE="${SKIP_CREATE:-false}" +SKIP_START="${SKIP_START:-false}" +VEEAM_SSH_HOST="${VEEAM_SSH_HOST:-}" +VEEAM_SSH_USER="${VEEAM_SSH_USER:-administrator}" +VEEAM_SSH_KEY="${VEEAM_SSH_KEY:-}" +VEEAM_INSTALL_DIR="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" +JOB_NAME="${JOB_NAME:-${VEEAM_JOB_NAME:-Mold KVM Backup}}" +KVM_IP="${KVM_IP:-10.10.31.2}" +KVM_HOSTNAME="${KVM_HOSTNAME:-}" +BACKUP_PATH="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" +VEEAM_REPO_NAME="${VEEAM_REPO_NAME:-}" + +[[ -n "$VEEAM_SSH_HOST" ]] || die "Set VEEAM_SSH_HOST=10.10.254.246 in mold-backup.env" + +VEEAM_SSH="${VEEAM_SSH_USER}@${VEEAM_SSH_HOST}" +SSH_OPTS=() +[[ -n "$VEEAM_SSH_KEY" && -f "$VEEAM_SSH_KEY" ]] && SSH_OPTS=(-i "$VEEAM_SSH_KEY") + +veeam_ssh() { + if ((${#SSH_OPTS[@]} > 0)); then + ssh "${SSH_OPTS[@]}" "$@" + else + ssh "$@" + fi +} + +veeam_scp() { + if ((${#SSH_OPTS[@]} > 0)); then + scp "${SSH_OPTS[@]}" "$@" + else + scp "$@" + fi +} + +if [[ "${VEEAM_BACKUP_TARGET:-}" == "guest" || -n "${VM_TARGETS:-}" ]]; then + die "Guest-mode Veeam jobs were removed. Use host/datadisk mode (VEEAM_BACKUP_TARGET=host, unset VM_TARGETS)." +fi + +VEEAM_FILES=( + setup-veeam-mold-job.ps1 + create-veeam-agent-job.ps1 + install-veeam-job.ps1 + veeam-job-pre-backup.ps1 + veeam-job-post-backup.ps1 + veeam-job-post-restore.ps1 + setup-veeam-host-repo.ps1 + mold-backup.windows.conf.default +) + +VEEAM_FILES_REQUIRED=( + setup-veeam-mold-job.ps1 + create-veeam-agent-job.ps1 + install-veeam-job.ps1 + mold-backup.windows.conf.default +) + +PS1_DIR="$(mold_push_resolve_ps1_dir)" +if ! grep -q 'SelectedFiles' "${PS1_DIR}/install-veeam-job.ps1" 2>/dev/null; then + die "Outdated PS1 in ${PS1_DIR} (missing SelectedFiles support). Run: bash install.sh from updated veeam/ package, or MOLD_BACKUP_SHARE_DIR=/path/to/repo/veeam bash push-to-veeam.sh ..." +fi +mold_push_sync_windows_conf "${ENV_FILE:-${ETC_DIR}/mold-backup.env}" + +echo "=== Prepare Veeam install dir: ${VEEAM_INSTALL_DIR} ===" +veeam_ssh "$VEEAM_SSH" "powershell -Command \"New-Item -ItemType Directory -Force -Path '${VEEAM_INSTALL_DIR}' | Out-Null\"" \ + || die "Cannot SSH to Veeam server ${VEEAM_SSH}" + +echo "=== SCP scripts → ${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/ (from ${PS1_DIR}) ===" +for f in "${VEEAM_FILES_REQUIRED[@]}"; do + [[ -f "${PS1_DIR}/${f}" ]] || die "Missing required ${PS1_DIR}/${f} — run install.sh or set MOLD_BACKUP_SHARE_DIR" +done +for f in "${VEEAM_FILES[@]}"; do + [[ -f "${PS1_DIR}/${f}" ]] || continue + veeam_scp "${PS1_DIR}/${f}" "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/${f}" +done + +# Push KVM-generated windows conf (local file or remote KVM_HOST) +if [[ -f "${ETC_DIR}/mold-backup.windows.conf" ]]; then + echo "=== Copy mold-backup.windows.conf from ${ETC_DIR} ===" + veeam_scp "${ETC_DIR}/mold-backup.windows.conf" \ + "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/mold-backup.windows.conf" +else + KVM_SSH="${KVM_HOST:-}" + if [[ -n "$KVM_SSH" && "$KVM_SSH" != *@* ]]; then + KVM_SSH="root@${KVM_SSH}" + fi + if [[ -n "$KVM_SSH" ]]; then + echo "=== Try copy mold-backup.windows.conf from KVM ===" + veeam_scp "${KVM_SSH}:/etc/ablestack/veeam/mold-backup.windows.conf" \ + "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/mold-backup.windows.conf" 2>/dev/null \ + || echo "NOTE: mold-backup.windows.conf not copied from KVM (run veeam_config.sh on KVM first)" + fi +fi + +if [[ "$SKIP_CREATE" == "true" ]]; then + echo "=== Skip job create (--skip-create) ===" + exit 0 +fi + +mold_push_ps_escape() { + # PowerShell single-quoted string: ' -> '' + printf '%s' "$1" | sed "s/'/''/g" +} + +mold_push_write_remote_runner() { + local setup_script="$1" + shift + local runner_local + runner_local="$(mktemp "${TMPDIR:-/tmp}/mold-push-remote-run.XXXXXX.ps1")" + { + printf '%s\n' '$ErrorActionPreference = "Stop"' + printf '& %s' "'$(mold_push_ps_escape "$setup_script")'" + while [[ $# -gt 0 ]]; do + local flag="$1" val="${2:-}" + if [[ "$flag" == -* && -n "$val" && "$val" != -* ]]; then + printf ' %s %s' "$flag" "'$(mold_push_ps_escape "$val")'" + shift 2 + else + printf ' %s' "$flag" + shift + fi + done + printf '\n' + } >"$runner_local" + veeam_scp "$runner_local" "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/mold-push-remote-run.ps1" + rm -f "$runner_local" +} + +mold_push_run_remote_setup() { + local setup_script="${VEEAM_INSTALL_DIR}/setup-veeam-mold-job.ps1" + local setup_name="setup-veeam-mold-job.ps1" + local runner_path="${VEEAM_INSTALL_DIR}/mold-push-remote-run.ps1" + local -a ps_args=( + -InstallDir "${VEEAM_INSTALL_DIR}" + -ConfPath "${VEEAM_INSTALL_DIR}/mold-backup.windows.conf" + ) + + echo "=== setup-veeam-mold-job.ps1 (file-level /tmp/mold/veeam + Pre/Post) ===" + [[ -n "$KVM_HOSTNAME" ]] && ps_args+=(-AgentHostName "$KVM_HOSTNAME") + [[ -n "$VEEAM_REPO_NAME" ]] && ps_args+=(-RepositoryName "$VEEAM_REPO_NAME") + if [[ "${VEEAM_START_JOBS:-true}" != "false" && "$SKIP_START" != "true" ]]; then + ps_args+=(-StartJob) + fi + + echo "=== Upload remote runner for ${setup_name} ===" + mold_push_write_remote_runner "$setup_script" "${ps_args[@]}" + + # No quotes around -File path: OpenSSH on Windows breaks ''path'' with nested quoting. + echo "=== Remote: ${setup_name} on ${VEEAM_SSH} ===" + if veeam_ssh "$VEEAM_SSH" pwsh -NoProfile -ExecutionPolicy Bypass -File "${runner_path}"; then + return 0 + fi + if veeam_ssh "$VEEAM_SSH" powershell -NoProfile -ExecutionPolicy Bypass -File "${runner_path}"; then + return 0 + fi + die "Veeam job setup failed — check Agent on KVM and mold-backup.windows.conf" +} + +mold_push_run_remote_setup + +echo "" +echo "=== Done ===" +echo "Veeam UI: jobs started automatically (set VEEAM_START_JOBS=false or --no-start to skip)" +echo "FLR→Mold: on KVM run enable-veeam-mold-restore.sh (mold-veeam-restore-agent.timer)" +echo "KVM log: ssh ${KVM_SSH:-root@${KVM_IP}} tail -f /var/log/mold/veeam-hook.log" diff --git a/scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh b/scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh new file mode 100755 index 000000000000..935ff110b449 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh @@ -0,0 +1,174 @@ +#!/usr/bin/bash +# Mold datadisk (KVM) + Veeam host repo (E:\opt1\veeam\) setup. +# +# Architecture (KVM hypervisor unit, e.g. 10.10.31.2 ablecube31-2): +# - Mold qcow2 on KVM data disk (BACKUP_REPO_TYPE=local, no NAS) +# - One Veeam Linux Agent job ON the KVM host → E:\opt1\veeam\\ +# - BACKUP_MODE=host: pre/post on KVM export VM disks; NOT per-guest Veeam jobs +# - Veeam UI Guest files restore (FLR) → KVM restore agent → Mold restoreBackup +# +# Run on KVM host: +# bash setup-datadisk-veeam-backup.sh --env-file /etc/ablestack/veeam/mold-backup.env +# bash setup-datadisk-veeam-backup.sh --datadisk-path /data/backup --kvm-hostname ablecube31-2 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +ENV_FILE="${ETC_DIR}/mold-backup.env" +DATADISK_PATH="" +KVM_HOSTNAME_OVERRIDE="" +SKIP_VEEAM_REPO=false +REGISTER_SCRIPTS_ONLY=false + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="$2"; shift 2 ;; + --datadisk-path) DATADISK_PATH="$2"; shift 2 ;; + --kvm-hostname) KVM_HOSTNAME_OVERRIDE="$2"; shift 2 ;; + --skip-veeam-repo) SKIP_VEEAM_REPO=true; shift ;; + --register-scripts-only) REGISTER_SCRIPTS_ONLY=true; shift ;; + -h|--help) + sed -n '2,18p' "$0" + exit 0 + ;; + *) die "Unknown: $1" ;; + esac +done + +[[ -f "$ENV_FILE" ]] || die "Missing $ENV_FILE" +# shellcheck source=/dev/null +set -a && source "$ENV_FILE" && set +a + +COMMON_LIB="${SCRIPT_DIR}/mold-guest-common.sh" +[[ -f "$COMMON_LIB" ]] || COMMON_LIB="${ETC_DIR}/mold-guest-common.sh" +[[ -f "$COMMON_LIB" ]] || die "mold-guest-common.sh not found" +MOLD_GUEST_SCRIPT_DIR="$SCRIPT_DIR" +# shellcheck source=mold-guest-common.sh +source "$COMMON_LIB" + +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +host="${KVM_HOSTNAME_OVERRIDE:-${KVM_HOSTNAME:-$(hostname -s)}}" +kvm_ip="$(mold_guest_resolve_kvm_ip_from_env "$ENV_FILE" 2>/dev/null || hostname -I 2>/dev/null | awk '{print $1}')" +disk="${DATADISK_PATH:-${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS:-/data/backup}}}" +# Legacy GFS bind mount — datadisk mode uses KVM data disk, not glue-gfs NAS path. +if [[ -z "$DATADISK_PATH" && "$disk" == *glue-gfs* ]]; then + if [[ -d /data/backup ]]; then + echo "WARN: GFS path ${disk} → /data/backup (datadisk mode; no NAS restore)" + disk="/data/backup" + else + echo "WARN: BACKUP_REPO still on glue-gfs (${disk}). Create /data/backup and re-run:" + echo " $0 --datadisk-path /data/backup --env-file ${ENV_FILE}" + fi +fi +veeam_root="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" +repo_name="${VEEAM_REPO_NAME:-Mold ${host}}" +job_name="${VEEAM_JOB_NAME:-Mold ${host}}" +host_conf="${ETC_DIR}/Mold_Host_Backup.conf" + +echo "=== Mold datadisk + Veeam host repo setup ===" +echo " KVM host : ${host} (${kvm_ip})" +echo " Veeam job : ${job_name}" +echo " Datadisk path: ${disk}" +echo " Veeam repo : ${veeam_root}/${host} (name: ${repo_name})" + +mkdir -p "$disk" +chmod 0755 "$disk" 2>/dev/null || true + +mold_host_ensure_conf + +set_kv() { + local k="$1" v="$2" f="$host_conf" + if grep -q "^${k}=" "$f" 2>/dev/null; then + sed -i "s|^${k}=.*|${k}=\"${v}\"|" "$f" + else + echo "${k}=\"${v}\"" >>"$f" + fi +} + +for f in "$host_conf" "${ETC_DIR}/mold-backup.conf"; do + [[ -f "$f" ]] || continue + host_conf="$f" + set_kv BACKUP_STORAGE_MODE datadisk + set_kv BACKUP_REPO_TYPE local + set_kv BACKUP_REPO_PROVIDER localfs + set_kv BACKUP_REPO_NAME "Ablestack Data Disk" + set_kv MOLD_DATADISK_PATH "$disk" + set_kv BACKUP_REPO_ADDRESS "$disk" + set_kv NAS_REPO_MOUNT "" + set_kv BACKUP_MODE host + set_kv KVM_HOSTNAME "$host" + set_kv KVM_HOST "$kvm_ip" + set_kv VEEAM_JOB_NAME "$job_name" + set_kv VEEAM_HOST_BACKUP_PATH "${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + set_kv KVM_PRE_NOTIFY_SCRIPT "${ETC_DIR}/ablestack_veeam_pre_notify.sh" + set_kv KVM_POST_NOTIFY_SCRIPT "${ETC_DIR}/ablestack_veeam_post_notify.sh" + set_kv VEEAM_HOST_REPO_ROOT "$veeam_root" + set_kv VEEAM_REPO_NAME "$repo_name" + set_kv RESTORE_SOURCE mold-only + set_kv VEEAM_UI_RESTORE_SOURCE mold-only + set_kv RESTORE_WATCH_TRIGGER_MOLD true + set_kv BACKUP_STORAGE_ENGINE "${BACKUP_STORAGE_ENGINE:-auto}" +done + +# Keep mold-backup.env aligned (many scripts source env before job conf). +env_sync_kv() { + local k="$1" v="$2" + if grep -q "^${k}=" "$ENV_FILE" 2>/dev/null; then + sed -i "s|^${k}=.*|${k}=\"${v}\"|" "$ENV_FILE" + else + echo "${k}=\"${v}\"" >>"$ENV_FILE" + fi +} +env_sync_kv BACKUP_STORAGE_MODE datadisk +env_sync_kv BACKUP_REPO_TYPE local +env_sync_kv BACKUP_REPO_PROVIDER localfs +env_sync_kv BACKUP_REPO_NAME "Ablestack Data Disk" +env_sync_kv MOLD_DATADISK_PATH "$disk" +env_sync_kv BACKUP_REPO_ADDRESS "$disk" +env_sync_kv BACKUP_MODE host +env_sync_kv RESTORE_SOURCE mold-only +echo "Synced ${ENV_FILE}: BACKUP_REPO_ADDRESS=${disk} (was glue-gfs if unset above)" + +grep -E 'BACKUP_MODE|BACKUP_STORAGE|DATADISK|BACKUP_REPO|VEEAM_JOB|VEEAM_HOST|KVM_HOST|KVM_PRE|KVM_POST|RESTORE_' "$host_conf" || true + +echo "=== KVM pre/post scripts (ablestack_veeam_*_notify.sh) ===" +mold_host_ensure_kvm_prepost_scripts + +if [[ "$REGISTER_SCRIPTS_ONLY" == "true" ]]; then + mold_host_register_veeam_scripts "$ENV_FILE" "$host" "$kvm_ip" +elif [[ "$SKIP_VEEAM_REPO" != "true" ]]; then + mold_host_setup_veeam_job "$ENV_FILE" "$host" "$kvm_ip" +fi + +echo "=== Enable restore agent (Veeam FLR → Mold datadisk restore) ===" +bash "${SCRIPT_DIR}/enable-veeam-mold-restore.sh" --env-file "$ENV_FILE" \ + ${VM_INCLUDE:+--vm-include "$VM_INCLUDE"} 2>/dev/null \ + || { + systemctl daemon-reload 2>/dev/null || true + systemctl enable mold-veeam-restore-agent.timer 2>/dev/null || true + systemctl start mold-veeam-restore-agent.timer 2>/dev/null || true + systemctl is-active mold-veeam-restore-agent.timer 2>/dev/null || echo "(timer not installed — run install.sh)" + } + +cat </dev/null || echo Ablestack Veeam)' to VMs in Mold UI (no Mold backup repository registration) + 2) Set VMs on this host: ${ETC_DIR}/veeam_config.sh --job-name '${job_name}' --backup-mode host --vm-include 'i-2-XX-VM' + 3) FLR→Mold: ${ETC_DIR}/enable-veeam-mold-restore.sh --vm-include 'i-2-XX-VM' + 4) Run backup from Veeam UI (job ${job_name}) or: Start-VBRComputerBackupJob + 5) After backup: cat ${ETC_DIR}/registry/.latest-backup-id + +EOF diff --git a/scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 new file mode 100644 index 000000000000..e192a8d98e24 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 @@ -0,0 +1,91 @@ +# Create / ensure a host-level Veeam backup repository on local disk (E:\opt1\veeam\). +# All guest Agent jobs for one KVM hypervisor share this repo (not per-VM folders on E:). +# +# pwsh -File setup-veeam-host-repo.ps1 -KvmHostname ablecube31-2 +# pwsh -File setup-veeam-host-repo.ps1 -KvmHostname ablecube31-2 -HostRepoRoot "E:\opt1\veeam" + +param( + [Parameter(Mandatory = $true)] + [string]$KvmHostname, + + [string]$HostRepoRoot = "E:\opt1\veeam", + + [string]$RepositoryName = "" +) + +$ErrorActionPreference = "Stop" + +if (-not $RepositoryName) { + $RepositoryName = "Mold $KvmHostname" +} + +$folder = Join-Path $HostRepoRoot $KvmHostname +New-Item -ItemType Directory -Force -Path $folder | Out-Null +Write-Host "Host repo folder: $folder" + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue +try { Connect-VBRServer -Server localhost -ErrorAction SilentlyContinue | Out-Null } catch {} + +function Get-MoldRepoPathString { + param($Repo) + if ($null -eq $Repo) { return "" } + foreach ($key in @("Path", "Folder", "Location")) { + if ($Repo.PSObject.Properties.Name -contains $key -and $Repo.$key) { + return [string]$Repo.$key + } + } + return "" +} + +$repo = Get-VBRBackupRepository -Name $RepositoryName -ErrorAction SilentlyContinue +if ($repo) { + Write-Host "Repository already exists: $RepositoryName" + Write-Output $RepositoryName + exit 0 +} + +$created = $false +$errors = @() + +function Try-AddRepo { + param([scriptblock]$Action, [string]$Label) + try { + $r = & $Action + if ($r) { return $r } + $r = Get-VBRBackupRepository -Name $RepositoryName -ErrorAction SilentlyContinue + if ($r) { return $r } + } catch { + $script:errors += "${Label}: $($_.Exception.Message)" + } + return $null +} + +$repo = Try-AddRepo { Add-VBRBackupRepository -Name $RepositoryName -Folder $folder } "Folder" +if (-not $repo) { + $repo = Try-AddRepo { Add-VBRBackupRepository -Name $RepositoryName -Path $folder } "Path" +} +if (-not $repo) { + $repo = Try-AddRepo { Add-VBRBackupRepository -Name $RepositoryName -Folder $folder -Type WinLocal } "WinLocal" +} + +if (-not $repo) { + $normalized = $folder.TrimEnd('\') + $repo = Get-VBRBackupRepository -ErrorAction SilentlyContinue | Where-Object { + $p = Get-MoldRepoPathString $_ + $p -and ($p -eq $folder -or $p -eq $normalized -or $p -like "*\$KvmHostname") + } | Select-Object -First 1 +} + +if ($repo) { + Write-Host "Using repository: $($repo.Name) -> $folder" + Write-Output $RepositoryName + exit 0 +} + +Write-Error @" +Failed to create Veeam repository '$RepositoryName' at '$folder'. +Create manually in Veeam UI: Backup Infrastructure -> Backup Repositories -> Add -> Windows -> $folder +Errors: +$($errors -join "`n") +"@ +exit 1 diff --git a/scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 new file mode 100644 index 000000000000..22336ddf886e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-shot: KVM hypervisor host backup (Linux Agent on 10.10.31.2). +# - Backup scope: /tmp/mold/veeam (VM disk export staging from Mold pre-script) +# - Pre/Post: Guest Processing on KVM Agent (ablestack_veeam_pre/post_notify.sh) +# +# Run on Veeam B&R server (Administrator, PowerShell 7+): +# .\setup-veeam-mold-job.ps1 +# .\setup-veeam-mold-job.ps1 -ConfPath C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf +# .\setup-veeam-mold-job.ps1 -StartJob +# +# Or from Mac/ccvm: bash push-to-veeam.sh --env-file mold-backup.env + +param( + [string]$ConfPath = "", + [string]$InstallDir = "C:\ProgramData\Mold\backup\veeam", + + [string]$JobName = "", + [string]$KvmHost = "", + [string]$AgentHostName = "", + [string]$BackupPath = "", + [string]$RepositoryName = "", + [string]$ProtectionGroupName = "", + + [switch]$StartJob, + [switch]$WhatIf +) + +$ErrorActionPreference = "Stop" + +if (-not $ConfPath) { + $ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF + if (-not $ConfPath) { + $ConfPath = Join-Path $InstallDir "mold-backup.windows.conf" + } +} + +function Read-MoldWindowsConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { return $cfg } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +$cfg = Read-MoldWindowsConf -Path $ConfPath + +if (-not $JobName) { $JobName = $cfg["VEEAM_JOB_NAME"] } +if (-not $KvmHost) { $KvmHost = $cfg["KVM_HOST"] } +if (-not $BackupPath) { + $BackupPath = $cfg["VEEAM_HOST_BACKUP_PATH"] +} +if (-not $BackupPath) { $BackupPath = "/tmp/mold/veeam" } +if ($BackupPath -match '^[A-Za-z]:[\\/]') { + throw @" +BackupPath must be a Linux path on the KVM agent (e.g. /tmp/mold/veeam), not a Windows path: $BackupPath + +STAGING_PATH in mold-backup.windows.conf is for Windows VM disk export (FLR), not Agent SelectedFiles. +Set VEEAM_HOST_BACKUP_PATH=/tmp/mold/veeam in $ConfPath, or pass -BackupPath '/tmp/mold/veeam'. +"@ +} +if (-not $RepositoryName) { $RepositoryName = $cfg["VEEAM_REPO_NAME"] } +if (-not $AgentHostName) { $AgentHostName = $cfg["KVM_HOSTNAME"] } +if (-not $ProtectionGroupName) { $ProtectionGroupName = $cfg["VEEAM_PROTECTION_GROUP_NAME"] } +if (-not $ProtectionGroupName) { $ProtectionGroupName = "Mold KVM Agents" } + +if (-not $JobName) { throw "JobName required (VEEAM_JOB_NAME in $ConfPath or -JobName)" } +if (-not $KvmHost) { throw "KvmHost required (KVM_HOST in $ConfPath or -KvmHost)" } + +$preScript = $cfg["KVM_PRE_NOTIFY_SCRIPT"] +if (-not $preScript) { $preScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" } +$postScript = $cfg["KVM_POST_NOTIFY_SCRIPT"] +if (-not $postScript) { $postScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" } + +$kvmSshUser = $cfg["KVM_SSH_USER"] +if (-not $kvmSshUser) { $kvmSshUser = "root" } +$kvmSshPassword = $cfg["KVM_SSH_PASSWORD"] +if (-not $kvmSshPassword) { $kvmSshPassword = $env:MOLD_KVM_SSH_PASSWORD } +if (-not $kvmSshPassword) { + throw @" +KVM_SSH_PASSWORD is required for Linux Agent backup on $KvmHost. + +Veeam fails with: Cannot find credentials for agent $KvmHost + +Set KVM_SSH_PASSWORD in $ConfPath (or MOLD_KVM_SSH_PASSWORD env) and re-run. +"@ +} + +Write-Host "=== Mold Veeam setup (KVM hypervisor host — Agent on $KvmHost) ===" +Write-Host " Job : $JobName" +Write-Host " KVM : $KvmHost (Linux Agent on hypervisor — backs up VM export staging)" +Write-Host " Agent : $AgentHostName" +Write-Host " Scope : $BackupPath (SelectedFiles — VM disks exported here by Mold Pre-script)" +Write-Host " Pre : $preScript" +Write-Host " Post : $postScript" +Write-Host " PG : $ProtectionGroupName" +if ($cfg["VM_NAME"]) { + Write-Host " VM : $($cfg['VM_NAME']) (set on KVM via veeam_config.sh --vm-include)" +} else { + Write-Host " VM : (set on KVM) veeam_config.sh --vm-include 'i-2-7-VM'" +} +Write-Host " Conf : $ConfPath" +Write-Host "" + +$createScript = Join-Path $InstallDir "create-veeam-agent-job.ps1" +if (-not (Test-Path $createScript)) { + $createScript = Join-Path $PSScriptRoot "create-veeam-agent-job.ps1" +} +if (-not (Test-Path $createScript)) { + throw "create-veeam-agent-job.ps1 not found. Run push-to-veeam.sh first." +} + +$createArgs = @{ + JobName = $JobName + KvmHost = $KvmHost + BackupPath = $BackupPath + InstallDir = $InstallDir + AgentPreNotifyScript = $preScript + AgentPostNotifyScript = $postScript +} +if ($AgentHostName) { $createArgs["AgentHostName"] = $AgentHostName } +if ($RepositoryName) { $createArgs["RepositoryName"] = $RepositoryName } +$createArgs["ProtectionGroupName"] = $ProtectionGroupName +$createArgs["KvmSshUser"] = $kvmSshUser +$createArgs["KvmSshPassword"] = $kvmSshPassword +if ($WhatIf) { $createArgs["WhatIf"] = $true } + +& $createScript @createArgs + +if ($WhatIf) { exit 0 } + +if ($StartJob) { + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + $job = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue + if (-not $job) { throw "Job not found after create: $JobName" } + Write-Host "Starting backup job: $JobName" + Start-VBRComputerBackupJob -Job $job | Out-Null + Write-Host "Job started. Check KVM: tail -f /var/log/mold/veeam-hook.log" +} + +Write-Host "" +Write-Host "Done. Veeam backs up VM exports under $BackupPath on KVM hypervisor $KvmHost ($AgentHostName)." +Write-Host "On KVM, set VMs on this host:" +Write-Host " /etc/ablestack/veeam/veeam_config.sh --job-name '$JobName' --backup-mode host --vm-include 'i-2-7-VM'" diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 new file mode 100644 index 000000000000..6a5686553832 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Veeam Backup Job post-script: invoke KVM post-notify (NetBackup bpend equivalent) over SSH. + +$ErrorActionPreference = "Stop" + +$ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF +if (-not $ConfPath) { + $ConfPath = "C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf" +} + +function Read-MoldConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { throw "Config not found: $Path" } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +$cfg = Read-MoldConf -Path $ConfPath +$kvmHost = $cfg["KVM_HOST"] +$kvmUser = $cfg["KVM_SSH_USER"] +$jobName = $cfg["VEEAM_JOB_NAME"] +if (-not $kvmHost -or -not $kvmUser) { + throw "KVM_HOST and KVM_SSH_USER are required in $ConfPath" +} +if (-not $jobName) { throw "VEEAM_JOB_NAME is required in $ConfPath" } + +$postScript = $cfg["KVM_POST_NOTIFY_SCRIPT"] +if (-not $postScript) { $postScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" } + +$sshKey = $cfg["KVM_SSH_KEY"] +$sshArgs = @() +if ($sshKey) { $sshArgs += @("-i", $sshKey) } + +$remote = "$kvmUser@$kvmHost" +$vmName = $cfg["VM_NAME"] +$vmEnv = "" +if ($vmName) { + $vmEnv = "VM_INCLUDE='$vmName' VM_NAME='$vmName' " +} +Write-Host "Running post-notify on $remote : $postScript (job=$jobName vm=$vmName)" +& ssh @sshArgs $remote "${vmEnv}bash '$postScript' '$(hostname)' '$jobName'" +if ($LASTEXITCODE -ne 0) { throw "post-notify failed with exit code $LASTEXITCODE" } + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 new file mode 100644 index 000000000000..df756f86150f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 @@ -0,0 +1,59 @@ +# Veeam post-restore hook: push restore event to KVM restore agent over SSH. +# The KVM agent checks VM ownership + flock before calling Mold restoreBackup. + +$ErrorActionPreference = "Stop" + +$ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF +if (-not $ConfPath) { + $ConfPath = "C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf" +} + +function Read-MoldConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { throw "Config not found: $Path" } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +$cfg = Read-MoldConf -Path $ConfPath +$kvmHost = $cfg["KVM_HOST"] +$kvmUser = $cfg["KVM_SSH_USER"] +$jobName = $cfg["VEEAM_JOB_NAME"] +$backupId = $cfg["BACKUP_ID"] +$vmName = $cfg["VM_NAME"] +if (-not $kvmHost -or -not $kvmUser) { + throw "KVM_HOST and KVM_SSH_USER are required in $ConfPath" +} +if (-not $vmName) { + throw "VM_NAME is required in $ConfPath for post-restore event" +} + +$eventScript = $cfg["KVM_RESTORE_EVENT_SCRIPT"] +if (-not $eventScript) { $eventScript = "/etc/ablestack/veeam/ablestack_veeam_restore_event.sh" } + +$sessionId = $env:VEEAM_RESTORE_SESSION_ID +if (-not $sessionId) { + $sessionId = "veeam-ps1-$(Get-Date -Format 'yyyyMMddHHmmss')" +} + +$sshKey = $cfg["KVM_SSH_KEY"] +$sshArgs = @() +if ($sshKey) { $sshArgs += @("-i", $sshKey) } + +$remote = "$kvmUser@$kvmHost" +$restoreSource = $cfg["VEEAM_UI_RESTORE_SOURCE"] +if (-not $restoreSource) { $restoreSource = "mold-only" } +$backupEnv = "" +if ($backupId) { $backupEnv = "BACKUP_ID='$backupId' " } +Write-Host "Pushing restore event to $remote (vm=$vmName session=$sessionId source=$restoreSource)" +& ssh @sshArgs $remote "${backupEnv}RESTORE_SOURCE='$restoreSource' VEEAM_JOB_NAME='$jobName' VM_NAME='$vmName' bash '$eventScript' veeam.restore.completed '$sessionId' '$vmName' source=veeam-ui-flr" +if ($LASTEXITCODE -ne 0) { throw "restore-event failed with exit code $LASTEXITCODE" } + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 b/scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 new file mode 100644 index 000000000000..473ae98a6420 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Veeam Backup Job pre-script: record restore point id; optionally export disks to staging. +# FileLevel Agent backups: no VMDK export — KVM seed uses live libvirt disks (VEEAM_BACKUP_MODE=filelevel). +# Configure: C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf + +$ErrorActionPreference = "Stop" + +if ($PSVersionTable.PSVersion.Major -lt 7) { + throw "Veeam.Backup.PowerShell requires PowerShell 7+. Run: pwsh -File $PSCommandPath" +} + +$ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF +if (-not $ConfPath) { + $ConfPath = "C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf" +} + +function Read-MoldConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { + throw "Config not found: $Path" + } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +function Write-MoldConfValue { + param([string]$Path, [string]$Key, [string]$Value) + $content = Get-Content $Path -ErrorAction SilentlyContinue + if (-not $content) { $content = @() } + $found = $false + $newContent = foreach ($line in $content) { + if ($line -match "^\s*$([regex]::Escape($Key))\s*=") { + $found = $true + "$Key=`"$Value`"" + } else { $line } + } + if (-not $found) { $newContent += "$Key=`"$Value`"" } + Set-Content -Path $Path -Value $newContent -Encoding UTF8 +} + +$cfg = Read-MoldConf -Path $ConfPath +$vmName = $cfg["VM_NAME"] +if (-not $vmName) { $vmName = $env:VEEAM_VM_NAME } + +$jobName = $cfg["VEEAM_JOB_NAME"] +if (-not $jobName) { $jobName = $env:VEEAM_JOB_NAME } + +$staging = $cfg["STAGING_PATH"] +if (-not $staging) { $staging = $cfg["VEEAM_HOST_BACKUP_PATH"] } +if (-not $staging) { $staging = "/tmp/mold/veeam" } + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + +# NetBackup-style: multi-VM job — pre-notify only (no VMDK export) +if ($cfg["VEEAM_BACKUP_MODE"] -eq "filelevel" -and $cfg["KVM_HOST"] -and $cfg["KVM_SSH_USER"] -and $jobName -and -not $vmName) { + $preScript = $cfg["KVM_PRE_NOTIFY_SCRIPT"] + if (-not $preScript) { $preScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" } + $sshKey = $cfg["KVM_SSH_KEY"] + $sshArgs = @() + if ($sshKey) { $sshArgs += @("-i", $sshKey) } + $remote = "$($cfg['KVM_SSH_USER'])@$($cfg['KVM_HOST'])" + Write-Host "Running pre-notify (multi-VM) on $remote : $preScript (job=$jobName)" + & ssh @sshArgs $remote "bash '$preScript' '$(hostname)' '$jobName'" + if ($LASTEXITCODE -ne 0) { throw "pre-notify failed with exit code $LASTEXITCODE" } + exit 0 +} + +if (-not $vmName) { throw "VM_NAME is required in $ConfPath or VEEAM_VM_NAME env" } + +$backups = @() +if ($jobName) { + # Agent jobs: Veeam requires a trailing wildcard on backup name. + $backups = @(Get-VBRBackup -Name "${jobName}*" -ErrorAction SilentlyContinue) +} +if ($backups.Count -eq 0) { + $backups = @(Get-VBRBackup | Where-Object { $_.JobType -eq "EpAgentBackup" }) +} +if ($backups.Count -eq 0) { + throw "No Veeam backups found. Set VEEAM_JOB_NAME in $ConfPath (e.g. Agent Backup Job 1)." +} + +$backup = $null +$vm = $null +foreach ($b in $backups) { + $objects = @(Get-VBRBackupObject -Backup $b -ErrorAction SilentlyContinue) + $match = $objects | Where-Object { $_.Name -eq $vmName } | Select-Object -First 1 + if ($match) { + $backup = $b + $vm = $match + break + } +} +if (-not $vm) { + $known = foreach ($b in $backups) { + Get-VBRBackupObject -Backup $b -ErrorAction SilentlyContinue | ForEach-Object { $_.Name } + } + $known = ($known | Sort-Object -Unique) -join ", " + throw "Veeam backup object not found for VM_NAME=$vmName. Known names: $known" +} + +# Start-VBRRestoreVirtualDisks expects COib from Get-VBRRestorePoint (not Get-VBRObjectRestorePoint). +$rp = $backup | Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +if (-not $rp) { + $rp = $vm | Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +} +if (-not $rp) { + $rp = $backup | Get-VBRObjectRestorePoint -Name $vmName | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +} +if (-not $rp) { + $rp = Get-VBRObjectRestorePoint -Backup $backup | + Where-Object { $_.Name -eq $vmName } | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +} +if (-not $rp) { throw "No restore point for VM: $vmName (backup: $($backup.Name))" } + +$rpId = $rp.Id +if ($rpId -is [guid]) { $rpId = $rpId.Guid } + +$rpOib = $backup | Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Where-Object { $_.Id -eq $rpId -or $_.Id.Guid -eq $rpId } | + Select-Object -First 1 +if (-not $rpOib) { + $rpOib = Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Where-Object { $_.Id -eq $rpId -or $_.Id.Guid -eq $rpId } | + Select-Object -First 1 +} +if (-not $rpOib) { $rpOib = $rp } + +function Test-MoldVeeamFileLevelRestorePoint { + param($RestorePoint) + $text = $RestorePoint | Format-List * -Force | Out-String + if ($text -match 'BackupMode\s*:\s*FileLevel') { return $true } + if ($text -match 'COibAuxDataLinuxBackup') { return $true } + if ($text -match 'ItemType\s*:\s*LinuxPhysicalDisk' -and $text -match 'IncludePaths') { return $true } + return $false +} + +$backupMode = $cfg["VEEAM_BACKUP_MODE"] +if (-not $backupMode) { $backupMode = "auto" } +$isFileLevel = ($backupMode -eq "filelevel") -or ( + $backupMode -eq "auto" -and (Test-MoldVeeamFileLevelRestorePoint -RestorePoint $rpOib) +) + +if ($isFileLevel) { + Write-Host "VEEAM_BACKUP_MODE=filelevel: NetBackup-style pre-notify on KVM (host path /tmp/mold/veeam)." + Write-MoldConfValue -Path $ConfPath -Key "VEEAM_RESTORE_POINT_ID" -Value $rpId + Write-Host "VEEAM_RESTORE_POINT_ID=$rpId written to $ConfPath" + if ($cfg["KVM_HOST"] -and $cfg["KVM_SSH_USER"] -and $jobName) { + $preScript = $cfg["KVM_PRE_NOTIFY_SCRIPT"] + if (-not $preScript) { $preScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" } + $sshKey = $cfg["KVM_SSH_KEY"] + $sshArgs = @() + if ($sshKey) { $sshArgs += @("-i", $sshKey) } + $remote = "$($cfg['KVM_SSH_USER'])@$($cfg['KVM_HOST'])" + Write-Host "Running pre-notify on $remote : $preScript (job=$jobName)" + & ssh @sshArgs $remote "bash '$preScript' '$(hostname)' '$jobName'" + if ($LASTEXITCODE -ne 0) { throw "pre-notify failed with exit code $LASTEXITCODE" } + } + exit 0 +} + +New-Item -ItemType Directory -Force -Path $staging | Out-Null +Get-ChildItem -Path $staging -File -Recurse -Include *.vmdk,*.vhd,*.vhdx -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue + +function Resolve-MoldVeeamExportServer { + param([hashtable]$Cfg) + $configured = $Cfg["STAGING_EXPORT_SERVER"] + if ($configured) { + $s = Get-VBRServer -Type Windows -Name $configured -ErrorAction SilentlyContinue + if ($s) { return $s } + } + $names = @($env:COMPUTERNAME) + if ($env:USERDNSDOMAIN) { + $names += "$($env:COMPUTERNAME).$($env:USERDNSDOMAIN)" + } + foreach ($n in $names) { + $s = Get-VBRServer -Type Windows -Name $n -ErrorAction SilentlyContinue + if ($s) { return $s } + } + $localhost = Get-VBRLocalhost -ErrorAction SilentlyContinue + if ($localhost) { return $localhost } + Get-VBRServer -ErrorAction SilentlyContinue | Select-Object -First 1 +} + +function Export-MoldVeeamDisksToStaging { + param( + [Parameter(Mandatory = $true)]$RestorePoint, + [Parameter(Mandatory = $true)][string]$StagingPath, + [hashtable]$Cfg = @{} + ) + if (Get-Command Start-VBRFLRSession -ErrorAction SilentlyContinue) { + $session = Start-VBRFLRSession -RestorePoint $RestorePoint + try { + $items = Get-VBRFLRItem -Session $session | Where-Object { $_.Type -eq "HardDisk" } + foreach ($item in $items) { + $dest = Join-Path $StagingPath ($item.Name + ".vmdk") + Copy-VBRFLRItem -FLRSession $session -Item $item -Destination $dest + Write-Host "Exported $($item.Name) -> $dest" + } + } finally { + Stop-VBRFLRSession -Session $session + } + return + } + if (Get-Command Start-VBRRestoreVirtualDisks -ErrorAction SilentlyContinue) { + $server = Resolve-MoldVeeamExportServer -Cfg $Cfg + if (-not $server) { + throw "No Windows managed server for export. Add $($env:COMPUTERNAME) under Backup Infrastructure > Managed Servers, or set STAGING_EXPORT_SERVER in mold-backup.windows.conf" + } + if (-not (Test-Path -LiteralPath $StagingPath)) { + New-Item -ItemType Directory -Force -Path $StagingPath | Out-Null + } + Write-Host "Exporting disks via Start-VBRRestoreVirtualDisks -> $StagingPath (server: $($server.Name))" + try { + Start-VBRRestoreVirtualDisks -RestorePoint $RestorePoint -Server $server ` + -Path $StagingPath -RestoreDiskType Vmdk | Out-Null + } catch { + throw "Start-VBRRestoreVirtualDisks failed on server '$($server.Name)': $_. Ensure the folder exists, Veeam service account has write access, and the server is a Managed Windows server in Veeam console." + } + Get-ChildItem -Path $StagingPath -Recurse -Include *.vmdk,*.vhd,*.vhdx -ErrorAction SilentlyContinue | + ForEach-Object { Write-Host "Exported $($_.FullName)" } + return + } + throw "No supported Veeam export cmdlet (Start-VBRFLRSession / Start-VBRRestoreVirtualDisks)" +} + +Export-MoldVeeamDisksToStaging -RestorePoint $rpOib -StagingPath $staging -Cfg $cfg +$exported = Get-ChildItem -Path $staging -Recurse -Include *.vmdk,*.vhd,*.vhdx -ErrorAction SilentlyContinue +if (-not $exported) { + throw "No virtual disk files exported under $staging" +} + +Write-MoldConfValue -Path $ConfPath -Key "VEEAM_RESTORE_POINT_ID" -Value $rpId +Write-Host "VEEAM_RESTORE_POINT_ID=$rpId written to $ConfPath" + +# Optional: push restore point id to KVM host config over SSH +if ($cfg["KVM_HOST"] -and $cfg["KVM_SSH_USER"]) { + $kvmConf = $cfg["KVM_MOLD_BACKUP_CONF"] + if (-not $kvmConf) { $kvmConf = "/etc/mold/backup/veeam/mold-backup.conf" } + $sshKey = $cfg["KVM_SSH_KEY"] + $sshArgs = @() + if ($sshKey) { $sshArgs += @("-i", $sshKey) } + $remote = "$($cfg['KVM_SSH_USER'])@$($cfg['KVM_HOST'])" + $cmd = "grep -q '^VEEAM_RESTORE_POINT_ID=' '$kvmConf' 2>/dev/null && sed -i 's|^VEEAM_RESTORE_POINT_ID=.*|VEEAM_RESTORE_POINT_ID=`"$rpId`"|' '$kvmConf' || echo 'VEEAM_RESTORE_POINT_ID=`"$rpId`"' >> '$kvmConf'" + & ssh @sshArgs $remote $cmd + Write-Host "Updated VEEAM_RESTORE_POINT_ID on KVM host $remote" +} + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam_config.sh b/scripts/vm/hypervisor/kvm/veeam/veeam_config.sh new file mode 100755 index 000000000000..5ad5d8aaab8b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam_config.sh @@ -0,0 +1,620 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup-style Veeam + Mold host configuration (veeam_config.sh). +# Creates /etc/ablestack/veeam/.conf, encrypts API secret, optional Mold global settings. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" +SECRET_SCRIPT="${SCRIPT_DIR}/mold-backup-secret.sh" +DEFAULT_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + +JOB_NAME="" +BACKUP_OFFERING_NAME="VeeamBackup" +VM_INCLUDE="" +VM_EXCLUDE="" +MAX_CHAIN="7" +MOLD_API_URL="" +MOLD_API_KEY="" +MOLD_API_SECRET="" +ZONE_ID="" +RETENTION_PERIOD="P7D" +VEEAM_URL="" +VEEAM_USERNAME="" +VEEAM_PASSWORD="" +BACKUP_MODE="host" +IMPORT_MODE="auto" + +VEEAM_SSH_HOST="" +VEEAM_SSH_USER="administrator" +VEEAM_SSH_KEY="/root/.ssh/veeam_id_rsa" +RESTORE_SOURCE="auto" +KVM_HOST="" +KVM_SSH_USER="root" +KVM_SSH_KEY="" +KVM_SSH_PASSWORD="" +VM_NAME_CFG="" +VM_UUID_CFG="" +BACKUP_REPO_TYPE="local" +BACKUP_REPO_NAME="Ablestack Data Disk" +BACKUP_REPO_ADDRESS="" +BACKUP_REPO_MOUNT_OPTS="" +BACKUP_REPO_PROVIDER="localfs" +NAS_REPO_MOUNT="" +BACKUP_STORAGE_MODE="datadisk" +MOLD_DATADISK_PATH="/data/backup" +VEEAM_HOST_REPO_ROOT="E:/opt1/veeam" +ENCRYPT_SECRET="true" +SECRET_KEY_FILE="" +VEEAM_BACKUP_TARGET="" +VM_TARGETS="" +VEEAM_GUEST_JOB_PREFIX="Mold VM" +GUEST_VM_SSH_USER="" +GUEST_VM_SSH_PASSWORD="" +RUN_INSTALL="false" +CONFIGURE_MOLD="true" +ENV_FILE="" +DEFAULT_ENV_FILE="${ETC_DIR}/mold-backup.env" +MOLD_API_SECRET_ENC_FILE_CANDIDATE="" + +usage() { + cat <<'EOF' +Usage: veeam_config.sh [options] + +Required: + --job-name NAME Veeam backup job / policy name (conf file name) + +Auto-filled (when omitted) from, in order: + 1) CLI options + 2) --env-file or /etc/ablestack/veeam/mold-backup.env + 3) Existing /etc/ablestack/veeam/*.conf + 4) secrets/secret.enc + /root/.ssh/ablestack.key + 5) Mold API listZones / listBackupRepositories + 6) /etc/cloudstack/agent/agent.properties (Mold API URL) + +Common optional overrides: + --offering-name NAME Mold backup offering name (default: VeeamBackup) + --mold-url URL Mold API URL (http://:8080/client/api) + --api-key KEY Mold API key + --api-secret SECRET Mold API secret + --zone-id UUID Zone for importBackupOffering + --nas-repo ADDR NAS repo address (host:/export; nfs:// prefix optional) + --env-file PATH Env file (default: /etc/ablestack/veeam/mold-backup.env) + +Other optional: + --vm-include LIST Comma-separated libvirt names (* = all running, default) + --vm-exclude LIST Comma-separated libvirt names to skip + --vm-name NAME libvirt name (mold-backup.sh status 등) + --vm-uuid UUID Mold VM UUID + --max-chain N Max incremental chain (default: 7) + --retention PERIOD Backup offering retention (default: P7D) + --backup-mode MODE host|api|local|auto (default: host = NetBackup-style /tmp/mold/veeam) + --backup-target MODE host only (guest mode removed) + --veeam-url URL Mold zone setting backup.plugin.ablestack-veeam.url + --veeam-user USER Mold zone setting backup.plugin.ablestack-veeam.username + --veeam-password PASS Mold zone setting backup.plugin.ablestack-veeam.password + --nas-repo ADDR NAS repo address (host:/export; nfs:// prefix optional) + --repo-name NAME Mold backup repository name (default: Ablestack Veeam NAS) + --repo-type TYPE nfs|cifs (default: nfs) + --repo-mount-opts OPTS Mount options for addBackupRepository + --kvm-host IP KVM host for Veeam post-job SSH + --kvm-ssh-user USER (default: root) + --kvm-ssh-key PATH SSH private key + --no-configure-mold Skip listConfigurations/updateConfiguration calls + --no-encrypt-secret Store API secret in plain text (not recommended) + --secret-key-file PATH Passphrase file (default: /root/.ssh/ablestack.key) + --env-file PATH Credentials/env file (default: /etc/ablestack/veeam/mold-backup.env) + --install Run install.sh after generating configs + -h, --help + +Example (minimal — reads mold-backup.env + existing conf): + veeam_config.sh --job-name "Mold KVM Backup" --kvm-host 10.10.31.2 + +Example (explicit): + veeam_config.sh \ + --job-name "VeeamBackup" \ + --offering-name "VeeamBackup" \ + --mold-url http://10.10.31.20:8080/client/api \ + --api-key KEY --api-secret 'SECRET' \ + --zone-id \ + --vm-include "i-2-3-VM,i-2-7-VM" \ + --max-chain 7 \ + --veeam-url https://veeam:9398/api/ \ + --kvm-host 10.10.31.30 \ + --install +EOF +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +veeam_set_if_empty() { + local name="$1" value="$2" + [[ -z "${value}" ]] && return 0 + [[ -n "${!name:-}" ]] && return 0 + printf -v "$name" '%s' "$value" +} + +veeam_config_read_conf_var() { + local file="$1" key="$2" line val + [[ -f "$file" ]] || return 1 + line="$(grep -E "^${key}=" "$file" 2>/dev/null | tail -1)" || return 1 + val="${line#*=}" + val="${val%$'\r'}" + if [[ "$val" == \"*\" ]]; then + val="${val#\"}"; val="${val%\"}" + elif [[ "$val" == \'*\' ]]; then + val="${val#\'}"; val="${val%\'}" + fi + [[ -n "$val" ]] || return 1 + echo "$val" +} + +veeam_config_import_env_file() { + local f="$1" line key val + [[ -f "$f" ]] || return 0 + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%%#*}" + line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -z "$line" || "$line" != *"="* ]] && continue + key="${line%%=*}" + key="$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + val="${line#*=}" + val="$(echo "$val" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ "$val" == \"*\" ]]; then val="${val#\"}"; val="${val%\"}"; fi + if [[ "$val" == \'*\' ]]; then val="${val#\'}"; val="${val%\'}"; fi + case "$key" in + VEEAM_USER) [[ -n "$val" ]] && VEEAM_USERNAME="$val" ;; + KVM_IP) veeam_set_if_empty KVM_HOST "$val" ;; + VEEAM_JOB_NAME) veeam_set_if_empty JOB_NAME "$val" ;; + MOLD_API_URL) [[ -n "$val" ]] && MOLD_API_URL="$val" ;; + MOLD_API_KEY) [[ -n "$val" ]] && MOLD_API_KEY="$val" ;; + MOLD_API_SECRET) [[ -n "$val" ]] && MOLD_API_SECRET="$val" ;; + ZONE_ID) [[ -n "$val" ]] && ZONE_ID="$val" ;; + JOB_NAME) veeam_set_if_empty JOB_NAME "$val" ;; + BACKUP_OFFERING_NAME) veeam_set_if_empty BACKUP_OFFERING_NAME "$val" ;; + VM_INCLUDE) veeam_set_if_empty VM_INCLUDE "$val" ;; + VM_EXCLUDE) veeam_set_if_empty VM_EXCLUDE "$val" ;; + VM_NAME) veeam_set_if_empty VM_NAME_CFG "$val" ;; + VM_UUID) veeam_set_if_empty VM_UUID_CFG "$val" ;; + MAX_CHAIN) [[ -n "$val" ]] && MAX_CHAIN="$val" ;; + RETENTION_PERIOD) [[ -n "$val" ]] && RETENTION_PERIOD="$val" ;; + BACKUP_MODE) veeam_set_if_empty BACKUP_MODE "$val" ;; + IMPORT_MODE) [[ -n "$val" ]] && IMPORT_MODE="$val" ;; + VEEAM_URL) [[ -n "$val" ]] && VEEAM_URL="$val" ;; + VEEAM_USERNAME) [[ -n "$val" ]] && VEEAM_USERNAME="$val" ;; + VEEAM_PASSWORD) [[ -n "$val" ]] && VEEAM_PASSWORD="$val" ;; + VEEAM_SSH_HOST) [[ -n "$val" ]] && VEEAM_SSH_HOST="$val" ;; + VEEAM_SSH_USER) [[ -n "$val" ]] && VEEAM_SSH_USER="$val" ;; + VEEAM_SSH_KEY) [[ -n "$val" ]] && VEEAM_SSH_KEY="$val" ;; + KVM_HOST) veeam_set_if_empty KVM_HOST "$val" ;; + KVM_SSH_USER) [[ -n "$val" ]] && KVM_SSH_USER="$val" ;; + KVM_SSH_KEY) [[ -n "$val" ]] && KVM_SSH_KEY="$val" ;; + KVM_SSH_PASSWORD) [[ -n "$val" ]] && KVM_SSH_PASSWORD="$val" ;; + BACKUP_REPO_ADDRESS) [[ -n "$val" ]] && BACKUP_REPO_ADDRESS="$val" ;; + BACKUP_REPO_NAME) [[ -n "$val" ]] && BACKUP_REPO_NAME="$val" ;; + BACKUP_REPO_TYPE) [[ -n "$val" ]] && BACKUP_REPO_TYPE="$val" ;; + BACKUP_REPO_MOUNT_OPTS) [[ -n "$val" ]] && BACKUP_REPO_MOUNT_OPTS="$val" ;; + NAS_REPO_MOUNT) [[ -n "$val" ]] && NAS_REPO_MOUNT="$val" ;; + VEEAM_BACKUP_TARGET) veeam_set_if_empty VEEAM_BACKUP_TARGET "$val" ;; + VM_TARGETS) [[ -n "$val" ]] && VM_TARGETS="$val" ;; + VEEAM_GUEST_JOB_PREFIX) [[ -n "$val" ]] && VEEAM_GUEST_JOB_PREFIX="$val" ;; + GUEST_VM_SSH_USER) [[ -n "$val" ]] && GUEST_VM_SSH_USER="$val" ;; + GUEST_VM_SSH_PASSWORD) [[ -n "$val" ]] && GUEST_VM_SSH_PASSWORD="$val" ;; + esac + done < "$f" +} + +veeam_config_load_from_conf() { + local file="$1" v + [[ -f "$file" ]] || return 0 + veeam_set_if_empty MOLD_API_URL "$(veeam_config_read_conf_var "$file" MOLD_API_URL || true)" + veeam_set_if_empty MOLD_API_KEY "$(veeam_config_read_conf_var "$file" MOLD_API_KEY || true)" + veeam_set_if_empty MOLD_API_SECRET "$(veeam_config_read_conf_var "$file" MOLD_API_SECRET || true)" + v="$(veeam_config_read_conf_var "$file" MOLD_API_SECRET_ENC_FILE || true)" + [[ -n "$v" ]] && MOLD_API_SECRET_ENC_FILE_CANDIDATE="$v" + veeam_set_if_empty SECRET_KEY_FILE "$(veeam_config_read_conf_var "$file" MOLD_SECRET_KEY_FILE || true)" + veeam_set_if_empty ZONE_ID "$(veeam_config_read_conf_var "$file" ZONE_ID || true)" + veeam_set_if_empty BACKUP_OFFERING_NAME "$(veeam_config_read_conf_var "$file" BACKUP_OFFERING_NAME || true)" + veeam_set_if_empty VM_INCLUDE "$(veeam_config_read_conf_var "$file" VM_INCLUDE || true)" + veeam_set_if_empty VM_EXCLUDE "$(veeam_config_read_conf_var "$file" VM_EXCLUDE || true)" + veeam_set_if_empty VM_NAME_CFG "$(veeam_config_read_conf_var "$file" VM_NAME || true)" + veeam_set_if_empty VM_UUID_CFG "$(veeam_config_read_conf_var "$file" VM_UUID || true)" + veeam_set_if_empty MAX_CHAIN "$(veeam_config_read_conf_var "$file" VEEAM_MAX_CHAIN || true)" + veeam_set_if_empty RETENTION_PERIOD "$(veeam_config_read_conf_var "$file" RETENTION_PERIOD || true)" + veeam_set_if_empty VEEAM_URL "$(veeam_config_read_conf_var "$file" VEEAM_URL || true)" + veeam_set_if_empty VEEAM_USERNAME "$(veeam_config_read_conf_var "$file" VEEAM_USERNAME || true)" + veeam_set_if_empty VEEAM_PASSWORD "$(veeam_config_read_conf_var "$file" VEEAM_PASSWORD || true)" + veeam_set_if_empty VEEAM_SSH_HOST "$(veeam_config_read_conf_var "$file" VEEAM_SSH_HOST || true)" + veeam_set_if_empty BACKUP_REPO_ADDRESS "$(veeam_config_read_conf_var "$file" BACKUP_REPO_ADDRESS || true)" + veeam_set_if_empty BACKUP_REPO_NAME "$(veeam_config_read_conf_var "$file" BACKUP_REPO_NAME || true)" + veeam_set_if_empty BACKUP_REPO_TYPE "$(veeam_config_read_conf_var "$file" BACKUP_REPO_TYPE || true)" + veeam_set_if_empty BACKUP_REPO_MOUNT_OPTS "$(veeam_config_read_conf_var "$file" BACKUP_REPO_MOUNT_OPTS || true)" + veeam_set_if_empty NAS_REPO_MOUNT "$(veeam_config_read_conf_var "$file" NAS_REPO_MOUNT || true)" +} + +veeam_config_auto_mold_url() { + local props host + [[ -n "$MOLD_API_URL" ]] && return 0 + for props in /etc/cloudstack/agent/agent.properties /etc/cloudstack/agent/agent.properties.override; do + [[ -f "$props" ]] || continue + host="$(grep -E '^[[:space:]]*host[[:space:]]*=' "$props" 2>/dev/null | tail -1 | cut -d= -f2- | tr -d ' \r')" + [[ -n "$host" ]] || continue + if [[ "$host" == http* ]]; then + MOLD_API_URL="${host%/}/client/api" + else + MOLD_API_URL="http://${host}:8080/client/api" + fi + return 0 + done +} + +veeam_config_resolve_api_secret() { + local enc key + [[ -n "$MOLD_API_SECRET" ]] && return 0 + enc="${MOLD_API_SECRET_ENC_FILE_CANDIDATE:-${ETC_DIR}/secrets/secret.enc}" + key="${SECRET_KEY_FILE:-$DEFAULT_SECRET_KEY_FILE}" + [[ -f "$enc" && -f "$key" && -x "$SECRET_SCRIPT" ]] || return 0 + MOLD_API_SECRET="$("$SECRET_SCRIPT" decrypt --enc-file "$enc" --key-file "$key" 2>/dev/null || true)" +} + +veeam_config_autofill_defaults() { + local env_path conf_path safe_job f + + for env_path in \ + "${ENV_FILE}" \ + "${DEFAULT_ENV_FILE}" \ + "${SCRIPT_DIR}/mold-backup.env" \ + "${SHARE_DIR}/mold-backup.env"; do + [[ -n "$env_path" && -f "$env_path" ]] || continue + veeam_config_import_env_file "$env_path" + done + + safe_job="" + if [[ -n "$JOB_NAME" ]]; then + safe_job="$(echo "$JOB_NAME" | tr ' /' '__')" + veeam_config_load_from_conf "${ETC_DIR}/${safe_job}.conf" + veeam_config_load_from_conf "${ETC_DIR}/${JOB_NAME}.conf" + fi + veeam_config_load_from_conf "${ETC_DIR}/mold-backup.conf" + + if [[ -z "$MOLD_API_KEY" || -z "$MOLD_API_SECRET" || -z "$ZONE_ID" || -z "$BACKUP_REPO_ADDRESS" ]]; then + for conf_path in "${ETC_DIR}"/*.conf; do + [[ -f "$conf_path" ]] || continue + [[ "$conf_path" == *mold-backup.windows.conf ]] && continue + veeam_config_load_from_conf "$conf_path" + done + fi + + veeam_config_resolve_api_secret + veeam_config_auto_mold_url + [[ -n "$KVM_HOST" ]] || KVM_HOST="$(hostname -I 2>/dev/null | awk '{print $1}')" + + if [[ -n "$MOLD_API_URL" && -n "$MOLD_API_KEY" && -n "$MOLD_API_SECRET" ]]; then + # shellcheck source=mold-backup.lib.sh + source "${SCRIPT_DIR}/mold-backup.lib.sh" + export MOLD_API_URL MOLD_API_KEY MOLD_API_SECRET ZONE_ID BACKUP_REPO_NAME BACKUP_REPO_ADDRESS + if [[ -z "$ZONE_ID" ]]; then + ZONE_ID="$(mold_backup_api_first_zone_id 2>/dev/null || true)" + fi + if [[ -z "$BACKUP_REPO_ADDRESS" ]]; then + if [[ "${BACKUP_STORAGE_MODE:-datadisk}" == "datadisk" ]]; then + BACKUP_REPO_ADDRESS="${MOLD_DATADISK_PATH:-/data/backup}" + else + BACKUP_REPO_ADDRESS="$(mold_backup_api_first_repo_address "${BACKUP_REPO_NAME}" 2>/dev/null || true)" + fi + fi + fi +} + +veeam_config_print_autofill_summary() { + echo "Configuration sources:" + [[ -f "${ENV_FILE:-$DEFAULT_ENV_FILE}" ]] && echo " env: ${ENV_FILE:-$DEFAULT_ENV_FILE}" + echo " API URL: ${MOLD_API_URL:-}" + [[ -n "${MOLD_API_KEY:-}" ]] && echo " API key: ${MOLD_API_KEY:0:12}..." + [[ -n "${MOLD_API_SECRET:-}" ]] && echo " API secret: (loaded)" + echo " Zone: ${ZONE_ID:-}" + echo " NAS repo: ${BACKUP_REPO_ADDRESS:-}" + echo " KVM host: ${KVM_HOST:-}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --job-name) JOB_NAME="$2"; shift 2 ;; + --offering-name) BACKUP_OFFERING_NAME="$2"; shift 2 ;; + --vm-include) VM_INCLUDE="$2"; shift 2 ;; + --vm-exclude) VM_EXCLUDE="$2"; shift 2 ;; + --vm-name) VM_NAME_CFG="$2"; shift 2 ;; + --vm-uuid) VM_UUID_CFG="$2"; shift 2 ;; + --max-chain) MAX_CHAIN="$2"; shift 2 ;; + --mold-url) MOLD_API_URL="$2"; shift 2 ;; + --api-key) MOLD_API_KEY="$2"; shift 2 ;; + --api-secret) MOLD_API_SECRET="$2"; shift 2 ;; + --zone-id) ZONE_ID="$2"; shift 2 ;; + --retention) RETENTION_PERIOD="$2"; shift 2 ;; + --backup-mode) BACKUP_MODE="$2"; shift 2 ;; + --backup-target) VEEAM_BACKUP_TARGET="$2"; shift 2 ;; + --vm-targets) die "Guest-mode --vm-targets removed. Use --vm-include with host/datadisk mode." ;; + --guest-job-prefix) die "Guest-mode --guest-job-prefix removed." ;; + --veeam-url) VEEAM_URL="$2"; shift 2 ;; + --veeam-user) VEEAM_USERNAME="$2"; shift 2 ;; + --veeam-password) VEEAM_PASSWORD="$2"; shift 2 ;; + --veeam-ssh-host) VEEAM_SSH_HOST="$2"; shift 2 ;; + --veeam-ssh-user) VEEAM_SSH_USER="$2"; shift 2 ;; + --veeam-ssh-key) VEEAM_SSH_KEY="$2"; shift 2 ;; + --nas-repo) BACKUP_REPO_ADDRESS="$2"; shift 2 ;; + --repo-name) BACKUP_REPO_NAME="$2"; shift 2 ;; + --repo-type) BACKUP_REPO_TYPE="$2"; shift 2 ;; + --repo-mount-opts) BACKUP_REPO_MOUNT_OPTS="$2"; shift 2 ;; + --nas-repo-mount) NAS_REPO_MOUNT="$2"; shift 2 ;; + --kvm-host) KVM_HOST="$2"; shift 2 ;; + --kvm-ssh-user) KVM_SSH_USER="$2"; shift 2 ;; + --kvm-ssh-key) KVM_SSH_KEY="$2"; shift 2 ;; + --no-configure-mold) CONFIGURE_MOLD="false"; shift ;; + --no-encrypt-secret) ENCRYPT_SECRET="false"; shift ;; + --secret-key-file) SECRET_KEY_FILE="$2"; shift 2 ;; + --env-file) ENV_FILE="$2"; shift 2 ;; + --install) RUN_INSTALL="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) die "Unknown option: $1" ;; + esac +done + +veeam_config_autofill_defaults + +if [[ "${VEEAM_BACKUP_TARGET:-host}" == "guest" || "${BACKUP_MODE:-}" =~ ^(guest|veeam-guest)$ ]]; then + die "Guest-mode Veeam jobs were removed. Use --backup-target host / --backup-mode host (datadisk)." +fi + +# Host mode (default): one Veeam Agent job on this KVM hypervisor (e.g. 10.10.31.2). +if [[ -z "$JOB_NAME" ]]; then + JOB_NAME="Mold ${KVM_HOSTNAME:-$(hostname -s)}" + BACKUP_MODE="host" +fi + +if [[ -z "$JOB_NAME" ]]; then + die "--job-name is required. Edit ${ENV_FILE:-$DEFAULT_ENV_FILE} and set JOB_NAME=... (or VEEAM_JOB_NAME=...), or pass --job-name 'Mold Guest Backup'. Shell variables are not read unless exported and listed in --env-file." +fi +[[ -n "$MOLD_API_URL" && -n "$MOLD_API_KEY" && -n "$MOLD_API_SECRET" ]] \ + || die "API credentials missing — set MOLD_API_KEY/MOLD_API_SECRET in ${DEFAULT_ENV_FILE} or pass --api-key/--api-secret" +[[ -n "$ZONE_ID" ]] \ + || die "zone-id missing — set ZONE_ID in ${DEFAULT_ENV_FILE}, pass --zone-id, or ensure listZones API is reachable" + +veeam_config_print_autofill_summary +echo "" + +if [[ -n "$BACKUP_REPO_ADDRESS" ]]; then + if [[ "$BACKUP_REPO_ADDRESS" == nfs://* ]]; then + BACKUP_REPO_TYPE="nfs" + elif [[ "$BACKUP_REPO_ADDRESS" == cifs://* ]]; then + BACKUP_REPO_TYPE="cifs" + fi + BACKUP_REPO_ADDRESS="${BACKUP_REPO_ADDRESS#nfs://}" + BACKUP_REPO_ADDRESS="${BACKUP_REPO_ADDRESS#cifs://}" +fi + +# Datadisk mode: Mold API may still point at glue-gfs; KVM backups go to /data/backup. +if [[ "${BACKUP_STORAGE_MODE:-datadisk}" == "datadisk" && "${BACKUP_REPO_ADDRESS:-}" == *glue-gfs* ]]; then + if [[ -d /data/backup ]]; then + echo "WARN: API/GFS repo ${BACKUP_REPO_ADDRESS} → /data/backup (datadisk mode; no NAS restore)" + BACKUP_REPO_ADDRESS="/data/backup" + MOLD_DATADISK_PATH="/data/backup" + fi +fi + +install -d -m 0700 "${ETC_DIR}" "${ETC_DIR}/secrets" "${ETC_DIR}/state" "${ETC_DIR}/registry" + +ENC_FILE="${ETC_DIR}/secrets/secret.enc" +MOLD_API_SECRET_LINE="" +MOLD_API_SECRET_ENC_FILE_LINE="" +MOLD_SECRET_KEY_FILE_LINE="" + +if [[ "$ENCRYPT_SECRET" == "true" ]]; then + [[ -x "$SECRET_SCRIPT" ]] || chmod +x "$SECRET_SCRIPT" + KEY_FILE="${SECRET_KEY_FILE:-$DEFAULT_SECRET_KEY_FILE}" + [[ -f "$KEY_FILE" ]] || die "Secret key file not found: $KEY_FILE" + "$SECRET_SCRIPT" encrypt --secret "$MOLD_API_SECRET" --key-file "$KEY_FILE" --out "$ENC_FILE" >/dev/null + MOLD_API_SECRET_LINE='MOLD_API_SECRET=""' + MOLD_API_SECRET_ENC_FILE_LINE="MOLD_API_SECRET_ENC_FILE=\"${ENC_FILE}\"" + MOLD_SECRET_KEY_FILE_LINE="MOLD_SECRET_KEY_FILE=\"${KEY_FILE}\"" +else + MOLD_API_SECRET_LINE="MOLD_API_SECRET=\"${MOLD_API_SECRET}\"" + MOLD_API_SECRET_ENC_FILE_LINE='MOLD_API_SECRET_ENC_FILE=""' + MOLD_SECRET_KEY_FILE_LINE='MOLD_SECRET_KEY_FILE=""' +fi + +safe_job="$(echo "$JOB_NAME" | tr ' /' '__')" +CONF="${ETC_DIR}/${safe_job}.conf" +cat > "$CONF" < start matching Veeam Agent job. +VEEAM_TRIGGER_ENABLED="${VEEAM_TRIGGER_ENABLED:-$([[ "${BACKUP_MODE}" == guest ]] && echo true || echo false)}" +VEEAM_TRIGGER_TTL="${VEEAM_TRIGGER_TTL:-1800}" +VEEAM_TRIGGER_METHOD="${VEEAM_TRIGGER_METHOD:-$([[ "${BACKUP_MODE}" == guest ]] && echo ssh || echo auto)}" +VEEAM_API_URL="${VEEAM_API_URL:-}" +VEEAM_API_VERSION="${VEEAM_API_VERSION:-1.2-rev0}" +VEEAM_API_USER="${VEEAM_API_USER:-${VEEAM_USERNAME:-}}" +VEEAM_API_PASSWORD="${VEEAM_API_PASSWORD:-${VEEAM_PASSWORD:-}}" +VM_TARGETS="${VM_TARGETS:-}" +VEEAM_GUEST_JOB_PREFIX="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + +VEEAM_HOST_BACKUP_PATH="/tmp/mold/veeam" +STAGING_PATH="/tmp/mold/veeam" +VEEAM_BACKUP_MODE="filelevel" +BACKUP_MODE="${BACKUP_MODE}" +IMPORT_MODE="${IMPORT_MODE}" + +BACKUP_REPO_TYPE="${BACKUP_REPO_TYPE}" +BACKUP_REPO_NAME="${BACKUP_REPO_NAME}" +BACKUP_REPO_ADDRESS="${BACKUP_REPO_ADDRESS}" +BACKUP_REPO_MOUNT_OPTS="${BACKUP_REPO_MOUNT_OPTS}" +BACKUP_REPO_PROVIDER="${BACKUP_REPO_PROVIDER}" +NAS_REPO_MOUNT="${NAS_REPO_MOUNT}" +BACKUP_STORAGE_MODE="${BACKUP_STORAGE_MODE:-datadisk}" +MOLD_DATADISK_PATH="${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS}}" +VEEAM_HOST_REPO_ROOT="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" +VEEAM_REPO_NAME="${VEEAM_REPO_NAME:-Mold ${KVM_HOSTNAME:-}}" + +BACKUP_ID="" +VM_UUID="${VM_UUID_CFG}" +VM_NAME="${VM_NAME_CFG}" + +CLEANUP_STAGING_AFTER_BACKUP="true" +CLEANUP_STAGING_ON_ERROR="true" + +NAS_BACKUP_SCRIPT="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh" +CVT_BACKUP_SCRIPT="/etc/ablestack/veeam/ablestack_cvtbackup.sh" +LOG_FILE="/var/log/mold/veeam-hook.log" +LOG_TAG="mold-veeam-hook" +EOF +chmod 0600 "$CONF" + +# Job-specific hook wrappers (NetBackup bpstart_notify. pattern) +HOOKS_DIR="${ETC_DIR}/hooks" +install -d -m 0755 "${HOOKS_DIR}" +for hook in pre post restore; do + target="${HOOKS_DIR}/${hook}-notify.${safe_job}" + cat > "$target" < "$target" < "$WIN_CONF" < "$MANIFEST" < '${ETC_DIR}/ablestack_veeam_pre_notify.sh' \$(hostname) '${JOB_NAME}' +# Post: ssh root@ '${ETC_DIR}/ablestack_veeam_post_notify.sh' \$(hostname) '${JOB_NAME}' + +# Or on Veeam server (VM file-level job — NOT entire host): +# pwsh -File setup-veeam-mold-job.ps1 -ConfPath mold-backup.windows.conf +# Or from Mac/ccvm: +# bash push-to-veeam.sh --env-file mold-backup.env +EOF + +if [[ "$CONFIGURE_MOLD" == "true" ]]; then + # shellcheck source=mold-backup.lib.sh + source "${SCRIPT_DIR}/mold-backup.lib.sh" + export MOLD_BACKUP_CONF="$CONF" + export VEEAM_JOB_NAME="$JOB_NAME" + mold_backup_load_config || true + if mold_backup_cmk_bin >/dev/null 2>&1 || command -v curl >/dev/null 2>&1; then + mold_backup_api_ensure_global_settings || true + if mold_backup_is_datadisk_mode; then + mold_backup_notify_log info "datadisk mode: skipping addBackupRepository/importBackupOffering (assign offering in Mold UI)" + else + mold_backup_api_ensure_backup_resources >/dev/null \ + || mold_backup_notify_log warn "addBackupRepository/importBackupOffering skipped or failed (Admin API key + BACKUP_REPO_ADDRESS + ZONE_ID required)" + fi + fi +fi + +echo "Created:" +echo " ${CONF}" +echo " ${WIN_CONF}" +echo " ${MANIFEST}" +echo " ${HOOKS_DIR}/pre-notify.${safe_job}" +echo "" +if [[ "${VEEAM_BACKUP_TARGET:-host}" == "guest" ]]; then + die "Guest-mode Veeam jobs were removed. Use --backup-target host (datadisk) instead." +else + echo "Veeam job backup selections: ${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}/" + echo "Next: bash push-to-veeam.sh (host Agent on KVM)" + echo "FLR→Mold: bash ${ETC_DIR}/enable-veeam-mold-restore.sh" +fi + +if [[ "$RUN_INSTALL" == "true" && -x "${SCRIPT_DIR}/install.sh" ]]; then + ABLESTACK_VEEAM_ETC_DIR="$ETC_DIR" bash "${SCRIPT_DIR}/install.sh" +fi + +echo "Done." From f9b4d784850b25abd8ad1a3175ff7207d5603c92 Mon Sep 17 00:00:00 2001 From: jeong da-young Date: Tue, 21 Jul 2026 10:48:06 +0900 Subject: [PATCH 12/12] fix AblestackNas getLatestBackedUpBackup overload for Veeam seed check Restore one-arg helper used by hasBackedUpSeed after NetBackup merge changed the method to require backupScheduleId. Co-authored-by: Cursor --- .../backup/AblestackNasBackupProvider.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java index f71a101172ff..0f719c8beae9 100644 --- a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java +++ b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java @@ -566,6 +566,18 @@ private String getCheckpointPath(String backupPath, String checkpointName, Strin return String.format("%s/checkpoints/%s.xml", backupPath, checkpointName); } + private BackupVO getLatestBackedUpBackup(VirtualMachine vm) { + List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), vm.getBackupOfferingId()); + return backups.stream() + .filter(BackupVO.class::isInstance) + .map(BackupVO.class::cast) + .filter(backup -> Backup.Status.BackedUp.equals(backup.getStatus())) + .peek(backupDao::loadDetails) + .filter(backup -> getBackupDetail(backup, DETAIL_CHECKPOINT_NAME) != null) + .max(Comparator.comparing(BackupVO::getDate)) + .orElse(null); + } + private BackupVO getLatestBackedUpBackup(VirtualMachine vm, Long backupScheduleId) { List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), vm.getBackupOfferingId()); return backups.stream()