diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index fb4b083198d0..f0e07626bf39 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -534,6 +534,10 @@ public class ApiConstants { public static final String QUALIFIERS = "qualifiers"; public static final String QUERY_FILTER = "queryfilter"; public static final String QUIESCE_VM = "quiescevm"; + public static final String STAGING_DISK_PATHS = "stagingdiskpaths"; + public static final String VEEAM_RESTORE_POINT_ID = "veeamrestorepointid"; + public static final String SOURCE_DISK_FORMAT = "sourcediskformat"; + public static final String BOOTSTRAP_CHECKPOINT = "bootstrapcheckpoint"; public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java new file mode 100644 index 000000000000..c1632bbb30ea --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateAblestackVeeamBackupCmd.java @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "createAblestackVeeamBackup", + description = "Create an incremental NAS backup for a VM assigned to the ablestack-veeam offering " + + "(typically after a Veeam job completes)", + responseObject = SuccessResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class CreateAblestackVeeamBackupCmd extends BaseAsyncCreateCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance ID") + private Long vmId; + + @Parameter(name = ApiConstants.QUIESCE_VM, + type = CommandType.BOOLEAN, + required = false, + description = "Quiesce VM via QEMU guest agent before backup") + private Boolean quiesceVM; + + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup (default: VM hostname + ISO-8601 timestamp)", + since = "4.22.0.0") + private String name; + + public Long getVmId() { + return vmId; + } + + public Boolean getQuiesceVM() { + return quiesceVM; + } + + public String getName() { + return name; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + boolean result = backupManager.createAblestackVeeamBackup(this, getJob()); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new CloudRuntimeException("Failed to create Ablestack Veeam backup"); + } + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Backup; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Creating Ablestack Veeam NAS backup for Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); + } + + @Override + public void create() throws ResourceAllocationException { + } + + @Override + public Long getEntityId() { + return vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java new file mode 100644 index 000000000000..90eda81f51ee --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ImportAblestackVeeamBackupSeedCmd.java @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; + +@APICommand(name = "importAblestackVeeamBackupSeed", + description = "Import a Veeam restore point as a NAS backup seed for Ablestack Veeam incremental KVM backups", + responseObject = BackupResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class ImportAblestackVeeamBackupSeedCmd extends BaseAsyncCreateCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance to register the seed backup for") + private Long vmId; + + @Parameter(name = ApiConstants.VEEAM_RESTORE_POINT_ID, + type = CommandType.STRING, + required = false, + description = "Veeam restore point ID. If omitted, stagingdiskpaths must be provided.") + private String veeamRestorePointId; + + @Parameter(name = ApiConstants.STAGING_DISK_PATHS, + type = CommandType.STRING, + required = false, + description = "Comma-separated disk file paths on the KVM host (from shared staging). " + + "If omitted, disks are exported from Veeam using veeamrestorepointid.") + private String stagingDiskPaths; + + @Parameter(name = ApiConstants.SOURCE_DISK_FORMAT, + type = CommandType.STRING, + required = false, + description = "Staging disk format: vmdk, flat, qcow2, or raw. Default: vmdk") + private String sourceDiskFormat; + + @Parameter(name = ApiConstants.BOOTSTRAP_CHECKPOINT, + type = CommandType.BOOLEAN, + required = false, + description = "Create libvirt checkpoint on the KVM VM after import. Default: true") + private Boolean bootstrapCheckpoint; + + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup (default: VM hostname + ISO-8601 timestamp)", + since = "4.22.0.0") + private String name; + + public Long getVmId() { + return vmId; + } + + public String getVeeamRestorePointId() { + return veeamRestorePointId; + } + + public String getStagingDiskPaths() { + return stagingDiskPaths; + } + + public String getSourceDiskFormat() { + return sourceDiskFormat; + } + + public Boolean getBootstrapCheckpoint() { + return bootstrapCheckpoint; + } + + public String getName() { + return name; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + Backup backup = backupManager.importAblestackVeeamBackupSeed(this); + BackupResponse response = backupManager.createBackupResponse(backup, false); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Backup; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_CREATE; + } + + @Override + public String getEventDescription() { + return "Importing Ablestack Veeam backup seed for Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); + } + + @Override + public void create() throws ResourceAllocationException { + } + + @Override + public Long getEntityId() { + return vmId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java new file mode 100644 index 000000000000..2f31289adb1d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListAblestackVeeamBackupsCmd.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.Pair; + +@APICommand(name = "listAblestackVeeamBackups", + description = "List NAS backups for a VM using the ablestack-veeam backup offering", + responseObject = BackupResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListAblestackVeeamBackupsCmd extends BaseListCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance ID") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + Pair, Integer> result = backupManager.listAblestackVeeamBackups(this); + ListResponse response = new ListResponse<>(); + List backupResponses = new ArrayList<>(); + for (Backup backup : result.first()) { + backupResponses.add(backupManager.createBackupResponse(backup, false)); + } + response.setResponses(backupResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java new file mode 100644 index 000000000000..b7b0cc6e686b --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListVeeamRestorePointsCmd.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRestorePointResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; + +@APICommand(name = "listVeeamRestorePoints", + description = "List Veeam B&R restore points for a KVM instance using the ablestack-veeam backup offering", + responseObject = BackupRestorePointResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class ListVeeamRestorePointsCmd extends BaseCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + required = true, + description = "KVM instance UUID") + private Long vmId; + + public Long getVmId() { + return vmId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + List points = backupManager.listVeeamRestorePoints(this); + ListResponse response = new ListResponse<>(); + List list = backupManager.createVeeamRestorePointResponses(points); + response.setResponses(list); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java new file mode 100644 index 000000000000..4e58344583b6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreAblestackVeeamBackupCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "restoreAblestackVeeamBackup", + description = "Restore a VM from an ablestack-veeam NAS backup entry", + responseObject = SuccessResponse.class, + since = "4.22.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin}) +public class RestoreAblestackVeeamBackupCmd extends BaseAsyncCmd { + + @Inject + private BackupManager backupManager; + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "Backup ID") + private Long backupId; + + public Long getBackupId() { + return backupId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + boolean result = backupManager.restoreAblestackVeeamBackup(backupId); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new CloudRuntimeException("Failed to restore Ablestack Veeam backup"); + } + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_BACKUP_RESTORE; + } + + @Override + public String getEventDescription() { + return "Restoring Instance from Ablestack Veeam backup ID: " + getResourceUuid(ApiConstants.ID); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 0a4241be270f..290cc8e97640 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -26,8 +26,13 @@ import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateNetBackupCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateAblestackVeeamBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateNetBackupCmd; +import org.apache.cloudstack.api.command.user.backup.ImportAblestackVeeamBackupSeedCmd; +import org.apache.cloudstack.api.command.user.backup.ListAblestackVeeamBackupsCmd; +import org.apache.cloudstack.api.command.user.backup.ListVeeamRestorePointsCmd; +import org.apache.cloudstack.api.response.BackupRestorePointResponse; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; @@ -251,6 +256,21 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllocationException; + /** + * Import a Veeam restore point as NAS seed for Ablestack Veeam incremental backups. + */ + Backup importAblestackVeeamBackupSeed(ImportAblestackVeeamBackupSeedCmd cmd) throws ResourceAllocationException; + + List listVeeamRestorePoints(ListVeeamRestorePointsCmd cmd); + + List createVeeamRestorePointResponses(List points); + + boolean createAblestackVeeamBackup(CreateAblestackVeeamBackupCmd cmd, Object job) throws ResourceAllocationException; + + boolean restoreAblestackVeeamBackup(Long backupId); + + Pair, Integer> listAblestackVeeamBackups(ListAblestackVeeamBackupsCmd cmd); + /** * List existing backups for a VM */ diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 98255212f285..15ec3b56a274 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -91,6 +91,14 @@ default String getCatalogBackupTime(Long zoneId, String backupId) { return null; } + /** + * Import a Veeam restore point as a NAS backup seed (Ablestack Veeam provider only). + */ + default Pair importAblestackVeeamBackupSeed(VirtualMachine vm, String veeamRestorePointId, + List stagingDiskPaths, String sourceDiskFormat, Boolean bootstrapCheckpoint) { + throw new UnsupportedOperationException("Provider " + getName() + " does not support Veeam seed import"); + } + /** * Delete an existing backup * @param backup The backup to exclude diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java index f34b89432edd..ab84c661835d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProviderNameUtils.java @@ -25,6 +25,7 @@ public final class BackupProviderNameUtils { public static final String ABLESTACK_NAS = "ablestack-nas"; public static final String ABLESTACK_COMMVAULT = "ablestack-commvault"; public static final String ABLESTACK_NETBACKUP = "ablestack-netbackup"; + public static final String ABLESTACK_VEEAM = "ablestack-veeam"; private BackupProviderNameUtils() { } @@ -42,6 +43,9 @@ public static String canonicalize(final String providerName) { if (NETBACKUP.equalsIgnoreCase(providerName) || ABLESTACK_NETBACKUP.equalsIgnoreCase(providerName)) { return ABLESTACK_NETBACKUP; } + if ("veeam".equalsIgnoreCase(providerName) || ABLESTACK_VEEAM.equalsIgnoreCase(providerName)) { + return ABLESTACK_VEEAM; + } return providerName; } @@ -58,6 +62,9 @@ public static String toDisplayName(final String providerName) { if (ABLESTACK_NETBACKUP.equalsIgnoreCase(providerName) || NETBACKUP.equalsIgnoreCase(providerName)) { return NETBACKUP; } + if (ABLESTACK_VEEAM.equalsIgnoreCase(providerName) || "veeam".equalsIgnoreCase(providerName)) { + return "veeam"; + } return providerName; } @@ -72,4 +79,8 @@ public static boolean isCommvaultFamily(final String providerName) { public static boolean isNetBackupFamily(final String providerName) { return ABLESTACK_NETBACKUP.equalsIgnoreCase(canonicalize(providerName)); } + + public static boolean isVeeamFamily(final String providerName) { + return ABLESTACK_VEEAM.equalsIgnoreCase(canonicalize(providerName)); + } } diff --git a/client/pom.xml b/client/pom.xml index e9d550e5ecc2..066d1c5c8f52 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -637,6 +637,11 @@ cloud-plugin-backup-ablestack-netbackup ${project.version} + + org.apache.cloudstack + cloud-plugin-backup-ablestack-veeam + ${project.version} + org.apache.cloudstack cloud-plugin-backup-bx diff --git a/core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java b/core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java new file mode 100644 index 000000000000..cdfff2f475ab --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/AblestackNasImportVeeamSeedCommand.java @@ -0,0 +1,158 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; + +import java.util.List; + +public class AblestackNasImportVeeamSeedCommand extends Command { + private String vmName; + private String backupPath; + private String checkpointName; + private String backupRepoType; + private String backupRepoAddress; + private List volumePools; + private List volumePaths; + private List backupFiles; + private List stagingDiskPaths; + private String sourceFormat; + private String veeamRestorePointId; + private Boolean bootstrapCheckpoint; + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + + public AblestackNasImportVeeamSeedCommand(String vmName, String backupPath) { + super(); + this.vmName = vmName; + this.backupPath = backupPath; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public String getCheckpointName() { + return checkpointName; + } + + public void setCheckpointName(String checkpointName) { + this.checkpointName = checkpointName; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public List getVolumePools() { + return volumePools; + } + + public void setVolumePools(List volumePools) { + this.volumePools = volumePools; + } + + public List getVolumePaths() { + return volumePaths; + } + + public void setVolumePaths(List volumePaths) { + this.volumePaths = volumePaths; + } + + public List getBackupFiles() { + return backupFiles; + } + + public void setBackupFiles(List backupFiles) { + this.backupFiles = backupFiles; + } + + public List getStagingDiskPaths() { + return stagingDiskPaths; + } + + public void setStagingDiskPaths(List stagingDiskPaths) { + this.stagingDiskPaths = stagingDiskPaths; + } + + public String getSourceFormat() { + return sourceFormat; + } + + public void setSourceFormat(String sourceFormat) { + this.sourceFormat = sourceFormat; + } + + public String getVeeamRestorePointId() { + return veeamRestorePointId; + } + + public void setVeeamRestorePointId(String veeamRestorePointId) { + this.veeamRestorePointId = veeamRestorePointId; + } + + public Boolean getBootstrapCheckpoint() { + return bootstrapCheckpoint; + } + + public void setBootstrapCheckpoint(Boolean bootstrapCheckpoint) { + this.bootstrapCheckpoint = bootstrapCheckpoint; + } + + public String getMountOptions() { + return mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java index eda1a8d5ab55..0f719c8beae9 100644 --- a/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java +++ b/plugins/backup/ablestack-nas/src/main/java/org/apache/cloudstack/backup/AblestackNasBackupProvider.java @@ -111,6 +111,9 @@ public class AblestackNasBackupProvider extends AdapterBase implements BackupPro private static final String DETAIL_FALLBACK_VOLUME_UUIDS = "nas.fallback.volume.uuids"; private static final String DETAIL_FAILURE_PHASE = "nas.failure.phase"; private static final String DETAIL_FAILURE_REASON = "nas.failure.reason"; + public static final String DETAIL_BACKUP_SOURCE = "nas.backup.source"; + public static final String DETAIL_VEEAM_RESTORE_POINT_ID = "nas.veeam.restore.point.id"; + public static final String DETAIL_VEEAM_IMPORTED = "nas.veeam.imported"; private static final String MISSING_PARENT_RBD_SNAPSHOT_ERROR = "Parent RBD snapshot"; private static final String MISSING_PARENT_QCOW2_BITMAP_ERROR = "Parent qcow2 bitmap"; private static final long STALE_BACKUP_THRESHOLD_MS = TimeUnit.DAYS.toMillis(1); @@ -288,6 +291,81 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce return new Pair<>(result.success, result.backup); } + /** + * Import Veeam-exported disks as a FULL NAS backup seed for subsequent incremental backups. + */ + public Pair importVeeamBackupSeed(final VirtualMachine vm, final List stagingDiskPaths, + final String veeamRestorePointId, final String sourceFormat, final Boolean bootstrapCheckpoint) { + if (CollectionUtils.isEmpty(stagingDiskPaths)) { + throw new CloudRuntimeException("Staging disk paths are required to import a Veeam backup seed"); + } + final Host host = getVMHypervisorHostForBackup(vm); + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException("No valid backup repository found for the VM"); + } + + validateNoKvmFileBasedVmSnapshots(vm); + List vmVolumes = volumeDao.findByInstance(vm.getId()); + vmVolumes.sort(Comparator.comparing(Volume::getDeviceId)); + Pair, List> volumePoolsAndPaths = getVolumePoolsAndPaths(vmVolumes); + validateVolumePoolTypes(volumePoolsAndPaths.first()); + + final String backupPath = buildBackupPath(vm); + final String checkpointName = backupPath.substring(backupPath.lastIndexOf("/") + 1); + final String backupEngine = areAllVolumesOnRbdPool(volumePoolsAndPaths.first()) ? BACKUP_ENGINE_RBD_DIFF : BACKUP_ENGINE_QCOW2; + final List backupFiles = buildBackupFileNames(vmVolumes, backupEngine, false); + + BackupVO backupVO = createBackupObject(vm, backupPath, BACKUP_TYPE_FULL, checkpointName, backupEngine, null, volumePoolsAndPaths.second()); + updateBackupDetail(backupVO, DETAIL_BACKUP_SOURCE, "ablestack-veeam"); + updateBackupDetail(backupVO, DETAIL_VEEAM_IMPORTED, "true"); + if (StringUtils.isNotBlank(veeamRestorePointId)) { + updateBackupDetail(backupVO, DETAIL_VEEAM_RESTORE_POINT_ID, veeamRestorePointId); + } + + AblestackNasImportVeeamSeedCommand command = new AblestackNasImportVeeamSeedCommand(vm.getInstanceName(), backupPath); + command.setCheckpointName(checkpointName); + command.setBackupFiles(backupFiles); + command.setVolumePools(volumePoolsAndPaths.first()); + command.setVolumePaths(volumePoolsAndPaths.second()); + command.setStagingDiskPaths(stagingDiskPaths); + command.setSourceFormat(StringUtils.defaultIfBlank(sourceFormat, "vmdk")); + command.setVeeamRestorePointId(veeamRestorePointId); + command.setBootstrapCheckpoint(bootstrapCheckpoint == null || bootstrapCheckpoint); + command.setBackupRepoType(backupRepository.getType()); + command.setBackupRepoAddress(backupRepository.getAddress()); + command.setMountOptions(backupRepository.getMountOptions()); + + BackupAnswer answer; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), command); + } catch (AgentUnavailableException e) { + removeBackupWithDetails(backupVO.getId()); + throw new CloudRuntimeException("Unable to contact hypervisor host to import Veeam backup seed"); + } catch (OperationTimedoutException e) { + removeBackupWithDetails(backupVO.getId()); + throw new CloudRuntimeException("Timed out importing Veeam backup seed, please try again"); + } + + if (answer != null && answer.getResult()) { + backupVO.setDate(new Date()); + backupVO.setSize(answer.getSize()); + backupVO.setStatus(Backup.Status.BackedUp); + backupVO.setBackedUpVolumes(createVolumeInfoFromVolumes(vmVolumes, backupFiles)); + backupDao.update(backupVO.getId(), backupVO); + return new Pair<>(true, backupVO); + } + + final String details = answer != null ? answer.getDetails() : "No answer received"; + logger.error("Failed to import Veeam backup seed for VM {}: {}", vm.getInstanceName(), details); + removeBackupWithDetails(backupVO.getId()); + throw new CloudRuntimeException("Failed to import Veeam backup seed: " + details); + } + + public boolean hasBackedUpSeed(final VirtualMachine vm) { + return getLatestBackedUpBackup(vm) != null; + } + private BackupExecutionResult executeBackup(VirtualMachine vm, Boolean quiesceVM, Host host, BackupRepository backupRepository, List vmVolumes, Pair, List> volumePoolsAndPaths, Backup parentBackup, boolean incrementalBackup, boolean retryAsFullOnFailure) { @@ -488,6 +566,18 @@ private String getCheckpointPath(String backupPath, String checkpointName, Strin return String.format("%s/checkpoints/%s.xml", backupPath, checkpointName); } + private BackupVO getLatestBackedUpBackup(VirtualMachine vm) { + List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), vm.getBackupOfferingId()); + return backups.stream() + .filter(BackupVO.class::isInstance) + .map(BackupVO.class::cast) + .filter(backup -> Backup.Status.BackedUp.equals(backup.getStatus())) + .peek(backupDao::loadDetails) + .filter(backup -> getBackupDetail(backup, DETAIL_CHECKPOINT_NAME) != null) + .max(Comparator.comparing(BackupVO::getDate)) + .orElse(null); + } + private BackupVO getLatestBackedUpBackup(VirtualMachine vm, Long backupScheduleId) { List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), vm.getBackupOfferingId()); return backups.stream() @@ -503,30 +593,57 @@ private BackupVO getLatestBackedUpBackup(VirtualMachine vm, Long backupScheduleI private boolean shouldUseIncrementalBackup(VirtualMachine vm, Backup latestBackup, List vmVolumes, Long backupScheduleId) { if (latestBackup == null) { + LOG.debug("NAS backup for VM [{}] will be FULL: no previous BackedUp backup.", vm.getInstanceName()); + return false; + } + loadBackupDetailsIfNeeded(latestBackup); + + if (Boolean.parseBoolean(getBackupDetail(latestBackup, DETAIL_CHAIN_SEALED))) { + LOG.info("NAS backup for VM [{}] will be FULL: backup chain [{}] is sealed ({})", + vm.getInstanceName(), latestBackup.getUuid(), getBackupDetail(latestBackup, DETAIL_CHAIN_SEAL_REASON)); + return false; + } + + if (backupScheduleId != null && !hasBackedUpBackupForSchedule(backupScheduleId)) { + LOG.debug("NAS backup for VM [{}] will be FULL: no BackedUp backup for schedule [{}].", vm.getInstanceName(), backupScheduleId); return false; } final Long clusterId = getClusterIdFromRootVolume(vm); if (clusterId == null) { - LOG.debug("Unable to resolve cluster for VM [{}], fallback to full backup.", vm); + LOG.info("NAS backup for VM [{}] will be FULL: unable to resolve cluster from root volume.", vm.getInstanceName()); return false; } - if (!KvmIncrementalBackup.valueIn(clusterId)) { + if (!Boolean.TRUE.equals(KvmIncrementalBackup.valueIn(clusterId))) { + LOG.info("NAS backup for VM [{}] will be FULL: kvm.incremental.backup is disabled for cluster [{}] " + + "(set cluster configuration kvm.incremental.backup=true or run veeam_config.sh / pre-notify).", + vm.getInstanceName(), clusterId); return false; } if (!hasHealthyIncrementalSource(latestBackup)) { markVolumeFallbackAndSeal(latestBackup, "unhealthy-chain"); + LOG.info("NAS backup for VM [{}] will be FULL: latest backup [{}] has an unhealthy volume chain.", + vm.getInstanceName(), latestBackup.getUuid()); return false; } if (getBackupChainSize(vm, latestBackup) >= BackupChainSize.value()) { sealBackupChain(latestBackup, "chain-size-limit"); + LOG.info("NAS backup for VM [{}] will be FULL: incremental chain size limit reached for backup [{}].", + vm.getInstanceName(), latestBackup.getUuid()); return false; } + LOG.info("NAS incremental backup selected for VM [{}] using parent backup [{}].", + vm.getInstanceName(), latestBackup.getUuid()); return true; } + private boolean hasBackedUpBackupForSchedule(Long backupScheduleId) { + return backupDao.listBySchedule(backupScheduleId).stream() + .anyMatch(backup -> Backup.Status.BackedUp.equals(backup.getStatus())); + } + private int getBackupChainSize(VirtualMachine vm, Backup latestBackup) { List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), vm.getBackupOfferingId()).stream() .filter(BackupVO.class::isInstance) @@ -575,7 +692,7 @@ private void sealBackupChain(Backup backup, String reason) { updateBackupDetail(backup, DETAIL_CHAIN_SEAL_REASON, reason); } - private void updateBackupDetail(Backup backup, String key, String value) { + public void updateBackupDetail(Backup backup, String key, String value) { if (backup == null || StringUtils.isBlank(key)) { return; } diff --git a/plugins/backup/ablestack-veeam/pom.xml b/plugins/backup/ablestack-veeam/pom.xml new file mode 100644 index 000000000000..06f19313539f --- /dev/null +++ b/plugins/backup/ablestack-veeam/pom.xml @@ -0,0 +1,48 @@ + + + 4.0.0 + cloud-plugin-backup-ablestack-veeam + Ablestack Plugin - Veeam + NAS KVM Backup Plugin + + cloudstack-plugins + org.apache.cloudstack + 4.22.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-plugin-backup-ablestack-nas + ${project.version} + + + org.apache.cloudstack + cloud-plugin-backup-veeam + ${project.version} + + + org.apache.commons + commons-lang3 + ${cs.commons-lang3.version} + + + diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java new file mode 100644 index 000000000000..a9f7a7d62645 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupOffering.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; + +public class AblestackVeeamBackupOffering implements BackupOffering { + + private final String name; + private final String uid; + + public AblestackVeeamBackupOffering(String name, String uid) { + this.name = name; + this.uid = uid; + } + + @Override + public String getExternalId() { + return uid; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "Ablestack Veeam + NAS KVM Backup Offering (Job)"; + } + + @Override + public long getZoneId() { + return -1; + } + + @Override + public boolean isUserDrivenBackupAllowed() { + return true; + } + + @Override + public String getProvider() { + return "ablestack-veeam"; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public String getUuid() { + return uid; + } + + @Override + public long getId() { + return -1; + } + + @Override + public String getRetentionPeriod() { + return null; + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java new file mode 100644 index 000000000000..f271717d92fc --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/AblestackVeeamBackupProvider.java @@ -0,0 +1,514 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamClient; +import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamRestClient; +import org.apache.cloudstack.backup.ablestackveeam.AblestackVeeamSshClient; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; + +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; + +/** + * Ablestack Veeam backup provider for KVM: seeds from Veeam restore points, then incremental NAS backups. + */ +public class AblestackVeeamBackupProvider extends AdapterBase implements BackupProvider, Configurable { + + public static final String PROVIDER_NAME = "ablestack-veeam"; + public static final String DETAIL_VEEAM_RESTORE_POINT_ID = "ablestack.veeam.restore.point.id"; + public static final String DETAIL_VEEAM_VM_NAME = "ablestack.veeam.vm.name"; + + public ConfigKey AblestackVeeamUrl = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.url", "https://localhost:9398/api/", + "The Ablestack Veeam B&R REST API URL.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamVersion = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.version", "0", + "Veeam server major version (0 = auto-detect via PowerShell).", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamUsername = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.username", "administrator", + "Veeam B&R username for Ablestack Veeam plugin.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamPassword = new ConfigKey<>("Secure", String.class, + "backup.plugin.ablestack-veeam.password", "", + "Veeam B&R password for Ablestack Veeam plugin.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamValidateSsl = new ConfigKey<>("Advanced", Boolean.class, + "backup.plugin.ablestack-veeam.validate.ssl", "false", + "Validate SSL when connecting to Veeam REST API.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamApiTimeout = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.request.timeout", "300", + "Veeam API request timeout in seconds.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.restore.timeout", "3600", + "Veeam export/restore operation timeout in seconds.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamTaskPollInterval = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.task.poll.interval", "5", + "Veeam task poll interval in seconds.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + private ConfigKey AblestackVeeamTaskPollMaxRetry = new ConfigKey<>("Advanced", Integer.class, + "backup.plugin.ablestack-veeam.task.poll.max.retry", "240", + "Max retries when polling Veeam tasks.", true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamStagingPath = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.staging.path", "/var/ablestack-veeam-staging", + "Shared staging directory on the Veeam server (must be visible to KVM hosts) for disk export before NAS seed import.", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamUseRestApi = new ConfigKey<>("Advanced", Boolean.class, + "backup.plugin.ablestack-veeam.use.rest.api", "false", + "Use the Veeam Backup & Replication REST API (port 9419) for restore point discovery instead of " + + "PowerShell-over-SSH. Disk export for seed/restore still uses PowerShell (no REST equivalent).", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamRestUrl = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.rest.url", "https://localhost:9419/api/", + "Veeam Backup & Replication REST API base URL (port 9419), used when use.rest.api is true.", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + public ConfigKey AblestackVeeamRestApiVersion = new ConfigKey<>("Advanced", String.class, + "backup.plugin.ablestack-veeam.rest.api.version", "1.3-rev1", + "Veeam Backup & Replication REST API version header (e.g. 1.3-rev1 for v13.0.1).", + true, ConfigKey.Scope.Zone, BackupFrameworkEnabled.key()); + + @Inject + private AblestackNasBackupProvider nasBackupProvider; + + @Inject + private BackupManager backupManager; + + protected AblestackVeeamClient getClient(final Long zoneId) { + try { + return new AblestackVeeamClient( + AblestackVeeamUrl.valueIn(zoneId), + AblestackVeeamVersion.valueIn(zoneId), + AblestackVeeamUsername.valueIn(zoneId), + AblestackVeeamPassword.valueIn(zoneId), + AblestackVeeamValidateSsl.valueIn(zoneId), + AblestackVeeamApiTimeout.valueIn(zoneId), + AblestackVeeamRestoreTimeout.valueIn(zoneId), + AblestackVeeamTaskPollInterval.valueIn(zoneId), + AblestackVeeamTaskPollMaxRetry.valueIn(zoneId)); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Ablestack Veeam API URL: " + e.getMessage()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new CloudRuntimeException("Failed to build Ablestack Veeam client: " + e.getMessage()); + } + } + + protected AblestackVeeamRestClient getRestClient(final Long zoneId) { + try { + return new AblestackVeeamRestClient( + AblestackVeeamRestUrl.valueIn(zoneId), + AblestackVeeamRestApiVersion.valueIn(zoneId), + AblestackVeeamUsername.valueIn(zoneId), + AblestackVeeamPassword.valueIn(zoneId), + AblestackVeeamValidateSsl.valueIn(zoneId), + AblestackVeeamApiTimeout.valueIn(zoneId)); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Ablestack Veeam REST API URL: " + e.getMessage()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new CloudRuntimeException("Failed to build Ablestack Veeam REST client: " + e.getMessage()); + } + } + + /** + * SSH/PowerShell client for disk export. Intentionally does NOT use Enterprise Manager + * (9398) so that REST-API (9419) deployments do not depend on EM at all. The Veeam SSH + * host is derived from the REST URL (falling back to the EM URL); credentials reuse the + * configured Veeam username/password (same as the EM/SSH credentials). + */ + protected AblestackVeeamSshClient getSshClient(final Long zoneId) { + String host = extractHost(AblestackVeeamRestUrl.valueIn(zoneId)); + if (StringUtils.isBlank(host)) { + host = extractHost(AblestackVeeamUrl.valueIn(zoneId)); + } + return new AblestackVeeamSshClient(host, AblestackVeeamUsername.valueIn(zoneId), AblestackVeeamPassword.valueIn(zoneId)); + } + + private String extractHost(final String url) { + if (StringUtils.isBlank(url)) { + return null; + } + try { + return new URI(url).getHost(); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Veeam URL for SSH host: " + e.getMessage()); + } + } + + @Override + public String getName() { + return PROVIDER_NAME; + } + + @Override + public String getDescription() { + return "Ablestack Veeam + NAS KVM Backup Plugin"; + } + + @Override + public List listBackupOfferings(Long zoneId) { + return nasBackupProvider.listBackupOfferings(zoneId); + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + return nasBackupProvider.isValidProviderOffering(zoneId, uuid); + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { + if (!Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType())) { + throw new CloudRuntimeException("Ablestack Veeam backup provider supports KVM instances only"); + } + return true; + } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + return true; + } + + @Override + public boolean willDeleteBackupsOnOfferingRemoval() { + return false; + } + + @Override + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { + if (!nasBackupProvider.hasBackedUpSeed(vm)) { + final String restorePointId = resolveRestorePointId(vm); + final String stagingSubDir = String.format("%s/%s", vm.getInstanceName(), System.currentTimeMillis()); + final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); + logger.info("No NAS seed found for VM [{}], exporting Veeam restore point [{}] to [{}]", + vm.getInstanceName(), restorePointId, stagingPath); + final List stagingDisks = getSshClient(vm.getDataCenterId()) + .exportRestorePointDisksToStaging(restorePointId, stagingPath); + final Pair seedResult = nasBackupProvider.importVeeamBackupSeed( + vm, stagingDisks, restorePointId, "vmdk", true); + if (!seedResult.first()) { + return seedResult; + } + tagBackupAsVeeamSourced(seedResult.second(), restorePointId, vm); + return seedResult; + } + final Pair result = nasBackupProvider.takeBackup(vm, quiesceVM); + if (result.second() != null) { + tagBackupAsVeeamSourced(result.second(), getVmRestorePointDetail(vm), vm); + } + return result; + } + + @Override + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM, Long backupScheduleId) { + return takeBackup(vm, quiesceVM); + } + + @Override + public Pair importAblestackVeeamBackupSeed(VirtualMachine vm, String veeamRestorePointId, + List stagingDiskPaths, String sourceFormat, Boolean bootstrapCheckpoint) { + List staging = stagingDiskPaths; + if (CollectionUtils.isEmpty(staging)) { + if (StringUtils.isBlank(veeamRestorePointId)) { + throw new CloudRuntimeException("Either veeam restore point id or staging disk paths are required"); + } + final String stagingSubDir = String.format("%s/%s-import", vm.getInstanceName(), System.currentTimeMillis()); + final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); + staging = getSshClient(vm.getDataCenterId()).exportRestorePointDisksToStaging(veeamRestorePointId, stagingPath); + } + final Pair result = nasBackupProvider.importVeeamBackupSeed( + vm, staging, veeamRestorePointId, sourceFormat, bootstrapCheckpoint); + if (result.second() != null) { + tagBackupAsVeeamSourced(result.second(), veeamRestorePointId, vm); + } + return result; + } + + private void tagBackupAsVeeamSourced(Backup backup, String restorePointId, VirtualMachine vm) { + if (backup == null) { + return; + } + nasBackupProvider.updateBackupDetail(backup, AblestackNasBackupProvider.DETAIL_BACKUP_SOURCE, PROVIDER_NAME); + if (StringUtils.isNotBlank(restorePointId)) { + nasBackupProvider.updateBackupDetail(backup, DETAIL_VEEAM_RESTORE_POINT_ID, restorePointId); + } + nasBackupProvider.updateBackupDetail(backup, DETAIL_VEEAM_VM_NAME, vm.getInstanceName()); + } + + private String resolveRestorePointId(VirtualMachine vm) { + final String fromVm = getVmRestorePointDetail(vm); + if (StringUtils.isNotBlank(fromVm)) { + return fromVm; + } + final List points = listRestorePoints(vm); + if (CollectionUtils.isEmpty(points)) { + throw new CloudRuntimeException(String.format( + "No Veeam restore point found for VM [%s]. Set detail [%s] or import a seed via API.", + vm.getInstanceName(), DETAIL_VEEAM_RESTORE_POINT_ID)); + } + return points.get(0).getId(); + } + + private String getVmRestorePointDetail(VirtualMachine vm) { + Map details = backupManager.getBackupDetailsFromVM(vm); + return details != null ? details.get(DETAIL_VEEAM_RESTORE_POINT_ID) : null; + } + + @Override + public boolean deleteBackup(Backup backup, boolean forced) { + return nasBackupProvider.deleteBackup(backup, forced); + } + + @Override + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + try { + final Pair local = nasBackupProvider.restoreBackupToVM(vm, backup, hostIp, dataStoreUuid); + if (local != null && Boolean.TRUE.equals(local.first())) { + return local; + } + logger.warn("Local NAS restore did not succeed for backup [{}]; attempting Veeam chain fallback.", backup.getUuid()); + } catch (Exception e) { + logger.warn(String.format("Local NAS restore failed for backup [%s]; attempting Veeam chain fallback: %s", + backup.getUuid(), e.getMessage())); + } + final Backup seed = rebuildBackupFromVeeam(vm, backup); + return nasBackupProvider.restoreBackupToVM(vm, seed, hostIp, dataStoreUuid); + } + + @Override + public Pair restoreBackupToVM(Long backupId, String vmName) { + // This entry point only carries ids; the Veeam fallback needs the full VM + // and Backup objects (restore-point detail, zone, staging path). The + // object-based overloads above provide the Veeam chain fallback. + return nasBackupProvider.restoreBackupToVM(backupId, vmName); + } + + @Override + public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + try { + if (nasBackupProvider.restoreVMFromBackup(vm, backup)) { + return true; + } + logger.warn("Local NAS restore did not succeed for backup [{}]; attempting Veeam chain fallback.", backup.getUuid()); + } catch (Exception e) { + logger.warn(String.format("Local NAS restore failed for backup [%s]; attempting Veeam chain fallback: %s", + backup.getUuid(), e.getMessage())); + } + final Backup seed = rebuildBackupFromVeeam(vm, backup); + return nasBackupProvider.restoreVMFromBackup(vm, seed); + } + + /** + * Veeam chain restore fallback used when the local qcow2 chain cannot satisfy a + * restore (e.g. the chain is incomplete because the authoritative data lives in + * Veeam). We re-export the exact Veeam restore point recorded on the backup, + * re-import it as a fresh NAS seed (a self-contained full point that does not + * depend on the local chain) and let the caller restore from that seed. + * + *

The happy path (local restore succeeds) never reaches here, so this adds no + * regression risk to working local restores.

+ */ + private Backup rebuildBackupFromVeeam(VirtualMachine vm, Backup backup) { + final String restorePointId = getVeeamRestorePointId(backup); + if (StringUtils.isBlank(restorePointId)) { + throw new CloudRuntimeException(String.format( + "Local restore failed for backup [%s] and no Veeam restore point id is recorded; cannot fall back to Veeam.", + backup.getUuid())); + } + logger.info("Rebuilding backup [{}] from Veeam restore point [{}] for restore.", backup.getUuid(), restorePointId); + final String stagingSubDir = String.format("restore-%s/%s", vm.getInstanceName(), System.currentTimeMillis()); + final String stagingPath = String.format("%s/%s", AblestackVeeamStagingPath.valueIn(vm.getDataCenterId()), stagingSubDir); + final List stagingDisks = getSshClient(vm.getDataCenterId()) + .exportRestorePointDisksToStaging(restorePointId, stagingPath); + final Pair seed = nasBackupProvider.importVeeamBackupSeed( + vm, stagingDisks, restorePointId, "vmdk", true); + if (seed == null || !Boolean.TRUE.equals(seed.first()) || seed.second() == null) { + throw new CloudRuntimeException(String.format( + "Failed to rebuild backup [%s] from Veeam restore point [%s].", backup.getUuid(), restorePointId)); + } + return seed.second(); + } + + private String getVeeamRestorePointId(Backup backup) { + final Map details = backup != null ? backup.getDetails() : null; + if (details == null) { + return null; + } + final String fromProvider = details.get(DETAIL_VEEAM_RESTORE_POINT_ID); + if (StringUtils.isNotBlank(fromProvider)) { + return fromProvider; + } + return details.get(AblestackNasBackupProvider.DETAIL_VEEAM_RESTORE_POINT_ID); + } + + @Override + public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, + String dataStoreUuid, Pair vmNameAndState) { + return nasBackupProvider.restoreBackedUpVolume(backup, backupVolumeInfo, hostIp, dataStoreUuid, vmNameAndState); + } + + @Override + public void syncBackupMetrics(Long zoneId) { + nasBackupProvider.syncBackupMetrics(zoneId); + } + + @Override + public List listRestorePoints(VirtualMachine vm) { + final String veeamVmName = getVeeamSourceVmName(vm); + if (Boolean.TRUE.equals(AblestackVeeamUseRestApi.valueIn(vm.getDataCenterId()))) { + return getRestClient(vm.getDataCenterId()).listRestorePointsForVm(veeamVmName); + } + final AblestackVeeamClient client = getClient(vm.getDataCenterId()); + client.syncBackupRepository(); + return client.listRestorePointsForVmDisplayName(veeamVmName); + } + + private String getVeeamSourceVmName(VirtualMachine vm) { + Map details = backupManager.getBackupDetailsFromVM(vm); + if (details != null && StringUtils.isNotBlank(details.get(DETAIL_VEEAM_VM_NAME))) { + return details.get(DETAIL_VEEAM_VM_NAME); + } + return vm.getInstanceName(); + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { + throw new CloudRuntimeException("Use importVeeamNasBackupSeed API to register a Veeam restore point as NAS seed"); + } + + @Override + public boolean supportsInstanceFromBackup() { + return nasBackupProvider.supportsInstanceFromBackup(); + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + return nasBackupProvider.getBackupStorageStats(zoneId); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + nasBackupProvider.syncBackupStorageStats(zoneId); + } + + @Override + public void syncBackups(VirtualMachine vm) { + nasBackupProvider.syncBackups(vm); + } + + /** + * Backups for this provider are NAS-managed (delegated to nasBackupProvider). + * Veeam restore points returned by {@link #listRestorePoints(VirtualMachine)} are + * a seed source only, not the authoritative backup list. The generic out-of-band + * sync reconciles DB backups against listRestorePoints() and DELETES any DB backup + * that has no matching Veeam restore point, which would wrongly wipe NAS/agent backups + * (e.g. when Veeam Enterprise Manager reports no restore points for the VM display name). + * Opt out so those backups are not destroyed; NAS sync is handled via syncBackups(vm). + */ + @Override + public boolean supportsOutOfBandBackupSync() { + return false; + } + + @Override + public boolean checkBackupAgent(Long zoneId) { + return true; + } + + @Override + public boolean installBackupAgent(Long zoneId) { + return true; + } + + @Override + public boolean importBackupPlan(Long zoneId, String retentionPeriod, String externalId) { + return true; + } + + @Override + public boolean updateBackupPlan(Long zoneId, String retentionPeriod, String externalId) { + return true; + } + + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return nasBackupProvider.crossZoneInstanceCreationEnabled(backupOffering); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + AblestackVeeamUrl, + AblestackVeeamVersion, + AblestackVeeamUsername, + AblestackVeeamPassword, + AblestackVeeamValidateSsl, + AblestackVeeamApiTimeout, + AblestackVeeamRestoreTimeout, + AblestackVeeamTaskPollInterval, + AblestackVeeamTaskPollMaxRetry, + AblestackVeeamStagingPath, + AblestackVeeamUseRestApi, + AblestackVeeamRestUrl, + AblestackVeeamRestApiVersion + }; + } + + @Override + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } + + @Override + public boolean supportsVolumeLevelChainState() { + return nasBackupProvider.supportsVolumeLevelChainState(); + } + + @Override + public boolean supportsRestorePlan() { + return nasBackupProvider.supportsRestorePlan(); + } + + @Override + public boolean supportsRestoreChainValidation() { + return nasBackupProvider.supportsRestoreChainValidation(); + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java new file mode 100644 index 000000000000..f1294a33ebfc --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamClient.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.ablestackveeam; + +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Date; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.veeam.VeeamClient; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; + +/** + * Ablestack Veeam client: extends the stock Veeam REST/SSH client with NAS seed export helpers. + */ +public class AblestackVeeamClient extends VeeamClient { + + /** + * PowerShell executable used on the Veeam server. Override with + * -Dveeam.powershell.bin=pwsh if the Veeam (v12.1+/v13) module requires + * PowerShell 7 instead of Windows PowerShell 5.1. + */ + private static final String POWERSHELL_BIN = System.getProperty("veeam.powershell.bin", "powershell"); + + public AblestackVeeamClient(final String url, final Integer version, final String username, final String password, + final boolean validateCertificate, final int timeout, final int restoreTimeout, final int taskPollInterval, + final int taskPollMaxRetry) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + super(url, version, username, password, validateCertificate, timeout, restoreTimeout, taskPollInterval, taskPollMaxRetry); + } + + /** + * Build the single SSH command that runs the given PowerShell statements on the + * Veeam server. + * + *

The stock implementation joins commands with ';' and prefixes a bare + * {@code PowerShell ...} token, relying on the Windows SSH default shell + * (cmd.exe) to leave the rest untouched. That breaks for any statement that + * contains cmd.exe metacharacters - most notably the pipeline operator + * {@code |} (e.g. {@code Get-VBRRestorePoint | Where-Object ...}) and script + * blocks {@code { }}. cmd.exe intercepts those, which is why the logs show + * errors like {@code 'Where-Object' is not recognized as an internal or + * external command}.

+ * + *

Instead we assemble one PowerShell script and pass it via + * {@code -EncodedCommand} (Base64 of UTF-16LE). The encoded payload contains + * only Base64 characters, so cmd.exe cannot mangle it and the whole script + * runs inside a single PowerShell process.

+ */ + @Override + protected String transformPowerShellCommandList(final List cmds) { + // The Ablestack Veeam integration targets modern Veeam (v12+/v13) and uses + // Veeam.Backup.PowerShell cmdlets, so always use the module import (non-legacy + // PSSnapin) path. Keeping this independent of the parent's legacy detection. + final StringJoiner script = new StringJoiner("\n"); + script.add("Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue"); + script.add("$ProgressPreference='SilentlyContinue'"); + for (final String cmd : cmds) { + script.add(normalizeToPowerShell(cmd)); + } + final String encoded = Base64.getEncoder().encodeToString(script.toString().getBytes(StandardCharsets.UTF_16LE)); + return String.format("%s -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand %s", POWERSHELL_BIN, encoded); + } + + /** + * Some inherited command strings are pre-escaped for cmd.exe passthrough + * (e.g. {@code ^|} for the pipeline operator and {@code \"} for quotes). + * When the script is run via -EncodedCommand it is pure PowerShell, so those + * cmd escapes must be reverted to their native PowerShell form. + */ + private String normalizeToPowerShell(final String cmd) { + if (cmd == null) { + return ""; + } + return cmd.replace("^|", "|").replace("\\\"", "\""); + } + + /** + * Export all hard disks from a Veeam restore point to a directory on the Veeam server. + * The directory must be reachable from the KVM hypervisor (shared NFS recommended). + * + * @return absolute paths of exported disk files on the Veeam server + */ + public List exportRestorePointDisksToStaging(final String restorePointId, final String stagingPath) { + logger.debug(String.format("Exporting Veeam restore point [%s] to staging [%s]", restorePointId, stagingPath)); + final String escapedStaging = stagingPath.replace("'", "''"); + final String escapedId = restorePointId.replace("'", "''"); + final List cmds = Arrays.asList( + String.format("$staging = '%s'", escapedStaging), + "New-Item -ItemType Directory -Force -Path $staging | Out-Null", + String.format("$restorePoint = Get-VBRRestorePoint | Where-Object { $_.Id -eq '%s' -or $_.Id.Guid -eq '%s' }", escapedId, escapedId), + "if (-not $restorePoint) { Write-Output 'Failed: restore point not found'; Exit 1 }", + "$session = Start-VBRFLRSession -RestorePoint $restorePoint", + "$items = Get-VBRFLRItem -Session $session", + "$exported = @()", + "foreach ($item in $items) {", + " if ($item.Type -eq 'HardDisk') {", + " $target = Join-Path $staging ($item.Name + '.vmdk')", + " Copy-VBRFLRItem -FLRSession $session -Item $item -Destination $target", + " $exported += $target", + " }", + "}", + "Stop-VBRFLRSession -Session $session", + "if ($exported.Count -eq 0) { Write-Output 'Failed: no disks exported'; Exit 1 }", + "$exported -join ','" + ); + Pair result = executePowerShellCommands(cmds); + if (result == null || !result.first() || StringUtils.isBlank(result.second())) { + throw new CloudRuntimeException(String.format("Failed to export Veeam restore point [%s] to [%s]", restorePointId, stagingPath)); + } + if (result.second().contains("Failed:")) { + throw new CloudRuntimeException(result.second().trim()); + } + return Arrays.stream(result.second().trim().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + /** + * List Veeam restore points for a VM display name (KVM migration; no vCenter hierarchy required). + */ + public List listRestorePointsForVmDisplayName(final String vmDisplayName) { + final String escapedName = vmDisplayName.replace("'", "''"); + final List cmds = Arrays.asList( + String.format("$points = Get-VBRRestorePoint | Where-Object { $_.VmName -eq '%s' -or $_.Name -like '*%s*' }", escapedName, escapedName), + "if (-not $points) { Exit 0 }", + "$points | Sort-Object CreationTime -Descending | ForEach-Object {", + " Write-Output $_.Id.Guid", + " Write-Output $_.CreationTime.ToString('yyyy-MM-ddTHH:mm:ss')", + " Write-Output $_.Type", + " Write-Output '-----'", + "}" + ); + Pair response = executePowerShellCommands(cmds); + if (response == null || !response.first() || StringUtils.isBlank(response.second())) { + return new ArrayList<>(); + } + final List restorePoints = new ArrayList<>(); + for (final String block : response.second().split("-----\r\n")) { + final String[] parts = block.trim().split("\r\n"); + if (parts.length < 3) { + continue; + } + try { + final SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + final Date created = fmt.parse(parts[1].trim()); + restorePoints.add(new Backup.RestorePoint(parts[0].trim(), created, parts[2].trim(), null, null)); + } catch (ParseException e) { + logger.warn("Skipping unparseable Veeam restore point block: {}", block); + } + } + return restorePoints; + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java new file mode 100644 index 000000000000..2703d95df9d2 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamRestClient.java @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.ablestackveeam; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; + +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.nio.TrustAllManager; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Veeam Backup & Replication (VBR) REST API client (port 9419, v12+/v13). + * + *

Unlike the Enterprise Manager XML API (9398) used by {@link org.apache.cloudstack.backup.veeam.VeeamClient}, + * the VBR REST API is JSON based, uses OAuth 2.0 bearer tokens and is available on every + * VBR server without Enterprise Manager. This client covers the read/query operations that + * can be served over REST (restore point discovery), removing the dependency on + * PowerShell-over-SSH for those calls.

+ * + *

Note: there is no VBR REST endpoint that exports backup disk content to a filesystem + * path. Disk export for NAS seed creation/restore must still be performed via PowerShell + * ({@link AblestackVeeamClient#exportRestorePointDisksToStaging}).

+ */ +public class AblestackVeeamRestClient { + + private static final Logger LOG = LogManager.getLogger(AblestackVeeamRestClient.class); + + private final URI apiURI; + private final String apiVersion; + private final String username; + private final String password; + private final HttpClient httpClient; + + private String accessToken; + + public AblestackVeeamRestClient(final String url, final String apiVersion, final String username, + final String password, final boolean validateCertificate, final int timeoutSeconds) + throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + this.apiURI = new URI(StringUtils.appendIfMissing(url, "/")); + this.apiVersion = StringUtils.defaultIfBlank(apiVersion, "1.3-rev1"); + this.username = username; + this.password = password; + + final RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeoutSeconds * 1000) + .setConnectionRequestTimeout(timeoutSeconds * 1000) + .setSocketTimeout(timeoutSeconds * 1000) + .build(); + + if (!validateCertificate) { + final SSLContext sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, new X509TrustManager[]{new TrustAllManager()}, new SecureRandom()); + final SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + this.httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).setSSLSocketFactory(factory).build(); + } else { + this.httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); + } + + authenticate(); + } + + private String endpoint(final String relativePath) { + return apiURI.toString() + relativePath; + } + + private void authenticate() { + final HttpPost request = new HttpPost(endpoint("oauth2/token")); + request.setHeader("x-api-version", apiVersion); + request.setHeader(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded"); + request.setHeader(HttpHeaders.ACCEPT, "application/json"); + final String body = String.format("grant_type=password&username=%s&password=%s", + urlEncode(username), urlEncode(password)); + request.setEntity(new StringEntity(body, StandardCharsets.UTF_8)); + try { + final HttpResponse response = httpClient.execute(request); + final int code = response.getStatusLine().getStatusCode(); + final String payload = readBody(response); + if (code != HttpStatus.SC_OK) { + throw new CloudRuntimeException(String.format( + "Failed to authenticate to Veeam VBR REST API [%s]: HTTP %d %s", apiURI, code, payload)); + } + final JsonNode node = new ObjectMapper().readTree(payload); + this.accessToken = node.path("access_token").asText(null); + if (StringUtils.isBlank(accessToken)) { + throw new CloudRuntimeException("Veeam VBR REST API returned no access_token"); + } + } catch (IOException e) { + throw new CloudRuntimeException("Error authenticating to Veeam VBR REST API: " + e.getMessage(), e); + } + } + + /** + * List restore points for a VM (object) name, newest first. The returned restore-point + * id is the bare GUID (urn:uuid: prefix stripped) so it stays compatible with the + * PowerShell disk-export path that looks up {@code Get-VBRRestorePoint} by Id. + */ + public List listRestorePointsForVm(final String vmName) { + final String path = String.format("v1/objectRestorePoints?nameFilter=%s&orderColumn=CreationTime&orderAsc=false", + urlEncode(vmName)); + final JsonNode root = getJson(path); + final List points = new ArrayList<>(); + final JsonNode data = root.has("data") ? root.get("data") : root; + if (data == null || !data.isArray()) { + return points; + } + for (final JsonNode rp : data) { + final String rawId = rp.path("id").asText(null); + if (StringUtils.isBlank(rawId)) { + continue; + } + final String name = rp.path("name").asText(""); + // Restore points are returned for all objects matching the name filter; keep + // only those whose name actually corresponds to the requested VM. + if (StringUtils.isNotBlank(vmName) && StringUtils.isNotBlank(name) + && !name.toLowerCase().contains(vmName.toLowerCase())) { + continue; + } + final Date created = parseDate(rp.path("creationTime").asText(null)); + final String type = firstNonBlank(rp.path("type").asText(null), rp.path("pointType").asText(null), ""); + points.add(new Backup.RestorePoint(stripUrn(rawId), created, type, null, null)); + } + return points; + } + + private JsonNode getJson(final String relativePath) { + try { + JsonNode node = executeGet(relativePath); + return node; + } catch (TokenExpiredException e) { + authenticate(); + return executeGet(relativePath); + } + } + + private JsonNode executeGet(final String relativePath) { + final HttpGet request = new HttpGet(endpoint(relativePath)); + request.setHeader("x-api-version", apiVersion); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken); + request.setHeader(HttpHeaders.ACCEPT, "application/json"); + try { + final HttpResponse response = httpClient.execute(request); + final int code = response.getStatusLine().getStatusCode(); + final String payload = readBody(response); + if (code == HttpStatus.SC_UNAUTHORIZED) { + throw new TokenExpiredException(); + } + if (code != HttpStatus.SC_OK) { + throw new CloudRuntimeException(String.format( + "Veeam VBR REST API GET [%s] failed: HTTP %d %s", relativePath, code, payload)); + } + return new ObjectMapper().readTree(payload); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Error calling Veeam VBR REST API [%s]: %s", relativePath, e.getMessage()), e); + } + } + + private static String readBody(final HttpResponse response) throws IOException { + final HttpEntity entity = response.getEntity(); + return entity == null ? "" : EntityUtils.toString(entity, StandardCharsets.UTF_8); + } + + private static String urlEncode(final String value) { + return URLEncoder.encode(value == null ? "" : value, StandardCharsets.UTF_8); + } + + private static String stripUrn(final String id) { + if (id == null) { + return null; + } + final int idx = id.lastIndexOf(':'); + return idx >= 0 ? id.substring(idx + 1) : id; + } + + private static String firstNonBlank(final String... values) { + for (final String v : values) { + if (StringUtils.isNotBlank(v)) { + return v; + } + } + return ""; + } + + private static Date parseDate(final String value) { + if (StringUtils.isBlank(value)) { + return null; + } + try { + return Date.from(Instant.parse(value)); + } catch (DateTimeParseException e) { + LOG.warn("Unable to parse Veeam restore point creationTime [{}]", value); + return null; + } + } + + private static final class TokenExpiredException extends RuntimeException { + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java new file mode 100644 index 000000000000..6e9a1a02b85f --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/java/org/apache/cloudstack/backup/ablestackveeam/AblestackVeeamSshClient.java @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.ablestackveeam; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; + +/** + * Standalone SSH/PowerShell client for the Veeam server that performs the operations + * which have no Veeam REST API equivalent - notably exporting restore point disks to a + * staging directory for NAS seed creation/restore. + * + *

Unlike {@link org.apache.cloudstack.backup.veeam.VeeamClient} (and its subclass + * {@link AblestackVeeamClient}), this client does NOT authenticate against the Veeam + * Enterprise Manager REST API (port 9398). It only opens an SSH session (port 22) and + * runs PowerShell. That decouples disk export from Enterprise Manager so deployments + * using the VBR REST API (port 9419) for queries do not need EM/9398 at all.

+ * + *

PowerShell is delivered via {@code -EncodedCommand} (Base64 of UTF-16LE) so the + * Windows SSH default shell (cmd.exe) cannot mangle PowerShell metacharacters such as + * {@code |}, {@code { }} and {@code >}.

+ */ +public class AblestackVeeamSshClient { + + protected Logger logger = LogManager.getLogger(getClass()); + + /** + * Override with -Dveeam.powershell.bin=pwsh if the Veeam (v12.1+/v13) module + * requires PowerShell 7 instead of Windows PowerShell 5.1. + */ + private static final String POWERSHELL_BIN = System.getProperty("veeam.powershell.bin", "powershell"); + + private static final int SSH_PORT = 22; + private static final int CONNECT_TIMEOUT_MS = 120000; + private static final int KEX_TIMEOUT_MS = 120000; + private static final int WAIT_TIMEOUT_MS = 3600000; + + private final String host; + private final String username; + private final String password; + private final boolean legacy; + + public AblestackVeeamSshClient(final String host, final String username, final String password) { + this(host, username, password, false); + } + + public AblestackVeeamSshClient(final String host, final String username, final String password, final boolean legacy) { + if (StringUtils.isBlank(host)) { + throw new CloudRuntimeException("Veeam SSH host is required for disk export"); + } + this.host = host; + this.username = username; + this.password = password; + this.legacy = legacy; + } + + /** + * Export all hard disks from a Veeam restore point to a directory on the Veeam server. + * The directory must be reachable from the KVM hypervisor (shared NFS recommended). + * + * @return absolute paths of exported disk files on the Veeam server + */ + public List exportRestorePointDisksToStaging(final String restorePointId, final String stagingPath) { + logger.debug(String.format("Exporting Veeam restore point [%s] to staging [%s] via SSH", restorePointId, stagingPath)); + final String escapedStaging = stagingPath.replace("'", "''"); + final String escapedId = restorePointId.replace("'", "''"); + final List cmds = Arrays.asList( + String.format("$staging = '%s'", escapedStaging), + "New-Item -ItemType Directory -Force -Path $staging | Out-Null", + String.format("$restorePoint = Get-VBRRestorePoint | Where-Object { $_.Id -eq '%s' -or $_.Id.Guid -eq '%s' }", escapedId, escapedId), + "if (-not $restorePoint) { Write-Output 'Failed: restore point not found'; Exit 1 }", + "$session = Start-VBRFLRSession -RestorePoint $restorePoint", + "$items = Get-VBRFLRItem -Session $session", + "$exported = @()", + "foreach ($item in $items) {", + " if ($item.Type -eq 'HardDisk') {", + " $target = Join-Path $staging ($item.Name + '.vmdk')", + " Copy-VBRFLRItem -FLRSession $session -Item $item -Destination $target", + " $exported += $target", + " }", + "}", + "Stop-VBRFLRSession -Session $session", + "if ($exported.Count -eq 0) { Write-Output 'Failed: no disks exported'; Exit 1 }", + "$exported -join ','" + ); + final Pair result = executePowerShellCommands(cmds); + if (result == null || !result.first() || StringUtils.isBlank(result.second())) { + throw new CloudRuntimeException(String.format("Failed to export Veeam restore point [%s] to [%s]", restorePointId, stagingPath)); + } + if (result.second().contains("Failed:")) { + throw new CloudRuntimeException(result.second().trim()); + } + return Arrays.stream(result.second().trim().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + private Pair executePowerShellCommands(final List cmds) { + final String command = transformPowerShellCommandList(cmds); + try { + final Pair response = SshHelper.sshExecute(host, SSH_PORT, username, null, password, + command, CONNECT_TIMEOUT_MS, KEX_TIMEOUT_MS, WAIT_TIMEOUT_MS); + if (response == null || !response.first()) { + logger.error(String.format("Veeam SSH PowerShell command failed: [%s]", + response != null ? response.second() : "no output returned")); + } + return response; + } catch (Exception e) { + throw new CloudRuntimeException("Error while executing Veeam SSH PowerShell command: " + e.getMessage(), e); + } + } + + /** + * Build the single SSH command that runs the given PowerShell statements, passed via + * {@code -EncodedCommand} so cmd.exe cannot intercept PowerShell metacharacters. + */ + private String transformPowerShellCommandList(final List cmds) { + final StringJoiner script = new StringJoiner("\n"); + if (legacy) { + script.add("Add-PSSnapin VeeamPSSnapin"); + } else { + script.add("Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue"); + script.add("$ProgressPreference='SilentlyContinue'"); + } + final List all = new ArrayList<>(cmds); + for (final String cmd : all) { + script.add(normalizeToPowerShell(cmd)); + } + final String encoded = Base64.getEncoder().encodeToString(script.toString().getBytes(StandardCharsets.UTF_16LE)); + return String.format("%s -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand %s", POWERSHELL_BIN, encoded); + } + + private String normalizeToPowerShell(final String cmd) { + if (cmd == null) { + return ""; + } + return cmd.replace("^|", "|").replace("\\\"", "\""); + } +} diff --git a/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties new file mode 100644 index 000000000000..1700af0e6132 --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=ablestack-veeam +parent=ablestack-nas diff --git a/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml new file mode 100644 index 000000000000..cbf9bbfb6f4e --- /dev/null +++ b/plugins/backup/ablestack-veeam/src/main/resources/META-INF/cloudstack/ablestack-veeam/spring-backup-ablestack-veeam-context.xml @@ -0,0 +1,26 @@ + + + + + + + diff --git a/plugins/backup/veeam/pom.xml b/plugins/backup/veeam/pom.xml index 0626c5f8e842..92a0011f6870 100644 --- a/plugins/backup/veeam/pom.xml +++ b/plugins/backup/veeam/pom.xml @@ -43,11 +43,6 @@ cloud-engine-components-api ${project.version} - - org.apache.cloudstack - cloud-plugin-hypervisor-vmware - ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-xml @@ -66,4 +61,49 @@ + + + + veeam-vmware-provider + + + noredist + + + + + org.apache.cloudstack + cloud-plugin-hypervisor-vmware + ${project.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org/apache/cloudstack/backup/VeeamBackupProvider.java + + + org/apache/cloudstack/backup/VeeamBackupProviderTest.java + + + + + + diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 840f336a10ca..6e692f2aa5ca 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -56,35 +56,35 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public static final String BACKUP_IDENTIFIER = "-CSBKP-"; - public ConfigKey VeeamUrl = new ConfigKey<>("Advanced", String.class, + public ConfigKey VeeamUrl = new ConfigKey("Advanced", String.class, "backup.plugin.veeam.url", "https://localhost:9398/api/", "The Veeam backup and recovery URL.", true, ConfigKey.Scope.Zone); - public ConfigKey VeeamVersion = new ConfigKey<>("Advanced", Integer.class, + public ConfigKey VeeamVersion = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.version", "0", "The version of Veeam backup and recovery. CloudStack will get Veeam server version via PowerShell commands if it is 0 or not set", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamUsername = new ConfigKey<>("Advanced", String.class, + private ConfigKey VeeamUsername = new ConfigKey("Advanced", String.class, "backup.plugin.veeam.username", "administrator", "The Veeam backup and recovery username.", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamPassword = new ConfigKey<>("Secure", String.class, + private ConfigKey VeeamPassword = new ConfigKey("Secure", String.class, "backup.plugin.veeam.password", "", "The Veeam backup and recovery password.", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamValidateSSLSecurity = new ConfigKey<>("Advanced", Boolean.class, "backup.plugin.veeam.validate.ssl", "false", + private ConfigKey VeeamValidateSSLSecurity = new ConfigKey("Advanced", Boolean.class, "backup.plugin.veeam.validate.ssl", "false", "When set to true, this will validate the SSL certificate when connecting to https/ssl enabled Veeam API service.", true, ConfigKey.Scope.Zone); - private ConfigKey VeeamApiRequestTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.request.timeout", "300", + private ConfigKey VeeamApiRequestTimeout = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.request.timeout", "300", "The Veeam B&R API request timeout in seconds.", true, ConfigKey.Scope.Zone); - private static ConfigKey VeeamRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.restore.timeout", "600", + private static ConfigKey VeeamRestoreTimeout = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.restore.timeout", "600", "The Veeam B&R API restore backup timeout in seconds.", true, ConfigKey.Scope.Zone); - private static ConfigKey VeeamTaskPollInterval = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.task.poll.interval", "5", + private static ConfigKey VeeamTaskPollInterval = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.task.poll.interval", "5", "The time interval in seconds when the management server polls for Veeam task status.", true, ConfigKey.Scope.Zone); - private static ConfigKey VeeamTaskPollMaxRetry = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.task.poll.max.retry", "120", + private static ConfigKey VeeamTaskPollMaxRetry = new ConfigKey("Advanced", Integer.class, "backup.plugin.veeam.task.poll.max.retry", "120", "The max number of retrying times when the management server polls for Veeam task status.", true, ConfigKey.Scope.Zone); @Inject @@ -104,7 +104,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, @Inject private VolumeDao volumeDao; - private Map backupFilesMetricsMap = new HashMap<>(); + private Map backupFilesMetricsMap = new HashMap(); protected VeeamClient getClient(final Long zoneId) { try { @@ -113,14 +113,16 @@ protected VeeamClient getClient(final Long zoneId) { VeeamTaskPollInterval.valueIn(zoneId), VeeamTaskPollMaxRetry.valueIn(zoneId)); } catch (URISyntaxException e) { throw new CloudRuntimeException("Failed to parse Veeam API URL: " + e.getMessage()); - } catch (NoSuchAlgorithmException | KeyManagementException e) { + } catch (NoSuchAlgorithmException e) { + logger.error("Failed to build Veeam API client due to: ", e); + } catch (KeyManagementException e) { logger.error("Failed to build Veeam API client due to: ", e); } throw new CloudRuntimeException("Failed to build Veeam API client"); } public List listBackupOfferings(final Long zoneId) { - List policies = new ArrayList<>(); + List policies = new ArrayList(); for (final BackupOffering policy : getClient(zoneId).listJobs()) { if (!policy.getName().contains(BACKUP_IDENTIFIER)) { policies.add(policy); @@ -222,7 +224,7 @@ public boolean willDeleteBackupsOnOfferingRemoval() { public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { final VeeamClient client = getClient(vm.getDataCenterId()); Boolean result = client.startBackupJob(vm.getBackupExternalId()); - return new Pair<>(result, null); + return new Pair(result, null); } @Override @@ -321,7 +323,7 @@ public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoi backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setName(backupManager.getBackupNameFromVM(vm)); - List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + List volumes = new ArrayList(volumeDao.findByInstance(vm.getId())); backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); Map details = backupManager.getBackupDetailsFromVM(vm); backup.setDetails(details); @@ -351,7 +353,7 @@ public boolean supportsInstanceFromBackup() { @Override public Pair getBackupStorageStats(Long zoneId) { - return new Pair<>(0L, 0L); + return new Pair(0L, 0L); } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 8a111f928680..179072ac13fe 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -395,7 +395,7 @@ protected void checkIfRestoreSessionFinished(String type, String path) throws IO private Pair getRelatedLinkPair(List links) { for (Link link : links) { if (link.getRel().equals("Related")) { - return new Pair<>(link.getHref(), link.getType()); + return new Pair(link.getHref(), link.getType()); } } return null; @@ -463,7 +463,7 @@ public List listJobs() { final HttpResponse response = get("/jobs"); checkResponseOK(response); final EntityReferences entityReferences = OBJECT_MAPPER.readValue(response.getEntity().getContent(), EntityReferences.class); - final List policies = new ArrayList<>(); + final List policies = new ArrayList(); if (entityReferences == null || entityReferences.getRefs() == null) { return policies; } @@ -475,7 +475,7 @@ public List listJobs() { logger.error("Failed to list Veeam jobs due to:", e); checkResponseTimeOut(e); } - return new ArrayList<>(); + return new ArrayList(); } public Job listJob(final String jobId) { @@ -698,11 +698,11 @@ public Map getBackupMetricsViaVeeamAPI() { logger.error("Failed to get backup metrics via Veeam B&R API due to:", e); checkResponseTimeOut(e); } - return new HashMap<>(); + return new HashMap(); } protected Map processHttpResponseForBackupMetrics(final InputStream content) { - Map metrics = new HashMap<>(); + Map metrics = new HashMap(); try { final BackupFiles backupFiles = OBJECT_MAPPER.readValue(content, BackupFiles.class); if (backupFiles == null || CollectionUtils.isEmpty(backupFiles.getBackupFiles())) { @@ -755,7 +755,7 @@ public Map getBackupMetricsLegacy() { protected Map processPowerShellResultForBackupMetrics(final String result) { logger.debug("Processing powershell result: " + result); final String separator = "====="; - Map metrics = new HashMap<>(); + Map metrics = new HashMap(); for (final String block : result.split(separator + "\r\n")) { final String[] parts = block.split("\r\n"); if (parts.length != 3) { @@ -811,7 +811,7 @@ public List listRestorePointsLegacy(String backupName, Stri return null; } - final List restorePoints = new ArrayList<>(); + final List restorePoints = new ArrayList(); for (final String block : response.second().split("\r\n\r\n")) { if (block.isEmpty()) { continue; @@ -842,11 +842,11 @@ public List listVmRestorePointsViaVeeamAPI(String vmwareDcN logger.error("Failed to list VM restore points via Veeam B&R API due to:", e); checkResponseTimeOut(e); } - return new ArrayList<>(); + return new ArrayList(); } public List processHttpResponseForVmRestorePoints(InputStream content, String vmwareDcName, String vmInternalName, Map metricsMap) { - List vmRestorePointList = new ArrayList<>(); + List vmRestorePointList = new ArrayList(); try { final VmRestorePoints vmRestorePoints = OBJECT_MAPPER.readValue(content, VmRestorePoints.class); final String hierarchyId = findDCHierarchy(vmwareDcName); @@ -891,7 +891,10 @@ public List processHttpResponseForVmRestorePoints(InputStre } vmRestorePointList.add(new Backup.RestorePoint(vmRestorePointId, created, type, backupSize, dataSize)); } - } catch (final IOException | ParseException e) { + } catch (final IOException e) { + logger.error("Failed to process response to get VM restore points via Veeam B&R API due to:", e); + checkResponseTimeOut(e); + } catch (final ParseException e) { logger.error("Failed to process response to get VM restore points via Veeam B&R API due to:", e); checkResponseTimeOut(e); } @@ -919,7 +922,7 @@ public Pair restoreVMToDifferentLocation(String restorePointId, if (result == null || !result.first()) { throw new CloudRuntimeException("Failed to restore VM to location " + restoreLocation); } - return new Pair<>(result.first(), restoreLocation); + return new Pair(result.first(), restoreLocation); } /** diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java index a82ff551b8a9..97a54713d53f 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java @@ -98,7 +98,7 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasJustOneBackup() { Mockito.when(vmInstanceDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(vmInstanceVO); Mockito.doReturn(client).when(backupProvider).getClient(2l); Mockito.doReturn(true).when(client).deleteBackup("abc"); - List backups = new ArrayList<>(); + List backups = new ArrayList(); backups.add(backup); Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(backups); Mockito.verify(backupDao, Mockito.never()).remove(Mockito.anyLong()); @@ -124,7 +124,10 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasMoreThanOneBackup() { Mockito.when(vmInstanceDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(vmInstanceVO); Mockito.doReturn(client).when(backupProvider).getClient(2l); Mockito.doReturn(true).when(client).deleteBackup("abc"); - Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(List.of(backup, backup2)); + List backups = new ArrayList(); + backups.add(backup); + backups.add(backup2); + Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(backups); boolean result = backupProvider.deleteBackup(backup, true); Mockito.verify(backupDao, Mockito.times(1)).remove(2l); assertEquals(true, result); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 333c3e16053a..de62dc9d46a3 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -61,6 +61,11 @@ public class VeeamClientTest { @Rule public WireMockRule wireMockRule = new WireMockRule(9399); + @SuppressWarnings("unchecked") + private static List anyCommandList() { + return (List) Mockito.any(List.class); + } + @Before public void setUp() throws Exception { wireMockRule.stubFor(post(urlMatching(".*/sessionMngr/.*")) @@ -107,7 +112,7 @@ public void testVeeamJobs() { public void getRepositoryNameFromJobTestExceptionCmdWithoutResult() throws Exception { String backupName = "TEST-BACKUP"; try { - Mockito.doReturn(null).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(null).when(mockClient).executePowerShellCommands(anyCommandList()); mockClient.getRepositoryNameFromJob(backupName); fail(); } catch (Exception e) { @@ -120,7 +125,7 @@ public void getRepositoryNameFromJobTestExceptionCmdWithoutResult() throws Excep public void getRepositoryNameFromJobTestExceptionCmdWithFalseResult() { String backupName = "TEST-BACKUP2"; Pair response = new Pair(Boolean.FALSE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); try { mockClient.getRepositoryNameFromJob(backupName); fail(); @@ -134,7 +139,7 @@ public void getRepositoryNameFromJobTestExceptionCmdWithFalseResult() { public void getRepositoryNameFromJobTestExceptionWhenResultIsInWrongFormat() { String backupName = "TEST-BACKUP3"; Pair response = new Pair(Boolean.TRUE, "\nName:\n\nName-test"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); try { mockClient.getRepositoryNameFromJob(backupName); fail(); @@ -148,7 +153,7 @@ public void getRepositoryNameFromJobTestExceptionWhenResultIsInWrongFormat() { public void getRepositoryNameFromJobTestSuccess() throws Exception { String backupName = "TEST-BACKUP3"; Pair response = new Pair(Boolean.TRUE, "\r\nName : test"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); String repositoryNameFromJob = mockClient.getRepositoryNameFromJob(backupName); Assert.assertEquals("test", repositoryNameFromJob); } @@ -173,18 +178,18 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti @Test public void getRestoreVmErrorDescriptionTestFindErrorDescription() { - Pair response = new Pair<>(true, "Example of error description found in Veeam."); + Pair response = new Pair(true, "Example of error description found in Veeam."); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Example of error description found in Veeam.", result); } @Test public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { - Pair response = new Pair<>(true, "Cannot find restore session with provided uid uuid"); + Pair response = new Pair(true, "Cannot find restore session with provided uid uuid"); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Cannot find restore session with provided uid uuid", result); } @@ -192,16 +197,16 @@ public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { @Test public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(null); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(null); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); } @Test public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { - Pair response = new Pair<>(false, null); + Pair response = new Pair(false, null); Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); - Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + Mockito.when(mockClient.executePowerShellCommands(anyCommandList())).thenReturn(response); String result = mockClient.getRestoreVmErrorDescription("uuid"); Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); } @@ -544,21 +549,21 @@ public void testListVmRestorePointsViaVeeamAPI() { @Test public void testGetVeeamServerVersionAllGood() { Pair response = new Pair(Boolean.TRUE, "12.0.0.1"); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); Assert.assertEquals(12, (int) mockClient.getVeeamServerVersion()); } @Test public void testGetVeeamServerVersionWithError() { Pair response = new Pair(Boolean.FALSE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); } @Test public void testGetVeeamServerVersionWithEmptyVersion() { Pair response = new Pair(Boolean.TRUE, ""); - Mockito.doReturn(response).when(mockClient).executePowerShellCommands(Mockito.anyList()); + Mockito.doReturn(response).when(mockClient).executePowerShellCommands(anyCommandList()); Assert.assertEquals(0, (int) mockClient.getVeeamServerVersion()); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index aec8abee565b..fb66e8ad6133 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -390,7 +390,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_PACKAGE = "virtio-win"; public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; - public static final String VDDK_AUTODETECT_PATH_CMD = "find / -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; + public static final String VDDK_AUTODETECT_PATH_CMD = + "find /opt /usr /usr/local -maxdepth 5 -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; + private static final int VDDK_AUTODETECT_TIMEOUT_SECONDS = 15; public static final int LIBVIRT_CGROUP_CPU_SHARES_MIN = 2; public static final int LIBVIRT_CGROUP_CPU_SHARES_MAX = 262144; @@ -6864,10 +6866,7 @@ public boolean hostSupportsVddk(String overriddenVddkLibDir) { if (StringUtils.isBlank(effectiveVddkLibDir)) { effectiveVddkLibDir = StringUtils.trimToNull(vddkLibDir); } - if (StringUtils.isBlank(effectiveVddkLibDir) || !isVddkLibDirValid(effectiveVddkLibDir)) { - effectiveVddkLibDir = detectVddkLibDir(); - } - return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(detectVddkVersion()); + return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(vddkVersion); } protected boolean isVddkLibDirValid(String path) { @@ -6883,7 +6882,8 @@ protected boolean isVddkLibDirValid(String path) { } protected String detectVddkLibDir() { - String detectedPath = StringUtils.trimToNull(Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD)); + String detectedPath = StringUtils.trimToNull( + Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD, VDDK_AUTODETECT_TIMEOUT_SECONDS)); if (StringUtils.isNotBlank(detectedPath) && isVddkLibDirValid(detectedPath)) { return detectedPath; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java index c04fb30f63fa..6f887f636163 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasBackupHelper.java @@ -27,7 +27,9 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.storage.Storage; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.AblestackNasImportVeeamSeedCommand; import org.apache.cloudstack.backup.AblestackNasTakeBackupCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.utils.security.ParserUtils; @@ -88,6 +90,16 @@ String getScriptOperation() { this.resource = resource; } + Pair executeImportVeeamSeed(AblestackNasImportVeeamSeedCommand command, List diskPaths) { + if (CollectionUtils.isNullOrEmpty(command.getStagingDiskPaths())) { + return new Pair<>(1, "Staging disk paths are required for Veeam seed import"); + } + String[] scriptCommand = buildImportVeeamSeedScriptCommand(command, diskPaths); + List commands = new ArrayList<>(); + commands.add(scriptCommand); + return Script.executePipedCommands(commands, resource.getCmdsTimeout()); + } + Pair executeBackup(AblestackNasTakeBackupCommand command) { LOGGER.info("LibvirtNasBackupHelper executeBackup entered for vm=[{}], backupPath=[{}], backupType=[{}]", command.getVmName(), command.getBackupPath(), command.getBackupType()); @@ -154,6 +166,25 @@ private boolean isWholeNumber(String value) { return value != null && !value.isEmpty() && value.chars().allMatch(Character::isDigit); } + private String[] buildImportVeeamSeedScriptCommand(AblestackNasImportVeeamSeedCommand command, List diskPaths) { + return new String[] { + resource.getAbleNasBackupPath(), + "-o", "import-veeam-seed", + "-v", command.getVmName(), + "-t", command.getBackupRepoType(), + "-s", command.getBackupRepoAddress(), + "-m", Objects.nonNull(command.getMountOptions()) ? command.getMountOptions() : "", + "-p", command.getBackupPath(), + "-c", Objects.nonNull(command.getCheckpointName()) ? command.getCheckpointName() : "", + "-f", CollectionUtils.isNullOrEmpty(command.getBackupFiles()) ? "" : String.join(",", command.getBackupFiles()), + "-d", diskPaths.isEmpty() ? "" : String.join(",", diskPaths), + "--staging-disks", String.join(",", command.getStagingDiskPaths()), + "--source-format", Objects.nonNull(command.getSourceFormat()) ? command.getSourceFormat() : "vmdk", + "--veeam-restore-point", Objects.nonNull(command.getVeeamRestorePointId()) ? command.getVeeamRestorePointId() : "", + "--bootstrap-checkpoint", command.getBootstrapCheckpoint() != null && command.getBootstrapCheckpoint() ? "true" : "false" + }; + } + private String[] buildBackupScriptCommand(AblestackNasTakeBackupCommand command, List diskPaths, BackupExecutionMode executionMode) { return new String[] { resource.getAbleNasBackupPath(), @@ -274,14 +305,17 @@ private Pair executeStoppedVmBackup(AblestackNasTakeBackupComma private Path mountRepository(AblestackNasTakeBackupCommand command) throws IOException { Path mountPoint = Files.createTempDirectory("csbackup."); - StringBuilder mount = new StringBuilder() - .append("mount -t ").append(shellQuote(command.getBackupRepoType())) - .append(" ").append(shellQuote(command.getBackupRepoAddress())) - .append(" ").append(shellQuote(mountPoint.toString())); - if (command.getMountOptions() != null && !command.getMountOptions().isEmpty()) { - mount.append(" -o ").append(shellQuote(command.getMountOptions())); - } - if (Script.runSimpleBashScriptForExitValue(mount.toString(), resource.getCmdsTimeout(), false) != 0) { + final String mount; + try { + mount = LibvirtBackupRepositoryMountHelper.buildMountCommand( + command.getBackupRepoAddress(), + command.getBackupRepoType(), + command.getMountOptions(), + mountPoint.toString()); + } catch (CloudRuntimeException e) { + throw new IOException(e.getMessage(), e); + } + if (Script.runSimpleBashScriptForExitValue(mount, resource.getCmdsTimeout(), false) != 0) { throw new IOException("Failed to mount backup repository"); } return mountPoint; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java new file mode 100644 index 000000000000..83eec305929a --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasImportVeeamSeedCommandWrapper.java @@ -0,0 +1,56 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import org.apache.cloudstack.backup.AblestackNasImportVeeamSeedCommand; +import org.apache.cloudstack.backup.BackupAnswer; + +import java.util.List; + +@ResourceWrapper(handles = AblestackNasImportVeeamSeedCommand.class) +public class LibvirtAblestackNasImportVeeamSeedCommandWrapper extends CommandWrapper { + @Override + public Answer execute(AblestackNasImportVeeamSeedCommand command, LibvirtComputingResource libvirtComputingResource) { + LibvirtAblestackNasBackupHelper backupHelper = new LibvirtAblestackNasBackupHelper(libvirtComputingResource); + List diskPaths = backupHelper.resolveDiskPaths(command.getVolumePools(), command.getVolumePaths()); + Pair result = backupHelper.executeImportVeeamSeed(command, diskPaths); + + if (result.first() != 0) { + BackupAnswer answer = new BackupAnswer(command, false, result.second().trim()); + if (result.first() == LibvirtAblestackNasBackupHelper.EXIT_CLEANUP_FAILED) { + answer.setNeedsCleanup(true); + } + return answer; + } + + BackupAnswer answer = new BackupAnswer(command, true, result.second().trim()); + try { + answer.setSize(backupHelper.parseBackupSize(result.second(), diskPaths)); + } catch (RuntimeException e) { + logger.warn("Failed to parse Veeam seed import size for vm=[{}]: {}", command.getVmName(), e.getMessage()); + } + return answer; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java index fd2108638ccf..47e03c60c04d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtAblestackNasRestoreBackupCommandWrapper.java @@ -227,17 +227,8 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } - String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); - if ("cifs".equals(backupRepoType)) { - if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) { - mountOptions = "nobrl"; - } else { - mountOptions += ",nobrl"; - } - } - if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) { - mount += " -o " + mountOptions; - } + final String mount = LibvirtBackupRepositoryMountHelper.buildMountCommand( + backupRepoAddress, backupRepoType, mountOptions, mountDirectory); int exitValue = Script.runSimpleBashScriptForExitValue(mount, mountTimeout, false); if (exitValue != 0) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java new file mode 100644 index 000000000000..8cebb62221d8 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBackupRepositoryMountHelper.java @@ -0,0 +1,72 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.StringUtils; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Locale; +import java.util.Objects; + +/** + * Mount helpers for backup repositories on KVM hosts. + * Local repositories use bind mounts (see ablestack_nasbackup.sh mount_operation). + */ +public final class LibvirtBackupRepositoryMountHelper { + + private static final String MOUNT_COMMAND = "sudo mount -t %s %s %s"; + + private LibvirtBackupRepositoryMountHelper() { + } + + public static boolean isLocalBackupRepositoryType(final String backupRepoType) { + if (StringUtils.isBlank(backupRepoType)) { + return false; + } + final String normalized = backupRepoType.toLowerCase(Locale.ROOT); + return "local".equals(normalized) || "dir".equals(normalized) || "localfs".equals(normalized); + } + + public static String buildMountCommand(final String backupRepoAddress, final String backupRepoType, + String mountOptions, final String mountDirectory) { + if (isLocalBackupRepositoryType(backupRepoType)) { + if (!Files.isDirectory(Paths.get(backupRepoAddress))) { + throw new CloudRuntimeException(String.format( + "Local backup directory does not exist on the KVM host: %s", backupRepoAddress)); + } + return String.format("sudo mount --bind %s %s", backupRepoAddress, mountDirectory); + } + + String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); + if ("cifs".equals(backupRepoType)) { + if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) { + mountOptions = "nobrl"; + } else { + mountOptions += ",nobrl"; + } + } + if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) { + mount += " -o " + mountOptions; + } + return mount; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 78c5346c8335..e0951c375de3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -205,17 +205,8 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } - String mount = String.format(MOUNT_COMMAND, backupRepoType, backupRepoAddress, mountDirectory); - if ("cifs".equals(backupRepoType)) { - if (Objects.isNull(mountOptions) || mountOptions.trim().isEmpty()) { - mountOptions = "nobrl"; - } else { - mountOptions += ",nobrl"; - } - } - if (Objects.nonNull(mountOptions) && !mountOptions.trim().isEmpty()) { - mount += " -o " + mountOptions; - } + final String mount = LibvirtBackupRepositoryMountHelper.buildMountCommand( + backupRepoAddress, backupRepoType, mountOptions, mountDirectory); int exitValue = Script.runSimpleBashScriptForExitValue(mount, mountTimeout, false); if (exitValue != 0) { diff --git a/plugins/pom.xml b/plugins/pom.xml index 95289b51e86c..41e265f0c905 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -66,6 +66,8 @@ backup/ablestack-nas backup/ablestack-commvault backup/ablestack-netbackup + backup/veeam + backup/ablestack-veeam backup/bx ca/root-ca @@ -240,7 +242,6 @@ api/vmware-sioc - backup/veeam hypervisors/vmware network-elements/cisco-vnmc network-elements/nsx diff --git a/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh b/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh index 016ca7642e7e..07672a2ca0b6 100755 --- a/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh @@ -38,6 +38,10 @@ PARENT_CHECKPOINT_NAME="" PARENT_CHECKPOINT_PATH="" BACKUP_FILES="" DISK_PATHS="" +STAGING_DISK_PATHS="" +SOURCE_FORMAT="vmdk" +VEEAM_RESTORE_POINT_ID="" +BOOTSTRAP_CHECKPOINT="true" QUIESCE="" FORCED="false" CLEANUP_CHECKPOINT_NAMES="" @@ -116,7 +120,7 @@ backup_running_vm() { cleanup exit 1 fi - redefine_checkpoint_if_needed "$VM" "$parent_checkpoint_file" + redefine_checkpoint_if_needed "$VM" "$parent_checkpoint_file" "$mount_point" fi echo "" > "$dest/backup.xml" @@ -148,7 +152,10 @@ backup_running_vm() { local backup_begin=0 local backup_begin_output="" if backup_begin_output=$(virsh -c qemu:///system backup-begin --domain "$VM" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" 2>&1); then - backup_begin=1; + backup_begin=1 + else + echo "backup-begin failed for VM $VM: $backup_begin_output" >> "$logFile" + echo "backup-begin failed for VM $VM: $backup_begin_output" fi if [[ $thaw -eq 1 ]]; then @@ -517,6 +524,27 @@ get_backup_stats() { mount_operation() { mount_point=$(mktemp -d -t csbackup.XXXXX) dest="$mount_point/${BACKUP_DIR}" + + # Local data disk: NAS_ADDRESS is a directory already mounted on this host + # (e.g. a dedicated data disk). Bind-mount it so the rest of the flow + # (dest/umount/df) keeps working without a network mount. + case "${NAS_TYPE}" in + local|dir|localfs) + if [[ ! -d "${NAS_ADDRESS}" ]]; then + echo "Local backup directory does not exist: ${NAS_ADDRESS}" + exit 1 + fi + mount --bind "${NAS_ADDRESS}" "${mount_point}" 2>&1 | tee -a "$logFile" + if [ ${PIPESTATUS[0]} -eq 0 ]; then + log -ne "Successfully bind-mounted local backup dir ${NAS_ADDRESS}" + else + echo "Failed to bind-mount local backup dir ${NAS_ADDRESS}" + exit 1 + fi + return 0 + ;; + esac + if [ ${NAS_TYPE} == "cifs" ]; then MOUNT_OPTS="${MOUNT_OPTS},nobrl" fi @@ -546,6 +574,7 @@ cleanup() { echo "Backup cleanup failed" exit $EXIT_CLEANUP_FAILED fi + exit 1 } split_csv() { @@ -582,19 +611,89 @@ dump_checkpoint_xml() { fi } +strip_checkpoint_parent_from_xml() { + local xml_file="$1" + [[ -f "$xml_file" ]] || return 0 + python3 - "$xml_file" <<'PY' 2>/dev/null || true +import sys +import xml.etree.ElementTree as ET + +path = sys.argv[1] +tree = ET.parse(path) +root = tree.getroot() +for parent in list(root.findall('parent')): + root.remove(parent) +tree.write(path, encoding='unicode', xml_declaration=True) +PY +} + +get_parent_checkpoint_name_from_xml() { + local xml_file="$1" + [[ -f "$xml_file" ]] || return 0 + python3 - "$xml_file" <<'PY' 2>/dev/null +import sys +import xml.etree.ElementTree as ET + +try: + root = ET.parse(sys.argv[1]).getroot() + parent = root.find('parent') + if parent is None: + sys.exit(0) + name = parent.findtext('name', '').strip() + if name: + print(name) +except Exception: + pass +PY +} + +find_checkpoint_xml_on_nas() { + local search_root="$1" checkpoint_name="$2" + [[ -n "$search_root" && -n "$checkpoint_name" && -d "$search_root" ]] || return 1 + find "$search_root" -maxdepth 5 -type f -name "${checkpoint_name}.xml" 2>/dev/null | head -1 +} + +redefine_checkpoint_chain_if_needed() { + local vm_name="$1" checkpoint_file="$2" search_root="$3" + local checkpoint_name parent_name parent_file visited_key + + [[ -n "$checkpoint_file" && -f "$checkpoint_file" ]] || return 0 + checkpoint_name="$(basename "$checkpoint_file" .xml)" + visited_key="|${checkpoint_name}|" + if [[ "${REDEFINE_VISITED_CHECKPOINTS:-}" == *"$visited_key"* ]]; then + return 0 + fi + REDEFINE_VISITED_CHECKPOINTS="${REDEFINE_VISITED_CHECKPOINTS:-}${visited_key}" + + parent_name="$(get_parent_checkpoint_name_from_xml "$checkpoint_file")" + if [[ -n "$parent_name" ]]; then + parent_file="$(find_checkpoint_xml_on_nas "$search_root" "$parent_name")" + if [[ -n "$parent_file" && -f "$parent_file" ]]; then + redefine_checkpoint_chain_if_needed "$vm_name" "$parent_file" "$search_root" + else + strip_checkpoint_parent_from_xml "$checkpoint_file" + fi + fi + + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$checkpoint_name" > /dev/null 2>&1; then + return 0 + fi + if ! virsh -c qemu:///system checkpoint-create --domain "$vm_name" --xmlfile "$checkpoint_file" --redefine >> "$logFile" 2>&1; then + echo "Failed to redefine checkpoint ${checkpoint_name} on domain ${vm_name}" + cleanup + fi +} + redefine_checkpoint_if_needed() { - local vm_name="$1" - local checkpoint_file="$2" + local vm_name="$1" checkpoint_file="$2" search_root="${3:-}" if [[ -z "$PARENT_CHECKPOINT_NAME" || -z "$checkpoint_file" || ! -f "$checkpoint_file" ]]; then return fi if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$PARENT_CHECKPOINT_NAME" > /dev/null 2>&1; then return fi - if ! virsh -c qemu:///system checkpoint-create --domain "$vm_name" --xmlfile "$checkpoint_file" --redefine > /dev/null 2>&1; then - echo "Failed to redefine checkpoint $PARENT_CHECKPOINT_NAME on domain $vm_name" - cleanup - fi + REDEFINE_VISITED_CHECKPOINTS="" + redefine_checkpoint_chain_if_needed "$vm_name" "$checkpoint_file" "${search_root:-$mount_point}" } parent_qcow2_bitmap_exists_on_all_disks() { @@ -782,6 +881,251 @@ EOF log -ne "Wrote RBD backup metadata to [$dest/rbd-backup.meta]" } +write_veeam_seed_metadata() { + local backup_engine="$1" + cat > "$dest/veeam-seed.meta" <> "$logFile" 2>&1; then + echo "Failed to convert staging disk $staging_path to $output" + cleanup + fi + ;; + *) + echo "Unsupported source format: $SOURCE_FORMAT" + cleanup + ;; + esac +} + +import_rbd_seed_disk() { + local staging_path="$1" + local output="$2" + local disk_uri="$3" + + if ! qemu-img convert -p -O raw "$staging_path" "$output" >> "$logFile" 2>&1; then + echo "Failed to convert staging disk $staging_path to $output" + cleanup + fi + + parse_rbd_uri "$disk_uri" + build_rbd_cmd + if [[ -z "$RBD_IMAGE" ]]; then + echo "Unable to parse RBD disk path for seed import: $disk_uri" + cleanup + fi + + if ! timeout 30s "${RBD_CMD[@]}" snap ls "$RBD_IMAGE" 2>>"$logFile" | awk 'NR>1 {print $2}' | grep -Fxq "$CHECKPOINT_NAME"; then + if ! timeout 30s "${RBD_CMD[@]}" snap create "${RBD_IMAGE}@${CHECKPOINT_NAME}" >> "$logFile" 2>&1; then + echo "Failed to create RBD baseline snapshot ${RBD_IMAGE}@${CHECKPOINT_NAME}" + cleanup + fi + fi +} + +bootstrap_qcow2_checkpoint() { + local vm_name="$1" + + if [[ -z "$vm_name" ]]; then + log -ne "Skip checkpoint bootstrap: VM name not set" + return 0 + fi + + if ! virsh -c qemu:///system dominfo "$vm_name" > /dev/null 2>&1; then + log -ne "Skip checkpoint bootstrap: VM [$vm_name] not found in libvirt" + return 0 + fi + + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$CHECKPOINT_NAME" > /dev/null 2>&1; then + dump_checkpoint_xml "$vm_name" + return 0 + fi + + echo "" > "$dest/backup.xml" + echo "" >> "$dest/backup.xml" + echo "$CHECKPOINT_NAME" > "$dest/checkpoint.xml" + local index=0 + while IFS='|' read -r disk target; do + [[ -z "$disk" ]] && continue + local backup_file + backup_file=$(get_backup_file_by_index "$index" "$(basename "$target").qcow2") + echo "" >> "$dest/backup.xml" + echo "" >> "$dest/checkpoint.xml" + index=$((index + 1)) + done < <(virsh -c qemu:///system domblklist "$vm_name" --details 2>/dev/null | awk '/disk/ {print $3 "|" $4}') + echo "" >> "$dest/backup.xml" + echo "" >> "$dest/checkpoint.xml" + + if ! virsh -c qemu:///system backup-begin --domain "$vm_name" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" >> "$logFile" 2>&1; then + echo "Failed to bootstrap checkpoint on VM $vm_name" + cleanup + fi + + while true; do + local status + status=$(virsh -c qemu:///system domjobinfo "$vm_name" --completed --keep-completed 2>/dev/null | awk '/Job type:/ {print $3}') + case "$status" in + Completed) break ;; + Failed) + echo "Virsh checkpoint bootstrap job failed for VM $vm_name" + cleanup ;; + esac + sleep 5 + done + + dump_checkpoint_xml "$vm_name" + rm -f "$dest/backup.xml" "$dest/checkpoint.xml" + log -ne "Bootstrapped libvirt checkpoint [$CHECKPOINT_NAME] on VM [$vm_name]" +} + +# Veeam seed: NAS qcow2 already exists from staging convert — only create live-disk bitmap checkpoint. +bootstrap_qcow2_checkpoint_seed() { + local vm_name="$1" + local -a diskspec_args=() + local disk + + if [[ -z "$vm_name" ]]; then + log -ne "Skip checkpoint bootstrap: VM name not set" + return 0 + fi + + if ! virsh -c qemu:///system dominfo "$vm_name" > /dev/null 2>&1; then + log -ne "Skip checkpoint bootstrap: VM [$vm_name] not found in libvirt" + return 0 + fi + + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$CHECKPOINT_NAME" > /dev/null 2>&1; then + dump_checkpoint_xml "$vm_name" + return 0 + fi + + while IFS='|' read -r disk _target; do + [[ -z "$disk" ]] && continue + diskspec_args+=(--diskspec "${disk},bitmap=${CHECKPOINT_NAME}") + done < <(virsh -c qemu:///system domblklist "$vm_name" --details 2>/dev/null | awk '/disk/ {print $3 "|" $4}') + + if [[ ${#diskspec_args[@]} -eq 0 ]]; then + echo "No disks found for checkpoint bootstrap on VM $vm_name" + cleanup + fi + + if ! virsh -c qemu:///system checkpoint-create-as "$vm_name" "$CHECKPOINT_NAME" \ + "${diskspec_args[@]}" >> "$logFile" 2>&1; then + log -ne "Warn: checkpoint-create-as failed for seed on VM [$vm_name] (glue-gfs/raw may not support bitmap); continuing without checkpoint" + return 0 + fi + + dump_checkpoint_xml "$vm_name" + strip_checkpoint_parent_from_xml "$dest/checkpoints/$CHECKPOINT_NAME.xml" + log -ne "Bootstrapped libvirt checkpoint (seed) [$CHECKPOINT_NAME] on VM [$vm_name]" +} + +import_veeam_seed() { + log -ne "Entered import_veeam_seed staging=[$STAGING_DISK_PATHS] backupDir=[$BACKUP_DIR]" + mount_operation + mkdir -p "$dest" "$dest/checkpoints" || { echo "Failed to create backup directory $dest"; exit 1; } + + if [[ -z "$STAGING_DISK_PATHS" ]]; then + echo "Staging disk paths are required for import-veeam-seed" + cleanup + fi + + local use_rbd=0 + if [[ -n "$DISK_PATHS" ]]; then + while IFS= read -r disk_path; do + [[ -z "$disk_path" ]] && continue + if is_rbd_disk_path "$disk_path"; then + use_rbd=1 + break + fi + done < <(split_csv "$DISK_PATHS") + fi + + local backup_engine="QCOW2" + [[ $use_rbd -eq 1 ]] && backup_engine="RBD_DIFF" + + local index=0 + local staging_index=0 + while IFS= read -r staging_disk; do + [[ -z "$staging_disk" ]] && continue + local backup_file live_disk="" + if [[ -n "$DISK_PATHS" ]]; then + live_disk=$(split_csv "$DISK_PATHS" | sed -n "$((staging_index + 1))p") + fi + if [[ $use_rbd -eq 1 && -n "$live_disk" ]]; then + backup_file=$(get_backup_file_by_index "$index" "${live_disk##*/}.raw") + else + backup_file=$(get_backup_file_by_index "$index" "disk-${index}.qcow2") + fi + local output="$dest/$backup_file" + if [[ $use_rbd -eq 1 ]]; then + import_rbd_seed_disk "$staging_disk" "$output" "$live_disk" + else + convert_staging_disk_to_backup "$staging_disk" "$output" + fi + stat -c %s "$output" + index=$((index + 1)) + staging_index=$((staging_index + 1)) + done < <(split_csv "$STAGING_DISK_PATHS") + + backup_domain_information "$VM" + + if [[ "$backup_engine" == "RBD_DIFF" ]]; then + write_rbd_backup_metadata "FULL" "$CHECKPOINT_NAME" "" + cat > "$dest/checkpoints/${CHECKPOINT_NAME}.meta" < -v|--vm -t -s -m -p -b -c -r -i -j -f -d -q|--quiesce -x|--forced " @@ -871,6 +1215,26 @@ while [[ $# -gt 0 ]]; do shift shift ;; + --staging-disks) + STAGING_DISK_PATHS="$2" + shift + shift + ;; + --source-format) + SOURCE_FORMAT="$2" + shift + shift + ;; + --veeam-restore-point) + VEEAM_RESTORE_POINT_ID="$2" + shift + shift + ;; + --bootstrap-checkpoint) + BOOTSTRAP_CHECKPOINT="$2" + shift + shift + ;; -h|--help) usage shift @@ -895,4 +1259,12 @@ elif [ "$OP" = "delete" ]; then delete_backup elif [ "$OP" = "stats" ]; then get_backup_stats +elif [ "$OP" = "import-veeam-seed" ]; then + import_veeam_seed +fi + +# Optional Mold->Veeam trigger (bidirectional mode C). Best-effort: never affects backup result. +VEEAM_TRIGGER_HOOK="${VEEAM_TRIGGER_HOOK:-/etc/ablestack/veeam/mold-veeam-trigger-hook.sh}" +if [[ -x "$VEEAM_TRIGGER_HOOK" ]]; then + "$VEEAM_TRIGGER_HOOK" "$OP" "$VM" "$BACKUP_TYPE" >/dev/null 2>&1 || true fi diff --git a/scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh b/scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh new file mode 100755 index 000000000000..c90dbb2b516f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/postinstall_mold_backup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Entry point for mold-agent or cloudstack-common RPM %post. +set -e +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -x "${DIR}/veeam/install.sh" ]]; then + "${DIR}/veeam/install.sh" +fi +# veeam/install.sh deploys /root/.ssh/ablestack.key from ablestack.key.default when missing (chmod 600). +# mold-backup hooks decrypt MOLD_API_SECRET via mold-backup-secret.sh + MOLD_SECRET_KEY_FILE. diff --git a/scripts/vm/hypervisor/kvm/setup_agent.sh b/scripts/vm/hypervisor/kvm/setup_agent.sh index 761f1109830f..a3de9691c821 100755 --- a/scripts/vm/hypervisor/kvm/setup_agent.sh +++ b/scripts/vm/hypervisor/kvm/setup_agent.sh @@ -56,6 +56,7 @@ install_cloud_agent() { then let retry=retry-1 else + install_mold_veeam_backup_hooks break fi done @@ -69,6 +70,7 @@ install_cloud_agent() { then let retry=retry-1 else + install_mold_veeam_backup_hooks break fi @@ -82,6 +84,22 @@ install_cloud_agent() { fi } +# Install Ablestack Veeam backup pre/post hooks (idempotent). +install_mold_veeam_backup_hooks() { + local candidates=" + /usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/veeam/install.sh + $(dirname "$0")/veeam/install.sh + " + local installer + for installer in $candidates; do + if [ -x "$installer" ]; then + "$installer" && return 0 + fi + done + printf "Note: mold Veeam backup installer not found (skip)\n" + return 0 +} + install_cloud_consoleP() { local dev=$1 local retry=10 diff --git a/scripts/vm/hypervisor/kvm/veeam/README.ko.md b/scripts/vm/hypervisor/kvm/veeam/README.ko.md new file mode 100644 index 000000000000..28c9cf65d0b7 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/README.ko.md @@ -0,0 +1,80 @@ +# Ablestack Veeam + Mold 백업 (호스트 / datadisk) + +## 핵심 + +| 항목 | 값 | +|------|-----| +| 모드 | **host** — KVM Agent + `/tmp/mold/veeam` (게스트별 Veeam Job 없음) | +| Mold 백업 오퍼링 | `BACKUP_OFFERING_NAME` (예: `VeeamBackup`) | +| Job conf | Veeam Job 이름과 맞추면 편함 (`--job-name`) | +| FLR→Mold | KVM `mold-veeam-restore-agent.timer` → `restoreBackup` | + +## 최초 설정 (KVM) + +```bash +/etc/ablestack/veeam/veeam_config.sh \ + --job-name "Mold ablecube31-2" \ + --offering-name "VeeamBackup" \ + --mold-url "http://10.10.31.20:8080/client/api" \ + --api-key KEY --api-secret 'SECRET' \ + --zone-id ZONE_UUID \ + --vm-include "*" \ + --kvm-host "10.10.31.2" \ + --backup-mode host \ + --install +``` + +Datadisk 한 번에: + +```bash +bash /etc/ablestack/veeam/setup-datadisk-veeam-backup.sh --env-file /etc/ablestack/veeam/mold-backup.env +``` + +## 백업 / 복원 + +```bash +# 전체(실행 중 VM) 또는 VM_INCLUDE에 맞춘 백업 +/etc/ablestack/veeam/mold-backup.sh backup-full --job "Mold ablecube31-2" + +# 백업 ID 확인 후 개별 복원 (VM 정지 필요) +/etc/ablestack/veeam/mold-backup.sh list-backups --job "Mold ablecube31-2" +virsh -c qemu:///system shutdown i-2-11-VM +/etc/ablestack/veeam/mold-backup.sh restore --job "Mold ablecube31-2" \ + --vm-name i-2-11-VM --backup-id "" +``` + +## Veeam Job (KVM file-level) + +```bash +# mold-backup.env 에 VEEAM_SSH_HOST=10.10.254.246 등 설정 후 +bash /etc/ablestack/veeam/push-to-veeam.sh --env-file /etc/ablestack/veeam/mold-backup.env +``` + +| 단계 | 내용 | +|------|------| +| PS1 배포 | `C:\ProgramData\Mold\backup\veeam\` | +| `setup-veeam-mold-job.ps1` | SelectedFiles `/tmp/mold/veeam` + Pre/Post | +| Pre/Post | KVM `ablestack_veeam_pre/post_notify.sh` → Mold API | + +## Veeam UI Guest files → Mold + +```bash +bash /etc/ablestack/veeam/enable-veeam-mold-restore.sh --vm-include 'i-2-XX-VM' +# timer: mold-veeam-restore-agent.timer → mold-backup.sh restore-watch --trigger-mold +``` + +## 배포 경로 + +| 목적 | 스크립트 | +|------|----------| +| KVM 훅 설치 | `push-to-kvm.sh` → `install.sh` | +| 설정 생성 | `veeam_config.sh` / `setup-datadisk-veeam-backup.sh` | +| Veeam PS1/Job | `push-to-veeam.sh` | +| UI FLR→Mold | `enable-veeam-mold-restore.sh` | + +## 유지 파일 (요약) + +- **KVM 코어**: `mold-backup.sh`, `mold-backup.lib.sh`, `veeam_config.sh`, `install.sh`, pre/post notify, `ablestack_cvtbackup.sh` +- **FLR**: `enable-veeam-mold-restore.sh`, `mold-veeam-restore-agent.*` +- **Veeam PS1**: `setup-veeam-mold-job.ps1`, `create-veeam-agent-job.ps1`, `install-veeam-job.ps1`, `veeam-job-*.ps1` +- **패키지/에이전트**: 상위 `ablestack_nasbackup.sh`, `postinstall_mold_backup.sh`, `setup_agent.sh` diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack.key.default b/scripts/vm/hypervisor/kvm/veeam/ablestack.key.default new file mode 100644 index 000000000000..99ad546ab3f5 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack.key.default @@ -0,0 +1 @@ +QWJsZWNsb3VkMSE= diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh new file mode 100644 index 000000000000..08366ff05aab --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_cvtbackup.sh @@ -0,0 +1,485 @@ +#!/usr/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +# CloudStack B&R Commvault Backup and Recovery Tool for KVM + +# TODO: do libvirt/logging etc checks + +### Declare variables ### + +OP="" +VM="" +BACKUP_DIR="" +DISK_PATHS="" +QUIESCE="" +BACKUP_TYPE="FULL" +CHECKPOINT_NAME="" +PARENT_BACKUP_DIR="" +PARENT_CHECKPOINT_NAME="" +PARENT_CHECKPOINT_PATH="" +BACKUP_FILES="" +FORCED="false" +logFile="/var/log/cloudstack/agent/agent.log" + +EXIT_CLEANUP_FAILED=20 + +log() { + [[ "$verb" -eq 1 ]] && builtin echo "$@" + if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then + builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile" + else + builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile" + fi +} + +vercomp() { + local IFS=. + local i ver1=($1) ver2=($3) + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 0 + elif ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 + fi + done + return 0 +} + +sanity_checks() { + hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}') + libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1) + apiVersion=$(virsh version | grep API | awk '{print $(NF)}') + + vercomp "$hvVersion" ">=" "4.2.0" + hvStatus=$? + vercomp "$libvVersion" ">=" "7.2.0" + libvStatus=$? + + if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then + log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" + else + echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" + exit 1 + fi +} + +cleanup() { + local status=0 + rm -rf "$dest" || { echo "Failed to delete $dest"; status=1; } + if [[ -e "$dest" ]]; then + echo "Backup directory still exists after cleanup: $dest" + status=1 + fi + if [[ $status -ne 0 ]]; then + echo "Backup cleanup failed" + exit $EXIT_CLEANUP_FAILED + fi +} + +split_csv() { + tr ',' '\n' <<< "$1" +} + +is_rbd_disk_path() { + local disk_path="$1" + [[ "$disk_path" == rbd:* || "$disk_path" == rbd/* ]] +} + +get_backup_file_by_index() { + local index="$1" + local fallback="$2" + if [[ -z "$BACKUP_FILES" ]]; then + echo "$fallback" + return + fi + local current=0 + while IFS= read -r value; do + if [[ "$current" -eq "$index" ]]; then + echo "$value" + return + fi + current=$((current + 1)) + done < <(split_csv "$BACKUP_FILES") + echo "$fallback" +} + +dump_checkpoint_xml() { + local vm_name="$1" + if [[ -n "$CHECKPOINT_NAME" ]]; then + virsh -c qemu:///system checkpoint-dumpxml --domain "$vm_name" --checkpointname "$CHECKPOINT_NAME" --no-domain > "$dest/checkpoints/$CHECKPOINT_NAME.xml" 2>/dev/null || true + fi +} + +redefine_checkpoint_if_needed() { + local vm_name="$1" + local checkpoint_file="$2" + if [[ -z "$PARENT_CHECKPOINT_NAME" || -z "$checkpoint_file" || ! -f "$checkpoint_file" ]]; then + return + fi + if virsh -c qemu:///system checkpoint-info --domain "$vm_name" --checkpointname "$PARENT_CHECKPOINT_NAME" > /dev/null 2>&1; then + return + fi + if ! virsh -c qemu:///system checkpoint-create --domain "$vm_name" --xmlfile "$checkpoint_file" --redefine > /dev/null 2>&1; then + echo "Failed to redefine checkpoint $PARENT_CHECKPOINT_NAME on domain $vm_name" + exit 1 + fi +} + + +parse_rbd_uri() { + local uri="$1" + log -ne "parse_rbd_uri called with uri=[$uri]" + + RBD_IMAGE="" + RBD_MON_HOST="" + RBD_USER="" + RBD_KEY="" + + if [[ "$uri" == rbd:* ]]; then + local payload="${uri#rbd:}" + RBD_IMAGE="${payload%%:*}" + + if [[ "$uri" =~ :mon_host=([^:]*) ]]; then + RBD_MON_HOST="${BASH_REMATCH[1]}" + RBD_MON_HOST="${RBD_MON_HOST//\\;/,}" + RBD_MON_HOST="${RBD_MON_HOST//\\:/:}" + fi + + if [[ "$uri" =~ :id=([^:]*) ]]; then + RBD_USER="${BASH_REMATCH[1]}" + fi + + if [[ "$uri" =~ :key=([^:]*) ]]; then + RBD_KEY="${BASH_REMATCH[1]}" + fi + elif [[ "$uri" == rbd/* ]]; then + RBD_IMAGE="$uri" + else + echo "Invalid RBD disk path: $uri" + cleanup + fi + + if [[ -z "$RBD_IMAGE" ]]; then + echo "Failed to parse RBD image from uri: $uri" + cleanup + fi + + log -ne "Parsed RBD uri -> IMAGE=[$RBD_IMAGE], MON=[$RBD_MON_HOST], USER=[$RBD_USER]" +} + +build_rbd_cmd() { + RBD_CMD=(rbd) + if [[ -n "$RBD_MON_HOST" ]]; then + RBD_CMD+=(-m "$RBD_MON_HOST") + fi + if [[ -n "$RBD_USER" ]]; then + RBD_CMD+=(--id "$RBD_USER") + fi + if [[ -n "$RBD_KEY" ]]; then + RBD_CMD+=(--key "$RBD_KEY") + fi +} + +write_rbd_backup_metadata() { + local backup_type="$1" + local checkpoint_name="$2" + local parent_checkpoint_name="$3" + + cat > "$dest/rbd-backup.meta" < "$dest/checkpoints/$checkpoint_name.meta" < /dev/null 2>&1; then + virsh -c qemu:///system dumpxml "$vm_name" > "$dest/domain-config.xml" 2>/dev/null || true + virsh -c qemu:///system dominfo "$vm_name" > "$dest/dominfo.xml" 2>/dev/null || true + virsh -c qemu:///system domiflist "$vm_name" > "$dest/domiflist.xml" 2>/dev/null || true + virsh -c qemu:///system domblklist "$vm_name" > "$dest/domblklist.xml" 2>/dev/null || true + + if [[ -n "$CHECKPOINT_NAME" ]]; then + cat > "$dest/checkpoints/$CHECKPOINT_NAME.meta" <" > "$dest/backup.xml" + if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_NAME" ]]; then + echo "$PARENT_CHECKPOINT_NAME" >> "$dest/backup.xml" + fi + echo "" >> "$dest/backup.xml" + local index=0 + for disk in $(virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '/disk/{print $3}'); do + local target_file="$dest/$(get_backup_file_by_index "$index")" + echo "" >> "$dest/backup.xml" + index=$((index + 1)) + done + echo "" >> "$dest/backup.xml" + + echo "$CHECKPOINT_NAME" > "$dest/checkpoint.xml" + for disk in $(virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '/disk/{print $3}'); do + echo "" >> "$dest/checkpoint.xml" + done + echo "" >> "$dest/checkpoint.xml" + + local thaw=0 + if [[ ${QUIESCE} == "true" ]]; then + if virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-freeze"}' > /dev/null 2>/dev/null; then + thaw=1 + fi + fi + + local backup_begin=0 + local backup_begin_output="" + if backup_begin_output=$(virsh -c qemu:///system backup-begin --domain "$VM" --backupxml "$dest/backup.xml" --checkpointxml "$dest/checkpoint.xml" 2>&1); then + backup_begin=1 + fi + + if [[ $thaw -eq 1 ]]; then + virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-thaw"}' > /dev/null 2>&1 || true + fi + + if [[ $backup_begin -ne 1 ]]; then + echo "Failed to start libvirt backup for VM [$VM]: ${backup_begin_output:-Unknown error}" + cleanup + exit 1 + fi + + backup_domain_information "$VM" + + while true; do + status=$(virsh -c qemu:///system domjobinfo "$VM" --completed --keep-completed | awk '/Job type:/ {print $3}') + case "$status" in + Completed) break ;; + Failed) echo "Virsh backup job failed"; cleanup ;; + esac + sleep 5 + done + + dump_checkpoint_xml "$VM" + rm -f "$dest/backup.xml" "$dest/checkpoint.xml" + sync +} + +backup_rbd_volumes() { + mkdir -p "$dest/checkpoints" || { echo "Failed to create backup directory $dest"; exit 1; } + backup_domain_information "$VM" + local index=0 + while IFS= read -r disk_path; do + [[ -z "$disk_path" ]] && continue + local created_snapshot="" + log -ne "Loop disk raw value=[$disk_path]" + parse_rbd_uri "$disk_path" + build_rbd_cmd + log -ne "Built RBD command: ${RBD_CMD[*]}" + + local output_file="$dest/$(get_backup_file_by_index "$index" "${RBD_IMAGE##*/}.raw")" + log -ne "Starting RBD backup for disk path [$disk_path], resolved image [$RBD_IMAGE], output [$output_file]" + + if ! timeout 30s "${RBD_CMD[@]}" info "$RBD_IMAGE" >> "$logFile" 2>&1; then + echo "Failed to access RBD image $RBD_IMAGE" + cleanup + fi + + if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_NAME" ]]; then + if ! timeout 30s "${RBD_CMD[@]}" snap ls "$RBD_IMAGE" 2>>"$logFile" | awk 'NR>1 {print $2}' | grep -Fxq "$PARENT_CHECKPOINT_NAME"; then + echo "Parent RBD snapshot ${RBD_IMAGE}@${PARENT_CHECKPOINT_NAME} not found for incremental backup" + cleanup + fi + fi + + if ! timeout 30s "${RBD_CMD[@]}" snap create "${RBD_IMAGE}@${CHECKPOINT_NAME}" >> "$logFile" 2>&1; then + echo "Failed to create RBD snapshot ${RBD_IMAGE}@${CHECKPOINT_NAME}" + cleanup + fi + created_snapshot="${RBD_IMAGE}@${CHECKPOINT_NAME}" + + if [[ "$BACKUP_TYPE" == "INCREMENTAL" && -n "$PARENT_CHECKPOINT_NAME" ]]; then + if ! timeout 6h "${RBD_CMD[@]}" export-diff --from-snap "$PARENT_CHECKPOINT_NAME" "${RBD_IMAGE}@${CHECKPOINT_NAME}" "$output_file" >> "$logFile" 2>&1; then + echo "Failed to export incremental RBD diff for ${RBD_IMAGE}@${CHECKPOINT_NAME}" + [[ -n "$created_snapshot" ]] && "${RBD_CMD[@]}" snap rm "$created_snapshot" >> "$logFile" 2>&1 || true + cleanup + fi + else + if ! timeout 6h "${RBD_CMD[@]}" export "${RBD_IMAGE}@${CHECKPOINT_NAME}" "$output_file" >> "$logFile" 2>&1; then + echo "Failed to export full RBD snapshot ${RBD_IMAGE}@${CHECKPOINT_NAME}" + [[ -n "$created_snapshot" ]] && "${RBD_CMD[@]}" snap rm "$created_snapshot" >> "$logFile" 2>&1 || true + cleanup + fi + fi + + log -ne "Finished exporting backup file [$output_file] size=[$(stat -c %s "$output_file" 2>/dev/null)]" + index=$((index + 1)) + done < <(split_csv "$DISK_PATHS") + + write_rbd_backup_metadata "$BACKUP_TYPE" "$CHECKPOINT_NAME" "$PARENT_CHECKPOINT_NAME" + write_rbd_checkpoint_metadata "$CHECKPOINT_NAME" "$PARENT_CHECKPOINT_NAME" +} + +has_child_backup() { + local checkpoint_name="$1" + [[ -z "$checkpoint_name" ]] && return 1 + grep -R -q "^parent_checkpoint_name=$checkpoint_name$" "$(dirname "$dest")"/*/rbd-backup.meta 2>/dev/null +} + +delete_rbd_snapshot_if_unreferenced() { + local disk_paths="$1" + local checkpoint_name="$2" + + [[ -z "$checkpoint_name" ]] && return 0 + + if has_child_backup "$checkpoint_name"; then + log -ne "Skip snapshot delete [$checkpoint_name] (child exists)" + return 0 + fi + + while IFS= read -r disk_path; do + [[ -z "$disk_path" ]] && continue + parse_rbd_uri "$disk_path" + build_rbd_cmd + + if timeout 30s "${RBD_CMD[@]}" snap ls "$RBD_IMAGE" 2>/dev/null | awk 'NR>1 {print $2}' | grep -Fxq "$checkpoint_name"; then + log -ne "Deleting snapshot [${RBD_IMAGE}@${checkpoint_name}]" + "${RBD_CMD[@]}" snap rm "${RBD_IMAGE}@${checkpoint_name}" >> "$logFile" 2>&1 || true + fi + done < <(split_csv "$disk_paths") +} + +delete_backup() { + if [[ -f "$dest/rbd-backup.meta" ]]; then + source "$dest/rbd-backup.meta" + + log -ne "Deleting backup with metadata [$dest]" + + if [[ "$FORCED" != "true" ]] && has_child_backup "$checkpoint_name"; then + echo "Cannot delete backup [$backup_dir]: child backup exists" + exit 1 + fi + + delete_rbd_snapshot_if_unreferenced "$disk_paths" "$checkpoint_name" + elif [[ -n "$CHECKPOINT_NAME" && -n "$DISK_PATHS" ]]; then + log -ne "Deleting backup using command metadata [$dest]" + delete_rbd_snapshot_if_unreferenced "$DISK_PATHS" "$CHECKPOINT_NAME" + fi + + rm -frv "$dest" + sync +} + +usage() { + echo "" + echo "Usage: $0 -o -v|--vm -p -b -c -r -i -j -f -d -q|--quiesce " + echo "" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + -o|--operation) OP="$2"; shift; shift ;; + -v|--vm) VM="$2"; shift; shift ;; + -p|--path) BACKUP_DIR="$2"; shift; shift ;; + -b|--backuptype) BACKUP_TYPE="$2"; shift; shift ;; + -c|--checkpoint) CHECKPOINT_NAME="$2"; shift; shift ;; + -r|--parentbackup) PARENT_BACKUP_DIR="$2"; shift; shift ;; + -i|--parentcheckpoint) PARENT_CHECKPOINT_NAME="$2"; shift; shift ;; + -j|--parentcheckpointpath) PARENT_CHECKPOINT_PATH="$2"; shift; shift ;; + -f|--backupfiles) BACKUP_FILES="$2"; shift; shift ;; + -q|--quiesce) QUIESCE="$2"; shift; shift ;; + -d|--diskpaths) DISK_PATHS="$2"; shift; shift ;; + -x|--forced) FORCED="$2"; shift; shift ;; + -h|--help) usage ;; + *) echo "Invalid option: $1"; usage ;; + esac +done + +if [[ -z "$BACKUP_DIR" ]]; then + echo "Backup path (-p|--path) is required" + exit 1 +fi + +dest="$BACKUP_DIR" +sanity_checks + +log -ne "ablestack_cvtbackup.sh start op=[$OP] vm=[$VM] backupDir=[$BACKUP_DIR] backupType=[$BACKUP_TYPE] checkpoint=[$CHECKPOINT_NAME] parentBackup=[$PARENT_BACKUP_DIR] parentCheckpoint=[$PARENT_CHECKPOINT_NAME] diskPaths=[$DISK_PATHS] backupFiles=[$BACKUP_FILES]" + +if [[ "$OP" == "backup-running" ]]; then + backup_running_vm +elif [[ "$OP" == "backup-rbd" ]]; then + backup_rbd_volumes +elif [[ "$OP" == "delete" ]]; then + delete_backup +else + echo "Unsupported operation: $OP" + exit 1 +fi diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh new file mode 100755 index 000000000000..997649254ff4 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_post_notify.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup bpend_notify equivalent for Veeam + Mold. +# Args: CLIENT JOB [SCHEDULE] [TYPE] + +set -euo pipefail + +CLIENT="${1:-$(hostname -s)}" +JOB="${2:-${VEEAM_JOB_NAME:-}}" +SCHEDULE="${3:-${VEEAM_SCHEDULE_NAME:-default}}" +TYPE="${4:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +export VEEAM_JOB_NAME="$JOB" +mold_backup_post_notify "$CLIENT" "$JOB" "$SCHEDULE" "$TYPE" diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh new file mode 100755 index 000000000000..91114ffd6da5 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_pre_notify.sh @@ -0,0 +1,35 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup bpstart_notify equivalent for Veeam + Mold. +# Args: CLIENT JOB [SCHEDULE] [TYPE] +# Log to file only; do not write to stdout (Veeam/NetBackup policy requirement). + +set -euo pipefail + +CLIENT="${1:-$(hostname -s)}" +JOB="${2:-${VEEAM_JOB_NAME:-}}" +SCHEDULE="${3:-${VEEAM_SCHEDULE_NAME:-default}}" +TYPE="${4:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +export VEEAM_JOB_NAME="$JOB" +mold_backup_pre_notify "$CLIENT" "$JOB" "$SCHEDULE" "$TYPE" diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh new file mode 100644 index 000000000000..909f4f094b2a --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_event.sh @@ -0,0 +1,68 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Restore event ingress (Veeam PS1 SSH push, restore-watch agent, or manual). +# +# Events: +# veeam.restore.completed Veeam UI/Agent restore session finished → Mold datadisk restore +# mold.restore.manual Operator/Mold UI path with BACKUP_ID already set +# +# Usage: +# ablestack_veeam_restore_event.sh veeam.restore.completed [detail] +# BACKUP_ID= ablestack_veeam_restore_event.sh mold.restore.manual manual + +set -euo pipefail + +EVENT="${1:-}" +SESSION_ID="${2:-}" +VM_NAME_E="${3:-}" +DETAIL="${4:-}" +JOB="${VEEAM_JOB_NAME:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +mold_backup_load_config || exit 1 +[[ -n "$JOB" ]] || JOB="${VEEAM_JOB_NAME:-}" + +case "$EVENT" in + veeam.restore.completed) + [[ -n "$SESSION_ID" && -n "$VM_NAME_E" ]] || { + mold_backup_notify_log err "restore-event: veeam.restore.completed needs session-id and vm-name" + exit 1 + } + mold_backup_handle_veeam_restore_session "$JOB" "$VM_NAME_E" "$SESSION_ID" "$DETAIL" true + ;; + mold.restore.manual) + [[ -n "$VM_NAME_E" ]] || { + mold_backup_notify_log err "restore-event: mold.restore.manual needs vm-name" + exit 1 + } + export VM_NAME="$VM_NAME_E" + [[ -n "${BACKUP_ID:-}" ]] || mold_backup_die "mold.restore.manual requires BACKUP_ID" + mold_backup_emit_restore_event "mold.restore.requested" "$VM_NAME_E" "backup_id=${BACKUP_ID};source=manual" + mold_backup_trigger_mark "mold-restore-active" "$VM_NAME_E" + mold_backup_restore_notify "$(hostname -s)" "$JOB" + mold_backup_emit_restore_event "mold.restore.completed" "$VM_NAME_E" "backup_id=${BACKUP_ID}" + ;; + *) + mold_backup_notify_log err "restore-event: unknown event '${EVENT}'" + exit 1 + ;; +esac diff --git a/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh new file mode 100755 index 000000000000..b371629eef79 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/ablestack_veeam_restore_notify.sh @@ -0,0 +1,32 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup restore_notify equivalent for Veeam + Mold. +# Set BACKUP_ID (and optional VM_NAME / VM_UUID) in job conf or environment. + +set -euo pipefail + +CLIENT="${1:-$(hostname -s)}" +JOB="${2:-${VEEAM_JOB_NAME:-}}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +export VEEAM_JOB_NAME="$JOB" +mold_backup_restore_notify "$CLIENT" "$JOB" diff --git a/scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 b/scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 new file mode 100644 index 000000000000..28b9ac393c04 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/create-veeam-agent-job.ps1 @@ -0,0 +1,627 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Create (or update) a Veeam Linux Agent file-level backup job for Mold integration. +# Idempotent: safe to re-run. Registers Pre/Post bash hooks via install-veeam-job.ps1. +# +# Prerequisite: Veeam Agent for Linux on KVM host, connected to this B&R server. +# +# Example (run on Veeam B&R server as Administrator, PowerShell 7+): +# .\create-veeam-agent-job.ps1 ` +# -JobName "Mold KVM Backup" ` +# -KvmHost "10.10.31.2" ` +# -AgentHostName "ablecube31-2" ` +# -BackupPath "/tmp/mold/veeam" ` +# -RepositoryName "Default Backup Repository" + +param( + [Parameter(Mandatory = $true)] + [string]$JobName, + + [Parameter(Mandatory = $true)] + [string]$KvmHost, + + [string]$AgentHostName = "", + [string]$BackupPath = "/tmp/mold/veeam", + [string]$RepositoryName = "", + + [string]$InstallDir = "C:\ProgramData\Mold\backup\veeam", + # Custom PG for PowerShell job creation (not "Manually Added"). + [string]$ProtectionGroupName = "Mold KVM Agents", + + [string]$AgentPreNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh", + [string]$AgentPostNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh", + + # Linux SSH for Protection Group (required — temporary certificate fails with + # "Cannot find credentials for agent " on managed Linux agents). + [string]$KvmSshUser = "root", + [string]$KvmSshPassword = "", + + [switch]$SkipScriptRegistration, + [switch]$StartJob, + [switch]$WhatIf +) + +$ErrorActionPreference = "Stop" + +if ($PSVersionTable.PSVersion.Major -lt 7) { + throw "Veeam.Backup.PowerShell requires PowerShell 7+. Run: pwsh -File $PSCommandPath" +} + +if (-not $KvmSshPassword -and $env:MOLD_KVM_SSH_PASSWORD) { + $KvmSshPassword = $env:MOLD_KVM_SSH_PASSWORD +} +if (-not $KvmSshUser) { $KvmSshUser = "root" } + +function Write-MoldVeeamInfo([string]$Message) { Write-Host "[mold-veeam] $Message" } +function Write-MoldVeeamWarn([string]$Message) { Write-Warning "[mold-veeam] $Message" } + +function Connect-MoldVeeamSession { + if (-not (Get-Command Connect-VBRServer -ErrorAction SilentlyContinue)) { return } + $sessions = @() + try { $sessions = @(Get-VBRServerSession -ErrorAction SilentlyContinue) } catch { } + if ($sessions.Count -eq 0) { + Write-MoldVeeamInfo "Connecting to local Veeam B&R (localhost)..." + Connect-VBRServer -Server localhost | Out-Null + } +} + +function Invoke-MoldVeeamCmdlet { + param( + [scriptblock]$Command, + [string]$Context = "Veeam cmdlet" + ) + try { + return & $Command + } catch { + if ($_.Exception.Message -match 'Identity service') { + throw @" +$Context failed: Veeam Identity service is unreachable. + +On this B&R server (Administrator pwsh): + Get-Service VeeamBackupSvc, VeeamBrokerSvc | Restart-Service + Connect-VBRServer -Server localhost + Get-VBRProtectionGroup | Select-Object -First 3 Name + +Then re-run setup-veeam-mold-job.ps1 +"@ + } + throw + } +} + +function Get-MoldVeeamProtectionGroupByName { + param([string]$Name) + if (-not $Name) { return $null } + try { + return Get-VBRProtectionGroup -Name $Name -ErrorAction Stop + } catch { + if ($_.Exception.Message -match 'does not exist|not found|Cannot find') { + return $null + } + if ($_.Exception.Message -match 'Identity service') { + throw @" +Get-VBRProtectionGroup -Name '$Name' failed: Veeam Identity service is unreachable. + +On this B&R server (Administrator pwsh): + Get-Service VeeamBackupSvc, VeeamBrokerSvc | Restart-Service + Connect-VBRServer -Server localhost + Get-VBRProtectionGroup | Select-Object -First 3 Name + +Then re-run setup-veeam-mold-job.ps1 +"@ + } + throw + } +} + +function Get-MoldVeeamPropertyValue { + param($Object, [string[]]$Names) + foreach ($name in $Names) { + if ($null -eq $Object) { continue } + if ($Object.PSObject.Properties.Name -contains $name) { + $val = $Object.$name + if ($null -ne $val -and "$val" -ne "") { return "$val" } + } + } + return "" +} + +function Test-MoldVeeamHostMatch { + param($Computer, [string]$HostIp, [string]$HostName) + $candidates = @( + (Get-MoldVeeamPropertyValue $Computer @("Name", "ComputerName", "HostName", "DnsName")) + (Get-MoldVeeamPropertyValue $Computer @("IPAddress", "IpAddress", "Address")) + (Get-MoldVeeamPropertyValue $Computer @("DisplayName", "Description")) + ) | Where-Object { $_ -ne "" } + + foreach ($c in $candidates) { + if ($HostName -and ($c -eq $HostName -or $c -like "*$HostName*")) { return $true } + if ($HostIp -and ($c -eq $HostIp -or $c -like "*$HostIp*")) { return $true } + } + return $false +} + +function Get-MoldVeeamDiscoveredComputers { + param([string]$GroupName = "") + + if ($GroupName) { + $group = Get-MoldVeeamProtectionGroupByName -Name $GroupName + if ($group) { + return @( + Invoke-MoldVeeamCmdlet -Context "Get-VBRDiscoveredComputer (group $GroupName)" { + Get-VBRDiscoveredComputer -ProtectionGroup $group -ErrorAction SilentlyContinue + } + ) + } + Write-MoldVeeamWarn "Protection group not found: $GroupName" + } + + return @( + Invoke-MoldVeeamCmdlet -Context "Get-VBRDiscoveredComputer" { + Get-VBRDiscoveredComputer -ErrorAction SilentlyContinue + } + ) +} + +function Format-MoldVeeamDiscoveredComputerLine { + param($Computer) + $name = Get-MoldVeeamPropertyValue $Computer @("Name", "ComputerName", "HostName") + $ip = Get-MoldVeeamPropertyValue $Computer @("IPAddress", "IpAddress", "Address") + if ($name -and $ip) { return "${name} (${ip})" } + if ($name) { return $name } + if ($ip) { return $ip } + return ($Computer | Out-String).Trim() +} + +function Find-MoldVeeamDiscoveredComputer { + param([string]$HostIp, [string]$HostName, [string]$GroupName) + + $computers = @(Get-MoldVeeamDiscoveredComputers -GroupName $GroupName) + + $match = $computers | Where-Object { Test-MoldVeeamHostMatch -Computer $_ -HostIp $HostIp -HostName $HostName } | Select-Object -First 1 + if ($match) { return $match } + + if ($HostName) { + $match = $computers | Where-Object { + $n = Get-MoldVeeamPropertyValue $_ @("Name", "ComputerName", "HostName") + $n -and ($n -eq $HostName) + } | Select-Object -First 1 + if ($match) { return $match } + } + + return $null +} + +function Test-MoldVeeamHostInContainer { + param($Container, [string[]]$Targets) + $hostNames = @() + if ($Container -and $Container.CustomCredentials) { + foreach ($c in @($Container.CustomCredentials)) { + $h = Get-MoldVeeamPropertyValue $c @("HostName", "Name") + if ($h) { $hostNames += $h } + } + } + foreach ($t in $Targets) { + if ($hostNames -contains $t) { return $true } + if (($hostNames | Where-Object { $_ -like "*$t*" }).Count -gt 0) { return $true } + } + return $false +} + +function Ensure-MoldVeeamLinuxKvmCredentialRecord { + param( + [string]$User, + [string]$Password + ) + if (-not $Password) { + throw "KvmSshPassword is required (KVM_SSH_PASSWORD in mold-backup.windows.conf or -KvmSshPassword)" + } + + $desc = "Mold KVM Linux SSH ($User)" + $records = @(Get-VBRCredentials -ErrorAction SilentlyContinue) + $match = $records | Where-Object { + "$($_.Description)" -like "*Mold KVM Linux SSH*" + } | Select-Object -First 1 + + if (-not $match) { + $match = $records | Where-Object { + (Get-MoldVeeamPropertyValue $_ @("User", "UserName")) -eq $User + } | Select-Object -First 1 + } + + if ($match) { + Write-MoldVeeamInfo "Updating Veeam credentials record for KVM Linux user: $User" + try { + Set-VBRCredentials -Credential $match -User $User -Password $Password -Description $desc -SshPort 22 | Out-Null + } catch { + Set-VBRCredentials -Credential $match -Password $Password | Out-Null + } + return $match + } + + Write-MoldVeeamInfo "Creating Veeam Linux credentials record for KVM user: $User" + return Add-VBRCredentials -Type Linux -User $User -Password $Password -Description $desc -SshPort 22 +} + +function New-MoldVeeamLinuxKvmComputerCredential { + param( + [string]$HostIp, + $VeeamCredentials, + [string]$User + ) + + $last = "" + $credName = Get-MoldVeeamPropertyValue $VeeamCredentials @("User", "UserName", "Name") + if (-not $credName) { $credName = $User } + + $attempts = @( + { New-VBRIndividualComputerCustomCredentials -HostName $HostIp -Credentials $VeeamCredentials }, + { New-VBRIndividualComputerCustomCredentials -HostName $HostIp -Credentials $credName } + ) + + foreach ($attempt in $attempts) { + try { + $cred = & $attempt + Write-MoldVeeamInfo "PG computer OK: ${User}@${HostIp}" + return $cred + } catch { + $last = $_.Exception.Message + } + } + throw "New-VBRIndividualComputerCustomCredentials failed for ${User}@${HostIp}: $last" +} + +function Ensure-MoldVeeamAgentProtectionGroup { + param( + [string]$HostIp, + [string]$HostName, + [string]$GroupName, + [string]$KvmSshUser = "root", + [string]$KvmSshPassword = "" + ) + + $targets = @($HostIp) + if ($HostName -and $HostName -ne $HostIp) { $targets += $HostName } + + $pg = Get-MoldVeeamProtectionGroupByName -Name $GroupName + + if ($KvmSshPassword) { + Write-MoldVeeamInfo "Protection group: $GroupName (Individual computers + Linux SSH for KVM)" + $veeamCredRecord = Ensure-MoldVeeamLinuxKvmCredentialRecord -User $KvmSshUser -Password $KvmSshPassword + + $targetIps = @($HostIp) + if ($pg) { + $existingIps = @( + Get-VBRDiscoveredComputer -ProtectionGroup $pg -ErrorAction SilentlyContinue | ForEach-Object { + $ip = $null + if ($_.IPAddress) { $ip = @($_.IPAddress)[0] } + if (-not $ip) { $ip = $_.Name } + $ip + } | Where-Object { $_ -match '^\d{1,3}(\.\d{1,3}){3}$' } + ) + if ($existingIps.Count -gt 0) { + Write-MoldVeeamInfo "Existing PG computers kept: $($existingIps -join ', ')" + } + $targetIps = @($existingIps + $HostIp | Select-Object -Unique) + } + + $creds = $targetIps | ForEach-Object { + New-MoldVeeamLinuxKvmComputerCredential -HostIp $_ -VeeamCredentials $veeamCredRecord -User $KvmSshUser + } + $container = New-VBRIndividualComputerContainer -CustomCredentials $creds + + if (-not $pg) { + $pg = Add-VBRProtectionGroup -Name $GroupName ` + -Description "Mold KVM Linux agents (Linux SSH credentials)" ` + -Container $container + } else { + Set-VBRProtectionGroup -ProtectionGroup $pg -Container $container | Out-Null + $pg = Get-MoldVeeamProtectionGroupByName -Name $GroupName + } + } else { + Write-MoldVeeamWarn @" +KvmSshPassword not set — using temporary certificate (often fails with: + Cannot find credentials for agent $HostIp + +Set KVM_SSH_PASSWORD in mold-backup.windows.conf and re-run setup-veeam-mold-job.ps1 +"@ + if (-not $pg) { + Write-MoldVeeamInfo "Creating protection group: $GroupName (Linux, certificate auth)" + $creds = @($HostIp) | ForEach-Object { + New-VBRIndividualComputerCustomCredentials -HostName $_ -UseTemporaryCertificate + } + $container = New-VBRIndividualComputerContainer -CustomCredentials $creds + $pg = Add-VBRProtectionGroup -Name $GroupName ` + -Description "Mold KVM Linux agents — used by setup-veeam-mold-job.ps1" ` + -Container $container + } elseif (-not (Test-MoldVeeamHostInContainer -Container $pg.Container -Targets $targets)) { + Write-MoldVeeamInfo "Adding $HostIp to protection group: $GroupName" + $comps = @($pg.Container.CustomCredentials) + $comps += New-VBRIndividualComputerCustomCredentials -HostName $HostIp -UseTemporaryCertificate + $newContainer = Set-VBRIndividualComputerContainer -Container $pg.Container -CustomCredentials $comps + Set-VBRProtectionGroup -ProtectionGroup $pg -Container $newContainer | Out-Null + $pg = Get-MoldVeeamProtectionGroupByName -Name $GroupName + } + } + + Write-MoldVeeamInfo "Rescanning protection group: $GroupName" + Rescan-VBREntity -Entity $pg | Out-Null + return $pg +} + +function Get-MoldVeeamSingleObject { + param($Value) + if ($null -eq $Value) { return $null } + if ($Value -is [System.Array]) { + if ($Value.Count -eq 0) { return $null } + return $Value[0] + } + return $Value +} + +function Get-MoldVeeamBackupRepository { + param([string]$PreferredName) + + $candidates = @() + if ($PreferredName) { + $candidates = @(Get-VBRBackupRepository -Name $PreferredName -ErrorAction SilentlyContinue) + } + if ($candidates.Count -eq 0) { + $candidates = @(Get-VBRBackupRepository -ErrorAction SilentlyContinue) + } + if ($candidates.Count -eq 0) { + throw "No Veeam backup repository found. Run: Get-VBRBackupRepository | Select-Object Name, Id" + } + + $pick = $candidates | Where-Object { $_.Name -like "*Default*" } | Select-Object -First 1 + if (-not $pick) { $pick = $candidates | Select-Object -First 1 } + $pick = Get-MoldVeeamSingleObject $pick + if (-not $pick -or -not $pick.Name) { + throw "Could not resolve a backup repository object." + } + + # Re-fetch by exact name — avoids pipeline/array wrapper issues with Add-VBRComputerBackupJob. + $repo = Get-MoldVeeamSingleObject (Get-VBRBackupRepository -Name $pick.Name -ErrorAction SilentlyContinue) + if (-not $repo) { $repo = $pick } + + Write-MoldVeeamInfo "Using backup repository: $($repo.Name) (type=$($repo.GetType().Name))" + return $repo +} + +function Get-MoldVeeamBackupServerName { + $server = $null + try { $server = Get-MoldVeeamSingleObject (Get-VBRServer -ErrorAction SilentlyContinue) } catch { } + if ($server) { + foreach ($key in @("Name", "DnsHostName", "DisplayName")) { + if ($server.PSObject.Properties.Name -contains $key -and $server.$key) { + return "$($server.$key)" + } + } + } + return $env:COMPUTERNAME +} + +function New-MoldVeeamComputerBackupJob { + param( + [string]$JobName, + [string]$Description, + $BackupObject, + $Scope, + $Repository + ) + + $common = @{ + OSPlatform = "Linux" + Type = "Server" + Mode = "ManagedByBackupServer" + Name = $JobName + Description = $Description + BackupObject = $BackupObject + BackupType = "SelectedFiles" + SelectedFilesOptions = $Scope + } + + try { + Add-VBRComputerBackupJob @common -BackupRepository $Repository | Out-Null + Write-MoldVeeamInfo "Created Agent backup job via -BackupRepository" + return + } catch { + $msg = $_.Exception.Message + if ($msg -notmatch 'BackupRepository|Destination') { throw } + Write-MoldVeeamWarn "Add-VBRComputerBackupJob -BackupRepository failed: $msg" + } + + $serverName = Get-MoldVeeamBackupServerName + Write-MoldVeeamInfo "Retrying with -DestinationOptions (BackupServerName=$serverName)" + $destination = New-VBRComputerDestinationOptions ` + -OSPlatform Linux ` + -BackupRepository $Repository ` + -BackupServerName $serverName + Add-VBRComputerBackupJob @common -DestinationOptions $destination -BackupRepository $Repository | Out-Null + Write-MoldVeeamInfo "Created Agent backup job via -DestinationOptions + -BackupRepository" +} + +function Assert-MoldVeeamLinuxAgentPath { + param([string]$Path) + if ($Path -match '^[A-Za-z]:[\\/]') { + throw "Linux Agent SelectedFiles path must be on the KVM host (e.g. /tmp/mold/veeam), not: $Path" + } + if (-not $Path.StartsWith("/")) { + throw "Linux Agent SelectedFiles path must be absolute (start with /): $Path" + } +} + +function New-MoldVeeamFileLevelScope { + param([string]$Path) + Assert-MoldVeeamLinuxAgentPath -Path $Path + New-VBRSelectedFilesBackupOptions -OSPlatform Linux -BackupSelectedFiles -SelectedFiles $Path +} + +function Update-MoldVeeamAgentJobScope { + param($Job, [string]$Path) + $scope = New-MoldVeeamFileLevelScope -Path $Path + Set-VBRComputerBackupJob -Job $Job -BackupType SelectedFiles -SelectedFilesOptions $scope | Out-Null + Write-MoldVeeamInfo "Updated job scope to file-level path: $Path" +} + +if (-not $AgentHostName) { + try { + $resolved = [System.Net.Dns]::GetHostEntry($KvmHost) + if ($resolved.HostName) { + $AgentHostName = ($resolved.HostName -split '\.')[0] + } + } catch { + $AgentHostName = $KvmHost + } +} + +Write-MoldVeeamInfo "Job=$JobName KVM=$KvmHost agentHost=$AgentHostName path=$BackupPath (script=mold-veeam-v3)" + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue +Connect-MoldVeeamSession + +# PowerShell cannot add individual computers from "Manually Added" — use a dedicated PG. +$protectionGroup = Ensure-MoldVeeamAgentProtectionGroup ` + -HostIp $KvmHost ` + -HostName $AgentHostName ` + -GroupName $ProtectionGroupName ` + -KvmSshUser $KvmSshUser ` + -KvmSshPassword $KvmSshPassword + +$discovered = Find-MoldVeeamDiscoveredComputer -HostIp $KvmHost -HostName $AgentHostName -GroupName $ProtectionGroupName +if ($discovered) { + $discName = Get-MoldVeeamPropertyValue $discovered @("Name", "ComputerName", "HostName") + Write-MoldVeeamInfo "Found discovered computer in '$ProtectionGroupName': $discName" +} else { + $all = @(Get-MoldVeeamDiscoveredComputers) + $knownList = if ($all.Count -gt 0) { + ($all | ForEach-Object { Format-MoldVeeamDiscoveredComputerLine $_ }) -join "`n - " + } else { + "(none)" + } + Write-MoldVeeamWarn @" +KVM not yet visible in '$ProtectionGroupName' after rescan. +Job will still be created with protection group '$ProtectionGroupName'. + +On KVM, ensure agent is connected to B&R: + veeamconfig vbrserver add --name vbr01 --address --port 10006 --login administrator --password '...' + veeamconfig vbrserver list + +Open firewall on KVM from B&R: tcp/6160, tcp/10006 + +Known discovered computers: + - $knownList +"@ +} + +$repository = Get-MoldVeeamBackupRepository -PreferredName $RepositoryName +$scope = New-MoldVeeamFileLevelScope -Path $BackupPath +$backupObject = @($protectionGroup) +Write-MoldVeeamInfo "Backup object: protection group '$ProtectionGroupName' (not Manually Added computer)" +Write-MoldVeeamInfo "Backup repository: $($repository.Name)" + +$existing = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue +if ($existing) { + Write-MoldVeeamInfo "Agent backup job already exists: $JobName" + if (-not $WhatIf) { + Update-MoldVeeamAgentJobScope -Job $existing -Path $BackupPath + } + } else { + Write-MoldVeeamInfo "Creating Agent backup job: $JobName" + if ($WhatIf) { + Write-MoldVeeamInfo "WhatIf: would call Add-VBRComputerBackupJob" + } else { + New-MoldVeeamComputerBackupJob ` + -JobName $JobName ` + -Description "Mold NAS backup + file-level staging ($BackupPath)" ` + -BackupObject $backupObject ` + -Scope $scope ` + -Repository $repository + Write-MoldVeeamInfo "Created Agent backup job: $JobName" + } + } + +if ($SkipScriptRegistration) { + Write-MoldVeeamInfo "SkipScriptRegistration set — done." + exit 0 +} + +$installScript = Join-Path $InstallDir "install-veeam-job.ps1" +if (-not (Test-Path $installScript)) { + $installScript = Join-Path $PSScriptRoot "install-veeam-job.ps1" +} +if (-not (Test-Path $installScript)) { + throw "install-veeam-job.ps1 not found under $InstallDir or $PSScriptRoot" +} + +if ($WhatIf) { + Write-MoldVeeamInfo "WhatIf: would register Pre/Post via $installScript" + exit 0 +} + +function Invoke-MoldVeeamInstallJob { + param( + [string]$ScriptPath, + [string]$JobName, + [string]$InstallDir, + [string]$KvmHost, + [string]$ProtectionGroupName, + [string]$AgentPreNotifyScript, + [string]$AgentPostNotifyScript + ) + + $installParams = @{ + JobName = $JobName + InstallDir = $InstallDir + KvmHost = $KvmHost + ProtectionGroupName = $ProtectionGroupName + AgentPreNotifyScript = $AgentPreNotifyScript + AgentPostNotifyScript = $AgentPostNotifyScript + } + + $cmd = Get-Command $ScriptPath -ErrorAction Stop + if ($cmd.Parameters.ContainsKey("LinuxAgent")) { + $installParams["LinuxAgent"] = $true + } else { + Write-MoldVeeamWarn "install-veeam-job.ps1 is outdated (no -LinuxAgent). Update from repo: curl install-veeam-job.ps1" + } + + & $ScriptPath @installParams +} + +Invoke-MoldVeeamInstallJob ` + -ScriptPath $installScript ` + -JobName $JobName ` + -InstallDir $InstallDir ` + -KvmHost $KvmHost ` + -ProtectionGroupName $ProtectionGroupName ` + -AgentPreNotifyScript $AgentPreNotifyScript ` + -AgentPostNotifyScript $AgentPostNotifyScript + +if ($StartJob) { + $jobToRun = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue + if ($jobToRun) { + Write-MoldVeeamInfo "Starting job: $JobName" + Start-VBRComputerBackupJob -Job $jobToRun | Out-Null + } +} + +Write-MoldVeeamInfo "Done. Scope=SelectedFiles path=$BackupPath (VM export only, not entire host)." +Write-MoldVeeamInfo "KVM: set VM targets — /etc/ablestack/veeam/veeam_config.sh --job-name '$JobName' --vm-include 'i-2-7-VM'" +Write-MoldVeeamInfo "Start: Veeam UI -> Jobs -> $JobName -> Start" +Write-MoldVeeamInfo "KVM log: tail -f /var/log/mold/veeam-hook.log" diff --git a/scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh b/scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh new file mode 100644 index 000000000000..8072f98bd32d --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/enable-veeam-mold-restore.sh @@ -0,0 +1,124 @@ +#!/usr/bin/bash +# Enable Veeam UI restore → Mold datadisk restore chain on this KVM host. +# +# bash enable-veeam-mold-restore.sh +# bash enable-veeam-mold-restore.sh --vm-include 'i-2-62-VM,i-2-64-VM' +# bash enable-veeam-mold-restore.sh --restore-vm i-2-62-VM # FLR 시 복원할 VM (host job, 다중 VM일 때) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +ENV_FILE="${ETC_DIR}/mold-backup.env" +HOST_CONF="${ETC_DIR}/Mold_Host_Backup.conf" +VM_INCLUDE_ARG="" +RESTORE_VM="" + +die() { echo "ERROR: $*" >&2; exit 1; } + +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" + +safe_install() { + local mode="$1" src="$2" dst="$3" + [[ -f "$src" ]] || die "Missing source file: $src" + if [[ -e "$src" && -e "$dst" ]] && [[ "$(stat -c '%d:%i' "$src" 2>/dev/null)" == "$(stat -c '%d:%i' "$dst" 2>/dev/null)" ]]; then + chmod "$mode" "$dst" 2>/dev/null || true + return 0 + fi + install -m "$mode" "$src" "$dst" +} + +resolve_bundle_file() { + local name="$1" d + for d in "${SCRIPT_DIR}" "${ETC_DIR}" "${SHARE_DIR}" "/tmp/veeam-install"; do + [[ -f "${d}/${name}" ]] && { echo "${d}/${name}"; return 0; } + done + if [[ -f "/etc/systemd/system/${name}" ]]; then + echo "/etc/systemd/system/${name}" + return 0 + fi + die "Missing ${name} — run: bash ${ETC_DIR}/install.sh or push-to-kvm.sh from Git repo" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="$2"; shift 2 ;; + --vm-include) VM_INCLUDE_ARG="$2"; shift 2 ;; + --restore-vm) RESTORE_VM="$2"; shift 2 ;; + -h|--help) + sed -n '2,12p' "$0" + exit 0 + ;; + *) die "Unknown: $1" ;; + esac +done + +[[ -f "$HOST_CONF" ]] || die "Missing ${HOST_CONF} — run setup-datadisk-veeam-backup.sh first" + +# shellcheck source=/dev/null +[[ -f "$ENV_FILE" ]] && set -a && source "$ENV_FILE" && set +a + +set_kv() { + local k="$1" v="$2" f="$HOST_CONF" + if grep -q "^${k}=" "$f" 2>/dev/null; then + sed -i "s|^${k}=.*|${k}=\"${v}\"|" "$f" + else + echo "${k}=\"${v}\"" >>"$f" + fi +} + +job="${VEEAM_JOB_NAME:-Mold ablecube31-2}" +[[ -n "$VM_INCLUDE_ARG" ]] && set_kv VM_INCLUDE "$VM_INCLUDE_ARG" +[[ -n "$RESTORE_VM" ]] && set_kv VEEAM_RESTORE_VM "$RESTORE_VM" + +set_kv RESTORE_WATCH_TRIGGER_MOLD true +set_kv VEEAM_UI_RESTORE_SOURCE mold-only +set_kv RESTORE_SOURCE mold-only +set_kv BACKUP_MODE host +set_kv VEEAM_JOB_NAME "$job" +set_kv VEEAM_RESTORE_WATCH_WINDOW_MIN "${VEEAM_RESTORE_WATCH_WINDOW_MIN:-10}" +[[ -n "${VEEAM_SSH_HOST:-}" ]] && set_kv VEEAM_SSH_HOST "$VEEAM_SSH_HOST" + +echo "=== Mold Veeam restore agent (KVM) ===" +grep -E 'VEEAM_JOB|VM_INCLUDE|VEEAM_RESTORE_VM|RESTORE_|VEEAM_SSH|BACKUP_MODE' "$HOST_CONF" || true + +bash "${SCRIPT_DIR}/install.sh" 2>/dev/null || bash "${ETC_DIR}/install.sh" 2>/dev/null || true + +mkdir -p "${ETC_DIR}/events" "${ETC_DIR}/registry" "${ETC_DIR}/state" 2>/dev/null || true +touch "${ETC_DIR}/events/restore.log" 2>/dev/null || true + +agent_sh="$(resolve_bundle_file mold-veeam-restore-agent.sh)" +agent_svc="$(resolve_bundle_file mold-veeam-restore-agent.service)" +agent_timer="$(resolve_bundle_file mold-veeam-restore-agent.timer)" + +safe_install 0755 "$agent_sh" "${ETC_DIR}/mold-veeam-restore-agent.sh" +safe_install 0644 "$agent_svc" "${ETC_DIR}/mold-veeam-restore-agent.service" +safe_install 0644 "$agent_timer" "${ETC_DIR}/mold-veeam-restore-agent.timer" +safe_install 0644 "$agent_svc" /etc/systemd/system/mold-veeam-restore-agent.service +safe_install 0644 "$agent_timer" /etc/systemd/system/mold-veeam-restore-agent.timer + +systemctl daemon-reload +# Legacy timer without --trigger-mold — disable to avoid duplicate polls. +systemctl disable mold-veeam-restore-watch.timer 2>/dev/null || true +systemctl stop mold-veeam-restore-watch.timer 2>/dev/null || true +systemctl enable mold-veeam-restore-agent.timer +systemctl start mold-veeam-restore-agent.timer + +echo "" +echo "Enabled: mold-veeam-restore-agent.timer ($(systemctl is-active mold-veeam-restore-agent.timer 2>/dev/null || echo unknown))" +echo "" +cat <&2 + echo " printf 'QWJsZWNsb3VkMSE=' > ${KEY_FILE} && chmod 600 ${KEY_FILE}" >&2 + exit 1 +fi +install -m 0600 "$src" "$KEY_FILE" +echo "Installed ${KEY_FILE} from ${src} ($(wc -c < "$KEY_FILE") bytes)" diff --git a/scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 b/scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 new file mode 100644 index 000000000000..87a7dcc380d0 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/install-veeam-job.ps1 @@ -0,0 +1,645 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Run on Veeam Backup & Replication server (Administrator PowerShell). +# 1) Copies Mold Veeam scripts + default config to ProgramData +# 2) Registers Pre-job / Post-job on a Veeam Agent backup job (Guest Processing scripts) +# +# Example: +# .\install-veeam-job.ps1 -JobName "Rocky Agent Backup" -SourceDir "\\git\...\veeam" +# .\install-veeam-job.ps1 -JobName "Rocky Agent Backup" -VmName "rocky94" -StagingPath "D:\veeam-staging\rocky94" -KvmHost "10.0.0.10" + +param( + [Parameter(Mandatory = $true)] + [string]$JobName, + + [string]$SourceDir = $PSScriptRoot, + + [string]$InstallDir = "C:\ProgramData\Mold\backup\veeam", + + [string]$VmName = "", + [string]$StagingPath = "", + [string]$KvmHost = "", + [string]$KvmSshUser = "root", + [string]$KvmSshKey = "", + + # Linux Agent jobs run pre/post scripts ON THE AGENT HOST (bash), not on this Windows server. + [switch]$LinuxAgent, + [string]$AgentPreNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh", + [string]$AgentPostNotifyScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh", + [string]$ProtectionGroupName = "Mold KVM Agents", + + # Guest VM mode: pre/post SSH to KVM for Mold; scripts run ON the guest agent. + [switch]$GuestVm, + [string]$VmLibvirtName = "", + [string]$VmIp = "", + $DiscoveredComputer = $null, + + # Only generate the pre/post wrapper scripts into $InstallDir on this Veeam + # server. Skips Veeam module load, job lookup, and job registration so you + # can point the Veeam UI Guest Processing scripts at the generated files. + [switch]$ScriptsOnly, + + [switch]$EnableRestoreScript +) + +$ErrorActionPreference = "Stop" + +function Get-MoldVeeamGuestWrapperPath { + param( + [string]$InstallDir, + [string]$Phase, + [string]$VmLibvirtName + ) + $safeVm = ($VmLibvirtName -replace '[^\w\-]', '_') + return (Join-Path $InstallDir "mold-guest-${Phase}-${safeVm}.sh") +} + +function Write-MoldAgentWrapperScript { + param( + [string]$Path, + [string]$AgentScript, + [string]$JobName + ) + $escapedJob = $JobName -replace "'", "'\''" + $escapedScript = $AgentScript -replace "'", "'\''" + $content = @" +#!/bin/bash +exec '$escapedScript' "`$(hostname -s)" '$escapedJob' +"@ + $utf8NoBom = New-Object System.Text.UTF8Encoding $false + [System.IO.File]::WriteAllText($Path, $content.Replace("`r`n", "`n"), $utf8NoBom) +} + +function Write-MoldGuestKvmSshWrapperScript { + param( + [string]$Path, + [string]$KvmHost, + [string]$KvmUser, + [string]$VmLibvirtName, + [string]$JobName, + [string]$RemoteScript, + [switch]$IsPost + ) + $escapedJob = $JobName -replace "'", "'\''" + $escapedVm = $VmLibvirtName -replace "'", "'\''" + $escapedRemote = $RemoteScript -replace "'", "'\''" + $escapedKvm = $KvmHost -replace "'", "'\''" + $escapedUser = $KvmUser -replace "'", "'\''" + $hookPhase = if ($IsPost) { "post" } else { "pre" } + if ($IsPost) { + $remoteBody = "MOLD_BACKUP_CONF=/etc/ablestack/veeam/Mold_Guest_Backup.conf VM_INCLUDE='${escapedVm}' VM_NAME='${escapedVm}' bash '${escapedRemote}' \`"`$CLIENT\`" '${escapedJob}' default" + } else { + $remoteBody = "MOLD_BACKUP_CONF=/etc/ablestack/veeam/Mold_Guest_Backup.conf VM_INCLUDE='${escapedVm}' bash '${escapedRemote}' \`"`$CLIENT\`" '${escapedJob}' default '${escapedVm}'" + } + $content = @" +#!/bin/bash +# Veeam Guest Processing: SSH from guest VM to KVM for Mold pre/post-notify. +GUEST_LOG=/var/log/mold/guest-veeam-hook.log +mkdir -p /var/log/mold 2>/dev/null || true +log() { echo "[`$(date '+%Y-%m-%d %H:%M:%S')] [$hookPhase] `$*" >>"`$GUEST_LOG"; } +SSH=/usr/bin/ssh +[[ -x "`$SSH" ]] || SSH=ssh +CLIENT="`$(hostname -s 2>/dev/null || hostname)" +log "start job=${escapedJob} vm=${escapedVm} kvm=${escapedKvm} client=`$CLIENT" +`$SSH -o BatchMode=yes -o ConnectTimeout=30 -o StrictHostKeyChecking=accept-new '${escapedUser}@${escapedKvm}' \ + "$remoteBody" +rc=`$? +if [[ `$rc -ne 0 ]]; then + log "kvm notify failed rc=`$rc" + exit `$rc +fi +log "kvm notify ok" +exit 0 +"@ + $utf8NoBom = New-Object System.Text.UTF8Encoding $false + [System.IO.File]::WriteAllText($Path, $content.Replace("`r`n", "`n"), $utf8NoBom) +} + +function Clear-MoldVeeamGuestJobIncompatibleOptions { + param( + $AgentJob, + $BackupObject + ) + + if ($BackupObject) { + try { + $indexDisable = New-VBRComputerIndexingOptions ` + -BackupObject $BackupObject ` + -OSPlatform Linux ` + -IndexingMode Disable + Set-VBRComputerBackupJob -Job $AgentJob ` + -EnableIndexing:$false ` + -IndexingOptions @($indexDisable) | Out-Null + Write-Host "Disabled guest file indexing (SelectedFiles incompatible with IndexIncludedOnly)." + } catch { + Write-Warning "Disable indexing: $($_.Exception.Message)" + } + } else { + try { + Set-VBRComputerBackupJob -Job $AgentJob -EnableIndexing:$false | Out-Null + } catch { + Write-Warning "Disable indexing (no BackupObject): $($_.Exception.Message)" + } + } + + try { + Set-VBRComputerBackupJob -Job $AgentJob -EnableApplicationProcessing:$false | Out-Null + } catch { + Write-Warning "Disable ApplicationProcessing: $($_.Exception.Message)" + } +} + +function Register-MoldVeeamGuestSelectedFilesScripts { + param( + $AgentJob, + [string]$VmIp, + $DiscoveredComputer, + [string]$PreWrapper, + [string]$PostWrapper + ) + + $backupObject = $DiscoveredComputer + if (-not $backupObject) { + $backupObject = $AgentJob.BackupObject + } + if ($backupObject -is [System.Array]) { + $backupObject = $backupObject | Select-Object -First 1 + } + if (-not $backupObject) { + throw "BackupObject not found for guest VM ${VmIp}." + } + + try { + Rescan-VBREntity -Entity $backupObject | Out-Null + } catch { + Write-Warning "Rescan backup object failed (continuing): $($_.Exception.Message)" + } + + Clear-MoldVeeamGuestJobIncompatibleOptions -AgentJob $AgentJob -BackupObject $backupObject + + $indexDisable = New-VBRComputerIndexingOptions ` + -BackupObject $backupObject ` + -OSPlatform Linux ` + -IndexingMode Disable + + # SelectedFiles: pre-job/post-job (not pre-freeze/post-thaw). ScriptOptions is for Windows agents. + $scriptProcessing = New-VBRScriptProcessingOptions ` + -ProcessingAction RequireSuccess ` + -ScriptPreJobCommand $PreWrapper ` + -ScriptPostJobCommand $PostWrapper + + $appProcessing = New-VBRApplicationProcessingOptions ` + -Enable ` + -OSPlatform Linux ` + -BackupObject $backupObject ` + -ScriptProcessingOptions $scriptProcessing + + $disabledJobScripts = New-VBRJobScriptOptions -PreScriptEnabled:$false -PostScriptEnabled:$false + + try { + Set-VBRComputerBackupJob -Job $AgentJob ` + -ScriptOptions $disabledJobScripts ` + -EnableIndexing:$false ` + -IndexingOptions @($indexDisable) ` + -EnableApplicationProcessing ` + -ApplicationProcessingOptions @($appProcessing) | Out-Null + } catch { + throw @" +SelectedFiles script registration failed for ${VmIp}: $($_.Exception.Message) + +If indexing was enabled in Veeam UI, open Job -> Guest Processing: + 1) Indexing tab -> disable file indexing + 2) Scripts tab -> Job scripts: + Pre-job : $PreWrapper + Post-job: $PostWrapper +Then re-run push-to-veeam.sh +"@ + } + + Write-Host "Registered pre-job/post-job on SelectedFiles guest job ${VmIp} (runs on guest -> SSH KVM for Mold)." + Write-Host " Pre-job : $PreWrapper" + Write-Host " Post-job : $PostWrapper" + Write-Host " On guest, Veeam uploads to /var/lib/veeam/scripts/ and runs as root." + Write-Host " Guest log: /var/log/mold/guest-veeam-hook.log on ${VmIp}" +} + +function Register-MoldVeeamGuestVmGuestScripts { + param( + $AgentJob, + [string]$JobName, + [string]$InstallDir, + [string]$KvmHost, + [string]$KvmSshUser, + [string]$VmLibvirtName, + [string]$VmIp, + $DiscoveredComputer, + [string]$AgentPreNotifyScript, + [string]$AgentPostNotifyScript + ) + + if (-not $VmLibvirtName -or -not $VmIp) { + throw "GuestVm mode requires -VmLibvirtName and -VmIp" + } + if (-not $KvmHost) { + throw "GuestVm mode requires -KvmHost (Mold trigger on hypervisor)" + } + + $safeVm = ($VmLibvirtName -replace '[^\w\-]', '_') + $preDeployed = Join-Path $InstallDir "mold-guest-pre-${safeVm}.sh" + $postDeployed = Join-Path $InstallDir "mold-guest-post-${safeVm}.sh" + if ((Test-Path $preDeployed) -and (Test-Path $postDeployed)) { + $preWrapper = $preDeployed + $postWrapper = $postDeployed + Write-Host "Using deployed guest wrappers from KVM (libvirt name):" + } else { + $preWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "pre" -VmLibvirtName $VmLibvirtName + $postWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "post" -VmLibvirtName $VmLibvirtName + Write-MoldGuestKvmSshWrapperScript -Path $preWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPreNotifyScript + Write-MoldGuestKvmSshWrapperScript -Path $postWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPostNotifyScript -IsPost + Write-Host "Created guest KVM-SSH wrappers (run on VM ${VmIp}, Mold on ${KvmHost}):" + } + Write-Host " $preWrapper" + Write-Host " $postWrapper" + + $backupType = "" + foreach ($key in @("BackupType", "Type", "BackupJobType")) { + if ($AgentJob.PSObject.Properties.Name -contains $key -and $AgentJob.$key) { + $backupType = "$($AgentJob.$key)" + break + } + } + + # SelectedFiles (file-level) Linux jobs: pre-job/post-job via Guest Processing scripts. + if ($backupType -eq "SelectedFiles") { + Register-MoldVeeamGuestSelectedFilesScripts ` + -AgentJob $AgentJob ` + -VmIp $VmIp ` + -DiscoveredComputer $DiscoveredComputer ` + -PreWrapper $preWrapper ` + -PostWrapper $postWrapper + return + } + + $backupObject = $DiscoveredComputer + if (-not $backupObject) { + $backupObject = $AgentJob.BackupObject + } + if ($backupObject -is [System.Array]) { + $backupObject = $backupObject | Select-Object -First 1 + } + if (-not $backupObject) { + throw "Job BackupObject not found for guest VM ${VmIp}. Re-run create-veeam-guest-vm-jobs.ps1 after agent rescan." + } + + try { + Rescan-VBREntity -Entity $backupObject | Out-Null + } catch { + Write-Warning "Rescan backup object failed (continuing): $($_.Exception.Message)" + } + + $scriptProcessing = New-VBRScriptProcessingOptions ` + -ProcessingAction RequireSuccess ` + -ScriptPrefreezeCommand $preWrapper ` + -ScriptPostthawCommand $postWrapper + + $appProcessing = New-VBRApplicationProcessingOptions ` + -Enable ` + -OSPlatform Linux ` + -BackupObject $backupObject ` + -ScriptProcessingOptions $scriptProcessing + + $disabledJobScripts = New-VBRJobScriptOptions -PreScriptEnabled:$false -PostScriptEnabled:$false + + try { + Set-VBRComputerBackupJob -Job $AgentJob ` + -ScriptOptions $disabledJobScripts ` + -EnableApplicationProcessing ` + -ApplicationProcessingOptions @($appProcessing) | Out-Null + } catch { + throw "Guest Processing registration failed for ${VmIp}: $($_.Exception.Message). For SelectedFiles jobs use pre-job/post-job (re-run push-to-veeam.sh with updated install-veeam-job.ps1)." + } + + Write-Host "Registered Guest Processing pre-freeze/post-thaw on VM ${VmIp} (runs on guest -> SSH KVM for Mold)." + Write-Host " Pre-freeze : $preWrapper" + Write-Host " Post-thaw : $postWrapper" + Write-Host " On guest, Veeam uploads to /var/lib/veeam/scripts/ and runs as root." + Write-Host " Guest log: /var/log/mold/guest-veeam-hook.log on ${VmIp}" +} + +function Register-MoldVeeamLinuxAgentGuestScripts { + param( + $AgentJob, + [string]$JobName, + [string]$InstallDir, + [string]$ProtectionGroupName, + [string]$AgentPreNotifyScript, + [string]$AgentPostNotifyScript + ) + + $safeJob = ($JobName -replace '[^\w\-]', '_') + $preWrapper = Join-Path $InstallDir "mold-agent-pre-$safeJob.sh" + $postWrapper = Join-Path $InstallDir "mold-agent-post-$safeJob.sh" + Write-MoldAgentWrapperScript -Path $preWrapper -AgentScript $AgentPreNotifyScript -JobName $JobName + Write-MoldAgentWrapperScript -Path $postWrapper -AgentScript $AgentPostNotifyScript -JobName $JobName + Write-Host "Created agent wrapper scripts (uploaded to KVM by Veeam on job run):" + Write-Host " $preWrapper" + Write-Host " $postWrapper" + + $backupObject = $null + if ($ProtectionGroupName) { + $backupObject = Get-VBRProtectionGroup -Name $ProtectionGroupName -ErrorAction SilentlyContinue + } + if (-not $backupObject -and $AgentJob.BackupObject) { + $backupObject = @($AgentJob.BackupObject) | Select-Object -First 1 + } + if (-not $backupObject) { + throw "Protection group not found for Guest Processing scripts. Pass -ProtectionGroupName or fix job BackupObject." + } + + if ($backupObject -is [System.Array]) { + $backupObject = $backupObject | Select-Object -First 1 + } + + Write-Host "Rescanning protection group before guest script registration..." + Rescan-VBREntity -Entity $backupObject | Out-Null + + # Pre-freeze/post-thaw run ON the Linux guest (Veeam uploads to /var/lib/veeam/scripts/). + $scriptProcessing = New-VBRScriptProcessingOptions ` + -ProcessingAction RequireSuccess ` + -ScriptPrefreezeCommand $preWrapper ` + -ScriptPostthawCommand $postWrapper + + $appProcessing = New-VBRApplicationProcessingOptions ` + -Enable ` + -OSPlatform Linux ` + -BackupObject $backupObject ` + -ScriptProcessingOptions $scriptProcessing + + # Job-level scripts (Storage -> Advanced -> Scripts) run on Windows B&R — disable them. + $disabledJobScripts = New-VBRJobScriptOptions -PreScriptEnabled:$false -PostScriptEnabled:$false + + Set-VBRComputerBackupJob -Job $AgentJob ` + -ScriptOptions $disabledJobScripts ` + -EnableApplicationProcessing ` + -ApplicationProcessingOptions @($appProcessing) | Out-Null + + Write-Host "Registered Guest Processing pre/post (run on KVM agent, not this Windows server)." +} + +if ($ScriptsOnly) { + # Generate ONLY the pre/post wrapper scripts on this Veeam server. + # No Veeam module, no job lookup, no job registration. Point the Veeam UI + # (Job -> Guest Processing -> Scripts) at the generated files manually. + New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + $safeJob = ($JobName -replace '[^\w\-]', '_') + + if ($GuestVm.IsPresent) { + if (-not $VmLibvirtName -or -not $VmIp) { + throw "ScriptsOnly + GuestVm requires -VmLibvirtName and -VmIp" + } + if (-not $KvmHost) { + throw "ScriptsOnly + GuestVm requires -KvmHost" + } + $preWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "pre" -VmLibvirtName $VmLibvirtName + $postWrapper = Get-MoldVeeamGuestWrapperPath -InstallDir $InstallDir -Phase "post" -VmLibvirtName $VmLibvirtName + Write-MoldGuestKvmSshWrapperScript -Path $preWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPreNotifyScript + Write-MoldGuestKvmSshWrapperScript -Path $postWrapper -KvmHost $KvmHost -KvmUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName -JobName $JobName -RemoteScript $AgentPostNotifyScript -IsPost + Write-Host "Generated guest KVM-SSH wrapper scripts (runs on VM ${VmIp}, Mold on ${KvmHost}):" + } else { + $preWrapper = Join-Path $InstallDir "mold-agent-pre-$safeJob.sh" + $postWrapper = Join-Path $InstallDir "mold-agent-post-$safeJob.sh" + Write-MoldAgentWrapperScript -Path $preWrapper -AgentScript $AgentPreNotifyScript -JobName $JobName + Write-MoldAgentWrapperScript -Path $postWrapper -AgentScript $AgentPostNotifyScript -JobName $JobName + Write-Host "Generated agent wrapper scripts (uploaded to the agent host by Veeam on job run):" + } + + Write-Host " Pre-freeze : $preWrapper" + Write-Host " Post-thaw : $postWrapper" + Write-Host "" + Write-Host "Next (Veeam UI): Job -> Guest Processing -> Applications -> (select VM) Edit ->" + Write-Host " Scripts tab -> Pre-freeze script = $preWrapper" + Write-Host " Post-thaw script = $postWrapper" + return +} + +$files = @( + "veeam-job-pre-backup.ps1", + "veeam-job-post-backup.ps1", + "veeam-job-post-restore.ps1", + "mold-backup.windows.conf.default" +) + +New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null + +$sourceRoot = [System.IO.Path]::GetFullPath($SourceDir).TrimEnd('\', '/') +$installRoot = [System.IO.Path]::GetFullPath($InstallDir).TrimEnd('\', '/') +$skipCopy = ($sourceRoot -ieq $installRoot) +if ($skipCopy) { + Write-Host "SourceDir equals InstallDir ($installRoot); skipping file copy." +} + +# Linux Agent: pre/post run on KVM (bash). Windows PS1 bundle is optional for guest/Linux agent jobs. +$requireWindowsBundle = (-not $LinuxAgent.IsPresent) -and (-not $GuestVm.IsPresent) + +if ($GuestVm.IsPresent -and [string]::IsNullOrWhiteSpace($SourceDir)) { + $SourceDir = $InstallDir +} +if ($GuestVm.IsPresent -and -not (Test-Path (Join-Path $SourceDir "veeam-job-pre-backup.ps1"))) { + if (Test-Path (Join-Path $InstallDir "veeam-job-pre-backup.ps1")) { + $SourceDir = $InstallDir + } +} + +foreach ($f in $files) { + $src = Join-Path $SourceDir $f + $dest = Join-Path $InstallDir $f + if (-not (Test-Path $src)) { + if ($requireWindowsBundle) { + throw "Missing file: $src" + } + Write-Warning "Skipping optional file (Linux Agent mode): $src" + continue + } + if ($skipCopy) { + continue + } + $srcFull = [System.IO.Path]::GetFullPath($src) + $destFull = [System.IO.Path]::GetFullPath($dest) + if ($srcFull -ieq $destFull) { + continue + } + Copy-Item -Path $src -Destination $dest -Force +} + +$confPath = Join-Path $InstallDir "mold-backup.windows.conf" +$confDefault = Join-Path $InstallDir "mold-backup.windows.conf.default" +if (-not (Test-Path $confPath) -and (Test-Path $confDefault)) { + Copy-Item $confDefault $confPath + Write-Host "Created $confPath — edit VM_NAME, STAGING_PATH, KVM_HOST if needed." +} + +function Set-ConfValue { + param([string]$Key, [string]$Value) + if ([string]::IsNullOrWhiteSpace($Value)) { return } + $lines = Get-Content $confPath + $found = $false + $out = foreach ($line in $lines) { + if ($line -match "^\s*$([regex]::Escape($Key))\s*=") { + $found = $true + "$Key=`"$Value`"" + } else { $line } + } + if (-not $found) { $out += "$Key=`"$Value`"" } + Set-Content -Path $confPath -Value $out -Encoding UTF8 +} + +Set-ConfValue -Key "VEEAM_JOB_NAME" -Value $JobName +Set-ConfValue -Key "VM_NAME" -Value $VmName +Set-ConfValue -Key "STAGING_PATH" -Value $StagingPath +Set-ConfValue -Key "KVM_HOST" -Value $KvmHost +Set-ConfValue -Key "KVM_SSH_USER" -Value $KvmSshUser +Set-ConfValue -Key "KVM_SSH_KEY" -Value $KvmSshKey + +$prePs1 = Join-Path $InstallDir "veeam-job-pre-backup.ps1" +$postPs1 = Join-Path $InstallDir "veeam-job-post-backup.ps1" +$restorePs1 = Join-Path $InstallDir "veeam-job-post-restore.ps1" + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + +$agentJob = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue +$job = $null +if (-not $agentJob) { + $job = Get-VBRJob -Name $JobName -ErrorAction SilentlyContinue +} +if (-not $agentJob -and -not $job) { + Write-Host "Available Agent backup jobs:" + Get-VBRComputerBackupJob | Select-Object -ExpandProperty Name + $createScript = Join-Path $InstallDir "create-veeam-agent-job.ps1" + if (-not (Test-Path $createScript)) { + $createScript = Join-Path $PSScriptRoot "create-veeam-agent-job.ps1" + } + throw @" +Job not found: $JobName + +Create the Agent backup job first (run on this Veeam B&R server): + pwsh -File '$createScript' -JobName '$JobName' -KvmHost '' -AgentHostName '' + +Or from Git repo / ccvm: + bash push-to-veeam.sh --env-file mold-backup.env +"@ +} + +# Linux Agent: pre/post scripts run ON THE AGENT HOST (bash), not on this Windows B&R server. +$useLinuxAgentScripts = $LinuxAgent.IsPresent +if ($agentJob -and -not $useLinuxAgentScripts) { + $osType = $agentJob.OSPlatform + if ($osType -eq "Linux") { $useLinuxAgentScripts = $true } +} +if ($useLinuxAgentScripts -and -not $GuestVm.IsPresent) { + Write-Host "Linux Agent job: Guest Processing scripts -> KVM (Mold VM backup pre/post)." + Write-Host "Ensure scripts exist on KVM:" + Write-Host " $AgentPreNotifyScript" + Write-Host " $AgentPostNotifyScript" + Write-Host "On KVM: /etc/veeam/veeam.ini [scripts] timeoutPrePost = 1800 ; systemctl restart veeamservice" +} else { + $psExe = "pwsh.exe" + if (-not (Get-Command $psExe -ErrorAction SilentlyContinue)) { + Write-Warning "pwsh.exe not found; falling back to powershell.exe (Veeam module may fail on PS 5.1)" + $psExe = "powershell.exe" + } + $preCmd = "$psExe -ExecutionPolicy Bypass -NoProfile -File `"$prePs1`"" + $postCmd = "$psExe -ExecutionPolicy Bypass -NoProfile -File `"$postPs1`"" + $restoreCmd = "$psExe -ExecutionPolicy Bypass -NoProfile -File `"$restorePs1`"" +} + +if ($agentJob) { + if ($GuestVm.IsPresent) { + Register-MoldVeeamGuestVmGuestScripts ` + -AgentJob $agentJob ` + -JobName $JobName ` + -InstallDir $InstallDir ` + -KvmHost $KvmHost ` + -KvmSshUser $KvmSshUser ` + -VmLibvirtName $VmLibvirtName ` + -VmIp $VmIp ` + -DiscoveredComputer $DiscoveredComputer ` + -AgentPreNotifyScript $AgentPreNotifyScript ` + -AgentPostNotifyScript $AgentPostNotifyScript + $preCmd = "Guest VM ${VmIp} -> SSH KVM Mold pre" + $postCmd = "Guest VM ${VmIp} -> SSH KVM Mold post" + } elseif ($useLinuxAgentScripts) { + Register-MoldVeeamLinuxAgentGuestScripts ` + -AgentJob $agentJob ` + -JobName $JobName ` + -InstallDir $InstallDir ` + -ProtectionGroupName $ProtectionGroupName ` + -AgentPreNotifyScript $AgentPreNotifyScript ` + -AgentPostNotifyScript $AgentPostNotifyScript + $preCmd = "Guest Processing pre-job -> KVM ($AgentPreNotifyScript)" + $postCmd = "Guest Processing post-job -> KVM ($AgentPostNotifyScript)" + } else { + $scriptOptions = New-VBRJobScriptOptions ` + -PreScriptEnabled ` + -PreCommand $preCmd ` + -PostScriptEnabled ` + -PostCommand $postCmd ` + -Periodicity Cycles ` + -Frequency 1 + + Set-VBRComputerBackupJob -Job $agentJob -ScriptOptions $scriptOptions | Out-Null + Write-Host "Registered Pre/Post scripts on Agent backup job: $JobName" + } +} elseif ($job) { + $opts = Get-VBRJobOptions -Job $job + $base = $opts.JobScriptOptions + if (-not $base) { + $base = New-VBRJobScriptOptions + } + $newScript = Set-VBRJobScriptOptions -JobScriptOptions $base ` + -PreScriptEnabled -PreCommand $preCmd ` + -PostScriptEnabled -PostCommand $postCmd ` + -Periodicity Cycles -Frequency 1 + $opts.JobScriptOptions = $newScript + Set-VBRJobOptions -Job $job -Options $opts | Out-Null + Write-Host "Registered Pre/Post scripts on backup job: $JobName" +} + +Write-Host "" +Write-Host "Installed to: $InstallDir" +Write-Host " Pre-job : $preCmd" +Write-Host " Post-job: $postCmd" +Write-Host "" +Write-Host "Verify in Veeam UI: Job -> Guest Processing -> Processing Settings -> Scripts (Pre-job / Post-job)" +if ($GuestVm.IsPresent) { + Write-Host "Guest VM ${VmIp}: ensure passwordless ssh ${KvmSshUser}@${KvmHost}" + Write-Host "SelectedFiles jobs: Veeam UI -> Job -> Storage -> Advanced -> Scripts (Pre-job / Post-job)" +} elseif ($useLinuxAgentScripts) { + Write-Host "KVM host: run veeam_config.sh then ensure Veeam job backs up /tmp/mold/veeam" +} else { + Write-Host "KVM host: run veeam_config.sh for Mold integration" +} +if ($EnableRestoreScript) { + Write-Host "" + Write-Host "Post-restore script (run manually after Veeam restore completes):" + Write-Host " $restoreCmd" + Write-Host "Set BACKUP_ID in mold-backup.windows.conf before running." +} diff --git a/scripts/vm/hypervisor/kvm/veeam/install.sh b/scripts/vm/hypervisor/kvm/veeam/install.sh new file mode 100755 index 000000000000..a40a4124958e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/install.sh @@ -0,0 +1,168 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Install Ablestack Veeam backup hooks on KVM host (mold-agent post-install). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +LEGACY_ETC="/etc/mold/backup/veeam" +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" +LOG_DIR="/var/log/mold" +ABLESTACK_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + +install -d -m 0755 "${SHARE_DIR}" "${ETC_DIR}" "${ETC_DIR}/secrets" "${ETC_DIR}/state" "${ETC_DIR}/hooks" "${ETC_DIR}/registry" "${LOG_DIR}" + +ABLESTACK_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE}" bash "${SCRIPT_DIR}/install-ablestack-secret-key.sh" +if [[ -f "${SCRIPT_DIR}/ablestack.key.default" ]]; then + install -m 0644 "${SCRIPT_DIR}/ablestack.key.default" "${SHARE_DIR}/ablestack.key.default" + install -m 0644 "${SCRIPT_DIR}/ablestack.key.default" "${ETC_DIR}/ablestack.key.default" +fi + +for f in install.sh install-ablestack-secret-key.sh mold-backup.lib.sh mold-backup-secret.sh mold-backup.sh veeam_config.sh \ + ablestack_veeam_pre_notify.sh ablestack_veeam_post_notify.sh ablestack_veeam_restore_notify.sh ablestack_veeam_restore_event.sh \ + mold-veeam-trigger-hook.sh; do + [[ -f "${SCRIPT_DIR}/${f}" ]] || continue + install -m 0755 "${SCRIPT_DIR}/${f}" "${ETC_DIR}/${f}" + install -m 0755 "${SCRIPT_DIR}/${f}" "${SHARE_DIR}/${f}" 2>/dev/null || true +done + +CVT_SRC="" +for _cvt_candidate in \ + "${ABLESTACK_CVT_BACKUP_SRC:-}" \ + "${SCRIPT_DIR}/ablestack_cvtbackup.sh" \ + "${SCRIPT_DIR}/../ablestack_cvtbackup.sh" \ + "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_cvtbackup.sh"; do + [[ -n "${_cvt_candidate}" && -f "${_cvt_candidate}" ]] || continue + CVT_SRC="${_cvt_candidate}" + break +done +if [[ -n "${CVT_SRC}" ]]; then + install -m 0755 "${CVT_SRC}" "${ETC_DIR}/ablestack_cvtbackup.sh" + install -m 0755 "${CVT_SRC}" "${SHARE_DIR}/ablestack_cvtbackup.sh" + CS_CVT_DIR="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm" + if [[ -d "${CS_CVT_DIR}" ]]; then + install -m 0755 "${CVT_SRC}" "${CS_CVT_DIR}/ablestack_cvtbackup.sh" + fi + echo "Installed host export script: ${ETC_DIR}/ablestack_cvtbackup.sh (from ${CVT_SRC})" +else + echo "ERROR: ablestack_cvtbackup.sh not found." >&2 + echo " Copy scripts/vm/hypervisor/kvm/veeam/ (includes ablestack_cvtbackup.sh) or set ABLESTACK_CVT_BACKUP_SRC=/path/to/ablestack_cvtbackup.sh" >&2 + exit 1 +fi + +NAS_SRC="" +for _nas_candidate in \ + "${ABLESTACK_NAS_BACKUP_SRC:-}" \ + "${SCRIPT_DIR}/ablestack_nasbackup.sh" \ + "${SCRIPT_DIR}/../ablestack_nasbackup.sh" \ + "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh"; do + [[ -n "${_nas_candidate}" && -f "${_nas_candidate}" ]] || continue + NAS_SRC="${_nas_candidate}" + break +done +if [[ -n "${NAS_SRC}" ]]; then + install -m 0755 "${NAS_SRC}" "${ETC_DIR}/ablestack_nasbackup.sh" + install -m 0755 "${NAS_SRC}" "${SHARE_DIR}/ablestack_nasbackup.sh" + CS_KVM_DIR="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm" + install -d -m 0755 "${CS_KVM_DIR}" + install -m 0755 "${NAS_SRC}" "${CS_KVM_DIR}/ablestack_nasbackup.sh" + echo "Installed NAS backup script: ${CS_KVM_DIR}/ablestack_nasbackup.sh (from ${NAS_SRC})" +else + echo "WARN: ablestack_nasbackup.sh not found — import-veeam-seed requires agent + this script" >&2 +fi + +install -m 0644 "${SCRIPT_DIR}/mold-ms-backup-schema-fix.sql" "${SHARE_DIR}/mold-ms-backup-schema-fix.sql" 2>/dev/null || true +install -m 0644 "${SCRIPT_DIR}/mold-ms-backup-schema-fix.sql" "${ETC_DIR}/mold-ms-backup-schema-fix.sql" 2>/dev/null || true + +install -m 0644 "${SCRIPT_DIR}/mold-backup.conf.default" "${SHARE_DIR}/mold-backup.conf.default" +install -m 0644 "${SCRIPT_DIR}/mold-backup.env.example" "${SHARE_DIR}/mold-backup.env.example" +if [[ ! -f "${ETC_DIR}/mold-backup.env" ]]; then + install -m 0600 "${SCRIPT_DIR}/mold-backup.env.example" "${ETC_DIR}/mold-backup.env" + echo "Created ${ETC_DIR}/mold-backup.env — set MOLD_API_KEY / MOLD_API_SECRET, then veeam_config.sh --job-name ..." +else + echo "Keeping existing ${ETC_DIR}/mold-backup.env" +fi +install -m 0644 "${SCRIPT_DIR}/mold-backup.windows.conf.default" "${SHARE_DIR}/mold-backup.windows.conf.default" +install -m 0644 "${SCRIPT_DIR}/veeam-job-pre-backup.ps1" "${SHARE_DIR}/veeam-job-pre-backup.ps1" +install -m 0644 "${SCRIPT_DIR}/veeam-job-post-backup.ps1" "${SHARE_DIR}/veeam-job-post-backup.ps1" +install -m 0644 "${SCRIPT_DIR}/veeam-job-post-restore.ps1" "${SHARE_DIR}/veeam-job-post-restore.ps1" +install -m 0644 "${SCRIPT_DIR}/install-veeam-job.ps1" "${SHARE_DIR}/install-veeam-job.ps1" +install -m 0644 "${SCRIPT_DIR}/create-veeam-agent-job.ps1" "${SHARE_DIR}/create-veeam-agent-job.ps1" 2>/dev/null || true +install -m 0644 "${SCRIPT_DIR}/setup-veeam-mold-job.ps1" "${SHARE_DIR}/setup-veeam-mold-job.ps1" 2>/dev/null || true +install -m 0755 "${SCRIPT_DIR}/setup-datadisk-veeam-backup.sh" "${ETC_DIR}/setup-datadisk-veeam-backup.sh" 2>/dev/null || true +install -m 0644 "${SCRIPT_DIR}/setup-veeam-host-repo.ps1" "${SHARE_DIR}/setup-veeam-host-repo.ps1" 2>/dev/null || true +install -m 0755 "${SCRIPT_DIR}/mold-guest-common.sh" "${ETC_DIR}/mold-guest-common.sh" +install -m 0755 "${SCRIPT_DIR}/push-to-veeam.sh" "${SHARE_DIR}/push-to-veeam.sh" 2>/dev/null || true +install -m 0755 "${SCRIPT_DIR}/push-to-veeam.sh" "${ETC_DIR}/push-to-veeam.sh" 2>/dev/null || true + +if [[ -f "${SCRIPT_DIR}/README.ko.md" ]]; then + install -m 0644 "${SCRIPT_DIR}/README.ko.md" "${SHARE_DIR}/README.ko.md" +fi + +if [[ ! -f "${ETC_DIR}/mold-backup.conf" ]]; then + install -m 0600 "${SCRIPT_DIR}/mold-backup.conf.default" "${ETC_DIR}/mold-backup.conf" + echo "Created ${ETC_DIR}/mold-backup.conf — run veeam_config.sh for job-based setup." +else + echo "Keeping existing ${ETC_DIR}/mold-backup.conf" +fi + +# Backward-compatible symlink for older paths +if [[ ! -e "${LEGACY_ETC}" ]]; then + ln -sfn "${ETC_DIR}" "${LEGACY_ETC}" 2>/dev/null || true +fi + +mkdir -p "${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" 2>/dev/null || true + +install_restore_agent_units() { + local unit_dir="/etc/systemd/system" + [[ -f "${SCRIPT_DIR}/mold-veeam-restore-agent.service" ]] || return 0 + for f in mold-veeam-restore-agent.sh enable-veeam-mold-restore.sh; do + [[ -f "${SCRIPT_DIR}/${f}" ]] || continue + install -m 0755 "${SCRIPT_DIR}/${f}" "${ETC_DIR}/${f}" + install -m 0755 "${SCRIPT_DIR}/${f}" "${SHARE_DIR}/${f}" 2>/dev/null || true + done + for f in mold-veeam-restore-agent.service mold-veeam-restore-agent.timer; do + [[ -f "${SCRIPT_DIR}/${f}" ]] || continue + install -m 0644 "${SCRIPT_DIR}/${f}" "${ETC_DIR}/${f}" + install -m 0644 "${SCRIPT_DIR}/${f}" "${SHARE_DIR}/${f}" 2>/dev/null || true + done + install -m 0644 "${SCRIPT_DIR}/mold-veeam-restore-agent.service" "${unit_dir}/mold-veeam-restore-agent.service" + install -m 0644 "${SCRIPT_DIR}/mold-veeam-restore-agent.timer" "${unit_dir}/mold-veeam-restore-agent.timer" + if command -v systemctl >/dev/null 2>&1; then + systemctl daemon-reload 2>/dev/null || true + systemctl disable mold-veeam-restore-watch.timer 2>/dev/null || true + systemctl stop mold-veeam-restore-watch.timer 2>/dev/null || true + systemctl enable mold-veeam-restore-agent.timer 2>/dev/null || true + systemctl start mold-veeam-restore-agent.timer 2>/dev/null || true + echo "Enabled mold-veeam-restore-agent.timer (3min poll → Mold restoreBackup on Veeam FLR)" + echo " Run: ${ETC_DIR}/enable-veeam-mold-restore.sh --vm-include 'i-2-XX-VM'" + fi +} +install_restore_agent_units + +echo "Installed Ablestack Veeam backup hooks (host/datadisk mode):" +echo " Active: ${ETC_DIR}/" +echo " Reference: ${SHARE_DIR}/" +echo " Secret key: ${ABLESTACK_SECRET_KEY_FILE}" +echo " Host backup path: /tmp/mold/veeam" +echo " Configure: veeam_config.sh --job-name ... --install" +echo " Datadisk setup: ${ETC_DIR}/setup-datadisk-veeam-backup.sh" +echo " FLR→Mold: ${ETC_DIR}/enable-veeam-mold-restore.sh" +echo " Veeam PS1/Job: ${ETC_DIR}/push-to-veeam.sh" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh b/scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh new file mode 100755 index 000000000000..c13c603311d6 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup-secret.sh @@ -0,0 +1,151 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Encrypt/decrypt Mold API secret for mold-backup hooks (AES-256-CBC + PBKDF2). +# Default host key: /root/.ssh/ablestack.key (base64-encoded passphrase, chmod 600). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OPENSSL_CIPHER="aes-256-cbc" +DEFAULT_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + +usage() { + cat < --key-file [--out ] + mold-backup-secret.sh decrypt --enc-file --key-file + mold-backup-secret.sh gen-key --key-file + +Files: + --key-file Passphrase file (chmod 600). Default on hosts: ${DEFAULT_KEY_FILE} + Ablestack installs a base64-encoded passphrase in this file; the script + decodes base64 when valid, otherwise uses the file contents as-is. + --enc-file Encrypted secret blob (base64 OpenSSL output). + --out Output path for encrypt (default: same dir as key-file, api-secret.enc) +EOF +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +require_openssl() { + command -v openssl >/dev/null 2>&1 || die "openssl is required" +} + +# Read passphrase from key file (base64 one-liner or raw text). +mold_secret_read_passphrase() { + local key_file="$1" + local raw decoded + raw="$(tr -d '\n\r' < "$key_file")" + [[ -n "$raw" ]] || die "Key file is empty: $key_file" + if decoded="$(printf '%s' "$raw" | base64 -d 2>/dev/null)" && [[ -n "$decoded" ]]; then + printf '%s' "$decoded" + return 0 + fi + printf '%s' "$raw" +} + +mold_secret_openssl_passfile() { + local pass="$1" + local tmp + tmp="$(mktemp)" + chmod 600 "$tmp" + printf '%s' "$pass" > "$tmp" + echo "$tmp" +} + +mold_secret_encrypt() { + local secret="" key_file="" out_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --secret) secret="$2"; shift 2 ;; + --key-file) key_file="$2"; shift 2 ;; + --out) out_file="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + [[ -n "$secret" ]] || die "--secret is required" + [[ -n "$key_file" ]] || die "--key-file is required" + [[ -f "$key_file" ]] || die "Key file not found: $key_file" + if [[ -z "$out_file" ]]; then + out_file="$(dirname "$key_file")/api-secret.enc" + fi + require_openssl + install -d -m 0700 "$(dirname "$out_file")" + local pass pass_file + pass="$(mold_secret_read_passphrase "$key_file")" + pass_file="$(mold_secret_openssl_passfile "$pass")" + echo -n "$secret" | openssl enc -"${OPENSSL_CIPHER}" -salt -pbkdf2 -pass "file:${pass_file}" -base64 -out "$out_file" + rm -f "$pass_file" + chmod 0600 "$out_file" + echo "$out_file" +} + +mold_secret_decrypt() { + local enc_file="" key_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --enc-file) enc_file="$2"; shift 2 ;; + --key-file) key_file="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + [[ -f "$enc_file" ]] || die "Encrypted file not found: $enc_file" + [[ -f "$key_file" ]] || die "Key file not found: $key_file" + require_openssl + local pass pass_file + pass="$(mold_secret_read_passphrase "$key_file")" + pass_file="$(mold_secret_openssl_passfile "$pass")" + local plain + plain="$(openssl enc -"${OPENSSL_CIPHER}" -d -salt -pbkdf2 -pass "file:${pass_file}" -base64 -in "$enc_file" 2>/dev/null)" \ + || die "Decrypt failed (wrong key file or corrupt enc file): $enc_file" + rm -f "$pass_file" + # Strip trailing newline — OpenSSL may add one; breaks CloudStack API HMAC. + plain="${plain//$'\r'/}" + plain="${plain%"${plain##*[![:space:]]}"}" + printf '%s' "$plain" +} + +mold_secret_gen_key() { + local key_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --key-file) key_file="$2"; shift 2 ;; + *) die "Unknown option: $1" ;; + esac + done + [[ -n "$key_file" ]] || die "--key-file is required" + require_openssl + install -d -m 0700 "$(dirname "$key_file")" + if [[ -f "$key_file" ]]; then + die "Key file already exists: $key_file (refuse to overwrite)" + fi + openssl rand -base64 32 > "$key_file" + chmod 0600 "$key_file" + echo "Created key file: $key_file" +} + +cmd="${1:-}" +shift || true +case "$cmd" in + encrypt) mold_secret_encrypt "$@" ;; + decrypt) mold_secret_decrypt "$@" ;; + gen-key) mold_secret_gen_key "$@" ;; + -h|--help|help|"") usage ;; + *) die "Unknown command: $cmd (use --help)" ;; +esac diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default b/scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default new file mode 100644 index 000000000000..2b7cb8a7398b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.conf.default @@ -0,0 +1,79 @@ +# Ablestack Veeam + Mold backup configuration (NetBackup-style) +# Per-job config: /etc/ablestack/veeam/.conf (created by veeam_config.sh) + +# --- API --- +MOLD_API_URL="https://127.0.0.1:8080/client/api" +MOLD_API_KEY="" +MOLD_API_SECRET="" +MOLD_API_SECRET_ENC_FILE="/etc/ablestack/veeam/secrets/secret.enc" +MOLD_SECRET_KEY_FILE="/root/.ssh/ablestack.key" + +# --- Mold backup offering (UI name; separate from Veeam job if needed) --- +BACKUP_OFFERING_NAME="VeeamBackup" + +# --- Job (policy) --- +VEEAM_JOB_NAME="" +VEEAM_SCHEDULE_NAME="default" +VEEAM_MAX_CHAIN="7" +ZONE_ID="" +RETENTION_PERIOD="P7D" + +# --- VM filter (libvirt names) --- +VM_INCLUDE="*" +VM_EXCLUDE="" + +# --- Legacy single-VM --- +VM_UUID="" +VM_NAME="" + +# --- Mold zone / Veeam plugin --- +VEEAM_URL="" +VEEAM_USERNAME="" +VEEAM_PASSWORD="" + +# --- Veeam server SSH (restore-watch polls Get-VBRRestoreSession) --- +VEEAM_SSH_HOST="" +VEEAM_SSH_USER="administrator" +VEEAM_SSH_KEY="/root/.ssh/veeam_id_rsa" + +# --- Storage model: datadisk (no NAS) --- +# GFS: qcow2 (libvirt checkpoint) | HCI: rbd (rbd export-diff on Ceph primary) +# BACKUP_STORAGE_ENGINE: auto (detect from VM disks) | qcow2 | rbd +BACKUP_STORAGE_ENGINE="auto" +BACKUP_STORAGE_MODE="datadisk" +BACKUP_REPO_TYPE="local" +BACKUP_REPO_PROVIDER="localfs" +MOLD_DATADISK_PATH="/data/backup" +BACKUP_REPO_ADDRESS="/data/backup" +BACKUP_REPO_NAME="Ablestack Data Disk" +NAS_REPO_MOUNT="" +VEEAM_HOST_REPO_ROOT="E:/opt1/veeam" +VEEAM_REPO_NAME="" + +VEEAM_HOST_BACKUP_PATH="/tmp/mold/veeam" +STAGING_PATH="/tmp/mold/veeam" +KVM_PRE_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" +KVM_POST_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" + +BACKUP_ID="" +# mold-only = Veeam FLR 후 datadisk(/data/backup)만 VM 볼륨에 반영 — NAS 마운트·Veeam chain export 없음 +RESTORE_SOURCE="mold-only" + +KVM_HOSTNAME="" +RESTORE_WATCH_TRIGGER_MOLD="true" +VEEAM_UI_RESTORE_SOURCE="mold-only" +RESTORE_LOCK_DIR="" +VEEAM_RESTORE_WATCH_WINDOW_MIN="10" +# Host job FLR: explicit libvirt VM to restore when VM_INCLUDE has multiple VMs +VEEAM_RESTORE_VM="" + +# guest = Veeam Agent on VM; pre/post SSH to KVM for Mold datadisk backup +# host = Veeam Linux Agent ON KVM (10.10.31.2) — one job per hypervisor +BACKUP_MODE="host" +IMPORT_MODE="auto" + +CLEANUP_STAGING_AFTER_BACKUP="true" +NAS_BACKUP_SCRIPT="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh" +CVT_BACKUP_SCRIPT="/etc/ablestack/veeam/ablestack_cvtbackup.sh" +LOG_FILE="/var/log/mold/veeam-hook.log" +LOG_TAG="mold-veeam-hook" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example b/scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example new file mode 100644 index 000000000000..30d1a58128bc --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.env.example @@ -0,0 +1,81 @@ +# Mold backup env — installed to /etc/ablestack/veeam/mold-backup.env by install.sh +# veeam_config.sh reads this file automatically (API key/secret, NAS repo, zone-id). +# Do not commit mold-backup.env (contains secrets). + +MS_HOST=root@10.10.31.20 +KVM_HOST=root@10.10.31.2 +MS_PLUGIN_DIR=/usr/share/cloudstack-management/lib + +MOLD_API_URL=http://10.10.31.20:8080/client/api +MOLD_API_KEY= +MOLD_API_SECRET= +ZONE_ID= + +# Veeam Agent Job 이름 (Veeam UI와 동일) +JOB_NAME="Mold ablecube31-2" +VEEAM_JOB_NAME="Mold ablecube31-2" +KVM_HOSTNAME="ablecube31-2" +# host = KVM Linux Agent on 10.10.31.2 (one job per hypervisor) +VEEAM_BACKUP_TARGET="host" +# KVM root SSH — Veeam Protection Group credentials (필수) +KVM_SSH_USER=root +KVM_SSH_PASSWORD= +# Optional: libvirt:ip for FLR session → VM matching (not guest-mode jobs) +# VM_TARGETS="i-2-5-VM:10.10.254.70,i-2-40-VM:10.10.254.61" +# setup 후 Job 자동 Start (false 로 끄기) +VEEAM_START_JOBS=true +BACKUP_OFFERING_NAME="VeeamBackup" + +# 양방향 연동 (모드 C): Mold 백업 완료 → 매칭 Veeam Agent Job 자동 실행 +# - Veeam→Mold (pre-freeze)와 Mold→Veeam (이 트리거)는 마커로 루프 차단 +VEEAM_TRIGGER_ENABLED=false +VEEAM_TRIGGER_TTL=1800 +# 트리거 방식: auto(REST 먼저, 실패 시 SSH) | rest(curl만, SSH 불필요) | ssh +VEEAM_TRIGGER_METHOD=auto +# REST API (포트 9419) — SSH 없이 curl로 Job 시작. user/password 비우면 VEEAM_USER/PASSWORD 사용 +VEEAM_API_URL=https://10.10.254.246:9419 +VEEAM_API_VERSION=1.2-rev0 +VEEAM_API_USER=administrator +VEEAM_API_PASSWORD= + +# Veeam B&R 서버 (push-to-veeam.sh) +# NOTE: use Preferred IP (.246). .245 is Duplicate on this host and breaks SSH. +VEEAM_SSH_HOST=10.10.254.246 +VEEAM_SSH_USER=administrator +VEEAM_SSH_KEY= +VEEAM_URL=https://10.10.254.246:9398/api/ +VEEAM_USER=administrator +VEEAM_PASSWORD= +VEEAM_REPO_NAME= +KVM_HOSTNAME=ablecube31-2 +KVM_IP=10.10.31.2 +VEEAM_HOST_BACKUP_PATH=/tmp/mold/veeam + +# 전체 VM: * 또는 콤마 구분 libvirt 이름 +VM_INCLUDE="i-2-5-VM,i-2-40-VM" +VM_EXCLUDE= + +# KVM data disk path for Mold qcow2 (local directory on hypervisor — not Mold addBackupRepository) +# GFS: BACKUP_STORAGE_ENGINE=auto|qcow2 | HCI (RBD primary): auto|rbd +BACKUP_STORAGE_ENGINE=auto +BACKUP_STORAGE_MODE=datadisk +MOLD_DATADISK_PATH=/data/backup +BACKUP_REPO_ADDRESS=/data/backup +BACKUP_REPO_NAME=Ablestack Data Disk +BACKUP_REPO_TYPE=local +BACKUP_REPO_PROVIDER=localfs +# Veeam B&R local disk — one folder per KVM hypervisor (not per VM) +VEEAM_HOST_REPO_ROOT=E:/opt1/veeam +VEEAM_REPO_NAME=Mold ablecube31-2 + +# Veeam UI guest-file restore → KVM Mold datadisk restore (mold-only = no NAS / no Veeam chain) +VEEAM_UI_RESTORE_SOURCE=mold-only +RESTORE_SOURCE=mold-only +RESTORE_WATCH_TRIGGER_MOLD=true + +RUN_BACKUP=true +RUN_BUILD=false +RESTART_MS=true +SKIP_CONFIGURE=false +SKIP_DEPLOY=false +RUN_OPERATION= diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh b/scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh new file mode 100755 index 000000000000..2d63c34ba2bb --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.lib.sh @@ -0,0 +1,4389 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Shared library for Ablestack Veeam backup/restore hooks on KVM hosts. +# NetBackup-style layout: /etc/ablestack/veeam/.conf, host staging /tmp/mold/veeam + +ABLESTACK_VEEAM_ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +MOLD_BACKUP_ETC_DIR="${MOLD_BACKUP_ETC_DIR:-${ABLESTACK_VEEAM_ETC_DIR}}" +MOLD_BACKUP_CONF="${MOLD_BACKUP_CONF:-}" +VEEAM_HOST_BACKUP_PATH="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" +CVT_BACKUP_SCRIPT="${CVT_BACKUP_SCRIPT:-/etc/ablestack/veeam/ablestack_cvtbackup.sh}" +VEEAM_PROVIDER_NAME="${VEEAM_PROVIDER_NAME:-ablestack-veeam}" + +# Resolve config: MOLD_BACKUP_CONF, or /etc/ablestack/veeam/.conf, or mold-backup.conf +mold_backup_resolve_conf_path() { + if [[ -n "${MOLD_BACKUP_CONF:-}" && -f "$MOLD_BACKUP_CONF" ]]; then + echo "$MOLD_BACKUP_CONF" + return 0 + fi + local job="${VEEAM_JOB_NAME:-${1:-}}" + if [[ -n "$job" ]]; then + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/${job}.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/${job}.conf" + return 0 + fi + local safe_job + safe_job="$(mold_backup_safe_job_name "$job")" + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/${safe_job}.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/${safe_job}.conf" + return 0 + fi + # Guest Veeam jobs: Mold VM 10-10-254-70 → shared KVM policy conf + if [[ "$job" == Mold\ VM\ * ]]; then + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Guest_Backup.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Guest_Backup.conf" + return 0 + fi + elif [[ "$job" == Mold\ * ]]; then + # Host Veeam job: Mold ablecube31-2 → Mold_Host_Backup.conf + if [[ -f "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Host_Backup.conf" ]]; then + echo "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Host_Backup.conf" + return 0 + fi + fi + fi + for candidate in \ + "${ABLESTACK_VEEAM_ETC_DIR}/Mold_Guest_Backup.conf" \ + "${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.conf" \ + "/etc/mold/backup/veeam/mold-backup.conf" \ + "${MOLD_BACKUP_ETC_DIR}/mold-backup.conf"; do + if [[ -f "$candidate" ]]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +# Allow runtime override: BACKUP_OPERATION=backup (Veeam post-notify path) +mold_backup_load_config() { + local resolved + local _preserve_backup_id="${BACKUP_ID:-}" + local _preserve_restore_source="${RESTORE_SOURCE:-}" + local _preserve_vm_name="${VM_NAME:-}" + local _preserve_vm_uuid="${VM_UUID:-}" + local _preserve_vm_include="${VM_INCLUDE:-}" + resolved="$(mold_backup_resolve_conf_path "${VEEAM_JOB_NAME:-}")" || { + echo "Config not found — run veeam_config.sh or install.sh" >&2 + return 1 + } + MOLD_BACKUP_CONF="$resolved" + # shellcheck source=/dev/null + source "$MOLD_BACKUP_CONF" + # Caller-provided selectors must survive sourcing the job conf (which may define them). + [[ -n "${_preserve_backup_id}" ]] && BACKUP_ID="${_preserve_backup_id}" + [[ -n "${_preserve_restore_source}" ]] && RESTORE_SOURCE="${_preserve_restore_source}" + [[ -n "${_preserve_vm_name}" ]] && VM_NAME="${_preserve_vm_name}" + [[ -n "${_preserve_vm_uuid}" ]] && VM_UUID="${_preserve_vm_uuid}" + [[ -n "${_preserve_vm_include}" && "${_preserve_vm_include}" != "*" ]] && VM_INCLUDE="${_preserve_vm_include}" + + LOG_FILE="${LOG_FILE:-/var/log/mold/backup-veeam.log}" + LOG_TAG="${LOG_TAG:-mold-veeam-backup}" + NAS_BACKUP_SCRIPT="${NAS_BACKUP_SCRIPT:-/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh}" + IMPORT_MODE="${IMPORT_MODE:-auto}" + BACKUP_MODE="${BACKUP_MODE:-host}" + VM_INCLUDE="${VM_INCLUDE:-*}" + VM_EXCLUDE="${VM_EXCLUDE:-}" + ZONE_ID="${ZONE_ID:-}" + RETENTION_PERIOD="${RETENTION_PERIOD:-}" + VEEAM_URL="${VEEAM_URL:-}" + VEEAM_USERNAME="${VEEAM_USERNAME:-}" + VEEAM_PASSWORD="${VEEAM_PASSWORD:-}" + VEEAM_HOST_BACKUP_PATH="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + STAGING_PATH="${STAGING_PATH:-${VEEAM_HOST_BACKUP_PATH}}" + NAS_REPO_MOUNT="${NAS_REPO_MOUNT:-}" + SOURCE_DISK_FORMAT="${SOURCE_DISK_FORMAT:-vmdk}" + BOOTSTRAP_CHECKPOINT="${BOOTSTRAP_CHECKPOINT:-true}" + QUIESCE_VM="${QUIESCE_VM:-false}" + BACKUP_OPERATION="${BACKUP_OPERATION:-seed-import}" + CLEANUP_STAGING_AFTER_BACKUP="${CLEANUP_STAGING_AFTER_BACKUP:-true}" + CLEANUP_STAGING_ON_ERROR="${CLEANUP_STAGING_ON_ERROR:-true}" + BACKUP_OFFERING_NAME="${BACKUP_OFFERING_NAME:-VeeamBackup}" + BACKUP_REPO_TYPE="${BACKUP_REPO_TYPE:-local}" + BACKUP_REPO_NAME="${BACKUP_REPO_NAME:-Ablestack Data Disk}" + BACKUP_REPO_PROVIDER="${BACKUP_REPO_PROVIDER:-localfs}" + MOLD_DATADISK_PATH="${MOLD_DATADISK_PATH:-}" + VEEAM_HOST_REPO_ROOT="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + VEEAM_REPO_NAME="${VEEAM_REPO_NAME:-}" + BACKUP_STORAGE_MODE="${BACKUP_STORAGE_MODE:-datadisk}" + BACKUP_STORAGE_ENGINE="${BACKUP_STORAGE_ENGINE:-auto}" + # Mold→Veeam trigger (bidirectional mode C): start the matching Veeam Agent job + # over SSH after a Mold backup completes. Loop is broken by veeam-active/mold-active markers. + VEEAM_TRIGGER_ENABLED="${VEEAM_TRIGGER_ENABLED:-false}" + VEEAM_TRIGGER_TTL="${VEEAM_TRIGGER_TTL:-1800}" + VEEAM_SSH_HOST="${VEEAM_SSH_HOST:-}" + VEEAM_SSH_USER="${VEEAM_SSH_USER:-administrator}" + VEEAM_SSH_KEY="${VEEAM_SSH_KEY:-}" + # Veeam VBR native REST API (port 9419) — used by VEEAM_TRIGGER_METHOD=rest/auto (no SSH). + VEEAM_API_URL="${VEEAM_API_URL:-}" + VEEAM_API_HOST="${VEEAM_API_HOST:-}" + VEEAM_API_PORT="${VEEAM_API_PORT:-9419}" + VEEAM_API_VERSION="${VEEAM_API_VERSION:-1.2-rev0}" + VEEAM_API_USER="${VEEAM_API_USER:-${VEEAM_USERNAME:-}}" + VEEAM_API_PASSWORD="${VEEAM_API_PASSWORD:-${VEEAM_PASSWORD:-}}" + VEEAM_GUEST_JOB_PREFIX="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + VM_TARGETS="${VM_TARGETS:-}" + KVM_HOSTNAME="${KVM_HOSTNAME:-}" + RESTORE_WATCH_TRIGGER_MOLD="${RESTORE_WATCH_TRIGGER_MOLD:-false}" + VEEAM_UI_RESTORE_SOURCE="${VEEAM_UI_RESTORE_SOURCE:-mold-only}" + RESTORE_LOCK_DIR="${RESTORE_LOCK_DIR:-}" + VEEAM_RESTORE_WATCH_WINDOW_MIN="${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}" + + mold_backup_resolve_api_secret + [[ -n "${MOLD_BACKUP_OPERATION:-}" ]] && BACKUP_OPERATION="${MOLD_BACKUP_OPERATION}" + mold_backup_supplement_guest_config + mold_backup_apply_datadisk_profile + # Guest mode is bidirectional: Mold UI backup must start the matching Veeam Agent job. + if [[ "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + VEEAM_TRIGGER_ENABLED=true + case "${VEEAM_TRIGGER_METHOD:-}" in + ''|auto) VEEAM_TRIGGER_METHOD=ssh ;; + esac + else + VEEAM_TRIGGER_METHOD="${VEEAM_TRIGGER_METHOD:-auto}" + fi + return 0 +} + +# Datadisk + Veeam E:\opt1\veeam\: Mold backups on KVM data disk (no NAS mount/restore). +mold_backup_apply_datadisk_profile() { + if [[ "${BACKUP_STORAGE_MODE:-}" != "datadisk" && "${BACKUP_REPO_TYPE:-}" != "local" ]]; then + return 0 + fi + [[ -z "${KVM_HOSTNAME:-}" ]] && KVM_HOSTNAME="$(hostname -s 2>/dev/null || hostname)" + BACKUP_REPO_TYPE=local + BACKUP_REPO_PROVIDER="${BACKUP_REPO_PROVIDER:-localfs}" + local disk="${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS:-/data/backup}}" + if [[ "$disk" == *glue-gfs* && -d /data/backup ]]; then + disk="/data/backup" + fi + MOLD_DATADISK_PATH="$disk" + BACKUP_REPO_ADDRESS="${MOLD_DATADISK_PATH}" + BACKUP_REPO_NAME="${BACKUP_REPO_NAME:-Ablestack Data Disk}" + NAS_REPO_MOUNT="" + VEEAM_HOST_REPO_ROOT="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + [[ -z "${VEEAM_REPO_NAME:-}" ]] && VEEAM_REPO_NAME="Mold ${KVM_HOSTNAME}" + # 복원: datadisk bind-mount만 사용 (NAS/GFS 마운트·Veeam chain export 없음) + RESTORE_SOURCE="mold-only" + VEEAM_UI_RESTORE_SOURCE="mold-only" + RESTORE_WATCH_TRIGGER_MOLD="${RESTORE_WATCH_TRIGGER_MOLD:-true}" +} + +mold_backup_is_datadisk_mode() { + [[ "${BACKUP_STORAGE_MODE:-}" == "datadisk" || "${BACKUP_REPO_TYPE:-}" == "local" ]] +} + +mold_backup_datadisk_root() { + mold_backup_apply_datadisk_profile + echo "${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS:-/data/backup}}" +} + +mold_backup_veeam_host_repo_folder() { + local host="${KVM_HOSTNAME:-$(hostname -s)}" + local root="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + echo "${root%/}/${host}" +} + +# Read one KEY=value from an env file (strips optional quotes). +mold_backup_read_env_var() { + local key="$1" file="$2" line val + [[ -f "$file" ]] || return 1 + line="$(grep -E "^[[:space:]]*${key}=" "$file" 2>/dev/null | head -1)" || return 1 + val="${line#*=}" + val="${val#\"}"; val="${val%\"}" + val="${val#\'}"; val="${val%\'}" + val="${val//$'\r'/}" + [[ -n "$val" ]] || return 1 + printf '%s' "$val" +} + +# Guest hooks use per-job .conf; VM_TARGETS often lives only in mold-backup.env. +mold_backup_vm_targets_merge() { + local combined="" pair name ip out="" k + declare -A _vm_target_map=() + for combined in "$1" "$2"; do + [[ -n "$combined" ]] || continue + IFS=',' read -ra _pairs <<<"${combined// /}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + [[ -n "$name" && -n "$ip" && "$ip" != "$name" ]] || continue + _vm_target_map["$name"]="$ip" + done + done + for k in "${!_vm_target_map[@]}"; do + [[ -n "$out" ]] && out+="," + out+="${k}:${_vm_target_map[$k]}" + done + echo "$out" +} + +# Write or update KEY="value" in a job .conf (best-effort; used to persist VM_TARGETS). +mold_backup_upsert_conf_var() { + local conf="$1" key="$2" val="$3" + [[ -f "$conf" && -n "$key" && -n "$val" ]] || return 0 + val="${val//\"/\\\"}" + if grep -qE "^${key}=" "$conf" 2>/dev/null; then + sed -i "s#^${key}=.*#${key}=\"${val}\"#" "$conf" + else + echo "${key}=\"${val}\"" >> "$conf" + fi +} + +mold_backup_supplement_guest_config() { + local env_file key val env_targets + for env_file in \ + "${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.env" \ + "${MOLD_BACKUP_ETC_DIR}/mold-backup.env" \ + "$(dirname "${BASH_SOURCE[0]}")/mold-backup.env"; do + [[ -f "$env_file" ]] || continue + env_targets="$(mold_backup_read_env_var VM_TARGETS "$env_file" 2>/dev/null || true)" + if [[ -n "$env_targets" ]]; then + if [[ -n "${VM_TARGETS:-}" ]]; then + VM_TARGETS="$(mold_backup_vm_targets_merge "$VM_TARGETS" "$env_targets")" + else + VM_TARGETS="$env_targets" + fi + fi + for key in VEEAM_SSH_HOST VEEAM_SSH_USER VEEAM_SSH_KEY VEEAM_GUEST_JOB_PREFIX \ + VEEAM_TRIGGER_ENABLED VEEAM_TRIGGER_METHOD VEEAM_TRIGGER_TTL \ + VEEAM_USERNAME VEEAM_PASSWORD VEEAM_API_URL; do + [[ -n "${!key:-}" ]] && continue + val="$(mold_backup_read_env_var "$key" "$env_file" 2>/dev/null || true)" + [[ -n "$val" ]] && export "$key=$val" + done + break + done + # Persist merged VM_TARGETS into guest policy conf so hooks do not depend on env alone. + if [[ -n "${VM_TARGETS:-}" && "${MOLD_BACKUP_CONF:-}" == *Mold_Guest_Backup.conf ]]; then + if ! grep -qE '^[[:space:]]*VM_TARGETS=' "$MOLD_BACKUP_CONF" 2>/dev/null; then + mold_backup_upsert_conf_var "$MOLD_BACKUP_CONF" VM_TARGETS "$VM_TARGETS" + fi + fi +} + +mold_backup_resolve_api_secret() { + if [[ -n "${MOLD_API_SECRET:-}" ]]; then + return 0 + fi + if [[ -z "${MOLD_API_SECRET_ENC_FILE:-}" ]]; then + return 0 + fi + if [[ -z "${MOLD_SECRET_KEY_FILE:-}" ]]; then + MOLD_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + fi + local secret_script="${MOLD_BACKUP_ETC_DIR}/mold-backup-secret.sh" + [[ -x "$secret_script" ]] || secret_script="$(dirname "${BASH_SOURCE[0]}")/mold-backup-secret.sh" + if [[ ! -x "$secret_script" ]]; then + mold_backup_log warn "Cannot decrypt API secret: mold-backup-secret.sh not found" + return 0 + fi + MOLD_API_SECRET=$("$secret_script" decrypt --enc-file "${MOLD_API_SECRET_ENC_FILE}" --key-file "${MOLD_SECRET_KEY_FILE}") \ + || mold_backup_die "Failed to decrypt MOLD_API_SECRET from ${MOLD_API_SECRET_ENC_FILE}" + # OpenSSL decrypt may append a newline; breaks CloudStack API HMAC signature. + MOLD_API_SECRET="${MOLD_API_SECRET//$'\r'/}" + MOLD_API_SECRET="${MOLD_API_SECRET%"${MOLD_API_SECRET##*[![:space:]]}"}" +} + +mold_backup_log() { + local level="$1" + shift + local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" + echo "$msg" >&2 + mkdir -p "$(dirname "${LOG_FILE}")" 2>/dev/null || true + echo "$msg" >> "${LOG_FILE}" 2>/dev/null || true + if command -v logger >/dev/null 2>&1; then + case "$level" in + err) logger -t "${LOG_TAG}" -p user.err "$*" ;; + warn) logger -t "${LOG_TAG}" -p user.warning "$*" ;; + *) logger -t "${LOG_TAG}" -p user.info "$*" ;; + esac + fi +} + +mold_backup_die() { + mold_backup_log err "$@" + exit 1 +} + +mold_backup_require_var() { + local name="$1" + local value="${!name:-}" + if [[ -z "$value" ]]; then + mold_backup_die "Required config [$name] is not set in ${MOLD_BACKUP_CONF}" + fi +} + +mold_backup_cmk_bin() { + command -v cmk >/dev/null 2>&1 && echo "cmk" && return 0 + command -v cloudmonkey >/dev/null 2>&1 && echo "cloudmonkey" && return 0 + return 1 +} + +mold_backup_require_cmd() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || mold_backup_die "Required command not found: $cmd" +} + +mold_backup_cloudstack_api_call() { + local cmd="$1" + shift + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + mold_backup_require_cmd curl + mold_backup_require_cmd python3 + + # Sign per ApiServer.verifyRequest: sort param names, URLEncode values (+ -> %20), + # lowercase the full unsigned string, HMAC-SHA256, Base64 signature. + local url + url="$(python3 - "$MOLD_API_URL" "$MOLD_API_KEY" "$MOLD_API_SECRET" "$cmd" "$@" <<'PY' +import base64, hashlib, hmac, sys +from urllib.parse import quote, quote_plus, urlsplit, urlunsplit, parse_qsl + +api_url, apikey, secret, command, *pairs = sys.argv[1:] + +params = { + "apikey": apikey, + "command": command, + "response": "json", +} + +for p in pairs: + if "=" not in p: + continue + k, v = p.split("=", 1) + if k and v: + params[k] = v + +parts = urlsplit(api_url) +base = urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + +for k, v in parse_qsl(parts.query, keep_blank_values=True): + if k and v and k not in params: + params[k] = v + +def enc_value(v: str) -> str: + # Match Java URLEncoder.encode(..., UTF_8).replaceAll("\\+", "%20") + return quote_plus(str(v), safe="").replace("+", "%20") + +# Case-sensitive sort (java.util.Collections.sort on param names) +req_items = sorted(params.items(), key=lambda kv: kv[0]) +req_query = "&".join([f"{k}={enc_value(v)}" for k, v in req_items]) + +unsigned = req_query.lower() +sig = base64.b64encode( + hmac.new(secret.encode("utf-8"), unsigned.encode("utf-8"), hashlib.sha256).digest() +).decode("ascii") + +signed = f"{base}?{req_query}&signature={quote(sig, safe='')}" +print(signed) +PY +)" || exit $? + + mold_backup_log info "API: ${cmd} (curl)" >&2 + # Show response body on errors for debugging. + local tmp rc http_code body api_err + tmp="$(mktemp)" + http_code="$(curl -sS --connect-timeout 10 --max-time 120 -o "$tmp" -w '%{http_code}' "$url")" || { + rc=$? + rm -f "$tmp" + return "$rc" + } + body="$(cat "$tmp")" + rm -f "$tmp" + if [[ "$http_code" -lt 200 || "$http_code" -ge 300 ]]; then + echo "$body" >&2 + return 22 + fi + api_err="$(mold_backup_api_extract_error "$body" 2>/dev/null || true)" + if [[ -n "$api_err" ]]; then + mold_backup_api_log_ms_schema_hint "$api_err" + mold_backup_log err "API ${cmd} failed: ${api_err}" >&2 + echo "$body" + return 1 + fi + echo "$body" +} + +mold_backup_cmk_run() { + local cmd="$1" + shift + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + local cmk + cmk=$(mold_backup_cmk_bin) || { + mold_backup_cloudstack_api_call "$cmd" "$@" + return $? + } + local -a args=(-u "${MOLD_API_URL}" -a "${MOLD_API_KEY}" -s "${MOLD_API_SECRET}" "${cmd}") + local pair key value + for pair in "$@"; do + key="${pair%%=*}" + value="${pair#*=}" + [[ -n "$key" && -n "$value" ]] && args+=("${key}=${value}") + done + mold_backup_log info "Executing: ${cmk} ${cmd} $*" >&2 + local out rc api_err + out="$("${cmk}" "${args[@]}" 2>/dev/null)" || rc=$? + api_err="$(mold_backup_api_extract_error "$out" 2>/dev/null || true)" + if [[ -n "$api_err" ]]; then + mold_backup_api_log_ms_schema_hint "$api_err" + mold_backup_log err "API ${cmd} failed: ${api_err}" >&2 + echo "$out" + return 1 + fi + echo "$out" + return "${rc:-0}" +} + +mold_backup_resolve_vm_name() { + if [[ -n "${VM_NAME:-}" ]]; then + return 0 + fi + mold_backup_require_var VM_UUID + local json name + json=$(mold_backup_cmk_run listVirtualMachines "id=${VM_UUID}" 2>/dev/null) || true + name=$(echo "$json" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + vms = d.get('listvirtualmachinesresponse', {}).get('virtualmachine', []) + if isinstance(vms, dict): vms = [vms] + print(vms[0].get('instancename','') if vms else '') +except Exception: + print('') +" 2>/dev/null) + if [[ -n "$name" ]]; then + VM_NAME="$name" + mold_backup_log info "Resolved VM_NAME=${VM_NAME} from API" + return 0 + fi + mold_backup_die "VM_NAME is empty and could not be resolved from VM_UUID via API" +} + +mold_backup_check_libvirt_vm() { + mold_backup_resolve_vm_name + if ! virsh -c qemu:///system dominfo "${VM_NAME}" >/dev/null 2>&1; then + mold_backup_die "Libvirt domain [${VM_NAME}] not found on this host" + fi + mold_backup_log info "Libvirt domain [${VM_NAME}] is ready" +} + +mold_backup_get_live_disk_paths() { + mold_backup_log info "Resolving live libvirt disk paths (file + RBD)" + mold_backup_get_all_disk_paths +} + +mold_backup_get_all_disk_paths() { + mold_backup_resolve_vm_name + local paths=() + local target + while IFS= read -r target; do + [[ -z "$target" ]] && continue + paths+=("$target") + done < <(virsh -c qemu:///system domblklist "${VM_NAME}" --details 2>/dev/null | awk '/disk/ {print $4}') + if [[ ${#paths[@]} -eq 0 ]]; then + mold_backup_die "No disks found for VM ${VM_NAME}" + fi + (IFS=,; echo "${paths[*]}") +} + +mold_backup_has_rbd_disk() { + local csv="$1" + [[ "$csv" == rbd:* ]] && return 0 + [[ "$csv" == *",rbd:"* ]] && return 0 + return 1 +} + +# auto | qcow2 (GFS/file) | rbd (HCI/Ceph primary) +mold_backup_detect_storage_engine() { + local disk_paths_csv="${1:-}" + case "${BACKUP_STORAGE_ENGINE:-auto}" in + qcow2|rbd) echo "${BACKUP_STORAGE_ENGINE}"; return 0 ;; + esac + if mold_backup_has_rbd_disk "$disk_paths_csv"; then + echo "rbd" + else + echo "qcow2" + fi +} + +mold_backup_parse_rbd_volume_id() { + local uri="$1" image="" + [[ -n "$uri" ]] || return 0 + if [[ "$uri" == rbd:* ]]; then + image="${uri#rbd:}" + image="${image%%:*}" + elif [[ "$uri" == rbd/* ]]; then + image="${uri##*/}" + fi + echo "$image" +} + +mold_backup_disk_target_kind() { + local target + target="$(echo "${1:-}" | tr '[:upper:]' '[:lower:]')" + case "$target" in + vda|sda|hda) echo "root" ;; + *) echo "datadisk" ;; + esac +} + +# Lines: target|source_path (libvirt domblklist) +mold_backup_list_disk_specs() { + mold_backup_resolve_vm_name + virsh -c qemu:///system domblklist "${VM_NAME}" --details 2>/dev/null \ + | awk '/disk/ {print $3 "|" $4}' +} + +mold_backup_qcow2_volume_id_from_path() { + local path="$1" base uuid + base="$(basename "$path")" + uuid="${base%.qcow2}" + uuid="${uuid%.raw}" + if [[ "$uuid" =~ ^[0-9a-fA-F-]{36}$ ]]; then + echo "$uuid" + return 0 + fi + echo "" +} + +mold_backup_is_vm_running() { + mold_backup_resolve_vm_name + local state + state=$(virsh -c qemu:///system dominfo "${VM_NAME}" 2>/dev/null | awk -F: '/^State:/ {gsub(/^[ \t]+/, "", $2); print $2; exit}') + [[ "$state" == "running" ]] +} + +mold_backup_clean_repo_address() { + local addr="${BACKUP_REPO_ADDRESS}" + addr="${addr#nfs://}" + addr="${addr#cifs://}" + echo "$addr" +} + +mold_backup_meta_field() { + local file="$1" key="$2" + [[ -f "$file" ]] || return 1 + grep -E "^${key}=" "$file" 2>/dev/null | head -1 | cut -d= -f2- +} + +mold_backup_with_repo_mount() { + local callback="$1" + if mold_backup_is_datadisk_mode; then + case "${BACKUP_REPO_TYPE:-local}" in + nfs|cifs|glusterfs) + mold_backup_die "datadisk mode: NAS/network restore disabled — use BACKUP_REPO_TYPE=local and MOLD_DATADISK_PATH=${MOLD_DATADISK_PATH:-/data/backup}" + ;; + esac + fi + if [[ -n "${NAS_REPO_MOUNT:-}" && -d "${NAS_REPO_MOUNT}" ]]; then + "$callback" "${NAS_REPO_MOUNT}" + return $? + fi + + # Local data disk repo: BACKUP_REPO_ADDRESS is a directory on this host. + case "${BACKUP_REPO_TYPE:-nfs}" in + local|dir|localfs) + local local_dir + local_dir="$(mold_backup_clean_repo_address)" + [[ -d "$local_dir" ]] || mold_backup_die "Local backup directory not found: ${local_dir}" + "$callback" "$local_dir" + return $? + ;; + esac + + mold_backup_require_var BACKUP_REPO_TYPE + mold_backup_require_var BACKUP_REPO_ADDRESS + [[ -x "${NAS_BACKUP_SCRIPT}" ]] || mold_backup_die "NAS backup script not found: ${NAS_BACKUP_SCRIPT}" + + local mount_point repo_addr nas_type mount_opts mopts=() + mount_point=$(mktemp -d -t moldbackup.XXXXX) + repo_addr=$(mold_backup_clean_repo_address) + nas_type="${BACKUP_REPO_TYPE}" + mount_opts="${BACKUP_REPO_MOUNT_OPTS:-}" + if [[ "$nas_type" == "cifs" && -n "$mount_opts" ]]; then + mount_opts="${mount_opts},nobrl" + elif [[ "$nas_type" == "cifs" ]]; then + mount_opts="nobrl" + fi + [[ -n "$mount_opts" ]] && mopts=(-o "$mount_opts") + + if ! mount -t "${nas_type}" "${repo_addr}" "${mount_point}" "${mopts[@]}" 2>/dev/null; then + rmdir "${mount_point}" 2>/dev/null || true + mold_backup_die "Failed to mount NAS repository ${repo_addr} for parent lookup" + fi + + local rc=0 + "$callback" "${mount_point}" || rc=$? + umount "${mount_point}" 2>/dev/null || true + rmdir "${mount_point}" 2>/dev/null || true + return "$rc" +} + +# Sets PARENT_BACKUP_DIR_REL, PARENT_CHECKPOINT_NAME, PARENT_CHECKPOINT_PATH_REL, PARENT_BACKUP_FILES. +mold_backup_find_latest_nas_parent() { + local mount_point="$1" + mold_backup_resolve_vm_name + + local vm_dir="${mount_point}/${VM_NAME}" + [[ -d "$vm_dir" ]] || { + mold_backup_log err "No backup directory for VM on NAS: ${vm_dir}" + return 1 + } + + local latest_name="" latest_dir="" d base + for d in "${vm_dir}"/*; do + [[ -d "$d" ]] || continue + base=$(basename "$d") + if [[ -f "${d}/veeam-seed.meta" || -d "${d}/checkpoints" ]]; then + if [[ -z "$latest_name" || "$base" > "$latest_name" ]]; then + latest_name="$base" + latest_dir="$d" + fi + fi + done + + [[ -n "$latest_dir" ]] || { + mold_backup_log err "No seed or checkpoint backup found under ${vm_dir}" + return 1 + } + + PARENT_BACKUP_DIR_REL="${VM_NAME}/${latest_name}" + PARENT_CHECKPOINT_NAME="" + PARENT_CHECKPOINT_PATH_REL="" + PARENT_BACKUP_FILES="" + + if [[ -f "${latest_dir}/veeam-seed.meta" ]]; then + PARENT_CHECKPOINT_NAME=$(mold_backup_meta_field "${latest_dir}/veeam-seed.meta" checkpoint_name || true) + PARENT_BACKUP_FILES=$(mold_backup_meta_field "${latest_dir}/veeam-seed.meta" backup_files || true) + elif [[ -f "${latest_dir}/rbd-backup.meta" ]]; then + PARENT_CHECKPOINT_NAME=$(mold_backup_meta_field "${latest_dir}/rbd-backup.meta" checkpoint_name || true) + PARENT_BACKUP_FILES=$(mold_backup_meta_field "${latest_dir}/rbd-backup.meta" backup_files || true) + fi + [[ -z "$PARENT_CHECKPOINT_NAME" ]] && PARENT_CHECKPOINT_NAME="$latest_name" + + if [[ -f "${latest_dir}/checkpoints/${PARENT_CHECKPOINT_NAME}.xml" ]]; then + PARENT_CHECKPOINT_PATH_REL="${PARENT_BACKUP_DIR_REL}/checkpoints/${PARENT_CHECKPOINT_NAME}.xml" + elif [[ -f "${latest_dir}/checkpoints/${PARENT_CHECKPOINT_NAME}.meta" ]]; then + PARENT_CHECKPOINT_PATH_REL="${PARENT_BACKUP_DIR_REL}/checkpoints/${PARENT_CHECKPOINT_NAME}.meta" + else + mold_backup_log warn "Parent checkpoint file not found under ${latest_dir}/checkpoints; incremental may fail" + PARENT_CHECKPOINT_PATH_REL="${PARENT_BACKUP_DIR_REL}/checkpoints/${PARENT_CHECKPOINT_NAME}.xml" + fi + + mold_backup_log info "Local backup repo parent backup=${PARENT_BACKUP_DIR_REL} checkpoint=${PARENT_CHECKPOINT_NAME}" + return 0 +} + +mold_backup_run_local_incremental_on_mount() { + local mount_point="$1" + mold_backup_find_latest_nas_parent "$mount_point" || mold_backup_die "Cannot resolve NAS parent for incremental backup" + + mold_backup_require_var BACKUP_REPO_TYPE + mold_backup_require_var BACKUP_REPO_ADDRESS + [[ -x "${NAS_BACKUP_SCRIPT}" ]] || mold_backup_die "NAS backup script not found: ${NAS_BACKUP_SCRIPT}" + + local disk_paths backup_path checkpoint backup_files repo_addr op quiesce btype + disk_paths=$(mold_backup_get_all_disk_paths) + backup_path=$(mold_backup_generate_backup_path) + checkpoint="${backup_path##*/}" + btype="FULL" + [[ -n "${PARENT_BACKUP_DIR_REL:-}" ]] && btype="INCREMENTAL" + if [[ -n "${PARENT_BACKUP_FILES:-}" ]]; then + backup_files="${PARENT_BACKUP_FILES}" + if mold_backup_has_rbd_disk "$disk_paths" && [[ "$btype" == "INCREMENTAL" ]]; then + backup_files="${backup_files//.raw/.rbdiff}" + fi + else + backup_files=$(mold_backup_build_backup_files "$disk_paths" "$btype") + fi + repo_addr=$(mold_backup_clean_repo_address) + quiesce="${QUIESCE_VM:-false}" + + if mold_backup_has_rbd_disk "$disk_paths"; then + op="backup-rbd" + mold_backup_log info "Storage engine=rbd (HCI/Ceph primary)" + elif mold_backup_is_vm_running; then + op="backup-running" + else + mold_backup_die "VM ${VM_NAME} is not running; local incremental needs backup-running (start VM or use BACKUP_MODE=api)" + fi + + mold_backup_log info "Local datadisk incremental op=${op} path=${backup_path} parent=${PARENT_BACKUP_DIR_REL}" + "${NAS_BACKUP_SCRIPT}" \ + -o "${op}" \ + -v "${VM_NAME}" \ + -t "${BACKUP_REPO_TYPE}" \ + -s "${repo_addr}" \ + -m "${BACKUP_REPO_MOUNT_OPTS:-}" \ + -p "${backup_path}" \ + -b "INCREMENTAL" \ + -c "${checkpoint}" \ + -r "${PARENT_BACKUP_DIR_REL}" \ + -i "${PARENT_CHECKPOINT_NAME}" \ + -j "${PARENT_CHECKPOINT_PATH_REL}" \ + -q "${quiesce}" \ + -f "${backup_files}" \ + -d "${disk_paths}" \ + || mold_backup_die "ablestack_nasbackup.sh ${op} failed" +} + +mold_backup_run_local_incremental() { + mold_backup_with_repo_mount mold_backup_run_local_incremental_on_mount +} + +mold_backup_run_backup() { + case "${BACKUP_MODE}" in + host) + mold_backup_die "host mode uses pre-notify/post-notify hooks, not mold_backup_run_backup" + ;; + api) + mold_backup_api_create_backup + ;; + local) + mold_backup_run_local_incremental + ;; + auto) + if mold_backup_cmk_bin >/dev/null 2>&1; then + mold_backup_api_create_backup || { + mold_backup_log warn "API incremental backup failed, trying local NAS script" + mold_backup_run_local_incremental + } + else + mold_backup_run_local_incremental + fi + ;; + *) + mold_backup_die "Invalid BACKUP_MODE=${BACKUP_MODE} (use api|local|auto)" + ;; + esac +} + +mold_backup_check_staging() { + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" ]]; then + mold_backup_log info "FileLevel Veeam backup: staging VMDK optional (NAS seed uses live libvirt disks)" + return 0 + fi + if [[ -n "${STAGING_DISK_PATHS:-}" ]]; then + mold_backup_log info "Using STAGING_DISK_PATHS from config" + return 0 + fi + mold_backup_require_var STAGING_PATH + if [[ ! -d "${STAGING_PATH}" ]]; then + mold_backup_die "Staging directory not found: ${STAGING_PATH}" + fi + mold_backup_log info "Staging directory OK: ${STAGING_PATH}" +} + +# Disk paths for NAS seed: staging VMDK files, or live libvirt disks (FileLevel Agent). +mold_backup_resolve_seed_disk_paths() { + local staging + staging=$(mold_backup_list_staging_disks) + if [[ -n "$staging" ]]; then + echo "$staging" + return 0 + fi + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" ]]; then + mold_backup_log info "No staging disks; using live libvirt disk paths for seed import" + mold_backup_get_live_disk_paths + return 0 + fi + return 1 +} + +mold_backup_list_staging_disks() { + if [[ -n "${STAGING_DISK_PATHS:-}" ]]; then + echo "${STAGING_DISK_PATHS}" + return 0 + fi + find "${STAGING_PATH}" -maxdepth 3 -type f \( -name '*.vmdk' -o -name '*.flat' -o -name '*.qcow2' -o -name '*.raw' \) 2>/dev/null \ + | sort | paste -sd, - +} + +mold_backup_generate_backup_path() { + if [[ -n "${BACKUP_PATH:-}" ]]; then + echo "${BACKUP_PATH}" + return 0 + fi + mold_backup_resolve_vm_name + echo "${VM_NAME}/$(date '+%Y.%m.%d.%H.%M.%S.%3N')" +} + +mold_backup_build_backup_files() { + local disk_paths_csv="$1" + local backup_type="${2:-FULL}" + local -a out=() + if [[ -n "${BACKUP_FILES:-}" ]]; then + echo "${BACKUP_FILES}" + return 0 + fi + local engine suffix target path kind vol_id + engine="$(mold_backup_detect_storage_engine "$disk_paths_csv")" + suffix=".qcow2" + [[ "$backup_type" == "INCREMENTAL" ]] && suffix=".rbdiff" || true + if [[ "$engine" == "rbd" ]]; then + [[ "$backup_type" == "INCREMENTAL" ]] && suffix=".rbdiff" || suffix=".raw" + while IFS='|' read -r target path; do + [[ -n "$path" ]] || continue + vol_id="$(mold_backup_parse_rbd_volume_id "$path")" + [[ -n "$vol_id" ]] || vol_id="$(basename "$path")" + kind="$(mold_backup_disk_target_kind "$target")" + out+=("${kind}.${vol_id}${suffix}") + done < <(mold_backup_list_disk_specs) + else + local i=0 + while IFS='|' read -r target path; do + [[ -n "$path" ]] || continue + vol_id="$(mold_backup_qcow2_volume_id_from_path "$path")" + kind="$(mold_backup_disk_target_kind "$target")" + if [[ -n "$vol_id" ]]; then + out+=("${kind}.${vol_id}.qcow2") + else + out+=("disk-${i}.qcow2") + i=$((i + 1)) + fi + done < <(mold_backup_list_disk_specs) + if [[ ${#out[@]} -eq 0 ]]; then + local -a disks + IFS=, read -ra disks <<< "$disk_paths_csv" + local j=0 + for _ in "${disks[@]}"; do + out+=("disk-${j}.qcow2") + j=$((j + 1)) + done + fi + fi + [[ ${#out[@]} -gt 0 ]] || mold_backup_die "Cannot build backup file names for disks: ${disk_paths_csv}" + (IFS=,; echo "${out[*]}") +} + +mold_backup_veeam_export_ssh() { + [[ "${ENABLE_VEEAM_SSH_EXPORT}" == "true" ]] || return 0 + mold_backup_require_var VEEAM_SSH_HOST + mold_backup_require_var VEEAM_RESTORE_POINT_ID + local remote_staging="${VEEAM_STAGING_PATH_ON_SERVER:-${STAGING_PATH}}" + mold_backup_log info "Triggering Veeam FLR export on ${VEEAM_SSH_HOST}" + ssh -i "${VEEAM_SSH_KEY}" -o StrictHostKeyChecking=no "${VEEAM_SSH_USER}@${VEEAM_SSH_HOST}" powershell -Command " + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + \$rp = Get-VBRRestorePoint | Where-Object { \$_.Id -eq '${VEEAM_RESTORE_POINT_ID}' -or \$_.Id.Guid -eq '${VEEAM_RESTORE_POINT_ID}' } + if (-not \$rp) { exit 1 } + New-Item -ItemType Directory -Force -Path '${remote_staging}' | Out-Null + \$session = Start-VBRFLRSession -RestorePoint \$rp + Get-VBRFLRItem -Session \$session | Where-Object { \$_.Type -eq 'HardDisk' } | ForEach-Object { + Copy-VBRFLRItem -FLRSession \$session -Item \$_ -Destination (Join-Path '${remote_staging}' (\$_.Name + '.vmdk')) + } + Stop-VBRFLRSession -Session \$session + " || mold_backup_die "Veeam SSH export failed" +} + +mold_backup_run_local_seed_import() { + mold_backup_require_var BACKUP_REPO_TYPE + mold_backup_require_var BACKUP_REPO_ADDRESS + [[ -x "${NAS_BACKUP_SCRIPT}" ]] || mold_backup_die "NAS backup script not found: ${NAS_BACKUP_SCRIPT}" + + mold_backup_resolve_vm_name + local disk_paths staging checkpoint backup_path backup_files repo_addr source_format btype + disk_paths=$(mold_backup_get_all_disk_paths) + staging=$(mold_backup_resolve_seed_disk_paths) || mold_backup_die "No seed disk paths (staging or live libvirt disks)" + backup_path=$(mold_backup_generate_backup_path) + checkpoint="${backup_path##*/}" + btype="FULL" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "$btype") + + repo_addr="${BACKUP_REPO_ADDRESS}" + repo_addr="${repo_addr#nfs://}" + repo_addr="${repo_addr#cifs://}" + + source_format="${SOURCE_DISK_FORMAT:-vmdk}" + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" ]]; then + local first_seed="${staging%%,*}" + if [[ -f "$first_seed" ]] && command -v qemu-img >/dev/null 2>&1; then + if qemu-img info "$first_seed" 2>/dev/null | grep -q 'file format: qcow2'; then + source_format="qcow2" + elif qemu-img info "$first_seed" 2>/dev/null | grep -q 'file format: raw'; then + source_format="raw" + fi + fi + fi + + mold_backup_log info "Local datadisk seed import path=${backup_path} checkpoint=${checkpoint} source_format=${source_format}" + "${NAS_BACKUP_SCRIPT}" \ + -o import-veeam-seed \ + -v "${VM_NAME}" \ + -t "${BACKUP_REPO_TYPE}" \ + -s "${repo_addr}" \ + -m "${BACKUP_REPO_MOUNT_OPTS:-}" \ + -p "${backup_path}" \ + -c "${checkpoint}" \ + -f "${backup_files}" \ + -d "${disk_paths}" \ + --staging-disks "${staging}" \ + --source-format "${source_format}" \ + --veeam-restore-point "${VEEAM_RESTORE_POINT_ID}" \ + --bootstrap-checkpoint "${BOOTSTRAP_CHECKPOINT}" \ + || mold_backup_die "ablestack_nasbackup.sh import-veeam-seed failed" +} + +mold_backup_api_import_seed() { + mold_backup_require_var VM_UUID + local staging backup_name + staging=$(mold_backup_resolve_seed_disk_paths) || mold_backup_die "No seed disk paths for API import" + backup_name="$(mold_backup_api_build_backup_name_for_vm "${VM_UUID}" "${VM_NAME:-}")" + if mold_backup_cmk_supports importAblestackVeeamBackupSeed 2>/dev/null; then + mold_backup_cmk_run importAblestackVeeamBackupSeed \ + "virtualmachineid=${VM_UUID}" \ + "name=${backup_name}" \ + "veeamrestorepointid=${VEEAM_RESTORE_POINT_ID}" \ + "stagingdiskpaths=${staging}" \ + "sourcediskformat=${SOURCE_DISK_FORMAT}" \ + "bootstrapcheckpoint=${BOOTSTRAP_CHECKPOINT}" + return 0 + fi + mold_backup_die "importAblestackVeeamBackupSeed API not available in cloudmonkey/cmk" +} + +mold_backup_api_create_backup() { + mold_backup_api_create_veeam_backup +} + +mold_backup_api_restore() { + mold_backup_require_var BACKUP_ID + local json job_id + json=$(mold_backup_cmk_run restoreAblestackVeeamBackup "id=${BACKUP_ID}" 2>/dev/null) \ + || json=$(mold_backup_cmk_run restoreBackup "id=${BACKUP_ID}" 2>/dev/null) \ + || return 1 + job_id="$(mold_backup_api_json_field "$json" "restoreablestackveeambackupresponse.jobid")" + [[ -z "$job_id" ]] && job_id="$(mold_backup_api_json_field "$json" "restorebackupresponse.jobid")" + if [[ -n "$job_id" ]]; then + mold_backup_notify_log info "restoreAblestackVeeamBackup job=${job_id}; waiting for MS/agent restore" + mold_backup_api_wait_async_job "$job_id" 3600 || return 1 + mold_backup_notify_log info "Restore async job completed: ${job_id}" + return 0 + fi + return 0 +} + +mold_backup_cmk_supports() { + local cmd="$1" + local cmk + cmk=$(mold_backup_cmk_bin) || return 1 + "${cmk}" -h 2>/dev/null | grep -q "${cmd}" || return 1 +} + +mold_backup_api_create_veeam_backup() { + mold_backup_require_var VM_UUID + local backup_name args + backup_name="$(mold_backup_api_build_backup_name_for_vm "${VM_UUID}" "${VM_NAME:-}")" + args=("virtualmachineid=${VM_UUID}" "name=${backup_name}") + [[ "${QUIESCE_VM}" == "true" ]] && args+=("quiescevm=true") + mold_backup_cmk_run createAblestackVeeamBackup "${args[@]}" \ + || mold_backup_cmk_run createBackup "${args[@]}" +} + +mold_backup_cleanup_staging() { + [[ "${CLEANUP_STAGING_AFTER_BACKUP}" == "true" ]] || return 0 + # Guest VM mode: no Windows FLR staging on KVM hypervisor. + if [[ "${BACKUP_MODE:-}" =~ ^(guest|veeam-guest)$ ]]; then + return 0 + fi + if [[ -n "${STAGING_DISK_PATHS:-}" ]]; then + mold_backup_log info "Skipping staging dir cleanup (STAGING_DISK_PATHS set)" + return 0 + fi + if [[ -z "${STAGING_PATH:-}" ]]; then + mold_backup_log info "Skipping staging cleanup (STAGING_PATH not set)" + return 0 + fi + if [[ ! -d "${STAGING_PATH}" ]]; then + mold_backup_log info "Staging path already absent: ${STAGING_PATH}" + return 0 + fi + mold_backup_log info "Cleaning staging directory: ${STAGING_PATH}" + find "${STAGING_PATH}" -mindepth 1 -maxdepth 3 \( -name '*.vmdk' -o -name '*.flat' -o -name '*.qcow2' -o -name '*.raw' -o -name '*.meta' \) -delete 2>/dev/null || true + find "${STAGING_PATH}" -mindepth 1 -maxdepth 2 -type d -empty -delete 2>/dev/null || true +} + +mold_backup_import_seed() { + mold_backup_check_staging + mold_backup_require_var VEEAM_RESTORE_POINT_ID + case "${IMPORT_MODE}" in + api) + mold_backup_api_import_seed + ;; + local) + mold_backup_run_local_seed_import + ;; + auto) + if mold_backup_cmk_bin >/dev/null 2>&1; then + mold_backup_api_import_seed || { + mold_backup_log warn "API import failed, trying local NAS import" + mold_backup_run_local_seed_import + } + else + mold_backup_run_local_seed_import + fi + ;; + *) + mold_backup_die "Invalid IMPORT_MODE=${IMPORT_MODE} (use api|local|auto)" + ;; + esac +} + +mold_backup_run_operation() { + case "${BACKUP_OPERATION}" in + seed-import) + mold_backup_veeam_export_ssh + mold_backup_import_seed + ;; + backup) + mold_backup_run_backup + ;; + restore) + mold_backup_require_var BACKUP_ID + mold_backup_api_restore + ;; + *) + mold_backup_die "Unknown BACKUP_OPERATION=${BACKUP_OPERATION}" + ;; + esac +} + +# --- NetBackup-style policy/job hooks (bpstart / bpend / restore_notify) --- + +mold_backup_notify_log() { + local level="$1" + shift + LOG_FILE="${LOG_FILE:-/var/log/mold/veeam-hook.log}" + LOG_TAG="${LOG_TAG:-mold-veeam-hook}" + mold_backup_log "$level" "$@" +} + +mold_backup_state_dir() { + echo "${ABLESTACK_VEEAM_ETC_DIR}/state" +} + +mold_backup_state_file_for_job() { + local job="$1" + local run_id="${2:-$(date '+%Y%m%d%H%M%S')}" + echo "$(mold_backup_state_dir)/${job}.${run_id}.state" +} + +mold_backup_latest_state_file() { + local job="$1" + local dir found + dir="$(mold_backup_state_dir)" + [[ -d "$dir" ]] || return 0 + found="$(ls -1t "${dir}/${job}".*.state 2>/dev/null | head -1 || true)" + echo "$found" +} + +mold_backup_list_running_domains() { + virsh -c qemu:///system list --name --state-running 2>/dev/null | awk 'NF' || true +} + +mold_backup_domain_exists() { + local vm_name="$1" + [[ -n "$vm_name" ]] || return 1 + virsh -c qemu:///system dominfo "$vm_name" >/dev/null 2>&1 && return 0 + virsh dominfo "$vm_name" >/dev/null 2>&1 +} + +# Restore-watch target: libvirt domain and/or Mold VM on this hypervisor (shut-off OK). +mold_backup_vm_restorable_on_local_host() { + local vm="$1" + [[ -n "$vm" ]] || return 1 + if mold_backup_domain_exists "$vm" 2>/dev/null; then + return 0 + fi + local vm_id + vm_id="$(mold_backup_api_get_vm_id "$vm" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || return 1 + mold_backup_vm_owned_by_local_host "$vm" 2>/dev/null +} + +# Pre-notify/list-backups targets: running VMs, plus explicit VM_INCLUDE names (even if shut off). +mold_backup_list_target_domains() { + local -a targets=() seen="" vm_name token + while IFS= read -r vm_name; do + [[ -z "$vm_name" ]] && continue + mold_backup_vm_in_filter "$vm_name" || continue + [[ "$seen" == *"|${vm_name}|"* ]] && continue + seen="${seen}|${vm_name}|" + targets+=("$vm_name") + done < <(mold_backup_list_running_domains) + + local include="${VM_INCLUDE:-*}" + if [[ "$include" != "*" ]]; then + IFS=',' read -ra _in <<< "$include" + for token in "${_in[@]}"; do + token="$(echo "$token" | xargs)" + [[ -z "$token" ]] && continue + mold_backup_vm_in_filter "$token" || continue + mold_backup_domain_exists "$token" || continue + [[ "$seen" == *"|${token}|"* ]] && continue + seen="${seen}|${token}|" + targets+=("$token") + done + fi + + if [[ ${#targets[@]} -eq 0 ]]; then + return 0 + fi + printf '%s\n' "${targets[@]}" +} + +mold_backup_vm_in_filter() { + local vm_name="$1" + local include="${VM_INCLUDE:-*}" + local exclude="${VM_EXCLUDE:-}" + local token + + if [[ -n "$exclude" ]]; then + IFS=',' read -ra _ex <<< "$exclude" + for token in "${_ex[@]}"; do + token="$(echo "$token" | xargs)" + [[ -z "$token" ]] && continue + [[ "$vm_name" == "$token" ]] && return 1 + done + fi + + [[ "$include" == "*" ]] && return 0 + IFS=',' read -ra _in <<< "$include" + for token in "${_in[@]}"; do + token="$(echo "$token" | xargs)" + [[ -z "$token" ]] && continue + [[ "$vm_name" == "$token" ]] && return 0 + done + return 1 +} + +mold_backup_api_json_field() { + local json="$1" path="$2" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + parts = sys.argv[1].split('.') + cur = d + for p in parts: + if isinstance(cur, list) and cur: + cur = cur[0] + if not isinstance(cur, dict): + cur = None + break + cur = cur.get(p) + if isinstance(cur, list) and cur: + cur = cur[0] + print('' if cur is None else cur) +except Exception: + print('') +" "$path" 2>/dev/null +} + +mold_backup_api_list_config_value() { + local name="$1" + local json val + json=$(mold_backup_cmk_run listConfigurations "name=${name}" 2>/dev/null) || return 1 + val=$(echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + cfgs = d.get('listconfigurationsresponse', {}).get('configuration', []) + if isinstance(cfgs, dict): cfgs = [cfgs] + print(cfgs[0].get('value','') if cfgs else '') +except Exception: + print('') +" 2>/dev/null) + echo "$val" +} + +mold_backup_api_update_config_if_needed() { + local name="$1" value="$2" + local current + current="$(mold_backup_api_list_config_value "$name" 2>/dev/null || true)" + [[ "$current" == "$value" ]] && return 0 + mold_backup_cmk_run updateConfiguration "name=${name}" "value=${value}" >/dev/null \ + || mold_backup_notify_log warn "updateConfiguration ${name} failed (may need admin API key)" +} + +mold_backup_api_list_cluster_config_value() { + local name="$1" cluster_id="$2" + local json val + json=$(mold_backup_cmk_run listConfigurations "name=${name}" "clusterid=${cluster_id}" 2>/dev/null) || return 1 + val=$(echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + cfgs = d.get('listconfigurationsresponse', {}).get('configuration', []) + if isinstance(cfgs, dict): cfgs = [cfgs] + print(cfgs[0].get('value','') if cfgs else '') +except Exception: + print('') +" 2>/dev/null) + echo "$val" +} + +mold_backup_api_update_cluster_config_if_needed() { + local name="$1" value="$2" cluster_id="$3" + local current + [[ -n "$cluster_id" ]] || return 0 + current="$(mold_backup_api_list_cluster_config_value "$name" "$cluster_id" 2>/dev/null || true)" + [[ "$current" == "$value" ]] && return 0 + mold_backup_cmk_run updateConfiguration "name=${name}" "value=${value}" "clusterid=${cluster_id}" >/dev/null \ + && mold_backup_notify_log info "Enabled ${name}=${value} for cluster ${cluster_id}" \ + || mold_backup_notify_log warn "updateConfiguration ${name} clusterid=${cluster_id} failed (admin API key required)" +} + +# kvm.incremental.backup defaults to false at cluster scope — MS always chooses FULL without this. +mold_backup_api_ensure_cluster_incremental_backup() { + local json cluster_id + [[ -n "${ZONE_ID:-}" ]] || return 0 + json=$(mold_backup_cmk_run listClusters "zoneid=${ZONE_ID}" 2>/dev/null) || return 0 + while IFS= read -r cluster_id; do + [[ -z "$cluster_id" ]] && continue + mold_backup_api_update_cluster_config_if_needed "kvm.incremental.backup" "true" "$cluster_id" + done < <(printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + cs = d.get('listclustersresponse', {}).get('cluster', []) + if isinstance(cs, dict): cs = [cs] + for c in cs: + cid = c.get('id') + if cid: + print(cid) +except Exception: + pass +" 2>/dev/null) +} + +mold_backup_api_ensure_global_settings() { + mold_backup_api_update_config_if_needed "backup.framework.enabled" "true" + mold_backup_api_update_config_if_needed "backup.enable.attach.detach.of.volumes" "true" + mold_backup_api_update_config_if_needed "backup.framework.provider.plugin" "${VEEAM_PROVIDER_NAME}" + [[ -n "${VEEAM_URL:-}" ]] && mold_backup_api_update_config_if_needed "backup.plugin.ablestack-veeam.url" "${VEEAM_URL}" + [[ -n "${VEEAM_USERNAME:-}" ]] && mold_backup_api_update_config_if_needed "backup.plugin.ablestack-veeam.username" "${VEEAM_USERNAME}" + [[ -n "${VEEAM_PASSWORD:-}" ]] && mold_backup_api_update_config_if_needed "backup.plugin.ablestack-veeam.password" "${VEEAM_PASSWORD}" + mold_backup_api_ensure_cluster_incremental_backup +} + +mold_backup_api_extract_error() { + local json="$1" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + err = d.get('errorresponse', {}) + if err: + print(err.get('errortext', err)) + sys.exit(0) + for k, v in d.items(): + if k.endswith('response') and isinstance(v, dict) and v.get('errortext'): + print(v.get('errortext')) + sys.exit(0) +except Exception: + pass +" 2>/dev/null +} + +mold_backup_api_log_ms_schema_hint() { + local msg="$1" + [[ "$msg" == *backup_offering_details* ]] || return 0 + mold_backup_log err "Mold MS DB is missing table cloud.backup_offering_details (schema 4.23+). On MS host run: mysql cloud < mold-ms-backup-schema-fix.sql ; restart management server" >&2 +} + +mold_backup_api_list_backup_offerings() { + local json count err + local -a args=() + [[ -n "${ZONE_ID:-}" ]] && args+=("zoneid=${ZONE_ID}") + json=$(mold_backup_cmk_run listBackupOfferings "${args[@]}" 2>/dev/null) || return 1 + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + if [[ -n "$err" ]]; then + mold_backup_api_log_ms_schema_hint "$err" + mold_backup_notify_log err "listBackupOfferings failed: ${err}" + return 1 + fi + count=$(echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + r = d.get('listbackupofferingsresponse', {}) + c = r.get('count') + if c is None: + offs = r.get('backupoffering', []) + if isinstance(offs, dict): offs = [offs] + c = len(offs) + print(int(c or 0)) +except Exception: + print(0) +" 2>/dev/null) + if [[ "${count:-0}" -eq 0 && -n "${ZONE_ID:-}" ]]; then + json=$(mold_backup_cmk_run listBackupOfferings 2>/dev/null) || return 1 + fi + echo "$json" +} + +mold_backup_api_list_backup_repositories() { + local -a args=() + [[ -n "${ZONE_ID:-}" ]] && args+=("zoneid=${ZONE_ID}") + mold_backup_cmk_run listBackupRepositories "${args[@]}" 2>/dev/null +} + +# First zone UUID (listZones) — used by veeam_config.sh auto-fill. +mold_backup_api_first_zone_id() { + local json + json=$(mold_backup_cmk_run listZones 2>/dev/null) || return 1 + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + zones = d.get('listzonesresponse', {}).get('zone', []) + if isinstance(zones, dict): + zones = [zones] + if zones: + print(zones[0].get('id', '')) +except Exception: + pass +" 2>/dev/null +} + +# First backup repository NFS/CIFS address — used by veeam_config.sh auto-fill. +mold_backup_api_first_repo_address() { + local json name="${1:-}" + json=$(mold_backup_api_list_backup_repositories) || return 1 + echo "$json" | python3 -c " +import json, sys +name = sys.argv[1] if len(sys.argv) > 1 else '' +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): + repos = [repos] + if name: + for r in repos: + if r.get('name') == name: + print(r.get('address', '')) + sys.exit(0) + if repos: + print(repos[0].get('address', '')) +except Exception: + pass +" "$name" 2>/dev/null +} + +# importBackupOffering externalid MUST equal backup repository UUID (see BackupRepositoryDaoImpl.findByBackupOfferingId). +mold_backup_api_find_backup_repository_uuid() { + local json name="${1:-}" + json=$(mold_backup_api_list_backup_repositories) || return 1 + echo "$json" | python3 -c " +import json, sys +name = sys.argv[1] if len(sys.argv) > 1 else '' +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): repos = [repos] + if name: + for r in repos: + if r.get('name') == name: + print(r.get('id', '')) + sys.exit(0) + if repos: + print(repos[0].get('id', '')) +except Exception: + pass +" "$name" 2>/dev/null +} + +mold_backup_api_find_backup_repository_uuid_by_address() { + local address="$1" json + [[ -n "$address" ]] || return 1 + address="${address#nfs://}" + address="${address#cifs://}" + json=$(mold_backup_api_list_backup_repositories) || return 1 + echo "$json" | python3 -c " +import json, sys +want = sys.argv[1] +def norm(a): + if not a: return '' + a = a.strip() + for p in ('nfs://', 'cifs://'): + if a.startswith(p): + a = a[len(p):] + return a +want = norm(want) +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): repos = [repos] + for r in repos: + if norm(r.get('address', '')) == want: + print(r.get('id', '')) + sys.exit(0) +except Exception: + pass +" "$address" 2>/dev/null +} + +# True when id is a backup repository UUID (not a backup offering id). +mold_backup_api_repository_exists() { + local want="$1" json + [[ -n "$want" ]] || return 1 + json=$(mold_backup_api_list_backup_repositories 2>/dev/null) || return 1 + echo "$json" | python3 -c " +import json, sys +want = sys.argv[1] +try: + d = json.load(sys.stdin) + repos = d.get('listbackuprepositoriesresponse', {}).get('backuprepository', []) + if isinstance(repos, dict): + repos = [repos] + for r in repos: + if r.get('id') == want: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$want" +} + +# Create Mold backup repository when BACKUP_REPO_ADDRESS is set (addBackupRepository). +mold_backup_api_ensure_repository() { + local repo_id name addr repo_type args json err + repo_id="${BACKUP_REPOSITORY_UUID:-}" + if [[ -n "$repo_id" ]]; then + if mold_backup_api_repository_exists "$repo_id"; then + echo "$repo_id" + return 0 + fi + mold_backup_notify_log warn "BACKUP_REPOSITORY_UUID=${repo_id} is not a repository id (maybe an offering id?) — resolving from address/name" + repo_id="" + fi + + name="${BACKUP_REPO_NAME:-Ablestack Veeam NAS}" + if [[ -n "${BACKUP_REPO_ADDRESS:-}" ]]; then + addr="$(mold_backup_clean_repo_address)" + repo_id="$(mold_backup_api_find_backup_repository_uuid_by_address "$addr" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + fi + repo_id="$(mold_backup_api_find_backup_repository_uuid "$name" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + repo_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + + [[ -n "${BACKUP_REPO_ADDRESS:-}" ]] || { + mold_backup_notify_log err "No backup repository — set BACKUP_REPO_ADDRESS in conf or create in Mold UI" + return 1 + } + [[ -n "${ZONE_ID:-}" ]] || { + mold_backup_notify_log err "ZONE_ID required to addBackupRepository" + return 1 + } + + addr="$(mold_backup_clean_repo_address)" + repo_type="${BACKUP_REPO_TYPE:-nfs}" + args=( + "name=${name}" + "address=${addr}" + "type=${repo_type}" + "zoneid=${ZONE_ID}" + ) + [[ -n "${BACKUP_REPO_MOUNT_OPTS:-}" ]] && args+=("mountoptions=${BACKUP_REPO_MOUNT_OPTS}") + [[ -n "${BACKUP_REPO_PROVIDER:-}" ]] && args+=("provider=${BACKUP_REPO_PROVIDER}") + + mold_backup_notify_log info "addBackupRepository name=${name} address=${addr} type=${repo_type}" + if ! json=$(mold_backup_cmk_run addBackupRepository "${args[@]}" 2>&1); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "addBackupRepository failed${err:+: ${err}}" + repo_id="$(mold_backup_api_find_backup_repository_uuid_by_address "$addr" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + return 1 + fi + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + if [[ -n "$err" ]]; then + mold_backup_notify_log err "addBackupRepository failed: ${err}" + repo_id="$(mold_backup_api_find_backup_repository_uuid_by_address "$addr" 2>/dev/null || true)" + [[ -n "$repo_id" ]] && { echo "$repo_id"; return 0; } + return 1 + fi + repo_id="$(mold_backup_api_json_field "$json" "addbackuprepositoryresponse.backuprepository.id")" + [[ -n "$repo_id" ]] || repo_id="$(mold_backup_api_find_backup_repository_uuid "$name" 2>/dev/null || true)" + [[ -n "$repo_id" ]] || { + mold_backup_notify_log err "addBackupRepository returned no repository id" + return 1 + } + mold_backup_notify_log info "Backup repository ready id=${repo_id}" + echo "$repo_id" +} + +# Ensure backup repository + offering exist (NAS/guest). Datadisk host mode uses KVM +# /data/backup only — assign backup offering in Mold UI; no addBackupRepository. +mold_backup_api_ensure_backup_resources() { + local offering_id repo_id + if mold_backup_is_datadisk_mode; then + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" "$(mold_backup_offering_name)" 2>/dev/null || true)" + [[ -n "$offering_id" ]] || offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + if [[ -n "$offering_id" ]]; then + echo "$offering_id" + return 0 + fi + mold_backup_notify_log warn "datadisk mode: assign backup offering '$(mold_backup_offering_name)' (${VEEAM_PROVIDER_NAME}) in Mold UI — Mold backup repository is not auto-created" + return 1 + fi + repo_id="$(mold_backup_api_ensure_repository 2>/dev/null || true)" + if [[ -n "$repo_id" ]]; then + BACKUP_REPOSITORY_UUID="$repo_id" + OFFERING_EXTERNAL_ID="${OFFERING_EXTERNAL_ID:-$repo_id}" + fi + offering_id="$(mold_backup_api_ensure_offering 2>/dev/null || true)" + [[ -n "$offering_id" ]] && { echo "$offering_id"; return 0; } + return 1 +} + +mold_backup_api_get_offering_external_id() { + local offering_id="$1" json + json=$(mold_backup_api_list_backup_offerings 2>/dev/null) || return 1 + echo "$json" | python3 -c " +import json, sys +oid = sys.argv[1] +try: + d = json.load(sys.stdin) + offs = d.get('listbackupofferingsresponse', {}).get('backupoffering', []) + if isinstance(offs, dict): offs = [offs] + for o in offs: + if o.get('id') == oid: + print(o.get('externalid', '')) + break +except Exception: + pass +" "$offering_id" 2>/dev/null +} + +mold_backup_api_validate_offering_repository() { + local offering_id="$1" + if mold_backup_is_datadisk_mode; then + [[ -n "$offering_id" ]] && return 0 + return 1 + fi + local ext_id repo_id + ext_id="$(mold_backup_api_get_offering_external_id "$offering_id" 2>/dev/null || true)" + repo_id="${BACKUP_REPOSITORY_UUID:-}" + [[ -n "$repo_id" ]] || repo_id="$(mold_backup_api_find_backup_repository_uuid "${BACKUP_REPO_NAME:-}" 2>/dev/null || true)" + [[ -n "$repo_id" ]] || repo_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + [[ -n "$repo_id" ]] || { + mold_backup_notify_log err "No backup repository in zone — create one in Mold UI (Infrastructure → Backup Repositories)" + return 1 + } + [[ "$ext_id" == "$repo_id" ]] && return 0 + mold_backup_notify_log err "Backup offering externalid=${ext_id:-} does not match repository id=${repo_id}. Re-import offering with: externalid=${repo_id} (or set BACKUP_REPOSITORY_UUID in conf)" + return 1 +} + +mold_backup_api_log_repository_hint() { + local msg="$1" + [[ "$msg" == *"backup repository"* ]] || return 0 + local repo_id + repo_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + mold_backup_notify_log err "Fix: importBackupOffering externalid must equal backup repository UUID${repo_id:+ (${repo_id})}" +} + +mold_backup_api_log_agent_import_seed_hint() { + local msg="$1" + [[ "$msg" == *UnsupportedAnswer* ]] || return 0 + mold_backup_notify_log err "KVM mold-agent does not handle AblestackNasImportVeeamSeedCommand (import-veeam-seed). Update mold-agent to a build that includes LibvirtAblestackNasImportVeeamSeedCommandWrapper, ensure ablestack_nasbackup.sh supports -o import-veeam-seed, then: systemctl restart cloudstack-agent" +} + +mold_backup_api_extract_async_job_error() { + local json="$1" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + r = d.get('queryasyncjobresultresponse', {}) + jr = r.get('jobresult') + if isinstance(jr, dict): + if jr.get('errortext'): + print(jr.get('errortext')) + elif jr.get('errorresponse', {}).get('errortext'): + print(jr['errorresponse']['errortext']) + elif isinstance(jr, str) and jr.strip(): + print(jr.strip()) + if r.get('errortext'): + print(r.get('errortext')) +except Exception: + pass +" 2>/dev/null +} + +mold_backup_offering_name() { + echo "${BACKUP_OFFERING_NAME:-VeeamBackup}" +} + +mold_backup_api_find_offering_id() { + local provider="${1:-${VEEAM_PROVIDER_NAME}}" + local offering_name="${2:-$(mold_backup_offering_name)}" + local json + json=$(mold_backup_api_list_backup_offerings) || return 1 + echo "$json" | python3 -c " +import json, sys +provider = (sys.argv[1] or '').lower() +name = sys.argv[2] if len(sys.argv) > 2 else '' +aliases = {provider} +if provider in ('ablestack-veeam', 'veeam'): + aliases.update(['ablestack-veeam', 'veeam']) +if provider == 'ablestack-nas': + aliases.update(['ablestack-nas', 'nas']) +try: + d = json.load(sys.stdin) + offs = d.get('listbackupofferingsresponse', {}).get('backupoffering', []) + if isinstance(offs, dict): offs = [offs] + if name: + for o in offs: + if o.get('name') == name: + print(o.get('id', '')) + sys.exit(0) + for o in offs: + p = (o.get('provider') or '').lower() + if p in aliases: + print(o.get('id', '')) + break +except Exception: + pass +" "$provider" "$offering_name" 2>/dev/null +} + +mold_backup_api_ensure_offering() { + local offering_id json err want_ext ext_id + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + want_ext="${OFFERING_EXTERNAL_ID:-${BACKUP_REPOSITORY_UUID:-}}" + if [[ -n "$offering_id" && -n "$want_ext" ]]; then + ext_id="$(mold_backup_api_get_offering_external_id "$offering_id" 2>/dev/null || true)" + if [[ -n "$ext_id" && "$ext_id" != "$want_ext" ]]; then + mold_backup_notify_log warn "Backup offering id=${offering_id} externalid=${ext_id} != repository ${want_ext}" + mold_backup_notify_log warn "Delete '$(mold_backup_offering_name)' in Mold UI (Infrastructure → Backup Offerings) then re-run: mold_backup_api_ensure_backup_resources" + offering_id="" + fi + fi + [[ -n "$offering_id" ]] && { echo "$offering_id"; return 0; } + [[ -n "${ZONE_ID:-}" ]] || { + mold_backup_notify_log err "ZONE_ID required to importBackupOffering" + return 1 + } + local name + name="$(mold_backup_offering_name)" + local ext_id="${OFFERING_EXTERNAL_ID:-${BACKUP_REPOSITORY_UUID:-}}" + if [[ -z "$ext_id" ]]; then + ext_id="$(mold_backup_api_find_backup_repository_uuid 2>/dev/null || true)" + fi + [[ -n "$ext_id" ]] || { + mold_backup_notify_log err "No backup repository UUID for importBackupOffering — create Backup Repository in Mold UI or set BACKUP_REPOSITORY_UUID" + return 1 + } + local retention="${RETENTION_PERIOD:-P7D}" + local args=( + "name=${name}" + "description=Ablestack Veeam backup offering (${name})" + "provider=${VEEAM_PROVIDER_NAME}" + "externalid=${ext_id}" + "zoneid=${ZONE_ID}" + "allowuserdrivenbackups=false" + "retentionperiod=${retention}" + ) + if ! json=$(mold_backup_cmk_run importBackupOffering "${args[@]}" 2>&1); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importBackupOffering failed${err:+: ${err}}" + return 1 + fi + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + if [[ -n "$err" ]]; then + mold_backup_notify_log err "importBackupOffering failed: ${err}" + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + [[ -n "$offering_id" ]] && { echo "$offering_id"; return 0; } + return 1 + fi + offering_id="$(mold_backup_api_json_field "$json" "importbackupofferingresponse.backupoffering.id")" + local job_id + job_id="$(mold_backup_api_json_field "$json" "importbackupofferingresponse.jobid")" + if [[ -z "$offering_id" && -n "$job_id" ]]; then + mold_backup_notify_log info "importBackupOffering async job=${job_id}; waiting" + json=$(mold_backup_api_wait_async_job "$job_id" 300) || return 1 + offering_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backupoffering.id")" + [[ -z "$offering_id" ]] && offering_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.id")" + fi + [[ -n "$offering_id" ]] || offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" 2>/dev/null || true)" + [[ -n "$offering_id" ]] || { + mold_backup_notify_log err "importBackupOffering returned no offering id (async job may still be running; check listBackupOfferings)" + return 1 + } + echo "$offering_id" +} + +mold_backup_safe_job_name() { + echo "$1" | tr ' /' '__' +} + +mold_backup_registry_dir() { + echo "${ABLESTACK_VEEAM_ETC_DIR}/registry" +} + +mold_backup_api_pick_vm_record() { + local json="$1" lookup="$2" + echo "$json" | python3 -c " +import json, sys +lookup = sys.argv[1] +try: + d = json.load(sys.stdin) + vms = d.get('listvirtualmachinesresponse', {}).get('virtualmachine', []) + if isinstance(vms, dict): vms = [vms] + for v in vms: + if v.get('instancename') == lookup or v.get('name') == lookup: + print(json.dumps({'listvirtualmachinesresponse': {'count': 1, 'virtualmachine': v}})) + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$lookup" 2>/dev/null +} + +mold_backup_api_get_vm_record() { + local lookup="$1" + local -a args=("listall=true") + local json picked + local try_zone="${2:-}" + + _mold_backup_list_vms() { + local -a call_args=("listall=true") + [[ -n "$1" ]] && call_args+=("zoneid=$1") + mold_backup_cmk_run listVirtualMachines "${call_args[@]}" 2>/dev/null + } + + if [[ -n "$try_zone" ]]; then + json="$(_mold_backup_list_vms "$try_zone")" || json="" + if picked=$(mold_backup_api_pick_vm_record "$json" "$lookup" 2>/dev/null); then + echo "$picked" + return 0 + fi + fi + + json=$(mold_backup_cmk_run listVirtualMachines "listall=true" "name=${lookup}" 2>/dev/null) || true + if picked=$(mold_backup_api_pick_vm_record "$json" "$lookup" 2>/dev/null); then + echo "$picked" + return 0 + fi + + json="$(_mold_backup_list_vms "")" || return 1 + mold_backup_api_pick_vm_record "$json" "$lookup" +} + +mold_backup_api_get_vm_id() { + local instance_name="$1" json + if [[ "${VM_NAME:-}" == "$instance_name" && -n "${VM_UUID:-}" ]]; then + echo "$VM_UUID" + return 0 + fi + json=$(mold_backup_api_get_vm_record "$instance_name" "${ZONE_ID:-}") || return 1 + mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.id" +} + +mold_backup_api_get_vm_offering_id() { + local instance_name="$1" json + json=$(mold_backup_api_get_vm_record "$instance_name") || return 1 + mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid" +} + +# Mold UI backup name: {vm-hostname}-{yyyy-MM-ddTHH:mm:ss+0000} (same as netbackup / getBackupNameFromVM) +mold_backup_api_format_backup_name() { + local vm_label="$1" + echo "${vm_label}-$(date -u +%Y-%m-%dT%H:%M:%S+0000)" +} + +mold_backup_api_get_vm_hostname() { + local vm_id="$1" json name + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null) || return 1 + name="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.name")" + [[ -n "$name" ]] || name="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.instancename")" + [[ -n "$name" ]] || return 1 + echo "$name" +} + +mold_backup_api_build_backup_name_for_vm() { + local vm_id="$1" fallback_label="${2:-}" + local vm_label + # Prefer Mold VM hostname (e.g. backup-test) over libvirt instance name (i-2-7-VM) + vm_label="$(mold_backup_api_get_vm_hostname "$vm_id" 2>/dev/null || true)" + [[ -n "$vm_label" ]] || vm_label="$fallback_label" + [[ -n "$vm_label" ]] || vm_label="$vm_id" + mold_backup_api_format_backup_name "$vm_label" +} + +mold_backup_api_assign_offering_if_needed() { + local vm_id="$1" offering_id="$2" + local json current + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null) || return 1 + current="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + [[ "$current" == "$offering_id" ]] && return 0 + mold_backup_cmk_run assignVirtualMachineToBackupOffering "virtualmachineid=${vm_id}" "backupofferingid=${offering_id}" >/dev/null \ + || mold_backup_notify_log warn "assignVirtualMachineToBackupOffering failed for vm=${vm_id}" +} + +mold_backup_api_wait_async_job() { + local job_id="$1" max_wait="${2:-600}" + local elapsed=0 json status result + [[ -z "$job_id" ]] && return 1 + while [[ "$elapsed" -lt "$max_wait" ]]; do + json=$(mold_backup_cmk_run queryAsyncJobResult "jobid=${job_id}" 2>/dev/null) || return 1 + status="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobstatus")" + if [[ "$status" == "1" ]]; then + result="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult")" + echo "$json" + return 0 + fi + if [[ "$status" == "2" ]]; then + local job_err + job_err="$(mold_backup_api_extract_async_job_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "Async job failed: ${job_id}${job_err:+ — ${job_err}}" + mold_backup_api_log_repository_hint "$job_err" + mold_backup_api_log_agent_import_seed_hint "$job_err" + return 1 + fi + if [[ $((elapsed % 30)) -eq 0 ]]; then + mold_backup_notify_log info "Async job ${job_id} pending (status=${status:-0}, elapsed=${elapsed}s/${max_wait}s)" + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + mold_backup_notify_log err "Async job timeout: ${job_id}" + return 1 +} + +mold_backup_api_create_veeam_and_wait() { + local vm_id="$1" vm_label="${2:-}" + local json job_id backup_id backup_type backup_name + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "createAblestackVeeamBackup vm=${vm_id} name=${backup_name} (MS→agent NAS backup)" + json=$(mold_backup_cmk_run createAblestackVeeamBackup "virtualmachineid=${vm_id}" "name=${backup_name}" 2>/dev/null) \ + || json=$(mold_backup_cmk_run createBackup "virtualmachineid=${vm_id}" "name=${backup_name}" 2>/dev/null) \ + || return 1 + job_id="$(mold_backup_api_json_field "$json" "createablestackveeambackupresponse.jobid")" + [[ -z "$job_id" ]] && job_id="$(mold_backup_api_json_field "$json" "createbackupresponse.jobid")" + if [[ -n "$job_id" ]]; then + mold_backup_notify_log info "createAblestackVeeamBackup job=${job_id}; waiting for MS/NAS backup" + json=$(mold_backup_api_wait_async_job "$job_id" 1200) || return 1 + backup_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.id")" + backup_type="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.type")" + if [[ -z "$backup_id" ]]; then + local latest + latest="$(mold_backup_api_find_latest_backed_up_backup_for_vm "$vm_id" 2>/dev/null || true)" + if [[ -n "$latest" ]]; then + backup_id="${latest%%|*}" + backup_type="${latest#*|}" + mold_backup_notify_log info "Resolved backup_id=${backup_id} type=${backup_type} via listAblestackVeeamBackups (API returns SuccessResponse only)" + fi + fi + [[ -n "$backup_id" ]] && { echo "${backup_id}|${backup_type:-User}"; return 0; } + fi + backup_id="$(mold_backup_api_json_field "$json" "createablestackveeambackupresponse.backup.id")" + backup_type="$(mold_backup_api_json_field "$json" "createablestackveeambackupresponse.backup.type")" + [[ -n "$backup_id" ]] && { echo "${backup_id}|${backup_type:-User}"; return 0; } + return 1 +} + +# Step 4 — environment check: target VM + incremental chain count (설계: 대상머신, Chain 수) +mold_backup_api_count_backed_up_from_json() { + local json="$1" + printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): b = [b] + backed = [x for x in b if str(x.get('status', '')).lower() == 'backedup'] + print(len(backed)) +except Exception: + print(0) +" 2>/dev/null +} + +mold_backup_api_check_vm_environment() { + local vm_id="$1" vm_name="$2" + local json chain_size max_chain offering_id + max_chain="${VEEAM_MAX_CHAIN:-7}" + json=$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null) || { + mold_backup_notify_log info "env vm=${vm_name} id=${vm_id} chain=0 max=${max_chain} (no BackedUp backups)" + return 0 + } + chain_size=$(mold_backup_api_count_backed_up_from_json "$json") + mold_backup_notify_log info "env vm=${vm_name} id=${vm_id} chain=${chain_size} max=${max_chain}" + if [[ "$chain_size" -ge "$max_chain" ]]; then + mold_backup_notify_log warn "Chain size ${chain_size} >= max ${max_chain}; next backup may be full" + fi +} + +mold_backup_api_veeam_backup_count() { + local vm_id="$1" + local json + json=$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null) || { + echo 0 + return 0 + } + mold_backup_api_count_backed_up_from_json "$json" +} + +# createAblestackVeeamBackup async job returns SuccessResponse (no backup id in jobresult). +mold_backup_api_find_latest_backed_up_backup_for_vm() { + local vm_id="$1" json + json=$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null) || return 1 + printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): + b = [b] + backed = [x for x in b if str(x.get('status', '')).lower() == 'backedup'] + if not backed: + sys.exit(1) + latest = max(backed, key=lambda x: x.get('created') or '') + bid = latest.get('id', '') + btype = latest.get('type', 'User') + if not bid: + sys.exit(1) + print(f\"{bid}|{btype}\") +except Exception: + sys.exit(1) +" 2>/dev/null +} + +mold_backup_collect_host_staging_paths() { + local host_dir="$1" + local -a files=() + local f seen="" key + [[ -d "$host_dir" ]] || return 1 + while IFS= read -r -d '' f; do + key=",${f}," + [[ "$seen" == *"$key"* ]] && continue + seen="${seen}${key}" + files+=("$f") + done < <(find "$host_dir" -maxdepth 1 -type f \( -name 'disk-*' -o -name '*.qcow2' -o -name '*.qcow' -o -name '*.vmdk' -o -name '*.raw' \) -print0 2>/dev/null) + [[ ${#files[@]} -gt 0 ]] || return 1 + (IFS=,; echo "${files[*]}") +} + +mold_backup_detect_staging_source_format() { + local staging_paths="$1" + local first="${staging_paths%%,*}" + [[ -n "$first" && -f "$first" ]] || { echo "qcow2"; return 0; } + if command -v qemu-img >/dev/null 2>&1; then + if qemu-img info "$first" 2>/dev/null | grep -q 'file format: raw'; then + echo "raw" + return 0 + fi + if qemu-img info "$first" 2>/dev/null | grep -q 'file format: qcow2'; then + echo "qcow2" + return 0 + fi + fi + case "$first" in + *.raw) echo "raw" ;; + *.vmdk) echo "vmdk" ;; + *) echo "qcow2" ;; + esac +} + +# cmk may prefix stderr noise when captured with 2>&1; keep the JSON object only. +mold_backup_api_sanitize_json() { + local raw="$1" + printf '%s' "$raw" | python3 -c " +import json, sys +raw = sys.stdin.read() +start = raw.find('{') +if start < 0: + print(raw) + sys.exit(0) +blob = raw[start:] +for end in range(len(blob), 0, -1): + try: + d = json.loads(blob[:end]) + print(json.dumps(d)) + sys.exit(0) + except Exception: + pass +print(blob) +" 2>/dev/null +} + +# Parse importAblestackVeeamBackupSeed response; waits on jobid when present. +# Prints backup_id|type on stdout. +mold_backup_api_finish_import_seed_response() { + local json="$1" + local job_id backup_id backup_type err + json="$(mold_backup_api_sanitize_json "$json")" + job_id="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.jobid")" + backup_id="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.backup.id")" + [[ -z "$backup_id" ]] && backup_id="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.id")" + backup_type="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.backup.type")" + [[ -z "$backup_type" ]] && backup_type="$(mold_backup_api_json_field "$json" "importablestackveeambackupseedresponse.type")" + if [[ -n "$job_id" ]]; then + mold_backup_notify_log info "importAblestackVeeamBackupSeed job=${job_id}; waiting for NAS seed import" + json=$(mold_backup_api_wait_async_job "$job_id" 1200) || return 1 + json="$(mold_backup_api_sanitize_json "$json")" + backup_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.id")" + backup_type="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.backup.type")" + [[ -z "$backup_id" ]] && backup_id="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.id")" + [[ -z "$backup_type" ]] && backup_type="$(mold_backup_api_json_field "$json" "queryasyncjobresultresponse.jobresult.type")" + fi + [[ -n "$backup_id" ]] && { echo "${backup_id}|${backup_type:-User}"; return 0; } + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importAblestackVeeamBackupSeed: no backup id in response${err:+ — ${err}} (raw=${json:0:240})" + return 1 +} + +mold_backup_api_import_seed_and_wait() { + local vm_id="$1" staging_paths="$2" source_format="${3:-qcow2}" vm_label="${4:-}" + local json backup_name + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "importAblestackVeeamBackupSeed vm=${vm_id} name=${backup_name} (host staging, no MS→Veeam API)" + json=$(mold_backup_cmk_run importAblestackVeeamBackupSeed \ + "virtualmachineid=${vm_id}" \ + "name=${backup_name}" \ + "stagingdiskpaths=${staging_paths}" \ + "sourcediskformat=${source_format}" \ + "bootstrapcheckpoint=true" 2>/dev/null) || return 1 + mold_backup_api_finish_import_seed_response "$json" +} + +# Import NAS seed from KVM host staging; optionally tag the Veeam restore point on the backup. +mold_backup_api_import_staging_rp_seed_and_wait() { + local vm_id="$1" staging_paths="$2" source_format="${3:-qcow2}" rp_id="${4:-}" vm_label="${5:-}" + local json backup_name err + [[ -n "$staging_paths" ]] || return 1 + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "importAblestackVeeamBackupSeed vm=${vm_id} rp=${rp_id:-n/a} name=${backup_name} (host staging)" + local -a api_args=( + "virtualmachineid=${vm_id}" + "name=${backup_name}" + "stagingdiskpaths=${staging_paths}" + "sourcediskformat=${source_format}" + "bootstrapcheckpoint=true" + ) + [[ -n "$rp_id" ]] && api_args+=("veeamrestorepointid=${rp_id}") + if ! json=$(mold_backup_cmk_run importAblestackVeeamBackupSeed "${api_args[@]}" 2>/dev/null); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importAblestackVeeamBackupSeed failed: ${err:-${json:0:200}}" + return 1 + fi + mold_backup_api_finish_import_seed_response "$json" +} + +# Import NAS seed from a Veeam restore point (MS exports disks from Veeam; no KVM staging). +mold_backup_api_import_rp_seed_and_wait() { + local vm_id="$1" rp_id="$2" vm_label="${3:-}" + local json backup_name err + [[ -n "$rp_id" ]] || return 1 + backup_name="$(mold_backup_api_build_backup_name_for_vm "$vm_id" "$vm_label")" + mold_backup_notify_log info "importAblestackVeeamBackupSeed vm=${vm_id} rp=${rp_id} name=${backup_name} (MS→Veeam export)" + if ! json=$(mold_backup_cmk_run importAblestackVeeamBackupSeed \ + "virtualmachineid=${vm_id}" \ + "name=${backup_name}" \ + "veeamrestorepointid=${rp_id}" \ + "bootstrapcheckpoint=true" 2>/dev/null); then + err="$(mold_backup_api_extract_error "$json" 2>/dev/null || true)" + mold_backup_notify_log err "importAblestackVeeamBackupSeed (MS→Veeam) failed: ${err:-${json:0:200}}" + return 1 + fi + mold_backup_api_finish_import_seed_response "$json" +} + +mold_backup_api_list_backup_details() { + local backup_id="$1" + mold_backup_cmk_run listBackups "id=${backup_id}" "listvmdetails=true" 2>/dev/null +} + +mold_backup_api_backup_detail_field() { + local backup_id="$1" key="$2" + local json + json=$(mold_backup_api_list_backup_details "$backup_id") || return 1 + echo "$json" | python3 -c " +import json, sys +key = sys.argv[1] +try: + d = json.load(sys.stdin) + b = d.get('listbackupsresponse', {}).get('backup', {}) + if isinstance(b, list): + b = b[0] if b else {} + details = b.get('vmdetails') or b.get('vmDetails') or {} + if isinstance(details, str): + details = json.loads(details) if details else {} + val = details.get(key, '') + if not val and isinstance(b.get('details'), dict): + val = b['details'].get(key, '') + print(val or '') +except Exception: + print('') +" "$key" 2>/dev/null +} + +mold_backup_api_get_backup_type() { + local backup_id="$1" json + json=$(mold_backup_api_list_backup_details "$backup_id") || return 1 + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listbackupsresponse', {}).get('backup', {}) + if isinstance(b, list): + b = b[0] if b else {} + print(b.get('type', '') or '') +except Exception: + print('') +" 2>/dev/null +} + +mold_backup_api_get_parent_backup_id() { + local backup_id="$1" + mold_backup_api_backup_detail_field "$backup_id" "nas.parent.backup.uuid" +} + +mold_backup_api_is_full_backup() { + local backup_id="$1" btype parent + btype="$(mold_backup_api_get_backup_type "$backup_id" 2>/dev/null || true)" + case "${btype^^}" in + INCREMENTAL) return 1 ;; + FULL) return 0 ;; + esac + parent="$(mold_backup_api_get_parent_backup_id "$backup_id" 2>/dev/null || true)" + [[ -z "$parent" ]] +} + +# Build restore chain oldest → newest (설계 5: inc 복원 시 필요한 백업본 배열) +mold_backup_api_build_restore_chain() { + local backup_id="$1" + local -a chain=() + local current="$backup_id" parent visited="" + while [[ -n "$current" ]]; do + if [[ ",${visited}," == *",${current},"* ]]; then + mold_backup_notify_log err "Restore chain cycle at backup ${current}" + return 1 + fi + visited="${visited},${current}" + chain=("$current" "${chain[@]}") + parent="$(mold_backup_api_get_parent_backup_id "$current" 2>/dev/null || true)" + current="$parent" + done + (IFS=,; echo "${chain[*]}") +} + +mold_backup_state_write_line() { + local state_file="$1" + shift + echo "$*" >> "$state_file" +} + +mold_backup_state_parse_field() { + local line="$1" key="$2" + if [[ "$line" != *"${key}="* ]]; then + echo "" + return 0 + fi + local val="${line#*${key}=}" + val="${val%% *}" + echo "$val" +} + +# Canonical per-VM latest backup id (used by restore-watch / Mold restore). +mold_backup_vm_backup_id_map_file() { + echo "$(mold_backup_registry_dir)/vm-backup-ids.map" +} + +mold_backup_registry_set_vm_backup_id() { + local vm="$1" backup_id="$2" rp_id="${3:-}" job="${4:-${VEEAM_JOB_NAME:-}}" + local map_file line key + [[ -n "$vm" && -n "$backup_id" ]] || return 1 + map_file="$(mold_backup_vm_backup_id_map_file)" + mkdir -p "$(dirname "$map_file")" + key="vm=${vm} backup_id=${backup_id}" + [[ -n "$rp_id" ]] && key="${key} rp=${rp_id}" + [[ -n "$job" ]] && key="${key} job=${job}" + if [[ -f "$map_file" ]] && grep -q " vm=${vm} " "$map_file" 2>/dev/null; then + sed -i "s|.* vm=${vm} .*|$(date -Iseconds) ${key}|" "$map_file" + else + echo "$(date -Iseconds) ${key}" >> "$map_file" + fi + echo "${backup_id}" > "$(mold_backup_registry_dir)/${vm}.latest-backup-id" + [[ -n "$rp_id" ]] && echo "${rp_id}" > "$(mold_backup_registry_dir)/${vm}.latest-rp-id" + [[ -n "$rp_id" ]] && mold_backup_registry_index_rp_backup "$vm" "$rp_id" "$backup_id" "$job" + mold_backup_sync_vm_backup_ids_conf "$vm" "$backup_id" +} + +mold_backup_normalize_rp_id() { + local rp="${1:-}" + rp="${rp//\{/}" + rp="${rp//\}/}" + rp="$(printf '%s' "$rp" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + echo "$rp" +} + +mold_backup_registry_rp_map_file() { + echo "$(mold_backup_registry_dir)/veeam-rp-backup.map" +} + +# Persist Veeam restore point → Mold backup_id (per VM) for FLR restore-watch. +mold_backup_registry_index_rp_backup() { + local vm="$1" rp_id="$2" backup_id="$3" job="${4:-${VEEAM_JOB_NAME:-}}" + local map_file norm_rp + [[ -n "$vm" && -n "$rp_id" && -n "$backup_id" ]] || return 0 + norm_rp="$(mold_backup_normalize_rp_id "$rp_id")" + [[ -n "$norm_rp" ]] || return 0 + map_file="$(mold_backup_registry_rp_map_file)" + mkdir -p "$(dirname "$map_file")" + if [[ -f "$map_file" ]]; then + grep -viE " rp=${norm_rp} vm=${vm} " "$map_file" > "${map_file}.tmp" 2>/dev/null || : >"${map_file}.tmp" + mv -f "${map_file}.tmp" "$map_file" 2>/dev/null || true + fi + echo "$(date -Iseconds) rp=${norm_rp} vm=${vm} backup_id=${backup_id} job=${job}" >>"$map_file" + echo "${backup_id}" >"$(mold_backup_registry_dir)/${vm}.rp-${norm_rp}.backup-id" +} + +mold_backup_registry_get_backup_id_by_rp() { + local vm="$1" rp_id="$2" + local norm_rp map_file line bid reg_dir + [[ -n "$vm" && -n "$rp_id" ]] || return 1 + norm_rp="$(mold_backup_normalize_rp_id "$rp_id")" + [[ -n "$norm_rp" ]] || return 1 + reg_dir="$(mold_backup_registry_dir)" + if [[ -f "${reg_dir}/${vm}.rp-${norm_rp}.backup-id" ]]; then + bid="$(tr -d '[:space:]' <"${reg_dir}/${vm}.rp-${norm_rp}.backup-id" 2>/dev/null || true)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + fi + map_file="$(mold_backup_registry_rp_map_file)" + if [[ -f "$map_file" ]]; then + line="$(grep -E " rp=${norm_rp} vm=${vm} " "$map_file" 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + fi + if [[ -d "$reg_dir" ]]; then + line="$(grep -hE "vm=${vm}.*backup_id=.*rp=${norm_rp}|vm=${vm}.*rp=${norm_rp}.*backup_id=" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" + fi +} + +mold_backup_registry_get_backup_id_by_checkpoint() { + local vm="$1" ckpt="$2" reg_dir line bid + [[ -n "$vm" && -n "$ckpt" ]] || return 1 + reg_dir="$(mold_backup_registry_dir)" + line="$(grep -hE "vm=${vm}.*backup_id=.*${ckpt}" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + line="$(grep "backup_id=.*${vm}.*${ckpt}\|${vm}/${ckpt}" /var/log/mold/veeam-hook.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" +} + +mold_backup_registry_get_vm_backup_id() { + local vm="$1" map_file line bid + [[ -n "$vm" ]] || return 1 + if [[ -f "$(mold_backup_registry_dir)/${vm}.latest-backup-id" ]]; then + bid="$(tr -d '[:space:]' < "$(mold_backup_registry_dir)/${vm}.latest-backup-id" 2>/dev/null || true)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + fi + map_file="$(mold_backup_vm_backup_id_map_file)" + [[ -f "$map_file" ]] || return 1 + line="$(grep -E "^[^ ]* vm=${vm} " "$map_file" 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" +} + +mold_backup_registry_get_backup_id_by_checkpoint() { + local vm="$1" ckpt="$2" reg_dir line bid + [[ -n "$vm" && -n "$ckpt" ]] || return 1 + reg_dir="$(mold_backup_registry_dir)" + line="$(grep -hE "vm=${vm}.*backup_id=.*${ckpt}|backup_id=.*vm=${vm}.*${ckpt}" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && { echo "$bid"; return 0; } + line="$(grep -rh "path=.*${vm}/${ckpt}\|${vm}/${ckpt}" /var/log/mold/veeam-hook.log 2>/dev/null | grep backup_id | tail -1 || true)" + bid="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$bid" ]] && echo "$bid" +} + +# Keep mold-backup.windows.conf VM_BACKUP_IDS in sync for Veeam restore-watch. +mold_backup_sync_vm_backup_ids_conf() { + local vm="$1" backup_id="$2" + local win_conf="${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.windows.conf" + local existing entry new_val + [[ -n "$vm" && -n "$backup_id" && -f "$win_conf" ]] || return 0 + entry="${vm}:${backup_id}" + existing="$(grep -E '^VM_BACKUP_IDS=' "$win_conf" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + if [[ -z "$existing" ]]; then + echo "VM_BACKUP_IDS=\"${entry}\"" >> "$win_conf" + return 0 + fi + new_val="$existing" + if [[ "$existing" == *"${vm}:"* ]]; then + new_val="$(echo "$existing" | sed -E "s#${vm}:[^,;]*#${vm}:${backup_id}#g")" + else + new_val="${existing},${entry}" + fi + if grep -qE '^VM_BACKUP_IDS=' "$win_conf" 2>/dev/null; then + sed -i "s#^VM_BACKUP_IDS=.*#VM_BACKUP_IDS=\"${new_val}\"#" "$win_conf" + else + echo "VM_BACKUP_IDS=\"${new_val}\"" >> "$win_conf" + fi +} + +mold_backup_registry_save_backup() { + local job="$1" vm_name="$2" backup_id="$3" status="${4:-success}" rp_id="${5:-}" + local reg_dir reg_file + reg_dir="$(mold_backup_registry_dir)" + mkdir -p "$reg_dir" + reg_file="${reg_dir}/$(mold_backup_safe_job_name "$job").log" + echo "$(date -Iseconds) job=${job} vm=${vm_name} backup_id=${backup_id} status=${status}" >> "$reg_file" + mold_backup_registry_set_vm_backup_id "$vm_name" "$backup_id" "$rp_id" "$job" + mold_backup_notify_log info "Saved backup registry: vm=${vm_name} backup_id=${backup_id} status=${status}" +} + +# Record a restore event in the Mold-side registry (separate file per job, suffix .restore.log). +# event: source (veeam|mold), session id, restore point/end time. Used to reflect a restore +# performed directly in the Veeam UI back into Mold's state view. +mold_backup_registry_save_restore() { + local job="$1" vm_name="$2" source="${3:-veeam}" session="${4:-}" detail="${5:-}" status="${6:-restored}" + local reg_dir reg_file + reg_dir="$(mold_backup_registry_dir)" + mkdir -p "$reg_dir" + reg_file="${reg_dir}/$(mold_backup_safe_job_name "$job").restore.log" + echo "$(date -Iseconds) job=${job} vm=${vm_name} source=${source} session=${session} detail=${detail} status=${status}" >> "$reg_file" + mold_backup_notify_log info "Saved restore registry: vm=${vm_name} source=${source} session=${session} status=${status}" +} + +mold_backup_process_vm_pre_notify() { + local vm_name="$1" offering_id="$2" state_file="$3" + local vm_id backup_result backup_id backup_type host_path + + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { + mold_backup_notify_log err "No Mold VM id for ${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} status=fail reason=no-vm-id" + return 1 + } + + mold_backup_api_check_vm_environment "$vm_id" "$vm_name" + + local vm_offering json + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null || true) + vm_offering="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + if [[ -z "$offering_id" && -n "$vm_offering" ]]; then + offering_id="$vm_offering" + mold_backup_notify_log info "Using VM-assigned backup offering id=${offering_id}" + fi + if [[ -n "$offering_id" ]]; then + mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null || true) + vm_offering="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + fi + [[ -n "$vm_offering" ]] || { + mold_backup_notify_log err "VM ${vm_name} has no backup offering (assign '$(mold_backup_offering_name)' / ${VEEAM_PROVIDER_NAME} in Mold UI)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=no-offering" + return 1 + } + + # Loop guard (bidirectional): if this Veeam run was itself triggered by a Mold + # backup, the Mold NAS backup already happened — let Veeam do disk-only and skip + # createBackup to avoid re-triggering Mold. + if mold_backup_trigger_active "mold-active" "$vm_name"; then + mold_backup_trigger_clear "mold-active" "$vm_name" + mold_backup_notify_log info "mold-active marker present for ${vm_name}: Mold already backed up; Veeam disk-only (skip createBackup)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=success reason=mold-triggered" + return 0 + fi + # Mark this VM as Veeam-driven so the Mold→Veeam hook does not start Veeam again. + mold_backup_trigger_mark "veeam-active" "$vm_name" + + local chain_count staging_paths source_format + chain_count="$(mold_backup_api_veeam_backup_count "$vm_id")" + + if [[ "${VEEAM_BACKUP_MODE:-}" == "filelevel" && "${chain_count:-0}" -eq 0 ]]; then + mold_backup_notify_log info "First file-level backup: host export → importAblestackVeeamBackupSeed (skip MS→Veeam on seed)" + if ! host_path=$(mold_backup_run_host_export "$vm_name" "1" 2>/dev/null); then + mold_backup_notify_log err "Host export failed for ${vm_name} (seed bootstrap; check /var/log/mold/veeam-hook.log and cvtbackup output)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=host-export" + return 1 + fi + if [[ ! -d "$host_path" ]]; then + mold_backup_notify_log err "Host export returned invalid path (not a directory): ${host_path}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=host-export" + return 1 + fi + staging_paths="$(mold_backup_collect_host_staging_paths "$host_path" 2>/dev/null || true)" + if [[ -z "$staging_paths" ]]; then + mold_backup_notify_log err "No staging disk files under ${host_path}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=no-staging-disks" + return 1 + fi + source_format="$(mold_backup_detect_staging_source_format "$staging_paths")" + mold_backup_api_validate_offering_repository "$vm_offering" || { + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=offering-repo-mismatch" + return 1 + } + backup_result="$(mold_backup_api_import_seed_and_wait "$vm_id" "$staging_paths" "$source_format" "$vm_name" 2>/dev/null || true)" + if [[ -z "$backup_result" ]]; then + mold_backup_notify_log err "importAblestackVeeamBackupSeed failed for ${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=import-seed" + return 1 + fi + backup_id="${backup_result%%|*}" + backup_type="${backup_result#*|}" + mold_backup_state_write_line "$state_file" \ + "vm=${vm_name} id=${vm_id} backup_id=${backup_id} type=${backup_type} path=${host_path} status=success" + mold_backup_notify_log info "Pre-notify OK vm=${vm_name} backup_id=${backup_id} type=${backup_type} path=${host_path} (seed import)" + return 0 + fi + + backup_result="$(mold_backup_api_create_veeam_and_wait "$vm_id" "$vm_name" || true)" + if [[ -z "$backup_result" ]]; then + mold_backup_notify_log err "Mold API backup request failed for ${vm_name} (see Async job failed above or agent.log)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=fail reason=create-backup" + return 1 + fi + backup_id="${backup_result%%|*}" + backup_type="${backup_result#*|}" + + mold_backup_notify_log info "Host export starting vm=${vm_name} backup_id=${backup_id}" + if ! host_path=$(mold_backup_run_host_export "$vm_name" 2>/dev/null); then + mold_backup_notify_log err "Host export failed for ${vm_name} (backup_id=${backup_id})" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} backup_id=${backup_id} status=fail reason=host-export" + return 1 + fi + if [[ ! -d "$host_path" ]]; then + mold_backup_notify_log err "Host export failed for ${vm_name} (backup_id=${backup_id})" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} backup_id=${backup_id} status=fail reason=host-export" + return 1 + fi + + mold_backup_state_write_line "$state_file" \ + "vm=${vm_name} id=${vm_id} backup_id=${backup_id} type=${backup_type} path=${host_path} status=success" + mold_backup_notify_log info "Pre-notify OK vm=${vm_name} backup_id=${backup_id} type=${backup_type} path=${host_path}" + return 0 +} + +mold_backup_veeam_restore_chain_to_host() { + local backup_id="$1" + local chain rp_ids rp_id script_dir ps1 + chain="$(mold_backup_api_build_restore_chain "$backup_id")" || return 1 + rp_ids="" + local bid + for bid in ${chain//,/ }; do + [[ -z "$bid" ]] && continue + rp_id="$(mold_backup_api_backup_detail_field "$bid" "ablestack.veeam.restore.point.id" 2>/dev/null || true)" + [[ -n "$rp_id" ]] && rp_ids="${rp_ids},${rp_id}" + done + rp_ids="${rp_ids#,}" + [[ -n "$rp_ids" ]] || { + mold_backup_notify_log warn "No Veeam restore point ids in chain; datadisk restore only" + return 0 + } + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "VEEAM_SSH_HOST not set; skip external Veeam chain restore" + return 0 + } + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ps1="${script_dir}/veeam-restore-chain.ps1" + [[ -f "$ps1" ]] || ps1="/usr/share/mold/backup/veeam/veeam-restore-chain.ps1" + [[ -f "$ps1" ]] || { + mold_backup_notify_log warn "veeam-restore-chain.ps1 not found; datadisk restore only (no Veeam chain export)" + return 0 + } + mold_backup_notify_log info "Veeam chain restore RPs=${rp_ids} → ${VEEAM_HOST_BACKUP_PATH}" + ssh -i "${VEEAM_SSH_KEY:-/root/.ssh/id_rsa}" -o StrictHostKeyChecking=no \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "powershell -ExecutionPolicy Bypass -File '${ps1}' -RestorePointIds '${rp_ids}' -DestinationPath '${VEEAM_HOST_BACKUP_PATH}'" \ + || mold_backup_notify_log warn "Veeam chain restore script failed (continuing Mold restore)" +} + +mold_backup_get_domain_disk_paths() { + local vm_name="$1" + VM_NAME="$vm_name" + mold_backup_get_all_disk_paths +} + +mold_backup_run_host_export() { + local vm_name="$1" + local backup_subdir checkpoint disk_paths backup_files parent_dir parent_ckpt parent_ckpt_path backup_type op + [[ -x "${CVT_BACKUP_SCRIPT}" ]] || { + mold_backup_notify_log err "Host backup script not found: ${CVT_BACKUP_SCRIPT} (run veeam/install.sh on this KVM host)" + return 1 + } + + backup_subdir="${VEEAM_HOST_BACKUP_PATH}/${vm_name}/$(date '+%Y.%m.%d.%H.%M.%S.%3N')" + checkpoint="$(basename "$backup_subdir")" + mkdir -p "${VEEAM_HOST_BACKUP_PATH}/${vm_name}" + disk_paths=$(mold_backup_get_domain_disk_paths "$vm_name") + parent_dir="" + parent_ckpt="" + parent_ckpt_path="" + backup_type="FULL" + op="backup-running" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "$backup_type") + + if mold_backup_has_rbd_disk "$disk_paths"; then + op="backup-rbd" + mold_backup_notify_log info "Host export storage engine=rbd (HCI)" + fi + + local latest_parent="${VEEAM_HOST_BACKUP_PATH}/${vm_name}" + local latest_name="" latest_path="" d base force_full="${2:-0}" + for d in "${latest_parent}"/*; do + [[ -d "$d" ]] || continue + base=$(basename "$d") + [[ "$base" == "$checkpoint" ]] && continue + if [[ -f "${d}/checkpoints/${base}.xml" || -f "${d}/checkpoints/${base}.meta" \ + || -f "${d}/rbd-backup.meta" || -f "${d}/veeam-seed.meta" ]]; then + if [[ -z "$latest_name" || "$base" > "$latest_name" ]]; then + latest_name="$base" + latest_path="$d" + fi + fi + done + if [[ "$force_full" == "1" ]]; then + backup_type="FULL" + parent_dir="" + parent_ckpt="" + parent_ckpt_path="" + elif [[ -n "$latest_path" && -f "${latest_path}/checkpoints/${latest_name}.xml" ]]; then + backup_type="INCREMENTAL" + parent_dir="${vm_name}/${latest_name}" + parent_ckpt="$latest_name" + parent_ckpt_path="${latest_path}/checkpoints/${latest_name}.xml" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "INCREMENTAL") + elif [[ -n "$latest_path" && -f "${latest_path}/checkpoints/${latest_name}.meta" ]]; then + backup_type="INCREMENTAL" + parent_dir="${vm_name}/${latest_name}" + parent_ckpt="$latest_name" + parent_ckpt_path="${latest_path}/checkpoints/${latest_name}.meta" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "INCREMENTAL") + elif [[ -n "$latest_path" && -f "${latest_path}/rbd-backup.meta" ]]; then + backup_type="INCREMENTAL" + parent_dir="${vm_name}/${latest_name}" + parent_ckpt="$(mold_backup_meta_field "${latest_path}/rbd-backup.meta" checkpoint_name || echo "$latest_name")" + parent_ckpt_path="${latest_path}/checkpoints/${parent_ckpt}.meta" + [[ -f "$parent_ckpt_path" ]] || parent_ckpt_path="${latest_path}/rbd-backup.meta" + backup_files=$(mold_backup_build_backup_files "$disk_paths" "INCREMENTAL") + elif [[ -n "$latest_path" ]]; then + mold_backup_notify_log warn "Prior export missing checkpoint xml under ${latest_path}; forcing FULL" + backup_type="FULL" + parent_dir="" + parent_ckpt="" + parent_ckpt_path="" + fi + + mold_backup_notify_log info "Host export vm=${vm_name} path=${backup_subdir} type=${backup_type} op=${op}" + local cvt_log + cvt_log="$(mktemp "${TMPDIR:-/tmp}/mold-cvt.XXXXXX")" + if ! "${CVT_BACKUP_SCRIPT}" \ + -o "${op}" \ + -v "${vm_name}" \ + -p "${backup_subdir}" \ + -b "${backup_type}" \ + -c "${checkpoint}" \ + -r "${parent_dir}" \ + -i "${parent_ckpt}" \ + -j "${parent_ckpt_path}" \ + -f "${backup_files}" \ + -d "${disk_paths}" \ + -q "${QUIESCE_VM:-false}" \ + >"$cvt_log" 2>&1; then + mold_backup_notify_log err "Host export cvtbackup failed: $(tail -5 "$cvt_log" | tr '\n' ' ')" + rm -f "$cvt_log" + return 1 + fi + rm -f "$cvt_log" + echo "${backup_subdir}" +} + +mold_backup_cleanup_host_path() { + [[ -d "${VEEAM_HOST_BACKUP_PATH}" ]] || return 0 + mold_backup_notify_log info "Cleaning host backup path: ${VEEAM_HOST_BACKUP_PATH}" + find "${VEEAM_HOST_BACKUP_PATH}" -mindepth 1 -maxdepth 4 -type f -delete 2>/dev/null || true + find "${VEEAM_HOST_BACKUP_PATH}" -mindepth 1 -maxdepth 3 -type d -empty -delete 2>/dev/null || true +} + +# --- Bidirectional Mold<->Veeam trigger loop guard --- +# Two short-lived markers under state/triggers/ break the trigger loop: +# veeam-active- : set by pre_notify before requesting the Mold backup. +# The Mold->Veeam hook skips Start-VBRJob while present. +# mold-active- : set by the Mold->Veeam hook before Start-VBRJob. +# pre_notify skips createBackup while present (Veeam disk-only). +mold_backup_trigger_dir() { + local d="$(mold_backup_state_dir)/triggers" + mkdir -p "$d" 2>/dev/null || true + echo "$d" +} + +mold_backup_trigger_mark() { + local kind="$1" vm="$2" + date +%s > "$(mold_backup_trigger_dir)/${kind}.$(mold_backup_safe_job_name "$vm")" 2>/dev/null || true +} + +mold_backup_trigger_active() { + local kind="$1" vm="$2" ttl="${3:-${VEEAM_TRIGGER_TTL:-1800}}" + local f ts now + f="$(mold_backup_trigger_dir)/${kind}.$(mold_backup_safe_job_name "$vm")" + [[ -f "$f" ]] || return 1 + ts="$(cat "$f" 2>/dev/null || echo 0)" + now="$(date +%s)" + if (( now - ts > ttl )); then + rm -f "$f" 2>/dev/null || true + return 1 + fi + return 0 +} + +mold_backup_trigger_clear() { + rm -f "$(mold_backup_trigger_dir)/${1}.$(mold_backup_safe_job_name "$2")" 2>/dev/null || true +} + +# libvirt VM name -> guest IP, from VM_TARGETS=i-2-5-VM:10.10.254.70,i-2-40-VM:10.10.254.61 +mold_backup_vm_guest_ip() { + local vm="$1" pair name ip targets env_file list + for list in "${VM_TARGETS:-}"; do + [[ -n "$list" ]] || continue + IFS=',' read -ra _pairs <<<"${list}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + if [[ "$name" == "$vm" && -n "$ip" && "$ip" != "$name" ]]; then + echo "$ip" + return 0 + fi + done + done + for env_file in \ + "${ABLESTACK_VEEAM_ETC_DIR}/mold-backup.env" \ + "${MOLD_BACKUP_ETC_DIR}/mold-backup.env" \ + "$(dirname "${BASH_SOURCE[0]}")/mold-backup.env"; do + [[ -f "$env_file" ]] || continue + targets="$(mold_backup_read_env_var VM_TARGETS "$env_file" 2>/dev/null || true)" + [[ -n "$targets" ]] || continue + IFS=',' read -ra _pairs <<<"${targets}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + if [[ "$name" == "$vm" && -n "$ip" && "$ip" != "$name" ]]; then + echo "$ip" + return 0 + fi + done + done + return 1 +} + +# guest IP -> Veeam job name (legacy): 10.10.254.70 -> "Mold VM 10-10-254-70" +mold_backup_veeam_job_name_for_ip() { + local ip="$1" prefix="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + echo "${prefix} ${ip//./-}" +} + +# libvirt VM name -> Veeam job name: i-2-61-VM -> "Mold VM i-2-61-VM" +mold_backup_veeam_job_name_for_vm() { + local vm="$1" prefix="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + echo "${prefix} ${vm}" +} + +# Veeam job name -> libvirt VM name: "Mold VM i-2-61-VM" -> i-2-61-VM +mold_backup_vm_name_for_job() { + local job="$1" prefix="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + [[ "$job" == "${prefix} "* ]] || return 1 + echo "${job#${prefix} }" +} + +# guest IP -> libvirt VM name (reverse of mold_backup_vm_guest_ip), from VM_TARGETS. +mold_backup_vm_name_for_ip() { + local want="$1" pair name ip + [[ -n "${VM_TARGETS:-}" ]] || return 1 + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + name="${pair%%:*}" + ip="${pair#*:}" + if [[ "$ip" == "$want" && -n "$name" && "$ip" != "$name" ]]; then + echo "$name" + return 0 + fi + done + return 1 +} + +# Resolve the Veeam B&R REST API base (https://host:9419) from explicit or SSH host. +mold_backup_veeam_api_base() { + if [[ -n "${VEEAM_API_URL:-}" ]]; then + echo "${VEEAM_API_URL%/}" + return 0 + fi + local host="${VEEAM_API_HOST:-${VEEAM_SSH_HOST:-}}" + [[ -n "$host" ]] || return 1 + echo "https://${host}:${VEEAM_API_PORT:-9419}" +} + +# Start a Veeam job via the native VBR REST API (port 9419) — no SSH required. +# Returns 0 on start (or already running), 1 on any failure (caller may fall back to SSH). +mold_backup_trigger_veeam_job_rest() { + local vm="$1" job="$2" + local api ver user pass token job_id running + api="$(mold_backup_veeam_api_base)" || { + mold_backup_notify_log warn "Mold→Veeam(REST): no VEEAM_API_URL/VEEAM_API_HOST/VEEAM_SSH_HOST" + return 1 + } + ver="${VEEAM_API_VERSION:-1.2-rev0}" + user="${VEEAM_API_USER:-${VEEAM_USERNAME:-administrator}}" + pass="${VEEAM_API_PASSWORD:-${VEEAM_PASSWORD:-}}" + [[ -n "$pass" ]] || { + mold_backup_notify_log warn "Mold→Veeam(REST): VEEAM_API_PASSWORD/VEEAM_PASSWORD not set" + return 1 + } + + token="$(curl -sk --max-time 30 -X POST "${api}/api/oauth2/token" \ + -H "x-api-version: ${ver}" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=password&username=${user}&password=${pass}" 2>/dev/null \ + | python3 -c "import sys,json +try: print(json.load(sys.stdin).get('access_token','')) +except Exception: print('')" 2>/dev/null)" + [[ -n "$token" ]] || { + mold_backup_notify_log warn "Mold→Veeam(REST): auth failed at ${api} (check x-api-version='${ver}', user, port 9419)" + return 1 + } + + # Find the job id + running state by exact name. + local jobs_json + jobs_json="$(curl -sk --max-time 30 "${api}/api/v1/jobs" \ + -H "x-api-version: ${ver}" -H "Authorization: Bearer ${token}" 2>/dev/null)" + read -r job_id running <<<"$(echo "$jobs_json" | python3 -c " +import sys,json +want='''${job}''' +try: + d=json.load(sys.stdin) +except Exception: + print(''); sys.exit() +items=d.get('data', d if isinstance(d,list) else []) +for j in items: + if j.get('name')==want: + st=str(j.get('status') or j.get('lastResult') or '') + print(j.get('id',''), 'running' if str(j.get('isRunning','')).lower()=='true' or st.lower()=='running' else 'idle'); break +" 2>/dev/null)" + [[ -n "$job_id" ]] || { + mold_backup_notify_log warn "Mold→Veeam(REST): job '${job}' not found in /api/v1/jobs (Agent jobs may need SSH); will fall back" + return 1 + } + if [[ "$running" == "running" ]]; then + mold_backup_notify_log info "Mold→Veeam(REST): job '${job}' already running" + return 0 + fi + + local http_code + http_code="$(curl -sk --max-time 30 -o /dev/null -w '%{http_code}' -X POST \ + "${api}/api/v1/jobs/${job_id}/start" \ + -H "x-api-version: ${ver}" -H "Authorization: Bearer ${token}" 2>/dev/null)" + if [[ "$http_code" =~ ^20[0-9]$ ]]; then + mold_backup_notify_log info "Mold→Veeam(REST): job '${job}' start accepted (HTTP ${http_code})" + return 0 + fi + mold_backup_notify_log warn "Mold→Veeam(REST): start failed for '${job}' (HTTP ${http_code})" + return 1 +} + +# Run a short PowerShell script on Veeam via SSH (UTF-16LE EncodedCommand). +mold_backup_veeam_ssh_ps() { + local ps_script="$1" + local ps_enc + [[ -n "${VEEAM_SSH_HOST:-}" ]] || return 1 + ps_enc="$(printf '%s' "$ps_script" | iconv -f UTF-8 -t UTF-16LE 2>/dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || return 1 + mold_backup_veeam_ssh_encoded "$ps_enc" >/dev/null +} + +# Start a Veeam job over SSH (PowerShell) — fallback when REST cannot manage agent jobs. +mold_backup_trigger_veeam_job_ssh() { + local vm="$1" job="$2" job_esc ps_script + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "Mold→Veeam(SSH): VEEAM_SSH_HOST not set" + return 1 + } + job_esc="${job//\'/\'\'}" + ps_script="$(cat <Veeam direction). +# Method: rest (curl, no SSH), ssh (PowerShell), or auto (REST then SSH fallback). +mold_backup_trigger_veeam_job() { + local vm="$1" ip job method rc=1 + [[ "${VEEAM_TRIGGER_ENABLED:-false}" == "true" ]] || return 0 + ip="$(mold_backup_vm_guest_ip "$vm" 2>/dev/null || true)" + [[ -n "$ip" ]] || { + mold_backup_notify_log warn "No guest IP for ${vm} in VM_TARGETS; skip Veeam job start" + return 0 + } + job="$(mold_backup_veeam_job_name_for_vm "$vm")" + if [[ "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + method="${VEEAM_TRIGGER_METHOD:-ssh}" + else + method="${VEEAM_TRIGGER_METHOD:-auto}" + fi + + mold_backup_trigger_mark "mold-active" "$vm" + mold_backup_notify_log info "Mold→Veeam: starting Veeam job '${job}' for ${vm} (${ip}) method=${method}" + + case "$method" in + rest) mold_backup_trigger_veeam_job_rest "$vm" "$job"; rc=$? ;; + ssh) mold_backup_trigger_veeam_job_ssh "$vm" "$job"; rc=$? ;; + auto|*) + mold_backup_trigger_veeam_job_rest "$vm" "$job"; rc=$? + if [[ $rc -ne 0 ]]; then + mold_backup_notify_log info "Mold→Veeam: REST failed/unavailable, trying SSH fallback" + mold_backup_trigger_veeam_job_ssh "$vm" "$job"; rc=$? + fi + ;; + esac + + if [[ $rc -ne 0 ]]; then + mold_backup_trigger_clear "mold-active" "$vm" + mold_backup_notify_log warn "Mold→Veeam: could not start job '${job}' (cleared mold-active for ${vm})" + fi + return $rc +} + +# --- Veeam UI restore -> Mold reflect (reverse restore sync) --- +# A restore performed directly in the Veeam UI restores guest data in-place (the Mold +# VM's disks are updated by the guest agent), so Mold storage is already current. To make +# Mold "aware" of it, the KVM host polls Veeam over the existing KVM->Veeam SSH channel for +# recently-completed restore sessions, maps the target computer IP back to a libvirt VM via +# VM_TARGETS, and records the event in the Mold restore registry (+ Mold hook log). +# Loop guard: a Mold-initiated restore sets the mold-restore-active marker, so the watcher +# skips reflecting Mold's own restore (avoids double-recording). + +# State dir holding the set of already-reflected Veeam restore session ids. +mold_backup_restore_watch_state() { + local d="$(mold_backup_state_dir)/restore-watch" + mkdir -p "$d" 2>/dev/null || true + echo "$d/processed-sessions" +} + +mold_backup_restore_session_seen() { + local sid="$1" f + f="$(mold_backup_restore_watch_state)" + [[ -f "$f" ]] || return 1 + grep -qxF "$sid" "$f" 2>/dev/null +} + +mold_backup_restore_session_mark_seen() { + local sid="$1" f + f="$(mold_backup_restore_watch_state)" + echo "$sid" >> "$f" 2>/dev/null || true + # keep the file bounded + if [[ -f "$f" ]] && (( $(wc -l <"$f" 2>/dev/null || echo 0) > 2000 )); then + tail -n 1000 "$f" > "${f}.tmp" 2>/dev/null && mv -f "${f}.tmp" "$f" 2>/dev/null || true + fi +} + +# Query Veeam for restore sessions that completed within the last N minutes. +# Emits one line per session: sessionId|targetIp|endTimeUTC|result|name|backupName|restorePointId +# The target IP/computer is parsed out of the session Options XML (FLR/restore specs put +# IpOrDnsName/MachineName/BackupName there even when the session Name is hostname-based). +# Result is NOT filtered (e.g. an FLR session can end as 'Failed' even though files were +# restored), only completion + recency. Uses the existing KVM->Veeam SSH channel. +mold_backup_query_veeam_restores() { + local since_min="${1:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}" + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(restore): VEEAM_SSH_HOST not set" + return 1 + } + local ssh_key_opt=() + if [[ -n "${VEEAM_SSH_KEY:-}" ]]; then + if [[ -f "${VEEAM_SSH_KEY}" ]]; then + ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + else + mold_backup_notify_log warn "Veeam→Mold(restore): VEEAM_SSH_KEY=${VEEAM_SSH_KEY} missing; using default SSH keys" + fi + fi + local -a ssh_host_opts=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) + if [[ "${VEEAM_SSH_STRICT_HOSTKEY:-false}" == "true" ]]; then + ssh_host_opts=(-o StrictHostKeyChecking=accept-new) + fi + # PowerShell run on the Veeam B&R server: connect (warm-up loop), then emit every + # completed restore session as "Id|ip|epoch|result|name|backup|rpId". Falls back to + # Export-VBRAudit when Get-VBRRestoreSession is empty (common for Agent FLR over SSH). + local ps_script since_min_ps + since_min_ps="${since_min}" + ps_script="$(cat </dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || { mold_backup_notify_log warn "Veeam→Mold(restore): failed to encode PS script"; return 1; } + # Veeam.Backup.PowerShell needs PS 7+: full-path pwsh first (SSH often only has + # Windows PowerShell 5.1 "powershell", which cannot Import-Module Veeam). + # Also retry SSH itself: ARP/L2 to the Veeam host intermittently flaps. + local attempt out rc ps_launcher + local -a ps_launchers=( + '"C:\Program Files\PowerShell\7\pwsh.exe" -NoProfile -EncodedCommand' + 'pwsh.exe -NoProfile -EncodedCommand' + 'pwsh -NoProfile -EncodedCommand' + ) + for attempt in 1 2 3; do + for ps_launcher in "${ps_launchers[@]}"; do + out="$(ssh "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 "${ssh_host_opts[@]}" \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "${ps_launcher} ${ps_enc}" 2>/dev/null)" + rc=$? + if [[ $rc -eq 0 ]]; then + # PowerShell emits CRLF; strip CR so it doesn't end up in the last field. + out="$(printf '%s' "$out" | tr -d '\r')" + if [[ -z "${out//[[:space:]]/}" ]]; then + mold_backup_notify_log info "Veeam→Mold(restore): SSH ok (pwsh), 0 sessions from Get-VBRRestoreSession/audit (window=${since_min}min)" + else + local _raw_n + _raw_n="$(printf '%s\n' "$out" | grep -c '|' 2>/dev/null || echo 0)" + mold_backup_notify_log info "Veeam→Mold(restore): raw session lines=${_raw_n}" + fi + # Apply the time window here (3rd field = Unix epoch seconds). Lines without a + # numeric epoch are kept (fail-open) so we never silently drop real sessions. + local now_epoch cutoff line ep + now_epoch=$(date +%s) + cutoff=$(( now_epoch - since_min * 60 )) + while IFS= read -r line; do + [[ -n "$line" ]] || continue + ep="$(printf '%s' "$line" | cut -d'|' -f3)" + if [[ "$ep" =~ ^[0-9]+$ ]]; then + [[ "$ep" -ge "$cutoff" ]] && printf '%s\n' "$line" + else + printf '%s\n' "$line" + fi + done <<< "$out" + return 0 + fi + done + mold_backup_notify_log warn "Veeam→Mold(restore): SSH/pwsh query attempt ${attempt} failed (rc=${rc}); retrying" + sleep 3 + done + mold_backup_notify_log warn "Veeam→Mold(restore): SSH query failed after retries (need pwsh 7+ path)" + return 1 +} + +# Host Agent FLR restores files back under /tmp/mold/veeam// — detect locally when Veeam SSH returns nothing. +# Emits: sessionId|ip|epoch|result|name|backupName|restorePointId +mold_backup_restore_preflight() { + local vm owner bid d + mold_backup_notify_log info "restore preflight: host=$(mold_backup_local_kvm_name) job=${VEEAM_JOB_NAME:-n/a}" + [[ -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]] || { + mold_backup_notify_log warn "restore preflight: VM_INCLUDE not set" + return 0 + } + for vm in ${VM_INCLUDE//,/ }; do + vm="$(echo "$vm" | xargs)" + [[ -n "$vm" ]] || continue + if mold_backup_domain_exists "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore preflight: ${vm} libvirt=running (stop VM in Mold UI before restore)" + elif mold_backup_vm_restorable_on_local_host "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore preflight: ${vm} Mold Stopped on $(mold_backup_local_kvm_name) — no libvirt domain (normal for Mold; OK to restore)" + else + owner="$(mold_backup_api_get_vm_host_name "$vm" 2>/dev/null || echo unknown)" + mold_backup_notify_log warn "restore preflight: ${vm} not on this host (Mold hostname=${owner}); run restore on owner KVM" + fi + d="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}/${vm}" + if [[ -d "$d" ]] && [[ -n "$(find "$d" -mindepth 1 -print -quit 2>/dev/null)" ]]; then + mold_backup_notify_log info "restore preflight: ${vm} staging=${d} has files (FLR or backup staging)" + else + mold_backup_notify_log info "restore preflight: ${vm} staging=${d} empty — Veeam FLR must complete within restore-watch --since-min window" + fi + bid="$(mold_backup_registry_get_vm_backup_id "$vm" 2>/dev/null || true)" + mold_backup_notify_log info "restore preflight: ${vm} registry backup_id=${bid:-none}" + done +} + +mold_backup_query_local_host_flr() { + local since_min="${1:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}" + [[ "${BACKUP_MODE:-host}" == "host" ]] || return 0 + [[ -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]] || return 0 + local base="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + local cutoff now vm d epoch newest last state_f sid kvm_ip ckpt rp_id + now=$(date +%s) + cutoff=$((now - since_min * 60)) + kvm_ip="${KVM_IP:-}" + [[ -z "$kvm_ip" && -n "${KVM_HOST:-}" && "$KVM_HOST" == *@* ]] && kvm_ip="${KVM_HOST#*@}" + for vm in ${VM_INCLUDE//,/ }; do + vm="$(echo "$vm" | xargs)" + [[ -n "$vm" ]] || continue + d="${base}/${vm}" + if [[ ! -d "$d" ]]; then + mold_backup_notify_log info "local FLR: ${vm} no staging dir ${d}" + continue + fi + mold_backup_trigger_active "veeam-active" "$vm" && continue + mold_backup_trigger_active "mold-restore-active" "$vm" && continue + newest="$(find "$d" -mindepth 1 \( -type f -o -type d \) -printf '%T@\n' 2>/dev/null | sort -rn | head -1 || true)" + if [[ -z "$newest" ]]; then + mold_backup_notify_log info "local FLR: ${vm} staging empty ${d}" + continue + fi + epoch="${newest%.*}" + [[ "$epoch" =~ ^[0-9]+$ ]] || continue + if [[ "$epoch" -lt "$cutoff" ]]; then + mold_backup_notify_log info "local FLR: ${vm} files older than ${since_min}min (mtime epoch=${epoch}; widen --since-min or re-FLR)" + continue + fi + state_f="$(mold_backup_state_dir)/restore-watch/flr-${vm}.last-epoch" + mkdir -p "$(dirname "$state_f")" 2>/dev/null || true + last=0 + [[ -f "$state_f" ]] && last="$(tr -d '[:space:]' <"$state_f" 2>/dev/null || echo 0)" + [[ "$epoch" -le "${last:-0}" ]] && continue + ckpt="$(find "$d" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -rn | head -1 || true)" + rp_id="" + if [[ -n "$ckpt" ]]; then + local reg_line + reg_line="$(grep -h "vm=${vm}.*rp=" "$(mold_backup_registry_dir)"/*.log 2>/dev/null | tail -1 || true)" + rp_id="$(sed -n 's/.*rp=\([^ ]*\).*/\1/p' <<<"$reg_line" | tail -1)" + [[ -z "$rp_id" ]] && rp_id="$(tr -d '[:space:]' <"$(mold_backup_registry_dir)/${vm}.latest-rp-id" 2>/dev/null || true)" + fi + [[ -z "$rp_id" ]] && rp_id="$(mold_backup_query_veeam_rp_near_epoch "${VEEAM_JOB_NAME:-}" "$epoch" 2>/dev/null || true)" + sid="local-flr-${vm}-${epoch}" + mold_backup_notify_log info "local FLR detect: vm=${vm} epoch=${epoch} ckpt=${ckpt:-n/a} rp=${rp_id:-n/a}" + printf '%s|%s|%s|Success|local-flr-%s|%s|%s\n' "$sid" "${kvm_ip:-}" "$epoch" "$vm" "${ckpt:-}" "${rp_id:-}" + done +} + +mold_backup_local_flr_mark_epoch() { + local vm="$1" epoch="$2" state_f + [[ -n "$vm" && -n "$epoch" ]] || return 0 + state_f="$(mold_backup_state_dir)/restore-watch/flr-${vm}.last-epoch" + mkdir -p "$(dirname "$state_f")" 2>/dev/null || true + echo "$epoch" >"$state_f" +} + +# Pick Veeam restore point GUID whose CreationTime is closest to a Unix epoch (FLR time). +mold_backup_query_veeam_rp_near_epoch() { + local job="$1" epoch="$2" + [[ -n "${VEEAM_SSH_HOST:-}" && -n "$job" && "$epoch" =~ ^[0-9]+$ ]] || return 1 + local job_esc epoch_ps ps_script ps_enc out + job_esc="${job//\'/\'\'}" + epoch_ps="${epoch}" + ps_script="$(cat </dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || return 1 + local ssh_key_opt=() + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + out="$(ssh "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 -o StrictHostKeyChecking=no \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "pwsh -NoProfile -EncodedCommand ${ps_enc}" 2>/dev/null | tr -d '\r' | head -1)" + [[ -n "$out" ]] && echo "$out" +} + +# Reflect a single Veeam restore session into Mold state for one VM. +mold_backup_reflect_one_restore() { + local job="$1" vm="$2" sid="$3" detail="$4" + # Loop guard: Mold itself initiated this restore -> Veeam restore is part of that flow. + if mold_backup_trigger_active "mold-restore-active" "$vm"; then + mold_backup_trigger_clear "mold-restore-active" "$vm" + mold_backup_notify_log info "mold-restore-active for ${vm}: restore initiated by Mold; skip reflect (session=${sid})" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "mold.restore.skipped.mold-active" "$vm" "session=${sid}" + return 0 + fi + mold_backup_registry_save_restore "$job" "$vm" "veeam" "$sid" "$detail" "veeam-restored" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "veeam.restore.reflected" "$vm" "session=${sid};${detail}" + mold_backup_notify_log info "Veeam→Mold: reflected restore for ${vm} (session=${sid})" +} + +# --- Restore agent: host ownership, cluster lock, Mold API trigger --- + +mold_backup_local_kvm_name() { + if [[ -n "${KVM_HOSTNAME:-}" ]]; then + echo "$KVM_HOSTNAME" + return 0 + fi + local agent_props="/etc/cloudstack/agent/agent.properties" h + if [[ -f "$agent_props" ]]; then + h="$(grep -E '^host\.name=' "$agent_props" 2>/dev/null | tail -1 | cut -d= -f2-)" + h="${h//$'\r'/}" + [[ -n "$h" ]] && { echo "$h"; return 0; } + h="$(grep -E '^resource=' "$agent_props" 2>/dev/null | tail -1 | cut -d= -f2-)" + h="${h//$'\r'/}" + [[ -n "$h" ]] && { echo "$h"; return 0; } + h="$(grep -E '^host=' "$agent_props" 2>/dev/null | tail -1 | cut -d= -f2-)" + h="${h//$'\r'/}" + # host= is often MS resource id (e.g. 10.10.31.20@static), not hypervisor name — skip @ forms + if [[ -n "$h" && "$h" != *@* ]]; then + echo "$h" + return 0 + fi + fi + hostname -s +} + +mold_backup_api_get_vm_host_name() { + local vm_name="$1" json host + json="$(mold_backup_api_get_vm_record "$vm_name" "${ZONE_ID:-}")" || return 1 + host="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.hostname")" + [[ -n "$host" ]] || return 1 + echo "$host" +} + +mold_backup_vm_owned_by_local_host() { + local vm_name="$1" vm_host local_host + vm_host="$(mold_backup_api_get_vm_host_name "$vm_name" 2>/dev/null || true)" + local_host="$(mold_backup_local_kvm_name)" + if [[ -z "$vm_host" ]]; then + mold_backup_notify_log warn "restore-agent: no Mold hostname for ${vm_name}; allow local=${local_host}" + return 0 + fi + [[ "$vm_host" == "$local_host" ]] +} + +mold_backup_restore_lock_dir() { + local d="${RESTORE_LOCK_DIR:-}" + if [[ -z "$d" && -n "${BACKUP_REPO_ADDRESS:-}" ]]; then + d="${BACKUP_REPO_ADDRESS%/}/.mold/restore-locks" + fi + if [[ -z "$d" ]]; then + d="$(mold_backup_state_dir)/restore-locks" + fi + mkdir -p "$d" 2>/dev/null || true + echo "$d" +} + +mold_backup_restore_lock_acquire() { + local vm="$1" lock_file + lock_file="$(mold_backup_restore_lock_dir)/$(mold_backup_safe_job_name "$vm").lock" + exec {MOLD_RESTORE_LOCK_FD}>"$lock_file" || return 1 + if ! flock -n "$MOLD_RESTORE_LOCK_FD"; then + exec {MOLD_RESTORE_LOCK_FD}>&- + unset MOLD_RESTORE_LOCK_FD + return 1 + fi + return 0 +} + +mold_backup_restore_lock_release() { + [[ -n "${MOLD_RESTORE_LOCK_FD:-}" ]] || return 0 + flock -u "$MOLD_RESTORE_LOCK_FD" 2>/dev/null || true + exec {MOLD_RESTORE_LOCK_FD}>&- + unset MOLD_RESTORE_LOCK_FD +} + +mold_backup_events_log_file() { + local d="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}/events" + mkdir -p "$d" 2>/dev/null || true + echo "$d/restore.log" +} + +mold_backup_emit_restore_event() { + local event="$1" vm="$2" detail="${3:-}" host + host="$(mold_backup_local_kvm_name)" + echo "$(date -Iseconds) event=${event} host=${host} vm=${vm} ${detail}" >> "$(mold_backup_events_log_file)" + mold_backup_notify_log info "restore-event ${event} vm=${vm} ${detail}" +} + +mold_backup_api_find_backup_by_veeam_rp() { + local vm_name="$1" rp_id="$2" + local vm_id json norm_rp bid detail_rp + [[ -n "$vm_name" && -n "$rp_id" ]] || return 1 + norm_rp="$(mold_backup_normalize_rp_id "$rp_id")" + [[ -n "$norm_rp" ]] || return 1 + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || return 1 + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + [[ -n "$json" ]] || return 1 + while IFS= read -r bid; do + [[ -n "$bid" ]] || continue + detail_rp="$(mold_backup_api_backup_detail_field "$bid" "ablestack.veeam.restore.point.id" 2>/dev/null || true)" + [[ -n "$detail_rp" ]] || continue + if [[ "$(mold_backup_normalize_rp_id "$detail_rp")" == "$norm_rp" ]]; then + echo "$bid" + return 0 + fi + done < <(printf '%s\n' "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): + b = [b] + for x in b: + if str(x.get('status', '')).lower() == 'backedup' and x.get('id'): + print(x['id']) +except Exception: + pass +" 2>/dev/null) + return 1 +} + +mold_backup_resolve_backup_id_for_vm() { + local vm_name="$1" job="${2:-${VEEAM_JOB_NAME:-}}" rp_id="${3:-}" ckpt="${4:-}" + local line backup_id vm_id json + if [[ -n "$ckpt" ]]; then + backup_id="$(mold_backup_registry_get_backup_id_by_checkpoint "$vm_name" "$ckpt" 2>/dev/null || true)" + if [[ -n "$backup_id" ]]; then + mold_backup_notify_log info "restore-watch: vm=${vm_name} ckpt=${ckpt} → backup_id=${backup_id} (registry checkpoint)" + echo "$backup_id" + return 0 + fi + fi + if [[ -n "$rp_id" ]]; then + backup_id="$(mold_backup_registry_get_backup_id_by_rp "$vm_name" "$rp_id" 2>/dev/null || true)" + if [[ -n "$backup_id" ]]; then + mold_backup_notify_log info "restore-watch: vm=${vm_name} rp=${rp_id} → backup_id=${backup_id} (registry)" + echo "$backup_id" + return 0 + fi + backup_id="$(mold_backup_api_find_backup_by_veeam_rp "$vm_name" "$rp_id" 2>/dev/null || true)" + if [[ -n "$backup_id" ]]; then + mold_backup_notify_log info "restore-watch: vm=${vm_name} rp=${rp_id} → backup_id=${backup_id} (Mold API)" + mold_backup_registry_index_rp_backup "$vm_name" "$rp_id" "$backup_id" "$job" + echo "$backup_id" + return 0 + fi + mold_backup_notify_log warn "restore-watch: no Mold backup for Veeam restore point ${rp_id} vm=${vm_name}; using latest" + fi + if [[ -n "${BACKUP_ID:-}" ]]; then + echo "$BACKUP_ID" + return 0 + fi + backup_id="$(mold_backup_registry_get_vm_backup_id "$vm_name" 2>/dev/null || true)" + [[ -n "$backup_id" ]] && { echo "$backup_id"; return 0; } + local reg_dir + reg_dir="$(mold_backup_registry_dir)" + if [[ -d "$reg_dir" ]]; then + line="$(grep -h "vm=${vm_name}.*backup_id=" "${reg_dir}"/*.log 2>/dev/null | tail -1 || true)" + backup_id="$(sed -n 's/.*backup_id=\([^ ]*\).*/\1/p' <<<"$line" | tail -1)" + [[ -n "$backup_id" ]] && { echo "$backup_id"; return 0; } + fi + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || return 1 + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + backup_id="$(python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): b = [b] + backed = [x for x in b if str(x.get('status','')).lower() == 'backedup'] + backed.sort(key=lambda x: x.get('date',''), reverse=True) + print(backed[0]['id'] if backed else '') +except Exception: + print('') +" <<<"$json" 2>/dev/null)" + [[ -n "$backup_id" ]] && echo "$backup_id" +} + +# Handle one Veeam restore session: dedup, host check, flock, optional Mold restore API. +mold_backup_handle_veeam_restore_session() { + local job="$1" vm="$2" sid="$3" detail="$4" trigger_mold="${5:-false}" rp_id="${6:-}" + if [[ -z "$rp_id" && "$detail" == *"rp="* ]]; then + rp_id="$(sed -n 's/.*rp=\([^;]*\).*/\1/p' <<<"$detail" | tail -1)" + rp_id="${rp_id// /}" + fi + if [[ -z "$rp_id" && "$detail" == *"end="* ]]; then + local _ep + _ep="$(sed -n 's/.*end=\([^;]*\).*/\1/p' <<<"$detail" | tail -1)" + if [[ "$_ep" =~ ^[0-9]+$ ]]; then + rp_id="$(mold_backup_query_veeam_rp_near_epoch "$job" "$_ep" 2>/dev/null || true)" + fi + fi + if mold_backup_restore_session_seen "$sid"; then + mold_backup_emit_restore_event "mold.restore.skipped.duplicate" "$vm" "session=${sid}" + return 0 + fi + if mold_backup_trigger_active "mold-restore-active" "$vm"; then + mold_backup_trigger_clear "mold-restore-active" "$vm" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "mold.restore.skipped.mold-active" "$vm" "session=${sid}" + return 0 + fi + if [[ "$trigger_mold" != "true" ]]; then + mold_backup_reflect_one_restore "$job" "$vm" "$sid" "$detail" + return 0 + fi + if ! mold_backup_vm_owned_by_local_host "$vm"; then + local owner + owner="$(mold_backup_api_get_vm_host_name "$vm" 2>/dev/null || echo unknown)" + mold_backup_emit_restore_event "mold.restore.skipped.not-owner" "$vm" \ + "session=${sid};owner=${owner};local=$(mold_backup_local_kvm_name)" + mold_backup_restore_session_mark_seen "$sid" + return 0 + fi + if ! mold_backup_restore_lock_acquire "$vm"; then + mold_backup_emit_restore_event "mold.restore.skipped.locked" "$vm" "session=${sid}" + return 0 + fi + local backup_id rc=0 ckpt="" + ckpt="$(sed -n 's/.*backup=\([^;]*\).*/\1/p' <<<"$detail" | tail -1)" + ckpt="${ckpt// /}" + backup_id="$(mold_backup_resolve_backup_id_for_vm "$vm" "$job" "$rp_id" "$ckpt" 2>/dev/null || true)" + if [[ -z "$backup_id" ]]; then + mold_backup_emit_restore_event "mold.restore.failed" "$vm" "session=${sid};reason=no-backup-id;rp=${rp_id:-n/a}" + mold_backup_restore_lock_release + return 1 + fi + export BACKUP_ID="$backup_id" VM_NAME="$vm" + [[ -n "$rp_id" ]] && export VEEAM_RESTORE_POINT_ID="$rp_id" + export RESTORE_SOURCE="${RESTORE_SOURCE:-${VEEAM_UI_RESTORE_SOURCE:-mold-only}}" + mold_backup_notify_log info "Veeam UI restore session=${sid} vm=${vm} rp=${rp_id:-n/a} → Mold datadisk restore backup_id=${backup_id} (RESTORE_SOURCE=${RESTORE_SOURCE})" + mold_backup_emit_restore_event "veeam.restore.completed" "$vm" "session=${sid};rp=${rp_id:-n/a};backup_id=${backup_id};${detail}" + mold_backup_emit_restore_event "mold.restore.requested" "$vm" "session=${sid};rp=${rp_id:-n/a};backup_id=${backup_id};source=${RESTORE_SOURCE}" + mold_backup_trigger_mark "mold-restore-active" "$vm" + if mold_backup_restore_notify "$(hostname -s)" "$job"; then + mold_backup_registry_save_restore "$job" "$vm" "veeam" "$sid" "${detail};backup_id=${backup_id}" "mold-restored" + mold_backup_restore_session_mark_seen "$sid" + mold_backup_emit_restore_event "mold.restore.completed" "$vm" "session=${sid};backup_id=${backup_id}" + else + mold_backup_emit_restore_event "mold.restore.failed" "$vm" "session=${sid};backup_id=${backup_id}" + rc=1 + fi + mold_backup_trigger_clear "mold-restore-active" "$vm" + mold_backup_restore_lock_release + return "$rc" +} + +# Poll Veeam restore sessions and reflect new ones for VMs we manage (VM_TARGETS). +# When trigger_mold=true, the owning KVM host acquires a cluster flock and calls Mold restore API. +mold_backup_watch_veeam_restores() { + local job="${1:-${VEEAM_JOB_NAME:-}}" + local since_min="${2:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}" + local trigger_mold="${3:-${RESTORE_WATCH_TRIGGER_MOLD:-false}}" + if [[ -z "${VM_TARGETS:-}" && "${BACKUP_MODE:-host}" == "host" && -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]]; then + # Host mode: build name:ip pairs from libvirt when VM_TARGETS unset. + local _vm _ip + VM_TARGETS="" + for _vm in ${VM_INCLUDE//,/ }; do + _vm="$(echo "$_vm" | xargs)" + [[ -n "$_vm" ]] || continue + _ip="$(virsh -c qemu:///system domifaddr "$_vm" 2>/dev/null | awk '/ipv4/ {print $4; exit}' | cut -d/ -f1)" + VM_TARGETS+="${VM_TARGETS:+,}${_vm}:${_ip:-${_vm}}" + done + fi + [[ -n "${VM_TARGETS:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(restore): VM_TARGETS empty; set VM_INCLUDE or VM_TARGETS" + return 0 + } + [[ "${trigger_mold}" == "true" ]] && mold_backup_restore_preflight + mold_backup_notify_log info "=== restore-watch job=${job} window=${since_min}min trigger_mold=${trigger_mold} host=$(mold_backup_local_kvm_name) ===" + local sid sip et result nm bn rp_id matched_ip vm + local processed=0 handled=0 + while IFS='|' read -r sid sip et result nm bn rp_id; do + [[ -n "$sid" ]] || continue + processed=$((processed+1)) + if mold_backup_restore_session_seen "$sid"; then + mold_backup_notify_log info "restore-watch: session ${sid} already processed (skip duplicate)" + continue + fi + # Match: prefer the IP parsed from the session Options; fall back to scanning + # VM_TARGETS IPs (dots or dashes form) against the session name / backup name. + matched_ip="" + if [[ -n "$sip" ]] && mold_backup_vm_name_for_ip "$sip" >/dev/null 2>&1; then + matched_ip="$sip" + else + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + local pair ip ipd vmn + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + ip="${pair#*:}" + [[ -n "$ip" && "$ip" != "$vmn" ]] || continue + ipd="${ip//./-}" + # Match by IP (dots/dashes) for legacy "Mold VM " jobs, or by the + # libvirt internal name (e.g. i-2-51-VM) for jobs named by internal name. + if [[ "$sip" == "$ip" || "$nm" == *"$ip"* || "$nm" == *"$ipd"* || "$bn" == *"$ip"* || "$bn" == *"$ipd"* \ + || ( -n "$vmn" && ( "$nm" == *"$vmn"* || "$bn" == *"$vmn"* ) ) ]]; then + matched_ip="$ip" + break + fi + done + fi + # Host backup (KVM agent e.g. 10.10.31.2): FLR session targets the hypervisor. + if [[ -z "$matched_ip" && "${BACKUP_MODE:-host}" == "host" ]]; then + local kvm_ip="${KVM_IP:-}" + [[ -z "$kvm_ip" && -n "${KVM_HOST:-}" && "$KVM_HOST" == *@* ]] && kvm_ip="${KVM_HOST#*@}" + [[ -z "$kvm_ip" && -n "${KVM_HOST:-}" && "$KVM_HOST" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] && kvm_ip="${KVM_HOST}" + [[ -z "$kvm_ip" ]] && kvm_ip="$(hostname -I 2>/dev/null | awk '{print $1}')" + local kvm_hn="${KVM_HOSTNAME:-$(hostname -s)}" + local host_hit=false + if [[ -n "$kvm_ip" && ( "$sip" == "$kvm_ip" || "$nm" == *"$kvm_ip"* || "$bn" == *"$kvm_ip"* ) ]]; then + host_hit=true + elif [[ -n "$kvm_hn" && ( "$nm" == *"$kvm_hn"* || "$bn" == *"$kvm_hn"* || "$bn" == *"$job"* ) ]]; then + host_hit=true + fi + if [[ "$host_hit" == "true" ]]; then + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + local pair vmn ip + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + [[ -n "$vmn" ]] || continue + if [[ "$nm" == *"$vmn"* || "$bn" == *"$vmn"* ]]; then + ip="${pair#*:}" + matched_ip="${ip:-$vmn}" + break + fi + done + if [[ -z "$matched_ip" && -n "${VM_INCLUDE:-}" && "${VM_INCLUDE}" != "*" ]]; then + local _one + if [[ -n "${VEEAM_RESTORE_VM:-}" ]]; then + _one="${VEEAM_RESTORE_VM}" + mold_backup_notify_log info "restore-watch: host FLR session ${sid} → VEEAM_RESTORE_VM=${_one}" + else + _one="$(echo "${VM_INCLUDE}" | tr ',' ' ' | awk '{print $1}')" + mold_backup_notify_log info "restore-watch: host FLR session ${sid} → VM_INCLUDE=${_one} (set VEEAM_RESTORE_VM for explicit target)" + fi + if [[ -n "$_one" ]]; then + matched_ip="$_one" + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + ip="${pair#*:}" + [[ "$vmn" == "$_one" && -n "$ip" && "$ip" != "$vmn" ]] && matched_ip="$ip" && break + done + fi + fi + fi + fi + # Agent FLR to hypervisor: audit/session metadata often lacks libvirt VM name — use explicit target. + if [[ -z "$matched_ip" && "${BACKUP_MODE:-host}" == "host" && -n "${VEEAM_RESTORE_VM:-}" ]]; then + local _pair _vmn _ip + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + for _pair in "${_pairs[@]}"; do + _pair="${_pair// /}" + _vmn="${_pair%%:*}" + _ip="${_pair#*:}" + [[ "$_vmn" == "${VEEAM_RESTORE_VM}" ]] || continue + matched_ip="${_ip:-$_vmn}" + [[ "$matched_ip" == "$_vmn" ]] && matched_ip="$_vmn" + break + done + [[ -z "$matched_ip" ]] && matched_ip="${VEEAM_RESTORE_VM}" + mold_backup_notify_log info "restore-watch: session ${sid} name='${nm}' → host FLR fallback VEEAM_RESTORE_VM=${VEEAM_RESTORE_VM}" + fi + if [[ -z "$matched_ip" ]]; then + mold_backup_notify_log info "restore-watch: session ${sid} ip='${sip}' name='${nm}' backup='${bn}' rp='${rp_id:-}' — no VM_TARGETS match (skip)" + continue + fi + vm="$(mold_backup_vm_name_for_ip "$matched_ip" 2>/dev/null || true)" + [[ -n "$vm" ]] || vm="$matched_ip" + if ! mold_backup_vm_restorable_on_local_host "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore-watch: session ${sid} vm='${vm}' — not on this Mold host (virsh empty when Stopped is normal)" + continue + fi + if ! mold_backup_domain_exists "$vm" 2>/dev/null; then + mold_backup_notify_log info "restore-watch: vm=${vm} Mold Stopped (no libvirt) — triggering Mold restoreBackup via API" + fi + [[ -n "$rp_id" ]] && mold_backup_notify_log info "restore-watch: session ${sid} vm=${vm} veeam_rp=${rp_id}" + mold_backup_handle_veeam_restore_session "$job" "$vm" "$sid" \ + "name=${nm};end=${et};result=${result};backup=${bn};ip=${matched_ip};rp=${rp_id}" "$trigger_mold" "$rp_id" \ + && { + handled=$((handled+1)) + [[ "$sid" == local-flr-* ]] && mold_backup_local_flr_mark_epoch "$vm" "$et" + } || mold_backup_notify_log warn "restore-watch: session ${sid} vm=${vm} handle failed (see restore.log)" + done < <( + mold_backup_query_veeam_restores "$since_min" 2>/dev/null || true + mold_backup_query_local_host_flr "$since_min" 2>/dev/null || true + ) + mold_backup_notify_log info "=== restore-watch done: scanned=${processed} handled=${handled} trigger_mold=${trigger_mold} ===" + return 0 +} + +# === Veeam UI backup reflection (backup-watch) ============================= +# Mirror of restore-watch for the *backup* direction: poll Veeam backup +# sessions and record new ones into the Mold-side registry, so a backup that +# was started directly from the Veeam console shows up in Mold without any +# per-job pre/post script. Dedup is by Veeam session id (separate state file). + +mold_backup_backup_watch_state() { + local d="$(mold_backup_state_dir)/backup-watch" + mkdir -p "$d" 2>/dev/null || true + echo "$d/processed-sessions" +} + +mold_backup_backup_session_seen() { + local sid="$1" f + f="$(mold_backup_backup_watch_state)" + [[ -f "$f" ]] || return 1 + grep -qxF "$sid" "$f" 2>/dev/null +} + +mold_backup_backup_session_mark_seen() { + local sid="$1" f + f="$(mold_backup_backup_watch_state)" + echo "$sid" >> "$f" 2>/dev/null || true + # keep the file bounded + if [[ -f "$f" ]] && (( $(wc -l <"$f" 2>/dev/null || echo 0) > 2000 )); then + tail -n 1000 "$f" > "${f}.tmp" 2>/dev/null && mv -f "${f}.tmp" "$f" 2>/dev/null || true + fi +} + +# Run PowerShell on Veeam B&R via SSH (UTF-16LE base64 -EncodedCommand). +# Tries pwsh full path, then pwsh.exe, then powershell.exe (Windows OpenSSH PATH quirks). +mold_backup_veeam_ssh_encoded() { + local ps_enc="$1" + local attempt out rc last_err="" ps_launcher + [[ -n "${VEEAM_SSH_HOST:-}" ]] || return 1 + [[ -n "$ps_enc" ]] || return 1 + local ssh_key_opt=() + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + local -a ssh_host_opts=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) + if [[ "${VEEAM_SSH_STRICT_HOSTKEY:-false}" == "true" ]]; then + ssh_host_opts=(-o StrictHostKeyChecking=accept-new) + fi + local -a ps_launchers=( + '"C:\Program Files\PowerShell\7\pwsh.exe" -NoProfile -EncodedCommand' + 'pwsh.exe -NoProfile -EncodedCommand' + 'pwsh -NoProfile -EncodedCommand' + 'powershell.exe -NoProfile -EncodedCommand' + ) + for attempt in 1 2 3; do + for ps_launcher in "${ps_launchers[@]}"; do + out="$(ssh -n "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 "${ssh_host_opts[@]}" \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "${ps_launcher} ${ps_enc}" 2>&1)" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + printf '%s' "$out" + return 0 + fi + last_err="$out" + done + sleep 2 + done + last_err="${last_err//$'\r'/}" + last_err="${last_err//$'\n'/; }" + mold_backup_notify_log warn "Veeam SSH failed (host=${VEEAM_SSH_HOST} rc=${rc}): ${last_err:0:240}" + return 1 +} + +# Query Veeam B&R for the newest restore point GUID for a guest Agent job. +# Guest Agent backups register under computer IP/hostname in Veeam, not libvirt i-2-XX-VM. +# Tries: computer backup job → backup chain → restore points, then name/IP filters. +# Prints restore point GUID on stdout; returns 1 if none found. +mold_backup_query_veeam_latest_restore_point() { + local job="$1" vm_name="$2" guest_ip="${3:-}" retries="${4:-6}" attempt rp_id + for ((attempt=1; attempt<=retries; attempt++)); do + rp_id="$(mold_backup_query_veeam_latest_restore_point_once "$job" "$vm_name" "$guest_ip" 2>/dev/null || true)" + [[ -n "$rp_id" ]] && { echo "$rp_id"; return 0; } + if [[ "$attempt" -lt "$retries" ]]; then + mold_backup_notify_log info "guest post: restore point not ready (attempt ${attempt}/${retries}); retry in 10s job=${job} vm=${vm_name}" + sleep 10 + fi + done + mold_backup_notify_log warn "guest post: no Veeam restore point after ${retries} attempts job=${job} vm=${vm_name}" + return 1 +} + +mold_backup_query_veeam_latest_restore_point_once() { + local job="$1" vm_name="$2" guest_ip="${3:-}" + local job_esc vm_esc ip_esc dash_ip ps_script ps_enc out + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "guest post: VEEAM_SSH_HOST not set; cannot query restore points" + return 1 + } + job_esc="${job//\'/\'\'}" + vm_esc="${vm_name//\'/\'\'}" + ip_esc="${guest_ip//\'/\'\'}" + dash_ip="${guest_ip//./-}" + ps_script="$(cat </dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || return 1 + local out + out="$(mold_backup_veeam_ssh_encoded "$ps_enc" 2>/dev/null || true)" + [[ -n "$out" ]] || return 1 + out="${out//$'\r'/}" + out="$(printf '%s' "$out" | grep -E '^[0-9a-fA-F-]{36}$' | tail -1)" + [[ -n "$out" ]] && { echo "$out"; return 0; } + mold_backup_notify_log warn "Veeam restore point query: SSH ok but no GUID (job=${job})" + return 1 +} + +# Query Veeam for backup sessions that completed within the last N minutes. +# Emits one line per session: sessionId|targetIp|endEpoch|result|name|jobName +# The target IP is parsed from the session/job name (guest VM jobs are named +# "Mold VM 10-10-254-70" — dash form — so the dash IP is recovered to dots). +# Time-window filtering is applied afterwards in bash on the epoch field, same +# as restore-watch (avoids PowerShell/Veeam timezone quirks). +mold_backup_query_veeam_backups() { + local since_min="${1:-${VEEAM_BACKUP_WATCH_WINDOW_MIN:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}}" + [[ -n "${VEEAM_SSH_HOST:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(backup): VEEAM_SSH_HOST not set" + return 1 + } + local ssh_key_opt=() + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_key_opt=(-i "${VEEAM_SSH_KEY}") + local -a ssh_host_opts=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null) + if [[ "${VEEAM_SSH_STRICT_HOSTKEY:-false}" == "true" ]]; then + ssh_host_opts=(-o StrictHostKeyChecking=accept-new) + fi + local ps_script + ps_script="$(cat <<'PS' +$ErrorActionPreference = 'SilentlyContinue' +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue +# Warm-up loop (same quirk as restore sessions). Union regular backup sessions +# with Agent (computer) backup job sessions so guest-VM Agent jobs are included. +$sessions = @() +for ($k = 0; $k -lt 12; $k++) { + try { Connect-VBRServer -Server localhost -ErrorAction Stop } catch {} + $tmp = @() + try { $tmp += @(Get-VBRBackupSession) } catch {} + try { $tmp += @(Get-VBRComputerBackupJobSession) } catch {} + $sessions = @($tmp) + if ($sessions.Count -gt 0) { break } + Start-Sleep -Milliseconds 700 +} +$tmp = @() +try { $tmp += @(Get-VBRBackupSession) } catch {} +try { $tmp += @(Get-VBRComputerBackupJobSession) } catch {} +$sessions = @($tmp) +# Dedup by Id inside the loop via a hashtable (a "| Select-Object -Unique" +# reassignment can make the following foreach emit nothing in pwsh). +$seen = @{} +foreach ($s in $sessions) { + if ($null -eq $s) { continue } + $sidKey = [string]$s.Id + if ($seen.ContainsKey($sidKey)) { continue } + $seen[$sidKey] = $true + # Completion differs by session type: CBackupSession has IsCompleted, while the + # Agent VBRSession (Get-VBRComputerBackupJobSession) only exposes State — treat + # State=Stopped/Completed as done. (Relying on IsCompleted alone skipped every + # agent session because that property does not exist on VBRSession.) + $done = $false + if ($s.IsCompleted -eq $true) { $done = $true } + $st = [string]$s.State + if ($st -eq 'Stopped' -or $st -eq 'Completed') { $done = $true } + if (-not $done) { continue } + # Agent VBRSession has no JobName; its Name holds the job/computer (dash-IP) name. + $nm = ([string]$s.Name) -replace '[\|\r\n]',' ' + $jn = ([string]$s.JobName) -replace '[\|\r\n]',' ' + $rs = [string]$s.Result + $hay = "$nm $jn" + $ip = '' + $m = [regex]::Match($hay, '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})') + if ($m.Success) { $ip = $m.Groups[1].Value } + if (-not $ip) { + $md = [regex]::Match($hay, '(\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})') + if ($md.Success) { $ip = ($md.Groups[1].Value -replace '-','.') } + } + $epoch = 0 + $etObj = if ($null -ne $s.EndTime) { $s.EndTime } else { $s.EndTimeUTC } + try { if ($null -ne $etObj) { $epoch = [int64]([DateTimeOffset]$etObj).ToUnixTimeSeconds() } } catch { $epoch = 0 } + "$($s.Id)|$ip|$epoch|$rs|$nm|$jn" +} +PS +)" + local ps_enc + ps_enc="$(printf '%s' "$ps_script" | iconv -f UTF-8 -t UTF-16LE 2>/dev/null | base64 -w0 2>/dev/null)" + [[ -n "$ps_enc" ]] || { mold_backup_notify_log warn "Veeam→Mold(backup): failed to encode PS script"; return 1; } + local attempt out rc + for attempt in 1 2 3; do + out="$(ssh "${ssh_key_opt[@]}" -o BatchMode=yes -o ConnectTimeout=30 "${ssh_host_opts[@]}" \ + "${VEEAM_SSH_USER:-administrator}@${VEEAM_SSH_HOST}" \ + "pwsh -NoProfile -EncodedCommand ${ps_enc}" 2>/dev/null)" + rc=$? + if [[ $rc -eq 0 ]]; then + out="${out//$'\r'/}" + local now_epoch cutoff line ep + now_epoch=$(date +%s) + cutoff=$(( now_epoch - since_min * 60 )) + while IFS= read -r line; do + [[ -n "$line" ]] || continue + ep="$(printf '%s' "$line" | cut -d'|' -f3)" + if [[ "$ep" =~ ^[0-9]+$ ]]; then + [[ "$ep" -ge "$cutoff" ]] && printf '%s\n' "$line" + else + printf '%s\n' "$line" + fi + done <<< "$out" + return 0 + fi + mold_backup_notify_log warn "Veeam→Mold(backup): SSH query attempt ${attempt} failed (rc=${rc}); retrying" + sleep 3 + done + mold_backup_notify_log warn "Veeam→Mold(backup): SSH query failed after retries" + return 1 +} + +# Reflect a single Veeam backup session into the Mold-side registry for one VM. +mold_backup_reflect_one_backup() { + local job="$1" vm="$2" sid="$3" detail="$4" + # Loop guard: this backup was initiated by Mold itself (Mold->Veeam trigger); + # the Mold record already exists, so don't double-record. + if mold_backup_trigger_active "mold-active" "$vm"; then + mold_backup_trigger_clear "mold-active" "$vm" + mold_backup_notify_log info "mold-active for ${vm}: backup initiated by Mold; skip reflect (session=${sid})" + mold_backup_backup_session_mark_seen "$sid" + return 0 + fi + mold_backup_registry_save_backup "$job" "$vm" "veeam:${sid}" "veeam-backed-up" + mold_backup_backup_session_mark_seen "$sid" + mold_backup_notify_log info "Veeam→Mold: reflected backup for ${vm} (session=${sid}) detail=${detail}" +} + +# Poll Veeam backup sessions and reflect new ones for VMs we manage (VM_TARGETS). +mold_backup_watch_veeam_backups() { + local job="${1:-${VEEAM_JOB_NAME:-}}" + local since_min="${2:-${VEEAM_BACKUP_WATCH_WINDOW_MIN:-${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}}}" + [[ -n "${VM_TARGETS:-}" ]] || { + mold_backup_notify_log warn "Veeam→Mold(backup): VM_TARGETS empty; nothing to watch" + return 0 + } + mold_backup_notify_log info "=== backup-watch job=${job} window=${since_min}min ===" + local sid sip et result nm jn matched_ip vm + local processed=0 reflected=0 + while IFS='|' read -r sid sip et result nm jn; do + [[ -n "$sid" ]] || continue + processed=$((processed+1)) + mold_backup_backup_session_seen "$sid" && continue + # Only reflect successful/warning backups (a failed backup is not a restore point). + case "$(printf '%s' "$result" | tr '[:upper:]' '[:lower:]')" in + success|warning) ;; + *) + mold_backup_notify_log info "backup-watch: session ${sid} result='${result}' (skip non-success)" + mold_backup_backup_session_mark_seen "$sid" + continue + ;; + esac + # Match: prefer the IP parsed from the session/job name; fall back to scanning + # VM_TARGETS IPs (dots or dashes form) against the session / job name. + matched_ip="" + if [[ -n "$sip" ]] && mold_backup_vm_name_for_ip "$sip" >/dev/null 2>&1; then + matched_ip="$sip" + else + IFS=',' read -ra _pairs <<<"${VM_TARGETS}" + local pair ip ipd vmn + for pair in "${_pairs[@]}"; do + pair="${pair// /}" + vmn="${pair%%:*}" + ip="${pair#*:}" + [[ -n "$ip" && "$ip" != "$vmn" ]] || continue + ipd="${ip//./-}" + # Match by IP (dots/dashes) for legacy "Mold VM " jobs, or by the + # libvirt internal name (e.g. i-2-51-VM) for jobs named by internal name. + if [[ "$sip" == "$ip" || "$nm" == *"$ip"* || "$nm" == *"$ipd"* || "$jn" == *"$ip"* || "$jn" == *"$ipd"* \ + || ( -n "$vmn" && ( "$nm" == *"$vmn"* || "$jn" == *"$vmn"* ) ) ]]; then + matched_ip="$ip" + break + fi + done + fi + if [[ -z "$matched_ip" ]]; then + mold_backup_notify_log info "backup-watch: session ${sid} ip='${sip}' name='${nm}' job='${jn}' — no VM_TARGETS match (skip)" + continue + fi + vm="$(mold_backup_vm_name_for_ip "$matched_ip" 2>/dev/null || true)" + [[ -n "$vm" ]] || continue + mold_backup_reflect_one_backup "$job" "$vm" "$sid" "name=${nm};job=${jn};end=${et};result=${result};ip=${matched_ip}" + reflected=$((reflected+1)) + done < <(mold_backup_query_veeam_backups "$since_min" || true) + mold_backup_notify_log info "=== backup-watch done: scanned=${processed} reflected=${reflected} ===" + return 0 +} + +# Guest Agent on VM: Veeam pre runs before restore point exists — defer Mold API to post. +mold_backup_guest_pre_notify_vm() { + local vm_name="$1" offering_id="$2" state_file="$3" + local vm_id + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { + mold_backup_notify_log err "No Mold VM id for ${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} status=fail reason=no-vm-id" + return 1 + } + [[ -n "$offering_id" ]] && mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + mold_backup_api_check_vm_environment "$vm_id" "$vm_name" + if mold_backup_trigger_active "mold-active" "$vm_name"; then + mold_backup_trigger_clear "mold-active" "$vm_name" + mold_backup_notify_log info "mold-active for ${vm_name}: skip guest pre (Mold already backed up)" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=success reason=mold-triggered" + return 0 + fi + mold_backup_trigger_mark "veeam-active" "$vm_name" + mold_backup_notify_log info "guest pre: defer Mold backup until Veeam post (restore point not ready yet) vm=${vm_name}" + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} status=pending reason=veeam-guest-pre" + return 0 +} + +# After Veeam backup completes, restore point exists — create Mold backup record now. +mold_backup_guest_post_notify_vm() { + local vm_name="$1" vm_id="$2" job="$3" + local backup_result backup_id backup_type offering_id guest_ip rp_id chain_count + local host_path staging_paths source_format vm_offering json + [[ -n "$vm_id" ]] || vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { + mold_backup_notify_log err "guest post: no Mold VM id for ${vm_name}" + return 1 + } + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" "$(mold_backup_offering_name)" 2>/dev/null || true)" + [[ -n "$offering_id" ]] && mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + [[ -n "$job" ]] || job="$(mold_backup_veeam_job_name_for_vm "$vm_name")" + guest_ip="$(mold_backup_vm_guest_ip "$vm_name" 2>/dev/null || true)" + [[ -n "$guest_ip" ]] || mold_backup_notify_log warn "guest post: VM_TARGETS missing ${vm_name}:ip in conf/env (see mold-backup.env)" + rp_id="$(mold_backup_query_veeam_latest_restore_point "$job" "$vm_name" "$guest_ip" 2>/dev/null || true)" + if [[ -z "$rp_id" ]]; then + mold_backup_notify_log err "guest post: no Veeam restore point for ${vm_name} (job=${job} ip=${guest_ip:-n/a} veeam=${VEEAM_SSH_HOST:-unset}; run Veeam backup to Success first)" + return 1 + fi + mold_backup_notify_log info "guest post: Veeam restore point=${rp_id} vm=${vm_name} job=${job}" + chain_count="$(mold_backup_api_veeam_backup_count "$vm_id")" + if [[ "${chain_count:-0}" -eq 0 ]]; then + # Guest SelectedFiles backups have file-level restore points (no exportable disks on Veeam). + # Seed NAS from live KVM disks (same as host file-level first backup), tag Veeam RP id. + mold_backup_notify_log info "guest post: first backup — host export + importSeed (SelectedFiles; skip MS→Veeam disk export)" + if ! host_path=$(mold_backup_run_host_export "$vm_name" "1" 2>/dev/null); then + mold_backup_notify_log err "guest post: host export failed for ${vm_name} (check ${CVT_BACKUP_SCRIPT} and /var/log/mold/veeam-hook.log)" + return 1 + fi + if [[ ! -d "$host_path" ]]; then + mold_backup_notify_log err "guest post: host export invalid path: ${host_path}" + return 1 + fi + staging_paths="$(mold_backup_collect_host_staging_paths "$host_path" 2>/dev/null || true)" + if [[ -z "$staging_paths" ]]; then + mold_backup_notify_log err "guest post: no staging disks under ${host_path}" + return 1 + fi + source_format="$(mold_backup_detect_staging_source_format "$staging_paths")" + json=$(mold_backup_cmk_run listVirtualMachines "id=${vm_id}" 2>/dev/null || true) + vm_offering="$(mold_backup_api_json_field "$json" "listvirtualmachinesresponse.virtualmachine.backupofferingid")" + mold_backup_api_validate_offering_repository "$vm_offering" || return 1 + backup_result="$(mold_backup_api_import_staging_rp_seed_and_wait "$vm_id" "$staging_paths" "$source_format" "$rp_id" "$vm_name" || true)" + else + mold_backup_notify_log info "guest post: incremental — createAblestackVeeamBackup vm=${vm_name} (chain=${chain_count})" + backup_result="$(mold_backup_api_create_veeam_and_wait "$vm_id" "$vm_name" 2>/dev/null || true)" + fi + backup_id="${backup_result%%|*}" + backup_type="${backup_result#*|}" + if [[ -n "$backup_id" ]]; then + mold_backup_registry_save_backup "$job" "$vm_name" "$backup_id" "veeam-backed-up rp=${rp_id}" "$rp_id" + export BACKUP_ID="$backup_id" VM_NAME="$vm_name" + mold_backup_notify_log info "guest post OK vm=${vm_name} backup_id=${backup_id} type=${backup_type} rp=${rp_id}" + return 0 + fi + mold_backup_notify_log err "guest post: Mold API backup failed for ${vm_name} (rp=${rp_id} chain=${chain_count:-0}; check MS/agent.log)" + return 1 +} + +mold_backup_pre_notify() { + local client="${1:-$(hostname -s)}" + local job="${2:-${VEEAM_JOB_NAME:-}}" + local schedule="${3:-${VEEAM_SCHEDULE_NAME:-default}}" + local single_vm="${4:-}" + local saved_include="" + [[ -n "$job" ]] || mold_backup_die "VEEAM_JOB_NAME is required for pre-notify" + VEEAM_JOB_NAME="$job" + mold_backup_load_config || exit 1 + + if [[ -n "$single_vm" ]]; then + saved_include="${VM_INCLUDE:-*}" + VM_INCLUDE="$single_vm" + mold_backup_notify_log info "single-vm scope: ${single_vm}" + fi + + mold_backup_notify_log info "=== pre-notify (설계4: Pre-script + Mold API 백업요청) client=${client} job=${job} schedule=${schedule} ===" + mkdir -p "$(mold_backup_state_dir)" "${VEEAM_HOST_BACKUP_PATH}" + + local offering_id="" vm_name + local success=0 fail=0 + local state_file run_id + run_id="$(date '+%Y%m%d%H%M%S')" + state_file="$(mold_backup_state_file_for_job "$job" "$run_id")" + : > "$state_file" + echo "run_id=${run_id}" >> "$state_file" + + if mold_backup_cmk_bin >/dev/null 2>&1 || command -v curl >/dev/null 2>&1; then + mold_backup_api_ensure_global_settings || true + offering_id="$(mold_backup_api_find_offering_id "${VEEAM_PROVIDER_NAME}" "$(mold_backup_offering_name)" 2>/dev/null || true)" + if [[ -z "$offering_id" ]]; then + if ! mold_backup_api_list_backup_offerings >/dev/null 2>&1; then + mold_backup_notify_log err "Cannot list/import backup offerings (check API key/secret or MS DB schema; see mold-ms-backup-schema-fix.sql)" + else + offering_id="$(mold_backup_api_ensure_backup_resources 2>/dev/null || true)" + fi + fi + [[ -n "$offering_id" ]] || { + if mold_backup_is_datadisk_mode; then + mold_backup_notify_log warn "No backup offering '$(mold_backup_offering_name)' for ${VEEAM_PROVIDER_NAME}; assign in Mold UI (datadisk mode does not auto-create backup repository)" + else + mold_backup_notify_log warn "No backup offering '$(mold_backup_offering_name)' for ${VEEAM_PROVIDER_NAME}; set BACKUP_REPO_ADDRESS + ZONE_ID and re-run veeam_config.sh" + fi + } + fi + + while IFS= read -r vm_name; do + [[ -z "$vm_name" ]] && continue + mold_backup_vm_in_filter "$vm_name" || continue + mold_backup_notify_log info "Target VM ${vm_name}" + + case "${BACKUP_MODE}" in + guest|veeam-guest) + if mold_backup_guest_pre_notify_vm "$vm_name" "$offering_id" "$state_file"; then + success=$((success + 1)) + else + fail=$((fail + 1)) + fi + ;; + host|policy) + if mold_backup_process_vm_pre_notify "$vm_name" "$offering_id" "$state_file"; then + success=$((success + 1)) + else + fail=$((fail + 1)) + fi + ;; + api|local|auto) + local vm_id backup_result backup_id + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { fail=$((fail + 1)); continue; } + [[ -n "$offering_id" ]] && mold_backup_api_assign_offering_if_needed "$vm_id" "$offering_id" + mold_backup_api_check_vm_environment "$vm_id" "$vm_name" + backup_result="$(mold_backup_api_create_veeam_and_wait "$vm_id" "$vm_name" 2>/dev/null || true)" + backup_id="${backup_result%%|*}" + if [[ -n "$backup_id" ]]; then + mold_backup_state_write_line "$state_file" "vm=${vm_name} id=${vm_id} backup_id=${backup_id} status=success" + success=$((success + 1)) + else + fail=$((fail + 1)) + fi + ;; + *) + mold_backup_die "Invalid BACKUP_MODE=${BACKUP_MODE} (use guest|host|policy|api|local|auto)" + ;; + esac + done < <(mold_backup_list_target_domains || true) + + if [[ "$success" -eq 0 && "$fail" -eq 0 ]]; then + mold_backup_notify_log warn "No target VMs for job=${job} (vm_include=${VM_INCLUDE:-*}). Start VM or set VM_INCLUDE to libvirt name(s)." + if [[ "${VM_INCLUDE:-*}" != "*" ]]; then + local _t + for _t in ${VM_INCLUDE//,/ }; do + _t="$(echo "$_t" | xargs)" + [[ -z "$_t" ]] && continue + if mold_backup_domain_exists "$_t"; then + if virsh -c qemu:///system dominfo "$_t" 2>/dev/null | grep -q 'State:.*shut off'; then + mold_backup_notify_log warn "VM ${_t} exists but is shut off — start it for host export: virsh start ${_t}" + fi + else + mold_backup_notify_log warn "VM ${_t} not found in libvirt on $(hostname -s)" + fi + done + fi + fi + + mold_backup_notify_log info "pre-notify done success=${success} fail=${fail} state=${state_file}" + [[ -n "$saved_include" ]] && VM_INCLUDE="$saved_include" + [[ "$success" -gt 0 ]] && return 0 + return 1 +} + +mold_backup_post_notify() { + local client="${1:-$(hostname -s)}" + local job="${2:-${VEEAM_JOB_NAME:-}}" + local schedule="${3:-${VEEAM_SCHEDULE_NAME:-default}}" + [[ -n "$job" ]] || mold_backup_die "VEEAM_JOB_NAME is required for post-notify" + VEEAM_JOB_NAME="$job" + mold_backup_load_config || exit 1 + + mold_backup_notify_log info "=== post-notify (설계4: Post-script + 백업ID 저장) client=${client} job=${job} ===" + local state_file line vm_name backup_id status guest_handled=0 guest_fail=0 vm_id reason rc=0 + local host_rp_id="" + state_file="$(mold_backup_latest_state_file "$job" || true)" + + if [[ "${BACKUP_MODE:-host}" == "host" || "${BACKUP_MODE:-host}" == "policy" ]]; then + host_rp_id="$(mold_backup_query_veeam_latest_restore_point "$job" "" "" 6 2>/dev/null || true)" + [[ -n "$host_rp_id" ]] && mold_backup_notify_log info "post-notify: host job Veeam restore point=${host_rp_id}" + fi + + if [[ -f "$state_file" ]]; then + while IFS= read -r line; do + [[ "$line" =~ ^vm= ]] || continue + vm_name="$(mold_backup_state_parse_field "$line" "vm")" + backup_id="$(mold_backup_state_parse_field "$line" "backup_id")" + status="$(mold_backup_state_parse_field "$line" "status")" + vm_id="$(mold_backup_state_parse_field "$line" "id")" + reason="$(mold_backup_state_parse_field "$line" "reason")" + # Mold schedule/UI backup → Veeam trigger: Mold NAS backup already exists (HOURLY/MANUAL). + if [[ "$status" == "success" && "$reason" == "mold-triggered" ]]; then + mold_backup_notify_log info "guest post: skip for ${vm_name} (Mold backup already done; no duplicate import)" + guest_handled=1 + [[ -n "$vm_name" ]] && mold_backup_trigger_clear "veeam-active" "$vm_name" + continue + fi + if [[ "$status" == "pending" && "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + mold_backup_notify_log info "guest post: pending vm=${vm_name} mode=${BACKUP_MODE} reason=${reason:-veeam-guest-pre}" + if mold_backup_guest_post_notify_vm "$vm_name" "$vm_id" "$job"; then + guest_handled=1 + else + guest_fail=$((guest_fail + 1)) + fi + [[ -n "$vm_name" ]] && mold_backup_trigger_clear "veeam-active" "$vm_name" + continue + fi + # Veeam job for this VM finished — clear the loop-guard marker. + [[ -n "$vm_name" ]] && mold_backup_trigger_clear "veeam-active" "$vm_name" + if [[ "$status" == "pending" ]]; then + mold_backup_notify_log warn "post-notify: pending vm=${vm_name} but BACKUP_MODE=${BACKUP_MODE:-host} (expected guest)" + fi + [[ "$status" == "success" && -n "$backup_id" ]] || continue + mold_backup_registry_save_backup "$job" "$vm_name" "$backup_id" "veeam-backed-up rp=${host_rp_id:-n/a}" "$host_rp_id" + done < "$state_file" + else + mold_backup_notify_log warn "No state file for job ${job}" + if [[ "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + vm_name="$(mold_backup_vm_name_for_job "$job" 2>/dev/null || true)" + if [[ -n "$vm_name" ]]; then + mold_backup_notify_log info "guest post: fallback (no state) vm=${vm_name}" + if mold_backup_guest_post_notify_vm "$vm_name" "" "$job"; then + guest_handled=1 + else + guest_fail=$((guest_fail + 1)) + fi + mold_backup_trigger_clear "veeam-active" "$vm_name" + fi + fi + fi + + if [[ "$guest_handled" -eq 0 && "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + vm_name="$(mold_backup_vm_name_for_job "$job" 2>/dev/null || true)" + if [[ -n "$vm_name" ]]; then + mold_backup_notify_log info "guest post: fallback (no pending state) vm=${vm_name}" + if mold_backup_guest_post_notify_vm "$vm_name" "" "$job"; then + guest_handled=1 + else + guest_fail=$((guest_fail + 1)) + fi + mold_backup_trigger_clear "veeam-active" "$vm_name" + fi + fi + + if [[ "$guest_fail" -gt 0 ]]; then + mold_backup_notify_log err "post-notify: guest Mold backup failed for job=${job} (restore will not work until post succeeds)" + rc=1 + else + rc=0 + fi + + if [[ "$guest_handled" -eq 0 && "${BACKUP_MODE}" =~ ^(guest|veeam-guest)$ ]]; then + mold_backup_notify_log warn "post-notify: no guest VM processed for job=${job} (re-run pre-notify before post, or check state dir)" + fi + + mold_backup_cleanup_host_path + mold_backup_cleanup_staging + [[ -f "$state_file" ]] && rm -f "$state_file" + mold_backup_notify_log info "=== post-notify done (guest_fail=${guest_fail}) ===" + return "$rc" +} + +mold_backup_api_backup_vm_id() { + local backup_id="$1" json + json=$(mold_backup_cmk_run listBackups "id=${backup_id}" 2>/dev/null) || return 1 + mold_backup_api_json_field "$json" "listbackupsresponse.backup.virtualmachineid" +} + +mold_backup_api_verify_backup_for_vm() { + local backup_id="$1" vm_id="$2" + local owner + owner="$(mold_backup_api_backup_vm_id "$backup_id" 2>/dev/null || true)" + [[ -n "$owner" ]] || return 0 + [[ "$owner" == "$vm_id" ]] || { + mold_backup_notify_log err "Backup ${backup_id} belongs to VM ${owner}, not ${vm_id}" + return 1 + } + return 0 +} + +mold_backup_restore_notify() { + local client="${1:-$(hostname -s)}" + local job="${2:-${VEEAM_JOB_NAME:-}}" + mold_backup_load_config || exit 1 + mold_backup_apply_datadisk_profile + local restore_source="${RESTORE_SOURCE:-auto}" + if mold_backup_is_datadisk_mode; then + restore_source="mold-only" + RESTORE_SOURCE="mold-only" + fi + mold_backup_notify_log info "=== restore-notify client=${client} job=${job} backup_id=${BACKUP_ID:-} vm=${VM_NAME:-} source=${restore_source} ===" + mold_backup_require_var BACKUP_ID + [[ -n "${VM_UUID:-}" ]] || { + [[ -n "${VM_NAME:-}" ]] && VM_UUID="$(mold_backup_api_get_vm_id "$VM_NAME" 2>/dev/null || true)" + } + [[ -n "${VM_UUID:-}" ]] || mold_backup_die "restore requires VM_NAME or VM_UUID (individual VM restore)" + mold_backup_api_verify_backup_for_vm "$BACKUP_ID" "$VM_UUID" || exit 1 + + # Loop guard for reverse restore-sync: mark this VM so the Veeam->Mold restore + # watcher skips reflecting a restore that Mold itself initiated. + [[ -n "${VM_NAME:-}" ]] && mold_backup_trigger_mark "mold-restore-active" "$VM_NAME" + + if mold_backup_api_is_full_backup "$BACKUP_ID"; then + mold_backup_notify_log info "Restore type=FULL → Mold restoreBackup (datadisk ${BACKUP_REPO_ADDRESS:-/data/backup})" + mold_backup_api_restore + else + mold_backup_notify_log info "Restore type=INCREMENTAL → datadisk only (qcow2/raw/rbdiff on ${BACKUP_REPO_ADDRESS:-/data/backup}, no NAS)" + if [[ "$restore_source" != "mold-only" ]]; then + mold_backup_veeam_restore_chain_to_host "$BACKUP_ID" + fi + mold_backup_api_restore + fi + mold_backup_notify_log info "=== restore-notify done ===" +} diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.sh b/scripts/vm/hypervisor/kvm/veeam/mold-backup.sh new file mode 100755 index 000000000000..0faf4d98513f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.sh @@ -0,0 +1,345 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Mold + Veeam backup operations on KVM. +# +# backup-full 전체 VM 백업 (VM_INCLUDE=* 또는 목록) +# list-backups VM별 BackedUp 백업 목록 +# restore 개별 VM 복원 (--vm-name + --backup-id) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ABLESTACK_VEEAM_ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +CMD="${1:-}" +shift || true + +JOB_NAME="${VEEAM_JOB_NAME:-VeeamBackup}" +BACKUP_ID="${BACKUP_ID:-}" +VM_NAME_ARG="" +VM_UUID_ARG="" +CLIENT="$(hostname -s)" +DRY_RUN="${DRY_RUN:-false}" +RESTORE_SOURCE="${RESTORE_SOURCE:-}" +SINCE_MIN="${VEEAM_RESTORE_WATCH_WINDOW_MIN:-60}" +TRIGGER_MOLD="${RESTORE_WATCH_TRIGGER_MOLD:-false}" +RESTORE_EVENT="" +RESTORE_EVENT_ARGS=() + +usage() { + cat <<'EOF' +Usage: mold-backup.sh [options] + +Commands: + backup Pre: Mold API + /tmp/mold/veeam export (대상: VM_INCLUDE) + backup-complete Post: registry + staging 정리 + backup-full backup + backup-complete (전체/다중 VM 한 번에) + list-backups VM별 BackedUp 백업 목록 (개별 복원용 ID 확인) + restore 개별 VM 복원 (--vm-name + --backup-id, VM 정지 권장) + restore-notify restore 와 동일 (ablestack_veeam_restore_notify.sh 호출) + restore-watch Veeam UI 복원 감지 → (옵션) Mold datadisk restoreBackup 호출 + restore-event 복원 이벤트 수동 주입 (veeam.restore.completed | mold.restore.manual) + backup-watch Veeam UI에서 직접 백업한 세션을 감지해 Mold 상태에 반영 + status 한 VM 백업 + registry + +Options: + --job NAME Job conf 이름 (default: VeeamBackup) + --backup-id UUID restore 시 백업 ID + --vm-name NAME libvirt 이름 (restore/status/list) + --vm-uuid UUID Mold VM UUID + --restore-source SRC 복원 소스: auto | veeam | mold-only (default: auto) + auto = FULL은 Mold, INCREMENTAL은 Veeam 체인 회수 후 Mold 복원 + veeam = Veeam 체인을 호스트로 회수 후 Mold 복원 + mold-only= datadisk 백업만으로 복원 (NAS·Veeam chain export 없음) + --since-min N restore-watch/backup-watch 조회 창(분, default: 60) + --trigger-mold restore-watch: VM 소유 호스트만 Mold restoreBackup 호출 + -n, --dry-run + +Mold backup offering: BACKUP_OFFERING_NAME=VeeamBackup (veeam_config --offering-name) + +Examples: + # 전체 실행 중 VM 백업 + mold-backup.sh backup-full --job VeeamBackup + + # VM별 백업 ID 확인 + mold-backup.sh list-backups --job VeeamBackup + + # i-2-11-VM 만 복원 (auto: FULL=Mold, INCREMENTAL=Veeam 회수 후 Mold) + virsh shutdown i-2-11-VM + mold-backup.sh restore --job VeeamBackup --vm-name i-2-11-VM --backup-id + + # Veeam 리포지토리에서 복원 (Veeam 체인 회수 후 Mold 복원) + mold-backup.sh restore --job VeeamBackup --vm-name i-2-11-VM --backup-id --restore-source veeam + + # Mold datadisk만으로 복원 (NAS/Veeam chain 없음) + mold-backup.sh restore --job VeeamBackup --vm-name i-2-11-VM --backup-id --restore-source mold-only + + # Veeam UI 복원 감지 (+ --trigger-mold 시 KVM이 datadisk restoreBackup 실행) + mold-backup.sh restore-watch --job Mold_Guest_Backup --since-min 10 --trigger-mold + + # 이벤트 직접 주입 (Veeam PS1 SSH push) + mold-backup.sh restore-event --job Mold_Guest_Backup veeam.restore.completed i-2-63-VM + + # Veeam UI에서 직접 백업한 내역을 Mold에 반영 (cron/systemd timer로 주기 실행 권장) + mold-backup.sh backup-watch --job VeeamBackup --since-min 60 +EOF +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --job) JOB_NAME="$2"; shift 2 ;; + --backup-id) BACKUP_ID="$2"; shift 2 ;; + --vm-name) VM_NAME_ARG="$2"; shift 2 ;; + --vm-uuid) VM_UUID_ARG="$2"; shift 2 ;; + --restore-source) RESTORE_SOURCE="$2"; shift 2 ;; + --since-min) SINCE_MIN="$2"; shift 2 ;; + --trigger-mold) TRIGGER_MOLD=true; shift ;; + -n|--dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + if [[ "$CMD" == "restore-event" ]]; then + RESTORE_EVENT_ARGS+=("$1") + shift + else + die "Unknown option: $1" + fi + ;; + esac +done + +[[ -n "$CMD" ]] || { usage; exit 1; } + +export VEEAM_JOB_NAME="$JOB_NAME" +mold_backup_load_config || die "Job config not found — run veeam_config.sh first" + +[[ -n "$VM_NAME_ARG" ]] && export VM_NAME="$VM_NAME_ARG" +[[ -n "$VM_UUID_ARG" ]] && export VM_UUID="$VM_UUID_ARG" +[[ -n "$BACKUP_ID" ]] && export BACKUP_ID +[[ -n "${RESTORE_SOURCE:-}" ]] && export RESTORE_SOURCE + +run_pre() { + echo "=== backup (pre-notify) job=${JOB_NAME} offering=$(mold_backup_offering_name) vm_include=${VM_INCLUDE:-*} ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] ablestack_veeam_pre_notify.sh ${CLIENT} ${JOB_NAME}" + return 0 + fi + "${ABLESTACK_VEEAM_ETC_DIR}/ablestack_veeam_pre_notify.sh" "$CLIENT" "$JOB_NAME" +} + +run_post() { + echo "=== backup-complete job=${JOB_NAME} ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] ablestack_veeam_post_notify.sh ${CLIENT} ${JOB_NAME}" + return 0 + fi + "${ABLESTACK_VEEAM_ETC_DIR}/ablestack_veeam_post_notify.sh" "$CLIENT" "$JOB_NAME" +} + +# Mold-initiated bidirectional (mode C): after a Mold backup, start the matching Veeam +# Agent job(s) over SSH. mold_backup_trigger_veeam_job sets the mold-active marker so the +# Veeam-driven pre_notify skips a duplicate createAblestackVeeamBackup (one Mold record). +# Requires VEEAM_TRIGGER_ENABLED/METHOD/SSH_* and VM_TARGETS in the loaded job conf. +run_veeam_trigger() { + if [[ "${VEEAM_TRIGGER_ENABLED:-false}" != "true" ]]; then + return 0 + fi + echo "=== Mold→Veeam trigger (job=${JOB_NAME}) ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] mold_backup_trigger_veeam_job for VM_INCLUDE=${VM_INCLUDE:-*}" + return 0 + fi + local vm + local -a _domains=() + while IFS= read -r vm; do + [[ -z "$vm" ]] && continue + _domains+=("$vm") + done < <(mold_backup_list_target_domains || true) + for vm in "${_domains[@]}"; do + mold_backup_vm_in_filter "$vm" || continue + mold_backup_trigger_veeam_job "$vm" [session-id] [vm-name] [detail]" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] ablestack_veeam_restore_event.sh $*" + return 0 + fi + "${ABLESTACK_VEEAM_ETC_DIR}/ablestack_veeam_restore_event.sh" "$@" +} + +# Veeam UI에서 직접 수행한 백업을 감지해 Mold 상태(registry/로그)에 반영. +# restore-watch와 동일하게 KVM→Veeam SSH로 최근 완료된 백업 세션을 조회하고, +# 대상 게스트 IP를 VM_TARGETS로 libvirt VM에 역매핑해 기록한다. +run_backup_watch() { + if [[ "${VEEAM_TRIGGER_ENABLED:-false}" != "true" && -z "${VEEAM_SSH_HOST:-}" ]]; then + die "backup-watch needs VEEAM_SSH_HOST (KVM→Veeam SSH) in job conf" + fi + echo "=== backup-watch job=${JOB_NAME} window=${SINCE_MIN}min ===" + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] mold_backup_watch_veeam_backups ${JOB_NAME} ${SINCE_MIN}" + return 0 + fi + mold_backup_watch_veeam_backups "$JOB_NAME" "$SINCE_MIN" + local safe_job reg + safe_job="$(mold_backup_safe_job_name "$JOB_NAME")" + reg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.log" + if [[ -f "$reg" ]]; then + echo "=== backup registry (last 10) ===" + tail -10 "$reg" + fi +} + +run_list_backups() { + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + echo "=== BackedUp backups (offering=$(mold_backup_offering_name)) ===" + local vm_name vm_id json + while IFS= read -r vm_name; do + [[ -z "$vm_name" ]] && continue + mold_backup_vm_in_filter "$vm_name" || continue + vm_id="$(mold_backup_api_get_vm_id "$vm_name" 2>/dev/null || true)" + [[ -n "$vm_id" ]] || { echo "${vm_name}: (no Mold VM id)"; continue; } + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + echo "--- ${vm_name} (${vm_id}) ---" + echo "$json" | python3 -c " +import json, sys +try: + d = json.load(sys.stdin) + b = d.get('listablestackveeambackupsresponse', {}).get('backup', []) + if isinstance(b, dict): b = [b] + backed = [x for x in b if str(x.get('status','')).lower() == 'backedup'] + if not backed: + print(' (no BackedUp backups)') + for x in backed: + print(f\" restore: mold-backup.sh restore --vm-name {sys.argv[1]} --backup-id {x.get('id')} # {x.get('created')} {x.get('type')}\") +except Exception as e: + print(' (parse error)', e) +" "$vm_name" 2>/dev/null || echo "$json" | head -c 200 + done < <(mold_backup_list_target_domains || true) + local safe_job reg + safe_job="$(mold_backup_safe_job_name "$JOB_NAME")" + reg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.log" + if [[ -f "$reg" ]]; then + echo "=== registry (last backup_id per VM) ===" + tail -30 "$reg" + fi +} + +run_status() { + mold_backup_require_var MOLD_API_URL + mold_backup_require_var MOLD_API_KEY + mold_backup_require_var MOLD_API_SECRET + local vm_id="${VM_UUID:-}" + if [[ -z "$vm_id" && -n "${VM_NAME:-}" ]]; then + vm_id="$(mold_backup_api_get_vm_id "$VM_NAME" 2>/dev/null || true)" + fi + [[ -n "$vm_id" ]] || die "status needs --vm-name, --vm-uuid, or VM_NAME in conf" + + echo "=== Mold backups VM ${vm_id} ===" + mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" \ + | python3 -m json.tool 2>/dev/null || true + local json cnt + json="$(mold_backup_cmk_run listAblestackVeeamBackups "virtualmachineid=${vm_id}" 2>/dev/null || true)" + cnt="$(mold_backup_api_count_backed_up_from_json "$json" 2>/dev/null || echo "?")" + echo "BackedUp chain count: ${cnt}" + local safe_job reg + safe_job="$(mold_backup_safe_job_name "$JOB_NAME")" + reg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.log" + echo "=== registry ===" + [[ -f "$reg" ]] && grep "vm=${VM_NAME:-}" "$reg" 2>/dev/null | tail -5 || tail -10 "$reg" 2>/dev/null || echo "(none)" + local rreg="${ABLESTACK_VEEAM_ETC_DIR}/registry/${safe_job}.restore.log" + echo "=== restore registry (Veeam UI 복원 반영) ===" + if [[ -f "$rreg" ]]; then + grep "vm=${VM_NAME:-}" "$rreg" 2>/dev/null | tail -5 || tail -10 "$rreg" 2>/dev/null + else + echo "(none)" + fi +} + +case "$CMD" in + backup|pre) run_pre ;; + backup-complete|post) run_post ;; + backup-full|full) + run_pre + run_post + run_veeam_trigger + ;; + restore|restore-notify) run_restore ;; + restore-watch|watch-restore) run_restore_watch ;; + restore-event) run_restore_event "${RESTORE_EVENT_ARGS[@]}" ;; + backup-watch|watch-backup) run_backup_watch ;; + list-backups|list) run_list_backups ;; + status) run_status ;; + -h|--help|help) usage ;; + *) die "Unknown command: ${CMD}" ;; +esac + +echo "=== mold-backup.sh ${CMD} done ===" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default b/scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default new file mode 100644 index 000000000000..09ad538b37f4 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-backup.windows.conf.default @@ -0,0 +1,46 @@ +# Veeam Backup & Replication server configuration +# Copy to C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf and edit. + +VEEAM_JOB_NAME="" +VEEAM_SCHEDULE_NAME="default" +VEEAM_MAX_CHAIN="7" +# filelevel = KVM Agent SelectedFiles /tmp/mold/veeam (host mode) +# guest = one Agent job per VM IP (SelectedFiles on guest; pre-job/post-job for Mold hooks) +VEEAM_BACKUP_MODE="filelevel" +VEEAM_BACKUP_TARGET="host" +# Guest mode: libvirt-name:guest-ip pairs (comma-separated) +VM_TARGETS="" +VEEAM_GUEST_JOB_PREFIX="Mold VM" +# Guest VM SSH (Veeam Protection Group — auto Individual computers + rescan) +GUEST_VM_SSH_USER=root +GUEST_VM_SSH_PASSWORD= +# Linux path on KVM — used by setup-veeam-mold-job.ps1 / create-veeam-agent-job.ps1 (host mode only) +VEEAM_HOST_BACKUP_PATH="/tmp/mold/veeam" +# Custom protection group for Agent job (PowerShell). Not "Manually Added". +# guest mode: create in Veeam UI first (Inventory -> Protection Groups) +VEEAM_PROTECTION_GROUP_NAME="Mold KVM Agents" + +VM_NAME="" +# Windows-only: FLR disk export on B&R server (veeam-job-pre-backup.ps1). Do NOT use for Linux Agent scope. +STAGING_PATH="D:\veeam-staging" + +KVM_HOST="" +KVM_SSH_USER="root" +KVM_SSH_KEY="" +KVM_SSH_PASSWORD="" +KVM_PRE_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" +KVM_POST_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" +KVM_RESTORE_NOTIFY_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_restore_notify.sh" +KVM_RESTORE_EVENT_SCRIPT="/etc/ablestack/veeam/ablestack_veeam_restore_event.sh" +CLEANUP_STAGING_AFTER_BACKUP="true" + +VEEAM_RESTORE_POINT_ID="" +BACKUP_ID="" +# Per-VM Mold backup UUID for restore-watch / post-restore (libvirt-name:uuid pairs) +# Example: VM_BACKUP_IDS=i-2-63-VM:2d48660f-de6b-42de-ac47-9f1337b9bfce +VM_BACKUP_IDS="" +# Datadisk + Veeam host repo (E:\opt1\veeam\) +VEEAM_HOST_REPO_ROOT="E:/opt1/veeam" +VEEAM_REPO_NAME="" +# After Veeam UI guest-file restore: Mold datadisk qcow2 only (Veeam already restored files) +VEEAM_UI_RESTORE_SOURCE="mold-only" diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh b/scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh new file mode 100644 index 000000000000..e44d777ee392 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-guest-common.sh @@ -0,0 +1,417 @@ +#!/usr/bin/bash +# Shared helpers for Mold guest VM + Veeam onboarding (sourced by other scripts). +# shellcheck shell=bash + +mold_guest_common_die() { echo "ERROR: $*" >&2; exit 1; } + +mold_guest_resolve_script() { + local name="$1" + local base="${MOLD_GUEST_SCRIPT_DIR:-}" + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local candidate + for candidate in \ + "${base}/${name}" \ + "${etc}/${name}" \ + "/usr/share/mold/backup/veeam/${name}" \ + "/tmp/veeam-install/${name}"; do + if [[ -f "$candidate" ]]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +mold_guest_sync_api_from_env() { + local env_file="$1" conf_file="$2" key val + [[ -f "$env_file" && -f "$conf_file" ]] || return 0 + for key in MOLD_API_URL MOLD_API_KEY MOLD_API_SECRET ZONE_ID BACKUP_REPO_ADDRESS BACKUP_OFFERING_NAME \ + VEEAM_SSH_HOST VM_TARGETS VEEAM_PASSWORD VEEAM_USERNAME VEEAM_TRIGGER_METHOD \ + MOLD_DATADISK_PATH VEEAM_HOST_REPO_ROOT VEEAM_REPO_NAME BACKUP_STORAGE_MODE KVM_HOSTNAME \ + KVM_HOST KVM_SSH_USER KVM_SSH_PASSWORD VEEAM_JOB_NAME VEEAM_PROTECTION_GROUP_NAME; do + val=$(grep -E "^${key}=" "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"') || true + [[ -n "$val" ]] || continue + if grep -qE "^${key}=" "$conf_file" 2>/dev/null; then + sed -i "s#^${key}=.*#${key}=\"${val}\"#" "$conf_file" + else + echo "${key}=\"${val}\"" >> "$conf_file" + fi + done +} + +# Push mold-backup.windows.conf (with KVM_SSH_PASSWORD) to Veeam B&R for setup-veeam-mold-job.ps1. +mold_guest_sync_windows_conf_to_veeam() { + local env_file="$1" + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local win_local="${etc}/mold-backup.windows.conf" + local veeam_host veeam_user install_dir + local -a ssh_opts=() scp_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + [[ -n "$veeam_host" ]] || return 0 + [[ -f "$win_local" ]] || return 0 + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + mold_guest_sync_api_from_env "$env_file" "$win_local" + echo "=== Sync windows conf → ${veeam_user}@${veeam_host}:${install_dir}/mold-backup.windows.conf ===" + scp "${scp_opts[@]}" "${ssh_opts[@]}" "$win_local" \ + "${veeam_user}@${veeam_host}:${install_dir}/mold-backup.windows.conf" +} + +mold_guest_ensure_conf() { + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local guest_conf="${etc}/Mold_Guest_Backup.conf" + local main_conf="" candidate + for candidate in "${etc}/mold-backup.conf" "${etc}/Mold_Guest_Backup.conf" "${etc}/"*.conf; do + [[ -f "$candidate" ]] || continue + main_conf="$candidate" + break + done + [[ -n "$main_conf" ]] || mold_guest_common_die "Missing KVM backup conf under ${etc} — run veeam_config.sh first" + if [[ ! -f "$guest_conf" ]]; then + cp -a "$main_conf" "$guest_conf" + chmod 0600 "$guest_conf" + echo "Created ${guest_conf} from $(basename "$main_conf")" + fi + local env_file="${etc}/mold-backup.env" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$guest_conf" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$main_conf" + sed -i 's/^BACKUP_MODE=.*/BACKUP_MODE="guest"/' "$guest_conf" + for key_val in \ + 'VEEAM_TRIGGER_ENABLED="true"' \ + 'VEEAM_TRIGGER_METHOD="ssh"'; do + key="${key_val%%=*}" + if grep -qE "^${key}=" "$guest_conf" 2>/dev/null; then + sed -i "s#^${key}=.*#${key_val}#" "$guest_conf" + else + echo "$key_val" >> "$guest_conf" + fi + done +} + +# Passwordless SSH guest VM -> KVM (required for Veeam pre/post hooks). +mold_guest_run_ssh_setup() { + local env_file="$1" + local vm_targets="$2" + local setup_script rc=0 + + [[ -n "$vm_targets" ]] || mold_guest_common_die "VM_TARGETS empty for guest SSH setup" + [[ -f "$env_file" ]] || mold_guest_common_die "Missing env file: $env_file" + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + if [[ -z "${GUEST_VM_SSH_PASSWORD:-}" ]]; then + mold_guest_common_die "GUEST_VM_SSH_PASSWORD is not set in ${env_file} (required for automatic guest->KVM SSH)" + fi + + setup_script="$(mold_guest_resolve_script setup-guest-kvm-ssh.sh)" \ + || mold_guest_common_die "setup-guest-kvm-ssh.sh not found" + + echo "=== Auto: guest -> KVM SSH (${vm_targets}) ===" + bash "$setup_script" --env-file "$env_file" --vm-targets "$vm_targets" || rc=$? + if [[ "$rc" -ne 0 ]]; then + mold_guest_common_die "guest->KVM SSH setup failed (check GUEST_VM_SSH_PASSWORD and guest reachability)" + fi +} + +# conf + SSH + optional Veeam .sh upload (called from deploy / push / veeam_config). +mold_guest_prepare_onboarding() { + local env_file="$1" + local vm_targets="$2" + local skip_ssh="${3:-false}" + + mold_guest_ensure_conf + if [[ "$skip_ssh" != "true" ]]; then + mold_guest_run_ssh_setup "$env_file" "$vm_targets" + fi +} + +mold_guest_resolve_kvm_ip_from_env() { + local env_file="$1" + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + if [[ -n "${KVM_IP:-}" ]]; then + echo "$KVM_IP" + return 0 + fi + local host="${KVM_HOST:-}" + if [[ "$host" =~ @(.+)$ ]]; then + echo "${BASH_REMATCH[1]}" + return 0 + fi + [[ -n "$host" ]] && echo "$host" && return 0 + mold_guest_common_die "Set KVM_IP or KVM_HOST in ${env_file}" +} + +# E:\opt1\veeam\ — one Veeam repository per hypervisor (not per VM). +mold_guest_setup_veeam_host_repo() { + local env_file="$1" kvm_hostname="${2:-}" + local veeam_host veeam_user install_dir host_root repo_name + local -a ssh_opts=() scp_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" ]] || mold_guest_common_die "VEEAM_SSH_HOST not set" + [[ -n "$kvm_hostname" ]] || kvm_hostname="${KVM_HOSTNAME:-$(hostname -s)}" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + host_root="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" + repo_name="${VEEAM_REPO_NAME:-Mold ${kvm_hostname}}" + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + mold_guest_sync_ps1_to_veeam "$env_file" + local ps1_local + ps1_local="$(mold_guest_resolve_script setup-veeam-host-repo.ps1 2>/dev/null || true)" + if [[ -n "$ps1_local" ]]; then + scp "${scp_opts[@]}" "${ssh_opts[@]}" "$ps1_local" \ + "${veeam_user}@${veeam_host}:${install_dir}/setup-veeam-host-repo.ps1" + fi + + echo "=== Veeam host repo: ${host_root}/${kvm_hostname} (name: ${repo_name}) ===" + local remote_cmd + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -ExecutionPolicy Bypass -File \"${install_dir}/setup-veeam-host-repo.ps1\" " \ + "-KvmHostname \"${kvm_hostname}\" " \ + "-HostRepoRoot \"${host_root}\" " \ + "-RepositoryName \"${repo_name}\"") + ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=120 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" \ + || mold_guest_common_die "setup-veeam-host-repo.ps1 failed — create E:\\opt1\\veeam\\${kvm_hostname} in Veeam UI" +} + +# Copy required PS1 from KVM share to Veeam (not installed under /etc/ablestack/veeam). +mold_guest_sync_ps1_to_veeam() { + local env_file="$1" + local veeam_host veeam_user install_dir ps1_dir f local_path + local -a ssh_opts=() scp_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + [[ -n "$veeam_host" ]] || return 0 + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + ps1_dir="" + for f in install-veeam-job.ps1 create-veeam-agent-job.ps1 setup-veeam-mold-job.ps1 \ + setup-veeam-host-repo.ps1; do + local_path="$(mold_guest_resolve_script "$f" 2>/dev/null || true)" + [[ -n "$local_path" ]] && ps1_dir="$(dirname "$local_path")" && break + done + [[ -n "$ps1_dir" ]] || { + echo "WARN: PS1 not found on KVM — run: bash /tmp/veeam-install/install.sh" + return 0 + } + + echo "=== Sync PS1 → ${veeam_user}@${veeam_host}:${install_dir} (from ${ps1_dir}) ===" + for f in install-veeam-job.ps1 create-veeam-agent-job.ps1 setup-veeam-mold-job.ps1 \ + setup-veeam-host-repo.ps1 veeam-job-pre-backup.ps1 veeam-job-post-backup.ps1 \ + veeam-job-post-restore.ps1; do + [[ -f "${ps1_dir}/${f}" ]] || continue + scp "${scp_opts[@]}" "${ssh_opts[@]}" "${ps1_dir}/${f}" \ + "${veeam_user}@${veeam_host}:${install_dir}/${f}" + done +} + +# KVM hypervisor (10.10.31.2) — one Veeam Agent job + Mold_Host_Backup.conf (not per-guest VM). +mold_host_ensure_conf() { + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local host_conf="${etc}/Mold_Host_Backup.conf" + local main_conf="" candidate + for candidate in "${etc}/mold-backup.conf" "${etc}/Mold_Host_Backup.conf" "${etc}/"*.conf; do + [[ -f "$candidate" ]] || continue + main_conf="$candidate" + break + done + [[ -n "$main_conf" ]] || mold_guest_common_die "Missing KVM backup conf under ${etc} — run veeam_config.sh first" + if [[ ! -f "$host_conf" ]]; then + cp -a "$main_conf" "$host_conf" + chmod 0600 "$host_conf" + echo "Created ${host_conf} from $(basename "$main_conf")" + fi + local env_file="${etc}/mold-backup.env" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$host_conf" + [[ -f "$env_file" ]] && mold_guest_sync_api_from_env "$env_file" "$main_conf" + sed -i 's/^BACKUP_MODE=.*/BACKUP_MODE="host"/' "$host_conf" + local pre="${etc}/ablestack_veeam_pre_notify.sh" + local post="${etc}/ablestack_veeam_post_notify.sh" + for key_val in \ + "KVM_PRE_NOTIFY_SCRIPT=\"${pre}\"" \ + "KVM_POST_NOTIFY_SCRIPT=\"${post}\"" \ + 'VEEAM_TRIGGER_ENABLED="false"' \ + 'VEEAM_TRIGGER_METHOD="auto"'; do + key="${key_val%%=*}" + if grep -qE "^${key}=" "$host_conf" 2>/dev/null; then + sed -i "s#^${key}=.*#${key_val}#" "$host_conf" + else + echo "$key_val" >> "$host_conf" + fi + done +} + +# Veeam Agent job on KVM (e.g. 10.10.31.2) → E:\opt1\veeam\\ repository. +mold_host_ensure_kvm_prepost_scripts() { + local etc="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + local pre="${KVM_PRE_NOTIFY_SCRIPT:-${etc}/ablestack_veeam_pre_notify.sh}" + local post="${KVM_POST_NOTIFY_SCRIPT:-${etc}/ablestack_veeam_post_notify.sh}" + local staging="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + local ini="/etc/veeam/veeam.ini" + local timeout="${VEEAM_SCRIPT_TIMEOUT:-1800}" + + [[ -x "$pre" ]] || mold_guest_common_die "Missing pre-notify on KVM: ${pre} — run install.sh" + [[ -x "$post" ]] || mold_guest_common_die "Missing post-notify on KVM: ${post} — run install.sh" + mkdir -p "$staging" + chmod 0755 "$staging" 2>/dev/null || true + + if [[ -f "$ini" ]]; then + if grep -q '^\[scripts\]' "$ini" 2>/dev/null; then + if grep -q '^timeoutPrePost' "$ini" 2>/dev/null; then + sed -i "s/^timeoutPrePost.*/timeoutPrePost = ${timeout}/" "$ini" + else + sed -i "/^\[scripts\]/a timeoutPrePost = ${timeout}" "$ini" + fi + else + printf '\n[scripts]\ntimeoutPrePost = %s\n' "$timeout" >>"$ini" + fi + if command -v systemctl >/dev/null 2>&1 && systemctl is-active veeamservice >/dev/null 2>&1; then + systemctl restart veeamservice 2>/dev/null || true + echo "Veeam Agent: ${ini} timeoutPrePost=${timeout}s (veeamservice restarted)" + else + echo "Veeam Agent: set ${ini} [scripts] timeoutPrePost = ${timeout}" + fi + else + echo "WARN: ${ini} not found — install Veeam Agent for Linux on this KVM host" + fi + echo "KVM pre/post ready: ${pre} | ${post} | staging=${staging}" +} + +# Register Guest Processing pre-freeze/post-thaw on the KVM Linux Agent job (runs ON 10.10.31.2). +mold_host_register_veeam_scripts() { + local env_file="$1" kvm_hostname="${2:-}" kvm_ip="${3:-}" + local veeam_host veeam_user install_dir job_name pg_name pre post + local -a ssh_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" ]] || mold_guest_common_die "VEEAM_SSH_HOST not set in ${env_file}" + [[ -n "$kvm_hostname" ]] || kvm_hostname="${KVM_HOSTNAME:-$(hostname -s)}" + [[ -n "$kvm_ip" ]] || kvm_ip="$(mold_guest_resolve_kvm_ip_from_env "$env_file")" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + job_name="${VEEAM_JOB_NAME:-Mold ${kvm_hostname}}" + pg_name="${VEEAM_PROTECTION_GROUP_NAME:-Mold KVM Agents}" + pre="${KVM_PRE_NOTIFY_SCRIPT:-/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh}" + post="${KVM_POST_NOTIFY_SCRIPT:-/etc/ablestack/veeam/ablestack_veeam_post_notify.sh}" + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + mold_guest_sync_ps1_to_veeam "$env_file" + + if ! mold_guest_veeam_job_exists "$env_file" "$job_name"; then + echo "WARN: Veeam job '${job_name}' not found — run mold_host_setup_veeam_job first" + return 1 + fi + + echo "=== Register pre/post on Veeam host job '${job_name}' (Linux Agent ${kvm_ip}) ===" + local remote_ps1="${install_dir}/install-veeam-job.ps1" + local remote_cmd + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -ExecutionPolicy Bypass -File \"${remote_ps1}\" " \ + "-JobName \"${job_name}\" " \ + "-LinuxAgent " \ + "-KvmHost \"${kvm_ip}\" " \ + "-ProtectionGroupName \"${pg_name}\" " \ + "-AgentPreNotifyScript \"${pre}\" " \ + "-AgentPostNotifyScript \"${post}\" " \ + "-InstallDir \"${install_dir}\" " \ + "-SourceDir \"${install_dir}\"") + ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=120 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" \ + || mold_guest_common_die "install-veeam-job.ps1 -LinuxAgent failed for '${job_name}'" + + echo "Registered Guest Processing scripts on '${job_name}':" + echo " Pre-freeze → ${pre} (Mold export → ${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam})" + echo " Post-thaw → ${post} (registry + staging cleanup)" +} + +mold_host_setup_veeam_job() { + local env_file="$1" kvm_hostname="${2:-}" kvm_ip="${3:-}" + local veeam_host veeam_user install_dir job_name repo_name backup_path pg_name + local -a ssh_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" ]] || mold_guest_common_die "VEEAM_SSH_HOST not set in ${env_file}" + [[ -n "$kvm_hostname" ]] || kvm_hostname="${KVM_HOSTNAME:-$(hostname -s)}" + [[ -n "$kvm_ip" ]] || kvm_ip="$(mold_guest_resolve_kvm_ip_from_env "$env_file")" + install_dir="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" + job_name="${VEEAM_JOB_NAME:-Mold ${kvm_hostname}}" + repo_name="${VEEAM_REPO_NAME:-Mold ${kvm_hostname}}" + backup_path="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + pg_name="${VEEAM_PROTECTION_GROUP_NAME:-Mold KVM Agents}" + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + [[ -n "${KVM_SSH_PASSWORD:-}" ]] || mold_guest_common_die "KVM_SSH_PASSWORD required in ${env_file} (Veeam: Cannot find credentials for agent)" + + mold_guest_sync_ps1_to_veeam "$env_file" + mold_guest_sync_windows_conf_to_veeam "$env_file" + mold_guest_setup_veeam_host_repo "$env_file" "$kvm_hostname" 2>/dev/null \ + || echo "WARN: Veeam host repo setup skipped" + + echo "=== Veeam KVM host job: ${job_name} (agent ${kvm_ip} / ${kvm_hostname}) ===" + local remote_cmd win_conf="${install_dir}/mold-backup.windows.conf" + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -ExecutionPolicy Bypass -File \"${install_dir}/setup-veeam-mold-job.ps1\" " \ + "-ConfPath \"${win_conf}\" " \ + "-JobName \"${job_name}\" " \ + "-KvmHost \"${kvm_ip}\" " \ + "-AgentHostName \"${kvm_hostname}\" " \ + "-BackupPath \"${backup_path}\" " \ + "-RepositoryName \"${repo_name}\" " \ + "-ProtectionGroupName \"${pg_name}\"") + ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" \ + || mold_guest_common_die "setup-veeam-mold-job.ps1 failed for KVM host ${kvm_ip}" + + mold_host_register_veeam_scripts "$env_file" "$kvm_hostname" "$kvm_ip" \ + || echo "WARN: pre/post registration skipped — re-run: mold_host_register_veeam_scripts" +} + +mold_guest_veeam_job_exists() { + local env_file="$1" job_name="$2" + local veeam_host veeam_user install_dir remote_cmd out + local -a ssh_opts=() + + # shellcheck source=/dev/null + set -a && source "$env_file" && set +a + veeam_host="${VEEAM_SSH_HOST:-}" + veeam_user="${VEEAM_SSH_USER:-administrator}" + [[ -n "$veeam_host" && -n "$job_name" ]] || return 1 + [[ -n "${VEEAM_SSH_KEY:-}" && -f "${VEEAM_SSH_KEY}" ]] && ssh_opts=(-i "${VEEAM_SSH_KEY}") + + remote_cmd=$(printf '%s' \ + "pwsh -NoProfile -Command \"Import-Module Veeam.Backup.PowerShell -ErrorAction SilentlyContinue; " \ + "if (Get-VBRComputerBackupJob -Name '${job_name//\'/\'\'\'}' -ErrorAction SilentlyContinue) { Write-Output 'true' } else { Write-Output 'false' }\"") + + out="$(ssh "${ssh_opts[@]}" -o BatchMode=yes -o ConnectTimeout=60 -o StrictHostKeyChecking=accept-new \ + "${veeam_user}@${veeam_host}" "$remote_cmd" 2>/dev/null | tr -d '[:space:]')" + [[ "$out" == "true" ]] +} + +# Create one guest Agent job on Veeam (+ register Guest Processing via install-veeam-job.ps1). +mold_guest_create_veeam_job_for_vm() { + mold_guest_common_die "Guest-mode Veeam jobs were removed. Use mold_host_setup_veeam_job / setup-datadisk-veeam-backup.sh." +} + +# Register Pre-job/Post-job on an EXISTING Veeam Agent job (UI-created job). +mold_guest_register_veeam_job_scripts() { + mold_guest_common_die "Guest-mode Veeam jobs were removed. Use mold_host_register_veeam_scripts / setup-datadisk-veeam-backup.sh." +} diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service new file mode 100644 index 000000000000..66ed6c3045f2 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.service @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Polls Veeam for completed restore sessions and triggers Mold datadisk restore on the +# KVM host that owns the VM. Uses the Veeam UI restore point id to select the matching +# Mold backup_id (not always the latest). +[Unit] +Description=Mold Veeam restore agent (detect Veeam restore, trigger Mold restore) +After=network-online.target cloudstack-agent.service +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=-/etc/ablestack/veeam/mold-backup.env +Environment="MOLD_BACKUP_CONF=/etc/ablestack/veeam/Mold_Host_Backup.conf" +ExecStart=/etc/ablestack/veeam/mold-veeam-restore-agent.sh + +[Install] +WantedBy=multi-user.target diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh new file mode 100644 index 000000000000..d40373d35131 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash +# Poll Veeam for completed restore sessions and trigger Mold restoreBackup on this KVM host. +# Matches the Veeam UI restore point (FLR) to the corresponding Mold backup_id. +# Invoked by mold-veeam-restore-agent.timer (every 3 min). +# +# Flow: Veeam UI Guest files restore (FLR) on host job → this agent → Mold datadisk restore +# +# Manual: +# bash /etc/ablestack/veeam/mold-veeam-restore-agent.sh +# bash /etc/ablestack/veeam/mold-backup.sh restore-watch --job 'Mold ablecube31-2' --trigger-mold --since-min 30 + +set -euo pipefail + +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +ENV_FILE="${ETC_DIR}/mold-backup.env" +HOST_CONF="${MOLD_BACKUP_CONF:-${ETC_DIR}/Mold_Host_Backup.conf}" +SINCE_MIN="${VEEAM_RESTORE_WATCH_WINDOW_MIN:-10}" + +[[ -f "$ENV_FILE" ]] && { set -a; # shellcheck source=/dev/null + source "$ENV_FILE"; set +a; } + +export MOLD_BACKUP_CONF="$HOST_CONF" +mkdir -p "${ETC_DIR}/events" "${ETC_DIR}/registry" "${ETC_DIR}/state" 2>/dev/null || true + +job="" +if [[ -f "$HOST_CONF" ]]; then + job="$(grep -E '^VEEAM_JOB_NAME=' "$HOST_CONF" 2>/dev/null | head -1 | cut -d= -f2- | sed -e 's/^"//' -e 's/"$//' -e "s/^'//" -e "s/'$//")" +fi +job="${job:-${VEEAM_JOB_NAME:-Mold ablecube31-2}}" + +exec "${ETC_DIR}/mold-backup.sh" restore-watch \ + --job "$job" \ + --since-min "${SINCE_MIN}" \ + --trigger-mold diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer new file mode 100644 index 000000000000..cbfce699848a --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-restore-agent.timer @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[Unit] +Description=Poll Veeam restores and trigger Mold datadisk restore every 3 minutes + +[Timer] +OnBootSec=1min +OnUnitActiveSec=3min +AccuracySec=30s +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh new file mode 100755 index 000000000000..4a74d4d99497 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/mold-veeam-trigger-hook.sh @@ -0,0 +1,63 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Mold -> Veeam trigger (bidirectional mode C). +# Invoked best-effort by ablestack_nasbackup.sh after a VM backup completes. +# Starts the matching Veeam Agent job over SSH unless the current backup was +# itself triggered by Veeam (veeam-active marker present). +# +# Args: [backup-type] +# Never fail the caller: always exit 0. + +OP="${1:-}" +VM="${2:-}" +BACKUP_TYPE="${3:-}" + +# Only react to the actual VM backup operation. +case "$OP" in + backup-running|backup-rbd) ;; + *) exit 0 ;; +esac +[[ -n "$VM" ]] || exit 0 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIB="${SCRIPT_DIR}/mold-backup.lib.sh" +[[ -f "$LIB" ]] || LIB="/etc/ablestack/veeam/mold-backup.lib.sh" +[[ -f "$LIB" ]] || exit 0 + +# shellcheck source=/dev/null +source "$LIB" +# Guest VMs use Mold_Guest_Backup.conf (VEEAM_TRIGGER_*, VM_TARGETS, VEEAM_SSH_*). +export MOLD_BACKUP_CONF="${MOLD_BACKUP_CONF:-/etc/ablestack/veeam/Mold_Guest_Backup.conf}" +mold_backup_load_config || exit 0 + +[[ "${VEEAM_TRIGGER_ENABLED:-false}" == "true" ]] || { + mold_backup_notify_log info "Mold→Veeam trigger disabled (set VEEAM_TRIGGER_ENABLED=true in Mold_Guest_Backup.conf)" + exit 0 +} + +# If Veeam started this backup (veeam-active marker), do NOT start Veeam again. +if mold_backup_trigger_active "veeam-active" "$VM"; then + mold_backup_notify_log info "veeam-active present for ${VM}: Veeam-driven backup, skip Mold→Veeam trigger" + exit 0 +fi + +mold_backup_notify_log info "Mold→Veeam trigger hook: vm=${VM} op=${OP}" +mold_backup_trigger_veeam_job "$VM" || mold_backup_notify_log warn "Mold→Veeam trigger failed for ${VM} (see log above)" + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh b/scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh new file mode 100755 index 000000000000..4e56035c770e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh @@ -0,0 +1,117 @@ +#!/usr/bin/bash +# Copy veeam scripts to KVM and run install.sh (+ optional veeam_config). +# Run from Git repo on Mac or any host with SSH to KVM. +# +# cp mold-backup.env.example mold-backup.env # edit API keys +# bash scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh --env-file mold-backup.env +# +# Or: +# KVM_HOST=root@10.10.31.2 bash scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh +# bash scripts/vm/hypervisor/kvm/veeam/push-to-kvm.sh --host root@10.10.31.2 --no-configure + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" +RUN_CONFIGURE="${RUN_CONFIGURE:-true}" + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) + [[ -f "$2" ]] || die "not found: $2" + # shellcheck source=/dev/null + set -a && source "$2" && set +a + shift 2 + ;; + --host|--kvm-host) + [[ -n "${2:-}" ]] || die "$1 requires a value (e.g. root@10.10.31.2)" + KVM_HOST="$2" + [[ "$KVM_HOST" == *@* ]] || KVM_HOST="root@${KVM_HOST}" + shift 2 + ;; + --jump|--jump-host) + [[ -n "${2:-}" ]] || die "$1 requires a value (e.g. root@10.10.31.20)" + JUMP_HOST="$2" + shift 2 + ;; + --no-configure) RUN_CONFIGURE=false; shift ;; + -h|--help) + echo "Usage: push-to-kvm.sh [--host root@KVM] [--jump root@JUMP] [--env-file mold-backup.env] [--no-configure]" + echo "" + echo "Examples:" + echo " bash push-to-kvm.sh --host root@10.10.31.2 --no-configure" + echo " KVM_HOST=root@10.10.31.2 bash push-to-kvm.sh" + echo " bash push-to-kvm.sh --env-file mold-backup.env" + exit 0 + ;; + *) die "Unknown: $1 (try --help)" ;; + esac +done + +KVM_HOST="${KVM_HOST:-}" +[[ -n "$KVM_HOST" ]] || die "Set KVM_HOST=root@10.10.31.2 (or use --env-file)" + +# SSH via jump host (e.g. Mac → ccvm → KVM): JUMP_HOST=root@10.10.31.20 +SSH_OPTS=() +SCP_OPTS=() +if [[ -n "${JUMP_HOST:-}" ]]; then + SSH_OPTS=(-o "ProxyJump=${JUMP_HOST}") + SCP_OPTS=(-o "ProxyJump=${JUMP_HOST}") + echo "Using jump host: ${JUMP_HOST}" +fi + +kvm_ssh() { + if ((${#SSH_OPTS[@]} > 0)); then + ssh "${SSH_OPTS[@]}" "$@" + else + ssh "$@" + fi +} + +kvm_scp() { + if ((${#SCP_OPTS[@]} > 0)); then + scp "${SCP_OPTS[@]}" "$@" + else + scp "$@" + fi +} + +echo "=== SCP veeam scripts → ${KVM_HOST}:/tmp/veeam-install/ ===" +kvm_ssh "$KVM_HOST" "mkdir -p /tmp/veeam-install" +# Include parent ablestack_nasbackup.sh (agent restore path) +NAS_PARENT="${SCRIPT_DIR}/../ablestack_nasbackup.sh" +kvm_scp -r "${SCRIPT_DIR}/"* "${KVM_HOST}:/tmp/veeam-install/" +[[ -f "$NAS_PARENT" ]] && kvm_scp "$NAS_PARENT" "${KVM_HOST}:/tmp/veeam-install/ablestack_nasbackup.sh" + +echo "=== install.sh on KVM ===" +kvm_ssh "$KVM_HOST" "bash /tmp/veeam-install/install.sh" + +if [[ "$RUN_CONFIGURE" == "true" ]]; then + JOB_NAME="${JOB_NAME:-Mold KVM Backup}" + CONFIG_ARGS=(--job-name "${JOB_NAME}" --install) + [[ -n "${KVM_IP:-}" ]] && CONFIG_ARGS+=(--kvm-host "${KVM_IP}") + if [[ -n "${MOLD_API_URL:-}" && -n "${MOLD_API_KEY:-}" && -n "${MOLD_API_SECRET:-}" ]]; then + CONFIG_ARGS+=( + --mold-url "${MOLD_API_URL}" + --api-key "${MOLD_API_KEY}" + --api-secret "${MOLD_API_SECRET}" + ) + [[ -n "${ZONE_ID:-}" ]] && CONFIG_ARGS+=(--zone-id "${ZONE_ID}") + [[ -n "${VM_INCLUDE:-}" ]] && CONFIG_ARGS+=(--vm-include "${VM_INCLUDE}") + [[ -n "${VM_NAME:-}" ]] && CONFIG_ARGS+=(--vm-name "${VM_NAME}") + [[ -n "${VM_UUID:-}" ]] && CONFIG_ARGS+=(--vm-uuid "${VM_UUID}") + [[ -n "${BACKUP_REPO_ADDRESS:-}" ]] && CONFIG_ARGS+=(--nas-repo "${BACKUP_REPO_ADDRESS}") + [[ -n "${VEEAM_URL:-}" ]] && CONFIG_ARGS+=(--veeam-url "${VEEAM_URL}") + else + echo "=== veeam_config.sh on KVM (reads /etc/ablestack/veeam/mold-backup.env) ===" + fi + echo "=== veeam_config.sh on KVM ===" + kvm_ssh "$KVM_HOST" "cd /etc/ablestack/veeam && ./veeam_config.sh $(printf '%q ' "${CONFIG_ARGS[@]}")" +fi + +echo "" +echo "=== Done. On KVM run: ===" +echo " /etc/ablestack/veeam/mold-backup.sh backup-full --job \"${JOB_NAME:-Mold KVM Backup}\"" +echo " /etc/ablestack/veeam/mold-backup.sh status --job \"${JOB_NAME:-Mold KVM Backup}\"" diff --git a/scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh b/scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh new file mode 100755 index 000000000000..6a86480d9935 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/push-to-veeam.sh @@ -0,0 +1,271 @@ +#!/usr/bin/bash +# Deploy Mold Veeam scripts to B&R server and create Agent backup job (idempotent). +# +# cp mold-backup.env.example mold-backup.env # edit VEEAM_SSH_HOST, KVM_IP, JOB_NAME +# bash push-to-veeam.sh --env-file mold-backup.env +# +# Requires: SSH from this host to Veeam Windows server (OpenSSH on Windows). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" + +mold_push_resolve_ps1_dir() { + local candidate + for candidate in "$SCRIPT_DIR" "$SHARE_DIR" "$ETC_DIR"; do + [[ -f "${candidate}/setup-veeam-mold-job.ps1" ]] && { echo "$candidate"; return 0; } + done + echo "$SCRIPT_DIR" +} + +mold_push_sync_windows_conf() { + local env_file="${1:-${ETC_DIR}/mold-backup.env}" + local win_conf="${ETC_DIR}/mold-backup.windows.conf" + [[ -f "$env_file" ]] || return 0 + [[ -f "$win_conf" ]] || return 0 + + local pw user targets kvm_host kvm_pw kvm_user + pw="$(grep -E '^GUEST_VM_SSH_PASSWORD=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + user="$(grep -E '^GUEST_VM_SSH_USER=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + kvm_pw="$(grep -E '^KVM_SSH_PASSWORD=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + kvm_user="$(grep -E '^KVM_SSH_USER=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + targets="$(grep -E '^VM_TARGETS=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + kvm_host="$(grep -E '^KVM_HOST=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + [[ -z "$kvm_host" ]] && kvm_host="$(grep -E '^KVM_IP=' "$env_file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"')" || true + + if [[ -n "$pw" ]]; then + if grep -qE '^GUEST_VM_SSH_PASSWORD=' "$win_conf" 2>/dev/null; then + sed -i "s#^GUEST_VM_SSH_PASSWORD=.*#GUEST_VM_SSH_PASSWORD=\"${pw}\"#" "$win_conf" + else + echo "GUEST_VM_SSH_PASSWORD=\"${pw}\"" >>"$win_conf" + fi + fi + if [[ -n "$user" ]]; then + if grep -qE '^GUEST_VM_SSH_USER=' "$win_conf" 2>/dev/null; then + sed -i "s#^GUEST_VM_SSH_USER=.*#GUEST_VM_SSH_USER=\"${user}\"#" "$win_conf" + else + echo "GUEST_VM_SSH_USER=\"${user}\"" >>"$win_conf" + fi + fi + if [[ -n "$targets" ]]; then + if grep -qE '^VM_TARGETS=' "$win_conf" 2>/dev/null; then + sed -i "s#^VM_TARGETS=.*#VM_TARGETS=\"${targets}\"#" "$win_conf" + else + echo "VM_TARGETS=\"${targets}\"" >>"$win_conf" + fi + fi + if [[ -n "$kvm_host" ]]; then + if grep -qE '^KVM_HOST=' "$win_conf" 2>/dev/null; then + sed -i "s#^KVM_HOST=.*#KVM_HOST=\"${kvm_host}\"#" "$win_conf" + else + echo "KVM_HOST=\"${kvm_host}\"" >>"$win_conf" + fi + fi + if [[ -n "$kvm_pw" ]]; then + if grep -qE '^KVM_SSH_PASSWORD=' "$win_conf" 2>/dev/null; then + sed -i "s#^KVM_SSH_PASSWORD=.*#KVM_SSH_PASSWORD=\"${kvm_pw}\"#" "$win_conf" + else + echo "KVM_SSH_PASSWORD=\"${kvm_pw}\"" >>"$win_conf" + fi + fi + if [[ -n "$kvm_user" ]]; then + if grep -qE '^KVM_SSH_USER=' "$win_conf" 2>/dev/null; then + sed -i "s#^KVM_SSH_USER=.*#KVM_SSH_USER=\"${kvm_user}\"#" "$win_conf" + else + echo "KVM_SSH_USER=\"${kvm_user}\"" >>"$win_conf" + fi + fi +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) + [[ -f "$2" ]] || die "not found: $2" + ENV_FILE="$2" + # shellcheck source=/dev/null + set -a && source "$2" && set +a + shift 2 + ;; + --vm-targets) + die "Guest-mode VM_TARGETS removed. Use host/datadisk mode only." + ;; + --skip-create) SKIP_CREATE=true; shift ;; + --no-start) SKIP_START=true; shift ;; + --skip-guest-onboard) shift ;; # ignored (compat) + -h|--help) + echo "Usage: push-to-veeam.sh [--env-file mold-backup.env] [--skip-create] [--no-start]" + echo " Deploys host-mode PS1 to Veeam and runs setup-veeam-mold-job.ps1" + exit 0 + ;; + *) die "Unknown: $1" ;; + esac +done + +SKIP_CREATE="${SKIP_CREATE:-false}" +SKIP_START="${SKIP_START:-false}" +VEEAM_SSH_HOST="${VEEAM_SSH_HOST:-}" +VEEAM_SSH_USER="${VEEAM_SSH_USER:-administrator}" +VEEAM_SSH_KEY="${VEEAM_SSH_KEY:-}" +VEEAM_INSTALL_DIR="${VEEAM_INSTALL_DIR:-C:/ProgramData/Mold/backup/veeam}" +JOB_NAME="${JOB_NAME:-${VEEAM_JOB_NAME:-Mold KVM Backup}}" +KVM_IP="${KVM_IP:-10.10.31.2}" +KVM_HOSTNAME="${KVM_HOSTNAME:-}" +BACKUP_PATH="${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" +VEEAM_REPO_NAME="${VEEAM_REPO_NAME:-}" + +[[ -n "$VEEAM_SSH_HOST" ]] || die "Set VEEAM_SSH_HOST=10.10.254.246 in mold-backup.env" + +VEEAM_SSH="${VEEAM_SSH_USER}@${VEEAM_SSH_HOST}" +SSH_OPTS=() +[[ -n "$VEEAM_SSH_KEY" && -f "$VEEAM_SSH_KEY" ]] && SSH_OPTS=(-i "$VEEAM_SSH_KEY") + +veeam_ssh() { + if ((${#SSH_OPTS[@]} > 0)); then + ssh "${SSH_OPTS[@]}" "$@" + else + ssh "$@" + fi +} + +veeam_scp() { + if ((${#SSH_OPTS[@]} > 0)); then + scp "${SSH_OPTS[@]}" "$@" + else + scp "$@" + fi +} + +if [[ "${VEEAM_BACKUP_TARGET:-}" == "guest" || -n "${VM_TARGETS:-}" ]]; then + die "Guest-mode Veeam jobs were removed. Use host/datadisk mode (VEEAM_BACKUP_TARGET=host, unset VM_TARGETS)." +fi + +VEEAM_FILES=( + setup-veeam-mold-job.ps1 + create-veeam-agent-job.ps1 + install-veeam-job.ps1 + veeam-job-pre-backup.ps1 + veeam-job-post-backup.ps1 + veeam-job-post-restore.ps1 + setup-veeam-host-repo.ps1 + mold-backup.windows.conf.default +) + +VEEAM_FILES_REQUIRED=( + setup-veeam-mold-job.ps1 + create-veeam-agent-job.ps1 + install-veeam-job.ps1 + mold-backup.windows.conf.default +) + +PS1_DIR="$(mold_push_resolve_ps1_dir)" +if ! grep -q 'SelectedFiles' "${PS1_DIR}/install-veeam-job.ps1" 2>/dev/null; then + die "Outdated PS1 in ${PS1_DIR} (missing SelectedFiles support). Run: bash install.sh from updated veeam/ package, or MOLD_BACKUP_SHARE_DIR=/path/to/repo/veeam bash push-to-veeam.sh ..." +fi +mold_push_sync_windows_conf "${ENV_FILE:-${ETC_DIR}/mold-backup.env}" + +echo "=== Prepare Veeam install dir: ${VEEAM_INSTALL_DIR} ===" +veeam_ssh "$VEEAM_SSH" "powershell -Command \"New-Item -ItemType Directory -Force -Path '${VEEAM_INSTALL_DIR}' | Out-Null\"" \ + || die "Cannot SSH to Veeam server ${VEEAM_SSH}" + +echo "=== SCP scripts → ${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/ (from ${PS1_DIR}) ===" +for f in "${VEEAM_FILES_REQUIRED[@]}"; do + [[ -f "${PS1_DIR}/${f}" ]] || die "Missing required ${PS1_DIR}/${f} — run install.sh or set MOLD_BACKUP_SHARE_DIR" +done +for f in "${VEEAM_FILES[@]}"; do + [[ -f "${PS1_DIR}/${f}" ]] || continue + veeam_scp "${PS1_DIR}/${f}" "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/${f}" +done + +# Push KVM-generated windows conf (local file or remote KVM_HOST) +if [[ -f "${ETC_DIR}/mold-backup.windows.conf" ]]; then + echo "=== Copy mold-backup.windows.conf from ${ETC_DIR} ===" + veeam_scp "${ETC_DIR}/mold-backup.windows.conf" \ + "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/mold-backup.windows.conf" +else + KVM_SSH="${KVM_HOST:-}" + if [[ -n "$KVM_SSH" && "$KVM_SSH" != *@* ]]; then + KVM_SSH="root@${KVM_SSH}" + fi + if [[ -n "$KVM_SSH" ]]; then + echo "=== Try copy mold-backup.windows.conf from KVM ===" + veeam_scp "${KVM_SSH}:/etc/ablestack/veeam/mold-backup.windows.conf" \ + "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/mold-backup.windows.conf" 2>/dev/null \ + || echo "NOTE: mold-backup.windows.conf not copied from KVM (run veeam_config.sh on KVM first)" + fi +fi + +if [[ "$SKIP_CREATE" == "true" ]]; then + echo "=== Skip job create (--skip-create) ===" + exit 0 +fi + +mold_push_ps_escape() { + # PowerShell single-quoted string: ' -> '' + printf '%s' "$1" | sed "s/'/''/g" +} + +mold_push_write_remote_runner() { + local setup_script="$1" + shift + local runner_local + runner_local="$(mktemp "${TMPDIR:-/tmp}/mold-push-remote-run.XXXXXX.ps1")" + { + printf '%s\n' '$ErrorActionPreference = "Stop"' + printf '& %s' "'$(mold_push_ps_escape "$setup_script")'" + while [[ $# -gt 0 ]]; do + local flag="$1" val="${2:-}" + if [[ "$flag" == -* && -n "$val" && "$val" != -* ]]; then + printf ' %s %s' "$flag" "'$(mold_push_ps_escape "$val")'" + shift 2 + else + printf ' %s' "$flag" + shift + fi + done + printf '\n' + } >"$runner_local" + veeam_scp "$runner_local" "${VEEAM_SSH}:${VEEAM_INSTALL_DIR}/mold-push-remote-run.ps1" + rm -f "$runner_local" +} + +mold_push_run_remote_setup() { + local setup_script="${VEEAM_INSTALL_DIR}/setup-veeam-mold-job.ps1" + local setup_name="setup-veeam-mold-job.ps1" + local runner_path="${VEEAM_INSTALL_DIR}/mold-push-remote-run.ps1" + local -a ps_args=( + -InstallDir "${VEEAM_INSTALL_DIR}" + -ConfPath "${VEEAM_INSTALL_DIR}/mold-backup.windows.conf" + ) + + echo "=== setup-veeam-mold-job.ps1 (file-level /tmp/mold/veeam + Pre/Post) ===" + [[ -n "$KVM_HOSTNAME" ]] && ps_args+=(-AgentHostName "$KVM_HOSTNAME") + [[ -n "$VEEAM_REPO_NAME" ]] && ps_args+=(-RepositoryName "$VEEAM_REPO_NAME") + if [[ "${VEEAM_START_JOBS:-true}" != "false" && "$SKIP_START" != "true" ]]; then + ps_args+=(-StartJob) + fi + + echo "=== Upload remote runner for ${setup_name} ===" + mold_push_write_remote_runner "$setup_script" "${ps_args[@]}" + + # No quotes around -File path: OpenSSH on Windows breaks ''path'' with nested quoting. + echo "=== Remote: ${setup_name} on ${VEEAM_SSH} ===" + if veeam_ssh "$VEEAM_SSH" pwsh -NoProfile -ExecutionPolicy Bypass -File "${runner_path}"; then + return 0 + fi + if veeam_ssh "$VEEAM_SSH" powershell -NoProfile -ExecutionPolicy Bypass -File "${runner_path}"; then + return 0 + fi + die "Veeam job setup failed — check Agent on KVM and mold-backup.windows.conf" +} + +mold_push_run_remote_setup + +echo "" +echo "=== Done ===" +echo "Veeam UI: jobs started automatically (set VEEAM_START_JOBS=false or --no-start to skip)" +echo "FLR→Mold: on KVM run enable-veeam-mold-restore.sh (mold-veeam-restore-agent.timer)" +echo "KVM log: ssh ${KVM_SSH:-root@${KVM_IP}} tail -f /var/log/mold/veeam-hook.log" diff --git a/scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh b/scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh new file mode 100755 index 000000000000..935ff110b449 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/setup-datadisk-veeam-backup.sh @@ -0,0 +1,174 @@ +#!/usr/bin/bash +# Mold datadisk (KVM) + Veeam host repo (E:\opt1\veeam\) setup. +# +# Architecture (KVM hypervisor unit, e.g. 10.10.31.2 ablecube31-2): +# - Mold qcow2 on KVM data disk (BACKUP_REPO_TYPE=local, no NAS) +# - One Veeam Linux Agent job ON the KVM host → E:\opt1\veeam\\ +# - BACKUP_MODE=host: pre/post on KVM export VM disks; NOT per-guest Veeam jobs +# - Veeam UI Guest files restore (FLR) → KVM restore agent → Mold restoreBackup +# +# Run on KVM host: +# bash setup-datadisk-veeam-backup.sh --env-file /etc/ablestack/veeam/mold-backup.env +# bash setup-datadisk-veeam-backup.sh --datadisk-path /data/backup --kvm-hostname ablecube31-2 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +ENV_FILE="${ETC_DIR}/mold-backup.env" +DATADISK_PATH="" +KVM_HOSTNAME_OVERRIDE="" +SKIP_VEEAM_REPO=false +REGISTER_SCRIPTS_ONLY=false + +die() { echo "ERROR: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) ENV_FILE="$2"; shift 2 ;; + --datadisk-path) DATADISK_PATH="$2"; shift 2 ;; + --kvm-hostname) KVM_HOSTNAME_OVERRIDE="$2"; shift 2 ;; + --skip-veeam-repo) SKIP_VEEAM_REPO=true; shift ;; + --register-scripts-only) REGISTER_SCRIPTS_ONLY=true; shift ;; + -h|--help) + sed -n '2,18p' "$0" + exit 0 + ;; + *) die "Unknown: $1" ;; + esac +done + +[[ -f "$ENV_FILE" ]] || die "Missing $ENV_FILE" +# shellcheck source=/dev/null +set -a && source "$ENV_FILE" && set +a + +COMMON_LIB="${SCRIPT_DIR}/mold-guest-common.sh" +[[ -f "$COMMON_LIB" ]] || COMMON_LIB="${ETC_DIR}/mold-guest-common.sh" +[[ -f "$COMMON_LIB" ]] || die "mold-guest-common.sh not found" +MOLD_GUEST_SCRIPT_DIR="$SCRIPT_DIR" +# shellcheck source=mold-guest-common.sh +source "$COMMON_LIB" + +# shellcheck source=mold-backup.lib.sh +source "${SCRIPT_DIR}/mold-backup.lib.sh" + +host="${KVM_HOSTNAME_OVERRIDE:-${KVM_HOSTNAME:-$(hostname -s)}}" +kvm_ip="$(mold_guest_resolve_kvm_ip_from_env "$ENV_FILE" 2>/dev/null || hostname -I 2>/dev/null | awk '{print $1}')" +disk="${DATADISK_PATH:-${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS:-/data/backup}}}" +# Legacy GFS bind mount — datadisk mode uses KVM data disk, not glue-gfs NAS path. +if [[ -z "$DATADISK_PATH" && "$disk" == *glue-gfs* ]]; then + if [[ -d /data/backup ]]; then + echo "WARN: GFS path ${disk} → /data/backup (datadisk mode; no NAS restore)" + disk="/data/backup" + else + echo "WARN: BACKUP_REPO still on glue-gfs (${disk}). Create /data/backup and re-run:" + echo " $0 --datadisk-path /data/backup --env-file ${ENV_FILE}" + fi +fi +veeam_root="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" +repo_name="${VEEAM_REPO_NAME:-Mold ${host}}" +job_name="${VEEAM_JOB_NAME:-Mold ${host}}" +host_conf="${ETC_DIR}/Mold_Host_Backup.conf" + +echo "=== Mold datadisk + Veeam host repo setup ===" +echo " KVM host : ${host} (${kvm_ip})" +echo " Veeam job : ${job_name}" +echo " Datadisk path: ${disk}" +echo " Veeam repo : ${veeam_root}/${host} (name: ${repo_name})" + +mkdir -p "$disk" +chmod 0755 "$disk" 2>/dev/null || true + +mold_host_ensure_conf + +set_kv() { + local k="$1" v="$2" f="$host_conf" + if grep -q "^${k}=" "$f" 2>/dev/null; then + sed -i "s|^${k}=.*|${k}=\"${v}\"|" "$f" + else + echo "${k}=\"${v}\"" >>"$f" + fi +} + +for f in "$host_conf" "${ETC_DIR}/mold-backup.conf"; do + [[ -f "$f" ]] || continue + host_conf="$f" + set_kv BACKUP_STORAGE_MODE datadisk + set_kv BACKUP_REPO_TYPE local + set_kv BACKUP_REPO_PROVIDER localfs + set_kv BACKUP_REPO_NAME "Ablestack Data Disk" + set_kv MOLD_DATADISK_PATH "$disk" + set_kv BACKUP_REPO_ADDRESS "$disk" + set_kv NAS_REPO_MOUNT "" + set_kv BACKUP_MODE host + set_kv KVM_HOSTNAME "$host" + set_kv KVM_HOST "$kvm_ip" + set_kv VEEAM_JOB_NAME "$job_name" + set_kv VEEAM_HOST_BACKUP_PATH "${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}" + set_kv KVM_PRE_NOTIFY_SCRIPT "${ETC_DIR}/ablestack_veeam_pre_notify.sh" + set_kv KVM_POST_NOTIFY_SCRIPT "${ETC_DIR}/ablestack_veeam_post_notify.sh" + set_kv VEEAM_HOST_REPO_ROOT "$veeam_root" + set_kv VEEAM_REPO_NAME "$repo_name" + set_kv RESTORE_SOURCE mold-only + set_kv VEEAM_UI_RESTORE_SOURCE mold-only + set_kv RESTORE_WATCH_TRIGGER_MOLD true + set_kv BACKUP_STORAGE_ENGINE "${BACKUP_STORAGE_ENGINE:-auto}" +done + +# Keep mold-backup.env aligned (many scripts source env before job conf). +env_sync_kv() { + local k="$1" v="$2" + if grep -q "^${k}=" "$ENV_FILE" 2>/dev/null; then + sed -i "s|^${k}=.*|${k}=\"${v}\"|" "$ENV_FILE" + else + echo "${k}=\"${v}\"" >>"$ENV_FILE" + fi +} +env_sync_kv BACKUP_STORAGE_MODE datadisk +env_sync_kv BACKUP_REPO_TYPE local +env_sync_kv BACKUP_REPO_PROVIDER localfs +env_sync_kv BACKUP_REPO_NAME "Ablestack Data Disk" +env_sync_kv MOLD_DATADISK_PATH "$disk" +env_sync_kv BACKUP_REPO_ADDRESS "$disk" +env_sync_kv BACKUP_MODE host +env_sync_kv RESTORE_SOURCE mold-only +echo "Synced ${ENV_FILE}: BACKUP_REPO_ADDRESS=${disk} (was glue-gfs if unset above)" + +grep -E 'BACKUP_MODE|BACKUP_STORAGE|DATADISK|BACKUP_REPO|VEEAM_JOB|VEEAM_HOST|KVM_HOST|KVM_PRE|KVM_POST|RESTORE_' "$host_conf" || true + +echo "=== KVM pre/post scripts (ablestack_veeam_*_notify.sh) ===" +mold_host_ensure_kvm_prepost_scripts + +if [[ "$REGISTER_SCRIPTS_ONLY" == "true" ]]; then + mold_host_register_veeam_scripts "$ENV_FILE" "$host" "$kvm_ip" +elif [[ "$SKIP_VEEAM_REPO" != "true" ]]; then + mold_host_setup_veeam_job "$ENV_FILE" "$host" "$kvm_ip" +fi + +echo "=== Enable restore agent (Veeam FLR → Mold datadisk restore) ===" +bash "${SCRIPT_DIR}/enable-veeam-mold-restore.sh" --env-file "$ENV_FILE" \ + ${VM_INCLUDE:+--vm-include "$VM_INCLUDE"} 2>/dev/null \ + || { + systemctl daemon-reload 2>/dev/null || true + systemctl enable mold-veeam-restore-agent.timer 2>/dev/null || true + systemctl start mold-veeam-restore-agent.timer 2>/dev/null || true + systemctl is-active mold-veeam-restore-agent.timer 2>/dev/null || echo "(timer not installed — run install.sh)" + } + +cat </dev/null || echo Ablestack Veeam)' to VMs in Mold UI (no Mold backup repository registration) + 2) Set VMs on this host: ${ETC_DIR}/veeam_config.sh --job-name '${job_name}' --backup-mode host --vm-include 'i-2-XX-VM' + 3) FLR→Mold: ${ETC_DIR}/enable-veeam-mold-restore.sh --vm-include 'i-2-XX-VM' + 4) Run backup from Veeam UI (job ${job_name}) or: Start-VBRComputerBackupJob + 5) After backup: cat ${ETC_DIR}/registry/.latest-backup-id + +EOF diff --git a/scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 new file mode 100644 index 000000000000..e192a8d98e24 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-host-repo.ps1 @@ -0,0 +1,91 @@ +# Create / ensure a host-level Veeam backup repository on local disk (E:\opt1\veeam\). +# All guest Agent jobs for one KVM hypervisor share this repo (not per-VM folders on E:). +# +# pwsh -File setup-veeam-host-repo.ps1 -KvmHostname ablecube31-2 +# pwsh -File setup-veeam-host-repo.ps1 -KvmHostname ablecube31-2 -HostRepoRoot "E:\opt1\veeam" + +param( + [Parameter(Mandatory = $true)] + [string]$KvmHostname, + + [string]$HostRepoRoot = "E:\opt1\veeam", + + [string]$RepositoryName = "" +) + +$ErrorActionPreference = "Stop" + +if (-not $RepositoryName) { + $RepositoryName = "Mold $KvmHostname" +} + +$folder = Join-Path $HostRepoRoot $KvmHostname +New-Item -ItemType Directory -Force -Path $folder | Out-Null +Write-Host "Host repo folder: $folder" + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue +try { Connect-VBRServer -Server localhost -ErrorAction SilentlyContinue | Out-Null } catch {} + +function Get-MoldRepoPathString { + param($Repo) + if ($null -eq $Repo) { return "" } + foreach ($key in @("Path", "Folder", "Location")) { + if ($Repo.PSObject.Properties.Name -contains $key -and $Repo.$key) { + return [string]$Repo.$key + } + } + return "" +} + +$repo = Get-VBRBackupRepository -Name $RepositoryName -ErrorAction SilentlyContinue +if ($repo) { + Write-Host "Repository already exists: $RepositoryName" + Write-Output $RepositoryName + exit 0 +} + +$created = $false +$errors = @() + +function Try-AddRepo { + param([scriptblock]$Action, [string]$Label) + try { + $r = & $Action + if ($r) { return $r } + $r = Get-VBRBackupRepository -Name $RepositoryName -ErrorAction SilentlyContinue + if ($r) { return $r } + } catch { + $script:errors += "${Label}: $($_.Exception.Message)" + } + return $null +} + +$repo = Try-AddRepo { Add-VBRBackupRepository -Name $RepositoryName -Folder $folder } "Folder" +if (-not $repo) { + $repo = Try-AddRepo { Add-VBRBackupRepository -Name $RepositoryName -Path $folder } "Path" +} +if (-not $repo) { + $repo = Try-AddRepo { Add-VBRBackupRepository -Name $RepositoryName -Folder $folder -Type WinLocal } "WinLocal" +} + +if (-not $repo) { + $normalized = $folder.TrimEnd('\') + $repo = Get-VBRBackupRepository -ErrorAction SilentlyContinue | Where-Object { + $p = Get-MoldRepoPathString $_ + $p -and ($p -eq $folder -or $p -eq $normalized -or $p -like "*\$KvmHostname") + } | Select-Object -First 1 +} + +if ($repo) { + Write-Host "Using repository: $($repo.Name) -> $folder" + Write-Output $RepositoryName + exit 0 +} + +Write-Error @" +Failed to create Veeam repository '$RepositoryName' at '$folder'. +Create manually in Veeam UI: Backup Infrastructure -> Backup Repositories -> Add -> Windows -> $folder +Errors: +$($errors -join "`n") +"@ +exit 1 diff --git a/scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 new file mode 100644 index 000000000000..22336ddf886e --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/setup-veeam-mold-job.ps1 @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-shot: KVM hypervisor host backup (Linux Agent on 10.10.31.2). +# - Backup scope: /tmp/mold/veeam (VM disk export staging from Mold pre-script) +# - Pre/Post: Guest Processing on KVM Agent (ablestack_veeam_pre/post_notify.sh) +# +# Run on Veeam B&R server (Administrator, PowerShell 7+): +# .\setup-veeam-mold-job.ps1 +# .\setup-veeam-mold-job.ps1 -ConfPath C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf +# .\setup-veeam-mold-job.ps1 -StartJob +# +# Or from Mac/ccvm: bash push-to-veeam.sh --env-file mold-backup.env + +param( + [string]$ConfPath = "", + [string]$InstallDir = "C:\ProgramData\Mold\backup\veeam", + + [string]$JobName = "", + [string]$KvmHost = "", + [string]$AgentHostName = "", + [string]$BackupPath = "", + [string]$RepositoryName = "", + [string]$ProtectionGroupName = "", + + [switch]$StartJob, + [switch]$WhatIf +) + +$ErrorActionPreference = "Stop" + +if (-not $ConfPath) { + $ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF + if (-not $ConfPath) { + $ConfPath = Join-Path $InstallDir "mold-backup.windows.conf" + } +} + +function Read-MoldWindowsConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { return $cfg } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +$cfg = Read-MoldWindowsConf -Path $ConfPath + +if (-not $JobName) { $JobName = $cfg["VEEAM_JOB_NAME"] } +if (-not $KvmHost) { $KvmHost = $cfg["KVM_HOST"] } +if (-not $BackupPath) { + $BackupPath = $cfg["VEEAM_HOST_BACKUP_PATH"] +} +if (-not $BackupPath) { $BackupPath = "/tmp/mold/veeam" } +if ($BackupPath -match '^[A-Za-z]:[\\/]') { + throw @" +BackupPath must be a Linux path on the KVM agent (e.g. /tmp/mold/veeam), not a Windows path: $BackupPath + +STAGING_PATH in mold-backup.windows.conf is for Windows VM disk export (FLR), not Agent SelectedFiles. +Set VEEAM_HOST_BACKUP_PATH=/tmp/mold/veeam in $ConfPath, or pass -BackupPath '/tmp/mold/veeam'. +"@ +} +if (-not $RepositoryName) { $RepositoryName = $cfg["VEEAM_REPO_NAME"] } +if (-not $AgentHostName) { $AgentHostName = $cfg["KVM_HOSTNAME"] } +if (-not $ProtectionGroupName) { $ProtectionGroupName = $cfg["VEEAM_PROTECTION_GROUP_NAME"] } +if (-not $ProtectionGroupName) { $ProtectionGroupName = "Mold KVM Agents" } + +if (-not $JobName) { throw "JobName required (VEEAM_JOB_NAME in $ConfPath or -JobName)" } +if (-not $KvmHost) { throw "KvmHost required (KVM_HOST in $ConfPath or -KvmHost)" } + +$preScript = $cfg["KVM_PRE_NOTIFY_SCRIPT"] +if (-not $preScript) { $preScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" } +$postScript = $cfg["KVM_POST_NOTIFY_SCRIPT"] +if (-not $postScript) { $postScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" } + +$kvmSshUser = $cfg["KVM_SSH_USER"] +if (-not $kvmSshUser) { $kvmSshUser = "root" } +$kvmSshPassword = $cfg["KVM_SSH_PASSWORD"] +if (-not $kvmSshPassword) { $kvmSshPassword = $env:MOLD_KVM_SSH_PASSWORD } +if (-not $kvmSshPassword) { + throw @" +KVM_SSH_PASSWORD is required for Linux Agent backup on $KvmHost. + +Veeam fails with: Cannot find credentials for agent $KvmHost + +Set KVM_SSH_PASSWORD in $ConfPath (or MOLD_KVM_SSH_PASSWORD env) and re-run. +"@ +} + +Write-Host "=== Mold Veeam setup (KVM hypervisor host — Agent on $KvmHost) ===" +Write-Host " Job : $JobName" +Write-Host " KVM : $KvmHost (Linux Agent on hypervisor — backs up VM export staging)" +Write-Host " Agent : $AgentHostName" +Write-Host " Scope : $BackupPath (SelectedFiles — VM disks exported here by Mold Pre-script)" +Write-Host " Pre : $preScript" +Write-Host " Post : $postScript" +Write-Host " PG : $ProtectionGroupName" +if ($cfg["VM_NAME"]) { + Write-Host " VM : $($cfg['VM_NAME']) (set on KVM via veeam_config.sh --vm-include)" +} else { + Write-Host " VM : (set on KVM) veeam_config.sh --vm-include 'i-2-7-VM'" +} +Write-Host " Conf : $ConfPath" +Write-Host "" + +$createScript = Join-Path $InstallDir "create-veeam-agent-job.ps1" +if (-not (Test-Path $createScript)) { + $createScript = Join-Path $PSScriptRoot "create-veeam-agent-job.ps1" +} +if (-not (Test-Path $createScript)) { + throw "create-veeam-agent-job.ps1 not found. Run push-to-veeam.sh first." +} + +$createArgs = @{ + JobName = $JobName + KvmHost = $KvmHost + BackupPath = $BackupPath + InstallDir = $InstallDir + AgentPreNotifyScript = $preScript + AgentPostNotifyScript = $postScript +} +if ($AgentHostName) { $createArgs["AgentHostName"] = $AgentHostName } +if ($RepositoryName) { $createArgs["RepositoryName"] = $RepositoryName } +$createArgs["ProtectionGroupName"] = $ProtectionGroupName +$createArgs["KvmSshUser"] = $kvmSshUser +$createArgs["KvmSshPassword"] = $kvmSshPassword +if ($WhatIf) { $createArgs["WhatIf"] = $true } + +& $createScript @createArgs + +if ($WhatIf) { exit 0 } + +if ($StartJob) { + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + $job = Get-VBRComputerBackupJob -Name $JobName -ErrorAction SilentlyContinue + if (-not $job) { throw "Job not found after create: $JobName" } + Write-Host "Starting backup job: $JobName" + Start-VBRComputerBackupJob -Job $job | Out-Null + Write-Host "Job started. Check KVM: tail -f /var/log/mold/veeam-hook.log" +} + +Write-Host "" +Write-Host "Done. Veeam backs up VM exports under $BackupPath on KVM hypervisor $KvmHost ($AgentHostName)." +Write-Host "On KVM, set VMs on this host:" +Write-Host " /etc/ablestack/veeam/veeam_config.sh --job-name '$JobName' --backup-mode host --vm-include 'i-2-7-VM'" diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 new file mode 100644 index 000000000000..6a5686553832 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-backup.ps1 @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Veeam Backup Job post-script: invoke KVM post-notify (NetBackup bpend equivalent) over SSH. + +$ErrorActionPreference = "Stop" + +$ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF +if (-not $ConfPath) { + $ConfPath = "C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf" +} + +function Read-MoldConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { throw "Config not found: $Path" } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +$cfg = Read-MoldConf -Path $ConfPath +$kvmHost = $cfg["KVM_HOST"] +$kvmUser = $cfg["KVM_SSH_USER"] +$jobName = $cfg["VEEAM_JOB_NAME"] +if (-not $kvmHost -or -not $kvmUser) { + throw "KVM_HOST and KVM_SSH_USER are required in $ConfPath" +} +if (-not $jobName) { throw "VEEAM_JOB_NAME is required in $ConfPath" } + +$postScript = $cfg["KVM_POST_NOTIFY_SCRIPT"] +if (-not $postScript) { $postScript = "/etc/ablestack/veeam/ablestack_veeam_post_notify.sh" } + +$sshKey = $cfg["KVM_SSH_KEY"] +$sshArgs = @() +if ($sshKey) { $sshArgs += @("-i", $sshKey) } + +$remote = "$kvmUser@$kvmHost" +$vmName = $cfg["VM_NAME"] +$vmEnv = "" +if ($vmName) { + $vmEnv = "VM_INCLUDE='$vmName' VM_NAME='$vmName' " +} +Write-Host "Running post-notify on $remote : $postScript (job=$jobName vm=$vmName)" +& ssh @sshArgs $remote "${vmEnv}bash '$postScript' '$(hostname)' '$jobName'" +if ($LASTEXITCODE -ne 0) { throw "post-notify failed with exit code $LASTEXITCODE" } + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 new file mode 100644 index 000000000000..df756f86150f --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam-job-post-restore.ps1 @@ -0,0 +1,59 @@ +# Veeam post-restore hook: push restore event to KVM restore agent over SSH. +# The KVM agent checks VM ownership + flock before calling Mold restoreBackup. + +$ErrorActionPreference = "Stop" + +$ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF +if (-not $ConfPath) { + $ConfPath = "C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf" +} + +function Read-MoldConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { throw "Config not found: $Path" } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +$cfg = Read-MoldConf -Path $ConfPath +$kvmHost = $cfg["KVM_HOST"] +$kvmUser = $cfg["KVM_SSH_USER"] +$jobName = $cfg["VEEAM_JOB_NAME"] +$backupId = $cfg["BACKUP_ID"] +$vmName = $cfg["VM_NAME"] +if (-not $kvmHost -or -not $kvmUser) { + throw "KVM_HOST and KVM_SSH_USER are required in $ConfPath" +} +if (-not $vmName) { + throw "VM_NAME is required in $ConfPath for post-restore event" +} + +$eventScript = $cfg["KVM_RESTORE_EVENT_SCRIPT"] +if (-not $eventScript) { $eventScript = "/etc/ablestack/veeam/ablestack_veeam_restore_event.sh" } + +$sessionId = $env:VEEAM_RESTORE_SESSION_ID +if (-not $sessionId) { + $sessionId = "veeam-ps1-$(Get-Date -Format 'yyyyMMddHHmmss')" +} + +$sshKey = $cfg["KVM_SSH_KEY"] +$sshArgs = @() +if ($sshKey) { $sshArgs += @("-i", $sshKey) } + +$remote = "$kvmUser@$kvmHost" +$restoreSource = $cfg["VEEAM_UI_RESTORE_SOURCE"] +if (-not $restoreSource) { $restoreSource = "mold-only" } +$backupEnv = "" +if ($backupId) { $backupEnv = "BACKUP_ID='$backupId' " } +Write-Host "Pushing restore event to $remote (vm=$vmName session=$sessionId source=$restoreSource)" +& ssh @sshArgs $remote "${backupEnv}RESTORE_SOURCE='$restoreSource' VEEAM_JOB_NAME='$jobName' VM_NAME='$vmName' bash '$eventScript' veeam.restore.completed '$sessionId' '$vmName' source=veeam-ui-flr" +if ($LASTEXITCODE -ne 0) { throw "restore-event failed with exit code $LASTEXITCODE" } + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 b/scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 new file mode 100644 index 000000000000..473ae98a6420 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam-job-pre-backup.ps1 @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Veeam Backup Job pre-script: record restore point id; optionally export disks to staging. +# FileLevel Agent backups: no VMDK export — KVM seed uses live libvirt disks (VEEAM_BACKUP_MODE=filelevel). +# Configure: C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf + +$ErrorActionPreference = "Stop" + +if ($PSVersionTable.PSVersion.Major -lt 7) { + throw "Veeam.Backup.PowerShell requires PowerShell 7+. Run: pwsh -File $PSCommandPath" +} + +$ConfPath = $env:MOLD_BACKUP_WINDOWS_CONF +if (-not $ConfPath) { + $ConfPath = "C:\ProgramData\Mold\backup\veeam\mold-backup.windows.conf" +} + +function Read-MoldConf { + param([string]$Path) + $cfg = @{} + if (-not (Test-Path $Path)) { + throw "Config not found: $Path" + } + Get-Content $Path | ForEach-Object { + $line = $_.Trim() + if ($line -match '^\s*#' -or $line -eq '') { return } + if ($line -match '^([^=]+)=(.*)$') { + $cfg[$Matches[1].Trim()] = $Matches[2].Trim().Trim('"') + } + } + return $cfg +} + +function Write-MoldConfValue { + param([string]$Path, [string]$Key, [string]$Value) + $content = Get-Content $Path -ErrorAction SilentlyContinue + if (-not $content) { $content = @() } + $found = $false + $newContent = foreach ($line in $content) { + if ($line -match "^\s*$([regex]::Escape($Key))\s*=") { + $found = $true + "$Key=`"$Value`"" + } else { $line } + } + if (-not $found) { $newContent += "$Key=`"$Value`"" } + Set-Content -Path $Path -Value $newContent -Encoding UTF8 +} + +$cfg = Read-MoldConf -Path $ConfPath +$vmName = $cfg["VM_NAME"] +if (-not $vmName) { $vmName = $env:VEEAM_VM_NAME } + +$jobName = $cfg["VEEAM_JOB_NAME"] +if (-not $jobName) { $jobName = $env:VEEAM_JOB_NAME } + +$staging = $cfg["STAGING_PATH"] +if (-not $staging) { $staging = $cfg["VEEAM_HOST_BACKUP_PATH"] } +if (-not $staging) { $staging = "/tmp/mold/veeam" } + +Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + +# NetBackup-style: multi-VM job — pre-notify only (no VMDK export) +if ($cfg["VEEAM_BACKUP_MODE"] -eq "filelevel" -and $cfg["KVM_HOST"] -and $cfg["KVM_SSH_USER"] -and $jobName -and -not $vmName) { + $preScript = $cfg["KVM_PRE_NOTIFY_SCRIPT"] + if (-not $preScript) { $preScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" } + $sshKey = $cfg["KVM_SSH_KEY"] + $sshArgs = @() + if ($sshKey) { $sshArgs += @("-i", $sshKey) } + $remote = "$($cfg['KVM_SSH_USER'])@$($cfg['KVM_HOST'])" + Write-Host "Running pre-notify (multi-VM) on $remote : $preScript (job=$jobName)" + & ssh @sshArgs $remote "bash '$preScript' '$(hostname)' '$jobName'" + if ($LASTEXITCODE -ne 0) { throw "pre-notify failed with exit code $LASTEXITCODE" } + exit 0 +} + +if (-not $vmName) { throw "VM_NAME is required in $ConfPath or VEEAM_VM_NAME env" } + +$backups = @() +if ($jobName) { + # Agent jobs: Veeam requires a trailing wildcard on backup name. + $backups = @(Get-VBRBackup -Name "${jobName}*" -ErrorAction SilentlyContinue) +} +if ($backups.Count -eq 0) { + $backups = @(Get-VBRBackup | Where-Object { $_.JobType -eq "EpAgentBackup" }) +} +if ($backups.Count -eq 0) { + throw "No Veeam backups found. Set VEEAM_JOB_NAME in $ConfPath (e.g. Agent Backup Job 1)." +} + +$backup = $null +$vm = $null +foreach ($b in $backups) { + $objects = @(Get-VBRBackupObject -Backup $b -ErrorAction SilentlyContinue) + $match = $objects | Where-Object { $_.Name -eq $vmName } | Select-Object -First 1 + if ($match) { + $backup = $b + $vm = $match + break + } +} +if (-not $vm) { + $known = foreach ($b in $backups) { + Get-VBRBackupObject -Backup $b -ErrorAction SilentlyContinue | ForEach-Object { $_.Name } + } + $known = ($known | Sort-Object -Unique) -join ", " + throw "Veeam backup object not found for VM_NAME=$vmName. Known names: $known" +} + +# Start-VBRRestoreVirtualDisks expects COib from Get-VBRRestorePoint (not Get-VBRObjectRestorePoint). +$rp = $backup | Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +if (-not $rp) { + $rp = $vm | Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +} +if (-not $rp) { + $rp = $backup | Get-VBRObjectRestorePoint -Name $vmName | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +} +if (-not $rp) { + $rp = Get-VBRObjectRestorePoint -Backup $backup | + Where-Object { $_.Name -eq $vmName } | + Sort-Object CreationTime -Descending | + Select-Object -First 1 +} +if (-not $rp) { throw "No restore point for VM: $vmName (backup: $($backup.Name))" } + +$rpId = $rp.Id +if ($rpId -is [guid]) { $rpId = $rpId.Guid } + +$rpOib = $backup | Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Where-Object { $_.Id -eq $rpId -or $_.Id.Guid -eq $rpId } | + Select-Object -First 1 +if (-not $rpOib) { + $rpOib = Get-VBRRestorePoint -ErrorAction SilentlyContinue | + Where-Object { $_.Id -eq $rpId -or $_.Id.Guid -eq $rpId } | + Select-Object -First 1 +} +if (-not $rpOib) { $rpOib = $rp } + +function Test-MoldVeeamFileLevelRestorePoint { + param($RestorePoint) + $text = $RestorePoint | Format-List * -Force | Out-String + if ($text -match 'BackupMode\s*:\s*FileLevel') { return $true } + if ($text -match 'COibAuxDataLinuxBackup') { return $true } + if ($text -match 'ItemType\s*:\s*LinuxPhysicalDisk' -and $text -match 'IncludePaths') { return $true } + return $false +} + +$backupMode = $cfg["VEEAM_BACKUP_MODE"] +if (-not $backupMode) { $backupMode = "auto" } +$isFileLevel = ($backupMode -eq "filelevel") -or ( + $backupMode -eq "auto" -and (Test-MoldVeeamFileLevelRestorePoint -RestorePoint $rpOib) +) + +if ($isFileLevel) { + Write-Host "VEEAM_BACKUP_MODE=filelevel: NetBackup-style pre-notify on KVM (host path /tmp/mold/veeam)." + Write-MoldConfValue -Path $ConfPath -Key "VEEAM_RESTORE_POINT_ID" -Value $rpId + Write-Host "VEEAM_RESTORE_POINT_ID=$rpId written to $ConfPath" + if ($cfg["KVM_HOST"] -and $cfg["KVM_SSH_USER"] -and $jobName) { + $preScript = $cfg["KVM_PRE_NOTIFY_SCRIPT"] + if (-not $preScript) { $preScript = "/etc/ablestack/veeam/ablestack_veeam_pre_notify.sh" } + $sshKey = $cfg["KVM_SSH_KEY"] + $sshArgs = @() + if ($sshKey) { $sshArgs += @("-i", $sshKey) } + $remote = "$($cfg['KVM_SSH_USER'])@$($cfg['KVM_HOST'])" + Write-Host "Running pre-notify on $remote : $preScript (job=$jobName)" + & ssh @sshArgs $remote "bash '$preScript' '$(hostname)' '$jobName'" + if ($LASTEXITCODE -ne 0) { throw "pre-notify failed with exit code $LASTEXITCODE" } + } + exit 0 +} + +New-Item -ItemType Directory -Force -Path $staging | Out-Null +Get-ChildItem -Path $staging -File -Recurse -Include *.vmdk,*.vhd,*.vhdx -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue + +function Resolve-MoldVeeamExportServer { + param([hashtable]$Cfg) + $configured = $Cfg["STAGING_EXPORT_SERVER"] + if ($configured) { + $s = Get-VBRServer -Type Windows -Name $configured -ErrorAction SilentlyContinue + if ($s) { return $s } + } + $names = @($env:COMPUTERNAME) + if ($env:USERDNSDOMAIN) { + $names += "$($env:COMPUTERNAME).$($env:USERDNSDOMAIN)" + } + foreach ($n in $names) { + $s = Get-VBRServer -Type Windows -Name $n -ErrorAction SilentlyContinue + if ($s) { return $s } + } + $localhost = Get-VBRLocalhost -ErrorAction SilentlyContinue + if ($localhost) { return $localhost } + Get-VBRServer -ErrorAction SilentlyContinue | Select-Object -First 1 +} + +function Export-MoldVeeamDisksToStaging { + param( + [Parameter(Mandatory = $true)]$RestorePoint, + [Parameter(Mandatory = $true)][string]$StagingPath, + [hashtable]$Cfg = @{} + ) + if (Get-Command Start-VBRFLRSession -ErrorAction SilentlyContinue) { + $session = Start-VBRFLRSession -RestorePoint $RestorePoint + try { + $items = Get-VBRFLRItem -Session $session | Where-Object { $_.Type -eq "HardDisk" } + foreach ($item in $items) { + $dest = Join-Path $StagingPath ($item.Name + ".vmdk") + Copy-VBRFLRItem -FLRSession $session -Item $item -Destination $dest + Write-Host "Exported $($item.Name) -> $dest" + } + } finally { + Stop-VBRFLRSession -Session $session + } + return + } + if (Get-Command Start-VBRRestoreVirtualDisks -ErrorAction SilentlyContinue) { + $server = Resolve-MoldVeeamExportServer -Cfg $Cfg + if (-not $server) { + throw "No Windows managed server for export. Add $($env:COMPUTERNAME) under Backup Infrastructure > Managed Servers, or set STAGING_EXPORT_SERVER in mold-backup.windows.conf" + } + if (-not (Test-Path -LiteralPath $StagingPath)) { + New-Item -ItemType Directory -Force -Path $StagingPath | Out-Null + } + Write-Host "Exporting disks via Start-VBRRestoreVirtualDisks -> $StagingPath (server: $($server.Name))" + try { + Start-VBRRestoreVirtualDisks -RestorePoint $RestorePoint -Server $server ` + -Path $StagingPath -RestoreDiskType Vmdk | Out-Null + } catch { + throw "Start-VBRRestoreVirtualDisks failed on server '$($server.Name)': $_. Ensure the folder exists, Veeam service account has write access, and the server is a Managed Windows server in Veeam console." + } + Get-ChildItem -Path $StagingPath -Recurse -Include *.vmdk,*.vhd,*.vhdx -ErrorAction SilentlyContinue | + ForEach-Object { Write-Host "Exported $($_.FullName)" } + return + } + throw "No supported Veeam export cmdlet (Start-VBRFLRSession / Start-VBRRestoreVirtualDisks)" +} + +Export-MoldVeeamDisksToStaging -RestorePoint $rpOib -StagingPath $staging -Cfg $cfg +$exported = Get-ChildItem -Path $staging -Recurse -Include *.vmdk,*.vhd,*.vhdx -ErrorAction SilentlyContinue +if (-not $exported) { + throw "No virtual disk files exported under $staging" +} + +Write-MoldConfValue -Path $ConfPath -Key "VEEAM_RESTORE_POINT_ID" -Value $rpId +Write-Host "VEEAM_RESTORE_POINT_ID=$rpId written to $ConfPath" + +# Optional: push restore point id to KVM host config over SSH +if ($cfg["KVM_HOST"] -and $cfg["KVM_SSH_USER"]) { + $kvmConf = $cfg["KVM_MOLD_BACKUP_CONF"] + if (-not $kvmConf) { $kvmConf = "/etc/mold/backup/veeam/mold-backup.conf" } + $sshKey = $cfg["KVM_SSH_KEY"] + $sshArgs = @() + if ($sshKey) { $sshArgs += @("-i", $sshKey) } + $remote = "$($cfg['KVM_SSH_USER'])@$($cfg['KVM_HOST'])" + $cmd = "grep -q '^VEEAM_RESTORE_POINT_ID=' '$kvmConf' 2>/dev/null && sed -i 's|^VEEAM_RESTORE_POINT_ID=.*|VEEAM_RESTORE_POINT_ID=`"$rpId`"|' '$kvmConf' || echo 'VEEAM_RESTORE_POINT_ID=`"$rpId`"' >> '$kvmConf'" + & ssh @sshArgs $remote $cmd + Write-Host "Updated VEEAM_RESTORE_POINT_ID on KVM host $remote" +} + +exit 0 diff --git a/scripts/vm/hypervisor/kvm/veeam/veeam_config.sh b/scripts/vm/hypervisor/kvm/veeam/veeam_config.sh new file mode 100755 index 000000000000..5ad5d8aaab8b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/veeam/veeam_config.sh @@ -0,0 +1,620 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NetBackup-style Veeam + Mold host configuration (veeam_config.sh). +# Creates /etc/ablestack/veeam/.conf, encrypts API secret, optional Mold global settings. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ETC_DIR="${ABLESTACK_VEEAM_ETC_DIR:-/etc/ablestack/veeam}" +SHARE_DIR="${MOLD_BACKUP_SHARE_DIR:-/usr/share/mold/backup/veeam}" +SECRET_SCRIPT="${SCRIPT_DIR}/mold-backup-secret.sh" +DEFAULT_SECRET_KEY_FILE="${ABLESTACK_SECRET_KEY_FILE:-/root/.ssh/ablestack.key}" + +JOB_NAME="" +BACKUP_OFFERING_NAME="VeeamBackup" +VM_INCLUDE="" +VM_EXCLUDE="" +MAX_CHAIN="7" +MOLD_API_URL="" +MOLD_API_KEY="" +MOLD_API_SECRET="" +ZONE_ID="" +RETENTION_PERIOD="P7D" +VEEAM_URL="" +VEEAM_USERNAME="" +VEEAM_PASSWORD="" +BACKUP_MODE="host" +IMPORT_MODE="auto" + +VEEAM_SSH_HOST="" +VEEAM_SSH_USER="administrator" +VEEAM_SSH_KEY="/root/.ssh/veeam_id_rsa" +RESTORE_SOURCE="auto" +KVM_HOST="" +KVM_SSH_USER="root" +KVM_SSH_KEY="" +KVM_SSH_PASSWORD="" +VM_NAME_CFG="" +VM_UUID_CFG="" +BACKUP_REPO_TYPE="local" +BACKUP_REPO_NAME="Ablestack Data Disk" +BACKUP_REPO_ADDRESS="" +BACKUP_REPO_MOUNT_OPTS="" +BACKUP_REPO_PROVIDER="localfs" +NAS_REPO_MOUNT="" +BACKUP_STORAGE_MODE="datadisk" +MOLD_DATADISK_PATH="/data/backup" +VEEAM_HOST_REPO_ROOT="E:/opt1/veeam" +ENCRYPT_SECRET="true" +SECRET_KEY_FILE="" +VEEAM_BACKUP_TARGET="" +VM_TARGETS="" +VEEAM_GUEST_JOB_PREFIX="Mold VM" +GUEST_VM_SSH_USER="" +GUEST_VM_SSH_PASSWORD="" +RUN_INSTALL="false" +CONFIGURE_MOLD="true" +ENV_FILE="" +DEFAULT_ENV_FILE="${ETC_DIR}/mold-backup.env" +MOLD_API_SECRET_ENC_FILE_CANDIDATE="" + +usage() { + cat <<'EOF' +Usage: veeam_config.sh [options] + +Required: + --job-name NAME Veeam backup job / policy name (conf file name) + +Auto-filled (when omitted) from, in order: + 1) CLI options + 2) --env-file or /etc/ablestack/veeam/mold-backup.env + 3) Existing /etc/ablestack/veeam/*.conf + 4) secrets/secret.enc + /root/.ssh/ablestack.key + 5) Mold API listZones / listBackupRepositories + 6) /etc/cloudstack/agent/agent.properties (Mold API URL) + +Common optional overrides: + --offering-name NAME Mold backup offering name (default: VeeamBackup) + --mold-url URL Mold API URL (http://:8080/client/api) + --api-key KEY Mold API key + --api-secret SECRET Mold API secret + --zone-id UUID Zone for importBackupOffering + --nas-repo ADDR NAS repo address (host:/export; nfs:// prefix optional) + --env-file PATH Env file (default: /etc/ablestack/veeam/mold-backup.env) + +Other optional: + --vm-include LIST Comma-separated libvirt names (* = all running, default) + --vm-exclude LIST Comma-separated libvirt names to skip + --vm-name NAME libvirt name (mold-backup.sh status 등) + --vm-uuid UUID Mold VM UUID + --max-chain N Max incremental chain (default: 7) + --retention PERIOD Backup offering retention (default: P7D) + --backup-mode MODE host|api|local|auto (default: host = NetBackup-style /tmp/mold/veeam) + --backup-target MODE host only (guest mode removed) + --veeam-url URL Mold zone setting backup.plugin.ablestack-veeam.url + --veeam-user USER Mold zone setting backup.plugin.ablestack-veeam.username + --veeam-password PASS Mold zone setting backup.plugin.ablestack-veeam.password + --nas-repo ADDR NAS repo address (host:/export; nfs:// prefix optional) + --repo-name NAME Mold backup repository name (default: Ablestack Veeam NAS) + --repo-type TYPE nfs|cifs (default: nfs) + --repo-mount-opts OPTS Mount options for addBackupRepository + --kvm-host IP KVM host for Veeam post-job SSH + --kvm-ssh-user USER (default: root) + --kvm-ssh-key PATH SSH private key + --no-configure-mold Skip listConfigurations/updateConfiguration calls + --no-encrypt-secret Store API secret in plain text (not recommended) + --secret-key-file PATH Passphrase file (default: /root/.ssh/ablestack.key) + --env-file PATH Credentials/env file (default: /etc/ablestack/veeam/mold-backup.env) + --install Run install.sh after generating configs + -h, --help + +Example (minimal — reads mold-backup.env + existing conf): + veeam_config.sh --job-name "Mold KVM Backup" --kvm-host 10.10.31.2 + +Example (explicit): + veeam_config.sh \ + --job-name "VeeamBackup" \ + --offering-name "VeeamBackup" \ + --mold-url http://10.10.31.20:8080/client/api \ + --api-key KEY --api-secret 'SECRET' \ + --zone-id \ + --vm-include "i-2-3-VM,i-2-7-VM" \ + --max-chain 7 \ + --veeam-url https://veeam:9398/api/ \ + --kvm-host 10.10.31.30 \ + --install +EOF +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +veeam_set_if_empty() { + local name="$1" value="$2" + [[ -z "${value}" ]] && return 0 + [[ -n "${!name:-}" ]] && return 0 + printf -v "$name" '%s' "$value" +} + +veeam_config_read_conf_var() { + local file="$1" key="$2" line val + [[ -f "$file" ]] || return 1 + line="$(grep -E "^${key}=" "$file" 2>/dev/null | tail -1)" || return 1 + val="${line#*=}" + val="${val%$'\r'}" + if [[ "$val" == \"*\" ]]; then + val="${val#\"}"; val="${val%\"}" + elif [[ "$val" == \'*\' ]]; then + val="${val#\'}"; val="${val%\'}" + fi + [[ -n "$val" ]] || return 1 + echo "$val" +} + +veeam_config_import_env_file() { + local f="$1" line key val + [[ -f "$f" ]] || return 0 + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%%#*}" + line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -z "$line" || "$line" != *"="* ]] && continue + key="${line%%=*}" + key="$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + val="${line#*=}" + val="$(echo "$val" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ "$val" == \"*\" ]]; then val="${val#\"}"; val="${val%\"}"; fi + if [[ "$val" == \'*\' ]]; then val="${val#\'}"; val="${val%\'}"; fi + case "$key" in + VEEAM_USER) [[ -n "$val" ]] && VEEAM_USERNAME="$val" ;; + KVM_IP) veeam_set_if_empty KVM_HOST "$val" ;; + VEEAM_JOB_NAME) veeam_set_if_empty JOB_NAME "$val" ;; + MOLD_API_URL) [[ -n "$val" ]] && MOLD_API_URL="$val" ;; + MOLD_API_KEY) [[ -n "$val" ]] && MOLD_API_KEY="$val" ;; + MOLD_API_SECRET) [[ -n "$val" ]] && MOLD_API_SECRET="$val" ;; + ZONE_ID) [[ -n "$val" ]] && ZONE_ID="$val" ;; + JOB_NAME) veeam_set_if_empty JOB_NAME "$val" ;; + BACKUP_OFFERING_NAME) veeam_set_if_empty BACKUP_OFFERING_NAME "$val" ;; + VM_INCLUDE) veeam_set_if_empty VM_INCLUDE "$val" ;; + VM_EXCLUDE) veeam_set_if_empty VM_EXCLUDE "$val" ;; + VM_NAME) veeam_set_if_empty VM_NAME_CFG "$val" ;; + VM_UUID) veeam_set_if_empty VM_UUID_CFG "$val" ;; + MAX_CHAIN) [[ -n "$val" ]] && MAX_CHAIN="$val" ;; + RETENTION_PERIOD) [[ -n "$val" ]] && RETENTION_PERIOD="$val" ;; + BACKUP_MODE) veeam_set_if_empty BACKUP_MODE "$val" ;; + IMPORT_MODE) [[ -n "$val" ]] && IMPORT_MODE="$val" ;; + VEEAM_URL) [[ -n "$val" ]] && VEEAM_URL="$val" ;; + VEEAM_USERNAME) [[ -n "$val" ]] && VEEAM_USERNAME="$val" ;; + VEEAM_PASSWORD) [[ -n "$val" ]] && VEEAM_PASSWORD="$val" ;; + VEEAM_SSH_HOST) [[ -n "$val" ]] && VEEAM_SSH_HOST="$val" ;; + VEEAM_SSH_USER) [[ -n "$val" ]] && VEEAM_SSH_USER="$val" ;; + VEEAM_SSH_KEY) [[ -n "$val" ]] && VEEAM_SSH_KEY="$val" ;; + KVM_HOST) veeam_set_if_empty KVM_HOST "$val" ;; + KVM_SSH_USER) [[ -n "$val" ]] && KVM_SSH_USER="$val" ;; + KVM_SSH_KEY) [[ -n "$val" ]] && KVM_SSH_KEY="$val" ;; + KVM_SSH_PASSWORD) [[ -n "$val" ]] && KVM_SSH_PASSWORD="$val" ;; + BACKUP_REPO_ADDRESS) [[ -n "$val" ]] && BACKUP_REPO_ADDRESS="$val" ;; + BACKUP_REPO_NAME) [[ -n "$val" ]] && BACKUP_REPO_NAME="$val" ;; + BACKUP_REPO_TYPE) [[ -n "$val" ]] && BACKUP_REPO_TYPE="$val" ;; + BACKUP_REPO_MOUNT_OPTS) [[ -n "$val" ]] && BACKUP_REPO_MOUNT_OPTS="$val" ;; + NAS_REPO_MOUNT) [[ -n "$val" ]] && NAS_REPO_MOUNT="$val" ;; + VEEAM_BACKUP_TARGET) veeam_set_if_empty VEEAM_BACKUP_TARGET "$val" ;; + VM_TARGETS) [[ -n "$val" ]] && VM_TARGETS="$val" ;; + VEEAM_GUEST_JOB_PREFIX) [[ -n "$val" ]] && VEEAM_GUEST_JOB_PREFIX="$val" ;; + GUEST_VM_SSH_USER) [[ -n "$val" ]] && GUEST_VM_SSH_USER="$val" ;; + GUEST_VM_SSH_PASSWORD) [[ -n "$val" ]] && GUEST_VM_SSH_PASSWORD="$val" ;; + esac + done < "$f" +} + +veeam_config_load_from_conf() { + local file="$1" v + [[ -f "$file" ]] || return 0 + veeam_set_if_empty MOLD_API_URL "$(veeam_config_read_conf_var "$file" MOLD_API_URL || true)" + veeam_set_if_empty MOLD_API_KEY "$(veeam_config_read_conf_var "$file" MOLD_API_KEY || true)" + veeam_set_if_empty MOLD_API_SECRET "$(veeam_config_read_conf_var "$file" MOLD_API_SECRET || true)" + v="$(veeam_config_read_conf_var "$file" MOLD_API_SECRET_ENC_FILE || true)" + [[ -n "$v" ]] && MOLD_API_SECRET_ENC_FILE_CANDIDATE="$v" + veeam_set_if_empty SECRET_KEY_FILE "$(veeam_config_read_conf_var "$file" MOLD_SECRET_KEY_FILE || true)" + veeam_set_if_empty ZONE_ID "$(veeam_config_read_conf_var "$file" ZONE_ID || true)" + veeam_set_if_empty BACKUP_OFFERING_NAME "$(veeam_config_read_conf_var "$file" BACKUP_OFFERING_NAME || true)" + veeam_set_if_empty VM_INCLUDE "$(veeam_config_read_conf_var "$file" VM_INCLUDE || true)" + veeam_set_if_empty VM_EXCLUDE "$(veeam_config_read_conf_var "$file" VM_EXCLUDE || true)" + veeam_set_if_empty VM_NAME_CFG "$(veeam_config_read_conf_var "$file" VM_NAME || true)" + veeam_set_if_empty VM_UUID_CFG "$(veeam_config_read_conf_var "$file" VM_UUID || true)" + veeam_set_if_empty MAX_CHAIN "$(veeam_config_read_conf_var "$file" VEEAM_MAX_CHAIN || true)" + veeam_set_if_empty RETENTION_PERIOD "$(veeam_config_read_conf_var "$file" RETENTION_PERIOD || true)" + veeam_set_if_empty VEEAM_URL "$(veeam_config_read_conf_var "$file" VEEAM_URL || true)" + veeam_set_if_empty VEEAM_USERNAME "$(veeam_config_read_conf_var "$file" VEEAM_USERNAME || true)" + veeam_set_if_empty VEEAM_PASSWORD "$(veeam_config_read_conf_var "$file" VEEAM_PASSWORD || true)" + veeam_set_if_empty VEEAM_SSH_HOST "$(veeam_config_read_conf_var "$file" VEEAM_SSH_HOST || true)" + veeam_set_if_empty BACKUP_REPO_ADDRESS "$(veeam_config_read_conf_var "$file" BACKUP_REPO_ADDRESS || true)" + veeam_set_if_empty BACKUP_REPO_NAME "$(veeam_config_read_conf_var "$file" BACKUP_REPO_NAME || true)" + veeam_set_if_empty BACKUP_REPO_TYPE "$(veeam_config_read_conf_var "$file" BACKUP_REPO_TYPE || true)" + veeam_set_if_empty BACKUP_REPO_MOUNT_OPTS "$(veeam_config_read_conf_var "$file" BACKUP_REPO_MOUNT_OPTS || true)" + veeam_set_if_empty NAS_REPO_MOUNT "$(veeam_config_read_conf_var "$file" NAS_REPO_MOUNT || true)" +} + +veeam_config_auto_mold_url() { + local props host + [[ -n "$MOLD_API_URL" ]] && return 0 + for props in /etc/cloudstack/agent/agent.properties /etc/cloudstack/agent/agent.properties.override; do + [[ -f "$props" ]] || continue + host="$(grep -E '^[[:space:]]*host[[:space:]]*=' "$props" 2>/dev/null | tail -1 | cut -d= -f2- | tr -d ' \r')" + [[ -n "$host" ]] || continue + if [[ "$host" == http* ]]; then + MOLD_API_URL="${host%/}/client/api" + else + MOLD_API_URL="http://${host}:8080/client/api" + fi + return 0 + done +} + +veeam_config_resolve_api_secret() { + local enc key + [[ -n "$MOLD_API_SECRET" ]] && return 0 + enc="${MOLD_API_SECRET_ENC_FILE_CANDIDATE:-${ETC_DIR}/secrets/secret.enc}" + key="${SECRET_KEY_FILE:-$DEFAULT_SECRET_KEY_FILE}" + [[ -f "$enc" && -f "$key" && -x "$SECRET_SCRIPT" ]] || return 0 + MOLD_API_SECRET="$("$SECRET_SCRIPT" decrypt --enc-file "$enc" --key-file "$key" 2>/dev/null || true)" +} + +veeam_config_autofill_defaults() { + local env_path conf_path safe_job f + + for env_path in \ + "${ENV_FILE}" \ + "${DEFAULT_ENV_FILE}" \ + "${SCRIPT_DIR}/mold-backup.env" \ + "${SHARE_DIR}/mold-backup.env"; do + [[ -n "$env_path" && -f "$env_path" ]] || continue + veeam_config_import_env_file "$env_path" + done + + safe_job="" + if [[ -n "$JOB_NAME" ]]; then + safe_job="$(echo "$JOB_NAME" | tr ' /' '__')" + veeam_config_load_from_conf "${ETC_DIR}/${safe_job}.conf" + veeam_config_load_from_conf "${ETC_DIR}/${JOB_NAME}.conf" + fi + veeam_config_load_from_conf "${ETC_DIR}/mold-backup.conf" + + if [[ -z "$MOLD_API_KEY" || -z "$MOLD_API_SECRET" || -z "$ZONE_ID" || -z "$BACKUP_REPO_ADDRESS" ]]; then + for conf_path in "${ETC_DIR}"/*.conf; do + [[ -f "$conf_path" ]] || continue + [[ "$conf_path" == *mold-backup.windows.conf ]] && continue + veeam_config_load_from_conf "$conf_path" + done + fi + + veeam_config_resolve_api_secret + veeam_config_auto_mold_url + [[ -n "$KVM_HOST" ]] || KVM_HOST="$(hostname -I 2>/dev/null | awk '{print $1}')" + + if [[ -n "$MOLD_API_URL" && -n "$MOLD_API_KEY" && -n "$MOLD_API_SECRET" ]]; then + # shellcheck source=mold-backup.lib.sh + source "${SCRIPT_DIR}/mold-backup.lib.sh" + export MOLD_API_URL MOLD_API_KEY MOLD_API_SECRET ZONE_ID BACKUP_REPO_NAME BACKUP_REPO_ADDRESS + if [[ -z "$ZONE_ID" ]]; then + ZONE_ID="$(mold_backup_api_first_zone_id 2>/dev/null || true)" + fi + if [[ -z "$BACKUP_REPO_ADDRESS" ]]; then + if [[ "${BACKUP_STORAGE_MODE:-datadisk}" == "datadisk" ]]; then + BACKUP_REPO_ADDRESS="${MOLD_DATADISK_PATH:-/data/backup}" + else + BACKUP_REPO_ADDRESS="$(mold_backup_api_first_repo_address "${BACKUP_REPO_NAME}" 2>/dev/null || true)" + fi + fi + fi +} + +veeam_config_print_autofill_summary() { + echo "Configuration sources:" + [[ -f "${ENV_FILE:-$DEFAULT_ENV_FILE}" ]] && echo " env: ${ENV_FILE:-$DEFAULT_ENV_FILE}" + echo " API URL: ${MOLD_API_URL:-}" + [[ -n "${MOLD_API_KEY:-}" ]] && echo " API key: ${MOLD_API_KEY:0:12}..." + [[ -n "${MOLD_API_SECRET:-}" ]] && echo " API secret: (loaded)" + echo " Zone: ${ZONE_ID:-}" + echo " NAS repo: ${BACKUP_REPO_ADDRESS:-}" + echo " KVM host: ${KVM_HOST:-}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --job-name) JOB_NAME="$2"; shift 2 ;; + --offering-name) BACKUP_OFFERING_NAME="$2"; shift 2 ;; + --vm-include) VM_INCLUDE="$2"; shift 2 ;; + --vm-exclude) VM_EXCLUDE="$2"; shift 2 ;; + --vm-name) VM_NAME_CFG="$2"; shift 2 ;; + --vm-uuid) VM_UUID_CFG="$2"; shift 2 ;; + --max-chain) MAX_CHAIN="$2"; shift 2 ;; + --mold-url) MOLD_API_URL="$2"; shift 2 ;; + --api-key) MOLD_API_KEY="$2"; shift 2 ;; + --api-secret) MOLD_API_SECRET="$2"; shift 2 ;; + --zone-id) ZONE_ID="$2"; shift 2 ;; + --retention) RETENTION_PERIOD="$2"; shift 2 ;; + --backup-mode) BACKUP_MODE="$2"; shift 2 ;; + --backup-target) VEEAM_BACKUP_TARGET="$2"; shift 2 ;; + --vm-targets) die "Guest-mode --vm-targets removed. Use --vm-include with host/datadisk mode." ;; + --guest-job-prefix) die "Guest-mode --guest-job-prefix removed." ;; + --veeam-url) VEEAM_URL="$2"; shift 2 ;; + --veeam-user) VEEAM_USERNAME="$2"; shift 2 ;; + --veeam-password) VEEAM_PASSWORD="$2"; shift 2 ;; + --veeam-ssh-host) VEEAM_SSH_HOST="$2"; shift 2 ;; + --veeam-ssh-user) VEEAM_SSH_USER="$2"; shift 2 ;; + --veeam-ssh-key) VEEAM_SSH_KEY="$2"; shift 2 ;; + --nas-repo) BACKUP_REPO_ADDRESS="$2"; shift 2 ;; + --repo-name) BACKUP_REPO_NAME="$2"; shift 2 ;; + --repo-type) BACKUP_REPO_TYPE="$2"; shift 2 ;; + --repo-mount-opts) BACKUP_REPO_MOUNT_OPTS="$2"; shift 2 ;; + --nas-repo-mount) NAS_REPO_MOUNT="$2"; shift 2 ;; + --kvm-host) KVM_HOST="$2"; shift 2 ;; + --kvm-ssh-user) KVM_SSH_USER="$2"; shift 2 ;; + --kvm-ssh-key) KVM_SSH_KEY="$2"; shift 2 ;; + --no-configure-mold) CONFIGURE_MOLD="false"; shift ;; + --no-encrypt-secret) ENCRYPT_SECRET="false"; shift ;; + --secret-key-file) SECRET_KEY_FILE="$2"; shift 2 ;; + --env-file) ENV_FILE="$2"; shift 2 ;; + --install) RUN_INSTALL="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) die "Unknown option: $1" ;; + esac +done + +veeam_config_autofill_defaults + +if [[ "${VEEAM_BACKUP_TARGET:-host}" == "guest" || "${BACKUP_MODE:-}" =~ ^(guest|veeam-guest)$ ]]; then + die "Guest-mode Veeam jobs were removed. Use --backup-target host / --backup-mode host (datadisk)." +fi + +# Host mode (default): one Veeam Agent job on this KVM hypervisor (e.g. 10.10.31.2). +if [[ -z "$JOB_NAME" ]]; then + JOB_NAME="Mold ${KVM_HOSTNAME:-$(hostname -s)}" + BACKUP_MODE="host" +fi + +if [[ -z "$JOB_NAME" ]]; then + die "--job-name is required. Edit ${ENV_FILE:-$DEFAULT_ENV_FILE} and set JOB_NAME=... (or VEEAM_JOB_NAME=...), or pass --job-name 'Mold Guest Backup'. Shell variables are not read unless exported and listed in --env-file." +fi +[[ -n "$MOLD_API_URL" && -n "$MOLD_API_KEY" && -n "$MOLD_API_SECRET" ]] \ + || die "API credentials missing — set MOLD_API_KEY/MOLD_API_SECRET in ${DEFAULT_ENV_FILE} or pass --api-key/--api-secret" +[[ -n "$ZONE_ID" ]] \ + || die "zone-id missing — set ZONE_ID in ${DEFAULT_ENV_FILE}, pass --zone-id, or ensure listZones API is reachable" + +veeam_config_print_autofill_summary +echo "" + +if [[ -n "$BACKUP_REPO_ADDRESS" ]]; then + if [[ "$BACKUP_REPO_ADDRESS" == nfs://* ]]; then + BACKUP_REPO_TYPE="nfs" + elif [[ "$BACKUP_REPO_ADDRESS" == cifs://* ]]; then + BACKUP_REPO_TYPE="cifs" + fi + BACKUP_REPO_ADDRESS="${BACKUP_REPO_ADDRESS#nfs://}" + BACKUP_REPO_ADDRESS="${BACKUP_REPO_ADDRESS#cifs://}" +fi + +# Datadisk mode: Mold API may still point at glue-gfs; KVM backups go to /data/backup. +if [[ "${BACKUP_STORAGE_MODE:-datadisk}" == "datadisk" && "${BACKUP_REPO_ADDRESS:-}" == *glue-gfs* ]]; then + if [[ -d /data/backup ]]; then + echo "WARN: API/GFS repo ${BACKUP_REPO_ADDRESS} → /data/backup (datadisk mode; no NAS restore)" + BACKUP_REPO_ADDRESS="/data/backup" + MOLD_DATADISK_PATH="/data/backup" + fi +fi + +install -d -m 0700 "${ETC_DIR}" "${ETC_DIR}/secrets" "${ETC_DIR}/state" "${ETC_DIR}/registry" + +ENC_FILE="${ETC_DIR}/secrets/secret.enc" +MOLD_API_SECRET_LINE="" +MOLD_API_SECRET_ENC_FILE_LINE="" +MOLD_SECRET_KEY_FILE_LINE="" + +if [[ "$ENCRYPT_SECRET" == "true" ]]; then + [[ -x "$SECRET_SCRIPT" ]] || chmod +x "$SECRET_SCRIPT" + KEY_FILE="${SECRET_KEY_FILE:-$DEFAULT_SECRET_KEY_FILE}" + [[ -f "$KEY_FILE" ]] || die "Secret key file not found: $KEY_FILE" + "$SECRET_SCRIPT" encrypt --secret "$MOLD_API_SECRET" --key-file "$KEY_FILE" --out "$ENC_FILE" >/dev/null + MOLD_API_SECRET_LINE='MOLD_API_SECRET=""' + MOLD_API_SECRET_ENC_FILE_LINE="MOLD_API_SECRET_ENC_FILE=\"${ENC_FILE}\"" + MOLD_SECRET_KEY_FILE_LINE="MOLD_SECRET_KEY_FILE=\"${KEY_FILE}\"" +else + MOLD_API_SECRET_LINE="MOLD_API_SECRET=\"${MOLD_API_SECRET}\"" + MOLD_API_SECRET_ENC_FILE_LINE='MOLD_API_SECRET_ENC_FILE=""' + MOLD_SECRET_KEY_FILE_LINE='MOLD_SECRET_KEY_FILE=""' +fi + +safe_job="$(echo "$JOB_NAME" | tr ' /' '__')" +CONF="${ETC_DIR}/${safe_job}.conf" +cat > "$CONF" < start matching Veeam Agent job. +VEEAM_TRIGGER_ENABLED="${VEEAM_TRIGGER_ENABLED:-$([[ "${BACKUP_MODE}" == guest ]] && echo true || echo false)}" +VEEAM_TRIGGER_TTL="${VEEAM_TRIGGER_TTL:-1800}" +VEEAM_TRIGGER_METHOD="${VEEAM_TRIGGER_METHOD:-$([[ "${BACKUP_MODE}" == guest ]] && echo ssh || echo auto)}" +VEEAM_API_URL="${VEEAM_API_URL:-}" +VEEAM_API_VERSION="${VEEAM_API_VERSION:-1.2-rev0}" +VEEAM_API_USER="${VEEAM_API_USER:-${VEEAM_USERNAME:-}}" +VEEAM_API_PASSWORD="${VEEAM_API_PASSWORD:-${VEEAM_PASSWORD:-}}" +VM_TARGETS="${VM_TARGETS:-}" +VEEAM_GUEST_JOB_PREFIX="${VEEAM_GUEST_JOB_PREFIX:-Mold VM}" + +VEEAM_HOST_BACKUP_PATH="/tmp/mold/veeam" +STAGING_PATH="/tmp/mold/veeam" +VEEAM_BACKUP_MODE="filelevel" +BACKUP_MODE="${BACKUP_MODE}" +IMPORT_MODE="${IMPORT_MODE}" + +BACKUP_REPO_TYPE="${BACKUP_REPO_TYPE}" +BACKUP_REPO_NAME="${BACKUP_REPO_NAME}" +BACKUP_REPO_ADDRESS="${BACKUP_REPO_ADDRESS}" +BACKUP_REPO_MOUNT_OPTS="${BACKUP_REPO_MOUNT_OPTS}" +BACKUP_REPO_PROVIDER="${BACKUP_REPO_PROVIDER}" +NAS_REPO_MOUNT="${NAS_REPO_MOUNT}" +BACKUP_STORAGE_MODE="${BACKUP_STORAGE_MODE:-datadisk}" +MOLD_DATADISK_PATH="${MOLD_DATADISK_PATH:-${BACKUP_REPO_ADDRESS}}" +VEEAM_HOST_REPO_ROOT="${VEEAM_HOST_REPO_ROOT:-E:/opt1/veeam}" +VEEAM_REPO_NAME="${VEEAM_REPO_NAME:-Mold ${KVM_HOSTNAME:-}}" + +BACKUP_ID="" +VM_UUID="${VM_UUID_CFG}" +VM_NAME="${VM_NAME_CFG}" + +CLEANUP_STAGING_AFTER_BACKUP="true" +CLEANUP_STAGING_ON_ERROR="true" + +NAS_BACKUP_SCRIPT="/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/ablestack_nasbackup.sh" +CVT_BACKUP_SCRIPT="/etc/ablestack/veeam/ablestack_cvtbackup.sh" +LOG_FILE="/var/log/mold/veeam-hook.log" +LOG_TAG="mold-veeam-hook" +EOF +chmod 0600 "$CONF" + +# Job-specific hook wrappers (NetBackup bpstart_notify. pattern) +HOOKS_DIR="${ETC_DIR}/hooks" +install -d -m 0755 "${HOOKS_DIR}" +for hook in pre post restore; do + target="${HOOKS_DIR}/${hook}-notify.${safe_job}" + cat > "$target" < "$target" < "$WIN_CONF" < "$MANIFEST" < '${ETC_DIR}/ablestack_veeam_pre_notify.sh' \$(hostname) '${JOB_NAME}' +# Post: ssh root@ '${ETC_DIR}/ablestack_veeam_post_notify.sh' \$(hostname) '${JOB_NAME}' + +# Or on Veeam server (VM file-level job — NOT entire host): +# pwsh -File setup-veeam-mold-job.ps1 -ConfPath mold-backup.windows.conf +# Or from Mac/ccvm: +# bash push-to-veeam.sh --env-file mold-backup.env +EOF + +if [[ "$CONFIGURE_MOLD" == "true" ]]; then + # shellcheck source=mold-backup.lib.sh + source "${SCRIPT_DIR}/mold-backup.lib.sh" + export MOLD_BACKUP_CONF="$CONF" + export VEEAM_JOB_NAME="$JOB_NAME" + mold_backup_load_config || true + if mold_backup_cmk_bin >/dev/null 2>&1 || command -v curl >/dev/null 2>&1; then + mold_backup_api_ensure_global_settings || true + if mold_backup_is_datadisk_mode; then + mold_backup_notify_log info "datadisk mode: skipping addBackupRepository/importBackupOffering (assign offering in Mold UI)" + else + mold_backup_api_ensure_backup_resources >/dev/null \ + || mold_backup_notify_log warn "addBackupRepository/importBackupOffering skipped or failed (Admin API key + BACKUP_REPO_ADDRESS + ZONE_ID required)" + fi + fi +fi + +echo "Created:" +echo " ${CONF}" +echo " ${WIN_CONF}" +echo " ${MANIFEST}" +echo " ${HOOKS_DIR}/pre-notify.${safe_job}" +echo "" +if [[ "${VEEAM_BACKUP_TARGET:-host}" == "guest" ]]; then + die "Guest-mode Veeam jobs were removed. Use --backup-target host (datadisk) instead." +else + echo "Veeam job backup selections: ${VEEAM_HOST_BACKUP_PATH:-/tmp/mold/veeam}/" + echo "Next: bash push-to-veeam.sh (host Agent on KVM)" + echo "FLR→Mold: bash ${ETC_DIR}/enable-veeam-mold-restore.sh" +fi + +if [[ "$RUN_INSTALL" == "true" && -x "${SCRIPT_DIR}/install.sh" ]]; then + ABLESTACK_VEEAM_ETC_DIR="$ETC_DIR" bash "${SCRIPT_DIR}/install.sh" +fi + +echo "Done." diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 3de48eede78b..7d7e9eefa722 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -53,8 +53,14 @@ import org.apache.cloudstack.api.command.admin.backup.UpdateNetBackupCmd; import org.apache.cloudstack.api.command.admin.vm.CreateVMFromBackupCmdByAdmin; import org.apache.cloudstack.api.command.user.backup.AssignVirtualMachineToBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateAblestackVeeamBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateNetBackupCmd; +import org.apache.cloudstack.api.command.user.backup.ImportAblestackVeeamBackupSeedCmd; +import org.apache.cloudstack.api.command.user.backup.ListAblestackVeeamBackupsCmd; +import org.apache.cloudstack.api.command.user.backup.ListVeeamRestorePointsCmd; +import org.apache.cloudstack.api.command.user.backup.RestoreAblestackVeeamBackupCmd; +import org.apache.cloudstack.api.response.BackupRestorePointResponse; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; @@ -795,8 +801,9 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { if (!BackupProviderNameUtils.isNasFamily(offering.getProvider()) && !BackupProviderNameUtils.isCommvaultFamily(offering.getProvider()) && + !BackupProviderNameUtils.isVeeamFamily(offering.getProvider()) && cmd.getQuiesceVM() != null) { - throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault backup provider"); + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault, and Ablestack Veeam backup providers"); } final String timezoneId = timeZone.getID(); @@ -1000,8 +1007,9 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo if (!BackupProviderNameUtils.isNasFamily(offering.getProvider()) && !BackupProviderNameUtils.isCommvaultFamily(offering.getProvider()) && + !BackupProviderNameUtils.isVeeamFamily(offering.getProvider()) && cmd.getQuiesceVM() != null) { - throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault backup provider"); + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS, Commvault, and Ablestack Veeam backup providers"); } Long backupScheduleId = getBackupScheduleId(job); @@ -1029,6 +1037,176 @@ public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllo return true; } + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "importing Ablestack Veeam backup seed", async = true) + public Backup importAblestackVeeamBackupSeed(ImportAblestackVeeamBackupSeedCmd cmd) throws ResourceAllocationException { + final VMInstanceVO vm = findVmById(cmd.getVmId()); + validateBackupForZone(vm.getDataCenterId()); + final Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, null, true, vm); + + if (vm.getBackupOfferingId() == null) { + throw new CloudRuntimeException("VM must be assigned to an Ablestack Veeam backup offering before importing a seed"); + } + + final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); + if (offering == null || !BackupProviderNameUtils.isVeeamFamily(offering.getProvider())) { + throw new CloudRuntimeException("VM backup offering must use the ablestack-veeam provider"); + } + + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + + List stagingPaths = null; + if (org.apache.commons.lang3.StringUtils.isNotBlank(cmd.getStagingDiskPaths())) { + stagingPaths = Arrays.stream(cmd.getStagingDiskPaths().split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + final Pair result = backupProvider.importAblestackVeeamBackupSeed( + vm, cmd.getVeeamRestorePointId(), stagingPaths, cmd.getSourceDiskFormat(), cmd.getBootstrapCheckpoint()); + + if (!result.first() || result.second() == null) { + throw new CloudRuntimeException("Failed to import Ablestack Veeam backup seed"); + } + + BackupVO backupVO = backupDao.findById(result.second().getId()); + if (cmd.getName() != null) { + backupVO.setName(cmd.getName()); + backupDao.update(backupVO.getId(), backupVO); + } + + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + if (result.second().getSize() != null) { + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, result.second().getSize()); + } + return backupVO; + } + + private BackupOffering validateVmAblestackVeeamOffering(final VMInstanceVO vm) { + if (vm.getBackupOfferingId() == null) { + throw new CloudRuntimeException("VM must be assigned to an Ablestack Veeam backup offering"); + } + final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); + if (offering == null || !BackupProviderNameUtils.isVeeamFamily(offering.getProvider())) { + throw new CloudRuntimeException("VM backup offering must use the ablestack-veeam provider"); + } + return offering; + } + + @Override + public List listVeeamRestorePoints(final ListVeeamRestorePointsCmd cmd) { + final VMInstanceVO vm = findVmById(cmd.getVmId()); + validateBackupForZone(vm.getDataCenterId()); + final Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, null, true, vm); + validateVmAblestackVeeamOffering(vm); + final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + return backupProvider.listRestorePoints(vm); + } + + @Override + public List createVeeamRestorePointResponses(final List points) { + final List responses = new ArrayList<>(); + if (points == null) { + return responses; + } + for (final Backup.RestorePoint point : points) { + final BackupRestorePointResponse response = new BackupRestorePointResponse(); + response.setId(point.getId()); + response.setCreated(point.getCreated()); + response.setType(point.getType()); + response.setObjectName(point.getId()); + responses.add(response); + } + return responses; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating Ablestack Veeam NAS backup", async = true) + public boolean createAblestackVeeamBackup(final CreateAblestackVeeamBackupCmd cmd, final Object job) + throws ResourceAllocationException { + final Long vmId = cmd.getVmId(); + final Account caller = CallContext.current().getCallingAccount(); + final VMInstanceVO vm = findVmById(vmId); + validateBackupForZone(vm.getDataCenterId()); + accountManager.checkAccess(caller, null, true, vm); + final BackupOffering offering = validateVmAblestackVeeamOffering(vm); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + final Account owner = accountManager.getAccount(vm.getAccountId()); + Long backupSize = 0L; + for (final Volume volume : volumeDao.findByInstance(vmId)) { + if (Volume.State.Ready.equals(volume.getState())) { + Long volumeSize = volumeApiService.getVolumePhysicalSize(volume.getFormat(), volume.getPath(), volume.getChainInfo()); + if (volumeSize == null) { + volumeSize = volume.getSize(); + } + backupSize += volumeSize; + } + } + createCheckedBackupVeeam(vm, vmId, backupProvider, cmd.getQuiesceVM(), backupSize, owner, getBackupScheduleId(job), + cmd.getName()); + return true; + } + + private void createCheckedBackupVeeam(final VMInstanceVO vm, final Long vmId, final BackupProvider backupProvider, + final Boolean quiesceVM, final Long backupSize, final Account owner, final Long backupScheduleId, + final String backupName) + throws ResourceAllocationException { + try (CheckedReservation backupReservation = new CheckedReservation(owner, Resource.ResourceType.backup, + 1L, reservationDao, resourceLimitMgr); + CheckedReservation backupStorageReservation = new CheckedReservation(owner, + Resource.ResourceType.backup_storage, backupSize, reservationDao, resourceLimitMgr)) { + + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), + EventTypes.EVENT_VM_BACKUP_CREATE, "creating Ablestack Veeam backup for VM ID:" + vm.getUuid(), + vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); + + final Pair result = backupProvider.takeBackup(vm, quiesceVM); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create Ablestack Veeam VM backup"); + } + final Backup backup = result.second(); + if (backup != null) { + final BackupVO vmBackup = backupDao.findById(backup.getId()); + vmBackup.setBackupScheduleId(backupScheduleId); + if (backupName != null) { + vmBackup.setName(backupName); + } + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + } + } + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring VM from Ablestack Veeam backup", async = true) + public boolean restoreAblestackVeeamBackup(final Long backupId) { + final BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup " + backupId + " does not exist"); + } + final BackupOffering backupOffering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (backupOffering == null || !BackupProviderNameUtils.isVeeamFamily(backupOffering.getProvider())) { + throw new CloudRuntimeException("Backup is not from an ablestack-veeam offering"); + } + return restoreBackup(backupId); + } + + @Override + public Pair, Integer> listAblestackVeeamBackups(final ListAblestackVeeamBackupsCmd cmd) { + final VMInstanceVO vm = findVmById(cmd.getVmId()); + validateBackupForZone(vm.getDataCenterId()); + final Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, null, true, vm); + final BackupOffering offering = validateVmAblestackVeeamOffering(vm); + final List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), offering.getId()); + return new Pair<>(backups, backups.size()); + } + private void createCheckedBackup(CreateBackupCmd cmd, Account owner, boolean isScheduledBackup, Long backupSize, VMInstanceVO vm, Long vmId, BackupProvider backupProvider, Long backupScheduleId) throws ResourceAllocationException { @@ -2739,6 +2917,11 @@ public List> getCommands() { // Operations cmdList.add(CreateBackupCmd.class); cmdList.add(CreateNetBackupCmd.class); + cmdList.add(ImportAblestackVeeamBackupSeedCmd.class); + cmdList.add(ListVeeamRestorePointsCmd.class); + cmdList.add(CreateAblestackVeeamBackupCmd.class); + cmdList.add(RestoreAblestackVeeamBackupCmd.class); + cmdList.add(ListAblestackVeeamBackupsCmd.class); cmdList.add(ListBackupsCmd.class); cmdList.add(RestoreBackupCmd.class); cmdList.add(PrepareNetBackupRestoreCmd.class);