From c7b618969c45890f331d768bdf2d8ff6e5b21aa6 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Thu, 5 Nov 2026 12:12:03 +0100 Subject: [PATCH 1/4] backup: let storage drivers export volumes for KVM backups (LINSTOR) The KVM backup export (Veeam) only works for file-based pools: the hypervisor reads the volume files through a libvirt backup job, and the volume path is put together from the pool's mount point. Volumes on block storage like LINSTOR can't be read that way. Storage drivers can now serve the volumes of their pools themselves (new default methods on PrimaryDataStoreDriver): startBackupExport takes one crash consistent storage snapshot of the VM's volumes and makes each volume readable as a raw device on a host, finishBackupExport releases it, prepareVolumeForRestore makes a volume writable for a restore. For such VMs KVMBackupExportServiceImpl skips the libvirt backup job and the hypervisor checkpoints, keeps the per-volume export in backup_details and serves each volume with its own qemu-nbd on the host the driver chose, running and stopped VMs alike. VMs whose volumes mix both kinds of pools are rejected. qemu-nbd exports a block device as raw up to the volume size, as the device can be larger than the volume. Backups of these volumes can be incremental. The driver gets, per volume, the change tracking id of the volume's last backup at the VM's active checkpoint (volume detail backup.change.tracking.id, ":") and can mark the export incremental. It can also name an extents source: the host's storage adaptor writes the extents of the export with the flags dirty (changed since the last backup) and zero (reads as zeros) to a file, which the image server serves as the extents of the transfer instead of a dirty bitmap and the allocation of the export. Without one, all of the export counts as changed data. LINSTOR implements it with changed block tracking snapshots (REST API 1.30): one multi snapshot with the CBT flag, a CBT session per volume on a host with a diskful copy, and the snapshot is deleted once the sessions are finished. The extents come from the CBT session of the snapshot: in mode dirty for an incremental backup, only if DRBD tracked the changes since the last backup's CBT id (else, e.g. after a failed backup, the volume is backed up in full), and in mode zero for a full backup if DRBD knows the never written extents (drbd-utils 9.36.0 and newer). Listing the dirty extents fails if they are not relative to that CBT id. The Veeam adapter accepts pools whose driver exports the volumes and reports LINSTOR pools as block storage domains. Their volumes are reported as cow (QCOW2) disks with backup incremental, whatever image format CloudStack records for them, as Veeam backs up only QCOW2 disks incrementally; their data goes through the image transfers as raw like for any disk. java-linstor 0.9.0 brings REST API 1.30; its resourceSnapshotDelete has an additional argument. --- .../backup/KVMBackupExportService.java | 7 + .../backup/CreateImageTransferCommand.java | 29 ++ .../backup/StartNBDServerCommand.java | 16 + .../api/storage/PrimaryDataStoreDriver.java | 38 +++ .../api/storage/VolumeBackupExport.java | 106 +++++++ ...virtCreateImageTransferCommandWrapper.java | 41 +++ .../LibvirtStartNBDServerCommandWrapper.java | 6 +- .../kvm/storage/KVMStoragePool.java | 14 + ...CreateImageTransferCommandWrapperTest.java | 64 ++++ ...bvirtStartNBDServerCommandWrapperTest.java | 29 ++ .../veeam/adapter/ServerAdapter.java | 56 +++- .../StoreVOToStorageDomainConverter.java | 2 + .../VolumeJoinVOToDiskConverter.java | 44 ++- .../VolumeJoinVOToDiskConverterTest.java | 18 ++ .../kvm/storage/LinstorStorageAdaptor.java | 61 ++++ .../kvm/storage/LinstorStoragePool.java | 5 + .../LinstorPrimaryDataStoreDriverImpl.java | 276 ++++++++++++++++- .../storage/datastore/util/LinstorUtil.java | 54 ++++ .../snapshot/LinstorVMSnapshotStrategy.java | 2 +- .../storage/LinstorStorageAdaptorTest.java | 92 ++++++ ...LinstorPrimaryDataStoreDriverImplTest.java | 278 +++++++++++++++++ .../datastore/util/LinstorUtilTest.java | 47 +++ pom.xml | 2 +- .../kvm/imageserver/backends/__init__.py | 1 + .../kvm/imageserver/backends/nbd.py | 20 +- .../vm/hypervisor/kvm/imageserver/config.py | 9 + .../vm/hypervisor/kvm/imageserver/handler.py | 4 +- .../kvm/imageserver/tests/test_nbd_backend.py | 62 ++++ .../imageserver/tests/test_registry_idle.py | 27 ++ .../kvm/imageserver/tests/test_util.py | 111 +++++++ scripts/vm/hypervisor/kvm/imageserver/util.py | 78 ++++- .../backup/KVMBackupExportServiceImpl.java | 293 ++++++++++++++++-- .../KVMBackupExportServiceImplTest.java | 263 +++++++++++++++- 33 files changed, 2097 insertions(+), 58 deletions(-) create mode 100644 engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index e1affb311ec6..e4df538fc189 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.storage.StoragePool; import com.cloud.utils.component.PluggableService; /** @@ -103,4 +104,10 @@ public interface KVMBackupExportService extends Configurable, PluggableService { * List Compatible Data Center Ids for the service */ List listCompatibleDataCenterIds(); + + /** + * Whether the storage driver of the pool exports the pool's volumes for backups itself (block storage), + * instead of the hypervisor reading the volume files + */ + boolean isStorageDriverBackupExportSupported(StoragePool pool); } diff --git a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java index 95b56c9a9c38..e6baa15f0be1 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/CreateImageTransferCommand.java @@ -17,7 +17,10 @@ package org.apache.cloudstack.backup; +import java.util.Map; + import com.cloud.agent.api.Command; +import com.cloud.storage.Storage; public class CreateImageTransferCommand extends Command { private String transferId; @@ -28,6 +31,9 @@ public class CreateImageTransferCommand extends Command { private String file; private ImageTransfer.Backend backend; private int idleTimeoutSeconds; + private Storage.StoragePoolType extentsPoolType; + private String extentsPoolUuid; + private Map extentsSource; public CreateImageTransferCommand() { } @@ -91,4 +97,27 @@ public String getCheckpointId() { public int getIdleTimeoutSeconds() { return idleTimeoutSeconds; } + + /** + * Has the pool's storage list the extents of the export, the ones that changed since the last backup and the + * ones that read as zeros, instead of a dirty bitmap (see the checkpoint id) and the allocation of the export. + * @param source driver specific parameters, see {@code KVMStoragePool.writeExtents} + */ + public void setExtentsSource(Storage.StoragePoolType poolType, String poolUuid, Map source) { + this.extentsPoolType = poolType; + this.extentsPoolUuid = poolUuid; + this.extentsSource = source; + } + + public Storage.StoragePoolType getExtentsPoolType() { + return extentsPoolType; + } + + public String getExtentsPoolUuid() { + return extentsPoolUuid; + } + + public Map getExtentsSource() { + return extentsSource; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java index 07e50f11fd37..7a87ddf1458f 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartNBDServerCommand.java @@ -29,6 +29,7 @@ public class StartNBDServerCommand extends Command { private String fromCheckpointId; @LogLevel(LogLevel.Log4jLevel.Off) private byte[] passphrase; + private Long blockDeviceSizeBytes; public StartNBDServerCommand() { } @@ -43,6 +44,14 @@ protected StartNBDServerCommand(String transferId, String exportName, String vol this.passphrase = passphrase; } + /** + * Exports a raw block device. Only its first sizeBytes are exported, as the device can be larger than the volume. + */ + protected StartNBDServerCommand(String transferId, String exportName, String devicePath, long sizeBytes, String socket, String direction) { + this(transferId, exportName, devicePath, socket, direction, null, null); + this.blockDeviceSizeBytes = sizeBytes; + } + public String getExportName() { return exportName; } @@ -75,4 +84,11 @@ public String getFromCheckpointId() { public byte[] getPassphrase() { return passphrase; } + + /** + * Size to export if the volume path is a raw block device, null if it is an image file. + */ + public Long getBlockDeviceSizeBytes() { + return blockDeviceSizeBytes; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java index 37e2d2dd4b12..d124caac462a 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java @@ -249,4 +249,42 @@ default List listVolumesForImport(StoragePool pool, String vo */ default void volumeManagementChanged(StoragePool pool, Volume volume, boolean imported) { } + + /** + * Whether the driver serves the volumes of the pool for image based backups (KVM backup export) from + * its own storage snapshots, see {@link #startBackupExport}, instead of the hypervisor reading the volume files. + */ + default boolean canExportVolumesForBackup(StoragePool pool) { + return false; + } + + /** + * Takes one crash consistent storage snapshot of the volumes and makes each volume of it readable as a raw + * device on a host. {@link #finishBackupExport} has to be called afterwards in any case, also if this fails. + * + * @param exportName name of the export, unique per backup + * @param volumes volumes of one VM, all on this pool + * @param changeTrackingIds per volume id, the change tracking id of the last successful backup of the volume + * (see {@link VolumeBackupExport#getChangeTrackingId()}), missing if there is none + * @return per volume where to read it + */ + default List startBackupExport(StoragePool pool, String exportName, List volumes, + Map changeTrackingIds) { + throw new UnsupportedOperationException("The storage driver does not export volumes for backups"); + } + + /** + * Releases everything {@link #startBackupExport} set up for the export (devices, snapshot). Does nothing for + * parts that are already released, so it can be called again after a failure. + */ + default void finishBackupExport(StoragePool pool, String exportName, List volumes) { + throw new UnsupportedOperationException("The storage driver does not export volumes for backups"); + } + + /** + * Makes the volume writable as a raw device on a host, to restore a backup into it. + */ + default VolumeBackupExport prepareVolumeForRestore(StoragePool pool, VolumeInfo volume) { + throw new UnsupportedOperationException("The storage driver does not export volumes for backups"); + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java new file mode 100644 index 000000000000..2f1c126611bc --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeBackupExport.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +import java.util.Map; + +/** + * Where a storage driver made a volume accessible as a raw device for an image based backup + * (read only, see {@link PrimaryDataStoreDriver#startBackupExport}) or a restore into it + * (see {@link PrimaryDataStoreDriver#prepareVolumeForRestore}). + */ +public class VolumeBackupExport { + private long volumeId; + private long hostId; + private String devicePath; + private long sizeBytes; + private boolean incremental; + private String changeTrackingId; + private Map extentsSource; + + public VolumeBackupExport() { + } + + public VolumeBackupExport(long volumeId, long hostId, String devicePath, long sizeBytes) { + this.volumeId = volumeId; + this.hostId = hostId; + this.devicePath = devicePath; + this.sizeBytes = sizeBytes; + } + + public long getVolumeId() { + return volumeId; + } + + /** + * The host the device is accessible on. + */ + public long getHostId() { + return hostId; + } + + public String getDevicePath() { + return devicePath; + } + + /** + * Size of the volume in bytes. The device can be larger, only this much of it is the volume's data. + */ + public long getSizeBytes() { + return sizeBytes; + } + + /** + * True if only the extents {@link #getExtentsSource()} lists as dirty changed since the backup with the + * change tracking id given to {@link PrimaryDataStoreDriver#startBackupExport}, false if the whole volume + * has to be read. + */ + public boolean isIncremental() { + return incremental; + } + + public void setIncremental(boolean incremental) { + this.incremental = incremental; + } + + /** + * The storage's change tracking id of this backup, to pass to the next {@link PrimaryDataStoreDriver#startBackupExport} + * once this backup succeeded. Null if the storage does not track changes for the volume. + */ + public String getChangeTrackingId() { + return changeTrackingId; + } + + public void setChangeTrackingId(String changeTrackingId) { + this.changeTrackingId = changeTrackingId; + } + + /** + * Driver specific parameters the host's storage adaptor uses to list the extents of the export: the ones that + * changed (of an incremental export) and the ones that read as zeros. Null if the storage lists none, then the + * whole volume counts as changed data. + */ + public Map getExtentsSource() { + return extentsSource; + } + + public void setExtentsSource(Map extentsSource) { + this.extentsSource = extentsSource; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java index 7cf05da9b211..481e54a09f82 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapper.java @@ -18,6 +18,9 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -25,15 +28,18 @@ import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.ImageTransfer; import org.apache.cloudstack.storage.resource.IpTablesHelper; +import org.apache.commons.collections.MapUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.StringUtils; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @ResourceWrapper(handles = CreateImageTransferCommand.class) @@ -42,6 +48,23 @@ public class LibvirtCreateImageTransferCommandWrapper extends CommandWrapper 0) { + if (cmd.getBlockDeviceSizeBytes() != null) { + // the device can be larger than the volume (e.g. DRBD), so only export the volume's size of it + imageArg = String.format("--image-opts driver=raw,size=%d,file.driver=host_device,file.filename=%s", + cmd.getBlockDeviceSizeBytes(), volumePath); + } else if (passphrase != null && passphrase.length > 0) { KeyFile srcKey = new KeyFile(passphrase); secretArg = String.format("--object secret,id=sec0,file=%s", srcKey); imageArg = String.format("--image-opts driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", volumePath); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 58665fdb5766..13d02aff24c7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -202,4 +202,18 @@ default void deleteBackupScratch(String name) { throw new CloudRuntimeException(String.format("Storage pool %s of type %s cannot provide backup scratch space", getUuid(), getType())); } + + /** + * Writes the extents of a volume's backup export, as the storage lists them, to {@code file}: a JSON array of + * extents like the ones of the ovirt-imageio extents API, objects with {@code start} and {@code length} in bytes + * and the flags {@code dirty} (changed since the last backup) and {@code zero} (reads as zeros). An extent + * without the dirty flag counts as changed, one without the zero flag as data. The image server reports them + * as the extents of the transfer. + * + * @param source the storage driver's parameters naming the volume and its backup + */ + default void writeExtents(Map source, String file) { + throw new CloudRuntimeException(String.format("Storage pool %s of type %s does not list the extents of its volumes", + getUuid(), getType())); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java index 8782f592aed3..8d04f45a4cb8 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateImageTransferCommandWrapperTest.java @@ -17,8 +17,11 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import java.util.Map; + import org.apache.cloudstack.backup.CreateImageTransferAnswer; import org.apache.cloudstack.backup.CreateImageTransferCommand; import org.apache.cloudstack.backup.ImageTransfer; @@ -34,6 +37,10 @@ import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.ImageServerControlSocket; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @RunWith(MockitoJUnitRunner.class) @@ -118,4 +125,61 @@ public void testExecuteRegisterTransferFailureReturnsFailure() { Assert.assertEquals("Failed to register transfer with image server.", answer.getDetails()); } } + + private KVMStoragePool mockNbdTransferWithExtentsSource(String transferId, Map source) { + Mockito.when(command.getTransferId()).thenReturn(transferId); + Mockito.when(command.getBackend()).thenReturn(ImageTransfer.Backend.nbd); + Mockito.when(command.getSocket()).thenReturn("sock-3"); + Mockito.when(command.getExportName()).thenReturn("vol-3"); + Mockito.when(command.getExtentsSource()).thenReturn(source); + Mockito.when(command.getExtentsPoolType()).thenReturn(Storage.StoragePoolType.Linstor); + Mockito.when(command.getExtentsPoolUuid()).thenReturn("pool-1"); + + Mockito.when(resource.getImageServerPath()).thenReturn("/opt/cloudstack/image/server.py"); + Mockito.when(resource.getImageServerListenAddress()).thenReturn("192.168.10.10"); + KVMStoragePoolManager poolManager = Mockito.mock(KVMStoragePoolManager.class); + KVMStoragePool pool = Mockito.mock(KVMStoragePool.class); + Mockito.when(resource.getStoragePoolMgr()).thenReturn(poolManager); + Mockito.when(poolManager.getStoragePool(Storage.StoragePoolType.Linstor, "pool-1")).thenReturn(pool); + return pool; + } + + @Test + public void testExecuteWithExtentsSourceHasImageServerReadTheListedExtents() { + Map source = Map.of("snapshot", "ckp-1"); + KVMStoragePool pool = mockNbdTransferWithExtentsSource("tr-3", source); + + try (MockedConstruction