diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..c5f68918fd56 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,12 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + +4.23.0.0 > 24.0.0: + * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd + LUKS2 (), for both data disks + and root disks. Encryption is transparent to the guest and reuses the + existing CloudStack volume-encryption passphrase handling, so no + additional key store is required. Note: attaching an encrypted RBD volume + to a running Instance requires libvirt >= 10.1.0; booting an Instance from + an encrypted RBD root disk works on older libvirt. diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index c110e4ca94e1..cca2edd70c6b 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -56,6 +56,7 @@ public static String[] toStrings(Host.Type... types) { String HOST_UEFI_ENABLE = "host.uefi.enable"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + String HOST_RBD_VOLUME_ENCRYPTION = "host.volume.encryption.rbd"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; String HOST_VDDK_SUPPORT = "host.vddk.support"; String HOST_VDDK_LIB_DIR = "vddk.lib.dir"; diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 3511b4e88cb9..275f6d25268d 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -172,7 +172,7 @@ public static enum StoragePoolType { LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR CLVM(true, false, EncryptionSupport.Unsupported), CLVM_NG(true, false, EncryptionSupport.Hypervisor), - RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD + RBD(true, true, EncryptionSupport.Hypervisor), // http://libvirt.org/storage.html#StorageBackendRBD ; encrypted natively by librbd (LUKS2, engine='librbd') SharedMountPoint(true, true, EncryptionSupport.Hypervisor), VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers. diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index db7dcc3fb44f..a32b8dd604e7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -92,7 +92,7 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { @Parameter(name = ApiConstants.DISK_PATH, type = CommandType.STRING, - description = "path of the disk image") + description = "path of the disk image. It is the file name on file based storage pools (NFS, Local, SharedMountPoint), and the image name on RBD storage pools") private String diskPath; @Parameter(name = ApiConstants.IMPORT_SOURCE, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index 50f4b9c1fbe5..db5961ac7f8b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -63,7 +63,7 @@ public class ImportVolumeCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.PATH, type = BaseCmd.CommandType.STRING, required = true, - description = "the path of the volume") + description = "the path of the volume. It is the file name on file based storage pools (NFS, Local, SharedMountPoint), and the image name on RBD storage pools") private String path; @Parameter(name = ApiConstants.NAME, diff --git a/debian/control b/debian/control index cdf663ef8906..64531543a6d0 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat, openssl, bzip2, gzip, unzip Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 141596407fe8..8614197347aa 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -183,10 +183,12 @@ List allocateTemplatedVolumes(Type type, String name, DiskOffering */ DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo); + Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, + Storage.ImageFormat format); DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile); + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile, + Storage.ImageFormat format); /** * Unmanage VM volumes diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index 53bfcce27038..f4f8d342eda7 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -33,7 +33,8 @@ */ public interface HighAvailabilityManager extends Manager { - List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint, + StoragePoolType.RBD); ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 2fad96ec1da2..4f163b9a1c9e 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -2626,7 +2626,8 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo, @Override public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, + ImageFormat format) { if (sizeInBytes == null) { sizeInBytes = offering.getDiskSize(); } @@ -2665,7 +2666,9 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(hypervisorType)); + // The format the hypervisor actually reported for the existing image wins; pools such as RBD + // hold raw images even though QCOW2 is the cluster default for KVM. + vol.setFormat(format != null ? format : getSupportedImageFormatForCluster(hypervisorType)); vol.setPoolId(poolId); vol.setPoolType(poolType); vol.setPath(path); @@ -2677,7 +2680,8 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L @Override public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile, + ImageFormat format) { VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId()); if (vm != null) { @@ -2709,7 +2713,9 @@ public DiskProfile updateImportedVolume(Type type, DiskOffering offering, Virtua vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); + // The format the hypervisor actually reported for the existing image wins; pools such as RBD + // hold raw images even though QCOW2 is the cluster default for KVM. + vol.setFormat(format != null ? format : getSupportedImageFormatForCluster(vm.getHypervisorType())); vol.setPoolId(poolId); vol.setPoolType(poolType); vol.setPath(path); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index d61caf16ae5d..8336c5ae5ef7 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -246,7 +246,7 @@ public void testImportVolume() { volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null, zoneId, hypervisorType, null, null, owner, - deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo); + deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo, null); VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0); Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong()); diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 3ba2e4d5789e..f0f9438da09b 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -129,6 +129,10 @@ Requires: (selinux-tools if selinux-tools) Requires: sysstat Requires: python3-libnbd Requires: socat +Requires: openssl +Requires: bzip2 +Requires: gzip +Requires: unzip Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec index cdfc5a72a34e..911215b13b92 100644 --- a/packaging/suse15/cloud.spec +++ b/packaging/suse15/cloud.spec @@ -127,6 +127,10 @@ Requires: rng-tools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Requires: (selinux-tools if selinux-tools) Requires: sysstat +Requires: openssl +Requires: bzip2 +Requires: gzip +Requires: unzip Provides: cloud-agent Group: System Environment/Libraries %description agent diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 9f1b849e9727..7a7662f8b72d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -130,7 +130,7 @@ private void checkForNotExistingLibvirtStoragePools(Set removedPools, St removedPools.add(uuid); } - logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); + logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 4281036d9456..9946c85cfd9d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -19,6 +19,7 @@ import static com.cloud.host.Host.HOST_CDROM_MAX_COUNT; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_RBD_VOLUME_ENCRYPTION; import static com.cloud.host.Host.HOST_VDDK_LIB_DIR; import static com.cloud.host.Host.HOST_VDDK_SUPPORT; import static com.cloud.host.Host.HOST_VDDK_VERSION; @@ -91,6 +92,7 @@ import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; import org.apache.cloudstack.utils.linux.CPUStat; import org.apache.cloudstack.utils.linux.KVMHostInfo; @@ -3882,7 +3884,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { if (volumeObjectTO.requiresEncryption() && pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) { String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase()); - DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat())); + // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. + String encryptEngine = (pool.getType() == StoragePoolType.RBD) ? "librbd" : null; + DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()), encryptEngine); disk.setLibvirtDiskEncryptDetails(encryptDetails); } } @@ -4410,6 +4414,7 @@ public StartupCommand[] initialize() { cmd.setGatewayIpAddress(localGateway); cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); + cmd.getHostDetails().put(HOST_RBD_VOLUME_ENCRYPTION, String.valueOf(hostSupportsRbdVolumeEncryption())); cmd.setHostTags(getHostTags()); boolean instanceConversionSupported = hostSupportsInstanceConversion(); cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); @@ -6195,7 +6200,10 @@ public boolean isHostSecured() { } /** - * Test host for volume encryption support + * Test host for qemu-native LUKS volume encryption (qemu-img LUKS support + cryptsetup), + * reported as {@code host.volume.encryption}. RBD/librbd encryption support is a separate + * capability, reported as {@code host.volume.encryption.rbd} + * (see {@link #hostSupportsRbdVolumeEncryption()}). * @return boolean */ public boolean hostSupportsVolumeEncryption() { @@ -6220,6 +6228,13 @@ public boolean hostSupportsVolumeEncryption() { return true; } + /** + * Test host for librbd native LUKS encryption support (rbd CLI with the encryption subcommand). + */ + public boolean hostSupportsRbdVolumeEncryption() { + return new RbdEncryption().isSupported(); + } + public boolean isSecureMode(String bootMode) { if (StringUtils.isNotBlank(bootMode) && "secure".equalsIgnoreCase(bootMode)) { return true; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 74529d9d5fa2..439e4f663416 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -788,14 +788,21 @@ public static class DiskDef { public static class LibvirtDiskEncryptDetails { String passphraseUuid; QemuObject.EncryptFormat encryptFormat; + String engine; // optional libvirt encryption engine (e.g. "librbd"); null => libvirt/qemu default public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat) { + this(passphraseUuid, encryptFormat, null); + } + + public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat, String engine) { this.passphraseUuid = passphraseUuid; this.encryptFormat = encryptFormat; + this.engine = engine; } public String getPassphraseUuid() { return this.passphraseUuid; } public QemuObject.EncryptFormat getEncryptFormat() { return this.encryptFormat; } + public String getEngine() { return this.engine; } } public static class DiskGeometry { @@ -1446,7 +1453,11 @@ public String toString() { } if (encryptDetails != null) { - diskBuilder.append("\n"); + diskBuilder.append("\n"); diskBuilder.append("\n"); diskBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 6788516df741..3a42f230f845 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -50,7 +50,8 @@ public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.SharedMountPoint); + Storage.StoragePoolType.SharedMountPoint, + Storage.StoragePoolType.RBD); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { @@ -64,14 +65,25 @@ public Answer execute(final CheckVolumeCommand command, final LibvirtComputingRe if (STORAGE_POOL_TYPES_SUPPORTED.contains(storageFilerTO.getType())) { final KVMPhysicalDisk vol = pool.getPhysicalDisk(srcFile); final String path = vol.getPath(); - try { - KVMPhysicalDisk.checkQcow2File(path); - } catch (final CloudRuntimeException e) { - return new CheckVolumeAnswer(command, false, "", 0, getVolumeDetails(pool, vol)); + final boolean isRbd = Storage.StoragePoolType.RBD.equals(storageFilerTO.getType()); + + Map volumeDetails = getVolumeDetails(pool, vol); + if (MapUtils.isEmpty(volumeDetails)) { + return new Answer(command, false, "Unable to read the volume on the storage pool"); + } + + if (!isRbd) { + try { + KVMPhysicalDisk.checkQcow2File(path); + } catch (final CloudRuntimeException e) { + return new CheckVolumeAnswer(command, false, "", 0, volumeDetails); + } } - long size = KVMPhysicalDisk.getVirtualSizeFromFile(path); - return new CheckVolumeAnswer(command, true, "", size, getVolumeDetails(pool, vol)); + // Images on RBD are raw and the path is an image name that qemu-img cannot open + // without the rbd: URI, so take the size libvirt already reported for the volume. + long size = isRbd ? vol.getVirtualSize() : KVMPhysicalDisk.getVirtualSizeFromFile(path); + return new CheckVolumeAnswer(command, true, "", size, volumeDetails); } else { return new Answer(command, false, "Unsupported Storage Pool"); } @@ -122,6 +134,9 @@ private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk try { QemuImg qemu = new QemuImg(0); QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + if (Storage.StoragePoolType.RBD.equals(pool.getType())) { + qemuFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath()), disk.getFormat()); + } return qemu.info(qemuFile, secure); } catch (QemuImgException | LibvirtException ex) { logger.error("Failed to get info of disk file: " + ex.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index a43b584dd6d6..20e3891478b4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -93,6 +94,13 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR final String path = vol.getPath(); String type = notifyOnlyType; + // Encrypted RBD volumes are encrypted natively by librbd; they must be resized via + // `rbd resize --encryption-passphrase-file` so librbd grows the encrypted payload and keeps + // the LUKS header consistent. The libvirt/qemu-img resize paths below are for qemu-native + // encryption and would not handle the librbd LUKS2 layout. + final boolean rbdEncrypted = pool.getType() == StoragePoolType.RBD + && command.getPassphrase() != null && command.getPassphrase().length > 0; + if (spool.getType().equals(StoragePoolType.PowerFlex) && vol.getFormat().equals(PhysicalDiskFormat.QCOW2)) { // PowerFlex QCOW2 sizing needs to consider overhead. newSize = ScaleIOStorageAdaptor.getUsableBytesFromRawBytes(newSize); @@ -115,7 +123,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR /* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via qemu-img */ if (pool.getType() != StoragePoolType.CLVM && pool.getType() != StoragePoolType.CLVM_NG && pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex - && vol.getFormat() != PhysicalDiskFormat.QCOW2) { + && vol.getFormat() != PhysicalDiskFormat.QCOW2 && !rbdEncrypted) { logger.debug("Volume " + path + " can be resized by libvirt. Asking libvirt to resize the volume."); try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); @@ -139,11 +147,15 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR boolean vmIsRunning = isVmRunning(vmInstanceName, libvirtComputingResource); - /* when VM is offline, we use qemu-img directly to resize encrypted volumes. - If VM is online, the existing resize script will call virsh blockresize which works - with both encrypted and non-encrypted volumes. + /* when VM is offline, we use qemu-img (or rbd, for librbd-encrypted RBD) directly to resize + encrypted volumes. If VM is online, the existing resize script calls virsh blockresize, + which for an librbd-encrypted RBD disk lets qemu/librbd grow the encrypted payload and + notify the guest in one step (no passphrase needed, qemu already has the secret loaded). */ - if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { + if (rbdEncrypted && !vmIsRunning) { + logger.debug("Invoking rbd to resize an offline, encrypted (librbd) RBD volume"); + resizeRbdEncryptedVolume(pool, vol, newSize, shrinkOk, command.getPassphrase()); + } else if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { logger.debug("Invoking qemu-img to resize an offline, encrypted volume"); QemuObject.EncryptFormat encryptFormat = QemuObject.EncryptFormat.enumValue(command.getEncryptFormat()); resizeEncryptedQcowFile(vol, encryptFormat,newSize, command.getPassphrase(), libvirtComputingResource); @@ -213,6 +225,16 @@ private void resizeEncryptedQcowFile(final KVMPhysicalDisk vol, final QemuObject } } + private void resizeRbdEncryptedVolume(final KVMStoragePool pool, final KVMPhysicalDisk vol, long newSize, + boolean shrinkOk, byte[] passphrase) throws CloudRuntimeException { + try { + new RbdEncryption().resize(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getAuthSecret(), pool.getSourceDir(), vol.getName(), newSize, shrinkOk, passphrase); + } finally { + Arrays.fill(passphrase, (byte) 0); + } + } + private Answer handleMultipathSCSIResize(ResizeVolumeCommand command, KVMStoragePool pool) { ((MultipathSCSIPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize()); return new ResizeVolumeAnswer(command, true, ""); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 3839d7f6bdbd..ccd0ec634525 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -20,6 +20,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -103,11 +104,11 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser newVolumeId = getVolumeUuidFromPath(volumePath, volumePool); Long size = command.getRestoreVolumeSizes().get(0); restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile, size, - new Pair<>(vmName, command.getVmState()), mountDirectory, timeout); + new Pair<>(vmName, command.getVmState()), mountDirectory, timeout, mountTimeout); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout, mountTimeout); } else { - restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout, mountTimeout); } } catch (CloudRuntimeException e) { String errorMessage = e.getMessage() != null ? e.getMessage() : ""; @@ -128,7 +129,7 @@ private void verifyBackupFile(String backupPath, String volUuid) { private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, List restoreVolumePools, List restoreVolumePaths, List backedVolumesUUIDs, - String backupPath, List backupFiles, String mountDirectory, int timeout) { + String backupPath, List backupFiles, String mountDirectory, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { @@ -145,13 +146,13 @@ private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, Li } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, List volumePools, - List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout) { + List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { @@ -167,13 +168,13 @@ private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile, - Long size, Pair vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout, Integer mountTimeout) { String bkpPath; String volumeUuid; try { @@ -190,7 +191,7 @@ private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPa } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } @@ -206,6 +207,7 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT logger.error("Failed to create the tmp mount directory {} for restore", mountDirectory, e); throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } + int exitValue; try { String mountPath = Script.getExecutableAbsolutePath("mount"); List mountCmd = new ArrayList<>(); @@ -226,23 +228,42 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT mountCmd.add("-o"); mountCmd.add(mountOptions); } - Script.executeCommand(mountCmd.toArray(new String[0])); + exitValue = Script.executeCommandForExitValue(mountTimeout, mountCmd.toArray(new String[0])); } catch (Exception e) { logger.error("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory, e); throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); } + if (exitValue != 0) { + logger.error("Failed to mount repository {} of type {} to the directory {}, mount exited with {}", backupRepoAddress, + backupRepoType, mountDirectory, exitValue); + removeTemporaryDirectoryQuietly(mountDirectory); + throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); + } return mountDirectory; } - private void unmountBackupDirectory(String backupDirectory) { + private void unmountBackupDirectory(String backupDirectory, Integer mountTimeout) { + int exitValue; try { String umountPath = Script.getExecutableAbsolutePath("umount"); String[] umountCmd = new String[] { "sudo", umountPath, backupDirectory }; - Script.executeCommand(umountCmd); + exitValue = Script.executeCommandForExitValue(mountTimeout, umountCmd); } catch (Exception e) { logger.error("Failed to unmount backup directory {}", backupDirectory, e); throw new CloudRuntimeException("Failed to unmount the backup directory"); } + if (exitValue != 0) { + logger.error("Failed to unmount backup directory {}, umount exited with {}", backupDirectory, exitValue); + throw new CloudRuntimeException("Failed to unmount the backup directory"); + } + } + + private void removeTemporaryDirectoryQuietly(String backupDirectory) { + try { + Files.deleteIfExists(Paths.get(backupDirectory)); + } catch (IOException e) { + logger.warn("Failed to remove the temporary mount directory {} after the mount failed.", backupDirectory, e); + } } private void deleteTemporaryDirectory(String backupDirectory) { @@ -293,7 +314,7 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr } String[] rsyncCmd = new String[] { Script.getExecutableAbsolutePath("rsync"), "-az", backupPath, volumePath }; - int exitValue = Script.executeCommandForExitValue(rsyncCmd); + int exitValue = Script.executeCommandForExitValue(timeout, rsyncCmd); return exitValue == 0; } @@ -362,38 +383,68 @@ private boolean replaceBlockDeviceWithBackup(KVMStoragePoolManager storagePoolMg private boolean attachVolumeToVm(KVMStoragePoolManager storagePoolMgr, String vmName, PrimaryDataStoreTO volumePool, String volumePath) { String deviceToAttachDiskTo = getDeviceToAttachDisk(vmName); + if (Storage.StoragePoolType.RBD.equals(volumePool.getPoolType())) { + return attachRbdVolumeToVm(storagePoolMgr, vmName, volumePool, volumePath, deviceToAttachDiskTo); + } List virshCmd = new ArrayList<>(); virshCmd.add(Script.getExecutableAbsolutePath("virsh")); - if (volumePool.getPoolType() == Storage.StoragePoolType.RBD) { - String xmlForRbdDisk = getXmlForRbdDisk(storagePoolMgr, volumePool, volumePath, deviceToAttachDiskTo); - logger.debug("RBD disk xml to attach: {}", xmlForRbdDisk); - virshCmd.add("attach-device"); - virshCmd.add(vmName); - virshCmd.add("/dev/stdin"); - virshCmd.add("< result = Script.executePipedCommands(Arrays.asList(domblkCmd, tailCmd, headCmd, awkCmd), 0); - String currentDevice = result.second(); + // executePipedCommands appends a line separator to every line it reads, so the device + // name has to be trimmed before the last character can be incremented. + String currentDevice = result.second() == null ? "" : result.second().trim(); + if (result.first() == null || result.first() != 0 || StringUtils.isBlank(currentDevice)) { + throw new CloudRuntimeException(String.format("Failed to determine the device to attach the restored volume to on VM [%s].", vmName)); + } char lastChar = currentDevice.charAt(currentDevice.length() - 1); char incrementedChar = (char) (lastChar + 1); return currentDevice.substring(0, currentDevice.length() - 1) + incrementedChar; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java index 8a9d69c97954..b3b55f484e4d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java @@ -54,9 +54,9 @@ public static String RBDStringBuilder(KVMStoragePool storagePool, String image) rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort); if (authUserName == null) { - rbdOpts += ":auth_supported=none"; + rbdOpts += ":auth_client_required=none"; } else { - rbdOpts += ":auth_supported=cephx"; + rbdOpts += ":auth_client_required=cephx"; rbdOpts += ":id=" + authUserName; rbdOpts += ":key=" + authSecret; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 11acb9546b53..b8a01b848343 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -182,6 +182,12 @@ public class KVMStorageProcessor implements StorageProcessor { private static final String CEPH_AUTH_KEY = "key"; private static final String CEPH_CLIENT_MOUNT_TIMEOUT = "client_mount_timeout"; private static final String CEPH_DEFAULT_MOUNT_TIMEOUT = "30"; + + // libvirt < 10.1.0 has an object apply-order bug (fixed in 10.1.0) that breaks hot-plug of an encrypted + // blockdev: on attach the disk is opened before its LUKS secret object is defined, so the attach fails with + // "No secret with id '...-format-encryption-secret0'". Booting a VM from an encrypted disk is unaffected (the + // QEMU command line resolves all -object before -blockdev). See qemuBlockStorageSourceAttachApply() in libvirt. + private static final long MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG = 10001000L; // libvirt 10.1.0 /** * Time interval before rechecking virsh commands */ @@ -1795,6 +1801,20 @@ protected DiskDef.DiskBus getAttachDiskBusType(int deviceId, List disks return DiskDef.DiskBus.VIRTIO; } + /** + * libvirt < 10.1.0 cannot hot-plug an encrypted rbd blockdev (the LUKS secret is applied after the disk is + * opened), so refuse the attach with a clear message rather than letting libvirt fail with an opaque + * "No secret with id ..." error. Only the RBD hot-plug path is affected; booting a VM from an encrypted RBD + * disk works on older libvirt, so this does not gate the boot/root path. + */ + protected void ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType poolType) { + if (poolType == StoragePoolType.RBD + && resource.getHypervisorLibvirtVersion() < MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG) { + throw new CloudRuntimeException("Libvirt version 10.1.0 required to attach an encrypted RBD volume to a running VM, but version " + + resource.getHypervisorLibvirtVersion() + " detected. Booting a VM from an encrypted RBD disk is not affected."); + } + } + @Override public Answer attachVolume(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); @@ -1807,8 +1827,13 @@ public Answer attachVolume(final AttachCommand cmd) { final Connect conn = LibvirtConnection.getConnectionByVmName(vmName); DiskDef.LibvirtDiskEncryptDetails encryptDetails = null; if (vol.requiresEncryption()) { + // Encrypted RBD is decrypted by librbd inside qemu; hot-plugging it needs a libvirt new enough to + // emit the LUKS secret before the rbd blockdev. Booting from an encrypted RBD disk is unaffected. + ensureLibvirtSupportsEncryptedRbdHotplug(primaryStore.getPoolType()); String secretUuid = resource.createLibvirtVolumeSecret(conn, vol.getPath(), vol.getPassphrase()); - encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat())); + // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. + String encryptEngine = (primaryStore.getPoolType() == StoragePoolType.RBD) ? "librbd" : null; + encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat()), encryptEngine); vol.clearPassphrase(); } @@ -2568,33 +2593,36 @@ private CreateObjectAnswer takeClvmVolumeSnapshotOfStoppedVm(KVMPhysicalDisk dis * barriers properly (>2.6.32) this won't be any different then pulling the power * cord out of a running machine. */ - private Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + protected Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { Long snapshotSize = null; + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage image = null; try { - Rados r = radosConnect(primaryPool); + r = radosConnect(primaryPool); - final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); - final Rbd rbd = new Rbd(io); - final RbdImage image = rbd.open(disk.getName()); + io = r.ioCtxCreate(primaryPool.getSourceDir()); + rbd = new Rbd(io); + image = rbd.open(disk.getName()); logger.debug("Attempting to create RBD snapshot {}@{}", disk.getName(), snapshotName); image.snapCreate(snapshotName); - image.snapCreate(snapshotName); long rbdSnapshotSize = getRbdSnapshotSize(primaryPool.getSourceDir(), disk.getName(), snapshotName, primaryPool.getSourceHost(), primaryPool.getAuthUserName(), primaryPool.getAuthSecret()); if (rbdSnapshotSize > 0) { snapshotSize = rbdSnapshotSize; } - - rbd.close(image); - r.ioCtxDestroy(io); } catch (final Exception e) { logger.error("A RBD snapshot operation on [{}] failed. The error was: {}", disk.getName(), e.getMessage(), e); + } finally { + closeRbdImage(rbd, image, disk.getName()); + destroyRadosIoCtx(r, io, disk.getName()); } return snapshotSize; } - private long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { + protected long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { logger.debug("Get RBD snapshot size for {}/{}@{}", poolPath, diskName, snapshotName); //cmd: rbd du /@ --format json --mon-host --id --key 2>/dev/null String snapshotDetailsInJson = Script.runSimpleBashScript(String.format("rbd du %s/%s@%s --format json --mon-host %s --id %s --key %s 2>/dev/null", poolPath, diskName, snapshotName, rbdMonitor, authUser, authSecret)); @@ -2881,7 +2909,7 @@ protected boolean isAvailablePoolSizeDividedByDiskSizeLesserThanMinRate(long ava return ((availablePoolSize * 1d) / (diskSize * 1d)) < MIN_RATE_BETWEEN_AVAILABLE_POOL_AND_DISK_SIZE_TO_TAKE_DISK_SNAPSHOT; } - private Rados radosConnect(final KVMStoragePool primaryPool) throws RadosException { + protected Rados radosConnect(final KVMStoragePool primaryPool) throws RadosException { Rados r = new Rados(primaryPool.getAuthUserName()); r.confSet(CEPH_MON_HOST, primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); r.confSet(CEPH_AUTH_KEY, primaryPool.getAuthSecret()); @@ -2891,6 +2919,50 @@ private Rados radosConnect(final KVMStoragePool primaryPool) throws RadosExcepti return r; } + /** + * Closes an RBD image if it was opened; never throws. An image left open keeps this client's RBD + * exclusive-lock, which later makes 'rbd snap rollback' (revertSnapshot) fail with EROFS and keeps + * the image busy so it cannot be removed. + */ + protected void closeRbdImage(Rbd rbd, RbdImage image, String imageName) { + if (image == null) { + return; + } + try { + rbd.close(image); + } catch (final Exception e) { + logger.warn("Failed to close RBD image [{}]. The error was: {}", imageName, e.getMessage(), e); + } + } + + /** Destroys a RADOS IO context if it was created; never throws. */ + protected void destroyRadosIoCtx(Rados r, IoCTX io, String contextDescription) { + if (io == null) { + return; + } + try { + r.ioCtxDestroy(io); + } catch (final Exception e) { + logger.warn("Failed to destroy the RADOS IO context used for [{}]. The error was: {}", contextDescription, e.getMessage(), e); + } + } + + /** + * Unprotects an RBD snapshot if it was protected; never throws. A snapshot left protected cannot + * be deleted, and neither can its volume. + */ + protected void unprotectRbdSnapshot(RbdImage image, String snapshotName, boolean snapProtected) { + if (!snapProtected) { + return; + } + try { + image.snapUnprotect(snapshotName); + } catch (final Exception e) { + logger.error("Failed to unprotect RBD snapshot [{}]; it and its volume cannot be deleted until this is resolved manually. The error was: {}", + snapshotName, e.getMessage(), e); + } + } + @Override public Answer deleteVolume(final DeleteCommand cmd) { final VolumeObjectTO vol = (VolumeObjectTO)cmd.getData(); @@ -3046,17 +3118,24 @@ private KVMPhysicalDisk createRBDvolumeFromRBDSnapshot(KVMPhysicalDisk volume, S disk.setSize(size > volume.getVirtualSize() ? size : volume.getVirtualSize()); disk.setVirtualSize(size > volume.getVirtualSize() ? size : disk.getSize()); + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage srcImage = null; + RbdImage diskImage = null; + boolean snapProtected = false; + try { - Rados r = new Rados(srcPool.getAuthUserName()); + r = new Rados(srcPool.getAuthUserName()); r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); r.confSet("key", srcPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); r.connect(); - IoCTX io = r.ioCtxCreate(srcPool.getSourceDir()); - Rbd rbd = new Rbd(io); - RbdImage srcImage = rbd.open(volume.getName()); + io = r.ioCtxCreate(srcPool.getSourceDir()); + rbd = new Rbd(io); + srcImage = rbd.open(volume.getName()); List snaps = srcImage.snapList(); boolean snapFound = false; @@ -3072,23 +3151,26 @@ private KVMPhysicalDisk createRBDvolumeFromRBDSnapshot(KVMPhysicalDisk volume, S return null; } srcImage.snapProtect(snapshotName); + snapProtected = true; logger.debug(String.format("Try to clone snapshot %s on RBD", snapshotName)); rbd.clone(volume.getName(), snapshotName, io, disk.getName(), LibvirtStorageAdaptor.RBD_FEATURES, 0); - RbdImage diskImage = rbd.open(disk.getName()); + diskImage = rbd.open(disk.getName()); if (disk.getVirtualSize() > volume.getVirtualSize()) { diskImage.resize(disk.getVirtualSize()); } diskImage.flatten(); - rbd.close(diskImage); - - srcImage.snapUnprotect(snapshotName); - rbd.close(srcImage); - r.ioCtxDestroy(io); } catch (RadosException | RbdException e) { logger.error(String.format("Failed due to %s", e.getMessage()), e); disk = null; + } finally { + // Every handle has to be released on all paths, including the "snapshot not found" return and + // any failure of clone/resize/flatten. + closeRbdImage(rbd, diskImage, newUuid); + unprotectRbdSnapshot(srcImage, snapshotName, snapProtected); + closeRbdImage(rbd, srcImage, volume.getName()); + destroyRadosIoCtx(r, io, snapshotName); } return disk; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 4bfac31b68f9..8a1a7b5bbe2e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -34,7 +34,9 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; @@ -55,7 +57,6 @@ import com.ceph.rados.IoCTX; import com.ceph.rados.Rados; -import com.ceph.rados.exceptions.ErrorCode; import com.ceph.rados.exceptions.RadosException; import com.ceph.rbd.Rbd; import com.ceph.rbd.RbdException; @@ -95,6 +96,11 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_DEEP_FLATTEN = 32; public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ + /* Space reserved at the front of an encrypted RBD image for the LUKS2 header/keyslots so the + usable (decrypted) size still matches the requested volume size. */ + private static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB + /* libvirt's VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS, not exposed as a constant by libvirt-java */ + private static final int VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS = 2; private static final Set QEMU_IMG_MANAGED_POOL_TYPES = Set.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint); @@ -988,8 +994,18 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); - return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : - createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase); + // Create the raw RBD image first. For encrypted volumes we apply a native librbd LUKS header + // afterwards via `rbd encryption format` (engine='librbd'). We deliberately do NOT hand the + // passphrase to qemu-img, which would instead produce a qemu-native LUKS container. + KVMPhysicalDisk disk = (dataPool == null) ? + createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : + createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, null); + + if (passphrase != null && passphrase.length > 0) { + formatRbdImageEncryption(pool, name, passphrase); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + } + return disk; } else if (QEMU_IMG_MANAGED_POOL_TYPES.contains(poolType)) { switch (format) { case QCOW2: @@ -1150,61 +1166,6 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag logger.info("Attempting to remove volume " + uuid + " from pool " + pool.getUuid()); - /** - * RBD volume can have snapshots and while they exist libvirt - * can't remove the RBD volume - * - * We have to remove those snapshots first - */ - if (pool.getType() == StoragePoolType.RBD) { - try { - logger.info("Unprotecting and Removing RBD snapshots of image " + pool.getSourceDir() + "/" + uuid + " prior to removing the image"); - - Rados r = new Rados(pool.getAuthUserName()); - r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); - r.confSet("key", pool.getAuthSecret()); - r.confSet("client_mount_timeout", "30"); - r.connect(); - logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host")); - - IoCTX io = r.ioCtxCreate(pool.getSourceDir()); - Rbd rbd = new Rbd(io); - RbdImage image = rbd.open(uuid); - logger.debug("Fetching list of snapshots of RBD image " + pool.getSourceDir() + "/" + uuid); - List snaps = image.snapList(); - try { - for (RbdSnapInfo snap : snaps) { - if (image.snapIsProtected(snap.name)) { - logger.debug("Unprotecting snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); - image.snapUnprotect(snap.name); - } else { - logger.debug("Snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name + " is not protected."); - } - logger.debug("Removing snapshot " + pool.getSourceDir() + "/" + uuid + "@" + snap.name); - image.snapRemove(snap.name); - } - logger.info("Successfully unprotected and removed any remaining snapshots (" + snaps.size() + ") of " - + pool.getSourceDir() + "/" + uuid + " Continuing to remove the RBD image"); - } catch (RbdException e) { - logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); - } finally { - logger.debug("Closing image and destroying context"); - rbd.close(image); - r.ioCtxDestroy(io); - } - } catch (RadosException e) { - logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); - } catch (RbdException e) { - logger.error("Failed to remove snapshot with exception: " + e.toString() + - ", RBD error: " + ErrorCode.getErrorMessage(e.getReturnValue())); - throw new CloudRuntimeException(e.toString() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); - } - } - LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), uuid); @@ -1244,7 +1205,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { - disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); + disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout, passphrase); } else { try (KeyFile keyFile = new KeyFile(passphrase)){ String newUuid = name; @@ -1324,7 +1285,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, } private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){ + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase){ /* With RBD you can't run qemu-img convert with an existing RBD image as destination @@ -1335,6 +1296,8 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ KVMStoragePool srcPool = template.getPool(); + Map destDetails = destPool.getDetails(); + String dataPool = (destDetails == null) ? null : destDetails.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); KVMPhysicalDisk disk = null; String newUuid = name; @@ -1351,6 +1314,16 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } + if (passphrase != null && passphrase.length > 0) { + boolean sameClusterRbd = srcPool.getType() == StoragePoolType.RBD + && srcPool.getSourceHost().equals(destPool.getSourceHost()) + && srcPool.getSourceDir().equals(destPool.getSourceDir()); + if (sameClusterRbd) { + return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase); + } + return createEncryptedRootFullCopy(srcPool, template, destPool, newUuid, disk, passphrase); + } + QemuImgFile srcFile; QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath())); destFile.setFormat(format); @@ -1383,6 +1356,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); r.confSet("key", srcPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); + if (dataPool != null) { + logger.debug("Setting RBD data pool to " + dataPool + " for the new image " + disk.getName()); + r.confSet(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL, dataPool); + } r.connect(); logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host")); @@ -1459,6 +1436,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, rDest.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); rDest.confSet("key", destPool.getAuthSecret()); rDest.confSet("client_mount_timeout", "30"); + if (dataPool != null) { + logger.debug("Setting RBD data pool to " + dataPool + " on the destination cluster for the new image " + disk.getName()); + rDest.confSet(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL, dataPool); + } rDest.connect(); logger.debug("Successfully connected to source Ceph cluster at " + rDest.confGet("mon_host")); @@ -1493,9 +1474,126 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, disk = null; } } + + // Encrypted volumes are handled by the early return above (create empty -> luks2 format -> + // import template through encryption); the clone/convert path here is for plaintext volumes. return disk; } + /** + * Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster + * as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to + * reserve LUKS2-header space, snapshot+protect that grown state, clone from it, apply a LUKS2 header, + * then resize the clone to the requested size. The inherited (plaintext) template data stays readable + * through the clone's encryption, and the clone is a thin CoW image (only the header is written). + * + * @return the encrypted CoW clone, or {@code null} if the Ceph operations failed + */ + private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KVMStoragePool destPool, + String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + String luksReservedSnapshotName = rbdTemplateSnapName + "-luks"; + Rados radosConnection = null; + IoCTX ioContext = null; + Rbd rbdClient = null; + RbdImage templateImage = null; + try { + radosConnection = new Rados(destPool.getAuthUserName()); + radosConnection.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + radosConnection.confSet("key", destPool.getAuthSecret()); + radosConnection.confSet("client_mount_timeout", "30"); + radosConnection.connect(); + ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); + rbdClient = new Rbd(ioContext); + templateImage = rbdClient.open(template.getName()); + boolean luksSnapshotExists = false; + for (RbdSnapInfo snapshotInfo : templateImage.snapList()) { + if (luksReservedSnapshotName.equals(snapshotInfo.name)) { + luksSnapshotExists = true; + break; + } + } + if (!luksSnapshotExists) { + templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); + templateImage.snapCreate(luksReservedSnapshotName); + templateImage.snapProtect(luksReservedSnapshotName); + logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName); + } + rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage()); + return null; + } finally { + if (rbdClient != null && templateImage != null) { + try { + rbdClient.close(templateImage); + } catch (RbdException ignored) { + // best-effort close of the template handle + } + } + if (radosConnection != null && ioContext != null) { + radosConnection.ioCtxDestroy(ioContext); + } + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + if (disk.getVirtualSize() > template.getVirtualSize()) { + // grow the clone to the requested root size (encryption-aware) + new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(), + destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), + newUuid, disk.getVirtualSize(), false, passphrase); + } + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + + /** + * Option B (full-copy encrypted root), used when the template is not on the same RBD cluster (e.g. first + * use from secondary storage). Create an empty image, apply a LUKS2 header, then import the template + * THROUGH the encryption layer (qemu-img convert -n). Correct but not thin (no CoW). + * + * @return the encrypted image, or {@code null} if the Ceph operations failed + */ + private KVMPhysicalDisk createEncryptedRootFullCopy(KVMStoragePool srcPool, KVMPhysicalDisk template, + KVMStoragePool destPool, String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + long imageSizeWithLuksHeader = disk.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES; + Rados radosConnection = null; + IoCTX ioContext = null; + try { + radosConnection = new Rados(destPool.getAuthUserName()); + radosConnection.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + radosConnection.confSet("key", destPool.getAuthSecret()); + radosConnection.confSet("client_mount_timeout", "30"); + radosConnection.connect(); + ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); + Rbd rbdClient = new Rbd(ioContext); + rbdClient.create(newUuid, imageSizeWithLuksHeader, RBD_FEATURES, rbdOrder); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted RBD image {}: {}", newUuid, e.getMessage()); + return null; + } finally { + if (radosConnection != null && ioContext != null) { + radosConnection.ioCtxDestroy(ioContext); + } + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + boolean sourceIsRbdPool = srcPool.getType() == StoragePoolType.RBD; + new RbdEncryption().importTemplate( + sourceIsRbdPool ? srcPool.getSourceDir() : null, sourceIsRbdPool ? template.getName() : null, + sourceIsRbdPool ? null : template.getPath(), sourceIsRbdPool ? null : template.getFormat().toString(), + destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), + destPool.getSourceDir(), newUuid, passphrase, CryptSetup.LuksType.LUKS2); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + + /** + * Apply native librbd LUKS encryption to an existing RBD image via the rbd CLI. + * Isolated here so the CLI dependency can later be swapped for a native (JNA) librbd binding. + */ + private void formatRbdImageEncryption(KVMStoragePool pool, String image, byte[] passphrase) { + new RbdEncryption().format(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getAuthSecret(), pool.getSourceDir(), image, passphrase, CryptSetup.LuksType.LUKS2); + } + @Override public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { return null; @@ -1721,7 +1819,19 @@ private void refreshPool(StoragePool pool) throws LibvirtException { } private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { - vol.delete(0); + /** + * RBD volumes can have snapshots, and libvirt refuses to remove a volume while + * they exist. VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS tells the RBD storage backend + * to unprotect and remove any snapshots before removing the volume itself. + * + * libvirt-java has no named constant for this flag (added upstream in libvirt 1.2.20, + * commit 3c7590e0a4), so it's passed as a raw flag value here. + */ + int flags = 0; + if (pool.getType() == StoragePoolType.RBD) { + flags |= VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS; + } + vol.delete(flags); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index a8c32baa6ef3..7053dc4e4d9b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -330,29 +330,55 @@ public boolean isPoolSupportHA() { public String getHearthBeatPath() { if (StoragePoolType.NetworkFilesystem.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmheartbeat.sh"); } else if (StoragePoolType.SharedMountPoint.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmsmpheartbeat.sh"); + } else if (StoragePoolType.RBD.equals(type)) { + return findKvmHaScript("kvmheartbeat_rbd.sh"); } return null; } + private String findKvmHaScript(String scriptName) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, scriptName); + if (scriptPath == null) { + throw new CloudRuntimeException(String.format("Unable to find heartbeat script '%s' in directory: %s", scriptName, kvmScriptsDir)); + } + return scriptPath; + } + + /** + * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, credentials) + * to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key" + * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}). + */ + private void addRbdConnectionArgs(Script cmd) { + cmd.add("-s", sourceHost); + cmd.add("-o", sourceDir); + if (authUsername != null) { + cmd.add("-n", authUsername); + cmd.add("-k", authSecret); + } + } + + /** + * Adds the arguments identifying the storage to a heartbeat/VM-activity check {@link Script}: + * the Ceph connection details for a RBD pool, or the NFS server, path and mount point otherwise. + */ + private void addPoolConnectionArgs(Script cmd, HAStoragePool pool) { + if (StoragePoolType.RBD.equals(type)) { + addRbdConnectionArgs(cmd); + } else { + cmd.add("-i", pool.getPoolIp()); + cmd.add("-p", pool.getPoolMountSourcePath()); + cmd.add("-m", pool.getMountDestPath()); + } + } public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); - cmd.add("-i", primaryStoragePool.getPoolIp()); - cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); - cmd.add("-m", primaryStoragePool.getMountDestPath()); + addPoolConnectionArgs(cmd, primaryStoragePool); if (hostValidation) { cmd.add("-h", hostPrivateIp); @@ -377,9 +403,7 @@ public String getStorageNodeId() { public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { String hostIp = host.getPrivateNetwork().getIp(); Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); + addPoolConnectionArgs(cmd, pool); cmd.add("-h", hostIp); cmd.add("-r"); cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); @@ -401,10 +425,12 @@ public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { @Override public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { String hostIp = host.getPrivateNetwork().getIp(); - Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); - cmd.add("-i", pool.getPoolIp()); - cmd.add("-p", pool.getPoolMountSourcePath()); - cmd.add("-m", pool.getMountDestPath()); + // RBD volumes have no shared mount point to stat(), so the RBD-specific script + // (using RBD watchers to detect activity) is used instead of the generic, + // NFS/SharedMountPoint-oriented script path passed in by the caller. + String scriptPath = StoragePoolType.RBD.equals(type) ? findKvmHaScript("kvmvmactivity_rbd.sh") : vmActivityCheckPath; + Script cmd = new Script(scriptPath, activityScriptTimeout.getStandardSeconds(), logger); + addPoolConnectionArgs(cmd, pool); cmd.add("-h", hostIp); cmd.add("-u", volumeUUIDListString); cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java index 4a577ef3400c..10e39cb5ffcb 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageOptions.java @@ -86,8 +86,17 @@ public String[] toCommandFlag() { return new String[] { params.get(FILENAME_PARAM_KEY) }; } } + return toCommandFlag(QemuImg.IMAGE_OPTS_FLAG); + } + + /** + * Converts QemuImageOptions into the command strings under the given qemu-img flag, + * e.g. {@link QemuImg#TARGET_IMAGE_OPTS_FLAG} for a convert destination. + * @return array of strings representing the flag and its options value + */ + public String[] toCommandFlag(String flagName) { Map sorted = new TreeMap<>(params); String paramString = Joiner.on(",").withKeyValueSeparator("=").join(sorted); - return new String[] {"--image-opts", paramString}; + return new String[] {flagName, paramString}; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index cae6832999eb..49f531ed7c11 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -52,6 +52,8 @@ public class QemuImg { public static final String ENCRYPT_FORMAT = "encrypt.format"; public static final String ENCRYPT_KEY_SECRET = "encrypt.key-secret"; public static final String TARGET_ZERO_FLAG = "--target-is-zero"; + public static final String IMAGE_OPTS_FLAG = "--image-opts"; + public static final String TARGET_IMAGE_OPTS_FLAG = "--target-image-opts"; public static final String PREALLOCATION = "preallocation"; public static final long QEMU_2_10 = 2010000; public static final long QEMU_5_1 = 5001000; @@ -402,6 +404,21 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, convert(srcFile, destFile, null, options, qemuObjects, srcImageOpts, snapshotName, forceSourceFormat, false, false, false, null, null); } + /** + * Converts an image into an existing destination that is described by explicit image options + * ({@code --target-image-opts}) instead of a plain filename - for example an RBD image written + * through librbd encryption ({@code driver=rbd,...,encrypt.format=...}). The destination is + * never created ({@code -n} is implied) and must already exist with the wanted size and format. + * + * @param destImageOpts + * image options describing the existing destination; passed as --target-image-opts. + */ + public void convertIntoExistingTarget(final QemuImgFile srcFile, final Map options, + final List qemuObjects, final QemuImageOptions srcImageOpts, final QemuImageOptions destImageOpts, + final boolean forceSourceFormat) throws QemuImgException { + convert(srcFile, null, null, options, qemuObjects, srcImageOpts, destImageOpts, null, forceSourceFormat, false, false, false, null, null); + } + protected Map getResizeOptionsFromConvertOptions(final Map options) { if (MapUtils.isEmpty(options)) { return null; @@ -450,6 +467,25 @@ protected Map getResizeOptionsFromConvertOptions(final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat, boolean keepBitmaps, boolean outOfOrderWrites, boolean compress, Integer coroutines, Integer rateLimit) throws QemuImgException { + convert(srcFile, destFile, backingFile, options, qemuObjects, srcImageOpts, null, snapshotName, forceSourceFormat, keepBitmaps, outOfOrderWrites, compress, coroutines, + rateLimit); + } + + /** + * Converts an image from source to destination, optionally into an existing destination described by explicit image options + * ({@code --target-image-opts}) instead of a plain filename; see {@link #convertIntoExistingTarget}. All other parameters + * behave as documented above. + * + * @param destImageOpts + * If not null, the destination is described by these image options and {@code destFile} is unused. + */ + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuImgFile backingFile, final Map options, final List qemuObjects, + final QemuImageOptions srcImageOpts, final QemuImageOptions destImageOpts, final String snapshotName, final boolean forceSourceFormat, boolean keepBitmaps, + boolean outOfOrderWrites, boolean compress, Integer coroutines, Integer rateLimit) throws QemuImgException { + if (destImageOpts != null && this.version < QEMU_2_10) { + throw new QemuImgException(String.format("qemu >= 2.10 is required to convert into a destination described by %s", TARGET_IMAGE_OPTS_FLAG)); + } + Script script = new Script(_qemuImgPath, timeout); if (StringUtils.isNotBlank(snapshotName)) { String qemuPath = Script.runSimpleBashScript(getQemuImgPathScript); @@ -458,7 +494,10 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("convert"); - if (skipZero && Files.exists(Paths.get(destFile.getFileName()))) { + if (destImageOpts != null) { + // a destination described by image options always exists already; qemu-img requires -n with --target-image-opts + script.add("-n"); + } else if (skipZero && Files.exists(Paths.get(destFile.getFileName()))) { script.add("-n"); script.add(TARGET_ZERO_FLAG); script.add("-W"); @@ -469,8 +508,10 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("-n"); } - script.add("-O"); - script.add(destFile.getFormat().toString()); + if (destImageOpts == null) { + script.add("-O"); + script.add(destFile.getFormat().toString()); + } addBackingFileToConvertCommand(script, backingFile); addScriptOptionsFromMap(options, script); @@ -524,14 +565,19 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("--bitmaps"); } - script.add(destFile.getFileName()); + if (destImageOpts != null) { + script.add(destImageOpts.toCommandFlag(TARGET_IMAGE_OPTS_FLAG)); + } else { + script.add(destFile.getFileName()); + } final String result = script.execute(); if (result != null) { throw new QemuImgException(result); } - if (srcFile.getSize() < destFile.getSize()) { + // an image-options destination already exists with its final size; 'qemu-img resize' cannot address it by filename + if (destImageOpts == null && srcFile.getSize() < destFile.getSize()) { this.resize(destFile, destFile.getSize(), getResizeOptionsFromConvertOptions(options)); } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java index efeee04cb90f..511e91074969 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java @@ -54,6 +54,7 @@ public enum ObjectParameter { */ public enum EncryptFormat { LUKS("luks"), + LUKS2("luks2"), AES("aes"); private final String format; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java new file mode 100644 index 000000000000..a23ce93d2b82 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.rbd; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.libvirt.LibvirtException; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Thin wrapper around the {@code rbd} CLI to apply native librbd LUKS encryption to an + * RBD image via {@code rbd encryption format}. This is only used at volume create time; + * runtime decryption is handled by libvirt/qemu through {@code }. + * + * The CLI dependency is intentionally isolated in this class so it can later be replaced + * by a native librbd (JNA) binding without touching callers. rados-java (0.x) does not + * expose the rbd_encryption_format API, hence the CLI for now. + * + * The command builders ({@code build*Script}) are separated from execution so the generated + * argv can be unit-tested without a live Ceph cluster (see {@code RbdEncryptionTest}). + */ +public class RbdEncryption { + protected Logger logger = LogManager.getLogger(getClass()); + + protected String commandPath = "rbd"; + + /** qemu secret id used to hand the LUKS passphrase to qemu-img during template import */ + protected static final String LUKS_SECRET_ID = "luks0"; + + public RbdEncryption() {} + + public RbdEncryption(String commandPath) { + this.commandPath = commandPath; + } + + private static String monSpec(String monHost, int monPort) { + return monPort > 0 ? monHost + ":" + monPort : monHost; + } + + /** + * Apply a LUKS header to an existing RBD image so librbd can transparently encrypt it. + *

+ * cephx authentication is supplied via {@code --id} plus a temporary keyfile so the secret + * never appears on the command line. The LUKS passphrase is supplied via a temporary file + * ({@link KeyFile}). Both temp files are deleted when this method returns. + * + * @param monHost ceph monitor host + * @param monPort ceph monitor port (0 to omit) + * @param authUser cephx user (e.g. "cloudstack"); null to skip --id + * @param authSecret cephx secret/key (as used for ceph "key" config); null to skip --keyfile + * @param cephPool ceph pool name + * @param image rbd image name + * @param passphrase LUKS passphrase + * @param luksType LUKS1/LUKS2 (librbd engine supports both; LUKS2 recommended) + */ + public void format(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, byte[] passphrase, CryptSetup.LuksType luksType) { + final String imageSpec = cephPool + "/" + image; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot LUKS-format RBD image " + imageSpec + ": empty passphrase"); + } + try (KeyFile passFile = new KeyFile(passphrase); + KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { + final Script script = buildFormatScript(imageSpec, luksType, passFile.toString(), + monSpec(monHost, monPort), authUser, cephKeyFile.isSet() ? cephKeyFile.toString() : null); + final String result = script.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s: %s", luksType, imageSpec, result)); + } + logger.debug("Applied {} encryption to RBD image {}", luksType, imageSpec); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s", luksType, imageSpec), ex); + } + } + + protected Script buildFormatScript(String imageSpec, CryptSetup.LuksType luksType, String passFilePath, + String monSpec, String authUser, String cephKeyFilePath) { + final Script script = new Script(commandPath); + script.add("encryption"); + script.add("format"); + script.add(imageSpec); + script.add(luksType.toString()); + script.add(passFilePath); + script.add("--mon-host"); + script.add(monSpec); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFilePath != null) { + script.add("--keyfile"); + script.add(cephKeyFilePath); + } + return script; + } + + /** + * Resize an encrypted RBD image. librbd needs the passphrase so it can resize the encrypted + * payload (not just the raw image) and keep the LUKS header consistent. {@code newSizeBytes} is + * the usable (decrypted) size requested; rbd {@code --size} is expressed in MiB. + * + * @param allowShrink pass --allow-shrink when shrinking is permitted + */ + public void resize(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, long newSizeBytes, boolean allowShrink, byte[] passphrase) { + final String imageSpec = cephPool + "/" + image; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot resize encrypted RBD image " + imageSpec + ": empty passphrase"); + } + // rbd --size is in MiB; round up so a non-MiB-aligned request never shrinks the volume below what was asked for. + final long sizeMiB = (newSizeBytes + (1024L * 1024L) - 1) / (1024L * 1024L); + try (KeyFile passFile = new KeyFile(passphrase); + KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { + final Script script = buildResizeScript(imageSpec, sizeMiB, passFile.toString(), allowShrink, + monSpec(monHost, monPort), authUser, cephKeyFile.isSet() ? cephKeyFile.toString() : null); + final String result = script.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s to %d MiB: %s", imageSpec, sizeMiB, result)); + } + logger.debug("Resized encrypted RBD image {} to {} MiB", imageSpec, sizeMiB); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s", imageSpec), ex); + } + } + + protected Script buildResizeScript(String imageSpec, long sizeMiB, String passFilePath, boolean allowShrink, + String monSpec, String authUser, String cephKeyFilePath) { + final Script script = new Script(commandPath); + script.add("resize"); + script.add("--size"); + script.add(String.valueOf(sizeMiB)); + script.add(imageSpec); + script.add("--encryption-passphrase-file"); + script.add(passFilePath); + if (allowShrink) { + script.add("--allow-shrink"); + } + script.add("--mon-host"); + script.add(monSpec); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFilePath != null) { + script.add("--keyfile"); + script.add(cephKeyFilePath); + } + return script; + } + + /** + * Import a template into an already-created, already-LUKS-formatted RBD image by writing it + * THROUGH the librbd encryption layer with {@code qemu-img convert -n} (so the data lands + * encrypted). This is how encrypted root disks are populated: we never clone-then-format a + * plaintext template (that leaves the inherited OS data unreadable) — instead we format an + * empty image and convert the template into it. + * + * Exactly one source must be given: an RBD image ({@code srcRbdPool}+{@code srcRbdImage}) or a + * local file ({@code srcFilePath}[+{@code srcFileFormat}]). cephx auth is provided to qemu-img + * via a temporary ceph.conf + keyring (deleted on return). The conversion itself goes through + * {@link QemuImg#convertIntoExistingTarget}. + */ + public void importTemplate(String srcRbdPool, String srcRbdImage, + String srcFilePath, String srcFileFormat, + String monHost, int monPort, String authUser, String authSecret, + String cephPool, String destImage, byte[] passphrase, CryptSetup.LuksType luksType) { + final String imageSpec = cephPool + "/" + destImage; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot import template into encrypted RBD image " + imageSpec + ": empty passphrase"); + } + Path conf = null; + Path keyring = null; + try (KeyFile passFile = new KeyFile(passphrase)) { + // These temp files hold the cephx secret; create them 0600 up front (matching KeyFile) rather than relying on the umask. + final FileAttribute ownerOnly = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")); + keyring = Files.createTempFile("cs-ceph-", ".keyring", ownerOnly); + Files.writeString(keyring, "[client." + authUser + "]\n\tkey = " + authSecret + "\n"); + conf = Files.createTempFile("cs-ceph-", ".conf", ownerOnly); + Files.writeString(conf, "[global]\nmon_host = " + monSpec(monHost, monPort) + "\nkeyring = " + keyring + "\n"); + + QemuImgFile srcQemuFile; + QemuImageOptions srcImageOpts; + boolean forceSourceFormat = false; + if (srcRbdImage != null) { + srcQemuFile = new QemuImgFile(srcRbdPool + "/" + srcRbdImage, QemuImg.PhysicalDiskFormat.RAW); + srcImageOpts = new QemuImageOptions(rbdImageOptions(srcRbdPool, srcRbdImage, conf.toString(), authUser)); + srcImageOpts.setImageOptsFlag(true); + } else { + QemuImg.PhysicalDiskFormat srcFormat = srcFileFormat != null ? QemuImg.PhysicalDiskFormat.valueOf(srcFileFormat.toUpperCase()) : null; + srcQemuFile = srcFormat != null ? new QemuImgFile(srcFilePath, srcFormat) : new QemuImgFile(srcFilePath); + srcImageOpts = new QemuImageOptions(srcFilePath); + if (srcFormat != null) { + // emit the source as --image-opts driver=,file.filename= (the -f equivalent) + srcImageOpts.setImageOptsFlag(true); + forceSourceFormat = true; + } + } + + Map destParams = rbdImageOptions(cephPool, destImage, conf.toString(), authUser); + destParams.put("encrypt.format", luksType.toString()); + destParams.put("encrypt.key-secret", LUKS_SECRET_ID); + QemuImageOptions destImageOpts = new QemuImageOptions(destParams); + + EnumMap secretParams = new EnumMap<>(QemuObject.ObjectParameter.class); + secretParams.put(QemuObject.ObjectParameter.ID, LUKS_SECRET_ID); + secretParams.put(QemuObject.ObjectParameter.FILE, passFile.toString()); + QemuObject luksSecret = new QemuObject(QemuObject.ObjectType.SECRET, secretParams); + + QemuImg qemu = createQemuImg(); + qemu.convertIntoExistingTarget(srcQemuFile, null, List.of(luksSecret), srcImageOpts, destImageOpts, forceSourceFormat); + logger.debug("Imported template into encrypted RBD image {}", imageSpec); + } catch (IOException | QemuImgException | LibvirtException ex) { + throw new CloudRuntimeException(String.format("Failed to import template into encrypted RBD image %s", imageSpec), ex); + } finally { + deleteQuietly(conf); + deleteQuietly(keyring); + } + } + + /** + * Seam for unit tests; {@link QemuImg} probes the qemu version through libvirt on construction. + */ + protected QemuImg createQemuImg() throws QemuImgException, LibvirtException { + return new QemuImg(0); + } + + /** qemu image options addressing an RBD image (as used with --image-opts / --target-image-opts). */ + private static Map rbdImageOptions(String cephPool, String image, String confPath, String authUser) { + Map opts = new HashMap<>(); + opts.put("driver", "rbd"); + opts.put("pool", cephPool); + opts.put("image", image); + opts.put("conf", confPath); + if (authUser != null) { + opts.put("user", authUser); + } + return opts; + } + + private static void deleteQuietly(Path p) { + if (p == null) { + return; + } + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort cleanup of the temporary ceph auth files + } + } + + /** + * Best-effort probe that the local rbd CLI supports the encryption subcommand. + */ + public boolean isSupported() { + final Script script = new Script(commandPath); + script.add("help"); + script.add("encryption"); + script.add("format"); + return script.execute() == null; + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java index fc2341632984..3bdc23d27a15 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -25,9 +25,11 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.List; import org.apache.cloudstack.backup.BackupAnswer; import org.apache.cloudstack.backup.RestoreBackupCommand; @@ -42,8 +44,11 @@ import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.storage.Storage; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import com.cloud.vm.VirtualMachine; @@ -261,8 +266,8 @@ public void testExecuteWithMountFailure() throws Exception { filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath); try (MockedStatic