diff --git a/conf/springConfigXml/VolumeManager.xml b/conf/springConfigXml/VolumeManager.xml index 4d8d6110207..cbdab5a0d4b 100644 --- a/conf/springConfigXml/VolumeManager.xml +++ b/conf/springConfigXml/VolumeManager.xml @@ -104,6 +104,7 @@ + diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java b/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java index 8be31aff4de..ebd4e92b06b 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/CreateVolumeSpec.java @@ -10,6 +10,8 @@ public class CreateVolumeSpec { private boolean encrypted; private String hostUuid; private String encryptedDek; + private String encryptionKeyResourceType; + private String encryptionKeyResourceUuid; private String allocatedUrl; private boolean dryRun; @@ -85,4 +87,20 @@ public String getEncryptedDek() { public void setEncryptedDek(String encryptedDek) { this.encryptedDek = encryptedDek; } + + public String getEncryptionKeyResourceType() { + return encryptionKeyResourceType; + } + + public void setEncryptionKeyResourceType(String encryptionKeyResourceType) { + this.encryptionKeyResourceType = encryptionKeyResourceType; + } + + public String getEncryptionKeyResourceUuid() { + return encryptionKeyResourceUuid; + } + + public void setEncryptionKeyResourceUuid(String encryptionKeyResourceUuid) { + this.encryptionKeyResourceUuid = encryptionKeyResourceUuid; + } } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index a1269c93b53..672ccebe134 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -6,6 +6,8 @@ import org.zstack.header.storage.addon.*; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageReply; import org.zstack.header.storage.snapshot.VolumeSnapshotStats; import org.zstack.header.volume.VolumeInventory; import org.zstack.header.volume.VolumeProtocol; @@ -34,6 +36,8 @@ public interface PrimaryStorageControllerSvc { void cloneVolume(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletioncomp); void copyVolume(String srcInstallPath, CreateVolumeSpec dst, ReturnValueCompletioncomp); void encryptVolumeBits(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion comp); + void convertVolumeEncryption(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion); void flattenVolume(String installPath, ReturnValueCompletioncomp); // support uri or path diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java index 7ffeb6668c9..59108ae528d 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionBackend.java @@ -21,5 +21,12 @@ public interface ZbsVolumeEncryptionBackend { void checkNoSnapshots(String installPath, Completion completion); + void validateConversionPaths(String sourceInstallPath, String targetInstallPath); + + void createConversionTarget(String targetInstallPath, long virtualSize, boolean targetEncrypted, + ReturnValueCompletion completion); + + void deleteConversionTarget(String targetInstallPath, Completion completion); + void stats(String installPath, ReturnValueCompletion completion); } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java index 59fb0a28519..81170e3139d 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ZbsVolumeEncryptionExtensionPoint.java @@ -3,6 +3,8 @@ import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageReply; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; import org.zstack.header.volume.VolumeInventory; import org.zstack.header.volume.VolumeStats; @@ -23,6 +25,9 @@ void copyEncryptedVolumeFromSnapshot(ZbsVolumeEncryptionBackend backend, String void encryptVolumeBits(ZbsVolumeEncryptionBackend backend, EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion); + void convertVolumeEncryption(ZbsVolumeEncryptionBackend backend, ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion); + void resizeEncryptedVolume(String primaryStorageUuid, VolumeInventory volume, long size, ReturnValueCompletion completion); diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index 0f6cb4abfc2..3953bec59cc 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -37,6 +37,8 @@ import org.zstack.header.storage.addon.primary.*; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageReply; import org.zstack.header.storage.primary.ImageCacheInventory; import org.zstack.header.storage.primary.VolumeSnapshotCapability; import org.zstack.header.storage.snapshot.VolumeSnapshotStats; @@ -1102,6 +1104,13 @@ public void encryptVolumeBits(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnVa comp.fail(new ErrorCode(SysErrors.UNIMPLEMENTED_OPERATION_ERROR.toString(), "unimplemented operation", details)); } + @Override + public void convertVolumeEncryption(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + String details = String.format("primary storage controller[%s] does not support volume encryption conversion", getIdentity()); + completion.fail(new ErrorCode(SysErrors.UNIMPLEMENTED_OPERATION_ERROR.toString(), "unimplemented operation", details)); + } + @Override public void flattenVolume(String installPath, ReturnValueCompletion comp) { // TODO flatten snapshot diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 70a5420c544..593bc1ea820 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -789,6 +789,16 @@ public void encryptVolumeBits(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnVa } } + @Override + public void convertVolumeEncryption(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + try { + getVolumeEncryptionExtension().convertVolumeEncryption(encryptionBackend(), msg, completion); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } + } + @Override public void flattenVolume(String installPath, ReturnValueCompletion comp) { FlattenVolumeCmd cmd = new FlattenVolumeCmd(); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java index b826f98c216..444cae71bcf 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsVolumeEncryptionBackendImpl.java @@ -13,6 +13,8 @@ import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; import org.zstack.header.volume.VolumeConstant; import org.zstack.header.volume.VolumeStats; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; import java.util.UUID; @@ -22,6 +24,7 @@ import static org.zstack.storage.zbs.ZbsHelper.getSizeUnit; class ZbsVolumeEncryptionBackendImpl implements ZbsVolumeEncryptionBackend { + private static final CLogger logger = Utils.getLogger(ZbsVolumeEncryptionBackendImpl.class); private static final String QUERY_SNAPSHOT_PATH = "/zbs/primarystorage/snapshot/query"; private static final long LUKS_PAYLOAD_OFFSET = 8L * 1024 * 1024; @@ -178,6 +181,97 @@ public void fail(ErrorCode errorCode) { }); } + @Override + public void validateConversionPaths(String sourceInstallPath, String targetInstallPath) { + String version = addonInfo.getClusterInfo() == null ? null : addonInfo.getClusterInfo().getVersion(); + if (StringUtils.isBlank(version) || !ZbsConstants.MEGABYTE_UNIT.equals(getSizeUnit(version))) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion requires ZBS version[%s] or later, but current version is[%s]", + ZbsConstants.MEGABYTE_SUPPORTED_VERSION, version)); + } + + CbdPath source = parseConversionCbdPath(sourceInstallPath); + CbdPath target = parseConversionCbdPath(targetInstallPath); + if (!source.physicalPool.equals(target.physicalPool) || !source.logicalPool.equals(target.logicalPool)) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion requires source[%s] and target[%s] in the same physical and logical pool", + sourceInstallPath, targetInstallPath)); + } + if (source.volume.equals(target.volume)) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion requires different source and target volumes, but both are[%s]", + sourceInstallPath)); + } + } + + @Override + public void createConversionTarget(String targetInstallPath, long virtualSize, boolean targetEncrypted, + ReturnValueCompletion completion) { + CbdPath target; + try { + target = parseConversionCbdPath(targetInstallPath); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + + ZbsStorageController.CreateVolumeCmd cmd = new ZbsStorageController.CreateVolumeCmd(); + cmd.setLogicalPool(target.logicalPool); + cmd.setVolume(target.volume); + cmd.setUnit(getSizeUnit(addonInfo.getClusterInfo().getVersion())); + long allocatedSize = targetEncrypted ? luksBackingSize(virtualSize) : virtualSize; + cmd.setSize(alignSizeTo(allocatedSize, cmd.getUnit())); + cmd.setSkipIfExisting(false); + + controller.httpCall(ZbsStorageController.CREATE_VOLUME_PATH, cmd, ZbsStorageController.CreateVolumeRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsStorageController.CreateVolumeRsp returnValue) { + String createdInstallPath = returnValue.getInstallPath(); + if (targetInstallPath.equals(createdInstallPath)) { + completion.success(createdInstallPath); + return; + } + + ErrorCode error = operr( + "ZBS volume encryption conversion target[%s] was created at unexpected path[%s]", + targetInstallPath, createdInstallPath); + String cleanupInstallPath = targetInstallPath; + try { + deleteConversionTarget(cleanupInstallPath, new Completion(completion) { + @Override + public void success() { + completion.fail(error); + } + + @Override + public void fail(ErrorCode cleanupError) { + logger.warn(String.format( + "failed to cleanup unexpected ZBS conversion target[installPath:%s] after create path mismatch: %s", + cleanupInstallPath, cleanupError)); + completion.fail(error); + } + }); + } catch (Exception e) { + logger.warn(String.format( + "failed to cleanup unexpected ZBS conversion target[installPath:%s] after create path mismatch: %s", + cleanupInstallPath, e.getMessage())); + completion.fail(error); + } + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void deleteConversionTarget(String targetInstallPath, Completion completion) { + controller.doDeleteVolume(targetInstallPath, true, completion); + } + @Override public void stats(String installPath, ReturnValueCompletion completion) { controller.stats(installPath, completion); @@ -207,6 +301,25 @@ private CbdPath parseActiveCbdPath(String installPath) { return path; } + private CbdPath parseConversionCbdPath(String installPath) { + if (StringUtils.isBlank(installPath) || !installPath.startsWith("cbd:") || installPath.contains("@")) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion requires an active CBD path, but got[%s]", installPath)); + } + + String[] parts = installPath.substring("cbd:".length()).split("/", -1); + if (parts.length != 3 || StringUtils.isBlank(parts[0]) || StringUtils.isBlank(parts[1]) || + StringUtils.isBlank(parts[2])) { + throw new OperationFailureException(operr("invalid ZBS CBD path[%s]", installPath)); + } + + CbdPath path = new CbdPath(); + path.physicalPool = parts[0]; + path.logicalPool = parts[1]; + path.volume = parts[2]; + return path; + } + private long luksBackingSize(long virtualSize) { return virtualSize + LUKS_PAYLOAD_OFFSET; } diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index ed60c4a1f01..c5ef28c4d5d 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -460,6 +460,24 @@ public void fail(ErrorCode errorCode) { }); } + @Override + protected void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + controller.convertVolumeEncryption(msg, + new ReturnValueCompletion(msg) { + @Override + public void success(ConvertVolumeEncryptionOnPrimaryStorageReply returnValue) { + bus.reply(msg, returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + protected void handle(final SetTrashExpirationTimeMsg msg) { controller.setTrashExpireTime(msg.getExpirationTime(), new Completion(msg) { @Override diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java index d234c4fbf05..d71ef4b73ff 100644 --- a/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsEncryptedEmptyVolumeCreator.java @@ -57,7 +57,7 @@ private void createOnPreparedBacking(String primaryStorageUuid, String installPa ReturnValueCompletion completion) { ZbsVolumeEncryptionMaterial material; try { - material = materialFactory.prepareVolumeEncryption(primaryStorageUuid, spec.getUuid(), + material = materialFactory.prepareVolumeEncryption(primaryStorageUuid, spec, String.format("create encrypted volume[uuid:%s]", spec.getUuid())); } catch (OperationFailureException e) { completion.fail(e.getErrorCode()); diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java index 50e61f24e65..d0494f25a1d 100644 --- a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionCommands.java @@ -42,6 +42,16 @@ static class KVMHostLuksResizeCmd extends KVMAgentCommands.AgentCommand implemen public Long virtualSize; } + static class KVMHostLuksConvertCmd extends KVMAgentCommands.AgentCommand implements Serializable { + public String psUuid; + @NoLogging + public String encryptedDek; + public String installPath; + public String targetInstallPath; + public boolean targetEncrypted; + public long virtualSize; + } + static class KVMHostLuksRsp extends KVMAgentCommands.AgentResponse { public Long actualSize; public String installPath; diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java index 1fdcb3fd8f6..f642176834b 100644 --- a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConstants.java @@ -4,6 +4,7 @@ final class ZbsVolumeEncryptionConstants { static final String KVM_HOST_LUKS_CLONE_PATH = "/zbs/primarystorage/kvmhost/luksclone"; static final String KVM_HOST_LUKS_CREATE_EMPTY_PATH = "/zbs/primarystorage/kvmhost/lukscreateempty"; static final String KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH = "/zbs/primarystorage/kvmhost/encryptinplace"; + static final String KVM_HOST_LUKS_CONVERT_PATH = "/zbs/primarystorage/kvmhost/luksconvert"; static final String KVM_HOST_LUKS_RESIZE_PATH = "/zbs/primarystorage/kvmhost/luksresize"; private ZbsVolumeEncryptionConstants() { diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConverter.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConverter.java new file mode 100644 index 00000000000..34fbcb196b3 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionConverter.java @@ -0,0 +1,207 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageReply; +import org.zstack.header.volume.VolumeStats; +import org.zstack.header.volume.VolumeVO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Collections; + +import static org.zstack.core.Platform.operr; + +public class ZbsVolumeEncryptionConverter { + private static final CLogger logger = Utils.getLogger(ZbsVolumeEncryptionConverter.class); + + @Autowired + private ZbsVolumeEncryptionMaterialFactory materialFactory; + @Autowired + private ZbsVolumeEncryptionKvmCaller kvmCaller; + + void convert(ZbsVolumeEncryptionBackend backend, ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item; + try { + item = validate(msg); + backend.validateConversionPaths(item.getSourceInstallPath(), item.getTargetInstallPath()); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + + try { + backend.createConversionTarget(item.getTargetInstallPath(), msg.getVolume().getSize(), + msg.isTargetEncrypted(), new ReturnValueCompletion(completion) { + @Override + public void success(String createdTargetInstallPath) { + convertCreatedTarget(backend, msg, item, createdTargetInstallPath, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } + } + + private ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem validate( + ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { + if (msg.getVolume() == null || StringUtils.isBlank(msg.getVolume().getUuid())) { + throw new OperationFailureException(operr("ZBS volume encryption conversion requires a valid volume")); + } + if (msg.getItems() == null || msg.getItems().size() != 1) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion requires exactly one active volume item, but got[%s]", + msg.getItems() == null ? 0 : msg.getItems().size())); + } + + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item = msg.getItems().get(0); + if (item == null || !VolumeVO.class.getSimpleName().equals(item.getResourceType()) || + !msg.getVolume().getUuid().equals(item.getResourceUuid())) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion item must identify active volume[uuid:%s]", + msg.getVolume().getUuid())); + } + if (StringUtils.isBlank(item.getSourceInstallPath()) || StringUtils.isBlank(item.getTargetInstallPath())) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion requires non-empty source and target paths for volume[uuid:%s]", + msg.getVolume().getUuid())); + } + if (StringUtils.isNotBlank(item.getTargetBackingInstallPath())) { + throw new OperationFailureException(operr( + "ZBS volume encryption conversion does not support backing path[%s] for volume[uuid:%s]", + item.getTargetBackingInstallPath(), msg.getVolume().getUuid())); + } + return item; + } + + private void convertCreatedTarget(ZbsVolumeEncryptionBackend backend, + ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item, + String createdTargetInstallPath, + ReturnValueCompletion completion) { + ZbsVolumeEncryptionMaterial material; + try { + material = materialFactory.prepareVolumeEncryption(backend.getPrimaryStorageUuid(), + msg.getVolume().getUuid(), String.format( + "convert ZBS volume[uuid:%s] encryption", msg.getVolume().getUuid())); + } catch (OperationFailureException e) { + cleanupAndFail(backend, createdTargetInstallPath, e.getErrorCode(), completion); + return; + } catch (RuntimeException e) { + cleanupAndFail(backend, createdTargetInstallPath, + operr("failed to prepare ZBS volume encryption material for volume[uuid:%s]", + msg.getVolume().getUuid()).withException(e.getMessage()), completion); + return; + } + + ZbsVolumeEncryptionCommands.KVMHostLuksConvertCmd cmd = + new ZbsVolumeEncryptionCommands.KVMHostLuksConvertCmd(); + cmd.psUuid = backend.getPrimaryStorageUuid(); + cmd.encryptedDek = material.encryptedDek; + cmd.installPath = item.getSourceInstallPath(); + cmd.targetInstallPath = createdTargetInstallPath; + cmd.targetEncrypted = msg.isTargetEncrypted(); + cmd.virtualSize = msg.getVolume().getSize(); + + try { + kvmCaller.call(material.hostUuid, ZbsVolumeEncryptionConstants.KVM_HOST_LUKS_CONVERT_PATH, + cmd, ZbsVolumeEncryptionCommands.KVMHostLuksRsp.class, "volume encryption conversion", + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsVolumeEncryptionCommands.KVMHostLuksRsp rsp) { + if (rsp.actualSize != null) { + complete(msg, rsp.actualSize, completion); + return; + } + queryTargetStats(backend, msg, createdTargetInstallPath, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + cleanupAndFail(backend, createdTargetInstallPath, errorCode, completion); + } + }); + } catch (OperationFailureException e) { + cleanupAndFail(backend, createdTargetInstallPath, e.getErrorCode(), completion); + } catch (RuntimeException e) { + cleanupAndFail(backend, createdTargetInstallPath, + operr("failed to dispatch ZBS volume encryption conversion for volume[uuid:%s]", + msg.getVolume().getUuid()).withException(e.getMessage()), completion); + } + } + + private void queryTargetStats(ZbsVolumeEncryptionBackend backend, + ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + String targetInstallPath, + ReturnValueCompletion completion) { + try { + backend.stats(targetInstallPath, new ReturnValueCompletion(completion) { + @Override + public void success(VolumeStats stats) { + if (stats == null || stats.getActualSize() == null) { + cleanupAndFail(backend, targetInstallPath, + operr("cannot determine converted ZBS volume[uuid:%s] actual size", + msg.getVolume().getUuid()), completion); + return; + } + complete(msg, stats.getActualSize(), completion); + } + + @Override + public void fail(ErrorCode errorCode) { + cleanupAndFail(backend, targetInstallPath, errorCode, completion); + } + }); + } catch (OperationFailureException e) { + cleanupAndFail(backend, targetInstallPath, e.getErrorCode(), completion); + } catch (RuntimeException e) { + cleanupAndFail(backend, targetInstallPath, + operr("failed to query converted ZBS volume[uuid:%s] stats", + msg.getVolume().getUuid()).withException(e.getMessage()), completion); + } + } + + private void complete(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, long actualSize, + ReturnValueCompletion completion) { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + reply.setActualSizes(Collections.singletonMap(msg.getVolume().getUuid(), actualSize)); + completion.success(reply); + } + + private void cleanupAndFail(ZbsVolumeEncryptionBackend backend, String targetInstallPath, ErrorCode originalError, + ReturnValueCompletion completion) { + try { + backend.deleteConversionTarget(targetInstallPath, new Completion(completion) { + @Override + public void success() { + completion.fail(originalError); + } + + @Override + public void fail(ErrorCode cleanupError) { + logger.warn(String.format( + "failed to cleanup converted ZBS target[installPath:%s] while preserving original conversion error: %s", + targetInstallPath, cleanupError)); + completion.fail(originalError); + } + }); + } catch (Exception e) { + logger.warn(String.format( + "failed to cleanup converted ZBS target[installPath:%s] while preserving original conversion error: %s", + targetInstallPath, e.getMessage())); + completion.fail(originalError); + } + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java index 0cb4033479c..62ab6f33d37 100644 --- a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionExtension.java @@ -7,6 +7,8 @@ import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionExtensionPoint; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageReply; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; import org.zstack.header.volume.VolumeInventory; import org.zstack.header.volume.VolumeStats; @@ -22,6 +24,8 @@ public class ZbsVolumeEncryptionExtension implements ZbsVolumeEncryptionExtensio @Autowired private ZbsVolumeEncryptInPlaceHandler encryptInPlaceHandler; @Autowired + private ZbsVolumeEncryptionConverter volumeEncryptionConverter; + @Autowired private ZbsEncryptedVolumeResizer volumeResizer; @Autowired private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; @@ -50,6 +54,13 @@ public void encryptVolumeBits(ZbsVolumeEncryptionBackend backend, EncryptVolumeB encryptInPlaceHandler.encrypt(backend, msg, completion); } + @Override + public void convertVolumeEncryption(ZbsVolumeEncryptionBackend backend, + ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + volumeEncryptionConverter.convert(backend, msg, completion); + } + @Override public void resizeEncryptedVolume(String primaryStorageUuid, VolumeInventory volume, long size, ReturnValueCompletion completion) { diff --git a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java index 9ddcc9f5f06..1733f3e53c1 100644 --- a/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java +++ b/storage/src/main/java/org/zstack/storage/encrypt/ZbsVolumeEncryptionMaterialFactory.java @@ -5,6 +5,8 @@ import org.zstack.core.db.Q; import org.zstack.header.errorcode.ErrorableValue; import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.image.ImageVO; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; import org.zstack.header.storage.addon.primary.CreateVolumeSpec; import org.zstack.header.storage.snapshot.VolumeSnapshotVO; import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; @@ -18,6 +20,10 @@ public class ZbsVolumeEncryptionMaterialFactory { @Autowired private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; @Autowired private VolumeManager volumeManager; @@ -37,6 +43,49 @@ ZbsVolumeEncryptionMaterial prepareVolumeEncryption(String primaryStorageUuid, S return new ZbsVolumeEncryptionMaterial(hostUuid, encryptedDek); } + ZbsVolumeEncryptionMaterial prepareVolumeEncryption(String primaryStorageUuid, CreateVolumeSpec spec, String operation) { + if (StringUtils.isBlank(spec.getEncryptionKeyResourceType()) && + StringUtils.isBlank(spec.getEncryptionKeyResourceUuid())) { + return prepareVolumeEncryption(primaryStorageUuid, spec.getUuid(), operation); + } + if (StringUtils.isBlank(spec.getEncryptionKeyResourceType()) || + StringUtils.isBlank(spec.getEncryptionKeyResourceUuid())) { + throw new OperationFailureException(operr( + "prepare encrypted ZBS volume requires complete key resource, type:%s uuid:%s", + spec.getEncryptionKeyResourceType(), spec.getEncryptionKeyResourceUuid())); + } + + String hostUuid = findConnectedKvmHost(primaryStorageUuid, operation); + String kpUuid = findKeyProviderUuid(spec.getEncryptionKeyResourceType(), spec.getEncryptionKeyResourceUuid()); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted resource[type:%s, uuid:%s] requires LUKS secret material but has no key provider binding", + spec.getEncryptionKeyResourceType(), spec.getEncryptionKeyResourceUuid())); + } + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(spec.getEncryptionKeyResourceUuid()); + ctx.setResourceType(spec.getEncryptionKeyResourceType()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("prepare-zbs-encryption-material"); + EncryptedResourceKeyManager.ResourceKeyResult keyResult = encryptedResourceKeyManager.getExistingKeySync(ctx); + if (keyResult == null || StringUtils.isBlank(keyResult.getDekBase64())) { + throw new OperationFailureException(operr( + "key manager returned empty DEK for encrypted resource[type:%s, uuid:%s]", + spec.getEncryptionKeyResourceType(), spec.getEncryptionKeyResourceUuid())); + } + + String encryptedDek = volumeEncryptedSecretHelper.verifyHostKeyAndHpkeSealDek( + hostUuid, spec.getEncryptionKeyResourceUuid(), keyResult.getDekBase64()); + if (StringUtils.isBlank(encryptedDek)) { + throw new OperationFailureException(operr( + "cannot prepare LUKS encryptedDek for encrypted resource[type:%s, uuid:%s] on host[uuid:%s]", + spec.getEncryptionKeyResourceType(), spec.getEncryptionKeyResourceUuid(), hostUuid)); + } + return new ZbsVolumeEncryptionMaterial(hostUuid, encryptedDek); + } + ZbsVolumeEncryptionMaterial prepareSnapshotCopyEncryption(String primaryStorageUuid, String srcInstallPath, CreateVolumeSpec dst) { if (StringUtils.isBlank(dst.getUuid())) { @@ -72,6 +121,17 @@ ZbsVolumeEncryptionMaterial prepareSnapshotCopyEncryption(String primaryStorageU return new ZbsVolumeEncryptionMaterial(hostUuid, encryptedDek); } + private String findKeyProviderUuid(String resourceType, String resourceUuid) { + if (VolumeVO.class.getSimpleName().equals(resourceType)) { + return volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(resourceUuid); + } + if (ImageVO.class.getSimpleName().equals(resourceType)) { + return volumeEncryptedResourceKeyBackend.findKeyProviderUuidByTemporarySnapshotImage(resourceUuid); + } + throw new OperationFailureException(operr( + "unsupported encrypted ZBS key resource[type:%s, uuid:%s]", resourceType, resourceUuid)); + } + private String findConnectedKvmHost(String primaryStorageUuid, String operation) { ErrorableValue host = volumeManager.findConnectedKvmHostByPrimaryStorage( primaryStorageUuid, "external primary storage", operation); diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java index 738bcbe80d8..ee6f015f4e4 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java @@ -3525,7 +3525,9 @@ private VolumeEncryptionConversionContext buildVolumeEncryptionConversionContext } private ErrorCode validateVolumeEncryptionConversion() { - if (!isCephInstallPath(self.getInstallPath())) { + boolean ceph = isCephInstallPath(self.getInstallPath()); + boolean cbd = isCbdInstallPath(self.getInstallPath()); + if (!ceph && !cbd) { return null; } @@ -3537,8 +3539,9 @@ private ErrorCode validateVolumeEncryptionConversion() { return null; } - return operr("cannot change encryption for Ceph/RBD volume[uuid:%s] because it has snapshots; delete snapshots before changing volume encryption", - self.getUuid()); + String storageType = cbd ? "ZBS/CBD" : "Ceph/RBD"; + return operr("cannot change encryption for %s volume[uuid:%s] because it has snapshots; delete snapshots before changing volume encryption", + storageType, self.getUuid()); } private String resolveVolumeSecretHostUuid(VolumeVO volume) { @@ -3661,6 +3664,9 @@ private String makeConvertedVolumeInstallPath(String installPath, String resourc if (isCephInstallPath(installPath)) { return makeCephVolumeInstallPath(installPath, convertedName); } + if (isCbdInstallPath(installPath)) { + return makeSiblingInstallPath(installPath, convertedName); + } if (isSharedBlockInstallPath(installPath) || installPath.startsWith("/dev/")) { return makeSiblingInstallPath(installPath, convertedName); } @@ -3723,6 +3729,10 @@ private boolean isCephInstallPath(String installPath) { return installPath != null && installPath.startsWith("ceph://"); } + private boolean isCbdInstallPath(String installPath) { + return installPath != null && installPath.startsWith("cbd:"); + } + private boolean isSharedBlockInstallPath(String installPath) { return installPath != null && installPath.startsWith("sharedblock://"); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVolumeEncryptionCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVolumeEncryptionCase.groovy index 1c05b49b5dc..49ab914f6f7 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVolumeEncryptionCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVolumeEncryptionCase.groovy @@ -1,16 +1,48 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity +import org.zstack.compute.vm.devices.DummyEncryptedResourceKeyManager import org.zstack.core.cloudbus.CloudBus import org.zstack.core.Platform +import org.zstack.core.db.DatabaseFacade +import org.zstack.header.core.Completion +import org.zstack.header.core.FutureReturnValueCompletion +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.errorcode.OperationFailureException +import org.zstack.header.image.ImageVO +import org.zstack.header.keyprovider.EncryptedResourceKeyManager import org.zstack.header.message.MessageReply +import org.zstack.header.storage.addon.primary.CreateVolumeSpec +import org.zstack.header.storage.addon.primary.ZbsVolumeEncryptionBackend +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageReply import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageReply import org.zstack.header.storage.primary.PrimaryStorageConstant +import org.zstack.header.storage.snapshot.VolumeSnapshotState +import org.zstack.header.storage.snapshot.VolumeSnapshotStatus +import org.zstack.header.storage.snapshot.VolumeSnapshotTreeStatus +import org.zstack.header.storage.snapshot.VolumeSnapshotTreeVO +import org.zstack.header.storage.snapshot.VolumeSnapshotVO +import org.zstack.header.volume.VolumeInventory +import org.zstack.header.volume.VolumeState +import org.zstack.header.volume.VolumeStatus +import org.zstack.header.volume.VolumeStats +import org.zstack.header.volume.VolumeType +import org.zstack.header.volume.VolumeVO import org.zstack.kvm.KVMHostAsyncHttpCallMsg import org.zstack.kvm.KVMHostAsyncHttpCallReply +import org.zstack.sdk.ChangeVolumeEncryptionAction +import org.zstack.sdk.ClusterInventory import org.zstack.sdk.KVMHostInventory import org.zstack.sdk.PrimaryStorageInventory +import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper +import org.zstack.storage.encrypt.DummyVolumeEncryptedResourceKeyBackend +import org.zstack.storage.encrypt.ZbsVolumeEncryptionExtension +import org.zstack.storage.encrypt.ZbsVolumeEncryptionKvmCaller +import org.zstack.storage.encrypt.ZbsVolumeEncryptionMaterialFactory +import org.zstack.storage.volume.VolumeBase import org.zstack.storage.zbs.ZbsConstants import org.zstack.storage.zbs.ZbsStorageController import org.zstack.test.integration.storage.StorageTest @@ -19,17 +51,30 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil +import java.util.concurrent.TimeUnit + class ZbsVolumeEncryptionCase extends SubCase { private static final String QUERY_SNAPSHOT_PATH = "/zbs/primarystorage/snapshot/query" + private static final String KVM_LUKS_CREATE_EMPTY_PATH = "/zbs/primarystorage/kvmhost/lukscreateempty" private static final String KVM_LUKS_ENCRYPT_IN_PLACE_PATH = "/zbs/primarystorage/kvmhost/encryptinplace" + private static final String KVM_LUKS_CONVERT_PATH = "/zbs/primarystorage/kvmhost/luksconvert" EnvSpec env PrimaryStorageInventory ps + ClusterInventory cluster KVMHostInventory host CloudBus bus + boolean primaryStorageAttached @Override void clean() { + if (primaryStorageAttached) { + detachPrimaryStorageFromCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + primaryStorageAttached = false + } env.delete() } @@ -73,9 +118,619 @@ class ZbsVolumeEncryptionCase extends SubCase { ps = env.inventoryByName("zbs") as PrimaryStorageInventory host = env.inventoryByName("kvm") as KVMHostInventory bus = bean(CloudBus.class) + cluster = env.inventoryByName("cluster") as ClusterInventory + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + primaryStorageAttached = true + testBuildNativeCbdConversionPaths() + testPrepareVolumeEncryptionFallsBackToTargetVolumeKey() + testEncryptedEmptyVolumeCreatorReusesImageKey() + testPrepareVolumeEncryptionRejectsIncompleteKeyResource() + testChangeZbsVolumeEncryptionThroughPublicApi() + testConvertPlainVolumeToEncryptedTarget() + testConvertEncryptedVolumeToPlainTarget() + testRejectBidirectionalConversionWhenZbsVersionDoesNotSupportMegabyteAllocation() + testConvertFailureDeletesOnlyCreatedTarget() + testConvertMaterialFailureDeletesCreatedTarget() + testConvertMaterialRuntimeFailureDeletesCreatedTargetOnce() + testConvertKvmRuntimeFailureDeletesCreatedTargetOnce() + testConvertTargetConflictDoesNotCallKvmOrDeleteTarget() + testConvertKeepsZeroKvmActualSize() + testConvertFallsBackToTargetStats() + testConvertStatsFailureDeletesCreatedTarget() + testConvertStatsRuntimeFailureDeletesCreatedTargetOnce() + testConvertCreatePathMismatchDeletesOnlyRequestedTarget() + testConvertCreatePathMismatchNeverDeletesSource() + testConvertBlankCreatePathDeletesRequestedTarget() + testConvertRejectsInvalidItemsBeforeMutation() testEncryptVolumeBitsCreatesEncryptedTarget() testEncryptVolumeBitsCleansCreatedTargetOnKvmFailure() + testRejectConversionWhenZbsVolumeHasSnapshots() + } + } + + void testBuildNativeCbdConversionPaths() { + String volumeUuid = Platform.uuid + VolumeVO volume = new VolumeVO() + volume.uuid = volumeUuid + volume.installPath = "cbd:physicalPool/logicalPool/source" + VolumeBase base = new VolumeBase(volume) + def method = VolumeBase.class.getDeclaredMethod("makeConvertedVolumeInstallPath", + String.class, String.class, Boolean.TYPE, String.class) + method.accessible = true + + String encryptedTarget = method.invoke(base, volume.installPath, volumeUuid, true, "conversion") + String plainTarget = method.invoke(base, volume.installPath, volumeUuid, false, "conversion") + + assert encryptedTarget == "cbd:physicalPool/logicalPool/${volumeUuid}.encrypted.conversion" + assert plainTarget == "cbd:physicalPool/logicalPool/${volumeUuid}.plain.conversion" + assert encryptedTarget != volume.installPath + assert plainTarget != volume.installPath + assert !encryptedTarget.endsWith(".qcow2") + assert !plainTarget.endsWith(".qcow2") + } + + void testPrepareVolumeEncryptionFallsBackToTargetVolumeKey() { + def method = prepareVolumeEncryptionWithSpecMethod() + String volumeUuid = Platform.uuid + CreateVolumeSpec spec = new CreateVolumeSpec() + spec.uuid = volumeUuid + StubVolumeEncryptedSecretHelper secretHelper = new StubVolumeEncryptedSecretHelper(sealedDek: "volume-sealed-dek") + StubEncryptedResourceKeyManager keyManager = new StubEncryptedResourceKeyManager() + StubVolumeEncryptedResourceKeyBackend keyBackend = new StubVolumeEncryptedResourceKeyBackend() + + def material = withMaterialFactoryStubs(secretHelper, keyManager, keyBackend) { factory -> + invokePrepareVolumeEncryption(method, factory, spec) + } + + assert secretHelper.requests == [[hostUuid: host.uuid, volumeUuid: volumeUuid]] + assert secretHelper.sealRequests.isEmpty() + assert keyManager.requests.isEmpty() + assert keyBackend.volumeRequests.isEmpty() + assert keyBackend.imageRequests.isEmpty() + assert readMaterialField(material, "encryptedDek") == "volume-sealed-dek" + } + + void testEncryptedEmptyVolumeCreatorReusesImageKey() { + String imageUuid = Platform.uuid + String keyProviderUuid = Platform.uuid + String installPath = "cbd:lpool1/target-volume" + CreateVolumeSpec spec = new CreateVolumeSpec() + spec.uuid = Platform.uuid + spec.name = "target-volume" + spec.size = SizeUnit.GIGABYTE.toByte(1) + spec.encryptionKeyResourceType = ImageVO.simpleName + spec.encryptionKeyResourceUuid = imageUuid + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = new EncryptedResourceKeyManager.ResourceKeyResult() + keyResult.dekBase64 = "image-dek" + keyResult.keyProviderUuid = keyProviderUuid + StubVolumeEncryptedSecretHelper secretHelper = new StubVolumeEncryptedSecretHelper(sealedDek: "image-sealed-dek") + StubEncryptedResourceKeyManager keyManager = new StubEncryptedResourceKeyManager(result: keyResult) + StubVolumeEncryptedResourceKeyBackend keyBackend = + new StubVolumeEncryptedResourceKeyBackend(keyProviderUuid: keyProviderUuid) + List deletedPaths = [] + ZbsVolumeEncryptionBackend backend = [ + getPrimaryStorageUuid : { ps.uuid }, + buildConfiguredVolumePath: { String volumeName -> installPath }, + createLuksBackingVolume: { String path, long size, ReturnValueCompletion completion -> + completion.success(path) + }, + deleteLuksBackingVolume: { String path -> deletedPaths << path }, + ] as ZbsVolumeEncryptionBackend + env.message(KVMHostAsyncHttpCallMsg.class) { KVMHostAsyncHttpCallMsg msg, CloudBus cloudBus -> + assert msg.path == KVM_LUKS_CREATE_EMPTY_PATH + LinkedHashMap cmd = JSONObjectUtil.toObject(msg.command, LinkedHashMap.class) + assert cmd.encryptedDek == "image-sealed-dek" + KVMHostAsyncHttpCallReply reply = new KVMHostAsyncHttpCallReply() + reply.response = [success: true] as LinkedHashMap + cloudBus.reply(msg, reply) + } + + FutureReturnValueCompletion completion = new FutureReturnValueCompletion(null) + withMaterialFactoryStubs(secretHelper, keyManager, keyBackend) { + bean(ZbsVolumeEncryptionExtension.class).createEncryptedEmptyVolume(backend, spec, completion) + } + completion.await(TimeUnit.SECONDS.toMillis(10)) + + assert completion.success : completion.errorCode + assert (completion.result as VolumeStats).installPath == installPath + assert deletedPaths.isEmpty() + assert keyBackend.imageRequests == [imageUuid] + assert keyManager.requests*.resourceUuid == [imageUuid] + } + + void testPrepareVolumeEncryptionRejectsIncompleteKeyResource() { + def method = prepareVolumeEncryptionWithSpecMethod() + CreateVolumeSpec spec = new CreateVolumeSpec() + spec.uuid = Platform.uuid + spec.encryptionKeyResourceType = ImageVO.simpleName + StubVolumeEncryptedSecretHelper secretHelper = new StubVolumeEncryptedSecretHelper() + StubEncryptedResourceKeyManager keyManager = new StubEncryptedResourceKeyManager() + StubVolumeEncryptedResourceKeyBackend keyBackend = new StubVolumeEncryptedResourceKeyBackend() + OperationFailureException failure = null + + withMaterialFactoryStubs(secretHelper, keyManager, keyBackend) { factory -> + try { + invokePrepareVolumeEncryption(method, factory, spec) + } catch (OperationFailureException e) { + failure = e + } + } + + assert failure != null + assert failure.message.contains("requires complete key resource") + assert secretHelper.requests.isEmpty() + assert secretHelper.sealRequests.isEmpty() + assert keyManager.requests.isEmpty() + assert keyBackend.volumeRequests.isEmpty() + assert keyBackend.imageRequests.isEmpty() + } + + void testChangeZbsVolumeEncryptionThroughPublicApi() { + DatabaseFacade dbf = bean(DatabaseFacade.class) + String volumeUuid = Platform.uuid + String sourceInstallPath = "cbd:pool1/lpool1/${volumeUuid}" + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + long actualSize = SizeUnit.MEGABYTE.toByte(768) + VolumeVO volume = new VolumeVO() + volume.uuid = volumeUuid + volume.name = "zbs-public-api-conversion" + volume.primaryStorageUuid = ps.uuid + volume.installPath = sourceInstallPath + volume.type = VolumeType.Data + volume.status = VolumeStatus.Ready + volume.state = VolumeState.Enabled + volume.format = "raw" + volume.size = sourceSize + volume.actualSize = SizeUnit.GIGABYTE.toByte(1) + volume.encrypted = true + dbf.persistAndRefresh(volume) + + Expando tracker = installPublicApiConversionSimulators(sourceInstallPath, sourceSize, actualSize) + try { + withSealedDekStub { List> requests -> + ChangeVolumeEncryptionAction action = new ChangeVolumeEncryptionAction() + action.uuid = volumeUuid + action.encrypted = false + action.sessionId = adminSession() + ChangeVolumeEncryptionAction.Result result = action.call() + + assert result.error == null + assert result.value.inventory.uuid == volumeUuid + assert !result.value.inventory.encrypted + assert result.value.inventory.installPath == tracker.targetInstallPath + assert result.value.inventory.actualSize == actualSize + assert tracker.createVolumeCount == 1 + assert tracker.kvmConvertCount == 1 + assert tracker.createCmd.logicalPool == "lpool1" + assert tracker.createCmd.size == sourceSizeInCreateUnit(sourceSize, tracker.createCmd.unit as String) + assert tracker.kvmCmd.installPath == sourceInstallPath + assert tracker.kvmCmd.targetInstallPath == tracker.targetInstallPath + assert !tracker.kvmCmd.targetEncrypted + assert tracker.kvmCmd.virtualSize == sourceSize + assert tracker.deletedInstallPaths.isEmpty() + assert tracker.targetInstallPath.startsWith("cbd:pool1/lpool1/") + assert !tracker.targetInstallPath.endsWith(".qcow2") + assert requests == [[hostUuid: host.uuid, volumeUuid: volumeUuid]] + + VolumeVO current = dbf.findByUuid(volumeUuid, VolumeVO.class) + assert !current.encrypted + assert current.installPath == tracker.targetInstallPath + assert current.actualSize == actualSize + } + } finally { + dbf.removeByPrimaryKey(volumeUuid, VolumeVO.class) + } + } + + void testRejectConversionWhenZbsVolumeHasSnapshots() { + DatabaseFacade dbf = bean(DatabaseFacade.class) + String volumeUuid = Platform.uuid + String treeUuid = Platform.uuid + String snapshotUuid = Platform.uuid + String sourceInstallPath = "cbd:pool1/lpool1/${volumeUuid}" + VolumeVO volume = new VolumeVO() + volume.uuid = volumeUuid + volume.name = "zbs-snapshot-conversion-guard" + volume.primaryStorageUuid = ps.uuid + volume.installPath = sourceInstallPath + volume.type = VolumeType.Data + volume.status = VolumeStatus.Ready + volume.state = VolumeState.Enabled + volume.format = "raw" + volume.size = SizeUnit.GIGABYTE.toByte(1) + volume.actualSize = SizeUnit.MEGABYTE.toByte(1) + volume.encrypted = false + dbf.persistAndRefresh(volume) + + VolumeSnapshotTreeVO tree = new VolumeSnapshotTreeVO() + tree.uuid = treeUuid + tree.volumeUuid = volumeUuid + tree.current = true + tree.status = VolumeSnapshotTreeStatus.Completed + dbf.persistAndRefresh(tree) + + VolumeSnapshotVO snapshot = new VolumeSnapshotVO() + snapshot.uuid = snapshotUuid + snapshot.name = "zbs-snapshot-conversion-guard" + snapshot.volumeUuid = volumeUuid + snapshot.treeUuid = treeUuid + snapshot.primaryStorageUuid = ps.uuid + snapshot.primaryStorageInstallPath = "${sourceInstallPath}@${snapshotUuid}" + snapshot.state = VolumeSnapshotState.Enabled + snapshot.status = VolumeSnapshotStatus.Ready + snapshot.format = "raw" + snapshot.volumeType = VolumeType.Data.toString() + snapshot.size = volume.size + dbf.persistAndRefresh(snapshot) + + Expando tracker = installSnapshotConversionGuardCounters() + try { + ChangeVolumeEncryptionAction action = new ChangeVolumeEncryptionAction() + action.uuid = volumeUuid + action.encrypted = true + action.sessionId = adminSession() + ChangeVolumeEncryptionAction.Result result = action.call() + + assert result.error != null + assert result.error.details.contains("snapshots") + assert result.error.details.contains("ZBS/CBD") + assert tracker.conversionMessageCount == 0 + assert tracker.createVolumeCount == 0 + assert tracker.kvmConvertCount == 0 + + VolumeVO current = dbf.findByUuid(volumeUuid, VolumeVO.class) + assert !current.encrypted + assert current.installPath == sourceInstallPath + } finally { + dbf.removeByPrimaryKey(snapshotUuid, VolumeSnapshotVO.class) + dbf.removeByPrimaryKey(treeUuid, VolumeSnapshotTreeVO.class) + dbf.removeByPrimaryKey(volumeUuid, VolumeVO.class) + } + } + + void testConvertPlainVolumeToEncryptedTarget() { + String sourceInstallPath = "cbd:pool1/lpool1/volume-source-convert" + String targetInstallPath = "cbd:pool1/lpool1/volume-target-convert" + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + long actualSize = SizeUnit.GIGABYTE.toByte(1) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + sourceInstallPath, targetInstallPath, sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, actualSize, sourceSize.intdiv(3)) + + withSealedDekStub { List> requests -> + ConvertVolumeEncryptionOnPrimaryStorageReply reply = + bus.call(msg) as ConvertVolumeEncryptionOnPrimaryStorageReply + + assert reply.success + assert reply.actualSizes == [(msg.volume.uuid): actualSize] + assert tracker.createVolumeCount == 1 + assert tracker.kvmConvertCount == 1 + assert tracker.statsCount == 0 + assert tracker.deletedInstallPaths.isEmpty() + assert tracker.createCmd.logicalPool == "lpool1" + assert tracker.createCmd.volume == "volume-target-convert" + assert !tracker.createCmd.skipIfExisting + assert tracker.createCmd.size == sourceSizeInCreateUnit( + sourceSize + SizeUnit.MEGABYTE.toByte(8), tracker.createCmd.unit as String) + assert tracker.kvmCmd.targetEncrypted + assert tracker.kvmCmd.encryptedDek == "sealed-dek" + assert requests == [[hostUuid: host.uuid, volumeUuid: msg.volume.uuid]] + } + } + + void testConvertEncryptedVolumeToPlainTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/encrypted-source", "cbd:pool1/lpool1/plain-target", + sourceSize, true, false) + Expando tracker = installConversionSimulators(msg, false, false, + sourceSize.intdiv(2), sourceSize.intdiv(3)) + + withSealedDekStub { List> requests -> + ConvertVolumeEncryptionOnPrimaryStorageReply reply = + bus.call(msg) as ConvertVolumeEncryptionOnPrimaryStorageReply + + assert reply.success + assert tracker.createCmd.size == sourceSizeInCreateUnit(sourceSize, tracker.createCmd.unit as String) + assert tracker.kvmCmd.containsKey("targetEncrypted") + assert !tracker.kvmCmd.targetEncrypted + assert tracker.kvmCmd.encryptedDek == "sealed-dek" + assert requests == [[hostUuid: host.uuid, volumeUuid: msg.volume.uuid]] + assert tracker.deletedInstallPaths.isEmpty() + } + } + + void testRejectBidirectionalConversionWhenZbsVersionDoesNotSupportMegabyteAllocation() { + env.cleanAfterSimulatorHandlers() + env.afterSimulator(ZbsStorageController.GET_FACTS_PATH) { ZbsStorageController.GetFactsRsp rsp, + HttpEntity e -> + rsp.version = "1.6.0" + return rsp + } + reconnectPrimaryStorage { + uuid = ps.uuid + } + env.cleanAfterSimulatorHandlers() + + try { + [[false, true], [true, false]].eachWithIndex { List encryption, int index -> + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/unsupported-version-source-${index}", + "cbd:pool1/lpool1/unsupported-version-target-${index}", + SizeUnit.GIGABYTE.toByte(2), encryption[0], encryption[1]) + Expando tracker = installConversionSimulators(msg, false, false, + SizeUnit.GIGABYTE.toByte(1), SizeUnit.GIGABYTE.toByte(1)) + + withSealedDekStub { + MessageReply reply = bus.call(msg) + + assert !reply.success + assert reply.error.details.contains("1.6.0") + assert reply.error.details.contains(ZbsConstants.MEGABYTE_SUPPORTED_VERSION) + assert tracker.createVolumeCount == 0 + assert tracker.kvmConvertCount == 0 + assert tracker.deletedInstallPaths.isEmpty() + } + } + } finally { + env.cleanSimulatorAndMessageHandlers() + reconnectPrimaryStorage { + uuid = ps.uuid + } + } + } + + void testConvertFailureDeletesOnlyCreatedTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/failure-source", "cbd:pool1/lpool1/failure-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, true, false, null, sourceSize.intdiv(3)) + + withSealedDekStub { + MessageReply reply = bus.call(msg) + + assert !reply.success + assert reply.error.details.contains("on purpose") + retryInSecs(3, 1) { + assert tracker.deletedInstallPaths == [msg.items[0].targetInstallPath] + assert !tracker.deletedInstallPaths.contains(msg.items[0].sourceInstallPath) + } + } + } + + void testConvertMaterialFailureDeletesCreatedTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/material-failure-source", "cbd:pool1/lpool1/material-failure-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, null, sourceSize.intdiv(3)) + + withSealedDekStub("") { List> requests -> + MessageReply reply = bus.call(msg) + + assert !reply.success + assert reply.error.details.contains("cannot prepare LUKS encryptedDek") + assert tracker.createVolumeCount == 1 + assert tracker.kvmConvertCount == 0 + assert requests == [[hostUuid: host.uuid, volumeUuid: msg.volume.uuid]] + retryInSecs(3, 1) { + assert tracker.deletedInstallPaths == [msg.items[0].targetInstallPath] + assert !tracker.deletedInstallPaths.contains(msg.items[0].sourceInstallPath) + } + } + } + + void testConvertMaterialRuntimeFailureDeletesCreatedTargetOnce() { + Expando tracker = runPostCreateRuntimeFailure("material") + + assert tracker.runtimeException == null + assert tracker.successCount == 0 + assert tracker.failureCount == 1 + assert tracker.error.details.contains("failed to prepare ZBS volume encryption material") + assert tracker.error.getFromOpaque("exception") == "material runtime failure" + assert tracker.deletedInstallPaths == [tracker.targetInstallPath] + } + + void testConvertKvmRuntimeFailureDeletesCreatedTargetOnce() { + Expando tracker = runPostCreateRuntimeFailure("kvm") + + assert tracker.runtimeException == null + assert tracker.successCount == 0 + assert tracker.failureCount == 1 + assert tracker.error.details.contains("kvm runtime failure") + assert tracker.deletedInstallPaths == [tracker.targetInstallPath] + } + + void testConvertKeepsZeroKvmActualSize() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/zero-source", "cbd:pool1/lpool1/zero-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, 0L, sourceSize.intdiv(3)) + + withSealedDekStub { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = + bus.call(msg) as ConvertVolumeEncryptionOnPrimaryStorageReply + + assert reply.success + assert reply.actualSizes == [(msg.volume.uuid): 0L] + assert tracker.statsCount == 0 + assert tracker.deletedInstallPaths.isEmpty() + } + } + + void testConvertTargetConflictDoesNotCallKvmOrDeleteTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/conflict-source", "cbd:pool1/lpool1/conflict-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, true, null, sourceSize.intdiv(3)) + + withSealedDekStub { List> requests -> + MessageReply reply = bus.call(msg) + + assert !reply.success + assert tracker.createVolumeCount == 1 + assert tracker.kvmConvertCount == 0 + assert tracker.deletedInstallPaths.isEmpty() + assert requests.isEmpty() + } + } + + void testConvertFallsBackToTargetStats() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + long targetActualSize = SizeUnit.MEGABYTE.toByte(768) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/stats-source", "cbd:pool1/lpool1/stats-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, null, targetActualSize) + + withSealedDekStub { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = + bus.call(msg) as ConvertVolumeEncryptionOnPrimaryStorageReply + + assert reply.success + assert reply.actualSizes == [(msg.volume.uuid): targetActualSize] + assert tracker.statsCount == 1 + assert tracker.deletedInstallPaths.isEmpty() + } + } + + void testConvertStatsFailureDeletesCreatedTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/stats-failure-source", "cbd:pool1/lpool1/stats-failure-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, null, 0, true) + + withSealedDekStub { + MessageReply reply = bus.call(msg) + + assert !reply.success + retryInSecs(3, 1) { + assert tracker.deletedInstallPaths == [msg.items[0].targetInstallPath] + } + } + } + + void testConvertStatsRuntimeFailureDeletesCreatedTargetOnce() { + Expando tracker = runPostCreateRuntimeFailure("stats") + + retryInSecs(3, 1) { + assert tracker.failureCount == 1 + assert tracker.deletedInstallPaths == [tracker.targetInstallPath] + } + assert tracker.runtimeException == null + assert tracker.successCount == 0 + assert tracker.error.details.contains("failed to query converted ZBS volume") + assert tracker.error.getFromOpaque("exception") == "stats runtime failure" + } + + void testConvertCreatePathMismatchDeletesOnlyRequestedTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + String returnedInstallPath = "cbd:pool1/lpool1/unexpected-created-target" + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/path-mismatch-source", "cbd:pool1/lpool1/path-mismatch-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, null, 0, + false, returnedInstallPath) + + withSealedDekStub { List> requests -> + MessageReply reply = bus.call(msg) + + assert !reply.success + retryInSecs(3, 1) { + assert tracker.deletedInstallPaths == [msg.items[0].targetInstallPath] + assert !tracker.deletedInstallPaths.contains(msg.items[0].sourceInstallPath) + assert !tracker.deletedInstallPaths.contains(returnedInstallPath) + } + assert tracker.kvmConvertCount == 0 + assert requests.isEmpty() + } + } + + void testConvertCreatePathMismatchNeverDeletesSource() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/source-returned-by-create", "cbd:pool1/lpool1/source-mismatch-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, null, 0, + false, msg.items[0].sourceInstallPath) + + withSealedDekStub { List> requests -> + MessageReply reply = bus.call(msg) + + assert !reply.success + retryInSecs(3, 1) { + assert tracker.deletedInstallPaths == [msg.items[0].targetInstallPath] + assert !tracker.deletedInstallPaths.contains(msg.items[0].sourceInstallPath) + } + assert tracker.kvmConvertCount == 0 + assert requests.isEmpty() + } + } + + void testConvertBlankCreatePathDeletesRequestedTarget() { + long sourceSize = SizeUnit.GIGABYTE.toByte(2) + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/blank-path-source", "cbd:pool1/lpool1/blank-path-target", + sourceSize, false, true) + Expando tracker = installConversionSimulators(msg, false, false, null, 0, + false, "") + + withSealedDekStub { List> requests -> + MessageReply reply = bus.call(msg) + + assert !reply.success + assert reply.error.details.contains("unexpected path") + retryInSecs(3, 1) { + assert tracker.deletedInstallPaths == [msg.items[0].targetInstallPath] + assert !tracker.deletedInstallPaths.contains("") + assert !tracker.deletedInstallPaths.contains(msg.items[0].sourceInstallPath) + } + assert tracker.kvmConvertCount == 0 + assert requests.isEmpty() + } + } + + void testConvertRejectsInvalidItemsBeforeMutation() { + Expando tracker = installConversionMutationCounters() + List invalidMutations = [ + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items = null }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items = [] }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items = [msg.items[0], copyConversionItem(msg.items[0])] }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].resourceType = VolumeSnapshotVO.simpleName }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].resourceUuid = Platform.getUuid() }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].sourceInstallPath = "" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetInstallPath = "" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetBackingInstallPath = "cbd:pool1/lpool1/backing" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].sourceInstallPath = "pool1/lpool1/invalid-source" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetInstallPath = "cbd:pool1//invalid-target" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].sourceInstallPath += "@snapshot" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetInstallPath += "@snapshot" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetInstallPath = "cbd:pool2/lpool1/invalid-target" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetInstallPath = "cbd:pool1/lpool2/invalid-target" }, + { ConvertVolumeEncryptionOnPrimaryStorageMsg msg -> msg.items[0].targetInstallPath = msg.items[0].sourceInstallPath } + ] + + invalidMutations.each { Closure mutation -> + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/valid-source", "cbd:pool1/lpool1/valid-target", + SizeUnit.GIGABYTE.toByte(2), false, true) + mutation.call(msg) + + MessageReply reply = bus.call(msg) + + assert !reply.success + assert tracker.createVolumeCount == 0 + assert tracker.kvmConvertCount == 0 + assert tracker.deletedInstallPaths.isEmpty() } } @@ -114,6 +769,424 @@ class ZbsVolumeEncryptionCase extends SubCase { } } + private Expando installSnapshotConversionGuardCounters() { + env.cleanSimulatorAndMessageHandlers() + Expando tracker = new Expando( + conversionMessageCount: 0, + createVolumeCount: 0, + kvmConvertCount: 0 + ) + + env.message(ConvertVolumeEncryptionOnPrimaryStorageMsg.class) { + ConvertVolumeEncryptionOnPrimaryStorageMsg msg, CloudBus cloudBus -> + tracker.conversionMessageCount++ + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply() + reply.setError(operr("conversion message must not be sent when the ZBS volume has snapshots")) + cloudBus.reply(msg, reply) + } + env.simulator(ZbsStorageController.CREATE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + tracker.createVolumeCount++ + return new ZbsStorageController.CreateVolumeRsp() + } + env.message(KVMHostAsyncHttpCallMsg.class) { KVMHostAsyncHttpCallMsg msg, CloudBus cloudBus -> + tracker.kvmConvertCount++ + KVMHostAsyncHttpCallReply reply = new KVMHostAsyncHttpCallReply() + reply.response = [success: true, actualSize: 1L] as LinkedHashMap + cloudBus.reply(msg, reply) + } + return tracker + } + + private Expando installPublicApiConversionSimulators(String sourceInstallPath, + long sourceSize, + long actualSize) { + env.cleanSimulatorAndMessageHandlers() + Expando tracker = new Expando( + createVolumeCount: 0, + kvmConvertCount: 0, + createCmd: null, + kvmCmd: null, + targetInstallPath: null, + deletedInstallPaths: [] + ) + + env.simulator(ZbsStorageController.CREATE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + tracker.createVolumeCount++ + tracker.createCmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd.class) + tracker.targetInstallPath = "cbd:pool1/${tracker.createCmd.logicalPool}/${tracker.createCmd.volume}" + ZbsStorageController.CreateVolumeRsp rsp = new ZbsStorageController.CreateVolumeRsp() + rsp.installPath = tracker.targetInstallPath + return rsp + } + env.simulator(ZbsStorageController.DELETE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, LinkedHashMap.class) + tracker.deletedInstallPaths << cmd.path + return new ZbsStorageController.DeleteVolumeRsp() + } + env.message(KVMHostAsyncHttpCallMsg.class) { KVMHostAsyncHttpCallMsg hmsg, CloudBus cloudBus -> + tracker.kvmConvertCount++ + assert hmsg.path == KVM_LUKS_CONVERT_PATH + assert hmsg.hostUuid == host.uuid + tracker.kvmCmd = JSONObjectUtil.toObject(hmsg.command, LinkedHashMap.class) + assert tracker.kvmCmd.psUuid == ps.uuid + assert tracker.kvmCmd.installPath == sourceInstallPath + assert tracker.kvmCmd.targetInstallPath == tracker.targetInstallPath + assert tracker.kvmCmd.virtualSize == sourceSize + + KVMHostAsyncHttpCallReply reply = new KVMHostAsyncHttpCallReply() + reply.response = [success: true, actualSize: actualSize] as LinkedHashMap + cloudBus.reply(hmsg, reply) + } + return tracker + } + + private ConvertVolumeEncryptionOnPrimaryStorageMsg buildConversionMsg(String sourceInstallPath, + String targetInstallPath, + long sourceSize, + boolean sourceEncrypted, + boolean targetEncrypted) { + VolumeInventory volume = new VolumeInventory() + volume.uuid = Platform.getUuid() + volume.primaryStorageUuid = ps.uuid + volume.size = sourceSize + volume.encrypted = sourceEncrypted + + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item = + new ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem() + item.resourceUuid = volume.uuid + item.resourceType = VolumeVO.simpleName + item.sourceInstallPath = sourceInstallPath + item.targetInstallPath = targetInstallPath + + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = new ConvertVolumeEncryptionOnPrimaryStorageMsg() + msg.volume = volume + msg.targetEncrypted = targetEncrypted + msg.items = [item] + bus.makeTargetServiceIdByResourceUuid(msg, PrimaryStorageConstant.SERVICE_ID, ps.uuid) + return msg + } + + private ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem copyConversionItem( + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem source) { + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item = + new ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem() + item.resourceUuid = source.resourceUuid + item.resourceType = source.resourceType + item.sourceInstallPath = source.sourceInstallPath + item.targetInstallPath = source.targetInstallPath + item.targetBackingInstallPath = source.targetBackingInstallPath + return item + } + + private Expando installConversionSimulators(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + boolean failKvm, + boolean targetConflict, + Long kvmActualSize, + long statsActualSize, + boolean failStats = false, + String createdInstallPath = null) { + env.cleanSimulatorAndMessageHandlers() + def item = msg.items[0] + Expando tracker = new Expando( + createVolumeCount: 0, + kvmConvertCount: 0, + statsCount: 0, + createCmd: null, + kvmCmd: null, + deletedInstallPaths: [] + ) + + env.simulator(ZbsStorageController.CREATE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + tracker.createVolumeCount++ + tracker.createCmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd.class) + if (targetConflict) { + return [success: false, error: "target already exists"] + } + + ZbsStorageController.CreateVolumeRsp rsp = new ZbsStorageController.CreateVolumeRsp() + rsp.installPath = createdInstallPath == null ? item.targetInstallPath : createdInstallPath + return rsp + } + + env.simulator(ZbsStorageController.DELETE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, LinkedHashMap.class) + tracker.deletedInstallPaths << cmd.path + return new ZbsStorageController.DeleteVolumeRsp() + } + + env.simulator(ZbsStorageController.QUERY_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + tracker.statsCount++ + ZbsStorageController.QueryVolumeCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.QueryVolumeCmd.class) + assert cmd.path == item.targetInstallPath + if (failStats) { + return [success: false, error: "stats failed on purpose"] + } + + ZbsStorageController.QueryVolumeRsp rsp = new ZbsStorageController.QueryVolumeRsp() + rsp.size = msg.volume.size + rsp.actualSize = statsActualSize + return rsp + } + + env.message(KVMHostAsyncHttpCallMsg.class) { KVMHostAsyncHttpCallMsg hmsg, CloudBus cloudBus -> + tracker.kvmConvertCount++ + assert hmsg.path == KVM_LUKS_CONVERT_PATH + assert hmsg.hostUuid == host.uuid + tracker.kvmCmd = JSONObjectUtil.toObject(hmsg.command, LinkedHashMap.class) + assert tracker.kvmCmd.psUuid == ps.uuid + assert tracker.kvmCmd.installPath == item.sourceInstallPath + assert tracker.kvmCmd.targetInstallPath == item.targetInstallPath + assert tracker.kvmCmd.virtualSize == msg.volume.size + + KVMHostAsyncHttpCallReply reply = new KVMHostAsyncHttpCallReply() + reply.response = [ + success: !failKvm, + error: failKvm ? "on purpose" : null, + actualSize: kvmActualSize + ] as LinkedHashMap + cloudBus.reply(hmsg, reply) + } + return tracker + } + + private Expando runPostCreateRuntimeFailure(String failurePoint) { + env.cleanSimulatorAndMessageHandlers() + String failureMessage = "${failurePoint} runtime failure" + ConvertVolumeEncryptionOnPrimaryStorageMsg msg = buildConversionMsg( + "cbd:pool1/lpool1/${failurePoint}-runtime-source", + "cbd:pool1/lpool1/${failurePoint}-runtime-target", + SizeUnit.GIGABYTE.toByte(2), false, true) + Expando tracker = new Expando( + targetInstallPath: msg.items[0].targetInstallPath, + deletedInstallPaths: [], + successCount: 0, + failureCount: 0, + error: null, + runtimeException: null + ) + ZbsVolumeEncryptionBackend backend = [ + getPrimaryStorageUuid : { ps.uuid }, + validateConversionPaths: { String source, String target -> }, + createConversionTarget : { String target, long size, boolean encrypted, + ReturnValueCompletion completion -> + completion.success(target) + }, + deleteConversionTarget : { String target, Completion completion -> + tracker.deletedInstallPaths << target + completion.success() + }, + stats : { String target, ReturnValueCompletion completion -> + if (failurePoint == "stats") { + throw new RuntimeException(failureMessage) + } + VolumeStats stats = new VolumeStats() + stats.actualSize = 1L + completion.success(stats) + } + ] as ZbsVolumeEncryptionBackend + + if (failurePoint == "stats") { + env.message(KVMHostAsyncHttpCallMsg.class) { KVMHostAsyncHttpCallMsg hmsg, CloudBus cloudBus -> + KVMHostAsyncHttpCallReply reply = new KVMHostAsyncHttpCallReply() + reply.response = [success: true, actualSize: null] as LinkedHashMap + cloudBus.reply(hmsg, reply) + } + } + + StubVolumeEncryptedSecretHelper secretHelper = new StubVolumeEncryptedSecretHelper( + sealedDek: "sealed-dek", + runtimeFailure: failurePoint == "material" ? new RuntimeException(failureMessage) : null) + Closure convert = { + try { + bean(ZbsVolumeEncryptionExtension.class).convertVolumeEncryption(backend, msg, + new ReturnValueCompletion(null) { + @Override + void success(ConvertVolumeEncryptionOnPrimaryStorageReply reply) { + tracker.successCount++ + } + + @Override + void fail(ErrorCode errorCode) { + tracker.failureCount++ + tracker.error = errorCode + } + }) + } catch (RuntimeException e) { + tracker.runtimeException = e + } + } + + withMaterialFactoryStubs(secretHelper, new StubEncryptedResourceKeyManager(), + new StubVolumeEncryptedResourceKeyBackend()) { + if (failurePoint == "kvm") { + CloudBus throwingBus = [ + makeTargetServiceIdByResourceUuid: { Object... ignored -> + throw new RuntimeException(failureMessage) + } + ] as CloudBus + withKvmCallerBus(throwingBus, convert) + } else { + convert.call() + } + } + return tracker + } + + private Expando installConversionMutationCounters() { + env.cleanSimulatorAndMessageHandlers() + Expando tracker = new Expando(createVolumeCount: 0, kvmConvertCount: 0, deletedInstallPaths: []) + + env.simulator(ZbsStorageController.CREATE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + tracker.createVolumeCount++ + return new ZbsStorageController.CreateVolumeRsp() + } + env.simulator(ZbsStorageController.DELETE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, LinkedHashMap.class) + tracker.deletedInstallPaths << cmd.path + return new ZbsStorageController.DeleteVolumeRsp() + } + env.message(KVMHostAsyncHttpCallMsg.class) { KVMHostAsyncHttpCallMsg hmsg, CloudBus cloudBus -> + tracker.kvmConvertCount++ + KVMHostAsyncHttpCallReply reply = new KVMHostAsyncHttpCallReply() + reply.response = [success: true, actualSize: 1L] as LinkedHashMap + cloudBus.reply(hmsg, reply) + } + return tracker + } + + private T withSealedDekStub(String sealedDek = "sealed-dek", Closure body) { + ZbsVolumeEncryptionMaterialFactory materialFactory = bean(ZbsVolumeEncryptionMaterialFactory.class) + def field = ZbsVolumeEncryptionMaterialFactory.getDeclaredField("volumeEncryptedSecretHelper") + field.accessible = true + VolumeEncryptedSecretHelper original = field.get(materialFactory) as VolumeEncryptedSecretHelper + StubVolumeEncryptedSecretHelper stub = new StubVolumeEncryptedSecretHelper() + stub.sealedDek = sealedDek + field.set(materialFactory, stub) + try { + return body.call(stub.requests) + } finally { + field.set(materialFactory, original) + } + } + + private def prepareVolumeEncryptionWithSpecMethod() { + def method = ZbsVolumeEncryptionMaterialFactory.declaredMethods.find { + it.name == "prepareVolumeEncryption" && + it.parameterTypes.toList() == [String.class, CreateVolumeSpec.class, String.class] + } + assert method != null: "ZbsVolumeEncryptionMaterialFactory must accept CreateVolumeSpec" + method.accessible = true + return method + } + + private def invokePrepareVolumeEncryption(def method, ZbsVolumeEncryptionMaterialFactory factory, + CreateVolumeSpec spec) { + try { + return method.invoke(factory, ps.uuid, spec, "test prepare ZBS encryption material") + } catch (java.lang.reflect.InvocationTargetException e) { + throw e.targetException + } + } + + private T withMaterialFactoryStubs(StubVolumeEncryptedSecretHelper secretHelper, + StubEncryptedResourceKeyManager keyManager, + StubVolumeEncryptedResourceKeyBackend keyBackend, + Closure body) { + ZbsVolumeEncryptionMaterialFactory factory = bean(ZbsVolumeEncryptionMaterialFactory.class) + Map replacements = [ + volumeEncryptedSecretHelper : secretHelper, + encryptedResourceKeyManager : keyManager, + volumeEncryptedResourceKeyBackend: keyBackend, + ] + Map originals = [:] + replacements.each { name, replacement -> + def field = ZbsVolumeEncryptionMaterialFactory.getDeclaredField(name) + field.accessible = true + originals[name] = field.get(factory) + field.set(factory, replacement) + } + try { + return body.call(factory) + } finally { + originals.each { name, original -> + def field = ZbsVolumeEncryptionMaterialFactory.getDeclaredField(name) + field.accessible = true + field.set(factory, original) + } + } + } + + private T withKvmCallerBus(CloudBus replacement, Closure body) { + ZbsVolumeEncryptionKvmCaller caller = bean(ZbsVolumeEncryptionKvmCaller.class) + def field = ZbsVolumeEncryptionKvmCaller.getDeclaredField("bus") + field.accessible = true + CloudBus original = field.get(caller) as CloudBus + field.set(caller, replacement) + try { + return body.call() + } finally { + field.set(caller, original) + } + } + + private static Object readMaterialField(Object material, String name) { + def field = material.class.getDeclaredField(name) + field.accessible = true + return field.get(material) + } + + private static class StubVolumeEncryptedSecretHelper extends VolumeEncryptedSecretHelper { + List> requests = [] + List> sealRequests = [] + String sealedDek + RuntimeException runtimeFailure + + @Override + String materializeAndSealVolumeDekForHost(String hostUuid, String volumeUuid) { + requests << [hostUuid: hostUuid, volumeUuid: volumeUuid] + if (runtimeFailure != null) { + throw runtimeFailure + } + return sealedDek + } + + @Override + String verifyHostKeyAndHpkeSealDek(String hostUuid, String resourceUuid, String dekBase64) { + sealRequests << [hostUuid: hostUuid, resourceUuid: resourceUuid, dekBase64: dekBase64] + return sealedDek + } + } + + private static class StubEncryptedResourceKeyManager extends DummyEncryptedResourceKeyManager { + List requests = [] + EncryptedResourceKeyManager.ResourceKeyResult result + + @Override + EncryptedResourceKeyManager.ResourceKeyResult getExistingKeySync( + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx) { + requests << ctx + return result + } + } + + private static class StubVolumeEncryptedResourceKeyBackend extends DummyVolumeEncryptedResourceKeyBackend { + List volumeRequests = [] + List imageRequests = [] + String keyProviderUuid + + @Override + String findKeyProviderUuidByVolume(String volumeUuid) { + volumeRequests << volumeUuid + return keyProviderUuid + } + + @Override + String findKeyProviderUuidByTemporarySnapshotImage(String imageUuid) { + imageRequests << imageUuid + return keyProviderUuid + } + } + private EncryptVolumeBitsOnPrimaryStorageMsg buildEncryptBitsMsg(String installPath) { EncryptVolumeBitsOnPrimaryStorageMsg msg = new EncryptVolumeBitsOnPrimaryStorageMsg() msg.primaryStorageUuid = ps.uuid