diff --git a/VERSION b/VERSION index b7c3a02aec6..d406ff05502 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=5 MINOR=4 -UPDATE=7 +UPDATE=10 diff --git a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java index bd79900c13c..b42e5a32edd 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java @@ -52,6 +52,10 @@ public class VmGlobalConfig { @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class}) public static GlobalConfig VM_PORT_OFF = new GlobalConfig(CATEGORY, "vmPortOff"); + @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable PMU for VM, disabled by default on aarch64 to avoid Kunpeng-920 kernel panic") + @GlobalConfigValidation(validValues = {"true", "false"}) + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) + public static GlobalConfig VM_PMU = new GlobalConfig(CATEGORY, "vm.pmu"); @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) public static GlobalConfig EMULATE_HYPERV = new GlobalConfig(CATEGORY, "emulateHyperV"); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index ed98aef34e0..9e03bfa61b3 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1004,16 +1004,38 @@ public void done(ErrorCodeList errorCodeList) { logger.debug(String.format("HaStartVmJudger[%s] says the VM[uuid:%s, name:%s] is qualified for HA start, now we are starting it", judger.getClass(), self.getUuid(), self.getName())); + String hostUuid = self.getHostUuid(); + String suspectHostUuid = StringUtils.trimToNull(hostUuid); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); UpdateQuery sql = SQL.New(VmInstanceVO.class) .eq(VmInstanceVO_.uuid, self.getUuid()) .set(VmInstanceVO_.state, VmInstanceState.Stopped) .set(VmInstanceVO_.hostUuid, null); - if (self.getHostUuid() != null) { - sql.set(VmInstanceVO_.lastHostUuid, self.getHostUuid()); + if (hostUuid != null) { + sql.set(VmInstanceVO_.lastHostUuid, hostUuid); } sql.update(); + refreshVO(); + if (suspectHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host is absent", + self.getUuid())); + } else if (peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because peer host is absent", + self.getUuid())); + } else { + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } startVm(msg, new Completion(msg, chain) { @Override @@ -8984,4 +9006,3 @@ public void run(MessageReply reply) { }); } } - diff --git a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java index 20965f61655..bf44adeb836 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java @@ -5,17 +5,26 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.asyncbatch.While; import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.workflow.FlowChainBuilder; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowChain; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; import org.zstack.header.core.workflow.FlowRollback; import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.host.HostConstant; import org.zstack.header.message.MessageReply; import org.zstack.header.vm.*; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.List; import java.util.Map; @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) public class VmStartOnHypervisorFlow implements Flow { @@ -26,34 +35,68 @@ public class VmStartOnHypervisorFlow implements Flow { @Autowired private PluginRegistry pluginRgty; - private final List exts = pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class);; - - private void fireExtensions(VmInstanceSpec spec) { - for (VmBeforeStartOnHypervisorExtensionPoint ext : exts) { - ext.beforeStartVmOnHypervisor(spec); - } - } - @Override public void run(final FlowTrigger chain, final Map data) { final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); + FlowChain fchain = FlowChainBuilder.newSimpleFlowChain(); + fchain.setName(String.format("vm-start-on-hypervisor-vm-%s", spec.getVmInventory().getUuid())); + fchain.then(new NoRollbackFlow() { + @Override + public void run(FlowTrigger trigger, Map d) { + new While<>(pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class)) + .each((ext, comp) -> ext.beforeStartVmOnHypervisor(spec, new Completion(comp) { + @Override + public void success() { + comp.done(); + } - fireExtensions(spec); - - StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); - msg.setVmSpec(spec); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); - bus.send(msg, new CloudBusCallBack(chain) { + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + })).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().size() > 0) { + trigger.fail(errorCodeList.getCauses().get(0)); + } else { + trigger.next(); + } + } + }); + } + }); + fchain.then(new NoRollbackFlow() { @Override - public void run(MessageReply reply) { - if (reply.isSuccess()) { - data.put(VmStartOnHypervisorFlow.class.getName(), true); - chain.next(); - } else { - chain.fail(reply.getError()); - } + public void run(FlowTrigger trigger, Map d) { + StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); + msg.setVmSpec(spec); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); + bus.send(msg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + data.put(VmStartOnHypervisorFlow.class.getName(), true); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); } }); + fchain.done(new FlowDoneHandler(chain) { + @Override + public void handle(Map d) { + chain.next(); + } + }).error(new FlowErrorHandler(chain) { + @Override + public void handle(ErrorCode errCode, Map d) { + chain.fail(errCode); + } + }).start(); } @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index 21c9668175a..6a146649119 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -310,4 +310,11 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); + + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; + public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; + public static PatternedSystemTag HA_PRE_FENCE_PENDING = + new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", + HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, + HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); } diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql new file mode 100644 index 00000000000..ab73deda0f6 --- /dev/null +++ b/conf/db/upgrade/V5.4.10__schema.sql @@ -0,0 +1,3 @@ +-- ZSTAC-86476: Add normalizedModelName column for GPU spec dedup +CALL ADD_COLUMN('GpuDeviceSpecVO', 'normalizedModelName', 'VARCHAR(255)', 1, NULL); +CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalizedModelName'); diff --git a/conf/globalConfig/virutalRouter.xml b/conf/globalConfig/virutalRouter.xml index 36e3b30ef2c..e084e3f925b 100755 --- a/conf/globalConfig/virutalRouter.xml +++ b/conf/globalConfig/virutalRouter.xml @@ -21,6 +21,20 @@ virtualRouter java.lang.Integer + + dnsmasq.dnsForwardMax + The maximum number of concurrent DNS queries forwarded by dnsmasq on a virtual router. + 1000 + virtualRouter + java.lang.Integer + + + dnsmasq.cacheSize + The DNS cache size used by dnsmasq on a virtual router. + 10000 + virtualRouter + java.lang.Integer + ping.interval The interval management nodes ping the virtual router agents running virtual router VMs, in seconds diff --git a/conf/springConfigXml/sugonSdnController.xml b/conf/springConfigXml/sugonSdnController.xml index f0ff6ad0033..c55e6da4903 100644 --- a/conf/springConfigXml/sugonSdnController.xml +++ b/conf/springConfigXml/sugonSdnController.xml @@ -67,11 +67,7 @@ - - - - - + diff --git a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java index 08a776f1db2..9ead578395d 100755 --- a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java +++ b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java @@ -27,27 +27,27 @@ public class ResourceDestinationMakerImpl implements ManagementNodeChangeListene private DatabaseFacade dbf; @Override - public void nodeJoin(ManagementNodeInventory inv) { + public synchronized void nodeJoin(ManagementNodeInventory inv) { nodeHash.add(inv.getUuid()); nodes.put(inv.getUuid(), new NodeInfo(inv)); } @Override - public void nodeLeft(ManagementNodeInventory inv) { + public synchronized void nodeLeft(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iAmDead(ManagementNodeInventory inv) { + public synchronized void iAmDead(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iJoin(ManagementNodeInventory inv) { + public synchronized void iJoin(ManagementNodeInventory inv) { List lst = Q.New(ManagementNodeVO.class).list(); lst.forEach((ManagementNodeVO node) -> { nodeHash.add(node.getUuid()); @@ -56,7 +56,7 @@ public void iJoin(ManagementNodeInventory inv) { } @Override - public String makeDestination(String resourceUuid) { + public synchronized String makeDestination(String resourceUuid) { String nodeUuid = nodeHash.get(resourceUuid); if (nodeUuid == null) { throw new CloudRuntimeException("Cannot find any available management node to send message"); @@ -66,18 +66,18 @@ public String makeDestination(String resourceUuid) { } @Override - public boolean isManagedByUs(String resourceUuid) { + public synchronized boolean isManagedByUs(String resourceUuid) { String nodeUuid = makeDestination(resourceUuid); return nodeUuid.equals(Platform.getManagementServerId()); } @Override - public Collection getManagementNodesInHashRing() { - return nodeHash.getNodes(); + public synchronized Collection getManagementNodesInHashRing() { + return new ArrayList<>(nodeHash.getNodes()); } @Override - public NodeInfo getNodeInfo(String nodeUuid) { + public synchronized NodeInfo getNodeInfo(String nodeUuid) { NodeInfo info = nodes.get(nodeUuid); if (info == null) { ManagementNodeVO vo = dbf.findByUuid(nodeUuid, ManagementNodeVO.class); @@ -93,17 +93,17 @@ public NodeInfo getNodeInfo(String nodeUuid) { } @Override - public Collection getAllNodeInfo() { - return nodes.values(); + public synchronized Collection getAllNodeInfo() { + return new ArrayList<>(nodes.values()); } @Override - public int getManagementNodeCount() { - return nodes.values().size(); + public synchronized int getManagementNodeCount() { + return nodes.size(); } - public boolean isNodeInCircle(String nodeId) { + public synchronized boolean isNodeInCircle(String nodeId) { return nodeHash.hasNode(nodeId); } } diff --git a/header/src/main/java/org/zstack/header/identity/RBACInfo.java b/header/src/main/java/org/zstack/header/identity/RBACInfo.java index 609a916d20c..81223064cd9 100755 --- a/header/src/main/java/org/zstack/header/identity/RBACInfo.java +++ b/header/src/main/java/org/zstack/header/identity/RBACInfo.java @@ -12,6 +12,7 @@ public void permissions() { .name("identity") .adminOnlyAPIs( APICreateAccountMsg.class, + APIQueryAccountMsg.class, APIShareResourceMsg.class, APIRevokeResourceSharingMsg.class, APIUpdateQuotaMsg.class, diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java index f292e694183..2ab0bcb4653 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java @@ -13,7 +13,7 @@ public interface PrimaryStorageNodeSvc { void deactivate(String installPath, String protocol, ActiveVolumeClient client, Completion comp); - void blacklist(String installPath, String protocol, HostInventory h, Completion comp); + void blacklist(String installPath, String protocol, HostInventory h); String getActivePath(BaseVolumeInfo v, HostInventory h, boolean shareable); BaseVolumeInfo getActiveVolumeInfo(String activePath, HostInventory h, boolean shareable); diff --git a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java index a358249cdf9..3ef49037137 100644 --- a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java @@ -12,6 +12,7 @@ public class HaStartVmInstanceMsg extends NeedReplyMessage implements VmInstance private String vmInstanceUuid; private String judgerClassName; private List softAvoidHostUuids; + private String accessiblePeerHostUuid; private String haReason; public String getJudgerClassName() { @@ -30,6 +31,14 @@ public void setSoftAvoidHostUuids(List softAvoidHostUuids) { this.softAvoidHostUuids = softAvoidHostUuids; } + public String getAccessiblePeerHostUuid() { + return accessiblePeerHostUuid; + } + + public void setAccessiblePeerHostUuid(String accessiblePeerHostUuid) { + this.accessiblePeerHostUuid = accessiblePeerHostUuid; + } + @Override public String getVmInstanceUuid() { return vmInstanceUuid; diff --git a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java index c56d99d38dd..1b3a393496f 100755 --- a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java @@ -1,7 +1,14 @@ package org.zstack.header.vm; +import org.zstack.header.core.Completion; + /** */ public interface VmBeforeStartOnHypervisorExtensionPoint { - void beforeStartVmOnHypervisor(VmInstanceSpec spec); + default void beforeStartVmOnHypervisor(VmInstanceSpec spec) {} + + default void beforeStartVmOnHypervisor(VmInstanceSpec spec, Completion completion) { + beforeStartVmOnHypervisor(spec); + completion.success(); + } } diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index 2a3762f98ff..8cb331a91de 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -614,14 +614,13 @@ public void cleanActiveRecord(ImageCacheInventory cache) { } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { + public void blacklist(String installPath, String protocol, HostInventory h) { logger.debug(String.format("blacklisting volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); UssGatewayModule uss = getUssGateway(VolumeProtocol.valueOf(protocol), h.getManagementIp()); VolumeModule exponVol = apiHelper.getVolume(getVolIdFromPath(installPath)); apiHelper.addVolumePathToBlacklist(buildExponVolumeBoundPath(uss, exponVol.getVolumeName())); - comp.success(); } @Override diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index c6eb04a8557..90aca223f1e 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -334,6 +334,7 @@ public void done(ErrorCodeList errorCodeList) { @Override public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { List vols = getManagerExclusiveVolume(spec); + ErrorCode[] deactivateErr = new ErrorCode[1]; for (VolumeInventory vol : vols) { PrimaryStorageNodeSvc nodeSvc = extPsFactory.getNodeSvc(vol.getPrimaryStorageUuid()); @@ -359,20 +360,31 @@ public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAg clientHost.getUuid(), clientHost.getManagementIp(), host.getUuid(), host.getManagementIp())); + deactivateErr[0] = null; nodeSvc.deactivate(vol.getInstallPath(), vol.getProtocol(), client, new Completion(null) { @Override public void success() { + boolean stillActive = nodeSvc.getActiveClients(vol.getInstallPath(), vol.getProtocol()).stream() + .anyMatch(c -> client.getManagerIp().equals(c.getManagerIp())); + if (stillActive) { + deactivateErr[0] = operr("deactivate reported success but volume[uuid:%s, installPath:%s] is still active on host[uuid:%s, ip:%s]", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp()); + return; + } logger.info(String.format("successfully deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]", vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); } @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s], add it to blacklist", - vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); - nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost), new NopeCompletion()); + deactivateErr[0] = errorCode; } }); + if (deactivateErr[0] != null) { + logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]: %s, add it to blacklist", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp(), deactivateErr[0].getDetails())); + nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost)); + } } } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 91e5d885bd8..2eb30e5040a 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -2282,6 +2282,8 @@ public static class StartVmCmd extends vdiCmd implements VmAddOnsCmd { @GrayVersion(value = "5.0.0") private boolean acpi; @GrayVersion(value = "5.0.0") + private boolean pmu = true; + @GrayVersion(value = "5.0.0") private boolean x2apic = true; // cpuid hypervisor feature @GrayVersion(value = "5.0.0") @@ -2789,6 +2791,14 @@ public void setAcpi(boolean acpi) { this.acpi = acpi; } + public boolean isPmu() { + return pmu; + } + + public void setPmu(boolean pmu) { + this.pmu = pmu; + } + public boolean getX2apic() { return x2apic; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index aa0fd787bed..4b1d393de48 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -120,6 +120,10 @@ public class KVMGlobalConfig { @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable install host shutdown hook") public static GlobalConfig INSTALL_HOST_SHUTDOWN_HOOK = new GlobalConfig(CATEGORY, "install.host.shutdown.hook"); + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip entries from departed management nodes") + public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "vm.orphanedSkipTimeout"); + @GlobalConfigValidation(validValues = {"true", "false"}) @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable memory auto balloon") @BindResourceConfig({VmInstanceVO.class, HostVO.class, ClusterVO.class}) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index ab8c29640f3..34e0c434296 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -4498,6 +4498,21 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi } nics = nics.stream().sorted(Comparator.comparing(NicTO::getDeviceId)).collect(Collectors.toList()); cmd.setNics(nics); + if ((platform.equals(ImagePlatform.Windows.toString()) || + platform.equals(ImagePlatform.WindowsVirtio.toString()))) { + if (cmd.getNics() != null) { + for (NicTO nicTo : cmd.getNics()) { + if (nicTo.getType() != null && + nicTo.getType().toLowerCase().contains(VmNicType.VmNicSubType.VHOSTUSER.toString().toLowerCase()) && + !VmNicDriverType.VIRTIO.toString().equalsIgnoreCase(nicTo.getDriverType())) { + logger.warn(String.format("Force updating nic[%s] driver from [%s] to [virtio] for " + + "Windows vhostuser compatibility.", + nicTo.getUuid(), nicTo.getDriverType())); + nicTo.setDriverType(VmNicDriverType.VIRTIO.toString()); + } + } + } + } for (VmInstanceSpec.CdRomSpec cdRomSpec : spec.getCdRomSpecs()) { CdRomTO cdRomTO = new CdRomTO(); @@ -4532,6 +4547,10 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi if (spec.isCreatePaused()) { cmd.setCreatePaused(true); } + if (ImageArchitecture.aarch64.toString().equals(architecture)) { + Boolean pmuEnabled = rcf.getResourceConfigValue(VmGlobalConfig.VM_PMU, spec.getVmInventory().getUuid(), Boolean.class); + cmd.setPmu(Boolean.TRUE.equals(pmuEnabled)); + } cmd.setAcpi(true); GuestOsCharacter.Config config = GuestOsHelper.getInstance().getGuestOsCharacter( @@ -5803,6 +5822,13 @@ public void success(Boolean run) { @Override public void fail(ErrorCode errorCode) { + if (KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(info.isNewAdded(), errorCode)) { + logger.warn(String.format( + "kvm ansible failed to mask libvirt sockets because systemd dbus timed out on existing host[uuid:%s, ip:%s], continue reconnect and verify kvmagent, error: %s", + self.getUuid(), self.getManagementIp(), errorCode)); + trigger.next(); + return; + } trigger.fail(errorCode); } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java index cf6d16e7560..3b9979caeb3 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java @@ -2,6 +2,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.network.l2.*; import org.zstack.header.tag.SystemTagVO; import org.zstack.header.tag.SystemTagVO_; @@ -13,6 +14,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; /** @@ -21,6 +23,37 @@ public class KVMHostUtils { private static final CLogger logger = CLoggerImpl.getLogger(KVMHostUtils.class); + public static boolean shouldContinueReconnectOnAnsibleFailure(boolean isNewAdded, ErrorCode errorCode) { + return !isNewAdded && isLibvirtSocketMaskSystemdTimeout(errorCode); + } + + public static boolean isLibvirtSocketMaskSystemdTimeout(ErrorCode errorCode) { + String errorText = collectErrorText(errorCode).toLowerCase(Locale.ROOT); + return errorText.contains("systemctl mask") + && errorText.contains("libvirtd.socket") + && errorText.contains("org.freedesktop.systemd1") + && errorText.contains("timed out") + && (errorText.contains("failed to get properties") + || errorText.contains("failed to activate service")); + } + + private static String collectErrorText(ErrorCode errorCode) { + StringBuilder sb = new StringBuilder(); + ErrorCode cursor = errorCode; + while (cursor != null) { + appendIfNotNull(sb, cursor.getDetails()); + appendIfNotNull(sb, cursor.getDescription()); + cursor = cursor.getCause(); + } + return sb.toString(); + } + + private static void appendIfNotNull(StringBuilder sb, String text) { + if (text != null) { + sb.append(text).append('\n'); + } + } + /** * Get normalized bridge name for l2 network, which at most has 15 chars. * - if l2 network has L2_BRIDGE_NAME tag, then return it's value directly; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index af1a8718800..0f3ceeae74e 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -67,6 +67,11 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private List> skipVmTracerMessages = new ArrayList<>(); private List skipVmTracerReplies = new ArrayList<>(); private Map vmInShutdownMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); + + private long getOrphanTtlMs() { + return KVMGlobalConfig.ORPHANED_VM_SKIP_TIMEOUT.value(Long.class) * 1000; + } { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { @@ -195,8 +200,11 @@ private void syncVm(final HostInventory host, final Completion completion) { // Get vms to skip before send command to host to confirm the vm will be skipped after sync command finished. // The problem is if one vm-sync skipped operation is started and finished during vm sync command's handling // vm state would still be sync to mn + cleanupExpiredOrphanedSkipVms(); + Set vmsToSkipSetHostSide = new HashSet<>(); vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); // if the vm is not running on host when sync command executing but started as soon as possible // before response handling of vm sync, mgmtSideStates will including the running vm but not result in @@ -227,6 +235,7 @@ public void run(MessageReply reply) { // Get vms to skip after sync result returned. vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); Collection vmUuidsInDeleteVmGC = DeleteVmGC.queryVmInGC(host.getUuid(), ret.getStates().keySet()); @@ -445,7 +454,15 @@ public void nodeJoin(ManagementNodeInventory inv) { @Override public void nodeLeft(ManagementNodeInventory inv) { vmApis.remove(inv.getUuid()); - vmsToSkip.remove(inv.getUuid()); + Set skippedVms = vmsToSkip.remove(inv.getUuid()); + if (skippedVms != null && !skippedVms.isEmpty()) { + long now = System.currentTimeMillis(); + for (String vmUuid : skippedVms) { + orphanedSkipVms.put(vmUuid, now); + logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + + " (will expire in %d minutes)", vmUuid, inv.getUuid(), getOrphanTtlMs() / 60000)); + } + } } @Override @@ -459,6 +476,36 @@ public void iJoin(ManagementNodeInventory inv) { } public boolean isVmDoNotNeedToTrace(String vmUuid) { - return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid)); + if (vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))) { + return true; + } + + Long orphanedAt = orphanedSkipVms.get(vmUuid); + if (orphanedAt != null) { + if (System.currentTimeMillis() - orphanedAt < getOrphanTtlMs()) { + logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); + return true; + } + + orphanedSkipVms.remove(vmUuid, orphanedAt); + logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", + vmUuid, getOrphanTtlMs() / 60000)); + } + + return false; + } + + private void cleanupExpiredOrphanedSkipVms() { + if (orphanedSkipVms.isEmpty()) { + return; + } + + long now = System.currentTimeMillis(); + for (Map.Entry entry : orphanedSkipVms.entrySet()) { + if (now - entry.getValue() >= getOrphanTtlMs()) { + orphanedSkipVms.remove(entry.getKey(), entry.getValue()); + logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); + } + } } } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ea1168da036..72af77c1070 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -39,6 +39,7 @@ import org.zstack.network.service.vip.VipVO_; import org.zstack.tag.PatternedSystemTag; import org.zstack.tag.TagManager; +import org.zstack.core.upgrade.UpgradeGlobalConfig; import org.zstack.utils.*; import org.zstack.utils.function.ForEachFunction; import org.zstack.utils.logging.CLogger; @@ -151,10 +152,22 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APIGetCandidateVmNicsForLoadBalancerServerGroupMsg)msg); } else if (msg instanceof APIChangeLoadBalancerBackendServerMsg) { validate((APIChangeLoadBalancerBackendServerMsg)msg); + } else if (msg instanceof APIDeleteLoadBalancerMsg) { + validate((APIDeleteLoadBalancerMsg) msg); } return msg; } + private void validate(APIDeleteLoadBalancerMsg msg) { + if (UpgradeGlobalConfig.GRAYSCALE_UPGRADE.value(Boolean.class)) { + LoadBalancerVO lb = dbf.findByUuid(msg.getUuid(), LoadBalancerVO.class); + if (lb != null && lb.getType() == LoadBalancerType.SLB) { + throw new ApiMessageInterceptionException(argerr( + "cannot delete the standalone load balancer[uuid:%s] during grayscale upgrade", msg.getUuid())); + } + } + } + private void validate(APIDeleteAccessControlListMsg msg) { /*List refs = Q.New(LoadBalancerListenerACLRefVO.class).select(LoadBalancerListenerACLRefVO_.listenerUuid) .eq(LoadBalancerListenerACLRefVO_.aclUuid, msg.getUuid()).isNull(LoadBalancerListenerACLRefVO_.serverGroupUuid).listValues(); diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java index f4733f6a833..8ff809fd308 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java @@ -86,6 +86,10 @@ public String getSdnControllerSignature() { return "sdn-controller-" + self.getUuid(); } + public String getSdnControllerHostSignature(String hostUuid) { + return "sdn-controller-" + self.getUuid() + "-host-" + hostUuid; + } + protected SdnController getSdnController() { SdnControllerFactory factory = sdnMgr.getSdnControllerFactory(self.getVendorType()); return factory.getSdnController(self); @@ -427,7 +431,7 @@ private void sdnControllerAddHost(APISdnControllerAddHostMsg msg, Completion com thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -508,7 +512,7 @@ private void sdnControllerRemoveHostInQueue(SdnControllerRemoveHostMsg msg, Comp thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -594,7 +598,7 @@ private void doControllerChangeHost(APISdnControllerChangeHostMsg msg, Completio thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 45a68537e38..e63d69927c4 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.MessageSafe; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.header.core.Completion; @@ -22,10 +23,13 @@ import org.zstack.network.l3.L3NetworkSystemTags; import org.zstack.sdnController.SdnController; import org.zstack.sdnController.SdnControllerL2; +import org.zstack.sdnController.SdnControllerPingMsg; +import org.zstack.sdnController.SdnControllerPingReply; import org.zstack.sdnController.header.*; import org.zstack.sugonSdnController.controller.api.*; import org.zstack.sugonSdnController.controller.api.types.*; import org.zstack.sugonSdnController.header.APICreateL2TfNetworkMsg; +import org.zstack.sugonSdnController.network.TfZstackPortSync; import org.zstack.utils.StringDSL; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; @@ -44,6 +48,8 @@ public class SugonSdnController implements TfSdnController, SdnController, SdnCo CloudBus bus; @Autowired DatabaseFacade dbf; + @Autowired + TfZstackPortSync tfZstackPortSync; private SdnControllerVO sdnControllerVO; private TfHttpClient client; @@ -55,8 +61,30 @@ public SugonSdnController(SdnControllerVO vo) { } @Override + @MessageSafe public void handleMessage(SdnControllerMessage msg) { - bus.dealWithUnknownMessage((Message) msg); + if (msg instanceof SdnControllerPingMsg) { + handMessage((SdnControllerPingMsg) msg); + } else { + bus.dealWithUnknownMessage((Message) msg); + } + } + + void handMessage(SdnControllerPingMsg msg) { + SdnControllerPingReply reply = new SdnControllerPingReply(); + try { + Domain domain = (Domain) client.getDomain(); + if (domain == null) { + reply.setError(operr("get default domain on tf controller failed")); + } else { + tfZstackPortSync.triggerSyncIfDue(msg.getSdnControllerUuid()); + reply.setSuccess(true); + } + } catch (Exception e) { + logger.warn("ping tf sdn controller failed", e); + reply.setError(operr("get default domain on tf controller failed")); + } + bus.reply(msg, reply); } @Override diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java index 1054f708696..2c531684d1b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java @@ -32,6 +32,22 @@ public TfPortClient(){ if (sdn == null){ throw new RuntimeException("Can not find a tf sdn controller."); } + init(sdn); + } + + public TfPortClient(String sdnControllerUuid) { + SdnControllerVO sdn = Q.New(SdnControllerVO.class).eq( + SdnControllerVO_.uuid, sdnControllerUuid).find(); + if (sdn == null) { + throw new RuntimeException(String.format("Can not find a tf sdn controller[uuid:%s].", sdnControllerUuid)); + } + if (!Objects.equals(sdn.getVendorType(), SdnControllerConstant.TF_CONTROLLER)) { + throw new RuntimeException(String.format("Sdn controller[uuid:%s] is not a tf sdn controller.", sdnControllerUuid)); + } + init(sdn); + } + + private void init(SdnControllerVO sdn) { client = new TfHttpClient(sdn.getIp()); tenantId = StringDSL.transToTfUuid(sdn.getAccountUuid()); } @@ -509,4 +525,3 @@ public Status updateTfPort(String tfPortUUid, String accountId, String deviceId, } } - diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java index c4767689bd1..e7a9f0c0419 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java @@ -33,6 +33,11 @@ public List getTfPortsDetail() { return tfPortClient.getVirtualMachineInterfaceDetail(); } + public List getTfPortsDetail(String sdnControllerUuid) { + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.getVirtualMachineInterfaceDetail(); + } + public TfPortResponse createTfPort(String tfPortUUid, String l2NetworkUuid, String l3NetworkUuid, String mac, String ip) { //invoke tf rest interface to retrieve real ip and mac and portId TfPortClient tfPortClient = new TfPortClient(); @@ -115,6 +120,12 @@ public TfPortResponse deleteTfPort(String portUUid) { return tfPortClient.deletePort(tfPortUUid); } + public TfPortResponse deleteTfPort(String sdnControllerUuid, String portUUid) { + String tfPortUUid = StringDSL.transToTfUuid(portUUid); + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.deletePort(tfPortUUid); + } + public Status updateTfPort(String portUUid, String bmUuid, KeyValuePairs bindInfo) { String accountId = StringDSL.transToTfUuid(acntMgr.getOwnerAccountUuidOfResource(bmUuid)); String tfBmUUid = StringDSL.transToTfUuid(bmUuid); diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java index 184b2e721ed..710b63cdb2b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java @@ -2,9 +2,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.db.Q; -import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.Task; import org.zstack.core.thread.ThreadFacade; -import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; import org.zstack.header.network.l2.L2NetworkVO; import org.zstack.header.network.l2.L2NetworkVO_; import org.zstack.header.vm.VmNicVO; @@ -19,39 +18,57 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.concurrent.Future; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -public class TfZstackPortSync implements ManagementNodeReadyExtensionPoint { +public class TfZstackPortSync { + private static final long SYNC_INTERVAL_MILLIS = TimeUnit.DAYS.toMillis(1); + private static final int MAX_DELETE_COUNT = 10; @Autowired protected ThreadFacade thdf; - private Future trackerThread = null; @Autowired private TfPortService tfPortService; private final static CLogger logger = Utils.getLogger(TfZstackPortSync.class); private final List excludeTypes = new ArrayList(Arrays.asList("neutron:LOADBALANCER", "VIP", "BMS")); + private final Map lastSyncTime = new ConcurrentHashMap<>(); + private final Set runningSyncs = Collections.newSetFromMap(new ConcurrentHashMap()); - @Override - public void managementNodeReady() { - if (trackerThread != null) { - trackerThread.cancel(true); + public void triggerSyncIfDue(String sdnControllerUuid) { + if (sdnControllerUuid == null) { + logger.warn("Port_Sync_Task: skip sync because sdn controller uuid is null."); + return; } - trackerThread = thdf.submitPeriodicTask(new SyncPort()); - } - private class SyncPort implements PeriodicTask { + long now = System.currentTimeMillis(); + Long lastSync = lastSyncTime.get(sdnControllerUuid); + if (lastSync != null && now - lastSync < SYNC_INTERVAL_MILLIS) { + return; + } + if (!runningSyncs.add(sdnControllerUuid)) { + return; + } - @Override - public TimeUnit getTimeUnit() { - return TimeUnit.DAYS; + lastSyncTime.put(sdnControllerUuid, now); + try { + thdf.submit(new SyncPort(sdnControllerUuid)); + } catch (RuntimeException e) { + lastSyncTime.remove(sdnControllerUuid); + runningSyncs.remove(sdnControllerUuid); + throw e; } + } - @Override - public long getInterval() { - return 1; + private class SyncPort implements Task { + private final String sdnControllerUuid; + + private SyncPort(String sdnControllerUuid) { + this.sdnControllerUuid = sdnControllerUuid; } @Override @@ -64,7 +81,10 @@ private HashSet getPortToDelete() { List zstackL2NetworksUuid = Q.New(L2NetworkVO.class).select(L2NetworkVO_.uuid).listValues(); List tfPortsUuid = new ArrayList<>(); try{ - List tfPorts = tfPortService.getTfPortsDetail(); + List tfPorts = tfPortService.getTfPortsDetail(sdnControllerUuid); + if (tfPorts == null) { + return new HashSet<>(); + } for (VirtualMachineInterface vmi : tfPorts) { // skip port if it's network not in zstack List> tfNetworks = vmi.getVirtualNetwork(); @@ -83,7 +103,7 @@ private HashSet getPortToDelete() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task: Fetch tf VirtualMachineInterface failed: %s.", e)); - return null; + return new HashSet<>(); } HashSet result = new HashSet<>(tfPortsUuid); result.removeAll(zstackPortsUuid); @@ -92,13 +112,13 @@ private HashSet getPortToDelete() { } @Override - public void run() { + public Void call() { logger.info("Port_Sync_Task: begin."); try { HashSet portsToDelete = getPortToDelete(); - int maxDeleteCount = 10; + int maxDeleteCount = MAX_DELETE_COUNT; for (String portUuid: portsToDelete) { - TfPortResponse response = tfPortService.deleteTfPort(portUuid); + TfPortResponse response = tfPortService.deleteTfPort(sdnControllerUuid, portUuid); if (response.getCode() == 200) { logger.info(String.format("Port_Sync_Task: VirtualMachineInterface: %s delete success.", portUuid)); @@ -113,7 +133,10 @@ public void run() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task failed: %s.", e)); + } finally { + runningSyncs.remove(sdnControllerUuid); } + return null; } } } diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java index 3b8d87bb8f0..7b38e6cc723 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java @@ -18,6 +18,12 @@ public class VirtualRouterGlobalConfig { public static GlobalConfig COMMANDS_PARALELLISM_DEGREE = new GlobalConfig(CATEGORY, "command.parallelismDegree"); @GlobalConfigValidation public static GlobalConfig RESTART_DNSMASQ_COUNT = new GlobalConfig(CATEGORY, "dnsmasq.restartAfterNumberOfSIGUSER1"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_DNS_FORWARD_MAX = new GlobalConfig(CATEGORY, "dnsmasq.dnsForwardMax"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_CACHE_SIZE = new GlobalConfig(CATEGORY, "dnsmasq.cacheSize"); @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig PING_INTERVAL = new GlobalConfig(CATEGORY, "ping.interval"); @GlobalConfigValidation diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java index d890e10ed33..1108c650975 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java @@ -180,12 +180,9 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setLogLevel(rcf.getResourceConfigValue(VirtualRouterGlobalConfig.LOG_LEVEL, vrUuid, String.class)); Map parms = new HashMap<>(); - long count = Q.New(VirtualRouterLoadBalancerRefVO.class) - .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, vrUuid) - .notNull(VirtualRouterLoadBalancerRefVO_.loadBalancerUuid) - .count(); - if(count > 0){ - parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(),VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); + if (!VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value().equals( + VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.defaultValue(String.class))){ + parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(), VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); } cmd.setParms(parms); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java index 3a90db3b043..2177283b48b 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java @@ -472,6 +472,11 @@ public void deleteBdcBdev(int bdevId, int bdcId) { } else { retryUtilResourceDeleted(gReq, GetBdcBdevResponse.class); } + + if (!call(gReq, GetBdcBdevResponse.class).resourceIsDeleted()) { + throw new OperationFailureException(operr("bdev[id:%s] still exists after deletion polling on bdc[id:%s], " + + "the old storage client may still hold the volume", bdevId, bdcId)); + } } public VolumeModule rollbackSnapshot(int volId, int snapId) { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java index 267d55f4039..53c8cca1751 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java @@ -202,18 +202,23 @@ public synchronized ActiveVolumeTO activeIscsiVolume(String clientIp, String cli public void deactivate(String installPath, String protocol, HostInventory h, Completion comp) { logger.debug(String.format("deactivating volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); - if (VolumeProtocol.Vhost.toString().equals(protocol)) { - deactivateVhost(installPath, h); - comp.success(); - return; - } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { - // iscsi target is shared by all hosts, we cannot control one volume on one host for now. - deactivateIscsi(installPath, h); - comp.success(); + try { + if (VolumeProtocol.Vhost.toString().equals(protocol)) { + deactivateVhost(installPath, h); + } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { + // iscsi target is shared by all hosts, we cannot control one volume on one host for now. + deactivateIscsi(installPath, h); + } else { + comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + return; + } + } catch (Exception e) { + comp.fail(operr("failed to deactivate volume[path:%s, protocol:%s] on host[uuid:%s, ip:%s]: %s", + installPath, protocol, h.getUuid(), h.getManagementIp(), e.getMessage())); return; } - comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + comp.success(); } private void deactivateIscsi(String installPath, HostInventory h) { @@ -255,7 +260,7 @@ private void deactivateVhost(String installPath, HostInventory h) { return; } - retry(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); + retryOrThrow(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); } @Override @@ -271,9 +276,10 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - // todo - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { + throw new OperationFailureException(operr("xinfini does not support volume path isolation yet, " + + "abort starting VM on host[uuid:%s, ip:%s] to prevent split-brain on volume[path:%s, protocol:%s]", + h.getUuid(), h.getManagementIp(), installPath, protocol)); } @Override @@ -1038,16 +1044,33 @@ private void retry(Runnable r) { private void retry(Runnable r, int retry) { - while (retry-- > 0) { + try { + retryOrThrow(r, retry); + } catch (RuntimeException ignored) { + } + } + + private void retryOrThrow(Runnable r) { + retryOrThrow(r, 3); + } + + private void retryOrThrow(Runnable r, int retry) { + RuntimeException lastError = null; + for (int i = 0; i < retry; i++) { try { r.run(); return; - } catch (Exception e) { + } catch (RuntimeException e) { + lastError = e; logger.warn("runnable failed, try ", e); try { TimeUnit.SECONDS.sleep(3); } catch (InterruptedException ignore) {} } } + + if (lastError != null) { + throw lastError; + } } } diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 4b725d7d952..faddfdbd5da 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -145,8 +145,7 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { } @Override diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java index 747a154919b..88bef7d8302 100755 --- a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java +++ b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java @@ -74,6 +74,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -106,6 +107,15 @@ public class ManagementNodeManagerImpl extends AbstractService implements Manage // A dictionary (nodeId -> ManagementNodeInventory) of joined management Node final private Map joinedManagementNodes = new ConcurrentHashMap<>(); + // Lock to serialize lifecycle events from heartbeat reconciliation and canonical event callbacks, + // preventing race conditions where a nodeJoin event is immediately followed by a stale nodeLeft + // from the heartbeat thread, or vice versa. See ZSTAC-77711. + private final Object lifecycleLock = new Object(); + + // Track nodes found in hash ring but missing from DB. Only call nodeLeft after a node + // is missing for two consecutive heartbeat cycles, to avoid removing nodes that just joined. + private final Set suspectedMissingFromDb = new HashSet<>(); + private static int NODE_STARTING = 0; private static int NODE_RUNNING = 1; private static int NODE_FAILED = -1; @@ -367,12 +377,16 @@ protected void run(Map tokens, Object data) { ManagementNodeLifeCycleData d = (ManagementNodeLifeCycleData) data; - if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeJoin(d.getInventory()); - } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeLeft(d.getInventory()); - } else { - throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + synchronized (lifecycleLock) { + if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { + // Clear from suspected set since the node is confirmed alive + suspectedMissingFromDb.remove(d.getInventory().getUuid()); + nodeLifeCycle.nodeJoin(d.getInventory()); + } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { + nodeLifeCycle.nodeLeft(d.getInventory()); + } else { + throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + } } } }; @@ -853,34 +867,55 @@ private void checkAllNodesHealth() { Set nodeUuidsInDb = nodesInDb.stream().map(ManagementNodeVO::getUuid).collect(Collectors.toSet()); - // When a node is dying, we may not receive the the dead notification because the message bus may be also dead - // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out - destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { - if (!nodeUuidsInDb.contains(nodeUuid)) { - logger.warn(String.format("found that a management node[uuid:%s] had no heartbeat in database but still in our hash ring," + - "notify that it's dead", nodeUuid)); - ManagementNodeInventory inv = new ManagementNodeInventory(); - inv.setUuid(nodeUuid); - inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); - - nodeLifeCycle.nodeLeft(inv); - } - }); - - // check if any node missing in our hash ring - nodesInDb.forEach(n -> { - if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { - return; - } - - new Runnable() { - @Override - @AsyncThread - public void run() { - nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + // Reconcile hash ring with DB under lifecycleLock to prevent race with + // canonical event callbacks (nodeJoin/nodeLeft). See ZSTAC-77711. + synchronized (lifecycleLock) { + // When a node is dying, we may not receive the dead notification because the message bus may be also dead + // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out. + // Use two-round confirmation: first round marks as suspected, second round actually removes. + Set currentSuspected = new HashSet<>(); + destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { + if (!nodeUuidsInDb.contains(nodeUuid)) { + if (suspectedMissingFromDb.contains(nodeUuid)) { + // Second consecutive detection — confirmed missing, remove from hash ring + logger.warn(String.format("management node[uuid:%s] confirmed missing from database for two consecutive" + + " heartbeat cycles, removing from hash ring", nodeUuid)); + ManagementNodeInventory inv = new ManagementNodeInventory(); + inv.setUuid(nodeUuid); + try { + inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); + } catch (Exception e) { + logger.warn(String.format("cannot get node info for node[uuid:%s], use empty hostname", nodeUuid)); + } + + nodeLifeCycle.nodeLeft(inv); + } else { + // First detection — mark as suspected, defer removal to next cycle + logger.warn(String.format("management node[uuid:%s] not found in database but still in hash ring," + + " marking as suspected (will remove on next heartbeat if still missing)", nodeUuid)); + currentSuspected.add(nodeUuid); + } } - }.run(); - }); + }); + // Update suspected set: only keep nodes that are newly suspected this round + suspectedMissingFromDb.clear(); + suspectedMissingFromDb.addAll(currentSuspected); + + // check if any node missing in our hash ring + nodesInDb.forEach(n -> { + if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { + return; + } + + new Runnable() { + @Override + @AsyncThread + public void run() { + nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + } + }.run(); + }); + } } @Override diff --git a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java index 8f98f870348..cd6fae86322 100644 --- a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java +++ b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java @@ -37,6 +37,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List types; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Integer minAvailableDevicesPerHost; + @Param(required = false) public java.util.List systemTags; diff --git a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java index 101684ad18c..df5e0e5a96c 100755 --- a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java +++ b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java @@ -729,7 +729,15 @@ String toJpql(boolean isCount) { throw new IllegalArgumentException(String.format("illegal sortBy[%s], entity[%s] doesn't have this field", msg.getSortBy(), info.entityClass.getName())); } - ret = String.format("%s order by %s.%s %s", ret, entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + String orderBy = String.format("order by %s.%s %s", entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + if (info.primaryKey != null && !msg.getSortBy().equals(info.primaryKey)) { + orderBy += String.format(", %s.%s ASC", entityName, info.primaryKey); + } + ret = String.format("%s %s", ret, orderBy); + } else if (msg.getStart() != null || msg.getLimit() != null) { + if (info.primaryKey != null) { + ret = String.format("%s order by %s.%s ASC", ret, entityName, info.primaryKey); + } } if (msg.getGroupBy() != null) { diff --git a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java index 59aa7ce7e7d..9beeb1dd675 100755 --- a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java +++ b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java @@ -30,6 +30,7 @@ import org.zstack.utils.*; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; +import org.zstack.core.db.EntityMetadata; import org.zstack.zql.ZQL; import org.zstack.zql.ZQLContext; import org.zstack.zql.ZQLQueryReturn; @@ -347,7 +348,7 @@ private Method getReplySetter(AutoQuery at) { } } - private void handle(APIQueryMessage msg) { + private APIQueryReply doQuery(APIQueryMessage msg) { AutoQuery at = autoQueryMap.get(msg.getClass()); if (at == null) { at = msg.getClass().getAnnotation(AutoQuery.class); @@ -372,7 +373,7 @@ private void handle(APIQueryMessage msg) { if (result.inventories != null) { replySetter.invoke(reply, result.inventories); } - bus.reply(msg, reply); + return reply; } catch (OperationFailureException of) { throw of; } catch (Exception e) { @@ -380,6 +381,39 @@ private void handle(APIQueryMessage msg) { } } + private void handle(APIQueryMessage msg) { + thdf.syncSubmit(new SyncTask() { + @Override + public Void call() { + APIQueryReply reply; + try { + reply = doQuery(msg); + } catch (OperationFailureException of) { + reply = new APIQueryReply(); + reply.setError(of.getErrorCode()); + } + + bus.reply(msg, reply); + return null; + } + + @Override + public String getName() { + return getSyncSignature(); + } + + @Override + public String getSyncSignature() { + return "api-query"; + } + + @Override + public int getSyncLevel() { + return getQuerySyncLevel(); + } + }); + } + private String toZQLConditionString(QueryCondition c) { // make every condition value as string, the ZQL // will put them in right type because it knows every field's type @@ -482,7 +516,17 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { } if (msg.getSortBy() != null) { - sb.add(String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection())); + String orderBy = String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection()); + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null && !msg.getSortBy().equals(pkFieldName)) { + orderBy += String.format(", %s asc", pkFieldName); + } + sb.add(orderBy); + } else if (!msg.isCount() && (msg.getLimit() != null || msg.getStart() != null)) { + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null) { + sb.add(String.format("order by %s asc", pkFieldName)); + } } if (msg.getLimit() != null) { @@ -514,6 +558,24 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { return result; } + private String getPrimaryKeyFieldNameSafely(Class inventoryClass) { + try { + ZQLMetadata.InventoryMetadata meta = ZQLMetadata.findInventoryMetadata( + ZQL.queryTargetNameFromInventoryClass(inventoryClass)); + Class voClass = meta.inventoryAnnotation.mappingVOClass(); + Field pkField = EntityMetadata.getPrimaryKeyField(voClass); + String pkName = pkField.getName(); + if (!meta.selfInventoryFieldNames.contains(pkName)) { + return null; + } + return pkName; + } catch (Exception e) { + logger.trace(String.format("cannot resolve PK for %s, skip pagination tiebreaker: %s", + inventoryClass.getSimpleName(), e.getMessage())); + return null; + } + } + private QueryBelongFilter validateFilterNameAndGetExp(String filterName) { if (filterName.split(":").length < 2) { throw new OperationFailureException(argerr("filterName must be formatted as [filterType:condition(s)]")); diff --git a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java index 6627311adb9..10ebab413b4 100644 --- a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java +++ b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java @@ -1,10 +1,10 @@ package org.zstack.zql.ast.visitors.plugin; -import org.apache.commons.lang.StringUtils; import org.zstack.header.zql.ASTNode; import java.util.List; -import java.util.stream.Collectors; + +import static org.zstack.zql.ast.visitors.constants.MySqlKeyword.isDistinct; public class SimpleCountPlugin extends AbstractQueryVisitorPlugin { public SimpleCountPlugin() { @@ -24,13 +24,13 @@ public String selectTarget() { String queryTarget; List fieldNames = super.targetFields(); - if (fieldNames.isEmpty()) { - queryTarget = entityAlias; - } else { + if (getClauseType() == ClauseType.COUNT || isDistinct(node.getTarget().getFunction())) { // TODO: Compatibility changes: hql count do not support multiple fields, even if distinct modified. - queryTarget = String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + queryTarget = fieldNames.isEmpty() ? entityAlias : String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + return String.format("count(%s)", String.format(functions(), queryTarget)); + } else { + queryTarget = entityAlias; + return String.format("count(%s)", queryTarget); } - - return String.format("count(%s)", String.format(functions(), queryTarget)); } } diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy new file mode 100644 index 00000000000..0340abb3109 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy @@ -0,0 +1,223 @@ +package org.zstack.test.integration.kvm.vm + +import org.springframework.http.HttpEntity +import org.zstack.compute.vm.VmGlobalConfig +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.flat.FlatNetworkServiceConstant +import org.zstack.network.service.userdata.UserdataConstant +import org.zstack.sdk.GlobalConfigInventory +import org.zstack.sdk.ImageInventory +import org.zstack.sdk.InstanceOfferingInventory +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +class VmPmuConfigCase extends SubCase { + EnvSpec env + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(2) + cpu = 1 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image_x86" + architecture = "x86_64" + url = "http://zstack.org/download/test.qcow2" + } + + image { + name = "image_arm64" + architecture = "aarch64" + url = "http://zstack.org/download/test-arm.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster_x86" + hypervisorType = "KVM" + architecture = "x86_64" + + kvm { + name = "kvm_x86" + managementIp = "localhost" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + cluster { + name = "cluster_arm64" + hypervisorType = "KVM" + architecture = "aarch64" + + kvm { + name = "kvm_arm64" + managementIp = "127.0.0.3" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + attachBackupStorage("sftp") + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + + service { + provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING + types = [NetworkServiceType.DHCP.toString(), EipConstant.EIP_NETWORK_SERVICE_TYPE, UserdataConstant.USERDATA_TYPE_STRING] + } + + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + } + } + } + + @Override + void test() { + env.create() + testPmuGlobalConfigExists() + testX86VmKeepsPmuEnabled() + testAarch64VmDisablesPmuByDefault() + testAarch64VmPmuResourceConfigOverride() + } + + void testPmuGlobalConfigExists() { + def configs = queryGlobalConfig { + conditions = ["category=${VmGlobalConfig.CATEGORY}", "name=${VmGlobalConfig.VM_PMU.name}"] + } + + assert configs.size() == 1 + assert (configs[0] as GlobalConfigInventory).defaultValue == "false" + } + + void testX86VmKeepsPmuEnabled() { + ImageInventory image = env.inventoryByName("image_x86") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-x86" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert cmd.pmu + } + + void testAarch64VmDisablesPmuByDefault() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-arm-default" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert !cmd.pmu + } + + void testAarch64VmPmuResourceConfigOverride() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + VmInstanceInventory vm = createVmInstance { + name = "pmu-arm-override" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } as VmInstanceInventory + + updateResourceConfig { + category = VmGlobalConfig.CATEGORY + name = VmGlobalConfig.VM_PMU.name + resourceUuid = vm.uuid + value = "true" + } + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + rebootVmInstance { + uuid = vm.uuid + } + } + + assert cmd.pmu + } + + KVMAgentCommands.StartVmCmd captureStartVmCmd(Closure trigger) { + KVMAgentCommands.StartVmCmd startCmd = null + env.afterSimulator(KVMConstant.KVM_START_VM_PATH) { rsp, HttpEntity e -> + startCmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.StartVmCmd.class) + return rsp + } + + trigger.call() + assert startCmd != null + env.cleanAfterSimulatorHandlers() + return startCmd + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy index 8e28757ba5a..e954020fa11 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -1,7 +1,12 @@ package org.zstack.test.integration.network.sdnController +import org.springframework.http.HttpEntity import org.zstack.core.db.DatabaseFacade +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.message.MessageReply import org.zstack.header.network.sdncontroller.SdnControllerVO +import org.zstack.header.network.sdncontroller.SdnControllerConstant +import org.zstack.sdnController.SdnControllerPingMsg import org.zstack.sdk.* import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant import org.zstack.sugonSdnController.controller.api.types.MacAddressesType @@ -16,6 +21,9 @@ import javax.persistence.TypedQuery; import org.springframework.http.ResponseEntity import org.springframework.http.HttpStatus +import javax.servlet.http.HttpServletRequest +import java.util.concurrent.atomic.AtomicInteger + class SugonSdnControllerCase extends SubCase { EnvSpec env @@ -37,6 +45,7 @@ class SugonSdnControllerCase extends SubCase { void test() { env.create { dbf = bean(DatabaseFacade.class) + testTfPortSyncTriggeredByPingSuccess() testTfApi() } } @@ -46,6 +55,53 @@ class SugonSdnControllerCase extends SubCase { env.delete() } + void testTfPortSyncTriggeredByPingSuccess() { + String sql = "select sdn" + + " from SdnControllerVO sdn" + + " where sdn.vendorType = :vendorType"; + TypedQuery q = dbf.getEntityManager().createQuery(sql, SdnControllerVO.class); + q.setParameter("vendorType", SugonSdnControllerConstant.TF_CONTROLLER); + SdnControllerVO sdn = q.getResultList().get(0); + AtomicInteger syncCount = new AtomicInteger(0) + + env.simulator(TfCommands.TF_CREATE_VMI) { HttpServletRequest req, HttpEntity e, EnvSpec spec -> + if (req.method == "GET") { + syncCount.incrementAndGet() + return '{"virtual-machine-interfaces":[]}' + } + + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = TfCommands.TEST_VMI_UUID + rsp.uuid = TfCommands.TEST_VMI_UUID + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.setParent(project) + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + sendTfPing(sdn.uuid) + retryInSecs { + assert syncCount.get() == 1 + } + + sleep(500) + sendTfPing(sdn.uuid) + assert syncCount.get() == 1 + } + + void sendTfPing(String sdnControllerUuid) { + CloudBus bus = bean(CloudBus.class) + SdnControllerPingMsg msg = new SdnControllerPingMsg() + msg.sdnControllerUuid = sdnControllerUuid + bus.makeTargetServiceIdByResourceUuid(msg, SdnControllerConstant.SERVICE_ID, sdnControllerUuid) + MessageReply reply = bus.call(msg) + assert reply.success + } + void testTfApi() { def zone = env.inventoryByName("zone") as ZoneInventory def cluster1 = env.inventoryByName("cluster1") as ClusterInventory diff --git a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy index 72cf1d6ebb8..e37785f7674 100644 --- a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy @@ -36,6 +36,7 @@ class CountGroupByCase extends SubCase { env.create { testCountGroupBy() testQuery() + testFieldQueryTotalWithNullableFirstField() testGroupLimit() testEmpty() } @@ -80,6 +81,13 @@ class CountGroupByCase extends SubCase { assert ret2.total == 102 } + void testFieldQueryTotalWithNullableFirstField() { + def ret = ZQL.fromString("query instanceoffering.description,uuid return with (total) limit 100").getSingleResult() as ZQLQueryReturn + assert ret.inventories.size() == 100 + assert ret.total == 102 + assert ret.inventories.every { it.description == null } + } + void testGroupLimit() { def ret = ZQL.fromString("count instanceoffering group by name,cpuNum limit 2").getSingleResult() as ZQLQueryReturn assert ret.inventoryCounts.size() == 2 diff --git a/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java new file mode 100644 index 00000000000..15216cb50bb --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java @@ -0,0 +1,43 @@ +package org.zstack.test.unittest.identity; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.zstack.header.identity.APIQueryAccountMsg; +import org.zstack.header.identity.RBACInfo; +import org.zstack.header.identity.rbac.RBAC; + +public class TestAPIQueryAccountMsgRBACCase { + + @Before + public void setUp() { + RBAC.permissions.clear(); + } + + @Test + public void testAPIQueryAccountMsgIsAdminOnly() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + boolean isAdminOnly = RBAC.isAdminOnlyAPI(APIQueryAccountMsg.class.getName()); + Assert.assertTrue( + "APIQueryAccountMsg should be admin-only to prevent privilege escalation, " + + "but it is currently accessible to normal IAM users via the wildcard normalAPIs pattern", + isAdminOnly + ); + } + + @Test + public void testAPIQueryAccountMsgNotInNormalAPIs() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + String apiName = APIQueryAccountMsg.class.getName(); + boolean isInNormalOnly = RBAC.permissions.stream() + .anyMatch(p -> p.getNormalAPIs().contains(apiName) && !p.getAdminOnlyAPIs().contains(apiName)); + Assert.assertFalse( + "APIQueryAccountMsg should NOT be accessible as a normal API", + isInNormalOnly + ); + } +} diff --git a/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java new file mode 100644 index 00000000000..2e0e06c7dfc --- /dev/null +++ b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java @@ -0,0 +1,25 @@ +package org.zstack.test.kvm; + +import org.junit.Assert; +import org.junit.Test; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.kvm.KVMHostUtils; + +public class KVMHostUtilsTest { + @Test + public void zstac86349_continueReconnectOnLibvirtSocketMaskSystemdTimeout() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl mask libvirtd.socket libvirtd-ro.socket libvirtd-admin.socket libvirtd-tls.socket libvirtd-tcp.socket failed! stderr: Failed to get properties: Failed to activate service 'org.freedesktop.systemd1': timed out (service_start_timeout=25000ms)"); + + Assert.assertTrue(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(true, error)); + } + + @Test + public void zstac86349_doNotContinueReconnectOnOtherAnsibleFailures() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl restart libvirtd failed! stderr: Job for libvirtd.service failed"); + + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + } +}