diff --git a/VERSION b/VERSION index b7c3a02aec6..529cf0d6103 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=5 MINOR=4 -UPDATE=7 +UPDATE=12 diff --git a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java index 49394e88b78..48ed6a7dc48 100755 --- a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java +++ b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java @@ -251,6 +251,11 @@ public void cancel() { public void trackHost(String hostUuid) { + if (!destMaker.isManagedByUs(hostUuid)) { + logger.debug(String.format("skip tracking host[uuid:%s], not managed by us", hostUuid)); + return; + } + Tracker t = trackers.get(hostUuid); if (t != null) { t.cancel(); diff --git a/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java b/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java index a55de8878cf..1c79a9035f0 100755 --- a/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java +++ b/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java @@ -51,6 +51,7 @@ protected void triggerNow(GCCompletion completion) { public void run(FlowTrigger trigger, Map data) { StopVmOnHypervisorMsg msg = new StopVmOnHypervisorMsg(); msg.setVmInventory(inventory); + msg.setIgnoreNotFoundError(true); bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); bus.send(msg, new CloudBusCallBack(trigger) { @Override @@ -66,6 +67,16 @@ public void run(MessageReply reply) { }).then(new NoRollbackFlow() { @Override public void run(FlowTrigger trigger, Map data) { + String vmHostUuid = Q.New(VmInstanceVO.class).select(VmInstanceVO_.hostUuid) + .eq(VmInstanceVO_.uuid, inventory.getUuid()).findValue(); + if (!hostUuid.equals(vmHostUuid)) { + logger.debug(String.format("skip changing vm[uuid:%s] state to stopped after StopVmGC, " + + "current host[uuid:%s] is not gc host[uuid:%s]", inventory.getUuid(), + vmHostUuid, hostUuid)); + trigger.next(); + return; + } + ChangeVmStateMsg cmsg = new ChangeVmStateMsg(); cmsg.setVmInstanceUuid(inventory.getUuid()); cmsg.setStateEvent(VmInstanceStateEvent.stopped.toString()); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java index bd79900c13c..b42e5a32edd 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java @@ -52,6 +52,10 @@ public class VmGlobalConfig { @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class}) public static GlobalConfig VM_PORT_OFF = new GlobalConfig(CATEGORY, "vmPortOff"); + @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable PMU for VM, disabled by default on aarch64 to avoid Kunpeng-920 kernel panic") + @GlobalConfigValidation(validValues = {"true", "false"}) + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) + public static GlobalConfig VM_PMU = new GlobalConfig(CATEGORY, "vm.pmu"); @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) public static GlobalConfig EMULATE_HYPERV = new GlobalConfig(CATEGORY, "emulateHyperV"); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index ed98aef34e0..18b40fda31d 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -928,6 +928,7 @@ private void getStartingCandidateHosts(final NeedReplyMessage msg, final ReturnV amsg.setL3NetworkUuids(VmNicHelper.getL3Uuids(VmNicInventory.valueOf(self.getVmNics()))); amsg.setDryRun(true); amsg.setListAllHosts(true); + amsg.setAllowNoL3Networks(true); bus.send(amsg, new CloudBusCallBack(completion) { @Override @@ -1004,16 +1005,38 @@ public void done(ErrorCodeList errorCodeList) { logger.debug(String.format("HaStartVmJudger[%s] says the VM[uuid:%s, name:%s] is qualified for HA start, now we are starting it", judger.getClass(), self.getUuid(), self.getName())); + String hostUuid = self.getHostUuid(); + String suspectHostUuid = StringUtils.trimToNull(hostUuid); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); UpdateQuery sql = SQL.New(VmInstanceVO.class) .eq(VmInstanceVO_.uuid, self.getUuid()) .set(VmInstanceVO_.state, VmInstanceState.Stopped) .set(VmInstanceVO_.hostUuid, null); - if (self.getHostUuid() != null) { - sql.set(VmInstanceVO_.lastHostUuid, self.getHostUuid()); + if (hostUuid != null) { + sql.set(VmInstanceVO_.lastHostUuid, hostUuid); } sql.update(); + refreshVO(); + if (suspectHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host is absent", + self.getUuid())); + } else if (peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because peer host is absent", + self.getUuid())); + } else { + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } startVm(msg, new Completion(msg, chain) { @Override @@ -8984,4 +9007,3 @@ public void run(MessageReply reply) { }); } } - diff --git a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java index 20965f61655..bf44adeb836 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java @@ -5,17 +5,26 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.asyncbatch.While; import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.workflow.FlowChainBuilder; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowChain; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; import org.zstack.header.core.workflow.FlowRollback; import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.host.HostConstant; import org.zstack.header.message.MessageReply; import org.zstack.header.vm.*; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.List; import java.util.Map; @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) public class VmStartOnHypervisorFlow implements Flow { @@ -26,34 +35,68 @@ public class VmStartOnHypervisorFlow implements Flow { @Autowired private PluginRegistry pluginRgty; - private final List exts = pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class);; - - private void fireExtensions(VmInstanceSpec spec) { - for (VmBeforeStartOnHypervisorExtensionPoint ext : exts) { - ext.beforeStartVmOnHypervisor(spec); - } - } - @Override public void run(final FlowTrigger chain, final Map data) { final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); + FlowChain fchain = FlowChainBuilder.newSimpleFlowChain(); + fchain.setName(String.format("vm-start-on-hypervisor-vm-%s", spec.getVmInventory().getUuid())); + fchain.then(new NoRollbackFlow() { + @Override + public void run(FlowTrigger trigger, Map d) { + new While<>(pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class)) + .each((ext, comp) -> ext.beforeStartVmOnHypervisor(spec, new Completion(comp) { + @Override + public void success() { + comp.done(); + } - fireExtensions(spec); - - StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); - msg.setVmSpec(spec); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); - bus.send(msg, new CloudBusCallBack(chain) { + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + })).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().size() > 0) { + trigger.fail(errorCodeList.getCauses().get(0)); + } else { + trigger.next(); + } + } + }); + } + }); + fchain.then(new NoRollbackFlow() { @Override - public void run(MessageReply reply) { - if (reply.isSuccess()) { - data.put(VmStartOnHypervisorFlow.class.getName(), true); - chain.next(); - } else { - chain.fail(reply.getError()); - } + public void run(FlowTrigger trigger, Map d) { + StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); + msg.setVmSpec(spec); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); + bus.send(msg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + data.put(VmStartOnHypervisorFlow.class.getName(), true); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); } }); + fchain.done(new FlowDoneHandler(chain) { + @Override + public void handle(Map d) { + chain.next(); + } + }).error(new FlowErrorHandler(chain) { + @Override + public void handle(ErrorCode errCode, Map d) { + chain.fail(errCode); + } + }).start(); } @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index 21c9668175a..6a146649119 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -310,4 +310,11 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); + + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; + public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; + public static PatternedSystemTag HA_PRE_FENCE_PENDING = + new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", + HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, + HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); } diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql new file mode 100644 index 00000000000..1d1915d1987 --- /dev/null +++ b/conf/db/upgrade/V5.4.10__schema.sql @@ -0,0 +1,20 @@ +-- ZSTAC-86476: Add normalizedModelName column for GPU spec dedup +CALL ADD_COLUMN('GpuDeviceSpecVO', 'normalizedModelName', 'VARCHAR(255)', 1, NULL); +CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalizedModelName'); + +UPDATE VolumeSnapshotVO AS sp, PrimaryStorageVO AS ps +SET sp.primaryStorageInstallPath = REPLACE(sp.primaryStorageInstallPath, '/dev/', 'sharedblock://') +WHERE sp.primaryStorageUuid = ps.uuid AND ps.type = 'SharedBlock' AND sp.volumeType = 'Memory' AND sp.primaryStorageInstallPath LIKE '/dev/%'; + +UPDATE `zstack`.`ActiveAlarmTemplateVO` +SET `metricName` = 'CPUUsedUtilization' +WHERE `uuid` = 'c9e6cdca107140bea62b4ca919ff9e88' + AND `metricName` = 'VRouterCPUAverageUsedUtilization'; + +UPDATE `zstack`.`AlarmVO` +SET `metricName` = 'CPUUsedUtilization' +WHERE `uuid` IN ( + SELECT `alarmUuid` FROM `zstack`.`ActiveAlarmVO` + WHERE `templateUuid` = 'c9e6cdca107140bea62b4ca919ff9e88' +) + AND `metricName` = 'VRouterCPUAverageUsedUtilization'; diff --git a/conf/db/upgrade/V5.4.12__schema.sql b/conf/db/upgrade/V5.4.12__schema.sql new file mode 100644 index 00000000000..405b7e018ac --- /dev/null +++ b/conf/db/upgrade/V5.4.12__schema.sql @@ -0,0 +1,7 @@ +ALTER TABLE `zstack`.`DRSVmMigrationActivityVO` MODIFY COLUMN `result` TEXT DEFAULT NULL; + +ALTER TABLE `GuestVmScriptExecutedRecordDetailVO` + MODIFY `stdout` MEDIUMTEXT CHARACTER SET `utf8mb4` COLLATE `utf8mb4_unicode_ci`; + +ALTER TABLE `GuestVmScriptExecutedRecordDetailVO` + MODIFY `stderr` MEDIUMTEXT CHARACTER SET `utf8mb4` COLLATE `utf8mb4_unicode_ci`; diff --git a/conf/globalConfig/virutalRouter.xml b/conf/globalConfig/virutalRouter.xml index 36e3b30ef2c..e084e3f925b 100755 --- a/conf/globalConfig/virutalRouter.xml +++ b/conf/globalConfig/virutalRouter.xml @@ -21,6 +21,20 @@ virtualRouter java.lang.Integer + + dnsmasq.dnsForwardMax + The maximum number of concurrent DNS queries forwarded by dnsmasq on a virtual router. + 1000 + virtualRouter + java.lang.Integer + + + dnsmasq.cacheSize + The DNS cache size used by dnsmasq on a virtual router. + 10000 + virtualRouter + java.lang.Integer + ping.interval The interval management nodes ping the virtual router agents running virtual router VMs, in seconds diff --git a/conf/serviceConfig/externalPrimaryStorage.xml b/conf/serviceConfig/externalPrimaryStorage.xml index d1687fe7685..e076cc55bc1 100644 --- a/conf/serviceConfig/externalPrimaryStorage.xml +++ b/conf/serviceConfig/externalPrimaryStorage.xml @@ -1,6 +1,7 @@ storage.primary + PrimaryStorageApiInterceptor org.zstack.header.storage.addon.primary.APIUpdateExternalPrimaryStorageMsg diff --git a/conf/springConfigXml/sugonSdnController.xml b/conf/springConfigXml/sugonSdnController.xml index f0ff6ad0033..c55e6da4903 100644 --- a/conf/springConfigXml/sugonSdnController.xml +++ b/conf/springConfigXml/sugonSdnController.xml @@ -67,11 +67,7 @@ - - - - - + diff --git a/conf/springConfigXml/volumeSnapshot.xml b/conf/springConfigXml/volumeSnapshot.xml index 09022186cb2..d26ca551061 100755 --- a/conf/springConfigXml/volumeSnapshot.xml +++ b/conf/springConfigXml/volumeSnapshot.xml @@ -26,6 +26,7 @@ + diff --git a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java index ad7be7f7bb9..5ef7b1f7b5a 100644 --- a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java @@ -71,6 +71,8 @@ public ErrorCode stopAnsible() { return useNcatAndNmapToTestConnection(ssh); } catch (SshException e) { return operr(e.getMessage()); + } finally { + ssh.close(); } } diff --git a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java index 08a776f1db2..9ead578395d 100755 --- a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java +++ b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java @@ -27,27 +27,27 @@ public class ResourceDestinationMakerImpl implements ManagementNodeChangeListene private DatabaseFacade dbf; @Override - public void nodeJoin(ManagementNodeInventory inv) { + public synchronized void nodeJoin(ManagementNodeInventory inv) { nodeHash.add(inv.getUuid()); nodes.put(inv.getUuid(), new NodeInfo(inv)); } @Override - public void nodeLeft(ManagementNodeInventory inv) { + public synchronized void nodeLeft(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iAmDead(ManagementNodeInventory inv) { + public synchronized void iAmDead(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iJoin(ManagementNodeInventory inv) { + public synchronized void iJoin(ManagementNodeInventory inv) { List lst = Q.New(ManagementNodeVO.class).list(); lst.forEach((ManagementNodeVO node) -> { nodeHash.add(node.getUuid()); @@ -56,7 +56,7 @@ public void iJoin(ManagementNodeInventory inv) { } @Override - public String makeDestination(String resourceUuid) { + public synchronized String makeDestination(String resourceUuid) { String nodeUuid = nodeHash.get(resourceUuid); if (nodeUuid == null) { throw new CloudRuntimeException("Cannot find any available management node to send message"); @@ -66,18 +66,18 @@ public String makeDestination(String resourceUuid) { } @Override - public boolean isManagedByUs(String resourceUuid) { + public synchronized boolean isManagedByUs(String resourceUuid) { String nodeUuid = makeDestination(resourceUuid); return nodeUuid.equals(Platform.getManagementServerId()); } @Override - public Collection getManagementNodesInHashRing() { - return nodeHash.getNodes(); + public synchronized Collection getManagementNodesInHashRing() { + return new ArrayList<>(nodeHash.getNodes()); } @Override - public NodeInfo getNodeInfo(String nodeUuid) { + public synchronized NodeInfo getNodeInfo(String nodeUuid) { NodeInfo info = nodes.get(nodeUuid); if (info == null) { ManagementNodeVO vo = dbf.findByUuid(nodeUuid, ManagementNodeVO.class); @@ -93,17 +93,17 @@ public NodeInfo getNodeInfo(String nodeUuid) { } @Override - public Collection getAllNodeInfo() { - return nodes.values(); + public synchronized Collection getAllNodeInfo() { + return new ArrayList<>(nodes.values()); } @Override - public int getManagementNodeCount() { - return nodes.values().size(); + public synchronized int getManagementNodeCount() { + return nodes.size(); } - public boolean isNodeInCircle(String nodeId) { + public synchronized boolean isNodeInCircle(String nodeId) { return nodeHash.hasNode(nodeId); } } diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 4f2157deca2..952ef0af20f 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -25,6 +25,7 @@ import org.zstack.core.thread.ThreadFacadeImpl.TimeoutTaskReceipt; import org.zstack.core.timeout.ApiTimeoutManager; import org.zstack.core.validation.ValidationFacade; +import org.zstack.header.Constants; import org.zstack.header.core.Completion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.errorcode.ErrorCode; @@ -80,6 +81,10 @@ private interface AsyncHttpWrapper { void fail(ErrorCode err); void success(HttpEntity responseEntity); + + String getResourceUuid(); + + long getSentAtMillis(); } private interface HttpCallHandlerWrapper { @@ -342,6 +347,9 @@ public void asyncJson(final String url, final String body, Map h HttpEntity req = new HttpEntity(body, requestHeaders); + final long sentAtMillis = System.currentTimeMillis(); + final String resourceUuid = (headers == null) ? null : headers.get(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID); + AsyncHttpWrapper wrapper = new AsyncHttpWrapper() { final AtomicBoolean called = new AtomicBoolean(false); @@ -436,6 +444,16 @@ public void fail(ErrorCode err) { public void success(HttpEntity responseEntity) { completion.success(responseEntity); } + + @Override + public String getResourceUuid() { + return resourceUuid; + } + + @Override + public long getSentAtMillis() { + return sentAtMillis; + } }; try { @@ -471,6 +489,29 @@ public void asyncJsonPost(String url, String body, AsyncRESTCallback callback) { asyncJsonPost(url, body, callback, TimeUnit.MILLISECONDS, timeout); } + @Override + public int failPendingCallsForResourceBefore(String resourceUuid, long cutoffMillis, ErrorCode err) { + if (resourceUuid == null) { + return 0; + } + int drained = 0; + for (AsyncHttpWrapper w : wrappers.values()) { + if (!resourceUuid.equals(w.getResourceUuid())) { + continue; + } + if (w.getSentAtMillis() >= cutoffMillis) { + continue; + } + try { + w.fail(err); + drained++; + } catch (Throwable t) { + logger.warn(String.format("fail injection threw for resource[uuid:%s]: %s", resourceUuid, t.getMessage()), t); + } + } + return drained; + } + @Override public HttpEntity httpServletRequestToHttpEntity(HttpServletRequest req) { try { diff --git a/header/src/main/java/org/zstack/header/identity/RBACInfo.java b/header/src/main/java/org/zstack/header/identity/RBACInfo.java index 609a916d20c..81223064cd9 100755 --- a/header/src/main/java/org/zstack/header/identity/RBACInfo.java +++ b/header/src/main/java/org/zstack/header/identity/RBACInfo.java @@ -12,6 +12,7 @@ public void permissions() { .name("identity") .adminOnlyAPIs( APICreateAccountMsg.class, + APIQueryAccountMsg.class, APIShareResourceMsg.class, APIRevokeResourceSharingMsg.class, APIUpdateQuotaMsg.class, diff --git a/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java b/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java index f9a911436d2..8e1848d9225 100644 --- a/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java +++ b/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java @@ -1,6 +1,7 @@ package org.zstack.header.network.sdncontroller; import org.zstack.header.identity.OwnedByAccount; +import org.zstack.header.log.NoLogging; import org.zstack.header.vo.BaseResource; import org.zstack.header.vo.NoView; import org.zstack.header.vo.ResourceVO; @@ -40,6 +41,7 @@ public class SdnControllerVO extends ResourceVO implements OwnedByAccount, ToInv private String username; @Column + @NoLogging private String password; @OneToMany(fetch=FetchType.EAGER) diff --git a/header/src/main/java/org/zstack/header/rest/RESTFacade.java b/header/src/main/java/org/zstack/header/rest/RESTFacade.java index e9d3120f9d6..a70d5d7e337 100755 --- a/header/src/main/java/org/zstack/header/rest/RESTFacade.java +++ b/header/src/main/java/org/zstack/header/rest/RESTFacade.java @@ -11,6 +11,7 @@ import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.web.client.RestTemplate; import org.zstack.header.core.Completion; +import org.zstack.header.errorcode.ErrorCode; import javax.net.ssl.SSLContext; import javax.servlet.http.HttpServletRequest; @@ -38,6 +39,8 @@ public interface RESTFacade { void asyncJson(final String url, final String body, Map headers, HttpMethod method, final AsyncRESTCallback callback, final TimeUnit unit, final long timeout); + int failPendingCallsForResourceBefore(String resourceUuid, long cutoffMillis, ErrorCode err); + T syncJsonPost(String url, Object body, Class returnClass); T syncJsonPost(String url, Object body, Class returnClass, TimeUnit unit, long timeout); diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java b/header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java new file mode 100644 index 00000000000..f14f5828ce0 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java @@ -0,0 +1,15 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.utils.gson.JSONObjectUtil; + +import java.io.Serializable; + +public interface AddonInfo extends Serializable { + default boolean changed(String infoJson) { + return !this.serialize().equals(infoJson); + } + + default String serialize() { + return JSONObjectUtil.toJsonString(this); + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java index 041438d7420..b2eb78ab2fc 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java @@ -9,7 +9,7 @@ public interface ExternalPrimaryStorageSvcBuilder { PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo); - void discover(String url, String config, ReturnValueCompletion completion); + void discover(String url, String config, ReturnValueCompletion completion); String getIdentity(); } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java new file mode 100644 index 00000000000..aefc52de2ac --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java @@ -0,0 +1,41 @@ +package org.zstack.header.storage.addon.primary; + +public class PingResult { + private AddonInfo addonInfo; + private boolean success = true; + private String error; + + public PingResult(AddonInfo addonInfo, String error) { + this.addonInfo = addonInfo; + this.success = false; + this.error = error; + } + + public PingResult(AddonInfo addonInfo) { + this.addonInfo = addonInfo; + } + + public AddonInfo getAddonInfo() { + return addonInfo; + } + + public void setAddonInfo(AddonInfo addonInfo) { + this.addonInfo = addonInfo; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index 5d6896960ce..be6174d197c 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -9,13 +9,38 @@ import org.zstack.header.volume.VolumeStats; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.List; public interface PrimaryStorageControllerSvc { String getIdentity(); - void connect(String config, String url, ReturnValueCompletion comp); - void ping(Completion completion); + + /** + * connect to storage system, and the AddonInfo may be changed after connection, + * report the latest AddonInfo via the completion, ZStack will persist it on all management nodes. + * @param config + * @param url + * @param comp + */ + void connect(String config, String url, ReturnValueCompletion comp); + + /** + * ping the storage system, and report the latest AddonInfo and result via the completion, + * ZStack will persist AddonInfo on all management nodes. + * @param completion + */ + void ping(ReturnValueCompletion completion); + + /** + * sync the addon info from other management node to this svc + * @param addonInfo + */ + void syncAddonInfo(String addonInfo); + + /** + * sync the config from other management node to this svc + * @param config + */ + void syncConfig(String config); void reportCapacity(ReturnValueCompletion comp); void reportHealthy(ReturnValueCompletion comp); diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java index f292e694183..2ab0bcb4653 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java @@ -13,7 +13,7 @@ public interface PrimaryStorageNodeSvc { void deactivate(String installPath, String protocol, ActiveVolumeClient client, Completion comp); - void blacklist(String installPath, String protocol, HostInventory h, Completion comp); + void blacklist(String installPath, String protocol, HostInventory h); String getActivePath(BaseVolumeInfo v, HostInventory h, boolean shareable); BaseVolumeInfo getActiveVolumeInfo(String activePath, HostInventory h, boolean shareable); diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java new file mode 100644 index 00000000000..9a86d0f29cf --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java @@ -0,0 +1,9 @@ +package org.zstack.header.storage.snapshot; + +import org.zstack.header.volume.VolumeInventory; + +public interface VolumeSnapshotDBSyncExtensionPoint { + VolumeSnapshotInventory syncVolumeSnapshotDBAfterTakeSnapshot(VolumeInventory volume, + VolumeSnapshotInventory snapshot, + String volumeNewInstallPath); +} diff --git a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java index a358249cdf9..3ef49037137 100644 --- a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java @@ -12,6 +12,7 @@ public class HaStartVmInstanceMsg extends NeedReplyMessage implements VmInstance private String vmInstanceUuid; private String judgerClassName; private List softAvoidHostUuids; + private String accessiblePeerHostUuid; private String haReason; public String getJudgerClassName() { @@ -30,6 +31,14 @@ public void setSoftAvoidHostUuids(List softAvoidHostUuids) { this.softAvoidHostUuids = softAvoidHostUuids; } + public String getAccessiblePeerHostUuid() { + return accessiblePeerHostUuid; + } + + public void setAccessiblePeerHostUuid(String accessiblePeerHostUuid) { + this.accessiblePeerHostUuid = accessiblePeerHostUuid; + } + @Override public String getVmInstanceUuid() { return vmInstanceUuid; diff --git a/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java index bffd82dce1f..966928e74d4 100755 --- a/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java +++ b/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java @@ -7,6 +7,7 @@ public class StopVmOnHypervisorMsg extends NeedReplyMessage implements HostMessa private VmInstanceInventory vmInventory; private String type; private boolean debug; + private Boolean ignoreNotFoundError; public VmInstanceInventory getVmInventory() { return vmInventory; @@ -36,4 +37,12 @@ public boolean isDebug() { public void setDebug(boolean debug) { this.debug = debug; } + + public Boolean isIgnoreNotFoundError() { + return ignoreNotFoundError; + } + + public void setIgnoreNotFoundError(Boolean ignoreNotFoundError) { + this.ignoreNotFoundError = ignoreNotFoundError; + } } diff --git a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java index c56d99d38dd..1b3a393496f 100755 --- a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java @@ -1,7 +1,14 @@ package org.zstack.header.vm; +import org.zstack.header.core.Completion; + /** */ public interface VmBeforeStartOnHypervisorExtensionPoint { - void beforeStartVmOnHypervisor(VmInstanceSpec spec); + default void beforeStartVmOnHypervisor(VmInstanceSpec spec) {} + + default void beforeStartVmOnHypervisor(VmInstanceSpec spec, Completion completion) { + beforeStartVmOnHypervisor(spec); + completion.success(); + } } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java index 96da4293fa8..9d2c3102222 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java @@ -830,7 +830,9 @@ public void setBootMode(String bootMode) { public long getRootDiskAllocateSize() { if (rootDiskOffering == null) { - return this.getImageSpec().getInventory().getSize(); + long virtualSize = this.getImageSpec().getInventory().getSize(); + long actualSize = this.getImageSpec().getInventory().getActualSize(); + return Math.max(virtualSize, actualSize); } return rootDiskOffering.getDiskSize(); } diff --git a/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java b/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java index c08f5c37c66..08f5ff08d16 100644 --- a/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java +++ b/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java @@ -1,5 +1,7 @@ package org.zstack.cbd; +import org.zstack.utils.gson.JSONObjectUtil; + import java.util.ArrayList; import java.util.List; @@ -7,7 +9,7 @@ * @author Xingwei Yu * @date 2024/4/1 18:12 */ -public class AddonInfo { +public class AddonInfo implements org.zstack.header.storage.addon.primary.AddonInfo { private ClusterInfo clusterInfo; private List mdsInfos = new ArrayList<>(); private List logicalPoolInfos = new ArrayList<>(); @@ -35,4 +37,13 @@ public List getLogicalPoolInfos() { public void setLogicalPoolInfos(List logicalPoolInfos) { this.logicalPoolInfos = logicalPoolInfos; } + + public void addLogicalPoolInfo(LogicalPoolInfo logicalPoolInfo) { + this.logicalPoolInfos.add(logicalPoolInfo); + } + + @Override + public boolean changed(String infoJson) { + return !JSONObjectUtil.toJsonString(this).equals(infoJson); + } } diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index 6579e941392..a6c091da5c3 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -5438,7 +5438,7 @@ private void deleteSnapshotOnPrimaryStorage(final DeleteSnapshotOnPrimaryStorage httpCall(DELETE_SNAPSHOT_PATH, cmd, DeleteSnapshotRsp.class, new ReturnValueCompletion(msg) { @Override public void success(DeleteSnapshotRsp returnValue) { - osdHelper.releaseAvailableCapWithRatio(msg.getSnapshot().getPrimaryStorageInstallPath(), msg.getSnapshot().getSize()); + osdHelper.releaseAvailableCapacity(msg.getSnapshot().getPrimaryStorageInstallPath(), msg.getSnapshot().getSize()); bus.reply(msg, reply); completion.done(); } diff --git a/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java b/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java index a44fc71f75a..eed1ceb33ed 100755 --- a/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java +++ b/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java @@ -576,7 +576,13 @@ private void handle(final APIAttachEipMsg msg) { return; } - EipInventory eip = EipInventory.valueOf(vo); + final String attachedVmNicUuid = nicInventory.getUuid(); + final String attachedGuestIp = guestIp.getIp(); + vo.setVmNicUuid(attachedVmNicUuid); + vo.setGuestIp(attachedGuestIp); + final EipVO updated = dbf.updateAndRefresh(vo); + + EipInventory eip = EipInventory.valueOf(updated); String l3NetworkUuid = getEipL3Network(nicInventory, eip); NetworkServiceProviderType providerType = nwServiceMgr.getTypeOfNetworkServiceProviderForService( l3NetworkUuid, EipConstant.EIP_TYPE); @@ -585,15 +591,25 @@ private void handle(final APIAttachEipMsg msg) { attachEip(struct, providerType.toString(), new Completion(msg) { @Override public void success() { - vo.setVmNicUuid(nicInventory.getUuid()); - vo.setGuestIp(guestIp.getIp()); - EipVO evo = dbf.updateAndRefresh(vo); - evt.setInventory(EipInventory.valueOf(evo)); + evt.setInventory(EipInventory.valueOf(dbf.reload(updated))); bus.publish(evt); } @Override public void fail(ErrorCode errorCode) { + int affectedRows = SQL.New(EipVO.class) + .eq(EipVO_.uuid, msg.getEipUuid()) + .eq(EipVO_.vmNicUuid, attachedVmNicUuid) + .eq(EipVO_.guestIp, attachedGuestIp) + .set(EipVO_.vmNicUuid, null) + .set(EipVO_.guestIp, null) + .update(); + if (affectedRows == 0) { + logger.warn(String.format( + "failed to roll back EIP[uuid:%s] attach binding[vmNicUuid:%s, guestIp:%s], " + + "the binding may have been changed by another flow", + msg.getEipUuid(), attachedVmNicUuid, attachedGuestIp)); + } evt.setError(errorCode); bus.publish(evt); } diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java b/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java index 07259255377..77c5be249af 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java @@ -2,12 +2,13 @@ import org.zstack.expon.sdk.cluster.TianshuClusterModule; import org.zstack.expon.sdk.pool.FailureDomainModule; +import org.zstack.header.storage.addon.primary.AddonInfo; import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; -public class ExponAddonInfo { +public class ExponAddonInfo implements AddonInfo { private List clusters = new ArrayList<>(); private List pools = new ArrayList<>(); diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index 2a3762f98ff..cb6d5b31b39 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -614,14 +614,13 @@ public void cleanActiveRecord(ImageCacheInventory cache) { } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { + public void blacklist(String installPath, String protocol, HostInventory h) { logger.debug(String.format("blacklisting volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); UssGatewayModule uss = getUssGateway(VolumeProtocol.valueOf(protocol), h.getManagementIp()); VolumeModule exponVol = apiHelper.getVolume(getVolIdFromPath(installPath)); apiHelper.addVolumePathToBlacklist(buildExponVolumeBoundPath(uss, exponVol.getVolumeName())); - comp.success(); } @Override @@ -840,7 +839,7 @@ public String getIdentity() { } @Override - public void connect(String config, String url, ReturnValueCompletion comp) { + public void connect(String config, String url, ReturnValueCompletion comp) { apiHelper.login(); ExponAddonInfo info = new ExponAddonInfo(); @@ -871,12 +870,22 @@ public void connect(String config, String url, ReturnValueCompletion completion) { + completion.success(new PingResult(addonInfo)); + } + + @Override + public void syncAddonInfo(String addonInfo) { + this.addonInfo = StringUtils.isEmpty(addonInfo) ? new ExponAddonInfo() : JSONObjectUtil.toObject(addonInfo, ExponAddonInfo.class); + } + + @Override + public void syncConfig(String config) { + this.config = StringUtils.isEmpty(config) ? new ExponConfig() : JSONObjectUtil.toObject(config, ExponConfig.class); } private void reloadDbInfo() { diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java index 6d24fbe824a..4086058d198 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java @@ -35,7 +35,7 @@ public PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo) { } @Override - public void discover(String url, String config, ReturnValueCompletion completion) { + public void discover(String url, String config, ReturnValueCompletion completion) { ExponStorageController controller = new ExponStorageController(url); MultiNodeSingleFlightImpl.register(controller.apiHelper); controller.connect(config, url, completion); diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index c6eb04a8557..a1e5a0aefe9 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -140,6 +140,26 @@ public void done(ErrorCodeList errorCodeList) { } }); } + }).then(new NoRollbackFlow() { + String __name__ = "ensure-heartbeat-volume"; + + @Override + public void run(FlowTrigger trigger, Map data) { + ensureHeartbeatVolume(context.getInventory(), extPss, new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().isEmpty()) { + trigger.next(); + } else { + logger.warn(String.format("failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection, %s", + context.getInventory().getUuid(), context.getInventory().getName(), errorCodeList.getReadableDetails())); + trigger.fail(operr(new ErrorCodeList().causedBy(errorCodeList.getCauses()), + "failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection", + context.getInventory().getUuid(), context.getInventory().getName())); + } + } + }); + } }).then(new NoRollbackFlow() { String __name__ = "check-host-status"; @@ -166,6 +186,25 @@ public void handle(Map data) { }).start(); } + private void ensureHeartbeatVolume(HostInventory host, List extPss, WhileDoneCompletion completion) { + new While<>(extPss).each((extPs, compl) -> { + logger.debug(String.format("ensuring heartbeat volume for external primary storage[uuid:%s, name:%s] before checking KVM host[uuid:%s, name:%s] storage connection", + extPs.getUuid(), extPs.getName(), host.getUuid(), host.getName())); + extPsFactory.getNodeSvc(extPs.getUuid()).activateHeartbeatVolume(host, new ReturnValueCompletion(compl) { + @Override + public void success(HeartbeatVolumeTO returnValue) { + compl.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + compl.addError(errorCode); + compl.done(); + } + }); + }).run(completion); + } + private void deployClient(final KVMHostConnectedContext context, List extPss, WhileDoneCompletion completion) { new While<>(extPss).each((extPs, compl) -> { logger.debug(String.format("deploying client for external primary storage[uuid:%s, name:%s] on KVM host[uuid:%s, name:%s]", @@ -334,6 +373,7 @@ public void done(ErrorCodeList errorCodeList) { @Override public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { List vols = getManagerExclusiveVolume(spec); + ErrorCode[] deactivateErr = new ErrorCode[1]; for (VolumeInventory vol : vols) { PrimaryStorageNodeSvc nodeSvc = extPsFactory.getNodeSvc(vol.getPrimaryStorageUuid()); @@ -359,20 +399,31 @@ public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAg clientHost.getUuid(), clientHost.getManagementIp(), host.getUuid(), host.getManagementIp())); + deactivateErr[0] = null; nodeSvc.deactivate(vol.getInstallPath(), vol.getProtocol(), client, new Completion(null) { @Override public void success() { + boolean stillActive = nodeSvc.getActiveClients(vol.getInstallPath(), vol.getProtocol()).stream() + .anyMatch(c -> client.getManagerIp().equals(c.getManagerIp())); + if (stillActive) { + deactivateErr[0] = operr("deactivate reported success but volume[uuid:%s, installPath:%s] is still active on host[uuid:%s, ip:%s]", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp()); + return; + } logger.info(String.format("successfully deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]", vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); } @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s], add it to blacklist", - vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); - nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost), new NopeCompletion()); + deactivateErr[0] = errorCode; } }); + if (deactivateErr[0] != null) { + logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]: %s, add it to blacklist", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp(), deactivateErr[0].getDetails())); + nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost)); + } } } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 91e5d885bd8..fef24886ba6 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -338,6 +338,8 @@ public void setVersion(String version) { public static class ConnectResponse extends AgentResponse { private String libvirtVersion; private String qemuVersion; + private boolean firstConnect; + private long agentStartTimeMillis; public String getLibvirtVersion() { return libvirtVersion; @@ -354,6 +356,22 @@ public String getQemuVersion() { public void setQemuVersion(String qemuVersion) { this.qemuVersion = qemuVersion; } + + public boolean isFirstConnect() { + return firstConnect; + } + + public void setFirstConnect(boolean firstConnect) { + this.firstConnect = firstConnect; + } + + public long getAgentStartTimeMillis() { + return agentStartTimeMillis; + } + + public void setAgentStartTimeMillis(long agentStartTimeMillis) { + this.agentStartTimeMillis = agentStartTimeMillis; + } } public static class PingCmd extends AgentCommand { @@ -2282,6 +2300,8 @@ public static class StartVmCmd extends vdiCmd implements VmAddOnsCmd { @GrayVersion(value = "5.0.0") private boolean acpi; @GrayVersion(value = "5.0.0") + private boolean pmu = true; + @GrayVersion(value = "5.0.0") private boolean x2apic = true; // cpuid hypervisor feature @GrayVersion(value = "5.0.0") @@ -2789,6 +2809,14 @@ public void setAcpi(boolean acpi) { this.acpi = acpi; } + public boolean isPmu() { + return pmu; + } + + public void setPmu(boolean pmu) { + this.pmu = pmu; + } + public boolean getX2apic() { return x2apic; } @@ -3188,6 +3216,8 @@ public static class StopVmCmd extends AgentCommand { private String type; @GrayVersion(value = "5.0.0") private long timeout; + @GrayVersion(value = "5.4.12") + private Boolean ignoreNotFoundError; private List vmNics; public String getUuid() { @@ -3214,6 +3244,14 @@ public void setType(String type) { this.type = type; } + public Boolean isIgnoreNotFoundError() { + return ignoreNotFoundError; + } + + public void setIgnoreNotFoundError(Boolean ignoreNotFoundError) { + this.ignoreNotFoundError = ignoreNotFoundError; + } + public List getVmNics() { return vmNics; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index aa0fd787bed..4b1d393de48 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -120,6 +120,10 @@ public class KVMGlobalConfig { @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable install host shutdown hook") public static GlobalConfig INSTALL_HOST_SHUTDOWN_HOOK = new GlobalConfig(CATEGORY, "install.host.shutdown.hook"); + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip entries from departed management nodes") + public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "vm.orphanedSkipTimeout"); + @GlobalConfigValidation(validValues = {"true", "false"}) @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable memory auto balloon") @BindResourceConfig({VmInstanceVO.class, HostVO.class, ClusterVO.class}) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index ab8c29640f3..786586ab0ee 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -71,6 +71,7 @@ import org.zstack.header.rest.JsonAsyncRESTCallback; import org.zstack.header.rest.RESTFacade; import org.zstack.header.storage.primary.*; +import org.zstack.header.storage.snapshot.*; import org.zstack.header.tag.SystemTagInventory; import org.zstack.header.vm.*; import org.zstack.header.vm.devices.DeviceAddress; @@ -3044,6 +3045,7 @@ public void success(TakeSnapshotResponse ret) { " data corruption may happen", ret.getNewVolumeInstallPath())); } + syncSnapshotMetadataAfterHypervisorSuccess(msg, ret); extEmitter.afterTakeSnapshot((KVMHostInventory) getSelfInventory(), msg, cmd, ret); reply.setNewVolumeInstallPath(ret.getNewVolumeInstallPath()); reply.setSnapshotInstallPath(ret.getSnapshotInstallPath()); @@ -3080,6 +3082,27 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private void syncSnapshotMetadataAfterHypervisorSuccess(TakeSnapshotOnHypervisorMsg msg, TakeSnapshotResponse ret) { + VolumeInventory volume = msg.getVolume(); + if (volume == null || msg.getSnapshotName() == null) { + return; + } + + VolumeSnapshotInventory snapshot = new VolumeSnapshotInventory(); + snapshot.setUuid(msg.getSnapshotName()); + snapshot.setVolumeUuid(volume.getUuid()); + snapshot.setPrimaryStorageUuid(volume.getPrimaryStorageUuid()); + snapshot.setPrimaryStorageInstallPath(ret.getSnapshotInstallPath()); + snapshot.setType(VolumeSnapshotConstant.HYPERVISOR_SNAPSHOT_TYPE.toString()); + snapshot.setStatus(VolumeSnapshotStatus.Ready.toString()); + snapshot.setSize(ret.getSize()); + snapshot.setFormat(volume.getFormat()); + + for (VolumeSnapshotDBSyncExtensionPoint ext : pluginRgty.getExtensionList(VolumeSnapshotDBSyncExtensionPoint.class)) { + ext.syncVolumeSnapshotDBAfterTakeSnapshot(volume, snapshot, ret.getNewVolumeInstallPath()); + } + } + private void migrateVm(final MigrateStruct s, final Completion completion) { final TaskProgressRange parentStage = getTaskStage(); final TaskProgressRange MIGRATE_VM_STAGE = new TaskProgressRange(0, 90); @@ -3855,6 +3878,7 @@ protected void stopVm(final StopVmOnHypervisorMsg msg, final NoErrorCompletion c cmd.setUuid(vminv.getUuid()); cmd.setType(msg.getType()); cmd.setTimeout(120); + cmd.setIgnoreNotFoundError(msg.isIgnoreNotFoundError()); cmd.setVmNics(vminv.getVmNics()); try { @@ -4498,6 +4522,21 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi } nics = nics.stream().sorted(Comparator.comparing(NicTO::getDeviceId)).collect(Collectors.toList()); cmd.setNics(nics); + if ((platform.equals(ImagePlatform.Windows.toString()) || + platform.equals(ImagePlatform.WindowsVirtio.toString()))) { + if (cmd.getNics() != null) { + for (NicTO nicTo : cmd.getNics()) { + if (nicTo.getType() != null && + nicTo.getType().toLowerCase().contains(VmNicType.VmNicSubType.VHOSTUSER.toString().toLowerCase()) && + !VmNicDriverType.VIRTIO.toString().equalsIgnoreCase(nicTo.getDriverType())) { + logger.warn(String.format("Force updating nic[%s] driver from [%s] to [virtio] for " + + "Windows vhostuser compatibility.", + nicTo.getUuid(), nicTo.getDriverType())); + nicTo.setDriverType(VmNicDriverType.VIRTIO.toString()); + } + } + } + } for (VmInstanceSpec.CdRomSpec cdRomSpec : spec.getCdRomSpecs()) { CdRomTO cdRomTO = new CdRomTO(); @@ -4532,6 +4571,10 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi if (spec.isCreatePaused()) { cmd.setCreatePaused(true); } + if (ImageArchitecture.aarch64.toString().equals(architecture)) { + Boolean pmuEnabled = rcf.getResourceConfigValue(VmGlobalConfig.VM_PMU, spec.getVmInventory().getUuid(), Boolean.class); + cmd.setPmu(Boolean.TRUE.equals(pmuEnabled)); + } cmd.setAcpi(true); GuestOsCharacter.Config config = GuestOsHelper.getInstance().getGuestOsCharacter( @@ -5147,6 +5190,16 @@ private ErrorCode connectToAgent() { errCode = operr("unable to connect to kvm host[uuid:%s, ip:%s, url:%s], because %s", self.getUuid(), self.getManagementIp(), connectPath, rsp.getError()); } else { + if (rsp.isFirstConnect()) { + long cutoffMillis = rsp.getAgentStartTimeMillis() - TimeUnit.SECONDS.toMillis(60); + int drained = restf.failPendingCallsForResourceBefore(self.getUuid(), cutoffMillis, + operr("kvmagent on host[uuid:%s] restarted at %s, pending HTTP call sent before %s aborted", + self.getUuid(), rsp.getAgentStartTimeMillis(), cutoffMillis)); + if (drained > 0) { + logger.info(String.format("kvmagent on host[uuid:%s] restarted at %d, drained %d pending HTTP call(s)", + self.getUuid(), rsp.getAgentStartTimeMillis(), drained)); + } + } VersionComparator libvirtVersion = new VersionComparator(rsp.getLibvirtVersion()); VersionComparator qemuVersion = new VersionComparator(rsp.getQemuVersion()); boolean liveSnapshot = libvirtVersion.compare(KVMConstant.MIN_LIBVIRT_LIVESNAPSHOT_VERSION) >= 0 @@ -5471,7 +5524,7 @@ public void run(FlowTrigger trigger, Map data) { sshShell.setPassword(getSelf().getPassword()); sshShell.setPort(getSelf().getPort()); sshShell.setWithSudo(false); - final String cmd = String.format("curl --connect-timeout 10 %s|| wget --spider -q --connect-timeout=10 %s|| test $? -eq 8", restf.getCallbackUrl(), restf.getCallbackUrl()); + final String cmd = String.format("curl --connect-timeout 10 --max-time 15 %s|| wget --spider -q --connect-timeout=10 --read-timeout=10 --tries=1 %s|| test $? -eq 8", restf.getCallbackUrl(), restf.getCallbackUrl()); SshResult ret = sshShell.runCommand(cmd); if (ret.getStderr() != null && ret.getStderr().contains("No route to host")) { // c.f. https://access.redhat.com/solutions/1120533 @@ -5533,10 +5586,10 @@ public boolean skip(Map data) { @Override public void run(FlowTrigger trigger, Map data) { + Ssh ssh = new Ssh().setUsername(getSelf().getUsername()) + .setPassword(getSelf().getPassword()).setPort(getSelf().getPort()) + .setHostname(getSelf().getManagementIp()); try { - Ssh ssh = new Ssh().setUsername(getSelf().getUsername()) - .setPassword(getSelf().getPassword()).setPort(getSelf().getPort()) - .setHostname(getSelf().getManagementIp()); ssh.command(String.format("grep -i ^uuid %s | sed 's/uuid://g'", hostTakeOverFlagPath)); SshResult hostRet = ssh.run(); if (hostRet.isSshFailure() || hostRet.getReturnCode() != 0) { @@ -5585,6 +5638,8 @@ public void run(FlowTrigger trigger, Map data) { logger.warn(e.getMessage(), e); trigger.next(); return; + } finally { + ssh.close(); } } }); @@ -5803,6 +5858,13 @@ public void success(Boolean run) { @Override public void fail(ErrorCode errorCode) { + if (KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(info.isNewAdded(), errorCode)) { + logger.warn(String.format( + "kvm ansible failed to mask libvirt sockets because systemd dbus timed out on existing host[uuid:%s, ip:%s], continue reconnect and verify kvmagent, error: %s", + self.getUuid(), self.getManagementIp(), errorCode)); + trigger.next(); + return; + } trigger.fail(errorCode); } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java index cf6d16e7560..3b9979caeb3 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java @@ -2,6 +2,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.network.l2.*; import org.zstack.header.tag.SystemTagVO; import org.zstack.header.tag.SystemTagVO_; @@ -13,6 +14,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; /** @@ -21,6 +23,37 @@ public class KVMHostUtils { private static final CLogger logger = CLoggerImpl.getLogger(KVMHostUtils.class); + public static boolean shouldContinueReconnectOnAnsibleFailure(boolean isNewAdded, ErrorCode errorCode) { + return !isNewAdded && isLibvirtSocketMaskSystemdTimeout(errorCode); + } + + public static boolean isLibvirtSocketMaskSystemdTimeout(ErrorCode errorCode) { + String errorText = collectErrorText(errorCode).toLowerCase(Locale.ROOT); + return errorText.contains("systemctl mask") + && errorText.contains("libvirtd.socket") + && errorText.contains("org.freedesktop.systemd1") + && errorText.contains("timed out") + && (errorText.contains("failed to get properties") + || errorText.contains("failed to activate service")); + } + + private static String collectErrorText(ErrorCode errorCode) { + StringBuilder sb = new StringBuilder(); + ErrorCode cursor = errorCode; + while (cursor != null) { + appendIfNotNull(sb, cursor.getDetails()); + appendIfNotNull(sb, cursor.getDescription()); + cursor = cursor.getCause(); + } + return sb.toString(); + } + + private static void appendIfNotNull(StringBuilder sb, String text) { + if (text != null) { + sb.append(text).append('\n'); + } + } + /** * Get normalized bridge name for l2 network, which at most has 15 chars. * - if l2 network has L2_BRIDGE_NAME tag, then return it's value directly; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index af1a8718800..0f3ceeae74e 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -67,6 +67,11 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private List> skipVmTracerMessages = new ArrayList<>(); private List skipVmTracerReplies = new ArrayList<>(); private Map vmInShutdownMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); + + private long getOrphanTtlMs() { + return KVMGlobalConfig.ORPHANED_VM_SKIP_TIMEOUT.value(Long.class) * 1000; + } { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { @@ -195,8 +200,11 @@ private void syncVm(final HostInventory host, final Completion completion) { // Get vms to skip before send command to host to confirm the vm will be skipped after sync command finished. // The problem is if one vm-sync skipped operation is started and finished during vm sync command's handling // vm state would still be sync to mn + cleanupExpiredOrphanedSkipVms(); + Set vmsToSkipSetHostSide = new HashSet<>(); vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); // if the vm is not running on host when sync command executing but started as soon as possible // before response handling of vm sync, mgmtSideStates will including the running vm but not result in @@ -227,6 +235,7 @@ public void run(MessageReply reply) { // Get vms to skip after sync result returned. vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); Collection vmUuidsInDeleteVmGC = DeleteVmGC.queryVmInGC(host.getUuid(), ret.getStates().keySet()); @@ -445,7 +454,15 @@ public void nodeJoin(ManagementNodeInventory inv) { @Override public void nodeLeft(ManagementNodeInventory inv) { vmApis.remove(inv.getUuid()); - vmsToSkip.remove(inv.getUuid()); + Set skippedVms = vmsToSkip.remove(inv.getUuid()); + if (skippedVms != null && !skippedVms.isEmpty()) { + long now = System.currentTimeMillis(); + for (String vmUuid : skippedVms) { + orphanedSkipVms.put(vmUuid, now); + logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + + " (will expire in %d minutes)", vmUuid, inv.getUuid(), getOrphanTtlMs() / 60000)); + } + } } @Override @@ -459,6 +476,36 @@ public void iJoin(ManagementNodeInventory inv) { } public boolean isVmDoNotNeedToTrace(String vmUuid) { - return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid)); + if (vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))) { + return true; + } + + Long orphanedAt = orphanedSkipVms.get(vmUuid); + if (orphanedAt != null) { + if (System.currentTimeMillis() - orphanedAt < getOrphanTtlMs()) { + logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); + return true; + } + + orphanedSkipVms.remove(vmUuid, orphanedAt); + logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", + vmUuid, getOrphanTtlMs() / 60000)); + } + + return false; + } + + private void cleanupExpiredOrphanedSkipVms() { + if (orphanedSkipVms.isEmpty()) { + return; + } + + long now = System.currentTimeMillis(); + for (Map.Entry entry : orphanedSkipVms.entrySet()) { + if (now - entry.getValue() >= getOrphanTtlMs()) { + orphanedSkipVms.remove(entry.getKey(), entry.getValue()); + logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); + } + } } } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ea1168da036..72af77c1070 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -39,6 +39,7 @@ import org.zstack.network.service.vip.VipVO_; import org.zstack.tag.PatternedSystemTag; import org.zstack.tag.TagManager; +import org.zstack.core.upgrade.UpgradeGlobalConfig; import org.zstack.utils.*; import org.zstack.utils.function.ForEachFunction; import org.zstack.utils.logging.CLogger; @@ -151,10 +152,22 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APIGetCandidateVmNicsForLoadBalancerServerGroupMsg)msg); } else if (msg instanceof APIChangeLoadBalancerBackendServerMsg) { validate((APIChangeLoadBalancerBackendServerMsg)msg); + } else if (msg instanceof APIDeleteLoadBalancerMsg) { + validate((APIDeleteLoadBalancerMsg) msg); } return msg; } + private void validate(APIDeleteLoadBalancerMsg msg) { + if (UpgradeGlobalConfig.GRAYSCALE_UPGRADE.value(Boolean.class)) { + LoadBalancerVO lb = dbf.findByUuid(msg.getUuid(), LoadBalancerVO.class); + if (lb != null && lb.getType() == LoadBalancerType.SLB) { + throw new ApiMessageInterceptionException(argerr( + "cannot delete the standalone load balancer[uuid:%s] during grayscale upgrade", msg.getUuid())); + } + } + } + private void validate(APIDeleteAccessControlListMsg msg) { /*List refs = Q.New(LoadBalancerListenerACLRefVO.class).select(LoadBalancerListenerACLRefVO_.listenerUuid) .eq(LoadBalancerListenerACLRefVO_.aclUuid, msg.getUuid()).isNull(LoadBalancerListenerACLRefVO_.serverGroupUuid).listValues(); diff --git a/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java b/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java index 96e51bc6d86..d4370ef6799 100755 --- a/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java +++ b/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java @@ -54,6 +54,7 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; +import static org.zstack.core.Platform.argerr; import static org.zstack.core.Platform.err; import static org.zstack.core.Platform.operr; import static org.zstack.utils.CollectionDSL.list; @@ -679,11 +680,59 @@ public void fail(ErrorCode errorCode) { private void handle(APICreatePortForwardingRuleMsg msg) { + thdf.chainSubmit(new ChainTask(msg) { + @Override + public String getSyncSignature() { + return String.format("portforwardingrule-vip-%s", msg.getVipUuid()); + } + + @Override + public void run(SyncTaskChain chain) { + doCreatePortForwardingRule(msg, chain); + } + + @Override + public String getName() { + return String.format("api-create-portforwardingrule-vip-%s", msg.getVipUuid()); + } + }); + } + + private ErrorCode validateVipPortConflict(String vipUuid, String protocolType, int vipPortStart, int vipPortEnd) { + List vos = Q.New(PortForwardingRuleVO.class) + .eq(PortForwardingRuleVO_.vipUuid, vipUuid) + .eq(PortForwardingRuleVO_.protocolType, PortForwardingProtocolType.valueOf(protocolType)) + .list(); + PortForwardingRuleVO vo = vos.stream() + .filter(it -> { + int existingStart = it.getVipPortStart(); + int existingEnd = it.getVipPortEnd(); + return existingStart <= vipPortEnd && vipPortStart <= existingEnd; + }) + .findFirst() + .orElse(null); + if (vo == null) { + return null; + } + + return argerr("vip port range[vipStartPort:%s, vipEndPort:%s] overlaps with rule[uuid:%s, vipStartPort:%s, vipEndPort:%s]", + vipPortStart, vipPortEnd, vo.getUuid(), vo.getVipPortStart(), vo.getVipPortEnd()); + } + + private void doCreatePortForwardingRule(APICreatePortForwardingRuleMsg msg, SyncTaskChain syncChain) { final APICreatePortForwardingRuleEvent evt = new APICreatePortForwardingRuleEvent(msg.getId()); int vipPortEnd = msg.getVipPortEnd() == null ? msg.getVipPortStart() : msg.getVipPortEnd(); int privatePortEnd = msg.getPrivatePortEnd() == null ? msg.getPrivatePortStart() : msg.getPrivatePortEnd(); + ErrorCode errCode = validateVipPortConflict(msg.getVipUuid(), msg.getProtocolType(), msg.getVipPortStart(), vipPortEnd); + if (errCode != null) { + evt.setError(errCode); + bus.publish(evt); + syncChain.next(); + return; + } + VipVO vip = dbf.findByUuid(msg.getVipUuid(), VipVO.class); final PortForwardingRuleVO vo = new PortForwardingRuleVO(); if (msg.getResourceUuid() != null) { @@ -712,8 +761,8 @@ protected void scripts() { } }.execute(); - FlowChain chain = FlowChainBuilder.newShareFlowChain(); - chain.setName("create-portforwading"); + FlowChain flowChain = FlowChainBuilder.newShareFlowChain(); + flowChain.setName("create-portforwading"); VipInventory vipInventory = VipInventory.valueOf(vip); if (msg.getVmNicUuid() == null) { ModifyVipAttributesStruct struct = new ModifyVipAttributesStruct(); @@ -726,6 +775,7 @@ protected void scripts() { public void success() { evt.setInventory(PortForwardingRuleInventory.valueOf(vo)); bus.publish(evt); + syncChain.next(); } @Override @@ -733,6 +783,7 @@ public void fail(ErrorCode errorCode) { dbf.remove(vo); evt.setError(errorCode); bus.publish(evt); + syncChain.next(); } }); @@ -756,6 +807,7 @@ public void fail(ErrorCode errorCode) { public void success() { evt.setInventory(PortForwardingRuleInventory.valueOf(vo)); bus.publish(evt); + syncChain.next(); } @Override @@ -763,13 +815,14 @@ public void fail(ErrorCode errorCode) { dbf.remove(vo); evt.setError(errorCode); bus.publish(evt); + syncChain.next(); } }); return; } - chain.then(new ShareFlow() { + flowChain.then(new ShareFlow() { @Override public void setup() { vo.setVmNicUuid(vmNic.getUuid()); @@ -817,7 +870,7 @@ public void run(AttachPortForwardingRuleExtensionPoint extp) { public void run(FlowTrigger trigger, Map data) { NetworkServiceProviderType providerType = (NetworkServiceProviderType)data.get("providerType"); final PortForwardingStruct struct = makePortForwardingStruct(ruleInv); - attachPortForwardingRule(struct, providerType.toString(), new Completion(msg) { + doAttachPortForwardingRule(struct, providerType.toString(), new Completion(msg) { @Override public void success() { CollectionUtils.safeForEach(attachRuleExts, new ForEachFunction() { @@ -852,11 +905,12 @@ public void fail(ErrorCode errorCode) { }); - chain.done(new FlowDoneHandler(msg) { + flowChain.done(new FlowDoneHandler(msg) { @Override public void handle(Map data) { evt.setInventory(PortForwardingRuleInventory.valueOf(dbf.reload(vo))); bus.publish(evt); + syncChain.next(); } }).error(new FlowErrorHandler(msg) { @Override @@ -864,6 +918,7 @@ public void handle(ErrorCode errCode, Map data) { dbf.remove(vo); evt.setError(errCode); bus.publish(evt); + syncChain.next(); } }).start(); } diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java index 5de07d899f7..57072cbaa57 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java @@ -13,10 +13,10 @@ public interface SdnController { 对sdn控制器的控制: event 有关sdn控制器的后置处理: post-event */ - void preInitSdnController(APIAddSdnControllerMsg msg, Completion completion); - void createSdnControllerDb(APIAddSdnControllerMsg msg, SdnControllerVO vo, Completion completion); + void preInitSdnController(AddSdnControllerMsg msg, Completion completion); + void createSdnControllerDb(AddSdnControllerMsg msg, SdnControllerVO vo, Completion completion); void deleteSdnControllerDb(SdnControllerVO vo); - void initSdnController(APIAddSdnControllerMsg msg, Completion completion); + void initSdnController(AddSdnControllerMsg msg, Completion completion); void postInitSdnController(SdnControllerVO vo, Completion completion); void deleteSdnController(SdnControllerDeletionMsg msg, SdnControllerInventory sdn, Completion completion); diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java index 741a99cdb70..8695e429a39 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java @@ -17,6 +17,7 @@ import org.zstack.header.network.sdncontroller.*; import org.zstack.header.vm.APIAttachL3NetworkToVmMsg; import org.zstack.header.vm.APIChangeVmNicNetworkMsg; +import org.zstack.header.vm.VmInstanceState; import org.zstack.header.vm.VmInstanceVO; import org.zstack.header.vm.VmNicVO; import org.zstack.network.l2.vxlan.vxlanNetwork.VxlanNetworkVO; @@ -125,7 +126,11 @@ private void validate(APIAttachL3NetworkToVmMsg msg) { VmInstanceVO vmVo = dbf.findByUuid(msg.getVmInstanceUuid(), VmInstanceVO.class); boolean found = false; for (SdnControllerHostRefVO ref : controllerVO.getHostRefVOS()) { - if (ref.getHostUuid().equals(vmVo.getHostUuid())) { + String currentHostUuid = vmVo.getHostUuid(); + if (VmInstanceState.Stopped.equals(vmVo.getState())) { + currentHostUuid = vmVo.getLastHostUuid(); + } + if (ref.getHostUuid().equals(currentHostUuid)) { found = true; break; } @@ -271,9 +276,25 @@ private void validate(APISdnControllerChangeHostMsg msg) { msg.setNicNames(new ArrayList<>(nicNamePciAddressMap.keySet())); } - if (msg.getNicNames().size() > 1 && msg.getBondMode() == null) { + if (msg.getBondMode() == null) { msg.setBondMode(refVO.getBondMode()); } + + if (msg.getBondMode().equals(refVO.getBondMode())) { + msg.setLacpMode(refVO.getLacpMode()); + } + + if (msg.getBondMode().equals(L2NetworkConstant.BONDING_MODE_TCP)) { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_ACTIVE); + } else if (msg.getBondMode().equals(L2NetworkConstant.BONDING_MODE_SLB)) { + if (msg.getLacpMode() == null) { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_OFF); + } else if (!(msg.getLacpMode().equals(L2NetworkConstant.LACP_MODE_PASSIVE))) { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_OFF); + } + } else { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_OFF); + } } private void validateH3cTenantStatus(String l3NetworkUuid, String sdnControllerUuid) { diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java index f4733f6a833..8ff809fd308 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java @@ -86,6 +86,10 @@ public String getSdnControllerSignature() { return "sdn-controller-" + self.getUuid(); } + public String getSdnControllerHostSignature(String hostUuid) { + return "sdn-controller-" + self.getUuid() + "-host-" + hostUuid; + } + protected SdnController getSdnController() { SdnControllerFactory factory = sdnMgr.getSdnControllerFactory(self.getVendorType()); return factory.getSdnController(self); @@ -427,7 +431,7 @@ private void sdnControllerAddHost(APISdnControllerAddHostMsg msg, Completion com thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -508,7 +512,7 @@ private void sdnControllerRemoveHostInQueue(SdnControllerRemoveHostMsg msg, Comp thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -594,7 +598,7 @@ private void doControllerChangeHost(APISdnControllerChangeHostMsg msg, Completio thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java index c9e939b63f4..c46d7be3ab3 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java @@ -6,6 +6,7 @@ import org.zstack.core.Platform; import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.workflow.FlowChainBuilder; @@ -19,6 +20,7 @@ import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; import org.zstack.header.network.NetworkException; import org.zstack.header.network.l2.*; import org.zstack.header.network.l3.*; @@ -80,6 +82,8 @@ public List getAliasIds() { public void handleMessage(Message msg) { if (msg instanceof APIAddSdnControllerMsg) { handle((APIAddSdnControllerMsg) msg); + } else if (msg instanceof AddSdnControllerMsg) { + handle((AddSdnControllerMsg) msg); } else if (msg instanceof SdnControllerMessage) { handleSdnControllerMessage((SdnControllerMessage) msg); } else { @@ -87,6 +91,22 @@ public void handleMessage(Message msg) { } } + private void handle(AddSdnControllerMsg msg) { + AddSdnControllerReply reply = new AddSdnControllerReply(); + doCreateSdnController(msg, new Completion(msg) { + @Override + public void success() { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + private void handleSdnControllerMessage(SdnControllerMessage msg) { SdnControllerVO vo = dbf.findByUuid(msg.getSdnControllerUuid(), SdnControllerVO.class); if (vo == null) { @@ -99,23 +119,23 @@ private void handleSdnControllerMessage(SdnControllerMessage msg) { sdnController.handleMessage(msg); } - private void doCreateSdnController(SdnControllerVO vo, APIAddSdnControllerMsg msg, Completion completion) { - SdnControllerFactory factory = getSdnControllerFactory(msg.getVendorType()); + private void doCreateSdnController(AddSdnControllerMsg msg, Completion completion) { + SdnControllerVO vo = msg.getSdnControllerVO(); + SdnControllerFactory factory = getSdnControllerFactory(vo.getVendorType()); SdnController controller = factory.getSdnController(vo); Map data = new HashMap(); FlowChain chain = FlowChainBuilder.newShareFlowChain(); chain.setData(data); - chain.setName(String.format("create-sdn-controller-%s", msg.getName())); + chain.setName(String.format("create-sdn-controller-%s", vo.getName())); chain.then(new ShareFlow() { @Override public void setup() { flow(new NoRollbackFlow() { - String __name__ = String.format("pre-process-for-create-sdn-controller-%s", msg.getName()); + String __name__ = String.format("pre-process-for-create-sdn-controller-%s", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { - msg.setResourceUuid(vo.getUuid()); controller.preInitSdnController(msg, new Completion(trigger) { @Override public void success() { @@ -130,7 +150,7 @@ public void fail(ErrorCode errorCode) { } }); flow(new Flow() { - String __name__ = String.format("create-sdn-controller-%s-on-db", msg.getName()); + String __name__ = String.format("create-sdn-controller-%s-on-db", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { @@ -155,7 +175,7 @@ public void rollback(FlowRollback trigger, Map data) { } }); flow(new Flow() { - String __name__ = String.format("init-sdn-controller-%s", msg.getName()); + String __name__ = String.format("init-sdn-controller-%s", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { @@ -179,7 +199,7 @@ public void rollback(FlowRollback trigger, Map data) { } }); flow(new NoRollbackFlow() { - String __name__ = String.format("post-process-for-create-sdn-controller--%s", msg.getName()); + String __name__ = String.format("post-process-for-create-sdn-controller-%s", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { @@ -238,17 +258,22 @@ private void handle(APIAddSdnControllerMsg msg) { } vo.setStatus(SdnControllerStatus.Connected); - doCreateSdnController(vo, msg, new Completion(msg) { - @Override - public void success() { - tagMgr.createTagsFromAPICreateMessage(msg, vo.getUuid(), SdnControllerVO.class.getSimpleName()); - event.setInventory(SdnControllerInventory.valueOf(dbf.findByUuid(vo.getUuid(), SdnControllerVO.class))); - bus.publish(event); - } - + AddSdnControllerMsg amsg = new AddSdnControllerMsg(); + amsg.setSdnControllerVO(vo); + amsg.setAccountUuid(msg.getSession().getAccountUuid()); + if (msg.getSystemTags() != null) { + amsg.setSystemTags(msg.getSystemTags()); + } + bus.makeTargetServiceIdByResourceUuid(amsg, SdnControllerConstant.SERVICE_ID, vo.getUuid()); + bus.send(amsg, new CloudBusCallBack(amsg) { @Override - public void fail(ErrorCode errorCode) { - event.setError(errorCode); + public void run(MessageReply reply) { + if (reply.isSuccess()) { + tagMgr.createTagsFromAPICreateMessage(msg, vo.getUuid(), SdnControllerVO.class.getSimpleName()); + event.setInventory(SdnControllerInventory.valueOf(dbf.findByUuid(vo.getUuid(), SdnControllerVO.class))); + } else { + event.setError(reply.getError()); + } bus.publish(event); } }); diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java index f9afed355d9..40cdc09d9eb 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java @@ -160,7 +160,7 @@ private void getH3cDefaultTenant(Completion completion) { } } - private void getH3cParameters(APIAddSdnControllerMsg msg, Completion completion) { + private void getH3cParameters(AddSdnControllerMsg msg, Completion completion) { FlowChain chain = FlowChainBuilder.newSimpleFlowChain(); chain.setName(String.format("get-h3c-parameters-%s", self.getIp())); chain.then(new NoRollbackFlow() { @@ -228,12 +228,12 @@ public void handleMessage(SdnControllerMessage msg) { @Override @SdnControllerLog - public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void preInitSdnController(AddSdnControllerMsg msg, Completion completion) { completion.success(); } @Override - public void createSdnControllerDb(APIAddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { + public void createSdnControllerDb(AddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { dbf.persist(vo); completion.success(); } @@ -245,7 +245,7 @@ public void deleteSdnControllerDb(SdnControllerVO vo) { @Override @SdnControllerLog - public void initSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void initSdnController(AddSdnControllerMsg msg, Completion completion) { getH3cControllerToken(new Completion(completion) { @Override public void success() { diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java index 778d641c5e4..4f057e51915 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java @@ -141,7 +141,7 @@ public void fail(ErrorCode errorCode) { @Override @SdnControllerLog - public void initSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void initSdnController(AddSdnControllerMsg msg, Completion completion) { getH3cControllerToken(new Completion(completion) { @Override public void success() { @@ -335,7 +335,7 @@ public void getH3cVniRanges(Completion completion) { } } - private void getH3cParameters(APIAddSdnControllerMsg msg, Completion completion) { + private void getH3cParameters(AddSdnControllerMsg msg, Completion completion) { FlowChain chain = FlowChainBuilder.newSimpleFlowChain(); chain.setName(String.format("get-h3c-parameters-%s", self.getIp())); chain.then(new NoRollbackFlow() { diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java new file mode 100644 index 00000000000..044b0a22e0f --- /dev/null +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java @@ -0,0 +1,31 @@ +package org.zstack.sdnController.header; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.network.sdncontroller.SdnControllerMessage; +import org.zstack.header.network.sdncontroller.SdnControllerVO; + +public class AddSdnControllerMsg extends NeedReplyMessage implements SdnControllerMessage { + SdnControllerVO sdnControllerVO; + String accountUuid; + + @Override + public String getSdnControllerUuid() { + return sdnControllerVO.getUuid(); + } + + public SdnControllerVO getSdnControllerVO() { + return sdnControllerVO; + } + + public void setSdnControllerVO(SdnControllerVO sdnControllerVO) { + this.sdnControllerVO = sdnControllerVO; + } + + public String getAccountUuid() { + return accountUuid; + } + + public void setAccountUuid(String accountUuid) { + this.accountUuid = accountUuid; + } +} diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java new file mode 100644 index 00000000000..a73448fdfe5 --- /dev/null +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java @@ -0,0 +1,16 @@ +package org.zstack.sdnController.header; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.network.sdncontroller.SdnControllerInventory; + +public class AddSdnControllerReply extends MessageReply { + private SdnControllerInventory inventory; + + public SdnControllerInventory getInventory() { + return inventory; + } + + public void setInventory(SdnControllerInventory inventory) { + this.inventory = inventory; + } +} diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 45a68537e38..dff73723041 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.MessageSafe; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.header.core.Completion; @@ -22,10 +23,13 @@ import org.zstack.network.l3.L3NetworkSystemTags; import org.zstack.sdnController.SdnController; import org.zstack.sdnController.SdnControllerL2; +import org.zstack.sdnController.SdnControllerPingMsg; +import org.zstack.sdnController.SdnControllerPingReply; import org.zstack.sdnController.header.*; import org.zstack.sugonSdnController.controller.api.*; import org.zstack.sugonSdnController.controller.api.types.*; import org.zstack.sugonSdnController.header.APICreateL2TfNetworkMsg; +import org.zstack.sugonSdnController.network.TfZstackPortSync; import org.zstack.utils.StringDSL; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; @@ -44,6 +48,8 @@ public class SugonSdnController implements TfSdnController, SdnController, SdnCo CloudBus bus; @Autowired DatabaseFacade dbf; + @Autowired + TfZstackPortSync tfZstackPortSync; private SdnControllerVO sdnControllerVO; private TfHttpClient client; @@ -55,12 +61,34 @@ public SugonSdnController(SdnControllerVO vo) { } @Override + @MessageSafe public void handleMessage(SdnControllerMessage msg) { - bus.dealWithUnknownMessage((Message) msg); + if (msg instanceof SdnControllerPingMsg) { + handMessage((SdnControllerPingMsg) msg); + } else { + bus.dealWithUnknownMessage((Message) msg); + } + } + + void handMessage(SdnControllerPingMsg msg) { + SdnControllerPingReply reply = new SdnControllerPingReply(); + try { + Domain domain = (Domain) client.getDomain(); + if (domain == null) { + reply.setError(operr("get default domain on tf controller failed")); + } else { + tfZstackPortSync.triggerSyncIfDue(msg.getSdnControllerUuid()); + reply.setSuccess(true); + } + } catch (Exception e) { + logger.warn("ping tf sdn controller failed", e); + reply.setError(operr("get default domain on tf controller failed")); + } + bus.reply(msg, reply); } @Override - public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void preInitSdnController(AddSdnControllerMsg msg, Completion completion) { try { long count = Q.New(SdnControllerVO.class).eq(SdnControllerVO_.vendorType, SugonSdnControllerConstant.TF_CONTROLLER).count(); if(count > 0) { @@ -73,7 +101,7 @@ public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completi return; } String accountUuid = StringDSL.transToTfUuid(accountVO.getUuid()); - client = new TfHttpClient(msg.getIp()); + client = new TfHttpClient(msg.getSdnControllerVO().getIp()); Domain domain = (Domain) client.getDomain(); if(domain == null){ completion.fail(operr("get default domain on tf controller failed")); @@ -105,7 +133,7 @@ public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completi } @Override - public void createSdnControllerDb(APIAddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { + public void createSdnControllerDb(AddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { dbf.persist(vo); completion.success(); } @@ -116,7 +144,7 @@ public void deleteSdnControllerDb(SdnControllerVO vo) { } @Override - public void initSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void initSdnController(AddSdnControllerMsg msg, Completion completion) { completion.success(); } diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java index 1054f708696..2c531684d1b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java @@ -32,6 +32,22 @@ public TfPortClient(){ if (sdn == null){ throw new RuntimeException("Can not find a tf sdn controller."); } + init(sdn); + } + + public TfPortClient(String sdnControllerUuid) { + SdnControllerVO sdn = Q.New(SdnControllerVO.class).eq( + SdnControllerVO_.uuid, sdnControllerUuid).find(); + if (sdn == null) { + throw new RuntimeException(String.format("Can not find a tf sdn controller[uuid:%s].", sdnControllerUuid)); + } + if (!Objects.equals(sdn.getVendorType(), SdnControllerConstant.TF_CONTROLLER)) { + throw new RuntimeException(String.format("Sdn controller[uuid:%s] is not a tf sdn controller.", sdnControllerUuid)); + } + init(sdn); + } + + private void init(SdnControllerVO sdn) { client = new TfHttpClient(sdn.getIp()); tenantId = StringDSL.transToTfUuid(sdn.getAccountUuid()); } @@ -509,4 +525,3 @@ public Status updateTfPort(String tfPortUUid, String accountId, String deviceId, } } - diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java index c4767689bd1..e7a9f0c0419 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java @@ -33,6 +33,11 @@ public List getTfPortsDetail() { return tfPortClient.getVirtualMachineInterfaceDetail(); } + public List getTfPortsDetail(String sdnControllerUuid) { + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.getVirtualMachineInterfaceDetail(); + } + public TfPortResponse createTfPort(String tfPortUUid, String l2NetworkUuid, String l3NetworkUuid, String mac, String ip) { //invoke tf rest interface to retrieve real ip and mac and portId TfPortClient tfPortClient = new TfPortClient(); @@ -115,6 +120,12 @@ public TfPortResponse deleteTfPort(String portUUid) { return tfPortClient.deletePort(tfPortUUid); } + public TfPortResponse deleteTfPort(String sdnControllerUuid, String portUUid) { + String tfPortUUid = StringDSL.transToTfUuid(portUUid); + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.deletePort(tfPortUUid); + } + public Status updateTfPort(String portUUid, String bmUuid, KeyValuePairs bindInfo) { String accountId = StringDSL.transToTfUuid(acntMgr.getOwnerAccountUuidOfResource(bmUuid)); String tfBmUUid = StringDSL.transToTfUuid(bmUuid); diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java index 184b2e721ed..710b63cdb2b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java @@ -2,9 +2,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.db.Q; -import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.Task; import org.zstack.core.thread.ThreadFacade; -import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; import org.zstack.header.network.l2.L2NetworkVO; import org.zstack.header.network.l2.L2NetworkVO_; import org.zstack.header.vm.VmNicVO; @@ -19,39 +18,57 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.concurrent.Future; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -public class TfZstackPortSync implements ManagementNodeReadyExtensionPoint { +public class TfZstackPortSync { + private static final long SYNC_INTERVAL_MILLIS = TimeUnit.DAYS.toMillis(1); + private static final int MAX_DELETE_COUNT = 10; @Autowired protected ThreadFacade thdf; - private Future trackerThread = null; @Autowired private TfPortService tfPortService; private final static CLogger logger = Utils.getLogger(TfZstackPortSync.class); private final List excludeTypes = new ArrayList(Arrays.asList("neutron:LOADBALANCER", "VIP", "BMS")); + private final Map lastSyncTime = new ConcurrentHashMap<>(); + private final Set runningSyncs = Collections.newSetFromMap(new ConcurrentHashMap()); - @Override - public void managementNodeReady() { - if (trackerThread != null) { - trackerThread.cancel(true); + public void triggerSyncIfDue(String sdnControllerUuid) { + if (sdnControllerUuid == null) { + logger.warn("Port_Sync_Task: skip sync because sdn controller uuid is null."); + return; } - trackerThread = thdf.submitPeriodicTask(new SyncPort()); - } - private class SyncPort implements PeriodicTask { + long now = System.currentTimeMillis(); + Long lastSync = lastSyncTime.get(sdnControllerUuid); + if (lastSync != null && now - lastSync < SYNC_INTERVAL_MILLIS) { + return; + } + if (!runningSyncs.add(sdnControllerUuid)) { + return; + } - @Override - public TimeUnit getTimeUnit() { - return TimeUnit.DAYS; + lastSyncTime.put(sdnControllerUuid, now); + try { + thdf.submit(new SyncPort(sdnControllerUuid)); + } catch (RuntimeException e) { + lastSyncTime.remove(sdnControllerUuid); + runningSyncs.remove(sdnControllerUuid); + throw e; } + } - @Override - public long getInterval() { - return 1; + private class SyncPort implements Task { + private final String sdnControllerUuid; + + private SyncPort(String sdnControllerUuid) { + this.sdnControllerUuid = sdnControllerUuid; } @Override @@ -64,7 +81,10 @@ private HashSet getPortToDelete() { List zstackL2NetworksUuid = Q.New(L2NetworkVO.class).select(L2NetworkVO_.uuid).listValues(); List tfPortsUuid = new ArrayList<>(); try{ - List tfPorts = tfPortService.getTfPortsDetail(); + List tfPorts = tfPortService.getTfPortsDetail(sdnControllerUuid); + if (tfPorts == null) { + return new HashSet<>(); + } for (VirtualMachineInterface vmi : tfPorts) { // skip port if it's network not in zstack List> tfNetworks = vmi.getVirtualNetwork(); @@ -83,7 +103,7 @@ private HashSet getPortToDelete() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task: Fetch tf VirtualMachineInterface failed: %s.", e)); - return null; + return new HashSet<>(); } HashSet result = new HashSet<>(tfPortsUuid); result.removeAll(zstackPortsUuid); @@ -92,13 +112,13 @@ private HashSet getPortToDelete() { } @Override - public void run() { + public Void call() { logger.info("Port_Sync_Task: begin."); try { HashSet portsToDelete = getPortToDelete(); - int maxDeleteCount = 10; + int maxDeleteCount = MAX_DELETE_COUNT; for (String portUuid: portsToDelete) { - TfPortResponse response = tfPortService.deleteTfPort(portUuid); + TfPortResponse response = tfPortService.deleteTfPort(sdnControllerUuid, portUuid); if (response.getCode() == 200) { logger.info(String.format("Port_Sync_Task: VirtualMachineInterface: %s delete success.", portUuid)); @@ -113,7 +133,10 @@ public void run() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task failed: %s.", e)); + } finally { + runningSyncs.remove(sdnControllerUuid); } + return null; } } } diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java index 3b8d87bb8f0..7b38e6cc723 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java @@ -18,6 +18,12 @@ public class VirtualRouterGlobalConfig { public static GlobalConfig COMMANDS_PARALELLISM_DEGREE = new GlobalConfig(CATEGORY, "command.parallelismDegree"); @GlobalConfigValidation public static GlobalConfig RESTART_DNSMASQ_COUNT = new GlobalConfig(CATEGORY, "dnsmasq.restartAfterNumberOfSIGUSER1"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_DNS_FORWARD_MAX = new GlobalConfig(CATEGORY, "dnsmasq.dnsForwardMax"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_CACHE_SIZE = new GlobalConfig(CATEGORY, "dnsmasq.cacheSize"); @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig PING_INTERVAL = new GlobalConfig(CATEGORY, "ping.interval"); @GlobalConfigValidation diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java index d890e10ed33..1108c650975 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java @@ -180,12 +180,9 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setLogLevel(rcf.getResourceConfigValue(VirtualRouterGlobalConfig.LOG_LEVEL, vrUuid, String.class)); Map parms = new HashMap<>(); - long count = Q.New(VirtualRouterLoadBalancerRefVO.class) - .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, vrUuid) - .notNull(VirtualRouterLoadBalancerRefVO_.loadBalancerUuid) - .count(); - if(count > 0){ - parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(),VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); + if (!VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value().equals( + VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.defaultValue(String.class))){ + parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(), VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); } cmd.setParms(parms); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java index e522a99fd38..4d35f379e74 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java @@ -1,5 +1,6 @@ package org.zstack.xinfini; +import org.zstack.header.storage.addon.primary.AddonInfo; import org.zstack.xinfini.sdk.node.NodeModule; import org.zstack.xinfini.sdk.pool.BsPolicyModule; import org.zstack.xinfini.sdk.pool.PoolCapacity; @@ -9,7 +10,7 @@ import java.util.ArrayList; import java.util.List; -public class XInfiniAddonInfo { +public class XInfiniAddonInfo implements AddonInfo { private List nodes; private List pools = new ArrayList<>(); @@ -72,6 +73,7 @@ public static class Node { private int id; private String ip; private String state; + private NodeStatus status; public String getIp() { return ip; @@ -97,6 +99,14 @@ public void setState(String state) { this.state = state; } + public NodeStatus getStatus() { + return status; + } + + public void setStatus(NodeStatus status) { + this.status = status; + } + public static Node valueOf(NodeModule nodeModule) { Node node = new Node(); node.setIp(nodeModule.getSpec().getAdminIp()); @@ -104,6 +114,7 @@ public static Node valueOf(NodeModule nodeModule) { node.setState(nodeModule.getMetadata().getState().getState()); return node; } + } public static class Pool { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java index 3a90db3b043..2177283b48b 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java @@ -472,6 +472,11 @@ public void deleteBdcBdev(int bdevId, int bdcId) { } else { retryUtilResourceDeleted(gReq, GetBdcBdevResponse.class); } + + if (!call(gReq, GetBdcBdevResponse.class).resourceIsDeleted()) { + throw new OperationFailureException(operr("bdev[id:%s] still exists after deletion polling on bdc[id:%s], " + + "the old storage client may still hold the volume", bdevId, bdcId)); + } } public VolumeModule rollbackSnapshot(int volId, int snapId) { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java index 88b498ce9ef..bc1fc53c597 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java @@ -52,15 +52,6 @@ public void setId(int id) { public static class Node { private String ip; private int port; - private NodeStatus status = NodeStatus.Connected; - - public NodeStatus getStatus() { - return status; - } - - public void setStatus(NodeStatus status) { - this.status = status; - } public String getIp() { return ip; diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java index 267d55f4039..8898bc800be 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java @@ -1,7 +1,6 @@ package org.zstack.xinfini; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.util.packed.DirectMonotonicReader; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; @@ -9,7 +8,6 @@ import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.core.db.SQL; -import org.zstack.core.thread.ThreadFacade; import org.zstack.header.core.Completion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.errorcode.OperationFailureException; @@ -26,8 +24,6 @@ import org.zstack.iscsi.kvm.IscsiHeartbeatVolumeTO; import org.zstack.iscsi.kvm.IscsiVolumeTO; import org.zstack.kvm.KVMConstant; -import org.zstack.resourceconfig.ResourceConfigFacade; -import org.zstack.storage.addon.primary.ExternalPrimaryStorageFactory; import org.zstack.storage.volume.VolumeConfigsGetter; import org.zstack.utils.CollectionUtils; import org.zstack.utils.DebugUtils; @@ -69,15 +65,13 @@ public class XInfiniStorageController implements PrimaryStorageControllerSvc, Pr private static CLogger logger = Utils.getLogger(XInfiniStorageController.class); @Autowired + @Deprecated private DatabaseFacade dbf; - @Autowired - private ThreadFacade thdf; + private ExternalPrimaryStorageVO self; private XInfiniConfig config; private XInfiniAddonInfo addonInfo; private final XInfiniApiHelper apiHelper; - @Autowired - private ResourceConfigFacade rcf; private String vhostSocketDir; @@ -122,11 +116,20 @@ public XInfiniStorageController(ExternalPrimaryStorageVO self) { public XInfiniStorageController(String configStr) { config = JSONObjectUtil.toObject(configStr, XInfiniConfig.class); + XInfiniConnectConfig clientConfig = new XInfiniConnectConfig(); clientConfig.readTimeout = TimeUnit.MINUTES.toMillis(10); clientConfig.writeTimeout = TimeUnit.MINUTES.toMillis(10); clientConfig.token = config.getToken(); - clientConfig.xInfiniConfig = config; + clientConfig.connectedNodeProvider = () -> { + Set connectedIps = this.addonInfo.getNodes().stream() + .filter(it -> it.getStatus() == NodeStatus.Connected) + .map(XInfiniAddonInfo.Node::getIp) + .collect(Collectors.toSet()); + return this.config.getNodes().stream() + .filter(it -> connectedIps.contains(it.getIp())) + .findFirst().orElse(null); + }; XInfiniClient client = new XInfiniClient(); client.configure(clientConfig); @@ -202,18 +205,23 @@ public synchronized ActiveVolumeTO activeIscsiVolume(String clientIp, String cli public void deactivate(String installPath, String protocol, HostInventory h, Completion comp) { logger.debug(String.format("deactivating volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); - if (VolumeProtocol.Vhost.toString().equals(protocol)) { - deactivateVhost(installPath, h); - comp.success(); - return; - } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { - // iscsi target is shared by all hosts, we cannot control one volume on one host for now. - deactivateIscsi(installPath, h); - comp.success(); + try { + if (VolumeProtocol.Vhost.toString().equals(protocol)) { + deactivateVhost(installPath, h); + } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { + // iscsi target is shared by all hosts, we cannot control one volume on one host for now. + deactivateIscsi(installPath, h); + } else { + comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + return; + } + } catch (Exception e) { + comp.fail(operr("failed to deactivate volume[path:%s, protocol:%s] on host[uuid:%s, ip:%s]: %s", + installPath, protocol, h.getUuid(), h.getManagementIp(), e.getMessage())); return; } - comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + comp.success(); } private void deactivateIscsi(String installPath, HostInventory h) { @@ -255,7 +263,7 @@ private void deactivateVhost(String installPath, HostInventory h) { return; } - retry(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); + retryOrThrow(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); } @Override @@ -271,9 +279,10 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - // todo - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { + throw new OperationFailureException(operr("xinfini does not support volume path isolation yet, " + + "abort starting VM on host[uuid:%s, ip:%s] to prevent split-brain on volume[path:%s, protocol:%s]", + h.getUuid(), h.getManagementIp(), installPath, protocol)); } @Override @@ -499,20 +508,17 @@ public String getIdentity() { } @Override - public void ping(Completion completion) { + public void ping(ReturnValueCompletion completion) { reloadDbInfo(); Map nodesStatus = apiHelper.checkNodesConnection(config.getNodes()); logger.info(String.format("get nodes status: %s", nodesStatus)); - config.getNodes().forEach(it -> it.setStatus(nodesStatus.getOrDefault(it.getIp(), NodeStatus.Disconnected))); - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.config, JSONObjectUtil.toJsonString(config)) - .update(); - completion.success(); + addonInfo.getNodes().forEach(it -> it.setStatus(nodesStatus.getOrDefault(it.getIp(), NodeStatus.Disconnected))); + completion.success(new PingResult(addonInfo)); } @Override - public void connect(String config, String url, ReturnValueCompletion comp) { + public void connect(String config, String url, ReturnValueCompletion comp) { DebugUtils.Assert(StringUtils.isNotEmpty(config), "config cannot be none"); XInfiniAddonInfo info = new XInfiniAddonInfo(); XInfiniConfig xConfig = JSONObjectUtil.toObject(config, XInfiniConfig.class); @@ -528,6 +534,8 @@ public void connect(String config, String url, ReturnValueCompletion new OperationFailureException(operr("fail to get node %s details, check ip address and role config", it.getIp())))); info.setNodes(nodes.stream().map(XInfiniAddonInfo.Node::valueOf).collect(Collectors.toList())); + Map nodeIpStatus = apiHelper.checkNodesConnection(xConfig.getNodes()); + info.getNodes().forEach(it -> it.setStatus(nodeIpStatus.get(it.getIp()))); List pools = apiHelper.queryPools(); if (CollectionUtils.isEmpty(pools)) { @@ -543,7 +551,7 @@ public void connect(String config, String url, ReturnValueCompletion comp) { - reloadDbInfo(); List pools = refreshPoolCapacity() .stream() .filter(it -> config.getPoolIds().contains(it.getId())) @@ -1038,16 +1056,33 @@ private void retry(Runnable r) { private void retry(Runnable r, int retry) { - while (retry-- > 0) { + try { + retryOrThrow(r, retry); + } catch (RuntimeException ignored) { + } + } + + private void retryOrThrow(Runnable r) { + retryOrThrow(r, 3); + } + + private void retryOrThrow(Runnable r, int retry) { + RuntimeException lastError = null; + for (int i = 0; i < retry; i++) { try { r.run(); return; - } catch (Exception e) { + } catch (RuntimeException e) { + lastError = e; logger.warn("runnable failed, try ", e); try { TimeUnit.SECONDS.sleep(3); } catch (InterruptedException ignore) {} } } + + if (lastError != null) { + throw lastError; + } } } diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java index 4faa1988e18..a21b84bf43d 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java @@ -36,7 +36,7 @@ public PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo) { } @Override - public void discover(String url, String config, ReturnValueCompletion completion) { + public void discover(String url, String config, ReturnValueCompletion completion) { // xinfini must set config if (StringUtils.isEmpty(config)) { completion.fail(operr("empty config, cannot discover xinfini")); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java index 8b486289dc7..3f15165b18f 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java @@ -6,15 +6,12 @@ import org.springframework.http.HttpMethod; import org.zstack.core.Platform; import org.zstack.externalStorage.sdk.ExternalStorageApiClient; -import org.zstack.externalStorage.sdk.ExternalStorageParam; import org.zstack.header.rest.DefaultSSLVerifier; import org.zstack.header.xinfini.XInfiniConstants; import org.zstack.utils.Utils; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; -import org.zstack.xinfini.NodeStatus; import org.zstack.xinfini.XInfiniConfig; -import org.zstack.xinfini.sdk.volume.FlattenVolumeRequest; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.X509TrustManager; @@ -139,12 +136,7 @@ XinfiniApiResult doCallWithNode(XInfiniConfig.Node withNode) { if (withNode != null) { node = withNode; } else { - node = config.xInfiniConfig - .getNodes() - .stream() - .filter(it -> it.getStatus() == NodeStatus.Connected) - .findAny() - .orElseThrow(() -> new XInfiniApiException("No connected node found")); + node = config.connectedNodeProvider.get(); } try { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java index 43668787513..dc42fe54473 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java @@ -3,6 +3,7 @@ import org.zstack.xinfini.XInfiniConfig; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; public class XInfiniConnectConfig { public String hostname = "localhost"; @@ -12,7 +13,7 @@ public class XInfiniConnectConfig { public Long readTimeout; public Long writeTimeout; public String token; - public XInfiniConfig xInfiniConfig; + public Supplier connectedNodeProvider; public void setPort(int port) { this.port = port; diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java index 8f547cfeb29..987138c3360 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java @@ -15,7 +15,8 @@ public interface ZbsConstants { long ZBS_HEARTBEAT_VOLUME_SIZE_IN_GIGABYTE = 1; String ZBS_CBD_LUN_PATH_FORMAT = "cbd:%s/%s/%s"; String ZBS_CBD_PREFIX_SCHEME = "cbd://"; - Integer PRIMARY_STORAGE_MDS_MAXIMUM_PING_FAILURE = 3; + Integer MDS_PING_RETRY_PER_CYCLE = 3; + Integer MDS_PING_FAIL_CYCLE_THRESHOLD = 3; String VOLUME_PHYSICAL_BLOCK_SIZE = "4096"; String MEGABYTE_SUPPORTED_VERSION = "1.6.1"; String MEGABYTE_UNIT = "M"; diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java index 81309d43642..986ad438b42 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java @@ -206,6 +206,12 @@ public void run(FlowTrigger trigger, Map data) { restf.asyncJsonPost(ZbsAgentUrl.primaryStorageUrl(getSelf().getAddr(), SYNC_METADATA_PATH), cmd, new JsonAsyncRESTCallback(trigger) { @Override public void success(SyncMetadataRsp ret) { + if (!ret.isSuccess()) { + trigger.fail(operr("unable to sync metadata from ZBS primary storage MDS[%s], because %s", + getSelf().getAddr(), ret.getError())); + return; + } + getSelf().setExternalAddr(ret.getExternalAddr()); trigger.next(); } @@ -307,7 +313,7 @@ public String getName() { } private void pingMds(ClusterInfo clusterInfo, final Completion completion) { - final Integer MAX_PING_CNT = ZbsConstants.PRIMARY_STORAGE_MDS_MAXIMUM_PING_FAILURE; + final Integer MAX_PING_CNT = ZbsConstants.MDS_PING_RETRY_PER_CYCLE; final List stepCount = new ArrayList<>(); for (int i = 1; i <= MAX_PING_CNT; i++) { stepCount.add(i); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 4b725d7d952..4db9baf0d1d 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.cbd.*; +import org.zstack.cbd.AddonInfo; import org.zstack.cbd.kvm.CbdHeartbeatVolumeTO; import org.zstack.cbd.kvm.CbdVolumeTo; import org.zstack.compute.host.HostGlobalConfig; @@ -56,10 +57,12 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.zstack.core.Platform.operr; +import static org.zstack.storage.zbs.ZbsConstants.MDS_PING_FAIL_CYCLE_THRESHOLD; import static org.zstack.storage.zbs.ZbsHelper.*; /** @@ -71,6 +74,7 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar private static final CLogger logger = Utils.getLogger(ZbsStorageController.class); @Autowired + @Deprecated private DatabaseFacade dbf; @Autowired protected RESTFacade restf; @@ -83,6 +87,7 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar private AddonInfo addonInfo; private Config config; + private final Map mdsConsecutivePingFailureCount = new ConcurrentHashMap<>(); public static final String DEPLOY_CLIENT_PATH = "/zbs/primarystorage/client/deploy"; public static final String GET_CAPACITY_PATH = "/zbs/primarystorage/capacity"; public static final String GET_FACTS_PATH = "/zbs/primarystorage/facts"; @@ -145,8 +150,7 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { } @Override @@ -333,7 +337,7 @@ public String getIdentity() { } @Override - public void connect(String cfg, String url, ReturnValueCompletion completion) { + public void connect(String cfg, String url, ReturnValueCompletion completion) { AddonInfo newAddonInfo = new AddonInfo(); Config current = JSONObjectUtil.toObject(cfg, Config.class); List mdsInfos = parseMdsInfos(current.getMdsUrls()); @@ -488,7 +492,7 @@ public void done(ErrorCodeList errorCodeList) { public void handle(Map data) { configUrl(self.getUuid()); addonInfo = newAddonInfo; - completion.success(JSONObjectUtil.rehashObject(newAddonInfo, LinkedHashMap.class)); + completion.success(newAddonInfo); } }); @@ -502,8 +506,103 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private void reconnectSingleMds(ZbsPrimaryStorageMdsBase mdsBase, Completion completion) { + mdsBase.connect(new Completion(completion) { + @Override + public void success() { + completion.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void reconnectMdss(List mdsBases, Completion completion) { + new While<>(mdsBases).all((mdsBase, comp) -> { + reconnectSingleMds(mdsBase, new Completion(comp) { + @Override + public void success() { + comp.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(errorCodeList.getCauses().get(0)); + return; + } + completion.success(); + } + }); + } + + private void incrementMdsPingFailureCount(ZbsPrimaryStorageMdsBase mds) { + // initialize the count if absent + mdsConsecutivePingFailureCount.putIfAbsent(mds.getSelf(), 0); + // increment the count only if it is below the threshold + // avoid unbounded growth of the count + if (mdsConsecutivePingFailureCount.get(mds.getSelf()) < MDS_PING_FAIL_CYCLE_THRESHOLD) { + mdsConsecutivePingFailureCount.merge(mds.getSelf(), 1, Integer::sum); + } + } + + private void resetMdsPingFailureCount(ZbsPrimaryStorageMdsBase mds) { + mdsConsecutivePingFailureCount.remove(mds.getSelf()); + } + + private List getDisconnectedMds(List mdsBases) { + return mdsBases.stream() + .filter(m -> MdsStatus.Disconnected.equals(m.getSelf().getStatus())) + .collect(Collectors.toList()); + } + + private boolean allMdssDisconnected(List mdsBases) { + List disconnectedMds = getDisconnectedMds(mdsBases); + return disconnectedMds.size() == mdsBases.size(); + } + + private List getMdssNeedToReconnect(List mdsBases) { + List disconnectedMds = getDisconnectedMds(mdsBases); + boolean allDisconnected = allMdssDisconnected(mdsBases); + + if (disconnectedMds.isEmpty()) { + return Collections.emptyList(); + } + + // If all MDS are disconnected, try to reconnect all of them + if (allDisconnected) { + logger.info("All MDS are disconnected, ignore the ping failure count threshold and all of them need be reconnected"); + return disconnectedMds; + } + + String disconnectedMdsInfo = disconnectedMds.stream() + .map(m -> String.format("%s (Consecutive Ping Failure: %d)", + m.getSelf().getAddr(), + mdsConsecutivePingFailureCount.computeIfAbsent(m.getSelf(), k -> 0))) + .collect(Collectors.joining(", ")); + + logger.info(String.format("Some MDS are disconnected for ZBS primary storage[uuid:%s], disconnected MDS: [%s]", + self.getUuid(), disconnectedMdsInfo)); + + return disconnectedMds.stream() + .filter(m -> mdsConsecutivePingFailureCount + .computeIfAbsent(m.getSelf(), k -> 0) >= MDS_PING_FAIL_CYCLE_THRESHOLD) + .collect(Collectors.toList()); + + } + @Override - public void ping(Completion completion) { + public void ping(ReturnValueCompletion completion) { reloadDbInfo(); if (addonInfo == null || addonInfo.getClusterInfo() == null) { @@ -516,39 +615,72 @@ public void ping(Completion completion) { @Override public void success() { m.getSelf().setStatus(MdsStatus.Connected); + resetMdsPingFailureCount(m); comp.done(); } @Override public void fail(ErrorCode errorCode) { m.getSelf().setStatus(MdsStatus.Disconnected); + incrementMdsPingFailureCount(m); comp.done(); } })).run(new WhileDoneCompletion(completion) { @Override public void done(ErrorCodeList errorCodeList) { - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) - .update(); - - boolean isConnected = addonInfo.getMdsInfos().stream().anyMatch(mdsInfo -> MdsStatus.Connected.equals(mdsInfo.getStatus())); - if (!isConnected) { - String notConnectedIps = addonInfo.getMdsInfos().stream() - .filter(mdsInfo -> !MdsStatus.Connected.equals(mdsInfo.getStatus())) - .map(MdsInfo::getAddr) - .collect(Collectors.joining(", ")); + List mdsNeedToReconnect = getMdssNeedToReconnect(mds); - completion.fail(operr("no MDS is Connected, the following MDS[%s] are not Connected.", notConnectedIps)); + if (mdsNeedToReconnect.isEmpty()) { + completion.success(new PingResult(addonInfo)); return; } - completion.success(); + + String mdsNeedToReconnectInfo = mdsNeedToReconnect.stream() + .map(m -> String.format("%s (Consecutive Ping Failure: %d)", + m.getSelf().getAddr(), + mdsConsecutivePingFailureCount.get(m.getSelf()))) + .collect(Collectors.joining(", ")); + + logger.warn(String.format("Some MDS need to reconnect for ZBS primary storage[uuid:%s], MDS : %s", + self.getUuid(), mdsNeedToReconnectInfo)); + + reconnectMdss(mdsNeedToReconnect, new Completion(completion) { + @Override + public void success() { + completion.success(new PingResult(addonInfo)); + } + + @Override + public void fail(ErrorCode errorCode) { + List stillDisconnectedMds = getDisconnectedMds(mds); + boolean allDisconnected = allMdssDisconnected(mds); + + if (allDisconnected) { + completion.success(new PingResult(addonInfo, + String.format("All MDS are still disconnected after reconnection for ZBS primary storage[uuid:%s]", + self.getUuid()))); + return; + } + + String stillDisconnectedMdsInfo = stillDisconnectedMds.stream() + .map(m -> String.format("%s (Consecutive Ping Failure: %d)", + m.getSelf().getAddr(), + mdsConsecutivePingFailureCount.get(m.getSelf()))) + .collect(Collectors.joining(", ")); + + logger.warn(String.format("Some MDS are still disconnected after reconnection for ZBS primary storage[uuid:%s], disconnected MDS : %s, error: %s", + self.getUuid(), stillDisconnectedMdsInfo, errorCode)); + + completion.success(new PingResult(addonInfo)); + } + }); + } }); } @Override public void reportCapacity(ReturnValueCompletion comp) { - reloadDbInfo(); GetCapacityCmd cmd = new GetCapacityCmd(); cmd.setLogicalPool(config.getLogicalPoolName()); @@ -556,6 +688,7 @@ public void reportCapacity(ReturnValueCompletion comp) { httpCall(GET_CAPACITY_PATH, cmd, GetCapacityRsp.class, new ReturnValueCompletion(comp) { @Override public void success(GetCapacityRsp returnValue) { + reloadDbInfo(); addonInfo.setLogicalPoolInfos(returnValue.getLogicalPoolInfos()); SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) @@ -1121,6 +1254,17 @@ public void fail(ErrorCode errorCode) { }); } + @Override + public void syncAddonInfo(String addonInfo) { + this.addonInfo = StringUtils.isEmpty(addonInfo) ? new AddonInfo() : JSONObjectUtil.toObject(addonInfo, AddonInfo.class); + } + + @Override + public void syncConfig(String config) { + this.config = StringUtils.isEmpty(config) ? new Config() : JSONObjectUtil.toObject(config, Config.class); + } + + @Deprecated private void reloadDbInfo() { self = dbf.reload(self); addonInfo = StringUtils.isEmpty(self.getAddonInfo()) ? new AddonInfo() : JSONObjectUtil.toObject(self.getAddonInfo(), AddonInfo.class); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java index df9fb479a2c..9fc5a72be5a 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java @@ -7,26 +7,15 @@ import org.zstack.core.db.Q; import org.zstack.core.trash.StorageTrash; import org.zstack.externalStorage.primary.ExternalStorageFencerType; -import org.zstack.header.core.Completion; -import org.zstack.header.core.NoErrorCompletion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.storage.addon.primary.*; -import org.zstack.header.storage.primary.DeleteVolumeBitsOnPrimaryStorageMsg; -import org.zstack.header.storage.primary.PrimaryStorageConstant; -import org.zstack.header.storage.snapshot.VolumeSnapshotAfterDeleteExtensionPoint; -import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; import org.zstack.header.volume.VolumeProtocol; import org.zstack.header.volume.VolumeVO; import org.zstack.header.volume.VolumeVO_; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.zstack.storage.zbs.ZbsHelper.*; import static org.zstack.core.Platform.operr; @@ -51,7 +40,7 @@ public PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo) { } @Override - public void discover(String url, String config, ReturnValueCompletion completion) { + public void discover(String url, String config, ReturnValueCompletion completion) { completion.fail(operr("zbs not support discover yet")); } diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java index 747a154919b..88bef7d8302 100755 --- a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java +++ b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java @@ -74,6 +74,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -106,6 +107,15 @@ public class ManagementNodeManagerImpl extends AbstractService implements Manage // A dictionary (nodeId -> ManagementNodeInventory) of joined management Node final private Map joinedManagementNodes = new ConcurrentHashMap<>(); + // Lock to serialize lifecycle events from heartbeat reconciliation and canonical event callbacks, + // preventing race conditions where a nodeJoin event is immediately followed by a stale nodeLeft + // from the heartbeat thread, or vice versa. See ZSTAC-77711. + private final Object lifecycleLock = new Object(); + + // Track nodes found in hash ring but missing from DB. Only call nodeLeft after a node + // is missing for two consecutive heartbeat cycles, to avoid removing nodes that just joined. + private final Set suspectedMissingFromDb = new HashSet<>(); + private static int NODE_STARTING = 0; private static int NODE_RUNNING = 1; private static int NODE_FAILED = -1; @@ -367,12 +377,16 @@ protected void run(Map tokens, Object data) { ManagementNodeLifeCycleData d = (ManagementNodeLifeCycleData) data; - if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeJoin(d.getInventory()); - } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeLeft(d.getInventory()); - } else { - throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + synchronized (lifecycleLock) { + if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { + // Clear from suspected set since the node is confirmed alive + suspectedMissingFromDb.remove(d.getInventory().getUuid()); + nodeLifeCycle.nodeJoin(d.getInventory()); + } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { + nodeLifeCycle.nodeLeft(d.getInventory()); + } else { + throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + } } } }; @@ -853,34 +867,55 @@ private void checkAllNodesHealth() { Set nodeUuidsInDb = nodesInDb.stream().map(ManagementNodeVO::getUuid).collect(Collectors.toSet()); - // When a node is dying, we may not receive the the dead notification because the message bus may be also dead - // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out - destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { - if (!nodeUuidsInDb.contains(nodeUuid)) { - logger.warn(String.format("found that a management node[uuid:%s] had no heartbeat in database but still in our hash ring," + - "notify that it's dead", nodeUuid)); - ManagementNodeInventory inv = new ManagementNodeInventory(); - inv.setUuid(nodeUuid); - inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); - - nodeLifeCycle.nodeLeft(inv); - } - }); - - // check if any node missing in our hash ring - nodesInDb.forEach(n -> { - if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { - return; - } - - new Runnable() { - @Override - @AsyncThread - public void run() { - nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + // Reconcile hash ring with DB under lifecycleLock to prevent race with + // canonical event callbacks (nodeJoin/nodeLeft). See ZSTAC-77711. + synchronized (lifecycleLock) { + // When a node is dying, we may not receive the dead notification because the message bus may be also dead + // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out. + // Use two-round confirmation: first round marks as suspected, second round actually removes. + Set currentSuspected = new HashSet<>(); + destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { + if (!nodeUuidsInDb.contains(nodeUuid)) { + if (suspectedMissingFromDb.contains(nodeUuid)) { + // Second consecutive detection — confirmed missing, remove from hash ring + logger.warn(String.format("management node[uuid:%s] confirmed missing from database for two consecutive" + + " heartbeat cycles, removing from hash ring", nodeUuid)); + ManagementNodeInventory inv = new ManagementNodeInventory(); + inv.setUuid(nodeUuid); + try { + inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); + } catch (Exception e) { + logger.warn(String.format("cannot get node info for node[uuid:%s], use empty hostname", nodeUuid)); + } + + nodeLifeCycle.nodeLeft(inv); + } else { + // First detection — mark as suspected, defer removal to next cycle + logger.warn(String.format("management node[uuid:%s] not found in database but still in hash ring," + + " marking as suspected (will remove on next heartbeat if still missing)", nodeUuid)); + currentSuspected.add(nodeUuid); + } } - }.run(); - }); + }); + // Update suspected set: only keep nodes that are newly suspected this round + suspectedMissingFromDb.clear(); + suspectedMissingFromDb.addAll(currentSuspected); + + // check if any node missing in our hash ring + nodesInDb.forEach(n -> { + if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { + return; + } + + new Runnable() { + @Override + @AsyncThread + public void run() { + nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + } + }.run(); + }); + } } @Override diff --git a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java index 8f98f870348..cd6fae86322 100644 --- a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java +++ b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java @@ -37,6 +37,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List types; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Integer minAvailableDevicesPerHost; + @Param(required = false) public java.util.List systemTags; diff --git a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java index 101684ad18c..df5e0e5a96c 100755 --- a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java +++ b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java @@ -729,7 +729,15 @@ String toJpql(boolean isCount) { throw new IllegalArgumentException(String.format("illegal sortBy[%s], entity[%s] doesn't have this field", msg.getSortBy(), info.entityClass.getName())); } - ret = String.format("%s order by %s.%s %s", ret, entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + String orderBy = String.format("order by %s.%s %s", entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + if (info.primaryKey != null && !msg.getSortBy().equals(info.primaryKey)) { + orderBy += String.format(", %s.%s ASC", entityName, info.primaryKey); + } + ret = String.format("%s %s", ret, orderBy); + } else if (msg.getStart() != null || msg.getLimit() != null) { + if (info.primaryKey != null) { + ret = String.format("%s order by %s.%s ASC", ret, entityName, info.primaryKey); + } } if (msg.getGroupBy() != null) { diff --git a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java index 59aa7ce7e7d..9beeb1dd675 100755 --- a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java +++ b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java @@ -30,6 +30,7 @@ import org.zstack.utils.*; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; +import org.zstack.core.db.EntityMetadata; import org.zstack.zql.ZQL; import org.zstack.zql.ZQLContext; import org.zstack.zql.ZQLQueryReturn; @@ -347,7 +348,7 @@ private Method getReplySetter(AutoQuery at) { } } - private void handle(APIQueryMessage msg) { + private APIQueryReply doQuery(APIQueryMessage msg) { AutoQuery at = autoQueryMap.get(msg.getClass()); if (at == null) { at = msg.getClass().getAnnotation(AutoQuery.class); @@ -372,7 +373,7 @@ private void handle(APIQueryMessage msg) { if (result.inventories != null) { replySetter.invoke(reply, result.inventories); } - bus.reply(msg, reply); + return reply; } catch (OperationFailureException of) { throw of; } catch (Exception e) { @@ -380,6 +381,39 @@ private void handle(APIQueryMessage msg) { } } + private void handle(APIQueryMessage msg) { + thdf.syncSubmit(new SyncTask() { + @Override + public Void call() { + APIQueryReply reply; + try { + reply = doQuery(msg); + } catch (OperationFailureException of) { + reply = new APIQueryReply(); + reply.setError(of.getErrorCode()); + } + + bus.reply(msg, reply); + return null; + } + + @Override + public String getName() { + return getSyncSignature(); + } + + @Override + public String getSyncSignature() { + return "api-query"; + } + + @Override + public int getSyncLevel() { + return getQuerySyncLevel(); + } + }); + } + private String toZQLConditionString(QueryCondition c) { // make every condition value as string, the ZQL // will put them in right type because it knows every field's type @@ -482,7 +516,17 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { } if (msg.getSortBy() != null) { - sb.add(String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection())); + String orderBy = String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection()); + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null && !msg.getSortBy().equals(pkFieldName)) { + orderBy += String.format(", %s asc", pkFieldName); + } + sb.add(orderBy); + } else if (!msg.isCount() && (msg.getLimit() != null || msg.getStart() != null)) { + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null) { + sb.add(String.format("order by %s asc", pkFieldName)); + } } if (msg.getLimit() != null) { @@ -514,6 +558,24 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { return result; } + private String getPrimaryKeyFieldNameSafely(Class inventoryClass) { + try { + ZQLMetadata.InventoryMetadata meta = ZQLMetadata.findInventoryMetadata( + ZQL.queryTargetNameFromInventoryClass(inventoryClass)); + Class voClass = meta.inventoryAnnotation.mappingVOClass(); + Field pkField = EntityMetadata.getPrimaryKeyField(voClass); + String pkName = pkField.getName(); + if (!meta.selfInventoryFieldNames.contains(pkName)) { + return null; + } + return pkName; + } catch (Exception e) { + logger.trace(String.format("cannot resolve PK for %s, skip pagination tiebreaker: %s", + inventoryClass.getSimpleName(), e.getMessage())); + return null; + } + } + private QueryBelongFilter validateFilterNameAndGetExp(String filterName) { if (filterName.split(":").length < 2) { throw new OperationFailureException(argerr("filterName must be formatted as [filterType:condition(s)]")); diff --git a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java index 6627311adb9..10ebab413b4 100644 --- a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java +++ b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java @@ -1,10 +1,10 @@ package org.zstack.zql.ast.visitors.plugin; -import org.apache.commons.lang.StringUtils; import org.zstack.header.zql.ASTNode; import java.util.List; -import java.util.stream.Collectors; + +import static org.zstack.zql.ast.visitors.constants.MySqlKeyword.isDistinct; public class SimpleCountPlugin extends AbstractQueryVisitorPlugin { public SimpleCountPlugin() { @@ -24,13 +24,13 @@ public String selectTarget() { String queryTarget; List fieldNames = super.targetFields(); - if (fieldNames.isEmpty()) { - queryTarget = entityAlias; - } else { + if (getClauseType() == ClauseType.COUNT || isDistinct(node.getTarget().getFunction())) { // TODO: Compatibility changes: hql count do not support multiple fields, even if distinct modified. - queryTarget = String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + queryTarget = fieldNames.isEmpty() ? entityAlias : String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + return String.format("count(%s)", String.format(functions(), queryTarget)); + } else { + queryTarget = entityAlias; + return String.format("count(%s)", queryTarget); } - - return String.format("count(%s)", String.format(functions(), queryTarget)); } } diff --git a/storage/pom.xml b/storage/pom.xml index d5fe306aec3..f4f07e048fe 100755 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -3,7 +3,7 @@ zstack org.zstack - 5.4.0 + 5.4.0 .. storage diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 79332acdc88..33f4f43a5f4 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.EventFacade; import org.zstack.core.cloudbus.ResourceDestinationMaker; import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.Q; @@ -87,6 +88,9 @@ public class ExternalPrimaryStorage extends PrimaryStorageBase { @Autowired private ResourceDestinationMaker destMaker; + @Autowired + private EventFacade evtf; + public ExternalPrimaryStorage(PrimaryStorageVO self, PrimaryStorageControllerSvc controller, PrimaryStorageNodeSvc node) { super(self); this.controller = controller; @@ -935,6 +939,7 @@ protected void handle(CreateTemplateFromVolumeOnPrimaryStorageMsg msg) { String snapshotPath; String bsInstallPath; + String actualFormat; long templateSize; @@ -1063,6 +1068,7 @@ public void run(MessageReply reply) { DownloadImageFromRemoteTargetReply r = reply.castReply(); bsInstallPath = r.getInstallPath(); templateSize = r.getSize(); + actualFormat = r.getFormat(); trigger.next(); } } @@ -1095,7 +1101,7 @@ public void fail(ErrorCode errorCode) { @Override public void handle(Map data) { reply.setTemplateBackupStorageInstallPath(bsInstallPath); - reply.setFormat(msg.getVolumeInventory().getFormat()); + reply.setFormat(actualFormat); reply.setActualSize(templateSize); bus.reply(msg, reply); } @@ -1965,15 +1971,28 @@ protected void handle(CheckVolumeSnapshotOperationOnPrimaryStorageMsg msg) { @Override protected void connectHook(ConnectParam param, Completion completion) { controller.validateConfig(externalVO.getConfig()); - controller.connect(externalVO.getConfig(), self.getUrl(), new ReturnValueCompletion(completion) { + controller.connect(externalVO.getConfig(), self.getUrl(), new ReturnValueCompletion(completion) { @Override - public void success(LinkedHashMap addonInfo) { + public void success(AddonInfo addonInfo) { if (param.isNewAdded()) { controller.onFirstAdditionConfigure(new NopeCompletion()); } SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) .update(); + + if (addonInfo != null && addonInfo.changed(externalVO.getAddonInfo())) { + String newAddonInfo = addonInfo.serialize(); + SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) + .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfo) + .update(); + externalVO.setAddonInfo(newAddonInfo); + // fire event when addonInfo changed + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = new org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); + evtData.setUuid(self.getUuid()); + evtData.setAddonInfo(newAddonInfo); + evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + } controller.setTrashExpireTime(PrimaryStorageGlobalConfig.TRASH_EXPIRATION_TIME.value(Integer.class), new NopeCompletion()); pingHook(completion); } @@ -1997,12 +2016,34 @@ public void setup() { @Override public void run(FlowTrigger trigger, Map data) { - controller.ping(new Completion(trigger) { + controller.ping(new ReturnValueCompletion(trigger) { @Override - public void success() { + public void success(PingResult ret) { + if (ret.getAddonInfo() != null && ret.getAddonInfo().changed(externalVO.getAddonInfo())) { + updateAddonInfo(ret.getAddonInfo()); + } + + if (!ret.isSuccess()) { + trigger.fail(operr("ping external primary storage[%s] failed, %s", self.getUuid(), ret.getError())); + return; + } + trigger.next(); } + private void updateAddonInfo(AddonInfo newAddonInfo) { + String newAddonInfoStr = newAddonInfo.serialize(); + externalVO.setAddonInfo(newAddonInfoStr); + SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) + .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfoStr) + .update(); + + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = new ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); + evtData.setUuid(self.getUuid()); + evtData.setAddonInfo(newAddonInfoStr); + evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + } + @Override public void fail(ErrorCode errorCode) { trigger.fail(errorCode); diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java new file mode 100644 index 00000000000..5fc99f89834 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java @@ -0,0 +1,26 @@ +package org.zstack.storage.addon.primary; + +public class ExternalPrimaryStorageCanonicalEvent { + public static final String ADDON_INFO_CHANGED_PATH = "/externalPrimaryStorage/addonInfo/changed"; + + public static class AddonInfoChangedData { + private String uuid; + private String addonInfo; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getAddonInfo() { + return addonInfo; + } + + public void setAddonInfo(String addonInfo) { + this.addonInfo = addonInfo; + } + } +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java index a8e00d3a0ad..974704efcd9 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java @@ -4,10 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.Platform; import org.zstack.core.asyncbatch.While; -import org.zstack.core.cloudbus.CloudBus; -import org.zstack.core.cloudbus.CloudBusCallBack; -import org.zstack.core.cloudbus.EventCallback; -import org.zstack.core.cloudbus.EventFacade; +import org.zstack.core.cloudbus.*; import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; @@ -52,6 +49,7 @@ import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; +import javax.persistence.Tuple; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -128,11 +126,46 @@ public void beforeDeliveryMessage(Message msg) { return false; }, new APIAddPrimaryStorageEvent()); - evtf.on(PrimaryStorageCanonicalEvent.PRIMARY_STORAGE_DELETED_PATH, new EventCallback() { + bus.subscribeEvent(e -> { + if (e instanceof APIUpdateExternalPrimaryStorageEvent) { + APIUpdateExternalPrimaryStorageEvent evt = (APIUpdateExternalPrimaryStorageEvent) e; + if (evt.isSuccess() && evt.getInventory() != null) { + logger.debug("receive event, update external ps controller config."); + PrimaryStorageControllerSvc controller = controllers.get(evt.getInventory().getUuid()); + ExternalPrimaryStorageVO vo = dbf.findByUuid(evt.getInventory().getUuid(), ExternalPrimaryStorageVO.class); + if (controller != null) { + controller.syncConfig(vo.getConfig()); + controller.syncAddonInfo(vo.getAddonInfo()); + } else { + saveControllerIfNeed(vo); + } + } + } + + return false; + }, new APIUpdateExternalPrimaryStorageEvent()); + + evtf.on(PrimaryStorageCanonicalEvent.PRIMARY_STORAGE_DELETED_PATH, new EventCallback() { @Override - protected void run(Map tokens, Object data) { - PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData d = (PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData) data; - MultiNodeSingleFlightImpl.unregister(d.getPrimaryStorageUuid()); + protected void run(Map tokens, PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData data) { + MultiNodeSingleFlightImpl.unregister(data.getPrimaryStorageUuid()); + } + }); + evtf.on(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, new EventCallback() { + @Override + protected void run(Map tokens, ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data) { + if (evtf.isFromThisManagementNode(tokens)) { + return; + } + + PrimaryStorageControllerSvc controller = controllers.get(data.getUuid()); + Tuple t = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.config, ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, data.getUuid()) + .findTuple(); + if (controller != null) { + controller.syncConfig(t.get(0, String.class)); + controller.syncAddonInfo(t.get(1, String.class)); + } } }); diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java index 0881cefa572..591f7c8982e 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java @@ -83,11 +83,11 @@ private void handle(APIDiscoverExternalPrimaryStorageMsg msg) { if (msg.getIdentity() != null) { ExternalPrimaryStorageSvcBuilder builder = pluginRgty.getExtensionFromMap(msg.getIdentity(), ExternalPrimaryStorageSvcBuilder.class); - builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(msg) { + builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(msg) { @Override - public void success(LinkedHashMap addonInfo) { + public void success(AddonInfo addonInfo) { inventory.setIdentity(msg.getIdentity()); - inventory.setAddonInfo(addonInfo); + inventory.setAddonInfo(JSONObjectUtil.rehashObject(addonInfo, LinkedHashMap.class)); event.setInventory(inventory); bus.publish(event); } @@ -102,11 +102,11 @@ public void fail(ErrorCode errorCode) { } new While<>(pluginRgty.getExtensionList(ExternalPrimaryStorageSvcBuilder.class)).each((builder, comp) -> { - builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(comp) { + builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(comp) { @Override - public void success(LinkedHashMap addonInfo) { + public void success(AddonInfo addonInfo) { inventory.setIdentity(builder.getIdentity()); - inventory.setAddonInfo(addonInfo); + inventory.setAddonInfo(JSONObjectUtil.rehashObject(addonInfo, LinkedHashMap.class)); event.setInventory(inventory); comp.allDone(); } diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java index 54b59efd08f..cd893b5c265 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java @@ -13,9 +13,12 @@ import org.zstack.header.apimediator.ApiMessageInterceptionException; import org.zstack.header.apimediator.ApiMessageInterceptor; import org.zstack.header.apimediator.StopRoutingException; +import org.json.JSONException; +import org.json.JSONObject; import org.zstack.header.cluster.ClusterVO; import org.zstack.header.cluster.ClusterVO_; import org.zstack.header.message.APIMessage; +import org.zstack.header.storage.addon.primary.APIAddExternalPrimaryStorageMsg; import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO; import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO_; import org.zstack.header.storage.primary.*; @@ -76,12 +79,27 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APICreateVolumeSnapshotGroupMsg) msg); } else if (msg instanceof APIAddStorageProtocolMsg) { validate((APIAddStorageProtocolMsg) msg); + } else if (msg instanceof APIAddExternalPrimaryStorageMsg) { + validate((APIAddExternalPrimaryStorageMsg) msg); } setServiceId(msg); return msg; } + private void validate(APIAddExternalPrimaryStorageMsg msg) { + String config = msg.getConfig(); + if (config == null || config.trim().isEmpty()) { + return; + } + try { + new JSONObject(config); + } catch (JSONException e) { + throw new ApiMessageInterceptionException(argerr( + "config is not a valid JSON object: %s", e.getMessage())); + } + } + private void validate(APIAddStorageProtocolMsg msg) { if (Q.New(PrimaryStorageOutputProtocolRefVO.class) .eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, msg.getUuid()) diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java index 7488f23e178..ee9b616a0e0 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java @@ -72,6 +72,7 @@ public class VolumeSnapshotManagerImpl extends AbstractService implements VolumeBeforeExpungeExtensionPoint, ResourceOwnerAfterChangeExtensionPoint, ReportQuotaExtensionPoint, + VolumeSnapshotDBSyncExtensionPoint, AfterReimageVmInstanceExtensionPoint, VmJustBeforeDeleteFromDbExtensionPoint, VolumeJustBeforeDeleteFromDbExtensionPoint, @@ -679,6 +680,12 @@ private void rollbackSnapshot(String uuid) { return; } + if (vo.getStatus() == VolumeSnapshotStatus.Ready) { + logger.warn(String.format("volume snapshot[uuid:%s] has been marked Ready, skip rollback to keep control plane metadata consistent with hypervisor snapshot", + uuid)); + return; + } + dbf.getEntityManager().remove(vo); String sql = "delete from AccountResourceRefVO where resourceUuid = :vsUuid and resourceType = 'VolumeSnapshotVO'"; @@ -696,6 +703,42 @@ private void rollbackSnapshot(String uuid) { } } + @Override + @Transactional + public VolumeSnapshotInventory syncVolumeSnapshotDBAfterTakeSnapshot(VolumeInventory volume, + VolumeSnapshotInventory snapshot, + String volumeNewInstallPath) { + if (volumeNewInstallPath != null) { + VolumeVO latestVol = dbf.findByUuid(volume.getUuid(), VolumeVO.class); + latestVol.setInstallPath(volumeNewInstallPath); + dbf.update(latestVol); + } + + VolumeSnapshotVO svo = dbf.findByUuid(snapshot.getUuid(), VolumeSnapshotVO.class); + if (svo == null) { + return null; + } + + svo.setType(snapshot.getType()); + svo.setPrimaryStorageUuid(snapshot.getPrimaryStorageUuid()); + svo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); + svo.setStatus(VolumeSnapshotStatus.Ready); + svo.setSize(snapshot.getSize()); + if (snapshot.getFormat() != null) { + svo.setFormat(snapshot.getFormat()); + } + svo = dbf.updateAndRefresh(svo); + + markSnapshotTreeCompleted(VolumeSnapshotInventory.valueOf(svo)); + if (svo.getParentUuid() == null) { + VolumeSnapshotReferenceUtils.updateReferenceAfterFirstSnapshot(svo); + } + + logger.debug(String.format("synced volume snapshot[uuid:%s] metadata after hypervisor snapshot succeeded", + svo.getUuid())); + return VolumeSnapshotInventory.valueOf(svo); + } + private void handle(final AskVolumeSnapshotStructMsg msg) { AskVolumeSnapshotStructReply reply = new AskVolumeSnapshotStructReply(); CreateVolumeSnapshotMsg cmsg = new CreateVolumeSnapshotMsg(); @@ -990,33 +1033,14 @@ public void run(MessageReply reply) { done(new FlowDoneHandler(msg) { @Override public void handle(Map data) { - markSnapshotTreeCompleted(snapshot); - if (volumeNewInstallPath != null) { - vol.setInstallPath(volumeNewInstallPath); - dbf.update(vol); - } - - VolumeSnapshotVO svo = dbf.findByUuid(snapshot.getUuid(), VolumeSnapshotVO.class); - svo.setType(snapshot.getType()); - svo.setPrimaryStorageUuid(snapshot.getPrimaryStorageUuid()); - svo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); - svo.setStatus(VolumeSnapshotStatus.Ready); - svo.setSize(snapshot.getSize()); - if (snapshot.getFormat() != null) { - svo.setFormat(snapshot.getFormat()); - } - svo = dbf.updateAndRefresh(svo); - - if (struct.isNewChain()) { - VolumeSnapshotReferenceUtils.updateReferenceAfterFirstSnapshot(svo); - } + VolumeSnapshotInventory sp = syncVolumeSnapshotDBAfterTakeSnapshot( + vol.toInventory(), snapshot, volumeNewInstallPath); new FireSnapShotCanonicalEvent(). fireSnapShotStatusChangedEvent( VolumeSnapshotStatus.valueOf(snapshot.getStatus()), - VolumeSnapshotInventory.valueOf(svo)); + sp); - VolumeSnapshotInventory sp = svo.toInventory(); callExtensionPoints(sp); ret.setInventory(sp); @@ -1033,8 +1057,18 @@ private void callExtensionPoints(VolumeSnapshotInventory sp) { error(new FlowErrorHandler(msg) { @Override public void handle(ErrorCode errCode, Map data) { - if (struct != null) { - rollbackSnapshot(struct.getCurrent().getUuid()); + if (snapshot != null) { + syncVolumeSnapshotDBAfterTakeSnapshot(vol.toInventory(), snapshot, volumeNewInstallPath); + logger.warn(String.format("volume snapshot[uuid:%s] has been created on primary storage, keep database record for recovery after error: %s", + snapshot.getUuid(), errCode)); + } else if (struct != null) { + String snapshotUuid = struct.getCurrent().getUuid(); + if (getCurrentSnapshotStatus(snapshotUuid) == VolumeSnapshotStatus.Ready) { + logger.warn(String.format("volume snapshot[uuid:%s] has been marked Ready, keep database record after error: %s", + snapshotUuid, errCode)); + } else { + rollbackSnapshot(snapshotUuid); + } } ret.setError(errCode); bus.reply(msg, ret); @@ -1044,6 +1078,13 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private VolumeSnapshotStatus getCurrentSnapshotStatus(String uuid) { + return Q.New(VolumeSnapshotVO.class) + .select(VolumeSnapshotVO_.status) + .eq(VolumeSnapshotVO_.uuid, uuid) + .findValue(); + } + private void handle(MarkRootVolumeAsSnapshotMsg msg) { final MarkRootVolumeAsSnapshotReply ret = new MarkRootVolumeAsSnapshotReply(); VolumeInventory vol = msg.getVolume(); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java index 49662466a46..30069bda97e 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java @@ -83,6 +83,14 @@ public static String getVolumeInstallUrlBackingOtherVolume(String volumeUuid) { return null; } + // use getVolumeAllSnapshotsReferencedOtherVolumes or isVolumeDirectlyReferenceByOthers + @Deprecated + public static List getVolumeSnapshotsReferencedByOtherVolumes(String volumeUuid) { + return getVolumeReferenceRef(volumeUuid).stream() + .map(VolumeSnapshotReferenceVO::getVolumeSnapshotInstallUrl).distinct() + .collect(Collectors.toList()); + } + // get volume snapshotUuids referenced by other volumes directly or indirectly // FIXME split different primary storage snapshot reference public static Set getVolumeAllSnapshotsReferencedByOtherVolumes(String volumeUuid) { diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy new file mode 100644 index 00000000000..c940a7c6d46 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy @@ -0,0 +1,117 @@ +package org.zstack.test.integration.kvm.host + +import org.springframework.http.HttpEntity +import org.zstack.header.Constants +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.errorcode.SysErrors +import org.zstack.header.rest.JsonAsyncRESTCallback +import org.zstack.header.rest.RESTFacade +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.sdk.HostInventory +import org.zstack.test.integration.kvm.Env +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +class HostAgentRestartDrainCase extends SubCase { + EnvSpec env + HostInventory host + RESTFacade restf + + static class StuckCall { + CountDownLatch entered = new CountDownLatch(1) + CountDownLatch release = new CountDownLatch(1) + CountDownLatch done = new CountDownLatch(1) + AtomicReference received = new AtomicReference<>() + } + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = Env.noVmEnv() + } + + @Override + void test() { + env.create { + host = env.inventoryByName("kvm") + restf = bean(RESTFacade.class) + testFirstConnectDrainsStaleCalls() + testNoFirstConnectKeepsStaleCalls() + testDifferentResourceUuidIsolated() + } + } + + @Override + void clean() { + env.delete() + } + + private StuckCall sendStuckCall(String resourceUuid) { + StuckCall h = new StuckCall() + def stuckPath = "/test/stuck/${UUID.randomUUID()}" + env.simulator(stuckPath) { + h.entered.countDown() + h.release.await(120, TimeUnit.SECONDS) + return new KVMAgentCommands.AgentResponse() + } + + Map headers = [(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID): resourceUuid] + restf.asyncJsonPost(restf.makeUrl(stuckPath), "{}", headers, new JsonAsyncRESTCallback(null) { + @Override void fail(ErrorCode err) { h.received.set(err); h.done.countDown() } + @Override void success(String ret) { h.done.countDown() } + @Override Class getReturnClass() { return String.class } + }, TimeUnit.MINUTES, 30) + assert h.entered.await(5, TimeUnit.SECONDS) + return h + } + + private void overrideConnectResponse(boolean firstConnect, long agentStartTimeMillis) { + env.afterSimulator(KVMConstant.KVM_CONNECT_PATH) { KVMAgentCommands.ConnectResponse rsp, HttpEntity e -> + rsp.firstConnect = firstConnect + rsp.agentStartTimeMillis = agentStartTimeMillis + return rsp + } + } + + void testFirstConnectDrainsStaleCalls() { + StuckCall h = sendStuckCall(host.uuid) + overrideConnectResponse(true, System.currentTimeMillis() + 70_000L) + reconnectHost { + uuid = host.uuid + } + assert h.done.await(5, TimeUnit.SECONDS) + assert h.received.get() != null + assert h.received.get().code == SysErrors.OPERATION_ERROR.toString() + h.release.countDown() + } + + void testNoFirstConnectKeepsStaleCalls() { + StuckCall h = sendStuckCall(host.uuid) + overrideConnectResponse(false, System.currentTimeMillis() + 70_000L) + reconnectHost { + uuid = host.uuid + } + assert !h.done.await(500, TimeUnit.MILLISECONDS) + h.release.countDown() + } + + void testDifferentResourceUuidIsolated() { + StuckCall h = sendStuckCall("unrelated-resource-uuid") + overrideConnectResponse(true, System.currentTimeMillis() + 70_000L) + reconnectHost { + uuid = host.uuid + } + assert !h.done.await(500, TimeUnit.MILLISECONDS) + h.release.countDown() + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy index eaf774fb84f..87a06e15abd 100644 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy @@ -16,6 +16,11 @@ import org.zstack.utils.gson.JSONObjectUtil import org.zstack.header.network.l2.BatchCheckNetworkPhysicalInterfaceMsg import org.zstack.header.network.l2.BatchCheckNetworkPhysicalInterfaceReply import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.cloudbus.ResourceDestinationMaker +import org.zstack.compute.host.HostTrackImpl + +import static org.mockito.Mockito.mock +import static org.mockito.Mockito.when class HostConnectedTimeCase extends SubCase { @@ -34,6 +39,7 @@ class HostConnectedTimeCase extends SubCase { @Override void test() { env.create { + testSkipTrackIfNotManagedByUs() testHostConnectedTime() } } @@ -43,6 +49,31 @@ class HostConnectedTimeCase extends SubCase { env.delete() } + /** + * Test for ZSTAC-76341: check isManagedByUs before trackHost to prevent dual MN tracking + */ + void testSkipTrackIfNotManagedByUs() { + HostInventory host = env.inventoryByName("kvm") as HostInventory + HostTrackImpl tracker = bean(HostTrackImpl.class) + + ResourceDestinationMaker originalDestMaker = tracker.@destMaker + + ResourceDestinationMaker mockDestMaker = mock(ResourceDestinationMaker.class) + when(mockDestMaker.isManagedByUs(host.uuid)).thenReturn(false) + tracker.@destMaker = mockDestMaker + + tracker.untrackHost(host.uuid) + assert !tracker.trackers.containsKey(host.uuid) + + tracker.trackHost(host.uuid) + assert !tracker.trackers.containsKey(host.uuid) : "Host should NOT be tracked when isManagedByUs returns false" + + tracker.@destMaker = originalDestMaker + + tracker.trackHost(host.uuid) + assert tracker.trackers.containsKey(host.uuid) : "Host should be tracked when isManagedByUs returns true" + } + void testHostConnectedTime() { HostInventory host = env.inventoryByName("kvm") diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy new file mode 100644 index 00000000000..0340abb3109 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy @@ -0,0 +1,223 @@ +package org.zstack.test.integration.kvm.vm + +import org.springframework.http.HttpEntity +import org.zstack.compute.vm.VmGlobalConfig +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.flat.FlatNetworkServiceConstant +import org.zstack.network.service.userdata.UserdataConstant +import org.zstack.sdk.GlobalConfigInventory +import org.zstack.sdk.ImageInventory +import org.zstack.sdk.InstanceOfferingInventory +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +class VmPmuConfigCase extends SubCase { + EnvSpec env + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(2) + cpu = 1 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image_x86" + architecture = "x86_64" + url = "http://zstack.org/download/test.qcow2" + } + + image { + name = "image_arm64" + architecture = "aarch64" + url = "http://zstack.org/download/test-arm.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster_x86" + hypervisorType = "KVM" + architecture = "x86_64" + + kvm { + name = "kvm_x86" + managementIp = "localhost" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + cluster { + name = "cluster_arm64" + hypervisorType = "KVM" + architecture = "aarch64" + + kvm { + name = "kvm_arm64" + managementIp = "127.0.0.3" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + attachBackupStorage("sftp") + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + + service { + provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING + types = [NetworkServiceType.DHCP.toString(), EipConstant.EIP_NETWORK_SERVICE_TYPE, UserdataConstant.USERDATA_TYPE_STRING] + } + + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + } + } + } + + @Override + void test() { + env.create() + testPmuGlobalConfigExists() + testX86VmKeepsPmuEnabled() + testAarch64VmDisablesPmuByDefault() + testAarch64VmPmuResourceConfigOverride() + } + + void testPmuGlobalConfigExists() { + def configs = queryGlobalConfig { + conditions = ["category=${VmGlobalConfig.CATEGORY}", "name=${VmGlobalConfig.VM_PMU.name}"] + } + + assert configs.size() == 1 + assert (configs[0] as GlobalConfigInventory).defaultValue == "false" + } + + void testX86VmKeepsPmuEnabled() { + ImageInventory image = env.inventoryByName("image_x86") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-x86" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert cmd.pmu + } + + void testAarch64VmDisablesPmuByDefault() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-arm-default" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert !cmd.pmu + } + + void testAarch64VmPmuResourceConfigOverride() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + VmInstanceInventory vm = createVmInstance { + name = "pmu-arm-override" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } as VmInstanceInventory + + updateResourceConfig { + category = VmGlobalConfig.CATEGORY + name = VmGlobalConfig.VM_PMU.name + resourceUuid = vm.uuid + value = "true" + } + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + rebootVmInstance { + uuid = vm.uuid + } + } + + assert cmd.pmu + } + + KVMAgentCommands.StartVmCmd captureStartVmCmd(Closure trigger) { + KVMAgentCommands.StartVmCmd startCmd = null + env.afterSimulator(KVMConstant.KVM_START_VM_PATH) { rsp, HttpEntity e -> + startCmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.StartVmCmd.class) + return rsp + } + + trigger.call() + assert startCmd != null + env.cleanAfterSimulatorHandlers() + return startCmd + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy index 8e28757ba5a..e954020fa11 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -1,7 +1,12 @@ package org.zstack.test.integration.network.sdnController +import org.springframework.http.HttpEntity import org.zstack.core.db.DatabaseFacade +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.message.MessageReply import org.zstack.header.network.sdncontroller.SdnControllerVO +import org.zstack.header.network.sdncontroller.SdnControllerConstant +import org.zstack.sdnController.SdnControllerPingMsg import org.zstack.sdk.* import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant import org.zstack.sugonSdnController.controller.api.types.MacAddressesType @@ -16,6 +21,9 @@ import javax.persistence.TypedQuery; import org.springframework.http.ResponseEntity import org.springframework.http.HttpStatus +import javax.servlet.http.HttpServletRequest +import java.util.concurrent.atomic.AtomicInteger + class SugonSdnControllerCase extends SubCase { EnvSpec env @@ -37,6 +45,7 @@ class SugonSdnControllerCase extends SubCase { void test() { env.create { dbf = bean(DatabaseFacade.class) + testTfPortSyncTriggeredByPingSuccess() testTfApi() } } @@ -46,6 +55,53 @@ class SugonSdnControllerCase extends SubCase { env.delete() } + void testTfPortSyncTriggeredByPingSuccess() { + String sql = "select sdn" + + " from SdnControllerVO sdn" + + " where sdn.vendorType = :vendorType"; + TypedQuery q = dbf.getEntityManager().createQuery(sql, SdnControllerVO.class); + q.setParameter("vendorType", SugonSdnControllerConstant.TF_CONTROLLER); + SdnControllerVO sdn = q.getResultList().get(0); + AtomicInteger syncCount = new AtomicInteger(0) + + env.simulator(TfCommands.TF_CREATE_VMI) { HttpServletRequest req, HttpEntity e, EnvSpec spec -> + if (req.method == "GET") { + syncCount.incrementAndGet() + return '{"virtual-machine-interfaces":[]}' + } + + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = TfCommands.TEST_VMI_UUID + rsp.uuid = TfCommands.TEST_VMI_UUID + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.setParent(project) + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + sendTfPing(sdn.uuid) + retryInSecs { + assert syncCount.get() == 1 + } + + sleep(500) + sendTfPing(sdn.uuid) + assert syncCount.get() == 1 + } + + void sendTfPing(String sdnControllerUuid) { + CloudBus bus = bean(CloudBus.class) + SdnControllerPingMsg msg = new SdnControllerPingMsg() + msg.sdnControllerUuid = sdnControllerUuid + bus.makeTargetServiceIdByResourceUuid(msg, SdnControllerConstant.SERVICE_ID, sdnControllerUuid) + MessageReply reply = bus.call(msg) + assert reply.success + } + void testTfApi() { def zone = env.inventoryByName("zone") as ZoneInventory def cluster1 = env.inventoryByName("cluster1") as ClusterInventory diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy new file mode 100644 index 00000000000..27123046860 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy @@ -0,0 +1,258 @@ +package org.zstack.test.integration.networkservice.provider.flat.eip + +import org.springframework.http.HttpEntity +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.eip.EipVO +import org.zstack.network.service.flat.FlatEipBackend +import org.zstack.network.service.flat.FlatNetworkServiceConstant +import org.zstack.network.service.userdata.UserdataConstant +import org.zstack.sdk.AttachEipAction +import org.zstack.sdk.EipInventory +import org.zstack.sdk.StopVmInstanceAction +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.HttpError +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class AttachEipRaceCase extends SubCase { + EnvSpec env + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(8) + cpu = 4 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image" + url = "http://zstack.org/download/test.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster" + hypervisorType = "KVM" + + kvm { + name = "kvm" + managementIp = "localhost" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + attachL2Network("l2-pub") + } + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + + service { + provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING + types = [NetworkServiceType.DHCP.toString(), EipConstant.EIP_NETWORK_SERVICE_TYPE, UserdataConstant.USERDATA_TYPE_STRING] + } + + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + + l2NoVlanNetwork { + name = "l2-pub" + physicalInterface = "eth1" + + l3Network { + name = "pubL3" + + ip { + startIp = "11.168.100.10" + endIp = "11.168.100.100" + netmask = "255.255.255.0" + gateway = "11.168.100.1" + } + } + } + + attachBackupStorage("sftp") + + eip { + name = "eip-race" + useVip("pubL3") + } + + eip { + name = "eip-fail" + useVip("pubL3") + } + } + + vm { + name = "vm-race" + useImage("image") + useL3Networks("l3") + useInstanceOffering("instanceOffering") + } + + vm { + name = "vm-fail" + useImage("image") + useL3Networks("l3") + useInstanceOffering("instanceOffering") + } + } + } + + @Override + void test() { + env.create { + testStopVmDuringAttachReleasesEip() + testAttachFailureRollsBackEarlyBinding() + } + } + + void testStopVmDuringAttachReleasesEip() { + EipInventory eip = env.inventoryByName("eip-race") as EipInventory + VmInstanceInventory vm = env.inventoryByName("vm-race") as VmInstanceInventory + String nicUuid = vm.vmNics[0].uuid + String guestIp = vm.vmNics[0].ip + String sessionId = adminSession() + + CountDownLatch applyEntered = new CountDownLatch(1) + CountDownLatch releaseApply = new CountDownLatch(1) + CountDownLatch deleteCalled = new CountDownLatch(1) + FlatEipBackend.ApplyEipCmd applyCmd = null + FlatEipBackend.DeleteEipCmd deleteCmd = null + + env.afterSimulator(FlatEipBackend.APPLY_EIP_PATH) { rsp, HttpEntity entity -> + FlatEipBackend.ApplyEipCmd cmd = json(entity.getBody(), FlatEipBackend.ApplyEipCmd.class) + if (cmd.eip.eipUuid == eip.uuid) { + applyCmd = cmd + applyEntered.countDown() + assert releaseApply.await(30, TimeUnit.SECONDS) + } + return rsp + } + + env.afterSimulator(FlatEipBackend.DELETE_EIP_PATH) { rsp, HttpEntity entity -> + FlatEipBackend.DeleteEipCmd cmd = json(entity.getBody(), FlatEipBackend.DeleteEipCmd.class) + if (cmd.eip.eipUuid == eip.uuid) { + deleteCmd = cmd + deleteCalled.countDown() + } + return rsp + } + + Throwable attachFailure = null + Thread attachThread = Thread.start { + try { + AttachEipAction action = new AttachEipAction() + action.eipUuid = eip.uuid + action.vmNicUuid = nicUuid + action.sessionId = sessionId + def result = action.call() + if (result.error != null) { + attachFailure = new AssertionError(result.error.toString()) + } + } catch (Throwable t) { + attachFailure = t + } + } + + assert applyEntered.await(30, TimeUnit.SECONDS) + try { + retryInSecs { + EipVO vo = dbFindByUuid(eip.uuid, EipVO.class) + assert vo.vmNicUuid == nicUuid + assert vo.guestIp == guestIp + } + } finally { + releaseApply.countDown() + } + + attachThread.join(30000) + + assert !attachThread.isAlive() + assert attachFailure == null + assert applyCmd != null + + StopVmInstanceAction action = new StopVmInstanceAction() + action.uuid = vm.uuid + action.sessionId = sessionId + def result = action.call() + + assert result.error == null + assert deleteCalled.await(30, TimeUnit.SECONDS) + assert deleteCmd != null + assert deleteCmd.eip.eipUuid == eip.uuid + } + + void testAttachFailureRollsBackEarlyBinding() { + EipInventory eip = env.inventoryByName("eip-fail") as EipInventory + VmInstanceInventory vm = env.inventoryByName("vm-fail") as VmInstanceInventory + + env.afterSimulator(FlatEipBackend.APPLY_EIP_PATH) { rsp, HttpEntity entity -> + FlatEipBackend.ApplyEipCmd cmd = json(entity.getBody(), FlatEipBackend.ApplyEipCmd.class) + if (cmd.eip.eipUuid == eip.uuid) { + throw new HttpError(500, "on purpose") + } + return rsp + } + + AttachEipAction action = new AttachEipAction() + action.eipUuid = eip.uuid + action.vmNicUuid = vm.vmNics[0].uuid + action.sessionId = adminSession() + def result = action.call() + + assert result.error != null + retryInSecs { + EipVO vo = dbFindByUuid(eip.uuid, EipVO.class) + assert vo.vmNicUuid == null + assert vo.guestIp == null + } + } + + @Override + void clean() { + env.delete() + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy new file mode 100644 index 00000000000..8e71b93e7fa --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy @@ -0,0 +1,183 @@ +package org.zstack.test.integration.networkservice.provider.virtualrouter.portforwarding + +import org.zstack.core.db.Q +import org.zstack.header.errorcode.SysErrors +import org.zstack.network.service.portforwarding.PortForwardingProtocolType +import org.zstack.network.service.portforwarding.PortForwardingRuleVO +import org.zstack.network.service.portforwarding.PortForwardingRuleVO_ +import org.zstack.sdk.CreatePortForwardingRuleAction +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.VipInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class ConcurrentCreatePortForwardingRuleCase extends SubCase { + EnvSpec env + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + zone { + name = "zone" + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "pubL3" + + ip { + startIp = "11.168.100.10" + endIp = "11.168.100.100" + netmask = "255.255.255.0" + gateway = "11.168.100.1" + } + } + } + } + } + } + + @Override + void test() { + env.create { + testConcurrentCreateSamePortForwardingRule() + } + } + + void testConcurrentCreateSamePortForwardingRule() { + L3NetworkInventory pubL3 = env.inventoryByName("pubL3") as L3NetworkInventory + VipInventory vip = createVip { + name = "vip" + l3NetworkUuid = pubL3.uuid + } as VipInventory + + int requestCount = 50 + String sessionId = adminSession() + Closure createRule = { String name, int vipPortStart, int vipPortEnd -> + CreatePortForwardingRuleAction action = new CreatePortForwardingRuleAction() + action.name = name + action.vipUuid = vip.uuid + action.vipPortStart = vipPortStart + action.vipPortEnd = vipPortEnd + action.privatePortStart = vipPortStart + action.privatePortEnd = vipPortEnd + action.protocolType = PortForwardingProtocolType.TCP.toString() + action.sessionId = sessionId + return action.call() + } + CountDownLatch ready = new CountDownLatch(requestCount) + CountDownLatch start = new CountDownLatch(1) + CountDownLatch done = new CountDownLatch(requestCount) + def results = new ConcurrentLinkedQueue() + def errors = new ConcurrentLinkedQueue() + + (0.. + Thread.start("create-same-port-forwarding-rule-${index}") { + try { + ready.countDown() + if (!start.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("not all createPortForwardingRule workers became ready within 10 seconds") + } + + results.add(createRule("pf-${index}", 8080, 8080)) + } catch (Throwable t) { + errors.add(t) + } finally { + done.countDown() + } + } + } + + assert ready.await(10, TimeUnit.SECONDS) : + "createPortForwardingRule workers did not all become ready: expected=${requestCount}" + start.countDown() + assert done.await(60, TimeUnit.SECONDS) : + "concurrent createPortForwardingRule requests did not finish within 60 seconds" + assert errors.isEmpty() : + "worker thread threw unexpected errors: ${errors.collect { it.message }.join('; ')}" + + def successes = results.findAll { it.error == null } + def failures = results.findAll { it.error != null } + assert successes.size() == 1 : + "same VIP/protocol/port create must succeed once: expectedSuccess=1, actualSuccess=${successes.size()}, results=${results.size()}" + assert failures.size() == requestCount - 1 : + "duplicate same VIP/protocol/port creates must fail: expectedFailures=${requestCount - 1}, actualFailures=${failures.size()}" + def formattedErrors = failures.collect { formatError(it.error) } + assert failures.every { isVipPortConflictError(it.error, 8080, 8080) } : + "duplicate failures should be VIP port conflict errors: errors=${formattedErrors}" + + def lowerBoundaryConflict = createRule("pf-lower-boundary-conflict", 8079, 8080) + assert lowerBoundaryConflict.error != null : + "closed VIP port range should conflict when new end touches existing start" + assert isVipPortConflictError(lowerBoundaryConflict.error, 8079, 8080) : + "lower boundary conflict should report VIP port conflict error: error=${lowerBoundaryConflict.error}" + + def upperBoundaryConflict = createRule("pf-upper-boundary-conflict", 8080, 8081) + assert upperBoundaryConflict.error != null : + "closed VIP port range should conflict when new start touches existing end" + assert isVipPortConflictError(upperBoundaryConflict.error, 8080, 8081) : + "upper boundary conflict should report VIP port conflict error: error=${upperBoundaryConflict.error}" + + def lowerAdjacent = createRule("pf-lower-adjacent", 8079, 8079) + assert lowerAdjacent.error == null : + "adjacent lower VIP port should not conflict with existing port 8080: error=${lowerAdjacent.error}" + + def upperAdjacent = createRule("pf-upper-adjacent", 8081, 8081) + assert upperAdjacent.error == null : + "adjacent upper VIP port should not conflict with existing port 8080: error=${upperAdjacent.error}" + + long sameRules = Q.New(PortForwardingRuleVO.class) + .eq(PortForwardingRuleVO_.vipUuid, vip.uuid) + .eq(PortForwardingRuleVO_.protocolType, PortForwardingProtocolType.TCP) + .eq(PortForwardingRuleVO_.vipPortStart, 8080) + .eq(PortForwardingRuleVO_.vipPortEnd, 8080) + .eq(PortForwardingRuleVO_.privatePortStart, 8080) + .eq(PortForwardingRuleVO_.privatePortEnd, 8080) + .count() + assert sameRules == 1L : + "database must not contain duplicate same VIP/protocol/port rules: expected=1, actual=${sameRules}, vipUuid=${vip.uuid}" + } + + boolean isVipPortConflictError(org.zstack.sdk.ErrorCode error, int vipPortStart, int vipPortEnd) { + return isPortForwardingOverlapError(error) || + isVipUsedPortRangeConflictError(error, vipPortStart, vipPortEnd) + } + + boolean isPortForwardingOverlapError(org.zstack.sdk.ErrorCode error) { + String details = error.details ?: "" + return error.code == SysErrors.INVALID_ARGUMENT_ERROR.toString() && + details.contains("vipStartPort") && + details.contains("vipEndPort") && + details.contains("uuid:") + } + + boolean isVipUsedPortRangeConflictError(org.zstack.sdk.ErrorCode error, int vipPortStart, int vipPortEnd) { + String details = error.details ?: "" + return error.code == SysErrors.OPERATION_ERROR.toString() && + details.contains("uuid:") && + details.contains(PortForwardingProtocolType.TCP.toString()) && + details.contains(vipPortStart.toString()) && + details.contains(vipPortEnd.toString()) + } + + String formatError(org.zstack.sdk.ErrorCode error) { + return "code=${error.code}, details=${error.details}" + } + + @Override + void clean() { + env.delete() + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 5fc8f14aa97..6f4fb8db734 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -1,15 +1,22 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity +import org.zstack.cbd.AddonInfo +import org.zstack.core.cloudbus.EventCallback +import org.zstack.core.cloudbus.EventFacade import org.zstack.core.db.Q import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ +import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO import org.zstack.header.storage.primary.PrimaryStorageHostRefVO import org.zstack.header.storage.primary.PrimaryStorageHostRefVO_ import org.zstack.header.storage.primary.PrimaryStorageStatus +import org.zstack.header.storage.primary.PrimaryStorageVO import org.zstack.cbd.MdsUri import org.zstack.kvm.KVMAgentCommands +import org.zstack.cbd.MdsStatus import org.zstack.sdk.* +import org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent import org.zstack.storage.primary.PrimaryStorageGlobalConfig import org.zstack.header.storage.primary.PrimaryStorageHostStatus import org.zstack.storage.volume.VolumeGlobalConfig @@ -23,6 +30,9 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger + /** * @author Xingwei Yu * @date 2024/4/19 10:09 @@ -35,6 +45,7 @@ class ZbsPrimaryStorageCase extends SubCase { DiskOfferingInventory diskOffering VolumeInventory vol, vol2 KVMHostInventory kvm + EventFacade evtf @Override void clean() { @@ -152,7 +163,7 @@ class ZbsPrimaryStorageCase extends SubCase { ps = env.inventoryByName("zbs-1") as PrimaryStorageInventory diskOffering = env.inventoryByName("diskOffering") as DiskOfferingInventory kvm = env.inventoryByName("kvm-1") as KVMHostInventory - + evtf = bean(EventFacade.class) testDefaultConfig() testUpdateExternalPrimaryStorage() testLifecycle() @@ -160,8 +171,10 @@ class ZbsPrimaryStorageCase extends SubCase { testMdsPing() testCheckHostStorageConnection() testNegativeScenario() + testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor() testDataVolumeNegativeScenario() testDecodeMdsUriWithSpecialPassword() + testMdsReconnectAfterMaximumPingFailures() } } @@ -171,8 +184,25 @@ class ZbsPrimaryStorageCase extends SubCase { clusterUuid = cluster.uuid } + Set heartbeatPools = Collections.synchronizedSet(new HashSet<>()) + AtomicBoolean checkStartedBeforeHeartbeatReady = new AtomicBoolean(false) + env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity e -> + ZbsStorageController.CreateVolumeCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd) + if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) { + heartbeatPools.add(cmd.logicalPool) + ZbsStorageController.CreateVolumeRsp createVolumeRsp = new ZbsStorageController.CreateVolumeRsp() + createVolumeRsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}" + return createVolumeRsp + } + + return rsp + } + env.afterSimulator(ZbsStorageController.CHECK_HOST_STORAGE_CONNECTION_PATH) { rsp, HttpEntity e -> ZbsStorageController.CheckHostStorageConnectionCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CheckHostStorageConnectionCmd) + if (!heartbeatPools.contains("lpool1")) { + checkStartedBeforeHeartbeatReady.set(true) + } ZbsStorageController.CheckHostStorageConnectionRsp checkHostStorageConnectionRsp = new ZbsStorageController.CheckHostStorageConnectionRsp() checkHostStorageConnectionRsp.error = "fake error" @@ -186,6 +216,10 @@ class ZbsPrimaryStorageCase extends SubCase { uuid = kvm.getUuid() } } + retryInSecs { + assert heartbeatPools.contains("lpool1") + assert !checkStartedBeforeHeartbeatReady.get() + } retryInSecs { assert Q.New(PrimaryStorageHostRefVO.class) @@ -315,6 +349,17 @@ class ZbsPrimaryStorageCase extends SubCase { } void testMdsPing() { + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data = null + long count = 0 + EventCallback callback = new EventCallback() { + @Override + protected void run(Map tokens, ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData d) { + data = d + count++ + } + } + + evtf.on(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, callback) PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(1) Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected @@ -322,54 +367,89 @@ class ZbsPrimaryStorageCase extends SubCase { def addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" + assert data == null env.afterSimulator(ZbsPrimaryStorageMdsBase.PING_PATH) { rsp, HttpEntity e -> def cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) - ZbsPrimaryStorageMdsBase.PingRsp pingRsp = new ZbsPrimaryStorageMdsBase.PingRsp() if (cmd.addr.equals("127.0.1.1")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } - return pingRsp + return rsp } - sleep(2000) + env.afterSimulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp, HttpEntity e -> + ZbsPrimaryStorageMdsBase.PingCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) + if (cmd.getAddr().equals("127.0.1.1")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } + + return rsp + } + + retryInSecs { + assert count == 1 + } addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected + assert data.uuid == ps.uuid env.afterSimulator(ZbsPrimaryStorageMdsBase.PING_PATH) { rsp, HttpEntity e -> def cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) - ZbsPrimaryStorageMdsBase.PingRsp pingRsp = new ZbsPrimaryStorageMdsBase.PingRsp() if (cmd.addr.equals("127.0.1.1")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } else if (cmd.addr.equals("127.0.1.2")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } else if (cmd.addr.equals("127.0.1.3")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } - return pingRsp + return rsp } - sleep(2000) + env.afterSimulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp, HttpEntity e -> + ZbsPrimaryStorageMdsBase.PingCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) + if (cmd.getAddr().equals("127.0.1.1")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } else if (cmd.getAddr().equals("127.0.1.2")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } else if (cmd.getAddr().equals("127.0.1.3")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } + + return rsp + } + + retryInSecs { + assert count == 2 + } addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" - assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Disconnected + retryInSecs { + assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Disconnected + } + assert data.uuid == ps.uuid env.cleanAfterSimulatorHandlers() - sleep(2000) + retryInSecs { + assert count >= 3 + } Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected @@ -451,6 +531,43 @@ class ZbsPrimaryStorageCase extends SubCase { } } + void testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor() { + long extPsCountBefore = Q.New(ExternalPrimaryStorageVO.class).count() + long primaryStorageCountBefore = Q.New(PrimaryStorageVO.class).count() + long protocolRefCountBefore = Q.New(PrimaryStorageOutputProtocolRefVO.class).count() + + expect(AssertionError.class) { + addExternalPrimaryStorage { + zoneUuid = zone.uuid + name = "zbs-bad-json" + identity = "zbs" + defaultOutputProtocol = "CBD" + config = "{this is not valid json" + url = "" + } + } + + long extPsCountAfter = Q.New(ExternalPrimaryStorageVO.class).count() + long primaryStorageCountAfter = Q.New(PrimaryStorageVO.class).count() + long protocolRefCountAfter = Q.New(PrimaryStorageOutputProtocolRefVO.class).count() + assert extPsCountAfter == extPsCountBefore : \ + "ExternalPrimaryStorageVO leaked despite marshal-time JSON rejection: " + + "before=${extPsCountBefore} after=${extPsCountAfter}" + assert primaryStorageCountAfter == primaryStorageCountBefore : \ + "PrimaryStorageVO leaked despite marshal-time JSON rejection: " + + "before=${primaryStorageCountBefore} after=${primaryStorageCountAfter}" + assert protocolRefCountAfter == protocolRefCountBefore : \ + "PrimaryStorageOutputProtocolRefVO leaked despite marshal-time JSON rejection: " + + "before=${protocolRefCountBefore} after=${protocolRefCountAfter}" + boolean stalePsByName = Q.New(ExternalPrimaryStorageVO.class) + .eq(ExternalPrimaryStorageVO_.name, "zbs-bad-json") + .isExists() + assert !stalePsByName : "stale ExternalPrimaryStorageVO[name=zbs-bad-json] reached DB despite interceptor rejection" + + def psList = queryPrimaryStorage {} as List + assert psList != null : "queryPrimaryStorage threw / returned null after attempted bad-JSON Add" + } + void testDataVolumeNegativeScenario() { env.simulator(ZbsStorageController.CREATE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> def rsp = new ZbsStorageController.CreateVolumeRsp() @@ -502,6 +619,57 @@ class ZbsPrimaryStorageCase extends SubCase { assert uri.password == specialPassword } + void testMdsReconnectAfterMaximumPingFailures() { + env.cleanSimulatorAndMessageHandlers() + Integer originalPingInterval = PrimaryStorageGlobalConfig.PING_INTERVAL.value().toInteger() + PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(1) + final Integer MDS_PING_RETRY_PER_CYCLE = ZbsConstants.MDS_PING_RETRY_PER_CYCLE + final Integer MDS_PING_FAIL_CYCLE_THRESHOLD = ZbsConstants.MDS_PING_FAIL_CYCLE_THRESHOLD + AtomicInteger pingFailureCount = new AtomicInteger(0) + Boolean reconnectTriggered = false + + env.simulator(ZbsPrimaryStorageMdsBase.PING_PATH) { HttpEntity e, EnvSpec spec -> + ZbsPrimaryStorageMdsBase.PingCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) + ZbsPrimaryStorageMdsBase.PingRsp rsp = new ZbsPrimaryStorageMdsBase.PingRsp() + if (cmd.getAddr().equals("127.0.1.1") && !reconnectTriggered) { + rsp.success = false + rsp.error = "Mds ping failed on purpose" + pingFailureCount.incrementAndGet() + return rsp + } + return rsp + + } + + env.simulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { HttpEntity e, EnvSpec spec -> + ZbsPrimaryStorageMdsBase.SyncMetadataCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.SyncMetadataCmd.class) + ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp = new ZbsPrimaryStorageMdsBase.SyncMetadataRsp() + rsp.setExternalAddr(cmd.getAddr()) + if (cmd.getAddr().equals("127.0.1.1") && pingFailureCount.intValue().equals(MDS_PING_RETRY_PER_CYCLE * MDS_PING_FAIL_CYCLE_THRESHOLD)) { + reconnectTriggered = true + } + return rsp + } + + sleep((MDS_PING_RETRY_PER_CYCLE * MDS_PING_FAIL_CYCLE_THRESHOLD + 1) * 1000 + 500) + retryInSecs { + assert reconnectTriggered + } + retryInSecs { + AddonInfo addonInfo = JSONObjectUtil.toObject( + Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, ps.uuid) + .findValue() + .toString(), + AddonInfo.class + ) + assert MdsStatus.Connected.equals(addonInfo.getMdsInfos().find { it.addr.equals("127.0.1.1") }.getStatus()) + } + + PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(originalPingInterval) + env.cleanAfterSimulatorHandlers() + } void deleteVolume(String volUuid) { deleteDataVolume { diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy new file mode 100644 index 00000000000..46979001b31 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy @@ -0,0 +1,205 @@ +package org.zstack.test.integration.storage.snapshot + +import org.springframework.http.HttpEntity +import org.zstack.core.Platform +import org.zstack.core.componentloader.PluginRegistry +import org.zstack.core.db.Q +import org.zstack.header.core.Completion +import org.zstack.header.core.workflow.Flow +import org.zstack.header.storage.snapshot.ConsistentType +import org.zstack.header.storage.snapshot.CreateVolumesSnapshotOverlayInnerMsg +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmReply +import org.zstack.header.storage.snapshot.VolumeSnapshotConstant +import org.zstack.header.storage.snapshot.VolumeSnapshotCreationExtensionPoint +import org.zstack.header.storage.snapshot.VolumeSnapshotStatus +import org.zstack.header.storage.snapshot.VolumeSnapshotTreeStatus +import org.zstack.header.storage.snapshot.VolumeSnapshotTreeVO +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory +import org.zstack.header.storage.snapshot.VolumeSnapshotVO +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_ +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupInventory +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_ +import org.zstack.header.volume.CreateVolumeSnapshotGroupMessage +import org.zstack.header.volume.VolumeVO +import org.zstack.header.volume.VolumeVO_ +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.ldap.Env +import org.zstack.test.integration.storage.StorageTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.gson.JSONObjectUtil + +class CreateSnapshotRollbackAfterDataPlaneSuccessCase extends SubCase { + EnvSpec env + VmInstanceInventory vm + + int takeSnapshotCount = 0 + String firstSnapshotInstallPath + String firstNewVolumeInstallPath + String secondVolumeInstallPath + String secondSnapshotInstallPath + String secondNewVolumeInstallPath + long snapshotSize = 1L + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(StorageTest.springSpec) + } + + @Override + void environment() { + env = Env.localStorageOneVmEnv() + } + + @Override + void test() { + env.create { + vm = env.inventoryByName("vm") as VmInstanceInventory + testKeepSnapshotMetadataWhenControlPlaneFailsAfterDataPlaneSuccess() + } + } + + void testKeepSnapshotMetadataWhenControlPlaneFailsAfterDataPlaneSuccess() { + String originalVolumeInstallPath = Q.New(VolumeVO.class) + .select(VolumeVO_.installPath) + .eq(VolumeVO_.uuid, vm.rootVolumeUuid) + .findValue() + + env.simulator(KVMConstant.KVM_TAKE_VOLUME_SNAPSHOT_PATH) { HttpEntity e, EnvSpec espec -> + KVMAgentCommands.TakeSnapshotCmd cmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.TakeSnapshotCmd.class) + takeSnapshotCount++ + + if (takeSnapshotCount == 1) { + assert cmd.volumeInstallPath == originalVolumeInstallPath + firstSnapshotInstallPath = cmd.volumeInstallPath + firstNewVolumeInstallPath = cmd.installPath + } else if (takeSnapshotCount == 2) { + secondVolumeInstallPath = cmd.volumeInstallPath + secondSnapshotInstallPath = cmd.volumeInstallPath + secondNewVolumeInstallPath = cmd.installPath + } + + KVMAgentCommands.TakeSnapshotResponse rsp = new KVMAgentCommands.TakeSnapshotResponse() + rsp.newVolumeInstallPath = cmd.installPath + rsp.snapshotInstallPath = cmd.volumeInstallPath + rsp.size = snapshotSize + return rsp + } + + PluginRegistry pluginRegistry = bean(PluginRegistry.class) + List extensions = pluginRegistry.getExtensionList(VolumeSnapshotCreationExtensionPoint.class) + FailAfterSnapshotCreatedExtension failureExtension = new FailAfterSnapshotCreatedExtension() + extensions.add(failureExtension) + + try { + expectError { + createVolumeSnapshot { + name = "snapshot-fail-after-metadata-synced" + volumeUuid = vm.rootVolumeUuid + } + } + + assert takeSnapshotCount == 1 + + VolumeSnapshotVO firstSnapshot + retryInSecs(10, 1) { + firstSnapshot = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.volumeUuid, vm.rootVolumeUuid) + .find() + assert firstSnapshot != null + assert firstSnapshot.status == VolumeSnapshotStatus.Ready + } + + assert firstSnapshot.latest + assert firstSnapshot.primaryStorageInstallPath == firstSnapshotInstallPath + assert firstSnapshot.type == VolumeSnapshotConstant.HYPERVISOR_SNAPSHOT_TYPE.toString() + assert firstSnapshot.size == snapshotSize + + VolumeSnapshotTreeVO firstTree = dbFindByUuid(firstSnapshot.treeUuid, VolumeSnapshotTreeVO.class) + assert firstTree.current + assert firstTree.status == VolumeSnapshotTreeStatus.Completed + + assert Q.New(VolumeVO.class) + .select(VolumeVO_.installPath) + .eq(VolumeVO_.uuid, vm.rootVolumeUuid) + .findValue() == firstNewVolumeInstallPath + + VolumeSnapshotReferenceVO ref = Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.volumeUuid, vm.rootVolumeUuid) + .limit(1) + .find() + if (ref != null && ref.referenceInstallUrl == firstSnapshotInstallPath) { + assert ref.referenceUuid == firstSnapshot.uuid + assert ref.referenceType == VolumeSnapshotVO.class.simpleName + } + + org.zstack.sdk.VolumeSnapshotInventory secondSnapshotInv = createVolumeSnapshot { + name = "snapshot-after-recovery" + volumeUuid = vm.rootVolumeUuid + } as org.zstack.sdk.VolumeSnapshotInventory + + assert takeSnapshotCount == 2 + assert secondVolumeInstallPath == firstNewVolumeInstallPath + + VolumeSnapshotVO firstSnapshotAfterRecovery = dbFindByUuid(firstSnapshot.uuid, VolumeSnapshotVO.class) + VolumeSnapshotVO secondSnapshot = dbFindByUuid(secondSnapshotInv.uuid, VolumeSnapshotVO.class) + assert !firstSnapshotAfterRecovery.latest + assert secondSnapshot.latest + assert secondSnapshot.parentUuid == firstSnapshot.uuid + assert secondSnapshot.primaryStorageInstallPath == secondSnapshotInstallPath + assert secondSnapshot.status == VolumeSnapshotStatus.Ready + + assert Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.volumeUuid, vm.rootVolumeUuid) + .count() == 2 + assert Q.New(VolumeVO.class) + .select(VolumeVO_.installPath) + .eq(VolumeVO_.uuid, vm.rootVolumeUuid) + .findValue() == secondNewVolumeInstallPath + } finally { + extensions.remove(failureExtension) + } + } + + private static class FailAfterSnapshotCreatedExtension implements VolumeSnapshotCreationExtensionPoint { + boolean failAfterSnapshotCreated = true + + @Override + void afterVolumeSnapshotCreated(VolumeSnapshotInventory snapshot, Completion completion) { + if (failAfterSnapshotCreated) { + failAfterSnapshotCreated = false + completion.fail(Platform.operr("TEST.ERROR", "fail after KVM snapshot has been created")) + return + } + + completion.success() + } + + @Override + void afterVolumeLiveSnapshotGroupCreatedOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, TakeVolumesSnapshotOnKvmReply treply, Completion completion) { + completion.success() + } + + @Override + void afterVolumeLiveSnapshotGroupCreationFailsOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, TakeVolumesSnapshotOnKvmReply treply) { + } + + @Override + void afterVolumeSnapshotGroupCreated(VolumeSnapshotGroupInventory snapshotGroup, ConsistentType consistentType, Completion completion) { + completion.success() + } + + @Override + List beforeCreateVolumeSnapshotFlow(CreateVolumeSnapshotGroupMessage msg) { + return null + } + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy index 72cf1d6ebb8..e37785f7674 100644 --- a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy @@ -36,6 +36,7 @@ class CountGroupByCase extends SubCase { env.create { testCountGroupBy() testQuery() + testFieldQueryTotalWithNullableFirstField() testGroupLimit() testEmpty() } @@ -80,6 +81,13 @@ class CountGroupByCase extends SubCase { assert ret2.total == 102 } + void testFieldQueryTotalWithNullableFirstField() { + def ret = ZQL.fromString("query instanceoffering.description,uuid return with (total) limit 100").getSingleResult() as ZQLQueryReturn + assert ret.inventories.size() == 100 + assert ret.total == 102 + assert ret.inventories.every { it.description == null } + } + void testGroupLimit() { def ret = ZQL.fromString("count instanceoffering group by name,cpuNum limit 2").getSingleResult() as ZQLQueryReturn assert ret.inventoryCounts.size() == 2 diff --git a/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java new file mode 100644 index 00000000000..15216cb50bb --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java @@ -0,0 +1,43 @@ +package org.zstack.test.unittest.identity; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.zstack.header.identity.APIQueryAccountMsg; +import org.zstack.header.identity.RBACInfo; +import org.zstack.header.identity.rbac.RBAC; + +public class TestAPIQueryAccountMsgRBACCase { + + @Before + public void setUp() { + RBAC.permissions.clear(); + } + + @Test + public void testAPIQueryAccountMsgIsAdminOnly() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + boolean isAdminOnly = RBAC.isAdminOnlyAPI(APIQueryAccountMsg.class.getName()); + Assert.assertTrue( + "APIQueryAccountMsg should be admin-only to prevent privilege escalation, " + + "but it is currently accessible to normal IAM users via the wildcard normalAPIs pattern", + isAdminOnly + ); + } + + @Test + public void testAPIQueryAccountMsgNotInNormalAPIs() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + String apiName = APIQueryAccountMsg.class.getName(); + boolean isInNormalOnly = RBAC.permissions.stream() + .anyMatch(p -> p.getNormalAPIs().contains(apiName) && !p.getAdminOnlyAPIs().contains(apiName)); + Assert.assertFalse( + "APIQueryAccountMsg should NOT be accessible as a normal API", + isInNormalOnly + ); + } +} diff --git a/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java new file mode 100644 index 00000000000..2e0e06c7dfc --- /dev/null +++ b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java @@ -0,0 +1,25 @@ +package org.zstack.test.kvm; + +import org.junit.Assert; +import org.junit.Test; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.kvm.KVMHostUtils; + +public class KVMHostUtilsTest { + @Test + public void zstac86349_continueReconnectOnLibvirtSocketMaskSystemdTimeout() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl mask libvirtd.socket libvirtd-ro.socket libvirtd-admin.socket libvirtd-tls.socket libvirtd-tcp.socket failed! stderr: Failed to get properties: Failed to activate service 'org.freedesktop.systemd1': timed out (service_start_timeout=25000ms)"); + + Assert.assertTrue(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(true, error)); + } + + @Test + public void zstac86349_doNotContinueReconnectOnOtherAnsibleFailures() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl restart libvirtd failed! stderr: Job for libvirtd.service failed"); + + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + } +} diff --git a/utils/src/main/java/org/zstack/utils/ssh/SshShell.java b/utils/src/main/java/org/zstack/utils/ssh/SshShell.java index 80ea35c94ce..0c333fdbfbc 100755 --- a/utils/src/main/java/org/zstack/utils/ssh/SshShell.java +++ b/utils/src/main/java/org/zstack/utils/ssh/SshShell.java @@ -29,6 +29,14 @@ public class SshShell { private String privateKey; private int port = 22; private Boolean withSudo = true; + private int connectTimeoutSeconds = 10; + private int serverAliveIntervalSeconds = 5; + private int serverAliveCountMax = 2; + + private String sshTimeoutOptions() { + return String.format("-o ConnectTimeout=%d -o ServerAliveInterval=%d -o ServerAliveCountMax=%d", + connectTimeoutSeconds, serverAliveIntervalSeconds, serverAliveCountMax); + } private void checkParams() { DebugUtils.Assert(hostname != null && !hostname.trim().equals(""), "hostname cannot be null"); @@ -45,13 +53,13 @@ public SshResult runCommand(String cmd) { if (privateKey != null) { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, privateKey); - ssh = String.format("ssh -q -i %s -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no -p %s %s@%s '%s'", - tempPasswordFile.getAbsolutePath(), port, username, hostname, cmd); + ssh = String.format("ssh -q -i %s -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no %s -p %s %s@%s '%s'", + tempPasswordFile.getAbsolutePath(), sshTimeoutOptions(), port, username, hostname, cmd); } else { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, password); - ssh = String.format("sshpass -f%s ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -p %s %s@%s '%s'", - tempPasswordFile.getAbsolutePath(), port, username, hostname, cmd); + ssh = String.format("sshpass -f%s ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no %s -p %s %s@%s '%s'", + tempPasswordFile.getAbsolutePath(), sshTimeoutOptions(), port, username, hostname, cmd); } if (logger.isTraceEnabled()) { @@ -89,7 +97,7 @@ public SshResult runScript(String script) { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, privateKey); ssh = ln( - "ssh -q -i {0} -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no -p {1} -T {2}@{3} << 'EOF'", + "ssh -q -i {0} -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no {5} -p {1} -T {2}@{3} << 'EOF'", "s=`mktemp`", "cat << 'EOT' > $s", "{4}", @@ -99,12 +107,12 @@ public SshResult runScript(String script) { "rm -f $s", "exit $ret", "EOF" - ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script); + ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script, sshTimeoutOptions()); } else { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, password); ssh = ln( - "sshpass -f{0} ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -p {1} -T {2}@{3} << 'EOF'", + "sshpass -f{0} ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no {5} -p {1} -T {2}@{3} << 'EOF'", "s=`mktemp`", "cat << 'EOT' > $s", "{4}", @@ -114,7 +122,7 @@ public SshResult runScript(String script) { "rm -f $s", "exit $ret", "EOF" - ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script); + ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script, sshTimeoutOptions()); } if (logger.isTraceEnabled()) { @@ -209,4 +217,28 @@ public Boolean getWithSudo() { public void setWithSudo(Boolean withSudo) { this.withSudo = withSudo; } + + public int getConnectTimeoutSeconds() { + return connectTimeoutSeconds; + } + + public void setConnectTimeoutSeconds(int connectTimeoutSeconds) { + this.connectTimeoutSeconds = connectTimeoutSeconds; + } + + public int getServerAliveIntervalSeconds() { + return serverAliveIntervalSeconds; + } + + public void setServerAliveIntervalSeconds(int serverAliveIntervalSeconds) { + this.serverAliveIntervalSeconds = serverAliveIntervalSeconds; + } + + public int getServerAliveCountMax() { + return serverAliveCountMax; + } + + public void setServerAliveCountMax(int serverAliveCountMax) { + this.serverAliveCountMax = serverAliveCountMax; + } } \ No newline at end of file