<fix>[ha]: ZSTAC-84992 direct-pick HA pre-fence core to 5.4.10 (@@3)

VERSION

@@ -1,3 +1,3 @@
 MAJOR=5
 MINOR=4
-UPDATE=7
+UPDATE=10

compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java

@@ -52,6 +52,10 @@ public class VmGlobalConfig {
     @GlobalConfigValidation(validValues = {"true", "false"})
     @BindResourceConfig(value = {VmInstanceVO.class})
     public static GlobalConfig VM_PORT_OFF = new GlobalConfig(CATEGORY, "vmPortOff");
+    @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable PMU for VM, disabled by default on aarch64 to avoid Kunpeng-920 kernel panic")
+    @GlobalConfigValidation(validValues = {"true", "false"})
+    @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class})
+    public static GlobalConfig VM_PMU = new GlobalConfig(CATEGORY, "vm.pmu");
     @GlobalConfigValidation(validValues = {"true", "false"})
     @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class})
     public static GlobalConfig EMULATE_HYPERV = new GlobalConfig(CATEGORY, "emulateHyperV");

compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java

@@ -1004,16 +1004,38 @@ public void done(ErrorCodeList errorCodeList) {
 
                         logger.debug(String.format("HaStartVmJudger[%s] says the VM[uuid:%s, name:%s] is qualified for HA start, now we are starting it",
                                 judger.getClass(), self.getUuid(), self.getName()));
+                        String hostUuid = self.getHostUuid();
+                        String suspectHostUuid = StringUtils.trimToNull(hostUuid);
+                        String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid());
                         UpdateQuery sql = SQL.New(VmInstanceVO.class)
                                 .eq(VmInstanceVO_.uuid, self.getUuid())
                                 .set(VmInstanceVO_.state, VmInstanceState.Stopped)
                                 .set(VmInstanceVO_.hostUuid, null);
 
-                        if (self.getHostUuid() != null) {
-                            sql.set(VmInstanceVO_.lastHostUuid, self.getHostUuid());
+                        if (hostUuid != null) {
+                            sql.set(VmInstanceVO_.lastHostUuid, hostUuid);
                         }
 
                         sql.update();
+                        refreshVO();
+                        if (suspectHostUuid == null) {
+                            logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host is absent",
+                                    self.getUuid()));
+                        } else if (peerHostUuid == null) {
+                            logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because peer host is absent",
+                                    self.getUuid()));
+                        } else {
+                            Map<String, String> tokens = new HashMap<>();
+                            tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid);
+                            tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid);
+
+                            SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid());
+                            creator.inherent = true;
+                            creator.recreate = true;
+                            creator.ignoreIfExisting = true;
+                            creator.setTagByTokens(tokens);
+                            creator.create();
+                        }
 
                         startVm(msg, new Completion(msg, chain) {
                             @Override
@@ -8984,4 +9006,3 @@ public void run(MessageReply reply) {
         });
     }
 }
-

compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java

@@ -5,17 +5,26 @@
 import org.springframework.beans.factory.annotation.Configurable;
 import org.zstack.core.cloudbus.CloudBus;
 import org.zstack.core.cloudbus.CloudBusCallBack;
+import org.zstack.core.asyncbatch.While;
 import org.zstack.core.componentloader.PluginRegistry;
+import org.zstack.core.workflow.FlowChainBuilder;
+import org.zstack.header.core.Completion;
+import org.zstack.header.core.WhileDoneCompletion;
 import org.zstack.header.core.workflow.Flow;
+import org.zstack.header.core.workflow.FlowChain;
+import org.zstack.header.core.workflow.FlowDoneHandler;
+import org.zstack.header.core.workflow.FlowErrorHandler;
 import org.zstack.header.core.workflow.FlowRollback;
 import org.zstack.header.core.workflow.FlowTrigger;
+import org.zstack.header.core.workflow.NoRollbackFlow;
+import org.zstack.header.errorcode.ErrorCode;
+import org.zstack.header.errorcode.ErrorCodeList;
 import org.zstack.header.host.HostConstant;
 import org.zstack.header.message.MessageReply;
 import org.zstack.header.vm.*;
 import org.zstack.utils.Utils;
 import org.zstack.utils.logging.CLogger;
 
-import java.util.List;
 import java.util.Map;
 @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
 public class VmStartOnHypervisorFlow implements Flow {
@@ -26,34 +35,68 @@ public class VmStartOnHypervisorFlow implements Flow {
     @Autowired
     private PluginRegistry pluginRgty;
 
-    private final List<VmBeforeStartOnHypervisorExtensionPoint> exts = pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class);;
-
-    private void fireExtensions(VmInstanceSpec spec) {
-        for (VmBeforeStartOnHypervisorExtensionPoint ext : exts) {
-            ext.beforeStartVmOnHypervisor(spec);
-        }
-    }
-
     @Override
     public void run(final FlowTrigger chain, final Map data) {
         final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());
+        FlowChain fchain = FlowChainBuilder.newSimpleFlowChain();
+        fchain.setName(String.format("vm-start-on-hypervisor-vm-%s", spec.getVmInventory().getUuid()));
+        fchain.then(new NoRollbackFlow() {
+            @Override
+            public void run(FlowTrigger trigger, Map d) {
+                new While<>(pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class))
+                        .each((ext, comp) -> ext.beforeStartVmOnHypervisor(spec, new Completion(comp) {
+                            @Override
+                            public void success() {
+                                comp.done();
+                            }
 
-        fireExtensions(spec);
-
-        StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg();
-        msg.setVmSpec(spec);
-        bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid());
-        bus.send(msg, new CloudBusCallBack(chain) {
+                            @Override
+                            public void fail(ErrorCode errorCode) {
+                                comp.addError(errorCode);
+                                comp.done();
+                            }
+                        })).run(new WhileDoneCompletion(trigger) {
+                            @Override
+                            public void done(ErrorCodeList errorCodeList) {
+                                if (errorCodeList.getCauses().size() > 0) {
+                                    trigger.fail(errorCodeList.getCauses().get(0));
+                                } else {
+                                    trigger.next();
+                                }
+                            }
+                        });
+            }
+        });
+        fchain.then(new NoRollbackFlow() {
             @Override
-            public void run(MessageReply reply) {
-                if (reply.isSuccess()) {
-                    data.put(VmStartOnHypervisorFlow.class.getName(), true);
-                    chain.next();
-                } else {
-                    chain.fail(reply.getError());
-                }
+            public void run(FlowTrigger trigger, Map d) {
+                StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg();
+                msg.setVmSpec(spec);
+                bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid());
+                bus.send(msg, new CloudBusCallBack(trigger) {
+                    @Override
+                    public void run(MessageReply reply) {
+                        if (reply.isSuccess()) {
+                            data.put(VmStartOnHypervisorFlow.class.getName(), true);
+                            trigger.next();
+                        } else {
+                            trigger.fail(reply.getError());
+                        }
+                    }
+                });
             }
         });
+        fchain.done(new FlowDoneHandler(chain) {
+            @Override
+            public void handle(Map d) {
+                chain.next();
+            }
+        }).error(new FlowErrorHandler(chain) {
+            @Override
+            public void handle(ErrorCode errCode, Map d) {
+                chain.fail(errCode);
+            }
+        }).start();
     }
 
     @Override

compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java

@@ -310,4 +310,11 @@ public String desensitizeTag(SystemTag systemTag, String tag) {
     public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class);
 
     public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class);
+
+    public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid";
+    public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid";
+    public static PatternedSystemTag HA_PRE_FENCE_PENDING =
+            new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}",
+                    HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN,
+                    HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class);
 }

conf/db/upgrade/V5.4.10__schema.sql

@@ -0,0 +1,3 @@
+-- ZSTAC-86476: Add normalizedModelName column for GPU spec dedup
+CALL ADD_COLUMN('GpuDeviceSpecVO', 'normalizedModelName', 'VARCHAR(255)', 1, NULL);
+CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalizedModelName');

conf/globalConfig/virutalRouter.xml

@@ -21,6 +21,20 @@
         <category>virtualRouter</category>
         <type>java.lang.Integer</type>
     </config>
+    <config>
+        <name>dnsmasq.dnsForwardMax</name>
+        <description>The maximum number of concurrent DNS queries forwarded by dnsmasq on a virtual router.</description>
+        <defaultValue>1000</defaultValue>
+        <category>virtualRouter</category>
+        <type>java.lang.Integer</type>
+    </config>
+    <config>
+        <name>dnsmasq.cacheSize</name>
+        <description>The DNS cache size used by dnsmasq on a virtual router.</description>
+        <defaultValue>10000</defaultValue>
+        <category>virtualRouter</category>
+        <type>java.lang.Integer</type>
+    </config>
     <config>
         <name>ping.interval</name>
         <description>The interval management nodes ping the virtual router agents running virtual router VMs, in seconds</description>

conf/springConfigXml/sugonSdnController.xml

@@ -67,11 +67,7 @@
         </zstack:plugin>
     </bean>
 
-    <bean id="TfZstackPortSync" class="org.zstack.sugonSdnController.network.TfZstackPortSync">
-        <zstack:plugin>
-            <zstack:extension interface="org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint" />
-        </zstack:plugin>
-    </bean>
+    <bean id="TfZstackPortSync" class="org.zstack.sugonSdnController.network.TfZstackPortSync" />
 
     <bean id="SugonApiInterceptor" class="org.zstack.sugonSdnController.header.SugonApiInterceptor">
         <zstack:plugin>

core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java

@@ -27,27 +27,27 @@ public class ResourceDestinationMakerImpl implements ManagementNodeChangeListene
     private DatabaseFacade dbf;
 
     @Override
-    public void nodeJoin(ManagementNodeInventory inv) {
+    public synchronized void nodeJoin(ManagementNodeInventory inv) {
         nodeHash.add(inv.getUuid());
         nodes.put(inv.getUuid(), new NodeInfo(inv));
     }
 
     @Override
-    public void nodeLeft(ManagementNodeInventory inv) {
+    public synchronized void nodeLeft(ManagementNodeInventory inv) {
         String nodeId = inv.getUuid();
         nodeHash.remove(nodeId);
         nodes.remove(nodeId);
     }
 
     @Override
-    public void iAmDead(ManagementNodeInventory inv) {
+    public synchronized void iAmDead(ManagementNodeInventory inv) {
         String nodeId = inv.getUuid();
         nodeHash.remove(nodeId);
         nodes.remove(nodeId);
     }
 
     @Override
-    public void iJoin(ManagementNodeInventory inv) {
+    public synchronized void iJoin(ManagementNodeInventory inv) {
         List<ManagementNodeVO> lst = Q.New(ManagementNodeVO.class).list();
         lst.forEach((ManagementNodeVO node) -> {
             nodeHash.add(node.getUuid());
@@ -56,7 +56,7 @@ public void iJoin(ManagementNodeInventory inv) {
     }
 
     @Override
-    public String makeDestination(String resourceUuid) {
+    public synchronized String makeDestination(String resourceUuid) {
         String nodeUuid = nodeHash.get(resourceUuid);
         if (nodeUuid == null) {
             throw new CloudRuntimeException("Cannot find any available management node to send message");
@@ -66,18 +66,18 @@ public String makeDestination(String resourceUuid) {
     }
 
     @Override
-    public boolean isManagedByUs(String resourceUuid) {
+    public synchronized boolean isManagedByUs(String resourceUuid) {
         String nodeUuid = makeDestination(resourceUuid);
         return nodeUuid.equals(Platform.getManagementServerId());
     }
 
     @Override
-    public Collection<String> getManagementNodesInHashRing() {
-        return nodeHash.getNodes();
+    public synchronized Collection<String> getManagementNodesInHashRing() {
+        return new ArrayList<>(nodeHash.getNodes());
     }
 
     @Override
-    public NodeInfo getNodeInfo(String nodeUuid) {
+    public synchronized NodeInfo getNodeInfo(String nodeUuid) {
         NodeInfo info = nodes.get(nodeUuid);
         if (info == null) {
             ManagementNodeVO vo = dbf.findByUuid(nodeUuid, ManagementNodeVO.class);
@@ -93,17 +93,17 @@ public NodeInfo getNodeInfo(String nodeUuid) {
     }
 
     @Override
-    public Collection<NodeInfo> getAllNodeInfo() {
-        return nodes.values();
+    public synchronized Collection<NodeInfo> getAllNodeInfo() {
+        return new ArrayList<>(nodes.values());
     }
 
     @Override
-    public int getManagementNodeCount() {
-        return nodes.values().size();
+    public synchronized int getManagementNodeCount() {
+        return nodes.size();
     }
 
 
-    public boolean isNodeInCircle(String nodeId) {
+    public synchronized boolean isNodeInCircle(String nodeId) {
         return nodeHash.hasNode(nodeId);
     }
 }

header/src/main/java/org/zstack/header/identity/RBACInfo.java

@@ -12,6 +12,7 @@ public void permissions() {
                 .name("identity")
                 .adminOnlyAPIs(
                         APICreateAccountMsg.class,
+                        APIQueryAccountMsg.class,
                         APIShareResourceMsg.class,
                         APIRevokeResourceSharingMsg.class,
                         APIUpdateQuotaMsg.class,

header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java

@@ -13,7 +13,7 @@ public interface PrimaryStorageNodeSvc {
 
     void deactivate(String installPath, String protocol, ActiveVolumeClient client, Completion comp);
 
-    void blacklist(String installPath, String protocol, HostInventory h, Completion comp);
+    void blacklist(String installPath, String protocol, HostInventory h);
 
     String getActivePath(BaseVolumeInfo v, HostInventory h, boolean shareable);
     BaseVolumeInfo getActiveVolumeInfo(String activePath, HostInventory h, boolean shareable);

header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java

@@ -12,6 +12,7 @@ public class HaStartVmInstanceMsg extends NeedReplyMessage implements VmInstance
     private String vmInstanceUuid;
     private String judgerClassName;
     private List<String> softAvoidHostUuids;
+    private String accessiblePeerHostUuid;
     private String haReason;
 
     public String getJudgerClassName() {
@@ -30,6 +31,14 @@ public void setSoftAvoidHostUuids(List<String> softAvoidHostUuids) {
         this.softAvoidHostUuids = softAvoidHostUuids;
     }
 
+    public String getAccessiblePeerHostUuid() {
+        return accessiblePeerHostUuid;
+    }
+
+    public void setAccessiblePeerHostUuid(String accessiblePeerHostUuid) {
+        this.accessiblePeerHostUuid = accessiblePeerHostUuid;
+    }
+
     @Override
     public String getVmInstanceUuid() {
         return vmInstanceUuid;

header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java

@@ -1,7 +1,14 @@
 package org.zstack.header.vm;
 
+import org.zstack.header.core.Completion;
+
 /**
  */
 public interface VmBeforeStartOnHypervisorExtensionPoint {
-    void beforeStartVmOnHypervisor(VmInstanceSpec spec);
+    default void beforeStartVmOnHypervisor(VmInstanceSpec spec) {}
+
+    default void beforeStartVmOnHypervisor(VmInstanceSpec spec, Completion completion) {
+        beforeStartVmOnHypervisor(spec);
+        completion.success();
+    }
 }

plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java

@@ -614,14 +614,13 @@ public void cleanActiveRecord(ImageCacheInventory cache) {
     }
 
     @Override
-    public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) {
+    public void blacklist(String installPath, String protocol, HostInventory h) {
         logger.debug(String.format("blacklisting volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]",
                 installPath, protocol, h.getUuid(), h.getManagementIp()));
 
         UssGatewayModule uss = getUssGateway(VolumeProtocol.valueOf(protocol), h.getManagementIp());
         VolumeModule exponVol = apiHelper.getVolume(getVolIdFromPath(installPath));
         apiHelper.addVolumePathToBlacklist(buildExponVolumeBoundPath(uss, exponVol.getVolumeName()));
-        comp.success();
     }
 
     @Override