ZSTAC-86567: select target-aware MN endpoint

core/src/main/java/org/zstack/core/ManagementEndpointData.java

@@ -0,0 +1,170 @@
+package org.zstack.core;
+
+import org.apache.commons.lang.StringUtils;
+import org.zstack.header.errorcode.ErrorableValue;
+import org.zstack.utils.network.IPv6Constants;
+import org.zstack.utils.network.IPv6NetworkUtils;
+import org.zstack.utils.network.NetworkUtils;
+import org.zstack.utils.zsha2.ZSha2Info;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_PLATFORM_10000;
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_PLATFORM_10001;
+import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_PLATFORM_10002;
+
+public class ManagementEndpointData {
+    public enum EndpointType {
+        NODE,
+        CANONICAL_NODE,
+        VIP
+    }
+
+    private final Map<Integer, String> nodeIps = new HashMap<>();
+    private final Map<Integer, String> haNodeIps = new HashMap<>();
+    private final Map<Integer, String> haVips = new HashMap<>();
+    private final Set<Integer> nestedHaFamilies = new HashSet<>();
+    private final boolean ha;
+
+    public ManagementEndpointData(Collection<String> nodeIps) {
+        this(nodeIps, null);
+    }
+
+    public ManagementEndpointData(Collection<String> nodeIps, ZSha2Info info) {
+        this.ha = info != null;
+        for (String ip : nodeIps) {
+            put(this.nodeIps, ip);
+        }
+        if (info != null) {
+            addHaFamily(IPv6Constants.IPv4, info.getIpv4());
+            addHaFamily(IPv6Constants.IPv6, info.getIpv6());
+            if (nestedHaFamilies.isEmpty()) {
+                addLegacyHaEndpoint(haNodeIps, info.getNodeip());
+                addLegacyHaEndpoint(haVips, info.getDbvip());
+            }
+        }
+    }
+
+    public ErrorableValue<String> selectForTarget(EndpointType endpointType, String targetIp) {
+        Integer family = getAddressFamily(targetIp);
+        if (family == null) {
+            return ErrorableValue.ofErrorCode(Platform.argerr(ORG_ZSTACK_CORE_PLATFORM_10000,
+                    "cannot select management %s endpoint because target[%s] is not an IPv4 or IPv6 address",
+                    endpointName(endpointType), targetIp));
+        }
+
+        String endpoint = getEndpoint(endpointType, family);
+        if (endpoint != null) {
+            return ErrorableValue.of(endpoint);
+        }
+
+        if (ha && endpointType != EndpointType.NODE) {
+            return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10002,
+                    "cannot select management %s endpoint for %s target[%s]: HA %s family record is missing or invalid",
+                    endpointName(endpointType), familyName(family), targetIp, familyName(family)));
+        }
+
+        return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10001,
+                "cannot select management %s endpoint for %s target[%s]: no configured %s endpoint exists",
+                endpointName(endpointType), familyName(family), targetIp, familyName(family)));
+    }
+
+    public ErrorableValue<String> selectDefault(EndpointType endpointType) {
+        String endpoint = getDefaultEndpoint(endpointType);
+        if (endpoint != null) {
+            return ErrorableValue.of(endpoint);
+        }
+
+        if (ha && endpointType != EndpointType.NODE) {
+            return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10002,
+                    "cannot select default management %s endpoint: HA default family record is missing or invalid",
+                    endpointName(endpointType)));
+        }
+
+        return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10001,
+                "cannot select default management %s endpoint because no configured endpoint exists",
+                endpointName(endpointType)));
+    }
+
+    public String getDefaultEndpoint(EndpointType endpointType) {
+        Integer defaultFamily = null;
+        if (nodeIps.containsKey(IPv6Constants.IPv4)) {
+            defaultFamily = IPv6Constants.IPv4;
+        } else if (nodeIps.containsKey(IPv6Constants.IPv6)) {
+            defaultFamily = IPv6Constants.IPv6;
+        }
+        return defaultFamily == null ? null : getEndpoint(endpointType, defaultFamily);
+    }
+
+    private String getEndpoint(EndpointType endpointType, int family) {
+        if (endpointType == EndpointType.NODE || !ha) {
+            return nodeIps.get(family);
+        }
+        return endpointType == EndpointType.CANONICAL_NODE ? haNodeIps.get(family) : haVips.get(family);
+    }
+
+    private void addHaFamily(int expectedFamily, ZSha2Info.HaAddressFamily family) {
+        if (family == null) {
+            return;
+        }
+        nestedHaFamilies.add(expectedFamily);
+        if (!family.isEnabled() || !hasFamily(family.getNodeIp(), expectedFamily)
+                || !hasFamily(family.getPeerIp(), expectedFamily)
+                || !hasFamily(family.getVirtualIp(), expectedFamily)) {
+            return;
+        }
+        haNodeIps.put(expectedFamily, normalize(family.getNodeIp()));
+        haVips.put(expectedFamily, normalize(family.getVirtualIp()));
+    }
+
+    private void addLegacyHaEndpoint(Map<Integer, String> endpoints, String ip) {
+        Integer family = getAddressFamily(ip);
+        if (family != null && !endpoints.containsKey(family)) {
+            endpoints.put(family, normalize(ip));
+        }
+    }
+
+    private static void put(Map<Integer, String> endpoints, String ip) {
+        Integer family = getAddressFamily(ip);
+        if (family != null) {
+            endpoints.put(family, normalize(ip));
+        }
+    }
+
+    private static boolean hasFamily(String ip, int expectedFamily) {
+        Integer actualFamily = getAddressFamily(ip);
+        return actualFamily != null && actualFamily == expectedFamily;
+    }
+
+    private static Integer getAddressFamily(String ip) {
+        if (StringUtils.isBlank(ip)) {
+            return null;
+        }
+
+        String normalized = normalize(ip);
+        if (NetworkUtils.isIpv4Address(normalized)) {
+            return IPv6Constants.IPv4;
+        }
+        if (IPv6NetworkUtils.isIpv6Address(normalized)) {
+            return IPv6Constants.IPv6;
+        }
+        return null;
+    }
+
+    private static String normalize(String ip) {
+        String normalized = ip.trim();
+        return IPv6NetworkUtils.isIpv6Address(normalized) ? IPv6NetworkUtils.normalizeIpv6(normalized) : normalized;
+    }
+
+    private static String endpointName(EndpointType endpointType) {
+        return endpointType.name().toLowerCase().replace('_', ' ');
+    }
+
+    private static String familyName(int family) {
+        return family == IPv6Constants.IPv4 ? "IPv4" : "IPv6";
+    }
+}

core/src/main/java/org/zstack/core/Platform.java

@@ -31,6 +31,7 @@
 import org.zstack.header.core.encrypt.ENCRYPT;
 import org.zstack.header.errorcode.ErrorCode;
 import org.zstack.header.errorcode.ErrorCodeList;
+import org.zstack.header.errorcode.ErrorableValue;
 import org.zstack.header.errorcode.SysErrors;
 import org.zstack.header.exception.CloudRuntimeException;
 import org.zstack.header.identity.IdentityErrors;
@@ -1034,13 +1035,26 @@ public static String getUuidFromBytes(byte[] name) {
     }
 
     public static String getManagementServerIp() {
+        String endpoint = getManagementNodeEndpointData().getDefaultEndpoint(ManagementEndpointData.EndpointType.NODE);
+        if (endpoint != null) {
+            return endpoint;
+        }
+
+        if (hasConfiguredManagementNodeIpProperty()) {
+            throw new CloudRuntimeException("management server IP configuration contains no valid IPv4 or IPv6 address");
+        }
+
         if (managementServerIp == null) {
             managementServerIp = getManagementServerIpInternal();
         }
 
         return managementServerIp;
     }
 
+    public static ErrorableValue<String> getManagementServerIp(String targetIp) {
+        return getManagementNodeEndpointData().selectForTarget(ManagementEndpointData.EndpointType.NODE, targetIp);
+    }
+
     public static int getManagementNodeServicePort() {
         return Integer.parseInt(System.getProperty("RESTFacade.port", "8080"));
     }
@@ -1049,7 +1063,15 @@ public static String getManagementServerVip() {
         if (!ZSha2Helper.isMNHaEnvironment()) {
             return getManagementServerIp();
         }
-        return ZSha2Helper.getInfo(false).getDbvip();
+        ErrorableValue<String> endpoint = getManagementEndpointData().selectDefault(ManagementEndpointData.EndpointType.VIP);
+        if (!endpoint.isSuccess()) {
+            throw new CloudRuntimeException(endpoint.error.getDetails());
+        }
+        return endpoint.result;
+    }
+
+    public static ErrorableValue<String> getManagementServerVip(String targetIp) {
+        return getManagementEndpointData().selectForTarget(ManagementEndpointData.EndpointType.VIP, targetIp);
     }
 
     public static String getManagementServerVipBaseUrl() {
@@ -1071,8 +1093,15 @@ public static String getCanonicalServerIp() {
         if (!ZSha2Helper.isMNHaEnvironment()) {
             return getManagementServerIp();
         }
+        ErrorableValue<String> endpoint = getManagementEndpointData().selectDefault(ManagementEndpointData.EndpointType.CANONICAL_NODE);
+        if (!endpoint.isSuccess()) {
+            throw new CloudRuntimeException(endpoint.error.getDetails());
+        }
+        return endpoint.result;
+    }
 
-        return ZSha2Helper.getInfo(false).getNodeip();
+    public static ErrorableValue<String> getCanonicalServerIp(String targetIp) {
+        return getManagementEndpointData().selectForTarget(ManagementEndpointData.EndpointType.CANONICAL_NODE, targetIp);
     }
 
     public static boolean isVIPNode() {
@@ -1234,19 +1263,61 @@ private static String selectManagementServerIpFromDefaultRoute(String defaultLin
     }
 
     public static String getManagementServerIp6() {
-        String ip = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP6_PROPERTY, IPv6Constants.IPv6);
-        if (ip != null) {
-            return ip;
-        }
-        return getManagementServerIpOnManagementInterface(IPv6Constants.IPv6);
+        return getConfiguredManagementServerIp(IPv6Constants.IPv6);
     }
 
     public static String getManagementServerIp4() {
-        String ip = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP4_PROPERTY, IPv6Constants.IPv4);
-        if (ip != null) {
-            return ip;
+        return getConfiguredManagementServerIp(IPv6Constants.IPv4);
+    }
+
+    private static ManagementEndpointData getManagementEndpointData() {
+        List<String> nodeIps = getConfiguredManagementNodeIps();
+        return ZSha2Helper.isMNHaEnvironment() ?
+                new ManagementEndpointData(nodeIps, ZSha2Helper.getInfo(false)) :
+                new ManagementEndpointData(nodeIps);
+    }
+
+    private static ManagementEndpointData getManagementNodeEndpointData() {
+        return new ManagementEndpointData(getConfiguredManagementNodeIps());
+    }
+
+    private static List<String> getConfiguredManagementNodeIps() {
+        List<String> nodeIps = new ArrayList<>();
+        String primaryIp = System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY);
+        if (!StringUtils.isBlank(primaryIp)) {
+            nodeIps.add(normalizeManagementIp(primaryIp));
+        }
+
+        String ipv4 = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP4_PROPERTY, IPv6Constants.IPv4);
+        if (ipv4 != null) {
+            nodeIps.add(ipv4);
+        }
+        String ipv6 = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP6_PROPERTY, IPv6Constants.IPv6);
+        if (ipv6 != null) {
+            nodeIps.add(ipv6);
         }
-        return getManagementServerIpOnManagementInterface(IPv6Constants.IPv4);
+        return nodeIps;
+    }
+
+    private static boolean hasConfiguredManagementNodeIpProperty() {
+        return !StringUtils.isBlank(System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY))
+                || !StringUtils.isBlank(System.getProperty(MANAGEMENT_SERVER_IP4_PROPERTY))
+                || !StringUtils.isBlank(System.getProperty(MANAGEMENT_SERVER_IP6_PROPERTY));
+    }
+
+    private static String getConfiguredManagementServerIp(int ipVersion) {
+        String primaryIp = System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY);
+        if (!StringUtils.isBlank(primaryIp)) {
+            String normalizedPrimaryIp = normalizeManagementIp(primaryIp);
+            if ((ipVersion == IPv6Constants.IPv4 && NetworkUtils.isIpv4Address(normalizedPrimaryIp)) ||
+                    (ipVersion == IPv6Constants.IPv6 && IPv6NetworkUtils.isIpv6Address(normalizedPrimaryIp))) {
+                return normalizedPrimaryIp;
+            }
+        }
+
+        return getManagementServerSecondaryIpProperty(
+                ipVersion == IPv6Constants.IPv4 ? MANAGEMENT_SERVER_IP4_PROPERTY : MANAGEMENT_SERVER_IP6_PROPERTY,
+                ipVersion);
     }
 
     private static String getManagementServerSecondaryIpProperty(String property, int ipVersion) {
@@ -1267,53 +1338,19 @@ private static String getManagementServerSecondaryIpProperty(String property, in
         return normalizedIp;
     }
 
-    private static String getManagementServerIpOnManagementInterface(int ipVersion) {
-        try {
-            NetworkInterface iface = findManagementServerInterface();
-            if (iface == null || !iface.isUp()) {
-                return null;
-            }
-
-            for (InetAddress address : Collections.list(iface.getInetAddresses())) {
-                if (address.isLoopbackAddress() || address.isLinkLocalAddress()) {
-                    continue;
-                }
-                if (ipVersion == IPv6Constants.IPv6 && !(address instanceof Inet4Address)) {
-                    return normalizeManagementIp(address.getHostAddress());
-                }
-                if (ipVersion == IPv6Constants.IPv4 && address instanceof Inet4Address) {
-                    return normalizeManagementIp(address.getHostAddress());
-                }
-            }
-        } catch (SocketException e) {
-            throw new CloudRuntimeException(e);
-        }
-
-        return null;
-    }
-
-    private static NetworkInterface findManagementServerInterface() throws SocketException {
-        String currentIp = normalizeManagementIp(getManagementServerIp());
-        Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
-        for (NetworkInterface iface : Collections.list(nets)) {
-            for (InetAddress address : Collections.list(iface.getInetAddresses())) {
-                if (currentIp.equals(normalizeManagementIp(address.getHostAddress()))) {
-                    return iface;
-                }
-            }
-        }
-
-        return null;
-    }
-
     public static String getManagementServerIp6Cidr() {
         String ip6 = getManagementServerIp6();
         return ip6 == null ? null : getManagementServerCidr(ip6);
     }
 
     public static List<String> getManagementServerIps() {
         LinkedHashSet<String> ips = new LinkedHashSet<>();
-        ips.add(getManagementServerIp());
+        String primaryIp = System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY);
+        if (!StringUtils.isBlank(primaryIp)) {
+            ips.add(normalizeManagementIp(primaryIp));
+        } else {
+            ips.add(getManagementServerIp());
+        }
         ips.add(getManagementServerIp4());
         ips.add(getManagementServerIp6());
         ips.remove(null);