diff --git a/core/src/main/java/org/zstack/core/ManagementEndpointData.java b/core/src/main/java/org/zstack/core/ManagementEndpointData.java new file mode 100644 index 00000000000..f47eb762e9a --- /dev/null +++ b/core/src/main/java/org/zstack/core/ManagementEndpointData.java @@ -0,0 +1,170 @@ +package org.zstack.core; + +import org.apache.commons.lang.StringUtils; +import org.zstack.header.errorcode.ErrorableValue; +import org.zstack.utils.network.IPv6Constants; +import org.zstack.utils.network.IPv6NetworkUtils; +import org.zstack.utils.network.NetworkUtils; +import org.zstack.utils.zsha2.ZSha2Info; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_PLATFORM_10000; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_PLATFORM_10001; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_PLATFORM_10002; + +public class ManagementEndpointData { + public enum EndpointType { + NODE, + CANONICAL_NODE, + VIP + } + + private final Map nodeIps = new HashMap<>(); + private final Map haNodeIps = new HashMap<>(); + private final Map haVips = new HashMap<>(); + private final Set nestedHaFamilies = new HashSet<>(); + private final boolean ha; + + public ManagementEndpointData(Collection nodeIps) { + this(nodeIps, null); + } + + public ManagementEndpointData(Collection nodeIps, ZSha2Info info) { + this.ha = info != null; + for (String ip : nodeIps) { + put(this.nodeIps, ip); + } + if (info != null) { + addHaFamily(IPv6Constants.IPv4, info.getIpv4()); + addHaFamily(IPv6Constants.IPv6, info.getIpv6()); + if (nestedHaFamilies.isEmpty()) { + addLegacyHaEndpoint(haNodeIps, info.getNodeip()); + addLegacyHaEndpoint(haVips, info.getDbvip()); + } + } + } + + public ErrorableValue selectForTarget(EndpointType endpointType, String targetIp) { + Integer family = getAddressFamily(targetIp); + if (family == null) { + return ErrorableValue.ofErrorCode(Platform.argerr(ORG_ZSTACK_CORE_PLATFORM_10000, + "cannot select management %s endpoint because target[%s] is not an IPv4 or IPv6 address", + endpointName(endpointType), targetIp)); + } + + String endpoint = getEndpoint(endpointType, family); + if (endpoint != null) { + return ErrorableValue.of(endpoint); + } + + if (ha && endpointType != EndpointType.NODE) { + return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10002, + "cannot select management %s endpoint for %s target[%s]: HA %s family record is missing or invalid", + endpointName(endpointType), familyName(family), targetIp, familyName(family))); + } + + return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10001, + "cannot select management %s endpoint for %s target[%s]: no configured %s endpoint exists", + endpointName(endpointType), familyName(family), targetIp, familyName(family))); + } + + public ErrorableValue selectDefault(EndpointType endpointType) { + String endpoint = getDefaultEndpoint(endpointType); + if (endpoint != null) { + return ErrorableValue.of(endpoint); + } + + if (ha && endpointType != EndpointType.NODE) { + return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10002, + "cannot select default management %s endpoint: HA default family record is missing or invalid", + endpointName(endpointType))); + } + + return ErrorableValue.ofErrorCode(Platform.operr(ORG_ZSTACK_CORE_PLATFORM_10001, + "cannot select default management %s endpoint because no configured endpoint exists", + endpointName(endpointType))); + } + + public String getDefaultEndpoint(EndpointType endpointType) { + Integer defaultFamily = null; + if (nodeIps.containsKey(IPv6Constants.IPv4)) { + defaultFamily = IPv6Constants.IPv4; + } else if (nodeIps.containsKey(IPv6Constants.IPv6)) { + defaultFamily = IPv6Constants.IPv6; + } + return defaultFamily == null ? null : getEndpoint(endpointType, defaultFamily); + } + + private String getEndpoint(EndpointType endpointType, int family) { + if (endpointType == EndpointType.NODE || !ha) { + return nodeIps.get(family); + } + return endpointType == EndpointType.CANONICAL_NODE ? haNodeIps.get(family) : haVips.get(family); + } + + private void addHaFamily(int expectedFamily, ZSha2Info.HaAddressFamily family) { + if (family == null) { + return; + } + nestedHaFamilies.add(expectedFamily); + if (!family.isEnabled() || !hasFamily(family.getNodeIp(), expectedFamily) + || !hasFamily(family.getPeerIp(), expectedFamily) + || !hasFamily(family.getVirtualIp(), expectedFamily)) { + return; + } + haNodeIps.put(expectedFamily, normalize(family.getNodeIp())); + haVips.put(expectedFamily, normalize(family.getVirtualIp())); + } + + private void addLegacyHaEndpoint(Map endpoints, String ip) { + Integer family = getAddressFamily(ip); + if (family != null && !endpoints.containsKey(family)) { + endpoints.put(family, normalize(ip)); + } + } + + private static void put(Map endpoints, String ip) { + Integer family = getAddressFamily(ip); + if (family != null) { + endpoints.put(family, normalize(ip)); + } + } + + private static boolean hasFamily(String ip, int expectedFamily) { + Integer actualFamily = getAddressFamily(ip); + return actualFamily != null && actualFamily == expectedFamily; + } + + private static Integer getAddressFamily(String ip) { + if (StringUtils.isBlank(ip)) { + return null; + } + + String normalized = normalize(ip); + if (NetworkUtils.isIpv4Address(normalized)) { + return IPv6Constants.IPv4; + } + if (IPv6NetworkUtils.isIpv6Address(normalized)) { + return IPv6Constants.IPv6; + } + return null; + } + + private static String normalize(String ip) { + String normalized = ip.trim(); + return IPv6NetworkUtils.isIpv6Address(normalized) ? IPv6NetworkUtils.normalizeIpv6(normalized) : normalized; + } + + private static String endpointName(EndpointType endpointType) { + return endpointType.name().toLowerCase().replace('_', ' '); + } + + private static String familyName(int family) { + return family == IPv6Constants.IPv4 ? "IPv4" : "IPv6"; + } +} diff --git a/core/src/main/java/org/zstack/core/Platform.java b/core/src/main/java/org/zstack/core/Platform.java index 34df20e3e12..e3c259d99b5 100755 --- a/core/src/main/java/org/zstack/core/Platform.java +++ b/core/src/main/java/org/zstack/core/Platform.java @@ -31,6 +31,7 @@ import org.zstack.header.core.encrypt.ENCRYPT; import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.errorcode.ErrorableValue; import org.zstack.header.errorcode.SysErrors; import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.identity.IdentityErrors; @@ -1034,6 +1035,15 @@ public static String getUuidFromBytes(byte[] name) { } public static String getManagementServerIp() { + String endpoint = getManagementNodeEndpointData().getDefaultEndpoint(ManagementEndpointData.EndpointType.NODE); + if (endpoint != null) { + return endpoint; + } + + if (hasConfiguredManagementNodeIpProperty()) { + throw new CloudRuntimeException("management server IP configuration contains no valid IPv4 or IPv6 address"); + } + if (managementServerIp == null) { managementServerIp = getManagementServerIpInternal(); } @@ -1041,6 +1051,10 @@ public static String getManagementServerIp() { return managementServerIp; } + public static ErrorableValue getManagementServerIp(String targetIp) { + return getManagementNodeEndpointData().selectForTarget(ManagementEndpointData.EndpointType.NODE, targetIp); + } + public static int getManagementNodeServicePort() { return Integer.parseInt(System.getProperty("RESTFacade.port", "8080")); } @@ -1049,7 +1063,15 @@ public static String getManagementServerVip() { if (!ZSha2Helper.isMNHaEnvironment()) { return getManagementServerIp(); } - return ZSha2Helper.getInfo(false).getDbvip(); + ErrorableValue endpoint = getManagementEndpointData().selectDefault(ManagementEndpointData.EndpointType.VIP); + if (!endpoint.isSuccess()) { + throw new CloudRuntimeException(endpoint.error.getDetails()); + } + return endpoint.result; + } + + public static ErrorableValue getManagementServerVip(String targetIp) { + return getManagementEndpointData().selectForTarget(ManagementEndpointData.EndpointType.VIP, targetIp); } public static String getManagementServerVipBaseUrl() { @@ -1071,8 +1093,15 @@ public static String getCanonicalServerIp() { if (!ZSha2Helper.isMNHaEnvironment()) { return getManagementServerIp(); } + ErrorableValue endpoint = getManagementEndpointData().selectDefault(ManagementEndpointData.EndpointType.CANONICAL_NODE); + if (!endpoint.isSuccess()) { + throw new CloudRuntimeException(endpoint.error.getDetails()); + } + return endpoint.result; + } - return ZSha2Helper.getInfo(false).getNodeip(); + public static ErrorableValue getCanonicalServerIp(String targetIp) { + return getManagementEndpointData().selectForTarget(ManagementEndpointData.EndpointType.CANONICAL_NODE, targetIp); } public static boolean isVIPNode() { @@ -1234,19 +1263,61 @@ private static String selectManagementServerIpFromDefaultRoute(String defaultLin } public static String getManagementServerIp6() { - String ip = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP6_PROPERTY, IPv6Constants.IPv6); - if (ip != null) { - return ip; - } - return getManagementServerIpOnManagementInterface(IPv6Constants.IPv6); + return getConfiguredManagementServerIp(IPv6Constants.IPv6); } public static String getManagementServerIp4() { - String ip = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP4_PROPERTY, IPv6Constants.IPv4); - if (ip != null) { - return ip; + return getConfiguredManagementServerIp(IPv6Constants.IPv4); + } + + private static ManagementEndpointData getManagementEndpointData() { + List nodeIps = getConfiguredManagementNodeIps(); + return ZSha2Helper.isMNHaEnvironment() ? + new ManagementEndpointData(nodeIps, ZSha2Helper.getInfo(false)) : + new ManagementEndpointData(nodeIps); + } + + private static ManagementEndpointData getManagementNodeEndpointData() { + return new ManagementEndpointData(getConfiguredManagementNodeIps()); + } + + private static List getConfiguredManagementNodeIps() { + List nodeIps = new ArrayList<>(); + String primaryIp = System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY); + if (!StringUtils.isBlank(primaryIp)) { + nodeIps.add(normalizeManagementIp(primaryIp)); + } + + String ipv4 = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP4_PROPERTY, IPv6Constants.IPv4); + if (ipv4 != null) { + nodeIps.add(ipv4); + } + String ipv6 = getManagementServerSecondaryIpProperty(MANAGEMENT_SERVER_IP6_PROPERTY, IPv6Constants.IPv6); + if (ipv6 != null) { + nodeIps.add(ipv6); } - return getManagementServerIpOnManagementInterface(IPv6Constants.IPv4); + return nodeIps; + } + + private static boolean hasConfiguredManagementNodeIpProperty() { + return !StringUtils.isBlank(System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY)) + || !StringUtils.isBlank(System.getProperty(MANAGEMENT_SERVER_IP4_PROPERTY)) + || !StringUtils.isBlank(System.getProperty(MANAGEMENT_SERVER_IP6_PROPERTY)); + } + + private static String getConfiguredManagementServerIp(int ipVersion) { + String primaryIp = System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY); + if (!StringUtils.isBlank(primaryIp)) { + String normalizedPrimaryIp = normalizeManagementIp(primaryIp); + if ((ipVersion == IPv6Constants.IPv4 && NetworkUtils.isIpv4Address(normalizedPrimaryIp)) || + (ipVersion == IPv6Constants.IPv6 && IPv6NetworkUtils.isIpv6Address(normalizedPrimaryIp))) { + return normalizedPrimaryIp; + } + } + + return getManagementServerSecondaryIpProperty( + ipVersion == IPv6Constants.IPv4 ? MANAGEMENT_SERVER_IP4_PROPERTY : MANAGEMENT_SERVER_IP6_PROPERTY, + ipVersion); } private static String getManagementServerSecondaryIpProperty(String property, int ipVersion) { @@ -1267,45 +1338,6 @@ private static String getManagementServerSecondaryIpProperty(String property, in return normalizedIp; } - private static String getManagementServerIpOnManagementInterface(int ipVersion) { - try { - NetworkInterface iface = findManagementServerInterface(); - if (iface == null || !iface.isUp()) { - return null; - } - - for (InetAddress address : Collections.list(iface.getInetAddresses())) { - if (address.isLoopbackAddress() || address.isLinkLocalAddress()) { - continue; - } - if (ipVersion == IPv6Constants.IPv6 && !(address instanceof Inet4Address)) { - return normalizeManagementIp(address.getHostAddress()); - } - if (ipVersion == IPv6Constants.IPv4 && address instanceof Inet4Address) { - return normalizeManagementIp(address.getHostAddress()); - } - } - } catch (SocketException e) { - throw new CloudRuntimeException(e); - } - - return null; - } - - private static NetworkInterface findManagementServerInterface() throws SocketException { - String currentIp = normalizeManagementIp(getManagementServerIp()); - Enumeration nets = NetworkInterface.getNetworkInterfaces(); - for (NetworkInterface iface : Collections.list(nets)) { - for (InetAddress address : Collections.list(iface.getInetAddresses())) { - if (currentIp.equals(normalizeManagementIp(address.getHostAddress()))) { - return iface; - } - } - } - - return null; - } - public static String getManagementServerIp6Cidr() { String ip6 = getManagementServerIp6(); return ip6 == null ? null : getManagementServerCidr(ip6); @@ -1313,7 +1345,12 @@ public static String getManagementServerIp6Cidr() { public static List getManagementServerIps() { LinkedHashSet ips = new LinkedHashSet<>(); - ips.add(getManagementServerIp()); + String primaryIp = System.getProperty(MANAGEMENT_SERVER_IP_PROPERTY); + if (!StringUtils.isBlank(primaryIp)) { + ips.add(normalizeManagementIp(primaryIp)); + } else { + ips.add(getManagementServerIp()); + } ips.add(getManagementServerIp4()); ips.add(getManagementServerIp6()); ips.remove(null); diff --git a/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java b/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java index bd7e06ec714..8daa203abb9 100755 --- a/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java +++ b/core/src/main/java/org/zstack/core/ansible/AnsibleRunner.java @@ -4,11 +4,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.header.core.Completion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorableValue; import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.message.MessageReply; @@ -63,6 +65,7 @@ public class AnsibleRunner { } private String targetIp; + private String managementNodeIp; /** targetUuid is an unique resource for cache (see /usr/local/zstack/ansible/.ansible.cache/$targetUuid/), * it should be a phsical resource, * it it option. @@ -137,6 +140,11 @@ public void setTargetIp(String targetIp) { this.targetIp = targetIp; } + public AnsibleRunner setManagementNodeIp(String managementNodeIp) { + this.managementNodeIp = managementNodeIp; + return this; + } + public String getTargetUuid() { return targetUuid; } @@ -388,14 +396,27 @@ public void run(ReturnValueCompletion completion) { } int port = new URI(restf.getBaseUrl()).getPort(); + String selectedManagementNodeIp = managementNodeIp; + if (selectedManagementNodeIp == null) { + if (NetworkUtils.isIpAddress(targetIp)) { + ErrorableValue managementNodeEndpoint = Platform.getManagementServerIp(targetIp); + if (!managementNodeEndpoint.isSuccess()) { + completion.fail(managementNodeEndpoint.error); + return; + } + selectedManagementNodeIp = managementNodeEndpoint.result; + } else { + selectedManagementNodeIp = restf.getHostName(); + } + } if (deployArguments == null) { deployArguments = new AnsibleBasicArguments(); } - deployArguments.setPipUrl(buildPipUrl(restf.getHostName(), port)); - deployArguments.setTrustedHost(restf.getHostName()); - deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(restf.getHostName(), port)); + deployArguments.setPipUrl(buildPipUrl(selectedManagementNodeIp, port)); + deployArguments.setTrustedHost(selectedManagementNodeIp); + deployArguments.setYumServer(IPv6NetworkUtils.formatHostPort(selectedManagementNodeIp, port)); deployArguments.setRemoteUser(username); if (password != null && !password.isEmpty()) { deployArguments.setRemotePass(password); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 226ebbfbcaa..2d98423bb40 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -2903,9 +2903,12 @@ public static String buildManagementNodeCallbackCheckCommand(String callbackUrl) return String.format(MANAGEMENT_NODE_CALLBACK_CHECK_COMMAND, callbackUrl, callbackUrl); } - public static String buildManagementNodeCallbackCheckCommand(String hostManagementIp, RESTFacade restf) { - String callbackHost = Platform.getManagementServerIpForRemote(hostManagementIp); - return buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(callbackHost)); + public static ErrorableValue buildManagementNodeCallbackCheckCommand(String hostManagementIp, RESTFacade restf) { + ErrorableValue callbackHost = Platform.getManagementServerIp(hostManagementIp); + if (!callbackHost.isSuccess()) { + return ErrorableValue.ofErrorCode(callbackHost.error); + } + return ErrorableValue.of(buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(callbackHost.result))); } private static String joinAgentPath(String rootPath, String path) { @@ -5808,8 +5811,11 @@ public void run(FlowTrigger trigger, Map data) { sshShell.setPassword(getSelf().getPassword()); sshShell.setPort(getSelf().getPort()); sshShell.setWithSudo(false); - final String callbackHost = Platform.getManagementServerIpForRemote(getSelf().getManagementIp()); - final String cmd = buildManagementNodeCallbackCheckCommand(getSelf().getManagementIp(), restf); + ErrorableValue callbackHost = Platform.getManagementServerIp(getSelf().getManagementIp()); + if (!callbackHost.isSuccess()) { + throw new OperationFailureException(callbackHost.error); + } + final String cmd = buildManagementNodeCallbackCheckCommand(restf.buildCallbackUrl(callbackHost.result)); SshResult ret = sshShell.runCommand(cmd); if (ret.getStderr() != null && ret.getStderr().contains("No route to host")) { // c.f. https://access.redhat.com/solutions/1120533 @@ -5827,7 +5833,7 @@ public void run(FlowTrigger trigger, Map data) { "please check if username/password is wrong; %s", self.getManagementIp(), getSelf().getUsername(), getSelf().getPort(), ret.getExitErrorMessage())); } else if (ret.getReturnCode() != 0) { throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10106, "the KVM host[ip:%s] cannot access the management node's callback url. It seems" + - " that the KVM host cannot reach the management IP[%s]. %s %s", self.getManagementIp(), callbackHost, + " that the KVM host cannot reach the management IP[%s]. %s %s", self.getManagementIp(), callbackHost.result, ret.getStderr(), ret.getExitErrorMessage())); } @@ -6063,6 +6069,12 @@ public boolean skip(Map data) { @Override public void run(final FlowTrigger trigger, Map data) { + ErrorableValue callbackIp = Platform.getManagementServerIp(getSelf().getManagementIp()); + if (!callbackIp.isSuccess()) { + trigger.fail(callbackIp.error); + return; + } + String srcPath = PathUtil.findFileOnClassPath(String.format("ansible/kvm/%s", agentPackageName), true).getAbsolutePath(); String destPath = String.format("/var/lib/zstack/kvm/package/%s", agentPackageName); SshFileMd5Checker checker = new SshFileMd5Checker(); @@ -6097,7 +6109,7 @@ public void run(final FlowTrigger trigger, Map data) { callbackChecker.setUsername(getSelf().getUsername()); callbackChecker.setPassword(getSelf().getPassword()); callbackChecker.setPort(getSelf().getPort()); - callbackChecker.setCallbackIp(Platform.getManagementServerIpForRemote(getSelf().getManagementIp())); + callbackChecker.setCallbackIp(callbackIp.result); callbackChecker.setCallBackPort(CloudBusGlobalProperty.HTTP_PORT); KvmHostConfigChecker kvmHostConfigChecker = new KvmHostConfigChecker(); @@ -6121,7 +6133,7 @@ public void run(final FlowTrigger trigger, Map data) { hostTcpConnectionCallbackChecker.setUsername(getSelf().getUsername()); hostTcpConnectionCallbackChecker.setPassword(getSelf().getPassword()); hostTcpConnectionCallbackChecker.setPort(getSelf().getPort()); - hostTcpConnectionCallbackChecker.setCallbackIp(Platform.getManagementServerIpForRemote(getSelf().getManagementIp())); + hostTcpConnectionCallbackChecker.setCallbackIp(callbackIp.result); hostTcpConnectionCallbackChecker.setCallBackPort(KVMGlobalProperty.TCP_SERVER_PORT); runner.installChecker(hostTcpConnectionCallbackChecker); } @@ -6134,6 +6146,7 @@ public void run(final FlowTrigger trigger, Map data) { } runner.setAgentPort(KVMGlobalProperty.AGENT_PORT); runner.setTargetIp(getSelf().getManagementIp()); + runner.setManagementNodeIp(callbackIp.result); runner.setTargetUuid(getSelf().getUuid()); runner.setPlayBookName(KVMConstant.ANSIBLE_PLAYBOOK_NAME); runner.setUsername(getSelf().getUsername()); diff --git a/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy b/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy index 2c75a9929af..aad5faee3d0 100644 --- a/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/core/ManagementNetworkIpv6Case.groovy @@ -7,6 +7,7 @@ import org.zstack.appliancevm.ApplianceVmGlobalProperty import org.zstack.core.ansible.CallBackNetworkChecker import org.zstack.core.ansible.AnsibleRunner import org.zstack.core.CoreGlobalProperty +import org.zstack.core.ManagementEndpointData import org.zstack.core.Platform import org.zstack.header.exception.CloudRuntimeException import org.zstack.core.agent.AgentManagerImpl @@ -31,10 +32,12 @@ import org.zstack.storage.primary.nfs.NfsApiParamChecker import org.zstack.testlib.SubCase import org.zstack.utils.TagUtils import org.zstack.utils.URLBuilder +import org.zstack.utils.gson.JSONObjectUtil import org.zstack.utils.ssh.SshShell import org.zstack.utils.network.IPv6Constants import org.zstack.utils.network.IPv6NetworkUtils import org.zstack.utils.network.NetworkUtils +import org.zstack.utils.zsha2.ZSha2Info import org.junit.Test import java.lang.reflect.Field @@ -132,6 +135,9 @@ class ManagementNetworkIpv6Case extends SubCase { testKvmIpmiAddressKeepsIpv6() testApplianceVmBootstrapParam() testZsha2SearchBackendSelection() + testTargetAwareManagementNodeSelectionDoesNotCrossAddressFamily() + testDefaultManagementNodeSelectionPrefersIpv4() + testHaManagementEndpointSelectionPreservesEndpointKind() } void testSelectManagementServerIpDualStackPolicy() { @@ -255,13 +261,14 @@ class ManagementNetworkIpv6Case extends SubCase { RESTFacadeImpl.buildCallbackUrl(host, REST_PORT, "zstack") } ] as RESTFacade - String command = KVMHost.buildManagementNodeCallbackCheckCommand(IPV6_2, restf) + def command = KVMHost.buildManagementNodeCallbackCheckCommand(IPV6_2, restf) String callbackUrl = RESTFacadeImpl.buildCallbackUrl(IPV6, REST_PORT, "zstack") + assert command.success assert callbackUrl == "http://[${IPV6}]:${REST_PORT}/zstack${RESTConstant.CALLBACK_PATH}" - assert command.contains("curl --connect-timeout 10 --max-time 15 ${callbackUrl}") - assert command.contains("wget --spider -q --connect-timeout=10 --read-timeout=10 --tries=1 ${callbackUrl}") - assert !command.contains("http://${IPV4}:${REST_PORT}/zstack${RESTConstant.CALLBACK_PATH}") + assert command.result.contains("curl --connect-timeout 10 --max-time 15 ${callbackUrl}") + assert command.result.contains("wget --spider -q --connect-timeout=10 --read-timeout=10 --tries=1 ${callbackUrl}") + assert !command.result.contains("http://${IPV4}:${REST_PORT}/zstack${RESTConstant.CALLBACK_PATH}") } } @@ -432,6 +439,69 @@ class ManagementNetworkIpv6Case extends SubCase { } } + void testTargetAwareManagementNodeSelectionDoesNotCrossAddressFamily() { + withManagementServerIpProperties([ + "management.server.ip": IPV4, + ]) { + assert Platform.getManagementServerIp("192.168.1.20").result == IPV4 + def missingIpv6 = Platform.getManagementServerIp(IPV6_2) + assert !missingIpv6.success + assert missingIpv6.error.globalErrorCode == "ORG_ZSTACK_CORE_PLATFORM_10001" + def hostnameTarget = Platform.getManagementServerIp("host.example.com") + assert !hostnameTarget.success + assert hostnameTarget.error.globalErrorCode == "ORG_ZSTACK_CORE_PLATFORM_10000" + } + } + + void testDefaultManagementNodeSelectionPrefersIpv4() { + withManagementServerIpProperties([ + "management.server.ip" : IPV6, + "management.server.ip4": IPV4, + ]) { + assert Platform.getManagementServerIp() == IPV4 + } + } + + void testHaManagementEndpointSelectionPreservesEndpointKind() { + ZSha2Info info = JSONObjectUtil.toObject(''' + { + "ipv4": {"enabled": true, "nodeIp": "192.168.1.11", "peerIp": "192.168.1.12", "virtualIp": "192.168.1.100"}, + "ipv6": {"enabled": true, "nodeIp": "2001:db8::11", "peerIp": "2001:db8::12", "virtualIp": "2001:db8::100"} + } + ''', ZSha2Info.class) + + ManagementEndpointData endpoints = new ManagementEndpointData([IPV4, IPV6], info) + assert endpoints.selectForTarget(ManagementEndpointData.EndpointType.NODE, IPV6_2).result == IPV6 + assert endpoints.selectForTarget(ManagementEndpointData.EndpointType.CANONICAL_NODE, IPV6_2).result == "2001:db8::11" + assert endpoints.selectForTarget(ManagementEndpointData.EndpointType.VIP, IPV6_2).result == "2001:db8::100" + + ZSha2Info missingIpv6Vip = new ZSha2Info() + missingIpv6Vip.setIpv6(new ZSha2Info.HaAddressFamily(nodeIp: "2001:db8::11", peerIp: "2001:db8::12", enabled: true)) + missingIpv6Vip.setDbvip("2001:db8::200") + def missingVip = new ManagementEndpointData([IPV4, IPV6], missingIpv6Vip) + .selectForTarget(ManagementEndpointData.EndpointType.VIP, IPV6_2) + assert !missingVip.success + assert missingVip.error.globalErrorCode == "ORG_ZSTACK_CORE_PLATFORM_10002" + + ZSha2Info missingIpv6Family = new ZSha2Info() + missingIpv6Family.setIpv4(new ZSha2Info.HaAddressFamily(nodeIp: "192.168.1.11", peerIp: "192.168.1.12", virtualIp: "192.168.1.100", enabled: true)) + missingIpv6Family.setDbvip("2001:db8::200") + assert !new ManagementEndpointData([IPV4, IPV6], missingIpv6Family) + .selectForTarget(ManagementEndpointData.EndpointType.VIP, IPV6_2).success + + ZSha2Info legacyIpv4 = new ZSha2Info(nodeip: "192.168.1.11", dbvip: "192.168.1.100") + ManagementEndpointData legacyEndpoints = new ManagementEndpointData([IPV4], legacyIpv4) + assert legacyEndpoints.selectForTarget(ManagementEndpointData.EndpointType.CANONICAL_NODE, "192.168.1.20").result == "192.168.1.11" + assert legacyEndpoints.selectForTarget(ManagementEndpointData.EndpointType.VIP, "192.168.1.20").result == "192.168.1.100" + + ZSha2Info ipv6OnlyHaRecord = new ZSha2Info() + ipv6OnlyHaRecord.setIpv6(new ZSha2Info.HaAddressFamily(nodeIp: "2001:db8::11", peerIp: "2001:db8::12", virtualIp: "2001:db8::100", enabled: true)) + def missingDefaultIpv4Vip = new ManagementEndpointData([IPV4, IPV6], ipv6OnlyHaRecord) + .selectDefault(ManagementEndpointData.EndpointType.VIP) + assert !missingDefaultIpv4Vip.success + assert missingDefaultIpv4Vip.error.globalErrorCode == "ORG_ZSTACK_CORE_PLATFORM_10002" + } + void testManagementServerSecondaryPropertyRejectsWrongAddressFamily() { withManagementServerIpProperties([ "management.server.ip" : IPV4, diff --git a/test/src/test/groovy/org/zstack/test/integration/core/PlatformManagementEndpointSelectionCase.groovy b/test/src/test/groovy/org/zstack/test/integration/core/PlatformManagementEndpointSelectionCase.groovy new file mode 100644 index 00000000000..c4435c053b9 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/core/PlatformManagementEndpointSelectionCase.groovy @@ -0,0 +1,55 @@ +package org.zstack.test.integration.core + +import org.junit.Test +import org.zstack.core.ManagementEndpointData +import org.zstack.core.Platform +import org.zstack.header.exception.CloudRuntimeException +import org.zstack.utils.zsha2.ZSha2Info + +import java.lang.reflect.Field + +class PlatformManagementEndpointSelectionCase { + private static final String IPV4 = "192.168.1.10" + private static final String IPV6 = "2001:db8::1" + + @Test + void testHaDefaultUsesTheConfiguredNodeFamily() { + ZSha2Info info = new ZSha2Info() + info.setIpv6(new ZSha2Info.HaAddressFamily( + nodeIp: "2001:db8::11", peerIp: "2001:db8::12", virtualIp: "2001:db8::100", enabled: true)) + + ManagementEndpointData endpoints = new ManagementEndpointData([IPV4, IPV6], info) + + assert endpoints.getDefaultEndpoint(ManagementEndpointData.EndpointType.VIP) == null + assert endpoints.getDefaultEndpoint(ManagementEndpointData.EndpointType.CANONICAL_NODE) == null + } + + @Test + void testInvalidConfiguredDefaultDoesNotFallBackToRoute() { + Platform.getManagementServerIp() + String oldManagementIp = System.getProperty("management.server.ip") + try { + System.setProperty("management.server.ip", "not-an-ip") + resetCachedManagementServerIp() + + try { + Platform.getManagementServerIp() + assert false + } catch (CloudRuntimeException ignored) { + } + } finally { + if (oldManagementIp == null) { + System.clearProperty("management.server.ip") + } else { + System.setProperty("management.server.ip", oldManagementIp) + } + resetCachedManagementServerIp() + } + } + + private static void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.setAccessible(true) + field.set(null, null) + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy b/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy new file mode 100644 index 00000000000..fbb1af2c383 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/core/ansible/AnsibleRunnerTargetAwareArgumentsCase.groovy @@ -0,0 +1,205 @@ +package org.zstack.test.integration.core.ansible + +import org.junit.Test +import org.zstack.core.ansible.AnsibleGlobalProperty +import org.zstack.core.ansible.AnsibleRunner +import org.zstack.core.ansible.PrepareAnsible +import org.zstack.core.ansible.RunAnsibleMsg +import org.zstack.core.CoreGlobalProperty +import org.zstack.core.Platform +import org.zstack.core.aspect.AsyncSafeAspect +import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.cloudbus.CloudBusCallBack +import org.zstack.core.componentloader.ComponentLoader +import org.zstack.core.errorcode.ErrorFacade +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.message.NeedReplyMessage +import org.zstack.header.message.MessageReply +import org.zstack.header.rest.RESTFacade + +import java.lang.reflect.Field +import java.util.concurrent.atomic.AtomicReference + +class AnsibleRunnerTargetAwareArgumentsCase { + private static final String IPV4 = "192.168.1.10" + private static final String IPV6 = "2001:db8::1" + private static final String IPV6_TARGET = "2001:db8::2" + private static final int REST_PORT = 8080 + + @Test + void test() { + testTargetAwareEndpointArgumentsUseTheSelectedIpv6Node() + testMissingFamilyFailsBeforeRunAnsibleDispatch() + } + + void testTargetAwareEndpointArgumentsUseTheSelectedIpv6Node() { + withManagementServerIpProperties([ + "management.server.ip" : IPV4, + "management.server.ip6": IPV6, + ]) { + withTemporaryAnsibleInventory { + AtomicReference received = new AtomicReference<>() + AtomicReference result = new AtomicReference<>() + AtomicReference error = new AtomicReference<>() + + runAnsible(IPV6_TARGET, received, result, error) + + assert result.get() == true + assert error.get() == null + assert received.get() != null + def arguments = received.get().deployArguments + assert arguments.pipUrl == "http://[${IPV6}]:${REST_PORT}/zstack/static/pypi/simple" + assert arguments.yumServer == "[${IPV6}]:${REST_PORT}" + assert arguments.trustedHost == IPV6 + assert received.get().targetIp == IPV6_TARGET + } + } + } + + void testMissingFamilyFailsBeforeRunAnsibleDispatch() { + withManagementServerIpProperties([ + "management.server.ip": IPV4, + ]) { + AtomicReference received = new AtomicReference<>() + AtomicReference result = new AtomicReference<>() + AtomicReference error = new AtomicReference<>() + + runAnsible(IPV6_TARGET, received, result, error) + + assert result.get() == null + assert error.get()?.globalErrorCode == "ORG_ZSTACK_CORE_PLATFORM_10001" + assert received.get() == null + } + } + + private void runAnsible(String targetIp, AtomicReference received, + AtomicReference result, AtomicReference error) { + AnsibleRunner runner = new AnsibleRunner() + setField(runner, "restf", [ + getBaseUrl : { String.format("http://127.0.0.1:%d", REST_PORT) }, + getHostName: { "fallback-management-host" as String }, + ] as RESTFacade) + setField(runner, "bus", [ + makeTargetServiceIdByResourceUuid: { NeedReplyMessage msg, String serviceId, String resourceUuid -> }, + send : { NeedReplyMessage msg, CloudBusCallBack callback -> + received.set(msg as RunAnsibleMsg) + callback.run(new MessageReply()) + return null + } + ] as CloudBus) + + boolean oldUnitTestOn = CoreGlobalProperty.UNIT_TEST_ON + try { + CoreGlobalProperty.UNIT_TEST_ON = true + runner.forceRun = true + runner.targetIp = targetIp + runner.targetUuid = Platform.uuid + runner.playBookName = "kvm.yml" + runner.username = "root" + runner.password = "password" + withErrorFacade { + runner.run(new ReturnValueCompletion(null) { + @Override + void success(Boolean returnValue) { + result.set(returnValue) + } + + @Override + void fail(ErrorCode errorCode) { + error.set(errorCode) + } + }) + } + } finally { + CoreGlobalProperty.UNIT_TEST_ON = oldUnitTestOn + } + } + + private void withManagementServerIpProperties(Map properties, Closure closure) { + List managedKeys = [ + "management.server.ip", + "management.server.ip4", + "management.server.ip6", + ] + Map oldValues = [:] + managedKeys.each { key -> oldValues[key] = System.getProperty(key) } + + try { + resetCachedManagementServerIp() + managedKeys.each { key -> System.clearProperty(key) } + properties.each { key, value -> System.setProperty(key, value) } + closure.call() + } finally { + managedKeys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private void withTemporaryAnsibleInventory(Closure closure) { + Field hostsFileField = PrepareAnsible.class.getDeclaredField("hostsFile") + Field hostIpsField = PrepareAnsible.class.getDeclaredField("hostIPs") + hostsFileField.setAccessible(true) + hostIpsField.setAccessible(true) + File oldHostsFile = hostsFileField.get(null) as File + List oldHostIps = hostIpsField.get(null) as List + boolean oldKeepHostsFileInMemory = AnsibleGlobalProperty.KEEP_HOSTS_FILE_IN_MEMORY + File temporaryHostsFile = File.createTempFile("zstack-ansible-hosts", ".tmp") + + try { + hostsFileField.set(null, temporaryHostsFile) + hostIpsField.set(null, []) + AnsibleGlobalProperty.KEEP_HOSTS_FILE_IN_MEMORY = true + closure.call() + } finally { + AnsibleGlobalProperty.KEEP_HOSTS_FILE_IN_MEMORY = oldKeepHostsFileInMemory + hostsFileField.set(null, oldHostsFile) + hostIpsField.set(null, oldHostIps) + temporaryHostsFile.delete() + } + } + + private void setField(Object target, String fieldName, Object value) { + Field field = AnsibleRunner.class.getDeclaredField(fieldName) + field.setAccessible(true) + field.set(target, value) + } + + private void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.setAccessible(true) + field.set(null, null) + } + + private void withErrorFacade(Closure closure) { + ErrorFacade errorFacade = [ + instantiateErrorCode : { Enum code, String details, ErrorCode cause -> new ErrorCode(code.name(), "test", details) }, + stringToInternalError: { String details -> new ErrorCode("TEST", "test", details) } + ] as ErrorFacade + + Field loaderField = Platform.class.getDeclaredField("loader") + loaderField.setAccessible(true) + Object oldLoader = loaderField.get(null) + + AsyncSafeAspect aspect = AsyncSafeAspect.aspectOf() + Field errorFacadeField = AsyncSafeAspect.class.getDeclaredField("errf") + errorFacadeField.setAccessible(true) + ErrorFacade oldErrorFacade = errorFacadeField.get(aspect) as ErrorFacade + try { + loaderField.set(null, [ + getComponent: { Class componentClass -> componentClass == ErrorFacade.class ? errorFacade : null } + ] as ComponentLoader) + errorFacadeField.set(aspect, errorFacade) + closure.call() + } finally { + errorFacadeField.set(aspect, oldErrorFacade) + loaderField.set(null, oldLoader) + } + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy new file mode 100644 index 00000000000..27d906ca80e --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/DualStackAddKvmHostCase.groovy @@ -0,0 +1,160 @@ +package org.zstack.test.integration.kvm.host + +import org.zstack.core.Platform +import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.db.DatabaseFacade +import org.zstack.core.db.Q +import org.zstack.core.rest.RESTFacadeImpl +import org.zstack.header.host.ConnectHostMsg +import org.zstack.header.host.ConnectHostReply +import org.zstack.header.host.CpuArchitecture +import org.zstack.header.host.HostVO +import org.zstack.header.host.HostVO_ +import org.zstack.header.rest.RESTConstant +import org.zstack.header.rest.RESTFacade +import org.zstack.header.tag.SystemTagVO +import org.zstack.header.tag.SystemTagVO_ +import org.zstack.header.zone.ZoneVO +import org.zstack.kvm.KVMHost +import org.zstack.kvm.KVMHostVO +import org.zstack.sdk.AddKVMHostAction +import org.zstack.sdk.ClusterInventory +import org.zstack.sdk.KVMHostInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +import java.lang.reflect.Field + +class DualStackAddKvmHostCase extends SubCase { + private static final String MN_IPV4 = "192.168.1.10" + private static final String MN_IPV6 = "2001:db8::1" + private static final String HOST_IPV6 = "2001:db8::10" + private static final int REST_PORT = 8080 + + private EnvSpec env + private ClusterInventory cluster + private DatabaseFacade dbf + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = HostEnv.noHostBasicEnv() + } + + @Override + void clean() { + env.delete() + } + + @Override + void test() { + env.create { + dbf = bean(DatabaseFacade.class) + cluster = env.inventoryByName("cluster") as ClusterInventory + updateZoneIpVersionToIpv6(cluster.zoneUuid) + prepareConnectHostReply() + + withManagementServerIpProperties([ + "management.server.ip" : MN_IPV4, + "management.server.ip6": MN_IPV6, + ]) { + addIpv6KvmHost() + assertIpv6CallbackPrecheckCommand() + } + } + } + + private void updateZoneIpVersionToIpv6(String zoneUuid) { + SystemTagVO currentTag = Q.New(SystemTagVO.class) + .eq(SystemTagVO_.resourceUuid, zoneUuid) + .eq(SystemTagVO_.resourceType, ZoneVO.simpleName) + .like(SystemTagVO_.tag, "managementNetwork::ipVersion::%") + .find() + assert currentTag != null + + updateSystemTag { + uuid = currentTag.uuid + tag = "managementNetwork::ipVersion::ipv6" + } + } + + private void prepareConnectHostReply() { + env.message(ConnectHostMsg.class) { ConnectHostMsg msg, CloudBus bus -> + KVMHostVO host = dbf.findByUuid(msg.uuid, KVMHostVO.class) + host.setArchitecture(CpuArchitecture.x86_64.name()) + host.setOsDistribution("centos") + host.setOsRelease("core") + host.setOsVersion("7.6.1810") + dbf.update(host) + bus.reply(msg, new ConnectHostReply()) + } + } + + private void addIpv6KvmHost() { + AddKVMHostAction action = new AddKVMHostAction() + action.sessionId = adminSession() + action.resourceUuid = Platform.uuid + action.clusterUuid = cluster.uuid + action.managementIp = HOST_IPV6 + action.name = "dual-stack-kvm" + action.username = "root" + action.password = "password" + + def result = action.call() + + assert result.error == null + assert (result.value.inventory as KVMHostInventory).managementIp == HOST_IPV6 + assert Q.New(HostVO.class).eq(HostVO_.managementIp, HOST_IPV6).isExists() + } + + private void assertIpv6CallbackPrecheckCommand() { + RESTFacade restf = [ + buildCallbackUrl: { String host -> RESTFacadeImpl.buildCallbackUrl(host, REST_PORT, "zstack") } + ] as RESTFacade + + def command = KVMHost.buildManagementNodeCallbackCheckCommand(HOST_IPV6, restf) + String callbackUrl = RESTFacadeImpl.buildCallbackUrl(MN_IPV6, REST_PORT, "zstack") + + assert command.success + assert command.result.contains("curl --connect-timeout 10 --max-time 15 ${callbackUrl}") + assert command.result.contains("wget --spider -q --connect-timeout=10 --read-timeout=10 --tries=1 ${callbackUrl}") + assert !command.result.contains("http://${MN_IPV4}:${REST_PORT}/zstack${RESTConstant.CALLBACK_PATH}") + } + + private void withManagementServerIpProperties(Map properties, Closure closure) { + List managedKeys = [ + "management.server.ip", + "management.server.ip4", + "management.server.ip6", + ] + Map oldValues = [:] + managedKeys.each { key -> oldValues[key] = System.getProperty(key) } + + try { + resetCachedManagementServerIp() + managedKeys.each { key -> System.clearProperty(key) } + properties.each { key, value -> System.setProperty(key, value) } + closure.call() + } finally { + managedKeys.each { key -> + if (oldValues[key] == null) { + System.clearProperty(key) + } else { + System.setProperty(key, oldValues[key]) + } + } + resetCachedManagementServerIp() + } + } + + private void resetCachedManagementServerIp() { + Field field = Platform.class.getDeclaredField("managementServerIp") + field.setAccessible(true) + field.set(null, null) + } +} diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 9fc175de2c1..bcc3bbf8903 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -2950,6 +2950,12 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_CORE_ANSIBLE_10007 = "ORG_ZSTACK_CORE_ANSIBLE_10007"; + public static final String ORG_ZSTACK_CORE_PLATFORM_10000 = "ORG_ZSTACK_CORE_PLATFORM_10000"; + + public static final String ORG_ZSTACK_CORE_PLATFORM_10001 = "ORG_ZSTACK_CORE_PLATFORM_10001"; + + public static final String ORG_ZSTACK_CORE_PLATFORM_10002 = "ORG_ZSTACK_CORE_PLATFORM_10002"; + public static final String ORG_ZSTACK_SCHEDULER_10000 = "ORG_ZSTACK_SCHEDULER_10000"; public static final String ORG_ZSTACK_SCHEDULER_10001 = "ORG_ZSTACK_SCHEDULER_10001"; diff --git a/utils/src/main/java/org/zstack/utils/zsha2/ZSha2Info.java b/utils/src/main/java/org/zstack/utils/zsha2/ZSha2Info.java index 3754b77abd9..a2e597a57a1 100644 --- a/utils/src/main/java/org/zstack/utils/zsha2/ZSha2Info.java +++ b/utils/src/main/java/org/zstack/utils/zsha2/ZSha2Info.java @@ -11,6 +11,47 @@ public class ZSha2Info { private int peerport; private boolean isMaster; private String execUser; + private HaAddressFamily ipv4; + private HaAddressFamily ipv6; + + public static class HaAddressFamily { + private boolean enabled; + private String virtualIp; + private String nodeIp; + private String peerIp; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getVirtualIp() { + return virtualIp; + } + + public void setVirtualIp(String virtualIp) { + this.virtualIp = virtualIp; + } + + public String getNodeIp() { + return nodeIp; + } + + public void setNodeIp(String nodeIp) { + this.nodeIp = nodeIp; + } + + public String getPeerIp() { + return peerIp; + } + + public void setPeerIp(String peerIp) { + this.peerIp = peerIp; + } + } public String getNodeip() { return nodeip; @@ -63,4 +104,20 @@ public String getExecUser() { public void setExecUser(String execUser) { this.execUser = execUser; } + + public HaAddressFamily getIpv4() { + return ipv4; + } + + public void setIpv4(HaAddressFamily ipv4) { + this.ipv4 = ipv4; + } + + public HaAddressFamily getIpv6() { + return ipv6; + } + + public void setIpv6(HaAddressFamily ipv6) { + this.ipv6 = ipv6; + } }