From 4aa9008eb97b63e6d91e5ce83749c6e33bc1ed74 Mon Sep 17 00:00:00 2001 From: lianghy Date: Tue, 20 Jan 2026 18:07:31 +0800 Subject: [PATCH 01/53] [conf]: bump version to 5.4.10 DBImpact Resolves: ZSTAC-81425 Change-Id: I627a6576686f6c6972636d67786e77716e637471 --- VERSION | 2 +- conf/db/upgrade/V5.4.10__schema.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 conf/db/upgrade/V5.4.10__schema.sql diff --git a/VERSION b/VERSION index b7c3a02aec6..d406ff05502 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=5 MINOR=4 -UPDATE=7 +UPDATE=10 diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql new file mode 100644 index 00000000000..e69de29bb2d From 11767587c9f03bd1ef4cff19ab3f3245e6f6133d Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Wed, 24 Jun 2026 18:26:46 +0900 Subject: [PATCH 02/53] [search]: fix zql field query total Use entity alias for simple return-with-total counts so nullable selected fields do not turn totals into zero. Keep explicit count(field) and distinct(field) behavior unchanged. Add CountGroupByCase coverage for nullable first selected field. Resolves: ZSTAC-78110 Change-Id: I3b437bc686c345ba17baf256a338ce88209571f8 (cherry picked from commit 0dfe389e6d5560dbc8e899a4330de039c84e2791) --- .../ast/visitors/plugin/SimpleCountPlugin.java | 16 ++++++++-------- .../test/integration/zql/CountGroupByCase.groovy | 8 ++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java index 6627311adb9..10ebab413b4 100644 --- a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java +++ b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java @@ -1,10 +1,10 @@ package org.zstack.zql.ast.visitors.plugin; -import org.apache.commons.lang.StringUtils; import org.zstack.header.zql.ASTNode; import java.util.List; -import java.util.stream.Collectors; + +import static org.zstack.zql.ast.visitors.constants.MySqlKeyword.isDistinct; public class SimpleCountPlugin extends AbstractQueryVisitorPlugin { public SimpleCountPlugin() { @@ -24,13 +24,13 @@ public String selectTarget() { String queryTarget; List fieldNames = super.targetFields(); - if (fieldNames.isEmpty()) { - queryTarget = entityAlias; - } else { + if (getClauseType() == ClauseType.COUNT || isDistinct(node.getTarget().getFunction())) { // TODO: Compatibility changes: hql count do not support multiple fields, even if distinct modified. - queryTarget = String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + queryTarget = fieldNames.isEmpty() ? entityAlias : String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + return String.format("count(%s)", String.format(functions(), queryTarget)); + } else { + queryTarget = entityAlias; + return String.format("count(%s)", queryTarget); } - - return String.format("count(%s)", String.format(functions(), queryTarget)); } } diff --git a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy index 72cf1d6ebb8..e37785f7674 100644 --- a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy @@ -36,6 +36,7 @@ class CountGroupByCase extends SubCase { env.create { testCountGroupBy() testQuery() + testFieldQueryTotalWithNullableFirstField() testGroupLimit() testEmpty() } @@ -80,6 +81,13 @@ class CountGroupByCase extends SubCase { assert ret2.total == 102 } + void testFieldQueryTotalWithNullableFirstField() { + def ret = ZQL.fromString("query instanceoffering.description,uuid return with (total) limit 100").getSingleResult() as ZQLQueryReturn + assert ret.inventories.size() == 100 + assert ret.total == 102 + assert ret.inventories.every { it.description == null } + } + void testGroupLimit() { def ret = ZQL.fromString("count instanceoffering group by name,cpuNum limit 2").getSingleResult() as ZQLQueryReturn assert ret.inventoryCounts.size() == 2 From 4ef0c8f8ec8166c0b05e184c5da7c8625c3d91dd Mon Sep 17 00:00:00 2001 From: "dejing.liu" Date: Thu, 15 Jan 2026 15:25:01 +0800 Subject: [PATCH 03/53] [kvm]: force virtio driver for windows vhostuser nic The vhost-user interface (used by OVS-DPDK) strictly requires the 'virtio-net' frontend driver in Libvirt/QEMU. However, ZStack defaults Windows NIC drivers to 'e1000' (or other emulated models) when the image platform is 'Windows', causing VM start failure with Libvirt error: "vhostuser requires the virtio-net* frontend". This fix forces the NIC driver to 'virtio' specifically when the interface type is 'vhostuser' on Windows platforms, regardless of the default platform settings. Satisfying both the legacy boot requirements of Windows and the vhost-user network requirements. Resolves: ZSTAC-76013 Change-Id: I737a6e637273796d776e726e7969786c7a6a6478 --- .../kvm/src/main/java/org/zstack/kvm/KVMHost.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index ab8c29640f3..cd58e7b58e9 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -4498,6 +4498,21 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi } nics = nics.stream().sorted(Comparator.comparing(NicTO::getDeviceId)).collect(Collectors.toList()); cmd.setNics(nics); + if ((platform.equals(ImagePlatform.Windows.toString()) || + platform.equals(ImagePlatform.WindowsVirtio.toString()))) { + if (cmd.getNics() != null) { + for (NicTO nicTo : cmd.getNics()) { + if (nicTo.getType() != null && + nicTo.getType().toLowerCase().contains(VmNicType.VmNicSubType.VHOSTUSER.toString().toLowerCase()) && + !VmNicDriverType.VIRTIO.toString().equalsIgnoreCase(nicTo.getDriverType())) { + logger.warn(String.format("Force updating nic[%s] driver from [%s] to [virtio] for " + + "Windows vhostuser compatibility.", + nicTo.getUuid(), nicTo.getDriverType())); + nicTo.setDriverType(VmNicDriverType.VIRTIO.toString()); + } + } + } + } for (VmInstanceSpec.CdRomSpec cdRomSpec : spec.getCdRomSpecs()) { CdRomTO cdRomTO = new CdRomTO(); From ada4787435dc7a56902fc314d15b836af3fc1734 Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Thu, 14 May 2026 22:45:49 +0800 Subject: [PATCH 04/53] [query]: limit API query concurrency Route AutoQuery handling through the query sync queue so bursty GraphQL refreshes cannot run unlimited APIQueryMessage requests concurrently. Resolves: ZSTAC-80742 Change-Id: If2c052e539526cd0511e037d1718838bdb65a7a3 --- .../org/zstack/query/QueryFacadeImpl.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java index 59aa7ce7e7d..3e110ae6dd1 100755 --- a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java +++ b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java @@ -347,7 +347,7 @@ private Method getReplySetter(AutoQuery at) { } } - private void handle(APIQueryMessage msg) { + private APIQueryReply doQuery(APIQueryMessage msg) { AutoQuery at = autoQueryMap.get(msg.getClass()); if (at == null) { at = msg.getClass().getAnnotation(AutoQuery.class); @@ -372,7 +372,7 @@ private void handle(APIQueryMessage msg) { if (result.inventories != null) { replySetter.invoke(reply, result.inventories); } - bus.reply(msg, reply); + return reply; } catch (OperationFailureException of) { throw of; } catch (Exception e) { @@ -380,6 +380,39 @@ private void handle(APIQueryMessage msg) { } } + private void handle(APIQueryMessage msg) { + thdf.syncSubmit(new SyncTask() { + @Override + public Void call() { + APIQueryReply reply; + try { + reply = doQuery(msg); + } catch (OperationFailureException of) { + reply = new APIQueryReply(); + reply.setError(of.getErrorCode()); + } + + bus.reply(msg, reply); + return null; + } + + @Override + public String getName() { + return getSyncSignature(); + } + + @Override + public String getSyncSignature() { + return "api-query"; + } + + @Override + public int getSyncLevel() { + return getQuerySyncLevel(); + } + }); + } + private String toZQLConditionString(QueryCondition c) { // make every condition value as string, the ZQL // will put them in right type because it knows every field's type From f25df0a6e9a85e47d44893c54ce69638c5dcb0d9 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 2 Jul 2026 12:14:53 +0800 Subject: [PATCH 05/53] [gpu]: add gpu normalized model schema Backport the normalizedModelName schema required by premium GPU spec deduplication to 5.4.10. This is the zstack companion change for the premium ZSTAC-86476 backport. Constraint: ZSTAC-86476 is the 5.4.10 clone of ZSTAC-75319. Confidence: high Scope-risk: narrow Tested: ./runMavenProfile premium in pr-build-bin Docker with zstack and premium mounted Not-tested: local stability case execution Resolves: ZSTAC-86476 Change-Id: Ic77e24e974a99842c40f29e4bd72851e3833795b --- conf/db/upgrade/V5.4.10__schema.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql index e69de29bb2d..ab73deda0f6 100644 --- a/conf/db/upgrade/V5.4.10__schema.sql +++ b/conf/db/upgrade/V5.4.10__schema.sql @@ -0,0 +1,3 @@ +-- ZSTAC-86476: Add normalizedModelName column for GPU spec dedup +CALL ADD_COLUMN('GpuDeviceSpecVO', 'normalizedModelName', 'VARCHAR(255)', 1, NULL); +CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalizedModelName'); From b5d630f15533ac69d3aed2ac50e917c62f2526bd Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 2 Jul 2026 14:37:09 +0800 Subject: [PATCH 06/53] [search]: stabilize paginated query order Backport ZSTAC-74748 pagination tiebreaker from 5.5.6/5.5.28 to 5.4.10. Paginated LIMIT/OFFSET queries can return duplicate resources when the requested sort field has duplicate values. Append the primary key as a deterministic tiebreaker in both ZQL and legacy HQL query paths, and skip hidden PK fields that ZQL cannot order by. Source-Commits: 284d04988a, b1156d87cc Tested: git diff --check upstream/5.4.10..HEAD Resolves: ZSTAC-86475 Change-Id: I14bef1d630a70711ef2605ea95fd37cb86e5273c --- .../zstack/query/MysqlQueryBuilderImpl3.java | 10 +++++- .../org/zstack/query/QueryFacadeImpl.java | 31 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java index 101684ad18c..df5e0e5a96c 100755 --- a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java +++ b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java @@ -729,7 +729,15 @@ String toJpql(boolean isCount) { throw new IllegalArgumentException(String.format("illegal sortBy[%s], entity[%s] doesn't have this field", msg.getSortBy(), info.entityClass.getName())); } - ret = String.format("%s order by %s.%s %s", ret, entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + String orderBy = String.format("order by %s.%s %s", entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + if (info.primaryKey != null && !msg.getSortBy().equals(info.primaryKey)) { + orderBy += String.format(", %s.%s ASC", entityName, info.primaryKey); + } + ret = String.format("%s %s", ret, orderBy); + } else if (msg.getStart() != null || msg.getLimit() != null) { + if (info.primaryKey != null) { + ret = String.format("%s order by %s.%s ASC", ret, entityName, info.primaryKey); + } } if (msg.getGroupBy() != null) { diff --git a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java index 59aa7ce7e7d..6908e4f2678 100755 --- a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java +++ b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java @@ -30,6 +30,7 @@ import org.zstack.utils.*; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; +import org.zstack.core.db.EntityMetadata; import org.zstack.zql.ZQL; import org.zstack.zql.ZQLContext; import org.zstack.zql.ZQLQueryReturn; @@ -482,7 +483,17 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { } if (msg.getSortBy() != null) { - sb.add(String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection())); + String orderBy = String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection()); + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null && !msg.getSortBy().equals(pkFieldName)) { + orderBy += String.format(", %s asc", pkFieldName); + } + sb.add(orderBy); + } else if (!msg.isCount() && (msg.getLimit() != null || msg.getStart() != null)) { + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null) { + sb.add(String.format("order by %s asc", pkFieldName)); + } } if (msg.getLimit() != null) { @@ -514,6 +525,24 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { return result; } + private String getPrimaryKeyFieldNameSafely(Class inventoryClass) { + try { + ZQLMetadata.InventoryMetadata meta = ZQLMetadata.findInventoryMetadata( + ZQL.queryTargetNameFromInventoryClass(inventoryClass)); + Class voClass = meta.inventoryAnnotation.mappingVOClass(); + Field pkField = EntityMetadata.getPrimaryKeyField(voClass); + String pkName = pkField.getName(); + if (!meta.selfInventoryFieldNames.contains(pkName)) { + return null; + } + return pkName; + } catch (Exception e) { + logger.trace(String.format("cannot resolve PK for %s, skip pagination tiebreaker: %s", + inventoryClass.getSimpleName(), e.getMessage())); + return null; + } + } + private QueryBelongFilter validateFilterNameAndGetExp(String filterName) { if (filterName.split(":").length < 2) { throw new OperationFailureException(argerr("filterName must be formatted as [filterType:condition(s)]")); From dc8697598dc3a63124aaa2e1951c851582ffc152 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Mon, 11 May 2026 21:45:42 +0800 Subject: [PATCH 07/53] [kvm]: skip known systemd timeout on reconnect Existing host reconnect can fail before kvmagent echo. Ansible masks libvirt sockets with systemctl during deploy. If host systemd D-Bus is stuck, that optional step times out. Continue only for this known mask timeout on reconnect. New host deploy and other ansible failures still fail. Resolves: ZSTAC-86349 Change-Id: I0ef4a535065ff797c9e4cfae5b39c2daa321a4cc --- .../src/main/java/org/zstack/kvm/KVMHost.java | 7 ++++ .../java/org/zstack/kvm/KVMHostUtils.java | 33 +++++++++++++++++++ .../org/zstack/test/kvm/KVMHostUtilsTest.java | 25 ++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index ab8c29640f3..fc0bb962f9a 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5803,6 +5803,13 @@ public void success(Boolean run) { @Override public void fail(ErrorCode errorCode) { + if (KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(info.isNewAdded(), errorCode)) { + logger.warn(String.format( + "kvm ansible failed to mask libvirt sockets because systemd dbus timed out on existing host[uuid:%s, ip:%s], continue reconnect and verify kvmagent, error: %s", + self.getUuid(), self.getManagementIp(), errorCode)); + trigger.next(); + return; + } trigger.fail(errorCode); } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java index cf6d16e7560..3b9979caeb3 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java @@ -2,6 +2,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.network.l2.*; import org.zstack.header.tag.SystemTagVO; import org.zstack.header.tag.SystemTagVO_; @@ -13,6 +14,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; /** @@ -21,6 +23,37 @@ public class KVMHostUtils { private static final CLogger logger = CLoggerImpl.getLogger(KVMHostUtils.class); + public static boolean shouldContinueReconnectOnAnsibleFailure(boolean isNewAdded, ErrorCode errorCode) { + return !isNewAdded && isLibvirtSocketMaskSystemdTimeout(errorCode); + } + + public static boolean isLibvirtSocketMaskSystemdTimeout(ErrorCode errorCode) { + String errorText = collectErrorText(errorCode).toLowerCase(Locale.ROOT); + return errorText.contains("systemctl mask") + && errorText.contains("libvirtd.socket") + && errorText.contains("org.freedesktop.systemd1") + && errorText.contains("timed out") + && (errorText.contains("failed to get properties") + || errorText.contains("failed to activate service")); + } + + private static String collectErrorText(ErrorCode errorCode) { + StringBuilder sb = new StringBuilder(); + ErrorCode cursor = errorCode; + while (cursor != null) { + appendIfNotNull(sb, cursor.getDetails()); + appendIfNotNull(sb, cursor.getDescription()); + cursor = cursor.getCause(); + } + return sb.toString(); + } + + private static void appendIfNotNull(StringBuilder sb, String text) { + if (text != null) { + sb.append(text).append('\n'); + } + } + /** * Get normalized bridge name for l2 network, which at most has 15 chars. * - if l2 network has L2_BRIDGE_NAME tag, then return it's value directly; diff --git a/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java new file mode 100644 index 00000000000..2e0e06c7dfc --- /dev/null +++ b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java @@ -0,0 +1,25 @@ +package org.zstack.test.kvm; + +import org.junit.Assert; +import org.junit.Test; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.kvm.KVMHostUtils; + +public class KVMHostUtilsTest { + @Test + public void zstac86349_continueReconnectOnLibvirtSocketMaskSystemdTimeout() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl mask libvirtd.socket libvirtd-ro.socket libvirtd-admin.socket libvirtd-tls.socket libvirtd-tcp.socket failed! stderr: Failed to get properties: Failed to activate service 'org.freedesktop.systemd1': timed out (service_start_timeout=25000ms)"); + + Assert.assertTrue(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(true, error)); + } + + @Test + public void zstac86349_doNotContinueReconnectOnOtherAnsibleFailures() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl restart libvirtd failed! stderr: Job for libvirtd.service failed"); + + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + } +} From e6d3189d79d0cf85e121703c0709bc99d658cca9 Mon Sep 17 00:00:00 2001 From: Jiajun Xu Date: Thu, 2 Jul 2026 17:08:23 +0800 Subject: [PATCH 08/53] [identity]: restrict APIQueryAccountMsg to admin-only in RBAC Resolves: ZSTAC-86350 Change-Id: I024ae51da6e5784fb336d8fbd5a4aeab44500565 --- .../org/zstack/header/identity/RBACInfo.java | 1 + .../TestAPIQueryAccountMsgRBACCase.java | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java diff --git a/header/src/main/java/org/zstack/header/identity/RBACInfo.java b/header/src/main/java/org/zstack/header/identity/RBACInfo.java index 609a916d20c..81223064cd9 100755 --- a/header/src/main/java/org/zstack/header/identity/RBACInfo.java +++ b/header/src/main/java/org/zstack/header/identity/RBACInfo.java @@ -12,6 +12,7 @@ public void permissions() { .name("identity") .adminOnlyAPIs( APICreateAccountMsg.class, + APIQueryAccountMsg.class, APIShareResourceMsg.class, APIRevokeResourceSharingMsg.class, APIUpdateQuotaMsg.class, diff --git a/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java new file mode 100644 index 00000000000..15216cb50bb --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java @@ -0,0 +1,43 @@ +package org.zstack.test.unittest.identity; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.zstack.header.identity.APIQueryAccountMsg; +import org.zstack.header.identity.RBACInfo; +import org.zstack.header.identity.rbac.RBAC; + +public class TestAPIQueryAccountMsgRBACCase { + + @Before + public void setUp() { + RBAC.permissions.clear(); + } + + @Test + public void testAPIQueryAccountMsgIsAdminOnly() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + boolean isAdminOnly = RBAC.isAdminOnlyAPI(APIQueryAccountMsg.class.getName()); + Assert.assertTrue( + "APIQueryAccountMsg should be admin-only to prevent privilege escalation, " + + "but it is currently accessible to normal IAM users via the wildcard normalAPIs pattern", + isAdminOnly + ); + } + + @Test + public void testAPIQueryAccountMsgNotInNormalAPIs() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + String apiName = APIQueryAccountMsg.class.getName(); + boolean isInNormalOnly = RBAC.permissions.stream() + .anyMatch(p -> p.getNormalAPIs().contains(apiName) && !p.getAdminOnlyAPIs().contains(apiName)); + Assert.assertFalse( + "APIQueryAccountMsg should NOT be accessible as a normal API", + isInNormalOnly + ); + } +} From 7e46fa568a88bff5b89d23bbecb2949f7aadbb8f Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 3 Jul 2026 10:43:56 +0800 Subject: [PATCH 09/53] [mn]: synchronize hash ring operations Backport the management node hash ring fix to 5.4.10. 1. Why? Management node lifecycle reconciliation can race with callbacks. This can cause NodeHash or Nodes inconsistency and task stalling. 2. How? Synchronize hash ring operations and use defensive copies. Add lifecycle reconciliation locking and two-round missing checks. 3. Side effects? Management node lifecycle operations are serialized by a local lock. # Summary of changes (by module): - core: synchronize resource destination hash ring operations. - portal: serialize management node lifecycle reconciliation. Source-Commit: 66ab572b3c0ff62d243518eb9808dc85aa7ea6ac Related: ZSTAC-86294, ZSTAC-78182 Change-Id: I3ba21ffc7b8f1284e17d50da7b027ffdad70828b --- .../ResourceDestinationMakerImpl.java | 28 ++--- .../ManagementNodeManagerImpl.java | 101 ++++++++++++------ 2 files changed, 82 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java index 08a776f1db2..9ead578395d 100755 --- a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java +++ b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java @@ -27,27 +27,27 @@ public class ResourceDestinationMakerImpl implements ManagementNodeChangeListene private DatabaseFacade dbf; @Override - public void nodeJoin(ManagementNodeInventory inv) { + public synchronized void nodeJoin(ManagementNodeInventory inv) { nodeHash.add(inv.getUuid()); nodes.put(inv.getUuid(), new NodeInfo(inv)); } @Override - public void nodeLeft(ManagementNodeInventory inv) { + public synchronized void nodeLeft(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iAmDead(ManagementNodeInventory inv) { + public synchronized void iAmDead(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iJoin(ManagementNodeInventory inv) { + public synchronized void iJoin(ManagementNodeInventory inv) { List lst = Q.New(ManagementNodeVO.class).list(); lst.forEach((ManagementNodeVO node) -> { nodeHash.add(node.getUuid()); @@ -56,7 +56,7 @@ public void iJoin(ManagementNodeInventory inv) { } @Override - public String makeDestination(String resourceUuid) { + public synchronized String makeDestination(String resourceUuid) { String nodeUuid = nodeHash.get(resourceUuid); if (nodeUuid == null) { throw new CloudRuntimeException("Cannot find any available management node to send message"); @@ -66,18 +66,18 @@ public String makeDestination(String resourceUuid) { } @Override - public boolean isManagedByUs(String resourceUuid) { + public synchronized boolean isManagedByUs(String resourceUuid) { String nodeUuid = makeDestination(resourceUuid); return nodeUuid.equals(Platform.getManagementServerId()); } @Override - public Collection getManagementNodesInHashRing() { - return nodeHash.getNodes(); + public synchronized Collection getManagementNodesInHashRing() { + return new ArrayList<>(nodeHash.getNodes()); } @Override - public NodeInfo getNodeInfo(String nodeUuid) { + public synchronized NodeInfo getNodeInfo(String nodeUuid) { NodeInfo info = nodes.get(nodeUuid); if (info == null) { ManagementNodeVO vo = dbf.findByUuid(nodeUuid, ManagementNodeVO.class); @@ -93,17 +93,17 @@ public NodeInfo getNodeInfo(String nodeUuid) { } @Override - public Collection getAllNodeInfo() { - return nodes.values(); + public synchronized Collection getAllNodeInfo() { + return new ArrayList<>(nodes.values()); } @Override - public int getManagementNodeCount() { - return nodes.values().size(); + public synchronized int getManagementNodeCount() { + return nodes.size(); } - public boolean isNodeInCircle(String nodeId) { + public synchronized boolean isNodeInCircle(String nodeId) { return nodeHash.hasNode(nodeId); } } diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java index 747a154919b..88bef7d8302 100755 --- a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java +++ b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java @@ -74,6 +74,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -106,6 +107,15 @@ public class ManagementNodeManagerImpl extends AbstractService implements Manage // A dictionary (nodeId -> ManagementNodeInventory) of joined management Node final private Map joinedManagementNodes = new ConcurrentHashMap<>(); + // Lock to serialize lifecycle events from heartbeat reconciliation and canonical event callbacks, + // preventing race conditions where a nodeJoin event is immediately followed by a stale nodeLeft + // from the heartbeat thread, or vice versa. See ZSTAC-77711. + private final Object lifecycleLock = new Object(); + + // Track nodes found in hash ring but missing from DB. Only call nodeLeft after a node + // is missing for two consecutive heartbeat cycles, to avoid removing nodes that just joined. + private final Set suspectedMissingFromDb = new HashSet<>(); + private static int NODE_STARTING = 0; private static int NODE_RUNNING = 1; private static int NODE_FAILED = -1; @@ -367,12 +377,16 @@ protected void run(Map tokens, Object data) { ManagementNodeLifeCycleData d = (ManagementNodeLifeCycleData) data; - if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeJoin(d.getInventory()); - } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeLeft(d.getInventory()); - } else { - throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + synchronized (lifecycleLock) { + if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { + // Clear from suspected set since the node is confirmed alive + suspectedMissingFromDb.remove(d.getInventory().getUuid()); + nodeLifeCycle.nodeJoin(d.getInventory()); + } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { + nodeLifeCycle.nodeLeft(d.getInventory()); + } else { + throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + } } } }; @@ -853,34 +867,55 @@ private void checkAllNodesHealth() { Set nodeUuidsInDb = nodesInDb.stream().map(ManagementNodeVO::getUuid).collect(Collectors.toSet()); - // When a node is dying, we may not receive the the dead notification because the message bus may be also dead - // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out - destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { - if (!nodeUuidsInDb.contains(nodeUuid)) { - logger.warn(String.format("found that a management node[uuid:%s] had no heartbeat in database but still in our hash ring," + - "notify that it's dead", nodeUuid)); - ManagementNodeInventory inv = new ManagementNodeInventory(); - inv.setUuid(nodeUuid); - inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); - - nodeLifeCycle.nodeLeft(inv); - } - }); - - // check if any node missing in our hash ring - nodesInDb.forEach(n -> { - if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { - return; - } - - new Runnable() { - @Override - @AsyncThread - public void run() { - nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + // Reconcile hash ring with DB under lifecycleLock to prevent race with + // canonical event callbacks (nodeJoin/nodeLeft). See ZSTAC-77711. + synchronized (lifecycleLock) { + // When a node is dying, we may not receive the dead notification because the message bus may be also dead + // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out. + // Use two-round confirmation: first round marks as suspected, second round actually removes. + Set currentSuspected = new HashSet<>(); + destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { + if (!nodeUuidsInDb.contains(nodeUuid)) { + if (suspectedMissingFromDb.contains(nodeUuid)) { + // Second consecutive detection — confirmed missing, remove from hash ring + logger.warn(String.format("management node[uuid:%s] confirmed missing from database for two consecutive" + + " heartbeat cycles, removing from hash ring", nodeUuid)); + ManagementNodeInventory inv = new ManagementNodeInventory(); + inv.setUuid(nodeUuid); + try { + inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); + } catch (Exception e) { + logger.warn(String.format("cannot get node info for node[uuid:%s], use empty hostname", nodeUuid)); + } + + nodeLifeCycle.nodeLeft(inv); + } else { + // First detection — mark as suspected, defer removal to next cycle + logger.warn(String.format("management node[uuid:%s] not found in database but still in hash ring," + + " marking as suspected (will remove on next heartbeat if still missing)", nodeUuid)); + currentSuspected.add(nodeUuid); + } } - }.run(); - }); + }); + // Update suspected set: only keep nodes that are newly suspected this round + suspectedMissingFromDb.clear(); + suspectedMissingFromDb.addAll(currentSuspected); + + // check if any node missing in our hash ring + nodesInDb.forEach(n -> { + if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { + return; + } + + new Runnable() { + @Override + @AsyncThread + public void run() { + nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + } + }.run(); + }); + } } @Override From d2b599dd5475d773a4235490eef93c35a1679a11 Mon Sep 17 00:00:00 2001 From: "yaohua.wu" Date: Fri, 3 Jul 2026 11:57:16 +0800 Subject: [PATCH 10/53] [externalStorage]: prevent split-brain on deactivate fail Backport external-storage split-brain guards to 5.4.10. 1. Why? Starting a VM on a new host may continue even when the old storage client still holds the volume. For xinfini Vhost volumes this can leave two QEMU processes using the same bdev. 2. How? Report xinfini deactivate failures instead of swallowing them, verify the bdev is really deleted, re-check active clients after deactivate success, and make blacklist failures propagate out of beforeStartVmOnKvm. 3. Side effects? Deactivate failure or a still-active old client now aborts VM start for that volume instead of silently continuing. Happy-path deactivate keeps the previous behavior. # Summary of changes (by module): - header: make PrimaryStorageNodeSvc.blacklist synchronous. - externalStorage: post-check active clients and propagate blacklist failures. - xinfini: surface deactivate/delete failures and abort unsupported isolation. - expon/zbs: adapt blacklist implementations to the synchronous signature. Related: ZSTAC-86444 Related: ZSTAC-85124 Change-Id: I59812c6d68eec6533e48a8c9cf05b7c55d6b85eb --- .../addon/primary/PrimaryStorageNodeSvc.java | 2 +- .../zstack/expon/ExponStorageController.java | 3 +- .../kvm/ExternalPrimaryStorageKvmFactory.java | 18 +++++-- .../org/zstack/xinfini/XInfiniApiHelper.java | 5 ++ .../xinfini/XInfiniStorageController.java | 53 +++++++++++++------ .../storage/zbs/ZbsStorageController.java | 3 +- 6 files changed, 61 insertions(+), 23 deletions(-) diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java index f292e694183..2ab0bcb4653 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java @@ -13,7 +13,7 @@ public interface PrimaryStorageNodeSvc { void deactivate(String installPath, String protocol, ActiveVolumeClient client, Completion comp); - void blacklist(String installPath, String protocol, HostInventory h, Completion comp); + void blacklist(String installPath, String protocol, HostInventory h); String getActivePath(BaseVolumeInfo v, HostInventory h, boolean shareable); BaseVolumeInfo getActiveVolumeInfo(String activePath, HostInventory h, boolean shareable); diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index 2a3762f98ff..8cb331a91de 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -614,14 +614,13 @@ public void cleanActiveRecord(ImageCacheInventory cache) { } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { + public void blacklist(String installPath, String protocol, HostInventory h) { logger.debug(String.format("blacklisting volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); UssGatewayModule uss = getUssGateway(VolumeProtocol.valueOf(protocol), h.getManagementIp()); VolumeModule exponVol = apiHelper.getVolume(getVolIdFromPath(installPath)); apiHelper.addVolumePathToBlacklist(buildExponVolumeBoundPath(uss, exponVol.getVolumeName())); - comp.success(); } @Override diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index c6eb04a8557..90aca223f1e 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -334,6 +334,7 @@ public void done(ErrorCodeList errorCodeList) { @Override public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { List vols = getManagerExclusiveVolume(spec); + ErrorCode[] deactivateErr = new ErrorCode[1]; for (VolumeInventory vol : vols) { PrimaryStorageNodeSvc nodeSvc = extPsFactory.getNodeSvc(vol.getPrimaryStorageUuid()); @@ -359,20 +360,31 @@ public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAg clientHost.getUuid(), clientHost.getManagementIp(), host.getUuid(), host.getManagementIp())); + deactivateErr[0] = null; nodeSvc.deactivate(vol.getInstallPath(), vol.getProtocol(), client, new Completion(null) { @Override public void success() { + boolean stillActive = nodeSvc.getActiveClients(vol.getInstallPath(), vol.getProtocol()).stream() + .anyMatch(c -> client.getManagerIp().equals(c.getManagerIp())); + if (stillActive) { + deactivateErr[0] = operr("deactivate reported success but volume[uuid:%s, installPath:%s] is still active on host[uuid:%s, ip:%s]", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp()); + return; + } logger.info(String.format("successfully deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]", vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); } @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s], add it to blacklist", - vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); - nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost), new NopeCompletion()); + deactivateErr[0] = errorCode; } }); + if (deactivateErr[0] != null) { + logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]: %s, add it to blacklist", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp(), deactivateErr[0].getDetails())); + nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost)); + } } } }); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java index 3a90db3b043..2177283b48b 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java @@ -472,6 +472,11 @@ public void deleteBdcBdev(int bdevId, int bdcId) { } else { retryUtilResourceDeleted(gReq, GetBdcBdevResponse.class); } + + if (!call(gReq, GetBdcBdevResponse.class).resourceIsDeleted()) { + throw new OperationFailureException(operr("bdev[id:%s] still exists after deletion polling on bdc[id:%s], " + + "the old storage client may still hold the volume", bdevId, bdcId)); + } } public VolumeModule rollbackSnapshot(int volId, int snapId) { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java index 267d55f4039..53c8cca1751 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java @@ -202,18 +202,23 @@ public synchronized ActiveVolumeTO activeIscsiVolume(String clientIp, String cli public void deactivate(String installPath, String protocol, HostInventory h, Completion comp) { logger.debug(String.format("deactivating volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); - if (VolumeProtocol.Vhost.toString().equals(protocol)) { - deactivateVhost(installPath, h); - comp.success(); - return; - } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { - // iscsi target is shared by all hosts, we cannot control one volume on one host for now. - deactivateIscsi(installPath, h); - comp.success(); + try { + if (VolumeProtocol.Vhost.toString().equals(protocol)) { + deactivateVhost(installPath, h); + } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { + // iscsi target is shared by all hosts, we cannot control one volume on one host for now. + deactivateIscsi(installPath, h); + } else { + comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + return; + } + } catch (Exception e) { + comp.fail(operr("failed to deactivate volume[path:%s, protocol:%s] on host[uuid:%s, ip:%s]: %s", + installPath, protocol, h.getUuid(), h.getManagementIp(), e.getMessage())); return; } - comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + comp.success(); } private void deactivateIscsi(String installPath, HostInventory h) { @@ -255,7 +260,7 @@ private void deactivateVhost(String installPath, HostInventory h) { return; } - retry(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); + retryOrThrow(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); } @Override @@ -271,9 +276,10 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - // todo - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { + throw new OperationFailureException(operr("xinfini does not support volume path isolation yet, " + + "abort starting VM on host[uuid:%s, ip:%s] to prevent split-brain on volume[path:%s, protocol:%s]", + h.getUuid(), h.getManagementIp(), installPath, protocol)); } @Override @@ -1038,16 +1044,33 @@ private void retry(Runnable r) { private void retry(Runnable r, int retry) { - while (retry-- > 0) { + try { + retryOrThrow(r, retry); + } catch (RuntimeException ignored) { + } + } + + private void retryOrThrow(Runnable r) { + retryOrThrow(r, 3); + } + + private void retryOrThrow(Runnable r, int retry) { + RuntimeException lastError = null; + for (int i = 0; i < retry; i++) { try { r.run(); return; - } catch (Exception e) { + } catch (RuntimeException e) { + lastError = e; logger.warn("runnable failed, try ", e); try { TimeUnit.SECONDS.sleep(3); } catch (InterruptedException ignore) {} } } + + if (lastError != null) { + throw lastError; + } } } diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 4b725d7d952..faddfdbd5da 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -145,8 +145,7 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { } @Override From b0ed121d80359c291fdfcecbfc476746ec45ae61 Mon Sep 17 00:00:00 2001 From: "yaohua.wu" Date: Fri, 3 Jul 2026 12:00:55 +0800 Subject: [PATCH 11/53] [kvm]: defer orphaned skip tracing cleanup Backport VM HA skip-trace protection to 5.4.10. 1. Why? When a management node leaves, its VM skip-trace entries are removed immediately. If those VMs are still being started by kvmagent, the next VM sync may treat them as stopped and trigger HA, causing split-brain. 2. How? Move departed management node skip entries into an orphaned set, keep them skipped for a configurable TTL, include them in VM sync skip lists, and remove expired entries with compare-and-remove cleanup. 3. Side effects? VMs whose controller MN departed stay skipped until the TTL expires. The default TTL is 600 seconds and can be changed by global config `kvm.vm.orphanedSkipTimeout`. # Summary of changes (by module): - kvm: add orphaned skip set, TTL global config, sync inclusion and CAS cleanup. Related: ZSTAC-84636 Related: ZSTAC-80821 Change-Id: I7862360c7318cedb51e38939467cf2a7836e61d0 --- .../java/org/zstack/kvm/KVMGlobalConfig.java | 4 ++ .../org/zstack/kvm/KvmVmSyncPingTask.java | 51 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index aa0fd787bed..4b1d393de48 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -120,6 +120,10 @@ public class KVMGlobalConfig { @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable install host shutdown hook") public static GlobalConfig INSTALL_HOST_SHUTDOWN_HOOK = new GlobalConfig(CATEGORY, "install.host.shutdown.hook"); + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip entries from departed management nodes") + public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "vm.orphanedSkipTimeout"); + @GlobalConfigValidation(validValues = {"true", "false"}) @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable memory auto balloon") @BindResourceConfig({VmInstanceVO.class, HostVO.class, ClusterVO.class}) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index af1a8718800..0f3ceeae74e 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -67,6 +67,11 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private List> skipVmTracerMessages = new ArrayList<>(); private List skipVmTracerReplies = new ArrayList<>(); private Map vmInShutdownMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); + + private long getOrphanTtlMs() { + return KVMGlobalConfig.ORPHANED_VM_SKIP_TIMEOUT.value(Long.class) * 1000; + } { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { @@ -195,8 +200,11 @@ private void syncVm(final HostInventory host, final Completion completion) { // Get vms to skip before send command to host to confirm the vm will be skipped after sync command finished. // The problem is if one vm-sync skipped operation is started and finished during vm sync command's handling // vm state would still be sync to mn + cleanupExpiredOrphanedSkipVms(); + Set vmsToSkipSetHostSide = new HashSet<>(); vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); // if the vm is not running on host when sync command executing but started as soon as possible // before response handling of vm sync, mgmtSideStates will including the running vm but not result in @@ -227,6 +235,7 @@ public void run(MessageReply reply) { // Get vms to skip after sync result returned. vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); Collection vmUuidsInDeleteVmGC = DeleteVmGC.queryVmInGC(host.getUuid(), ret.getStates().keySet()); @@ -445,7 +454,15 @@ public void nodeJoin(ManagementNodeInventory inv) { @Override public void nodeLeft(ManagementNodeInventory inv) { vmApis.remove(inv.getUuid()); - vmsToSkip.remove(inv.getUuid()); + Set skippedVms = vmsToSkip.remove(inv.getUuid()); + if (skippedVms != null && !skippedVms.isEmpty()) { + long now = System.currentTimeMillis(); + for (String vmUuid : skippedVms) { + orphanedSkipVms.put(vmUuid, now); + logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + + " (will expire in %d minutes)", vmUuid, inv.getUuid(), getOrphanTtlMs() / 60000)); + } + } } @Override @@ -459,6 +476,36 @@ public void iJoin(ManagementNodeInventory inv) { } public boolean isVmDoNotNeedToTrace(String vmUuid) { - return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid)); + if (vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))) { + return true; + } + + Long orphanedAt = orphanedSkipVms.get(vmUuid); + if (orphanedAt != null) { + if (System.currentTimeMillis() - orphanedAt < getOrphanTtlMs()) { + logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); + return true; + } + + orphanedSkipVms.remove(vmUuid, orphanedAt); + logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", + vmUuid, getOrphanTtlMs() / 60000)); + } + + return false; + } + + private void cleanupExpiredOrphanedSkipVms() { + if (orphanedSkipVms.isEmpty()) { + return; + } + + long now = System.currentTimeMillis(); + for (Map.Entry entry : orphanedSkipVms.entrySet()) { + if (now - entry.getValue() >= getOrphanTtlMs()) { + orphanedSkipVms.remove(entry.getKey(), entry.getValue()); + logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); + } + } } } From dadf250fdfcab78f53d974631f16bb5133d844b7 Mon Sep 17 00:00:00 2001 From: Jiajun Xu Date: Thu, 2 Jul 2026 17:22:15 +0800 Subject: [PATCH 12/53] [kvm]: disable arm vm pmu by default Backport PMU-off handling from ZSTAC-76375 to 5.4.10 for Kunpeng openEuler guest panic. Resolves: ZSTAC-86338 Change-Id: I3c2dbe1944c300a803f50198b4c7ba575bd77dcc --- .../org/zstack/compute/vm/VmGlobalConfig.java | 4 + .../java/org/zstack/kvm/KVMAgentCommands.java | 10 + .../src/main/java/org/zstack/kvm/KVMHost.java | 4 + .../integration/kvm/vm/VmPmuConfigCase.groovy | 223 ++++++++++++++++++ 4 files changed, 241 insertions(+) create mode 100644 test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy diff --git a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java index bd79900c13c..b42e5a32edd 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java @@ -52,6 +52,10 @@ public class VmGlobalConfig { @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class}) public static GlobalConfig VM_PORT_OFF = new GlobalConfig(CATEGORY, "vmPortOff"); + @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable PMU for VM, disabled by default on aarch64 to avoid Kunpeng-920 kernel panic") + @GlobalConfigValidation(validValues = {"true", "false"}) + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) + public static GlobalConfig VM_PMU = new GlobalConfig(CATEGORY, "vm.pmu"); @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) public static GlobalConfig EMULATE_HYPERV = new GlobalConfig(CATEGORY, "emulateHyperV"); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 91e5d885bd8..2eb30e5040a 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -2282,6 +2282,8 @@ public static class StartVmCmd extends vdiCmd implements VmAddOnsCmd { @GrayVersion(value = "5.0.0") private boolean acpi; @GrayVersion(value = "5.0.0") + private boolean pmu = true; + @GrayVersion(value = "5.0.0") private boolean x2apic = true; // cpuid hypervisor feature @GrayVersion(value = "5.0.0") @@ -2789,6 +2791,14 @@ public void setAcpi(boolean acpi) { this.acpi = acpi; } + public boolean isPmu() { + return pmu; + } + + public void setPmu(boolean pmu) { + this.pmu = pmu; + } + public boolean getX2apic() { return x2apic; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index cd58e7b58e9..cf62a6396cb 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -4547,6 +4547,10 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi if (spec.isCreatePaused()) { cmd.setCreatePaused(true); } + if (ImageArchitecture.aarch64.toString().equals(architecture)) { + Boolean pmuEnabled = rcf.getResourceConfigValue(VmGlobalConfig.VM_PMU, spec.getVmInventory().getUuid(), Boolean.class); + cmd.setPmu(Boolean.TRUE.equals(pmuEnabled)); + } cmd.setAcpi(true); GuestOsCharacter.Config config = GuestOsHelper.getInstance().getGuestOsCharacter( diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy new file mode 100644 index 00000000000..0340abb3109 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy @@ -0,0 +1,223 @@ +package org.zstack.test.integration.kvm.vm + +import org.springframework.http.HttpEntity +import org.zstack.compute.vm.VmGlobalConfig +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.flat.FlatNetworkServiceConstant +import org.zstack.network.service.userdata.UserdataConstant +import org.zstack.sdk.GlobalConfigInventory +import org.zstack.sdk.ImageInventory +import org.zstack.sdk.InstanceOfferingInventory +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +class VmPmuConfigCase extends SubCase { + EnvSpec env + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(2) + cpu = 1 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image_x86" + architecture = "x86_64" + url = "http://zstack.org/download/test.qcow2" + } + + image { + name = "image_arm64" + architecture = "aarch64" + url = "http://zstack.org/download/test-arm.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster_x86" + hypervisorType = "KVM" + architecture = "x86_64" + + kvm { + name = "kvm_x86" + managementIp = "localhost" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + cluster { + name = "cluster_arm64" + hypervisorType = "KVM" + architecture = "aarch64" + + kvm { + name = "kvm_arm64" + managementIp = "127.0.0.3" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + attachBackupStorage("sftp") + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + + service { + provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING + types = [NetworkServiceType.DHCP.toString(), EipConstant.EIP_NETWORK_SERVICE_TYPE, UserdataConstant.USERDATA_TYPE_STRING] + } + + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + } + } + } + + @Override + void test() { + env.create() + testPmuGlobalConfigExists() + testX86VmKeepsPmuEnabled() + testAarch64VmDisablesPmuByDefault() + testAarch64VmPmuResourceConfigOverride() + } + + void testPmuGlobalConfigExists() { + def configs = queryGlobalConfig { + conditions = ["category=${VmGlobalConfig.CATEGORY}", "name=${VmGlobalConfig.VM_PMU.name}"] + } + + assert configs.size() == 1 + assert (configs[0] as GlobalConfigInventory).defaultValue == "false" + } + + void testX86VmKeepsPmuEnabled() { + ImageInventory image = env.inventoryByName("image_x86") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-x86" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert cmd.pmu + } + + void testAarch64VmDisablesPmuByDefault() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-arm-default" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert !cmd.pmu + } + + void testAarch64VmPmuResourceConfigOverride() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + VmInstanceInventory vm = createVmInstance { + name = "pmu-arm-override" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } as VmInstanceInventory + + updateResourceConfig { + category = VmGlobalConfig.CATEGORY + name = VmGlobalConfig.VM_PMU.name + resourceUuid = vm.uuid + value = "true" + } + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + rebootVmInstance { + uuid = vm.uuid + } + } + + assert cmd.pmu + } + + KVMAgentCommands.StartVmCmd captureStartVmCmd(Closure trigger) { + KVMAgentCommands.StartVmCmd startCmd = null + env.afterSimulator(KVMConstant.KVM_START_VM_PATH) { rsp, HttpEntity e -> + startCmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.StartVmCmd.class) + return rsp + } + + trigger.call() + assert startCmd != null + env.cleanAfterSimulatorHandlers() + return startCmd + } +} From 42a035de901ec07fffa166d932cda7c9094872c1 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 13 Feb 2026 12:40:04 +0800 Subject: [PATCH 13/53] [loadBalancer]: block SLB deletion during grayscale upgrade Resolves: ZSTAC-78989 Change-Id: I0fe3a56ab724978944c69afadaab7ff7353e4c0f (cherry picked from commit f19223a6e72678f20610ae04226fc20c0cf1bb5b) --- .../service/lb/LoadBalancerApiInterceptor.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ea1168da036..72af77c1070 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -39,6 +39,7 @@ import org.zstack.network.service.vip.VipVO_; import org.zstack.tag.PatternedSystemTag; import org.zstack.tag.TagManager; +import org.zstack.core.upgrade.UpgradeGlobalConfig; import org.zstack.utils.*; import org.zstack.utils.function.ForEachFunction; import org.zstack.utils.logging.CLogger; @@ -151,10 +152,22 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APIGetCandidateVmNicsForLoadBalancerServerGroupMsg)msg); } else if (msg instanceof APIChangeLoadBalancerBackendServerMsg) { validate((APIChangeLoadBalancerBackendServerMsg)msg); + } else if (msg instanceof APIDeleteLoadBalancerMsg) { + validate((APIDeleteLoadBalancerMsg) msg); } return msg; } + private void validate(APIDeleteLoadBalancerMsg msg) { + if (UpgradeGlobalConfig.GRAYSCALE_UPGRADE.value(Boolean.class)) { + LoadBalancerVO lb = dbf.findByUuid(msg.getUuid(), LoadBalancerVO.class); + if (lb != null && lb.getType() == LoadBalancerType.SLB) { + throw new ApiMessageInterceptionException(argerr( + "cannot delete the standalone load balancer[uuid:%s] during grayscale upgrade", msg.getUuid())); + } + } + } + private void validate(APIDeleteAccessControlListMsg msg) { /*List refs = Q.New(LoadBalancerListenerACLRefVO.class).select(LoadBalancerListenerACLRefVO_.listenerUuid) .eq(LoadBalancerListenerACLRefVO_.aclUuid, msg.getUuid()).isNull(LoadBalancerListenerACLRefVO_.serverGroupUuid).listValues(); From 3079c287c63433e57d8ea4614be375a435f008f1 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 4 Jun 2026 14:59:56 +0800 Subject: [PATCH 14/53] [virtualrouter]: configure dnsmasq query limits Pass dns-forward-max and cache-size through VR DNS commands. Render both options in zstack-vyos dnsmasq config. Resolves: ZSTAC-85197 Change-Id: Ib78504edc564a2ab7dbd39bcb2e0ee396d019240 (cherry picked from commit 4ada5a52ee58d17b238ea1e22abb3609d6d47104) --- conf/globalConfig/virutalRouter.xml | 14 ++++++++++++++ .../virtualrouter/VirtualRouterGlobalConfig.java | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/conf/globalConfig/virutalRouter.xml b/conf/globalConfig/virutalRouter.xml index 36e3b30ef2c..e084e3f925b 100755 --- a/conf/globalConfig/virutalRouter.xml +++ b/conf/globalConfig/virutalRouter.xml @@ -21,6 +21,20 @@ virtualRouter java.lang.Integer + + dnsmasq.dnsForwardMax + The maximum number of concurrent DNS queries forwarded by dnsmasq on a virtual router. + 1000 + virtualRouter + java.lang.Integer + + + dnsmasq.cacheSize + The DNS cache size used by dnsmasq on a virtual router. + 10000 + virtualRouter + java.lang.Integer + ping.interval The interval management nodes ping the virtual router agents running virtual router VMs, in seconds diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java index 3b8d87bb8f0..7b38e6cc723 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java @@ -18,6 +18,12 @@ public class VirtualRouterGlobalConfig { public static GlobalConfig COMMANDS_PARALELLISM_DEGREE = new GlobalConfig(CATEGORY, "command.parallelismDegree"); @GlobalConfigValidation public static GlobalConfig RESTART_DNSMASQ_COUNT = new GlobalConfig(CATEGORY, "dnsmasq.restartAfterNumberOfSIGUSER1"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_DNS_FORWARD_MAX = new GlobalConfig(CATEGORY, "dnsmasq.dnsForwardMax"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_CACHE_SIZE = new GlobalConfig(CATEGORY, "dnsmasq.cacheSize"); @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig PING_INTERVAL = new GlobalConfig(CATEGORY, "ping.interval"); @GlobalConfigValidation From c9fbad0fff54cc52d39274d0a4e0826968b9505e Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 21 May 2026 21:17:17 +0800 Subject: [PATCH 15/53] [sugonSdn]: fix port sync plugin declaration Change-Id: If07dce918c4d7d9f62445184ff5984e30b83ce64 (cherry picked from commit 893999410fc56d1518b076d3c6a74880acaa4aa6) --- conf/springConfigXml/sugonSdnController.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/springConfigXml/sugonSdnController.xml b/conf/springConfigXml/sugonSdnController.xml index f0ff6ad0033..c55e6da4903 100644 --- a/conf/springConfigXml/sugonSdnController.xml +++ b/conf/springConfigXml/sugonSdnController.xml @@ -67,11 +67,7 @@ - - - - - + From 43e3374b4852e82bdbb7e98895127c697da837d7 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 21 May 2026 12:21:15 +0900 Subject: [PATCH 16/53] [sugonSdn]: sync tf ports after ping Trigger TF port sync from TF controller ping success instead of MN startup. Keep the original one day interval and port cleanup algorithm, while scoping sync to the current controller. Resolves: ZSTAC-85281 Change-Id: I84a88a34135cd47fc79b3019d2f0fc2953f06ac9 (cherry picked from commit e44d41a0f130d4ccccef1a0e27e511a469765a01) --- .../controller/SugonSdnController.java | 49 ++++++++++++- .../neutronClient/TfPortClient.java | 17 ++++- .../network/TfPortService.java | 11 +++ .../network/TfZstackPortSync.java | 69 ++++++++++++------- .../SugonSdnControllerCase.groovy | 56 +++++++++++++++ 5 files changed, 177 insertions(+), 25 deletions(-) diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 45a68537e38..57ac4406908 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -22,10 +22,13 @@ import org.zstack.network.l3.L3NetworkSystemTags; import org.zstack.sdnController.SdnController; import org.zstack.sdnController.SdnControllerL2; +import org.zstack.sdnController.SdnControllerPingMsg; +import org.zstack.sdnController.SdnControllerPingReply; import org.zstack.sdnController.header.*; import org.zstack.sugonSdnController.controller.api.*; import org.zstack.sugonSdnController.controller.api.types.*; import org.zstack.sugonSdnController.header.APICreateL2TfNetworkMsg; +import org.zstack.sugonSdnController.network.TfZstackPortSync; import org.zstack.utils.StringDSL; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; @@ -44,6 +47,8 @@ public class SugonSdnController implements TfSdnController, SdnController, SdnCo CloudBus bus; @Autowired DatabaseFacade dbf; + @Autowired + TfZstackPortSync tfZstackPortSync; private SdnControllerVO sdnControllerVO; private TfHttpClient client; @@ -56,7 +61,49 @@ public SugonSdnController(SdnControllerVO vo) { @Override public void handleMessage(SdnControllerMessage msg) { - bus.dealWithUnknownMessage((Message) msg); + if (msg instanceof SdnControllerPingMsg) { + handMessage((SdnControllerPingMsg) msg); + } else if (msg instanceof SdnControllerEnableDHCPMsg) { + handMessage((SdnControllerEnableDHCPMsg) msg); + } else if (msg instanceof SdnControllerDisableDHCPMsg) { + handMessage((SdnControllerDisableDHCPMsg) msg); + } else if (msg instanceof SdnControllerUpdateDHCPMsg) { + handMessage((SdnControllerUpdateDHCPMsg) msg); + } else { + bus.dealWithUnknownMessage((Message) msg); + } + } + + void handMessage(SdnControllerEnableDHCPMsg msg) { + SdnControllerEnableDHCPReply reply = new SdnControllerEnableDHCPReply(); + bus.reply(msg, reply); + } + + void handMessage(SdnControllerDisableDHCPMsg msg) { + SdnControllerDisableDHCPReply reply = new SdnControllerDisableDHCPReply(); + bus.reply(msg, reply); + } + + void handMessage(SdnControllerUpdateDHCPMsg msg) { + SdnControllerUpdateDHCPReply reply = new SdnControllerUpdateDHCPReply(); + bus.reply(msg, reply); + } + + void handMessage(SdnControllerPingMsg msg) { + SdnControllerPingReply reply = new SdnControllerPingReply(); + try { + Domain domain = (Domain) client.getDomain(); + if (domain == null) { + reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + } else { + tfZstackPortSync.triggerSyncIfDue(msg.getSdnControllerUuid()); + reply.setSuccess(true); + } + } catch (Exception e) { + logger.warn("ping tf sdn controller failed", e); + reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + } + bus.reply(msg, reply); } @Override diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java index 1054f708696..2c531684d1b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java @@ -32,6 +32,22 @@ public TfPortClient(){ if (sdn == null){ throw new RuntimeException("Can not find a tf sdn controller."); } + init(sdn); + } + + public TfPortClient(String sdnControllerUuid) { + SdnControllerVO sdn = Q.New(SdnControllerVO.class).eq( + SdnControllerVO_.uuid, sdnControllerUuid).find(); + if (sdn == null) { + throw new RuntimeException(String.format("Can not find a tf sdn controller[uuid:%s].", sdnControllerUuid)); + } + if (!Objects.equals(sdn.getVendorType(), SdnControllerConstant.TF_CONTROLLER)) { + throw new RuntimeException(String.format("Sdn controller[uuid:%s] is not a tf sdn controller.", sdnControllerUuid)); + } + init(sdn); + } + + private void init(SdnControllerVO sdn) { client = new TfHttpClient(sdn.getIp()); tenantId = StringDSL.transToTfUuid(sdn.getAccountUuid()); } @@ -509,4 +525,3 @@ public Status updateTfPort(String tfPortUUid, String accountId, String deviceId, } } - diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java index c4767689bd1..e7a9f0c0419 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java @@ -33,6 +33,11 @@ public List getTfPortsDetail() { return tfPortClient.getVirtualMachineInterfaceDetail(); } + public List getTfPortsDetail(String sdnControllerUuid) { + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.getVirtualMachineInterfaceDetail(); + } + public TfPortResponse createTfPort(String tfPortUUid, String l2NetworkUuid, String l3NetworkUuid, String mac, String ip) { //invoke tf rest interface to retrieve real ip and mac and portId TfPortClient tfPortClient = new TfPortClient(); @@ -115,6 +120,12 @@ public TfPortResponse deleteTfPort(String portUUid) { return tfPortClient.deletePort(tfPortUUid); } + public TfPortResponse deleteTfPort(String sdnControllerUuid, String portUUid) { + String tfPortUUid = StringDSL.transToTfUuid(portUUid); + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.deletePort(tfPortUUid); + } + public Status updateTfPort(String portUUid, String bmUuid, KeyValuePairs bindInfo) { String accountId = StringDSL.transToTfUuid(acntMgr.getOwnerAccountUuidOfResource(bmUuid)); String tfBmUUid = StringDSL.transToTfUuid(bmUuid); diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java index 184b2e721ed..710b63cdb2b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java @@ -2,9 +2,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.db.Q; -import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.Task; import org.zstack.core.thread.ThreadFacade; -import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; import org.zstack.header.network.l2.L2NetworkVO; import org.zstack.header.network.l2.L2NetworkVO_; import org.zstack.header.vm.VmNicVO; @@ -19,39 +18,57 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.concurrent.Future; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -public class TfZstackPortSync implements ManagementNodeReadyExtensionPoint { +public class TfZstackPortSync { + private static final long SYNC_INTERVAL_MILLIS = TimeUnit.DAYS.toMillis(1); + private static final int MAX_DELETE_COUNT = 10; @Autowired protected ThreadFacade thdf; - private Future trackerThread = null; @Autowired private TfPortService tfPortService; private final static CLogger logger = Utils.getLogger(TfZstackPortSync.class); private final List excludeTypes = new ArrayList(Arrays.asList("neutron:LOADBALANCER", "VIP", "BMS")); + private final Map lastSyncTime = new ConcurrentHashMap<>(); + private final Set runningSyncs = Collections.newSetFromMap(new ConcurrentHashMap()); - @Override - public void managementNodeReady() { - if (trackerThread != null) { - trackerThread.cancel(true); + public void triggerSyncIfDue(String sdnControllerUuid) { + if (sdnControllerUuid == null) { + logger.warn("Port_Sync_Task: skip sync because sdn controller uuid is null."); + return; } - trackerThread = thdf.submitPeriodicTask(new SyncPort()); - } - private class SyncPort implements PeriodicTask { + long now = System.currentTimeMillis(); + Long lastSync = lastSyncTime.get(sdnControllerUuid); + if (lastSync != null && now - lastSync < SYNC_INTERVAL_MILLIS) { + return; + } + if (!runningSyncs.add(sdnControllerUuid)) { + return; + } - @Override - public TimeUnit getTimeUnit() { - return TimeUnit.DAYS; + lastSyncTime.put(sdnControllerUuid, now); + try { + thdf.submit(new SyncPort(sdnControllerUuid)); + } catch (RuntimeException e) { + lastSyncTime.remove(sdnControllerUuid); + runningSyncs.remove(sdnControllerUuid); + throw e; } + } - @Override - public long getInterval() { - return 1; + private class SyncPort implements Task { + private final String sdnControllerUuid; + + private SyncPort(String sdnControllerUuid) { + this.sdnControllerUuid = sdnControllerUuid; } @Override @@ -64,7 +81,10 @@ private HashSet getPortToDelete() { List zstackL2NetworksUuid = Q.New(L2NetworkVO.class).select(L2NetworkVO_.uuid).listValues(); List tfPortsUuid = new ArrayList<>(); try{ - List tfPorts = tfPortService.getTfPortsDetail(); + List tfPorts = tfPortService.getTfPortsDetail(sdnControllerUuid); + if (tfPorts == null) { + return new HashSet<>(); + } for (VirtualMachineInterface vmi : tfPorts) { // skip port if it's network not in zstack List> tfNetworks = vmi.getVirtualNetwork(); @@ -83,7 +103,7 @@ private HashSet getPortToDelete() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task: Fetch tf VirtualMachineInterface failed: %s.", e)); - return null; + return new HashSet<>(); } HashSet result = new HashSet<>(tfPortsUuid); result.removeAll(zstackPortsUuid); @@ -92,13 +112,13 @@ private HashSet getPortToDelete() { } @Override - public void run() { + public Void call() { logger.info("Port_Sync_Task: begin."); try { HashSet portsToDelete = getPortToDelete(); - int maxDeleteCount = 10; + int maxDeleteCount = MAX_DELETE_COUNT; for (String portUuid: portsToDelete) { - TfPortResponse response = tfPortService.deleteTfPort(portUuid); + TfPortResponse response = tfPortService.deleteTfPort(sdnControllerUuid, portUuid); if (response.getCode() == 200) { logger.info(String.format("Port_Sync_Task: VirtualMachineInterface: %s delete success.", portUuid)); @@ -113,7 +133,10 @@ public void run() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task failed: %s.", e)); + } finally { + runningSyncs.remove(sdnControllerUuid); } + return null; } } } diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy index 8e28757ba5a..c11fb00950b 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -1,7 +1,12 @@ package org.zstack.test.integration.network.sdnController +import org.springframework.http.HttpEntity import org.zstack.core.db.DatabaseFacade +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.message.MessageReply import org.zstack.header.network.sdncontroller.SdnControllerVO +import org.zstack.header.network.sdncontroller.SdnControllerConstant +import org.zstack.sdnController.SdnControllerPingMsg import org.zstack.sdk.* import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant import org.zstack.sugonSdnController.controller.api.types.MacAddressesType @@ -16,6 +21,9 @@ import javax.persistence.TypedQuery; import org.springframework.http.ResponseEntity import org.springframework.http.HttpStatus +import javax.servlet.http.HttpServletRequest +import java.util.concurrent.atomic.AtomicInteger + class SugonSdnControllerCase extends SubCase { EnvSpec env @@ -37,6 +45,7 @@ class SugonSdnControllerCase extends SubCase { void test() { env.create { dbf = bean(DatabaseFacade.class) + testTfPortSyncTriggeredByPingSuccess() testTfApi() } } @@ -46,6 +55,53 @@ class SugonSdnControllerCase extends SubCase { env.delete() } + void testTfPortSyncTriggeredByPingSuccess() { + String sql = "select sdn" + + " from SdnControllerVO sdn" + + " where sdn.vendorType = :vendorType"; + TypedQuery q = dbf.getEntityManager().createQuery(sql, SdnControllerVO.class); + q.setParameter("vendorType", SugonSdnControllerConstant.TF_CONTROLLER); + SdnControllerVO sdn = q.getResultList().get(0); + AtomicInteger syncCount = new AtomicInteger(0) + + env.simulator(TfCommands.TF_CREATE_VMI) { HttpServletRequest req, HttpEntity e, EnvSpec spec -> + if (req.method == "GET") { + syncCount.incrementAndGet() + return '{"virtual-machine-interfaces":[]}' + } + + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = TfCommands.TEST_VMI_UUID + rsp.uuid = TfCommands.TEST_VMI_UUID + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.setParent(project) + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + sendTfPing(sdn.uuid) + retryInSecs { + assert syncCount.get() == 1 + } + + sendTfPing(sdn.uuid) + sleep(500) + assert syncCount.get() == 1 + } + + void sendTfPing(String sdnControllerUuid) { + CloudBus bus = bean(CloudBus.class) + SdnControllerPingMsg msg = new SdnControllerPingMsg() + msg.sdnControllerUuid = sdnControllerUuid + bus.makeTargetServiceIdByResourceUuid(msg, SdnControllerConstant.SERVICE_ID, sdnControllerUuid) + MessageReply reply = bus.call(msg) + assert reply.success + } + void testTfApi() { def zone = env.inventoryByName("zone") as ZoneInventory def cluster1 = env.inventoryByName("cluster1") as ClusterInventory From 31c9dbb1f83c8056618a953ff2685cd51998cbef Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Tue, 30 Jun 2026 22:23:23 +0900 Subject: [PATCH 17/53] [sugonSdn]: adapt ping sync backport Remove DHCP message handlers and generated error code reference unavailable on 5.4.10. Change-Id: I1a15fb1eeeab60ca9c6fa239904bda89839b64e5 --- .../controller/SugonSdnController.java | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 57ac4406908..4d9c66f50fc 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -63,45 +63,24 @@ public SugonSdnController(SdnControllerVO vo) { public void handleMessage(SdnControllerMessage msg) { if (msg instanceof SdnControllerPingMsg) { handMessage((SdnControllerPingMsg) msg); - } else if (msg instanceof SdnControllerEnableDHCPMsg) { - handMessage((SdnControllerEnableDHCPMsg) msg); - } else if (msg instanceof SdnControllerDisableDHCPMsg) { - handMessage((SdnControllerDisableDHCPMsg) msg); - } else if (msg instanceof SdnControllerUpdateDHCPMsg) { - handMessage((SdnControllerUpdateDHCPMsg) msg); } else { bus.dealWithUnknownMessage((Message) msg); } } - void handMessage(SdnControllerEnableDHCPMsg msg) { - SdnControllerEnableDHCPReply reply = new SdnControllerEnableDHCPReply(); - bus.reply(msg, reply); - } - - void handMessage(SdnControllerDisableDHCPMsg msg) { - SdnControllerDisableDHCPReply reply = new SdnControllerDisableDHCPReply(); - bus.reply(msg, reply); - } - - void handMessage(SdnControllerUpdateDHCPMsg msg) { - SdnControllerUpdateDHCPReply reply = new SdnControllerUpdateDHCPReply(); - bus.reply(msg, reply); - } - void handMessage(SdnControllerPingMsg msg) { SdnControllerPingReply reply = new SdnControllerPingReply(); try { Domain domain = (Domain) client.getDomain(); if (domain == null) { - reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + reply.setError(operr("get default domain on tf controller failed")); } else { tfZstackPortSync.triggerSyncIfDue(msg.getSdnControllerUuid()); reply.setSuccess(true); } } catch (Exception e) { logger.warn("ping tf sdn controller failed", e); - reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + reply.setError(operr("get default domain on tf controller failed")); } bus.reply(msg, reply); } From 16667ce8c2759315794b2cfc2c2fc57e0ee41016 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Wed, 1 Jul 2026 09:58:35 +0900 Subject: [PATCH 18/53] [sugonSdn]: address review comments Add MessageSafe to the SDN message entrypoint. Move the second ping after the first async sync window in the focused test. Resolves: ZSTAC-86445 Change-Id: I4d7748a0de3b5d6a052a47966d2265249b2779e2 --- .../sugonSdnController/controller/SugonSdnController.java | 2 ++ .../network/sdnController/SugonSdnControllerCase.groovy | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 4d9c66f50fc..e63d69927c4 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.MessageSafe; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.header.core.Completion; @@ -60,6 +61,7 @@ public SugonSdnController(SdnControllerVO vo) { } @Override + @MessageSafe public void handleMessage(SdnControllerMessage msg) { if (msg instanceof SdnControllerPingMsg) { handMessage((SdnControllerPingMsg) msg); diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy index c11fb00950b..e954020fa11 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -88,8 +88,8 @@ class SugonSdnControllerCase extends SubCase { assert syncCount.get() == 1 } - sendTfPing(sdn.uuid) sleep(500) + sendTfPing(sdn.uuid) assert syncCount.get() == 1 } From 57209eaeaa6ff71c11d3372d764bfe4b19a65fae Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Mon, 22 Dec 2025 19:37:55 +0800 Subject: [PATCH 19/53] [sdnController]: put sdn host crud in seperate queue Resolves: ZSTAC-79353 Change-Id: I63706d727173706e796b64666963776a74736c61 (cherry picked from commit 2ef761d0222ee0b39166d636e190e729ee45966c) --- .../org/zstack/sdnController/SdnControllerBase.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java index f4733f6a833..8ff809fd308 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java @@ -86,6 +86,10 @@ public String getSdnControllerSignature() { return "sdn-controller-" + self.getUuid(); } + public String getSdnControllerHostSignature(String hostUuid) { + return "sdn-controller-" + self.getUuid() + "-host-" + hostUuid; + } + protected SdnController getSdnController() { SdnControllerFactory factory = sdnMgr.getSdnControllerFactory(self.getVendorType()); return factory.getSdnController(self); @@ -427,7 +431,7 @@ private void sdnControllerAddHost(APISdnControllerAddHostMsg msg, Completion com thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -508,7 +512,7 @@ private void sdnControllerRemoveHostInQueue(SdnControllerRemoveHostMsg msg, Comp thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -594,7 +598,7 @@ private void doControllerChangeHost(APISdnControllerChangeHostMsg msg, Completio thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override From f9777240c2f50b6d42e7249d39fbb0010ceda257 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Tue, 20 Jan 2026 14:36:43 +0800 Subject: [PATCH 20/53] [virtualRouterProvider]: change vrouter ip_local_port_range Resolves: ZSTAC-81450 Change-Id: I79666f616d64797a747677656762626d6a71786d (cherry picked from commit 5c3fab5568c2647987978ce1b7cb0c41580d4dfb) --- .../service/virtualrouter/vyos/VyosConnectFlow.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java index d890e10ed33..1108c650975 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java @@ -180,12 +180,9 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setLogLevel(rcf.getResourceConfigValue(VirtualRouterGlobalConfig.LOG_LEVEL, vrUuid, String.class)); Map parms = new HashMap<>(); - long count = Q.New(VirtualRouterLoadBalancerRefVO.class) - .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, vrUuid) - .notNull(VirtualRouterLoadBalancerRefVO_.loadBalancerUuid) - .count(); - if(count > 0){ - parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(),VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); + if (!VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value().equals( + VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.defaultValue(String.class))){ + parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(), VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); } cmd.setParms(parms); From adf11eb193160523c036b30518856d231696b1d9 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Mon, 6 Jul 2026 14:39:16 +0800 Subject: [PATCH 21/53] [sdk]: update mdev candidates sdk The premium backport adds minAvailableDevicesPerHost to GetMdevDeviceCandidates. Commit the generated SDK action field on the companion zstack branch so update-sdk CI no longer reports an uncommitted SDK diff. Constraint: premium and zstack companion branches must use the same @@2 suffix for cross-repo SDK validation Tested: git diff --check Not-tested: Full ./runMavenProfile sdk locally Resolves: ZSTAC-86497 Change-Id: Iae56b01a4c533eeba05d10358d4944f655c5a6d4 --- .../java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java index 8f98f870348..cd6fae86322 100644 --- a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java +++ b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java @@ -37,6 +37,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List types; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Integer minAvailableDevicesPerHost; + @Param(required = false) public java.util.List systemTags; From 6757ae7d095d5ba19cf90b8f17f73782aba8d995 Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Mon, 22 Dec 2025 18:07:28 +0800 Subject: [PATCH 22/53] [migration]: fix failed to start vm after ceph to ceph offline migration. support starting vm without nic in other clusters. Resolves/Related: ZSTAC-80468 Change-Id: I6370646f7a796265677a6a656c716f6867706d69 --- compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index ed98aef34e0..b9f2b334cc2 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -928,6 +928,7 @@ private void getStartingCandidateHosts(final NeedReplyMessage msg, final ReturnV amsg.setL3NetworkUuids(VmNicHelper.getL3Uuids(VmNicInventory.valueOf(self.getVmNics()))); amsg.setDryRun(true); amsg.setListAllHosts(true); + amsg.setAllowNoL3Networks(true); bus.send(amsg, new CloudBusCallBack(completion) { @Override From dafd9816feca24b7908eb226eab941e4f53d437b Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Tue, 11 Nov 2025 14:22:29 +0800 Subject: [PATCH 23/53] [sharedblock]: convert memory snapshot install path from absolute path to protocol path convert memory snapshot install path from absolute path to protocol path Resolves/Related: ZSTAC-79756 Change-Id: I6e626d68626461627a737765786a676e6b617064 --- conf/db/upgrade/V5.4.10__schema.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql index ab73deda0f6..71efe026193 100644 --- a/conf/db/upgrade/V5.4.10__schema.sql +++ b/conf/db/upgrade/V5.4.10__schema.sql @@ -1,3 +1,7 @@ -- ZSTAC-86476: Add normalizedModelName column for GPU spec dedup CALL ADD_COLUMN('GpuDeviceSpecVO', 'normalizedModelName', 'VARCHAR(255)', 1, NULL); CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalizedModelName'); + +UPDATE VolumeSnapshotVO AS sp, PrimaryStorageVO AS ps +SET sp.primaryStorageInstallPath = REPLACE(sp.primaryStorageInstallPath, '/dev/', 'sharedblock://') +WHERE sp.primaryStorageUuid = ps.uuid AND ps.type = 'SharedBlock' AND sp.volumeType = 'Memory' AND sp.primaryStorageInstallPath LIKE '/dev/%'; From eabf4766f58fcf009aa01207a5b692e30051ec72 Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Sun, 17 May 2026 12:37:21 +0800 Subject: [PATCH 24/53] [snapshot]: ZSTAC-61889 handle late snapshot success Keep snapshot DB records when the hypervisor snapshot succeeds but later control-plane flow fails. KVMHost and VolumeSnapshotManagerImpl now share syncVolumeSnapshotDBAfterTakeSnapshot for the DB update. The regression case injects failure after data-plane success through VolumeSnapshotCreationExtensionPoint. Resolves: ZSTAC-61889 Change-Id: Ida2ef0ad95e7a2faf4d8db705d97b8807586802d --- conf/springConfigXml/volumeSnapshot.xml | 1 + .../VolumeSnapshotDBSyncExtensionPoint.java | 9 + .../src/main/java/org/zstack/kvm/KVMHost.java | 23 ++ .../snapshot/VolumeSnapshotManagerImpl.java | 89 ++++++-- ...otRollbackAfterDataPlaneSuccessCase.groovy | 205 ++++++++++++++++++ 5 files changed, 303 insertions(+), 24 deletions(-) create mode 100644 header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java create mode 100644 test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy diff --git a/conf/springConfigXml/volumeSnapshot.xml b/conf/springConfigXml/volumeSnapshot.xml index 09022186cb2..d26ca551061 100755 --- a/conf/springConfigXml/volumeSnapshot.xml +++ b/conf/springConfigXml/volumeSnapshot.xml @@ -26,6 +26,7 @@ + diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java new file mode 100644 index 00000000000..9a86d0f29cf --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotDBSyncExtensionPoint.java @@ -0,0 +1,9 @@ +package org.zstack.header.storage.snapshot; + +import org.zstack.header.volume.VolumeInventory; + +public interface VolumeSnapshotDBSyncExtensionPoint { + VolumeSnapshotInventory syncVolumeSnapshotDBAfterTakeSnapshot(VolumeInventory volume, + VolumeSnapshotInventory snapshot, + String volumeNewInstallPath); +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 34e0c434296..8be6d8a0e95 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -71,6 +71,7 @@ import org.zstack.header.rest.JsonAsyncRESTCallback; import org.zstack.header.rest.RESTFacade; import org.zstack.header.storage.primary.*; +import org.zstack.header.storage.snapshot.*; import org.zstack.header.tag.SystemTagInventory; import org.zstack.header.vm.*; import org.zstack.header.vm.devices.DeviceAddress; @@ -3044,6 +3045,7 @@ public void success(TakeSnapshotResponse ret) { " data corruption may happen", ret.getNewVolumeInstallPath())); } + syncSnapshotMetadataAfterHypervisorSuccess(msg, ret); extEmitter.afterTakeSnapshot((KVMHostInventory) getSelfInventory(), msg, cmd, ret); reply.setNewVolumeInstallPath(ret.getNewVolumeInstallPath()); reply.setSnapshotInstallPath(ret.getSnapshotInstallPath()); @@ -3080,6 +3082,27 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private void syncSnapshotMetadataAfterHypervisorSuccess(TakeSnapshotOnHypervisorMsg msg, TakeSnapshotResponse ret) { + VolumeInventory volume = msg.getVolume(); + if (volume == null || msg.getSnapshotName() == null) { + return; + } + + VolumeSnapshotInventory snapshot = new VolumeSnapshotInventory(); + snapshot.setUuid(msg.getSnapshotName()); + snapshot.setVolumeUuid(volume.getUuid()); + snapshot.setPrimaryStorageUuid(volume.getPrimaryStorageUuid()); + snapshot.setPrimaryStorageInstallPath(ret.getSnapshotInstallPath()); + snapshot.setType(VolumeSnapshotConstant.HYPERVISOR_SNAPSHOT_TYPE.toString()); + snapshot.setStatus(VolumeSnapshotStatus.Ready.toString()); + snapshot.setSize(ret.getSize()); + snapshot.setFormat(volume.getFormat()); + + for (VolumeSnapshotDBSyncExtensionPoint ext : pluginRgty.getExtensionList(VolumeSnapshotDBSyncExtensionPoint.class)) { + ext.syncVolumeSnapshotDBAfterTakeSnapshot(volume, snapshot, ret.getNewVolumeInstallPath()); + } + } + private void migrateVm(final MigrateStruct s, final Completion completion) { final TaskProgressRange parentStage = getTaskStage(); final TaskProgressRange MIGRATE_VM_STAGE = new TaskProgressRange(0, 90); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java index 7488f23e178..ee9b616a0e0 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java @@ -72,6 +72,7 @@ public class VolumeSnapshotManagerImpl extends AbstractService implements VolumeBeforeExpungeExtensionPoint, ResourceOwnerAfterChangeExtensionPoint, ReportQuotaExtensionPoint, + VolumeSnapshotDBSyncExtensionPoint, AfterReimageVmInstanceExtensionPoint, VmJustBeforeDeleteFromDbExtensionPoint, VolumeJustBeforeDeleteFromDbExtensionPoint, @@ -679,6 +680,12 @@ private void rollbackSnapshot(String uuid) { return; } + if (vo.getStatus() == VolumeSnapshotStatus.Ready) { + logger.warn(String.format("volume snapshot[uuid:%s] has been marked Ready, skip rollback to keep control plane metadata consistent with hypervisor snapshot", + uuid)); + return; + } + dbf.getEntityManager().remove(vo); String sql = "delete from AccountResourceRefVO where resourceUuid = :vsUuid and resourceType = 'VolumeSnapshotVO'"; @@ -696,6 +703,42 @@ private void rollbackSnapshot(String uuid) { } } + @Override + @Transactional + public VolumeSnapshotInventory syncVolumeSnapshotDBAfterTakeSnapshot(VolumeInventory volume, + VolumeSnapshotInventory snapshot, + String volumeNewInstallPath) { + if (volumeNewInstallPath != null) { + VolumeVO latestVol = dbf.findByUuid(volume.getUuid(), VolumeVO.class); + latestVol.setInstallPath(volumeNewInstallPath); + dbf.update(latestVol); + } + + VolumeSnapshotVO svo = dbf.findByUuid(snapshot.getUuid(), VolumeSnapshotVO.class); + if (svo == null) { + return null; + } + + svo.setType(snapshot.getType()); + svo.setPrimaryStorageUuid(snapshot.getPrimaryStorageUuid()); + svo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); + svo.setStatus(VolumeSnapshotStatus.Ready); + svo.setSize(snapshot.getSize()); + if (snapshot.getFormat() != null) { + svo.setFormat(snapshot.getFormat()); + } + svo = dbf.updateAndRefresh(svo); + + markSnapshotTreeCompleted(VolumeSnapshotInventory.valueOf(svo)); + if (svo.getParentUuid() == null) { + VolumeSnapshotReferenceUtils.updateReferenceAfterFirstSnapshot(svo); + } + + logger.debug(String.format("synced volume snapshot[uuid:%s] metadata after hypervisor snapshot succeeded", + svo.getUuid())); + return VolumeSnapshotInventory.valueOf(svo); + } + private void handle(final AskVolumeSnapshotStructMsg msg) { AskVolumeSnapshotStructReply reply = new AskVolumeSnapshotStructReply(); CreateVolumeSnapshotMsg cmsg = new CreateVolumeSnapshotMsg(); @@ -990,33 +1033,14 @@ public void run(MessageReply reply) { done(new FlowDoneHandler(msg) { @Override public void handle(Map data) { - markSnapshotTreeCompleted(snapshot); - if (volumeNewInstallPath != null) { - vol.setInstallPath(volumeNewInstallPath); - dbf.update(vol); - } - - VolumeSnapshotVO svo = dbf.findByUuid(snapshot.getUuid(), VolumeSnapshotVO.class); - svo.setType(snapshot.getType()); - svo.setPrimaryStorageUuid(snapshot.getPrimaryStorageUuid()); - svo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); - svo.setStatus(VolumeSnapshotStatus.Ready); - svo.setSize(snapshot.getSize()); - if (snapshot.getFormat() != null) { - svo.setFormat(snapshot.getFormat()); - } - svo = dbf.updateAndRefresh(svo); - - if (struct.isNewChain()) { - VolumeSnapshotReferenceUtils.updateReferenceAfterFirstSnapshot(svo); - } + VolumeSnapshotInventory sp = syncVolumeSnapshotDBAfterTakeSnapshot( + vol.toInventory(), snapshot, volumeNewInstallPath); new FireSnapShotCanonicalEvent(). fireSnapShotStatusChangedEvent( VolumeSnapshotStatus.valueOf(snapshot.getStatus()), - VolumeSnapshotInventory.valueOf(svo)); + sp); - VolumeSnapshotInventory sp = svo.toInventory(); callExtensionPoints(sp); ret.setInventory(sp); @@ -1033,8 +1057,18 @@ private void callExtensionPoints(VolumeSnapshotInventory sp) { error(new FlowErrorHandler(msg) { @Override public void handle(ErrorCode errCode, Map data) { - if (struct != null) { - rollbackSnapshot(struct.getCurrent().getUuid()); + if (snapshot != null) { + syncVolumeSnapshotDBAfterTakeSnapshot(vol.toInventory(), snapshot, volumeNewInstallPath); + logger.warn(String.format("volume snapshot[uuid:%s] has been created on primary storage, keep database record for recovery after error: %s", + snapshot.getUuid(), errCode)); + } else if (struct != null) { + String snapshotUuid = struct.getCurrent().getUuid(); + if (getCurrentSnapshotStatus(snapshotUuid) == VolumeSnapshotStatus.Ready) { + logger.warn(String.format("volume snapshot[uuid:%s] has been marked Ready, keep database record after error: %s", + snapshotUuid, errCode)); + } else { + rollbackSnapshot(snapshotUuid); + } } ret.setError(errCode); bus.reply(msg, ret); @@ -1044,6 +1078,13 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private VolumeSnapshotStatus getCurrentSnapshotStatus(String uuid) { + return Q.New(VolumeSnapshotVO.class) + .select(VolumeSnapshotVO_.status) + .eq(VolumeSnapshotVO_.uuid, uuid) + .findValue(); + } + private void handle(MarkRootVolumeAsSnapshotMsg msg) { final MarkRootVolumeAsSnapshotReply ret = new MarkRootVolumeAsSnapshotReply(); VolumeInventory vol = msg.getVolume(); diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy new file mode 100644 index 00000000000..46979001b31 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/storage/snapshot/CreateSnapshotRollbackAfterDataPlaneSuccessCase.groovy @@ -0,0 +1,205 @@ +package org.zstack.test.integration.storage.snapshot + +import org.springframework.http.HttpEntity +import org.zstack.core.Platform +import org.zstack.core.componentloader.PluginRegistry +import org.zstack.core.db.Q +import org.zstack.header.core.Completion +import org.zstack.header.core.workflow.Flow +import org.zstack.header.storage.snapshot.ConsistentType +import org.zstack.header.storage.snapshot.CreateVolumesSnapshotOverlayInnerMsg +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmReply +import org.zstack.header.storage.snapshot.VolumeSnapshotConstant +import org.zstack.header.storage.snapshot.VolumeSnapshotCreationExtensionPoint +import org.zstack.header.storage.snapshot.VolumeSnapshotStatus +import org.zstack.header.storage.snapshot.VolumeSnapshotTreeStatus +import org.zstack.header.storage.snapshot.VolumeSnapshotTreeVO +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory +import org.zstack.header.storage.snapshot.VolumeSnapshotVO +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_ +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupInventory +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO_ +import org.zstack.header.volume.CreateVolumeSnapshotGroupMessage +import org.zstack.header.volume.VolumeVO +import org.zstack.header.volume.VolumeVO_ +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.ldap.Env +import org.zstack.test.integration.storage.StorageTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.gson.JSONObjectUtil + +class CreateSnapshotRollbackAfterDataPlaneSuccessCase extends SubCase { + EnvSpec env + VmInstanceInventory vm + + int takeSnapshotCount = 0 + String firstSnapshotInstallPath + String firstNewVolumeInstallPath + String secondVolumeInstallPath + String secondSnapshotInstallPath + String secondNewVolumeInstallPath + long snapshotSize = 1L + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(StorageTest.springSpec) + } + + @Override + void environment() { + env = Env.localStorageOneVmEnv() + } + + @Override + void test() { + env.create { + vm = env.inventoryByName("vm") as VmInstanceInventory + testKeepSnapshotMetadataWhenControlPlaneFailsAfterDataPlaneSuccess() + } + } + + void testKeepSnapshotMetadataWhenControlPlaneFailsAfterDataPlaneSuccess() { + String originalVolumeInstallPath = Q.New(VolumeVO.class) + .select(VolumeVO_.installPath) + .eq(VolumeVO_.uuid, vm.rootVolumeUuid) + .findValue() + + env.simulator(KVMConstant.KVM_TAKE_VOLUME_SNAPSHOT_PATH) { HttpEntity e, EnvSpec espec -> + KVMAgentCommands.TakeSnapshotCmd cmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.TakeSnapshotCmd.class) + takeSnapshotCount++ + + if (takeSnapshotCount == 1) { + assert cmd.volumeInstallPath == originalVolumeInstallPath + firstSnapshotInstallPath = cmd.volumeInstallPath + firstNewVolumeInstallPath = cmd.installPath + } else if (takeSnapshotCount == 2) { + secondVolumeInstallPath = cmd.volumeInstallPath + secondSnapshotInstallPath = cmd.volumeInstallPath + secondNewVolumeInstallPath = cmd.installPath + } + + KVMAgentCommands.TakeSnapshotResponse rsp = new KVMAgentCommands.TakeSnapshotResponse() + rsp.newVolumeInstallPath = cmd.installPath + rsp.snapshotInstallPath = cmd.volumeInstallPath + rsp.size = snapshotSize + return rsp + } + + PluginRegistry pluginRegistry = bean(PluginRegistry.class) + List extensions = pluginRegistry.getExtensionList(VolumeSnapshotCreationExtensionPoint.class) + FailAfterSnapshotCreatedExtension failureExtension = new FailAfterSnapshotCreatedExtension() + extensions.add(failureExtension) + + try { + expectError { + createVolumeSnapshot { + name = "snapshot-fail-after-metadata-synced" + volumeUuid = vm.rootVolumeUuid + } + } + + assert takeSnapshotCount == 1 + + VolumeSnapshotVO firstSnapshot + retryInSecs(10, 1) { + firstSnapshot = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.volumeUuid, vm.rootVolumeUuid) + .find() + assert firstSnapshot != null + assert firstSnapshot.status == VolumeSnapshotStatus.Ready + } + + assert firstSnapshot.latest + assert firstSnapshot.primaryStorageInstallPath == firstSnapshotInstallPath + assert firstSnapshot.type == VolumeSnapshotConstant.HYPERVISOR_SNAPSHOT_TYPE.toString() + assert firstSnapshot.size == snapshotSize + + VolumeSnapshotTreeVO firstTree = dbFindByUuid(firstSnapshot.treeUuid, VolumeSnapshotTreeVO.class) + assert firstTree.current + assert firstTree.status == VolumeSnapshotTreeStatus.Completed + + assert Q.New(VolumeVO.class) + .select(VolumeVO_.installPath) + .eq(VolumeVO_.uuid, vm.rootVolumeUuid) + .findValue() == firstNewVolumeInstallPath + + VolumeSnapshotReferenceVO ref = Q.New(VolumeSnapshotReferenceVO.class) + .eq(VolumeSnapshotReferenceVO_.volumeUuid, vm.rootVolumeUuid) + .limit(1) + .find() + if (ref != null && ref.referenceInstallUrl == firstSnapshotInstallPath) { + assert ref.referenceUuid == firstSnapshot.uuid + assert ref.referenceType == VolumeSnapshotVO.class.simpleName + } + + org.zstack.sdk.VolumeSnapshotInventory secondSnapshotInv = createVolumeSnapshot { + name = "snapshot-after-recovery" + volumeUuid = vm.rootVolumeUuid + } as org.zstack.sdk.VolumeSnapshotInventory + + assert takeSnapshotCount == 2 + assert secondVolumeInstallPath == firstNewVolumeInstallPath + + VolumeSnapshotVO firstSnapshotAfterRecovery = dbFindByUuid(firstSnapshot.uuid, VolumeSnapshotVO.class) + VolumeSnapshotVO secondSnapshot = dbFindByUuid(secondSnapshotInv.uuid, VolumeSnapshotVO.class) + assert !firstSnapshotAfterRecovery.latest + assert secondSnapshot.latest + assert secondSnapshot.parentUuid == firstSnapshot.uuid + assert secondSnapshot.primaryStorageInstallPath == secondSnapshotInstallPath + assert secondSnapshot.status == VolumeSnapshotStatus.Ready + + assert Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.volumeUuid, vm.rootVolumeUuid) + .count() == 2 + assert Q.New(VolumeVO.class) + .select(VolumeVO_.installPath) + .eq(VolumeVO_.uuid, vm.rootVolumeUuid) + .findValue() == secondNewVolumeInstallPath + } finally { + extensions.remove(failureExtension) + } + } + + private static class FailAfterSnapshotCreatedExtension implements VolumeSnapshotCreationExtensionPoint { + boolean failAfterSnapshotCreated = true + + @Override + void afterVolumeSnapshotCreated(VolumeSnapshotInventory snapshot, Completion completion) { + if (failAfterSnapshotCreated) { + failAfterSnapshotCreated = false + completion.fail(Platform.operr("TEST.ERROR", "fail after KVM snapshot has been created")) + return + } + + completion.success() + } + + @Override + void afterVolumeLiveSnapshotGroupCreatedOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, TakeVolumesSnapshotOnKvmReply treply, Completion completion) { + completion.success() + } + + @Override + void afterVolumeLiveSnapshotGroupCreationFailsOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, TakeVolumesSnapshotOnKvmReply treply) { + } + + @Override + void afterVolumeSnapshotGroupCreated(VolumeSnapshotGroupInventory snapshotGroup, ConsistentType consistentType, Completion completion) { + completion.success() + } + + @Override + List beforeCreateVolumeSnapshotFlow(CreateVolumeSnapshotGroupMessage msg) { + return null + } + } +} From 04d48a45d11c2b628e31ed46c49afb8545090ed3 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 12:02:19 +0800 Subject: [PATCH 25/53] [vm]: use max of virtual and actual size for root disk when no disk offering Resolves: ZSTAC-74683 Change-Id: Id0339ed0221e92e506f60745cde972cc3ee6d9ae (cherry picked from commit 6b4521b38a688cf6072d6fc849171022af1e1490) --- header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java index 96da4293fa8..9d2c3102222 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java @@ -830,7 +830,9 @@ public void setBootMode(String bootMode) { public long getRootDiskAllocateSize() { if (rootDiskOffering == null) { - return this.getImageSpec().getInventory().getSize(); + long virtualSize = this.getImageSpec().getInventory().getSize(); + long actualSize = this.getImageSpec().getInventory().getActualSize(); + return Math.max(virtualSize, actualSize); } return rootDiskOffering.getDiskSize(); } From 013dbea9cd618d7d5243477dd56d3e09cac4da12 Mon Sep 17 00:00:00 2001 From: J M Date: Thu, 14 May 2026 18:07:47 +0800 Subject: [PATCH 26/53] [utils]: SshShell add ssh client timeouts SshShell.runCommand and runScript now inject ssh client timeout options into the underlying ssh command line: -o ConnectTimeout=10 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 Subprocess self-exits in ~10-20s on unreachable or non-responsive remote, instead of waiting forever in Process.waitFor (the underlying ShellUtils.runAndReturn has no JVM side timeout). The three values are tunable via setConnectTimeoutSeconds / setServerAliveIntervalSeconds / setServerAliveCountMax. Defaults are deliberately aggressive so existing call sites get protection without any code change. Background: in ZSTAC-72837 collect-log jstack, 222 KVMHost reconnect flows (checkConnectConditions step check-if-host-can-reach-management -node) had ssh subprocesses stuck in Process.waitFor for 30+ minutes, exhausting the ThreadFacadeImpl worker pool and contributing to MN unknown + VIP failover. host/ping HTTP itself already has a 10s RESTFacade timeout; the gap was on the post ping-fail reconnect path where SshShell.runCommand had no timeout. Adding ssh client side timeouts breaks the indefinite wait regardless of upstream trigger (slow remote sshd / DNS / PAM / TCP handshake). Resolves: ZSTAC-72837 Change-Id: I1f999a8404352a178910920e92c56bbceb2c1e5f (cherry picked from commit 3cffcd3c82e01ba3a1c8f8be5c3b38fda9379333) --- .../java/org/zstack/utils/ssh/SshShell.java | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/utils/src/main/java/org/zstack/utils/ssh/SshShell.java b/utils/src/main/java/org/zstack/utils/ssh/SshShell.java index 80ea35c94ce..0c333fdbfbc 100755 --- a/utils/src/main/java/org/zstack/utils/ssh/SshShell.java +++ b/utils/src/main/java/org/zstack/utils/ssh/SshShell.java @@ -29,6 +29,14 @@ public class SshShell { private String privateKey; private int port = 22; private Boolean withSudo = true; + private int connectTimeoutSeconds = 10; + private int serverAliveIntervalSeconds = 5; + private int serverAliveCountMax = 2; + + private String sshTimeoutOptions() { + return String.format("-o ConnectTimeout=%d -o ServerAliveInterval=%d -o ServerAliveCountMax=%d", + connectTimeoutSeconds, serverAliveIntervalSeconds, serverAliveCountMax); + } private void checkParams() { DebugUtils.Assert(hostname != null && !hostname.trim().equals(""), "hostname cannot be null"); @@ -45,13 +53,13 @@ public SshResult runCommand(String cmd) { if (privateKey != null) { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, privateKey); - ssh = String.format("ssh -q -i %s -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no -p %s %s@%s '%s'", - tempPasswordFile.getAbsolutePath(), port, username, hostname, cmd); + ssh = String.format("ssh -q -i %s -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no %s -p %s %s@%s '%s'", + tempPasswordFile.getAbsolutePath(), sshTimeoutOptions(), port, username, hostname, cmd); } else { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, password); - ssh = String.format("sshpass -f%s ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -p %s %s@%s '%s'", - tempPasswordFile.getAbsolutePath(), port, username, hostname, cmd); + ssh = String.format("sshpass -f%s ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no %s -p %s %s@%s '%s'", + tempPasswordFile.getAbsolutePath(), sshTimeoutOptions(), port, username, hostname, cmd); } if (logger.isTraceEnabled()) { @@ -89,7 +97,7 @@ public SshResult runScript(String script) { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, privateKey); ssh = ln( - "ssh -q -i {0} -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no -p {1} -T {2}@{3} << 'EOF'", + "ssh -q -i {0} -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o StrictHostKeyChecking=no {5} -p {1} -T {2}@{3} << 'EOF'", "s=`mktemp`", "cat << 'EOT' > $s", "{4}", @@ -99,12 +107,12 @@ public SshResult runScript(String script) { "rm -f $s", "exit $ret", "EOF" - ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script); + ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script, sshTimeoutOptions()); } else { tempPasswordFile = File.createTempFile("zstack", "tmp"); writeSecretFile(tempPasswordFile, password); ssh = ln( - "sshpass -f{0} ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -p {1} -T {2}@{3} << 'EOF'", + "sshpass -f{0} ssh -q -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o StrictHostKeyChecking=no {5} -p {1} -T {2}@{3} << 'EOF'", "s=`mktemp`", "cat << 'EOT' > $s", "{4}", @@ -114,7 +122,7 @@ public SshResult runScript(String script) { "rm -f $s", "exit $ret", "EOF" - ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script); + ).format(tempPasswordFile.getAbsolutePath(), port, username, hostname, script, sshTimeoutOptions()); } if (logger.isTraceEnabled()) { @@ -209,4 +217,28 @@ public Boolean getWithSudo() { public void setWithSudo(Boolean withSudo) { this.withSudo = withSudo; } + + public int getConnectTimeoutSeconds() { + return connectTimeoutSeconds; + } + + public void setConnectTimeoutSeconds(int connectTimeoutSeconds) { + this.connectTimeoutSeconds = connectTimeoutSeconds; + } + + public int getServerAliveIntervalSeconds() { + return serverAliveIntervalSeconds; + } + + public void setServerAliveIntervalSeconds(int serverAliveIntervalSeconds) { + this.serverAliveIntervalSeconds = serverAliveIntervalSeconds; + } + + public int getServerAliveCountMax() { + return serverAliveCountMax; + } + + public void setServerAliveCountMax(int serverAliveCountMax) { + this.serverAliveCountMax = serverAliveCountMax; + } } \ No newline at end of file From 203b822ce83aaa1bed015e008b024ca00b99f9e0 Mon Sep 17 00:00:00 2001 From: J M Date: Thu, 28 May 2026 18:42:54 +0800 Subject: [PATCH 27/53] [kvm]: bound host reachability check timeout Resolves: ZSTAC-72837 Change-Id: Ic9ab3cd09ef6bdc37979605374c93742533cf23d (cherry picked from commit a6b84f720cfb7c163ade0b44299912f3188d0b67) --- plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 34e0c434296..192f9259465 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5490,7 +5490,7 @@ public void run(FlowTrigger trigger, Map data) { sshShell.setPassword(getSelf().getPassword()); sshShell.setPort(getSelf().getPort()); sshShell.setWithSudo(false); - final String cmd = String.format("curl --connect-timeout 10 %s|| wget --spider -q --connect-timeout=10 %s|| test $? -eq 8", restf.getCallbackUrl(), restf.getCallbackUrl()); + final String cmd = String.format("curl --connect-timeout 10 --max-time 15 %s|| wget --spider -q --connect-timeout=10 --read-timeout=10 --tries=1 %s|| test $? -eq 8", restf.getCallbackUrl(), restf.getCallbackUrl()); SshResult ret = sshShell.runCommand(cmd); if (ret.getStderr() != null && ret.getStderr().contains("No route to host")) { // c.f. https://access.redhat.com/solutions/1120533 From 43e485ef42782461f2a4d69f223669a96c51ebb2 Mon Sep 17 00:00:00 2001 From: "Wen, Yubo" Date: Mon, 25 Mar 2024 10:03:04 +0000 Subject: [PATCH 28/53] [storage]: Add a task to promptly check,update the latest status of ps and host After a Ceph dead and fail, the host dis connects from the PS, potentially becoming offline. Post-failure, if the host-storage link isn't restored promptly, issues may arise where healthy hosts are wrongly filtered out. Thus,Mn needs to actively identify these states of disconnections. so wrote a scheduled task in PrimaryStorage -ManagerImpl to check and update the latest connection status between the host and ps on a scheduled basis. GlobalConfigImpact Resolves: ZSTAC-64050 Change-Id: I76676a656d6a61796e6a657161757a6a64706568 (cherry picked from commit 9e1256705525d6e30769b77d8b4419074a55b557) --- storage/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/pom.xml b/storage/pom.xml index d5fe306aec3..f4f07e048fe 100755 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -3,7 +3,7 @@ zstack org.zstack - 5.4.0 + 5.4.0 .. storage From cda7d94ad3e97a06e52189e6eceb692f72a9f1e8 Mon Sep 17 00:00:00 2001 From: J M Date: Thu, 29 May 2025 14:45:31 +0800 Subject: [PATCH 29/53] [storage]: trash top after storage live migration top is unused after storage migration even if its snapshot referenced by other volumes. only skip trash volume when its snapshots are inner snapshot. Resolves: ZSTAC-71514 Resolves: ZSV-10468 Change-Id: I636b6d726d6c757478737869726f6a777a736663 (cherry picked from commit d6f74e59eb0616bc603650c43d14cd095994096c) --- .../snapshot/reference/VolumeSnapshotReferenceUtils.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java index 49662466a46..30069bda97e 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/reference/VolumeSnapshotReferenceUtils.java @@ -83,6 +83,14 @@ public static String getVolumeInstallUrlBackingOtherVolume(String volumeUuid) { return null; } + // use getVolumeAllSnapshotsReferencedOtherVolumes or isVolumeDirectlyReferenceByOthers + @Deprecated + public static List getVolumeSnapshotsReferencedByOtherVolumes(String volumeUuid) { + return getVolumeReferenceRef(volumeUuid).stream() + .map(VolumeSnapshotReferenceVO::getVolumeSnapshotInstallUrl).distinct() + .collect(Collectors.toList()); + } + // get volume snapshotUuids referenced by other volumes directly or indirectly // FIXME split different primary storage snapshot reference public static Set getVolumeAllSnapshotsReferencedByOtherVolumes(String volumeUuid) { From 070a637e7d4c05f6fd7439f47281cf5204fb8cb5 Mon Sep 17 00:00:00 2001 From: J M Date: Thu, 19 Mar 2026 13:23:52 +0800 Subject: [PATCH 30/53] [core,kvm]: fix SSH session leak in CallBackNetworkChecker and KVMHost 1. CallBackNetworkChecker.stopAnsible() creates Ssh object but never closes it, leaking JSch Session thread on every call. 2. KVMHost "check-host-is-taken-over" flow has the same pattern. When a host continuously fails reconnect (e.g. sharedblock VG error), each reconnect cycle leaks 1-2 SSH sessions. Over days this exhausts heap memory (18000+ threads observed) causing MN OOM and Unknown status. Fix: add finally { ssh.close() } to both code paths. Resolves: ZSTAC-83305 Resolves: ZSTAC-82731 Change-Id: I786e766e79776d6b7a75786d776278696c76666f (cherry picked from commit 7b6ceead7ac67eaba829989d20fe796d857d5f4a) --- .../org/zstack/core/ansible/CallBackNetworkChecker.java | 2 ++ plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java index ad7be7f7bb9..5ef7b1f7b5a 100644 --- a/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java +++ b/core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java @@ -71,6 +71,8 @@ public ErrorCode stopAnsible() { return useNcatAndNmapToTestConnection(ssh); } catch (SshException e) { return operr(e.getMessage()); + } finally { + ssh.close(); } } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 192f9259465..857f21e163f 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5552,10 +5552,10 @@ public boolean skip(Map data) { @Override public void run(FlowTrigger trigger, Map data) { + Ssh ssh = new Ssh().setUsername(getSelf().getUsername()) + .setPassword(getSelf().getPassword()).setPort(getSelf().getPort()) + .setHostname(getSelf().getManagementIp()); try { - Ssh ssh = new Ssh().setUsername(getSelf().getUsername()) - .setPassword(getSelf().getPassword()).setPort(getSelf().getPort()) - .setHostname(getSelf().getManagementIp()); ssh.command(String.format("grep -i ^uuid %s | sed 's/uuid://g'", hostTakeOverFlagPath)); SshResult hostRet = ssh.run(); if (hostRet.isSshFailure() || hostRet.getReturnCode() != 0) { @@ -5604,6 +5604,8 @@ public void run(FlowTrigger trigger, Map data) { logger.warn(e.getMessage(), e); trigger.next(); return; + } finally { + ssh.close(); } } }); From 643a40c23f9e84d5817d47224d9698bc5ab6e603 Mon Sep 17 00:00:00 2001 From: J M Date: Tue, 28 Apr 2026 14:21:50 +0800 Subject: [PATCH 31/53] [primaryStorage]: validate ExternalPrimaryStorage config JSON at marshal time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per ZSTAC-84817, malformed JSON in APIAddExternalPrimaryStorage.config slipped past the API boundary and surfaced as SYS.1000 InternalError from a deep Gson parse, after the message had already reached the cloud-bus handler. Fix: * PrimaryStorageApiInterceptor now validates msg.getConfig() with org.json (strict — rejects unquoted keys / missing colons that Gson lenient mode silently tolerates) and throws ApiMessageInterceptionException with ORG_ZSTACK_STORAGE_PRIMARY_10053 if it isn't a valid JSON object. * conf/serviceConfig/externalPrimaryStorage.xml wires PrimaryStorageApiInterceptor at service level so the interceptor actually runs for APIAddExternalPrimaryStorageMsg (the existing wiring in primaryStorage.xml does not cover messages declared in this file). * CloudOperationsErrorCode.ORG_ZSTACK_STORAGE_PRIMARY_10053 constant added. Test: ZbsPrimaryStorageCase gains testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor — sends "{this is not valid json" via API, asserts the call fails AND no PrimaryStorageVO/ExternalPrimaryStorageVO/PrimaryStorageOutputProtocolRefVO row leaks. Verified locally: Tests run: 1, Failures: 0, Errors: 0, Time elapsed: 92.788 sec, BUILD SUCCESS. Resolves: ZSTAC-84817 Change-Id: Icf6c648133d7866edf35940d56a28f74f4c64817 (cherry picked from commit f8e039a061f305a952e8cda92f73ce15cbf3eb94) --- conf/serviceConfig/externalPrimaryStorage.xml | 1 + .../primary/PrimaryStorageApiInterceptor.java | 18 +++++++++ .../addon/zbs/ZbsPrimaryStorageCase.groovy | 40 +++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/conf/serviceConfig/externalPrimaryStorage.xml b/conf/serviceConfig/externalPrimaryStorage.xml index d1687fe7685..e076cc55bc1 100644 --- a/conf/serviceConfig/externalPrimaryStorage.xml +++ b/conf/serviceConfig/externalPrimaryStorage.xml @@ -1,6 +1,7 @@ storage.primary + PrimaryStorageApiInterceptor org.zstack.header.storage.addon.primary.APIUpdateExternalPrimaryStorageMsg diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java index 54b59efd08f..cd893b5c265 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageApiInterceptor.java @@ -13,9 +13,12 @@ import org.zstack.header.apimediator.ApiMessageInterceptionException; import org.zstack.header.apimediator.ApiMessageInterceptor; import org.zstack.header.apimediator.StopRoutingException; +import org.json.JSONException; +import org.json.JSONObject; import org.zstack.header.cluster.ClusterVO; import org.zstack.header.cluster.ClusterVO_; import org.zstack.header.message.APIMessage; +import org.zstack.header.storage.addon.primary.APIAddExternalPrimaryStorageMsg; import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO; import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO_; import org.zstack.header.storage.primary.*; @@ -76,12 +79,27 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APICreateVolumeSnapshotGroupMsg) msg); } else if (msg instanceof APIAddStorageProtocolMsg) { validate((APIAddStorageProtocolMsg) msg); + } else if (msg instanceof APIAddExternalPrimaryStorageMsg) { + validate((APIAddExternalPrimaryStorageMsg) msg); } setServiceId(msg); return msg; } + private void validate(APIAddExternalPrimaryStorageMsg msg) { + String config = msg.getConfig(); + if (config == null || config.trim().isEmpty()) { + return; + } + try { + new JSONObject(config); + } catch (JSONException e) { + throw new ApiMessageInterceptionException(argerr( + "config is not a valid JSON object: %s", e.getMessage())); + } + } + private void validate(APIAddStorageProtocolMsg msg) { if (Q.New(PrimaryStorageOutputProtocolRefVO.class) .eq(PrimaryStorageOutputProtocolRefVO_.primaryStorageUuid, msg.getUuid()) diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 5fc8f14aa97..75db97e4b55 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -4,9 +4,11 @@ import org.springframework.http.HttpEntity import org.zstack.core.db.Q import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ +import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO import org.zstack.header.storage.primary.PrimaryStorageHostRefVO import org.zstack.header.storage.primary.PrimaryStorageHostRefVO_ import org.zstack.header.storage.primary.PrimaryStorageStatus +import org.zstack.header.storage.primary.PrimaryStorageVO import org.zstack.cbd.MdsUri import org.zstack.kvm.KVMAgentCommands import org.zstack.sdk.* @@ -160,6 +162,7 @@ class ZbsPrimaryStorageCase extends SubCase { testMdsPing() testCheckHostStorageConnection() testNegativeScenario() + testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor() testDataVolumeNegativeScenario() testDecodeMdsUriWithSpecialPassword() } @@ -451,6 +454,43 @@ class ZbsPrimaryStorageCase extends SubCase { } } + void testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor() { + long extPsCountBefore = Q.New(ExternalPrimaryStorageVO.class).count() + long primaryStorageCountBefore = Q.New(PrimaryStorageVO.class).count() + long protocolRefCountBefore = Q.New(PrimaryStorageOutputProtocolRefVO.class).count() + + expect(AssertionError.class) { + addExternalPrimaryStorage { + zoneUuid = zone.uuid + name = "zbs-bad-json" + identity = "zbs" + defaultOutputProtocol = "CBD" + config = "{this is not valid json" + url = "" + } + } + + long extPsCountAfter = Q.New(ExternalPrimaryStorageVO.class).count() + long primaryStorageCountAfter = Q.New(PrimaryStorageVO.class).count() + long protocolRefCountAfter = Q.New(PrimaryStorageOutputProtocolRefVO.class).count() + assert extPsCountAfter == extPsCountBefore : \ + "ExternalPrimaryStorageVO leaked despite marshal-time JSON rejection: " + + "before=${extPsCountBefore} after=${extPsCountAfter}" + assert primaryStorageCountAfter == primaryStorageCountBefore : \ + "PrimaryStorageVO leaked despite marshal-time JSON rejection: " + + "before=${primaryStorageCountBefore} after=${primaryStorageCountAfter}" + assert protocolRefCountAfter == protocolRefCountBefore : \ + "PrimaryStorageOutputProtocolRefVO leaked despite marshal-time JSON rejection: " + + "before=${protocolRefCountBefore} after=${protocolRefCountAfter}" + boolean stalePsByName = Q.New(ExternalPrimaryStorageVO.class) + .eq(ExternalPrimaryStorageVO_.name, "zbs-bad-json") + .isExists() + assert !stalePsByName : "stale ExternalPrimaryStorageVO[name=zbs-bad-json] reached DB despite interceptor rejection" + + def psList = queryPrimaryStorage {} as List + assert psList != null : "queryPrimaryStorage threw / returned null after attempted bad-JSON Add" + } + void testDataVolumeNegativeScenario() { env.simulator(ZbsStorageController.CREATE_VOLUME_PATH) { HttpEntity e, EnvSpec spec -> def rsp = new ZbsStorageController.CreateVolumeRsp() From 4c2d3abb8849d9b3d6623c07a9efa5acb3b38566 Mon Sep 17 00:00:00 2001 From: zhangjianjun Date: Fri, 26 Dec 2025 18:08:12 +0800 Subject: [PATCH 32/53] [sdnController]: set lacp mode when change host Resolves: ZSTAC-76537 Change-Id: I61636a6e78746b7166796572677a7a746c6d7469 Signed-off-by: zhangjianjun (cherry picked from commit 1407260a5cd567c29e6a1a32b5a699910df830ec) --- .../SdnControllerApiInterceptor.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java index 741a99cdb70..8f395c2bb63 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java @@ -271,9 +271,25 @@ private void validate(APISdnControllerChangeHostMsg msg) { msg.setNicNames(new ArrayList<>(nicNamePciAddressMap.keySet())); } - if (msg.getNicNames().size() > 1 && msg.getBondMode() == null) { + if (msg.getBondMode() == null) { msg.setBondMode(refVO.getBondMode()); } + + if (msg.getBondMode().equals(refVO.getBondMode())) { + msg.setLacpMode(refVO.getLacpMode()); + } + + if (msg.getBondMode().equals(L2NetworkConstant.BONDING_MODE_TCP)) { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_ACTIVE); + } else if (msg.getBondMode().equals(L2NetworkConstant.BONDING_MODE_SLB)) { + if (msg.getLacpMode() == null) { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_OFF); + } else if (!(msg.getLacpMode().equals(L2NetworkConstant.LACP_MODE_PASSIVE))) { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_OFF); + } + } else { + msg.setLacpMode(L2NetworkConstant.LACP_MODE_OFF); + } } private void validateH3cTenantStatus(String l3NetworkUuid, String sdnControllerUuid) { From 3de9afaddd4a341e023b3818405aa4760df0377b Mon Sep 17 00:00:00 2001 From: "dejing.liu" Date: Tue, 2 Jun 2026 14:09:51 +0800 Subject: [PATCH 33/53] [eip]: fix EIP attach release race Persist vmNicUuid/guestIp before backend apply so VM release can see in-flight EIP bindings. Roll back only when the failed attach still matches the same EIP/NIC/IP tuple. Test: real Flat EIP flow and AttachEipRaceCase JIRA: ZSTAC-86435 Related: ZSTAC-83309 Change-Id: Ie8795d5bd1c3e90aa10e18cfef19d4968c73cfee --- .../network/service/eip/EipManagerImpl.java | 26 +- .../flat/eip/AttachEipRaceCase.groovy | 258 ++++++++++++++++++ 2 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy diff --git a/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java b/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java index a44fc71f75a..eed1ceb33ed 100755 --- a/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java +++ b/plugin/eip/src/main/java/org/zstack/network/service/eip/EipManagerImpl.java @@ -576,7 +576,13 @@ private void handle(final APIAttachEipMsg msg) { return; } - EipInventory eip = EipInventory.valueOf(vo); + final String attachedVmNicUuid = nicInventory.getUuid(); + final String attachedGuestIp = guestIp.getIp(); + vo.setVmNicUuid(attachedVmNicUuid); + vo.setGuestIp(attachedGuestIp); + final EipVO updated = dbf.updateAndRefresh(vo); + + EipInventory eip = EipInventory.valueOf(updated); String l3NetworkUuid = getEipL3Network(nicInventory, eip); NetworkServiceProviderType providerType = nwServiceMgr.getTypeOfNetworkServiceProviderForService( l3NetworkUuid, EipConstant.EIP_TYPE); @@ -585,15 +591,25 @@ private void handle(final APIAttachEipMsg msg) { attachEip(struct, providerType.toString(), new Completion(msg) { @Override public void success() { - vo.setVmNicUuid(nicInventory.getUuid()); - vo.setGuestIp(guestIp.getIp()); - EipVO evo = dbf.updateAndRefresh(vo); - evt.setInventory(EipInventory.valueOf(evo)); + evt.setInventory(EipInventory.valueOf(dbf.reload(updated))); bus.publish(evt); } @Override public void fail(ErrorCode errorCode) { + int affectedRows = SQL.New(EipVO.class) + .eq(EipVO_.uuid, msg.getEipUuid()) + .eq(EipVO_.vmNicUuid, attachedVmNicUuid) + .eq(EipVO_.guestIp, attachedGuestIp) + .set(EipVO_.vmNicUuid, null) + .set(EipVO_.guestIp, null) + .update(); + if (affectedRows == 0) { + logger.warn(String.format( + "failed to roll back EIP[uuid:%s] attach binding[vmNicUuid:%s, guestIp:%s], " + + "the binding may have been changed by another flow", + msg.getEipUuid(), attachedVmNicUuid, attachedGuestIp)); + } evt.setError(errorCode); bus.publish(evt); } diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy new file mode 100644 index 00000000000..27123046860 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/flat/eip/AttachEipRaceCase.groovy @@ -0,0 +1,258 @@ +package org.zstack.test.integration.networkservice.provider.flat.eip + +import org.springframework.http.HttpEntity +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.eip.EipVO +import org.zstack.network.service.flat.FlatEipBackend +import org.zstack.network.service.flat.FlatNetworkServiceConstant +import org.zstack.network.service.userdata.UserdataConstant +import org.zstack.sdk.AttachEipAction +import org.zstack.sdk.EipInventory +import org.zstack.sdk.StopVmInstanceAction +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.HttpError +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class AttachEipRaceCase extends SubCase { + EnvSpec env + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(8) + cpu = 4 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image" + url = "http://zstack.org/download/test.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster" + hypervisorType = "KVM" + + kvm { + name = "kvm" + managementIp = "localhost" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + attachL2Network("l2-pub") + } + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + + service { + provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING + types = [NetworkServiceType.DHCP.toString(), EipConstant.EIP_NETWORK_SERVICE_TYPE, UserdataConstant.USERDATA_TYPE_STRING] + } + + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + + l2NoVlanNetwork { + name = "l2-pub" + physicalInterface = "eth1" + + l3Network { + name = "pubL3" + + ip { + startIp = "11.168.100.10" + endIp = "11.168.100.100" + netmask = "255.255.255.0" + gateway = "11.168.100.1" + } + } + } + + attachBackupStorage("sftp") + + eip { + name = "eip-race" + useVip("pubL3") + } + + eip { + name = "eip-fail" + useVip("pubL3") + } + } + + vm { + name = "vm-race" + useImage("image") + useL3Networks("l3") + useInstanceOffering("instanceOffering") + } + + vm { + name = "vm-fail" + useImage("image") + useL3Networks("l3") + useInstanceOffering("instanceOffering") + } + } + } + + @Override + void test() { + env.create { + testStopVmDuringAttachReleasesEip() + testAttachFailureRollsBackEarlyBinding() + } + } + + void testStopVmDuringAttachReleasesEip() { + EipInventory eip = env.inventoryByName("eip-race") as EipInventory + VmInstanceInventory vm = env.inventoryByName("vm-race") as VmInstanceInventory + String nicUuid = vm.vmNics[0].uuid + String guestIp = vm.vmNics[0].ip + String sessionId = adminSession() + + CountDownLatch applyEntered = new CountDownLatch(1) + CountDownLatch releaseApply = new CountDownLatch(1) + CountDownLatch deleteCalled = new CountDownLatch(1) + FlatEipBackend.ApplyEipCmd applyCmd = null + FlatEipBackend.DeleteEipCmd deleteCmd = null + + env.afterSimulator(FlatEipBackend.APPLY_EIP_PATH) { rsp, HttpEntity entity -> + FlatEipBackend.ApplyEipCmd cmd = json(entity.getBody(), FlatEipBackend.ApplyEipCmd.class) + if (cmd.eip.eipUuid == eip.uuid) { + applyCmd = cmd + applyEntered.countDown() + assert releaseApply.await(30, TimeUnit.SECONDS) + } + return rsp + } + + env.afterSimulator(FlatEipBackend.DELETE_EIP_PATH) { rsp, HttpEntity entity -> + FlatEipBackend.DeleteEipCmd cmd = json(entity.getBody(), FlatEipBackend.DeleteEipCmd.class) + if (cmd.eip.eipUuid == eip.uuid) { + deleteCmd = cmd + deleteCalled.countDown() + } + return rsp + } + + Throwable attachFailure = null + Thread attachThread = Thread.start { + try { + AttachEipAction action = new AttachEipAction() + action.eipUuid = eip.uuid + action.vmNicUuid = nicUuid + action.sessionId = sessionId + def result = action.call() + if (result.error != null) { + attachFailure = new AssertionError(result.error.toString()) + } + } catch (Throwable t) { + attachFailure = t + } + } + + assert applyEntered.await(30, TimeUnit.SECONDS) + try { + retryInSecs { + EipVO vo = dbFindByUuid(eip.uuid, EipVO.class) + assert vo.vmNicUuid == nicUuid + assert vo.guestIp == guestIp + } + } finally { + releaseApply.countDown() + } + + attachThread.join(30000) + + assert !attachThread.isAlive() + assert attachFailure == null + assert applyCmd != null + + StopVmInstanceAction action = new StopVmInstanceAction() + action.uuid = vm.uuid + action.sessionId = sessionId + def result = action.call() + + assert result.error == null + assert deleteCalled.await(30, TimeUnit.SECONDS) + assert deleteCmd != null + assert deleteCmd.eip.eipUuid == eip.uuid + } + + void testAttachFailureRollsBackEarlyBinding() { + EipInventory eip = env.inventoryByName("eip-fail") as EipInventory + VmInstanceInventory vm = env.inventoryByName("vm-fail") as VmInstanceInventory + + env.afterSimulator(FlatEipBackend.APPLY_EIP_PATH) { rsp, HttpEntity entity -> + FlatEipBackend.ApplyEipCmd cmd = json(entity.getBody(), FlatEipBackend.ApplyEipCmd.class) + if (cmd.eip.eipUuid == eip.uuid) { + throw new HttpError(500, "on purpose") + } + return rsp + } + + AttachEipAction action = new AttachEipAction() + action.eipUuid = eip.uuid + action.vmNicUuid = vm.vmNics[0].uuid + action.sessionId = adminSession() + def result = action.call() + + assert result.error != null + retryInSecs { + EipVO vo = dbFindByUuid(eip.uuid, EipVO.class) + assert vo.vmNicUuid == null + assert vo.guestIp == null + } + } + + @Override + void clean() { + env.delete() + } +} From 4a6b97d2b8203ba2932bd5a6505f3aaf594e53b1 Mon Sep 17 00:00:00 2001 From: "pengchao.liu" Date: Tue, 2 Jun 2026 16:31:13 +0800 Subject: [PATCH 34/53] [portForwarding]: serialize rule creation Concurrent creates could pass interceptor validation before any rule was persisted. Recheck VIP port conflicts inside the VIP sync chain with closed-range boundary handling. Resolves: ZSTAC-77673 Change-Id: I7666646b7567696e6e7568736666766b61617766 --- .../PortForwardingManagerImpl.java | 65 ++++++- ...currentCreatePortForwardingRuleCase.groovy | 183 ++++++++++++++++++ 2 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy diff --git a/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java b/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java index 96e51bc6d86..d4370ef6799 100755 --- a/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java +++ b/plugin/portForwarding/src/main/java/org/zstack/network/service/portforwarding/PortForwardingManagerImpl.java @@ -54,6 +54,7 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; +import static org.zstack.core.Platform.argerr; import static org.zstack.core.Platform.err; import static org.zstack.core.Platform.operr; import static org.zstack.utils.CollectionDSL.list; @@ -679,11 +680,59 @@ public void fail(ErrorCode errorCode) { private void handle(APICreatePortForwardingRuleMsg msg) { + thdf.chainSubmit(new ChainTask(msg) { + @Override + public String getSyncSignature() { + return String.format("portforwardingrule-vip-%s", msg.getVipUuid()); + } + + @Override + public void run(SyncTaskChain chain) { + doCreatePortForwardingRule(msg, chain); + } + + @Override + public String getName() { + return String.format("api-create-portforwardingrule-vip-%s", msg.getVipUuid()); + } + }); + } + + private ErrorCode validateVipPortConflict(String vipUuid, String protocolType, int vipPortStart, int vipPortEnd) { + List vos = Q.New(PortForwardingRuleVO.class) + .eq(PortForwardingRuleVO_.vipUuid, vipUuid) + .eq(PortForwardingRuleVO_.protocolType, PortForwardingProtocolType.valueOf(protocolType)) + .list(); + PortForwardingRuleVO vo = vos.stream() + .filter(it -> { + int existingStart = it.getVipPortStart(); + int existingEnd = it.getVipPortEnd(); + return existingStart <= vipPortEnd && vipPortStart <= existingEnd; + }) + .findFirst() + .orElse(null); + if (vo == null) { + return null; + } + + return argerr("vip port range[vipStartPort:%s, vipEndPort:%s] overlaps with rule[uuid:%s, vipStartPort:%s, vipEndPort:%s]", + vipPortStart, vipPortEnd, vo.getUuid(), vo.getVipPortStart(), vo.getVipPortEnd()); + } + + private void doCreatePortForwardingRule(APICreatePortForwardingRuleMsg msg, SyncTaskChain syncChain) { final APICreatePortForwardingRuleEvent evt = new APICreatePortForwardingRuleEvent(msg.getId()); int vipPortEnd = msg.getVipPortEnd() == null ? msg.getVipPortStart() : msg.getVipPortEnd(); int privatePortEnd = msg.getPrivatePortEnd() == null ? msg.getPrivatePortStart() : msg.getPrivatePortEnd(); + ErrorCode errCode = validateVipPortConflict(msg.getVipUuid(), msg.getProtocolType(), msg.getVipPortStart(), vipPortEnd); + if (errCode != null) { + evt.setError(errCode); + bus.publish(evt); + syncChain.next(); + return; + } + VipVO vip = dbf.findByUuid(msg.getVipUuid(), VipVO.class); final PortForwardingRuleVO vo = new PortForwardingRuleVO(); if (msg.getResourceUuid() != null) { @@ -712,8 +761,8 @@ protected void scripts() { } }.execute(); - FlowChain chain = FlowChainBuilder.newShareFlowChain(); - chain.setName("create-portforwading"); + FlowChain flowChain = FlowChainBuilder.newShareFlowChain(); + flowChain.setName("create-portforwading"); VipInventory vipInventory = VipInventory.valueOf(vip); if (msg.getVmNicUuid() == null) { ModifyVipAttributesStruct struct = new ModifyVipAttributesStruct(); @@ -726,6 +775,7 @@ protected void scripts() { public void success() { evt.setInventory(PortForwardingRuleInventory.valueOf(vo)); bus.publish(evt); + syncChain.next(); } @Override @@ -733,6 +783,7 @@ public void fail(ErrorCode errorCode) { dbf.remove(vo); evt.setError(errorCode); bus.publish(evt); + syncChain.next(); } }); @@ -756,6 +807,7 @@ public void fail(ErrorCode errorCode) { public void success() { evt.setInventory(PortForwardingRuleInventory.valueOf(vo)); bus.publish(evt); + syncChain.next(); } @Override @@ -763,13 +815,14 @@ public void fail(ErrorCode errorCode) { dbf.remove(vo); evt.setError(errorCode); bus.publish(evt); + syncChain.next(); } }); return; } - chain.then(new ShareFlow() { + flowChain.then(new ShareFlow() { @Override public void setup() { vo.setVmNicUuid(vmNic.getUuid()); @@ -817,7 +870,7 @@ public void run(AttachPortForwardingRuleExtensionPoint extp) { public void run(FlowTrigger trigger, Map data) { NetworkServiceProviderType providerType = (NetworkServiceProviderType)data.get("providerType"); final PortForwardingStruct struct = makePortForwardingStruct(ruleInv); - attachPortForwardingRule(struct, providerType.toString(), new Completion(msg) { + doAttachPortForwardingRule(struct, providerType.toString(), new Completion(msg) { @Override public void success() { CollectionUtils.safeForEach(attachRuleExts, new ForEachFunction() { @@ -852,11 +905,12 @@ public void fail(ErrorCode errorCode) { }); - chain.done(new FlowDoneHandler(msg) { + flowChain.done(new FlowDoneHandler(msg) { @Override public void handle(Map data) { evt.setInventory(PortForwardingRuleInventory.valueOf(dbf.reload(vo))); bus.publish(evt); + syncChain.next(); } }).error(new FlowErrorHandler(msg) { @Override @@ -864,6 +918,7 @@ public void handle(ErrorCode errCode, Map data) { dbf.remove(vo); evt.setError(errCode); bus.publish(evt); + syncChain.next(); } }).start(); } diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy new file mode 100644 index 00000000000..8e71b93e7fa --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/portforwarding/ConcurrentCreatePortForwardingRuleCase.groovy @@ -0,0 +1,183 @@ +package org.zstack.test.integration.networkservice.provider.virtualrouter.portforwarding + +import org.zstack.core.db.Q +import org.zstack.header.errorcode.SysErrors +import org.zstack.network.service.portforwarding.PortForwardingProtocolType +import org.zstack.network.service.portforwarding.PortForwardingRuleVO +import org.zstack.network.service.portforwarding.PortForwardingRuleVO_ +import org.zstack.sdk.CreatePortForwardingRuleAction +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.VipInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class ConcurrentCreatePortForwardingRuleCase extends SubCase { + EnvSpec env + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + zone { + name = "zone" + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "pubL3" + + ip { + startIp = "11.168.100.10" + endIp = "11.168.100.100" + netmask = "255.255.255.0" + gateway = "11.168.100.1" + } + } + } + } + } + } + + @Override + void test() { + env.create { + testConcurrentCreateSamePortForwardingRule() + } + } + + void testConcurrentCreateSamePortForwardingRule() { + L3NetworkInventory pubL3 = env.inventoryByName("pubL3") as L3NetworkInventory + VipInventory vip = createVip { + name = "vip" + l3NetworkUuid = pubL3.uuid + } as VipInventory + + int requestCount = 50 + String sessionId = adminSession() + Closure createRule = { String name, int vipPortStart, int vipPortEnd -> + CreatePortForwardingRuleAction action = new CreatePortForwardingRuleAction() + action.name = name + action.vipUuid = vip.uuid + action.vipPortStart = vipPortStart + action.vipPortEnd = vipPortEnd + action.privatePortStart = vipPortStart + action.privatePortEnd = vipPortEnd + action.protocolType = PortForwardingProtocolType.TCP.toString() + action.sessionId = sessionId + return action.call() + } + CountDownLatch ready = new CountDownLatch(requestCount) + CountDownLatch start = new CountDownLatch(1) + CountDownLatch done = new CountDownLatch(requestCount) + def results = new ConcurrentLinkedQueue() + def errors = new ConcurrentLinkedQueue() + + (0.. + Thread.start("create-same-port-forwarding-rule-${index}") { + try { + ready.countDown() + if (!start.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("not all createPortForwardingRule workers became ready within 10 seconds") + } + + results.add(createRule("pf-${index}", 8080, 8080)) + } catch (Throwable t) { + errors.add(t) + } finally { + done.countDown() + } + } + } + + assert ready.await(10, TimeUnit.SECONDS) : + "createPortForwardingRule workers did not all become ready: expected=${requestCount}" + start.countDown() + assert done.await(60, TimeUnit.SECONDS) : + "concurrent createPortForwardingRule requests did not finish within 60 seconds" + assert errors.isEmpty() : + "worker thread threw unexpected errors: ${errors.collect { it.message }.join('; ')}" + + def successes = results.findAll { it.error == null } + def failures = results.findAll { it.error != null } + assert successes.size() == 1 : + "same VIP/protocol/port create must succeed once: expectedSuccess=1, actualSuccess=${successes.size()}, results=${results.size()}" + assert failures.size() == requestCount - 1 : + "duplicate same VIP/protocol/port creates must fail: expectedFailures=${requestCount - 1}, actualFailures=${failures.size()}" + def formattedErrors = failures.collect { formatError(it.error) } + assert failures.every { isVipPortConflictError(it.error, 8080, 8080) } : + "duplicate failures should be VIP port conflict errors: errors=${formattedErrors}" + + def lowerBoundaryConflict = createRule("pf-lower-boundary-conflict", 8079, 8080) + assert lowerBoundaryConflict.error != null : + "closed VIP port range should conflict when new end touches existing start" + assert isVipPortConflictError(lowerBoundaryConflict.error, 8079, 8080) : + "lower boundary conflict should report VIP port conflict error: error=${lowerBoundaryConflict.error}" + + def upperBoundaryConflict = createRule("pf-upper-boundary-conflict", 8080, 8081) + assert upperBoundaryConflict.error != null : + "closed VIP port range should conflict when new start touches existing end" + assert isVipPortConflictError(upperBoundaryConflict.error, 8080, 8081) : + "upper boundary conflict should report VIP port conflict error: error=${upperBoundaryConflict.error}" + + def lowerAdjacent = createRule("pf-lower-adjacent", 8079, 8079) + assert lowerAdjacent.error == null : + "adjacent lower VIP port should not conflict with existing port 8080: error=${lowerAdjacent.error}" + + def upperAdjacent = createRule("pf-upper-adjacent", 8081, 8081) + assert upperAdjacent.error == null : + "adjacent upper VIP port should not conflict with existing port 8080: error=${upperAdjacent.error}" + + long sameRules = Q.New(PortForwardingRuleVO.class) + .eq(PortForwardingRuleVO_.vipUuid, vip.uuid) + .eq(PortForwardingRuleVO_.protocolType, PortForwardingProtocolType.TCP) + .eq(PortForwardingRuleVO_.vipPortStart, 8080) + .eq(PortForwardingRuleVO_.vipPortEnd, 8080) + .eq(PortForwardingRuleVO_.privatePortStart, 8080) + .eq(PortForwardingRuleVO_.privatePortEnd, 8080) + .count() + assert sameRules == 1L : + "database must not contain duplicate same VIP/protocol/port rules: expected=1, actual=${sameRules}, vipUuid=${vip.uuid}" + } + + boolean isVipPortConflictError(org.zstack.sdk.ErrorCode error, int vipPortStart, int vipPortEnd) { + return isPortForwardingOverlapError(error) || + isVipUsedPortRangeConflictError(error, vipPortStart, vipPortEnd) + } + + boolean isPortForwardingOverlapError(org.zstack.sdk.ErrorCode error) { + String details = error.details ?: "" + return error.code == SysErrors.INVALID_ARGUMENT_ERROR.toString() && + details.contains("vipStartPort") && + details.contains("vipEndPort") && + details.contains("uuid:") + } + + boolean isVipUsedPortRangeConflictError(org.zstack.sdk.ErrorCode error, int vipPortStart, int vipPortEnd) { + String details = error.details ?: "" + return error.code == SysErrors.OPERATION_ERROR.toString() && + details.contains("uuid:") && + details.contains(PortForwardingProtocolType.TCP.toString()) && + details.contains(vipPortStart.toString()) && + details.contains(vipPortEnd.toString()) + } + + String formatError(org.zstack.sdk.ErrorCode error) { + return "code=${error.code}, details=${error.details}" + } + + @Override + void clean() { + env.delete() + } +} From 6e046332a3561a9e92e31205cafacac74e0d09e2 Mon Sep 17 00:00:00 2001 From: J M Date: Mon, 18 May 2026 13:34:28 +0800 Subject: [PATCH 35/53] [kvm]: drain stale http calls on agent restart When kvmagent crashes/restarts, MN-side in-flight HTTP wrappers wait until the 5min default timeout before failing, blocking ChainTask queues (e.g. volume backup stuck). Add a drain trigger on the connect reply: when agent reports firstConnect=true, all pending HTTP calls scoped to that host with sentAtMillis < agentStartTime - 60s are failed immediately. - RESTFacade.failPendingCallsForResourceBefore(resourceUuid, cutoffMillis, err): single-pass scan over wrappers, filters by X-Resource-UUID header + send time; calls wrapper.fail(err) (same path as natural timeout, idempotent via called.compareAndSet). - AsyncHttpWrapper exposes getResourceUuid() + getSentAtMillis(); closure stores both at construction. - KVMAgentCommands.ConnectResponse adds firstConnect + agentStartTimeMillis. PingResponse unchanged. - KVMHost.connectToAgent drains stale calls inline when rsp.isFirstConnect() after a successful connect. - New i18n error code ORG_ZSTACK_CORE_REST_10013 (en/zh). - Groovy IT HostAgentRestartDrainCase drives the path via reconnectHost SDK action with KVM_CONNECT_PATH simulator override. Resolves: ZSTAC-58310 Change-Id: I1c2d5e3f7a8b9c0d2e4f5a6b7c8d9e0f1a2b3c4d (cherry picked from commit b569092d728babf40fbd342ee2a0eb2e3cb9c805) --- .../org/zstack/core/rest/RESTFacadeImpl.java | 41 ++++++ .../org/zstack/header/rest/RESTFacade.java | 3 + .../java/org/zstack/kvm/KVMAgentCommands.java | 18 +++ .../src/main/java/org/zstack/kvm/KVMHost.java | 10 ++ .../kvm/host/HostAgentRestartDrainCase.groovy | 117 ++++++++++++++++++ 5 files changed, 189 insertions(+) create mode 100644 test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 4f2157deca2..952ef0af20f 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -25,6 +25,7 @@ import org.zstack.core.thread.ThreadFacadeImpl.TimeoutTaskReceipt; import org.zstack.core.timeout.ApiTimeoutManager; import org.zstack.core.validation.ValidationFacade; +import org.zstack.header.Constants; import org.zstack.header.core.Completion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.errorcode.ErrorCode; @@ -80,6 +81,10 @@ private interface AsyncHttpWrapper { void fail(ErrorCode err); void success(HttpEntity responseEntity); + + String getResourceUuid(); + + long getSentAtMillis(); } private interface HttpCallHandlerWrapper { @@ -342,6 +347,9 @@ public void asyncJson(final String url, final String body, Map h HttpEntity req = new HttpEntity(body, requestHeaders); + final long sentAtMillis = System.currentTimeMillis(); + final String resourceUuid = (headers == null) ? null : headers.get(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID); + AsyncHttpWrapper wrapper = new AsyncHttpWrapper() { final AtomicBoolean called = new AtomicBoolean(false); @@ -436,6 +444,16 @@ public void fail(ErrorCode err) { public void success(HttpEntity responseEntity) { completion.success(responseEntity); } + + @Override + public String getResourceUuid() { + return resourceUuid; + } + + @Override + public long getSentAtMillis() { + return sentAtMillis; + } }; try { @@ -471,6 +489,29 @@ public void asyncJsonPost(String url, String body, AsyncRESTCallback callback) { asyncJsonPost(url, body, callback, TimeUnit.MILLISECONDS, timeout); } + @Override + public int failPendingCallsForResourceBefore(String resourceUuid, long cutoffMillis, ErrorCode err) { + if (resourceUuid == null) { + return 0; + } + int drained = 0; + for (AsyncHttpWrapper w : wrappers.values()) { + if (!resourceUuid.equals(w.getResourceUuid())) { + continue; + } + if (w.getSentAtMillis() >= cutoffMillis) { + continue; + } + try { + w.fail(err); + drained++; + } catch (Throwable t) { + logger.warn(String.format("fail injection threw for resource[uuid:%s]: %s", resourceUuid, t.getMessage()), t); + } + } + return drained; + } + @Override public HttpEntity httpServletRequestToHttpEntity(HttpServletRequest req) { try { diff --git a/header/src/main/java/org/zstack/header/rest/RESTFacade.java b/header/src/main/java/org/zstack/header/rest/RESTFacade.java index e9d3120f9d6..a70d5d7e337 100755 --- a/header/src/main/java/org/zstack/header/rest/RESTFacade.java +++ b/header/src/main/java/org/zstack/header/rest/RESTFacade.java @@ -11,6 +11,7 @@ import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.web.client.RestTemplate; import org.zstack.header.core.Completion; +import org.zstack.header.errorcode.ErrorCode; import javax.net.ssl.SSLContext; import javax.servlet.http.HttpServletRequest; @@ -38,6 +39,8 @@ public interface RESTFacade { void asyncJson(final String url, final String body, Map headers, HttpMethod method, final AsyncRESTCallback callback, final TimeUnit unit, final long timeout); + int failPendingCallsForResourceBefore(String resourceUuid, long cutoffMillis, ErrorCode err); + T syncJsonPost(String url, Object body, Class returnClass); T syncJsonPost(String url, Object body, Class returnClass, TimeUnit unit, long timeout); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 2eb30e5040a..8c73ab18386 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -338,6 +338,8 @@ public void setVersion(String version) { public static class ConnectResponse extends AgentResponse { private String libvirtVersion; private String qemuVersion; + private boolean firstConnect; + private long agentStartTimeMillis; public String getLibvirtVersion() { return libvirtVersion; @@ -354,6 +356,22 @@ public String getQemuVersion() { public void setQemuVersion(String qemuVersion) { this.qemuVersion = qemuVersion; } + + public boolean isFirstConnect() { + return firstConnect; + } + + public void setFirstConnect(boolean firstConnect) { + this.firstConnect = firstConnect; + } + + public long getAgentStartTimeMillis() { + return agentStartTimeMillis; + } + + public void setAgentStartTimeMillis(long agentStartTimeMillis) { + this.agentStartTimeMillis = agentStartTimeMillis; + } } public static class PingCmd extends AgentCommand { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 857f21e163f..eaa694d0242 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5166,6 +5166,16 @@ private ErrorCode connectToAgent() { errCode = operr("unable to connect to kvm host[uuid:%s, ip:%s, url:%s], because %s", self.getUuid(), self.getManagementIp(), connectPath, rsp.getError()); } else { + if (rsp.isFirstConnect()) { + long cutoffMillis = rsp.getAgentStartTimeMillis() - TimeUnit.SECONDS.toMillis(60); + int drained = restf.failPendingCallsForResourceBefore(self.getUuid(), cutoffMillis, + operr("kvmagent on host[uuid:%s] restarted at %s, pending HTTP call sent before %s aborted", + self.getUuid(), rsp.getAgentStartTimeMillis(), cutoffMillis)); + if (drained > 0) { + logger.info(String.format("kvmagent on host[uuid:%s] restarted at %d, drained %d pending HTTP call(s)", + self.getUuid(), rsp.getAgentStartTimeMillis(), drained)); + } + } VersionComparator libvirtVersion = new VersionComparator(rsp.getLibvirtVersion()); VersionComparator qemuVersion = new VersionComparator(rsp.getQemuVersion()); boolean liveSnapshot = libvirtVersion.compare(KVMConstant.MIN_LIBVIRT_LIVESNAPSHOT_VERSION) >= 0 diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy new file mode 100644 index 00000000000..c940a7c6d46 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostAgentRestartDrainCase.groovy @@ -0,0 +1,117 @@ +package org.zstack.test.integration.kvm.host + +import org.springframework.http.HttpEntity +import org.zstack.header.Constants +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.errorcode.SysErrors +import org.zstack.header.rest.JsonAsyncRESTCallback +import org.zstack.header.rest.RESTFacade +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.sdk.HostInventory +import org.zstack.test.integration.kvm.Env +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +class HostAgentRestartDrainCase extends SubCase { + EnvSpec env + HostInventory host + RESTFacade restf + + static class StuckCall { + CountDownLatch entered = new CountDownLatch(1) + CountDownLatch release = new CountDownLatch(1) + CountDownLatch done = new CountDownLatch(1) + AtomicReference received = new AtomicReference<>() + } + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = Env.noVmEnv() + } + + @Override + void test() { + env.create { + host = env.inventoryByName("kvm") + restf = bean(RESTFacade.class) + testFirstConnectDrainsStaleCalls() + testNoFirstConnectKeepsStaleCalls() + testDifferentResourceUuidIsolated() + } + } + + @Override + void clean() { + env.delete() + } + + private StuckCall sendStuckCall(String resourceUuid) { + StuckCall h = new StuckCall() + def stuckPath = "/test/stuck/${UUID.randomUUID()}" + env.simulator(stuckPath) { + h.entered.countDown() + h.release.await(120, TimeUnit.SECONDS) + return new KVMAgentCommands.AgentResponse() + } + + Map headers = [(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID): resourceUuid] + restf.asyncJsonPost(restf.makeUrl(stuckPath), "{}", headers, new JsonAsyncRESTCallback(null) { + @Override void fail(ErrorCode err) { h.received.set(err); h.done.countDown() } + @Override void success(String ret) { h.done.countDown() } + @Override Class getReturnClass() { return String.class } + }, TimeUnit.MINUTES, 30) + assert h.entered.await(5, TimeUnit.SECONDS) + return h + } + + private void overrideConnectResponse(boolean firstConnect, long agentStartTimeMillis) { + env.afterSimulator(KVMConstant.KVM_CONNECT_PATH) { KVMAgentCommands.ConnectResponse rsp, HttpEntity e -> + rsp.firstConnect = firstConnect + rsp.agentStartTimeMillis = agentStartTimeMillis + return rsp + } + } + + void testFirstConnectDrainsStaleCalls() { + StuckCall h = sendStuckCall(host.uuid) + overrideConnectResponse(true, System.currentTimeMillis() + 70_000L) + reconnectHost { + uuid = host.uuid + } + assert h.done.await(5, TimeUnit.SECONDS) + assert h.received.get() != null + assert h.received.get().code == SysErrors.OPERATION_ERROR.toString() + h.release.countDown() + } + + void testNoFirstConnectKeepsStaleCalls() { + StuckCall h = sendStuckCall(host.uuid) + overrideConnectResponse(false, System.currentTimeMillis() + 70_000L) + reconnectHost { + uuid = host.uuid + } + assert !h.done.await(500, TimeUnit.MILLISECONDS) + h.release.countDown() + } + + void testDifferentResourceUuidIsolated() { + StuckCall h = sendStuckCall("unrelated-resource-uuid") + overrideConnectResponse(true, System.currentTimeMillis() + 70_000L) + reconnectHost { + uuid = host.uuid + } + assert !h.done.await(500, TimeUnit.MILLISECONDS) + h.release.countDown() + } +} From af6b04b42f24b378a32ba397d64e49181a0dec42 Mon Sep 17 00:00:00 2001 From: J M Date: Mon, 6 Jul 2026 17:51:59 +0800 Subject: [PATCH 36/53] [ceph]: release snapshot capacity without over-provisioning ratio deleteSnapshotOnPrimaryStorage released CephOsdGroupVO capacity via releaseAvailableCapWithRatio (size / ratio), but recalculate counts snapshot size at actual size. The mismatch leaks size*(1-1/ratio) on each snapshot deletion when over-provisioning is enabled. Align with recalculate by using releaseAvailableCapacity (actual size). This matches the final conclusion of ZSTAC-79709: the original fix (MR !9162 / a84a36e251) was itself a regression and was reverted on mainline (MR !9603), restoring the ZSTAC-67556 behavior. 5.4.10 never carried the regression but still had the pre-67556 buggy call, so this applies the correct one-line fix directly. Resolves: ZSTAC-86485 Change-Id: Icfc1e2b17da17d6de6f8d059468c72021b73eb1e --- .../org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index 6579e941392..a6c091da5c3 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -5438,7 +5438,7 @@ private void deleteSnapshotOnPrimaryStorage(final DeleteSnapshotOnPrimaryStorage httpCall(DELETE_SNAPSHOT_PATH, cmd, DeleteSnapshotRsp.class, new ReturnValueCompletion(msg) { @Override public void success(DeleteSnapshotRsp returnValue) { - osdHelper.releaseAvailableCapWithRatio(msg.getSnapshot().getPrimaryStorageInstallPath(), msg.getSnapshot().getSize()); + osdHelper.releaseAvailableCapacity(msg.getSnapshot().getPrimaryStorageInstallPath(), msg.getSnapshot().getSize()); bus.reply(msg, reply); completion.done(); } From e09ce8390c35acaa4b5ed36b50403509a6831f8f Mon Sep 17 00:00:00 2001 From: "lin.ma" Date: Wed, 4 Feb 2026 18:06:11 +0800 Subject: [PATCH 37/53] [zwatch]: VPC Router CPU alarm use external monitoring Backport the VPC router CPU alarm metric migration to 5.4.10. Move the source 5.5.12 schema change into the 5.4.10 schema file. JIRA: ZSTAC-86427 Related: ZSTAC-81171 Change-Id: I7778676171646874706164777869707279776172 (cherry picked from commit 0bfbf4544c99462ef85235d83c834b4b9b965780) --- conf/db/upgrade/V5.4.10__schema.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql index 71efe026193..1d1915d1987 100644 --- a/conf/db/upgrade/V5.4.10__schema.sql +++ b/conf/db/upgrade/V5.4.10__schema.sql @@ -5,3 +5,16 @@ CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalize UPDATE VolumeSnapshotVO AS sp, PrimaryStorageVO AS ps SET sp.primaryStorageInstallPath = REPLACE(sp.primaryStorageInstallPath, '/dev/', 'sharedblock://') WHERE sp.primaryStorageUuid = ps.uuid AND ps.type = 'SharedBlock' AND sp.volumeType = 'Memory' AND sp.primaryStorageInstallPath LIKE '/dev/%'; + +UPDATE `zstack`.`ActiveAlarmTemplateVO` +SET `metricName` = 'CPUUsedUtilization' +WHERE `uuid` = 'c9e6cdca107140bea62b4ca919ff9e88' + AND `metricName` = 'VRouterCPUAverageUsedUtilization'; + +UPDATE `zstack`.`AlarmVO` +SET `metricName` = 'CPUUsedUtilization' +WHERE `uuid` IN ( + SELECT `alarmUuid` FROM `zstack`.`ActiveAlarmVO` + WHERE `templateUuid` = 'c9e6cdca107140bea62b4ca919ff9e88' +) + AND `metricName` = 'VRouterCPUAverageUsedUtilization'; From 93bc8df79e4c5deb12ef4dbb9c90bbd032af8b91 Mon Sep 17 00:00:00 2001 From: "pengchao.liu" Date: Thu, 11 Dec 2025 11:44:14 +0800 Subject: [PATCH 38/53] [sdnController]: attach dpdk vnic when vm stopped Resolves: ZSTAC-77713 Change-Id: I7867616b6c6b66626f6b716f79637a616864726f (cherry picked from commit 5cad79c36fc8892fbf138b70cb1212e0b93f8145) --- .../zstack/sdnController/SdnControllerApiInterceptor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java index 741a99cdb70..01b7171c202 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerApiInterceptor.java @@ -17,6 +17,7 @@ import org.zstack.header.network.sdncontroller.*; import org.zstack.header.vm.APIAttachL3NetworkToVmMsg; import org.zstack.header.vm.APIChangeVmNicNetworkMsg; +import org.zstack.header.vm.VmInstanceState; import org.zstack.header.vm.VmInstanceVO; import org.zstack.header.vm.VmNicVO; import org.zstack.network.l2.vxlan.vxlanNetwork.VxlanNetworkVO; @@ -125,7 +126,11 @@ private void validate(APIAttachL3NetworkToVmMsg msg) { VmInstanceVO vmVo = dbf.findByUuid(msg.getVmInstanceUuid(), VmInstanceVO.class); boolean found = false; for (SdnControllerHostRefVO ref : controllerVO.getHostRefVOS()) { - if (ref.getHostUuid().equals(vmVo.getHostUuid())) { + String currentHostUuid = vmVo.getHostUuid(); + if (VmInstanceState.Stopped.equals(vmVo.getState())) { + currentHostUuid = vmVo.getLastHostUuid(); + } + if (ref.getHostUuid().equals(currentHostUuid)) { found = true; break; } From 825b45dd61965e66131a05f89b3c077d6a562b38 Mon Sep 17 00:00:00 2001 From: J M Date: Thu, 11 Dec 2025 18:45:44 +0800 Subject: [PATCH 39/53] [xinfini]: fix ping addon info sync between mn do use refresh db. use event notify after addon info changed by ping. Resolves: ZSTAC-80578 Change-Id: I7463686d6f736879726976706c736e6a63626d6b (cherry picked from commit 93940b9d499f9669a3e877997542fed12db7b08b) --- .../storage/addon/primary/AddonInfo.java | 15 ++++++ .../ExternalPrimaryStorageSvcBuilder.java | 2 +- .../storage/addon/primary/PingResult.java | 41 ++++++++++++++++ .../primary/PrimaryStorageControllerSvc.java | 31 ++++++++++-- .../main/java/org/zstack/cbd/AddonInfo.java | 13 ++++- .../java/org/zstack/expon/ExponAddonInfo.java | 3 +- .../zstack/expon/ExponStorageController.java | 18 +++++-- .../org/zstack/expon/ExponStorageFactory.java | 2 +- .../org/zstack/xinfini/XInfiniAddonInfo.java | 13 ++++- .../org/zstack/xinfini/XInfiniConfig.java | 9 ---- .../xinfini/XInfiniStorageController.java | 48 +++++++++++------- .../zstack/xinfini/XInfiniStorageFactory.java | 2 +- .../org/zstack/xinfini/sdk/XInfiniClient.java | 10 +--- .../xinfini/sdk/XInfiniConnectConfig.java | 3 +- .../storage/zbs/ZbsStorageController.java | 30 ++++++++---- .../zstack/storage/zbs/ZbsStorageFactory.java | 13 +---- .../addon/primary/ExternalPrimaryStorage.java | 47 ++++++++++++++++-- .../ExternalPrimaryStorageCanonicalEvent.java | 26 ++++++++++ .../ExternalPrimaryStorageFactory.java | 49 ++++++++++++++++--- .../ExternalPrimaryStorageManagerImpl.java | 12 ++--- .../addon/zbs/ZbsPrimaryStorageCase.groovy | 23 ++++++++- 21 files changed, 319 insertions(+), 91 deletions(-) create mode 100644 header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java create mode 100644 header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java create mode 100644 storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java b/header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java new file mode 100644 index 00000000000..f14f5828ce0 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/AddonInfo.java @@ -0,0 +1,15 @@ +package org.zstack.header.storage.addon.primary; + +import org.zstack.utils.gson.JSONObjectUtil; + +import java.io.Serializable; + +public interface AddonInfo extends Serializable { + default boolean changed(String infoJson) { + return !this.serialize().equals(infoJson); + } + + default String serialize() { + return JSONObjectUtil.toJsonString(this); + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java index 041438d7420..b2eb78ab2fc 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/ExternalPrimaryStorageSvcBuilder.java @@ -9,7 +9,7 @@ public interface ExternalPrimaryStorageSvcBuilder { PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo); - void discover(String url, String config, ReturnValueCompletion completion); + void discover(String url, String config, ReturnValueCompletion completion); String getIdentity(); } diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java new file mode 100644 index 00000000000..aefc52de2ac --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PingResult.java @@ -0,0 +1,41 @@ +package org.zstack.header.storage.addon.primary; + +public class PingResult { + private AddonInfo addonInfo; + private boolean success = true; + private String error; + + public PingResult(AddonInfo addonInfo, String error) { + this.addonInfo = addonInfo; + this.success = false; + this.error = error; + } + + public PingResult(AddonInfo addonInfo) { + this.addonInfo = addonInfo; + } + + public AddonInfo getAddonInfo() { + return addonInfo; + } + + public void setAddonInfo(AddonInfo addonInfo) { + this.addonInfo = addonInfo; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index 5d6896960ce..be6174d197c 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -9,13 +9,38 @@ import org.zstack.header.volume.VolumeStats; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.List; public interface PrimaryStorageControllerSvc { String getIdentity(); - void connect(String config, String url, ReturnValueCompletion comp); - void ping(Completion completion); + + /** + * connect to storage system, and the AddonInfo may be changed after connection, + * report the latest AddonInfo via the completion, ZStack will persist it on all management nodes. + * @param config + * @param url + * @param comp + */ + void connect(String config, String url, ReturnValueCompletion comp); + + /** + * ping the storage system, and report the latest AddonInfo and result via the completion, + * ZStack will persist AddonInfo on all management nodes. + * @param completion + */ + void ping(ReturnValueCompletion completion); + + /** + * sync the addon info from other management node to this svc + * @param addonInfo + */ + void syncAddonInfo(String addonInfo); + + /** + * sync the config from other management node to this svc + * @param config + */ + void syncConfig(String config); void reportCapacity(ReturnValueCompletion comp); void reportHealthy(ReturnValueCompletion comp); diff --git a/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java b/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java index c08f5c37c66..08f5ff08d16 100644 --- a/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java +++ b/plugin/cbd/src/main/java/org/zstack/cbd/AddonInfo.java @@ -1,5 +1,7 @@ package org.zstack.cbd; +import org.zstack.utils.gson.JSONObjectUtil; + import java.util.ArrayList; import java.util.List; @@ -7,7 +9,7 @@ * @author Xingwei Yu * @date 2024/4/1 18:12 */ -public class AddonInfo { +public class AddonInfo implements org.zstack.header.storage.addon.primary.AddonInfo { private ClusterInfo clusterInfo; private List mdsInfos = new ArrayList<>(); private List logicalPoolInfos = new ArrayList<>(); @@ -35,4 +37,13 @@ public List getLogicalPoolInfos() { public void setLogicalPoolInfos(List logicalPoolInfos) { this.logicalPoolInfos = logicalPoolInfos; } + + public void addLogicalPoolInfo(LogicalPoolInfo logicalPoolInfo) { + this.logicalPoolInfos.add(logicalPoolInfo); + } + + @Override + public boolean changed(String infoJson) { + return !JSONObjectUtil.toJsonString(this).equals(infoJson); + } } diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java b/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java index 07259255377..77c5be249af 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponAddonInfo.java @@ -2,12 +2,13 @@ import org.zstack.expon.sdk.cluster.TianshuClusterModule; import org.zstack.expon.sdk.pool.FailureDomainModule; +import org.zstack.header.storage.addon.primary.AddonInfo; import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; -public class ExponAddonInfo { +public class ExponAddonInfo implements AddonInfo { private List clusters = new ArrayList<>(); private List pools = new ArrayList<>(); diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index 8cb331a91de..cb6d5b31b39 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -839,7 +839,7 @@ public String getIdentity() { } @Override - public void connect(String config, String url, ReturnValueCompletion comp) { + public void connect(String config, String url, ReturnValueCompletion comp) { apiHelper.login(); ExponAddonInfo info = new ExponAddonInfo(); @@ -870,12 +870,22 @@ public void connect(String config, String url, ReturnValueCompletion completion) { + completion.success(new PingResult(addonInfo)); + } + + @Override + public void syncAddonInfo(String addonInfo) { + this.addonInfo = StringUtils.isEmpty(addonInfo) ? new ExponAddonInfo() : JSONObjectUtil.toObject(addonInfo, ExponAddonInfo.class); + } + + @Override + public void syncConfig(String config) { + this.config = StringUtils.isEmpty(config) ? new ExponConfig() : JSONObjectUtil.toObject(config, ExponConfig.class); } private void reloadDbInfo() { diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java index 6d24fbe824a..4086058d198 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageFactory.java @@ -35,7 +35,7 @@ public PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo) { } @Override - public void discover(String url, String config, ReturnValueCompletion completion) { + public void discover(String url, String config, ReturnValueCompletion completion) { ExponStorageController controller = new ExponStorageController(url); MultiNodeSingleFlightImpl.register(controller.apiHelper); controller.connect(config, url, completion); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java index e522a99fd38..4d35f379e74 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniAddonInfo.java @@ -1,5 +1,6 @@ package org.zstack.xinfini; +import org.zstack.header.storage.addon.primary.AddonInfo; import org.zstack.xinfini.sdk.node.NodeModule; import org.zstack.xinfini.sdk.pool.BsPolicyModule; import org.zstack.xinfini.sdk.pool.PoolCapacity; @@ -9,7 +10,7 @@ import java.util.ArrayList; import java.util.List; -public class XInfiniAddonInfo { +public class XInfiniAddonInfo implements AddonInfo { private List nodes; private List pools = new ArrayList<>(); @@ -72,6 +73,7 @@ public static class Node { private int id; private String ip; private String state; + private NodeStatus status; public String getIp() { return ip; @@ -97,6 +99,14 @@ public void setState(String state) { this.state = state; } + public NodeStatus getStatus() { + return status; + } + + public void setStatus(NodeStatus status) { + this.status = status; + } + public static Node valueOf(NodeModule nodeModule) { Node node = new Node(); node.setIp(nodeModule.getSpec().getAdminIp()); @@ -104,6 +114,7 @@ public static Node valueOf(NodeModule nodeModule) { node.setState(nodeModule.getMetadata().getState().getState()); return node; } + } public static class Pool { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java index 88b498ce9ef..bc1fc53c597 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniConfig.java @@ -52,15 +52,6 @@ public void setId(int id) { public static class Node { private String ip; private int port; - private NodeStatus status = NodeStatus.Connected; - - public NodeStatus getStatus() { - return status; - } - - public void setStatus(NodeStatus status) { - this.status = status; - } public String getIp() { return ip; diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java index 53c8cca1751..8898bc800be 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java @@ -1,7 +1,6 @@ package org.zstack.xinfini; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.util.packed.DirectMonotonicReader; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; @@ -9,7 +8,6 @@ import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.core.db.SQL; -import org.zstack.core.thread.ThreadFacade; import org.zstack.header.core.Completion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.errorcode.OperationFailureException; @@ -26,8 +24,6 @@ import org.zstack.iscsi.kvm.IscsiHeartbeatVolumeTO; import org.zstack.iscsi.kvm.IscsiVolumeTO; import org.zstack.kvm.KVMConstant; -import org.zstack.resourceconfig.ResourceConfigFacade; -import org.zstack.storage.addon.primary.ExternalPrimaryStorageFactory; import org.zstack.storage.volume.VolumeConfigsGetter; import org.zstack.utils.CollectionUtils; import org.zstack.utils.DebugUtils; @@ -69,15 +65,13 @@ public class XInfiniStorageController implements PrimaryStorageControllerSvc, Pr private static CLogger logger = Utils.getLogger(XInfiniStorageController.class); @Autowired + @Deprecated private DatabaseFacade dbf; - @Autowired - private ThreadFacade thdf; + private ExternalPrimaryStorageVO self; private XInfiniConfig config; private XInfiniAddonInfo addonInfo; private final XInfiniApiHelper apiHelper; - @Autowired - private ResourceConfigFacade rcf; private String vhostSocketDir; @@ -122,11 +116,20 @@ public XInfiniStorageController(ExternalPrimaryStorageVO self) { public XInfiniStorageController(String configStr) { config = JSONObjectUtil.toObject(configStr, XInfiniConfig.class); + XInfiniConnectConfig clientConfig = new XInfiniConnectConfig(); clientConfig.readTimeout = TimeUnit.MINUTES.toMillis(10); clientConfig.writeTimeout = TimeUnit.MINUTES.toMillis(10); clientConfig.token = config.getToken(); - clientConfig.xInfiniConfig = config; + clientConfig.connectedNodeProvider = () -> { + Set connectedIps = this.addonInfo.getNodes().stream() + .filter(it -> it.getStatus() == NodeStatus.Connected) + .map(XInfiniAddonInfo.Node::getIp) + .collect(Collectors.toSet()); + return this.config.getNodes().stream() + .filter(it -> connectedIps.contains(it.getIp())) + .findFirst().orElse(null); + }; XInfiniClient client = new XInfiniClient(); client.configure(clientConfig); @@ -505,20 +508,17 @@ public String getIdentity() { } @Override - public void ping(Completion completion) { + public void ping(ReturnValueCompletion completion) { reloadDbInfo(); Map nodesStatus = apiHelper.checkNodesConnection(config.getNodes()); logger.info(String.format("get nodes status: %s", nodesStatus)); - config.getNodes().forEach(it -> it.setStatus(nodesStatus.getOrDefault(it.getIp(), NodeStatus.Disconnected))); - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.config, JSONObjectUtil.toJsonString(config)) - .update(); - completion.success(); + addonInfo.getNodes().forEach(it -> it.setStatus(nodesStatus.getOrDefault(it.getIp(), NodeStatus.Disconnected))); + completion.success(new PingResult(addonInfo)); } @Override - public void connect(String config, String url, ReturnValueCompletion comp) { + public void connect(String config, String url, ReturnValueCompletion comp) { DebugUtils.Assert(StringUtils.isNotEmpty(config), "config cannot be none"); XInfiniAddonInfo info = new XInfiniAddonInfo(); XInfiniConfig xConfig = JSONObjectUtil.toObject(config, XInfiniConfig.class); @@ -534,6 +534,8 @@ public void connect(String config, String url, ReturnValueCompletion new OperationFailureException(operr("fail to get node %s details, check ip address and role config", it.getIp())))); info.setNodes(nodes.stream().map(XInfiniAddonInfo.Node::valueOf).collect(Collectors.toList())); + Map nodeIpStatus = apiHelper.checkNodesConnection(xConfig.getNodes()); + info.getNodes().forEach(it -> it.setStatus(nodeIpStatus.get(it.getIp()))); List pools = apiHelper.queryPools(); if (CollectionUtils.isEmpty(pools)) { @@ -549,7 +551,7 @@ public void connect(String config, String url, ReturnValueCompletion comp) { - reloadDbInfo(); List pools = refreshPoolCapacity() .stream() .filter(it -> config.getPoolIds().contains(it.getId())) diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java index 4faa1988e18..a21b84bf43d 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageFactory.java @@ -36,7 +36,7 @@ public PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo) { } @Override - public void discover(String url, String config, ReturnValueCompletion completion) { + public void discover(String url, String config, ReturnValueCompletion completion) { // xinfini must set config if (StringUtils.isEmpty(config)) { completion.fail(operr("empty config, cannot discover xinfini")); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java index 8b486289dc7..3f15165b18f 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniClient.java @@ -6,15 +6,12 @@ import org.springframework.http.HttpMethod; import org.zstack.core.Platform; import org.zstack.externalStorage.sdk.ExternalStorageApiClient; -import org.zstack.externalStorage.sdk.ExternalStorageParam; import org.zstack.header.rest.DefaultSSLVerifier; import org.zstack.header.xinfini.XInfiniConstants; import org.zstack.utils.Utils; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; -import org.zstack.xinfini.NodeStatus; import org.zstack.xinfini.XInfiniConfig; -import org.zstack.xinfini.sdk.volume.FlattenVolumeRequest; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.X509TrustManager; @@ -139,12 +136,7 @@ XinfiniApiResult doCallWithNode(XInfiniConfig.Node withNode) { if (withNode != null) { node = withNode; } else { - node = config.xInfiniConfig - .getNodes() - .stream() - .filter(it -> it.getStatus() == NodeStatus.Connected) - .findAny() - .orElseThrow(() -> new XInfiniApiException("No connected node found")); + node = config.connectedNodeProvider.get(); } try { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java index 43668787513..dc42fe54473 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/sdk/XInfiniConnectConfig.java @@ -3,6 +3,7 @@ import org.zstack.xinfini.XInfiniConfig; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; public class XInfiniConnectConfig { public String hostname = "localhost"; @@ -12,7 +13,7 @@ public class XInfiniConnectConfig { public Long readTimeout; public Long writeTimeout; public String token; - public XInfiniConfig xInfiniConfig; + public Supplier connectedNodeProvider; public void setPort(int port) { this.port = port; diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index faddfdbd5da..512712a3cf3 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.cbd.*; +import org.zstack.cbd.AddonInfo; import org.zstack.cbd.kvm.CbdHeartbeatVolumeTO; import org.zstack.cbd.kvm.CbdVolumeTo; import org.zstack.compute.host.HostGlobalConfig; @@ -71,6 +72,7 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar private static final CLogger logger = Utils.getLogger(ZbsStorageController.class); @Autowired + @Deprecated private DatabaseFacade dbf; @Autowired protected RESTFacade restf; @@ -332,7 +334,7 @@ public String getIdentity() { } @Override - public void connect(String cfg, String url, ReturnValueCompletion completion) { + public void connect(String cfg, String url, ReturnValueCompletion completion) { AddonInfo newAddonInfo = new AddonInfo(); Config current = JSONObjectUtil.toObject(cfg, Config.class); List mdsInfos = parseMdsInfos(current.getMdsUrls()); @@ -487,7 +489,7 @@ public void done(ErrorCodeList errorCodeList) { public void handle(Map data) { configUrl(self.getUuid()); addonInfo = newAddonInfo; - completion.success(JSONObjectUtil.rehashObject(newAddonInfo, LinkedHashMap.class)); + completion.success(newAddonInfo); } }); @@ -502,7 +504,7 @@ public void handle(ErrorCode errCode, Map data) { } @Override - public void ping(Completion completion) { + public void ping(ReturnValueCompletion completion) { reloadDbInfo(); if (addonInfo == null || addonInfo.getClusterInfo() == null) { @@ -526,10 +528,6 @@ public void fail(ErrorCode errorCode) { })).run(new WhileDoneCompletion(completion) { @Override public void done(ErrorCodeList errorCodeList) { - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) - .update(); - boolean isConnected = addonInfo.getMdsInfos().stream().anyMatch(mdsInfo -> MdsStatus.Connected.equals(mdsInfo.getStatus())); if (!isConnected) { String notConnectedIps = addonInfo.getMdsInfos().stream() @@ -537,17 +535,16 @@ public void done(ErrorCodeList errorCodeList) { .map(MdsInfo::getAddr) .collect(Collectors.joining(", ")); - completion.fail(operr("no MDS is Connected, the following MDS[%s] are not Connected.", notConnectedIps)); + completion.success(new PingResult(addonInfo, String.format("all MDS are not connected, disconnected MDS addresses: %s", notConnectedIps))); return; } - completion.success(); + completion.success(new PingResult(addonInfo)); } }); } @Override public void reportCapacity(ReturnValueCompletion comp) { - reloadDbInfo(); GetCapacityCmd cmd = new GetCapacityCmd(); cmd.setLogicalPool(config.getLogicalPoolName()); @@ -555,6 +552,7 @@ public void reportCapacity(ReturnValueCompletion comp) { httpCall(GET_CAPACITY_PATH, cmd, GetCapacityRsp.class, new ReturnValueCompletion(comp) { @Override public void success(GetCapacityRsp returnValue) { + reloadDbInfo(); addonInfo.setLogicalPoolInfos(returnValue.getLogicalPoolInfos()); SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) @@ -1120,6 +1118,18 @@ public void fail(ErrorCode errorCode) { }); } + + @Override + public void syncAddonInfo(String addonInfo) { + this.addonInfo = StringUtils.isEmpty(addonInfo) ? new AddonInfo() : JSONObjectUtil.toObject(addonInfo, AddonInfo.class); + } + + @Override + public void syncConfig(String config) { + this.config = StringUtils.isEmpty(config) ? new Config() : JSONObjectUtil.toObject(config, Config.class); + } + + @Deprecated private void reloadDbInfo() { self = dbf.reload(self); addonInfo = StringUtils.isEmpty(self.getAddonInfo()) ? new AddonInfo() : JSONObjectUtil.toObject(self.getAddonInfo(), AddonInfo.class); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java index df9fb479a2c..9fc5a72be5a 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageFactory.java @@ -7,26 +7,15 @@ import org.zstack.core.db.Q; import org.zstack.core.trash.StorageTrash; import org.zstack.externalStorage.primary.ExternalStorageFencerType; -import org.zstack.header.core.Completion; -import org.zstack.header.core.NoErrorCompletion; import org.zstack.header.core.ReturnValueCompletion; import org.zstack.header.storage.addon.primary.*; -import org.zstack.header.storage.primary.DeleteVolumeBitsOnPrimaryStorageMsg; -import org.zstack.header.storage.primary.PrimaryStorageConstant; -import org.zstack.header.storage.snapshot.VolumeSnapshotAfterDeleteExtensionPoint; -import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; import org.zstack.header.volume.VolumeProtocol; import org.zstack.header.volume.VolumeVO; import org.zstack.header.volume.VolumeVO_; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.zstack.storage.zbs.ZbsHelper.*; import static org.zstack.core.Platform.operr; @@ -51,7 +40,7 @@ public PrimaryStorageNodeSvc buildNodeSvc(ExternalPrimaryStorageVO vo) { } @Override - public void discover(String url, String config, ReturnValueCompletion completion) { + public void discover(String url, String config, ReturnValueCompletion completion) { completion.fail(operr("zbs not support discover yet")); } diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 79332acdc88..48c367acbe0 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.EventFacade; import org.zstack.core.cloudbus.ResourceDestinationMaker; import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.Q; @@ -87,6 +88,9 @@ public class ExternalPrimaryStorage extends PrimaryStorageBase { @Autowired private ResourceDestinationMaker destMaker; + @Autowired + private EventFacade evtf; + public ExternalPrimaryStorage(PrimaryStorageVO self, PrimaryStorageControllerSvc controller, PrimaryStorageNodeSvc node) { super(self); this.controller = controller; @@ -1965,15 +1969,28 @@ protected void handle(CheckVolumeSnapshotOperationOnPrimaryStorageMsg msg) { @Override protected void connectHook(ConnectParam param, Completion completion) { controller.validateConfig(externalVO.getConfig()); - controller.connect(externalVO.getConfig(), self.getUrl(), new ReturnValueCompletion(completion) { + controller.connect(externalVO.getConfig(), self.getUrl(), new ReturnValueCompletion(completion) { @Override - public void success(LinkedHashMap addonInfo) { + public void success(AddonInfo addonInfo) { if (param.isNewAdded()) { controller.onFirstAdditionConfigure(new NopeCompletion()); } SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) .update(); + + if (addonInfo != null && addonInfo.changed(externalVO.getAddonInfo())) { + String newAddonInfo = addonInfo.serialize(); + SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) + .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfo) + .update(); + externalVO.setAddonInfo(newAddonInfo); + // fire event when addonInfo changed + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = new org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); + evtData.setUuid(self.getUuid()); + evtData.setAddonInfo(newAddonInfo); + evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + } controller.setTrashExpireTime(PrimaryStorageGlobalConfig.TRASH_EXPIRATION_TIME.value(Integer.class), new NopeCompletion()); pingHook(completion); } @@ -1997,12 +2014,34 @@ public void setup() { @Override public void run(FlowTrigger trigger, Map data) { - controller.ping(new Completion(trigger) { + controller.ping(new ReturnValueCompletion(trigger) { @Override - public void success() { + public void success(PingResult ret) { + if (ret.getAddonInfo() != null && ret.getAddonInfo().changed(externalVO.getAddonInfo())) { + updateAddonInfo(ret.getAddonInfo()); + } + + if (!ret.isSuccess()) { + trigger.fail(operr("ping external primary storage[%s] failed, %s", self.getUuid(), ret.getError())); + return; + } + trigger.next(); } + private void updateAddonInfo(AddonInfo newAddonInfo) { + String newAddonInfoStr = newAddonInfo.serialize(); + externalVO.setAddonInfo(newAddonInfoStr); + SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) + .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfoStr) + .update(); + + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = new ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); + evtData.setUuid(self.getUuid()); + evtData.setAddonInfo(newAddonInfoStr); + evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + } + @Override public void fail(ErrorCode errorCode) { trigger.fail(errorCode); diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java new file mode 100644 index 00000000000..5fc99f89834 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageCanonicalEvent.java @@ -0,0 +1,26 @@ +package org.zstack.storage.addon.primary; + +public class ExternalPrimaryStorageCanonicalEvent { + public static final String ADDON_INFO_CHANGED_PATH = "/externalPrimaryStorage/addonInfo/changed"; + + public static class AddonInfoChangedData { + private String uuid; + private String addonInfo; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getAddonInfo() { + return addonInfo; + } + + public void setAddonInfo(String addonInfo) { + this.addonInfo = addonInfo; + } + } +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java index a8e00d3a0ad..974704efcd9 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java @@ -4,10 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.Platform; import org.zstack.core.asyncbatch.While; -import org.zstack.core.cloudbus.CloudBus; -import org.zstack.core.cloudbus.CloudBusCallBack; -import org.zstack.core.cloudbus.EventCallback; -import org.zstack.core.cloudbus.EventFacade; +import org.zstack.core.cloudbus.*; import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; @@ -52,6 +49,7 @@ import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; +import javax.persistence.Tuple; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -128,11 +126,46 @@ public void beforeDeliveryMessage(Message msg) { return false; }, new APIAddPrimaryStorageEvent()); - evtf.on(PrimaryStorageCanonicalEvent.PRIMARY_STORAGE_DELETED_PATH, new EventCallback() { + bus.subscribeEvent(e -> { + if (e instanceof APIUpdateExternalPrimaryStorageEvent) { + APIUpdateExternalPrimaryStorageEvent evt = (APIUpdateExternalPrimaryStorageEvent) e; + if (evt.isSuccess() && evt.getInventory() != null) { + logger.debug("receive event, update external ps controller config."); + PrimaryStorageControllerSvc controller = controllers.get(evt.getInventory().getUuid()); + ExternalPrimaryStorageVO vo = dbf.findByUuid(evt.getInventory().getUuid(), ExternalPrimaryStorageVO.class); + if (controller != null) { + controller.syncConfig(vo.getConfig()); + controller.syncAddonInfo(vo.getAddonInfo()); + } else { + saveControllerIfNeed(vo); + } + } + } + + return false; + }, new APIUpdateExternalPrimaryStorageEvent()); + + evtf.on(PrimaryStorageCanonicalEvent.PRIMARY_STORAGE_DELETED_PATH, new EventCallback() { @Override - protected void run(Map tokens, Object data) { - PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData d = (PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData) data; - MultiNodeSingleFlightImpl.unregister(d.getPrimaryStorageUuid()); + protected void run(Map tokens, PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData data) { + MultiNodeSingleFlightImpl.unregister(data.getPrimaryStorageUuid()); + } + }); + evtf.on(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, new EventCallback() { + @Override + protected void run(Map tokens, ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data) { + if (evtf.isFromThisManagementNode(tokens)) { + return; + } + + PrimaryStorageControllerSvc controller = controllers.get(data.getUuid()); + Tuple t = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.config, ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, data.getUuid()) + .findTuple(); + if (controller != null) { + controller.syncConfig(t.get(0, String.class)); + controller.syncAddonInfo(t.get(1, String.class)); + } } }); diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java index 0881cefa572..591f7c8982e 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageManagerImpl.java @@ -83,11 +83,11 @@ private void handle(APIDiscoverExternalPrimaryStorageMsg msg) { if (msg.getIdentity() != null) { ExternalPrimaryStorageSvcBuilder builder = pluginRgty.getExtensionFromMap(msg.getIdentity(), ExternalPrimaryStorageSvcBuilder.class); - builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(msg) { + builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(msg) { @Override - public void success(LinkedHashMap addonInfo) { + public void success(AddonInfo addonInfo) { inventory.setIdentity(msg.getIdentity()); - inventory.setAddonInfo(addonInfo); + inventory.setAddonInfo(JSONObjectUtil.rehashObject(addonInfo, LinkedHashMap.class)); event.setInventory(inventory); bus.publish(event); } @@ -102,11 +102,11 @@ public void fail(ErrorCode errorCode) { } new While<>(pluginRgty.getExtensionList(ExternalPrimaryStorageSvcBuilder.class)).each((builder, comp) -> { - builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(comp) { + builder.discover(msg.getUrl(), msg.getConfig(), new ReturnValueCompletion(comp) { @Override - public void success(LinkedHashMap addonInfo) { + public void success(AddonInfo addonInfo) { inventory.setIdentity(builder.getIdentity()); - inventory.setAddonInfo(addonInfo); + inventory.setAddonInfo(JSONObjectUtil.rehashObject(addonInfo, LinkedHashMap.class)); event.setInventory(inventory); comp.allDone(); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 75db97e4b55..f2394a852ba 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -1,6 +1,8 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity +import org.zstack.core.cloudbus.EventCallback +import org.zstack.core.cloudbus.EventFacade import org.zstack.core.db.Q import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ @@ -12,6 +14,7 @@ import org.zstack.header.storage.primary.PrimaryStorageVO import org.zstack.cbd.MdsUri import org.zstack.kvm.KVMAgentCommands import org.zstack.sdk.* +import org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent import org.zstack.storage.primary.PrimaryStorageGlobalConfig import org.zstack.header.storage.primary.PrimaryStorageHostStatus import org.zstack.storage.volume.VolumeGlobalConfig @@ -37,6 +40,7 @@ class ZbsPrimaryStorageCase extends SubCase { DiskOfferingInventory diskOffering VolumeInventory vol, vol2 KVMHostInventory kvm + EventFacade evtf @Override void clean() { @@ -154,6 +158,7 @@ class ZbsPrimaryStorageCase extends SubCase { ps = env.inventoryByName("zbs-1") as PrimaryStorageInventory diskOffering = env.inventoryByName("diskOffering") as DiskOfferingInventory kvm = env.inventoryByName("kvm-1") as KVMHostInventory + evtf = bean(EventFacade.class) testDefaultConfig() testUpdateExternalPrimaryStorage() @@ -318,6 +323,17 @@ class ZbsPrimaryStorageCase extends SubCase { } void testMdsPing() { + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data = null + long count = 0 + EventCallback callback = new EventCallback() { + @Override + protected void run(Map tokens, ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData d) { + data = d + count++ + } + } + + evtf.on(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, callback) PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(1) Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected @@ -325,6 +341,7 @@ class ZbsPrimaryStorageCase extends SubCase { def addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" + assert data == null env.afterSimulator(ZbsPrimaryStorageMdsBase.PING_PATH) { rsp, HttpEntity e -> def cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) @@ -344,6 +361,8 @@ class ZbsPrimaryStorageCase extends SubCase { assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected + assert count == 1 + assert data.uuid == ps.uuid env.afterSimulator(ZbsPrimaryStorageMdsBase.PING_PATH) { rsp, HttpEntity e -> def cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) @@ -362,13 +381,15 @@ class ZbsPrimaryStorageCase extends SubCase { return pingRsp } - sleep(2000) + sleep(2500) addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Disconnected + assert count == 2 + assert data.uuid == ps.uuid env.cleanAfterSimulatorHandlers() From e1a38d33f6b5d5142ada5bfb7731ad6bd6a98877 Mon Sep 17 00:00:00 2001 From: gitlab Date: Fri, 10 Jul 2026 03:59:01 +0000 Subject: [PATCH 40/53] bump version to 5.4.12 DBImpact Change-Id: I6c756579746e756e6b7a706b7979736773686566 --- VERSION | 2 +- conf/db/upgrade/V5.4.12__schema.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 conf/db/upgrade/V5.4.12__schema.sql diff --git a/VERSION b/VERSION index d406ff05502..529cf0d6103 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=5 MINOR=4 -UPDATE=10 +UPDATE=12 diff --git a/conf/db/upgrade/V5.4.12__schema.sql b/conf/db/upgrade/V5.4.12__schema.sql new file mode 100644 index 00000000000..e69de29bb2d From b866d5c34ebaf09dafb996fde5f51201168ab6b6 Mon Sep 17 00:00:00 2001 From: "haidong.pang" Date: Wed, 24 Dec 2025 09:52:22 +0000 Subject: [PATCH 41/53] [storage]: Set the actual image format for ExternalPrimaryStorage 1. Creating an image from a volume defaults to the source format, which may be unexpected. 2. Use the actual format from backup storage reply. Resolves: ZSTAC-78094 Change-Id: I716d747961666979766372626d7679716262667a --- .../zstack/storage/addon/primary/ExternalPrimaryStorage.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 48c367acbe0..33f4f43a5f4 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -939,6 +939,7 @@ protected void handle(CreateTemplateFromVolumeOnPrimaryStorageMsg msg) { String snapshotPath; String bsInstallPath; + String actualFormat; long templateSize; @@ -1067,6 +1068,7 @@ public void run(MessageReply reply) { DownloadImageFromRemoteTargetReply r = reply.castReply(); bsInstallPath = r.getInstallPath(); templateSize = r.getSize(); + actualFormat = r.getFormat(); trigger.next(); } } @@ -1099,7 +1101,7 @@ public void fail(ErrorCode errorCode) { @Override public void handle(Map data) { reply.setTemplateBackupStorageInstallPath(bsInstallPath); - reply.setFormat(msg.getVolumeInventory().getFormat()); + reply.setFormat(actualFormat); reply.setActualSize(templateSize); bus.reply(msg, reply); } From 34b61c73e5bb6f854ed1eea3e389222a98a30054 Mon Sep 17 00:00:00 2001 From: Haidong Pang Date: Fri, 15 May 2026 20:02:11 +0800 Subject: [PATCH 42/53] [vm]: handle missing VM in StopVmGC Resolves: ZSTAC-84158 Change-Id: I134a9c60e40d5bc9f4ad2b3cc27424b29858f0a9 --- .../src/main/java/org/zstack/compute/vm/StopVmGC.java | 11 +++++++++++ .../org/zstack/header/vm/StopVmOnHypervisorMsg.java | 9 +++++++++ .../main/java/org/zstack/kvm/KVMAgentCommands.java | 10 ++++++++++ plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java | 1 + 4 files changed, 31 insertions(+) diff --git a/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java b/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java index a55de8878cf..1c79a9035f0 100755 --- a/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java +++ b/compute/src/main/java/org/zstack/compute/vm/StopVmGC.java @@ -51,6 +51,7 @@ protected void triggerNow(GCCompletion completion) { public void run(FlowTrigger trigger, Map data) { StopVmOnHypervisorMsg msg = new StopVmOnHypervisorMsg(); msg.setVmInventory(inventory); + msg.setIgnoreNotFoundError(true); bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); bus.send(msg, new CloudBusCallBack(trigger) { @Override @@ -66,6 +67,16 @@ public void run(MessageReply reply) { }).then(new NoRollbackFlow() { @Override public void run(FlowTrigger trigger, Map data) { + String vmHostUuid = Q.New(VmInstanceVO.class).select(VmInstanceVO_.hostUuid) + .eq(VmInstanceVO_.uuid, inventory.getUuid()).findValue(); + if (!hostUuid.equals(vmHostUuid)) { + logger.debug(String.format("skip changing vm[uuid:%s] state to stopped after StopVmGC, " + + "current host[uuid:%s] is not gc host[uuid:%s]", inventory.getUuid(), + vmHostUuid, hostUuid)); + trigger.next(); + return; + } + ChangeVmStateMsg cmsg = new ChangeVmStateMsg(); cmsg.setVmInstanceUuid(inventory.getUuid()); cmsg.setStateEvent(VmInstanceStateEvent.stopped.toString()); diff --git a/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java index bffd82dce1f..690454f798e 100755 --- a/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java +++ b/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java @@ -7,6 +7,7 @@ public class StopVmOnHypervisorMsg extends NeedReplyMessage implements HostMessa private VmInstanceInventory vmInventory; private String type; private boolean debug; + private boolean ignoreNotFoundError; public VmInstanceInventory getVmInventory() { return vmInventory; @@ -36,4 +37,12 @@ public boolean isDebug() { public void setDebug(boolean debug) { this.debug = debug; } + + public boolean isIgnoreNotFoundError() { + return ignoreNotFoundError; + } + + public void setIgnoreNotFoundError(boolean ignoreNotFoundError) { + this.ignoreNotFoundError = ignoreNotFoundError; + } } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 8c73ab18386..25af200ac1b 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -3216,6 +3216,8 @@ public static class StopVmCmd extends AgentCommand { private String type; @GrayVersion(value = "5.0.0") private long timeout; + @GrayVersion(value = "5.5.22") + private boolean ignoreNotFoundError; private List vmNics; public String getUuid() { @@ -3242,6 +3244,14 @@ public void setType(String type) { this.type = type; } + public boolean isIgnoreNotFoundError() { + return ignoreNotFoundError; + } + + public void setIgnoreNotFoundError(boolean ignoreNotFoundError) { + this.ignoreNotFoundError = ignoreNotFoundError; + } + public List getVmNics() { return vmNics; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index b14e863a0e4..786586ab0ee 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -3878,6 +3878,7 @@ protected void stopVm(final StopVmOnHypervisorMsg msg, final NoErrorCompletion c cmd.setUuid(vminv.getUuid()); cmd.setType(msg.getType()); cmd.setTimeout(120); + cmd.setIgnoreNotFoundError(msg.isIgnoreNotFoundError()); cmd.setVmNics(vminv.getVmNics()); try { From f6b93e0c6b6def0d361fe497a5cefcbb61afee07 Mon Sep 17 00:00:00 2001 From: Haidong Pang Date: Mon, 18 May 2026 16:07:37 +0800 Subject: [PATCH 43/53] [vm]: allow nullable stop not-found flag Resolves: ZSTAC-84158 Change-Id: I10d2fba8187a7ebcac48063dae5cf71fa2686d79 --- .../java/org/zstack/header/vm/StopVmOnHypervisorMsg.java | 6 +++--- .../src/main/java/org/zstack/kvm/KVMAgentCommands.java | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java index 690454f798e..966928e74d4 100755 --- a/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java +++ b/header/src/main/java/org/zstack/header/vm/StopVmOnHypervisorMsg.java @@ -7,7 +7,7 @@ public class StopVmOnHypervisorMsg extends NeedReplyMessage implements HostMessa private VmInstanceInventory vmInventory; private String type; private boolean debug; - private boolean ignoreNotFoundError; + private Boolean ignoreNotFoundError; public VmInstanceInventory getVmInventory() { return vmInventory; @@ -38,11 +38,11 @@ public void setDebug(boolean debug) { this.debug = debug; } - public boolean isIgnoreNotFoundError() { + public Boolean isIgnoreNotFoundError() { return ignoreNotFoundError; } - public void setIgnoreNotFoundError(boolean ignoreNotFoundError) { + public void setIgnoreNotFoundError(Boolean ignoreNotFoundError) { this.ignoreNotFoundError = ignoreNotFoundError; } } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 25af200ac1b..fef24886ba6 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -3216,8 +3216,8 @@ public static class StopVmCmd extends AgentCommand { private String type; @GrayVersion(value = "5.0.0") private long timeout; - @GrayVersion(value = "5.5.22") - private boolean ignoreNotFoundError; + @GrayVersion(value = "5.4.12") + private Boolean ignoreNotFoundError; private List vmNics; public String getUuid() { @@ -3244,11 +3244,11 @@ public void setType(String type) { this.type = type; } - public boolean isIgnoreNotFoundError() { + public Boolean isIgnoreNotFoundError() { return ignoreNotFoundError; } - public void setIgnoreNotFoundError(boolean ignoreNotFoundError) { + public void setIgnoreNotFoundError(Boolean ignoreNotFoundError) { this.ignoreNotFoundError = ignoreNotFoundError; } From bacad971c857f7fd3f984bbacd5f37904e4c1910 Mon Sep 17 00:00:00 2001 From: Haidong Pang Date: Thu, 14 May 2026 14:19:59 +0800 Subject: [PATCH 44/53] [zbs]: ensure heartbeat before host check Ensure external primary storage heartbeat volumes in the KVM host connect path before host-storage status checks run. ZBS creates missing heartbeat volumes through the MN/controller side, including all configured pools. Host-side checks remain data-plane reads. Aggregate heartbeat ensure failures so reconnect diagnostics retain all failed primary storage causes. Resolves: ZSTAC-79201 Change-Id: I9b592bf4c48573f6e6951735eef4b39463b57fa6 --- .../kvm/ExternalPrimaryStorageKvmFactory.java | 39 +++++++++++++++++++ .../addon/zbs/ZbsPrimaryStorageCase.groovy | 23 +++++++++++ 2 files changed, 62 insertions(+) diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index 90aca223f1e..a1e5a0aefe9 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -140,6 +140,26 @@ public void done(ErrorCodeList errorCodeList) { } }); } + }).then(new NoRollbackFlow() { + String __name__ = "ensure-heartbeat-volume"; + + @Override + public void run(FlowTrigger trigger, Map data) { + ensureHeartbeatVolume(context.getInventory(), extPss, new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().isEmpty()) { + trigger.next(); + } else { + logger.warn(String.format("failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection, %s", + context.getInventory().getUuid(), context.getInventory().getName(), errorCodeList.getReadableDetails())); + trigger.fail(operr(new ErrorCodeList().causedBy(errorCodeList.getCauses()), + "failed to ensure heartbeat volumes before checking KVM host[uuid:%s, name:%s] storage connection", + context.getInventory().getUuid(), context.getInventory().getName())); + } + } + }); + } }).then(new NoRollbackFlow() { String __name__ = "check-host-status"; @@ -166,6 +186,25 @@ public void handle(Map data) { }).start(); } + private void ensureHeartbeatVolume(HostInventory host, List extPss, WhileDoneCompletion completion) { + new While<>(extPss).each((extPs, compl) -> { + logger.debug(String.format("ensuring heartbeat volume for external primary storage[uuid:%s, name:%s] before checking KVM host[uuid:%s, name:%s] storage connection", + extPs.getUuid(), extPs.getName(), host.getUuid(), host.getName())); + extPsFactory.getNodeSvc(extPs.getUuid()).activateHeartbeatVolume(host, new ReturnValueCompletion(compl) { + @Override + public void success(HeartbeatVolumeTO returnValue) { + compl.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + compl.addError(errorCode); + compl.done(); + } + }); + }).run(completion); + } + private void deployClient(final KVMHostConnectedContext context, List extPss, WhileDoneCompletion completion) { new While<>(extPss).each((extPs, compl) -> { logger.debug(String.format("deploying client for external primary storage[uuid:%s, name:%s] on KVM host[uuid:%s, name:%s]", diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index f2394a852ba..5a85f0fc4d4 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -28,6 +28,8 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil +import java.util.concurrent.atomic.AtomicBoolean + /** * @author Xingwei Yu * @date 2024/4/19 10:09 @@ -179,8 +181,25 @@ class ZbsPrimaryStorageCase extends SubCase { clusterUuid = cluster.uuid } + Set heartbeatPools = Collections.synchronizedSet(new HashSet<>()) + AtomicBoolean checkStartedBeforeHeartbeatReady = new AtomicBoolean(false) + env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity e -> + ZbsStorageController.CreateVolumeCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd) + if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) { + heartbeatPools.add(cmd.logicalPool) + ZbsStorageController.CreateVolumeRsp createVolumeRsp = new ZbsStorageController.CreateVolumeRsp() + createVolumeRsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}" + return createVolumeRsp + } + + return rsp + } + env.afterSimulator(ZbsStorageController.CHECK_HOST_STORAGE_CONNECTION_PATH) { rsp, HttpEntity e -> ZbsStorageController.CheckHostStorageConnectionCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CheckHostStorageConnectionCmd) + if (!heartbeatPools.contains("lpool1")) { + checkStartedBeforeHeartbeatReady.set(true) + } ZbsStorageController.CheckHostStorageConnectionRsp checkHostStorageConnectionRsp = new ZbsStorageController.CheckHostStorageConnectionRsp() checkHostStorageConnectionRsp.error = "fake error" @@ -194,6 +213,10 @@ class ZbsPrimaryStorageCase extends SubCase { uuid = kvm.getUuid() } } + retryInSecs { + assert heartbeatPools.contains("lpool1") + assert !checkStartedBeforeHeartbeatReady.get() + } retryInSecs { assert Q.New(PrimaryStorageHostRefVO.class) From b2ac784d757bf730aca460e6d72874289bb50dca Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 15 Jan 2026 14:42:35 +0800 Subject: [PATCH 45/53] [sdnController]: add AddSdnControllerMsg DBImpact Resolves: ZSTAC-73622 JIRA: ZSTAC-86415 Change-Id: I667461766363677373636a6e626f70766e736f6f (cherry picked from commit 6f33d216223c9c007413c442e88e3c9627b98c60) --- .../sdncontroller/SdnControllerVO.java | 2 + .../zstack/sdnController/SdnController.java | 6 +- .../SdnControllerManagerImpl.java | 61 +++++++++++++------ .../h3cVcfc/H3cVcfcSdnController.java | 8 +-- .../h3cVcfc/H3cVcfcV2SdnController.java | 4 +- .../header/AddSdnControllerMsg.java | 31 ++++++++++ .../header/AddSdnControllerReply.java | 16 +++++ .../controller/SugonSdnController.java | 8 +-- 8 files changed, 105 insertions(+), 31 deletions(-) create mode 100644 plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java create mode 100644 plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java diff --git a/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java b/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java index f9a911436d2..8e1848d9225 100644 --- a/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java +++ b/header/src/main/java/org/zstack/header/network/sdncontroller/SdnControllerVO.java @@ -1,6 +1,7 @@ package org.zstack.header.network.sdncontroller; import org.zstack.header.identity.OwnedByAccount; +import org.zstack.header.log.NoLogging; import org.zstack.header.vo.BaseResource; import org.zstack.header.vo.NoView; import org.zstack.header.vo.ResourceVO; @@ -40,6 +41,7 @@ public class SdnControllerVO extends ResourceVO implements OwnedByAccount, ToInv private String username; @Column + @NoLogging private String password; @OneToMany(fetch=FetchType.EAGER) diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java index 5de07d899f7..57072cbaa57 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnController.java @@ -13,10 +13,10 @@ public interface SdnController { 对sdn控制器的控制: event 有关sdn控制器的后置处理: post-event */ - void preInitSdnController(APIAddSdnControllerMsg msg, Completion completion); - void createSdnControllerDb(APIAddSdnControllerMsg msg, SdnControllerVO vo, Completion completion); + void preInitSdnController(AddSdnControllerMsg msg, Completion completion); + void createSdnControllerDb(AddSdnControllerMsg msg, SdnControllerVO vo, Completion completion); void deleteSdnControllerDb(SdnControllerVO vo); - void initSdnController(APIAddSdnControllerMsg msg, Completion completion); + void initSdnController(AddSdnControllerMsg msg, Completion completion); void postInitSdnController(SdnControllerVO vo, Completion completion); void deleteSdnController(SdnControllerDeletionMsg msg, SdnControllerInventory sdn, Completion completion); diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java index c9e939b63f4..c46d7be3ab3 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerManagerImpl.java @@ -6,6 +6,7 @@ import org.zstack.core.Platform; import org.zstack.core.asyncbatch.While; import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.workflow.FlowChainBuilder; @@ -19,6 +20,7 @@ import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; import org.zstack.header.network.NetworkException; import org.zstack.header.network.l2.*; import org.zstack.header.network.l3.*; @@ -80,6 +82,8 @@ public List getAliasIds() { public void handleMessage(Message msg) { if (msg instanceof APIAddSdnControllerMsg) { handle((APIAddSdnControllerMsg) msg); + } else if (msg instanceof AddSdnControllerMsg) { + handle((AddSdnControllerMsg) msg); } else if (msg instanceof SdnControllerMessage) { handleSdnControllerMessage((SdnControllerMessage) msg); } else { @@ -87,6 +91,22 @@ public void handleMessage(Message msg) { } } + private void handle(AddSdnControllerMsg msg) { + AddSdnControllerReply reply = new AddSdnControllerReply(); + doCreateSdnController(msg, new Completion(msg) { + @Override + public void success() { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + private void handleSdnControllerMessage(SdnControllerMessage msg) { SdnControllerVO vo = dbf.findByUuid(msg.getSdnControllerUuid(), SdnControllerVO.class); if (vo == null) { @@ -99,23 +119,23 @@ private void handleSdnControllerMessage(SdnControllerMessage msg) { sdnController.handleMessage(msg); } - private void doCreateSdnController(SdnControllerVO vo, APIAddSdnControllerMsg msg, Completion completion) { - SdnControllerFactory factory = getSdnControllerFactory(msg.getVendorType()); + private void doCreateSdnController(AddSdnControllerMsg msg, Completion completion) { + SdnControllerVO vo = msg.getSdnControllerVO(); + SdnControllerFactory factory = getSdnControllerFactory(vo.getVendorType()); SdnController controller = factory.getSdnController(vo); Map data = new HashMap(); FlowChain chain = FlowChainBuilder.newShareFlowChain(); chain.setData(data); - chain.setName(String.format("create-sdn-controller-%s", msg.getName())); + chain.setName(String.format("create-sdn-controller-%s", vo.getName())); chain.then(new ShareFlow() { @Override public void setup() { flow(new NoRollbackFlow() { - String __name__ = String.format("pre-process-for-create-sdn-controller-%s", msg.getName()); + String __name__ = String.format("pre-process-for-create-sdn-controller-%s", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { - msg.setResourceUuid(vo.getUuid()); controller.preInitSdnController(msg, new Completion(trigger) { @Override public void success() { @@ -130,7 +150,7 @@ public void fail(ErrorCode errorCode) { } }); flow(new Flow() { - String __name__ = String.format("create-sdn-controller-%s-on-db", msg.getName()); + String __name__ = String.format("create-sdn-controller-%s-on-db", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { @@ -155,7 +175,7 @@ public void rollback(FlowRollback trigger, Map data) { } }); flow(new Flow() { - String __name__ = String.format("init-sdn-controller-%s", msg.getName()); + String __name__ = String.format("init-sdn-controller-%s", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { @@ -179,7 +199,7 @@ public void rollback(FlowRollback trigger, Map data) { } }); flow(new NoRollbackFlow() { - String __name__ = String.format("post-process-for-create-sdn-controller--%s", msg.getName()); + String __name__ = String.format("post-process-for-create-sdn-controller-%s", vo.getName()); @Override public void run(FlowTrigger trigger, Map data) { @@ -238,17 +258,22 @@ private void handle(APIAddSdnControllerMsg msg) { } vo.setStatus(SdnControllerStatus.Connected); - doCreateSdnController(vo, msg, new Completion(msg) { - @Override - public void success() { - tagMgr.createTagsFromAPICreateMessage(msg, vo.getUuid(), SdnControllerVO.class.getSimpleName()); - event.setInventory(SdnControllerInventory.valueOf(dbf.findByUuid(vo.getUuid(), SdnControllerVO.class))); - bus.publish(event); - } - + AddSdnControllerMsg amsg = new AddSdnControllerMsg(); + amsg.setSdnControllerVO(vo); + amsg.setAccountUuid(msg.getSession().getAccountUuid()); + if (msg.getSystemTags() != null) { + amsg.setSystemTags(msg.getSystemTags()); + } + bus.makeTargetServiceIdByResourceUuid(amsg, SdnControllerConstant.SERVICE_ID, vo.getUuid()); + bus.send(amsg, new CloudBusCallBack(amsg) { @Override - public void fail(ErrorCode errorCode) { - event.setError(errorCode); + public void run(MessageReply reply) { + if (reply.isSuccess()) { + tagMgr.createTagsFromAPICreateMessage(msg, vo.getUuid(), SdnControllerVO.class.getSimpleName()); + event.setInventory(SdnControllerInventory.valueOf(dbf.findByUuid(vo.getUuid(), SdnControllerVO.class))); + } else { + event.setError(reply.getError()); + } bus.publish(event); } }); diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java index f9afed355d9..40cdc09d9eb 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcSdnController.java @@ -160,7 +160,7 @@ private void getH3cDefaultTenant(Completion completion) { } } - private void getH3cParameters(APIAddSdnControllerMsg msg, Completion completion) { + private void getH3cParameters(AddSdnControllerMsg msg, Completion completion) { FlowChain chain = FlowChainBuilder.newSimpleFlowChain(); chain.setName(String.format("get-h3c-parameters-%s", self.getIp())); chain.then(new NoRollbackFlow() { @@ -228,12 +228,12 @@ public void handleMessage(SdnControllerMessage msg) { @Override @SdnControllerLog - public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void preInitSdnController(AddSdnControllerMsg msg, Completion completion) { completion.success(); } @Override - public void createSdnControllerDb(APIAddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { + public void createSdnControllerDb(AddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { dbf.persist(vo); completion.success(); } @@ -245,7 +245,7 @@ public void deleteSdnControllerDb(SdnControllerVO vo) { @Override @SdnControllerLog - public void initSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void initSdnController(AddSdnControllerMsg msg, Completion completion) { getH3cControllerToken(new Completion(completion) { @Override public void success() { diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java index 778d641c5e4..4f057e51915 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/h3cVcfc/H3cVcfcV2SdnController.java @@ -141,7 +141,7 @@ public void fail(ErrorCode errorCode) { @Override @SdnControllerLog - public void initSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void initSdnController(AddSdnControllerMsg msg, Completion completion) { getH3cControllerToken(new Completion(completion) { @Override public void success() { @@ -335,7 +335,7 @@ public void getH3cVniRanges(Completion completion) { } } - private void getH3cParameters(APIAddSdnControllerMsg msg, Completion completion) { + private void getH3cParameters(AddSdnControllerMsg msg, Completion completion) { FlowChain chain = FlowChainBuilder.newSimpleFlowChain(); chain.setName(String.format("get-h3c-parameters-%s", self.getIp())); chain.then(new NoRollbackFlow() { diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java new file mode 100644 index 00000000000..044b0a22e0f --- /dev/null +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerMsg.java @@ -0,0 +1,31 @@ +package org.zstack.sdnController.header; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.network.sdncontroller.SdnControllerMessage; +import org.zstack.header.network.sdncontroller.SdnControllerVO; + +public class AddSdnControllerMsg extends NeedReplyMessage implements SdnControllerMessage { + SdnControllerVO sdnControllerVO; + String accountUuid; + + @Override + public String getSdnControllerUuid() { + return sdnControllerVO.getUuid(); + } + + public SdnControllerVO getSdnControllerVO() { + return sdnControllerVO; + } + + public void setSdnControllerVO(SdnControllerVO sdnControllerVO) { + this.sdnControllerVO = sdnControllerVO; + } + + public String getAccountUuid() { + return accountUuid; + } + + public void setAccountUuid(String accountUuid) { + this.accountUuid = accountUuid; + } +} diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java new file mode 100644 index 00000000000..a73448fdfe5 --- /dev/null +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/header/AddSdnControllerReply.java @@ -0,0 +1,16 @@ +package org.zstack.sdnController.header; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.network.sdncontroller.SdnControllerInventory; + +public class AddSdnControllerReply extends MessageReply { + private SdnControllerInventory inventory; + + public SdnControllerInventory getInventory() { + return inventory; + } + + public void setInventory(SdnControllerInventory inventory) { + this.inventory = inventory; + } +} diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index e63d69927c4..dff73723041 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -88,7 +88,7 @@ void handMessage(SdnControllerPingMsg msg) { } @Override - public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void preInitSdnController(AddSdnControllerMsg msg, Completion completion) { try { long count = Q.New(SdnControllerVO.class).eq(SdnControllerVO_.vendorType, SugonSdnControllerConstant.TF_CONTROLLER).count(); if(count > 0) { @@ -101,7 +101,7 @@ public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completi return; } String accountUuid = StringDSL.transToTfUuid(accountVO.getUuid()); - client = new TfHttpClient(msg.getIp()); + client = new TfHttpClient(msg.getSdnControllerVO().getIp()); Domain domain = (Domain) client.getDomain(); if(domain == null){ completion.fail(operr("get default domain on tf controller failed")); @@ -133,7 +133,7 @@ public void preInitSdnController(APIAddSdnControllerMsg msg, Completion completi } @Override - public void createSdnControllerDb(APIAddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { + public void createSdnControllerDb(AddSdnControllerMsg msg, SdnControllerVO vo, Completion completion) { dbf.persist(vo); completion.success(); } @@ -144,7 +144,7 @@ public void deleteSdnControllerDb(SdnControllerVO vo) { } @Override - public void initSdnController(APIAddSdnControllerMsg msg, Completion completion) { + public void initSdnController(AddSdnControllerMsg msg, Completion completion) { completion.success(); } From e6d9f5976052212ab2c6ac054eaf95c202f5030a Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Tue, 19 May 2026 00:52:45 +0800 Subject: [PATCH 46/53] [ha]: stabilize suspect host pre-fence flow Move premium-only HA pre-fence details out of community while keeping the generic async before-start extension point support. Jira: ZSTAC-83890 Test: mvn -pl header,compute,plugin/kvm,simulator/simulatorImpl,testlib -DskipTests compile Change-Id: I168adf82338f9df9e76287619b7f76a8e5be695f --- .../org/zstack/compute/vm/VmInstanceBase.java | 1 - .../compute/vm/VmStartOnHypervisorFlow.java | 87 ++++++++++++++----- .../header/vm/HaStartVmInstanceMsg.java | 9 ++ ...BeforeStartOnHypervisorExtensionPoint.java | 9 +- 4 files changed, 82 insertions(+), 24 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index b9f2b334cc2..6333cd49e50 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -8985,4 +8985,3 @@ public void run(MessageReply reply) { }); } } - diff --git a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java index 20965f61655..bf44adeb836 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java @@ -5,17 +5,26 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.asyncbatch.While; import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.workflow.FlowChainBuilder; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowChain; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; import org.zstack.header.core.workflow.FlowRollback; import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.host.HostConstant; import org.zstack.header.message.MessageReply; import org.zstack.header.vm.*; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.List; import java.util.Map; @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) public class VmStartOnHypervisorFlow implements Flow { @@ -26,34 +35,68 @@ public class VmStartOnHypervisorFlow implements Flow { @Autowired private PluginRegistry pluginRgty; - private final List exts = pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class);; - - private void fireExtensions(VmInstanceSpec spec) { - for (VmBeforeStartOnHypervisorExtensionPoint ext : exts) { - ext.beforeStartVmOnHypervisor(spec); - } - } - @Override public void run(final FlowTrigger chain, final Map data) { final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); + FlowChain fchain = FlowChainBuilder.newSimpleFlowChain(); + fchain.setName(String.format("vm-start-on-hypervisor-vm-%s", spec.getVmInventory().getUuid())); + fchain.then(new NoRollbackFlow() { + @Override + public void run(FlowTrigger trigger, Map d) { + new While<>(pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class)) + .each((ext, comp) -> ext.beforeStartVmOnHypervisor(spec, new Completion(comp) { + @Override + public void success() { + comp.done(); + } - fireExtensions(spec); - - StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); - msg.setVmSpec(spec); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); - bus.send(msg, new CloudBusCallBack(chain) { + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + })).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().size() > 0) { + trigger.fail(errorCodeList.getCauses().get(0)); + } else { + trigger.next(); + } + } + }); + } + }); + fchain.then(new NoRollbackFlow() { @Override - public void run(MessageReply reply) { - if (reply.isSuccess()) { - data.put(VmStartOnHypervisorFlow.class.getName(), true); - chain.next(); - } else { - chain.fail(reply.getError()); - } + public void run(FlowTrigger trigger, Map d) { + StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); + msg.setVmSpec(spec); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); + bus.send(msg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + data.put(VmStartOnHypervisorFlow.class.getName(), true); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); } }); + fchain.done(new FlowDoneHandler(chain) { + @Override + public void handle(Map d) { + chain.next(); + } + }).error(new FlowErrorHandler(chain) { + @Override + public void handle(ErrorCode errCode, Map d) { + chain.fail(errCode); + } + }).start(); } @Override diff --git a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java index a358249cdf9..3ef49037137 100644 --- a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java @@ -12,6 +12,7 @@ public class HaStartVmInstanceMsg extends NeedReplyMessage implements VmInstance private String vmInstanceUuid; private String judgerClassName; private List softAvoidHostUuids; + private String accessiblePeerHostUuid; private String haReason; public String getJudgerClassName() { @@ -30,6 +31,14 @@ public void setSoftAvoidHostUuids(List softAvoidHostUuids) { this.softAvoidHostUuids = softAvoidHostUuids; } + public String getAccessiblePeerHostUuid() { + return accessiblePeerHostUuid; + } + + public void setAccessiblePeerHostUuid(String accessiblePeerHostUuid) { + this.accessiblePeerHostUuid = accessiblePeerHostUuid; + } + @Override public String getVmInstanceUuid() { return vmInstanceUuid; diff --git a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java index c56d99d38dd..1b3a393496f 100755 --- a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java @@ -1,7 +1,14 @@ package org.zstack.header.vm; +import org.zstack.header.core.Completion; + /** */ public interface VmBeforeStartOnHypervisorExtensionPoint { - void beforeStartVmOnHypervisor(VmInstanceSpec spec); + default void beforeStartVmOnHypervisor(VmInstanceSpec spec) {} + + default void beforeStartVmOnHypervisor(VmInstanceSpec spec, Completion completion) { + beforeStartVmOnHypervisor(spec); + completion.success(); + } } From 3845ee6cfb57631d1b4c70f1d24e5e6dc58c037b Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Wed, 20 May 2026 16:57:11 +0800 Subject: [PATCH 47/53] [ha]: create pre-fence tag in core Move HA pre-fence pending tag creation into HaStartVmInstanceMsg handling after the VM state update, so later pre-fence execution observes the same database state. Resolves: ZSTAC-83890 Change-Id: I7f046732e0245319ed4cf188d1342b65dad11767 --- .../org/zstack/compute/vm/VmInstanceBase.java | 24 +++++++++++++++++++ .../org/zstack/compute/vm/VmSystemTags.java | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index 6333cd49e50..9755c654e23 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1015,6 +1015,8 @@ public void done(ErrorCodeList errorCodeList) { } sql.update(); + refreshVO(); + createPreFencePendingTag(msg); startVm(msg, new Completion(msg, chain) { @Override @@ -1045,6 +1047,28 @@ public String getName() { }); } + private void createPreFencePendingTag(HaStartVmInstanceMsg msg) { + String suspectHostUuid = StringUtils.trimToNull(CollectionUtils.isEmpty(msg.getSoftAvoidHostUuids()) ? + self.getLastHostUuid() : msg.getSoftAvoidHostUuids().get(0)); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); + if (suspectHostUuid == null || peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host[%s] or peer host[%s] is absent", + self.getUuid(), suspectHostUuid, peerHostUuid)); + return; + } + + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } + private void changeVmIp(final String l3Uuid, final Map staticIpMap, final Completion completion) { final VmNicVO targetNic = CollectionUtils.find(self.getVmNics(), new Function() { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index 21c9668175a..ebb34e5970f 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -309,5 +309,12 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; + public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; + public static PatternedSystemTag HA_PRE_FENCE_PENDING = + new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", + HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, + HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); + public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); } From b89400911911713ae14ea435291b14a4e9e33706 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Thu, 28 May 2026 15:43:20 +0800 Subject: [PATCH 48/53] [ha]: require peer for pre-fence tag Create HA pre-fence pending tag only when the checker provides an accessible peer host. The peer should be covered by the HA dead-host and storage-inactive paths, so do not persist a fallback marker when it is absent. Resolves: ZSTAC-83890 Change-Id: I36426b3386f1080ddeae8396ea9c4f7a81e25711 --- .../org/zstack/compute/vm/VmInstanceBase.java | 48 +++++++++---------- .../org/zstack/compute/vm/VmSystemTags.java | 4 +- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index 9755c654e23..18b40fda31d 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1005,18 +1005,38 @@ public void done(ErrorCodeList errorCodeList) { logger.debug(String.format("HaStartVmJudger[%s] says the VM[uuid:%s, name:%s] is qualified for HA start, now we are starting it", judger.getClass(), self.getUuid(), self.getName())); + String hostUuid = self.getHostUuid(); + String suspectHostUuid = StringUtils.trimToNull(hostUuid); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); UpdateQuery sql = SQL.New(VmInstanceVO.class) .eq(VmInstanceVO_.uuid, self.getUuid()) .set(VmInstanceVO_.state, VmInstanceState.Stopped) .set(VmInstanceVO_.hostUuid, null); - if (self.getHostUuid() != null) { - sql.set(VmInstanceVO_.lastHostUuid, self.getHostUuid()); + if (hostUuid != null) { + sql.set(VmInstanceVO_.lastHostUuid, hostUuid); } sql.update(); refreshVO(); - createPreFencePendingTag(msg); + if (suspectHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host is absent", + self.getUuid())); + } else if (peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because peer host is absent", + self.getUuid())); + } else { + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } startVm(msg, new Completion(msg, chain) { @Override @@ -1047,28 +1067,6 @@ public String getName() { }); } - private void createPreFencePendingTag(HaStartVmInstanceMsg msg) { - String suspectHostUuid = StringUtils.trimToNull(CollectionUtils.isEmpty(msg.getSoftAvoidHostUuids()) ? - self.getLastHostUuid() : msg.getSoftAvoidHostUuids().get(0)); - String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); - if (suspectHostUuid == null || peerHostUuid == null) { - logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host[%s] or peer host[%s] is absent", - self.getUuid(), suspectHostUuid, peerHostUuid)); - return; - } - - Map tokens = new HashMap<>(); - tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); - tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); - - SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); - creator.inherent = true; - creator.recreate = true; - creator.ignoreIfExisting = true; - creator.setTagByTokens(tokens); - creator.create(); - } - private void changeVmIp(final String l3Uuid, final Map staticIpMap, final Completion completion) { final VmNicVO targetNic = CollectionUtils.find(self.getVmNics(), new Function() { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index ebb34e5970f..6a146649119 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -309,12 +309,12 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); + public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; public static PatternedSystemTag HA_PRE_FENCE_PENDING = new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); - - public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); } From 9059e8269b52f7a42f861cbccdab3cc2e6e692ce Mon Sep 17 00:00:00 2001 From: "lin.ma" Date: Fri, 3 Jul 2026 10:40:47 +0800 Subject: [PATCH 49/53] [compute]: avoid dual MN host tracking Backport the host tracking guard to 5.4.10. 1. Why? ZSTAC-86365 tracks dual management nodes tracking the same host. The target branch does not contain the source fix yet. 2. How? Check isManagedByUs before trackHost to avoid duplicate tracking. 3. Side effects? None. The change only skips host tracking on non-owner MNs. # Summary of changes (by module): - compute: guard host tracking by management node ownership. - test: cover host connected time tracking behavior. Source-Commit: bd5119ae0dc46eef7dcab2e5f4fc403a85574c0e Related: ZSTAC-86365, ZSTAC-76341 Change-Id: I6de8611b596a775d893d2dde91ac89410f69bf2c --- .../zstack/compute/host/HostTrackImpl.java | 5 +++ .../kvm/host/HostConnectedTimeCase.groovy | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java index 49394e88b78..48ed6a7dc48 100755 --- a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java +++ b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java @@ -251,6 +251,11 @@ public void cancel() { public void trackHost(String hostUuid) { + if (!destMaker.isManagedByUs(hostUuid)) { + logger.debug(String.format("skip tracking host[uuid:%s], not managed by us", hostUuid)); + return; + } + Tracker t = trackers.get(hostUuid); if (t != null) { t.cancel(); diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy index eaf774fb84f..87a06e15abd 100644 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostConnectedTimeCase.groovy @@ -16,6 +16,11 @@ import org.zstack.utils.gson.JSONObjectUtil import org.zstack.header.network.l2.BatchCheckNetworkPhysicalInterfaceMsg import org.zstack.header.network.l2.BatchCheckNetworkPhysicalInterfaceReply import org.zstack.core.cloudbus.CloudBus +import org.zstack.core.cloudbus.ResourceDestinationMaker +import org.zstack.compute.host.HostTrackImpl + +import static org.mockito.Mockito.mock +import static org.mockito.Mockito.when class HostConnectedTimeCase extends SubCase { @@ -34,6 +39,7 @@ class HostConnectedTimeCase extends SubCase { @Override void test() { env.create { + testSkipTrackIfNotManagedByUs() testHostConnectedTime() } } @@ -43,6 +49,31 @@ class HostConnectedTimeCase extends SubCase { env.delete() } + /** + * Test for ZSTAC-76341: check isManagedByUs before trackHost to prevent dual MN tracking + */ + void testSkipTrackIfNotManagedByUs() { + HostInventory host = env.inventoryByName("kvm") as HostInventory + HostTrackImpl tracker = bean(HostTrackImpl.class) + + ResourceDestinationMaker originalDestMaker = tracker.@destMaker + + ResourceDestinationMaker mockDestMaker = mock(ResourceDestinationMaker.class) + when(mockDestMaker.isManagedByUs(host.uuid)).thenReturn(false) + tracker.@destMaker = mockDestMaker + + tracker.untrackHost(host.uuid) + assert !tracker.trackers.containsKey(host.uuid) + + tracker.trackHost(host.uuid) + assert !tracker.trackers.containsKey(host.uuid) : "Host should NOT be tracked when isManagedByUs returns false" + + tracker.@destMaker = originalDestMaker + + tracker.trackHost(host.uuid) + assert tracker.trackers.containsKey(host.uuid) : "Host should be tracked when isManagedByUs returns true" + } + void testHostConnectedTime() { HostInventory host = env.inventoryByName("kvm") From ee01712dc84362642a79442c43983c2be4956106 Mon Sep 17 00:00:00 2001 From: "haidong.pang" Date: Tue, 13 Jan 2026 06:50:19 +0000 Subject: [PATCH 50/53] [zbs]: Try to reconnect after zbs ping failure Resolves: ZSTAC-79447 Change-Id: I757275676874656b727a6172646961687a726870 --- .../storage/zbs/ZbsPrimaryStorageMdsBase.java | 6 + .../storage/zbs/ZbsStorageController.java | 88 +++++++++++-- .../addon/zbs/ZbsPrimaryStorageCase.groovy | 122 +++++++++++++++--- 3 files changed, 188 insertions(+), 28 deletions(-) diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java index 81309d43642..292beca133a 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java @@ -206,6 +206,12 @@ public void run(FlowTrigger trigger, Map data) { restf.asyncJsonPost(ZbsAgentUrl.primaryStorageUrl(getSelf().getAddr(), SYNC_METADATA_PATH), cmd, new JsonAsyncRESTCallback(trigger) { @Override public void success(SyncMetadataRsp ret) { + if (!ret.isSuccess()) { + trigger.fail(operr("unable to sync metadata from ZBS primary storage MDS[%s], because %s", + getSelf().getAddr(), ret.getError())); + return; + } + getSelf().setExternalAddr(ret.getExternalAddr()); trigger.next(); } diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 512712a3cf3..49e5924b5ab 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -57,6 +57,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -503,6 +504,46 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private void reconnectSingleMds(ZbsPrimaryStorageMdsBase mdsBase, Completion completion) { + mdsBase.connect(new Completion(completion) { + @Override + public void success() { + completion.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void reconnectMdss(List mdsBases, Completion completion) { + new While<>(mdsBases).all((mdsBase, comp) -> { + reconnectSingleMds(mdsBase, new Completion(comp) { + @Override + public void success() { + comp.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(errorCodeList.getCauses().get(0)); + return; + } + completion.success(); + } + }); + } + @Override public void ping(ReturnValueCompletion completion) { reloadDbInfo(); @@ -528,17 +569,46 @@ public void fail(ErrorCode errorCode) { })).run(new WhileDoneCompletion(completion) { @Override public void done(ErrorCodeList errorCodeList) { - boolean isConnected = addonInfo.getMdsInfos().stream().anyMatch(mdsInfo -> MdsStatus.Connected.equals(mdsInfo.getStatus())); - if (!isConnected) { - String notConnectedIps = addonInfo.getMdsInfos().stream() - .filter(mdsInfo -> !MdsStatus.Connected.equals(mdsInfo.getStatus())) - .map(MdsInfo::getAddr) - .collect(Collectors.joining(", ")); - - completion.success(new PingResult(addonInfo, String.format("all MDS are not connected, disconnected MDS addresses: %s", notConnectedIps))); + List disconnectedMds = mds.stream() + .filter(m -> MdsStatus.Disconnected.equals(m.getSelf().getStatus())) + .collect(Collectors.toList()); + + if (disconnectedMds.isEmpty()) { + completion.success(new PingResult(addonInfo)); return; } - completion.success(new PingResult(addonInfo)); + + final boolean allDisconnected = disconnectedMds.size() == mds.size(); + String notConnectedIps = disconnectedMds.stream() + .map(ZbsPrimaryStorageMdsBase::getSelf) + .map(MdsInfo::getAddr) + .collect(Collectors.joining(", ")); + + // it should be covered in primary storage reconnect phase + if (allDisconnected) { + completion.success(new PingResult(addonInfo, String.format("All MDS are not connected, disconnected MDS addresses: %s", notConnectedIps))); + return; + } + + // Try to reconnect disconnected MDS, the reconnection results are recorded in addonInfo + // and will not cause the ping operation to fail. + logger.warn(String.format("Some MDS are not connected for ZBS primary storage[uuid:%s], disconnected MDS addresses: %s, try to reconnect them", + self.getUuid(), notConnectedIps)); + + reconnectMdss(disconnectedMds, new Completion(completion) { + @Override + public void success() { + completion.success(new PingResult(addonInfo)); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("Some MDS are still not connected after reconnection for ZBS primary storage[uuid:%s], error: %s", + self.getUuid(), errorCode)); + completion.success(new PingResult(addonInfo)); + } + }); + } }); } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 5a85f0fc4d4..753302b4bc8 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -1,6 +1,7 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity +import org.zstack.cbd.AddonInfo import org.zstack.core.cloudbus.EventCallback import org.zstack.core.cloudbus.EventFacade import org.zstack.core.db.Q @@ -13,6 +14,7 @@ import org.zstack.header.storage.primary.PrimaryStorageStatus import org.zstack.header.storage.primary.PrimaryStorageVO import org.zstack.cbd.MdsUri import org.zstack.kvm.KVMAgentCommands +import org.zstack.cbd.MdsStatus import org.zstack.sdk.* import org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent import org.zstack.storage.primary.PrimaryStorageGlobalConfig @@ -29,6 +31,7 @@ import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger /** * @author Xingwei Yu @@ -161,7 +164,6 @@ class ZbsPrimaryStorageCase extends SubCase { diskOffering = env.inventoryByName("diskOffering") as DiskOfferingInventory kvm = env.inventoryByName("kvm-1") as KVMHostInventory evtf = bean(EventFacade.class) - testDefaultConfig() testUpdateExternalPrimaryStorage() testLifecycle() @@ -172,6 +174,7 @@ class ZbsPrimaryStorageCase extends SubCase { testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor() testDataVolumeNegativeScenario() testDecodeMdsUriWithSpecialPassword() + testMdsReconnectAfterMaximumPingFailures() } } @@ -368,55 +371,85 @@ class ZbsPrimaryStorageCase extends SubCase { env.afterSimulator(ZbsPrimaryStorageMdsBase.PING_PATH) { rsp, HttpEntity e -> def cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) - ZbsPrimaryStorageMdsBase.PingRsp pingRsp = new ZbsPrimaryStorageMdsBase.PingRsp() if (cmd.addr.equals("127.0.1.1")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" + } + + return rsp + } + + env.afterSimulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp, HttpEntity e -> + ZbsPrimaryStorageMdsBase.PingCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) + if (cmd.getAddr().equals("127.0.1.1")) { + rsp.setSuccess(false) + rsp.setError("on purpose") } - return pingRsp + return rsp } - sleep(2000) + retryInSecs { + assert count == 1 + } addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Connected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected - assert count == 1 assert data.uuid == ps.uuid env.afterSimulator(ZbsPrimaryStorageMdsBase.PING_PATH) { rsp, HttpEntity e -> def cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) - ZbsPrimaryStorageMdsBase.PingRsp pingRsp = new ZbsPrimaryStorageMdsBase.PingRsp() if (cmd.addr.equals("127.0.1.1")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } else if (cmd.addr.equals("127.0.1.2")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } else if (cmd.addr.equals("127.0.1.3")) { - pingRsp.success = false - pingRsp.error = "on purpose" + rsp.success = false + rsp.error = "on purpose" } - return pingRsp + return rsp } - sleep(2500) + env.afterSimulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp, HttpEntity e -> + ZbsPrimaryStorageMdsBase.PingCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) + if (cmd.getAddr().equals("127.0.1.1")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } else if (cmd.getAddr().equals("127.0.1.2")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } else if (cmd.getAddr().equals("127.0.1.3")) { + rsp.setSuccess(false) + rsp.setError("on purpose") + } + + return rsp + } + + retryInSecs { + assert count == 2 + } addonInfo = Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.addonInfo).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() assert addonInfo == "{\"clusterInfo\":{\"uuid\":\"123456789\",\"version\":\"1.6.1-for-test\"},\"mdsInfos\":[{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.1\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.2\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"},{\"username\":\"root\",\"password\":\"password\",\"port\":22,\"addr\":\"127.0.1.3\",\"externalAddr\":\"127.0.0.1\",\"status\":\"Disconnected\"}],\"logicalPoolInfos\":[{\"physicalPoolID\":1,\"redundanceAndPlaceMentPolicy\":{\"copysetNum\":300,\"replicaNum\":3,\"zoneNum\":3},\"logicalPoolID\":1,\"usedSize\":322961408,\"quota\":0,\"createTime\":1735875794,\"type\":0,\"rawWalUsedSize\":0,\"allocateStatus\":0,\"rawUsedSize\":968884224,\"physicalPoolName\":\"pool1\",\"capacity\":579933831168,\"logicalPoolName\":\"lpool1\",\"userPolicy\":\"eyJwb2xpY3kiIDogMX0=\",\"allocatedSize\":3221225472}]}" - assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Disconnected - assert count == 2 + retryInSecs { + assert Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Disconnected + } assert data.uuid == ps.uuid env.cleanAfterSimulatorHandlers() - sleep(2000) + retryInSecs { + assert count >= 3 + } Q.New(ExternalPrimaryStorageVO.class).select(ExternalPrimaryStorageVO_.status).eq(ExternalPrimaryStorageVO_.uuid, ps.uuid).findValue() == PrimaryStorageStatus.Connected @@ -586,6 +619,57 @@ class ZbsPrimaryStorageCase extends SubCase { assert uri.password == specialPassword } + void testMdsReconnectAfterMaximumPingFailures() { + env.cleanSimulatorAndMessageHandlers() + Integer originalPingInterval = PrimaryStorageGlobalConfig.PING_INTERVAL.value().toInteger() + PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(1) + final Integer MAX_PING_CNT = ZbsConstants.PRIMARY_STORAGE_MDS_MAXIMUM_PING_FAILURE + AtomicInteger pingFailureCount = new AtomicInteger(0) + Boolean reconnectTriggered = false + + env.simulator(ZbsPrimaryStorageMdsBase.PING_PATH) { HttpEntity e, EnvSpec spec -> + ZbsPrimaryStorageMdsBase.PingCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.PingCmd.class) + ZbsPrimaryStorageMdsBase.PingRsp rsp = new ZbsPrimaryStorageMdsBase.PingRsp() + if (cmd.getAddr().equals("127.0.1.1") && !reconnectTriggered) { + rsp.success = false + rsp.error = "Mds ping failed on purpose" + pingFailureCount.incrementAndGet() + return rsp + } + return rsp + + } + + env.simulator(ZbsPrimaryStorageMdsBase.SYNC_METADATA_PATH) { HttpEntity e, EnvSpec spec -> + ZbsPrimaryStorageMdsBase.SyncMetadataCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.SyncMetadataCmd.class) + ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp = new ZbsPrimaryStorageMdsBase.SyncMetadataRsp() + rsp.setExternalAddr(cmd.getAddr()) + if (cmd.getAddr().equals("127.0.1.1")) { + assert pingFailureCount.intValue().equals(MAX_PING_CNT) + reconnectTriggered = true + } + return rsp + } + + sleep((MAX_PING_CNT + 1) * 1000 + 500) + retryInSecs { + assert reconnectTriggered + } + retryInSecs { + AddonInfo addonInfo = JSONObjectUtil.toObject( + Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, ps.uuid) + .findValue() + .toString(), + AddonInfo.class + ) + assert MdsStatus.Connected.equals(addonInfo.getMdsInfos().find { it.addr.equals("127.0.1.1") }.getStatus()) + } + + PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(originalPingInterval) + env.cleanAfterSimulatorHandlers() + } void deleteVolume(String volUuid) { deleteDataVolume { From 450bf57ff476145d68703087fc44e2128d21f374 Mon Sep 17 00:00:00 2001 From: "haidong.pang" Date: Wed, 28 Jan 2026 05:33:30 +0000 Subject: [PATCH 51/53] [zbs]: Try to reconnect after all mds disconnected - After the ZBS primary storage agent on the MDS restarts, it may fail to start up independently due to lost firewall rules or metadata. - Redeploy by the MN is required. Resolves: ZSTAC-79447 Change-Id: I726b7467627269627779696d6e72756f75706e76 --- .../org/zstack/storage/zbs/ZbsConstants.java | 3 +- .../storage/zbs/ZbsPrimaryStorageMdsBase.java | 2 +- .../storage/zbs/ZbsStorageController.java | 109 ++++++++++++++---- .../addon/zbs/ZbsPrimaryStorageCase.groovy | 8 +- 4 files changed, 94 insertions(+), 28 deletions(-) diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java index 8f547cfeb29..987138c3360 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsConstants.java @@ -15,7 +15,8 @@ public interface ZbsConstants { long ZBS_HEARTBEAT_VOLUME_SIZE_IN_GIGABYTE = 1; String ZBS_CBD_LUN_PATH_FORMAT = "cbd:%s/%s/%s"; String ZBS_CBD_PREFIX_SCHEME = "cbd://"; - Integer PRIMARY_STORAGE_MDS_MAXIMUM_PING_FAILURE = 3; + Integer MDS_PING_RETRY_PER_CYCLE = 3; + Integer MDS_PING_FAIL_CYCLE_THRESHOLD = 3; String VOLUME_PHYSICAL_BLOCK_SIZE = "4096"; String MEGABYTE_SUPPORTED_VERSION = "1.6.1"; String MEGABYTE_UNIT = "M"; diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java index 292beca133a..986ad438b42 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsPrimaryStorageMdsBase.java @@ -313,7 +313,7 @@ public String getName() { } private void pingMds(ClusterInfo clusterInfo, final Completion completion) { - final Integer MAX_PING_CNT = ZbsConstants.PRIMARY_STORAGE_MDS_MAXIMUM_PING_FAILURE; + final Integer MAX_PING_CNT = ZbsConstants.MDS_PING_RETRY_PER_CYCLE; final List stepCount = new ArrayList<>(); for (int i = 1; i <= MAX_PING_CNT; i++) { stepCount.add(i); diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 49e5924b5ab..4db9baf0d1d 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -62,6 +62,7 @@ import java.util.stream.Collectors; import static org.zstack.core.Platform.operr; +import static org.zstack.storage.zbs.ZbsConstants.MDS_PING_FAIL_CYCLE_THRESHOLD; import static org.zstack.storage.zbs.ZbsHelper.*; /** @@ -86,6 +87,7 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar private AddonInfo addonInfo; private Config config; + private final Map mdsConsecutivePingFailureCount = new ConcurrentHashMap<>(); public static final String DEPLOY_CLIENT_PATH = "/zbs/primarystorage/client/deploy"; public static final String GET_CAPACITY_PATH = "/zbs/primarystorage/capacity"; public static final String GET_FACTS_PATH = "/zbs/primarystorage/facts"; @@ -544,6 +546,61 @@ public void done(ErrorCodeList errorCodeList) { }); } + private void incrementMdsPingFailureCount(ZbsPrimaryStorageMdsBase mds) { + // initialize the count if absent + mdsConsecutivePingFailureCount.putIfAbsent(mds.getSelf(), 0); + // increment the count only if it is below the threshold + // avoid unbounded growth of the count + if (mdsConsecutivePingFailureCount.get(mds.getSelf()) < MDS_PING_FAIL_CYCLE_THRESHOLD) { + mdsConsecutivePingFailureCount.merge(mds.getSelf(), 1, Integer::sum); + } + } + + private void resetMdsPingFailureCount(ZbsPrimaryStorageMdsBase mds) { + mdsConsecutivePingFailureCount.remove(mds.getSelf()); + } + + private List getDisconnectedMds(List mdsBases) { + return mdsBases.stream() + .filter(m -> MdsStatus.Disconnected.equals(m.getSelf().getStatus())) + .collect(Collectors.toList()); + } + + private boolean allMdssDisconnected(List mdsBases) { + List disconnectedMds = getDisconnectedMds(mdsBases); + return disconnectedMds.size() == mdsBases.size(); + } + + private List getMdssNeedToReconnect(List mdsBases) { + List disconnectedMds = getDisconnectedMds(mdsBases); + boolean allDisconnected = allMdssDisconnected(mdsBases); + + if (disconnectedMds.isEmpty()) { + return Collections.emptyList(); + } + + // If all MDS are disconnected, try to reconnect all of them + if (allDisconnected) { + logger.info("All MDS are disconnected, ignore the ping failure count threshold and all of them need be reconnected"); + return disconnectedMds; + } + + String disconnectedMdsInfo = disconnectedMds.stream() + .map(m -> String.format("%s (Consecutive Ping Failure: %d)", + m.getSelf().getAddr(), + mdsConsecutivePingFailureCount.computeIfAbsent(m.getSelf(), k -> 0))) + .collect(Collectors.joining(", ")); + + logger.info(String.format("Some MDS are disconnected for ZBS primary storage[uuid:%s], disconnected MDS: [%s]", + self.getUuid(), disconnectedMdsInfo)); + + return disconnectedMds.stream() + .filter(m -> mdsConsecutivePingFailureCount + .computeIfAbsent(m.getSelf(), k -> 0) >= MDS_PING_FAIL_CYCLE_THRESHOLD) + .collect(Collectors.toList()); + + } + @Override public void ping(ReturnValueCompletion completion) { reloadDbInfo(); @@ -558,44 +615,36 @@ public void ping(ReturnValueCompletion completion) { @Override public void success() { m.getSelf().setStatus(MdsStatus.Connected); + resetMdsPingFailureCount(m); comp.done(); } @Override public void fail(ErrorCode errorCode) { m.getSelf().setStatus(MdsStatus.Disconnected); + incrementMdsPingFailureCount(m); comp.done(); } })).run(new WhileDoneCompletion(completion) { @Override public void done(ErrorCodeList errorCodeList) { - List disconnectedMds = mds.stream() - .filter(m -> MdsStatus.Disconnected.equals(m.getSelf().getStatus())) - .collect(Collectors.toList()); + List mdsNeedToReconnect = getMdssNeedToReconnect(mds); - if (disconnectedMds.isEmpty()) { + if (mdsNeedToReconnect.isEmpty()) { completion.success(new PingResult(addonInfo)); return; } - final boolean allDisconnected = disconnectedMds.size() == mds.size(); - String notConnectedIps = disconnectedMds.stream() - .map(ZbsPrimaryStorageMdsBase::getSelf) - .map(MdsInfo::getAddr) + String mdsNeedToReconnectInfo = mdsNeedToReconnect.stream() + .map(m -> String.format("%s (Consecutive Ping Failure: %d)", + m.getSelf().getAddr(), + mdsConsecutivePingFailureCount.get(m.getSelf()))) .collect(Collectors.joining(", ")); - // it should be covered in primary storage reconnect phase - if (allDisconnected) { - completion.success(new PingResult(addonInfo, String.format("All MDS are not connected, disconnected MDS addresses: %s", notConnectedIps))); - return; - } - - // Try to reconnect disconnected MDS, the reconnection results are recorded in addonInfo - // and will not cause the ping operation to fail. - logger.warn(String.format("Some MDS are not connected for ZBS primary storage[uuid:%s], disconnected MDS addresses: %s, try to reconnect them", - self.getUuid(), notConnectedIps)); + logger.warn(String.format("Some MDS need to reconnect for ZBS primary storage[uuid:%s], MDS : %s", + self.getUuid(), mdsNeedToReconnectInfo)); - reconnectMdss(disconnectedMds, new Completion(completion) { + reconnectMdss(mdsNeedToReconnect, new Completion(completion) { @Override public void success() { completion.success(new PingResult(addonInfo)); @@ -603,8 +652,25 @@ public void success() { @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("Some MDS are still not connected after reconnection for ZBS primary storage[uuid:%s], error: %s", - self.getUuid(), errorCode)); + List stillDisconnectedMds = getDisconnectedMds(mds); + boolean allDisconnected = allMdssDisconnected(mds); + + if (allDisconnected) { + completion.success(new PingResult(addonInfo, + String.format("All MDS are still disconnected after reconnection for ZBS primary storage[uuid:%s]", + self.getUuid()))); + return; + } + + String stillDisconnectedMdsInfo = stillDisconnectedMds.stream() + .map(m -> String.format("%s (Consecutive Ping Failure: %d)", + m.getSelf().getAddr(), + mdsConsecutivePingFailureCount.get(m.getSelf()))) + .collect(Collectors.joining(", ")); + + logger.warn(String.format("Some MDS are still disconnected after reconnection for ZBS primary storage[uuid:%s], disconnected MDS : %s, error: %s", + self.getUuid(), stillDisconnectedMdsInfo, errorCode)); + completion.success(new PingResult(addonInfo)); } }); @@ -1188,7 +1254,6 @@ public void fail(ErrorCode errorCode) { }); } - @Override public void syncAddonInfo(String addonInfo) { this.addonInfo = StringUtils.isEmpty(addonInfo) ? new AddonInfo() : JSONObjectUtil.toObject(addonInfo, AddonInfo.class); diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 753302b4bc8..6f4fb8db734 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -623,7 +623,8 @@ class ZbsPrimaryStorageCase extends SubCase { env.cleanSimulatorAndMessageHandlers() Integer originalPingInterval = PrimaryStorageGlobalConfig.PING_INTERVAL.value().toInteger() PrimaryStorageGlobalConfig.PING_INTERVAL.updateValue(1) - final Integer MAX_PING_CNT = ZbsConstants.PRIMARY_STORAGE_MDS_MAXIMUM_PING_FAILURE + final Integer MDS_PING_RETRY_PER_CYCLE = ZbsConstants.MDS_PING_RETRY_PER_CYCLE + final Integer MDS_PING_FAIL_CYCLE_THRESHOLD = ZbsConstants.MDS_PING_FAIL_CYCLE_THRESHOLD AtomicInteger pingFailureCount = new AtomicInteger(0) Boolean reconnectTriggered = false @@ -644,14 +645,13 @@ class ZbsPrimaryStorageCase extends SubCase { ZbsPrimaryStorageMdsBase.SyncMetadataCmd cmd = JSONObjectUtil.toObject(e.body, ZbsPrimaryStorageMdsBase.SyncMetadataCmd.class) ZbsPrimaryStorageMdsBase.SyncMetadataRsp rsp = new ZbsPrimaryStorageMdsBase.SyncMetadataRsp() rsp.setExternalAddr(cmd.getAddr()) - if (cmd.getAddr().equals("127.0.1.1")) { - assert pingFailureCount.intValue().equals(MAX_PING_CNT) + if (cmd.getAddr().equals("127.0.1.1") && pingFailureCount.intValue().equals(MDS_PING_RETRY_PER_CYCLE * MDS_PING_FAIL_CYCLE_THRESHOLD)) { reconnectTriggered = true } return rsp } - sleep((MAX_PING_CNT + 1) * 1000 + 500) + sleep((MDS_PING_RETRY_PER_CYCLE * MDS_PING_FAIL_CYCLE_THRESHOLD + 1) * 1000 + 500) retryInSecs { assert reconnectTriggered } From a71fd09e49405af8acf67e0d8863b10ba66e224a Mon Sep 17 00:00:00 2001 From: "hanyu.liang" Date: Thu, 16 Jul 2026 12:06:27 +0800 Subject: [PATCH 52/53] [drs]: enlarge migration result column DBImpact Resolves: ZSTAC-86507 Change-Id: I170cd7bad481cb3d6d7e2dfcef12ef94cc132d40 --- conf/db/upgrade/V5.4.12__schema.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/db/upgrade/V5.4.12__schema.sql b/conf/db/upgrade/V5.4.12__schema.sql index e69de29bb2d..c5cb79342f0 100644 --- a/conf/db/upgrade/V5.4.12__schema.sql +++ b/conf/db/upgrade/V5.4.12__schema.sql @@ -0,0 +1 @@ +ALTER TABLE `zstack`.`DRSVmMigrationActivityVO` MODIFY COLUMN `result` TEXT DEFAULT NULL; From a9091ff7044c087cdfe4a84245414c4dfbb7f5d5 Mon Sep 17 00:00:00 2001 From: "yaohua.wu" Date: Thu, 16 Jul 2026 14:12:59 +0800 Subject: [PATCH 53/53] [guesttools]: support emoji script output on 5.4.12 Backport script output utf8mb4 storage to 5.4.12. 1. Why? Guest script stdout and stderr can contain four-byte UTF-8 characters. The old column charset cannot store emoji output correctly. 2. How? Change stdout and stderr columns to utf8mb4 in the V5.4.12 schema. The ALTER MODIFY statements are idempotent for repeated upgrades. 3. Side effects? Column charset changes to utf8mb4 with utf8mb4_unicode_ci collation. - db: alter guest script stdout and stderr columns to utf8mb4. Source-Commit: da699ea9498d745d0058daafa4136bc50f42bdb7 Related: ZSTAC-86425, ZSTAC-80478 Change-Id: I9e8458c2c6b497afb3d00545019f1c6419acccf1 --- conf/db/upgrade/V5.4.12__schema.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/db/upgrade/V5.4.12__schema.sql b/conf/db/upgrade/V5.4.12__schema.sql index c5cb79342f0..405b7e018ac 100644 --- a/conf/db/upgrade/V5.4.12__schema.sql +++ b/conf/db/upgrade/V5.4.12__schema.sql @@ -1 +1,7 @@ ALTER TABLE `zstack`.`DRSVmMigrationActivityVO` MODIFY COLUMN `result` TEXT DEFAULT NULL; + +ALTER TABLE `GuestVmScriptExecutedRecordDetailVO` + MODIFY `stdout` MEDIUMTEXT CHARACTER SET `utf8mb4` COLLATE `utf8mb4_unicode_ci`; + +ALTER TABLE `GuestVmScriptExecutedRecordDetailVO` + MODIFY `stderr` MEDIUMTEXT CHARACTER SET `utf8mb4` COLLATE `utf8mb4_unicode_ci`;