From 4aa9008eb97b63e6d91e5ce83749c6e33bc1ed74 Mon Sep 17 00:00:00 2001 From: lianghy Date: Tue, 20 Jan 2026 18:07:31 +0800 Subject: [PATCH 01/24] [conf]: bump version to 5.4.10 DBImpact Resolves: ZSTAC-81425 Change-Id: I627a6576686f6c6972636d67786e77716e637471 --- VERSION | 2 +- conf/db/upgrade/V5.4.10__schema.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 conf/db/upgrade/V5.4.10__schema.sql diff --git a/VERSION b/VERSION index b7c3a02aec6..d406ff05502 100755 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MAJOR=5 MINOR=4 -UPDATE=7 +UPDATE=10 diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql new file mode 100644 index 00000000000..e69de29bb2d From 11767587c9f03bd1ef4cff19ab3f3245e6f6133d Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Wed, 24 Jun 2026 18:26:46 +0900 Subject: [PATCH 02/24] [search]: fix zql field query total Use entity alias for simple return-with-total counts so nullable selected fields do not turn totals into zero. Keep explicit count(field) and distinct(field) behavior unchanged. Add CountGroupByCase coverage for nullable first selected field. Resolves: ZSTAC-78110 Change-Id: I3b437bc686c345ba17baf256a338ce88209571f8 (cherry picked from commit 0dfe389e6d5560dbc8e899a4330de039c84e2791) --- .../ast/visitors/plugin/SimpleCountPlugin.java | 16 ++++++++-------- .../test/integration/zql/CountGroupByCase.groovy | 8 ++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java index 6627311adb9..10ebab413b4 100644 --- a/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java +++ b/search/src/main/java/org/zstack/zql/ast/visitors/plugin/SimpleCountPlugin.java @@ -1,10 +1,10 @@ package org.zstack.zql.ast.visitors.plugin; -import org.apache.commons.lang.StringUtils; import org.zstack.header.zql.ASTNode; import java.util.List; -import java.util.stream.Collectors; + +import static org.zstack.zql.ast.visitors.constants.MySqlKeyword.isDistinct; public class SimpleCountPlugin extends AbstractQueryVisitorPlugin { public SimpleCountPlugin() { @@ -24,13 +24,13 @@ public String selectTarget() { String queryTarget; List fieldNames = super.targetFields(); - if (fieldNames.isEmpty()) { - queryTarget = entityAlias; - } else { + if (getClauseType() == ClauseType.COUNT || isDistinct(node.getTarget().getFunction())) { // TODO: Compatibility changes: hql count do not support multiple fields, even if distinct modified. - queryTarget = String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + queryTarget = fieldNames.isEmpty() ? entityAlias : String.format("%s.%s", inventory.simpleInventoryName(), fieldNames.get(0)); + return String.format("count(%s)", String.format(functions(), queryTarget)); + } else { + queryTarget = entityAlias; + return String.format("count(%s)", queryTarget); } - - return String.format("count(%s)", String.format(functions(), queryTarget)); } } diff --git a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy index 72cf1d6ebb8..e37785f7674 100644 --- a/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/zql/CountGroupByCase.groovy @@ -36,6 +36,7 @@ class CountGroupByCase extends SubCase { env.create { testCountGroupBy() testQuery() + testFieldQueryTotalWithNullableFirstField() testGroupLimit() testEmpty() } @@ -80,6 +81,13 @@ class CountGroupByCase extends SubCase { assert ret2.total == 102 } + void testFieldQueryTotalWithNullableFirstField() { + def ret = ZQL.fromString("query instanceoffering.description,uuid return with (total) limit 100").getSingleResult() as ZQLQueryReturn + assert ret.inventories.size() == 100 + assert ret.total == 102 + assert ret.inventories.every { it.description == null } + } + void testGroupLimit() { def ret = ZQL.fromString("count instanceoffering group by name,cpuNum limit 2").getSingleResult() as ZQLQueryReturn assert ret.inventoryCounts.size() == 2 From 4ef0c8f8ec8166c0b05e184c5da7c8625c3d91dd Mon Sep 17 00:00:00 2001 From: "dejing.liu" Date: Thu, 15 Jan 2026 15:25:01 +0800 Subject: [PATCH 03/24] [kvm]: force virtio driver for windows vhostuser nic The vhost-user interface (used by OVS-DPDK) strictly requires the 'virtio-net' frontend driver in Libvirt/QEMU. However, ZStack defaults Windows NIC drivers to 'e1000' (or other emulated models) when the image platform is 'Windows', causing VM start failure with Libvirt error: "vhostuser requires the virtio-net* frontend". This fix forces the NIC driver to 'virtio' specifically when the interface type is 'vhostuser' on Windows platforms, regardless of the default platform settings. Satisfying both the legacy boot requirements of Windows and the vhost-user network requirements. Resolves: ZSTAC-76013 Change-Id: I737a6e637273796d776e726e7969786c7a6a6478 --- .../kvm/src/main/java/org/zstack/kvm/KVMHost.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index ab8c29640f3..cd58e7b58e9 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -4498,6 +4498,21 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi } nics = nics.stream().sorted(Comparator.comparing(NicTO::getDeviceId)).collect(Collectors.toList()); cmd.setNics(nics); + if ((platform.equals(ImagePlatform.Windows.toString()) || + platform.equals(ImagePlatform.WindowsVirtio.toString()))) { + if (cmd.getNics() != null) { + for (NicTO nicTo : cmd.getNics()) { + if (nicTo.getType() != null && + nicTo.getType().toLowerCase().contains(VmNicType.VmNicSubType.VHOSTUSER.toString().toLowerCase()) && + !VmNicDriverType.VIRTIO.toString().equalsIgnoreCase(nicTo.getDriverType())) { + logger.warn(String.format("Force updating nic[%s] driver from [%s] to [virtio] for " + + "Windows vhostuser compatibility.", + nicTo.getUuid(), nicTo.getDriverType())); + nicTo.setDriverType(VmNicDriverType.VIRTIO.toString()); + } + } + } + } for (VmInstanceSpec.CdRomSpec cdRomSpec : spec.getCdRomSpecs()) { CdRomTO cdRomTO = new CdRomTO(); From ada4787435dc7a56902fc314d15b836af3fc1734 Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Thu, 14 May 2026 22:45:49 +0800 Subject: [PATCH 04/24] [query]: limit API query concurrency Route AutoQuery handling through the query sync queue so bursty GraphQL refreshes cannot run unlimited APIQueryMessage requests concurrently. Resolves: ZSTAC-80742 Change-Id: If2c052e539526cd0511e037d1718838bdb65a7a3 --- .../org/zstack/query/QueryFacadeImpl.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java index 59aa7ce7e7d..3e110ae6dd1 100755 --- a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java +++ b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java @@ -347,7 +347,7 @@ private Method getReplySetter(AutoQuery at) { } } - private void handle(APIQueryMessage msg) { + private APIQueryReply doQuery(APIQueryMessage msg) { AutoQuery at = autoQueryMap.get(msg.getClass()); if (at == null) { at = msg.getClass().getAnnotation(AutoQuery.class); @@ -372,7 +372,7 @@ private void handle(APIQueryMessage msg) { if (result.inventories != null) { replySetter.invoke(reply, result.inventories); } - bus.reply(msg, reply); + return reply; } catch (OperationFailureException of) { throw of; } catch (Exception e) { @@ -380,6 +380,39 @@ private void handle(APIQueryMessage msg) { } } + private void handle(APIQueryMessage msg) { + thdf.syncSubmit(new SyncTask() { + @Override + public Void call() { + APIQueryReply reply; + try { + reply = doQuery(msg); + } catch (OperationFailureException of) { + reply = new APIQueryReply(); + reply.setError(of.getErrorCode()); + } + + bus.reply(msg, reply); + return null; + } + + @Override + public String getName() { + return getSyncSignature(); + } + + @Override + public String getSyncSignature() { + return "api-query"; + } + + @Override + public int getSyncLevel() { + return getQuerySyncLevel(); + } + }); + } + private String toZQLConditionString(QueryCondition c) { // make every condition value as string, the ZQL // will put them in right type because it knows every field's type From f25df0a6e9a85e47d44893c54ce69638c5dcb0d9 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 2 Jul 2026 12:14:53 +0800 Subject: [PATCH 05/24] [gpu]: add gpu normalized model schema Backport the normalizedModelName schema required by premium GPU spec deduplication to 5.4.10. This is the zstack companion change for the premium ZSTAC-86476 backport. Constraint: ZSTAC-86476 is the 5.4.10 clone of ZSTAC-75319. Confidence: high Scope-risk: narrow Tested: ./runMavenProfile premium in pr-build-bin Docker with zstack and premium mounted Not-tested: local stability case execution Resolves: ZSTAC-86476 Change-Id: Ic77e24e974a99842c40f29e4bd72851e3833795b --- conf/db/upgrade/V5.4.10__schema.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/db/upgrade/V5.4.10__schema.sql b/conf/db/upgrade/V5.4.10__schema.sql index e69de29bb2d..ab73deda0f6 100644 --- a/conf/db/upgrade/V5.4.10__schema.sql +++ b/conf/db/upgrade/V5.4.10__schema.sql @@ -0,0 +1,3 @@ +-- ZSTAC-86476: Add normalizedModelName column for GPU spec dedup +CALL ADD_COLUMN('GpuDeviceSpecVO', 'normalizedModelName', 'VARCHAR(255)', 1, NULL); +CALL CREATE_INDEX('GpuDeviceSpecVO', 'idx_gpu_spec_normalized_model', 'normalizedModelName'); From b5d630f15533ac69d3aed2ac50e917c62f2526bd Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 2 Jul 2026 14:37:09 +0800 Subject: [PATCH 06/24] [search]: stabilize paginated query order Backport ZSTAC-74748 pagination tiebreaker from 5.5.6/5.5.28 to 5.4.10. Paginated LIMIT/OFFSET queries can return duplicate resources when the requested sort field has duplicate values. Append the primary key as a deterministic tiebreaker in both ZQL and legacy HQL query paths, and skip hidden PK fields that ZQL cannot order by. Source-Commits: 284d04988a, b1156d87cc Tested: git diff --check upstream/5.4.10..HEAD Resolves: ZSTAC-86475 Change-Id: I14bef1d630a70711ef2605ea95fd37cb86e5273c --- .../zstack/query/MysqlQueryBuilderImpl3.java | 10 +++++- .../org/zstack/query/QueryFacadeImpl.java | 31 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java index 101684ad18c..df5e0e5a96c 100755 --- a/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java +++ b/search/src/main/java/org/zstack/query/MysqlQueryBuilderImpl3.java @@ -729,7 +729,15 @@ String toJpql(boolean isCount) { throw new IllegalArgumentException(String.format("illegal sortBy[%s], entity[%s] doesn't have this field", msg.getSortBy(), info.entityClass.getName())); } - ret = String.format("%s order by %s.%s %s", ret, entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + String orderBy = String.format("order by %s.%s %s", entityName, msg.getSortBy(), msg.getSortDirection().toUpperCase()); + if (info.primaryKey != null && !msg.getSortBy().equals(info.primaryKey)) { + orderBy += String.format(", %s.%s ASC", entityName, info.primaryKey); + } + ret = String.format("%s %s", ret, orderBy); + } else if (msg.getStart() != null || msg.getLimit() != null) { + if (info.primaryKey != null) { + ret = String.format("%s order by %s.%s ASC", ret, entityName, info.primaryKey); + } } if (msg.getGroupBy() != null) { diff --git a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java index 59aa7ce7e7d..6908e4f2678 100755 --- a/search/src/main/java/org/zstack/query/QueryFacadeImpl.java +++ b/search/src/main/java/org/zstack/query/QueryFacadeImpl.java @@ -30,6 +30,7 @@ import org.zstack.utils.*; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; +import org.zstack.core.db.EntityMetadata; import org.zstack.zql.ZQL; import org.zstack.zql.ZQLContext; import org.zstack.zql.ZQLQueryReturn; @@ -482,7 +483,17 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { } if (msg.getSortBy() != null) { - sb.add(String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection())); + String orderBy = String.format("order by %s %s", msg.getSortBy(), msg.getSortDirection()); + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null && !msg.getSortBy().equals(pkFieldName)) { + orderBy += String.format(", %s asc", pkFieldName); + } + sb.add(orderBy); + } else if (!msg.isCount() && (msg.getLimit() != null || msg.getStart() != null)) { + String pkFieldName = getPrimaryKeyFieldNameSafely(targetInventoryClass); + if (pkFieldName != null) { + sb.add(String.format("order by %s asc", pkFieldName)); + } } if (msg.getLimit() != null) { @@ -514,6 +525,24 @@ public ZQLQueryReturn queryUseZQL(APIQueryMessage msg, Class inventoryClass) { return result; } + private String getPrimaryKeyFieldNameSafely(Class inventoryClass) { + try { + ZQLMetadata.InventoryMetadata meta = ZQLMetadata.findInventoryMetadata( + ZQL.queryTargetNameFromInventoryClass(inventoryClass)); + Class voClass = meta.inventoryAnnotation.mappingVOClass(); + Field pkField = EntityMetadata.getPrimaryKeyField(voClass); + String pkName = pkField.getName(); + if (!meta.selfInventoryFieldNames.contains(pkName)) { + return null; + } + return pkName; + } catch (Exception e) { + logger.trace(String.format("cannot resolve PK for %s, skip pagination tiebreaker: %s", + inventoryClass.getSimpleName(), e.getMessage())); + return null; + } + } + private QueryBelongFilter validateFilterNameAndGetExp(String filterName) { if (filterName.split(":").length < 2) { throw new OperationFailureException(argerr("filterName must be formatted as [filterType:condition(s)]")); From dc8697598dc3a63124aaa2e1951c851582ffc152 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Mon, 11 May 2026 21:45:42 +0800 Subject: [PATCH 07/24] [kvm]: skip known systemd timeout on reconnect Existing host reconnect can fail before kvmagent echo. Ansible masks libvirt sockets with systemctl during deploy. If host systemd D-Bus is stuck, that optional step times out. Continue only for this known mask timeout on reconnect. New host deploy and other ansible failures still fail. Resolves: ZSTAC-86349 Change-Id: I0ef4a535065ff797c9e4cfae5b39c2daa321a4cc --- .../src/main/java/org/zstack/kvm/KVMHost.java | 7 ++++ .../java/org/zstack/kvm/KVMHostUtils.java | 33 +++++++++++++++++++ .../org/zstack/test/kvm/KVMHostUtilsTest.java | 25 ++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index ab8c29640f3..fc0bb962f9a 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -5803,6 +5803,13 @@ public void success(Boolean run) { @Override public void fail(ErrorCode errorCode) { + if (KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(info.isNewAdded(), errorCode)) { + logger.warn(String.format( + "kvm ansible failed to mask libvirt sockets because systemd dbus timed out on existing host[uuid:%s, ip:%s], continue reconnect and verify kvmagent, error: %s", + self.getUuid(), self.getManagementIp(), errorCode)); + trigger.next(); + return; + } trigger.fail(errorCode); } }); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java index cf6d16e7560..3b9979caeb3 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostUtils.java @@ -2,6 +2,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.network.l2.*; import org.zstack.header.tag.SystemTagVO; import org.zstack.header.tag.SystemTagVO_; @@ -13,6 +14,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Set; /** @@ -21,6 +23,37 @@ public class KVMHostUtils { private static final CLogger logger = CLoggerImpl.getLogger(KVMHostUtils.class); + public static boolean shouldContinueReconnectOnAnsibleFailure(boolean isNewAdded, ErrorCode errorCode) { + return !isNewAdded && isLibvirtSocketMaskSystemdTimeout(errorCode); + } + + public static boolean isLibvirtSocketMaskSystemdTimeout(ErrorCode errorCode) { + String errorText = collectErrorText(errorCode).toLowerCase(Locale.ROOT); + return errorText.contains("systemctl mask") + && errorText.contains("libvirtd.socket") + && errorText.contains("org.freedesktop.systemd1") + && errorText.contains("timed out") + && (errorText.contains("failed to get properties") + || errorText.contains("failed to activate service")); + } + + private static String collectErrorText(ErrorCode errorCode) { + StringBuilder sb = new StringBuilder(); + ErrorCode cursor = errorCode; + while (cursor != null) { + appendIfNotNull(sb, cursor.getDetails()); + appendIfNotNull(sb, cursor.getDescription()); + cursor = cursor.getCause(); + } + return sb.toString(); + } + + private static void appendIfNotNull(StringBuilder sb, String text) { + if (text != null) { + sb.append(text).append('\n'); + } + } + /** * Get normalized bridge name for l2 network, which at most has 15 chars. * - if l2 network has L2_BRIDGE_NAME tag, then return it's value directly; diff --git a/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java new file mode 100644 index 00000000000..2e0e06c7dfc --- /dev/null +++ b/test/src/test/java/org/zstack/test/kvm/KVMHostUtilsTest.java @@ -0,0 +1,25 @@ +package org.zstack.test.kvm; + +import org.junit.Assert; +import org.junit.Test; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.kvm.KVMHostUtils; + +public class KVMHostUtilsTest { + @Test + public void zstac86349_continueReconnectOnLibvirtSocketMaskSystemdTimeout() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl mask libvirtd.socket libvirtd-ro.socket libvirtd-admin.socket libvirtd-tls.socket libvirtd-tcp.socket failed! stderr: Failed to get properties: Failed to activate service 'org.freedesktop.systemd1': timed out (service_start_timeout=25000ms)"); + + Assert.assertTrue(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(true, error)); + } + + @Test + public void zstac86349_doNotContinueReconnectOnOtherAnsibleFailures() { + ErrorCode error = new ErrorCode(); + error.setDetails("[HOST: 192.168.51.12] ERROR: run shell command: systemctl restart libvirtd failed! stderr: Job for libvirtd.service failed"); + + Assert.assertFalse(KVMHostUtils.shouldContinueReconnectOnAnsibleFailure(false, error)); + } +} From e6d3189d79d0cf85e121703c0709bc99d658cca9 Mon Sep 17 00:00:00 2001 From: Jiajun Xu Date: Thu, 2 Jul 2026 17:08:23 +0800 Subject: [PATCH 08/24] [identity]: restrict APIQueryAccountMsg to admin-only in RBAC Resolves: ZSTAC-86350 Change-Id: I024ae51da6e5784fb336d8fbd5a4aeab44500565 --- .../org/zstack/header/identity/RBACInfo.java | 1 + .../TestAPIQueryAccountMsgRBACCase.java | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java diff --git a/header/src/main/java/org/zstack/header/identity/RBACInfo.java b/header/src/main/java/org/zstack/header/identity/RBACInfo.java index 609a916d20c..81223064cd9 100755 --- a/header/src/main/java/org/zstack/header/identity/RBACInfo.java +++ b/header/src/main/java/org/zstack/header/identity/RBACInfo.java @@ -12,6 +12,7 @@ public void permissions() { .name("identity") .adminOnlyAPIs( APICreateAccountMsg.class, + APIQueryAccountMsg.class, APIShareResourceMsg.class, APIRevokeResourceSharingMsg.class, APIUpdateQuotaMsg.class, diff --git a/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java new file mode 100644 index 00000000000..15216cb50bb --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/unittest/identity/TestAPIQueryAccountMsgRBACCase.java @@ -0,0 +1,43 @@ +package org.zstack.test.unittest.identity; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.zstack.header.identity.APIQueryAccountMsg; +import org.zstack.header.identity.RBACInfo; +import org.zstack.header.identity.rbac.RBAC; + +public class TestAPIQueryAccountMsgRBACCase { + + @Before + public void setUp() { + RBAC.permissions.clear(); + } + + @Test + public void testAPIQueryAccountMsgIsAdminOnly() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + boolean isAdminOnly = RBAC.isAdminOnlyAPI(APIQueryAccountMsg.class.getName()); + Assert.assertTrue( + "APIQueryAccountMsg should be admin-only to prevent privilege escalation, " + + "but it is currently accessible to normal IAM users via the wildcard normalAPIs pattern", + isAdminOnly + ); + } + + @Test + public void testAPIQueryAccountMsgNotInNormalAPIs() { + RBACInfo rbacInfo = new RBACInfo(); + rbacInfo.permissions(); + + String apiName = APIQueryAccountMsg.class.getName(); + boolean isInNormalOnly = RBAC.permissions.stream() + .anyMatch(p -> p.getNormalAPIs().contains(apiName) && !p.getAdminOnlyAPIs().contains(apiName)); + Assert.assertFalse( + "APIQueryAccountMsg should NOT be accessible as a normal API", + isInNormalOnly + ); + } +} From 7e46fa568a88bff5b89d23bbecb2949f7aadbb8f Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 3 Jul 2026 10:43:56 +0800 Subject: [PATCH 09/24] [mn]: synchronize hash ring operations Backport the management node hash ring fix to 5.4.10. 1. Why? Management node lifecycle reconciliation can race with callbacks. This can cause NodeHash or Nodes inconsistency and task stalling. 2. How? Synchronize hash ring operations and use defensive copies. Add lifecycle reconciliation locking and two-round missing checks. 3. Side effects? Management node lifecycle operations are serialized by a local lock. # Summary of changes (by module): - core: synchronize resource destination hash ring operations. - portal: serialize management node lifecycle reconciliation. Source-Commit: 66ab572b3c0ff62d243518eb9808dc85aa7ea6ac Related: ZSTAC-86294, ZSTAC-78182 Change-Id: I3ba21ffc7b8f1284e17d50da7b027ffdad70828b --- .../ResourceDestinationMakerImpl.java | 28 ++--- .../ManagementNodeManagerImpl.java | 101 ++++++++++++------ 2 files changed, 82 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java index 08a776f1db2..9ead578395d 100755 --- a/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java +++ b/core/src/main/java/org/zstack/core/cloudbus/ResourceDestinationMakerImpl.java @@ -27,27 +27,27 @@ public class ResourceDestinationMakerImpl implements ManagementNodeChangeListene private DatabaseFacade dbf; @Override - public void nodeJoin(ManagementNodeInventory inv) { + public synchronized void nodeJoin(ManagementNodeInventory inv) { nodeHash.add(inv.getUuid()); nodes.put(inv.getUuid(), new NodeInfo(inv)); } @Override - public void nodeLeft(ManagementNodeInventory inv) { + public synchronized void nodeLeft(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iAmDead(ManagementNodeInventory inv) { + public synchronized void iAmDead(ManagementNodeInventory inv) { String nodeId = inv.getUuid(); nodeHash.remove(nodeId); nodes.remove(nodeId); } @Override - public void iJoin(ManagementNodeInventory inv) { + public synchronized void iJoin(ManagementNodeInventory inv) { List lst = Q.New(ManagementNodeVO.class).list(); lst.forEach((ManagementNodeVO node) -> { nodeHash.add(node.getUuid()); @@ -56,7 +56,7 @@ public void iJoin(ManagementNodeInventory inv) { } @Override - public String makeDestination(String resourceUuid) { + public synchronized String makeDestination(String resourceUuid) { String nodeUuid = nodeHash.get(resourceUuid); if (nodeUuid == null) { throw new CloudRuntimeException("Cannot find any available management node to send message"); @@ -66,18 +66,18 @@ public String makeDestination(String resourceUuid) { } @Override - public boolean isManagedByUs(String resourceUuid) { + public synchronized boolean isManagedByUs(String resourceUuid) { String nodeUuid = makeDestination(resourceUuid); return nodeUuid.equals(Platform.getManagementServerId()); } @Override - public Collection getManagementNodesInHashRing() { - return nodeHash.getNodes(); + public synchronized Collection getManagementNodesInHashRing() { + return new ArrayList<>(nodeHash.getNodes()); } @Override - public NodeInfo getNodeInfo(String nodeUuid) { + public synchronized NodeInfo getNodeInfo(String nodeUuid) { NodeInfo info = nodes.get(nodeUuid); if (info == null) { ManagementNodeVO vo = dbf.findByUuid(nodeUuid, ManagementNodeVO.class); @@ -93,17 +93,17 @@ public NodeInfo getNodeInfo(String nodeUuid) { } @Override - public Collection getAllNodeInfo() { - return nodes.values(); + public synchronized Collection getAllNodeInfo() { + return new ArrayList<>(nodes.values()); } @Override - public int getManagementNodeCount() { - return nodes.values().size(); + public synchronized int getManagementNodeCount() { + return nodes.size(); } - public boolean isNodeInCircle(String nodeId) { + public synchronized boolean isNodeInCircle(String nodeId) { return nodeHash.hasNode(nodeId); } } diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java index 747a154919b..88bef7d8302 100755 --- a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java +++ b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodeManagerImpl.java @@ -74,6 +74,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -106,6 +107,15 @@ public class ManagementNodeManagerImpl extends AbstractService implements Manage // A dictionary (nodeId -> ManagementNodeInventory) of joined management Node final private Map joinedManagementNodes = new ConcurrentHashMap<>(); + // Lock to serialize lifecycle events from heartbeat reconciliation and canonical event callbacks, + // preventing race conditions where a nodeJoin event is immediately followed by a stale nodeLeft + // from the heartbeat thread, or vice versa. See ZSTAC-77711. + private final Object lifecycleLock = new Object(); + + // Track nodes found in hash ring but missing from DB. Only call nodeLeft after a node + // is missing for two consecutive heartbeat cycles, to avoid removing nodes that just joined. + private final Set suspectedMissingFromDb = new HashSet<>(); + private static int NODE_STARTING = 0; private static int NODE_RUNNING = 1; private static int NODE_FAILED = -1; @@ -367,12 +377,16 @@ protected void run(Map tokens, Object data) { ManagementNodeLifeCycleData d = (ManagementNodeLifeCycleData) data; - if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeJoin(d.getInventory()); - } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { - nodeLifeCycle.nodeLeft(d.getInventory()); - } else { - throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + synchronized (lifecycleLock) { + if (LifeCycle.NodeJoin.toString().equals(d.getLifeCycle())) { + // Clear from suspected set since the node is confirmed alive + suspectedMissingFromDb.remove(d.getInventory().getUuid()); + nodeLifeCycle.nodeJoin(d.getInventory()); + } else if (LifeCycle.NodeLeft.toString().equals(d.getLifeCycle())) { + nodeLifeCycle.nodeLeft(d.getInventory()); + } else { + throw new CloudRuntimeException(String.format("unknown lifecycle[%s]", d.getLifeCycle())); + } } } }; @@ -853,34 +867,55 @@ private void checkAllNodesHealth() { Set nodeUuidsInDb = nodesInDb.stream().map(ManagementNodeVO::getUuid).collect(Collectors.toSet()); - // When a node is dying, we may not receive the the dead notification because the message bus may be also dead - // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out - destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { - if (!nodeUuidsInDb.contains(nodeUuid)) { - logger.warn(String.format("found that a management node[uuid:%s] had no heartbeat in database but still in our hash ring," + - "notify that it's dead", nodeUuid)); - ManagementNodeInventory inv = new ManagementNodeInventory(); - inv.setUuid(nodeUuid); - inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); - - nodeLifeCycle.nodeLeft(inv); - } - }); - - // check if any node missing in our hash ring - nodesInDb.forEach(n -> { - if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { - return; - } - - new Runnable() { - @Override - @AsyncThread - public void run() { - nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + // Reconcile hash ring with DB under lifecycleLock to prevent race with + // canonical event callbacks (nodeJoin/nodeLeft). See ZSTAC-77711. + synchronized (lifecycleLock) { + // When a node is dying, we may not receive the dead notification because the message bus may be also dead + // at that moment. By checking if the node UUID is still in our hash ring, we know what nodes should be kicked out. + // Use two-round confirmation: first round marks as suspected, second round actually removes. + Set currentSuspected = new HashSet<>(); + destinationMaker.getManagementNodesInHashRing().forEach(nodeUuid -> { + if (!nodeUuidsInDb.contains(nodeUuid)) { + if (suspectedMissingFromDb.contains(nodeUuid)) { + // Second consecutive detection — confirmed missing, remove from hash ring + logger.warn(String.format("management node[uuid:%s] confirmed missing from database for two consecutive" + + " heartbeat cycles, removing from hash ring", nodeUuid)); + ManagementNodeInventory inv = new ManagementNodeInventory(); + inv.setUuid(nodeUuid); + try { + inv.setHostName(destinationMaker.getNodeInfo(nodeUuid).getNodeIP()); + } catch (Exception e) { + logger.warn(String.format("cannot get node info for node[uuid:%s], use empty hostname", nodeUuid)); + } + + nodeLifeCycle.nodeLeft(inv); + } else { + // First detection — mark as suspected, defer removal to next cycle + logger.warn(String.format("management node[uuid:%s] not found in database but still in hash ring," + + " marking as suspected (will remove on next heartbeat if still missing)", nodeUuid)); + currentSuspected.add(nodeUuid); + } } - }.run(); - }); + }); + // Update suspected set: only keep nodes that are newly suspected this round + suspectedMissingFromDb.clear(); + suspectedMissingFromDb.addAll(currentSuspected); + + // check if any node missing in our hash ring + nodesInDb.forEach(n -> { + if (n.getUuid().equals(node().getUuid()) || suspects.contains(n)) { + return; + } + + new Runnable() { + @Override + @AsyncThread + public void run() { + nodeLifeCycle.nodeJoin(ManagementNodeInventory.valueOf(n)); + } + }.run(); + }); + } } @Override From d2b599dd5475d773a4235490eef93c35a1679a11 Mon Sep 17 00:00:00 2001 From: "yaohua.wu" Date: Fri, 3 Jul 2026 11:57:16 +0800 Subject: [PATCH 10/24] [externalStorage]: prevent split-brain on deactivate fail Backport external-storage split-brain guards to 5.4.10. 1. Why? Starting a VM on a new host may continue even when the old storage client still holds the volume. For xinfini Vhost volumes this can leave two QEMU processes using the same bdev. 2. How? Report xinfini deactivate failures instead of swallowing them, verify the bdev is really deleted, re-check active clients after deactivate success, and make blacklist failures propagate out of beforeStartVmOnKvm. 3. Side effects? Deactivate failure or a still-active old client now aborts VM start for that volume instead of silently continuing. Happy-path deactivate keeps the previous behavior. # Summary of changes (by module): - header: make PrimaryStorageNodeSvc.blacklist synchronous. - externalStorage: post-check active clients and propagate blacklist failures. - xinfini: surface deactivate/delete failures and abort unsupported isolation. - expon/zbs: adapt blacklist implementations to the synchronous signature. Related: ZSTAC-86444 Related: ZSTAC-85124 Change-Id: I59812c6d68eec6533e48a8c9cf05b7c55d6b85eb --- .../addon/primary/PrimaryStorageNodeSvc.java | 2 +- .../zstack/expon/ExponStorageController.java | 3 +- .../kvm/ExternalPrimaryStorageKvmFactory.java | 18 +++++-- .../org/zstack/xinfini/XInfiniApiHelper.java | 5 ++ .../xinfini/XInfiniStorageController.java | 53 +++++++++++++------ .../storage/zbs/ZbsStorageController.java | 3 +- 6 files changed, 61 insertions(+), 23 deletions(-) diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java index f292e694183..2ab0bcb4653 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageNodeSvc.java @@ -13,7 +13,7 @@ public interface PrimaryStorageNodeSvc { void deactivate(String installPath, String protocol, ActiveVolumeClient client, Completion comp); - void blacklist(String installPath, String protocol, HostInventory h, Completion comp); + void blacklist(String installPath, String protocol, HostInventory h); String getActivePath(BaseVolumeInfo v, HostInventory h, boolean shareable); BaseVolumeInfo getActiveVolumeInfo(String activePath, HostInventory h, boolean shareable); diff --git a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java index 2a3762f98ff..8cb331a91de 100644 --- a/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java +++ b/plugin/expon/src/main/java/org/zstack/expon/ExponStorageController.java @@ -614,14 +614,13 @@ public void cleanActiveRecord(ImageCacheInventory cache) { } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { + public void blacklist(String installPath, String protocol, HostInventory h) { logger.debug(String.format("blacklisting volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); UssGatewayModule uss = getUssGateway(VolumeProtocol.valueOf(protocol), h.getManagementIp()); VolumeModule exponVol = apiHelper.getVolume(getVolIdFromPath(installPath)); apiHelper.addVolumePathToBlacklist(buildExponVolumeBoundPath(uss, exponVol.getVolumeName())); - comp.success(); } @Override diff --git a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java index c6eb04a8557..90aca223f1e 100644 --- a/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java +++ b/plugin/externalStorage/src/main/java/org/zstack/externalStorage/primary/kvm/ExternalPrimaryStorageKvmFactory.java @@ -334,6 +334,7 @@ public void done(ErrorCodeList errorCodeList) { @Override public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { List vols = getManagerExclusiveVolume(spec); + ErrorCode[] deactivateErr = new ErrorCode[1]; for (VolumeInventory vol : vols) { PrimaryStorageNodeSvc nodeSvc = extPsFactory.getNodeSvc(vol.getPrimaryStorageUuid()); @@ -359,20 +360,31 @@ public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAg clientHost.getUuid(), clientHost.getManagementIp(), host.getUuid(), host.getManagementIp())); + deactivateErr[0] = null; nodeSvc.deactivate(vol.getInstallPath(), vol.getProtocol(), client, new Completion(null) { @Override public void success() { + boolean stillActive = nodeSvc.getActiveClients(vol.getInstallPath(), vol.getProtocol()).stream() + .anyMatch(c -> client.getManagerIp().equals(c.getManagerIp())); + if (stillActive) { + deactivateErr[0] = operr("deactivate reported success but volume[uuid:%s, installPath:%s] is still active on host[uuid:%s, ip:%s]", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp()); + return; + } logger.info(String.format("successfully deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]", vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); } @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s], add it to blacklist", - vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp())); - nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost), new NopeCompletion()); + deactivateErr[0] = errorCode; } }); + if (deactivateErr[0] != null) { + logger.warn(String.format("failed to deactivate volume[uuid:%s, installPath:%s] on host[uuid:%s, ip:%s]: %s, add it to blacklist", + vol.getUuid(), vol.getInstallPath(), clientHost.getUuid(), clientHost.getManagementIp(), deactivateErr[0].getDetails())); + nodeSvc.blacklist(vol.getInstallPath(), vol.getProtocol(), HostInventory.valueOf(clientHost)); + } } } }); diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java index 3a90db3b043..2177283b48b 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniApiHelper.java @@ -472,6 +472,11 @@ public void deleteBdcBdev(int bdevId, int bdcId) { } else { retryUtilResourceDeleted(gReq, GetBdcBdevResponse.class); } + + if (!call(gReq, GetBdcBdevResponse.class).resourceIsDeleted()) { + throw new OperationFailureException(operr("bdev[id:%s] still exists after deletion polling on bdc[id:%s], " + + "the old storage client may still hold the volume", bdevId, bdcId)); + } } public VolumeModule rollbackSnapshot(int volId, int snapId) { diff --git a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java index 267d55f4039..53c8cca1751 100644 --- a/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java +++ b/plugin/xinfini/src/main/java/org/zstack/xinfini/XInfiniStorageController.java @@ -202,18 +202,23 @@ public synchronized ActiveVolumeTO activeIscsiVolume(String clientIp, String cli public void deactivate(String installPath, String protocol, HostInventory h, Completion comp) { logger.debug(String.format("deactivating volume[path: %s, protocol:%s] on host[uuid:%s, ip:%s]", installPath, protocol, h.getUuid(), h.getManagementIp())); - if (VolumeProtocol.Vhost.toString().equals(protocol)) { - deactivateVhost(installPath, h); - comp.success(); - return; - } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { - // iscsi target is shared by all hosts, we cannot control one volume on one host for now. - deactivateIscsi(installPath, h); - comp.success(); + try { + if (VolumeProtocol.Vhost.toString().equals(protocol)) { + deactivateVhost(installPath, h); + } else if (VolumeProtocol.iSCSI.toString().equals(protocol)) { + // iscsi target is shared by all hosts, we cannot control one volume on one host for now. + deactivateIscsi(installPath, h); + } else { + comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + return; + } + } catch (Exception e) { + comp.fail(operr("failed to deactivate volume[path:%s, protocol:%s] on host[uuid:%s, ip:%s]: %s", + installPath, protocol, h.getUuid(), h.getManagementIp(), e.getMessage())); return; } - comp.fail(operr("not supported protocol[%s] for deactivate", protocol)); + comp.success(); } private void deactivateIscsi(String installPath, HostInventory h) { @@ -255,7 +260,7 @@ private void deactivateVhost(String installPath, HostInventory h) { return; } - retry(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); + retryOrThrow(() -> apiHelper.deleteBdcBdev(bdev.getSpec().getId(), bdc.getSpec().getId())); } @Override @@ -271,9 +276,10 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - // todo - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { + throw new OperationFailureException(operr("xinfini does not support volume path isolation yet, " + + "abort starting VM on host[uuid:%s, ip:%s] to prevent split-brain on volume[path:%s, protocol:%s]", + h.getUuid(), h.getManagementIp(), installPath, protocol)); } @Override @@ -1038,16 +1044,33 @@ private void retry(Runnable r) { private void retry(Runnable r, int retry) { - while (retry-- > 0) { + try { + retryOrThrow(r, retry); + } catch (RuntimeException ignored) { + } + } + + private void retryOrThrow(Runnable r) { + retryOrThrow(r, 3); + } + + private void retryOrThrow(Runnable r, int retry) { + RuntimeException lastError = null; + for (int i = 0; i < retry; i++) { try { r.run(); return; - } catch (Exception e) { + } catch (RuntimeException e) { + lastError = e; logger.warn("runnable failed, try ", e); try { TimeUnit.SECONDS.sleep(3); } catch (InterruptedException ignore) {} } } + + if (lastError != null) { + throw lastError; + } } } diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 4b725d7d952..faddfdbd5da 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -145,8 +145,7 @@ public void deactivate(String installPath, String protocol, ActiveVolumeClient c } @Override - public void blacklist(String installPath, String protocol, HostInventory h, Completion comp) { - comp.success(); + public void blacklist(String installPath, String protocol, HostInventory h) { } @Override From b0ed121d80359c291fdfcecbfc476746ec45ae61 Mon Sep 17 00:00:00 2001 From: "yaohua.wu" Date: Fri, 3 Jul 2026 12:00:55 +0800 Subject: [PATCH 11/24] [kvm]: defer orphaned skip tracing cleanup Backport VM HA skip-trace protection to 5.4.10. 1. Why? When a management node leaves, its VM skip-trace entries are removed immediately. If those VMs are still being started by kvmagent, the next VM sync may treat them as stopped and trigger HA, causing split-brain. 2. How? Move departed management node skip entries into an orphaned set, keep them skipped for a configurable TTL, include them in VM sync skip lists, and remove expired entries with compare-and-remove cleanup. 3. Side effects? VMs whose controller MN departed stay skipped until the TTL expires. The default TTL is 600 seconds and can be changed by global config `kvm.vm.orphanedSkipTimeout`. # Summary of changes (by module): - kvm: add orphaned skip set, TTL global config, sync inclusion and CAS cleanup. Related: ZSTAC-84636 Related: ZSTAC-80821 Change-Id: I7862360c7318cedb51e38939467cf2a7836e61d0 --- .../java/org/zstack/kvm/KVMGlobalConfig.java | 4 ++ .../org/zstack/kvm/KvmVmSyncPingTask.java | 51 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index aa0fd787bed..4b1d393de48 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -120,6 +120,10 @@ public class KVMGlobalConfig { @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable install host shutdown hook") public static GlobalConfig INSTALL_HOST_SHUTDOWN_HOOK = new GlobalConfig(CATEGORY, "install.host.shutdown.hook"); + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "600", type = Long.class, description = "timeout in seconds for orphaned VM skip entries from departed management nodes") + public static GlobalConfig ORPHANED_VM_SKIP_TIMEOUT = new GlobalConfig(CATEGORY, "vm.orphanedSkipTimeout"); + @GlobalConfigValidation(validValues = {"true", "false"}) @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable memory auto balloon") @BindResourceConfig({VmInstanceVO.class, HostVO.class, ClusterVO.class}) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java index af1a8718800..0f3ceeae74e 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmVmSyncPingTask.java @@ -67,6 +67,11 @@ public class KvmVmSyncPingTask extends VmTracer implements KVMPingAgentNoFailure private List> skipVmTracerMessages = new ArrayList<>(); private List skipVmTracerReplies = new ArrayList<>(); private Map vmInShutdownMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap orphanedSkipVms = new ConcurrentHashMap<>(); + + private long getOrphanTtlMs() { + return KVMGlobalConfig.ORPHANED_VM_SKIP_TIMEOUT.value(Long.class) * 1000; + } { getReflections().getTypesAnnotatedWith(SkipVmTracer.class).forEach(clz -> { @@ -195,8 +200,11 @@ private void syncVm(final HostInventory host, final Completion completion) { // Get vms to skip before send command to host to confirm the vm will be skipped after sync command finished. // The problem is if one vm-sync skipped operation is started and finished during vm sync command's handling // vm state would still be sync to mn + cleanupExpiredOrphanedSkipVms(); + Set vmsToSkipSetHostSide = new HashSet<>(); vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); // if the vm is not running on host when sync command executing but started as soon as possible // before response handling of vm sync, mgmtSideStates will including the running vm but not result in @@ -227,6 +235,7 @@ public void run(MessageReply reply) { // Get vms to skip after sync result returned. vmsToSkip.values().forEach(vmsToSkipSetHostSide::addAll); + vmsToSkipSetHostSide.addAll(orphanedSkipVms.keySet()); Collection vmUuidsInDeleteVmGC = DeleteVmGC.queryVmInGC(host.getUuid(), ret.getStates().keySet()); @@ -445,7 +454,15 @@ public void nodeJoin(ManagementNodeInventory inv) { @Override public void nodeLeft(ManagementNodeInventory inv) { vmApis.remove(inv.getUuid()); - vmsToSkip.remove(inv.getUuid()); + Set skippedVms = vmsToSkip.remove(inv.getUuid()); + if (skippedVms != null && !skippedVms.isEmpty()) { + long now = System.currentTimeMillis(); + for (String vmUuid : skippedVms) { + orphanedSkipVms.put(vmUuid, now); + logger.info(String.format("moved VM[uuid:%s] from departed MN[uuid:%s] skip list to orphaned set" + + " (will expire in %d minutes)", vmUuid, inv.getUuid(), getOrphanTtlMs() / 60000)); + } + } } @Override @@ -459,6 +476,36 @@ public void iJoin(ManagementNodeInventory inv) { } public boolean isVmDoNotNeedToTrace(String vmUuid) { - return vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid)); + if (vmsToSkip.values().stream().anyMatch(vmsToSkipSet -> vmsToSkipSet.contains(vmUuid))) { + return true; + } + + Long orphanedAt = orphanedSkipVms.get(vmUuid); + if (orphanedAt != null) { + if (System.currentTimeMillis() - orphanedAt < getOrphanTtlMs()) { + logger.debug(String.format("VM[uuid:%s] is in orphaned skip set, skipping trace", vmUuid)); + return true; + } + + orphanedSkipVms.remove(vmUuid, orphanedAt); + logger.info(String.format("orphaned skip entry for VM[uuid:%s] expired after %d minutes, resuming trace", + vmUuid, getOrphanTtlMs() / 60000)); + } + + return false; + } + + private void cleanupExpiredOrphanedSkipVms() { + if (orphanedSkipVms.isEmpty()) { + return; + } + + long now = System.currentTimeMillis(); + for (Map.Entry entry : orphanedSkipVms.entrySet()) { + if (now - entry.getValue() >= getOrphanTtlMs()) { + orphanedSkipVms.remove(entry.getKey(), entry.getValue()); + logger.info(String.format("cleaned up expired orphaned skip entry for VM[uuid:%s]", entry.getKey())); + } + } } } From dadf250fdfcab78f53d974631f16bb5133d844b7 Mon Sep 17 00:00:00 2001 From: Jiajun Xu Date: Thu, 2 Jul 2026 17:22:15 +0800 Subject: [PATCH 12/24] [kvm]: disable arm vm pmu by default Backport PMU-off handling from ZSTAC-76375 to 5.4.10 for Kunpeng openEuler guest panic. Resolves: ZSTAC-86338 Change-Id: I3c2dbe1944c300a803f50198b4c7ba575bd77dcc --- .../org/zstack/compute/vm/VmGlobalConfig.java | 4 + .../java/org/zstack/kvm/KVMAgentCommands.java | 10 + .../src/main/java/org/zstack/kvm/KVMHost.java | 4 + .../integration/kvm/vm/VmPmuConfigCase.groovy | 223 ++++++++++++++++++ 4 files changed, 241 insertions(+) create mode 100644 test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy diff --git a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java index bd79900c13c..b42e5a32edd 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java @@ -52,6 +52,10 @@ public class VmGlobalConfig { @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class}) public static GlobalConfig VM_PORT_OFF = new GlobalConfig(CATEGORY, "vmPortOff"); + @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable PMU for VM, disabled by default on aarch64 to avoid Kunpeng-920 kernel panic") + @GlobalConfigValidation(validValues = {"true", "false"}) + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) + public static GlobalConfig VM_PMU = new GlobalConfig(CATEGORY, "vm.pmu"); @GlobalConfigValidation(validValues = {"true", "false"}) @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) public static GlobalConfig EMULATE_HYPERV = new GlobalConfig(CATEGORY, "emulateHyperV"); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 91e5d885bd8..2eb30e5040a 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -2282,6 +2282,8 @@ public static class StartVmCmd extends vdiCmd implements VmAddOnsCmd { @GrayVersion(value = "5.0.0") private boolean acpi; @GrayVersion(value = "5.0.0") + private boolean pmu = true; + @GrayVersion(value = "5.0.0") private boolean x2apic = true; // cpuid hypervisor feature @GrayVersion(value = "5.0.0") @@ -2789,6 +2791,14 @@ public void setAcpi(boolean acpi) { this.acpi = acpi; } + public boolean isPmu() { + return pmu; + } + + public void setPmu(boolean pmu) { + this.pmu = pmu; + } + public boolean getX2apic() { return x2apic; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index cd58e7b58e9..cf62a6396cb 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -4547,6 +4547,10 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi if (spec.isCreatePaused()) { cmd.setCreatePaused(true); } + if (ImageArchitecture.aarch64.toString().equals(architecture)) { + Boolean pmuEnabled = rcf.getResourceConfigValue(VmGlobalConfig.VM_PMU, spec.getVmInventory().getUuid(), Boolean.class); + cmd.setPmu(Boolean.TRUE.equals(pmuEnabled)); + } cmd.setAcpi(true); GuestOsCharacter.Config config = GuestOsHelper.getInstance().getGuestOsCharacter( diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy new file mode 100644 index 00000000000..0340abb3109 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/vm/VmPmuConfigCase.groovy @@ -0,0 +1,223 @@ +package org.zstack.test.integration.kvm.vm + +import org.springframework.http.HttpEntity +import org.zstack.compute.vm.VmGlobalConfig +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.kvm.KVMAgentCommands +import org.zstack.kvm.KVMConstant +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.flat.FlatNetworkServiceConstant +import org.zstack.network.service.userdata.UserdataConstant +import org.zstack.sdk.GlobalConfigInventory +import org.zstack.sdk.ImageInventory +import org.zstack.sdk.InstanceOfferingInventory +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.VmInstanceInventory +import org.zstack.test.integration.kvm.KvmTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +class VmPmuConfigCase extends SubCase { + EnvSpec env + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(KvmTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(2) + cpu = 1 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image_x86" + architecture = "x86_64" + url = "http://zstack.org/download/test.qcow2" + } + + image { + name = "image_arm64" + architecture = "aarch64" + url = "http://zstack.org/download/test-arm.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster_x86" + hypervisorType = "KVM" + architecture = "x86_64" + + kvm { + name = "kvm_x86" + managementIp = "localhost" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + cluster { + name = "cluster_arm64" + hypervisorType = "KVM" + architecture = "aarch64" + + kvm { + name = "kvm_arm64" + managementIp = "127.0.0.3" + username = "root" + password = "password" + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + + attachBackupStorage("sftp") + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + + service { + provider = FlatNetworkServiceConstant.FLAT_NETWORK_SERVICE_TYPE_STRING + types = [NetworkServiceType.DHCP.toString(), EipConstant.EIP_NETWORK_SERVICE_TYPE, UserdataConstant.USERDATA_TYPE_STRING] + } + + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + } + } + } + + @Override + void test() { + env.create() + testPmuGlobalConfigExists() + testX86VmKeepsPmuEnabled() + testAarch64VmDisablesPmuByDefault() + testAarch64VmPmuResourceConfigOverride() + } + + void testPmuGlobalConfigExists() { + def configs = queryGlobalConfig { + conditions = ["category=${VmGlobalConfig.CATEGORY}", "name=${VmGlobalConfig.VM_PMU.name}"] + } + + assert configs.size() == 1 + assert (configs[0] as GlobalConfigInventory).defaultValue == "false" + } + + void testX86VmKeepsPmuEnabled() { + ImageInventory image = env.inventoryByName("image_x86") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-x86" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert cmd.pmu + } + + void testAarch64VmDisablesPmuByDefault() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + createVmInstance { + name = "pmu-arm-default" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } + } + + assert !cmd.pmu + } + + void testAarch64VmPmuResourceConfigOverride() { + ImageInventory image = env.inventoryByName("image_arm64") + InstanceOfferingInventory instanceOffering = env.inventoryByName("instanceOffering") + L3NetworkInventory l3 = env.inventoryByName("l3") + + VmInstanceInventory vm = createVmInstance { + name = "pmu-arm-override" + imageUuid = image.uuid + instanceOfferingUuid = instanceOffering.uuid + l3NetworkUuids = [l3.uuid] + } as VmInstanceInventory + + updateResourceConfig { + category = VmGlobalConfig.CATEGORY + name = VmGlobalConfig.VM_PMU.name + resourceUuid = vm.uuid + value = "true" + } + + KVMAgentCommands.StartVmCmd cmd = captureStartVmCmd { + rebootVmInstance { + uuid = vm.uuid + } + } + + assert cmd.pmu + } + + KVMAgentCommands.StartVmCmd captureStartVmCmd(Closure trigger) { + KVMAgentCommands.StartVmCmd startCmd = null + env.afterSimulator(KVMConstant.KVM_START_VM_PATH) { rsp, HttpEntity e -> + startCmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.StartVmCmd.class) + return rsp + } + + trigger.call() + assert startCmd != null + env.cleanAfterSimulatorHandlers() + return startCmd + } +} From 42a035de901ec07fffa166d932cda7c9094872c1 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 13 Feb 2026 12:40:04 +0800 Subject: [PATCH 13/24] [loadBalancer]: block SLB deletion during grayscale upgrade Resolves: ZSTAC-78989 Change-Id: I0fe3a56ab724978944c69afadaab7ff7353e4c0f (cherry picked from commit f19223a6e72678f20610ae04226fc20c0cf1bb5b) --- .../service/lb/LoadBalancerApiInterceptor.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ea1168da036..72af77c1070 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -39,6 +39,7 @@ import org.zstack.network.service.vip.VipVO_; import org.zstack.tag.PatternedSystemTag; import org.zstack.tag.TagManager; +import org.zstack.core.upgrade.UpgradeGlobalConfig; import org.zstack.utils.*; import org.zstack.utils.function.ForEachFunction; import org.zstack.utils.logging.CLogger; @@ -151,10 +152,22 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APIGetCandidateVmNicsForLoadBalancerServerGroupMsg)msg); } else if (msg instanceof APIChangeLoadBalancerBackendServerMsg) { validate((APIChangeLoadBalancerBackendServerMsg)msg); + } else if (msg instanceof APIDeleteLoadBalancerMsg) { + validate((APIDeleteLoadBalancerMsg) msg); } return msg; } + private void validate(APIDeleteLoadBalancerMsg msg) { + if (UpgradeGlobalConfig.GRAYSCALE_UPGRADE.value(Boolean.class)) { + LoadBalancerVO lb = dbf.findByUuid(msg.getUuid(), LoadBalancerVO.class); + if (lb != null && lb.getType() == LoadBalancerType.SLB) { + throw new ApiMessageInterceptionException(argerr( + "cannot delete the standalone load balancer[uuid:%s] during grayscale upgrade", msg.getUuid())); + } + } + } + private void validate(APIDeleteAccessControlListMsg msg) { /*List refs = Q.New(LoadBalancerListenerACLRefVO.class).select(LoadBalancerListenerACLRefVO_.listenerUuid) .eq(LoadBalancerListenerACLRefVO_.aclUuid, msg.getUuid()).isNull(LoadBalancerListenerACLRefVO_.serverGroupUuid).listValues(); From 3079c287c63433e57d8ea4614be375a435f008f1 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 4 Jun 2026 14:59:56 +0800 Subject: [PATCH 14/24] [virtualrouter]: configure dnsmasq query limits Pass dns-forward-max and cache-size through VR DNS commands. Render both options in zstack-vyos dnsmasq config. Resolves: ZSTAC-85197 Change-Id: Ib78504edc564a2ab7dbd39bcb2e0ee396d019240 (cherry picked from commit 4ada5a52ee58d17b238ea1e22abb3609d6d47104) --- conf/globalConfig/virutalRouter.xml | 14 ++++++++++++++ .../virtualrouter/VirtualRouterGlobalConfig.java | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/conf/globalConfig/virutalRouter.xml b/conf/globalConfig/virutalRouter.xml index 36e3b30ef2c..e084e3f925b 100755 --- a/conf/globalConfig/virutalRouter.xml +++ b/conf/globalConfig/virutalRouter.xml @@ -21,6 +21,20 @@ virtualRouter java.lang.Integer + + dnsmasq.dnsForwardMax + The maximum number of concurrent DNS queries forwarded by dnsmasq on a virtual router. + 1000 + virtualRouter + java.lang.Integer + + + dnsmasq.cacheSize + The DNS cache size used by dnsmasq on a virtual router. + 10000 + virtualRouter + java.lang.Integer + ping.interval The interval management nodes ping the virtual router agents running virtual router VMs, in seconds diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java index 3b8d87bb8f0..7b38e6cc723 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterGlobalConfig.java @@ -18,6 +18,12 @@ public class VirtualRouterGlobalConfig { public static GlobalConfig COMMANDS_PARALELLISM_DEGREE = new GlobalConfig(CATEGORY, "command.parallelismDegree"); @GlobalConfigValidation public static GlobalConfig RESTART_DNSMASQ_COUNT = new GlobalConfig(CATEGORY, "dnsmasq.restartAfterNumberOfSIGUSER1"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_DNS_FORWARD_MAX = new GlobalConfig(CATEGORY, "dnsmasq.dnsForwardMax"); + @GlobalConfigValidation(numberGreaterThan = 0) + @BindResourceConfig({VmInstanceVO.class}) + public static GlobalConfig DNSMASQ_CACHE_SIZE = new GlobalConfig(CATEGORY, "dnsmasq.cacheSize"); @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig PING_INTERVAL = new GlobalConfig(CATEGORY, "ping.interval"); @GlobalConfigValidation From c9fbad0fff54cc52d39274d0a4e0826968b9505e Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 21 May 2026 21:17:17 +0800 Subject: [PATCH 15/24] [sugonSdn]: fix port sync plugin declaration Change-Id: If07dce918c4d7d9f62445184ff5984e30b83ce64 (cherry picked from commit 893999410fc56d1518b076d3c6a74880acaa4aa6) --- conf/springConfigXml/sugonSdnController.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/springConfigXml/sugonSdnController.xml b/conf/springConfigXml/sugonSdnController.xml index f0ff6ad0033..c55e6da4903 100644 --- a/conf/springConfigXml/sugonSdnController.xml +++ b/conf/springConfigXml/sugonSdnController.xml @@ -67,11 +67,7 @@ - - - - - + From 43e3374b4852e82bdbb7e98895127c697da837d7 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Thu, 21 May 2026 12:21:15 +0900 Subject: [PATCH 16/24] [sugonSdn]: sync tf ports after ping Trigger TF port sync from TF controller ping success instead of MN startup. Keep the original one day interval and port cleanup algorithm, while scoping sync to the current controller. Resolves: ZSTAC-85281 Change-Id: I84a88a34135cd47fc79b3019d2f0fc2953f06ac9 (cherry picked from commit e44d41a0f130d4ccccef1a0e27e511a469765a01) --- .../controller/SugonSdnController.java | 49 ++++++++++++- .../neutronClient/TfPortClient.java | 17 ++++- .../network/TfPortService.java | 11 +++ .../network/TfZstackPortSync.java | 69 ++++++++++++------- .../SugonSdnControllerCase.groovy | 56 +++++++++++++++ 5 files changed, 177 insertions(+), 25 deletions(-) diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 45a68537e38..57ac4406908 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -22,10 +22,13 @@ import org.zstack.network.l3.L3NetworkSystemTags; import org.zstack.sdnController.SdnController; import org.zstack.sdnController.SdnControllerL2; +import org.zstack.sdnController.SdnControllerPingMsg; +import org.zstack.sdnController.SdnControllerPingReply; import org.zstack.sdnController.header.*; import org.zstack.sugonSdnController.controller.api.*; import org.zstack.sugonSdnController.controller.api.types.*; import org.zstack.sugonSdnController.header.APICreateL2TfNetworkMsg; +import org.zstack.sugonSdnController.network.TfZstackPortSync; import org.zstack.utils.StringDSL; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; @@ -44,6 +47,8 @@ public class SugonSdnController implements TfSdnController, SdnController, SdnCo CloudBus bus; @Autowired DatabaseFacade dbf; + @Autowired + TfZstackPortSync tfZstackPortSync; private SdnControllerVO sdnControllerVO; private TfHttpClient client; @@ -56,7 +61,49 @@ public SugonSdnController(SdnControllerVO vo) { @Override public void handleMessage(SdnControllerMessage msg) { - bus.dealWithUnknownMessage((Message) msg); + if (msg instanceof SdnControllerPingMsg) { + handMessage((SdnControllerPingMsg) msg); + } else if (msg instanceof SdnControllerEnableDHCPMsg) { + handMessage((SdnControllerEnableDHCPMsg) msg); + } else if (msg instanceof SdnControllerDisableDHCPMsg) { + handMessage((SdnControllerDisableDHCPMsg) msg); + } else if (msg instanceof SdnControllerUpdateDHCPMsg) { + handMessage((SdnControllerUpdateDHCPMsg) msg); + } else { + bus.dealWithUnknownMessage((Message) msg); + } + } + + void handMessage(SdnControllerEnableDHCPMsg msg) { + SdnControllerEnableDHCPReply reply = new SdnControllerEnableDHCPReply(); + bus.reply(msg, reply); + } + + void handMessage(SdnControllerDisableDHCPMsg msg) { + SdnControllerDisableDHCPReply reply = new SdnControllerDisableDHCPReply(); + bus.reply(msg, reply); + } + + void handMessage(SdnControllerUpdateDHCPMsg msg) { + SdnControllerUpdateDHCPReply reply = new SdnControllerUpdateDHCPReply(); + bus.reply(msg, reply); + } + + void handMessage(SdnControllerPingMsg msg) { + SdnControllerPingReply reply = new SdnControllerPingReply(); + try { + Domain domain = (Domain) client.getDomain(); + if (domain == null) { + reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + } else { + tfZstackPortSync.triggerSyncIfDue(msg.getSdnControllerUuid()); + reply.setSuccess(true); + } + } catch (Exception e) { + logger.warn("ping tf sdn controller failed", e); + reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + } + bus.reply(msg, reply); } @Override diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java index 1054f708696..2c531684d1b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java @@ -32,6 +32,22 @@ public TfPortClient(){ if (sdn == null){ throw new RuntimeException("Can not find a tf sdn controller."); } + init(sdn); + } + + public TfPortClient(String sdnControllerUuid) { + SdnControllerVO sdn = Q.New(SdnControllerVO.class).eq( + SdnControllerVO_.uuid, sdnControllerUuid).find(); + if (sdn == null) { + throw new RuntimeException(String.format("Can not find a tf sdn controller[uuid:%s].", sdnControllerUuid)); + } + if (!Objects.equals(sdn.getVendorType(), SdnControllerConstant.TF_CONTROLLER)) { + throw new RuntimeException(String.format("Sdn controller[uuid:%s] is not a tf sdn controller.", sdnControllerUuid)); + } + init(sdn); + } + + private void init(SdnControllerVO sdn) { client = new TfHttpClient(sdn.getIp()); tenantId = StringDSL.transToTfUuid(sdn.getAccountUuid()); } @@ -509,4 +525,3 @@ public Status updateTfPort(String tfPortUUid, String accountId, String deviceId, } } - diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java index c4767689bd1..e7a9f0c0419 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfPortService.java @@ -33,6 +33,11 @@ public List getTfPortsDetail() { return tfPortClient.getVirtualMachineInterfaceDetail(); } + public List getTfPortsDetail(String sdnControllerUuid) { + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.getVirtualMachineInterfaceDetail(); + } + public TfPortResponse createTfPort(String tfPortUUid, String l2NetworkUuid, String l3NetworkUuid, String mac, String ip) { //invoke tf rest interface to retrieve real ip and mac and portId TfPortClient tfPortClient = new TfPortClient(); @@ -115,6 +120,12 @@ public TfPortResponse deleteTfPort(String portUUid) { return tfPortClient.deletePort(tfPortUUid); } + public TfPortResponse deleteTfPort(String sdnControllerUuid, String portUUid) { + String tfPortUUid = StringDSL.transToTfUuid(portUUid); + TfPortClient tfPortClient = new TfPortClient(sdnControllerUuid); + return tfPortClient.deletePort(tfPortUUid); + } + public Status updateTfPort(String portUUid, String bmUuid, KeyValuePairs bindInfo) { String accountId = StringDSL.transToTfUuid(acntMgr.getOwnerAccountUuidOfResource(bmUuid)); String tfBmUUid = StringDSL.transToTfUuid(bmUuid); diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java index 184b2e721ed..710b63cdb2b 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfZstackPortSync.java @@ -2,9 +2,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.zstack.core.db.Q; -import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.Task; import org.zstack.core.thread.ThreadFacade; -import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; import org.zstack.header.network.l2.L2NetworkVO; import org.zstack.header.network.l2.L2NetworkVO_; import org.zstack.header.vm.VmNicVO; @@ -19,39 +18,57 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.concurrent.Future; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -public class TfZstackPortSync implements ManagementNodeReadyExtensionPoint { +public class TfZstackPortSync { + private static final long SYNC_INTERVAL_MILLIS = TimeUnit.DAYS.toMillis(1); + private static final int MAX_DELETE_COUNT = 10; @Autowired protected ThreadFacade thdf; - private Future trackerThread = null; @Autowired private TfPortService tfPortService; private final static CLogger logger = Utils.getLogger(TfZstackPortSync.class); private final List excludeTypes = new ArrayList(Arrays.asList("neutron:LOADBALANCER", "VIP", "BMS")); + private final Map lastSyncTime = new ConcurrentHashMap<>(); + private final Set runningSyncs = Collections.newSetFromMap(new ConcurrentHashMap()); - @Override - public void managementNodeReady() { - if (trackerThread != null) { - trackerThread.cancel(true); + public void triggerSyncIfDue(String sdnControllerUuid) { + if (sdnControllerUuid == null) { + logger.warn("Port_Sync_Task: skip sync because sdn controller uuid is null."); + return; } - trackerThread = thdf.submitPeriodicTask(new SyncPort()); - } - private class SyncPort implements PeriodicTask { + long now = System.currentTimeMillis(); + Long lastSync = lastSyncTime.get(sdnControllerUuid); + if (lastSync != null && now - lastSync < SYNC_INTERVAL_MILLIS) { + return; + } + if (!runningSyncs.add(sdnControllerUuid)) { + return; + } - @Override - public TimeUnit getTimeUnit() { - return TimeUnit.DAYS; + lastSyncTime.put(sdnControllerUuid, now); + try { + thdf.submit(new SyncPort(sdnControllerUuid)); + } catch (RuntimeException e) { + lastSyncTime.remove(sdnControllerUuid); + runningSyncs.remove(sdnControllerUuid); + throw e; } + } - @Override - public long getInterval() { - return 1; + private class SyncPort implements Task { + private final String sdnControllerUuid; + + private SyncPort(String sdnControllerUuid) { + this.sdnControllerUuid = sdnControllerUuid; } @Override @@ -64,7 +81,10 @@ private HashSet getPortToDelete() { List zstackL2NetworksUuid = Q.New(L2NetworkVO.class).select(L2NetworkVO_.uuid).listValues(); List tfPortsUuid = new ArrayList<>(); try{ - List tfPorts = tfPortService.getTfPortsDetail(); + List tfPorts = tfPortService.getTfPortsDetail(sdnControllerUuid); + if (tfPorts == null) { + return new HashSet<>(); + } for (VirtualMachineInterface vmi : tfPorts) { // skip port if it's network not in zstack List> tfNetworks = vmi.getVirtualNetwork(); @@ -83,7 +103,7 @@ private HashSet getPortToDelete() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task: Fetch tf VirtualMachineInterface failed: %s.", e)); - return null; + return new HashSet<>(); } HashSet result = new HashSet<>(tfPortsUuid); result.removeAll(zstackPortsUuid); @@ -92,13 +112,13 @@ private HashSet getPortToDelete() { } @Override - public void run() { + public Void call() { logger.info("Port_Sync_Task: begin."); try { HashSet portsToDelete = getPortToDelete(); - int maxDeleteCount = 10; + int maxDeleteCount = MAX_DELETE_COUNT; for (String portUuid: portsToDelete) { - TfPortResponse response = tfPortService.deleteTfPort(portUuid); + TfPortResponse response = tfPortService.deleteTfPort(sdnControllerUuid, portUuid); if (response.getCode() == 200) { logger.info(String.format("Port_Sync_Task: VirtualMachineInterface: %s delete success.", portUuid)); @@ -113,7 +133,10 @@ public void run() { } } catch (Exception e) { logger.error(String.format("Port_Sync_Task failed: %s.", e)); + } finally { + runningSyncs.remove(sdnControllerUuid); } + return null; } } } diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy index 8e28757ba5a..c11fb00950b 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -1,7 +1,12 @@ package org.zstack.test.integration.network.sdnController +import org.springframework.http.HttpEntity import org.zstack.core.db.DatabaseFacade +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.message.MessageReply import org.zstack.header.network.sdncontroller.SdnControllerVO +import org.zstack.header.network.sdncontroller.SdnControllerConstant +import org.zstack.sdnController.SdnControllerPingMsg import org.zstack.sdk.* import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant import org.zstack.sugonSdnController.controller.api.types.MacAddressesType @@ -16,6 +21,9 @@ import javax.persistence.TypedQuery; import org.springframework.http.ResponseEntity import org.springframework.http.HttpStatus +import javax.servlet.http.HttpServletRequest +import java.util.concurrent.atomic.AtomicInteger + class SugonSdnControllerCase extends SubCase { EnvSpec env @@ -37,6 +45,7 @@ class SugonSdnControllerCase extends SubCase { void test() { env.create { dbf = bean(DatabaseFacade.class) + testTfPortSyncTriggeredByPingSuccess() testTfApi() } } @@ -46,6 +55,53 @@ class SugonSdnControllerCase extends SubCase { env.delete() } + void testTfPortSyncTriggeredByPingSuccess() { + String sql = "select sdn" + + " from SdnControllerVO sdn" + + " where sdn.vendorType = :vendorType"; + TypedQuery q = dbf.getEntityManager().createQuery(sql, SdnControllerVO.class); + q.setParameter("vendorType", SugonSdnControllerConstant.TF_CONTROLLER); + SdnControllerVO sdn = q.getResultList().get(0); + AtomicInteger syncCount = new AtomicInteger(0) + + env.simulator(TfCommands.TF_CREATE_VMI) { HttpServletRequest req, HttpEntity e, EnvSpec spec -> + if (req.method == "GET") { + syncCount.incrementAndGet() + return '{"virtual-machine-interfaces":[]}' + } + + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = TfCommands.TEST_VMI_UUID + rsp.uuid = TfCommands.TEST_VMI_UUID + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.setParent(project) + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + sendTfPing(sdn.uuid) + retryInSecs { + assert syncCount.get() == 1 + } + + sendTfPing(sdn.uuid) + sleep(500) + assert syncCount.get() == 1 + } + + void sendTfPing(String sdnControllerUuid) { + CloudBus bus = bean(CloudBus.class) + SdnControllerPingMsg msg = new SdnControllerPingMsg() + msg.sdnControllerUuid = sdnControllerUuid + bus.makeTargetServiceIdByResourceUuid(msg, SdnControllerConstant.SERVICE_ID, sdnControllerUuid) + MessageReply reply = bus.call(msg) + assert reply.success + } + void testTfApi() { def zone = env.inventoryByName("zone") as ZoneInventory def cluster1 = env.inventoryByName("cluster1") as ClusterInventory From 31c9dbb1f83c8056618a953ff2685cd51998cbef Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Tue, 30 Jun 2026 22:23:23 +0900 Subject: [PATCH 17/24] [sugonSdn]: adapt ping sync backport Remove DHCP message handlers and generated error code reference unavailable on 5.4.10. Change-Id: I1a15fb1eeeab60ca9c6fa239904bda89839b64e5 --- .../controller/SugonSdnController.java | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 57ac4406908..4d9c66f50fc 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -63,45 +63,24 @@ public SugonSdnController(SdnControllerVO vo) { public void handleMessage(SdnControllerMessage msg) { if (msg instanceof SdnControllerPingMsg) { handMessage((SdnControllerPingMsg) msg); - } else if (msg instanceof SdnControllerEnableDHCPMsg) { - handMessage((SdnControllerEnableDHCPMsg) msg); - } else if (msg instanceof SdnControllerDisableDHCPMsg) { - handMessage((SdnControllerDisableDHCPMsg) msg); - } else if (msg instanceof SdnControllerUpdateDHCPMsg) { - handMessage((SdnControllerUpdateDHCPMsg) msg); } else { bus.dealWithUnknownMessage((Message) msg); } } - void handMessage(SdnControllerEnableDHCPMsg msg) { - SdnControllerEnableDHCPReply reply = new SdnControllerEnableDHCPReply(); - bus.reply(msg, reply); - } - - void handMessage(SdnControllerDisableDHCPMsg msg) { - SdnControllerDisableDHCPReply reply = new SdnControllerDisableDHCPReply(); - bus.reply(msg, reply); - } - - void handMessage(SdnControllerUpdateDHCPMsg msg) { - SdnControllerUpdateDHCPReply reply = new SdnControllerUpdateDHCPReply(); - bus.reply(msg, reply); - } - void handMessage(SdnControllerPingMsg msg) { SdnControllerPingReply reply = new SdnControllerPingReply(); try { Domain domain = (Domain) client.getDomain(); if (domain == null) { - reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + reply.setError(operr("get default domain on tf controller failed")); } else { tfZstackPortSync.triggerSyncIfDue(msg.getSdnControllerUuid()); reply.setSuccess(true); } } catch (Exception e) { logger.warn("ping tf sdn controller failed", e); - reply.setError(operr(ORG_ZSTACK_SUGONSDNCONTROLLER_CONTROLLER_10002, "get default domain on tf controller failed")); + reply.setError(operr("get default domain on tf controller failed")); } bus.reply(msg, reply); } From 16667ce8c2759315794b2cfc2c2fc57e0ee41016 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Wed, 1 Jul 2026 09:58:35 +0900 Subject: [PATCH 18/24] [sugonSdn]: address review comments Add MessageSafe to the SDN message entrypoint. Move the second ping after the first async sync window in the focused test. Resolves: ZSTAC-86445 Change-Id: I4d7748a0de3b5d6a052a47966d2265249b2779e2 --- .../sugonSdnController/controller/SugonSdnController.java | 2 ++ .../network/sdnController/SugonSdnControllerCase.groovy | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 4d9c66f50fc..e63d69927c4 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -6,6 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.MessageSafe; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.header.core.Completion; @@ -60,6 +61,7 @@ public SugonSdnController(SdnControllerVO vo) { } @Override + @MessageSafe public void handleMessage(SdnControllerMessage msg) { if (msg instanceof SdnControllerPingMsg) { handMessage((SdnControllerPingMsg) msg); diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy index c11fb00950b..e954020fa11 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -88,8 +88,8 @@ class SugonSdnControllerCase extends SubCase { assert syncCount.get() == 1 } - sendTfPing(sdn.uuid) sleep(500) + sendTfPing(sdn.uuid) assert syncCount.get() == 1 } From 57209eaeaa6ff71c11d3372d764bfe4b19a65fae Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Mon, 22 Dec 2025 19:37:55 +0800 Subject: [PATCH 19/24] [sdnController]: put sdn host crud in seperate queue Resolves: ZSTAC-79353 Change-Id: I63706d727173706e796b64666963776a74736c61 (cherry picked from commit 2ef761d0222ee0b39166d636e190e729ee45966c) --- .../org/zstack/sdnController/SdnControllerBase.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java index f4733f6a833..8ff809fd308 100644 --- a/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java +++ b/plugin/sdnController/src/main/java/org/zstack/sdnController/SdnControllerBase.java @@ -86,6 +86,10 @@ public String getSdnControllerSignature() { return "sdn-controller-" + self.getUuid(); } + public String getSdnControllerHostSignature(String hostUuid) { + return "sdn-controller-" + self.getUuid() + "-host-" + hostUuid; + } + protected SdnController getSdnController() { SdnControllerFactory factory = sdnMgr.getSdnControllerFactory(self.getVendorType()); return factory.getSdnController(self); @@ -427,7 +431,7 @@ private void sdnControllerAddHost(APISdnControllerAddHostMsg msg, Completion com thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -508,7 +512,7 @@ private void sdnControllerRemoveHostInQueue(SdnControllerRemoveHostMsg msg, Comp thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override @@ -594,7 +598,7 @@ private void doControllerChangeHost(APISdnControllerChangeHostMsg msg, Completio thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return getSdnControllerSignature(); + return getSdnControllerHostSignature(msg.getHostUuid()); } @Override From f9777240c2f50b6d42e7249d39fbb0010ceda257 Mon Sep 17 00:00:00 2001 From: "shixin.ruan" Date: Tue, 20 Jan 2026 14:36:43 +0800 Subject: [PATCH 20/24] [virtualRouterProvider]: change vrouter ip_local_port_range Resolves: ZSTAC-81450 Change-Id: I79666f616d64797a747677656762626d6a71786d (cherry picked from commit 5c3fab5568c2647987978ce1b7cb0c41580d4dfb) --- .../service/virtualrouter/vyos/VyosConnectFlow.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java index d890e10ed33..1108c650975 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConnectFlow.java @@ -180,12 +180,9 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setLogLevel(rcf.getResourceConfigValue(VirtualRouterGlobalConfig.LOG_LEVEL, vrUuid, String.class)); Map parms = new HashMap<>(); - long count = Q.New(VirtualRouterLoadBalancerRefVO.class) - .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, vrUuid) - .notNull(VirtualRouterLoadBalancerRefVO_.loadBalancerUuid) - .count(); - if(count > 0){ - parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(),VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); + if (!VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value().equals( + VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.defaultValue(String.class))){ + parms.put(VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.getName(), VirtualRouterGlobalConfig.IPV4_LOCAL_PORT_RANGE.value()); } cmd.setParms(parms); From adf11eb193160523c036b30518856d231696b1d9 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Mon, 6 Jul 2026 14:39:16 +0800 Subject: [PATCH 21/24] [sdk]: update mdev candidates sdk The premium backport adds minAvailableDevicesPerHost to GetMdevDeviceCandidates. Commit the generated SDK action field on the companion zstack branch so update-sdk CI no longer reports an uncommitted SDK diff. Constraint: premium and zstack companion branches must use the same @@2 suffix for cross-repo SDK validation Tested: git diff --check Not-tested: Full ./runMavenProfile sdk locally Resolves: ZSTAC-86497 Change-Id: Iae56b01a4c533eeba05d10358d4944f655c5a6d4 --- .../java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java index 8f98f870348..cd6fae86322 100644 --- a/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java +++ b/sdk/src/main/java/org/zstack/sdk/GetMdevDeviceCandidatesAction.java @@ -37,6 +37,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List types; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Integer minAvailableDevicesPerHost; + @Param(required = false) public java.util.List systemTags; From fc5e3818e11e3182115fc26c4dbfe2477338d431 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Tue, 19 May 2026 00:52:45 +0800 Subject: [PATCH 22/24] [ha]: stabilize suspect host pre-fence flow Move premium-only HA pre-fence details out of community while keeping the generic async before-start extension point support. Jira: ZSTAC-83890 Test: mvn -pl header,compute,plugin/kvm,simulator/simulatorImpl,testlib -DskipTests compile Change-Id: I168adf82338f9df9e76287619b7f76a8e5be695f --- .../org/zstack/compute/vm/VmInstanceBase.java | 1 - .../compute/vm/VmStartOnHypervisorFlow.java | 87 ++++++++++++++----- .../header/vm/HaStartVmInstanceMsg.java | 9 ++ ...BeforeStartOnHypervisorExtensionPoint.java | 9 +- 4 files changed, 82 insertions(+), 24 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index ed98aef34e0..d77842b8099 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -8984,4 +8984,3 @@ public void run(MessageReply reply) { }); } } - diff --git a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java index 20965f61655..bf44adeb836 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmStartOnHypervisorFlow.java @@ -5,17 +5,26 @@ import org.springframework.beans.factory.annotation.Configurable; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.asyncbatch.While; import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.workflow.FlowChainBuilder; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowChain; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; import org.zstack.header.core.workflow.FlowRollback; import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.host.HostConstant; import org.zstack.header.message.MessageReply; import org.zstack.header.vm.*; import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; -import java.util.List; import java.util.Map; @Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) public class VmStartOnHypervisorFlow implements Flow { @@ -26,34 +35,68 @@ public class VmStartOnHypervisorFlow implements Flow { @Autowired private PluginRegistry pluginRgty; - private final List exts = pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class);; - - private void fireExtensions(VmInstanceSpec spec) { - for (VmBeforeStartOnHypervisorExtensionPoint ext : exts) { - ext.beforeStartVmOnHypervisor(spec); - } - } - @Override public void run(final FlowTrigger chain, final Map data) { final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); + FlowChain fchain = FlowChainBuilder.newSimpleFlowChain(); + fchain.setName(String.format("vm-start-on-hypervisor-vm-%s", spec.getVmInventory().getUuid())); + fchain.then(new NoRollbackFlow() { + @Override + public void run(FlowTrigger trigger, Map d) { + new While<>(pluginRgty.getExtensionList(VmBeforeStartOnHypervisorExtensionPoint.class)) + .each((ext, comp) -> ext.beforeStartVmOnHypervisor(spec, new Completion(comp) { + @Override + public void success() { + comp.done(); + } - fireExtensions(spec); - - StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); - msg.setVmSpec(spec); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); - bus.send(msg, new CloudBusCallBack(chain) { + @Override + public void fail(ErrorCode errorCode) { + comp.addError(errorCode); + comp.done(); + } + })).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().size() > 0) { + trigger.fail(errorCodeList.getCauses().get(0)); + } else { + trigger.next(); + } + } + }); + } + }); + fchain.then(new NoRollbackFlow() { @Override - public void run(MessageReply reply) { - if (reply.isSuccess()) { - data.put(VmStartOnHypervisorFlow.class.getName(), true); - chain.next(); - } else { - chain.fail(reply.getError()); - } + public void run(FlowTrigger trigger, Map d) { + StartVmOnHypervisorMsg msg = new StartVmOnHypervisorMsg(); + msg.setVmSpec(spec); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, spec.getDestHost().getUuid()); + bus.send(msg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + data.put(VmStartOnHypervisorFlow.class.getName(), true); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); } }); + fchain.done(new FlowDoneHandler(chain) { + @Override + public void handle(Map d) { + chain.next(); + } + }).error(new FlowErrorHandler(chain) { + @Override + public void handle(ErrorCode errCode, Map d) { + chain.fail(errCode); + } + }).start(); } @Override diff --git a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java index a358249cdf9..3ef49037137 100644 --- a/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/HaStartVmInstanceMsg.java @@ -12,6 +12,7 @@ public class HaStartVmInstanceMsg extends NeedReplyMessage implements VmInstance private String vmInstanceUuid; private String judgerClassName; private List softAvoidHostUuids; + private String accessiblePeerHostUuid; private String haReason; public String getJudgerClassName() { @@ -30,6 +31,14 @@ public void setSoftAvoidHostUuids(List softAvoidHostUuids) { this.softAvoidHostUuids = softAvoidHostUuids; } + public String getAccessiblePeerHostUuid() { + return accessiblePeerHostUuid; + } + + public void setAccessiblePeerHostUuid(String accessiblePeerHostUuid) { + this.accessiblePeerHostUuid = accessiblePeerHostUuid; + } + @Override public String getVmInstanceUuid() { return vmInstanceUuid; diff --git a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java index c56d99d38dd..1b3a393496f 100755 --- a/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/vm/VmBeforeStartOnHypervisorExtensionPoint.java @@ -1,7 +1,14 @@ package org.zstack.header.vm; +import org.zstack.header.core.Completion; + /** */ public interface VmBeforeStartOnHypervisorExtensionPoint { - void beforeStartVmOnHypervisor(VmInstanceSpec spec); + default void beforeStartVmOnHypervisor(VmInstanceSpec spec) {} + + default void beforeStartVmOnHypervisor(VmInstanceSpec spec, Completion completion) { + beforeStartVmOnHypervisor(spec); + completion.success(); + } } From d9911551f11be8b512d24e09cb93bee65c257e39 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Wed, 20 May 2026 16:57:11 +0800 Subject: [PATCH 23/24] [ha]: create pre-fence tag in core Move HA pre-fence pending tag creation into HaStartVmInstanceMsg handling after the VM state update, so later pre-fence execution observes the same database state. Resolves: ZSTAC-83890 Change-Id: I7f046732e0245319ed4cf188d1342b65dad11767 --- .../org/zstack/compute/vm/VmInstanceBase.java | 24 +++++++++++++++++++ .../org/zstack/compute/vm/VmSystemTags.java | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index d77842b8099..7232d478680 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1014,6 +1014,8 @@ public void done(ErrorCodeList errorCodeList) { } sql.update(); + refreshVO(); + createPreFencePendingTag(msg); startVm(msg, new Completion(msg, chain) { @Override @@ -1044,6 +1046,28 @@ public String getName() { }); } + private void createPreFencePendingTag(HaStartVmInstanceMsg msg) { + String suspectHostUuid = StringUtils.trimToNull(CollectionUtils.isEmpty(msg.getSoftAvoidHostUuids()) ? + self.getLastHostUuid() : msg.getSoftAvoidHostUuids().get(0)); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); + if (suspectHostUuid == null || peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host[%s] or peer host[%s] is absent", + self.getUuid(), suspectHostUuid, peerHostUuid)); + return; + } + + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } + private void changeVmIp(final String l3Uuid, final Map staticIpMap, final Completion completion) { final VmNicVO targetNic = CollectionUtils.find(self.getVmNics(), new Function() { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index 21c9668175a..ebb34e5970f 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -309,5 +309,12 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; + public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; + public static PatternedSystemTag HA_PRE_FENCE_PENDING = + new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", + HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, + HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); + public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); } From 118e53d14a81ca48fa32af00b69aca7c51b7b7d0 Mon Sep 17 00:00:00 2001 From: "yingzhe.hu" Date: Thu, 28 May 2026 15:43:20 +0800 Subject: [PATCH 24/24] [ha]: require peer for pre-fence tag Create HA pre-fence pending tag only when the checker provides an accessible peer host. The peer should be covered by the HA dead-host and storage-inactive paths, so do not persist a fallback marker when it is absent. Resolves: ZSTAC-83890 Change-Id: I36426b3386f1080ddeae8396ea9c4f7a81e25711 --- .../org/zstack/compute/vm/VmInstanceBase.java | 48 +++++++++---------- .../org/zstack/compute/vm/VmSystemTags.java | 4 +- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index 7232d478680..9e03bfa61b3 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -1004,18 +1004,38 @@ public void done(ErrorCodeList errorCodeList) { logger.debug(String.format("HaStartVmJudger[%s] says the VM[uuid:%s, name:%s] is qualified for HA start, now we are starting it", judger.getClass(), self.getUuid(), self.getName())); + String hostUuid = self.getHostUuid(); + String suspectHostUuid = StringUtils.trimToNull(hostUuid); + String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); UpdateQuery sql = SQL.New(VmInstanceVO.class) .eq(VmInstanceVO_.uuid, self.getUuid()) .set(VmInstanceVO_.state, VmInstanceState.Stopped) .set(VmInstanceVO_.hostUuid, null); - if (self.getHostUuid() != null) { - sql.set(VmInstanceVO_.lastHostUuid, self.getHostUuid()); + if (hostUuid != null) { + sql.set(VmInstanceVO_.lastHostUuid, hostUuid); } sql.update(); refreshVO(); - createPreFencePendingTag(msg); + if (suspectHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host is absent", + self.getUuid())); + } else if (peerHostUuid == null) { + logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because peer host is absent", + self.getUuid())); + } else { + Map tokens = new HashMap<>(); + tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); + tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); + + SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); + creator.inherent = true; + creator.recreate = true; + creator.ignoreIfExisting = true; + creator.setTagByTokens(tokens); + creator.create(); + } startVm(msg, new Completion(msg, chain) { @Override @@ -1046,28 +1066,6 @@ public String getName() { }); } - private void createPreFencePendingTag(HaStartVmInstanceMsg msg) { - String suspectHostUuid = StringUtils.trimToNull(CollectionUtils.isEmpty(msg.getSoftAvoidHostUuids()) ? - self.getLastHostUuid() : msg.getSoftAvoidHostUuids().get(0)); - String peerHostUuid = StringUtils.trimToNull(msg.getAccessiblePeerHostUuid()); - if (suspectHostUuid == null || peerHostUuid == null) { - logger.debug(String.format("HA-start vm[%s]: skip creating pre-fence tag because suspect host[%s] or peer host[%s] is absent", - self.getUuid(), suspectHostUuid, peerHostUuid)); - return; - } - - Map tokens = new HashMap<>(); - tokens.put(VmSystemTags.HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, suspectHostUuid); - tokens.put(VmSystemTags.HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN, peerHostUuid); - - SystemTagCreator creator = VmSystemTags.HA_PRE_FENCE_PENDING.newSystemTagCreator(self.getUuid()); - creator.inherent = true; - creator.recreate = true; - creator.ignoreIfExisting = true; - creator.setTagByTokens(tokens); - creator.create(); - } - private void changeVmIp(final String l3Uuid, final Map staticIpMap, final Completion completion) { final VmNicVO targetNic = CollectionUtils.find(self.getVmNics(), new Function() { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java index ebb34e5970f..6a146649119 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmSystemTags.java @@ -309,12 +309,12 @@ public String desensitizeTag(SystemTag systemTag, String tag) { public static PatternedSystemTag VM_STATE_PAUSED_AFTER_MIGRATE = new PatternedSystemTag(("vmPausedAfterMigrate"), VmInstanceVO.class); + public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); + public static String HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN = "suspectHostUuid"; public static String HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN = "accessiblePeerHostUuid"; public static PatternedSystemTag HA_PRE_FENCE_PENDING = new PatternedSystemTag(String.format("haPreFencePending::{%s}::{%s}", HA_PRE_FENCE_SUSPECT_HOST_UUID_TOKEN, HA_PRE_FENCE_ACCESSIBLE_PEER_HOST_UUID_TOKEN), VmInstanceVO.class); - - public static PatternedSystemTag VM_MEMORY_ACCESS_MODE_SHARED = new PatternedSystemTag(("vmMemoryAccessModeShared"), VmInstanceVO.class); }