diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 28613904211b23..b8d300d49a5fa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -1346,7 +1346,11 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) { break; case ALTER_PARTITION_STATES: // Replay only, like ALTER_IVM_INFO: a live change journals itself from inside MTMV. - mtmv.alterPartitionStates(alterMTMV.getPartitionStates()); + // The states and the snapshot removal land in one lock acquisition: a reader that saw + // the new requirement but still found the snapshot could let a transparent rewrite + // serve rows the rebuild has not replaced yet. + mtmv.replayAlterPartitionStates(alterMTMV.getPartitionStates(), + alterMTMV.getRemovedSnapshotPartitions()); break; default: throw new RuntimeException("Unknown type value: " + alterMTMV.getOpType()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index ab594e7487db5d..ac65375b12a22e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -162,7 +162,6 @@ import org.apache.doris.mtmv.MTMVRefreshPartitionSnapshot; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVService; -import org.apache.doris.mtmv.MTMVStatus; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mtmv.ivm.IvmUtil; import org.apache.doris.mysql.authenticate.AuthenticateType; @@ -7704,12 +7703,6 @@ public void alterMTMVProperty(AlterMTMVPropertyInfo info) throws UserException { this.alter.processAlterMTMVProperty(alter, false); } - public void alterMTMVStatus(TableNameInfo mvName, MTMVStatus status) { - AlterMTMV alter = new AlterMTMV(mvName, MTMVAlterOpType.ALTER_STATUS); - alter.setStatus(status); - this.alter.processAlterMTMV(alter, false); - } - public void addMTMVTaskResult(TableNameInfo mvName, MTMVTask task, MTMVRelation relation, Map partitionSnapshots) { AlterMTMV alter = new AlterMTMV(mvName, MTMVAlterOpType.ADD_TASK); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 586d9341aa60c5..294fa5bf198545 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -59,7 +59,6 @@ import org.apache.doris.mtmv.ivm.IvmUtil; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.rules.analysis.SessionVarGuardRewriter; -import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo.RefreshMode; import org.apache.doris.persist.AlterMTMV; import org.apache.doris.persist.EditLog.EditLogItem; import org.apache.doris.persist.OperationType; @@ -69,6 +68,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -109,9 +109,9 @@ public class MTMV extends OlapTable { @SerializedName("mpi") private MTMVPartitionInfo mvPartitionInfo; @SerializedName("rs") - private MTMVRefreshSnapshot refreshSnapshot; + private MTMVRefreshSnapshot refreshSnapshot = new MTMVRefreshSnapshot(); @SerializedName("ii") - private IvmInfo ivmInfo; + private IvmInfo ivmInfo = new IvmInfo(); /** * The refresh epoch of every MV partition, keyed by MV partition name. * @@ -120,13 +120,12 @@ public class MTMV extends OlapTable { * the ADD_TASK payload and ALTER_PARTITION_STATES are all no-ops for a non-IVM MV, so for one an * empty map is the complete answer. * - *

Null means the same thing -- no state -- and has three causes: an image written before the - * field existed, a non-IVM MV, and a live MV that has not been aligned yet. Only - * {@link #gsonPostProcess()} turns it into an empty map, on load; nothing else needs to, because a - * reader treats the two the same. + *

Never null, so a reader has no null case to answer: an MV is created with an empty map, and + * {@link #gsonPostProcess()} gives an MV loaded from an image written before the field existed the + * same one. */ @SerializedName("pst") - private Map partitionStates; + private Map partitionStates = Maps.newLinkedHashMap(); // Increased every time rewrite cache is invalidated to prevent publishing stale in-flight cache builds. private transient long rewriteCacheGeneration; private long schemaChangeVersion; @@ -229,6 +228,17 @@ public MTMVRefreshInfo alterRefreshInfo(MTMVRefreshInfo newRefreshInfo) { } } + /** + * Applies a status change, together with the invalidation it stands for: the version bump that + * discards a task result computed against the state being replaced, and the snapshot drop that stops + * the transparent rewrite serving rows from it. + * + *

This is the step that applies a change, not the one that records it -- it takes the MV + * lock and moves all three, but nothing here is journaled. It is what {@code Alter#processAlterMTMV} + * calls for an {@code ALTER_STATUS} op, both live and on replay; a live caller reaches it through + * {@link #invalidateWholeMv}, which goes the journaled way round. A new invalidation belongs there: + * calling this directly would leave the MV in a state a restart forgets. + */ public MTMVStatus alterStatus(MTMVStatus newStatus) { writeMvLock(); try { @@ -297,6 +307,10 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) { EditLogItem editLogItem; writeMvLock(); try { + // Read once, here: the task's worker thread may still be merging into this map, and the two + // places that use it -- applying the epochs and journaling them -- have to describe the same + // set of partitions. The getter hands out a detached copy for the same reason. + Map capturedEpochs = task.getIvmCapturedEpochs(); if (!isReplay && task.getMtmvSchemaChangeVersion() != this.schemaChangeVersion) { LOG.warn( "addTaskResult failed, schemaChangeVersion has changed. " @@ -305,14 +319,31 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) { name, task.getTaskId(), task.getMtmvSchemaChangeVersion(), this.schemaChangeVersion); return false; } - if (isReplay && alterMTMV.getIvmInfo() != null) { - // Replay the final IVM state; ADD_TASK does not change schemaChangeVersion. - ivmInfo = new IvmInfo(alterMTMV.getIvmInfo()); - } - if (isReplay && alterMTMV.getPartitionStates() != null) { - // A journal written before the field existed carries no state at all: leave the - // partition states alone rather than clearing them. - partitionStates = MTMVPartitionState.copyOf(alterMTMV.getPartitionStates()); + if (isReplay) { + if (alterMTMV.getIvmInfo() != null) { + // Replay the final IVM state; ADD_TASK does not change schemaChangeVersion. + ivmInfo = new IvmInfo(alterMTMV.getIvmInfo()); + } + if (alterMTMV.getPartitionStates() != null) { + // A journal written before the field existed carries no state at all: leave the + // partition states alone rather than clearing them. What a payload does carry is + // merged rather than assigned: a task result journals only the partitions it + // published, so the entries it does not mention belong to other records -- an + // invalidation that ran during the task, or an entry alignment added -- and + // assigning would drop them. The state-map channel proper + // (ALTER_PARTITION_STATES) still replaces, because that one carries the whole map. + for (Entry entry : alterMTMV.getPartitionStates().entrySet()) { + partitionStates.put(entry.getKey(), new MTMVPartitionState(entry.getValue())); + } + } + } else { + if (ivmInfo.isEnableIvm()) { + // The batches this task committed now hold data read at the epoch they captured, so + // the requirement is met for exactly those partitions. Recorded for a failed task + // too: its snapshots and epochs only ever cover the batches that succeeded, and + // leaving their rebuilt work unrecorded would only make the next refresh redo it. + applyRefreshedEpochs(capturedEpochs); + } } if (task.getStatus() == TaskStatus.SUCCESS) { this.status.setState(MTMVState.NORMAL); @@ -324,7 +355,6 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) { if (refreshedIvmPlanSignature != null) { ivmInfo.setPlanSignature(refreshedIvmPlanSignature); } - ivmInfo.clearBaselineRebuild(); } // The refresh publishes a new plan, so every cache built before this commit is stale. // Bump before publishing so an in-flight build cannot pass its generation check later. @@ -346,7 +376,14 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) { } this.jobInfo.addHistoryTask(task); compatiblePctSnapshot(partitionSnapshots); - this.refreshSnapshot.updateSnapshots(partitionSnapshots, getPartitionNames()); + // What this task wrote is described by the epochs just recorded, so a partition the result + // left dirty is left out: its snapshot would otherwise come back after an invalidation + // dropped it, and transparent rewrite reads that map to decide what it may serve. + Map snapshotsToWrite = partitionSnapshots; + if (!isReplay && ivmInfo.isEnableIvm()) { + snapshotsToWrite = snapshotsOfCleanPartitions(partitionSnapshots); + } + this.refreshSnapshot.updateSnapshots(snapshotsToWrite, getPartitionNames()); Env.getCurrentEnv().getMtmvService() .refreshComplete(this, relation, task); if (isReplay) { @@ -354,10 +391,17 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) { } if (ivmInfo.isEnableIvm()) { alterMTMV.setIvmInfo(ivmInfo); - // Same condition as ivmInfo, so the journal of a non-IVM MV stays byte-for-byte what it - // was. The map is null until the states are first aligned, and a payload without the - // member means the same as one carrying an empty map. - alterMTMV.setPartitionStates(partitionStates); + // Only the partitions this result published, not the whole map: the map has one entry per + // MV partition, so a scheduled refresh of an MV with many partitions would deep-copy and + // journal all of them on every run to say what almost all of them already said. What the + // record has to carry is the change; the replay merges it. A result that published nothing + // carries nothing, which is what a payload without the member already means. + alterMTMV.setPartitionStates(publishedPartitionStates(capturedEpochs)); + // Journal the map that was applied, not the one the task proposed: a partition this result + // left dirty was dropped from it above, and a replay that restored the raw map would put + // back the snapshot of a partition an invalidation has just cleared. The replay skips the + // filter, so what the payload carries is exactly what a restart ends up with. + alterMTMV.setPartitionSnapshots(snapshotsToWrite); } editLogItem = submitAlterLog(alterMTMV); } finally { @@ -370,102 +414,141 @@ public boolean addTaskResult(AlterMTMV alterMTMV, boolean isReplay) { public void alterMvProperties(AlterMTMV alterMTMV, boolean isReplay) { EditLogItem editLogItem; + EditLogItem invalidation = null; writeMvLock(); try { Map mvProperties = alterMTMV.getMvProperties(); - boolean containsExcludedTriggerTables = mvProperties.containsKey( - PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES); - Set oldExcludedTriggerTables = containsExcludedTriggerTables - ? parseExcludedTriggerTables() - : Sets.newHashSet(); - // Enlarging or removing ivm_partition_window_limit brings previously lossy - // partitions back into the refresh range. Their stream backlog was skipped by - // the windowed refreshes, so a strict incremental refresh would wrongly judge - // "all partitions are synced" and return SUCCESS with stale data. Force the - // next refresh to rebuild a complete baseline instead. - boolean containsPartitionWindowLimit = mvProperties.containsKey( - PropertyAnalyzer.PROPERTIES_IVM_PARTITION_WINDOW_LIMIT); - Map oldWindowLimits = containsPartitionWindowLimit - ? MTMVPropertyUtil.getIvmPartitionWindowLimit(this.mvProperties) - : Maps.newHashMap(); - // A partition_sync_limit window decides which base partitions the MV maintains. Only a change - // that can bring a partition back into that set needs a complete baseline rebuild -- a removed - // or wider limit -- because its deltas were skipped while it was outside and nothing - // incremental can repair them. That is the same trade as the two properties around it. A - // window that starts applying, a narrower one, and one that describes the same set as before - // leave the applied deltas intact; the partitions they take out are dropped by partition sync - // before the refresh plans, and taking one back in is the widening this answers. Doing it here, - // in the critical section that applies the ALTER, is also what keeps a window set and cleared - // while an invalidation reads the mapping from making that mapping look unwindowed. - boolean containsSyncWindow = MTMVPropertyUtil.containsPartitionSyncWindow(mvProperties); - Map oldSyncWindow = containsSyncWindow - ? MTMVPropertyUtil.partitionSyncWindowOf(this.mvProperties) : null; + // Read the old values before the properties are applied, and unconditionally: a property that is + // not part of this ALTER has to be compared against the value the MV actually holds. Reading it + // only when its key is present would compare an empty default against the real value, report a + // change that is not there, and drop the snapshot of an unrelated ALTER. + Set oldExcludedTriggerTables = parseExcludedTriggerTables(); + Map oldWindowLimits = + MTMVPropertyUtil.getIvmPartitionWindowLimit(this.mvProperties); + Map oldSyncWindow = MTMVPropertyUtil.partitionSyncWindowOf(this.mvProperties); this.mvProperties.putAll(mvProperties); - // Both excluded_trigger_tables changes and window limit enlargement/removal - // change the refresh baseline semantics: partitions previously skipped become - // refreshable again, and their stream backlog was not applied. Invalidate the - // snapshots (once) and require a complete baseline rebuild so the next refresh - // covers the new range instead of wrongly judging "all partitions are synced". - boolean invalidateRefreshSnapshot = false; - boolean requireCompleteBaselineRebuild = false; - if (containsExcludedTriggerTables) { - Set newExcludedTriggerTables = parseExcludedTriggerTables(); - if (!oldExcludedTriggerTables.equals(newExcludedTriggerTables)) { - invalidateRefreshSnapshot = true; - if (ivmInfo != null && ivmInfo.isEnableIvm() - && relation != null && relation.getBaseTables() != null) { - for (BaseTableInfo baseTableInfo : relation.getBaseTables()) { - TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(), - baseTableInfo.getDbName(), baseTableInfo.getTableName()); - if (MTMVPartitionUtil.isTableExcluded(oldExcludedTriggerTables, baseTableName) - && !MTMVPartitionUtil.isTableExcluded(newExcludedTriggerTables, baseTableName)) { - requireCompleteBaselineRebuild = true; - break; - } - } - } - } - } - if (containsPartitionWindowLimit && ivmInfo != null && ivmInfo.isEnableIvm() - && relation != null && relation.getBaseTables() != null) { - Map newWindowLimits = - MTMVPropertyUtil.getIvmPartitionWindowLimit(this.mvProperties); - for (BaseTableInfo baseTableInfo : relation.getBaseTables()) { - TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(), - baseTableInfo.getDbName(), baseTableInfo.getTableName()); - int oldLimit = MTMVPropertyUtil.getPartitionWindowLimit(oldWindowLimits, baseTableName); - if (oldLimit == -1) { - continue; - } - int newLimit = MTMVPropertyUtil.getPartitionWindowLimit(newWindowLimits, baseTableName); - if (newLimit == -1 || newLimit > oldLimit) { - requireCompleteBaselineRebuild = true; - break; - } - } - } - if (containsSyncWindow && ivmInfo != null && ivmInfo.isEnableIvm() - && MTMVPropertyUtil.partitionSyncWindowWidens(oldSyncWindow, - MTMVPropertyUtil.partitionSyncWindowOf(this.mvProperties))) { - requireCompleteBaselineRebuild = true; - } - if (invalidateRefreshSnapshot || requireCompleteBaselineRebuild) { - this.schemaChangeVersion++; - this.refreshSnapshot = new MTMVRefreshSnapshot(); - } - if (requireCompleteBaselineRebuild) { - ivmInfo.requireCompleteBaselineRebuild(); - } + // The one thing a property change can owe the refresh baseline: a whole-MV rebuild, when it + // brings base table partitions back into the set the MV maintains. Their stream backlog was + // skipped while they were outside that set, so no delta can repair them -- and the rebuild is + // whole-MV rather than per-partition because it covers the partitions partition sync has not + // created yet. invalidateWholeMv owns all of it: the state the refresh reads, the version bump + // that discards a task result computed before the change, and the snapshot drop that stops + // transparent rewrite serving rows from it. + // + // Narrowing that set owes nothing. The MV's rows for a table it no longer maintains are allowed + // to be stale by design, and the snapshot entry describing them is skipped by the next + // incremental refresh anyway, so dropping the whole snapshot and discarding a running task + // result for them buys nothing. A change that leaves the maintained set alone owes nothing + // either. if (isReplay) { + // The property change itself is applied above. Nothing else on this path has to run for a + // replay: the state, the version and the snapshot a whole-MV invalidation moves come back + // from the status record that precedes this one, through MTMV#alterStatus, and this + // property record never carried a snapshot. return; } + if (rebuildsWholeMv(oldExcludedTriggerTables, oldWindowLimits, oldSyncWindow)) { + // Journaled on its own record, ahead of the property change below; a replay applies both + // in that order. Submitted here and awaited below, outside the lock: the order is the + // enqueue order, which the lock already fixes, so there is nothing to gain by holding the + // lock across the flush. + invalidation = invalidateWholeMv("The MV's refresh baseline changed with its properties"); + } editLogItem = submitAlterLog(alterMTMV); } finally { writeMvUnlock(); } + if (invalidation != null) { + invalidation.await(); + } editLogItem.await(); } + /** + * Whether a property change brings base table partitions back into the set the MV maintains, and so + * owes a whole-MV rebuild. + * + *

Takes the values the MV held before the change; see the call site for why they are read + * unconditionally. Widening decides on its own: a change that both takes a partition out of the + * maintained set and puts one back is the rebuild, because the partition coming back is the one whose + * backlog was skipped. + */ + private boolean rebuildsWholeMv(Set oldExcludedTriggerTables, + Map oldWindowLimits, Map oldSyncWindow) { + // Judged once here rather than in each of the three: they answer "did this property move in the + // direction that owes a rebuild", which is only a question an MV maintaining an IVM baseline has. + if (!maintainsIvmBaseline()) { + return false; + } + return unexcludesABaseTable(oldExcludedTriggerTables) + || widensPartitionWindowLimit(oldWindowLimits) + || widensSyncWindow(oldSyncWindow); + } + + /** + * Whether this MV has an IVM baseline to maintain at all, which every widening check needs. + * + *

No null check on {@code ivmInfo}: it is initialized where an MV is built and + * {@link #gsonPostProcess()} gives an MV loaded from an image written before the field existed the + * same one, so it is non-null by the time anything reads it. + */ + private boolean maintainsIvmBaseline() { + return ivmInfo.isEnableIvm() && relation != null && relation.getBaseTables() != null; + } + + /** + * Whether a base table of this MV stopped being excluded. + * + *

An excluded table has no stream, so the partitions the MV read from it have no backlog to apply; + * while it was excluded the MV did not maintain them. + */ + private boolean unexcludesABaseTable(Set oldExcludedTriggerTables) { + Set newExcludedTriggerTables = parseExcludedTriggerTables(); + for (BaseTableInfo baseTableInfo : relation.getBaseTables()) { + TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(), + baseTableInfo.getDbName(), baseTableInfo.getTableName()); + if (MTMVPartitionUtil.isTableExcluded(oldExcludedTriggerTables, baseTableName) + && !MTMVPartitionUtil.isTableExcluded(newExcludedTriggerTables, baseTableName)) { + return true; + } + } + return false; + } + + /** + * Whether an ivm_partition_window_limit was removed or enlarged for some base table, which brings the + * partitions the windowed refreshes skipped back into range with their backlog unapplied. + */ + private boolean widensPartitionWindowLimit(Map oldWindowLimits) { + Map newWindowLimits = + MTMVPropertyUtil.getIvmPartitionWindowLimit(this.mvProperties); + for (BaseTableInfo baseTableInfo : relation.getBaseTables()) { + TableNameInfo baseTableName = new TableNameInfo(baseTableInfo.getCtlName(), + baseTableInfo.getDbName(), baseTableInfo.getTableName()); + int oldLimit = MTMVPropertyUtil.getPartitionWindowLimit(oldWindowLimits, baseTableName); + if (oldLimit == -1) { + continue; + } + int newLimit = MTMVPropertyUtil.getPartitionWindowLimit(newWindowLimits, baseTableName); + if (newLimit == -1 || newLimit > oldLimit) { + return true; + } + } + return false; + } + + /** + * Whether a partition_sync_limit window was widened. + * + *

A window that starts applying, a narrower one, and one that describes the same set as before all + * leave the applied deltas intact: the partitions they take out are dropped by partition sync before + * the refresh plans, and taking one back in is the widening this answers. + */ + private boolean widensSyncWindow(Map oldSyncWindow) { + return MTMVPropertyUtil.partitionSyncWindowWidens(oldSyncWindow, + MTMVPropertyUtil.partitionSyncWindowOf(this.mvProperties)); + } + public long getGracePeriod() { readMvLock(); try { @@ -626,7 +709,7 @@ public boolean hasRefreshSnapshot() { // IVM only needs to know whether a baseline has ever been built. // A newly added MV partition legitimately has no PCT snapshot yet, // but that must not block row-level incremental refresh. - return refreshSnapshot != null && !MapUtils.isEmpty(refreshSnapshot.getPartitionSnapshots()); + return !MapUtils.isEmpty(refreshSnapshot.getPartitionSnapshots()); } finally { readMvUnlock(); } @@ -663,54 +746,287 @@ public void alterIvmInfo(IvmInfo ivmInfo) { * into the journal, and a replay that replaces the field would leave the caller's reference * pointing at state that is no longer the MV's. Changing the states is the MV's own job, under its * write lock. - * - *

A missing map -- an image written before the field existed, or a non-IVM MV -- reads as empty. */ public Map getPartitionStates() { readMvLock(); try { - if (partitionStates == null) { - return Collections.emptyMap(); - } return Collections.unmodifiableMap(MTMVPartitionState.copyOf(partitionStates)); } finally { readMvUnlock(); } } + /** + * The partitions whose requirement has been raised and not met, which a refresh has to rebuild rather + * than catch up. + * + *

Detached names rather than the states themselves: a caller that only routes by them has no + * business holding the map the MV journals, and it needs nothing else from an entry. + */ + public Set getPartitionsNeedingRebuild() { + // Built before the lock, like the map getLatestEpochs returns: which entries go in is what needs + // the lock, not having somewhere to put them. + Set res = Sets.newLinkedHashSet(); + readMvLock(); + try { + for (Entry entry : partitionStates.entrySet()) { + if (entry.getValue().isDirty()) { + res.add(entry.getKey()); + } + } + return res; + } finally { + readMvUnlock(); + } + } + + /** + * Whether every partition the MV holds needs a rebuild, which is when a whole-MV refresh does nothing + * the per-partition routing would not. + * + *

A partition that holds data and does not need one makes this false: a whole-MV refresh would + * recompute it for nothing, which is the waste the per-partition routing exists to avoid. A partition + * that was never refreshed does not count against it -- a whole-MV refresh fills it, which its + * per-partition branch would do as well -- and it needs no clause of its own: an aligned entry is + * {@code {0, 1}}, so it is behind its requirement already. An MV with no partitions is not an + * escalation either. + * + *

Read in place rather than through {@link #getPartitionStates()}: the caller asks a yes/no + * question, and copying the map to answer it would allocate a state object per partition, under this + * lock, on every refresh -- including the ones that escalate nothing. + */ + public boolean allPartitionsNeedRebuild() { + readMvLock(); + try { + return !partitionStates.isEmpty() + && partitionStates.values().stream().allMatch(MTMVPartitionState::isDirty); + } finally { + readMvUnlock(); + } + } + // ALTER_PARTITION_STATES replay applies a detached snapshot here, mirroring alterIvmInfo(). Live // invalidation changes submit their journal from the mutating method instead. // // A payload without the member carries no state at all, which is not the same as an empty map that // says the states are now empty: leaving them alone is the only answer that cannot lose state. public void alterPartitionStates(Map partitionStates) { - if (partitionStates == null) { + replayAlterPartitionStates(partitionStates, null); + } + + /** + * ALTER_PARTITION_STATES replay: applies the states the payload carries, and drops the snapshots it + * names. Both in one lock acquisition, because a reader that saw the new requirement while the + * snapshot was still there could let a transparent rewrite serve rows the rebuild has to replace. + * + *

A payload without the states carries none, which is not the same as an empty map that says the + * states are now empty: leaving them alone is the only answer that cannot lose state. + */ + public void replayAlterPartitionStates(Map partitionStates, + Set removedSnapshotPartitions) { + writeMvLock(); + try { + if (partitionStates != null) { + this.partitionStates = MTMVPartitionState.copyOf(partitionStates); + } + refreshSnapshot.removeSnapshots(removedSnapshotPartitions); + } finally { + writeMvUnlock(); + } + } + + /** + * The {@code latestEpoch} of the given MV partitions, taken under the MV read lock. + * + *

This is the value a refresh has to remember: what it read from the base tables is described by + * the requirement in force when it started reading, so writing that value back as the new + * {@code refreshEpoch} is what keeps an invalidation arriving mid-refresh from being swallowed. A + * partition without an entry is left out -- a caller writes an epoch only for what it captured. + */ + public Map getLatestEpochs(Set partitionNames) { + if (CollectionUtils.isEmpty(partitionNames)) { + return Collections.emptyMap(); + } + // Sized before the lock: the state map is what needs it, and building the map is not part of that. + Map res = Maps.newHashMapWithExpectedSize(partitionNames.size()); + readMvLock(); + try { + for (String partitionName : partitionNames) { + MTMVPartitionState state = partitionStates.get(partitionName); + if (state != null) { + res.put(partitionName, state.getLatestEpoch()); + } + } + return res; + } finally { + readMvUnlock(); + } + } + + /** + * Brings the partition states in line with the MV's partitions: every partition gets an entry, and + * every entry whose partition is gone is dropped. + * + *

Alignment is what makes "the partition exists" and "the entry exists" the same thing, and it is + * why an invalidation cannot miss: rows are only written by a refresh, and every refresh aligns + * before it reads a base table, so a partition that holds rows always has an entry for the mark to + * land on. The other direction is what makes the criterion safe -- an entry created here describes a + * partition with no rows yet, so requiring one generation of it discards no requirement that was + * made earlier. + * + *

What it changes is journaled, because the entry has to be on disk before the rows it describes + * can be: a crash between this and the task result would otherwise leave a partition that holds rows + * with no entry at all, and every later invalidation of it would find nothing to land on. That is the + * one shape in which the criterion cannot be read -- "no entry" is supposed to mean "no rows" -- so + * the entry is made durable before any base table is read rather than derived again on the next run. + * + *

It is deliberately not a hook on every path that creates or drops a partition. An entry is + * derived state, and rebuilding it from the live partition set also repairs whatever a crash left + * behind: the drop of a partition and the removal of its entry are two journal records, and only + * their order -- partition first -- is safe, which leaves at most a stale entry that the next + * alignment drops. + * + *

Only an IVM MV is aligned. For a non-IVM MV the map stays as it is, and every reader treats + * "empty" and "no state" the same. + */ + public void alignPartitionStates() { + if (!isIvm()) { return; } + EditLogItem editLogItem = null; writeMvLock(); try { - this.partitionStates = MTMVPartitionState.copyOf(partitionStates); + // Read here rather than handed in by the caller: a caller has to read the names before it takes + // this lock, and a partition created in between -- by a concurrent refresh's partition sync -- + // would then be dropped by the retainAll below, taking with it the state a following + // invalidation has to land on. The read is cheap and takes no lock of its own, so doing it here + // does not add an edge to the lock order. + Set livePartitions = Sets.newHashSet(getPartitionNames()); + boolean changed = partitionStates.keySet().retainAll(livePartitions); + for (String partitionName : livePartitions) { + if (!partitionStates.containsKey(partitionName)) { + partitionStates.put(partitionName, MTMVPartitionState.initial()); + changed = true; + } + } + if (changed) { + editLogItem = submitPartitionStatesChange(Collections.emptySet()); + } } finally { writeMvUnlock(); } + if (editLogItem != null) { + editLogItem.await(); + } } - public void invalidateIvmBaseline() { - EditLogItem editLogItem; + /** + * The snapshots of the partitions that are clean after this result's epochs were applied. + * + *

An invalidation that reached a partition while the task ran leaves it dirty, and its snapshot + * must stay gone: dropping the entry is what keeps transparent rewrite away from rows the rebuild has + * to replace, and a result written back afterwards would undo exactly that. Removing only the entry + * keeps the rest of the map, which the removal on the invalidation side cannot express. + * + *

The caller holds the MV write lock and has already applied the epochs, so {@code isDirty} here + * reads the state the data is actually described by. + * + *

Only an IVM MV has partition states, so only its write-back is narrowed here: every entry of a + * non-IVM MV has no state to be dirty in and is written back as it always was. + */ + private Map snapshotsOfCleanPartitions( + Map snapshots) { + if (MapUtils.isEmpty(snapshots)) { + return snapshots; + } + Map res = Maps.newHashMapWithExpectedSize(snapshots.size()); + for (Entry entry : snapshots.entrySet()) { + MTMVPartitionState state = partitionStates.get(entry.getKey()); + // No entry means the partition was created after the alignment, so it can only hold rows this + // task wrote; a dirty one needs its rebuild before anything may read it through the MV. + if (state == null || !state.isDirty()) { + res.put(entry.getKey(), entry.getValue()); + } + } + return res; + } + + /** + * Records the epochs the given partitions were read at, which is how a refresh turns a requirement + * into the state of the data. + * + *

Only {@code refreshEpoch} is written: a refresh writes back the requirement it captured, and the + * requirement may have been raised again since that capture. A payload built from the captured map + * would overwrite the newer value and lose the rebuild it asks for, so {@code latestEpoch} is left + * alone here. + * + *

The caller holds the MV write lock (it is applied together with the rest of a task result). + */ + private void applyRefreshedEpochs(Map capturedEpochs) { + if (MapUtils.isEmpty(capturedEpochs)) { + return; + } + for (Entry entry : capturedEpochs.entrySet()) { + MTMVPartitionState state = partitionStates.get(entry.getKey()); + if (state == null) { + // The partition was dropped while the task ran, so its state went with it. + continue; + } + state.setRefreshEpoch(entry.getValue()); + } + } + + /** + * The states a task result publishes: the partitions whose epochs this result just wrote. + * + *

Read under the MV write lock, after {@link #applyRefreshedEpochs}, so what it captures is the state + * as published. A requirement raised during the task is carried along rather than recomputed: the + * write-back only moves {@code refreshEpoch}, and a payload that omitted the newer {@code latestEpoch} + * would let a replay restore the older one and lose the rebuild it asks for. + */ + private Map publishedPartitionStates(Map capturedEpochs) { + if (MapUtils.isEmpty(capturedEpochs)) { + return Collections.emptyMap(); + } + Map published = Maps.newLinkedHashMapWithExpectedSize(capturedEpochs.size()); + for (String partitionName : capturedEpochs.keySet()) { + MTMVPartitionState state = partitionStates.get(partitionName); + if (state != null) { + published.put(partitionName, state); + } + } + return published; + } + + /** + * Invalidates the whole MV: the state the refresh reads, the version bump that discards a task result + * computed against the state being replaced, and the snapshot drop that stops the transparent rewrite + * serving rows from it. + * + *

Applies the change and submits its journal record, and hands back the write for the caller to + * await. Both happen under one acquisition of the MV write lock, which is what keeps a refresh from + * publishing its result in between: the record has to be enqueued in the same critical section as the + * state it stands for, or a task result that slips into the gap is enqueued first and a replay applies + * it first -- leaving the follower in SCHEMA_CHANGE where the leader ended NORMAL. The callers that + * hold no outer MV lock are the ones this matters for; {@link #alterStatus} takes the same lock + * reentrantly, so holding it here is free. + * + *

The caller awaits outside the MV lock. It does not have to hold the lock across the flush to keep + * the order -- the record is enqueued in call order, so submitting this one before the next one is what + * puts it first -- and holding the lock across a journal wait is what the rest of this class avoids. + */ + public EditLogItem invalidateWholeMv(String detail) { + MTMVStatus status = new MTMVStatus(MTMVState.SCHEMA_CHANGE, detail); writeMvLock(); try { - if (ivmInfo == null) { - ivmInfo = new IvmInfo(); - } - ivmInfo.requireCompleteBaselineRebuild(); - // Bump the version even when a rebuild is already pending, so a task that started before - // this visible base-table change cannot clear the barrier with an old result. - schemaChangeVersion++; - editLogItem = submitIvmInfoChange(); + alterStatus(status); + AlterMTMV alterMTMV = new AlterMTMV(new TableNameInfo(getQualifiedDbName(), getName()), + MTMVAlterOpType.ALTER_STATUS); + alterMTMV.setStatus(status); + return submitAlterLog(alterMTMV); } finally { writeMvUnlock(); } - editLogItem.await(); } /** @@ -721,7 +1037,8 @@ public void invalidateIvmBaseline() { * @return whether a barrier was recorded. The caller reports the two outcomes differently: a change * that no MV partition reads leaves nothing to rebuild and must not be logged as one. */ - public boolean invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map changedPartitions) { + public boolean invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map changedPartitions, + String reason) { // Computed before the MV lock is taken, not inside it: the mapping reads the partition items of the // MV and of every PCT table, so it takes those tables' locks, and the MV lock has to stay a leaf // (nothing may be acquired under it) the way the rest of this class assumes. The selection does not @@ -738,21 +1055,28 @@ public boolean invalidateIvmBaseline(BaseTableInfo baseTableInfo, Map marked = markIvmPartitionsInvalidated(affectedMvPartitions.get()); + if (marked.isEmpty()) { + LOG.debug("No MV partition holds the changed base partitions, mv={}, baseTable={}, " + + "changedPartitions={}", name, baseTableInfo, changedPartitions); + return false; } - schemaChangeVersion++; - editLogItem = submitIvmInfoChange(); + editLogItem = submitPartitionStatesChange(marked); } finally { writeMvUnlock(); } @@ -760,6 +1084,32 @@ public boolean invalidateIvmBaseline(BaseTableInfo baseTableInfo, MapOnly partitions that have an entry are marked: an entry is created before anything reads a base + * table, so a partition without one holds no rows and there is nothing of its to rebuild. The two + * halves belong together -- the requirement is what sends the partition to a rebuild, and the missing + * snapshot is what keeps a transparent rewrite away from rows that are about to be replaced. + * + *

The caller holds the MV write lock, which is what keeps this read-modify-write of + * {@code latestEpoch} from losing a concurrent invalidation, and which makes the journal enqueue + * follow the mutation order. + */ + private Set markIvmPartitionsInvalidated(Set mvPartitionNames) { + Set marked = Sets.newLinkedHashSet(); + for (String partitionName : mvPartitionNames) { + MTMVPartitionState state = partitionStates.get(partitionName); + if (state == null) { + continue; + } + state.setLatestEpoch(state.getLatestEpoch() + 1); + marked.add(partitionName); + } + refreshSnapshot.removeSnapshots(marked); + return marked; + } + /** * Select the MV partitions that may hold rows read from the changed base table partitions. * @@ -947,67 +1297,62 @@ private MTMVRelatedTableIf findPctTable(BaseTableInfo baseTableInfo) { return null; } + private EditLogItem submitIvmInfoChange() { + // The caller has already mutated ivmInfo under the MV write lock. Submit its snapshot directly; + // replay later applies the payload through alterIvmInfo(). + AlterMTMV alterMTMV = new AlterMTMV( + new TableNameInfo(getQualifiedDbName(), getName()), MTMVAlterOpType.ALTER_IVM_INFO); + alterMTMV.setIvmInfo(ivmInfo); + return submitAlterLog(alterMTMV); + } + /** - * Release the IVM baseline barrier after the partitions it named have been rebuilt, or after - * partition sync removed them (a dropped partition resolves its own entry: the partition and its - * IVM offsets are both gone). - * - *

Guarded by schemaChangeVersion, like {@link #persistIvmBaselineGuard}: a base-table change - * landing while the rebuild runs carries its own barrier entry, and a blind clear would swallow - * it. Failing instead preserves that entry -- the next refresh rebuilds it together with the - * partitions this task handled. + * Raises the requirement of the given MV partitions, so the next refresh rebuilds them. * - *

Journals the new state right away, like every other ivmInfo mutation here. A task that dies - * before {@link #addTaskResult} would otherwise leave the release in memory only, and a restart - * would resurrect the barrier from disk. + *

This is an invalidation-shaped mutation, journaled as the whole state map before whatever needs + * it is done. A caller about to make a partition's rows unusable says so with it: the raised + * requirement survives a crash, so a refresh that never got to publish its rebuild leaves partitions + * naming a generation they do not hold, and the next refresh rebuilds them. */ - public void releaseIvmBaselineRebuild(long expectedSchemaChangeVersion) throws JobException { - EditLogItem editLogItem; - writeMvLock(); - try { - if (ivmInfo == null || !ivmInfo.isBaselineRebuildRequired()) { - // Nothing to release: skip both the mutation and the journal entry. Any base-table - // change that raced us in is still caught by validateIvmRefreshStart() below. - return; - } - if (schemaChangeVersion != expectedSchemaChangeVersion) { - throw new JobException("Base table metadata changed before IVM baseline refresh, mv=" - + getName()); - } - ivmInfo.clearBaselineRebuild(); - editLogItem = submitIvmInfoChange(); - } finally { - writeMvUnlock(); + public void markPartitionsForRebuild(Set partitionNames) { + if (CollectionUtils.isEmpty(partitionNames)) { + return; } - editLogItem.await(); - } - - public void persistIvmBaselineGuard(RefreshMode refreshMode, Set baselinePartitions, - long expectedSchemaChangeVersion) throws JobException { EditLogItem editLogItem; writeMvLock(); try { - if (schemaChangeVersion != expectedSchemaChangeVersion) { - throw new JobException("Base table metadata changed before IVM baseline refresh, mv=" + getName()); + boolean changed = false; + for (String partitionName : partitionNames) { + MTMVPartitionState state = partitionStates.get(partitionName); + if (state == null) { + // A partition dropped since the caller planned it has no rows to protect. + continue; + } + state.setLatestEpoch(state.getLatestEpoch() + 1); + changed = true; } - if (refreshMode == RefreshMode.COMPLETE) { - ivmInfo.requireCompleteBaselineRebuild(); - } else { - ivmInfo.addPendingBaselineRebuildPartitions(baselinePartitions); + if (!changed) { + return; } - editLogItem = submitIvmInfoChange(); + editLogItem = submitPartitionStatesChange(Collections.emptySet()); } finally { writeMvUnlock(); } editLogItem.await(); } - private EditLogItem submitIvmInfoChange() { - // The caller has already mutated ivmInfo under the MV write lock. Submit its snapshot directly; - // replay later applies the payload through alterIvmInfo(). + /** + * Journals the current states, and the MV partitions whose snapshots the same change dropped. + * + *

Same shape as submitIvmInfoChange: the caller mutated under the MV write lock, and replay applies + * this payload through replayAlterPartitionStates(). The states ride as the MV's own map -- the setter + * copies them -- so the payload cannot be written out half-mutated. + */ + private EditLogItem submitPartitionStatesChange(Set removedSnapshotPartitions) { AlterMTMV alterMTMV = new AlterMTMV( - new TableNameInfo(getQualifiedDbName(), getName()), MTMVAlterOpType.ALTER_IVM_INFO); - alterMTMV.setIvmInfo(ivmInfo); + new TableNameInfo(getQualifiedDbName(), getName()), MTMVAlterOpType.ALTER_PARTITION_STATES); + alterMTMV.setPartitionStates(partitionStates); + alterMTMV.setRemovedSnapshotPartitions(removedSnapshotPartitions); return submitAlterLog(alterMTMV); } @@ -1042,9 +1387,6 @@ public void validateIvmRefreshStart(long expectedSchemaChangeVersion) throws Job if (schemaChangeVersion != expectedSchemaChangeVersion) { throw new JobException("Base table metadata changed before IVM refresh, mv=" + getName()); } - if (ivmInfo != null && ivmInfo.isBaselineRebuildRequired()) { - throw new JobException("IVM baseline rebuild is pending, mv=" + getName()); - } } finally { readMvUnlock(); } @@ -1284,11 +1626,12 @@ public void gsonPostProcess() throws IOException { sessionVariables = Maps.newHashMap(); } if (ivmInfo == null) { + // Created with the MV as well; this covers an image that carries the member as null. ivmInfo = new IvmInfo(); } if (partitionStates == null) { - // An image written before the field existed deserializes it as null, and so does a non-IVM MV. - // Both mean "no state", so an empty map is the whole answer. + // The field is created with the MV, so an image that leaves it out keeps that empty map. This + // covers the one image that carries it as null, which reader code has no case for. partitionStates = Maps.newLinkedHashMap(); } if (refreshInfo != null && refreshInfo.getRefreshMethod() == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index f0445240602c41..c097c988daf4f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -54,6 +54,7 @@ import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVBaseTableIf; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVPartitionState; import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.MTMVRefreshContext; @@ -68,7 +69,6 @@ import org.apache.doris.mtmv.ivm.IvmIncrRefreshContext; import org.apache.doris.mtmv.ivm.IvmIncrRefreshManager; import org.apache.doris.mtmv.ivm.IvmIncrRefreshResult; -import org.apache.doris.mtmv.ivm.IvmInfo; import org.apache.doris.mtmv.ivm.IvmPlanSignature; import org.apache.doris.mtmv.ivm.IvmRewriteContext; import org.apache.doris.mtmv.ivm.IvmUtil; @@ -100,6 +100,8 @@ import java.math.BigDecimal; import java.math.RoundingMode; +import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -108,6 +110,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -142,7 +145,8 @@ public class MTMVTask extends AbstractTask { new Column("Progress", ScalarType.createStringType()), new Column("LastQueryId", ScalarType.createStringType()), new Column("ComputeGroup", ScalarType.createStringType()), - new Column("IvmFallbackReason", ScalarType.createStringType())); + new Column("IvmFallbackReason", ScalarType.createStringType()), + new Column("IvmRebuiltPartitions", ScalarType.createStringType())); public static final ImmutableMap COLUMN_TO_INDEX; @@ -240,10 +244,17 @@ private PartitionPlanningException(String message, Throwable cause) { private long mtmvId; @SerializedName("taskContext") private MTMVTaskContext taskContext; + // What this task reports as refreshed, and what it reports as done, for the whole task rather than for + // its last attempt: an attempt takes a scope of its own and a later one would otherwise replace it, + // hiding the partitions an earlier one committed -- which the MV has already published. Sets, because + // a later attempt's scope can cover an earlier one's (a whole-MV rebuild after a per-partition one), + // and a partition counted twice would report more work than the MV has partitions. Sorted, so that the + // column a task reports with reads the same on every look. The record is an array of names either way, + // which is what it was when these were lists. @SerializedName("needRefreshPartitions") - List needRefreshPartitions; + Set needRefreshPartitions; @SerializedName("completedPartitions") - List completedPartitions; + Set completedPartitions; @SerializedName("refreshMode") MTMVTaskRefreshMode refreshMode; @SerializedName("lastQueryId") @@ -259,7 +270,25 @@ private PartitionPlanningException(String message, Throwable cause) { // Written by the executing (Disruptor worker) thread via the executeCommand consumer // callback and read by the cancel (command) thread, so it must be volatile. private volatile StmtExecutor executor; - private Map partitionSnapshots; + // What this task has committed, per MV partition: the snapshot each partition's rows were read at. + // One accumulator for the whole task rather than one per phase, because that is what the MV publishes + // at the end of it -- a phase that started from empty would publish its own work and drop the work of + // the phases before it, leaving partitions a preceding rebuild replaced looking unsynced. + private Map partitionSnapshots = Maps.newConcurrentMap(); + // The requirement each refreshed partition was read under, captured before the base tables were read + // and recorded only once that batch's data committed (see commitCapturedEpochs). In memory only: the + // journal carries the resulting states, and a replay applies those instead of recomputing anything. + private transient Map ivmCapturedEpochs = Maps.newConcurrentMap(); + // The requirement every partition had when this refresh planned its work. What a batch records is + // clamped to it (see commitCapturedEpochs): a mark that lands after the plan must leave its partition + // dirty rather than be written back as satisfied. Empty on a path that does not plan partition work, + // which is the plain COMPLETE path -- a whole-MV rebuild replaces every partition, so whatever it read + // is what it repaired. + private transient Map ivmPlannedEpochs = Maps.newHashMap(); + // How many partitions this refresh rebuilt because the criterion demanded it, which a strict + // INCREMENTAL request reports so that "the request was incremental but the work was not" is visible. + @SerializedName("irp") + private int ivmRebuiltPartitions; private long mtmvSchemaChangeVersion; // Published only after a signature-mismatch fallback succeeds and its task result is accepted. private transient String refreshedIvmPlanSignature; @@ -312,15 +341,20 @@ public void run() throws JobException { // refresh fallback: incompatible MV definitions must fail directly. ensureQueryUsableIfNeeded(ctx, tableIfs); RefreshRequest request = resolveRefreshRequest(); - validateIvmBaselineBeforePartitionSync(request); - List attempts = buildAttempts(request, queryAnalysis.containsOneRowRelation()); try { syncPartitionsIfNeeded(ctx, tableIfs); } catch (PartitionPlanningException e) { throw new JobException(e.getMessage(), e); } + // Partition sync has decided which partitions exist, and nothing has read a base table yet: + // this is the point where an entry and the partition it describes become the same thing. + // Doing it any later would let a partition that sync has just added be refreshed without an + // entry, and an invalidation arriving in between would have nothing to land on. + mtmv.alignPartitionStates(); + // Decided after the sync and the alignment, because the escalation it can take reads the + // partition states and only then is the partition set they describe final. + List attempts = buildAttempts(request, queryAnalysis.containsOneRowRelation()); MTMVRefreshContext refreshContext = buildRefreshContext(tableIfs); - handlePendingIvmBaselineRebuild(refreshContext, request, ctx, attempts); boolean disablePartitionRefresh = false; for (RefreshAttemptType attemptType : attempts) { switch (attemptType) { @@ -342,7 +376,16 @@ public void run() throws JobException { } break; case COMPLETE: - executeCompleteAttempt(refreshContext, ctx); + try { + executeCompleteAttempt(refreshContext, ctx); + } finally { + // Counted from the groups that committed, like the rebuild phase above: a + // whole-MV rebuild that failed in a later group has still replaced the groups + // before it, and those keep the epochs and snapshots they published (see + // MTMV#addTaskResult), so reporting none of them would hide the partial work + // this count exists to expose. + recordRebuiltPartitions(request); + } return; default: throw new JobException("Unsupported refresh attempt type: " + attemptType); @@ -444,7 +487,33 @@ private RefreshRequest resolveRefreshRequest() throws JobException { Lists.newArrayList(), false); } - private List buildAttempts(RefreshRequest request, boolean containsOneRowRelation) { + private List buildAttempts(RefreshRequest request, boolean containsOneRowRelation) + throws JobException { + // A schema-level invalidation is not a set of dirty partitions: it means every partition, including + // the ones partition sync has not created yet, and no per-partition requirement can express that. + // IVM only -- a non-IVM MV reaches the same effect through its cleared snapshot, which its own + // refresh already depends on. + // + // Judged before the initial-refresh shortcut below, which also answers COMPLETE: an MV that has + // never been refreshed and reads an excluded trigger table (or a one-row relation) has to be built + // by a whole-MV refresh, and a request that may not fall back has to hear that rather than have it + // decided for it -- otherwise the refusal here would be unreachable in exactly the state it names. + if (mtmv.isIvm() && !request.explicitPartitions + && mtmv.getStatus().getState() == MTMVState.SCHEMA_CHANGE) { + if (request.refreshMode == RefreshMode.PARTITIONS && !request.allowFallback) { + // A PARTITIONS request that may not fall back asked for a scope, and this invalidation needs + // one it did not ask for. Refreshing the partitions it names would leave the MV in + // SCHEMA_CHANGE with rows nothing rebuilt, and widening it here would rebuild partitions the + // caller deliberately kept out -- the answer is that this request cannot do it, and the + // three that can are the ones whose scope already includes a whole-MV rebuild. + throw new JobException("The refresh baseline of MV " + mtmv.getName() + + " was invalidated, so the partitions it needs to rebuild are not the ones a" + + " PARTITIONS refresh names. Use COMPLETE, AUTO, or PARTITIONS FALLBACK."); + } + LOG.info("IVM MV is in SCHEMA_CHANGE, rebuilding the whole MV, mv={}, taskId={}", + mtmv.getName(), getTaskId()); + return Lists.newArrayList(RefreshAttemptType.COMPLETE); + } if (shouldUseCompleteForInitialIvmRefresh(containsOneRowRelation)) { return Lists.newArrayList(RefreshAttemptType.COMPLETE); } @@ -510,9 +579,58 @@ && hasUnusableIvmStream()) { mtmv.getName(), getTaskId()); return Lists.newArrayList(RefreshAttemptType.COMPLETE); } + // Every partition either needs a rebuild or was never filled, and at least one needs a rebuild: + // COMPLETE then does nothing the per-partition routing would not, in one read of the MV. + // + // Only for a request that may fall back, like the stream shortcut above. The routing those + // partitions would take is the incremental attempt, which rebuilds every one of them and then finds + // no scope left to catch up -- and it reads the streams, so a request that may not fall back still + // fails there when one is unusable. Answering COMPLETE instead does what such a request forbids: it + // reconciles the streams and resets their baselines, which is the same reset the shortcut above + // gates on allowing a fallback. A strict request reaches the incremental attempt, as its intent is + // stated there. + // + // No clause for an explicit partition list: the attempt list says it already. Such a list is + // rejected for an IVM MV when the statement is analyzed, and it becomes a PARTITIONS request here, + // which never reaches the incremental attempt -- so an attempt list that holds one cannot hold the + // other. The schema-change shortcut above needs its own clause because it is judged for requests + // that do name partitions. + if (request.allowFallback && attempts.contains(RefreshAttemptType.IVM) + && mtmv.allPartitionsNeedRebuild()) { + LOG.info("Every MV partition needs a rebuild or has no data yet, mv={}, taskId={}. " + + "Continuing with COMPLETE refresh.", mtmv.getName(), getTaskId()); + return Lists.newArrayList(RefreshAttemptType.COMPLETE); + } return attempts; } + /** + * Notes that this refresh rebuilds partitions the request did not ask to rebuild, which is what a + * strict INCREMENTAL request cannot tell from its result otherwise: it reports the count, and a request + * that asked for a complete refresh reports nothing because rebuilding everything is what it asked for. + * + *

What it counts is the partitions this task has replaced by now -- the batches that committed, read + * from the same accumulator the task reports its progress with, so a refresh that failed part-way + * through cannot report partitions it never replaced. None of them is the empty case: an attempt that + * found nothing to refresh, and one the stream reconciliation threw out of before it reached the + * executor, both report on a task that has committed nothing. That has no count, and it reads as zero + * rather than throwing where the refresh is being reported. + * + *

The value describes the refresh rather than its last attempt: a later attempt's count covers the + * partitions an earlier one rebuilt -- a whole-MV attempt replaces every partition it commits -- so the + * largest any of them reported is the number this refresh replaced, and an attempt that replaced fewer + * of them must not lower it. + */ + private void recordRebuiltPartitions(RefreshRequest request) { + // Only an IVM MV has a baseline to rebuild: a plain MV's COMPLETE is the only way it refreshes at + // all, so reporting it there would put a rebuild count on every ordinary refresh. + if (!mtmv.isIvm() || request.refreshMode == RefreshMode.COMPLETE) { + return; + } + int replaced = CollectionUtils.isEmpty(completedPartitions) ? 0 : completedPartitions.size(); + ivmRebuiltPartitions = Math.max(ivmRebuiltPartitions, replaced); + } + private boolean shouldUseCompleteForInitialIvmRefresh(boolean containsOneRowRelation) { if (!mtmv.isIvm() || mtmv.hasRefreshSnapshot()) { return false; @@ -532,6 +650,14 @@ private PartitionRefreshPlan planPartitionRefresh(MTMVRefreshContext context, if (request.explicitPartitions) { return PartitionRefreshPlan.success(context, request.partitions); } + // The partitions the criterion says have to be rebuilt, whatever the snapshots say: a snapshot + // records what a refresh last read, and a raised requirement says that read cannot be caught up. + // The two disagree in the one state a whole-MV rebuild that did not finish leaves behind -- every + // requirement raised, every snapshot kept -- and planning by snapshots alone would report + // NOT_REFRESH over partitions whose rows nothing repaired. A partition refresh is what rebuilds + // them (the incremental path rebuilds its own dirty set the same way), so they are planned here + // rather than left to an attempt this request may never take. + Set rebuildRequired = mtmv.getPartitionsNeedingRebuild(); boolean fresh; try { fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevelAndFromView(), @@ -539,7 +665,7 @@ private PartitionRefreshPlan planPartitionRefresh(MTMVRefreshContext context, } catch (Exception e) { return PartitionRefreshPlan.fallback(e.getMessage()); } - if (fresh) { + if (fresh && rebuildRequired.isEmpty()) { return PartitionRefreshPlan.success(context, Lists.newArrayList()); } if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.SELF_MANAGE) { @@ -550,14 +676,43 @@ private PartitionRefreshPlan planPartitionRefresh(MTMVRefreshContext context, + "does not support refreshing by partition"); } try { + Set planned = Sets.newLinkedHashSet(MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, + relation.getBaseTablesOneLevelAndFromView())); + planned.addAll(rebuildRequired); return PartitionRefreshPlan.success(context, - MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, - relation.getBaseTablesOneLevelAndFromView())); + excludingRebuiltPartitions(Lists.newArrayList(planned))); } catch (Exception e) { return PartitionRefreshPlan.fallback(e.getMessage()); } } + /** + * Takes the partitions this task has already replaced out of a planned set. + * + *

The plan is computed from the snapshot the MV holds, which this task has not published yet, so a + * partition this task has just filled still looks unsynced to it. Refreshing it here would replace the + * rows it holds with a second read of the same base table, and would do it in the one case that has + * already paid for it: the same refresh falling back out of the incremental attempt. What the + * accumulator holds at this point is exactly those partitions -- the fallback runs after an attempt + * that committed nothing, and a plan is built before anything is written. + * + *

An explicit partition list is left alone. It is the request itself rather than an inference from + * the MV's snapshot, and the rebuild phase does not take partitions out of it either: what the request + * names is refreshed, and at worst it is refreshed twice within one task. + */ + private List excludingRebuiltPartitions(List plannedPartitions) { + if (partitionSnapshots.isEmpty()) { + return plannedPartitions; + } + List remaining = Lists.newArrayListWithCapacity(plannedPartitions.size()); + for (String partitionName : plannedPartitions) { + if (!partitionSnapshots.containsKey(partitionName)) { + remaining.add(partitionName); + } + } + return remaining; + } + private MTMVRefreshContext buildRefreshContext(List tableIfs) throws AnalysisException { MetaLockUtils.readLockTables(tableIfs); try { @@ -567,142 +722,40 @@ private MTMVRefreshContext buildRefreshContext(List tableIfs) throws An } } - /** - * Makes the barrier that says "these MV partitions must be rebuilt before their IVM offsets may be - * used again" durable. Every caller writes it as soon as it has decided the partition set and - * before anything that touches MV data or base table streams, so that a crash or a rejection can - * only ever leave a barrier with no rebuild behind it, which merely costs one extra rebuild, and - * never a rebuild with no barrier, which silently loses rows. - */ - private void writeIvmBaselineBarrier(RefreshMode refreshMode) throws JobException { - if (mtmv.isIvm()) { - // Persist the guard before the first baseline data transaction. - mtmv.persistIvmBaselineGuard(refreshMode, Sets.newHashSet(needRefreshPartitions), - mtmvSchemaChangeVersion); - } - } - private void executeCompleteAttempt(MTMVRefreshContext context, ConnectContext ctx) throws JobException, AnalysisException { - this.needRefreshPartitions = Lists.newArrayList(mtmv.getPartitionNames()); - this.refreshMode = generateRefreshMode(needRefreshPartitions); + List partitions = Lists.newArrayList(mtmv.getPartitionNames()); + // Reported here rather than by the executor alone: a whole-MV attempt covers every partition from + // this point on, and the reconciliation below can throw before the executor is reached -- leaving a + // report that names what the attempt was about to rebuild rather than one that names nothing. + recordRefreshScope(partitions); + // Nothing to clamp a captured epoch against: a whole-MV rebuild replaces every partition, so what + // this refresh read is what it repaired. Dropped rather than kept, so the partitions an earlier + // partial rebuild did replace do not stay looking like they still owe one. + this.ivmPlannedEpochs = Maps.newHashMap(); + this.refreshMode = generateRefreshMode(partitions); if (refreshMode == MTMVTaskRefreshMode.NOT_REFRESH) { return; } - // The barrier goes first: a stream this rebuild reconciles carries the base table's current - // rows as its initial snapshot, and a later incremental refresh that consumed it as a delta - // against data still built from the old baseline would double-count them. - writeIvmBaselineBarrier(RefreshMode.COMPLETE); + // Marked before the streams are reconciled, not merely before the rebuild: reconciling durably + // creates a replacement stream whose historical rows are read as an append, so a crash after that + // create and before this rebuild publishes its epochs would leave a populated MV with clean epochs + // and a usable stream -- and the next incremental refresh would add those rows to the old baseline + // again. The requirement is raised as a real mark on the partitions, which is what "these must be + // rebuilt" means to the refresh, and it is journaled as the whole map before the reconcile below. + mtmv.markPartitionsForRebuild(Sets.newHashSet(partitions)); // A complete rebuild resets the stream baselines, so reconcile missing or unusable streams // before reading anything. Only COMPLETE may do this: a stream baseline is global, resetting // it during a partial refresh would corrupt the partitions that refresh does not touch. if (mtmv.isIvm()) { reconcileIvmStreams(ctx); } - executePartitionBasedRefresh(context, RefreshMode.COMPLETE, ctx); - } - - /** - * Rebuild the MV partitions whose IVM baseline is broken, before the normal refresh runs. - * - *

This is a pre-step, not a terminal branch: the caller keeps running {@code attempts} - * afterwards, so a broken baseline no longer skips the refresh entirely. The list is rewritten - * in place when the baseline demands a different set of attempts. - * - *

Partition sync drops the MV partitions whose base partition disappeared, which is exactly - * what the barrier recorded when that base partition was dropped. Those partitions are resolved - * by the drop itself (the partition and its IVM offsets are both gone), so only the partitions - * that still exist need a rebuild. The barrier is released either way, otherwise the IVM attempt - * that follows would be rejected by {@link MTMV#validateIvmRefreshStart}. - */ - private void handlePendingIvmBaselineRebuild(MTMVRefreshContext context, - RefreshRequest request, ConnectContext ctx, List attempts) - throws JobException, AnalysisException { - if (!mtmv.isIvm() || request.refreshMode == RefreshMode.COMPLETE - || !mtmv.getIvmInfo().isBaselineRebuildRequired()) { - return; - } - ivmFallbackReason = IvmFailureReason.BINLOG_BROKEN.name(); - IvmInfo ivmInfo = mtmv.getIvmInfo(); - // A lone COMPLETE attempt rebuilds every partition anyway, so a partial pre-rebuild here - // would be redundant; it also releases the barrier by itself once it succeeds. - if (attempts.size() == 1 && attempts.get(0) == RefreshAttemptType.COMPLETE) { - LOG.info("IVM baseline barrier is covered by the pending COMPLETE attempt, mv={}, taskId={}", - mtmv.getName(), getTaskId()); - return; - } - if (ivmInfo.requiresCompleteBaselineRebuild()) { - LOG.warn("IVM baseline requires a complete rebuild, mv={}, taskId={}. " - + "Continuing with COMPLETE refresh.", mtmv.getName(), getTaskId()); - attempts.clear(); - attempts.add(RefreshAttemptType.COMPLETE); - return; - } - List baselinePartitions = Lists.newArrayList(Sets.intersection( - ivmInfo.getPendingBaselineRebuildPartitions(), mtmv.getPartitionNames())); - if (baselinePartitions.isEmpty()) { - // Partition sync has already dropped every partition the barrier named, so there is - // nothing left to rebuild. The surviving partitions are picked up by the attempts below. - LOG.info("IVM baseline partitions were removed by partition sync, mv={}, taskId={}", - mtmv.getName(), getTaskId()); - } else { - baselinePartitions.sort(String::compareTo); - // This rebuild reads the streams of the partitions it rebuilds, exactly like any other - // partition refresh, so it judges them before it commits to the rebuild. A request that may - // not fall back fails instead of rebuilding less than it asked for; one that may reaches the - // COMPLETE attempt, which is also the only attempt that reconciles the stream this rebuild - // cannot read. Judging it here rather than in buildAttempts matters for a request whose - // attempt list holds no IVM attempt -- PARTITIONS FALLBACK is exactly that -- because the - // pre-step runs before the attempts do. - if (mtmv.isIvm() - && hasUnusableIvmStreamForPartitions(context, baselinePartitions)) { - if (!request.allowFallback) { - throw new JobException("IVM stream is unusable for the partitions of this refresh, mv=" - + mtmv.getName()); - } - ivmFallbackReason = IvmFailureReason.STREAM_UNSUPPORTED.name(); - LOG.warn("IVM stream is unusable for the partitions this baseline rebuild plans, mv={}, " - + "taskId={}. Continuing with COMPLETE refresh.", mtmv.getName(), getTaskId()); - attempts.clear(); - attempts.add(RefreshAttemptType.COMPLETE); - return; - } - this.needRefreshPartitions = baselinePartitions; - this.refreshMode = generateRefreshMode(baselinePartitions); - writeIvmBaselineBarrier(RefreshMode.PARTITIONS); - // Anything else that fails here is reported as it is -- leaving the barrier behind would - // make the IVM attempt that follows reject the task with "baseline rebuild is pending" - // instead of the real reason. - executePartitionBasedRefresh(context, RefreshMode.PARTITIONS, ctx); - } - mtmv.releaseIvmBaselineRebuild(mtmvSchemaChangeVersion); - } - - private void validateIvmBaselineBeforePartitionSync(RefreshRequest request) throws JobException { - if (!mtmv.isIvm() || request.refreshMode == RefreshMode.COMPLETE) { - return; - } - IvmInfo ivmInfo = mtmv.getIvmInfo(); - if (!ivmInfo.isBaselineRebuildRequired()) { - return; - } - ivmFallbackReason = IvmFailureReason.BINLOG_BROKEN.name(); - if ((request.refreshMode == RefreshMode.INCREMENTAL && !request.allowFallback) - || request.explicitPartitions) { - refreshMode = MTMVTaskRefreshMode.NOT_REFRESH; - throw new JobException("IVM baseline rebuild is pending for mv=" + mtmv.getName() - + "; run an AUTO or COMPLETE refresh first"); - } - if (request.refreshMode == RefreshMode.PARTITIONS && !request.allowFallback - && ivmInfo.requiresCompleteBaselineRebuild()) { - refreshMode = MTMVTaskRefreshMode.NOT_REFRESH; - throw new JobException("COMPLETE IVM baseline rebuild is pending for mv=" + mtmv.getName() - + "; run a PARTITIONS FALLBACK, AUTO, or COMPLETE refresh"); - } + executePartitionBasedRefresh(context, RefreshMode.COMPLETE, ctx, partitions); } private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext, - RefreshRequest request, ConnectContext ctx, List tableIfs) throws JobException { + RefreshRequest request, ConnectContext ctx, List tableIfs) + throws JobException, AnalysisException { if (!mtmv.isIvm()) { throw new JobException("Cannot use " + request.refreshMode + " refresh on a materialized view without INCREMENTAL capability."); @@ -715,12 +768,47 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext, + "Continuing with COMPLETE refresh.", mtmv.getName(), getTaskId()); return AttemptResultType.FALLBACK_TO_COMPLETE; } + // The partitions the criterion says must be rebuilt rather than caught up: the delta path can only + // append, so a partition it treated as current would record that in its epoch while its rows still + // come from before the change. Rebuilt first, with the partition executor, because that is the + // full recomputation they need -- and only in this task's batches, so a change that arrives while + // it runs leaves them dirty for the next round instead of being swallowed. + // One read of the states decides both what has to be rebuilt and the requirement each batch may + // write back. Reading them separately would leave a window between the two in which a mark lands, + // the routing decision does not see it, and the batch that follows captures the raised requirement + // and records it as met by a delta that cannot remove the rows that mark made unusable. + Map plannedStates = mtmv.getPartitionStates(); + Set livePartitionNames = mtmv.getPartitionNames(); + Set dirtyPartitions = Sets.newLinkedHashSet(); + Map plannedEpochs = Maps.newHashMap(); + for (Entry plannedState : plannedStates.entrySet()) { + if (!livePartitionNames.contains(plannedState.getKey())) { + continue; + } + plannedEpochs.put(plannedState.getKey(), plannedState.getValue().getLatestEpoch()); + if (plannedState.getValue().isDirty()) { + dirtyPartitions.add(plannedState.getKey()); + } + } + this.ivmPlannedEpochs = plannedEpochs; + if (!dirtyPartitions.isEmpty()) { + LOG.info("Rebuilding {} invalidated MV partitions before the incremental refresh, mv={}, taskId={}", + dirtyPartitions.size(), mtmv.getName(), getTaskId()); + List toRebuild = Lists.newArrayList(dirtyPartitions); + toRebuild.sort(Comparator.naturalOrder()); + this.refreshMode = generateRefreshMode(toRebuild); + try { + executePartitionBasedRefresh(refreshContext, RefreshMode.PARTITIONS, ctx, toRebuild); + } finally { + recordRebuiltPartitions(request); + } + } MTMVRefreshContext currentRefreshContext = refreshContext; int ivmAttemptLimit = Math.max(Config.max_query_retry_time, 0) + 1; IvmIncrRefreshResult ivmResult = null; for (int partitionSyncRetryCount = 0; partitionSyncRetryCount < ivmAttemptLimit; partitionSyncRetryCount++) { - ivmResult = executeSingleIvmAttempt(currentRefreshContext); + ivmResult = executeSingleIvmAttempt(currentRefreshContext, dirtyPartitions); if (ivmResult.isSuccess()) { return AttemptResultType.SUCCESS; } @@ -732,6 +820,16 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext, } try { syncPartitionsIfNeeded(ctx, tableIfs); + // The retry can add a partition that did not exist at the first alignment. It has to get + // its entry before the retried refresh reads a base table, or an invalidation arriving + // in between would have nothing to land on for rows this task is about to write. + mtmv.alignPartitionStates(); + // Alignment gives a partition it creates {0, 1} -- behind its requirement -- and the routing + // decision was taken before that partition existed. Without a fresh read the retried attempt + // would hand it to the delta path with no ceiling to be clamped against, and its capture, + // which is all that path can produce, would be recorded as the partition being caught up: + // a baseline it never received, and no repairing delta to notice. + adoptPartitionsCreatedByTheRetry(dirtyPartitions); currentRefreshContext = buildRefreshContext(tableIfs); } catch (Exception e) { throw new JobException("Failed to synchronize MV partitions before IVM retry for mv=" @@ -744,14 +842,17 @@ private AttemptResultType executeIvmAttempt(MTMVRefreshContext refreshContext, + mtmv.getName() + ", detail=" + ivmResult.getDetailMessage()); } - private IvmIncrRefreshResult executeSingleIvmAttempt(MTMVRefreshContext refreshContext) + private IvmIncrRefreshResult executeSingleIvmAttempt(MTMVRefreshContext refreshContext, + Set dirtyPartitions) throws JobException { - this.completedPartitions = Lists.newCopyOnWriteArrayList(); - this.partitionSnapshots = Maps.newConcurrentMap(); - // Determine which partitions need refresh, same as partition-based flow. - this.needRefreshPartitions = MTMVPartitionUtil.getMTMVNeedRefreshPartitions(refreshContext, - relation.getBaseTablesOneLevelAndFromView()); - if (CollectionUtils.isEmpty(needRefreshPartitions)) { + // Determine which partitions need refresh, same as partition-based flow. The partitions the + // rebuild above handled are taken out: an incremental refresh of one of them would record it as + // caught up while its rows are exactly what the rebuild had to replace. + Set incrementalScope = Sets.newLinkedHashSet(MTMVPartitionUtil.getMTMVNeedRefreshPartitions( + refreshContext, relation.getBaseTablesOneLevelAndFromView())); + incrementalScope.removeAll(dirtyPartitions); + recordRefreshScope(incrementalScope); + if (CollectionUtils.isEmpty(incrementalScope)) { LOG.info("IVM incremental refresh skipped for mv={}: all partitions are synced, taskId={}", mtmv.getName(), getTaskId()); return IvmIncrRefreshResult.success(); @@ -764,10 +865,13 @@ private IvmIncrRefreshResult executeSingleIvmAttempt(MTMVRefreshContext refreshC try { capturedSnapshots = MTMVPartitionUtil.generatePartitionSnapshots( refreshContext, relation.getBaseTablesOneLevelAndFromView(), - Sets.newHashSet(needRefreshPartitions)); + Sets.newHashSet(incrementalScope)); } catch (Exception e) { throw new JobException("IVM snapshot generation failed for mv=" + mtmv.getName(), e); } + // The requirement these partitions are read under, captured before the read inside doRefresh and + // recorded only if the refresh commits; see captureLatestEpochs. + Map capturedEpochs = captureLatestEpochs(incrementalScope); IvmIncrRefreshResult ivmResult; try { ivmResult = executeWithRetry(() -> { @@ -777,7 +881,7 @@ private IvmIncrRefreshResult executeSingleIvmAttempt(MTMVRefreshContext refreshC setupComputeGroup(ivmConnectContext); IvmIncrRefreshContext ivmIncrRefreshContext = new IvmIncrRefreshContext(mtmv, ivmConnectContext, - getRefreshAuditStmt(RefreshMode.INCREMENTAL, Sets.newHashSet(needRefreshPartitions)), + getRefreshAuditStmt(RefreshMode.INCREMENTAL, Sets.newHashSet(incrementalScope)), this::recordQueryId, this::registerExecutor); mtmv.validateIvmRefreshStart(mtmvSchemaChangeVersion); @@ -792,13 +896,117 @@ private IvmIncrRefreshResult executeSingleIvmAttempt(MTMVRefreshContext refreshC } if (ivmResult.isSuccess()) { this.partitionSnapshots.putAll(capturedSnapshots); - this.completedPartitions.addAll(needRefreshPartitions); + recordRefreshCompleted(incrementalScope); + commitCapturedEpochs(capturedEpochs); LOG.info("IVM incremental refresh succeeded for mv={}, taskId={}", mtmv.getName(), getTaskId()); } return ivmResult; } + /** + * Adds a phase's scope to what this task reports as refreshed, and the partitions it committed to what + * this task reports as done. Both are the task's; see the fields for why. + */ + private void recordRefreshScope(Collection partitions) { + // Created by the first phase that records one: a task that has not refreshed anything reports + // nothing, which is the same state as one that has not run yet. Concurrent because the columns it + // feeds are read while the worker fills them -- the tasks() table function reports a running task + // -- and ordered because what this reports is persisted and has to read the same on every look. + if (needRefreshPartitions == null) { + needRefreshPartitions = new ConcurrentSkipListSet<>(); + } + needRefreshPartitions.addAll(partitions); + } + + private void recordRefreshCompleted(Collection partitions) { + if (completedPartitions == null) { + completedPartitions = new ConcurrentSkipListSet<>(); + } + completedPartitions.addAll(partitions); + } + + /** + * Brings the routing decision up to date after a retry has synchronized and aligned the MV's partitions. + * + *

A partition the alignment creates is dirty by construction -- {@code {0, 1}}, behind its + * requirement -- and the decision was taken before it existed. Reading the states again is what makes + * the retried attempt treat it as such: it joins the dirty set, so the incremental attempt leaves it + * out rather than recording what a delta captured as the partition being caught up, and it gets the + * entry the batches are clamped against, so a mark landing later in this task cannot be written back as + * satisfied either. + * + *

A partition this leaves dirty is not rebuilt here. The rebuild phase has already run, and the + * partition the alignment created holds no rows yet -- there is nothing to replace in it -- so what it + * needs is a build, which is what the next refresh's rebuild gives it. + * + *

The planned value of a partition that already had one is kept: that is the value the routing + * decision was made on, which is what the clamp is for. + */ + private void adoptPartitionsCreatedByTheRetry(Set dirtyPartitions) { + Set livePartitionNames = mtmv.getPartitionNames(); + for (Entry entry : mtmv.getPartitionStates().entrySet()) { + if (!livePartitionNames.contains(entry.getKey())) { + continue; + } + ivmPlannedEpochs.putIfAbsent(entry.getKey(), entry.getValue().getLatestEpoch()); + if (entry.getValue().isDirty()) { + dirtyPartitions.add(entry.getKey()); + } + } + } + + /** + * Captures the requirement these partitions are about to be read under: the epoch in force at the + * moment the refresh starts reading, which is what the data it writes will be described by. + * + *

A refresh writes back the requirement it captured, not the one in force when it finishes, which + * is what keeps an invalidation that arrives while the refresh runs from being swallowed: the + * requirement it raises stays above the value the task writes, so the partition still counts as + * needing a rebuild. + * + *

It has to run before the base tables are read and never after. An epoch captured after the read + * could claim data newer than what the read saw, and the partition would then look caught up while + * it holds rows from before the change. + * + *

The caller keeps the result and hands it to {@link #commitCapturedEpochs} only once that batch's + * data has committed. Recording it here would credit a batch whose write never happened with data + * that does not exist, which is the one direction the epoch must never be wrong in. + * + *

A non-IVM MV carries no states, so this captures nothing for it. + */ + private Map captureLatestEpochs(Set partitionNames) { + if (CollectionUtils.isEmpty(partitionNames)) { + return Maps.newHashMap(); + } + return mtmv.getLatestEpochs(partitionNames); + } + + /** + * Commits the captured epochs of a batch whose data has landed, so its work is not repeated after a + * restart; see {@link #captureLatestEpochs} for what a captured value is. A partition read by two + * phases of one task keeps the higher value, which is the requirement its surviving data was read + * under. + */ + private void commitCapturedEpochs(Map capturedEpochs) { + for (Entry entry : capturedEpochs.entrySet()) { + ivmCapturedEpochs.merge(entry.getKey(), plannedCeiling(entry), Math::max); + } + } + + /** + * The epoch to record for a captured partition: the one it was read at, or the one the routing decision + * saw when that is lower -- see {@link #captureLatestEpochs} for the rule this serves. + * + *

The clamp is what keeps an invalidation that lands between that decision and this read from being + * recorded as satisfied: the delta cannot remove the rows it made unusable, so the partition has to + * stay dirty for the next refresh to rebuild. + */ + private long plannedCeiling(Entry captured) { + Long planned = ivmPlannedEpochs.get(captured.getKey()); + return planned == null ? captured.getValue() : Math.min(captured.getValue(), planned); + } + private AttemptResultType handleIvmFallbackResult(IvmIncrRefreshResult ivmResult, RefreshRequest request) throws JobException { ivmFallbackReason = ivmResult.getFailureReason().name(); @@ -841,15 +1049,15 @@ private boolean executePartitionBasedRefresh(MTMVRefreshContext refreshContext, } throw new JobException(partitionPlan.fallbackReason); } - this.needRefreshPartitions = partitionPlan.partitions; - this.refreshMode = generateRefreshMode(needRefreshPartitions); + recordRefreshScope(partitionPlan.partitions); + this.refreshMode = generateRefreshMode(partitionPlan.partitions); // This attempt now knows which partitions it refreshes, and with them which streams it reads. // Judged here rather than in buildAttempts because only the plan knows that scope, and judged // before the NOT_REFRESH return below so that a fallback-capable request still reaches the only // attempt that reconciles streams. Falling back here continues to COMPLETE, which is what repairs // them; a request that may not fall back fails instead of quietly refreshing less than it asked. if (mtmv.isIvm() - && hasUnusableIvmStreamForPartitions(partitionPlan.context, needRefreshPartitions)) { + && hasUnusableIvmStreamForPartitions(partitionPlan.context, partitionPlan.partitions)) { if (!request.allowFallback) { throw new JobException("IVM stream is unusable for the partitions of this refresh, mv=" + mtmv.getName()); @@ -862,22 +1070,27 @@ && hasUnusableIvmStreamForPartitions(partitionPlan.context, needRefreshPartition if (refreshMode == MTMVTaskRefreshMode.NOT_REFRESH) { return true; } - writeIvmBaselineBarrier(RefreshMode.PARTITIONS); - executePartitionBasedRefresh(partitionPlan.context, RefreshMode.PARTITIONS, ctx); + executePartitionBasedRefresh(partitionPlan.context, RefreshMode.PARTITIONS, ctx, + partitionPlan.partitions); return true; } private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMode refreshMode, - ConnectContext ctx) + ConnectContext ctx, List partitions) throws JobException, AnalysisException { + // Reported here rather than by the callers alone: this is the phase that runs, so this is the scope + // the report cannot do without -- a caller that forgot would leave the partitions it committed in + // the completed side and nothing in the scope, which reads as a refresh of no partitions. The + // callers name a scope as well, and only where they decide one and may not get here: a whole-MV + // attempt before it reconciles the streams, and a partition plan before it judges their streams. + recordRefreshScope(partitions); boolean useIvmFallbackStreams = mtmv.isIvm(); Map tableWithPartKey = getIncrementalTableMap(); - this.completedPartitions = Lists.newCopyOnWriteArrayList(); try { // Snapshot persistence happens after refresh partitions are split into execution groups. Load the // complete union here so the default one-partition group size cannot turn a large Hive MTMV into // one metadata request per MV partition; generatePartitionSnapshots reuses this context cache. - context.preparePartitionSnapshots(Sets.newHashSet(needRefreshPartitions)); + context.preparePartitionSnapshots(Sets.newHashSet(partitions)); } catch (Exception e) { // Preloading is only a batching optimization. Retrying through the existing per-group load below // preserves completed-group progress when a later chunk of the union fails. @@ -885,18 +1098,25 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod + "falling back to per-group loading", mtmv.getName(), getTaskId(), e); } int refreshPartitionNum = mtmv.getRefreshPartitionNum(); - long execNum = (needRefreshPartitions.size() / refreshPartitionNum) + ((needRefreshPartitions.size() + long execNum = (partitions.size() / refreshPartitionNum) + ((partitions.size() % refreshPartitionNum) > 0 ? 1 : 0); - boolean refreshAllPartitions = Sets.newHashSet(needRefreshPartitions).equals(mtmv.getPartitionNames()); + boolean refreshAllPartitions = Sets.newHashSet(partitions).equals(mtmv.getPartitionNames()); + // Only a COMPLETE that the signature-mismatch fallback asked for publishes its signature. Widening + // this to every COMPLETE an IVM MV runs -- the escalation an invalidated MV takes is the other one + // -- is not the free saving it looks like: the fresh signature lets the following refresh take the + // incremental path instead of that fallback, and the incremental then re-applies rows the COMPLETE + // had just rebuilt. Three suites report their delta twice once it is widened (test_ivm_snapshot, + // test_ivm_bitmap_agg_2, test_ivm_agg_array_1), so the second COMPLETE is protection rather than + // waste. What has to be understood first is how the incremental path baselines itself against a + // plan that has changed; until then this condition stays narrow. Non-IVM MVs produce no signature. boolean capturePlanSignature = refreshMode == RefreshMode.COMPLETE && IvmFailureReason.PLAN_SIGNATURE_MISMATCH.name().equals(ivmFallbackReason); - this.partitionSnapshots = Maps.newConcurrentMap(); IvmPlanSignature refreshedPlanSignature = null; for (int i = 0; i < execNum; i++) { int start = i * refreshPartitionNum; int end = start + refreshPartitionNum; - Set execPartitionNames = Sets.newHashSet(needRefreshPartitions - .subList(start, Math.min(end, needRefreshPartitions.size()))); + Set execPartitionNames = Sets.newHashSet(partitions + .subList(start, Math.min(end, partitions.size()))); Map> batchResetPartitionIds = useIvmFallbackStreams ? collectPctResetPartitionIds(context, execPartitionNames) : Maps.newHashMap(); Optional rewriteContext = Optional.empty(); @@ -906,6 +1126,11 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod rewriteContext = Optional.of( IvmRewriteContext.full(mtmv, batchResetPartitionIds, nonPctReadMode)); } + // The requirement this batch is read under, captured before the read below and recorded once + // the read's data has committed, next to its snapshots. Capturing it per batch keeps an + // invalidation that arrives during the refresh from holding back the whole round: only the + // batches already read keep a requirement above their captured value. + Map batchCapturedEpochs = captureLatestEpochs(execPartitionNames); // need get names before exec Map execPartitionSnapshots = MTMVPartitionUtil .generatePartitionSnapshots(context, relation.getBaseTablesOneLevelAndFromView(), @@ -928,8 +1153,9 @@ private void executePartitionBasedRefresh(MTMVRefreshContext context, RefreshMod mtmv.getName(), getTaskId(), e); throw new JobException(e.getMessage(), e); } - completedPartitions.addAll(execPartitionNames); + recordRefreshCompleted(execPartitionNames); partitionSnapshots.putAll(execPartitionSnapshots); + commitCapturedEpochs(batchCapturedEpochs); } if (capturePlanSignature) { refreshedIvmPlanSignature = refreshedPlanSignature.getSha256(); @@ -1455,6 +1681,7 @@ public TRow getTvfInfo(String jobName) { computeGroup == null || computeGroup.isEmpty() ? FeConstants.null_string : computeGroup)); trow.addToColumnValue(new TCell().setStringVal( ivmFallbackReason == null ? FeConstants.null_string : ivmFallbackReason)); + trow.addToColumnValue(new TCell().setStringVal(String.valueOf(ivmRebuiltPartitions))); return trow; } @@ -1532,6 +1759,22 @@ public long getMtmvSchemaChangeVersion() { return mtmvSchemaChangeVersion; } + /** + * The epochs this task's committed batches were read at -- the requirement each refreshed partition was + * read under, see {@link #captureLatestEpochs} -- as a detached copy. + * + *

Detached rather than live: the batches merge into the map on the thread running the task, while a + * STOP publishes what it holds from the callback thread -- `cancel(false)` does not wait for the + * execution to stop -- so a caller reading the field itself would iterate a map still being written, and + * would journal a result that goes on changing after it was read. + */ + public Map getIvmCapturedEpochs() { + // A task read back from the journal has none: the field is transient, so gson leaves it null and the + // constructor that would have initialized it never runs. What a replay applies is the states the + // record carries, not this. + return ivmCapturedEpochs == null ? Collections.emptyMap() : Maps.newHashMap(ivmCapturedEpochs); + } + public String getRefreshedIvmPlanSignature() { return refreshedIvmPlanSignature; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionState.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionState.java index edb30f84177477..2627d85dd626e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionState.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionState.java @@ -48,6 +48,7 @@ public class MTMVPartitionState { @SerializedName("le") private long latestEpoch; + public MTMVPartitionState() { } @@ -61,6 +62,35 @@ public MTMVPartitionState(MTMVPartitionState other) { this.latestEpoch = other.latestEpoch; } + /** + * The state a partition gets when it is first aligned: never refreshed, one generation required. + * + *

Alignment only ever creates this pair, so "no entry yet" and "this pair" say the same thing + * about the past -- the partition was never marked and holds no rows. + */ + public static MTMVPartitionState initial() { + return new MTMVPartitionState(0, 1); + } + + /** + * Whether this partition holds rows that a metadata-only change of a base table has made unusable, + * so it has to be rebuilt rather than caught up incrementally. + * + *

{@code refreshEpoch == 0} carries no exemption. It reads as "the partition was never refreshed, + * so it holds no rows", and that is not durable: a refresh commits the MV data transaction before its + * task result publishes the epochs, so a crash in between leaves rows in a partition whose state says + * never refreshed ({@code {0, 1}}). Reading that pair as clean would let a later invalidation raising + * {@code latestEpoch} -- {@code {0, 2}} -- go unnoticed, and a strict INCREMENTAL refresh would keep + * rows nothing can remove. Without the exemption the cut closes on its own: {@code 2 > 0} is dirty. + * + *

The cost is that a fresh MV's first refresh rebuilds every partition instead of skipping the + * partitions whose base partitions have no rows. That is the safe direction. + */ + public boolean isDirty() { + return latestEpoch > refreshEpoch; + } + + /** * Deep-copies a state map, or returns null for null. * @@ -95,4 +125,5 @@ public long getLatestEpoch() { public void setLatestEpoch(long latestEpoch) { this.latestEpoch = latestEpoch; } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 45c528449a86a2..b647652354568b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -107,6 +107,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -1075,13 +1076,49 @@ private static void checkColumnIfChange(MTMV mtmv, List analyz + "original length is: %s, current length is: %s", originalColumns.size(), analyzedColumns.size())); } - for (int i = 0; i < originalColumns.size(); i++) { - if (!isTypeLike(originalColumns.get(i).getType(), analyzedColumns.get(i).getType())) { + if (!mtmv.isIvm()) { + // Positional, as this check has always been for a plain MV: nothing lays its schema out twice, + // so the analysed list is the stored one in order. Matching these by name is not a refinement + // -- it is a different answer -- because a plain MV keeps the query as it was written, and an + // MV created with column names of its own (`CREATE MATERIALIZED VIEW mv (x, y) AS SELECT a, b`) + // persists x/y while re-analysing that query yields a/b. The change this check is for is a + // column that disappeared or changed type, and position plus type is what detects it. + for (int i = 0; i < originalColumns.size(); i++) { + if (!isTypeLike(originalColumns.get(i).getType(), analyzedColumns.get(i).getType())) { + throw new JobException(String.format( + "column type not same, please check whether columns of base table have changed, " + + "column name is: %s, original type is: %s, current type is: %s", + originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(), + analyzedColumns.get(i).getType().toSql())); + } + } + return; + } + // An IVM MV is matched by name, not by position. Its schema is laid out by two passes: + // MTMVPlanUtil#applyIvmPhysicalKeyLayout puts the final key columns first when the MV is created, + // and the analysed list comes from running that same layout again with the stored key columns as + // its input. The two agree except for a chained IVM MV whose base tables carry row-id columns of + // their own: the create pass derives the visible key prefix from the identity key slots, the + // analysed one takes it from the stored keys, and the base tables' row-id columns end up in a + // different block. Where a column sits is not part of what this check is for. + Map originalByName = Maps.newHashMap(); + for (Column column : originalColumns) { + originalByName.put(column.getName().toLowerCase(Locale.ROOT), column); + } + for (Column analyzedColumn : analyzedColumns) { + Column originalColumn = originalByName.get(analyzedColumn.getName().toLowerCase(Locale.ROOT)); + if (originalColumn == null) { + throw new JobException(String.format( + "column not found, please check whether columns of base table have changed, " + + "column name is: %s", + analyzedColumn.getName())); + } + if (!isTypeLike(originalColumn.getType(), analyzedColumn.getType())) { throw new JobException(String.format( "column type not same, please check whether columns of base table have changed, " + "column name is: %s, original type is: %s, current type is: %s", - originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(), - analyzedColumns.get(i).getType().toSql())); + analyzedColumn.getName(), originalColumn.getType().toSql(), + analyzedColumn.getType().toSql())); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java index bc6e1827c9e35e..4d99de5ef17660 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import java.util.Iterator; @@ -86,6 +87,17 @@ public void updateSnapshots(Map addPartiti } } + /** + * Drops these MV partitions' snapshots. An invalidated partition must not be able to back a + * transparent rewrite until it has been rebuilt: what it holds is exactly what the rebuild replaces. + */ + public void removeSnapshots(Set mvPartitionNames) { + if (CollectionUtils.isEmpty(mvPartitionNames)) { + return; + } + partitionSnapshots.keySet().removeAll(mvPartitionNames); + } + public Map getPartitionSnapshots() { return partitionSnapshots; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index 41a53adf77bfd9..51360b36790faf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -17,7 +17,6 @@ package org.apache.doris.mtmv; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; @@ -28,7 +27,6 @@ import org.apache.doris.job.common.TaskStatus; import org.apache.doris.job.exception.JobException; import org.apache.doris.job.extensions.mtmv.MTMVTask; -import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator; import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo; import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo; @@ -117,10 +115,12 @@ private void markIvmBaselineRebuild(BaseTableInfo baseTableInfo, boolean allPart } boolean invalidated; if (allPartitionsChanged) { - mtmv.invalidateIvmBaseline(); + // Awaited here, where no MV lock is held: the DDL does not return until the invalidation is + // durable, which is what it was before the record was handed back to the caller. + mtmv.invalidateWholeMv(reason).await(); invalidated = true; } else { - invalidated = mtmv.invalidateIvmBaseline(baseTableInfo, changedPartitions); + invalidated = mtmv.invalidateIvmBaseline(baseTableInfo, changedPartitions, reason); } // A partition change that no MV partition reads leaves nothing to rebuild, and saying that it // invalidated the baseline would claim a persisted barrier that does not exist. @@ -344,13 +344,16 @@ public void refreshComplete(MTMV mtmv, MTMVRelation relation, MTMVTask task) { */ @Override public void dropTable(Table table) { - // A dropped base table is already caught by the IVM stream guard (the stream records the - // base table id, so it stops being usable once the table is gone), no need to re-analyze. + // The message below names the table and what became of it, which is what an MV that reads it has to + // know; the query check would replace that with the weaker "the query is no longer analyzable", + // because a dropped table is the one change whose query is gone beyond doubt. What the two record + // is the same state either way. Unlike a rename it stays an invalidation: the table is gone for + // good, so the state is not something a later alter can make obsolete. processBaseTableChange(new BaseTableInfo(table), "The base table has been deleted:", false); } /** - * update mtmv status to `SCHEMA_CHANGE` + * update mtmv status to `SCHEMA_CHANGE`. * * @param isReplace */ @@ -361,35 +364,43 @@ public void alterTable(BaseTableInfo oldTableInfo, Optional newTa // REPLACE TABLE already invalidates the IVM baseline explicitly, see Alter#processReplaceTable processBaseTableChange(newTableInfo.get(), "The base table has been updated:", false); } - // A RENAME leaves every column alone, and the failure it does cause -- the MV query still - // spells the old name -- is already reported by the refresh itself (MTMVTask#run resolves - // the base tables from the query before it ever looks at the baseline). Invalidating here - // would only leave a stale flag behind: rename the table back and the query is analyzable - // again, yet every strict INCREMENTAL refresh would stay rejected until a COMPLETE one ran. boolean renamed = !isReplace && newTableInfo.isPresent() && !Objects.equals(oldTableInfo.getTableName(), newTableInfo.get().getTableName()); - processBaseTableChange(oldTableInfo, "The base table has been updated:", !renamed); + // A rename is the one change whose query check is skipped: the MV query keeps spelling the old + // name, so it is unanalyzable by construction, and the reason it would be invalidated with -- + // "the query is no longer analyzable" -- says less than the message this call records anyway. + boolean checkQueryUsable = !renamed; + processBaseTableChange(oldTableInfo, "The base table has been updated:", checkQueryUsable); } + /** - * An IVM baseline is only valid while the MV query can still be analyzed against the current - * base table schema. Re-analyzing the MV query here (right after the alter was applied) is what - * detects a changed column identity: dropping or renaming a column the MV uses makes the query - * unanalyzable, and a column re-added with the same name is a different column, so pre-existing - * rows read its default value instead. + * An MV's query is only as good as the base table schema it was analyzed against. Re-analyzing the + * MV query here (right after the alter was applied) is what detects a changed column identity: + * dropping or renaming a column the MV uses makes the query unanalyzable, and a column re-added with + * the same name is a different column, so pre-existing rows read its default value instead. * *

Such a change is metadata-only for light schema changes and emits no binlog, so an * incremental refresh would consume an empty delta and report SUCCESS while silently keeping the - * rows computed under the old column epoch. Invalidating the baseline makes a strict INCREMENTAL - * refresh fail and tell the user to run a COMPLETE refresh instead. + * rows computed under the old column epoch. Invalidating the MV is what keeps that from being + * reported as current. + * + *

Every MV is checked, not only an IVM one: whether the query still analyzes is a property of + * the MV and of the base table it reads, not of how the MV refreshes, and the invalidation is the + * same one a change to that table records. What an IVM MV has on top of it is a per-partition + * requirement, and that is decided elsewhere, from a query that analyzed. * - *

Only IVM is covered: a plain MTMV keeps its previous behaviour (status only). + * @return whether the MV was invalidated. That is the whole record for this change: the invalidation + * carries the reason, and the caller has nothing left to write -- a second record would land + * on the same state, and MTMVStatus#updateStateAndDetail would overwrite the detail with the + * blunter "the base table has been updated", which is what knowing the query is unusable is + * for. It would also bump the version and drop the snapshot twice for one change. */ - private void invalidateIvmBaselineIfQueryUnusable(BaseTableInfo baseTableInfo, Table mtmvTable) { - if (!(mtmvTable instanceof MTMV) || !((MTMV) mtmvTable).isIvm()) { - return; + private boolean invalidateMvIfQueryUnusable(BaseTableInfo baseTableInfo, Table mvTable) { + if (!(mvTable instanceof MTMV)) { + return false; } - MTMV mtmv = (MTMV) mtmvTable; + MTMV mtmv = (MTMV) mvTable; // Analyse in a context owned by this check, never the session that issued the alter: the check // must not disturb the running statement, and it has to work on threads that have no session. // Setting a thread local is how a context is made current, so restore the previous one. @@ -398,9 +409,10 @@ private void invalidateIvmBaselineIfQueryUnusable(BaseTableInfo baseTableInfo, T MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK)); } catch (Exception e) { - LOG.info("Invalidate IVM baseline, the MV query is no longer usable. baseTable={}, mtmv={}, " - + "reason={}", baseTableInfo, mtmv.getName(), e.getMessage()); - mtmv.invalidateIvmBaseline(); + LOG.info("Invalidate MV, the MV query is no longer usable. baseTable={}, mtmv={}, reason={}", + baseTableInfo, mtmv.getName(), e.getMessage()); + mtmv.invalidateWholeMv("The MV query is no longer analyzable: " + baseTableInfo).await(); + return true; } finally { if (previousCtx != null) { previousCtx.setThreadLocalInfo(); @@ -408,6 +420,7 @@ private void invalidateIvmBaselineIfQueryUnusable(BaseTableInfo baseTableInfo, T ConnectContext.remove(); } } + return false; } @Override @@ -469,28 +482,40 @@ private void processBaseViewChange(BaseTableInfo baseViewInfo, String msgPrefix) } } + /** + * Puts every MV that reads this base table into {@code SCHEMA_CHANGE}. + * + * @param checkQueryUsable whether to re-analyze each MV's query first; see + * {@link #invalidateMvIfQueryUnusable} + */ private void processBaseTableChange(BaseTableInfo baseTableInfo, String msgPrefix, - boolean checkIvmQueryUsable) { + boolean checkQueryUsable) { Set mtmvsByBaseTable = getMtmvsByBaseTableOneLevelAndFromView(baseTableInfo); if (CollectionUtils.isEmpty(mtmvsByBaseTable)) { return; } for (BaseTableInfo mtmvInfo : mtmvsByBaseTable) { - Table mtmv = null; + Table mvTable = null; try { - mtmv = (Table) MTMVUtil.getTable(mtmvInfo); + mvTable = (Table) MTMVUtil.getTable(mtmvInfo); } catch (AnalysisException e) { LOG.warn(e); continue; } - if (checkIvmQueryUsable) { - invalidateIvmBaselineIfQueryUnusable(baseTableInfo, mtmv); + if (checkQueryUsable && invalidateMvIfQueryUnusable(baseTableInfo, mvTable)) { + // Invalidated with the reason, which is the more specific of the two messages and the one + // this change is worth recording: the state is the same one the generic record below + // would set, so writing it too would only bury the reason. + continue; + } + if (!(mvTable instanceof MTMV)) { + continue; } - TableNameInfo tableNameInfo = new TableNameInfo(mtmv.getQualifiedDbName(), - mtmv.getName()); - MTMVStatus status = new MTMVStatus(MTMVState.SCHEMA_CHANGE, - msgPrefix + baseTableInfo); - Env.getCurrentEnv().alterMTMVStatus(tableNameInfo, status); + // Applied and enqueued in one MV-lock critical section, like the invalidation above: they are + // one change, and a task result enqueued between them would be replayed on a follower after + // this record rather than before it -- leaving the follower in SCHEMA_CHANGE where this FE + // ended NORMAL, which is a whole-MV rebuild the next refresh does not need. + ((MTMV) mvTable).invalidateWholeMv(msgPrefix + baseTableInfo).await(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java index 2708810770e1a6..ac9c70469aed8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java @@ -17,12 +17,8 @@ package org.apache.doris.mtmv.ivm; -import com.google.common.base.Preconditions; import com.google.gson.annotations.SerializedName; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; /** * Thin persistent IVM metadata stored on MTMV. @@ -35,16 +31,6 @@ public class IvmInfo { @SerializedName("en") private boolean enableIvm = false; - @SerializedName("brr") - // Keep an explicit COMPLETE requirement because persisted partition names are a point-in-time snapshot. - // For example, if the MV has {p1, p2} when invalidated and partition sync adds p3 before refresh, - // COMPLETE must rebuild p3 too. - private boolean completeBaselineRebuildRequired; - - @SerializedName("brp") - // MV partitions that must be rebuilt before their IVM offsets can be used again. - private Set pendingBaselineRebuildPartitions = new HashSet<>(); - /** Persisted ivm_use_full_keys flag: true means the MV unique keys include identity key columns. */ @SerializedName("ukf") private boolean useFullKeys = false; @@ -62,8 +48,6 @@ public IvmInfo() { public IvmInfo(IvmInfo other) { this.enableIvm = other.enableIvm; - this.completeBaselineRebuildRequired = other.completeBaselineRebuildRequired; - this.pendingBaselineRebuildPartitions = new HashSet<>(other.pendingBaselineRebuildPartitions); this.useFullKeys = other.useFullKeys; this.planSignature = other.planSignature; this.sequencePrefix = other.sequencePrefix; @@ -77,35 +61,6 @@ public void setEnableIvm(boolean enableIvm) { this.enableIvm = enableIvm; } - public boolean isBaselineRebuildRequired() { - return completeBaselineRebuildRequired || !pendingBaselineRebuildPartitions.isEmpty(); - } - - public boolean requiresCompleteBaselineRebuild() { - return completeBaselineRebuildRequired; - } - - public Set getPendingBaselineRebuildPartitions() { - return Collections.unmodifiableSet(new HashSet<>(pendingBaselineRebuildPartitions)); - } - - public void requireCompleteBaselineRebuild() { - completeBaselineRebuildRequired = true; - pendingBaselineRebuildPartitions.clear(); - } - - public void addPendingBaselineRebuildPartitions(Set partitions) { - Preconditions.checkArgument(!partitions.isEmpty(), "baseline rebuild partitions can not be empty"); - if (!completeBaselineRebuildRequired) { - pendingBaselineRebuildPartitions.addAll(partitions); - } - } - - public void clearBaselineRebuild() { - completeBaselineRebuildRequired = false; - pendingBaselineRebuildPartitions.clear(); - } - public boolean isUseFullKeys() { return useFullKeys; } @@ -134,8 +89,6 @@ public void advanceSequencePrefix() { public String toString() { return "IvmInfo{" + "enableIvm=" + enableIvm - + ", completeBaselineRebuildRequired=" + completeBaselineRebuildRequired - + ", pendingBaselineRebuildPartitions=" + pendingBaselineRebuildPartitions + ", useFullKeys=" + useFullKeys + ", planSignature='" + planSignature + '\'' + '}'; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMTMV.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMTMV.java index 208417698b8cca..0736672fce33f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMTMV.java @@ -30,6 +30,7 @@ import org.apache.doris.mtmv.ivm.IvmInfo; import org.apache.doris.persist.gson.GsonUtils; +import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; import java.io.DataInput; @@ -37,6 +38,7 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.Set; public class AlterMTMV implements Writable { @SerializedName("ot") @@ -61,6 +63,9 @@ public class AlterMTMV implements Writable { private IvmInfo ivmInfo; @SerializedName("pst") private Map partitionStates; + // MV partitions whose refresh snapshot the same change dropped; see MTMV.markIvmPartitionsInvalidated. + @SerializedName("rsp") + private Set removedSnapshotPartitions; public AlterMTMV(TableNameInfo mvName, MTMVRefreshInfo refreshInfo, MTMVAlterOpType opType) { this.mvName = Objects.requireNonNull(mvName, "require mvName object"); @@ -159,6 +164,15 @@ public void setPartitionStates(Map partitionStates) this.partitionStates = MTMVPartitionState.copyOf(partitionStates); } + public Set getRemovedSnapshotPartitions() { + return removedSnapshotPartitions; + } + + public void setRemovedSnapshotPartitions(Set removedSnapshotPartitions) { + this.removedSnapshotPartitions = removedSnapshotPartitions == null ? null + : Sets.newLinkedHashSet(removedSnapshotPartitions); + } + @Override public String toString() { return "AlterMTMV{" diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java index 8daa44e63d5c7b..cb029d7233e9a4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java @@ -35,6 +35,8 @@ import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.junit.jupiter.api.Assertions; @@ -399,18 +401,15 @@ public void testAlterIvmInfoPersistence() throws Exception { IvmInfo newInfo = new IvmInfo(initialInfo); newInfo.setPlanSignature("sig-1"); - newInfo.requireCompleteBaselineRebuild(); TableNameInfo tableName = new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName()); AlterMTMV replayAlter = new AlterMTMV(tableName, MTMVAlterOpType.ALTER_IVM_INFO); replayAlter.setIvmInfo(newInfo); long schemaChangeVersion = mtmv.getSchemaChangeVersion(); - newInfo.clearBaselineRebuild(); Env.getCurrentEnv().getAlterInstance().processAlterMTMV(replayAlter, true); IvmInfo updatedInfo = mtmv.getIvmInfo(); Assertions.assertEquals("sig-1", updatedInfo.getPlanSignature()); - Assertions.assertTrue(updatedInfo.isBaselineRebuildRequired()); Assertions.assertEquals(schemaChangeVersion, mtmv.getSchemaChangeVersion()); } @@ -484,6 +483,45 @@ private static byte[] journalBytes(AlterMTMV alter) throws IOException { } /** Replays an alter record the way a restart does: from what the journal wrote, not from memory. */ + @Test + public void testReplayAlterPartitionStatesRemovesSnapshots() throws Exception { + Config.enable_table_stream = true; + createDatabaseAndUse("alter_partition_states_snapshot_test"); + createTable("CREATE TABLE alter_partition_states_snapshot_test.states_base (k1 int, v1 int)\n" + + "DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')"); + createMvByNereids("CREATE MATERIALIZED VIEW states_snapshot_mv\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS SELECT k1, v1 FROM states_base"); + + MTMV mtmv = (MTMV) Env.getCurrentInternalCatalog() + .getDb("alter_partition_states_snapshot_test").get() + .getTableOrMetaException("states_snapshot_mv"); + String partitionName = mtmv.getPartitionNames().iterator().next(); + mtmv.getRefreshSnapshot().updateSnapshots( + Maps.newHashMap(Map.of(partitionName, new MTMVRefreshPartitionSnapshot())), + Sets.newHashSet(partitionName)); + Assertions.assertEquals(Sets.newHashSet(partitionName), + mtmv.getRefreshSnapshot().getPartitionSnapshots().keySet()); + + // The invalidation journaled the raised requirement and the snapshot it dropped together, so a + // replay has to apply both: a reader that saw the requirement while the snapshot was still there + // could let a transparent rewrite serve rows the rebuild has to replace. + AlterMTMV payload = new AlterMTMV( + new TableNameInfo(mtmv.getQualifiedDbName(), mtmv.getName()), + MTMVAlterOpType.ALTER_PARTITION_STATES); + payload.setPartitionStates(Map.of(partitionName, new MTMVPartitionState(1, 2))); + payload.setRemovedSnapshotPartitions(Sets.newHashSet(partitionName)); + + replayFromJournal(payload); + + Assertions.assertEquals(2, mtmv.getPartitionStates().get(partitionName).getLatestEpoch()); + Assertions.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); + } + private static void replayFromJournal(AlterMTMV alter) throws Exception { AlterMTMV replayed; try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(journalBytes(alter)))) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshSnapshotTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshSnapshotTest.java index 385a95c00d87b5..8d5e9739bb2c3d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshSnapshotTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshSnapshotTest.java @@ -39,6 +39,24 @@ public class MTMVRefreshSnapshotTest { private BaseTableInfo existTable = Mockito.mock(BaseTableInfo.class); private BaseTableInfo nonExistTable = Mockito.mock(BaseTableInfo.class); + @Test + public void testRemoveSnapshotsDropsOnlyTheNamedPartitions() { + Map others = Maps.newHashMap(); + others.put("mvp2", new MTMVRefreshPartitionSnapshot()); + refreshSnapshot.updateSnapshots(others, Sets.newHashSet(mvExistPartitionName, "mvp2")); + + refreshSnapshot.removeSnapshots(Sets.newHashSet(mvExistPartitionName)); + + // A dropped entry is an invalidation's mark: transparent rewrite reads this map to decide what it + // may serve, so the entry has to stay gone until the partition is rebuilt. + Assertions.assertTrue(refreshSnapshot.getPctSnapshots(mvExistPartitionName, existTable).isEmpty()); + Assertions.assertEquals(Sets.newHashSet("mvp2"), refreshSnapshot.getPartitionSnapshots().keySet()); + + // Nothing named, nothing dropped. + refreshSnapshot.removeSnapshots(Sets.newHashSet()); + Assertions.assertEquals(Sets.newHashSet("mvp2"), refreshSnapshot.getPartitionSnapshots().keySet()); + } + @BeforeEach public void setUp() throws NoSuchMethodException, SecurityException, AnalysisException { Mockito.when(existTable.getCtlName()).thenReturn("ctl1"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java index 5c44960cc118d1..b58137c9564b9f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java @@ -20,15 +20,18 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; +import org.apache.doris.persist.EditLog.EditLogItem; import com.google.common.collect.Sets; import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.Optional; import java.util.Set; public class MTMVRelationManagerTest { @@ -158,7 +161,7 @@ public void testBaselineBarrierOnlyInvalidatesIvm() { manager.markIvmBaselineRebuild(t3, "test"); } - Mockito.verify(mtmv, Mockito.never()).invalidateIvmBaseline(); + Mockito.verify(mtmv, Mockito.never()).invalidateWholeMv(Mockito.anyString()); } @Test @@ -175,6 +178,36 @@ public void testBaselineBarrierSkipsExcludedTable() { manager.markIvmBaselineRebuild(t3, "test"); } - Mockito.verify(mtmv, Mockito.never()).invalidateIvmBaseline(); + Mockito.verify(mtmv, Mockito.never()).invalidateWholeMv(Mockito.anyString()); + } + + /** + * The generic base-table change is an invalidation, and it is recorded the way the invalidation above + * is: applied and enqueued in one MV-lock critical section, awaited where no MV lock is held. A record + * enqueued while the state is still being applied would be replayed on a follower on the other side of + * a concurrent task result, leaving the follower in SCHEMA_CHANGE where this FE ended NORMAL. + * + *

A rename is the shortest way into this path: it skips the query check, so what the hook does is + * exactly the record under test. + */ + @Test + public void testABaseTableRenameIsRecordedThroughTheInvalidation() { + MTMVRelationManager manager = new MTMVRelationManager(); + manager.refreshMTMVCache(new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), + Sets.newHashSet(t3), Sets.newHashSet(), Sets.newHashSet()), mv1); + MTMV mtmv = Mockito.mock(MTMV.class); + EditLogItem editLogItem = Mockito.mock(EditLogItem.class); + Mockito.when(mtmv.invalidateWholeMv(Mockito.anyString())).thenReturn(editLogItem); + try (MockedStatic util = Mockito.mockStatic(MTMVUtil.class)) { + util.when(() -> MTMVUtil.getTable(Mockito.any(BaseTableInfo.class))).thenReturn(mtmv); + + manager.alterTable(t3, Optional.of(t4), false); + } + + ArgumentCaptor detail = ArgumentCaptor.forClass(String.class); + Mockito.verify(mtmv).invalidateWholeMv(detail.capture()); + Assertions.assertTrue(detail.getValue().startsWith("The base table has been updated:"), + detail.getValue()); + Mockito.verify(editLogItem).await(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 47a0d2403e45b4..7e5fbe6558b426 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -92,6 +92,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -137,6 +138,13 @@ public void setUp() Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.COMPLETE); Mockito.when(mtmv.hasRefreshSnapshot()).thenReturn(true); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus()); + // Sane defaults for the epoch state: no partition needs a rebuild unless a case says so. The + // routing reads the states themselves, so a case that wants a rebuild gives it a state whose + // requirement is ahead of what it holds. What the escalation reads is the MV's own verdict over + // those states (MTMVTest covers it), so a case that wants it stubs the verdict. + Mockito.when(mtmv.getPartitionStates()).thenReturn(Collections.emptyMap()); + Mockito.when(mtmv.allPartitionsNeedRebuild()).thenReturn(false); } @AfterEach @@ -159,6 +167,173 @@ public void testGenerateRefreshModeDistinguishesFullAndPartialScope() { Assertions.assertEquals(MTMVTask.MTMVTaskRefreshMode.PARTIAL, differentPartitions); } + /** + * An epoch a batch captures is clamped to the one the routing decision saw. + * + *

An invalidation that lands between that decision and the batch's read raises the partition's + * requirement above what the batch is about to read; recording what it read would describe the data as + * current, and the delta this refresh applies cannot remove the rows the invalidation made unusable. + * Recording the planned value leaves the partition dirty, so the next refresh rebuilds it. + */ + @Test + public void testCapturedEpochIsClampedToTheOneTheRoutingDecisionSaw() { + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + // The routing decided on p1 at epoch 2, and an invalidation raised it to 5 before the read. + Deencapsulation.setField(task, "ivmPlannedEpochs", Maps.newHashMap(Map.of(poneName, 2L))); + Deencapsulation.invoke(task, "commitCapturedEpochs", Maps.newHashMap(Map.of(poneName, 5L))); + + Assertions.assertEquals(Map.of(poneName, 2L), Deencapsulation.getField(task, "ivmCapturedEpochs")); + + // A partition no routing decision named keeps what the batch read: there is no ceiling to clamp to. + MTMVTask unnamed = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + Deencapsulation.invoke(unnamed, "commitCapturedEpochs", Maps.newHashMap(Map.of(ptwoName, 7L))); + + Assertions.assertEquals(Map.of(ptwoName, 7L), Deencapsulation.getField(unnamed, "ivmCapturedEpochs")); + } + + /** + * A PARTITIONS request that may not fall back is not widened by an invalidated baseline. + * + *

What the invalidation needs rebuilt is not what the request names -- it covers partitions partition + * sync has not created yet -- so refreshing the named ones would leave the MV in SCHEMA_CHANGE with rows + * nothing rebuilt, and widening to COMPLETE would rebuild partitions the caller deliberately kept out. + * The forms whose scope already includes a whole-MV rebuild are the ones that can answer it. + */ + @Test + public void testAStrictPartitionsRefreshIsRefusedRatherThanWidened() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmv.getName()).thenReturn("test_mv"); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus(MTMVState.SCHEMA_CHANGE, "invalidated")); + MTMVTask task = new MTMVTask(mtmv, relation, + MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, false, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + + JobException exception = Assertions.assertThrows(JobException.class, + () -> Deencapsulation.invoke(task, "buildAttempts", request, false)); + + Assertions.assertTrue(exception.getMessage().contains("PARTITIONS FALLBACK"), exception.getMessage()); + + // The same request with fallback allowed reaches the COMPLETE its scope already carries. + MTMVTask fallbackTask = new MTMVTask(mtmv, relation, + MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, true, null)); + Object fallbackRequest = Deencapsulation.invoke(fallbackTask, "resolveRefreshRequest"); + + Assertions.assertEquals(Lists.newArrayList("COMPLETE"), + toNames((List) Deencapsulation.invoke(fallbackTask, "buildAttempts", fallbackRequest, false))); + } + + /** + * The captured epochs are handed out as a copy: a caller writing through the getter would be editing + * what the task publishes, and a STOP publishes while the executing worker may still be merging into + * the map the caller would be iterating. + */ + @Test + public void testCapturedEpochsAreHandedOutAsACopy() { + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + + task.getIvmCapturedEpochs().put(poneName, 3L); + + Assertions.assertTrue(((Map) Deencapsulation.getField(task, "ivmCapturedEpochs")).isEmpty()); + } + + /** + * A task read back from the journal carries no captured epochs: the field is transient, so gson leaves + * it null and the constructor that would have initialized it never runs. A replay applies the states + * the record carries instead, so the getter has to answer for that case rather than throw. + */ + @Test + public void testCapturedEpochsOfATaskReadBackFromTheJournalAreEmpty() { + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + Deencapsulation.setField(task, "ivmCapturedEpochs", null); + + Assertions.assertTrue(task.getIvmCapturedEpochs().isEmpty()); + } + + /** + * The rebuilt-partition count is an IVM diagnostic, and a plain MV reaches the COMPLETE success path + * through an ordinary AUTO refresh -- where rebuilding is what the refresh does, not a side effect of an + * invalidated baseline. + */ + @Test + public void testANonIvmRefreshReportsNoRebuiltPartitions() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(false); + MTMVTask task = new MTMVTask(mtmv, relation, + MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, RefreshMode.AUTO, false, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + + Deencapsulation.invoke(task, "recordRebuiltPartitions", request); + + Assertions.assertEquals(0, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + /** + * The count is what the refresh has replaced, so it is read from the same accumulator the task reports + * its progress with. That accumulator is created by the first phase that reports one, which leaves the + * attempt that reports before any phase has: a whole-MV refresh of an MV with nothing to refresh, and + * one the stream reconciliation threw out of, both report on a task that has committed nothing. That is + * no partitions, and it must read as zero where the refresh is reported rather than throw there. + */ + @Test + public void testARefreshThatReplacedNothingReportsNoRebuiltPartitions() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + MTMVTask task = new MTMVTask(mtmv, relation, + MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, RefreshMode.AUTO, true, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Assertions.assertNull(Deencapsulation.getField(task, "completedPartitions")); + + Deencapsulation.invoke(task, "recordRebuiltPartitions", request); + + Assertions.assertEquals(0, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + /** + * And the count is what committed, not what was planned: the accumulator grows as batches commit, so a + * rebuild that replaced one of its partitions and failed on the next reports one. + */ + @Test + public void testTheRebuiltCountFollowsThePartitionsThatCommitted() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + MTMVTask task = new MTMVTask(mtmv, relation, + MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, RefreshMode.AUTO, true, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Deencapsulation.invoke(task, "recordRefreshCompleted", Lists.newArrayList(poneName)); + + Deencapsulation.invoke(task, "recordRebuiltPartitions", request); + + Assertions.assertEquals(1, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + + // A later attempt that replaced both keeps the larger count rather than the last one it read. + Deencapsulation.invoke(task, "recordRefreshCompleted", Lists.newArrayList(poneName, ptwoName)); + Deencapsulation.invoke(task, "recordRebuiltPartitions", request); + + Assertions.assertEquals(2, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + /** + * A retry that synchronized partitions has to be judged again: alignment gives a partition it creates + * {@code {0, 1}} -- behind its requirement -- and the routing decision was taken before it existed. + * Without the fresh read the retried attempt would hand it to the delta path with no ceiling to be + * clamped against, and its capture is all that path can produce: the partition would be recorded as + * caught up while it has never received a baseline. + */ + @Test + public void testRetryAdoptsThePartitionsAlignmentCreated() { + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + // The routing saw p1 as caught up at epoch 2; p2 is what the retry's alignment added, and + // alignment gives a partition it creates {0, 1}. + Deencapsulation.setField(task, "ivmPlannedEpochs", Maps.newHashMap(Map.of(poneName, 2L))); + Mockito.when(mtmv.getPartitionStates()).thenReturn(Maps.newHashMap(Map.of( + poneName, new MTMVPartitionState(2, 2), + ptwoName, MTMVPartitionState.initial()))); + Set dirtyPartitions = Sets.newLinkedHashSet(); + + Deencapsulation.invoke(task, "adoptPartitionsCreatedByTheRetry", dirtyPartitions); + + Assertions.assertEquals(Sets.newHashSet(ptwoName), dirtyPartitions); + Assertions.assertEquals(Map.of(poneName, 2L, ptwoName, 1L), + Deencapsulation.getField(task, "ivmPlannedEpochs")); + } + @Test public void testBuildAttemptsAutoCompleteMethodSkipsPartitionsAttempt() { // setUp stubs refreshMethod=COMPLETE. The PARTITIONS attempt must be skipped: @@ -391,6 +566,211 @@ private MTMVRelation relationWithOneBaseTable() { Sets.newHashSet(baseTable), Sets.newHashSet(), Sets.newHashSet()); } + @Test + public void testBuildAttemptsEscalatesToCompleteWhenEveryPartitionNeedsARebuild() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + // Every partition either holds rows read before a change or was never filled: COMPLETE does exactly + // what their routing branches would, in a single read of the MV. Which partition states make that + // verdict true is MTMVTest's, next to the predicate that reads them. + Mockito.when(mtmv.allPartitionsNeedRebuild()).thenReturn(true); + + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, true, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + List attempts = (List) Deencapsulation.invoke(task, "buildAttempts", request, false); + + Assertions.assertEquals(Lists.newArrayList("COMPLETE"), toNames(attempts)); + } + + /** + * The all-dirty shortcut answers a whole-MV refresh, which a request that may not fall back has not + * authorized -- and COMPLETE is not merely more work: it reconciles the IVM streams and resets their + * baselines, the same reset the unusable-stream shortcut above permits only when a fallback is allowed. + * The routing those partitions would take is the incremental attempt, which rebuilds every one of them, + * finds nothing left to catch up, and fails on an unusable stream instead of resetting it. So the + * shortcut is gated on that permission, and a strict request keeps the chain that reaches the attempt. + */ + @Test + public void testTheAllDirtyShortcutNeedsARequestThatMayFallBack() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + Mockito.when(mtmv.allPartitionsNeedRebuild()).thenReturn(true); + + MTMVTask strictTask = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, false, null)); + Object strictRequest = Deencapsulation.invoke(strictTask, "resolveRefreshRequest"); + + List strictAttempts = (List) Deencapsulation.invoke(strictTask, "buildAttempts", strictRequest, + false); + + Assertions.assertEquals(Lists.newArrayList("IVM"), toNames(strictAttempts)); + + // Same MV, same verdict, a request that may fall back: the shortcut still applies. + MTMVTask fallbackTask = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, true, null)); + Object fallbackRequest = Deencapsulation.invoke(fallbackTask, "resolveRefreshRequest"); + + List fallbackAttempts = (List) Deencapsulation.invoke(fallbackTask, "buildAttempts", + fallbackRequest, false); + + Assertions.assertEquals(Lists.newArrayList("COMPLETE"), toNames(fallbackAttempts)); + } + + @Test + public void testBuildAttemptsDoesNotEscalateWithoutAnInvalidatedPartition() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + // Nothing needs a rebuild, so COMPLETE would be a full recomputation for no reason. The verdict is + // the default in setUp (false); asserted here so a change to it is a failure rather than a shrug. + Assertions.assertFalse(mtmv.allPartitionsNeedRebuild()); + + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, true, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + List attempts = (List) Deencapsulation.invoke(task, "buildAttempts", request, false); + + Assertions.assertEquals(Lists.newArrayList("IVM", "PARTITIONS", "COMPLETE"), toNames(attempts)); + } + + @Test + public void testBuildAttemptsEscalatesAnIvmMvInSchemaChangeToComplete() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus( + MTMVState.SCHEMA_CHANGE, "the base table has been updated")); + + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.AUTO, true, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + List attempts = (List) Deencapsulation.invoke(task, "buildAttempts", request, false); + + // A schema-level invalidation is not a set of dirty partitions: it covers the partitions partition + // sync has not created yet as well, so no per-partition branch can express it. The task says how + // many partitions the request did not ask to rebuild, which is what the INCREMENTAL attempt would + // have left out of the trace otherwise. + Assertions.assertEquals(Lists.newArrayList("COMPLETE"), toNames(attempts)); + // Planning does not claim the work: the count is recorded once the rebuild has actually run, so a + // refresh that fails before replacing anything reports nothing rather than the whole MV. What this + // route owes the request is the escalation itself, which is the assertion above. + Assertions.assertEquals(0, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + /** + * The initial-refresh shortcut also answers COMPLETE, and it is judged after the schema-change + * refusal: an MV that has never been refreshed and reads an excluded trigger table needs a whole-MV + * rebuild, so a strict `PARTITIONS` request must hear that rather than have that shortcut widen it + * silently. Judging the shortcut first is what makes the refusal unreachable in exactly the state it + * names. + */ + @Test + public void testABuildInitialRefreshDoesNotWidenAStrictPartitionsRequest() { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + Mockito.when(mtmv.hasRefreshSnapshot()).thenReturn(false); + Mockito.when(mtmv.getExcludedTriggerTables()).thenReturn(Sets.newHashSet( + new TableNameInfo("internal", "test_db", "excluded_agg"))); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus( + MTMVState.SCHEMA_CHANGE, "the base table has been updated")); + + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, false, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + // The shortcut is what this state would take without the refusal, so this is the ordering under + // test rather than a request that would never have reached it. + Assertions.assertTrue( + (Boolean) Deencapsulation.invoke(task, "shouldUseCompleteForInitialIvmRefresh", false)); + + JobException refusal = Assertions.assertThrows(JobException.class, + () -> Deencapsulation.invoke(task, "buildAttempts", request, false)); + Assertions.assertTrue(refusal.getMessage().contains("Use COMPLETE, AUTO, or PARTITIONS FALLBACK"), + refusal.getMessage()); + } + + @Test + public void testBuildAttemptsLeavesANonIvmMvInSchemaChangeOnItsOwnChain() throws Exception { + // setUp stubs a non-IVM MV. Its state is not what its attempt chain is built from: the cleared + // snapshot of a schema change already sends every partition to a rebuild, so the chain stays as it + // was, which is what keeps this change from moving a non-IVM MV's behaviour. + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.AUTO); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus( + MTMVState.SCHEMA_CHANGE, "the base table has been updated")); + + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + List attempts = (List) Deencapsulation.invoke(task, "buildAttempts", request, false); + + Assertions.assertEquals(Lists.newArrayList("PARTITIONS", "COMPLETE"), toNames(attempts)); + Assertions.assertEquals(0, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + @Test + public void testBuildAttemptsKeepsAnExplicitPartitionListOutOfTheSchemaChangeEscalation() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus( + MTMVState.SCHEMA_CHANGE, "the base table has been updated")); + + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, Lists.newArrayList(poneName), RefreshMode.AUTO)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + List attempts = (List) Deencapsulation.invoke(task, "buildAttempts", request, false); + + // An explicit partition list is an exact request and is never widened to COMPLETE. (An IVM MV + // rejects one at analysis time, so this is the belt to that suspender.) + Assertions.assertEquals(Lists.newArrayList("PARTITIONS"), toNames(attempts)); + Assertions.assertEquals(0, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + @Test + public void testBuildAttemptsRebuildsTheWholeMvForAStrictIncrementalInSchemaChange() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmvRefreshInfo.getRefreshMethod()).thenReturn(RefreshMethod.INCREMENTAL); + Mockito.when(mtmv.getStatus()).thenReturn(new MTMVStatus( + MTMVState.SCHEMA_CHANGE, "the base table has been updated")); + + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( + MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, false, null)); + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + List attempts = (List) Deencapsulation.invoke(task, "buildAttempts", request, false); + + // Asking against the fallback does not refuse this one: the baseline the incremental attempt would + // read is gone, while a COMPLETE refresh restores exactly it. Reporting the count is what keeps the + // request honest -- the result says the partitions it did not ask for were rebuilt. + Assertions.assertEquals(Lists.newArrayList("COMPLETE"), toNames(attempts)); + // The request's count is reported by the result, not decided here: recorded once the rebuild has + // run, so a refresh that never replaced anything reports nothing. + Assertions.assertEquals(0, (int) Deencapsulation.getField(task, "ivmRebuiltPartitions")); + } + + @Test + public void testIncrementalAttemptLeavesTheRebuiltPartitionsOutOfItsScope() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmv.getName()).thenReturn("test_mv"); + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + MTMVRefreshContext refreshContext = Mockito.mock(MTMVRefreshContext.class); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.getMTMVNeedRefreshPartitions( + Mockito.same(refreshContext), Mockito.nullable(Set.class))) + .thenReturn(Lists.newArrayList(poneName, ptwoName)); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.generatePartitionSnapshots( + Mockito.same(refreshContext), Mockito.nullable(Set.class), Mockito.nullable(Set.class))) + .thenReturn(Collections.emptyMap()); + + try (MockedConstruction ignored = Mockito.mockConstruction(IvmIncrRefreshManager.class, + (mock, context) -> Mockito.when(mock.doRefresh(Mockito.any())) + .thenReturn(IvmIncrRefreshResult.success()))) { + // poneName was rebuilt by the partition executor before this attempt, so the incremental + // refresh must not treat it as needing a catch-up: it can only append, and it would record the + // partition as current while its rows are exactly what the rebuild replaced. + IvmIncrRefreshResult result = (IvmIncrRefreshResult) Deencapsulation.invoke( + task, "executeSingleIvmAttempt", refreshContext, Sets.newHashSet(poneName)); + Assertions.assertTrue(result.isSuccess()); + } + + Assertions.assertEquals(Sets.newHashSet(ptwoName), + Deencapsulation.getField(task, "needRefreshPartitions")); + } + private static List toNames(List attempts) { List names = Lists.newArrayList(); for (Object attempt : attempts) { @@ -427,6 +807,221 @@ public void testIncrementalFallbackOnNonIvmKeepsIvmAttempt() throws JobException .map(Object::toString).collect(Collectors.toList())); } + /** + * A refresh that rebuilt a partition and then fell back out of the incremental attempt must not plan + * to refresh that partition again. The plan is computed from the snapshot the MV holds, which such a + * refresh has not published yet, so without the rebuild's own record the partition still looks unsynced + * and the fallback replaces the work the rebuild just did. + */ + @Test + public void testFallbackPlanLeavesTheRebuiltPartitionsOut() throws Exception { + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVSync( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class), + Mockito.nullable(Set.class))).thenReturn(false); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.getMTMVNeedRefreshPartitions( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class))) + .thenReturn(Lists.newArrayList(poneName, ptwoName)); + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, + RefreshMode.INCREMENTAL, true, null)); + // poneName was rebuilt by the partition executor before this attempt, so the task's accumulator + // already holds it. + Map accumulated = Maps.newConcurrentMap(); + accumulated.put(poneName, Mockito.mock(MTMVRefreshPartitionSnapshot.class)); + Deencapsulation.setField(task, "partitionSnapshots", accumulated); + + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Object plan = Deencapsulation.invoke(task, "planPartitionRefresh", + Mockito.mock(MTMVRefreshContext.class), request); + + Assertions.assertTrue((Boolean) Deencapsulation.getField(plan, "canRefreshByPartitions")); + Assertions.assertEquals(Lists.newArrayList(ptwoName), + Deencapsulation.getField(plan, "partitions")); + } + + /** + * A partition the criterion says must be rebuilt is planned even though the snapshots say the MV is in + * sync. The two answer different questions -- what a refresh last read, and what a later read cannot + * catch up -- and they disagree in the one state a whole-MV rebuild that did not finish leaves behind: + * every requirement raised, every snapshot kept. Planning by snapshots alone would report NOT_REFRESH + * over partitions whose rows nothing repaired, and only the entry points that reach the incremental + * attempt would ever recover them. + */ + @Test + public void testPlanPartitionRefreshPlansThePartitionsThatNeedARebuild() throws Exception { + // setUp stubs isMTMVSync true and getMTMVNeedRefreshPartitions empty: by themselves they say there + // is nothing to refresh, which is the early return this has to get past. + Mockito.when(mtmv.getPartitionsNeedingRebuild()) + .thenReturn(Sets.newLinkedHashSet(Sets.newHashSet(poneName))); + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, + RefreshMode.PARTITIONS, true, null)); + + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Object plan = Deencapsulation.invoke(task, "planPartitionRefresh", + Mockito.mock(MTMVRefreshContext.class), request); + + Assertions.assertTrue((Boolean) Deencapsulation.getField(plan, "canRefreshByPartitions")); + Assertions.assertEquals(Lists.newArrayList(poneName), + Deencapsulation.getField(plan, "partitions")); + } + + /** + * The two answers are both planned when they disagree in the other direction as well: a partition the + * snapshots call unsynced joins the ones that need a rebuild, rather than replacing them. + */ + @Test + public void testPlanPartitionRefreshKeepsTheCriterionAlongsideTheSnapshots() throws Exception { + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.isMTMVSync( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class), + Mockito.nullable(Set.class))).thenReturn(false); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.getMTMVNeedRefreshPartitions( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class))) + .thenReturn(Lists.newArrayList(ptwoName)); + Mockito.when(mtmv.getPartitionsNeedingRebuild()) + .thenReturn(Sets.newLinkedHashSet(Sets.newHashSet(poneName))); + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, + RefreshMode.PARTITIONS, true, null)); + + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Object plan = Deencapsulation.invoke(task, "planPartitionRefresh", + Mockito.mock(MTMVRefreshContext.class), request); + + Assertions.assertEquals(Sets.newHashSet(poneName, ptwoName), + Sets.newHashSet((List) Deencapsulation.getField(plan, "partitions"))); + } + + /** + * The partitions an earlier phase replaced survive the incremental attempt that falls back after them. + * The task keeps one accumulator for the whole of it, because that is what the MV publishes: a phase + * that started from empty would drop the work of the phases before it, and the next refresh would find + * those partitions unsynced and replace them once more. + */ + @Test + public void testFallbackKeepsTheSnapshotsOfThePartitionsTheRebuildReplaced() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmv.getName()).thenReturn("test_mv"); + MTMVRefreshPartitionSnapshot rebuiltSnapshot = Mockito.mock(MTMVRefreshPartitionSnapshot.class); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.getMTMVNeedRefreshPartitions( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class))) + .thenReturn(Lists.newArrayList(ptwoName)); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.generatePartitionSnapshots( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class), + Mockito.nullable(Set.class))).thenReturn(Collections.emptyMap()); + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, + RefreshMode.INCREMENTAL, true, null)); + Map accumulated = Maps.newConcurrentMap(); + accumulated.put(poneName, rebuiltSnapshot); + Deencapsulation.setField(task, "partitionSnapshots", accumulated); + + try (MockedConstruction ignored = Mockito.mockConstruction( + IvmIncrRefreshManager.class, (mock, context) -> Mockito.when(mock.doRefresh(Mockito.any())) + .thenReturn(IvmIncrRefreshResult.fallback( + IvmFailureReason.INCREMENTAL_EXECUTION_FAILED, "forced")))) { + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Object result = Deencapsulation.invoke(task, "executeIvmAttempt", + Mockito.mock(MTMVRefreshContext.class), request, Mockito.mock(ConnectContext.class), + Lists.newArrayList()); + Assertions.assertEquals("FALLBACK_ALLOWED", result.toString()); + } + + Assertions.assertSame(rebuiltSnapshot, + ((Map) Deencapsulation.getField(task, "partitionSnapshots")).get(poneName)); + } + + /** + * The same holds for what the task reports: an attempt adds to it rather than replacing it. The + * partition the rebuild phase committed is part of the refresh the user asked for -- and the MV has + * published it, along with its epoch -- so a report that dropped it would describe work this task did + * as work it never did. + */ + @Test + public void testALaterAttemptAddsToTheReportInsteadOfReplacingIt() throws Exception { + Mockito.when(mtmv.isIvm()).thenReturn(true); + Mockito.when(mtmv.getName()).thenReturn("test_mv"); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.getMTMVNeedRefreshPartitions( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class))) + .thenReturn(Lists.newArrayList(ptwoName)); + mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.generatePartitionSnapshots( + Mockito.nullable(MTMVRefreshContext.class), Mockito.nullable(Set.class), + Mockito.nullable(Set.class))).thenReturn(Collections.emptyMap()); + MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of(MTMVTaskTriggerMode.MANUAL, null, + RefreshMode.INCREMENTAL, true, null)); + // The rebuild phase ran first and reported poneName as both its scope and its result. + Deencapsulation.setField(task, "needRefreshPartitions", + new ConcurrentSkipListSet<>(Sets.newHashSet(poneName))); + Deencapsulation.setField(task, "completedPartitions", + new ConcurrentSkipListSet<>(Sets.newHashSet(poneName))); + + try (MockedConstruction ignored = Mockito.mockConstruction( + IvmIncrRefreshManager.class, (mock, context) -> Mockito.when(mock.doRefresh(Mockito.any())) + .thenReturn(IvmIncrRefreshResult.fallback( + IvmFailureReason.INCREMENTAL_EXECUTION_FAILED, "forced")))) { + Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); + Object result = Deencapsulation.invoke(task, "executeIvmAttempt", + Mockito.mock(MTMVRefreshContext.class), request, Mockito.mock(ConnectContext.class), + Lists.newArrayList()); + Assertions.assertEquals("FALLBACK_ALLOWED", result.toString()); + } + + // The incremental attempt took a scope of its own and committed nothing, which must leave the + // rebuild's partition in both sets rather than replacing them with its own. + Assertions.assertEquals(Sets.newHashSet(poneName, ptwoName), + Deencapsulation.getField(task, "needRefreshPartitions")); + Assertions.assertEquals(Sets.newHashSet(poneName), + Deencapsulation.getField(task, "completedPartitions")); + } + + /** + * And a partition is reported once, however many attempts cover it: a whole-MV rebuild after a + * per-partition one names the partitions the rebuild already reported, and counting them twice would + * report more work than the MV has partitions. + */ + @Test + public void testTheReportCountsAPartitionOnce() { + MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); + + Deencapsulation.invoke(task, "recordRefreshScope", Lists.newArrayList(poneName, ptwoName)); + Deencapsulation.invoke(task, "recordRefreshCompleted", Lists.newArrayList(poneName)); + Deencapsulation.invoke(task, "recordRefreshScope", Lists.newArrayList(ptwoName)); + Deencapsulation.invoke(task, "recordRefreshCompleted", Lists.newArrayList(poneName, ptwoName)); + + Assertions.assertEquals(Sets.newHashSet(poneName, ptwoName), + Deencapsulation.getField(task, "needRefreshPartitions")); + Assertions.assertEquals(Sets.newHashSet(poneName, ptwoName), + Deencapsulation.getField(task, "completedPartitions")); + } + + /** + * The two columns these sets feed are persisted, so what changes about them is the field's type and not + * the record it reads: a task written by an older FE carries them as arrays of names, which is what a + * set is written as, and reads back into either shape. + */ + @Test + public void testThePartitionColumnsAreStillArraysInTheJournal() { + MTMVTask task = GsonUtils.GSON.fromJson("{\"di\":1,\"mi\":2}", MTMVTask.class); + Deencapsulation.invoke(task, "recordRefreshScope", Lists.newArrayList(ptwoName, poneName)); + Deencapsulation.invoke(task, "recordRefreshCompleted", Lists.newArrayList(poneName)); + + String json = GsonUtils.GSON.toJson(task); + Assertions.assertTrue( + json.contains("\"needRefreshPartitions\":[\"" + poneName + "\",\"" + ptwoName + "\"]"), json); + Assertions.assertTrue(json.contains("\"completedPartitions\":[\"" + poneName + "\"]"), json); + + // And the reader gets a set back, whichever version wrote the record. + MTMVTask readBack = GsonUtils.GSON.fromJson(json, MTMVTask.class); + Assertions.assertEquals(Sets.newHashSet(poneName, ptwoName), + Deencapsulation.getField(readBack, "needRefreshPartitions")); + Assertions.assertEquals(Sets.newHashSet(poneName), + Deencapsulation.getField(readBack, "completedPartitions")); + + // Which is the same record an older task carries, read the same way. + MTMVTask older = GsonUtils.GSON.fromJson("{\"di\":1,\"mi\":2,\"needRefreshPartitions\":[\"p1\",\"p2\"]," + + "\"completedPartitions\":[\"p1\"]}", MTMVTask.class); + Assertions.assertEquals(Sets.newHashSet(poneName, ptwoName), + Deencapsulation.getField(older, "needRefreshPartitions")); + Assertions.assertEquals(Sets.newHashSet(poneName), + Deencapsulation.getField(older, "completedPartitions")); + } + @Test public void testManualIvmWithOneRowRelationWithoutSnapshotUsesComplete() throws JobException { Mockito.when(mtmv.isIvm()).thenReturn(true); @@ -472,12 +1067,12 @@ public void testMvDefaultUnknownRefreshMethodRejected() { @Test public void testTaskSchemaContainsComputeGroup() { - Column computeGroupColumn = MTMVTask.SCHEMA.get(MTMVTask.SCHEMA.size() - 2); - Column fallbackReasonColumn = MTMVTask.SCHEMA.get(MTMVTask.SCHEMA.size() - 1); + Column computeGroupColumn = MTMVTask.SCHEMA.get(MTMVTask.COLUMN_TO_INDEX.get(COMPUTE_GROUP.toLowerCase())); + Column fallbackReasonColumn = MTMVTask.SCHEMA.get(MTMVTask.COLUMN_TO_INDEX.get("ivmfallbackreason")); + Column rebuiltPartitionsColumn = MTMVTask.SCHEMA.get(MTMVTask.COLUMN_TO_INDEX.get("ivmrebuiltpartitions")); Assertions.assertEquals(COMPUTE_GROUP, computeGroupColumn.getName()); Assertions.assertEquals("IvmFallbackReason", fallbackReasonColumn.getName()); - Assertions.assertEquals(MTMVTask.SCHEMA.size() - 2, - MTMVTask.COLUMN_TO_INDEX.get(COMPUTE_GROUP.toLowerCase()).intValue()); + Assertions.assertEquals("IvmRebuiltPartitions", rebuiltPartitionsColumn.getName()); } @Test @@ -850,183 +1445,9 @@ public void testExecuteIvmAttemptFallsBackToCompleteForBrokenBaseline() throws E } } - @Test - public void testStrictIncrementalRejectsPendingBaselineBeforePartitionSync() throws Exception { - Mockito.when(mtmv.isIvm()).thenReturn(true); - IvmInfo ivmInfo = new IvmInfo(); - ivmInfo.requireCompleteBaselineRebuild(); - Mockito.when(mtmv.getIvmInfo()).thenReturn(ivmInfo); - Mockito.when(mtmv.getPartitionNames()).thenReturn(Collections.singleton(poneName)); - MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( - MTMVTaskTriggerMode.MANUAL, null, RefreshMode.INCREMENTAL, false, null)); - Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); - - JobException exception = Assertions.assertThrows(JobException.class, - () -> Deencapsulation.invoke(task, "validateIvmBaselineBeforePartitionSync", request)); - - Assertions.assertTrue(exception.getMessage().contains("run an AUTO or COMPLETE refresh first")); - Assertions.assertEquals(IvmFailureReason.BINLOG_BROKEN.name(), - Deencapsulation.getField(task, "ivmFallbackReason")); - } - - @Test - public void testBarePartitionsRejectsCompletePendingBaseline() throws Exception { - Mockito.when(mtmv.isIvm()).thenReturn(true); - IvmInfo ivmInfo = new IvmInfo(); - ivmInfo.requireCompleteBaselineRebuild(); - Mockito.when(mtmv.getIvmInfo()).thenReturn(ivmInfo); - MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( - MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, false, null)); - Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); - - JobException exception = Assertions.assertThrows(JobException.class, - () -> Deencapsulation.invoke(task, "validateIvmBaselineBeforePartitionSync", request)); - - Assertions.assertTrue(exception.getMessage().contains("run a PARTITIONS FALLBACK, AUTO, or COMPLETE refresh")); - } - - @Test - public void testPartitionsFallbackRebuildsPendingBaselineWithComplete() throws Exception { - Mockito.when(mtmv.isIvm()).thenReturn(true); - IvmInfo ivmInfo = new IvmInfo(); - ivmInfo.requireCompleteBaselineRebuild(); - Mockito.when(mtmv.getIvmInfo()).thenReturn(ivmInfo); - Mockito.when(mtmv.getPartitionNames()).thenReturn(Collections.emptySet()); - MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( - MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, true, null)); - Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); - Deencapsulation.invoke(task, "validateIvmBaselineBeforePartitionSync", request); - List attempts = Lists.newArrayList(); - attempts.addAll(Deencapsulation.invoke(task, "buildAttempts", request, false)); - Assertions.assertEquals("[PARTITIONS, COMPLETE]", attempts.toString()); - Deencapsulation.invoke(task, "handlePendingIvmBaselineRebuild", - Mockito.mock(MTMVRefreshContext.class), request, new ConnectContext(), attempts); - // A pending COMPLETE rebuild reshapes the attempt list instead of rebuilding inline, so - // PARTITIONS FALLBACK rebuilds the whole MV through the COMPLETE attempt it keeps. - Assertions.assertEquals("[COMPLETE]", attempts.toString()); - Assertions.assertEquals(IvmFailureReason.BINLOG_BROKEN.name(), - Deencapsulation.getField(task, "ivmFallbackReason")); - // The barrier is released by the caller once the reshaped attempts have run. - Mockito.verify(mtmv, Mockito.never()).releaseIvmBaselineRebuild(Mockito.anyLong()); - } - - @Test - public void testPendingBaselineRebuildChecksTheStreamsItsPartitionsRead() throws Exception { - // A partial barrier left by an earlier failed refresh. The pre-step rebuilds those partitions - // before the attempts run, and that rebuild reads their streams, so a stream missing for them - // decides the request just as it does for the partition attempt -- and PARTITIONS FALLBACK - // reaches this pre-step without an IVM attempt for buildAttempts to have judged. - Mockito.when(mtmv.isIvm()).thenReturn(true); - Mockito.when(mtmv.getName()).thenReturn("test_mv"); - Mockito.when(mtmv.getId()).thenReturn(7L); - Mockito.when(mtmv.getExcludedTriggerTables()).thenReturn(Collections.emptySet()); - IvmInfo ivmInfo = new IvmInfo(); - ivmInfo.addPendingBaselineRebuildPartitions(Sets.newHashSet(poneName)); - Mockito.when(mtmv.getIvmInfo()).thenReturn(ivmInfo); - OlapTable t1 = mockBaseTable("t1"); - BaseTableInfo t1Info = Mockito.mock(BaseTableInfo.class); - mtmvUtilStatic.when(() -> MTMVUtil.getTable(t1Info)).thenReturn(t1); - // t1 is not a PCT table, so every partition this rebuild refreshes reads it through its stream, - // and the MV's database holds no stream for it. - Mockito.when(mtmv.getDatabase()).thenReturn(Mockito.mock(Database.class)); - MTMVRefreshContext context = Mockito.mock(MTMVRefreshContext.class); - Mockito.when(context.getByPartitionName(Mockito.anyString())).thenReturn(Maps.newHashMap()); - - MTMVRelation relation = new MTMVRelation(Sets.newHashSet(t1Info), Sets.newHashSet(t1Info), - Sets.newHashSet(t1Info), Sets.newHashSet(), Sets.newHashSet()); - MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( - MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, true, null)); - Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); - List attempts = Lists.newArrayList(); - attempts.addAll(Deencapsulation.invoke(task, "buildAttempts", request, false)); - Assertions.assertEquals("[PARTITIONS, COMPLETE]", attempts.toString()); - - try { - Deencapsulation.invoke(task, "handlePendingIvmBaselineRebuild", context, request, - new ConnectContext(), attempts); - } catch (Exception expected) { - // Without the stream check the pre-step rebuilds inline, and how far that rebuild gets - // against these mocks is not what this test is about; the attempts it leaves behind are. - } - - // The rebuild that cannot read its streams is skipped rather than attempted: its own barrier - // would have guarded nothing but the data it never wrote, and the COMPLETE attempt left in the - // list reconciles the stream and clears the barrier that is already pending. - Assertions.assertEquals("[COMPLETE]", attempts.toString()); - Assertions.assertEquals(IvmFailureReason.STREAM_UNSUPPORTED.name(), - Deencapsulation.getField(task, "ivmFallbackReason")); - Mockito.verify(mtmv, Mockito.never()).persistIvmBaselineGuard(Mockito.any(), Mockito.anySet(), - Mockito.anyLong()); - Mockito.verify(mtmv, Mockito.never()).releaseIvmBaselineRebuild(Mockito.anyLong()); - - // The same rebuild without fallback is not covered by a COMPLETE attempt, so it fails here - // rather than starting a rebuild that cannot read its streams. - MTMVTask strictTask = new MTMVTask(mtmv, relation, MTMVTaskContext.of( - MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, false, null)); - Object strictRequest = Deencapsulation.invoke(strictTask, "resolveRefreshRequest"); - List strictAttempts = Lists.newArrayList(); - strictAttempts.addAll(Deencapsulation.invoke(strictTask, "buildAttempts", strictRequest, false)); - Assertions.assertEquals("[PARTITIONS]", strictAttempts.toString()); - - JobException exception = Assertions.assertThrows(JobException.class, - () -> Deencapsulation.invoke(strictTask, "handlePendingIvmBaselineRebuild", context, - strictRequest, new ConnectContext(), strictAttempts)); - - Assertions.assertTrue(exception.getMessage().contains("IVM stream is unusable")); - } - - @Test - public void testCompleteAttemptWritesTheBarrierBeforeReconcilingStreams() throws Exception { - Mockito.when(mtmv.isIvm()).thenReturn(true); - Mockito.when(mtmv.getPartitionNames()).thenReturn(Sets.newHashSet(poneName)); - // The reconcile starts from the MV's database, which is what makes it visible to the order check. - Mockito.when(mtmv.getDatabase()).thenReturn(Mockito.mock(Database.class)); - MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); - InOrder inOrder = Mockito.inOrder(mtmv); - - try { - Deencapsulation.invoke(task, "executeCompleteAttempt", - Mockito.mock(MTMVRefreshContext.class), new ConnectContext()); - } catch (Exception expected) { - // How far the rebuild itself gets is not what this test is about. - } - - // A recreated stream starts from the base table's current rows, so the barrier that makes the - // next refresh rebuild the MV has to be durable before the stream is replaced. The other order - // loses those rows with no error anywhere. - inOrder.verify(mtmv).persistIvmBaselineGuard(Mockito.any(), Mockito.anySet(), Mockito.anyLong()); - inOrder.verify(mtmv).getDatabase(); - } - - - @Test - public void testDroppedBaselinePartitionsReleaseBarrierWithoutRebuild() throws Exception { - Mockito.when(mtmv.isIvm()).thenReturn(true); - IvmInfo ivmInfo = new IvmInfo(); - ivmInfo.addPendingBaselineRebuildPartitions(Sets.newHashSet(poneName)); - Mockito.when(mtmv.getIvmInfo()).thenReturn(ivmInfo); - // Partition sync already dropped the partition the barrier named, so nothing is left to - // pre-rebuild and the surviving partitions catch up through the attempts themselves. - Mockito.when(mtmv.getPartitionNames()).thenReturn(Sets.newHashSet(ptwoName)); - MTMVTask task = new MTMVTask(mtmv, relation, MTMVTaskContext.of( - MTMVTaskTriggerMode.MANUAL, null, RefreshMode.PARTITIONS, true, null)); - Deencapsulation.setField(task, "mtmvSchemaChangeVersion", 7L); - Object request = Deencapsulation.invoke(task, "resolveRefreshRequest"); - - List attempts = Lists.newArrayList(); - attempts.addAll(Deencapsulation.invoke(task, "buildAttempts", request, false)); - Deencapsulation.invoke(task, "handlePendingIvmBaselineRebuild", - Mockito.mock(MTMVRefreshContext.class), request, new ConnectContext(), attempts); - - Assertions.assertEquals("[PARTITIONS, COMPLETE]", attempts.toString()); - Assertions.assertNull(Deencapsulation.getField(task, "refreshMode")); - Assertions.assertEquals(IvmFailureReason.BINLOG_BROKEN.name(), - Deencapsulation.getField(task, "ivmFallbackReason")); - Mockito.verify(mtmv).releaseIvmBaselineRebuild(7L); - } @Test public void testExecuteIvmAttemptKeepsRefreshScopeForNonSignatureFallbackInAutoMode() throws Exception { @@ -1034,7 +1455,8 @@ public void testExecuteIvmAttemptKeepsRefreshScopeForNonSignatureFallbackInAutoM Mockito.when(mtmv.getName()).thenReturn("test_mv"); MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); MTMVRefreshContext refreshContext = mockIvmIncrRefreshContext(); - Deencapsulation.setField(task, "needRefreshPartitions", Lists.newArrayList(poneName)); + // What an earlier phase had already put in the report: the fallback must not take it back out. + Deencapsulation.setField(task, "needRefreshPartitions", new ConcurrentSkipListSet<>(Sets.newHashSet(poneName))); Deencapsulation.setField(task, "refreshMode", MTMVTask.MTMVTaskRefreshMode.PARTIAL); try (MockedConstruction ignored = Mockito.mockConstruction(IvmIncrRefreshManager.class, @@ -1047,7 +1469,7 @@ public void testExecuteIvmAttemptKeepsRefreshScopeForNonSignatureFallbackInAutoM Assertions.assertEquals("FALLBACK_ALLOWED", result.toString()); } - Assertions.assertEquals(Lists.newArrayList(poneName), + Assertions.assertEquals(Sets.newHashSet(poneName), Deencapsulation.getField(task, "needRefreshPartitions")); Assertions.assertEquals(MTMVTask.MTMVTaskRefreshMode.PARTIAL, Deencapsulation.getField(task, "refreshMode")); @@ -1074,7 +1496,8 @@ public void testDebugPlanSignatureDriftFallsBackToFullRefresh() throws Exception MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); MTMVRefreshContext refreshContext = mockIvmIncrRefreshContext(); - Deencapsulation.setField(task, "needRefreshPartitions", Lists.newArrayList(poneName)); + Deencapsulation.setField(task, "needRefreshPartitions", + new ConcurrentSkipListSet<>(Sets.newHashSet(poneName))); Deencapsulation.setField(task, "refreshMode", MTMVTask.MTMVTaskRefreshMode.PARTIAL); try (MockedConstruction ignored = Mockito.mockConstruction(IvmIncrRefreshManager.class, @@ -1137,7 +1560,6 @@ public void testUnionPreloadFailurePreservesCompletedGroupProgress() throws Exce Mockito.when(mtmv.getRefreshPartitionNum()).thenReturn(1); Mockito.when(mtmv.getExcludedTriggerTables()).thenReturn(Collections.emptySet()); MTMVTask task = new MTMVTask(mtmv, relation, new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL)); - Deencapsulation.setField(task, "needRefreshPartitions", Lists.newArrayList(poneName, ptwoName)); MTMVRefreshContext refreshContext = Mockito.mock(MTMVRefreshContext.class); Mockito.when(refreshContext.preparePartitionSnapshots(Sets.newHashSet(poneName, ptwoName))) @@ -1164,11 +1586,12 @@ public void testUnionPreloadFailurePreservesCompletedGroupProgress() throws Exce AnalysisException failure = Assertions.assertThrows(AnalysisException.class, () -> Deencapsulation.invoke(task, "executePartitionBasedRefresh", - refreshContext, RefreshMode.COMPLETE, mtmvCtx)); + refreshContext, RefreshMode.COMPLETE, mtmvCtx, + Lists.newArrayList(poneName, ptwoName))); Assertions.assertTrue(failure.getMessage().contains("second group failed")); } - Assertions.assertEquals(Collections.singletonList(poneName), + Assertions.assertEquals(Sets.newHashSet(poneName), Deencapsulation.getField(task, "completedPartitions")); Map snapshots = Deencapsulation.getField(task, "partitionSnapshots"); Assertions.assertSame(firstSnapshot, snapshots.get(poneName)); @@ -1248,7 +1671,6 @@ private void executeCompleteRefresh(MTMVTask task, IvmPlanSignature firstBatchSi mtmvPartitionUtilStatic.when(() -> MTMVPartitionUtil.generatePartitionSnapshots( Mockito.same(refreshContext), Mockito.anySet(), Mockito.anySet())) .thenReturn(Collections.emptyMap()); - Deencapsulation.setField(task, "needRefreshPartitions", Lists.newArrayList(poneName, ptwoName)); ConnectContext mtmvCtx = new ConnectContext(); mtmvCtx.setQueryId(new TUniqueId(1L, 2L)); @@ -1277,7 +1699,7 @@ private void executeCompleteRefresh(MTMVTask task, IvmPlanSignature firstBatchSi }); Deencapsulation.invoke(task, "executePartitionBasedRefresh", - refreshContext, RefreshMode.COMPLETE, mtmvCtx); + refreshContext, RefreshMode.COMPLETE, mtmvCtx, Lists.newArrayList(poneName, ptwoName)); } finally { ConnectContext.remove(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java index 5e7ea127b4612b..81c715ff50f9c2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java @@ -58,10 +58,12 @@ import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; +import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -227,8 +229,11 @@ public void testAlterMvPropertiesWithExcludedTriggerTablesChange() { replayAlterMvProperties(mtmv, newProperties); Assertions.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState()); - Assertions.assertEquals(oldSchemaChangeVersion + 1, mtmv.getSchemaChangeVersion()); - Assertions.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); + Assertions.assertEquals(oldSchemaChangeVersion, mtmv.getSchemaChangeVersion()); + // Only excluded_trigger_tables changed, and nothing here brings a base table back into + // what the MV maintains, so the change owes neither a version bump nor a snapshot + // drop: the rows the MV holds stay valid and no rebuild is asked for. + Assertions.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); mtmv.getRefreshSnapshot().getPartitionSnapshots().put("p1", new MTMVRefreshPartitionSnapshot()); oldSchemaChangeVersion = mtmv.getSchemaChangeVersion(); @@ -237,8 +242,11 @@ public void testAlterMvPropertiesWithExcludedTriggerTablesChange() { replayAlterMvProperties(mtmv, newProperties); Assertions.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState()); - Assertions.assertEquals(oldSchemaChangeVersion + 1, mtmv.getSchemaChangeVersion()); - Assertions.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); + Assertions.assertEquals(oldSchemaChangeVersion, mtmv.getSchemaChangeVersion()); + // Only excluded_trigger_tables changed, and nothing here brings a base table back into + // what the MV maintains, so the change owes neither a version bump nor a snapshot + // drop: the rows the MV holds stay valid and no rebuild is asked for. + Assertions.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); } @Test @@ -279,8 +287,11 @@ public void testAlterMvPropertiesWithReducedExcludedTriggerTables() { replayAlterMvProperties(mtmv, newProperties); Assertions.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState()); - Assertions.assertEquals(oldSchemaChangeVersion + 1, mtmv.getSchemaChangeVersion()); - Assertions.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); + Assertions.assertEquals(oldSchemaChangeVersion, mtmv.getSchemaChangeVersion()); + // Only excluded_trigger_tables changed, and nothing here brings a base table back into + // what the MV maintains, so the change owes neither a version bump nor a snapshot + // drop: the rows the MV holds stay valid and no rebuild is asked for. + Assertions.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); mtmv.getRefreshSnapshot().getPartitionSnapshots().put("p1", new MTMVRefreshPartitionSnapshot()); oldSchemaChangeVersion = mtmv.getSchemaChangeVersion(); @@ -289,8 +300,11 @@ public void testAlterMvPropertiesWithReducedExcludedTriggerTables() { replayAlterMvProperties(mtmv, newProperties); Assertions.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState()); - Assertions.assertEquals(oldSchemaChangeVersion + 1, mtmv.getSchemaChangeVersion()); - Assertions.assertTrue(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); + Assertions.assertEquals(oldSchemaChangeVersion, mtmv.getSchemaChangeVersion()); + // Only excluded_trigger_tables changed, and nothing here brings a base table back into + // what the MV maintains, so the change owes neither a version bump nor a snapshot + // drop: the rows the MV holds stay valid and no rebuild is asked for. + Assertions.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); } @Test @@ -300,12 +314,16 @@ public void testIncludingExcludedIvmBaseTableRequiresCompleteBaselineRebuild() { Map.of(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, "t1,t2"))); BaseTableInfo includedBaseTable = new BaseTableInfo(new TableNameInfo("internal", "db1", "t2")); mtmv.setRelation(new MTMVRelation(Set.of(includedBaseTable), Set.of(), Set.of(), Set.of(), Set.of())); + mtmv.setStatus(new MTMVStatus(MTMVState.NORMAL, "seed")); mtmv.getIvmInfo().setEnableIvm(true); replayAlterMvProperties(mtmv, Map.of(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, "t1")); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + // The property record does not carry the invalidation: the live change journals an ALTER_STATUS + // record ahead of it, which is what puts the MV into SCHEMA_CHANGE -- every partition, including + // the ones it has not created yet, which no per-partition requirement can express. + Assertions.assertEquals(MTMVState.NORMAL, mtmv.getStatus().getState()); } @Test @@ -367,6 +385,48 @@ public void testAlterStatus() { Assertions.assertEquals(MTMVRefreshState.SUCCESS, status.getRefreshState()); } + /** + * The generic base-table change puts every MV that reads the table into SCHEMA_CHANGE, and it has to do + * it the way the invalidation above does: applied and enqueued in one MV-lock critical section. A task + * result enqueued in between is replayed on a follower after this record rather than before it, which + * leaves the follower in SCHEMA_CHANGE where this FE ended NORMAL -- a whole-MV rebuild the next + * refresh does not need. + */ + @Test + public void testWholeMvInvalidationSubmitsJournalWhileHoldingMvLock() { + MTMV mtmv = buildSerializableMTMV(); + // A journaling path names the MV, and this fixture is built through the constructor that leaves the + // name unset; setName() cannot be used because it rekeys the index map by the current (null) name. + Deencapsulation.setField(mtmv, "name", "mv1"); + ReentrantReadWriteLock mvRwLock = Deencapsulation.getField(mtmv, "mvRwLock"); + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + EditLogItem editLogItem = Mockito.mock(EditLogItem.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(editLog.submitEdit(Mockito.eq(OperationType.OP_ALTER_MTMV), Mockito.any(AlterMTMV.class))) + .thenAnswer(invocation -> { + Assertions.assertTrue(mvRwLock.isWriteLockedByCurrentThread()); + return editLogItem; + }); + Mockito.when(editLogItem.await()).thenAnswer(invocation -> { + Assertions.assertFalse(mvRwLock.isWriteLockedByCurrentThread()); + return 1L; + }); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mtmv.invalidateWholeMv("The base table has been updated: db.t1").await(); + } + + // The record carries the status the insert above applied, so the two are one change. + ArgumentCaptor captor = ArgumentCaptor.forClass(AlterMTMV.class); + Mockito.verify(editLog).submitEdit(Mockito.eq(OperationType.OP_ALTER_MTMV), captor.capture()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, captor.getValue().getStatus().getState()); + Assertions.assertEquals("The base table has been updated: db.t1", + captor.getValue().getStatus().getSchemaChangeDetail()); + Mockito.verify(editLogItem).await(); + } + @Test public void testAlterPropertiesSubmitsJournalWhileHoldingMvLock() { MTMV mtmv = new MTMV(); @@ -748,10 +808,24 @@ public void testPartitionStatesEmptyOnImageWrittenBeforeTheFieldExisted() { JsonObject image = JsonParser.parseString(GsonUtils.GSON.toJson(mtmv)).getAsJsonObject(); Assertions.assertNotNull(image.remove("pst")); - // The field is gone from the image, so gsonPostProcess() is the only thing that can make it a map. MTMV restored = GsonUtils.GSON.fromJson(image.toString(), MTMV.class); - // Read the field itself: the getter lazily creates the map, so it would hide a missing init. + // Read the field itself rather than through the getter, which copies whatever is there. + Assertions.assertNotNull(Deencapsulation.getField(restored, "partitionStates")); + Assertions.assertTrue(restored.getPartitionStates().isEmpty()); + } + + @Test + public void testPartitionStatesImageThatCarriesTheFieldAsNullLoadsAsAnEmptyMap() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 5))); + JsonObject image = JsonParser.parseString(GsonUtils.GSON.toJson(mtmv)).getAsJsonObject(); + // An MV is created with an empty map and an image that leaves the member out keeps it, so a + // member that is there and null is the one case gsonPostProcess() has to answer for. + image.add("pst", JsonNull.INSTANCE); + + MTMV restored = GsonUtils.GSON.fromJson(image.toString(), MTMV.class); + Assertions.assertNotNull(Deencapsulation.getField(restored, "partitionStates")); Assertions.assertTrue(restored.getPartitionStates().isEmpty()); } @@ -827,23 +901,79 @@ public void testAddTaskResultReplayAppliesPartitionStates() { Assertions.assertEquals(5, state.getLatestEpoch()); } + @Test + public void testTaskResultLeavesTheSnapshotOfADirtyPartitionOut() { + MTMV mtmv = Mockito.spy(buildSerializableMTMV()); + mtmv.getIvmInfo().setEnableIvm(true); + Mockito.doReturn(Sets.newHashSet("p202601", "p202602")).when(mtmv).getPartitionNames(); + // p202601 was invalidated while the task ran, p202602 was not. + mtmv.alterPartitionStates(Maps.newHashMap(Map.of( + "p202601", new MTMVPartitionState(1, 2), + "p202602", new MTMVPartitionState(2, 2)))); + Map snapshots = Maps.newHashMap(Map.of( + "p202601", new MTMVRefreshPartitionSnapshot(), + "p202602", new MTMVRefreshPartitionSnapshot())); + + runAddTaskResult(mtmv, snapshots, null, false, Map.of()); + + // The invalidation dropped p202601's snapshot so that no transparent rewrite serves its rows, and + // a result written back afterwards must not put it there again -- while the partition that was not + // invalidated keeps the snapshot the task produced. + Assertions.assertEquals(Sets.newHashSet("p202602"), + mtmv.getRefreshSnapshot().getPartitionSnapshots().keySet()); + } + + @Test + public void testReplayIgnoresTheEpochsTheTaskCaptured() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.getIvmInfo().setEnableIvm(true); + mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 3))); + + // A replayed record carries the states that were decided when it was written, and the epochs a + // task captured live in memory only. Applying them on replay would re-decide a result that is + // already fixed, and the payload's value -- not the capture -- is what the record means. + runAddTaskResult(mtmv, Map.of("p202601", new MTMVPartitionState(7, 9)), true, Map.of("p202601", 1L)); + + Assertions.assertEquals(7, mtmv.getPartitionStates().get("p202601").getRefreshEpoch()); + Assertions.assertEquals(9, mtmv.getPartitionStates().get("p202601").getLatestEpoch()); + } + + @Test + public void testAFreshMvHasAnEmptyStateMapAndAlignmentFillsIt() { + MTMV mtmv = ivmMvWithPartitions(Sets.newHashSet("p202601")); + + // The map is created with the MV, so a reader has no null case to answer; alignment is what puts + // the MV's partitions into it. + Assertions.assertNotNull(Deencapsulation.getField(mtmv, "partitionStates")); + Assertions.assertTrue(mtmv.getPartitionStates().isEmpty()); + + runAlignPartitionStates(mtmv); + + Assertions.assertEquals(Sets.newHashSet("p202601"), mtmv.getPartitionStates().keySet()); + Assertions.assertEquals(1, mtmv.getPartitionStates().get("p202601").getLatestEpoch()); + Assertions.assertEquals(0, mtmv.getPartitionStates().get("p202601").getRefreshEpoch()); + } + @Test public void testIvmTaskResultJournalsPartitionStates() { MTMV mtmv = buildSerializableMTMV(); mtmv.getIvmInfo().setEnableIvm(true); mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 5))); - List journaled = runAddTaskResult(mtmv, null, false); + // The task published p202601, so that is what reaches the journal: the partition at the epoch it + // was published with, and the requirement it was read under. It carries no more than that -- see + // testTaskResultJournalsOnlyThePartitionsItPublished. + List journaled = runAddTaskResult(mtmv, null, false, Map.of("p202601", 6L)); Assertions.assertEquals(1, journaled.size()); - MTMVPartitionState journaledState = journaled.get(0).getPartitionStates().get("p202601"); - Assertions.assertEquals(3, journaledState.getRefreshEpoch()); - Assertions.assertEquals(5, journaledState.getLatestEpoch()); + Map published = journaled.get(0).getPartitionStates(); + Assertions.assertEquals(6, published.get("p202601").getRefreshEpoch()); + Assertions.assertEquals(5, published.get("p202601").getLatestEpoch()); // The payload reaches the journal as JSON, so it has to survive that trip to be replayable. AlterMTMV readBack = GsonUtils.GSON.fromJson( GsonUtils.GSON.toJson(journaled.get(0)), AlterMTMV.class); - Assertions.assertEquals(3, readBack.getPartitionStates().get("p202601").getRefreshEpoch()); + Assertions.assertEquals(6, readBack.getPartitionStates().get("p202601").getRefreshEpoch()); Assertions.assertEquals(5, readBack.getPartitionStates().get("p202601").getLatestEpoch()); } @@ -859,6 +989,215 @@ public void testNonIvmTaskResultDoesNotJournalPartitionStates() { Assertions.assertNull(journaled.get(0).getPartitionStates()); } + @Test + public void testPartitionsNeedingRebuildAreTheDirtyOnes() { + MTMV mtmv = ivmMvWithPartitions(Sets.newHashSet("p202601", "p202602", "p202603")); + mtmv.alterPartitionStates(Map.of( + "p202601", new MTMVPartitionState(1, 2), + "p202602", new MTMVPartitionState(4, 4), + "p202603", MTMVPartitionState.initial())); + + // The names a refresh has to rebuild rather than catch up: the state the whole-MV-rebuild-did-not- + // finish case leaves behind is exactly this one, so a reader that routes by it gets the partitions + // the snapshots cannot name. + Assertions.assertEquals(Sets.newHashSet("p202601", "p202603"), mtmv.getPartitionsNeedingRebuild()); + + // A plain MV keeps no states, so it has no requirement to report: the plan of a partition refresh + // reads this, and the answer has to be empty rather than anything the snapshots do not say. + Assertions.assertTrue(buildSerializableMTMV().getPartitionsNeedingRebuild().isEmpty()); + } + + @Test + public void testAllPartitionsNeedARebuildOnlyWhenNoneOfThemIsClean() { + MTMV mtmv = ivmMvWithPartitions(Sets.newHashSet("p202601", "p202602")); + mtmv.alterPartitionStates(Map.of( + "p202601", new MTMVPartitionState(1, 2), + "p202602", MTMVPartitionState.initial())); + + // Every partition is either behind its requirement or never filled, so a whole-MV refresh does + // nothing the per-partition routing would not. The second is what a fresh partition looks like: + // {0, 1} is dirty, so it needs no clause of its own. + Assertions.assertTrue(mtmv.allPartitionsNeedRebuild()); + + // A partition that holds data and is caught up is what makes a whole-MV refresh waste: it would be + // recomputed for nothing. Which is why the verdict is read for the escalation only -- the routing + // leaves such a partition alone. + mtmv.alterPartitionStates(Map.of( + "p202601", new MTMVPartitionState(2, 2), + "p202602", new MTMVPartitionState(2, 2))); + Assertions.assertFalse(mtmv.allPartitionsNeedRebuild()); + + // An MV with no partitions is not an escalation either: the empty answer must not read as "all of + // them". + Assertions.assertFalse(buildSerializableMTMV().allPartitionsNeedRebuild()); + } + + @Test + public void testPartitionStateIsDirtyWhenItIsBehindItsRequirement() { + Assertions.assertFalse(new MTMVPartitionState(1, 1).isDirty()); + Assertions.assertFalse(new MTMVPartitionState(4, 4).isDirty()); + Assertions.assertTrue(new MTMVPartitionState(1, 2).isDirty()); + Assertions.assertTrue(new MTMVPartitionState(5, 6).isDirty()); + // A refreshEpoch of 0 is not an exemption. It reads as "never refreshed, so no rows", and that is + // not durable: a refresh commits the MV data transaction before its task result publishes the + // epochs, so those rows can be there while the state still says 0. Reading the pair as clean would + // let a later invalidation raising latestEpoch go unnoticed. + Assertions.assertTrue(new MTMVPartitionState(0, 1).isDirty()); + Assertions.assertTrue(new MTMVPartitionState(0, 2).isDirty()); + // Which leaves "never refreshed" as a description of the past and no longer a category of its own: + // an aligned entry is (0, 1), so a refreshEpoch of 0 can never be the reason a partition is clean. + Assertions.assertEquals(0, MTMVPartitionState.initial().getRefreshEpoch()); + Assertions.assertTrue(MTMVPartitionState.initial().isDirty()); + Assertions.assertEquals(1, MTMVPartitionState.initial().getLatestEpoch()); + } + + @Test + public void testAlignPartitionStatesCreatesAndDropsEntries() { + MTMV mtmv = ivmMvWithPartitions(Sets.newHashSet("p202601", "p202602")); + mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 5))); + + // A partition that is already there keeps its requirement: alignment creates and destroys + // entries, it never rewrites one. + List journaled = runAlignPartitionStates(mtmv); + Assertions.assertEquals(1, journaled.size()); + Assertions.assertEquals(MTMVAlterOpType.ALTER_PARTITION_STATES, journaled.get(0).getOpType()); + Assertions.assertEquals(3, journaled.get(0).getPartitionStates().get("p202601").getRefreshEpoch()); + Assertions.assertEquals(5, journaled.get(0).getPartitionStates().get("p202601").getLatestEpoch()); + Assertions.assertEquals(0, journaled.get(0).getPartitionStates().get("p202602").getRefreshEpoch()); + Assertions.assertEquals(1, journaled.get(0).getPartitionStates().get("p202602").getLatestEpoch()); + Assertions.assertEquals(2, mtmv.getPartitionStates().size()); + + // Aligning onto the same set changes nothing, so it journals nothing either. + Assertions.assertTrue(runAlignPartitionStates(mtmv).isEmpty()); + + // A partition that is gone loses its entry, which is what keeps a mark from landing on state + // that no partition can hold rows for. + Mockito.doReturn(Sets.newHashSet("p202602")).when(mtmv).getPartitionNames(); + runAlignPartitionStates(mtmv); + Assertions.assertEquals(Sets.newHashSet("p202602"), mtmv.getPartitionStates().keySet()); + } + + @Test + public void testAlignPartitionStatesDoesNothingForANonIvmMv() { + MTMV mtmv = buildSerializableMTMV(); + Assertions.assertFalse(mtmv.getIvmInfo().isEnableIvm()); + + Assertions.assertTrue(runAlignPartitionStates(mtmv).isEmpty()); + + Assertions.assertTrue(mtmv.getPartitionStates().isEmpty()); + } + + @Test + public void testTaskResultRecordsTheCapturedEpochWithoutTouchingTheRequirement() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.getIvmInfo().setEnableIvm(true); + // The partition was read at requirement 5, and a base-table change raised it to 6 before the + // result was written back. Writing the captured requirement back would swallow that rebuild. + mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 6))); + + List journaled = runAddTaskResult(mtmv, null, false, Map.of("p202601", 5L)); + + Assertions.assertEquals(1, journaled.size()); + MTMVPartitionState journaledState = journaled.get(0).getPartitionStates().get("p202601"); + Assertions.assertEquals(5, journaledState.getRefreshEpoch()); + Assertions.assertEquals(6, journaledState.getLatestEpoch()); + Assertions.assertTrue(journaledState.isDirty()); + + Assertions.assertEquals(5, mtmv.getPartitionStates().get("p202601").getRefreshEpoch()); + Assertions.assertEquals(6, mtmv.getPartitionStates().get("p202601").getLatestEpoch()); + } + + @Test + public void testTaskResultSkipsPartitionsItDidNotCapture() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.getIvmInfo().setEnableIvm(true); + mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 3))); + + // The task captured only p202602, which is not a partition of this MV any more -- and p202601, + // which it did not capture, must keep the epoch of the data it still holds. + runAddTaskResult(mtmv, null, false, Map.of("p202602", 9L)); + + Assertions.assertEquals(Sets.newHashSet("p202601"), mtmv.getPartitionStates().keySet()); + Assertions.assertEquals(3, mtmv.getPartitionStates().get("p202601").getRefreshEpoch()); + } + + @Test + public void testLatestEpochsOmitsPartitionsWithoutAnEntry() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.alterPartitionStates(Map.of("p202601", new MTMVPartitionState(3, 5))); + + Assertions.assertEquals(Map.of("p202601", 5L), + mtmv.getLatestEpochs(Sets.newHashSet("p202601", "p202602"))); + Assertions.assertTrue(mtmv.getLatestEpochs(Sets.newHashSet()).isEmpty()); + } + + @Test + public void testTaskResultJournalsOnlyThePartitionsItPublished() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.getIvmInfo().setEnableIvm(true); + mtmv.alterPartitionStates(Map.of( + "p202601", new MTMVPartitionState(3, 5), + "p202602", new MTMVPartitionState(4, 4))); + + // Only p202601 was published by this task; p202602 belongs to another record. + List journaled = runAddTaskResult(mtmv, null, false, Map.of("p202601", 6L)); + + Assertions.assertEquals(1, journaled.size()); + Map published = journaled.get(0).getPartitionStates(); + Assertions.assertEquals(Set.of("p202601"), published.keySet()); + Assertions.assertEquals(6, published.get("p202601").getRefreshEpoch()); + // The requirement raised while the task ran rides along: a payload that dropped it would let a + // replay restore the older one and lose the rebuild it asks for. + Assertions.assertEquals(5, published.get("p202601").getLatestEpoch()); + } + + @Test + public void testTaskResultReplayMergesThePartitionsItCarries() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.getIvmInfo().setEnableIvm(true); + Map current = new HashMap<>(); + current.put("p202601", new MTMVPartitionState(3, 5)); + current.put("p202602", new MTMVPartitionState(9, 9)); + mtmv.alterPartitionStates(current); + + // A payload carries only the partitions its task published, so a replay merges it. Assigning + // would drop p202602, which another record -- an invalidation that ran during the task -- owns. + runAddTaskResult(mtmv, Map.of("p202601", new MTMVPartitionState(6, 5)), true); + + Map replayed = mtmv.getPartitionStates(); + Assertions.assertEquals(2, replayed.size()); + Assertions.assertEquals(6, replayed.get("p202601").getRefreshEpoch()); + Assertions.assertEquals(9, replayed.get("p202602").getRefreshEpoch()); + } + + @Test + public void testMarkPartitionsForRebuildIsJournaledAndClearedByTheTaskResult() { + MTMV mtmv = buildSerializableMTMV(); + mtmv.getIvmInfo().setEnableIvm(true); + // The journaling path names the MV, and the fixture is built through the deserialization + // constructor, which leaves the name unset. + Deencapsulation.setField(mtmv, "name", "mv1"); + Map current = new HashMap<>(); + current.put("p202601", new MTMVPartitionState(1, 1)); + current.put("p202602", new MTMVPartitionState(1, 1)); + mtmv.alterPartitionStates(current); + + List marked = Lists.newArrayList(); + withMockedEditLog(marked, () -> mtmv.markPartitionsForRebuild(Set.of("p202601"))); + + // Journaled as the full map: this is an invalidation-shaped record, not a task result. + Assertions.assertEquals(1, marked.size()); + Assertions.assertEquals(2, marked.get(0).getPartitionStates().get("p202601").getLatestEpoch()); + Assertions.assertEquals(1, marked.get(0).getPartitionStates().get("p202602").getLatestEpoch()); + Assertions.assertTrue(mtmv.getPartitionStates().get("p202601").isDirty()); + Assertions.assertFalse(mtmv.getPartitionStates().get("p202602").isDirty()); + + // Publishing the partition meets the raised requirement. + runAddTaskResult(mtmv, null, false, Map.of("p202601", 2L)); + Assertions.assertEquals(2, mtmv.getPartitionStates().get("p202601").getRefreshEpoch()); + Assertions.assertFalse(mtmv.getPartitionStates().get("p202601").isDirty()); + } + /** * Runs one ADD_TASK result through {@link MTMV#addTaskResult}, optionally carrying {@code * journaledStates} in its payload the way a real journal would, and returns the payloads that @@ -866,10 +1205,71 @@ public void testNonIvmTaskResultDoesNotJournalPartitionStates() { */ private List runAddTaskResult(MTMV mtmv, Map journaledStates, boolean isReplay) { + return runAddTaskResult(mtmv, journaledStates, isReplay, Map.of()); + } + + /** + * Same, with the epochs the task captured, which a live task result turns into {@code refreshEpoch} + * (the task carries them in memory; the journal carries the resulting states). + */ + private List runAddTaskResult(MTMV mtmv, Map journaledStates, + boolean isReplay, Map capturedEpochs) { + return runAddTaskResult(mtmv, Map.of(), journaledStates, isReplay, capturedEpochs); + } + + /** + * Same, with the snapshots the task produced, which a live task result merges into the MV's own map + * unless the partition it describes came out dirty. + */ + private List runAddTaskResult(MTMV mtmv, + Map partitionSnapshots, + Map journaledStates, boolean isReplay, + Map capturedEpochs) { + List journaled = Lists.newArrayList(); + withMockedEditLog(journaled, () -> { + MTMVTask task = new MTMVTask(mtmv, mtmv.getRelation(), null); + task.setStatus(TaskStatus.FAILED); + // Set on the task rather than through its getter: the getter hands out a detached copy, which + // is the point of it -- a cancelled task's worker may still be merging into the field. + Map taskCapturedEpochs = Maps.newConcurrentMap(); + taskCapturedEpochs.putAll(capturedEpochs); + Deencapsulation.setField(task, "ivmCapturedEpochs", taskCapturedEpochs); + AlterMTMV alterMTMV = new AlterMTMV(new TableNameInfo("db1", "mv1"), MTMVAlterOpType.ADD_TASK); + alterMTMV.setTask(task); + alterMTMV.setRelation(mtmv.getRelation()); + alterMTMV.setPartitionSnapshots(partitionSnapshots); + alterMTMV.setPartitionStates(journaledStates); + Assertions.assertTrue(mtmv.addTaskResult(alterMTMV, isReplay)); + }); + return journaled; + } + + /** + * An IVM MV whose partition names are the given ones: alignment reads them from the MV now, so a case + * that wants entries created or dropped has to say which partitions the MV has. + */ + private MTMV ivmMvWithPartitions(Set partitionNames) { + MTMV mtmv = Mockito.spy(buildSerializableMTMV()); + mtmv.getIvmInfo().setEnableIvm(true); + Mockito.doReturn(partitionNames).when(mtmv).getPartitionNames(); + return mtmv; + } + + private List runAlignPartitionStates(MTMV mtmv) { + // A journaling path names the MV, and the fixture is built through the deserialization + // constructor, which leaves the name unset (setName() cannot be used: it rekeys the index map + // by the current name, which is still null at that point). + Deencapsulation.setField(mtmv, "name", "mv1"); + List journaled = Lists.newArrayList(); + withMockedEditLog(journaled, () -> mtmv.alignPartitionStates()); + return journaled; + } + + /** Runs {@code action} against a mocked edit log and collects the payloads it journals. */ + private void withMockedEditLog(List journaled, Runnable action) { Env env = Mockito.mock(Env.class); EditLog editLog = Mockito.mock(EditLog.class); EditLogItem editLogItem = Mockito.mock(EditLogItem.class); - List journaled = Lists.newArrayList(); Mockito.when(env.getEditLog()).thenReturn(editLog); Mockito.when(env.getMtmvService()).thenReturn(Mockito.mock(MTMVService.class)); Mockito.when(editLog.submitEdit(Mockito.eq(OperationType.OP_ALTER_MTMV), Mockito.any(AlterMTMV.class))) @@ -878,18 +1278,9 @@ private List runAddTaskResult(MTMV mtmv, Map mockedEnv = Mockito.mockStatic(Env.class)) { mockedEnv.when(Env::getCurrentEnv).thenReturn(env); - Assertions.assertTrue(mtmv.addTaskResult(alterMTMV, isReplay)); + action.run(); } - return journaled; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java index af53ab71b6ba43..3935c0f6055fca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmBaselineRebuildTest.java @@ -35,9 +35,13 @@ import org.apache.doris.job.extensions.mtmv.MTMVTaskContext; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVAlterOpType; +import org.apache.doris.mtmv.MTMVPartitionState; import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; +import org.apache.doris.mtmv.MTMVRefreshPartitionSnapshot; import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVStatus; import org.apache.doris.persist.AlterMTMV; import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.persist.RecoverInfo; @@ -46,6 +50,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -54,6 +59,7 @@ import java.time.LocalDate; import java.util.Collections; +import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -74,7 +80,7 @@ public void testTruncateMarksBaselineRebuild() throws Exception { executeSql("TRUNCATE TABLE ivm_base"); - Assertions.assertTrue(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -84,7 +90,7 @@ public void testTruncatePartitionMarksBaselineRebuild() throws Exception { executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001)"); - Assertions.assertTrue(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -99,7 +105,7 @@ public void testRepeatedBrokenEventsAdvanceSchemaChangeVersion() throws Exceptio executeSql("TRUNCATE TABLE ivm_base PARTITION(p202002)"); Assertions.assertEquals(initialSchemaChangeVersion + 2, mtmv.getSchemaChangeVersion()); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } @Test @@ -110,7 +116,7 @@ public void testDropPartitionMarksBaselineRebuild() throws Exception { // SELF_MANAGE: the single MV partition reads every base partition, and the partition mapping API // answers nothing for it, so the whole MV has to be rebuilt. - Assertions.assertTrue(getMtmv(db).getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -119,16 +125,116 @@ public void testDropColumnMarksBaselineRebuildOnlyWhenReferenced() throws Except createPartitionedIvmTableAndMv(db); MTMV mtmv = getMtmv(db); - // ivm_mv selects dt, k1, v1. Dropping a column it does not use must leave the baseline alone. + // ivm_mv selects dt, k1, v1. Dropping a column it does not use must leave the baseline alone: no + // partition's requirement is raised, so no partition is sent to a rebuild. The MV state is not the + // witness here -- a column change puts any MV into SCHEMA_CHANGE through the shared base-table hook, + // IVM or not (see testRenameTableMarksBaselineRebuild), so telling a referenced column from an + // unreferenced one is that hook's criterion to refine and not this one's. + alignStatesOf(mtmv); + Map before = latestEpochsOf(mtmv); executeSql("ALTER TABLE ivm_base ADD COLUMN spare int"); executeSql("ALTER TABLE ivm_base DROP COLUMN spare"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(before, latestEpochsOf(mtmv), + "a column the MV does not use must not raise any partition's requirement"); // Dropping a column the MV uses makes the MV query unanalyzable: the change is metadata-only // and emits no binlog, so an incremental refresh would silently keep the rows of the old // column. The baseline has to be invalidated instead. executeSql("ALTER TABLE ivm_base DROP COLUMN v1"); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + // And the state is the only record: the invalidation stands for the change, so the detail names + // the reason rather than the alter that caused it. + assertUnanalyzableDetail(mtmv); + } + + /** + * The sync window is a property any MV can carry, but only an IVM MV maintains a baseline through it. + * Widening it on a plain MV brings base partitions back into a set it never stopped maintaining, so a + * whole-MV rebuild would recompute partitions it already has -- which is why the widening checks are + * judged together, under the guard that the MV maintains an IVM baseline at all. + */ + @Test + public void testWideningTheSyncWindowDoesNotInvalidateANonIvmMv() throws Exception { + String db = "ivm_sync_window_non_ivm"; + createPartitionedIvmTable(db); + createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv\n" + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1')\n" + + "AS SELECT dt, k1, v1 FROM ivm_base"); + MTMV mtmv = getMtmv(db); + Assertions.assertFalse(mtmv.isIvm()); + + // A window starts applying, is narrowed, then widened. The widen is what owes a rebuild for an IVM + // MV; for this one it is a change to a window nothing maintains. + executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '10')"); + executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1')"); + executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '10')"); + + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + // The window is still applied: it is the rebuild it owes that is IVM's, not the property. + Assertions.assertEquals("10", mtmv.getMvProperties().get("partition_sync_limit")); + } + + /** + * A plain MV can name its columns itself, and the check has to keep matching those by position. The MV + * persists the names it was given while re-analysing its query yields the names it selects -- and only + * an IVM MV has the stored query rewritten with its aliases -- so a name lookup finds nothing and + * reports a column that exists as missing. The check runs on every refresh of an MV in SCHEMA_CHANGE, + * so that MV would stop refreshing for good after an otherwise compatible base-table change. + */ + @Test + public void testAnMvWithItsOwnColumnNamesIsStillUsable() throws Exception { + String db = "ivm_mv_column_aliases"; + createPartitionedIvmTable(db); + createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv (c_dt, c_k1, c_v1)\n" + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1')\n" + + "AS SELECT dt, k1, v1 FROM ivm_base"); + MTMV mtmv = getMtmv(db); + Assertions.assertFalse(mtmv.isIvm()); + Assertions.assertEquals("c_v1", mtmv.getBaseSchema(true).get(2).getName()); + + Assertions.assertDoesNotThrow(() -> MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, + MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK))); + } + + /** + * The query check belongs to the MV, not to IVM. A plain MV's query is taken away by the same base + * table change as an IVM MV's is, and it is left in the same state -- the state its refresh re-analyzes + * the query under, which is how it finds out. The detail is what the two have to agree on, and it has + * to say the query is gone: "the base table has been updated" is true of every alter, including the + * ones the query survives. + */ + @Test + public void testDroppingAReadColumnInvalidatesANonIvmMv() throws Exception { + String db = "ivm_query_unusable_non_ivm"; + createPartitionedIvmTable(db); + createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv\n" + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1')\n" + + "AS SELECT dt, k1, v1 FROM ivm_base"); + MTMV mtmv = getMtmv(db); + Assertions.assertFalse(mtmv.isIvm()); + + // A column this MV does not read leaves its query alone, so the MV is put into SCHEMA_CHANGE for + // the ordinary reason -- which is what the shared hook does for any column change, IVM or not -- + // and not because its query went away. The detail is what tells the two apart here, because the + // state is the same either way. + executeSql("ALTER TABLE ivm_base ADD COLUMN spare int"); + executeSql("ALTER TABLE ivm_base DROP COLUMN spare"); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + Assertions.assertFalse( + mtmv.getStatus().getSchemaChangeDetail().contains("no longer analyzable"), + "a column the MV does not read leaves the query analyzable, was: " + + mtmv.getStatus().getSchemaChangeDetail()); + + // A column it does read takes the query away, and that is what the MV is invalidated with. + executeSql("ALTER TABLE ivm_base DROP COLUMN v1"); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + assertUnanalyzableDetail(mtmv); } /** @@ -145,10 +251,42 @@ public void testDropPartitionMarksOnlyMvPartitionsThatReadIt() throws Exception Set expected = mvPartitionsWithSameRange(mtmv, getBaseTable(db), "p202001"); Assertions.assertEquals(1, expected.size()); + alignStatesOf(mtmv); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - Assertions.assertFalse(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); - Assertions.assertEquals(expected, mtmv.getIvmInfo().getPendingBaselineRebuildPartitions()); + // The partitions that read the dropped one have their requirement raised, and only those: every + // other partition keeps catching up incrementally, and no whole-MV barrier is raised. + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + for (String partitionName : mtmv.getPartitionNames()) { + long expectedLatest = expected.contains(partitionName) ? 2 : 1; + Assertions.assertEquals(expectedLatest, + mtmv.getPartitionStates().get(partitionName).getLatestEpoch()); + } + } + + /** + * The other half of an invalidation: the partitions it marks lose their refresh snapshot, which is what + * keeps transparent rewrite away from them until the rebuild has replaced their rows. + */ + @Test + public void testInvalidationDropsTheSnapshotsOfThePartitionsItMarks() throws Exception { + String db = "ivm_invalidation_drops_snapshots"; + createPartitionedIvmTableAndPartitionedMv(db); + MTMV mtmv = getMtmv(db); + Set expected = mvPartitionsWithSameRange(mtmv, getBaseTable(db), "p202001"); + Assertions.assertEquals(1, expected.size()); + Map snapshots = Maps.newHashMap(); + for (String partitionName : mtmv.getPartitionNames()) { + snapshots.put(partitionName, new MTMVRefreshPartitionSnapshot()); + } + mtmv.getRefreshSnapshot().updateSnapshots(snapshots, mtmv.getPartitionNames()); + alignStatesOf(mtmv); + + executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); + + Assertions.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().keySet().stream() + .anyMatch(expected::contains)); + Assertions.assertFalse(mtmv.getRefreshSnapshot().getPartitionSnapshots().isEmpty()); } /** @@ -168,7 +306,7 @@ public void testDropPartitionOutsideMvPartitionsMarksNothing() throws Exception executeSql("ALTER TABLE ivm_base DROP PARTITION p202003"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -218,7 +356,7 @@ public void testChangedPartitionOutsideTheSyncWindowRebuildsTheWholeMv() throws + " 'partition_sync_time_unit' = 'YEAR')"); executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001)"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -267,7 +405,7 @@ public void testChangeThatMixesInWindowAndOutOfWindowPartitionsRebuildsTheWholeM // p202001 is not, which is exactly the mix a non-empty selection must not be allowed to hide. executeSql("TRUNCATE TABLE ivm_base PARTITION(p202001, pThisYear)"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -282,37 +420,37 @@ public void testOnlyAWiderSyncWindowRequiresCompleteBaselineRebuild() throws Exc String db = "ivm_sync_window_property_change"; createPartitionedIvmTableAndPartitionedMv(db); MTMV mtmv = getMtmv(db); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); // No limit is in effect, so the unit it is paired with decides nothing. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_time_unit' = 'YEAR')"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); // The window starts applying: it takes partitions out of what the MV maintains, it brings none back. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '10')"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); // The same window, restated. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '10')"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); // Narrower: it only removes partitions from the maintained set. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '1')"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); // Wider: the partitions it takes back in skipped their deltas while they were outside. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '10')"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); - clearBaselineRebuild(mtmv); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + resetMvState(mtmv); // The limit is gone: every partition comes back. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_limit' = '0')"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); - clearBaselineRebuild(mtmv); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + resetMvState(mtmv); // Still no limit in effect, so the unit decides nothing again. executeSql("ALTER MATERIALIZED VIEW ivm_mv SET ('partition_sync_time_unit' = 'DAY')"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -339,14 +477,15 @@ public void testTheSyncLimitIsReadOnBothSidesOfTheMapping() throws Exception { try (MockedStatic partitionUtil = Mockito.mockStatic(MTMVPartitionUtil.class, Mockito.CALLS_REAL_METHODS)) { Assertions.assertTrue(mtmv.invalidateIvmBaseline(new BaseTableInfo(baseTable), - Collections.singletonMap("p202001", baseTable.getPartition("p202001").getId()))); + Collections.singletonMap("p202001", baseTable.getPartition("p202001").getId()), + "test partition change")); partitionUtil.verify(() -> MTMVPartitionUtil.isPartitionSyncLimitActive(Mockito.any()), Mockito.times(2)); } // p202001 is outside the window while it is in effect, so its rows are described by no mapping // entry and only the limit can tell that apart from "no MV partition reads it". - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -387,7 +526,7 @@ public void testNonPctBaseTablePartitionChangeRequiresCompleteBaselineRebuild() executeSql("ALTER TABLE ivm_dim DROP PARTITION d202001"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -404,10 +543,15 @@ public void testMultiPctTablePartitionChangeStillNarrows() throws Exception { Set expected = mvPartitionsWithSameRange(mtmv, getBaseTable(db), "p202001"); Assertions.assertEquals(1, expected.size()); + alignStatesOf(mtmv); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - Assertions.assertFalse(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); - Assertions.assertEquals(expected, mtmv.getIvmInfo().getPendingBaselineRebuildPartitions()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + for (String partitionName : mtmv.getPartitionNames()) { + long expectedLatest = expected.contains(partitionName) ? 2 : 1; + Assertions.assertEquals(expectedLatest, + mtmv.getPartitionStates().get(partitionName).getLatestEpoch()); + } } /** @@ -449,7 +593,7 @@ public void testMultiPctTableBusyOtherTableRebuildsWholeMv() throws Exception { // The batch fails on its first table here, so what this covers is the whole-MV fallback; the // release of the locks taken before the busy one is covered in MetaLockUtilsTest. - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -491,7 +635,7 @@ public void testReplacePartitionMarksBaselineRebuild() throws Exception { executeSql("ALTER TABLE ivm_base REPLACE PARTITION (p202001) " + "WITH TEMPORARY PARTITION (tp202001)"); - Assertions.assertTrue(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -499,11 +643,11 @@ public void testRecoverPartitionMarksBaselineRebuild() throws Exception { String db = "ivm_broken_recover_partition"; createPartitionedIvmTableAndMv(db); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - clearBaselineRebuild(getMtmv(db)); + resetMvState(getMtmv(db)); executeSql("RECOVER PARTITION p202001 FROM ivm_base"); - Assertions.assertTrue(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } /** @@ -517,11 +661,11 @@ public void testRecoverPartitionOnPartitionedMvRequiresCompleteBaselineRebuild() createPartitionedIvmTableAndPartitionedMv(db); MTMV mtmv = getMtmv(db); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - clearBaselineRebuild(mtmv); + resetMvState(mtmv); executeSql("RECOVER PARTITION p202001 FROM ivm_base"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } @Test @@ -531,13 +675,13 @@ public void testRecoverAndDropKeepGlobalBrokenState() throws Exception { MTMV mtmv = getMtmv(db); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); executeSql("RECOVER PARTITION p202001 FROM ivm_base"); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); executeSql("ALTER TABLE ivm_base DROP PARTITION p202002"); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } /** @@ -556,14 +700,14 @@ public void testRecoveredPartitionWhoseNameWasReusedRebuildsTheWholeMv() throws MTMV mtmv = getMtmv(db); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - clearBaselineRebuild(mtmv); + resetMvState(mtmv); // Live again under the dropped name, with a range no MV partition covers: the RECOVER below is // still about the recycled partition, not about this one. executeSql("ALTER TABLE ivm_base ADD PARTITION p202001 VALUES [('2020-04-01'), ('2020-05-01'))"); executeSql("RECOVER PARTITION p202001 AS p202003 FROM ivm_base"); - Assertions.assertTrue(mtmv.getIvmInfo().requiresCompleteBaselineRebuild()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } @Test @@ -574,7 +718,7 @@ public void testAddPartitionDoesNotMarkBaselineRebuild() throws Exception { executeSql("ALTER TABLE ivm_base ADD PARTITION p202003 " + "VALUES [('2020-03-01'), ('2020-04-01'))"); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -586,7 +730,7 @@ public void testDropTempPartitionDoesNotMarkBaselineRebuild() throws Exception { executeSql("ALTER TABLE ivm_base DROP TEMPORARY PARTITION tp202001"); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -596,35 +740,71 @@ public void testDropMissingPartitionIfExistsDoesNotMarkBaselineRebuild() throws executeSql("ALTER TABLE ivm_base DROP PARTITION IF EXISTS p_missing"); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test - public void testRenameTableDoesNotMarkBaselineRebuild() throws Exception { + public void testRenameTableMarksBaselineRebuild() throws Exception { String db = "ivm_broken_rename_table"; createPartitionedIvmTableAndMv(db); + MTMV mtmv = getMtmv(db); + alignStatesOf(mtmv); + Map before = latestEpochsOf(mtmv); executeSql("ALTER TABLE ivm_base RENAME ivm_base_renamed"); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + // A rename leaves every column alone, so it raises no partition's requirement: no partition's rows + // have to be recomputed, and an epoch is not the place to record this change. What the rename does + // move is the MV state, through the shared base-table hook: the MV query still spells the old name, + // so it no longer analyzes, and the state is what sends the next refresh to a whole-MV COMPLETE + // rather than let it report SUCCESS over rows it can no longer recompute. + Assertions.assertEquals(before, latestEpochsOf(mtmv)); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } + /** + * The state a rename sets is the shared hook's, not IVM's: a non-IVM MV gets it for the same reason -- + * its refresh re-analyzes the query under it -- and an IVM MV is not exempt. What an IVM MV has instead + * of a re-analysis is the epoch state, and that is untouched by a rename, which is the division of + * labour the two tests around this one pin. + */ @Test - public void testRenameTableBackKeepsIncrementalRefreshStartable() throws Exception { + public void testRenameStillInvalidatesANonIvmMv() throws Exception { + String db = "ivm_broken_rename_non_ivm"; + createPartitionedIvmTable(db); + createMvByNereids("CREATE MATERIALIZED VIEW ivm_mv\n" + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + "DISTRIBUTED BY RANDOM BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1')\n" + + "AS SELECT dt, k1, v1 FROM ivm_base"); + MTMV mtmv = getMtmv(db); + Assertions.assertFalse(mtmv.isIvm()); + + executeSql("ALTER TABLE ivm_base RENAME ivm_base_renamed"); + + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); + } + + @Test + public void testRenameTableBackStillRequiresAWholeMvRefresh() throws Exception { String db = "ivm_broken_rename_table_back"; createPartitionedIvmTableAndMv(db); + MTMV mtmv = getMtmv(db); + alignStatesOf(mtmv); + Map before = latestEpochsOf(mtmv); executeSql("ALTER TABLE ivm_base RENAME ivm_base_renamed"); executeSql("ALTER TABLE ivm_base_renamed RENAME ivm_base"); - // A rename changes no column, so it must not invalidate the baseline in either direction: - // once the table is renamed back, the MV query is analyzable again and a strict INCREMENTAL - // refresh has to be able to start. A "baseline rebuild required" flag left behind by the - // rename would reject every one of them until a COMPLETE refresh had been run, even though - // nothing the MV depends on ever changed. - MTMV mtmv = getMtmv(db); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); - Assertions.assertDoesNotThrow(() -> mtmv.validateIvmRefreshStart(mtmv.getSchemaChangeVersion())); + // Renaming the table back makes the MV query analyzable again, but the state stays where the first + // rename put it, and the second rename is why: the dependencies are registered under the name the MV + // query spells, so a rename of the table away from that name finds nothing to update and the rename + // back finds nothing to clear. The cost of the round trip is one whole-MV COMPLETE refresh, paid on + // the next refresh, which is what a rename is worth here -- the state is the only durable thing that + // can carry the fact that the MV was un-analyzable in between, and a strict INCREMENTAL refresh is + // not the way to find that out. Neither direction raises a partition requirement: no rows moved. + Assertions.assertEquals(before, latestEpochsOf(mtmv)); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } @Test @@ -647,7 +827,7 @@ public void testReplaceTableMarksBaselineRebuild() throws Exception { executeSql("ALTER TABLE ivm_base REPLACE WITH TABLE ivm_new_base PROPERTIES('swap' = 'false')"); - Assertions.assertTrue(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -720,13 +900,13 @@ public void testReplaceTableSwapMarksBothSidesBaselineRebuild() throws Exception + "AS SELECT dt, k1, v1 FROM ivm_new_base"); MTMV oldSideMtmv = getMtmv(db); MTMV newSideMtmv = (MTMV) getDb(db).getTableOrMetaException("ivm_new_mv"); - Assertions.assertFalse(oldSideMtmv.getIvmInfo().isBaselineRebuildRequired()); - Assertions.assertFalse(newSideMtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, oldSideMtmv.getStatus().getState()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, newSideMtmv.getStatus().getState()); executeSql("ALTER TABLE ivm_base REPLACE WITH TABLE ivm_new_base PROPERTIES('swap' = 'true')"); - Assertions.assertTrue(oldSideMtmv.getIvmInfo().isBaselineRebuildRequired()); - Assertions.assertTrue(newSideMtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, oldSideMtmv.getStatus().getState()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, newSideMtmv.getStatus().getState()); } @Test @@ -741,7 +921,7 @@ public void testReplayDropPartitionDoesNotCreateBarrier() throws Exception { "p202001", false, false, 0L, table.getVisibleVersion(), table.getVisibleVersionTime()); Env.getCurrentInternalCatalog().replayDropPartition(info); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -761,7 +941,7 @@ public void testReplayTruncateDoesNotCreateBarrier() throws Exception { Collections.emptyMap(), table.getNextVersion(), System.currentTimeMillis()); Env.getCurrentInternalCatalog().replayTruncateTable(info); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -779,7 +959,7 @@ public void testReplayReplacePartitionDoesNotCreateBarrier() throws Exception { table.getVisibleVersion(), table.getVisibleVersionTime(), false); Env.getCurrentEnv().replayReplaceTempPartition(log); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -790,13 +970,13 @@ public void testReplayRecoverPartitionDoesNotCreateBarrier() throws Exception { OlapTable table = getBaseTable(db); long partitionId = table.getPartition("p202001").getId(); executeSql("ALTER TABLE ivm_base DROP PARTITION p202001"); - clearBaselineRebuild(getMtmv(db)); + resetMvState(getMtmv(db)); RecoverInfo info = new RecoverInfo(database.getId(), table.getId(), partitionId, "", table.getName(), "", "p202001", null); Env.getCurrentInternalCatalog().replayRecoverPartition(info); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } @Test @@ -804,10 +984,9 @@ public void testStaleTaskResultDoesNotMutateMtmv() throws Exception { String db = "ivm_stale_task_result"; createPartitionedIvmTableAndMv(db); MTMV mtmv = getMtmv(db); + mtmv.invalidateWholeMv("seed").await(); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); long taskVersion = mtmv.getSchemaChangeVersion(); - IvmInfo pending = mtmv.getIvmInfo(); - pending.requireCompleteBaselineRebuild(); - mtmv.alterIvmInfo(pending); Deencapsulation.setField(mtmv, "schemaChangeVersion", taskVersion + 1); int historySize = mtmv.getHistoryTasks().size(); @@ -817,13 +996,13 @@ public void testStaleTaskResultDoesNotMutateMtmv() throws Exception { Assertions.assertFalse(mtmv.addTaskResult(result, false)); Assertions.assertEquals(historySize, mtmv.getHistoryTasks().size()); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); Assertions.assertEquals(planSignature, mtmv.getIvmInfo().getPlanSignature()); Assertions.assertEquals(taskVersion + 1, mtmv.getSchemaChangeVersion()); } @Test - public void testIvmRefreshStartRejectsStaleVersionOrPendingBaseline() throws Exception { + public void testIvmRefreshStartRejectsAStaleSchemaChangeVersion() throws Exception { String db = "ivm_refresh_start_validation"; createPartitionedIvmTableAndMv(db); MTMV mtmv = getMtmv(db); @@ -831,11 +1010,6 @@ public void testIvmRefreshStartRejectsStaleVersionOrPendingBaseline() throws Exc mtmv.validateIvmRefreshStart(version); Assertions.assertThrows(JobException.class, () -> mtmv.validateIvmRefreshStart(version + 1)); - - IvmInfo pending = mtmv.getIvmInfo(); - pending.requireCompleteBaselineRebuild(); - mtmv.alterIvmInfo(pending); - Assertions.assertThrows(JobException.class, () -> mtmv.validateIvmRefreshStart(version)); } @Test @@ -846,27 +1020,19 @@ public void testReplayTaskResultAppliesIvmStateWithoutChangingVersion() throws E long schemaChangeVersion = mtmv.getSchemaChangeVersion(); AlterMTMV result = taskResult(mtmv, TaskStatus.FAILED, schemaChangeVersion); IvmInfo replayedInfo = mtmv.getIvmInfo(); - replayedInfo.requireCompleteBaselineRebuild(); replayedInfo.setPlanSignature("replayed_signature"); result.setIvmInfo(replayedInfo); Assertions.assertTrue(mtmv.addTaskResult(result, true)); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); Assertions.assertEquals("replayed_signature", mtmv.getIvmInfo().getPlanSignature()); Assertions.assertEquals(schemaChangeVersion, mtmv.getSchemaChangeVersion()); - - replayedInfo.clearBaselineRebuild(); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); } @Test - public void testSuccessfulBaselineResultClearsPendingState() throws Exception { + public void testSuccessfulResultKeepsThePlanSignature() throws Exception { String db = "ivm_successful_baseline_result"; createPartitionedIvmTableAndMv(db); MTMV mtmv = getMtmv(db); - IvmInfo pending = mtmv.getIvmInfo(); - pending.requireCompleteBaselineRebuild(); - mtmv.alterIvmInfo(pending); String planSignature = mtmv.getIvmInfo().getPlanSignature(); AlterMTMV result = taskResult(mtmv, TaskStatus.SUCCESS, mtmv.getSchemaChangeVersion()); boolean compatibilityMode = Config.enable_check_compatibility_mode; @@ -877,8 +1043,7 @@ public void testSuccessfulBaselineResultClearsPendingState() throws Exception { Config.enable_check_compatibility_mode = compatibilityMode; } - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); - Assertions.assertFalse(result.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); Assertions.assertEquals(planSignature, mtmv.getIvmInfo().getPlanSignature()); } @@ -887,9 +1052,6 @@ public void testSuccessfulSignatureFallbackPublishesNewPlanSignature() throws Ex String db = "ivm_successful_signature_fallback"; createPartitionedIvmTableAndMv(db); MTMV mtmv = getMtmv(db); - IvmInfo pending = mtmv.getIvmInfo(); - pending.requireCompleteBaselineRebuild(); - mtmv.alterIvmInfo(pending); AlterMTMV result = taskResult(mtmv, TaskStatus.SUCCESS, mtmv.getSchemaChangeVersion()); Deencapsulation.setField(result.getTask(), "refreshedIvmPlanSignature", "new_signature"); boolean compatibilityMode = Config.enable_check_compatibility_mode; @@ -902,27 +1064,9 @@ public void testSuccessfulSignatureFallbackPublishesNewPlanSignature() throws Ex Assertions.assertEquals("new_signature", mtmv.getIvmInfo().getPlanSignature()); Assertions.assertEquals("new_signature", result.getIvmInfo().getPlanSignature()); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState()); } - @Test - public void testFailedBaselineResultKeepsGuard() throws Exception { - String db = "ivm_failed_baseline_result"; - createPartitionedIvmTableAndMv(db); - MTMV mtmv = getMtmv(db); - IvmInfo pending = mtmv.getIvmInfo(); - pending.requireCompleteBaselineRebuild(); - mtmv.alterIvmInfo(pending); - AlterMTMV result = taskResult(mtmv, TaskStatus.FAILED, mtmv.getSchemaChangeVersion()); - Deencapsulation.setField(result.getTask(), "refreshedIvmPlanSignature", "new_signature"); - String planSignature = mtmv.getIvmInfo().getPlanSignature(); - - Assertions.assertTrue(mtmv.addTaskResult(result, false)); - - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired()); - Assertions.assertTrue(result.getIvmInfo().isBaselineRebuildRequired()); - Assertions.assertEquals(planSignature, mtmv.getIvmInfo().getPlanSignature()); - } private void createPartitionedIvmTableAndMv(String db) throws Exception { createPartitionedIvmTable(db); @@ -952,7 +1096,7 @@ private void createPartitionedIvmTableAndPartitionedMv(String db) throws Excepti private void assertFreshMv(String db) throws Exception { Assertions.assertTrue(getMtmv(db).isIvm()); - Assertions.assertFalse(getMtmv(db).getIvmInfo().isBaselineRebuildRequired()); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, getMtmv(db).getStatus().getState()); } private void createPartitionedIvmTable(String db) throws Exception { @@ -971,12 +1115,47 @@ private void createPartitionedIvmTable(String db) throws Exception { + "PROPERTIES ('replication_num' = '1', 'binlog.enable' = 'true', 'binlog.format' = 'ROW')"); } + /** + * Gives every MV partition the entry a refresh would have created. A refresh task aligns before it + * reads a base table, so an MV that has been refreshed has one entry per partition; the marker tests + * drive the marker on its own, so they set that state up directly. + */ + private void alignStatesOf(MTMV mtmv) { + Map aligned = Maps.newHashMap(); + for (String partitionName : mtmv.getPartitionNames()) { + aligned.put(partitionName, MTMVPartitionState.initial()); + } + mtmv.alterPartitionStates(aligned); + } + + /** + * What each MV partition currently requires, keyed by partition name. The requirement is what an + * invalidation raises, so comparing two of these is how "nothing was marked" is observed. + */ + private Map latestEpochsOf(MTMV mtmv) { + Map res = Maps.newHashMap(); + for (Entry entry : mtmv.getPartitionStates().entrySet()) { + res.put(entry.getKey(), entry.getValue().getLatestEpoch()); + } + return res; + } + private MTMV getMtmv(String db) throws Exception { return (MTMV) Env.getCurrentInternalCatalog() .getDb(db).get() .getTableOrMetaException("ivm_mv"); } + /** + * The detail a whole-MV invalidation records when the query can no longer be analyzed. It is the only + * record of that change, so the reason has to survive in it: a state alone cannot say why. + */ + private void assertUnanalyzableDetail(MTMV mtmv) { + Assertions.assertTrue( + mtmv.getStatus().getSchemaChangeDetail().contains("no longer analyzable"), + "the detail must name the reason, was: " + mtmv.getStatus().getSchemaChangeDetail()); + } + private Database getDb(String db) { return Env.getCurrentInternalCatalog().getDb(db).get(); } @@ -1001,10 +1180,9 @@ private Set mvPartitionsWithSameRange(MTMV mtmv, OlapTable baseTable, St return res; } - private void clearBaselineRebuild(MTMV mtmv) { - IvmInfo info = new IvmInfo(mtmv.getIvmInfo()); - info.clearBaselineRebuild(); - mtmv.alterIvmInfo(info); + /** Puts the MV back to a state where only a new invalidation can move it. */ + private void resetMvState(MTMV mtmv) { + mtmv.alterStatus(new MTMVStatus(MTMVState.NORMAL, "reset")); } private AlterMTMV taskResult(MTMV mtmv, TaskStatus status, long schemaChangeVersion) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmInfoTest.java index fdd1c899bce0be..926fd1976c1dc4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmInfoTest.java @@ -22,7 +22,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Collections; class IvmInfoTest { @Test @@ -66,40 +65,16 @@ void testUseFullKeysPersistsThroughGson() { void testCopyConstructor() { IvmInfo info = new IvmInfo(); info.setEnableIvm(true); - info.requireCompleteBaselineRebuild(); info.setUseFullKeys(true); info.setPlanSignature("abc123"); info.advanceSequencePrefix(); IvmInfo copy = new IvmInfo(info); - info.clearBaselineRebuild(); Assertions.assertTrue(copy.isEnableIvm()); - Assertions.assertTrue(copy.isBaselineRebuildRequired()); Assertions.assertTrue(copy.isUseFullKeys()); Assertions.assertEquals("abc123", copy.getPlanSignature()); Assertions.assertEquals(1, copy.getSequencePrefix()); } - @Test - void testBaselineRebuildStatePersistsThroughGson() { - IvmInfo info = new IvmInfo(); - Assertions.assertFalse(roundTrip(info).isBaselineRebuildRequired()); - - info.addPendingBaselineRebuildPartitions(Collections.singleton("p1")); - IvmInfo partitionsRebuild = roundTrip(info); - Assertions.assertTrue(partitionsRebuild.isBaselineRebuildRequired()); - Assertions.assertFalse(partitionsRebuild.requiresCompleteBaselineRebuild()); - Assertions.assertEquals(Collections.singleton("p1"), - partitionsRebuild.getPendingBaselineRebuildPartitions()); - - partitionsRebuild.requireCompleteBaselineRebuild(); - IvmInfo completeRebuild = roundTrip(partitionsRebuild); - Assertions.assertTrue(completeRebuild.requiresCompleteBaselineRebuild()); - Assertions.assertTrue(completeRebuild.getPendingBaselineRebuildPartitions().isEmpty()); - } - - private static IvmInfo roundTrip(IvmInfo info) { - return GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(info), IvmInfo.class); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java index 33c2967486336a..263d945d4ad804 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java @@ -32,6 +32,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.mtmv.MTMVAlterOpType; +import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.mtmv.ivm.IvmRewriteContext; import org.apache.doris.mtmv.ivm.IvmUtil; @@ -2286,8 +2287,8 @@ public void testAlterExcludedTriggerTablesReconcilesStreams() throws Exception { alterMtmv("ALTER MATERIALIZED VIEW ivm_alter_excl_stream_mv " + "SET ('excluded_trigger_tables' = 'ivm_alter_excl_stream_base2')"); - Assertions.assertFalse(mtmv.getIvmInfo().isBaselineRebuildRequired(), - "Excluding a base table should not require rebuilding the IVM baseline"); + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState(), + "Excluding a base table does not invalidate the MV"); Assertions.assertNotNull(db.getTableNullable(stream1), "Stream should remain for non-excluded table"); Assertions.assertNull(db.getTableNullable(stream2), @@ -2313,8 +2314,8 @@ public void testAlterExcludedTriggerTablesReconcilesStreams() throws Exception { "Stream should be dropped for newly excluded table"); Assertions.assertNotNull(db.getTableNullable(stream2), "Stream should be created for a table removed from excluded_trigger_tables"); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired(), - "Including a base table should require rebuilding the IVM baseline"); + Assertions.assertEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState(), + "Including a base table invalidates the whole MV"); } @Test @@ -2343,8 +2344,10 @@ public void testAlterExcludedTriggerTablesReplayDoesNotCreateStream() throws Exc Env.getCurrentEnv().getAlterInstance().processAlterMTMV(replayAlter, true); Assertions.assertTrue(mtmv.getExcludedTriggerTables().isEmpty()); - Assertions.assertTrue(mtmv.getIvmInfo().isBaselineRebuildRequired(), - "ALTER replay should restore the IVM baseline invalidation state"); + // The property record does not carry the invalidation: the live change journals an ALTER_STATUS + // record ahead of it, so a replay of this one alone leaves the state where it was. + Assertions.assertNotEquals(MTMVState.SCHEMA_CHANGE, mtmv.getStatus().getState(), + "replaying the property record does not invalidate the MV by itself"); Assertions.assertNull(db.getTableNullable(streamName), "ALTER replay should rely on OP_CREATE_TABLE replay instead of creating a new stream"); } diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_baseline_marker_scope.out b/regression-test/data/mtmv_p0/ivm/test_ivm_baseline_marker_scope.out index ed1179718df2ef..92ecd6924ee516 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_baseline_marker_scope.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_baseline_marker_scope.out @@ -7,7 +7,7 @@ SUCCESS COMPLETE NONE 2 2026-02-10 200 dim-b -- !non_pct_drop_task -- -SUCCESS COMPLETE BINLOG_BROKEN +SUCCESS COMPLETE NONE -- !non_pct_drop_mv -- 1 2026-01-10 100 \N @@ -18,3 +18,4 @@ SUCCESS -- !narrowed_mv -- 2 2026-02-10 200 dim-b + diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_chained_mtmv_2.out b/regression-test/data/mtmv_p0/ivm/test_ivm_chained_mtmv_2.out index 44a7f4ef0591a3..7da38e6c1ccaf3 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_chained_mtmv_2.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_chained_mtmv_2.out @@ -35,10 +35,10 @@ 5 50 -- !child_incremental_after_root_complete -- -FAILED IVM baseline rebuild is pending for mv=child_ivm_commit_tso; run an AUTO or COMPLETE refresh first +SUCCESS -- !child_auto_refresh_mode -- -COMPLETE +NONE -- !child_after_auto_refresh -- 3 35 diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.out b/regression-test/data/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.out index 8793c2b56bc319..999d68410b907c 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.out @@ -3,6 +3,9 @@ 10 2 300 20 1 300 +-- !mv_rows_after_aba_strict -- +0 3 600 + -- !mv_rows_after_aba -- 0 3 600 diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_excluded_trigger_table.out b/regression-test/data/mtmv_p0/ivm/test_ivm_excluded_trigger_table.out index 061164c55803e0..1c8b53f4e16762 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_excluded_trigger_table.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_excluded_trigger_table.out @@ -23,10 +23,9 @@ -- !alter_after_incremental_fallback -- 1 10 2 20 -3 30 -- !alter_fallback_refresh_mode -- -COMPLETE +NONE -- !reinclude_refresh_mode -- COMPLETE diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_baseline_rebuild_dup_keys.out b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_baseline_rebuild_dup_keys.out index 271a5f56d8e39b..caaea19e161c36 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_baseline_rebuild_dup_keys.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_baseline_rebuild_dup_keys.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !baseline_task -- -SUCCESS NONE NONE +SUCCESS COMPLETE NONE -- !baseline_base -- 2026-01-10 1 10 @@ -15,10 +15,10 @@ SUCCESS NONE NONE 2026-02-10 3 30 -- !strict_task -- -FAILED NOT_REFRESH BINLOG_BROKEN +SUCCESS PARTIAL NONE -- !fallback_task -- -SUCCESS PARTIAL BINLOG_BROKEN +SUCCESS NONE NONE -- !fallback_base -- 2026-02-10 3 30 diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.out b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.out index fab4869556a1d7..6b79a6125b435d 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.out @@ -13,10 +13,20 @@ SUCCESS NONE 2026-03-10 3 30 -- !strict_task -- -FAILED BINLOG_BROKEN +SUCCESS NONE + +-- !strict_base -- +2026-02-10 2 20 +2026-02-15 4 40 +2026-03-10 3 30 + +-- !strict_mv -- +2026-02-10 2 20 +2026-02-15 4 40 +2026-03-10 3 30 -- !fallback_task -- -SUCCESS BINLOG_BROKEN +SUCCESS NONE -- !fallback_base -- 2026-02-10 2 20 diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_epoch_rebuild.out b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_epoch_rebuild.out new file mode 100644 index 00000000000000..4563aba73f3850 --- /dev/null +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_epoch_rebuild.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !baseline_task -- +SUCCESS COMPLETE 0 + +-- !baseline_mv -- +1 2026-01-10 100 +2 2026-02-10 200 + +-- !incremental_task -- +SUCCESS NONE 0 + +-- !incremental_mv -- +1 2026-01-10 100 +2 2026-02-10 200 +3 2026-01-20 300 + +-- !truncate_task -- +SUCCESS PARTIAL 1 + +-- !truncate_mv -- +2 2026-02-10 200 +4 2026-02-20 400 + +-- !rename_task -- +SUCCESS COMPLETE 2 + +-- !rename_mv -- +2 2026-02-10 200 +4 2026-02-20 400 +5 2026-02-21 500 + +-- !second_truncate_task -- +SUCCESS PARTIAL 1 + +-- !second_truncate_mv -- + diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_window_remove.out b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_window_remove.out index 0db56ae6072eba..0e7d7c4cfe4fdb 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_partition_window_remove.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_partition_window_remove.out @@ -10,15 +10,15 @@ 2026-01-03 3 30 -- !pwr_strict_after_remove -- -FAILED true +SUCCESS false --- !pwr_stale_after_strict -- -2026-01-01 1 10 +-- !pwr_caught_up_after_strict -- +2026-01-01 1 11 2026-01-02 2 20 2026-01-03 3 30 -- !pwr_removed_refresh_mode -- -COMPLETE +NONE -- !pwr_window_removed_catchup -- 2026-01-01 1 11 diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_partitions_after_failed_complete.out b/regression-test/data/mtmv_p0/ivm/test_ivm_partitions_after_failed_complete.out new file mode 100644 index 00000000000000..897bb29d570c46 --- /dev/null +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_partitions_after_failed_complete.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !baseline_mv -- +1 2026-01-10 100 +2 2026-02-10 200 + +-- !failed_complete_task -- +FAILED COMPLETE 0 + +-- !partitions_task -- +SUCCESS COMPLETE 0 + +-- !partitions_mv -- +1 2026-01-10 100 +2 2026-02-10 200 +3 2026-01-20 300 + +-- !incremental_task -- +SUCCESS NONE 0 + +-- !partitions_fallback_task -- +SUCCESS COMPLETE 0 + +-- !partitions_fallback_mv -- +1 2026-01-10 100 +2 2026-02-10 200 +3 2026-01-20 300 +4 2026-02-20 400 + diff --git a/regression-test/data/mtmv_p0/ivm/test_ivm_strict_failure_partition_atomicity.out b/regression-test/data/mtmv_p0/ivm/test_ivm_strict_incremental_rebuilds_invalidated_partitions.out similarity index 68% rename from regression-test/data/mtmv_p0/ivm/test_ivm_strict_failure_partition_atomicity.out rename to regression-test/data/mtmv_p0/ivm/test_ivm_strict_incremental_rebuilds_invalidated_partitions.out index 5d670c0c69582c..135ddfdfb4e5ba 100644 --- a/regression-test/data/mtmv_p0/ivm/test_ivm_strict_failure_partition_atomicity.out +++ b/regression-test/data/mtmv_p0/ivm/test_ivm_strict_incremental_rebuilds_invalidated_partitions.out @@ -4,12 +4,10 @@ 2026-02-10 2 20 2026-03-10 3 30 --- !strict_task -- -FAILED BINLOG_BROKEN true +-- !strict_rebuild_task -- +SUCCESS NONE --- !after_strict_failure -- -2026-01-10 1 10 -2026-02-10 2 20 +-- !after_strict_rebuild -- 2026-03-10 3 30 -- !after_auto_recovery -- diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_chained_mtmv_2.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_chained_mtmv_2.groovy index f55896b1474171..3692534d3fa389 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_chained_mtmv_2.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_chained_mtmv_2.groovy @@ -123,6 +123,10 @@ suite("test_ivm_chained_mtmv_2") { sql """REFRESH MATERIALIZED VIEW root_ivm_commit_tso COMPLETE;""" waitingMTMVTaskFinishedByMvName("root_ivm_commit_tso") + // A COMPLETE refresh of the root rebuilds it, which is a base-table change of the child that emits no + // row binlog: the child's own baseline is invalidated by it. A strict INCREMENTAL request then rebuilds + // the child -- the rebuild is the refresh's own work -- instead of being refused until an AUTO refresh + // had run, so the task below succeeds and its error message is empty. sql """REFRESH MATERIALIZED VIEW child_ivm_commit_tso INCREMENTAL;""" waitingMTMVTaskFinishedNotNeedSuccess(getJobName(context.dbName, "child_ivm_commit_tso")) order_qt_child_incremental_after_root_complete """ @@ -134,8 +138,12 @@ suite("test_ivm_chained_mtmv_2") { sql """REFRESH MATERIALIZED VIEW child_ivm_commit_tso AUTO;""" waitingMTMVTaskFinishedByMvName("child_ivm_commit_tso") + // A refresh that only ran the incremental rewrite leaves RefreshMode unset, and an unset column comes + // back as the literal two-character string "\N", which does not survive the .out round trip, so fold + // every value that is not a scope into a printable token. order_qt_child_auto_refresh_mode """ - SELECT RefreshMode + SELECT CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END FROM tasks('type'='mv') WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'child_ivm_commit_tso' ORDER BY CreateTime DESC LIMIT 1; diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.groovy index 26dd37ff2d00a0..1bb6cb99c269fc 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_drop_referenced_column_baseline_rebuild.groovy @@ -25,9 +25,14 @@ import static java.util.concurrent.TimeUnit.SECONDS // The base table change is metadata-only (light schema change) and emits no binlog, so the // delta is empty and the refresh has nothing to apply -- the MV baseline is simply stale. // -// Expected: dropping a referenced column invalidates the IVM baseline, so a strict -// INCREMENTAL refresh is rejected and the user is told to run a COMPLETE refresh. -// Dropping an unreferenced column must still leave the incremental path untouched. +// Expected: dropping a referenced column invalidates the IVM baseline. The invalidation puts the MV +// in SCHEMA_CHANGE, so a strict INCREMENTAL refresh runs as a whole-MV COMPLETE instead of being +// refused -- while the column is gone the query cannot be analysed at all, so that refresh fails on +// the analysis error, and once a same-name column is added back the query analyses again and the +// rebuild succeeds. Rebuilding is what keeps the ABA case safe: the rows are recomputed under the +// current column semantics rather than an empty delta being applied to rows computed under the old +// ones, which is the silent staleness this case exists to catch. +// Dropping an unreferenced column must not invalidate the IVM baseline. suite("test_ivm_drop_referenced_column_baseline_rebuild") { def tableName = "ivm_drop_ref_col_t" def mvName = "ivm_drop_ref_col_mv" @@ -83,7 +88,7 @@ suite("test_ivm_drop_referenced_column_baseline_rebuild") { def taskResult Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ taskResult = sql_return_maparray(""" - SELECT TaskId, Status, RefreshMode, IvmFallbackReason, ErrorMsg + SELECT TaskId, Status, RefreshMode, IvmFallbackReason, ErrorMsg, IvmRebuiltPartitions FROM tasks('type'='mv') WHERE MvDatabaseName = '${context.dbName}' AND MvName = '${mv}' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 @@ -104,6 +109,11 @@ suite("test_ivm_drop_referenced_column_baseline_rebuild") { order_qt_mv_rows_baseline "SELECT grp, cnt, total FROM ${mvName}" // ------------------------------------- 2. unreferenced column: no baseline invalidation + // The IVM baseline itself is untouched -- no partition requirement is raised. The shared base-table + // change hook still moves the MV into SCHEMA_CHANGE for a column change, though, and that state is + // what the refresh below reads: it is escalated to a whole-MV COMPLETE. Narrowing the hook so a + // change that re-analyses cleanly leaves an IVM MV alone is PR 4's S1-5; pinned here so the + // escalation cannot pass unnoticed until then. def before = ddlJobCount(tableName) sql """ALTER TABLE ${tableName} DROP COLUMN spare""" waitDdlFinished(tableName, before) @@ -112,8 +122,12 @@ suite("test_ivm_drop_referenced_column_baseline_rebuild") { task = waitTerminalTask(mvName) assertEquals("SUCCESS", task.Status.toString(), "dropping an unreferenced column must not invalidate the IVM baseline: " + task.ErrorMsg) + assertEquals("COMPLETE", task.RefreshMode.toString(), + "the shared hook still moves the MV into SCHEMA_CHANGE, so this refresh is escalated") + assertEquals("1", task.IvmRebuiltPartitions.toString(), + "and the escalated refresh reports the partition it rebuilt instead of the request it got") - // ---------------------------------------- 3. referenced column: strict INCREMENTAL rejected + // --------------------------- 3. referenced column: the refresh can no longer analyse before = ddlJobCount(tableName) sql """ALTER TABLE ${tableName} DROP COLUMN grp""" waitDdlFinished(tableName, before) @@ -123,17 +137,25 @@ suite("test_ivm_drop_referenced_column_baseline_rebuild") { assertEquals("FAILED", task.Status.toString(), "dropping a referenced column must reject a strict INCREMENTAL refresh") - // -------------------------------- 4. same-name re-add (schema ABA) is still rejected + // --------------------------------- 4. same-name re-add (schema ABA): rebuilt, not accepted + // The re-added column makes the MV query analysable again while the MV is still in + // SCHEMA_CHANGE, so the strict INCREMENTAL runs as a whole-MV COMPLETE refresh. That is the + // difference the ABA case turns on: every row is recomputed under the current column semantics, + // instead of an empty delta being applied to rows computed under the old ones. Succeeding from + // the incremental path here would be exactly the silent staleness this case exists to catch, + // which is why the refresh mode is asserted and not just the status. before = ddlJobCount(tableName) sql """ALTER TABLE ${tableName} ADD COLUMN grp INT NULL DEFAULT '0'""" waitDdlFinished(tableName, before) sql """REFRESH MATERIALIZED VIEW ${mvName} INCREMENTAL""" task = waitTerminalTask(mvName) - assertEquals("FAILED", task.Status.toString(), - "schema ABA must not be silently accepted by a strict INCREMENTAL refresh") - assertTrue(task.ErrorMsg.toString().contains("baseline rebuild is pending"), - "expected a pending baseline rebuild hint, got: " + task.ErrorMsg) + assertEquals("SUCCESS", task.Status.toString(), + "with the column re-added the query analyses, so the escalated refresh succeeds: " + + task.ErrorMsg) + assertEquals("COMPLETE", task.RefreshMode.toString(), + "the ABA refresh must rebuild the whole MV rather than apply an empty delta") + order_qt_mv_rows_after_aba_strict "SELECT grp, cnt, total FROM ${mvName}" // ------------------------------------ 5. COMPLETE rebuild reflects current base semantics // Every pre-existing row now reads the re-added column's default value. diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_excluded_trigger_table.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_excluded_trigger_table.groovy index 3081682e3bb630..0fdd7c975e441f 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_excluded_trigger_table.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_excluded_trigger_table.groovy @@ -141,12 +141,23 @@ suite("test_ivm_excluded_trigger_table", "mtmv") { """ sql """REFRESH MATERIALIZED VIEW test_ivm_excluded_trigger_table_alt_mv INCREMENTAL FALLBACK""" waitingMTMVTaskFinishedByMvName("test_ivm_excluded_trigger_table_alt_mv") + // (3, 30) is absent on purpose. b is excluded by the ALTER above, and the property is read by the + // refresh that follows it, not by the one that preceded the INSERT: an excluded table is out of what + // the MV maintains, so this refresh leaves its rows where the last complete refresh left them. The + // request was an INCREMENTAL, and it runs as one rather than being escalated by the property change. + // The remedy for an excluded table is a manual COMPLETE, which is what makes the property's + // documentation load-bearing: an ALTER no longer sweeps the table's pending changes in for free. order_qt_alter_after_incremental_fallback """ SELECT k1, v1 FROM test_ivm_excluded_trigger_table_alt_mv """ + // This refresh runs as a plain incremental rewrite, which leaves RefreshMode unset. An unset column + // comes back as the literal two-character string "\N", which does not survive the .out round trip, so + // fold every value that is not a scope into a printable token. qt_alter_fallback_refresh_mode """ - SELECT RefreshMode FROM tasks('type'='mv') + SELECT CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END + FROM tasks('type'='mv') WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'test_ivm_excluded_trigger_table_alt_mv' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.groovy index bc9f182e306685..bea9eca24efbbe 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_drop_live_delta.groovy @@ -22,13 +22,19 @@ import static java.util.concurrent.TimeUnit.SECONDS /** * Dropping a base-table partition invalidates the IVM baseline, because the rows disappear through * metadata rather than through row binlog entries. The MV partition built from that base partition - * is then removed by partition sync, which is exactly what the baseline barrier recorded. + * is then removed by partition sync, and the requirement left on it is what records that its rows + * could not be removed incrementally. * *

The refresh must still consume the delta that accumulated on the *surviving* partitions: it * may not report SUCCESS while leaving those partitions stale. This case inserts a row into a * surviving partition after the drop, so an EMPTY baseline-rebuild intersection cannot be mistaken * for "nothing to do". * + *

A strict INCREMENTAL request is not refused when it meets that requirement: the refresh runs as + * a partition rebuild for the invalidated partition and applies the surviving partitions' delta in + * the same run, so the MV matches the base table as soon as the strict refresh returns -- which the + * FALLBACK refresh after it then confirms is a state it did not have to repair. + * *

Partitions are managed by hand (no dynamic partition scheduler) and every dt is a literal, so * the case is fully deterministic. */ @@ -110,14 +116,20 @@ suite("test_ivm_partition_drop_live_delta") { sql """ALTER TABLE ${tableName} DROP PARTITION p202601""" sql """INSERT INTO ${tableName} VALUES ('2026-02-15', 4, 40)""" - // A strict incremental refresh must refuse to run against a broken baseline. + // A strict incremental refresh meets the requirement the drop left behind and rebuilds the + // invalidated partition, instead of refusing to run until a COMPLETE refresh has been issued. sql """REFRESH MATERIALIZED VIEW ${mvName} INCREMENTAL""" taskId = waitForNewTask(taskId) qt_strict_task taskQuery(taskId) - // The fallback reports SUCCESS, so the MV has to match the base table afterwards: the expired - // partition is gone AND the row written to the surviving partition has been consumed. An MV - // that is missing that row means the refresh silently skipped the surviving partitions' delta. + // The strict refresh reported SUCCESS, so the MV has to match the base table already: the + // expired partition is gone AND the row written to the surviving partition has been consumed in + // the same run. An MV that is missing that row means the rebuild replaced the invalidated + // partition but silently skipped the surviving partitions' delta. + order_qt_strict_base """SELECT dt, id, v FROM ${tableName} ORDER BY dt, id""" + order_qt_strict_mv """SELECT dt, id, v FROM ${mvName} ORDER BY dt, id""" + + // The fallback refresh finds a baseline that is already repaired, and has to leave it that way. sql """REFRESH MATERIALIZED VIEW ${mvName} INCREMENTAL FALLBACK""" taskId = waitForNewTask(taskId) qt_fallback_task taskQuery(taskId) diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_epoch_rebuild.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_epoch_rebuild.groovy new file mode 100644 index 00000000000000..1d39e16313dfba --- /dev/null +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_epoch_rebuild.groovy @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +/** + * What a refresh does with the partitions a metadata-only base-table change left behind. + * + *

A partition drop / truncate / replace / recover emits no row binlog, so an MV partition that read those + * rows keeps them forever unless it is rebuilt. Such a partition carries a raised requirement, which sends + * it to a rebuild, while the partitions nothing changed for keep catching up incrementally -- and the + * request does not have to ask for it: a strict INCREMENTAL refresh rebuilds what the requirement names and + * reports how many partitions that was in IvmRebuiltPartitions, rather than reporting the stale rows as + * current. + * + *

Cases pinned here: + *

    + *
  1. the baseline: both MV partitions filled by a COMPLETE refresh, then one INCREMENTAL refresh that + * rebuilds nothing (IvmRebuiltPartitions 0) and still applies its delta;
    2. + *
  2. a truncated base partition: its rows go away, the other partition's delta is applied, and the + * refresh reports the partition it had to rebuild;
    3. + *
  3. a rename of the base table: it changes no column, so it raises no partition requirement -- an epoch + * is not where a rename belongs -- but it does put the MV into SCHEMA_CHANGE, and that state is what + * makes the strict INCREMENTAL refresh after it run as a whole-MV COMPLETE;
    4. + *
  4. a second truncated partition: the requirement keeps naming the partitions it belongs to.
    5. + *
+ * + *

All dates are literals: no current_date(), so the expectation does not depend on the run date. + */ +suite("test_ivm_partition_epoch_rebuild") { + def baseTable = "ivm_epoch_f" + def mvName = "ivm_epoch_mv" + + def waitForNewTask = { previousTaskId -> + def taskResult + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + taskResult = sql_return_maparray(""" + SELECT TaskId, Status + FROM tasks('type'='mv') + WHERE MvDatabaseName = '${context.dbName}' + AND MvName = '${mvName}' + ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 + """) + return !taskResult.isEmpty() + && taskResult[0].TaskId.toString() != previousTaskId + && taskResult[0].Status.toString() != 'PENDING' + && taskResult[0].Status.toString() != 'RUNNING' + }) + return taskResult[0].TaskId.toString() + } + + // The route a refresh took, in one row: whether it succeeded, which scope refreshed, and how many + // partitions it rebuilt although the request did not ask for them. + // + // A refresh that only ran the incremental rewrite leaves RefreshMode unset, and an unset column comes + // back as the literal two-character string "\N", which does not survive the .out round trip, so fold + // every value that is not a scope into a printable token. + def taskQuery = { String taskId -> + """ + SELECT Status, + CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END, + IvmRebuiltPartitions + FROM tasks('type'='mv') + WHERE TaskId = '${taskId}' + """ + } + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mvName}""" + sql """DROP TABLE IF EXISTS ${baseTable}""" + + sql """ + CREATE TABLE ${baseTable} ( + order_id BIGINT NOT NULL, + dt DATE NOT NULL, + amount INT + ) + UNIQUE KEY(order_id, dt) + PARTITION BY RANGE(dt) () + DISTRIBUTED BY HASH(order_id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + sql """ALTER TABLE ${baseTable} ADD PARTITION p202601 VALUES [('2026-01-01'), ('2026-02-01'))""" + sql """ALTER TABLE ${baseTable} ADD PARTITION p202602 VALUES [('2026-02-01'), ('2026-03-01'))""" + + sql """INSERT INTO ${baseTable} VALUES + (1, '2026-01-10', 100), + (2, '2026-02-10', 200)""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL + KEY(order_id, dt) + PARTITION BY(dt) + DISTRIBUTED BY HASH(order_id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + AS SELECT order_id, dt, amount FROM ${baseTable} + """ + + sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE""" + def taskId = waitForNewTask(null) + qt_baseline_task taskQuery(taskId) + order_qt_baseline_mv """SELECT order_id, dt, amount FROM ${mvName}""" + + // The ordinary incremental refresh: nothing was invalidated, so no partition is rebuilt -- and the + // delta still lands, which is what makes the count meaningful rather than a constant. + sql """INSERT INTO ${baseTable} VALUES (3, '2026-01-20', 300)""" + sql """REFRESH MATERIALIZED VIEW ${mvName} INCREMENTAL""" + taskId = waitForNewTask(taskId) + qt_incremental_task taskQuery(taskId) + order_qt_incremental_mv """SELECT order_id, dt, amount FROM ${mvName}""" + + // A truncated base partition emits no binlog, so the MV partition that read it keeps rows that no + // longer exist anywhere. It is rebuilt; the row inserted into the other partition in the same window + // arrives incrementally, and the refresh says it rebuilt one partition the request did not ask for. + sql """TRUNCATE TABLE ${baseTable} PARTITION(p202601)""" + sql """INSERT INTO ${baseTable} VALUES (4, '2026-02-20', 400)""" + sql """REFRESH MATERIALIZED VIEW ${mvName} INCREMENTAL""" + taskId = waitForNewTask(taskId) + qt_truncate_task taskQuery(taskId) + order_qt_truncate_mv """SELECT order_id, dt, amount FROM ${mvName}""" + + // A rename leaves every column alone and names nothing new to read, so it raises no partition + // requirement: an epoch is not where this change belongs. What it does move is the MV state. The + // shared base-table hook puts every MV that reads the table into SCHEMA_CHANGE -- the MV query still + // spells the old name, so it no longer analyzes -- and renaming the table back does not clear it: the + // dependencies are registered under the name the query spells, so the rename back finds nothing to + // update. The state is therefore what widens the strict INCREMENTAL below into a whole-MV COMPLETE. + // The rows it leaves are the same either way; the columns this query reports are not. The count is the + // size of the MV because the request was an INCREMENTAL that ran as a COMPLETE -- a request that was + // itself a COMPLETE reports 0, since rebuilding everything is what it asked for. + sql """ALTER TABLE ${baseTable} RENAME ivm_epoch_renamed""" + sql """ALTER TABLE ivm_epoch_renamed RENAME ${baseTable}""" + sql """INSERT INTO ${baseTable} VALUES (5, '2026-02-21', 500)""" + sql """REFRESH MATERIALIZED VIEW ${mvName} INCREMENTAL""" + taskId = waitForNewTask(taskId) + qt_rename_task taskQuery(taskId) + order_qt_rename_mv """SELECT order_id, dt, amount FROM ${mvName}""" + + // The requirement keeps naming its own partition: truncating the other one rebuilds that one. + sql """TRUNCATE TABLE ${baseTable} PARTITION(p202602)""" + sql """REFRESH MATERIALIZED VIEW ${mvName} AUTO""" + taskId = waitForNewTask(taskId) + qt_second_truncate_task taskQuery(taskId) + order_qt_second_truncate_mv """SELECT order_id, dt, amount FROM ${mvName}""" +} diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_window_remove.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_window_remove.groovy index 8fbcd05fed9fac..a8747670baf53d 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_window_remove.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_partition_window_remove.groovy @@ -73,8 +73,10 @@ suite("test_ivm_partition_window_remove") { waitingMTMVTaskFinishedByMvName("test_ivm_pwr_mv") order_qt_pwr_window_ignores_p1 """SELECT dt, k1, v1 FROM test_ivm_pwr_mv ORDER BY dt""" - // Remove the window. A strict manual INCREMENTAL cannot replay the lossy backlog - // safely: it must fail explicitly instead of returning SUCCESS with stale data. + // Remove the window. The ALTER changes the refresh baseline the MV's properties describe, so + // the MV goes into SCHEMA_CHANGE and the strict manual INCREMENTAL below runs as a whole-MV + // COMPLETE refresh. It must not report SUCCESS while leaving p1 stale: the widened range means + // p1's backlog -- skipped by the windowed refreshes -- has to be replayed by this refresh. sql """ALTER MATERIALIZED VIEW test_ivm_pwr_mv SET ("ivm_partition_window_limit" = "");""" def previousTaskId = sql(""" SELECT TaskId FROM tasks('type'='mv') @@ -100,13 +102,19 @@ suite("test_ivm_partition_window_remove") { WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'test_ivm_pwr_mv' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 """ - order_qt_pwr_stale_after_strict """SELECT dt, k1, v1 FROM test_ivm_pwr_mv ORDER BY dt""" + // The upgraded refresh already replayed the p1 backlog, so the MV is caught up rather than + // stale: this is the assertion DORIS-28376 turns on. + order_qt_pwr_caught_up_after_strict """SELECT dt, k1, v1 FROM test_ivm_pwr_mv ORDER BY dt""" - // The next AUTO refresh rebuilds a complete baseline and replays the p1 backlog. + // The next AUTO refresh has nothing left to repair; the escalation above did the rebuild. sql """REFRESH MATERIALIZED VIEW test_ivm_pwr_mv AUTO""" waitingMTMVTaskFinishedByMvName("test_ivm_pwr_mv") + // The AUTO refresh has nothing left to do, so it leaves RefreshMode unset, which comes back as + // the literal two-character string "\N" that does not survive the .out round trip; fold it. order_qt_pwr_removed_refresh_mode """ - SELECT RefreshMode FROM tasks('type'='mv') + SELECT CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END + FROM tasks('type'='mv') WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'test_ivm_pwr_mv' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 """ diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_partitions_after_failed_complete.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_partitions_after_failed_complete.groovy new file mode 100644 index 00000000000000..20880782c11a2b --- /dev/null +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_partitions_after_failed_complete.groovy @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +/** + * What a partition refresh does after a whole-MV refresh failed partway through. + * + *

A whole-MV refresh raises every partition's requirement before it reconciles the streams and before it + * reads anything, so a failure in between leaves the one state where the two criteria a refresh can plan by + * disagree: every partition needs a rebuild, and every snapshot still says the MV is in sync with its base + * tables. Planning by snapshots alone would report NOT_REFRESH and repair nothing, and only the requests + * that reach an incremental attempt would ever recover the MV. + * + *

The failure is injected (a debug point on the IVM insert), so it lands on a real refresh rather than on + * a crash window, and it is removed before the refreshes that have to recover from it. + * + *

Cases pinned here, over an MV with two partitions: + *

    + *
  1. a strict `PARTITIONS` refresh after the failed `COMPLETE`: it refreshes what the requirement names + * -- both partitions, not only the one the snapshots call unsynced -- and the increment the failed + * refresh left behind is applied;
    2. + *
  2. the incremental refresh after it rebuilds nothing, which is the requirement having been cleared + * rather than merely deferred to whichever request comes next;
    3. + *
  3. the same recovery for `PARTITIONS FALLBACK`, which does not have to reach the whole-MV attempt + * behind it because a partition refresh is what the requirement asks for.
    4. + *
+ * + *

All dates are literals: no current_date(), so the expectation does not depend on the run date. + */ +suite("test_ivm_partitions_after_failed_complete", "nonConcurrent") { + // Cloud mode: the injection targets the local insert path, which the cloud transaction manager never + // takes, so the refresh would succeed and there would be nothing to recover from. + if (isCloudMode()) { + logger.info("skip test_ivm_partitions_after_failed_complete on cloud mode: " + + "the insert failure injection only fires on the local txn path") + return + } + def mvName = "ivm_failed_complete_mv" + def rpcFailureDebugPoint = "AbstractInsertExecutor.executeSingleInsert.ivm_rpc_failure" + def rpcFailureFilterDebugPoint = "AbstractInsertExecutor.executeSingleInsert.ivm_rpc_failure.filter" + + def waitForNewTask = { previousTaskId -> + def taskResult + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + taskResult = sql_return_maparray(""" + SELECT TaskId, Status + FROM tasks('type'='mv') + WHERE MvDatabaseName = '${context.dbName}' + AND MvName = '${mvName}' + ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 + """) + return !taskResult.isEmpty() + && taskResult[0].TaskId.toString() != previousTaskId + && taskResult[0].Status.toString() != 'PENDING' + && taskResult[0].Status.toString() != 'RUNNING' + }) + return taskResult[0].TaskId.toString() + } + + // The route a refresh took, in one row: whether it succeeded, which scope refreshed, and how many + // partitions it rebuilt although the request did not ask for them. A refresh that only ran the + // incremental rewrite leaves RefreshMode unset, and an unset column comes back as the literal + // two-character string "\N", which does not survive the .out round trip, so fold every value that is + // not a scope into a printable token. + def taskQuery = { String taskId -> + """ + SELECT Status, + CASE WHEN RefreshMode IN ('COMPLETE', 'PARTIAL', 'NOT_REFRESH') + THEN RefreshMode ELSE 'NONE' END, + IvmRebuiltPartitions + FROM tasks('type'='mv') + WHERE TaskId = '${taskId}' + """ + } + + // A whole-MV refresh that fails on its first batch: the requirement is raised and journaled before the + // reconciliation below it, and the snapshots it would have replaced are never written, which is the + // state the partition refreshes have to plan by the requirement to get out of. What the failure wrote + // is pinned by the task queries below, which report the status of the task they ran as. + def failCompleteRefresh = { previousTaskId -> + try { + GetDebugPoint().enableDebugPointForAllFEs(rpcFailureFilterDebugPoint, [mv_name: mvName]) + GetDebugPoint().enableDebugPointForAllFEs(rpcFailureDebugPoint) + sql """REFRESH MATERIALIZED VIEW ivm_failed_complete_mv COMPLETE""" + return waitForNewTask(previousTaskId) + } finally { + GetDebugPoint().disableDebugPointForAllFEs(rpcFailureFilterDebugPoint) + GetDebugPoint().disableDebugPointForAllFEs(rpcFailureDebugPoint) + } + } + + sql """DROP MATERIALIZED VIEW IF EXISTS ivm_failed_complete_mv""" + sql """DROP TABLE IF EXISTS ivm_failed_complete_f""" + + sql """ + CREATE TABLE ivm_failed_complete_f ( + order_id BIGINT NOT NULL, + dt DATE NOT NULL, + amount INT + ) + UNIQUE KEY(order_id, dt) + PARTITION BY RANGE(dt) () + DISTRIBUTED BY HASH(order_id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + sql """ALTER TABLE ivm_failed_complete_f ADD PARTITION p202601 VALUES [('2026-01-01'), ('2026-02-01'))""" + sql """ALTER TABLE ivm_failed_complete_f ADD PARTITION p202602 VALUES [('2026-02-01'), ('2026-03-01'))""" + + sql """INSERT INTO ivm_failed_complete_f VALUES + (1, '2026-01-10', 100), + (2, '2026-02-10', 200)""" + + sql """ + CREATE MATERIALIZED VIEW ivm_failed_complete_mv + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL + KEY(order_id, dt) + PARTITION BY(dt) + DISTRIBUTED BY HASH(order_id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + AS SELECT order_id, dt, amount FROM ivm_failed_complete_f + """ + + sql """REFRESH MATERIALIZED VIEW ivm_failed_complete_mv COMPLETE""" + def taskId = waitForNewTask(null) + order_qt_baseline_mv """SELECT order_id, dt, amount FROM ivm_failed_complete_mv""" + + // The row the failed whole-MV refresh never got to read: it is what makes the recovery visible rather + // than a refresh that happens to report a different scope. + sql """INSERT INTO ivm_failed_complete_f VALUES (3, '2026-01-20', 300)""" + taskId = failCompleteRefresh(taskId) + qt_failed_complete_task taskQuery(taskId) + + // Both partitions were marked and neither snapshot moved, so the strict form plans the requirement on + // top of what the snapshots say: the whole MV is its scope, and the row above lands. + sql """REFRESH MATERIALIZED VIEW ivm_failed_complete_mv PARTITIONS""" + taskId = waitForNewTask(taskId) + qt_partitions_task taskQuery(taskId) + order_qt_partitions_mv """SELECT order_id, dt, amount FROM ivm_failed_complete_mv""" + + // Nothing is left to rebuild: the requirement the failed COMPLETE raised was cleared by the partition + // refresh that honoured it, not deferred to this one, which reports no scope of its own. + sql """REFRESH MATERIALIZED VIEW ivm_failed_complete_mv INCREMENTAL""" + taskId = waitForNewTask(taskId) + qt_incremental_task taskQuery(taskId) + + // And the fallback form recovers the same state without reaching the whole-MV attempt behind it. + sql """INSERT INTO ivm_failed_complete_f VALUES (4, '2026-02-20', 400)""" + taskId = failCompleteRefresh(taskId) + sql """REFRESH MATERIALIZED VIEW ivm_failed_complete_mv PARTITIONS FALLBACK""" + taskId = waitForNewTask(taskId) + qt_partitions_fallback_task taskQuery(taskId) + order_qt_partitions_fallback_mv """SELECT order_id, dt, amount FROM ivm_failed_complete_mv""" +} diff --git a/regression-test/suites/mtmv_p0/ivm/test_ivm_strict_failure_partition_atomicity.groovy b/regression-test/suites/mtmv_p0/ivm/test_ivm_strict_incremental_rebuilds_invalidated_partitions.groovy similarity index 58% rename from regression-test/suites/mtmv_p0/ivm/test_ivm_strict_failure_partition_atomicity.groovy rename to regression-test/suites/mtmv_p0/ivm/test_ivm_strict_incremental_rebuilds_invalidated_partitions.groovy index 13b9edc7e5c487..ede825a44584fc 100644 --- a/regression-test/suites/mtmv_p0/ivm/test_ivm_strict_failure_partition_atomicity.groovy +++ b/regression-test/suites/mtmv_p0/ivm/test_ivm_strict_incremental_rebuilds_invalidated_partitions.groovy @@ -18,11 +18,11 @@ import org.awaitility.Awaitility import static java.util.concurrent.TimeUnit.SECONDS -suite("test_ivm_strict_failure_partition_atomicity") { - sql """DROP MATERIALIZED VIEW IF EXISTS ivm_strict_atomicity_mv""" - sql """DROP TABLE IF EXISTS ivm_strict_atomicity_t""" +suite("test_ivm_strict_incremental_rebuilds_invalidated_partitions") { + sql """DROP MATERIALIZED VIEW IF EXISTS ivm_strict_rebuild_mv""" + sql """DROP TABLE IF EXISTS ivm_strict_rebuild_t""" sql """ - CREATE TABLE ivm_strict_atomicity_t ( + CREATE TABLE ivm_strict_rebuild_t ( dt DATE NOT NULL, id BIGINT NOT NULL, v INT @@ -42,37 +42,37 @@ suite("test_ivm_strict_failure_partition_atomicity") { "binlog.need_historical_value" = "true" ) """ - sql """INSERT INTO ivm_strict_atomicity_t VALUES + sql """INSERT INTO ivm_strict_rebuild_t VALUES ('2026-01-10', 1, 10), ('2026-02-10', 2, 20), ('2026-03-10', 3, 30)""" sql """ - CREATE MATERIALIZED VIEW ivm_strict_atomicity_mv + CREATE MATERIALIZED VIEW ivm_strict_rebuild_mv BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL PARTITION BY(dt) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ("replication_num" = "1") - AS SELECT dt, id, v FROM ivm_strict_atomicity_t + AS SELECT dt, id, v FROM ivm_strict_rebuild_t """ - sql """REFRESH MATERIALIZED VIEW ivm_strict_atomicity_mv COMPLETE""" - waitingMTMVTaskFinishedByMvName("ivm_strict_atomicity_mv") + sql """REFRESH MATERIALIZED VIEW ivm_strict_rebuild_mv COMPLETE""" + waitingMTMVTaskFinishedByMvName("ivm_strict_rebuild_mv") order_qt_before_ddl """ - SELECT dt, id, v FROM ivm_strict_atomicity_mv ORDER BY dt, id + SELECT dt, id, v FROM ivm_strict_rebuild_mv ORDER BY dt, id """ - sql """TRUNCATE TABLE ivm_strict_atomicity_t PARTITION(p1)""" - sql """ALTER TABLE ivm_strict_atomicity_t DROP PARTITION p2""" + sql """TRUNCATE TABLE ivm_strict_rebuild_t PARTITION(p1)""" + sql """ALTER TABLE ivm_strict_rebuild_t DROP PARTITION p2""" def previousTaskId = sql(""" SELECT TaskId FROM tasks('type'='mv') - WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_atomicity_mv' + WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_rebuild_mv' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 """)[0][0].toString() - sql """REFRESH MATERIALIZED VIEW ivm_strict_atomicity_mv INCREMENTAL""" + sql """REFRESH MATERIALIZED VIEW ivm_strict_rebuild_mv INCREMENTAL""" Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ def task = sql_return_maparray(""" SELECT TaskId, Status FROM tasks('type'='mv') - WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_atomicity_mv' + WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_rebuild_mv' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 """) return !task.isEmpty() @@ -80,20 +80,27 @@ suite("test_ivm_strict_failure_partition_atomicity") { && task[0].Status.toString() != 'PENDING' && task[0].Status.toString() != 'RUNNING' }) - order_qt_strict_task """ - SELECT Status, IvmFallbackReason, - ErrorMsg LIKE '%baseline rebuild is pending%' + // A strict INCREMENTAL request that meets the invalidated partitions rebuilds them and succeeds: what + // the truncated partition holds can no longer be removed incrementally, so the rebuild is the refresh's + // own work and not a reason to refuse. The state the change left behind -- the MV reads it as "rebuild + // the whole MV" -- is what such a refresh reports instead, and the rows below are the base tables'. + // + // IvmFallbackReason is unset for a refresh that never had to fall back, and an unset column comes back + // as the literal two-character string "\N", which does not survive the .out round trip, so fold it. + order_qt_strict_rebuild_task """ + SELECT Status, + CASE WHEN IvmFallbackReason = 'BINLOG_BROKEN' THEN IvmFallbackReason ELSE 'NONE' END FROM tasks('type'='mv') - WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_atomicity_mv' + WHERE MvDatabaseName = '${context.dbName}' AND MvName = 'ivm_strict_rebuild_mv' ORDER BY CreateTime DESC, TaskId DESC LIMIT 1 """ - order_qt_after_strict_failure """ - SELECT dt, id, v FROM ivm_strict_atomicity_mv ORDER BY dt, id + order_qt_after_strict_rebuild """ + SELECT dt, id, v FROM ivm_strict_rebuild_mv ORDER BY dt, id """ - sql """REFRESH MATERIALIZED VIEW ivm_strict_atomicity_mv AUTO""" - waitingMTMVTaskFinishedByMvName("ivm_strict_atomicity_mv") + sql """REFRESH MATERIALIZED VIEW ivm_strict_rebuild_mv AUTO""" + waitingMTMVTaskFinishedByMvName("ivm_strict_rebuild_mv") order_qt_after_auto_recovery """ - SELECT dt, id, v FROM ivm_strict_atomicity_mv ORDER BY dt, id + SELECT dt, id, v FROM ivm_strict_rebuild_mv ORDER BY dt, id """ }