From 2a31d25d02577f1415694fdae60e26222e6baa56 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 24 Apr 2026 11:41:16 -0700 Subject: [PATCH 1/7] feat: add partial load rule types changes: * adds `PartialLoadRule` abstract class to capture load rules that should partially load a segment on some tier * adds `IntervalPartialLoadRule, `ForeverPartialLoadRule`, and `PeriodPartialLoadRule` implementations to mirror non-partial load rules * adds `PartialLoadMatcher` interface to match and select what to partial load * adds `ExactProjectionPartialLoadMatcher` and `WildcardProjectionPartialLoadMatcher` to do partial loading of projections * adds `CannotMatchBehavior` enum to describe behavior of `PartialLoadRule` when `PartialLoadMatcher` is unable to match a segment * since partial loading is not available yet, partial rules function as regular load rules until follow-up work * tests --- .../rules/CannotMatchBehavior.java | 60 +++ .../ExactProjectionPartialLoadMatcher.java | 100 ++++ .../rules/ForeverPartialLoadRule.java | 61 +++ .../rules/IntervalPartialLoadRule.java | 94 ++++ .../coordinator/rules/PartialLoadMatcher.java | 60 +++ .../coordinator/rules/PartialLoadRule.java | 113 +++++ .../rules/PeriodPartialLoadRule.java | 104 +++++ .../rules/ProjectionPartialLoadMatcher.java | 85 ++++ .../druid/server/coordinator/rules/Rule.java | 3 + .../WildcardProjectionPartialLoadMatcher.java | 125 +++++ .../rules/PartialLoadMatcherTest.java | 437 ++++++++++++++++++ .../rules/PartialLoadRuleTest.java | 389 ++++++++++++++++ 12 files changed, 1631 insertions(+) create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcher.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverPartialLoadRule.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalPartialLoadRule.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadRule.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodPartialLoadRule.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java create mode 100644 server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java create mode 100644 server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadRuleTest.java diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java new file mode 100644 index 000000000000..cac9db690e84 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import javax.annotation.Nullable; + +/** + * Controls what happens when a {@link PartialLoadRule}'s {@link PartialLoadMatcher} does not apply to a given segment, + * for example, a {@link ProjectionPartialLoadMatcher} when faced with a segment that doesn't have projections. + *

+ * Unknown values deserialize to {@code null} so that an older coordinator encountering a rule authored on a newer + * version that introduces a new behavior falls back to the constructor's default ({@link #FULL_LOAD}) rather than + * failing to parse the rule. + */ +public enum CannotMatchBehavior +{ + /** + * The rule does not apply; the cascade continues to the next rule. + */ + FALL_THROUGH, + + /** + * The rule applies and the segment is loaded in full on this tier. + */ + FULL_LOAD; + + @JsonCreator + @Nullable + public static CannotMatchBehavior fromString(String value) + { + if (value == null) { + return null; + } + for (CannotMatchBehavior behavior : values()) { + if (behavior.name().equals(value)) { + return behavior; + } + } + return null; + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcher.java new file mode 100644 index 000000000000..4526db3a6a36 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcher.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.timeline.DataSegment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** + * Selects partial load of projections whose names appear in the configured {@link #names} list. + */ +public class ExactProjectionPartialLoadMatcher extends ProjectionPartialLoadMatcher +{ + public static final String TYPE = "exactProjection"; + + private final List names; + + @JsonCreator + public ExactProjectionPartialLoadMatcher(@JsonProperty("names") List names) + { + if (names == null || names.isEmpty()) { + throw InvalidInput.exception("names must not be null or empty for exactProjection matcher"); + } + this.names = List.copyOf(names); + } + + @JsonProperty + public List getNames() + { + return names; + } + + @Override + protected List resolveProjectionNames(DataSegment segment) + { + final List segmentProjections = segment.getProjections(); + if (segmentProjections == null || segmentProjections.isEmpty()) { + return Collections.emptyList(); + } + final Set present = new HashSet<>(segmentProjections); + final TreeSet intersected = new TreeSet<>(); + for (String name : names) { + if (present.contains(name)) { + intersected.add(name); + } + } + return new ArrayList<>(intersected); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ExactProjectionPartialLoadMatcher that = (ExactProjectionPartialLoadMatcher) o; + return Objects.equals(names, that.names); + } + + @Override + public int hashCode() + { + return Objects.hash(names); + } + + @Override + public String toString() + { + return "ExactProjectionPartialLoadMatcher{names=" + names + "}"; + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverPartialLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverPartialLoadRule.java new file mode 100644 index 000000000000..c11873cc9410 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ForeverPartialLoadRule.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.DateTime; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.util.Map; + +/** + * Forever variant of {@link PartialLoadRule}. Mirrors {@link ForeverLoadRule} (always load) and layers on + * {@link PartialLoadMatcher} for selection. + */ +public class ForeverPartialLoadRule extends PartialLoadRule +{ + public static final String TYPE = "loadPartialForever"; + + @JsonCreator + public ForeverPartialLoadRule( + @JsonProperty("tieredReplicants") Map tieredReplicants, + @JsonProperty("useDefaultTierForNull") @Nullable Boolean useDefaultTierForNull, + @JsonProperty("matcher") PartialLoadMatcher matcher, + @JsonProperty("onCannotMatch") @Nullable CannotMatchBehavior onCannotMatch + ) + { + super(tieredReplicants, useDefaultTierForNull, matcher, onCannotMatch); + } + + @Override + @JsonProperty + public String getType() + { + return TYPE; + } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) + { + return true; + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalPartialLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalPartialLoadRule.java new file mode 100644 index 000000000000..3df00418bac4 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/IntervalPartialLoadRule.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.DateTime; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.util.Map; +import java.util.Objects; + +/** + * Interval-based variant of {@link PartialLoadRule}. Mirrors {@link IntervalLoadRule} for time-window semantics and + * layers on {@link PartialLoadMatcher} for selection. + */ +public class IntervalPartialLoadRule extends PartialLoadRule +{ + public static final String TYPE = "loadPartialByInterval"; + + private final Interval interval; + + @JsonCreator + public IntervalPartialLoadRule( + @JsonProperty("interval") Interval interval, + @JsonProperty("tieredReplicants") Map tieredReplicants, + @JsonProperty("useDefaultTierForNull") @Nullable Boolean useDefaultTierForNull, + @JsonProperty("matcher") PartialLoadMatcher matcher, + @JsonProperty("onCannotMatch") @Nullable CannotMatchBehavior onCannotMatch + ) + { + super(tieredReplicants, useDefaultTierForNull, matcher, onCannotMatch); + this.interval = interval; + } + + @Override + @JsonProperty + public String getType() + { + return TYPE; + } + + @JsonProperty + public Interval getInterval() + { + return interval; + } + + @Override + public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) + { + return Rules.eligibleForLoad(interval, theInterval); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + IntervalPartialLoadRule that = (IntervalPartialLoadRule) o; + return Objects.equals(interval, that.interval); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), interval); + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java new file mode 100644 index 000000000000..31fc49290232 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.segment.loading.LoadSpec; +import org.apache.druid.server.coordination.SegmentChangeRequestLoad; +import org.apache.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.util.Map; + +/** + * Decides whether a {@link PartialLoadRule} should partially load a given segment and, when it should, produces the + * wire-form load-spec wrapper plus a fingerprint identifying that request. Implementations encapsulate both the + * configuration that drives the decision and the wire format of their corresponding {@link LoadSpec} wrapper, so the + * rule layer stays scheme-agnostic. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(name = ExactProjectionPartialLoadMatcher.TYPE, value = ExactProjectionPartialLoadMatcher.class), + @JsonSubTypes.Type(name = WildcardProjectionPartialLoadMatcher.TYPE, value = WildcardProjectionPartialLoadMatcher.class) +}) +public interface PartialLoadMatcher +{ + /** + * Returns the {@link MatchResult} this matcher produces for the given segment, or null if the matcher does not apply + * to the segment. When null, {@link PartialLoadRule} consults {@link CannotMatchBehavior} to decide whether the rule + * falls through or full-loads. + */ + @Nullable + MatchResult match(DataSegment segment, Map baseLoadSpec); + + /** + * Output of {@link #match(DataSegment, Map)} when the matcher applies. Carries the wrapped load-spec map (ready to + * be stamped onto an outbound {@link SegmentChangeRequestLoad}) and the fingerprint used by the coordinator to + * reconcile loaded replicas against the rule that requested them. + */ + record MatchResult(Map wrappedLoadSpec, String fingerprint) + { + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadRule.java new file mode 100644 index 000000000000..ad3c79faf835 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadRule.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.common.config.Configs; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.DateTime; + +import javax.annotation.Nullable; +import java.util.Map; +import java.util.Objects; + +/** + * Base class for rules that load only a subset of a segment on a tier. Pairs a {@link PartialLoadMatcher} (which + * produces the wrapped load-spec wire form and an accounting fingerprint when it applies to a segment) with a + * {@link CannotMatchBehavior} that controls whether the rule falls through or full-loads when the matcher does not + * apply. + */ +public abstract class PartialLoadRule extends LoadRule +{ + private final PartialLoadMatcher matcher; + private final CannotMatchBehavior onCannotMatch; + + protected PartialLoadRule( + Map tieredReplicants, + @Nullable Boolean useDefaultTierForNull, + PartialLoadMatcher matcher, + @Nullable CannotMatchBehavior onCannotMatch + ) + { + super(tieredReplicants, useDefaultTierForNull); + if (matcher == null) { + throw InvalidInput.exception("matcher must not be null for a partial load rule"); + } + this.matcher = matcher; + this.onCannotMatch = Configs.valueOrDefault(onCannotMatch, CannotMatchBehavior.FULL_LOAD); + } + + @JsonProperty + public PartialLoadMatcher getMatcher() + { + return matcher; + } + + @JsonProperty + public CannotMatchBehavior getOnCannotMatch() + { + return onCannotMatch; + } + + @Override + public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) + { + if (!appliesTo(segment.getInterval(), referenceTimestamp)) { + return false; + } + final PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + if (result != null) { + return true; + } + return onCannotMatch == CannotMatchBehavior.FULL_LOAD; + } + + @Override + public void run(DataSegment segment, SegmentActionHandler handler) + { + // Partial plumbing is added in future work. For now, a partial rule that applies to a segment full-loads it, + // identical behavior to the corresponding non-partial rule + handler.replicateSegment(segment, getTieredReplicants()); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PartialLoadRule that = (PartialLoadRule) o; + return onCannotMatch == that.onCannotMatch + && Objects.equals(matcher, that.matcher); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), matcher, onCannotMatch); + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodPartialLoadRule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodPartialLoadRule.java new file mode 100644 index 000000000000..db8d17c77450 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PeriodPartialLoadRule.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; + +import javax.annotation.Nullable; +import java.util.Map; +import java.util.Objects; + +/** + * Period-based variant of {@link PartialLoadRule}. Mirrors {@link PeriodLoadRule} for time-window semantics and layers + * on {@link PartialLoadMatcher} for selection. + */ +public class PeriodPartialLoadRule extends PartialLoadRule +{ + public static final String TYPE = "loadPartialByPeriod"; + + private final Period period; + private final boolean includeFuture; + + @JsonCreator + public PeriodPartialLoadRule( + @JsonProperty("period") Period period, + @JsonProperty("includeFuture") @Nullable Boolean includeFuture, + @JsonProperty("tieredReplicants") Map tieredReplicants, + @JsonProperty("useDefaultTierForNull") @Nullable Boolean useDefaultTierForNull, + @JsonProperty("matcher") PartialLoadMatcher matcher, + @JsonProperty("onCannotMatch") @Nullable CannotMatchBehavior onCannotMatch + ) + { + super(tieredReplicants, useDefaultTierForNull, matcher, onCannotMatch); + this.period = period; + this.includeFuture = includeFuture == null ? PeriodLoadRule.DEFAULT_INCLUDE_FUTURE : includeFuture; + } + + @Override + @JsonProperty + public String getType() + { + return TYPE; + } + + @JsonProperty + public Period getPeriod() + { + return period; + } + + @JsonProperty + public boolean isIncludeFuture() + { + return includeFuture; + } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) + { + return Rules.eligibleForLoad(period, interval, referenceTimestamp, includeFuture); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PeriodPartialLoadRule that = (PeriodPartialLoadRule) o; + return includeFuture == that.includeFuture && Objects.equals(period, that.period); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), period, includeFuture); + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java new file mode 100644 index 000000000000..cf5f6ef7cf0e --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; +import org.apache.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Base for {@link PartialLoadMatcher} implementations that decide which of a segment's V10 projections to load. + * Subclasses supply the resolution policy via {@link #resolveProjectionNames(DataSegment)}; this base handles + * fingerprint computation and wraps the result into the {@code partialProjection} load-spec wire form consumed + * by the historical-side {@code PartialProjectionLoadSpec} (which does not exist yet, supplied in future work). + *

+ * The fingerprint is a hash of what projections are partially loaded on a segment by this rule; the data node will + * include this value in the segment announcement so that it can be used as a lightweight value to compare against + * to handle things like rule change so that we can ensure that the 'right' partial load is in place from run to run. + */ +public abstract class ProjectionPartialLoadMatcher implements PartialLoadMatcher +{ + static final String WIRE_TYPE = "partialProjection"; + static final String FINGERPRINT_VERSION = "v1"; + + /** + * Returns the sorted, deduped list of projection names from {@link DataSegment#getProjections()} that this matcher + * selects. Returns an empty list when nothing matches (the segment exposes no projections, or no configured pattern + * intersects what the segment has). + */ + protected abstract List resolveProjectionNames(DataSegment segment); + + @Override + @Nullable + public MatchResult match(DataSegment segment, Map baseLoadSpec) + { + final List resolved = resolveProjectionNames(segment); + if (resolved.isEmpty()) { + return null; + } + final String fingerprint = computeFingerprint(resolved); + final Map wrapped = Map.of( + "type", WIRE_TYPE, + "delegate", baseLoadSpec, + "projections", resolved, + "ruleFingerprint", fingerprint + ); + return new MatchResult(wrapped, fingerprint); + } + + static String computeFingerprint(List sortedDedupedNames) + { + final Hasher hasher = Hashing.sha256().newHasher(); + for (String name : sortedDedupedNames) { + hasher.putString(name, StandardCharsets.UTF_8); + hasher.putByte((byte) 0); + } + final String hex = BaseEncoding.base16().encode(hasher.hash().asBytes()).toLowerCase(Locale.ROOT); + // should be good enough without dragging the whole thing around for every segment + return FINGERPRINT_VERSION + ":" + hex.substring(0, 16); + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java index a66101d0fe14..c6badd77b41b 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/Rule.java @@ -33,6 +33,9 @@ @JsonSubTypes.Type(name = "loadByPeriod", value = PeriodLoadRule.class), @JsonSubTypes.Type(name = "loadByInterval", value = IntervalLoadRule.class), @JsonSubTypes.Type(name = "loadForever", value = ForeverLoadRule.class), + @JsonSubTypes.Type(name = PeriodPartialLoadRule.TYPE, value = PeriodPartialLoadRule.class), + @JsonSubTypes.Type(name = IntervalPartialLoadRule.TYPE, value = IntervalPartialLoadRule.class), + @JsonSubTypes.Type(name = ForeverPartialLoadRule.TYPE, value = ForeverPartialLoadRule.class), @JsonSubTypes.Type(name = "dropByPeriod", value = PeriodDropRule.class), @JsonSubTypes.Type(name = "dropBeforeByPeriod", value = PeriodDropBeforeRule.class), @JsonSubTypes.Type(name = "dropByInterval", value = IntervalDropRule.class), diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java new file mode 100644 index 000000000000..775c9865bd53 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.io.FilenameUtils; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** + * Selects projections whose names match any of the configured glob patterns, minus any names listed in + * {@code excludes}. Matching uses {@link FilenameUtils#wildcardMatch(String, String)}; supported glob metacharacters + * are {@code *} (any sequence of characters) and {@code ?} (single character), all other characters literal. Excludes + * are literal projection names and let an operator carve out specific names from a broad pattern, for example, a + * long-retention rule {@code patterns=["user_*"]} combined with {@code excludes=["user_daily"]} keeps every + * {@code user_*} projection except {@code user_daily}, which the operator intends to retain only via a + * shorter-retention exact-match rule. + */ +public class WildcardProjectionPartialLoadMatcher extends ProjectionPartialLoadMatcher +{ + public static final String TYPE = "globProjection"; + + private final List patterns; + private final List excludes; + private final Set excludeSet; + + @JsonCreator + public WildcardProjectionPartialLoadMatcher( + @JsonProperty("patterns") List patterns, + @JsonProperty("excludes") @Nullable List excludes + ) + { + if (patterns == null || patterns.isEmpty()) { + throw InvalidInput.exception("patterns must not be null or empty for globProjection matcher"); + } + this.patterns = List.copyOf(patterns); + this.excludes = excludes == null ? List.of() : List.copyOf(excludes); + this.excludeSet = this.excludes.isEmpty() ? Set.of() : Set.copyOf(this.excludes); + } + + @JsonProperty + public List getPatterns() + { + return patterns; + } + + @JsonProperty + public List getExcludes() + { + return excludes; + } + + @Override + protected List resolveProjectionNames(DataSegment segment) + { + final List segmentProjections = segment.getProjections(); + if (segmentProjections == null || segmentProjections.isEmpty()) { + return Collections.emptyList(); + } + final TreeSet matched = new TreeSet<>(); + for (String name : segmentProjections) { + if (excludeSet.contains(name)) { + continue; + } + for (String pattern : patterns) { + if (FilenameUtils.wildcardMatch(name, pattern)) { + matched.add(name); + break; + } + } + } + return new ArrayList<>(matched); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + WildcardProjectionPartialLoadMatcher that = (WildcardProjectionPartialLoadMatcher) o; + return Objects.equals(patterns, that.patterns) && Objects.equals(excludes, that.excludes); + } + + @Override + public int hashCode() + { + return Objects.hash(patterns, excludes); + } + + @Override + public String toString() + { + return "WildcardProjectionPartialLoadMatcher{patterns=" + patterns + ", excludes=" + excludes + "}"; + } +} diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java new file mode 100644 index 000000000000..4265522434bd --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.hamcrest.MatcherAssert; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class PartialLoadMatcherTest +{ + private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); + + private static final Map BASE_LOAD_SPEC = Map.of( + "type", "local", + "path", "/var/druid/segments/foo" + ); + + private static final DataSegment.Builder BUILDER = DataSegment + .builder( + SegmentId.of( + "test", + Intervals.of("2024-01-01/2024-02-01"), + DateTimes.nowUtc().toString(), + new NumberedShardSpec(0, 0) + ) + ) + .loadSpec(BASE_LOAD_SPEC) + .size(0); + + @Test + void testExactMatchProducesResultWhenIntersectionNonEmpty() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("c", "a", "x") + ); + DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); + + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + + Map wrapped = result.wrappedLoadSpec(); + Assertions.assertEquals("partialProjection", wrapped.get("type")); + Assertions.assertEquals(BASE_LOAD_SPEC, wrapped.get("delegate")); + Assertions.assertEquals(List.of("a", "c"), wrapped.get("projections")); + Assertions.assertEquals(result.fingerprint(), wrapped.get("ruleFingerprint")); + Assertions.assertTrue(result.fingerprint().startsWith("v1:")); + Assertions.assertEquals("v1:".length() + 16, result.fingerprint().length()); + } + + @Test + void testExactMatchReturnsNullWhenNoIntersection() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("x", "y") + ); + DataSegment segment = segmentWithProjections(List.of("a", "b")); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testExactMatchReturnsNullForProjectionAgnosticSegment() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("a") + ); + DataSegment segment = segmentWithProjections(null); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testExactMatchReturnsNullForEmptyProjectionsList() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("a") + ); + DataSegment segment = segmentWithProjections(Collections.emptyList()); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testExactConstructorRejectsNullNames() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new ExactProjectionPartialLoadMatcher(null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("names must not be null or empty") + ); + } + + @Test + void testExactConstructorRejectsEmptyNames() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new ExactProjectionPartialLoadMatcher(Collections.emptyList()) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("names must not be null or empty") + ); + } + + @Test + void testExactMatchSortsAndDeduplicates() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("c", "a", "a", "b") + ); + DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("a", "b", "c"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testExactSerde() throws Exception + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("a", "b") + ); + String json = OBJECT_MAPPER.writeValueAsString(matcher); + PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertEquals(matcher, reread); + Assertions.assertInstanceOf(ExactProjectionPartialLoadMatcher.class, reread); + } + + @Test + void testExactEquals() + { + EqualsVerifier.forClass(ExactProjectionPartialLoadMatcher.class).usingGetClass().verify(); + } + + @Test + void testWildcardConstructorRejectsNullPatterns() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new WildcardProjectionPartialLoadMatcher(null, null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("patterns must not be null or empty") + ); + } + + @Test + void testWildcardConstructorRejectsEmptyPatterns() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new WildcardProjectionPartialLoadMatcher(Collections.emptyList(), null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("patterns must not be null or empty") + ); + } + + @Test + void testWildcardMatchStarMatchesAll() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + null + ); + DataSegment segment = segmentWithProjections(List.of("alpha", "beta", "gamma")); + + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("alpha", "beta", "gamma"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardMatchPrefixGlob() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + null + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "session_daily") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("user_daily", "user_hourly"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardMatchSingleCharGlob() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("p?"), + null + ); + DataSegment segment = segmentWithProjections(List.of("p1", "p2", "pxx", "q1")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("p1", "p2"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardMultiplePatternsUnioned() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*", "session_d*"), + null + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "session_daily", "session_hourly", "other") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("session_daily", "user_daily"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardReturnsNullForProjectionAgnosticSegment() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + null + ); + Assertions.assertNull(matcher.match(segmentWithProjections(null), BASE_LOAD_SPEC)); + Assertions.assertNull(matcher.match(segmentWithProjections(Collections.emptyList()), BASE_LOAD_SPEC)); + } + + @Test + void testWildcardEscapesRegexMetachars() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("a.b"), + null + ); + DataSegment segment = segmentWithProjections(List.of("a.b", "axb")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("a.b"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testWildcardSerde() throws Exception + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*", "p?"), + null + ); + String json = OBJECT_MAPPER.writeValueAsString(matcher); + PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertEquals(matcher, reread); + Assertions.assertInstanceOf(WildcardProjectionPartialLoadMatcher.class, reread); + } + + @Test + void testWildcardExcludeRemovesMatchedName() + { + // Long-retention rule that loads all user_* projections except user_daily — the latter is + // intended to live only on a shorter-retention exact-match rule. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_daily") + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "user_weekly") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("user_hourly", "user_weekly"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardExcludeNameNotMatchedByPatternIsNoop() + { + // Excluding a name that wouldn't have matched anyway is a no-op. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("session_daily") + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "session_daily") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("user_daily", "user_hourly"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardExcludeAllMatchedReturnsNull() + { + // If excludes consume every match the result is empty; the matcher reports "does not match". + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_daily", "user_hourly") + ); + DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testWildcardExcludeChangesFingerprint() + { + DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); + String withoutExcludes = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + null + ).match(segment, segment.getLoadSpec()).fingerprint(); + String withExcludes = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_daily") + ).match(segment, segment.getLoadSpec()).fingerprint(); + Assertions.assertNotEquals(withoutExcludes, withExcludes); + } + + @Test + void testWildcardSerdeWithExcludes() throws Exception + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_daily") + ); + String json = OBJECT_MAPPER.writeValueAsString(matcher); + PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertEquals(matcher, reread); + Assertions.assertEquals( + List.of("user_daily"), + ((WildcardProjectionPartialLoadMatcher) reread).getExcludes() + ); + } + + @Test + void testWildcardEquals() + { + EqualsVerifier.forClass(WildcardProjectionPartialLoadMatcher.class) + .withIgnoredFields("excludeSet") + .usingGetClass() + .verify(); + } + + @Test + void testFingerprintStableAcrossEquivalentInputOrderAndDuplicates() + { + // Two matcher configurations resolving to the same set on the same segment should yield identical fingerprints + // (the cascade should not thrash on equivalent rule rewordings). + DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); + ExactProjectionPartialLoadMatcher orderA = new ExactProjectionPartialLoadMatcher( + List.of("a", "b") + ); + ExactProjectionPartialLoadMatcher orderB = new ExactProjectionPartialLoadMatcher( + List.of("b", "b", "a") + ); + String fingerprintA = orderA.match(segment, segment.getLoadSpec()).fingerprint(); + String fingerprintB = orderB.match(segment, segment.getLoadSpec()).fingerprint(); + Assertions.assertEquals(fingerprintA, fingerprintB); + } + + @Test + void testFingerprintDiffersAcrossDifferentResolvedSets() + { + DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); + String fingerprintAB = new ExactProjectionPartialLoadMatcher(List.of("a", "b")) + .match(segment, segment.getLoadSpec()).fingerprint(); + String fingerprintAC = new ExactProjectionPartialLoadMatcher(List.of("a", "c")) + .match(segment, segment.getLoadSpec()).fingerprint(); + Assertions.assertNotEquals(fingerprintAB, fingerprintAC); + } + + @Test + void testFingerprintMatchesAcrossExactAndWildcardWhenResolvedSetIdentical() + { + // Exact {a,b} and Wildcard {a*} resolving to the same set on this segment should produce the same fingerprint, + // since fingerprinting is over the resolved list, not the rule text. + DataSegment segment = segmentWithProjections(List.of("a", "ab")); + String fingerprintExact = new ExactProjectionPartialLoadMatcher(List.of("a", "ab")) + .match(segment, segment.getLoadSpec()).fingerprint(); + String fingerprintGlob = new WildcardProjectionPartialLoadMatcher(List.of("a*"), null) + .match(segment, segment.getLoadSpec()).fingerprint(); + Assertions.assertEquals(fingerprintExact, fingerprintGlob); + } + + @Test + void testMatchResultEquals() + { + EqualsVerifier.forClass(PartialLoadMatcher.MatchResult.class).usingGetClass().verify(); + } + + private static DataSegment segmentWithProjections(List projections) + { + return BUILDER.projections(projections).build(); + } +} diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadRuleTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadRuleTest.java new file mode 100644 index 000000000000..f7cf0f2bc07a --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadRuleTest.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.client.DruidServer; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.hamcrest.MatcherAssert; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class PartialLoadRuleTest +{ + private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); + private static final DateTime NOW = DateTimes.of("2024-06-01T00:00:00"); + private static final Interval IN_WINDOW = Intervals.of("2024-05-15/2024-05-20"); + private static final Interval OUT_OF_WINDOW = Intervals.of("2020-01-01/2020-02-01"); + + @Test + void testAppliesToOutsideWindowReturnsFalseRegardlessOfMatcher() + { + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("a"), + null + ); + DataSegment segment = segmentWithProjections(OUT_OF_WINDOW, List.of("a", "b")); + Assertions.assertFalse(rule.appliesTo(segment, NOW)); + } + + @Test + void testAppliesToWindowMatchAndMatcherProducesResultReturnsTrue() + { + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("a"), + null + ); + DataSegment segment = segmentWithProjections(IN_WINDOW, List.of("a", "b")); + Assertions.assertTrue(rule.appliesTo(segment, NOW)); + } + + @Test + void testAppliesToMatcherDoesNotApplyFallThroughReturnsFalse() + { + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("nonexistent"), + CannotMatchBehavior.FALL_THROUGH + ); + DataSegment segment = segmentWithProjections(IN_WINDOW, List.of("a", "b")); + Assertions.assertFalse(rule.appliesTo(segment, NOW)); + } + + @Test + void testAppliesToMatcherDoesNotApplyFullLoadIsDefault() + { + // FULL_LOAD is the default. Segment has projections but matcher resolves to nothing, + // so the rule still applies and the segment gets full-loaded on this tier. + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("nonexistent"), + null + ); + DataSegment segment = segmentWithProjections(IN_WINDOW, List.of("a", "b")); + Assertions.assertTrue(rule.appliesTo(segment, NOW)); + } + + @Test + void testAppliesToProjectionAgnosticSegmentFallThrough() + { + // Pre-Druid-32 segment: projections == null. With FALL_THROUGH, rule does not apply. + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("a"), + CannotMatchBehavior.FALL_THROUGH + ); + DataSegment segment = segmentWithProjections(IN_WINDOW, null); + Assertions.assertFalse(rule.appliesTo(segment, NOW)); + } + + @Test + void testAppliesToProjectionAgnosticSegmentFullLoad() + { + // Default behavior: pre-Druid-32 segments fall into full-load on this tier. + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("a"), + null + ); + DataSegment segment = segmentWithProjections(IN_WINDOW, null); + Assertions.assertTrue(rule.appliesTo(segment, NOW)); + } + + @Test + void testIntervalOverloadIgnoresMatcher() + { + // The Interval-based appliesTo overload has no segment context, so it can only check the time window. Matcher + // logic does not run. + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("nonexistent"), + null + ); + Assertions.assertTrue(rule.appliesTo(IN_WINDOW, NOW)); + Assertions.assertFalse(rule.appliesTo(OUT_OF_WINDOW, NOW)); + } + + @Test + void testCascadeFallThroughToFullLoad() + { + // Rule 1: partial { "a" } 30 days, explicit FALL_THROUGH when matcher cannot match + PartialLoadRule partial = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(1), + null, + exact("a"), + CannotMatchBehavior.FALL_THROUGH + ); + // Rule 2: forever full load (no projections required) + ForeverLoadRule full = new ForeverLoadRule(tier(1), null); + + // Segment with no projections. Partial rule falls through; cascade lands on full. + DataSegment legacy = segmentWithProjections(IN_WINDOW, null); + Assertions.assertFalse(partial.appliesTo(legacy, NOW)); + Assertions.assertTrue(full.appliesTo(legacy, NOW)); + + // Segment with matching projection. Partial rule applies, cascade stops there. + DataSegment modern = segmentWithProjections(IN_WINDOW, List.of("a", "b")); + Assertions.assertTrue(partial.appliesTo(modern, NOW)); + } + + @Test + void testConstructorRejectsNullMatcher() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new PeriodPartialLoadRule(new Period("P1D"), false, tier(1), null, null, null) + ), + DruidExceptionMatcher.invalidInput() + .expectMessageContains("matcher must not be null") + ); + } + + @Test + void testPeriodSerde() throws Exception + { + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(2), + true, + exact("a", "b"), + CannotMatchBehavior.FULL_LOAD + ); + Rule reread = OBJECT_MAPPER.readValue(OBJECT_MAPPER.writeValueAsString(rule), Rule.class); + Assertions.assertEquals(rule, reread); + Assertions.assertInstanceOf(PeriodPartialLoadRule.class, reread); + } + + @Test + void testPeriodSerdeDefaults() throws Exception + { + String json = """ + {\ + "type": "loadPartialByPeriod",\ + "period": "P30D",\ + "matcher": {"type": "exactProjection", "names": ["a"]}\ + }"""; + PeriodPartialLoadRule rule = (PeriodPartialLoadRule) OBJECT_MAPPER.readValue(json, Rule.class); + Assertions.assertEquals(CannotMatchBehavior.FULL_LOAD, rule.getOnCannotMatch()); + Assertions.assertEquals(PeriodLoadRule.DEFAULT_INCLUDE_FUTURE, rule.isIncludeFuture()); + Assertions.assertEquals( + Map.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS), + rule.getTieredReplicants() + ); + } + + @Test + void testUnknownOnCannotMatchValueDeserializesToFullLoadDefault() throws Exception + { + // Simulates an older coordinator reading a rule authored by a newer version that introduced + // a new CannotMatchBehavior value. The rule should parse, with the unknown value falling + // back to the constructor's default (FULL_LOAD) rather than failing deserialization. + String json = """ + {\ + "type": "loadPartialByPeriod",\ + "period": "P30D",\ + "matcher": {"type": "exactProjection", "names": ["a"]},\ + "onCannotMatch": "SOME_FUTURE_BEHAVIOR"\ + }"""; + PeriodPartialLoadRule rule = (PeriodPartialLoadRule) OBJECT_MAPPER.readValue(json, Rule.class); + Assertions.assertEquals(CannotMatchBehavior.FULL_LOAD, rule.getOnCannotMatch()); + } + + @Test + void testIntervalSerde() throws Exception + { + IntervalPartialLoadRule rule = new IntervalPartialLoadRule( + Intervals.of("2024-01-01/2024-02-01"), + tier(1), + null, + new WildcardProjectionPartialLoadMatcher(List.of("user_*"), null), + CannotMatchBehavior.FALL_THROUGH + ); + Rule reread = OBJECT_MAPPER.readValue(OBJECT_MAPPER.writeValueAsString(rule), Rule.class); + Assertions.assertEquals(rule, reread); + Assertions.assertInstanceOf(IntervalPartialLoadRule.class, reread); + } + + @Test + void testForeverSerde() throws Exception + { + ForeverPartialLoadRule rule = new ForeverPartialLoadRule( + tier(1), + null, + exact("a"), + CannotMatchBehavior.FULL_LOAD + ); + Rule reread = OBJECT_MAPPER.readValue(OBJECT_MAPPER.writeValueAsString(rule), Rule.class); + Assertions.assertEquals(rule, reread); + Assertions.assertInstanceOf(ForeverPartialLoadRule.class, reread); + } + + @Test + void testForeverAlwaysAppliesToTimeWindow() + { + ForeverPartialLoadRule rule = new ForeverPartialLoadRule( + tier(1), + null, + exact("a"), + CannotMatchBehavior.FULL_LOAD + ); + DataSegment legacy = segmentWithProjections(OUT_OF_WINDOW, null); + Assertions.assertTrue(rule.appliesTo(legacy, NOW)); + } + + @Test + void testRunForwardsToReplicateSegment() + { + PeriodPartialLoadRule rule = new PeriodPartialLoadRule( + new Period("P30D"), + false, + tier(2), + null, + exact("a"), + null + ); + DataSegment segment = segmentWithProjections(IN_WINDOW, List.of("a", "b")); + RecordingHandler handler = new RecordingHandler(); + rule.run(segment, handler); + Assertions.assertEquals(1, handler.replicateCalls); + Assertions.assertEquals(0, handler.broadcastCalls); + Assertions.assertEquals(0, handler.deleteCalls); + Assertions.assertEquals(tier(2), handler.lastTierToReplicaCount); + } + + @Test + void testPeriodEquals() + { + EqualsVerifier.forClass(PeriodPartialLoadRule.class) + .withNonnullFields("tieredReplicants", "matcher", "onCannotMatch") + .withIgnoredFields("shouldSegmentBeLoaded") + .usingGetClass() + .verify(); + } + + @Test + void testIntervalEquals() + { + EqualsVerifier.forClass(IntervalPartialLoadRule.class) + .withNonnullFields("tieredReplicants", "matcher", "onCannotMatch") + .withIgnoredFields("shouldSegmentBeLoaded") + .usingGetClass() + .verify(); + } + + @Test + void testForeverEquals() + { + EqualsVerifier.forClass(ForeverPartialLoadRule.class) + .withNonnullFields("tieredReplicants", "matcher", "onCannotMatch") + .withIgnoredFields("shouldSegmentBeLoaded") + .usingGetClass() + .verify(); + } + + private static DataSegment segmentWithProjections(Interval interval, List projections) + { + return DataSegment + .builder(SegmentId.of("test", interval, DateTimes.nowUtc().toString(), new NumberedShardSpec(0, 0))) + .loadSpec(Map.of("type", "local", "path", "/var/druid/segments/foo")) + .projections(projections) + .size(0) + .build(); + } + + private static PartialLoadMatcher exact(String... names) + { + return new ExactProjectionPartialLoadMatcher(Arrays.asList(names)); + } + + private static Map tier(int n) + { + return Map.of(DruidServer.DEFAULT_TIER, n); + } + + private static class RecordingHandler implements SegmentActionHandler + { + int replicateCalls; + int broadcastCalls; + int deleteCalls; + Map lastTierToReplicaCount; + + @Override + public void replicateSegment(DataSegment segment, Map tierToReplicaCount) + { + replicateCalls++; + lastTierToReplicaCount = tierToReplicaCount; + } + + @Override + public void deleteSegment(DataSegment segment) + { + deleteCalls++; + } + + @Override + public void broadcastSegment(DataSegment segment) + { + broadcastCalls++; + } + } +} From f77c277643849887be23e787db079bfe61721867 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 24 Apr 2026 12:21:46 -0700 Subject: [PATCH 2/7] switch exclude to patterns too, fix style --- .../rules/ProjectionPartialLoadMatcher.java | 1 - .../WildcardProjectionPartialLoadMatcher.java | 55 +++++++++++-------- .../rules/PartialLoadMatcherTest.java | 47 +++++++++++----- 3 files changed, 64 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java index cf5f6ef7cf0e..48cafde7b973 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java @@ -19,7 +19,6 @@ package org.apache.druid.server.coordinator.rules; -import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java index 775c9865bd53..b47aaad019d9 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java @@ -30,38 +30,38 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.TreeSet; /** - * Selects projections whose names match any of the configured glob patterns, minus any names listed in - * {@code excludes}. Matching uses {@link FilenameUtils#wildcardMatch(String, String)}; supported glob metacharacters - * are {@code *} (any sequence of characters) and {@code ?} (single character), all other characters literal. Excludes - * are literal projection names and let an operator carve out specific names from a broad pattern, for example, a - * long-retention rule {@code patterns=["user_*"]} combined with {@code excludes=["user_daily"]} keeps every - * {@code user_*} projection except {@code user_daily}, which the operator intends to retain only via a - * shorter-retention exact-match rule. + * Selects projections whose names match any of the configured glob patterns, minus any names matching an entry in + * {@code excludePatterns}. Matching uses {@link FilenameUtils#wildcardMatch(String, String)}; supported glob + * metacharacters are {@code *} (any sequence of characters) and {@code ?} (single character), all other characters + * literal. A literal projection name is also a valid glob (with no wildcards it matches exactly itself), so the same + * field covers both "exclude this specific name" and "exclude anything matching this pattern." + *

+ * For example, a long-retention rule {@code patterns=["user_*"], excludePatterns=["user_daily"]} keeps every + * {@code user_*} projection except {@code user_daily} (which is expected to live on a shorter-retention rule). A + * broad rule {@code patterns=["*"], excludePatterns=["user_*"]} loads every projection except those handled by a + * more specific {@code user_*} rule elsewhere in the cascade. */ public class WildcardProjectionPartialLoadMatcher extends ProjectionPartialLoadMatcher { public static final String TYPE = "globProjection"; private final List patterns; - private final List excludes; - private final Set excludeSet; + private final List excludePatterns; @JsonCreator public WildcardProjectionPartialLoadMatcher( @JsonProperty("patterns") List patterns, - @JsonProperty("excludes") @Nullable List excludes + @JsonProperty("excludePatterns") @Nullable List excludePatterns ) { if (patterns == null || patterns.isEmpty()) { throw InvalidInput.exception("patterns must not be null or empty for globProjection matcher"); } this.patterns = List.copyOf(patterns); - this.excludes = excludes == null ? List.of() : List.copyOf(excludes); - this.excludeSet = this.excludes.isEmpty() ? Set.of() : Set.copyOf(this.excludes); + this.excludePatterns = excludePatterns == null ? List.of() : List.copyOf(excludePatterns); } @JsonProperty @@ -71,9 +71,9 @@ public List getPatterns() } @JsonProperty - public List getExcludes() + public List getExcludePatterns() { - return excludes; + return excludePatterns; } @Override @@ -85,19 +85,26 @@ protected List resolveProjectionNames(DataSegment segment) } final TreeSet matched = new TreeSet<>(); for (String name : segmentProjections) { - if (excludeSet.contains(name)) { + if (matchesAny(name, excludePatterns)) { continue; } - for (String pattern : patterns) { - if (FilenameUtils.wildcardMatch(name, pattern)) { - matched.add(name); - break; - } + if (matchesAny(name, patterns)) { + matched.add(name); } } return new ArrayList<>(matched); } + private static boolean matchesAny(String name, List patterns) + { + for (String pattern : patterns) { + if (FilenameUtils.wildcardMatch(name, pattern)) { + return true; + } + } + return false; + } + @Override public boolean equals(Object o) { @@ -108,18 +115,18 @@ public boolean equals(Object o) return false; } WildcardProjectionPartialLoadMatcher that = (WildcardProjectionPartialLoadMatcher) o; - return Objects.equals(patterns, that.patterns) && Objects.equals(excludes, that.excludes); + return Objects.equals(patterns, that.patterns) && Objects.equals(excludePatterns, that.excludePatterns); } @Override public int hashCode() { - return Objects.hash(patterns, excludes); + return Objects.hash(patterns, excludePatterns); } @Override public String toString() { - return "WildcardProjectionPartialLoadMatcher{patterns=" + patterns + ", excludes=" + excludes + "}"; + return "WildcardProjectionPartialLoadMatcher{patterns=" + patterns + ", excludePatterns=" + excludePatterns + "}"; } } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java index 4265522434bd..e640a5c48634 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java @@ -293,10 +293,11 @@ void testWildcardSerde() throws Exception } @Test - void testWildcardExcludeRemovesMatchedName() + void testWildcardExcludeLiteralRemovesMatchedName() { - // Long-retention rule that loads all user_* projections except user_daily — the latter is - // intended to live only on a shorter-retention exact-match rule. + // Long-retention rule that loads all user_* projections except user_daily — the latter is intended to live only on + // a shorter-retention exact-match rule. A literal name is a zero-wildcard glob, so the same excludePatterns field + // covers this case. WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( List.of("user_*"), List.of("user_daily") @@ -313,12 +314,31 @@ void testWildcardExcludeRemovesMatchedName() } @Test - void testWildcardExcludeNameNotMatchedByPatternIsNoop() + void testWildcardExcludePatternRemovesMatchingNames() { - // Excluding a name that wouldn't have matched anyway is a no-op. + // Broad rule that loads everything except names handled by a different, more specific rule. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + List.of("user_*") + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "session_daily", "other") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("other", "session_daily"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testWildcardExcludeNotMatchedIsNoop() + { + // Excluding a pattern that doesn't match anything in the segment is a no-op. WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( List.of("user_*"), - List.of("session_daily") + List.of("session_*") ); DataSegment segment = segmentWithProjections( List.of("user_daily", "user_hourly", "session_daily") @@ -334,10 +354,10 @@ void testWildcardExcludeNameNotMatchedByPatternIsNoop() @Test void testWildcardExcludeAllMatchedReturnsNull() { - // If excludes consume every match the result is empty; the matcher reports "does not match". + // If excludePatterns consume every match the result is empty; the matcher reports "does not match". WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( List.of("user_*"), - List.of("user_daily", "user_hourly") + List.of("user_*") ); DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); @@ -359,18 +379,18 @@ void testWildcardExcludeChangesFingerprint() } @Test - void testWildcardSerdeWithExcludes() throws Exception + void testWildcardSerdeWithExcludePatterns() throws Exception { WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - List.of("user_daily") + List.of("*"), + List.of("user_*") ); String json = OBJECT_MAPPER.writeValueAsString(matcher); PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); Assertions.assertEquals(matcher, reread); Assertions.assertEquals( - List.of("user_daily"), - ((WildcardProjectionPartialLoadMatcher) reread).getExcludes() + List.of("user_*"), + ((WildcardProjectionPartialLoadMatcher) reread).getExcludePatterns() ); } @@ -378,7 +398,6 @@ void testWildcardSerdeWithExcludes() throws Exception void testWildcardEquals() { EqualsVerifier.forClass(WildcardProjectionPartialLoadMatcher.class) - .withIgnoredFields("excludeSet") .usingGetClass() .verify(); } From 79ab5453bfd51f36741533c4eb3a4221ae871708 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 27 Apr 2026 03:01:32 -0700 Subject: [PATCH 3/7] better --- .../rules/CannotMatchBehavior.java | 31 +- .../coordinator/rules/PartialLoadMatcher.java | 2 +- .../rules/ProjectionPartialLoadMatcher.java | 8 +- .../rules/UnknownPartialLoadMatcher.java | 78 ++++ .../WildcardProjectionPartialLoadMatcher.java | 106 ++++- .../server/http/DataSourcesResource.java | 19 +- ...ExactProjectionPartialLoadMatcherTest.java | 169 ++++++++ .../rules/PartialLoadMatcherTest.java | 385 ++---------------- ...dcardProjectionPartialLoadMatcherTest.java | 372 +++++++++++++++++ .../server/http/DataSourcesResourceTest.java | 173 +++++++- 10 files changed, 962 insertions(+), 381 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/server/coordinator/rules/UnknownPartialLoadMatcher.java create mode 100644 server/src/test/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcherTest.java create mode 100644 server/src/test/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcherTest.java diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java index cac9db690e84..4da44b7361c2 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/CannotMatchBehavior.java @@ -20,6 +20,7 @@ package org.apache.druid.server.coordinator.rules; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; import javax.annotation.Nullable; @@ -36,25 +37,45 @@ public enum CannotMatchBehavior /** * The rule does not apply; the cascade continues to the next rule. */ - FALL_THROUGH, + FALL_THROUGH("fallThrough"), /** * The rule applies and the segment is loaded in full on this tier. */ - FULL_LOAD; + FULL_LOAD("fullLoad"); + + private final String id; + + CannotMatchBehavior(String id) + { + this.id = id; + } @JsonCreator @Nullable - public static CannotMatchBehavior fromString(String value) + public static CannotMatchBehavior fromString(final String id) { - if (value == null) { + if (id == null) { return null; } for (CannotMatchBehavior behavior : values()) { - if (behavior.name().equals(value)) { + if (behavior.id.equals(id)) { return behavior; } } + return null; } + + @JsonValue + public String getId() + { + return id; + } + + @Override + public String toString() + { + return id; + } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java index 31fc49290232..fd7cfc3cc5ba 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcher.java @@ -34,7 +34,7 @@ * configuration that drives the decision and the wire format of their corresponding {@link LoadSpec} wrapper, so the * rule layer stays scheme-agnostic. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = UnknownPartialLoadMatcher.class) @JsonSubTypes({ @JsonSubTypes.Type(name = ExactProjectionPartialLoadMatcher.TYPE, value = ExactProjectionPartialLoadMatcher.class), @JsonSubTypes.Type(name = WildcardProjectionPartialLoadMatcher.TYPE, value = WildcardProjectionPartialLoadMatcher.class) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java index 48cafde7b973..6098a6c11af3 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/ProjectionPartialLoadMatcher.java @@ -34,7 +34,7 @@ * Base for {@link PartialLoadMatcher} implementations that decide which of a segment's V10 projections to load. * Subclasses supply the resolution policy via {@link #resolveProjectionNames(DataSegment)}; this base handles * fingerprint computation and wraps the result into the {@code partialProjection} load-spec wire form consumed - * by the historical-side {@code PartialProjectionLoadSpec} (which does not exist yet, supplied in future work). + * by the historical-side {@code PartialProjectionLoadSpec}. *

* The fingerprint is a hash of what projections are partially loaded on a segment by this rule; the data node will * include this value in the segment announcement so that it can be used as a lightweight value to compare against @@ -42,7 +42,7 @@ */ public abstract class ProjectionPartialLoadMatcher implements PartialLoadMatcher { - static final String WIRE_TYPE = "partialProjection"; + static final String LOAD_SPEC_TYPE = "partialProjection"; static final String FINGERPRINT_VERSION = "v1"; /** @@ -62,10 +62,10 @@ public MatchResult match(DataSegment segment, Map baseLoadSpec) } final String fingerprint = computeFingerprint(resolved); final Map wrapped = Map.of( - "type", WIRE_TYPE, + "type", LOAD_SPEC_TYPE, "delegate", baseLoadSpec, "projections", resolved, - "ruleFingerprint", fingerprint + "fingerprint", fingerprint ); return new MatchResult(wrapped, fingerprint); } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/UnknownPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/UnknownPartialLoadMatcher.java new file mode 100644 index 000000000000..0ca2169196a9 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/UnknownPartialLoadMatcher.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Fallback {@link PartialLoadMatcher} used as the {@code defaultImpl} so that older Druid versions can still parse a + * rule whose matcher type was introduced in a newer version. {@link #match} always returns {@code null} (matcher does + * not apply); the rule then defers to its configured {@link CannotMatchBehavior}, so whether the segment falls through + * the cascade or gets full-loaded on this tier is up to the rule author. A warning is logged once per instance + * lifetime to surface the configuration mismatch without flooding the logs across coordinator passes. + *

+ * Note: this matcher is not lossless to round-trip. An older coordinator that reads a rule with an unknown matcher + * type and then re-serializes it will not preserve the original {@code type} discriminator or any matcher-specific + * configuration. The expected operational pattern is to upgrade the coordinator to a version that recognizes the + * matcher rather than rely on round-trip. + */ +public class UnknownPartialLoadMatcher implements PartialLoadMatcher +{ + private static final Logger log = new Logger(UnknownPartialLoadMatcher.class); + + private final AtomicBoolean warned = new AtomicBoolean(false); + + @Override + @Nullable + public MatchResult match(DataSegment segment, Map baseLoadSpec) + { + if (warned.compareAndSet(false, true)) { + log.warn( + "Encountered an unknown PartialLoadMatcher type in a partial load rule. The matcher will be treated as" + + " not applicable; the rule's onCannotMatch behavior determines the outcome. Upgrade Druid to a version" + + " that supports this matcher." + ); + } + return null; + } + + @Override + public boolean equals(Object o) + { + return o instanceof UnknownPartialLoadMatcher; + } + + @Override + public int hashCode() + { + return UnknownPartialLoadMatcher.class.hashCode(); + } + + @Override + public String toString() + { + return "UnknownPartialLoadMatcher{}"; + } +} diff --git a/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java b/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java index b47aaad019d9..934eaf6f9589 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcher.java @@ -20,8 +20,8 @@ package org.apache.druid.server.coordinator.rules; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.io.FilenameUtils; import org.apache.druid.error.InvalidInput; import org.apache.druid.timeline.DataSegment; @@ -31,13 +31,21 @@ import java.util.List; import java.util.Objects; import java.util.TreeSet; +import java.util.regex.Pattern; /** * Selects projections whose names match any of the configured glob patterns, minus any names matching an entry in - * {@code excludePatterns}. Matching uses {@link FilenameUtils#wildcardMatch(String, String)}; supported glob - * metacharacters are {@code *} (any sequence of characters) and {@code ?} (single character), all other characters - * literal. A literal projection name is also a valid glob (with no wildcards it matches exactly itself), so the same - * field covers both "exclude this specific name" and "exclude anything matching this pattern." + * {@code excludePatterns}. Supported glob metacharacters: + *

+ * All other characters are literal; regex metacharacters are escaped automatically. A literal projection name is + * a valid (zero-wildcard) glob, so the same {@code excludePatterns} field covers both "exclude this specific name" + * and "exclude anything matching this pattern." *

* For example, a long-retention rule {@code patterns=["user_*"], excludePatterns=["user_daily"]} keeps every * {@code user_*} projection except {@code user_daily} (which is expected to live on a shorter-retention rule). A @@ -50,6 +58,8 @@ public class WildcardProjectionPartialLoadMatcher extends ProjectionPartialLoadM private final List patterns; private final List excludePatterns; + private final List compiledPatterns; + private final List compiledExcludePatterns; @JsonCreator public WildcardProjectionPartialLoadMatcher( @@ -62,6 +72,8 @@ public WildcardProjectionPartialLoadMatcher( } this.patterns = List.copyOf(patterns); this.excludePatterns = excludePatterns == null ? List.of() : List.copyOf(excludePatterns); + this.compiledPatterns = compileAll(this.patterns); + this.compiledExcludePatterns = compileAll(this.excludePatterns); } @JsonProperty @@ -71,6 +83,7 @@ public List getPatterns() } @JsonProperty + @JsonInclude(JsonInclude.Include.NON_EMPTY) public List getExcludePatterns() { return excludePatterns; @@ -85,26 +98,99 @@ protected List resolveProjectionNames(DataSegment segment) } final TreeSet matched = new TreeSet<>(); for (String name : segmentProjections) { - if (matchesAny(name, excludePatterns)) { + if (matchesAny(name, compiledExcludePatterns)) { continue; } - if (matchesAny(name, patterns)) { + if (matchesAny(name, compiledPatterns)) { matched.add(name); } } return new ArrayList<>(matched); } - private static boolean matchesAny(String name, List patterns) + private static List compileAll(List globs) { - for (String pattern : patterns) { - if (FilenameUtils.wildcardMatch(name, pattern)) { + if (globs.isEmpty()) { + return List.of(); + } + final List compiled = new ArrayList<>(globs.size()); + for (String glob : globs) { + compiled.add(Pattern.compile(globToRegex(glob))); + } + return List.copyOf(compiled); + } + + private static boolean matchesAny(String name, List patterns) + { + for (Pattern pattern : patterns) { + if (pattern.matcher(name).matches()) { return true; } } return false; } + /** + * Translates a glob pattern with {@code *}, {@code ?}, and {@code \} escape semantics into an equivalent regex + * pattern that matches the entire input string. Regex metacharacters in literal positions are escaped. + * + * @throws org.apache.druid.error.DruidException if {@code glob} ends with an unescaped backslash + */ + static String globToRegex(String glob) + { + final StringBuilder sb = new StringBuilder(glob.length() + 4); + boolean escaping = false; + for (int i = 0; i < glob.length(); i++) { + final char c = glob.charAt(i); + if (escaping) { + appendLiteral(sb, c); + escaping = false; + continue; + } + switch (c) { + case '\\': + escaping = true; + break; + case '*': + sb.append(".*"); + break; + case '?': + sb.append('.'); + break; + default: + appendLiteral(sb, c); + } + } + if (escaping) { + throw InvalidInput.exception("Glob pattern [%s] ends with an unescaped backslash", glob); + } + return sb.toString(); + } + + private static void appendLiteral(StringBuilder sb, char c) + { + switch (c) { + case '.': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '+': + case '|': + case '^': + case '$': + case '\\': + case '*': + case '?': + sb.append('\\').append(c); + break; + default: + sb.append(c); + } + } + @Override public boolean equals(Object o) { diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 3fdf3eee9c73..a9b31fb94150 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -918,11 +918,28 @@ public Response isHandOffComplete( final Interval theInterval = Intervals.of(interval); final SegmentDescriptor descriptor = new SegmentDescriptor(theInterval, version, partitionNumber); final DateTime now = DateTimes.nowUtc(); + // Look up the segment in the metadata snapshot so the rule cascade can be evaluated against the real segment + // (necessary for partial load rules whose matcher inspects the segment's projection list). If the snapshot is + // not yet available, we cannot decide, let the realtime task retry. + final DataSourcesSnapshot snapshot = segmentsMetadataManager.getRecentDataSourcesSnapshot(); + if (snapshot == null) { + return Response.ok(false).build(); + } + final ImmutableDruidDataSource immutableDruidDataSource = snapshot.getDataSource(dataSourceName); + final DataSegment segment = immutableDruidDataSource == null + ? null + : immutableDruidDataSource.getSegment( + SegmentId.of(dataSourceName, theInterval, version, partitionNumber) + ); + // Segment isn't published in metadata; it will never be handed off. + if (segment == null) { + return Response.ok(true).build(); + } // A segment that is not eligible for load will never be handed off boolean eligibleForLoad = false; for (Rule rule : rules) { - if (rule.appliesTo(theInterval, now)) { + if (rule.appliesTo(segment, now)) { eligibleForLoad = rule instanceof LoadRule && ((LoadRule) rule).shouldMatchingSegmentBeLoaded(); break; } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcherTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcherTest.java new file mode 100644 index 000000000000..ade05388eb1a --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/ExactProjectionPartialLoadMatcherTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.hamcrest.MatcherAssert; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ExactProjectionPartialLoadMatcherTest +{ + private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); + + private static final Map BASE_LOAD_SPEC = Map.of( + "type", "local", + "path", "/var/druid/segments/foo" + ); + + private static final DataSegment.Builder BUILDER = DataSegment + .builder( + SegmentId.of( + "test", + Intervals.of("2024-01-01/2024-02-01"), + DateTimes.nowUtc().toString(), + new NumberedShardSpec(0, 0) + ) + ) + .loadSpec(BASE_LOAD_SPEC) + .size(0); + + @Test + void testMatchProducesResultWhenIntersectionNonEmpty() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("c", "a", "x") + ); + DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); + + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + + Map wrapped = result.wrappedLoadSpec(); + Assertions.assertEquals("partialProjection", wrapped.get("type")); + Assertions.assertEquals(BASE_LOAD_SPEC, wrapped.get("delegate")); + Assertions.assertEquals(List.of("a", "c"), wrapped.get("projections")); + Assertions.assertEquals(result.fingerprint(), wrapped.get("fingerprint")); + Assertions.assertTrue(result.fingerprint().startsWith("v1:")); + Assertions.assertEquals("v1:".length() + 16, result.fingerprint().length()); + } + + @Test + void testMatchReturnsNullWhenNoIntersection() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("x", "y") + ); + DataSegment segment = segmentWithProjections(List.of("a", "b")); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testMatchReturnsNullForProjectionAgnosticSegment() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("a") + ); + DataSegment segment = segmentWithProjections(null); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testMatchReturnsNullForEmptyProjectionsList() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("a") + ); + DataSegment segment = segmentWithProjections(Collections.emptyList()); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testConstructorRejectsNullNames() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new ExactProjectionPartialLoadMatcher(null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("names must not be null or empty") + ); + } + + @Test + void testConstructorRejectsEmptyNames() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new ExactProjectionPartialLoadMatcher(Collections.emptyList()) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("names must not be null or empty") + ); + } + + @Test + void testMatchSortsAndDeduplicates() + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("c", "a", "a", "b") + ); + DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("a", "b", "c"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testSerde() throws Exception + { + ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( + List.of("a", "b") + ); + String json = OBJECT_MAPPER.writeValueAsString(matcher); + PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertEquals(matcher, reread); + Assertions.assertInstanceOf(ExactProjectionPartialLoadMatcher.class, reread); + } + + @Test + void testEquals() + { + EqualsVerifier.forClass(ExactProjectionPartialLoadMatcher.class).usingGetClass().verify(); + } + + private static DataSegment segmentWithProjections(List projections) + { + return BUILDER.projections(projections).build(); + } +} diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java index e640a5c48634..9f861a9d3e49 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/PartialLoadMatcherTest.java @@ -21,22 +21,25 @@ import com.fasterxml.jackson.databind.ObjectMapper; import nl.jqno.equalsverifier.EqualsVerifier; -import org.apache.druid.error.DruidException; -import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.NumberedShardSpec; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import java.util.Map; +/** + * Cross-cutting tests for the {@link PartialLoadMatcher} interface itself: fingerprint stability across matcher + * implementations, the {@link PartialLoadMatcher.MatchResult} value type, and {@link UnknownPartialLoadMatcher} + * (the {@code defaultImpl} fallback for unrecognized matcher types). Per-matcher behavior tests live in their own + * test classes (see {@link ExactProjectionPartialLoadMatcherTest}, + * {@link WildcardProjectionPartialLoadMatcherTest}). + */ public class PartialLoadMatcherTest { private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); @@ -58,350 +61,6 @@ public class PartialLoadMatcherTest .loadSpec(BASE_LOAD_SPEC) .size(0); - @Test - void testExactMatchProducesResultWhenIntersectionNonEmpty() - { - ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( - List.of("c", "a", "x") - ); - DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); - - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - - Map wrapped = result.wrappedLoadSpec(); - Assertions.assertEquals("partialProjection", wrapped.get("type")); - Assertions.assertEquals(BASE_LOAD_SPEC, wrapped.get("delegate")); - Assertions.assertEquals(List.of("a", "c"), wrapped.get("projections")); - Assertions.assertEquals(result.fingerprint(), wrapped.get("ruleFingerprint")); - Assertions.assertTrue(result.fingerprint().startsWith("v1:")); - Assertions.assertEquals("v1:".length() + 16, result.fingerprint().length()); - } - - @Test - void testExactMatchReturnsNullWhenNoIntersection() - { - ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( - List.of("x", "y") - ); - DataSegment segment = segmentWithProjections(List.of("a", "b")); - Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); - } - - @Test - void testExactMatchReturnsNullForProjectionAgnosticSegment() - { - ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( - List.of("a") - ); - DataSegment segment = segmentWithProjections(null); - Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); - } - - @Test - void testExactMatchReturnsNullForEmptyProjectionsList() - { - ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( - List.of("a") - ); - DataSegment segment = segmentWithProjections(Collections.emptyList()); - Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); - } - - @Test - void testExactConstructorRejectsNullNames() - { - MatcherAssert.assertThat( - Assertions.assertThrows( - DruidException.class, - () -> new ExactProjectionPartialLoadMatcher(null) - ), - DruidExceptionMatcher.invalidInput().expectMessageContains("names must not be null or empty") - ); - } - - @Test - void testExactConstructorRejectsEmptyNames() - { - MatcherAssert.assertThat( - Assertions.assertThrows( - DruidException.class, - () -> new ExactProjectionPartialLoadMatcher(Collections.emptyList()) - ), - DruidExceptionMatcher.invalidInput().expectMessageContains("names must not be null or empty") - ); - } - - @Test - void testExactMatchSortsAndDeduplicates() - { - ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( - List.of("c", "a", "a", "b") - ); - DataSegment segment = segmentWithProjections(List.of("a", "b", "c")); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals(List.of("a", "b", "c"), result.wrappedLoadSpec().get("projections")); - } - - @Test - void testExactSerde() throws Exception - { - ExactProjectionPartialLoadMatcher matcher = new ExactProjectionPartialLoadMatcher( - List.of("a", "b") - ); - String json = OBJECT_MAPPER.writeValueAsString(matcher); - PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); - Assertions.assertEquals(matcher, reread); - Assertions.assertInstanceOf(ExactProjectionPartialLoadMatcher.class, reread); - } - - @Test - void testExactEquals() - { - EqualsVerifier.forClass(ExactProjectionPartialLoadMatcher.class).usingGetClass().verify(); - } - - @Test - void testWildcardConstructorRejectsNullPatterns() - { - MatcherAssert.assertThat( - Assertions.assertThrows( - DruidException.class, - () -> new WildcardProjectionPartialLoadMatcher(null, null) - ), - DruidExceptionMatcher.invalidInput().expectMessageContains("patterns must not be null or empty") - ); - } - - @Test - void testWildcardConstructorRejectsEmptyPatterns() - { - MatcherAssert.assertThat( - Assertions.assertThrows( - DruidException.class, - () -> new WildcardProjectionPartialLoadMatcher(Collections.emptyList(), null) - ), - DruidExceptionMatcher.invalidInput().expectMessageContains("patterns must not be null or empty") - ); - } - - @Test - void testWildcardMatchStarMatchesAll() - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("*"), - null - ); - DataSegment segment = segmentWithProjections(List.of("alpha", "beta", "gamma")); - - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("alpha", "beta", "gamma"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardMatchPrefixGlob() - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - null - ); - DataSegment segment = segmentWithProjections( - List.of("user_daily", "user_hourly", "session_daily") - ); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("user_daily", "user_hourly"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardMatchSingleCharGlob() - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("p?"), - null - ); - DataSegment segment = segmentWithProjections(List.of("p1", "p2", "pxx", "q1")); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("p1", "p2"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardMultiplePatternsUnioned() - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*", "session_d*"), - null - ); - DataSegment segment = segmentWithProjections( - List.of("user_daily", "session_daily", "session_hourly", "other") - ); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("session_daily", "user_daily"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardReturnsNullForProjectionAgnosticSegment() - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("*"), - null - ); - Assertions.assertNull(matcher.match(segmentWithProjections(null), BASE_LOAD_SPEC)); - Assertions.assertNull(matcher.match(segmentWithProjections(Collections.emptyList()), BASE_LOAD_SPEC)); - } - - @Test - void testWildcardEscapesRegexMetachars() - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("a.b"), - null - ); - DataSegment segment = segmentWithProjections(List.of("a.b", "axb")); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals(List.of("a.b"), result.wrappedLoadSpec().get("projections")); - } - - @Test - void testWildcardSerde() throws Exception - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*", "p?"), - null - ); - String json = OBJECT_MAPPER.writeValueAsString(matcher); - PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); - Assertions.assertEquals(matcher, reread); - Assertions.assertInstanceOf(WildcardProjectionPartialLoadMatcher.class, reread); - } - - @Test - void testWildcardExcludeLiteralRemovesMatchedName() - { - // Long-retention rule that loads all user_* projections except user_daily — the latter is intended to live only on - // a shorter-retention exact-match rule. A literal name is a zero-wildcard glob, so the same excludePatterns field - // covers this case. - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - List.of("user_daily") - ); - DataSegment segment = segmentWithProjections( - List.of("user_daily", "user_hourly", "user_weekly") - ); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("user_hourly", "user_weekly"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardExcludePatternRemovesMatchingNames() - { - // Broad rule that loads everything except names handled by a different, more specific rule. - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("*"), - List.of("user_*") - ); - DataSegment segment = segmentWithProjections( - List.of("user_daily", "user_hourly", "session_daily", "other") - ); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("other", "session_daily"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardExcludeNotMatchedIsNoop() - { - // Excluding a pattern that doesn't match anything in the segment is a no-op. - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - List.of("session_*") - ); - DataSegment segment = segmentWithProjections( - List.of("user_daily", "user_hourly", "session_daily") - ); - PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); - Assertions.assertNotNull(result); - Assertions.assertEquals( - List.of("user_daily", "user_hourly"), - result.wrappedLoadSpec().get("projections") - ); - } - - @Test - void testWildcardExcludeAllMatchedReturnsNull() - { - // If excludePatterns consume every match the result is empty; the matcher reports "does not match". - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - List.of("user_*") - ); - DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); - Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); - } - - @Test - void testWildcardExcludeChangesFingerprint() - { - DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); - String withoutExcludes = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - null - ).match(segment, segment.getLoadSpec()).fingerprint(); - String withExcludes = new WildcardProjectionPartialLoadMatcher( - List.of("user_*"), - List.of("user_daily") - ).match(segment, segment.getLoadSpec()).fingerprint(); - Assertions.assertNotEquals(withoutExcludes, withExcludes); - } - - @Test - void testWildcardSerdeWithExcludePatterns() throws Exception - { - WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( - List.of("*"), - List.of("user_*") - ); - String json = OBJECT_MAPPER.writeValueAsString(matcher); - PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); - Assertions.assertEquals(matcher, reread); - Assertions.assertEquals( - List.of("user_*"), - ((WildcardProjectionPartialLoadMatcher) reread).getExcludePatterns() - ); - } - - @Test - void testWildcardEquals() - { - EqualsVerifier.forClass(WildcardProjectionPartialLoadMatcher.class) - .usingGetClass() - .verify(); - } - @Test void testFingerprintStableAcrossEquivalentInputOrderAndDuplicates() { @@ -449,6 +108,36 @@ void testMatchResultEquals() EqualsVerifier.forClass(PartialLoadMatcher.MatchResult.class).usingGetClass().verify(); } + @Test + void testUnknownTypeDeserializesToFallback() throws Exception + { + // A matcher type unknown to this Druid version should fall back to UnknownPartialLoadMatcher rather than fail to + // parse, so the rules cascade can keep working when an older coordinator reads a rule authored on a newer version. + String json = "{\"type\": \"someFutureMatcherType\", \"someFutureField\": [\"x\", \"y\"]}"; + PartialLoadMatcher matcher = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertInstanceOf(UnknownPartialLoadMatcher.class, matcher); + } + + @Test + void testUnknownMatcherDoesNotApply() + { + // An unknown matcher returns null from match() so the rule's appliesTo defers to the rule's onCannotMatch + // behavior (FALL_THROUGH continues the cascade; FULL_LOAD applies the rule as a full load). + UnknownPartialLoadMatcher matcher = new UnknownPartialLoadMatcher(); + DataSegment segment = segmentWithProjections(List.of("a", "b")); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testUnknownMatcherEquals() + { + Assertions.assertEquals(new UnknownPartialLoadMatcher(), new UnknownPartialLoadMatcher()); + Assertions.assertEquals( + new UnknownPartialLoadMatcher().hashCode(), + new UnknownPartialLoadMatcher().hashCode() + ); + } + private static DataSegment segmentWithProjections(List projections) { return BUILDER.projections(projections).build(); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcherTest.java b/server/src/test/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcherTest.java new file mode 100644 index 000000000000..91d62ad4eecf --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/coordinator/rules/WildcardProjectionPartialLoadMatcherTest.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.coordinator.rules; + +import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.hamcrest.MatcherAssert; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class WildcardProjectionPartialLoadMatcherTest +{ + private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); + + private static final Map BASE_LOAD_SPEC = Map.of( + "type", "local", + "path", "/var/druid/segments/foo" + ); + + private static final DataSegment.Builder BUILDER = DataSegment + .builder( + SegmentId.of( + "test", + Intervals.of("2024-01-01/2024-02-01"), + DateTimes.nowUtc().toString(), + new NumberedShardSpec(0, 0) + ) + ) + .loadSpec(BASE_LOAD_SPEC) + .size(0); + + @Test + void testConstructorRejectsNullPatterns() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new WildcardProjectionPartialLoadMatcher(null, null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("patterns must not be null or empty") + ); + } + + @Test + void testConstructorRejectsEmptyPatterns() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new WildcardProjectionPartialLoadMatcher(Collections.emptyList(), null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("patterns must not be null or empty") + ); + } + + @Test + void testMatchStarMatchesAll() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + null + ); + DataSegment segment = segmentWithProjections(List.of("alpha", "beta", "gamma")); + + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("alpha", "beta", "gamma"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testMatchPrefixGlob() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + null + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "session_daily") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("user_daily", "user_hourly"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testMatchSingleCharGlob() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("p?"), + null + ); + DataSegment segment = segmentWithProjections(List.of("p1", "p2", "pxx", "q1")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("p1", "p2"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testMultiplePatternsUnioned() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*", "session_d*"), + null + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "session_daily", "session_hourly", "other") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("session_daily", "user_daily"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testReturnsNullForProjectionAgnosticSegment() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + null + ); + Assertions.assertNull(matcher.match(segmentWithProjections(null), BASE_LOAD_SPEC)); + Assertions.assertNull(matcher.match(segmentWithProjections(Collections.emptyList()), BASE_LOAD_SPEC)); + } + + @Test + void testEscapesRegexMetachars() + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("a.b"), + null + ); + DataSegment segment = segmentWithProjections(List.of("a.b", "axb")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("a.b"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testEscapesAsterisk() + { + // \\* in the pattern matches a literal asterisk, not the glob wildcard. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("foo\\*bar"), + null + ); + DataSegment segment = segmentWithProjections(List.of("foo*bar", "fooXYZbar", "foo*bar*baz")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("foo*bar"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testEscapesQuestionMark() + { + // \\? in the pattern matches a literal question mark, not the glob single-char wildcard. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("foo\\?bar"), + null + ); + DataSegment segment = segmentWithProjections(List.of("foo?bar", "fooXbar")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("foo?bar"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testEscapesBackslash() + { + // \\\\ in the Java string is \\ in the pattern, which represents a single literal backslash. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("foo\\\\bar"), + null + ); + DataSegment segment = segmentWithProjections(List.of("foo\\bar", "foobar")); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals(List.of("foo\\bar"), result.wrappedLoadSpec().get("projections")); + } + + @Test + void testRejectsTrailingBackslash() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new WildcardProjectionPartialLoadMatcher(List.of("foo\\"), null) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("ends with an unescaped backslash") + ); + } + + @Test + void testRejectsTrailingBackslashInExcludePatterns() + { + MatcherAssert.assertThat( + Assertions.assertThrows( + DruidException.class, + () -> new WildcardProjectionPartialLoadMatcher(List.of("*"), List.of("foo\\")) + ), + DruidExceptionMatcher.invalidInput().expectMessageContains("ends with an unescaped backslash") + ); + } + + @Test + void testSerde() throws Exception + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*", "p?"), + null + ); + String json = OBJECT_MAPPER.writeValueAsString(matcher); + PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertEquals(matcher, reread); + Assertions.assertInstanceOf(WildcardProjectionPartialLoadMatcher.class, reread); + } + + @Test + void testExcludeLiteralRemovesMatchedName() + { + // Long-retention rule that loads all user_* projections except user_daily, the latter is intended to live only on + // a shorter-retention exact-match rule. A literal name is a zero-wildcard glob, so the same excludePatterns field + // covers this case. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_daily") + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "user_weekly") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("user_hourly", "user_weekly"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testExcludePatternRemovesMatchingNames() + { + // Broad rule that loads everything except names handled by a different, more specific rule. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + List.of("user_*") + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "session_daily", "other") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("other", "session_daily"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testExcludeNotMatchedIsNoop() + { + // Excluding a pattern that doesn't match anything in the segment is a no-op. + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("session_*") + ); + DataSegment segment = segmentWithProjections( + List.of("user_daily", "user_hourly", "session_daily") + ); + PartialLoadMatcher.MatchResult result = matcher.match(segment, segment.getLoadSpec()); + Assertions.assertNotNull(result); + Assertions.assertEquals( + List.of("user_daily", "user_hourly"), + result.wrappedLoadSpec().get("projections") + ); + } + + @Test + void testExcludeAllMatchedReturnsNull() + { + // If excludePatterns consume every match the result is empty; the matcher reports "does not match". + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_*") + ); + DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); + Assertions.assertNull(matcher.match(segment, segment.getLoadSpec())); + } + + @Test + void testExcludeChangesFingerprint() + { + DataSegment segment = segmentWithProjections(List.of("user_daily", "user_hourly")); + String withoutExcludes = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + null + ).match(segment, segment.getLoadSpec()).fingerprint(); + String withExcludes = new WildcardProjectionPartialLoadMatcher( + List.of("user_*"), + List.of("user_daily") + ).match(segment, segment.getLoadSpec()).fingerprint(); + Assertions.assertNotEquals(withoutExcludes, withExcludes); + } + + @Test + void testSerdeWithExcludePatterns() throws Exception + { + WildcardProjectionPartialLoadMatcher matcher = new WildcardProjectionPartialLoadMatcher( + List.of("*"), + List.of("user_*") + ); + String json = OBJECT_MAPPER.writeValueAsString(matcher); + PartialLoadMatcher reread = OBJECT_MAPPER.readValue(json, PartialLoadMatcher.class); + Assertions.assertEquals(matcher, reread); + Assertions.assertEquals( + List.of("user_*"), + ((WildcardProjectionPartialLoadMatcher) reread).getExcludePatterns() + ); + } + + @Test + void testEquals() + { + EqualsVerifier.forClass(WildcardProjectionPartialLoadMatcher.class) + .withIgnoredFields("compiledPatterns", "compiledExcludePatterns") + .usingGetClass() + .verify(); + } + + private static DataSegment segmentWithProjections(List projections) + { + return BUILDER.projections(projections).build(); + } +} diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 183d785e9795..59a5c1d2f22d 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -54,8 +54,11 @@ import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.coordinator.DruidCoordinator; +import org.apache.druid.server.coordinator.rules.CannotMatchBehavior; +import org.apache.druid.server.coordinator.rules.ExactProjectionPartialLoadMatcher; import org.apache.druid.server.coordinator.rules.IntervalDropRule; import org.apache.druid.server.coordinator.rules.IntervalLoadRule; +import org.apache.druid.server.coordinator.rules.IntervalPartialLoadRule; import org.apache.druid.server.coordinator.rules.Rule; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; @@ -655,38 +658,49 @@ public void testIsHandOffComplete() Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-02T00:00:00Z/2013-01-03T00:00:00Z"), null, null); Rule dropRule = new IntervalDropRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z")); DataSourcesResource dataSourcesResource = - new DataSourcesResource(inventoryView, null, databaseRuleManager, null, null, null, auditManager); + new DataSourcesResource( + inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + ); - // test dropped + // test dropped: segment exists in metadata, drop rule applies → won't be handed off + String interval1 = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; + DataSegment segment1 = buildHandoffSegment(TestDataSource.WIKI, Intervals.of(interval1), "v1", 1); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) .andReturn(ImmutableList.of(loadRule, dropRule)) .once(); - EasyMock.replay(databaseRuleManager); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of(segment1))) + .once(); + EasyMock.replay(databaseRuleManager, segmentsMetadataManager); - String interval1 = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; Response response1 = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval1, 1, "v1"); Assert.assertTrue((boolean) response1.getEntity()); - EasyMock.verify(databaseRuleManager); + EasyMock.verify(databaseRuleManager, segmentsMetadataManager); // test isn't dropped and no timeline found - EasyMock.reset(databaseRuleManager); + String interval2 = "2013-01-02T01:00:00Z/2013-01-02T02:00:00Z"; + DataSegment segment2 = buildHandoffSegment(TestDataSource.WIKI, Intervals.of(interval2), "v1", 1); + EasyMock.reset(databaseRuleManager, segmentsMetadataManager); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) .andReturn(ImmutableList.of(loadRule, dropRule)) .once(); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of(segment2))) + .once(); EasyMock.expect(inventoryView.getTimeline(new TableDataSource(TestDataSource.WIKI))) .andReturn(null) .once(); - EasyMock.replay(inventoryView, databaseRuleManager); + EasyMock.replay(inventoryView, databaseRuleManager, segmentsMetadataManager); - String interval2 = "2013-01-02T01:00:00Z/2013-01-02T02:00:00Z"; Response response2 = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval2, 1, "v1"); Assert.assertFalse((boolean) response2.getEntity()); - EasyMock.verify(inventoryView, databaseRuleManager); + EasyMock.verify(inventoryView, databaseRuleManager, segmentsMetadataManager); // test isn't dropped and timeline exist String interval3 = "2013-01-02T02:00:00Z/2013-01-02T03:00:00Z"; + DataSegment segment3 = buildHandoffSegment(TestDataSource.WIKI, Intervals.of(interval3), "v1", 1); SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(createSegment(Intervals.of(interval3), "v1", 1)); segmentLoadInfo.addServer(createHistoricalServerMetadata("test")); VersionedIntervalTimeline timeline = @@ -702,19 +716,154 @@ public List> lookupWithIncompleteP return ret; } }; - EasyMock.reset(inventoryView, databaseRuleManager); + EasyMock.reset(inventoryView, databaseRuleManager, segmentsMetadataManager); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) .andReturn(ImmutableList.of(loadRule, dropRule)) .once(); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of(segment3))) + .once(); EasyMock.expect(inventoryView.getTimeline(new TableDataSource(TestDataSource.WIKI))) .andReturn(timeline) .once(); - EasyMock.replay(inventoryView, databaseRuleManager); + EasyMock.replay(inventoryView, databaseRuleManager, segmentsMetadataManager); Response response3 = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval3, 1, "v1"); Assert.assertTrue((boolean) response3.getEntity()); - EasyMock.verify(inventoryView, databaseRuleManager); + EasyMock.verify(inventoryView, databaseRuleManager, segmentsMetadataManager); + } + + @Test + public void testIsHandOffCompleteSegmentNotInMetadataReturnsTrue() + { + // A segment that hasn't been published to metadata yet (or has been removed) is reported as definitively + // never-handed-off so the realtime task can move on. + MetadataRuleManager databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class); + Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z"), null, null); + DataSourcesResource dataSourcesResource = + new DataSourcesResource( + inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + ); + EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) + .andReturn(ImmutableList.of(loadRule)) + .once(); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of())) + .once(); + EasyMock.replay(databaseRuleManager, segmentsMetadataManager); + + String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; + Response response = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval, 1, "v1"); + Assert.assertTrue((boolean) response.getEntity()); + + EasyMock.verify(databaseRuleManager, segmentsMetadataManager); + } + + @Test + public void testIsHandOffCompleteWithPartialLoadRuleFallThrough() + { + // A FALL_THROUGH partial rule whose matcher does not resolve on the segment (the projection it asks for is not + // present) should not halt the cascade. The next rule (drop) catches the segment, so the response is true. + MetadataRuleManager databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class); + Interval ruleInterval = Intervals.of("2013-01-01T00:00:00Z/2013-01-03T00:00:00Z"); + Rule partialRule = new IntervalPartialLoadRule( + ruleInterval, + null, + null, + new ExactProjectionPartialLoadMatcher(ImmutableList.of("user_daily")), + CannotMatchBehavior.FALL_THROUGH + ); + Rule dropRule = new IntervalDropRule(ruleInterval); + DataSourcesResource dataSourcesResource = + new DataSourcesResource( + inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + ); + + String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; + // Segment exposes projections [other_daily] which the partial rule's matcher (asking for "user_daily") cannot + // resolve, so the partial rule falls through and the drop rule catches it. + DataSegment segment = buildHandoffSegment( + TestDataSource.WIKI, Intervals.of(interval), "v1", 1, ImmutableList.of("other_daily") + ); + + EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) + .andReturn(ImmutableList.of(partialRule, dropRule)) + .once(); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of(segment))) + .once(); + EasyMock.replay(databaseRuleManager, segmentsMetadataManager); + + Response response = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval, 1, "v1"); + Assert.assertTrue((boolean) response.getEntity()); + + EasyMock.verify(databaseRuleManager, segmentsMetadataManager); + } + + @Test + public void testIsHandOffCompleteWithPartialLoadRuleMatcherResolves() + { + // A partial rule whose matcher does resolve on the segment applies (loads), so the segment is "still waiting for + // handoff", the response is false until the timeline reflects it. + MetadataRuleManager databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class); + Interval ruleInterval = Intervals.of("2013-01-01T00:00:00Z/2013-01-03T00:00:00Z"); + Rule partialRule = new IntervalPartialLoadRule( + ruleInterval, + null, + null, + new ExactProjectionPartialLoadMatcher(ImmutableList.of("user_daily")), + CannotMatchBehavior.FALL_THROUGH + ); + Rule dropRule = new IntervalDropRule(ruleInterval); + DataSourcesResource dataSourcesResource = + new DataSourcesResource( + inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + ); + + String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; + DataSegment segment = buildHandoffSegment( + TestDataSource.WIKI, Intervals.of(interval), "v1", 1, ImmutableList.of("user_daily") + ); + + EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) + .andReturn(ImmutableList.of(partialRule, dropRule)) + .once(); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of(segment))) + .once(); + EasyMock.expect(inventoryView.getTimeline(new TableDataSource(TestDataSource.WIKI))) + .andReturn(null) + .once(); + EasyMock.replay(inventoryView, databaseRuleManager, segmentsMetadataManager); + + Response response = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval, 1, "v1"); + Assert.assertFalse((boolean) response.getEntity()); + + EasyMock.verify(inventoryView, databaseRuleManager, segmentsMetadataManager); + } + + private static DataSegment buildHandoffSegment(String dataSource, Interval interval, String version, int partitionNumber) + { + return buildHandoffSegment(dataSource, interval, version, partitionNumber, null); + } + + private static DataSegment buildHandoffSegment( + String dataSource, + Interval interval, + String version, + int partitionNumber, + List projections + ) + { + return DataSegment.builder() + .dataSource(dataSource) + .interval(interval) + .version(version) + .shardSpec(new NumberedShardSpec(partitionNumber, 100)) + .projections(projections) + .size(0) + .build(); } @Test From 05256b0ee20c8c7c2b945f44ab88d7274c75298c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 27 Apr 2026 03:30:03 -0700 Subject: [PATCH 4/7] improve a bunch of stuff --- .../server/http/DataSourcesResourceTest.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 59a5c1d2f22d..18512ba93143 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -659,7 +659,13 @@ public void testIsHandOffComplete() Rule dropRule = new IntervalDropRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z")); DataSourcesResource dataSourcesResource = new DataSourcesResource( - inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + inventoryView, + segmentsMetadataManager, + databaseRuleManager, + null, + null, + null, + auditManager ); // test dropped: segment exists in metadata, drop rule applies → won't be handed off @@ -743,7 +749,13 @@ public void testIsHandOffCompleteSegmentNotInMetadataReturnsTrue() Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z"), null, null); DataSourcesResource dataSourcesResource = new DataSourcesResource( - inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + inventoryView, + segmentsMetadataManager, + databaseRuleManager, + null, + null, + null, + auditManager ); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) .andReturn(ImmutableList.of(loadRule)) @@ -777,14 +789,24 @@ public void testIsHandOffCompleteWithPartialLoadRuleFallThrough() Rule dropRule = new IntervalDropRule(ruleInterval); DataSourcesResource dataSourcesResource = new DataSourcesResource( - inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + inventoryView, + segmentsMetadataManager, + databaseRuleManager, + null, + null, + null, + auditManager ); String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; // Segment exposes projections [other_daily] which the partial rule's matcher (asking for "user_daily") cannot // resolve, so the partial rule falls through and the drop rule catches it. DataSegment segment = buildHandoffSegment( - TestDataSource.WIKI, Intervals.of(interval), "v1", 1, ImmutableList.of("other_daily") + TestDataSource.WIKI, + Intervals.of(interval), + "v1", + 1, + ImmutableList.of("other_daily") ); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) @@ -818,12 +840,22 @@ public void testIsHandOffCompleteWithPartialLoadRuleMatcherResolves() Rule dropRule = new IntervalDropRule(ruleInterval); DataSourcesResource dataSourcesResource = new DataSourcesResource( - inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + inventoryView, + segmentsMetadataManager, + databaseRuleManager, + null, + null, + null, + auditManager ); String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; DataSegment segment = buildHandoffSegment( - TestDataSource.WIKI, Intervals.of(interval), "v1", 1, ImmutableList.of("user_daily") + TestDataSource.WIKI, + Intervals.of(interval), + "v1", + 1, + ImmutableList.of("user_daily") ); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) From e04eb70a04dc6ae50a9ab4d14cc64fab2a3e10b8 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 27 Apr 2026 10:41:09 -0700 Subject: [PATCH 5/7] refresh snapshot if segment missing --- .../server/http/DataSourcesResource.java | 30 ++++++++----- .../server/http/DataSourcesResourceTest.java | 44 +++++++++++++++++-- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index a9b31fb94150..6fd2611e8aa4 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -919,19 +919,15 @@ public Response isHandOffComplete( final SegmentDescriptor descriptor = new SegmentDescriptor(theInterval, version, partitionNumber); final DateTime now = DateTimes.nowUtc(); // Look up the segment in the metadata snapshot so the rule cascade can be evaluated against the real segment - // (necessary for partial load rules whose matcher inspects the segment's projection list). If the snapshot is - // not yet available, we cannot decide, let the realtime task retry. - final DataSourcesSnapshot snapshot = segmentsMetadataManager.getRecentDataSourcesSnapshot(); - if (snapshot == null) { - return Response.ok(false).build(); + // (necessary for partial load rules whose matcher inspects the segment's projection list). If the cached + // snapshot is missing the segment, force a refresh and re-check before declaring it never-handed-off, since + // the cache may simply not have caught up to a recent publish. + final SegmentId segmentId = SegmentId.of(dataSourceName, theInterval, version, partitionNumber); + DataSegment segment = lookupSegment(segmentsMetadataManager.getRecentDataSourcesSnapshot(), segmentId); + if (segment == null) { + segment = lookupSegment(segmentsMetadataManager.forceUpdateDataSourcesSnapshot(), segmentId); } - final ImmutableDruidDataSource immutableDruidDataSource = snapshot.getDataSource(dataSourceName); - final DataSegment segment = immutableDruidDataSource == null - ? null - : immutableDruidDataSource.getSegment( - SegmentId.of(dataSourceName, theInterval, version, partitionNumber) - ); - // Segment isn't published in metadata; it will never be handed off. + // Still not in metadata after a refresh; it will never be handed off. if (segment == null) { return Response.ok(true).build(); } @@ -978,6 +974,16 @@ public Response isHandOffComplete( } } + @Nullable + private static DataSegment lookupSegment(@Nullable DataSourcesSnapshot snapshot, SegmentId segmentId) + { + if (snapshot == null) { + return null; + } + final ImmutableDruidDataSource ds = snapshot.getDataSource(segmentId.getDataSource()); + return ds == null ? null : ds.getSegment(segmentId); + } + static boolean isSegmentLoaded(Iterable servedSegments, SegmentDescriptor descriptor) { for (ImmutableSegmentLoadInfo segmentLoadInfo : servedSegments) { diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 18512ba93143..bd4b1ed4cbac 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -743,10 +743,9 @@ public List> lookupWithIncompleteP @Test public void testIsHandOffCompleteSegmentNotInMetadataReturnsTrue() { - // A segment that hasn't been published to metadata yet (or has been removed) is reported as definitively - // never-handed-off so the realtime task can move on. + // A segment that hasn't been published to metadata (and is not found after a forced refresh) is reported as + // definitively never-handed-off so the realtime task can move on. MetadataRuleManager databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class); - Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z"), null, null); DataSourcesResource dataSourcesResource = new DataSourcesResource( inventoryView, @@ -758,11 +757,14 @@ public void testIsHandOffCompleteSegmentNotInMetadataReturnsTrue() auditManager ); EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) - .andReturn(ImmutableList.of(loadRule)) + .andReturn(ImmutableList.of()) .once(); EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of())) .once(); + EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of())) + .once(); EasyMock.replay(databaseRuleManager, segmentsMetadataManager); String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; @@ -772,6 +774,40 @@ public void testIsHandOffCompleteSegmentNotInMetadataReturnsTrue() EasyMock.verify(databaseRuleManager, segmentsMetadataManager); } + @Test + public void testIsHandOffCompleteForcesMetadataRefreshOnSnapshotMiss() + { + // A cache miss on the recent snapshot triggers a forced refresh; if the segment turns up after the refresh, the + // rule cascade evaluates against it just as if it had been in the cached snapshot all along. + MetadataRuleManager databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class); + Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z"), null, null); + DataSourcesResource dataSourcesResource = + new DataSourcesResource( + inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + ); + String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; + DataSegment segment = buildHandoffSegment(TestDataSource.WIKI, Intervals.of(interval), "v1", 1); + + EasyMock.expect(databaseRuleManager.getRulesWithDefault(TestDataSource.WIKI)) + .andReturn(ImmutableList.of(loadRule)) + .once(); + EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of())) + .once(); + EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot()) + .andReturn(DataSourcesSnapshot.fromUsedSegments(ImmutableList.of(segment))) + .once(); + EasyMock.expect(inventoryView.getTimeline(new TableDataSource(TestDataSource.WIKI))) + .andReturn(null) + .once(); + EasyMock.replay(inventoryView, databaseRuleManager, segmentsMetadataManager); + + Response response = dataSourcesResource.isHandOffComplete(TestDataSource.WIKI, interval, 1, "v1"); + Assert.assertFalse((boolean) response.getEntity()); + + EasyMock.verify(inventoryView, databaseRuleManager, segmentsMetadataManager); + } + @Test public void testIsHandOffCompleteWithPartialLoadRuleFallThrough() { From 83eef8b86845896af67794729fbe8b182fbe46f8 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 27 Apr 2026 12:46:28 -0700 Subject: [PATCH 6/7] fix style --- .../druid/server/http/DataSourcesResourceTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index bd4b1ed4cbac..8e684ebcb1e0 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -783,7 +783,14 @@ public void testIsHandOffCompleteForcesMetadataRefreshOnSnapshotMiss() Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z"), null, null); DataSourcesResource dataSourcesResource = new DataSourcesResource( - inventoryView, segmentsMetadataManager, databaseRuleManager, null, null, null, auditManager + inventoryView, + segmentsMetadataManager, + databaseRuleManager, + null, + null, + null, + auditManager + auditManager ); String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; DataSegment segment = buildHandoffSegment(TestDataSource.WIKI, Intervals.of(interval), "v1", 1); From 150daa6592e8d6fcaee24218253c56e0b2ff04be Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 27 Apr 2026 13:16:02 -0700 Subject: [PATCH 7/7] oops --- .../org/apache/druid/server/http/DataSourcesResourceTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 8e684ebcb1e0..db96a55e0ad6 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -790,7 +790,6 @@ public void testIsHandOffCompleteForcesMetadataRefreshOnSnapshotMiss() null, null, auditManager - auditManager ); String interval = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z"; DataSegment segment = buildHandoffSegment(TestDataSource.WIKI, Intervals.of(interval), "v1", 1);