diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java index b61006f6de9d..73b39275f226 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSource.java @@ -34,87 +34,74 @@ public ImmutableDataSource(ColumnMetadata columnMetadata, ColumnIndexContainer c super(new ImmutableDataSourceMetadata(columnMetadata), columnIndexContainer); } + /// Exposes the segment's [ColumnMetadata] through the [DataSourceMetadata] view by delegating every accessor. + /// Holding a single reference instead of copying the ten fields the view exposes keeps this object at one + /// reference per column, which matters for wide segments where every loaded column retains one. The delegation + /// is observationally equivalent to a snapshot because every [ColumnMetadata] handed to an immutable data source + /// is itself immutable once built. private static class ImmutableDataSourceMetadata implements DataSourceMetadata { - final FieldSpec _fieldSpec; - final boolean _sorted; - final int _numDocs; - final int _numValues; - final int _maxNumValuesPerMVEntry; - final int _cardinality; - final Comparable _minValue; - final Comparable _maxValue; - final PartitionFunction _partitionFunction; - final Set _partitions; + final ColumnMetadata _columnMetadata; ImmutableDataSourceMetadata(ColumnMetadata columnMetadata) { - _fieldSpec = columnMetadata.getFieldSpec(); - _sorted = columnMetadata.isSorted(); - _numDocs = columnMetadata.getTotalDocs(); - _numValues = columnMetadata.getTotalNumberOfEntries(); - if (_fieldSpec.isSingleValueField()) { - _maxNumValuesPerMVEntry = -1; - } else { - _maxNumValuesPerMVEntry = columnMetadata.getMaxNumberOfMultiValues(); - } - _minValue = columnMetadata.getMinValue(); - _maxValue = columnMetadata.getMaxValue(); - _partitionFunction = columnMetadata.getPartitionFunction(); - _partitions = columnMetadata.getPartitions(); - _cardinality = columnMetadata.getCardinality(); + _columnMetadata = columnMetadata; } @Override public FieldSpec getFieldSpec() { - return _fieldSpec; + return _columnMetadata.getFieldSpec(); } @Override public boolean isSorted() { - return _sorted; + return _columnMetadata.isSorted(); } @Override public int getNumDocs() { - return _numDocs; + return _columnMetadata.getTotalDocs(); } @Override public int getNumValues() { - return _numValues; + return _columnMetadata.getTotalNumberOfEntries(); } @Override public int getMaxNumValuesPerMVEntry() { - return _maxNumValuesPerMVEntry; + // DataSourceMetadata reports -1 for single-value columns, whereas ColumnMetadata reports 0 + return _columnMetadata.getFieldSpec().isSingleValueField() ? -1 : _columnMetadata.getMaxNumberOfMultiValues(); } @Nullable @Override public Comparable getMinValue() { - return _minValue; + return _columnMetadata.getMinValue(); } @Nullable @Override public Comparable getMaxValue() { - return _maxValue; + return _columnMetadata.getMaxValue(); } @Nullable @Override public PartitionFunction getPartitionFunction() { - return _partitionFunction; + return _columnMetadata.getPartitionFunction(); } @Nullable @Override public Set getPartitions() { - return _partitions; + return _columnMetadata.getPartitions(); } @Override public int getCardinality() { - return _cardinality; + return _columnMetadata.getCardinality(); } + + // getMaxRowLengthInBytes() is deliberately not delegated: DataSourceMetadata defines it as -1 for immutable + // columns, while ColumnMetadata computes the serialized row length. } } diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSource.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSource.java index 93c32659ec43..64d7e8b6c75c 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSource.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSource.java @@ -62,36 +62,19 @@ public ColumnIndexContainer getIndexContainer(String key) { return null; } + /// Exposes the MAP column's [ColumnMetadata] through the [DataSourceMetadata] view by delegating every accessor + /// through a single reference instead of copying the fields (see the equivalent adapter in `ImmutableDataSource`). + /// A MAP column is never reported as sorted and does not support the max row length. private static class ImmutableMapDataSourceMetadata implements DataSourceMetadata { - final FieldSpec _fieldSpec; - final int _numDocs; - final int _numValues; - final int _maxNumValuesPerMVEntry; - final int _cardinality; - final PartitionFunction _partitionFunction; - final Set _partitions; - final Comparable _minValue; - final Comparable _maxValue; + final ColumnMetadata _columnMetadata; ImmutableMapDataSourceMetadata(ColumnMetadata columnMetadata) { - _fieldSpec = columnMetadata.getFieldSpec(); - _numDocs = columnMetadata.getTotalDocs(); - _numValues = columnMetadata.getTotalNumberOfEntries(); - if (_fieldSpec.isSingleValueField()) { - _maxNumValuesPerMVEntry = -1; - } else { - _maxNumValuesPerMVEntry = columnMetadata.getMaxNumberOfMultiValues(); - } - _minValue = columnMetadata.getMinValue(); - _maxValue = columnMetadata.getMaxValue(); - _partitionFunction = columnMetadata.getPartitionFunction(); - _partitions = columnMetadata.getPartitions(); - _cardinality = columnMetadata.getCardinality(); + _columnMetadata = columnMetadata; } @Override public FieldSpec getFieldSpec() { - return _fieldSpec; + return _columnMetadata.getFieldSpec(); } @Override @@ -101,45 +84,47 @@ public boolean isSorted() { @Override public int getNumDocs() { - return _numDocs; + return _columnMetadata.getTotalDocs(); } @Override public int getNumValues() { - return _numValues; + return _columnMetadata.getTotalNumberOfEntries(); } @Override public int getMaxNumValuesPerMVEntry() { - return _maxNumValuesPerMVEntry; + // DataSourceMetadata reports -1 for single-value columns, whereas ColumnMetadata reports 0 + return _columnMetadata.getFieldSpec().isSingleValueField() ? -1 : _columnMetadata.getMaxNumberOfMultiValues(); } @Nullable @Override public Comparable getMinValue() { - return _minValue; + return _columnMetadata.getMinValue(); } + @Nullable @Override public Comparable getMaxValue() { - return _maxValue; + return _columnMetadata.getMaxValue(); } @Nullable @Override public PartitionFunction getPartitionFunction() { - return _partitionFunction; + return _columnMetadata.getPartitionFunction(); } @Nullable @Override public Set getPartitions() { - return _partitions; + return _columnMetadata.getPartitions(); } @Override public int getCardinality() { - return _cardinality; + return _columnMetadata.getCardinality(); } @Override diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSourceTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSourceTest.java new file mode 100644 index 000000000000..0c58e20b9a0c --- /dev/null +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/datasource/ImmutableDataSourceTest.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.segment.local.segment.index.datasource; + +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import java.util.Set; +import org.apache.pinot.segment.local.segment.index.map.SimpleColumnMetadata; +import org.apache.pinot.segment.local.segment.virtualcolumn.DocIdVirtualColumnProvider; +import org.apache.pinot.segment.local.segment.virtualcolumn.VirtualColumnContext; +import org.apache.pinot.segment.spi.ColumnMetadata; +import org.apache.pinot.segment.spi.datasource.DataSource; +import org.apache.pinot.segment.spi.datasource.DataSourceMetadata; +import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer; +import org.apache.pinot.segment.spi.index.metadata.ColumnMetadataImpl; +import org.apache.pinot.segment.spi.partition.PartitionFunction; +import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory; +import org.apache.pinot.spi.data.DimensionFieldSpec; +import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.spi.data.FieldSpec.DataType; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; + + +/// Covers the [DataSourceMetadata] view an [ImmutableDataSource] exposes over the segment's [ColumnMetadata]. The +/// view delegates to the column metadata instead of copying it, so every accessor must report the value (and, for +/// reference types, the instance) the column metadata holds, while keeping the two contracts that differ between the +/// interfaces: `getMaxNumValuesPerMVEntry()` is `-1` for single-value columns and `getMaxRowLengthInBytes()` stays +/// at the [DataSourceMetadata] default of `-1`. +public class ImmutableDataSourceTest { + private static final int NUM_DOCS = 1000; + + @Test + public void testSingleValueColumn() { + FieldSpec fieldSpec = new DimensionFieldSpec("sv", DataType.INT, true); + PartitionFunction partitionFunction = PartitionFunctionFactory.getPartitionFunction("Modulo", 4, null); + Set partitions = new IntOpenHashSet(new int[]{1, 3}); + ColumnMetadata columnMetadata = new ColumnMetadataImpl.Builder().setFieldSpec(fieldSpec) + .setTotalDocs(NUM_DOCS) + .setCardinality(37) + .setHasDictionary(true) + .setSorted(true) + .setMinValue(-5) + .setMaxValue(123456) + .setPartitionFunction(partitionFunction) + .setPartitions(partitions) + .build(); + + DataSourceMetadata metadata = dataSourceMetadata(columnMetadata); + assertSame(metadata.getFieldSpec(), fieldSpec); + assertEquals(metadata.getDataType(), DataType.INT); + assertTrue(metadata.isSingleValue()); + assertTrue(metadata.isSorted()); + assertEquals(metadata.getNumDocs(), NUM_DOCS); + assertEquals(metadata.getNumValues(), NUM_DOCS); + assertEquals(metadata.getNumValues(), columnMetadata.getTotalNumberOfEntries()); + assertEquals(metadata.getCardinality(), 37); + assertSame(metadata.getMinValue(), columnMetadata.getMinValue()); + assertEquals(metadata.getMinValue(), -5); + assertSame(metadata.getMaxValue(), columnMetadata.getMaxValue()); + assertEquals(metadata.getMaxValue(), 123456); + assertSame(metadata.getPartitionFunction(), partitionFunction); + assertSame(metadata.getPartitions(), partitions); + + // Single-value columns report -1 through the data source view although the column metadata canonicalises to 0 + assertEquals(columnMetadata.getMaxNumberOfMultiValues(), 0); + assertEquals(metadata.getMaxNumValuesPerMVEntry(), -1); + + // The row length is not delegated: the column metadata computes it, the data source view keeps its default + assertEquals(columnMetadata.getMaxRowLengthInBytes(), Integer.BYTES); + assertEquals(metadata.getMaxRowLengthInBytes(), -1); + } + + @Test + public void testMultiValueColumn() { + FieldSpec fieldSpec = new DimensionFieldSpec("mv", DataType.STRING, false); + ColumnMetadata columnMetadata = new ColumnMetadataImpl.Builder().setFieldSpec(fieldSpec) + .setTotalDocs(NUM_DOCS) + .setCardinality(11) + .setHasDictionary(true) + .setSorted(false) + .setMinValue("apple") + .setMaxValue("zebra") + .setTotalNumberOfEntries(4 * NUM_DOCS) + .setMaxNumberOfMultiValues(7) + .setMaxRowLengthInBytes(42) + .setLengthOfLongestElement(6) + .build(); + + DataSourceMetadata metadata = dataSourceMetadata(columnMetadata); + assertSame(metadata.getFieldSpec(), fieldSpec); + assertEquals(metadata.getDataType(), DataType.STRING); + assertFalse(metadata.isSingleValue()); + assertFalse(metadata.isSorted()); + assertEquals(metadata.getNumDocs(), NUM_DOCS); + assertEquals(metadata.getNumValues(), 4 * NUM_DOCS); + assertEquals(metadata.getMaxNumValuesPerMVEntry(), 7); + assertEquals(metadata.getMaxNumValuesPerMVEntry(), columnMetadata.getMaxNumberOfMultiValues()); + assertEquals(metadata.getCardinality(), 11); + assertSame(metadata.getMinValue(), columnMetadata.getMinValue()); + assertEquals(metadata.getMinValue(), "apple"); + assertSame(metadata.getMaxValue(), columnMetadata.getMaxValue()); + assertEquals(metadata.getMaxValue(), "zebra"); + assertNull(metadata.getPartitionFunction()); + assertNull(metadata.getPartitions()); + + assertEquals(columnMetadata.getMaxRowLengthInBytes(), 42); + assertEquals(metadata.getMaxRowLengthInBytes(), -1); + } + + @Test + public void testVirtualColumn() { + FieldSpec fieldSpec = new DimensionFieldSpec("$docId", DataType.INT, true); + DataSource dataSource = + new DocIdVirtualColumnProvider().buildDataSource(new VirtualColumnContext(fieldSpec, NUM_DOCS)); + assertTrue(dataSource instanceof ImmutableDataSource); + + DataSourceMetadata metadata = dataSource.getDataSourceMetadata(); + assertSame(metadata.getFieldSpec(), fieldSpec); + assertTrue(metadata.isSingleValue()); + assertTrue(metadata.isSorted()); + assertEquals(metadata.getNumDocs(), NUM_DOCS); + assertEquals(metadata.getNumValues(), NUM_DOCS); + assertEquals(metadata.getMaxNumValuesPerMVEntry(), -1); + assertEquals(metadata.getCardinality(), NUM_DOCS); + assertNull(metadata.getMinValue()); + assertNull(metadata.getMaxValue()); + assertNull(metadata.getPartitionFunction()); + assertNull(metadata.getPartitions()); + assertEquals(metadata.getMaxRowLengthInBytes(), -1); + } + + /// A MAP key's data source is built over a [SimpleColumnMetadata] whose stats are all unavailable; the view must + /// pass them through unchanged rather than re-deriving them. + @Test + public void testUnavailableStatsPassThrough() { + FieldSpec fieldSpec = new DimensionFieldSpec("key", DataType.LONG, false); + ColumnMetadata columnMetadata = new SimpleColumnMetadata(fieldSpec, NUM_DOCS); + + DataSourceMetadata metadata = dataSourceMetadata(columnMetadata); + assertSame(metadata.getFieldSpec(), fieldSpec); + assertFalse(metadata.isSorted()); + assertEquals(metadata.getNumDocs(), NUM_DOCS); + assertEquals(metadata.getNumValues(), ColumnMetadata.UNAVAILABLE); + assertEquals(metadata.getMaxNumValuesPerMVEntry(), ColumnMetadata.UNAVAILABLE); + assertEquals(metadata.getCardinality(), ColumnMetadata.UNAVAILABLE); + assertNull(metadata.getMinValue()); + assertNull(metadata.getMaxValue()); + assertNull(metadata.getPartitionFunction()); + assertNull(metadata.getPartitions()); + assertEquals(metadata.getMaxRowLengthInBytes(), -1); + } + + private static DataSourceMetadata dataSourceMetadata(ColumnMetadata columnMetadata) { + return new ImmutableDataSource(columnMetadata, ColumnIndexContainer.Empty.INSTANCE).getDataSourceMetadata(); + } +} diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSourceTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSourceTest.java new file mode 100644 index 000000000000..8b4f6a176741 --- /dev/null +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/map/ImmutableMapDataSourceTest.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.segment.local.segment.index.map; + +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import java.util.Map; +import java.util.Set; +import org.apache.pinot.segment.spi.ColumnMetadata; +import org.apache.pinot.segment.spi.datasource.DataSourceMetadata; +import org.apache.pinot.segment.spi.index.IndexReader; +import org.apache.pinot.segment.spi.index.IndexType; +import org.apache.pinot.segment.spi.index.StandardIndexes; +import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer; +import org.apache.pinot.segment.spi.index.metadata.ColumnMetadataImpl; +import org.apache.pinot.segment.spi.index.reader.MapIndexReader; +import org.apache.pinot.segment.spi.partition.PartitionFunction; +import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory; +import org.apache.pinot.spi.data.ComplexFieldSpec; +import org.apache.pinot.spi.data.DimensionFieldSpec; +import org.apache.pinot.spi.data.FieldSpec.DataType; +import org.testng.annotations.Test; + +import static org.apache.pinot.spi.data.ComplexFieldSpec.KEY_FIELD; +import static org.apache.pinot.spi.data.ComplexFieldSpec.VALUE_FIELD; +import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + + +/// Covers the [DataSourceMetadata] view an [ImmutableMapDataSource] exposes over the MAP column's [ColumnMetadata]. +/// The view delegates to the column metadata instead of copying it, except for the two MAP-specific contracts: the +/// column is never reported as sorted, and `getMaxRowLengthInBytes()` is unsupported. +public class ImmutableMapDataSourceTest { + private static final int NUM_DOCS = 1000; + + @Test + public void testDelegatesToColumnMetadata() { + ComplexFieldSpec fieldSpec = new ComplexFieldSpec("m", DataType.MAP, true, Map.of( + KEY_FIELD, new DimensionFieldSpec(KEY_FIELD, DataType.STRING, true), + VALUE_FIELD, new DimensionFieldSpec(VALUE_FIELD, DataType.LONG, true) + )); + PartitionFunction partitionFunction = PartitionFunctionFactory.getPartitionFunction("Modulo", 2, null); + Set partitions = new IntOpenHashSet(new int[]{0}); + // Sorted is set on purpose: the MAP view must ignore it + ColumnMetadata columnMetadata = new ColumnMetadataImpl.Builder().setFieldSpec(fieldSpec) + .setTotalDocs(NUM_DOCS) + .setCardinality(5) + .setSorted(true) + .setMinValue("a") + .setMaxValue("z") + .setPartitionFunction(partitionFunction) + .setPartitions(partitions) + .build(); + Map indexes = Map.of(StandardIndexes.forward(), mock(MapIndexReader.class)); + ColumnIndexContainer indexContainer = new ColumnIndexContainer.FromMap(indexes); + + DataSourceMetadata metadata = new ImmutableMapDataSource(columnMetadata, indexContainer).getDataSourceMetadata(); + assertSame(metadata.getFieldSpec(), fieldSpec); + assertEquals(metadata.getDataType(), DataType.MAP); + assertTrue(metadata.isSingleValue()); + assertTrue(columnMetadata.isSorted()); + assertFalse(metadata.isSorted()); + assertEquals(metadata.getNumDocs(), NUM_DOCS); + assertEquals(metadata.getNumValues(), NUM_DOCS); + assertEquals(metadata.getMaxNumValuesPerMVEntry(), -1); + assertEquals(metadata.getCardinality(), 5); + assertSame(metadata.getMinValue(), columnMetadata.getMinValue()); + assertEquals(metadata.getMinValue(), "a"); + assertSame(metadata.getMaxValue(), columnMetadata.getMaxValue()); + assertEquals(metadata.getMaxValue(), "z"); + assertSame(metadata.getPartitionFunction(), partitionFunction); + assertSame(metadata.getPartitions(), partitions); + assertThrows(UnsupportedOperationException.class, metadata::getMaxRowLengthInBytes); + } +}