You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2026/05/12 22:45:59 UTC
(pinot) branch master updated: Introduce ColumnShape; align IndexCreationContext APIs (#18470)
This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new aec54289c51 Introduce ColumnShape; align IndexCreationContext APIs (#18470)
aec54289c51 is described below
commit aec54289c51d9188cd2e5d5fe3d34550cf20503d
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Tue May 12 15:45:53 2026 -0700
Introduce ColumnShape; align IndexCreationContext APIs (#18470)
- Extract a common ColumnShape interface from ColumnStatistics, ColumnMetadata,
and IndexCreationContext so column-shape attributes (cardinality, element
lengths, isAscii, maxRowLengthInBytes, partition info, etc.) flow through a
single accessor surface. Add EmptyColumnShape / EmptyColumnMetadata for
zero-row segments.
- Rework IndexCreationContext.Builder to take (File indexDir, TableConfig
tableConfig, ColumnStatistics | ColumnMetadata) at construction. tableName
and continueOnError are derived from the TableConfig; a String/ColumnShape
fallback constructor exists for callers without a TableConfig. Remove dead
forwardIndexDisabled plumbing.
- ColumnMetadataImpl gains a _maxRowLengthInBytes field derived in
Builder.build from canonicalized shape fields (correct for SV and
fixed-width MV, plus uniform-length var-width MV; UNAVAILABLE for
varying-length var-width MV). extractFieldSpec / extractPartitionFunction /
extractPartitions are exposed as public static helpers.
- SegmentMetadataImpl branches on _totalDocs == 0 and uses
EmptyColumnMetadata.fromPropertiesConfiguration for empty segments,
skipping V3 index-map loading.
- ForwardIndexHandler.rewriteDictToRawForwardIndex leans on the metadata
getMaxRowLengthInBytes and only scans when UNAVAILABLE.
- Fix a missing _totalDocs++ in NoDictColumnStatisticsCollector that was
causing BufferOverflowException at segment build.
- Hoist BaseSegmentCreator.seal's updatePostSegmentCreationIndexes call into
the existing `_totalDocs > 0` guard so empty segments skip post-creation
index handlers (matching SegmentPreProcessor.process's totalDocs==0
short-circuit on the reload path). Index handlers may now assume the
segment has at least one document.
Co-authored-by: Claude Opus 4.7 (1M context) <no...@anthropic.com>
---
.../core/common/RowBasedBlockValueFetcher.java | 4 +-
.../predicate/RangePredicateEvaluatorFactory.java | 5 +-
.../raw/RawMultiColumnDistinctExecutor.java | 12 +-
.../BaseSegmentProcessorFrameworkTest.java | 4 +-
.../tests/CommitTimeCompactionIntegrationTest.java | 1 -
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 24 +-
.../converter/stats/CompactedColumnStatistics.java | 25 +-
.../stats/CompactedNoDictColumnStatistics.java | 47 +-
.../converter/stats/MutableColumnStatistics.java | 14 +-
.../stats/MutableNoDictColumnStatistics.java | 24 +-
.../segment/creator/impl/BaseSegmentCreator.java | 27 +-
.../impl/inv/text/LuceneFSTIndexCreator.java | 7 -
.../impl/inv/text/LuceneIFSTIndexCreator.java | 7 -
.../stats/AbstractColumnStatisticsCollector.java | 14 +-
.../BigDecimalColumnPreIndexStatsCollector.java | 1 +
.../stats/BytesColumnPreIndexStatsCollector.java | 1 +
.../stats/DoubleColumnPreIndexStatsCollector.java | 2 +
.../creator/impl/stats/EmptyColumnStatistics.java | 79 +---
.../stats/FloatColumnPreIndexStatsCollector.java | 2 +
.../stats/IntColumnPreIndexStatsCollector.java | 2 +
.../stats/LongColumnPreIndexStatsCollector.java | 2 +
.../stats/MapColumnPreIndexStatsCollector.java | 1 +
.../stats/NoDictColumnStatisticsCollector.java | 20 +-
.../stats/StringColumnPreIndexStatsCollector.java | 1 +
.../index/forward/ForwardIndexCreatorFactory.java | 4 +-
.../local/segment/index/fst/FstIndexType.java | 11 +-
.../local/segment/index/fst/IFSTIndexType.java | 11 +-
.../segment/index/loader/BaseIndexHandler.java | 6 +-
.../segment/index/loader/ForwardIndexHandler.java | 43 +-
...IndexAndDictionaryBasedForwardIndexCreator.java | 138 +++---
.../loader/bloomfilter/BloomFilterHandler.java | 16 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 21 +-
.../defaultcolumn/DefaultColumnStatistics.java | 18 +-
.../loader/invertedindex/FSTIndexHandler.java | 8 +-
.../index/loader/invertedindex/H3IndexHandler.java | 16 +-
.../loader/invertedindex/IFSTIndexHandler.java | 3 +-
.../loader/invertedindex/InvertedIndexHandler.java | 8 +-
.../loader/invertedindex/JsonIndexHandler.java | 16 +-
.../loader/invertedindex/RangeIndexHandler.java | 8 +-
.../loader/invertedindex/TextIndexHandler.java | 8 +-
.../loader/invertedindex/VectorIndexHandler.java | 18 +-
.../segment/index/map/SimpleColumnMetadata.java | 24 +-
...apPartitionDedupMetadataManagerWithTTLTest.java | 3 +-
.../local/dedup/DedupWithTimestampColumnTest.java | 3 +-
.../local/segment/creator/DictionariesTest.java | 4 +-
.../AbstractColumnStatisticsCollectorTest.java | 2 +-
.../impl/stats/EmptyColumnStatisticsTest.java | 2 +-
.../forward/ForwardIndexCreatorFactoryTest.java | 25 +-
.../index/loader/ForwardIndexHandlerTest.java | 3 +-
.../index/loader/SegmentPreProcessorTest.java | 4 +-
.../defaultcolumn/DefaultColumnStatisticsTest.java | 8 +-
...rrentMapPartitionUpsertMetadataManagerTest.java | 2 +-
.../apache/pinot/segment/spi/ColumnMetadata.java | 71 +--
.../ColumnStatistics.java => ColumnShape.java} | 118 +++--
.../apache/pinot/segment/spi/EmptyColumnShape.java | 45 +-
.../segment/spi/creator/ColumnStatistics.java | 102 +----
.../segment/spi/creator/IndexCreationContext.java | 510 ++++++++++-----------
.../spi/index/metadata/ColumnMetadataImpl.java | 146 ++++--
.../spi/index/metadata/EmptyColumnMetadata.java | 101 ++++
.../spi/index/metadata/SegmentMetadataImpl.java | 62 +--
60 files changed, 861 insertions(+), 1053 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java b/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java
index d2620973f87..e914332fa65 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/RowBasedBlockValueFetcher.java
@@ -74,7 +74,7 @@ public class RowBasedBlockValueFetcher {
case UNKNOWN:
return new UnknownValueFetcher();
default:
- throw new IllegalStateException("Unsupported value type: " + storedType + " for single-value column");
+ throw new IllegalStateException("Unsupported SV stored type: " + storedType);
}
} else {
switch (storedType) {
@@ -93,7 +93,7 @@ public class RowBasedBlockValueFetcher {
case BYTES:
return new BytesMultiValueFetcher(blockValSet.getBytesValuesMV());
default:
- throw new IllegalStateException("Unsupported value type: " + storedType + " for multi-value column");
+ throw new IllegalStateException("Unsupported MV stored type: " + storedType);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
index 6b7c12be359..883507e0f9a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
@@ -303,7 +303,8 @@ public class RangePredicateEvaluatorFactory {
if (_dictIdSetBased) {
return _matchingDictIdSet.contains(dictId);
} else {
- switch (_dictionary.getValueType()) {
+ DataType storedType = _dictionary.getValueType();
+ switch (storedType) {
case INT:
return _rawValueBasedEvaluator.applySV(_dictionary.getIntValue(dictId));
case LONG:
@@ -319,7 +320,7 @@ public class RangePredicateEvaluatorFactory {
case BYTES:
return _rawValueBasedEvaluator.applySV(_dictionary.getBytesValue(dictId));
default:
- throw new IllegalStateException("Unsupported value type: " + _dictionary.getValueType());
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/RawMultiColumnDistinctExecutor.java b/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/RawMultiColumnDistinctExecutor.java
index 76c7cb4fd5b..126ebcbcc21 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/RawMultiColumnDistinctExecutor.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/raw/RawMultiColumnDistinctExecutor.java
@@ -142,8 +142,8 @@ public class RawMultiColumnDistinctExecutor implements DistinctExecutor {
private Object[] getSVValues(BlockValSet blockValueSet, int numDocs) {
Object[] values;
- DataType valueType = blockValueSet.getValueType();
- switch (valueType.getStoredType()) {
+ DataType storedType = blockValueSet.getValueType().getStoredType();
+ switch (storedType) {
case INT:
int[] intValues = blockValueSet.getIntValuesSV();
values = new Object[numDocs];
@@ -188,7 +188,7 @@ public class RawMultiColumnDistinctExecutor implements DistinctExecutor {
}
break;
default:
- throw new IllegalStateException("Unsupported value type: " + valueType + " for single-value column");
+ throw new IllegalStateException("Unsupported SV stored type: " + storedType);
}
if (_nullHandlingEnabled) {
RoaringBitmap nullBitmap = blockValueSet.getNullBitmap();
@@ -202,8 +202,8 @@ public class RawMultiColumnDistinctExecutor implements DistinctExecutor {
// TODO(https://github.com/apache/pinot/issues/10882): support NULL for multi-value
private Object[][] getMVValues(BlockValSet blockValueSet, int numDocs) {
Object[][] values;
- DataType valueType = blockValueSet.getValueType();
- switch (valueType.getStoredType()) {
+ DataType storedType = blockValueSet.getValueType().getStoredType();
+ switch (storedType) {
case INT:
int[][] intValues = blockValueSet.getIntValuesMV();
values = new Object[numDocs][];
@@ -248,7 +248,7 @@ public class RawMultiColumnDistinctExecutor implements DistinctExecutor {
}
break;
default:
- throw new IllegalStateException("Unsupported value type: " + valueType + " for multi-value column");
+ throw new IllegalStateException("Unsupported MV stored type: " + storedType);
}
return values;
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/BaseSegmentProcessorFrameworkTest.java b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/BaseSegmentProcessorFrameworkTest.java
index 9092c5eb92b..321d188a9ce 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/BaseSegmentProcessorFrameworkTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/BaseSegmentProcessorFrameworkTest.java
@@ -823,8 +823,8 @@ public abstract class BaseSegmentProcessorFrameworkTest {
SegmentMetadata metadata = new SegmentMetadataImpl(segmentDir);
ColumnMetadata timeMetadata = metadata.getColumnMetadataFor("time");
assertNotNull(timeMetadata, "Time column metadata should not be null");
- long startTime = (long) timeMetadata.getMinValue();
- long endTime = (long) timeMetadata.getMaxValue();
+ long startTime = (Long) timeMetadata.getMinValue();
+ long endTime = (Long) timeMetadata.getMaxValue();
// Verify time range is exactly 1 day
assertTrue(endTime - startTime <= 86400000, "Segment time range should be 1 day");
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CommitTimeCompactionIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CommitTimeCompactionIntegrationTest.java
index fb41b73f56d..b5c5c91c91e 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CommitTimeCompactionIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CommitTimeCompactionIntegrationTest.java
@@ -65,7 +65,6 @@ public class CommitTimeCompactionIntegrationTest extends BaseClusterIntegrationT
public static final String UPSERT_SCHEMA_FILE_NAME = "upsert_table_test.schema";
private static final List<String> COLUMNS_TO_COMPARE =
List.of("name", "game", "score", "timestampInEpoch", "deleted");
- private String _kafkaTopicName;
@BeforeClass
public void setUp()
diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtimetoofflinesegments/RealtimeToOfflineSegmentsTaskExecutorTest.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtimetoofflinesegments/RealtimeToOfflineSegmentsTaskExecutorTest.java
index d17b6c540dc..2f40da4782a 100644
--- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtimetoofflinesegments/RealtimeToOfflineSegmentsTaskExecutorTest.java
+++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/realtimetoofflinesegments/RealtimeToOfflineSegmentsTaskExecutorTest.java
@@ -242,8 +242,8 @@ public class RealtimeToOfflineSegmentsTaskExecutorTest {
assertEquals(segmentMetadata.getTotalDocs(), 30);
ColumnMetadata columnMetadataForT = segmentMetadata.getColumnMetadataFor(T);
assertEquals(columnMetadataForT.getCardinality(), 3);
- assertTrue((long) columnMetadataForT.getMinValue() >= 1600473600000L);
- assertTrue((long) columnMetadataForT.getMaxValue() < 1600560000000L);
+ assertTrue((Long) columnMetadataForT.getMinValue() >= 1600473600000L);
+ assertTrue((Long) columnMetadataForT.getMaxValue() < 1600560000000L);
}
@Test
@@ -271,8 +271,8 @@ public class RealtimeToOfflineSegmentsTaskExecutorTest {
assertEquals(segmentMetadata.getTotalDocs(), 3);
ColumnMetadata columnMetadataForT = segmentMetadata.getColumnMetadataFor(T);
assertEquals(columnMetadataForT.getCardinality(), 3);
- assertTrue((long) columnMetadataForT.getMinValue() >= 1600473600000L);
- assertTrue((long) columnMetadataForT.getMaxValue() < 1600560000000L);
+ assertTrue((Long) columnMetadataForT.getMinValue() >= 1600473600000L);
+ assertTrue((Long) columnMetadataForT.getMaxValue() < 1600560000000L);
}
@Test
@@ -301,7 +301,7 @@ public class RealtimeToOfflineSegmentsTaskExecutorTest {
assertEquals(segmentMetadata.getTotalDocs(), 2);
ColumnMetadata columnMetadataForT = segmentMetadata.getColumnMetadataFor(T);
assertEquals(columnMetadataForT.getCardinality(), 1);
- assertEquals((long) columnMetadataForT.getMinValue(), 1600473600000L);
+ assertEquals(columnMetadataForT.getMinValue(), 1600473600000L);
}
@Test
@@ -332,11 +332,11 @@ public class RealtimeToOfflineSegmentsTaskExecutorTest {
assertEquals(segmentMetadata.getTotalDocs(), 2);
ColumnMetadata columnMetadataForT = segmentMetadata.getColumnMetadataFor(T);
assertEquals(columnMetadataForT.getCardinality(), 1);
- assertEquals((long) columnMetadataForT.getMinValue(), 1600473600000L);
+ assertEquals(columnMetadataForT.getMinValue(), 1600473600000L);
ColumnMetadata columnMetadataForM1 = segmentMetadata.getColumnMetadataFor(M1);
assertEquals(columnMetadataForM1.getCardinality(), 2);
- assertEquals((int) columnMetadataForM1.getMinValue(), 1);
- assertEquals((int) columnMetadataForM1.getMaxValue(), 3);
+ assertEquals(columnMetadataForM1.getMinValue(), 1);
+ assertEquals(columnMetadataForM1.getMaxValue(), 3);
}
@Test
@@ -426,8 +426,8 @@ public class RealtimeToOfflineSegmentsTaskExecutorTest {
assertEquals(segmentMetadata.getTotalDocs(), 3);
ColumnMetadata columnMetadataForT = segmentMetadata.getColumnMetadataFor(T_TRX);
assertEquals(columnMetadataForT.getCardinality(), 3);
- assertTrue((int) columnMetadataForT.getMinValue() >= 444576);
- assertTrue((int) columnMetadataForT.getMaxValue() < 444600);
+ assertTrue((Integer) columnMetadataForT.getMinValue() >= 444576);
+ assertTrue((Integer) columnMetadataForT.getMaxValue() < 444600);
}
@Test
@@ -456,8 +456,8 @@ public class RealtimeToOfflineSegmentsTaskExecutorTest {
assertEquals(segmentMetadata.getTotalDocs(), 3);
ColumnMetadata columnMetadataForT = segmentMetadata.getColumnMetadataFor(T_TRX);
assertEquals(columnMetadataForT.getCardinality(), 3);
- assertTrue((int) columnMetadataForT.getMinValue() >= 2020091900);
- assertTrue((int) columnMetadataForT.getMaxValue() < 2020092000);
+ assertTrue((Integer) columnMetadataForT.getMinValue() >= 2020091900);
+ assertTrue((Integer) columnMetadataForT.getMaxValue() < 2020092000);
}
@AfterClass
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
index 42f0307eb64..95ad415eb04 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedColumnStatistics.java
@@ -38,6 +38,7 @@ import org.roaringbitmap.RoaringBitmap;
/// When commit-time compaction is enabled, only valid (non-deleted) documents should be considered.
@SuppressWarnings("rawtypes")
public class CompactedColumnStatistics extends MutableColumnStatistics {
+ private final int _totalDocs;
private final Comparable _minValue;
private final Comparable _maxValue;
private final Object _uniqueValues;
@@ -55,9 +56,10 @@ public class CompactedColumnStatistics extends MutableColumnStatistics {
super(dataSource, sortedDocIds, isSortedColumn);
Preconditions.checkState(!validDocIds.isEmpty(), "Use EmptyColumnStatistics for empty column: %s",
_fieldSpec.getName());
- DataType valueType = getValueType();
+ _totalDocs = validDocIds.getCardinality();
+ DataType storedType = getStoredType();
boolean isSingleValue = isSingleValue();
- boolean isFixedWidth = valueType.isFixedWidth();
+ boolean isFixedWidth = storedType.isFixedWidth();
boolean isVarBytesMV = !isSingleValue && !isFixedWidth;
MutableForwardIndex forwardIndex = (MutableForwardIndex) dataSource.getForwardIndex();
Preconditions.checkState(forwardIndex != null, "Failed to find forward index for column: %s", _fieldSpec.getName());
@@ -135,10 +137,10 @@ public class CompactedColumnStatistics extends MutableColumnStatistics {
Comparable minValue = null;
Comparable maxValue = null;
// For fixed-width types element length is constant; for variable-width it is tracked per entry
- int minElementLength = isFixedWidth ? valueType.size() : Integer.MAX_VALUE;
- int maxElementLength = isFixedWidth ? valueType.size() : 0;
+ int minElementLength = isFixedWidth ? storedType.size() : Integer.MAX_VALUE;
+ int maxElementLength = isFixedWidth ? storedType.size() : 0;
boolean isAscii = false;
- switch (valueType) {
+ switch (storedType) {
case INT: {
int[] values = new int[_cardinality];
if (_cardinality > 0) {
@@ -257,7 +259,7 @@ public class CompactedColumnStatistics extends MutableColumnStatistics {
break;
}
default:
- throw new IllegalStateException("Unsupported value type: " + valueType);
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
_minValue = minValue;
_maxValue = maxValue;
@@ -269,7 +271,7 @@ public class CompactedColumnStatistics extends MutableColumnStatistics {
} else if (isVarBytesMV) {
_maxRowLength = maxRowLength;
} else {
- _maxRowLength = maxMultiValues * valueType.size();
+ _maxRowLength = maxMultiValues * storedType.size();
}
if (_isSortedColumn) {
_isSorted = true;
@@ -281,12 +283,17 @@ public class CompactedColumnStatistics extends MutableColumnStatistics {
}
@Override
- public Comparable getMinValue() {
+ public int getTotalDocs() {
+ return _totalDocs;
+ }
+
+ @Override
+ public Comparable<?> getMinValue() {
return _minValue;
}
@Override
- public Comparable getMaxValue() {
+ public Comparable<?> getMaxValue() {
return _maxValue;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
index 836adad5651..c93205b3768 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/CompactedNoDictColumnStatistics.java
@@ -34,6 +34,7 @@ import org.roaringbitmap.RoaringBitmap;
/// When commit-time compaction is enabled, only valid (non-deleted) documents should be considered.
@SuppressWarnings({"rawtypes", "unchecked"})
public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatistics {
+ private final int _totalDocs;
private final Comparable _minValue;
private final Comparable _maxValue;
private final int _minElementLength;
@@ -49,10 +50,11 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
super(dataSource, sortedDocIds, isSortedColumn);
Preconditions.checkState(!validDocIds.isEmpty(), "Use EmptyColumnStatistics for empty column: %s",
_fieldSpec.getName());
+ _totalDocs = validDocIds.getCardinality();
- DataType valueType = _forwardIndex.getStoredType();
+ DataType storedType = _forwardIndex.getStoredType();
boolean isSingleValue = _forwardIndex.isSingleValue();
- boolean isVariableWidth = !valueType.isFixedWidth();
+ boolean isVariableWidth = !storedType.isFixedWidth();
// Single pass over valid documents to collect stats.
// For SV columns, sort order is tracked inline: when sortedDocIds is provided, iterate in that order; when null,
@@ -62,9 +64,9 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
Comparable minValue = null;
Comparable maxValue = null;
// For fixed-width types element length is constant; for variable-width it is tracked per entry
- int minElementLength = isVariableWidth ? Integer.MAX_VALUE : valueType.size();
- int maxElementLength = isVariableWidth ? 0 : valueType.size();
- boolean isAscii = valueType == DataType.STRING;
+ int minElementLength = isVariableWidth ? Integer.MAX_VALUE : storedType.size();
+ int maxElementLength = isVariableWidth ? 0 : storedType.size();
+ boolean isAscii = storedType == DataType.STRING;
boolean isSorted = !_isSortedColumn;
Comparable prevValue = null;
int totalEntries = 0;
@@ -79,7 +81,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
continue;
}
totalEntries++;
- Comparable value = readValue(docId, valueType);
+ Comparable value = readValue(docId, storedType);
if (minValue == null || value.compareTo(minValue) < 0) {
minValue = value;
}
@@ -93,7 +95,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
prevValue = value;
}
if (isVariableWidth) {
- int length = getElementLength(value, valueType);
+ int length = getElementLength(value, storedType);
minElementLength = Math.min(minElementLength, length);
maxElementLength = Math.max(maxElementLength, length);
if (isAscii) {
@@ -106,7 +108,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
while (iterator.hasNext()) {
int docId = iterator.next();
totalEntries++;
- Comparable value = readValue(docId, valueType);
+ Comparable value = readValue(docId, storedType);
if (minValue == null || value.compareTo(minValue) < 0) {
minValue = value;
}
@@ -120,7 +122,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
prevValue = value;
}
if (isVariableWidth) {
- int length = getElementLength(value, valueType);
+ int length = getElementLength(value, storedType);
minElementLength = Math.min(minElementLength, length);
maxElementLength = Math.max(maxElementLength, length);
if (isAscii) {
@@ -130,7 +132,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
}
}
} else {
- switch (valueType) {
+ switch (storedType) {
case INT: {
int minInt = Integer.MAX_VALUE;
int maxInt = Integer.MIN_VALUE;
@@ -282,7 +284,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
break;
}
default:
- throw new IllegalStateException("Unsupported value type: " + valueType);
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
}
@@ -298,7 +300,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
} else if (isVariableWidth) {
_maxRowLength = maxRowLength;
} else {
- _maxRowLength = maxMultiValues * valueType.size();
+ _maxRowLength = maxMultiValues * storedType.size();
}
if (_isSortedColumn) {
@@ -311,12 +313,17 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
}
@Override
- public Comparable getMinValue() {
+ public int getTotalDocs() {
+ return _totalDocs;
+ }
+
+ @Override
+ public Comparable<?> getMinValue() {
return _minValue;
}
@Override
- public Comparable getMaxValue() {
+ public Comparable<?> getMaxValue() {
return _maxValue;
}
@@ -355,8 +362,8 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
return _maxRowLength;
}
- private Comparable readValue(int docId, DataType valueType) {
- switch (valueType) {
+ private Comparable readValue(int docId, DataType storedType) {
+ switch (storedType) {
case INT:
return _forwardIndex.getInt(docId);
case LONG:
@@ -372,12 +379,12 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
case BYTES:
return new ByteArray(_forwardIndex.getBytes(docId));
default:
- throw new IllegalStateException("Unsupported value type: " + valueType);
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
}
- private static int getElementLength(Comparable value, DataType valueType) {
- switch (valueType) {
+ private static int getElementLength(Comparable value, DataType storedType) {
+ switch (storedType) {
case BIG_DECIMAL:
return BigDecimalUtils.byteSize((BigDecimal) value);
case STRING:
@@ -385,7 +392,7 @@ public class CompactedNoDictColumnStatistics extends MutableNoDictColumnStatisti
case BYTES:
return ((ByteArray) value).length();
default:
- throw new IllegalStateException("Unsupported variable-width value type: " + valueType);
+ throw new IllegalStateException("Unsupported variable-width stored type: " + storedType);
}
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java
index 2d2cbb1cb5d..c4e7e52fdee 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java
@@ -33,7 +33,6 @@ import org.apache.pinot.spi.data.FieldSpec;
/**
* Column statistics for a column coming from an in-memory realtime segment.
*/
-@SuppressWarnings("rawtypes")
public class MutableColumnStatistics implements ColumnStatistics {
protected final DataSource _dataSource;
protected final DataSourceMetadata _dataSourceMetadata;
@@ -64,13 +63,18 @@ public class MutableColumnStatistics implements ColumnStatistics {
}
@Override
- public Comparable getMinValue() {
- return _dictionary.getMinVal();
+ public int getTotalDocs() {
+ return _dataSourceMetadata.getNumDocs();
}
@Override
- public Comparable getMaxValue() {
- return _dictionary.getMaxVal();
+ public Comparable<?> getMinValue() {
+ return (Comparable<?>) _dictionary.getMinVal();
+ }
+
+ @Override
+ public Comparable<?> getMaxValue() {
+ return (Comparable<?>) _dictionary.getMaxVal();
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
index 43f878fe0e2..92d3455425f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableNoDictColumnStatistics.java
@@ -37,7 +37,6 @@ import org.apache.pinot.spi.utils.ByteArray;
import static org.apache.pinot.segment.spi.Constants.UNKNOWN_CARDINALITY;
-@SuppressWarnings("rawtypes")
public class MutableNoDictColumnStatistics implements ColumnStatistics, CLPStatsProvider {
protected final DataSourceMetadata _dataSourceMetadata;
protected final FieldSpec _fieldSpec;
@@ -64,13 +63,18 @@ public class MutableNoDictColumnStatistics implements ColumnStatistics, CLPStats
}
@Override
- public Comparable getMinValue() {
- return _dataSourceMetadata.getMinValue();
+ public int getTotalDocs() {
+ return _dataSourceMetadata.getNumDocs();
+ }
+
+ @Override
+ public Comparable<?> getMinValue() {
+ return (Comparable<?>) _dataSourceMetadata.getMinValue();
}
@Override
- public Comparable getMaxValue() {
- return _dataSourceMetadata.getMaxValue();
+ public Comparable<?> getMaxValue() {
+ return (Comparable<?>) _dataSourceMetadata.getMaxValue();
}
@Nullable
@@ -114,9 +118,9 @@ public class MutableNoDictColumnStatistics implements ColumnStatistics, CLPStats
int numDocs = _dataSourceMetadata.getNumDocs();
// Verify that values are non-decreasing when iterated in the given order
- DataType valueType = getValueType();
+ DataType storedType = getStoredType();
if (_sortedDocIds != null) {
- switch (valueType) {
+ switch (storedType) {
case INT: {
int prev = _forwardIndex.getInt(_sortedDocIds[0]);
for (int i = 1; i < numDocs; i++) {
@@ -195,10 +199,10 @@ public class MutableNoDictColumnStatistics implements ColumnStatistics, CLPStats
return true;
}
default:
- throw new IllegalStateException("Unsupported value type: " + valueType);
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
} else {
- switch (valueType) {
+ switch (storedType) {
case INT: {
int prev = _forwardIndex.getInt(0);
for (int i = 1; i < numDocs; i++) {
@@ -277,7 +281,7 @@ public class MutableNoDictColumnStatistics implements ColumnStatistics, CLPStats
return true;
}
default:
- throw new IllegalStateException("Unsupported value type: " + valueType);
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
index a5118be2fd4..ca98d274d08 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
@@ -188,26 +188,16 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
}
private IndexCreationContext.Common getIndexCreationContext(FieldSpec fieldSpec, boolean dictEnabledColumn) {
- ColumnStatistics columnStatistics = _columnStatisticsMap.get(fieldSpec.getName());
- FieldIndexConfigs fieldIndexConfig = _config.getIndexConfigsByColName().get(fieldSpec.getName());
- boolean forwardIndexDisabled = !fieldIndexConfig.getConfig(StandardIndexes.forward()).isEnabled();
-
- return IndexCreationContext.builder()
- .withIndexDir(_indexDir)
- .withDictionary(dictEnabledColumn)
- .withFieldSpec(fieldSpec)
- .withTotalDocs(_totalDocs)
- .withColumnStatistics(columnStatistics)
+ ColumnStatistics columnStats = _columnStatisticsMap.get(fieldSpec.getName());
+ return new IndexCreationContext.Builder(_indexDir, _config.getTableConfig(), columnStats, dictEnabledColumn)
+ .withOnHeap(_config.isOnHeap())
.withOptimizedDictionary(_config.isOptimizeDictionary()
|| _config.isOptimizeDictionaryForMetrics() && fieldSpec.getFieldType() == FieldSpec.FieldType.METRIC)
- .onHeap(_config.isOnHeap())
- .withForwardIndexDisabled(forwardIndexDisabled)
.withTextCommitOnClose(true)
.withRealtimeConversion(_config.isRealtimeConversion())
.withConsumerDir(_config.getConsumerDir())
.withMutableSegmentCompacted(_config.isMutableSegmentCompacted())
.withMutableToImmutableDocIdMap(_config.getMutableToImmutableDocIdMap())
- .withTableNameWithType(_config.getTableConfig().getTableName())
.withContinueOnError(_config.isContinueOnError())
.build();
}
@@ -240,10 +230,10 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
new DictionaryIndexPlugin().getIndexType().createIndexCreator(context, dictConfig);
try {
- dictionaryCreator.build(context.getSortedUniqueElementsArray());
+ dictionaryCreator.build(columnStatistics.getUniqueValuesSet());
} catch (Exception e) {
LOGGER.error("Error building dictionary for field: {}, cardinality: {}, number of bytes per entry: {}",
- context.getFieldSpec().getName(), context.getCardinality(), dictionaryCreator.getNumBytesPerEntry());
+ columnName, columnStatistics.getCardinality(), dictionaryCreator.getNumBytesPerEntry());
throw e;
}
return dictionaryCreator;
@@ -781,15 +771,14 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
// Format conversion
convertFormatIfNecessary(_indexDir);
- // Build indexes if there are documents
+ // Build indexes if there are documents. Empty segments have no values to index; index handlers (mirroring
+ // [SegmentPreProcessor#process]'s `totalDocs == 0` short-circuit) assume at least one doc.
if (_totalDocs > 0) {
buildStarTreeV2IfNecessary(_indexDir);
buildMultiColumnTextIndex(_indexDir);
+ updatePostSegmentCreationIndexes(_indexDir);
}
- // Update post-creation indexes
- updatePostSegmentCreationIndexes(_indexDir);
-
// Persist creation metadata
persistCreationMeta(_indexDir, dataCrc);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
index 8479b33ec7f..82a881e2a76 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
@@ -29,7 +29,6 @@ import org.apache.pinot.segment.local.segment.index.fst.FstIndexType;
import org.apache.pinot.segment.local.utils.MetricUtils;
import org.apache.pinot.segment.local.utils.fst.FSTBuilder;
import org.apache.pinot.segment.spi.V1Constants;
-import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.creator.FSTIndexCreator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,12 +97,6 @@ public class LuceneFSTIndexCreator implements FSTIndexCreator {
}
}
- public LuceneFSTIndexCreator(IndexCreationContext context)
- throws IOException {
- this(context.getIndexDir(), context.getFieldSpec().getName(), context.getTableNameWithType(),
- context.isContinueOnError(), (String[]) context.getSortedUniqueElementsArray());
- }
-
// Expects dictionary entries in sorted order.
@Override
public void add(String document)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneIFSTIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneIFSTIndexCreator.java
index 178df45d8f4..efa7a1d822c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneIFSTIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/text/LuceneIFSTIndexCreator.java
@@ -29,7 +29,6 @@ import org.apache.pinot.segment.local.segment.index.fst.FstIndexType;
import org.apache.pinot.segment.local.utils.MetricUtils;
import org.apache.pinot.segment.local.utils.fst.IFSTBuilder;
import org.apache.pinot.segment.spi.V1Constants;
-import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.creator.FSTIndexCreator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,12 +89,6 @@ public class LuceneIFSTIndexCreator implements FSTIndexCreator {
}
}
- public LuceneIFSTIndexCreator(IndexCreationContext context)
- throws IOException {
- this(context.getIndexDir(), context.getFieldSpec().getName(), context.getTableNameWithType(),
- context.isContinueOnError(), (String[]) context.getSortedUniqueElementsArray());
- }
-
// Expects dictionary entries in sorted order.
@Override
public void add(String document)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java
index 14db5917e07..e548248dea1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollector.java
@@ -44,11 +44,12 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
public abstract class AbstractColumnStatisticsCollector implements ColumnStatistics {
protected static final int INITIAL_HASH_SET_SIZE = 1000;
protected final FieldSpec _fieldSpec;
- protected final DataType _valueType;
+ protected final DataType _storedType;
protected final FieldConfig _fieldConfig;
protected final PartitionFunction _partitionFunction;
protected final Set<Integer> _partitions;
+ protected int _totalDocs = 0;
protected int _totalNumberOfEntries = 0;
protected int _maxNumberOfMultiValues = 0;
protected boolean _sorted;
@@ -57,7 +58,7 @@ public abstract class AbstractColumnStatisticsCollector implements ColumnStatist
public AbstractColumnStatisticsCollector(String column, StatsCollectorConfig statsCollectorConfig) {
_fieldSpec = statsCollectorConfig.getFieldSpecForColumn(column);
Preconditions.checkArgument(_fieldSpec != null, "Failed to find column: %s", column);
- _valueType = _fieldSpec.getDataType().getStoredType();
+ _storedType = _fieldSpec.getDataType().getStoredType();
_sorted = _fieldSpec.isSingleValueField();
_fieldConfig = statsCollectorConfig.getFieldConfigForColumn(column);
_partitionFunction = statsCollectorConfig.getPartitionFunction(column);
@@ -111,8 +112,13 @@ public abstract class AbstractColumnStatisticsCollector implements ColumnStatist
}
@Override
- public DataType getValueType() {
- return _valueType;
+ public DataType getStoredType() {
+ return _storedType;
+ }
+
+ @Override
+ public int getTotalDocs() {
+ return _totalDocs;
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java
index 68f79f0be0f..dd0dcfe00b9 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BigDecimalColumnPreIndexStatsCollector.java
@@ -43,6 +43,7 @@ public class BigDecimalColumnPreIndexStatsCollector extends AbstractColumnStatis
@Override
public void collect(Object entry) {
assert !_sealed;
+ _totalDocs++;
if (entry instanceof Object[]) {
Object[] values = (Object[]) entry;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BytesColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BytesColumnPreIndexStatsCollector.java
index 5b2f41cdbc2..371e5bbf51d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BytesColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/BytesColumnPreIndexStatsCollector.java
@@ -43,6 +43,7 @@ public class BytesColumnPreIndexStatsCollector extends AbstractColumnStatisticsC
@Override
public void collect(Object entry) {
assert !_sealed;
+ _totalDocs++;
if (entry instanceof Object[]) {
Object[] values = (Object[]) entry;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/DoubleColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/DoubleColumnPreIndexStatsCollector.java
index ad22f2902cd..8024460c004 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/DoubleColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/DoubleColumnPreIndexStatsCollector.java
@@ -57,6 +57,7 @@ public class DoubleColumnPreIndexStatsCollector extends AbstractColumnStatistics
updatePartition(Double.toString(value));
}
}
+ _totalDocs++;
_totalNumberOfEntries++;
}
@@ -71,6 +72,7 @@ public class DoubleColumnPreIndexStatsCollector extends AbstractColumnStatistics
_values.add(valueGetter.apply(i));
}
_maxNumberOfMultiValues = Math.max(_maxNumberOfMultiValues, length);
+ _totalDocs++;
updateTotalNumberOfEntries(length);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java
index a791fa3f039..8829206308a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java
@@ -20,42 +20,18 @@ package org.apache.pinot.segment.local.segment.creator.impl.stats;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.EmptyColumnShape;
import org.apache.pinot.segment.spi.creator.ColumnStatistics;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
/// Column statistics for an empty column (zero rows).
-@SuppressWarnings("rawtypes")
-public class EmptyColumnStatistics implements ColumnStatistics {
- private final FieldSpec _fieldSpec;
- private final PartitionFunction _partitionFunction;
- private final Set<Integer> _partitions;
+public class EmptyColumnStatistics extends EmptyColumnShape implements ColumnStatistics {
- // TODO: Revisit if we need to maintain partition info for empty columns
public EmptyColumnStatistics(FieldSpec fieldSpec, @Nullable PartitionFunction partitionFunction,
@Nullable Set<Integer> partitions) {
- _fieldSpec = fieldSpec;
- _partitionFunction = partitionFunction;
- _partitions = partitions;
- }
-
- @Override
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
- }
-
- @Nullable
- @Override
- public Comparable getMinValue() {
- return null;
- }
-
- @Nullable
- @Override
- public Comparable getMaxValue() {
- return null;
+ super(fieldSpec, partitionFunction, partitions);
}
@Nullable
@@ -63,53 +39,4 @@ public class EmptyColumnStatistics implements ColumnStatistics {
public Object getUniqueValuesSet() {
return null;
}
-
- @Override
- public int getCardinality() {
- return 0;
- }
-
- @Override
- public int getLengthOfShortestElement() {
- DataType valueType = getValueType();
- return valueType.isFixedWidth() ? valueType.size() : 0;
- }
-
- @Override
- public int getLengthOfLongestElement() {
- DataType valueType = getValueType();
- return valueType.isFixedWidth() ? valueType.size() : 0;
- }
-
- @Override
- public boolean isSorted() {
- return isSingleValue();
- }
-
- @Override
- public int getTotalNumberOfEntries() {
- return 0;
- }
-
- @Override
- public int getMaxNumberOfMultiValues() {
- return 0;
- }
-
- @Override
- public int getMaxRowLengthInBytes() {
- return 0;
- }
-
- @Nullable
- @Override
- public PartitionFunction getPartitionFunction() {
- return _partitionFunction;
- }
-
- @Nullable
- @Override
- public Set<Integer> getPartitions() {
- return _partitions;
- }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/FloatColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/FloatColumnPreIndexStatsCollector.java
index 54419d867d3..e26f1529335 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/FloatColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/FloatColumnPreIndexStatsCollector.java
@@ -57,6 +57,7 @@ public class FloatColumnPreIndexStatsCollector extends AbstractColumnStatisticsC
updatePartition(Float.toString(value));
}
}
+ _totalDocs++;
_totalNumberOfEntries++;
}
@@ -71,6 +72,7 @@ public class FloatColumnPreIndexStatsCollector extends AbstractColumnStatisticsC
_values.add(valueGetter.apply(i));
}
_maxNumberOfMultiValues = Math.max(_maxNumberOfMultiValues, length);
+ _totalDocs++;
updateTotalNumberOfEntries(length);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/IntColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/IntColumnPreIndexStatsCollector.java
index 80622739d83..249bf894c55 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/IntColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/IntColumnPreIndexStatsCollector.java
@@ -57,6 +57,7 @@ public class IntColumnPreIndexStatsCollector extends AbstractColumnStatisticsCol
updatePartition(Integer.toString(value));
}
}
+ _totalDocs++;
_totalNumberOfEntries++;
}
@@ -71,6 +72,7 @@ public class IntColumnPreIndexStatsCollector extends AbstractColumnStatisticsCol
_values.add(valueGetter.apply(i));
}
_maxNumberOfMultiValues = Math.max(_maxNumberOfMultiValues, length);
+ _totalDocs++;
updateTotalNumberOfEntries(length);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/LongColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/LongColumnPreIndexStatsCollector.java
index a9e90b3f133..c8d98f1d5d5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/LongColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/LongColumnPreIndexStatsCollector.java
@@ -57,6 +57,7 @@ public class LongColumnPreIndexStatsCollector extends AbstractColumnStatisticsCo
updatePartition(Long.toString(value));
}
}
+ _totalDocs++;
_totalNumberOfEntries++;
}
@@ -71,6 +72,7 @@ public class LongColumnPreIndexStatsCollector extends AbstractColumnStatisticsCo
_values.add(valueGetter.apply(i));
}
_maxNumberOfMultiValues = Math.max(_maxNumberOfMultiValues, length);
+ _totalDocs++;
updateTotalNumberOfEntries(length);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/MapColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/MapColumnPreIndexStatsCollector.java
index e2d720ad1fe..c845c73cd80 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/MapColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/MapColumnPreIndexStatsCollector.java
@@ -89,6 +89,7 @@ public class MapColumnPreIndexStatsCollector extends AbstractColumnStatisticsCol
@Override
public void collect(Object entry) {
assert !_sealed;
+ _totalDocs++;
if (entry instanceof Map) {
//noinspection unchecked
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
index 02a80b918fa..4aa23fa791f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
@@ -66,8 +66,8 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
public NoDictColumnStatisticsCollector(String column, StatsCollectorConfig statsCollectorConfig) {
super(column, statsCollectorConfig);
- _isFixedWidth = _valueType.isFixedWidth();
- _isAscii = _valueType == DataType.STRING;
+ _isFixedWidth = _storedType.isFixedWidth();
+ _isAscii = _storedType == DataType.STRING;
// Use default p and sp; can be made configurable via StatsCollectorConfig later if needed
_hllPlus = new HyperLogLogPlus(
CommonConstants.Helix.DEFAULT_HYPERLOGLOG_PLUS_P,
@@ -78,6 +78,8 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
@Override
public void collect(Object entry) {
assert !_sealed;
+ _totalDocs++;
+
if (entry instanceof Object[]) {
Object[] values = (Object[]) entry;
int rowLength = 0;
@@ -179,7 +181,7 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
}
private int getValueLength(Object value) {
- switch (_valueType) {
+ switch (_storedType) {
case BIG_DECIMAL:
return BigDecimalUtils.byteSize((BigDecimal) value);
case STRING:
@@ -187,13 +189,13 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
case BYTES:
return ((byte[]) value).length;
default:
- throw new IllegalStateException("Unsupported variable-width value type: " + _valueType);
+ throw new IllegalStateException("Unsupported variable-width value type: " + _storedType);
}
}
@Nullable
@Override
- public Comparable getMinValue() {
+ public Comparable<?> getMinValue() {
if (_sealed) {
return _minValue;
}
@@ -202,7 +204,7 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
@Nullable
@Override
- public Comparable getMaxValue() {
+ public Comparable<?> getMaxValue() {
if (_sealed) {
return _maxValue;
}
@@ -231,7 +233,7 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
@Override
public int getLengthOfShortestElement() {
if (_isFixedWidth) {
- return _valueType.size();
+ return _storedType.size();
} else {
return _minLength != Integer.MAX_VALUE ? _minLength : 0;
}
@@ -239,7 +241,7 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
@Override
public int getLengthOfLongestElement() {
- return _isFixedWidth ? _valueType.size() : _maxLength;
+ return _isFixedWidth ? _storedType.size() : _maxLength;
}
@Override
@@ -250,7 +252,7 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
@Override
public int getMaxRowLengthInBytes() {
if (_isFixedWidth) {
- int elementSize = _valueType.size();
+ int elementSize = _storedType.size();
return isSingleValue() ? elementSize : _maxNumberOfMultiValues * elementSize;
} else {
return isSingleValue() ? _maxLength : _maxRowLength;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
index 9c1631930ee..a01afe7b49a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
@@ -53,6 +53,7 @@ public class StringColumnPreIndexStatsCollector extends AbstractColumnStatistics
@Override
public void collect(Object entry) {
assert !_sealed;
+ _totalDocs++;
if (entry instanceof Object[]) {
Object[] values = (Object[]) entry;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
index 28c06c2a3c3..eaf669be34e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
@@ -97,11 +97,11 @@ public class ForwardIndexCreatorFactory {
int targetDocsPerChunk = indexConfig.getTargetDocsPerChunk();
if (fieldSpec.isSingleValueField()) {
return getRawIndexCreatorForSVColumn(indexDir, chunkCompressionType, columnName, storedType, numTotalDocs,
- context.getLengthOfLongestEntry(), deriveNumDocsPerChunk, writerVersion, targetMaxChunkSize,
+ context.getLengthOfLongestElement(), deriveNumDocsPerChunk, writerVersion, targetMaxChunkSize,
targetDocsPerChunk);
} else {
return getRawIndexCreatorForMVColumn(indexDir, chunkCompressionType, columnName, storedType, numTotalDocs,
- context.getMaxNumberOfMultiValueElements(), deriveNumDocsPerChunk, writerVersion,
+ context.getMaxNumberOfMultiValues(), deriveNumDocsPerChunk, writerVersion,
context.getMaxRowLengthInBytes(), targetMaxChunkSize, targetDocsPerChunk);
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
index 2f634d161f7..d7114c9f815 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
@@ -29,6 +29,7 @@ import org.apache.pinot.segment.local.segment.index.loader.invertedindex.FSTInde
import org.apache.pinot.segment.local.segment.index.readers.LuceneFSTIndexReader;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.creator.ColumnStatistics;
import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.AbstractIndexType;
import org.apache.pinot.segment.spi.index.ColumnConfigDeserializer;
@@ -100,13 +101,17 @@ public class FstIndexType extends AbstractIndexType<FstIndexConfig, TextIndexRea
@Override
public FSTIndexCreator createIndexCreator(IndexCreationContext context, FstIndexConfig indexConfig)
throws IOException {
- Preconditions.checkState(context.getFieldSpec().isSingleValueField(),
+ FieldSpec fieldSpec = context.getFieldSpec();
+ Preconditions.checkState(fieldSpec.isSingleValueField(),
"FST index is currently only supported on single-value columns");
- Preconditions.checkState(context.getFieldSpec().getDataType().getStoredType() == FieldSpec.DataType.STRING,
+ Preconditions.checkState(fieldSpec.getDataType().getStoredType() == FieldSpec.DataType.STRING,
"FST index is currently only supported on STRING type columns");
Preconditions.checkState(context.hasDictionary(),
"FST index is currently only supported on dictionary-encoded columns");
- return new LuceneFSTIndexCreator(context);
+ ColumnStatistics columnStatistics = context.getColumnStatistics();
+ String[] sortedEntries = columnStatistics != null ? (String[]) columnStatistics.getUniqueValuesSet() : null;
+ return new LuceneFSTIndexCreator(context.getIndexDir(), fieldSpec.getName(), context.getTableNameWithType(),
+ context.isContinueOnError(), sortedEntries);
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/IFSTIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/IFSTIndexType.java
index 3c580c479e2..9ec8683b820 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/IFSTIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/IFSTIndexType.java
@@ -28,6 +28,7 @@ import org.apache.pinot.segment.local.segment.index.loader.invertedindex.IFSTInd
import org.apache.pinot.segment.local.segment.index.readers.LuceneIFSTIndexReader;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.creator.ColumnStatistics;
import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.AbstractIndexType;
import org.apache.pinot.segment.spi.index.ColumnConfigDeserializer;
@@ -96,13 +97,17 @@ public class IFSTIndexType extends AbstractIndexType<FstIndexConfig, TextIndexRe
@Override
public FSTIndexCreator createIndexCreator(IndexCreationContext context, FstIndexConfig indexConfig)
throws IOException {
- Preconditions.checkState(context.getFieldSpec().isSingleValueField(),
+ FieldSpec fieldSpec = context.getFieldSpec();
+ Preconditions.checkState(fieldSpec.isSingleValueField(),
"IFST index is currently only supported on single-value columns");
- Preconditions.checkState(context.getFieldSpec().getDataType().getStoredType() == FieldSpec.DataType.STRING,
+ Preconditions.checkState(fieldSpec.getDataType().getStoredType() == FieldSpec.DataType.STRING,
"IFST index is currently only supported on STRING type columns");
Preconditions.checkState(context.hasDictionary(),
"IFST index is currently only supported on dictionary-encoded columns");
- return new LuceneIFSTIndexCreator(context);
+ ColumnStatistics columnStatistics = context.getColumnStatistics();
+ String[] sortedEntries = columnStatistics != null ? (String[]) columnStatistics.getUniqueValuesSet() : null;
+ return new LuceneIFSTIndexCreator(context.getIndexDir(), fieldSpec.getName(), context.getTableNameWithType(),
+ context.isContinueOnError(), sortedEntries);
}
@Override
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
index 98b98acf5dc..a5c68d21c8f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/BaseIndexHandler.java
@@ -116,10 +116,8 @@ public abstract class BaseIndexHandler implements IndexHandler {
boolean dictionaryBasedForwardIndex = forwardIndexConfig.getEncodingType() == FieldConfig.EncodingType.DICTIONARY;
InvertedIndexAndDictionaryBasedForwardIndexCreator creator =
- new InvertedIndexAndDictionaryBasedForwardIndexCreator(columnName, _segmentDirectory, dictionaryEnabled,
- dictionaryBasedForwardIndex, forwardIndexConfig, segmentWriter, isTemporaryForwardIndex,
- _tableConfig.getTableName(),
- _tableConfig.getIngestionConfig() != null && _tableConfig.getIngestionConfig().isContinueOnError());
+ new InvertedIndexAndDictionaryBasedForwardIndexCreator(_segmentDirectory, segmentWriter, _tableConfig,
+ columnName, dictionaryEnabled, dictionaryBasedForwardIndex, forwardIndexConfig, isTemporaryForwardIndex);
creator.regenerateForwardIndex();
// Validate that the forward index is created.
if (!segmentWriter.hasIndexFor(columnName, StandardIndexes.forward())) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
index c6cc8ef0505..0aba4f1a52e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
@@ -585,11 +585,7 @@ public class ForwardIndexHandler extends BaseIndexHandler {
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
try (ForwardIndexReader<?> reader = readerFactory.createIndexReader(segmentWriter, _fieldIndexConfigs.get(column),
columnMetadata)) {
- IndexCreationContext.Builder builder =
- IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError());
+ IndexCreationContext.Builder builder = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata);
// Encoding flows through ForwardIndexConfig; for compression-change rewrite the encoding does not change so
// the config in _fieldIndexConfigs already carries the correct encoding.
// Set entry length info for raw index creators. No need to set this when changing dictionary id compression type.
@@ -600,7 +596,7 @@ public class ForwardIndexHandler extends BaseIndexHandler {
lengthOfLongestEntry = getMaxRowLength(columnMetadata, reader, null);
}
if (columnMetadata.isSingleValue()) {
- builder.withLengthOfLongestEntry(lengthOfLongestEntry);
+ builder.withLengthOfLongestElement(lengthOfLongestEntry);
} else {
// For VarByte MV columns like String and Bytes, the storage representation of each row contains the following
// components:
@@ -984,16 +980,8 @@ public class ForwardIndexHandler extends BaseIndexHandler {
LOGGER.info("Built dictionary. Rewriting dictionary enabled forward index for segment={} and column={}",
segmentName, column);
ForwardIndexConfig config = _fieldIndexConfigs.get(column).getConfig(StandardIndexes.forward());
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withFieldSpec(fieldSpec)
- .withColumnStatistics(statsCollector)
- .withTotalDocs(numDocs)
- .withDictionary(true)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(
- _tableConfig.getIngestionConfig() != null && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context =
+ new IndexCreationContext.Builder(indexDir, _tableConfig, statsCollector, true).build();
try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, config)) {
forwardIndexRewriteHelper(column, existingColMetadata, reader, creator, numDocs, dictionaryCreator, null);
}
@@ -1212,24 +1200,15 @@ public class ForwardIndexHandler extends BaseIndexHandler {
columnMetadata)) {
Dictionary dictionary = DictionaryIndexType.read(segmentWriter, columnMetadata);
IndexCreationContext.Builder builder =
- IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError());
- builder.withDictionary(false);
- // Encoding flows through ForwardIndexConfig set below.
- if (!columnMetadata.getDataType().getStoredType().isFixedWidth()) {
- if (columnMetadata.isSingleValue()) {
- // lengthOfLongestEntry is available for dict columns from metadata.
- builder.withLengthOfLongestEntry(columnMetadata.getLengthOfLongestElement());
- } else {
- // maxRowLength can only be determined by scanning the column.
- builder.withMaxRowLengthInBytes(getMaxRowLength(columnMetadata, reader, dictionary));
- }
+ new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).withDictionary(false);
+ // Encoding flows through ForwardIndexConfig set below. The row length is derived from persisted metadata via
+ // the Builder constructor; only fall back to scanning when metadata returns UNAVAILABLE (var-width MV columns
+ // with varying element lengths).
+ if (columnMetadata.getMaxRowLengthInBytes() == ColumnMetadata.UNAVAILABLE) {
+ builder.withMaxRowLengthInBytes(getMaxRowLength(columnMetadata, reader, dictionary));
}
ForwardIndexConfig config = _fieldIndexConfigs.get(column).getConfig(StandardIndexes.forward());
- IndexCreationContext context = builder.build();
- try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, config)) {
+ try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(builder.build(), config)) {
forwardIndexRewriteHelper(column, columnMetadata, reader, creator, columnMetadata.getTotalDocs(), null,
dictionary);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
index c823b87822e..9ffc34b5089 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
@@ -40,7 +40,9 @@ import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.segment.spi.utils.SegmentMetadataUtils;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.PeekableIntIterator;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,34 +78,24 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
private static final String FORWARD_INDEX_LENGTH_BUFFER_SUFFIX = ".fwd.idx.len.buf";
private static final String FORWARD_INDEX_MAX_SIZE_BUFFER_SUFFIX = ".fwd.idx.maxsize.buf";
+ private final SegmentDirectory _segmentDirectory;
+ private final SegmentDirectory.Writer _segmentWriter;
+ private final TableConfig _tableConfig;
private final String _columnName;
+ /// `true` if a standalone dictionary file exists for the column. Independent of how the forward index is
+ /// encoded — both the standard "dict-encoded forward index" and the new "shared-dictionary + raw forward index"
+ /// configurations have `_dictionaryPresent == true`.
+ private final boolean _dictionaryPresent;
private final ForwardIndexConfig _forwardIndexConfig;
- private final SegmentDirectory.Writer _segmentWriter;
private final boolean _isTemporaryForwardIndex;
- private final String _tableNameWithType;
- private final boolean _continueOnError;
// Metadata
- private final SegmentDirectory _segmentDirectory;
private final ColumnMetadata _columnMetadata;
private final boolean _singleValue;
private final int _cardinality;
private final int _numDocs;
private final int _maxNumberOfMultiValues;
- private final FieldSpec.DataType _storedType;
private final int _totalNumberOfEntries;
- /// `true` if a standalone dictionary file exists for the column. Independent of how the forward index is
- /// encoded — both the standard "dict-encoded forward index" and the new "shared-dictionary + raw forward index"
- /// configurations have `_dictionaryPresent == true`.
- private final boolean _dictionaryPresent;
- /// `true` if the forward index itself stores dict IDs (`FieldConfig.EncodingType.DICTIONARY`),
- /// `false` if it stores raw values (`FieldConfig.EncodingType.RAW`).
- ///
- /// Pre-shared-dict, this field used to be a single `_dictionaryEnabled` that conflated both meanings.
- /// After the shared-dictionary feature, the two boolean dimensions are independent: a column can have a dictionary
- /// AND a raw forward index simultaneously. Subclasses or callers that previously expected a single combined flag
- /// must update to the two-flag model.
- private final boolean _dictionaryBasedForwardIndex;
private final boolean _useMMapBuffer;
// Files and temporary buffers
@@ -123,28 +115,25 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
// For multi-valued column only to track max row size
private PinotDataBuffer _forwardIndexMaxSizeBuffer;
- public InvertedIndexAndDictionaryBasedForwardIndexCreator(String columnName, SegmentDirectory segmentDirectory,
- boolean dictionaryPresent, boolean dictionaryBasedForwardIndex, ForwardIndexConfig fwdConf,
- SegmentDirectory.Writer segmentWriter,
- boolean isTemporaryForwardIndex, String tableNameWithType, boolean continueOnError)
+ public InvertedIndexAndDictionaryBasedForwardIndexCreator(SegmentDirectory segmentDirectory,
+ SegmentDirectory.Writer segmentWriter, TableConfig tableConfig, String columnName, boolean dictionaryPresent,
+ boolean dictionaryBasedForwardIndex, ForwardIndexConfig forwardIndexConfig, boolean isTemporaryForwardIndex)
throws IOException {
- _columnName = columnName;
_segmentDirectory = segmentDirectory;
- _segmentMetadata = segmentDirectory.getSegmentMetadata();
_segmentWriter = segmentWriter;
+ _tableConfig = tableConfig;
+ _columnName = columnName;
+ _dictionaryPresent = dictionaryPresent;
+ _forwardIndexConfig = forwardIndexConfig;
_isTemporaryForwardIndex = isTemporaryForwardIndex;
- _tableNameWithType = tableNameWithType;
- _continueOnError = continueOnError;
+ _segmentMetadata = segmentDirectory.getSegmentMetadata();
_columnMetadata = _segmentMetadata.getColumnMetadataFor(columnName);
_singleValue = _columnMetadata.isSingleValue();
_cardinality = _columnMetadata.getCardinality();
_numDocs = _columnMetadata.getTotalDocs();
_totalNumberOfEntries = _columnMetadata.getTotalNumberOfEntries();
_maxNumberOfMultiValues = _columnMetadata.getMaxNumberOfMultiValues();
- _storedType = _columnMetadata.getFieldSpec().getDataType().getStoredType();
- _dictionaryPresent = dictionaryPresent;
- _dictionaryBasedForwardIndex = dictionaryBasedForwardIndex;
int numValues = _singleValue ? _numDocs : _totalNumberOfEntries;
_useMMapBuffer = numValues > NUM_VALUES_THRESHOLD_FOR_MMAP_BUFFER;
@@ -152,14 +141,13 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
// a no-op for sorted columns
File indexDir = _segmentMetadata.getIndexDir();
String fileExtension;
- if (_dictionaryBasedForwardIndex) {
+ if (dictionaryBasedForwardIndex) {
fileExtension = _singleValue ? V1Constants.Indexes.UNSORTED_SV_FORWARD_INDEX_FILE_EXTENSION
: V1Constants.Indexes.UNSORTED_MV_FORWARD_INDEX_FILE_EXTENSION;
} else {
fileExtension = _singleValue ? V1Constants.Indexes.RAW_SV_FORWARD_INDEX_FILE_EXTENSION
: V1Constants.Indexes.RAW_MV_FORWARD_INDEX_FILE_EXTENSION;
}
- _forwardIndexConfig = fwdConf;
_forwardIndexFile = new File(indexDir, columnName + fileExtension);
_forwardIndexValueBufferFile = new File(indexDir, columnName + FORWARD_INDEX_VALUE_BUFFER_SUFFIX);
_forwardIndexLengthBufferFile = new File(indexDir, columnName + FORWARD_INDEX_LENGTH_BUFFER_SUFFIX);
@@ -267,27 +255,17 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
(BitmapInvertedIndexReader) InvertedIndexType.ReaderFactory
.INSTANCE.createSkippingForward(_segmentWriter, _columnMetadata);
Dictionary dictionary = DictionaryIndexType.read(_segmentWriter, _columnMetadata)) {
- boolean isFixedWidth = _columnMetadata.getFieldSpec().getDataType().isFixedWidth();
- int lengthOfLongestEntry = isFixedWidth ? -1 : 0;
// Construct the forward index in the values buffer
for (int dictId = 0; dictId < _cardinality; dictId++) {
ImmutableRoaringBitmap docIdsBitmap = invertedIndexReader.getDocIds(dictId);
int finalDictId = dictId;
docIdsBitmap.stream().forEach(docId -> putInt(_forwardIndexValueBuffer, docId, finalDictId));
- if (!isFixedWidth) {
- lengthOfLongestEntry = Math.max(lengthOfLongestEntry, dictionary.getValueSize(dictId));
- }
}
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(_segmentMetadata.getIndexDir())
- .withColumnMetadata(_columnMetadata)
- .withForwardIndexDisabled(false)
- .withDictionary(_dictionaryPresent)
- .withLengthOfLongestEntry(lengthOfLongestEntry)
- .withTableNameWithType(_tableNameWithType)
- .withContinueOnError(_continueOnError)
- .build();
+ IndexCreationContext context =
+ new IndexCreationContext.Builder(_segmentMetadata.getIndexDir(), _tableConfig, _columnMetadata)
+ .withDictionary(_dictionaryPresent)
+ .build();
// note: this method closes buffers and removes files
writeToForwardIndex(dictionary, context);
@@ -314,21 +292,16 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
.createSkippingForward(_segmentWriter, _columnMetadata);
Dictionary dictionary = DictionaryIndexType.read(_segmentWriter, _columnMetadata)) {
// Construct the forward index length buffer and create the inverted index values and length buffers
- int[] maxNumberOfMultiValues = new int[]{0};
- final boolean isFixedWidth = _columnMetadata.getFieldSpec().getDataType().isFixedWidth();
- int lengthOfLongestEntry = isFixedWidth ? -1 : 0;
- int[] maxRowLengthInBytes = isFixedWidth ? new int[]{-1} : new int[]{0};
+ int maxNumberOfMultiValues = 0;
for (int dictId = 0; dictId < _cardinality; dictId++) {
ImmutableRoaringBitmap docIdsBitmap = invertedIndexReader.getDocIds(dictId);
- docIdsBitmap.stream().forEach(docId -> {
- int newRowLength = getInt(_forwardIndexLengthBuffer, docId) + 1;
- maxNumberOfMultiValues[0] = Math.max(maxNumberOfMultiValues[0], newRowLength);
- putInt(_forwardIndexLengthBuffer, docId, newRowLength);
+ PeekableIntIterator intIterator = docIdsBitmap.getIntIterator();
+ while (intIterator.hasNext()) {
+ int docId = intIterator.next();
+ int numMultiValues = getInt(_forwardIndexLengthBuffer, docId) + 1;
+ putInt(_forwardIndexLengthBuffer, docId, numMultiValues);
+ maxNumberOfMultiValues = Math.max(maxNumberOfMultiValues, numMultiValues);
_nextValueId++;
- });
-
- if (!isFixedWidth) {
- lengthOfLongestEntry = Math.max(lengthOfLongestEntry, dictionary.getValueSize(dictId));
}
}
@@ -336,14 +309,14 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
LOGGER.warn("Total number of entries: {} less than expected total number of entries: {}, multi-value column: "
+ "{} duplicates detected, duplicate entries within each row lost! Expected maxNumberOfMultiValues: "
+ "{}, actual maxNumberOfMultiValues: {}", _nextValueId, _totalNumberOfEntries, _columnName,
- _maxNumberOfMultiValues, maxNumberOfMultiValues[0]);
+ _maxNumberOfMultiValues, maxNumberOfMultiValues);
} else {
Preconditions.checkState(_nextValueId == _totalNumberOfEntries,
String.format("Number of entries found %d cannot be higher than expected total number of entries: %d for "
+ "column: %s", _nextValueId, _totalNumberOfEntries, _columnName));
- Preconditions.checkState(maxNumberOfMultiValues[0] == _maxNumberOfMultiValues,
+ Preconditions.checkState(maxNumberOfMultiValues == _maxNumberOfMultiValues,
String.format("Actual maxNumberOfMultiValues: %d doesn't match expected maxNumberOfMultiValues: %d for "
- + "column %s", maxNumberOfMultiValues[0], _maxNumberOfMultiValues, _columnName));
+ + "column %s", maxNumberOfMultiValues, _maxNumberOfMultiValues, _columnName));
}
// Calculate value index for each docId in the forward index value buffer
@@ -357,38 +330,42 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
// Construct the forward index values buffer from the inverted index using the length buffer for index tracking
+ DataType storedType = _columnMetadata.getStoredType();
+ boolean isFixedWidth = storedType.isFixedWidth();
+ int maxRowLengthInBytes = isFixedWidth ? maxNumberOfMultiValues * storedType.size() : 0;
for (int dictId = 0; dictId < _cardinality; dictId++) {
ImmutableRoaringBitmap docIdsBitmap = invertedIndexReader.getDocIds(dictId);
- int finalDictId = dictId;
- docIdsBitmap.stream().forEach(docId -> {
+ PeekableIntIterator intIterator = docIdsBitmap.getIntIterator();
+ while (intIterator.hasNext()) {
+ int docId = intIterator.next();
int index = getInt(_forwardIndexLengthBuffer, docId);
- putInt(_forwardIndexValueBuffer, index, finalDictId);
+ putInt(_forwardIndexValueBuffer, index, dictId);
putInt(_forwardIndexLengthBuffer, docId, index + 1);
if (!isFixedWidth) {
- trackMaxRowLengthInBytes(dictionary, maxRowLengthInBytes, docId, finalDictId);
+ int currentRowLength = getInt(_forwardIndexMaxSizeBuffer, docId);
+ int newRowLength = currentRowLength + dictionary.getValueSize(dictId);
+ putInt(_forwardIndexMaxSizeBuffer, docId, newRowLength);
+ maxRowLengthInBytes = Math.max(maxRowLengthInBytes, newRowLength);
}
- });
+ }
}
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(_segmentMetadata.getIndexDir())
- .withColumnMetadata(_columnMetadata)
- .withForwardIndexDisabled(false)
- .withDictionary(_dictionaryPresent)
+ // When the duplicate-detection branch above fires, `_nextValueId` and `maxNumberOfMultiValues` are the new
+ // post-dedup values and differ from the source metadata; override so the forward index creator sees the correct
+ // sizes (consistent with the persisted metadata properties below).
+ File indexDir = _segmentMetadata.getIndexDir();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, _columnMetadata)
.withTotalNumberOfEntries(_nextValueId)
- .withMaxNumberOfMultiValueElements(maxNumberOfMultiValues[0])
- .withMaxRowLengthInBytes(maxRowLengthInBytes[0])
- .withLengthOfLongestEntry(lengthOfLongestEntry)
- .withTableNameWithType(_tableNameWithType)
- .withContinueOnError(_continueOnError)
+ .withMaxNumberOfMultiValues(maxNumberOfMultiValues)
+ .withMaxRowLengthInBytes(maxRowLengthInBytes)
+ .withDictionary(_dictionaryPresent)
.build();
writeToForwardIndex(dictionary, context);
// Setup and return the metadata properties to update
Map<String, String> metadataProperties = new HashMap<>();
- metadataProperties.put(getKeyFor(_columnName, MAX_MULTI_VALUE_ELEMENTS),
- String.valueOf(maxNumberOfMultiValues[0]));
+ metadataProperties.put(getKeyFor(_columnName, MAX_MULTI_VALUE_ELEMENTS), String.valueOf(maxNumberOfMultiValues));
metadataProperties.put(getKeyFor(_columnName, TOTAL_NUMBER_OF_ENTRIES), String.valueOf(_nextValueId));
metadataProperties.put(getKeyFor(_columnName, FORWARD_INDEX_ENCODING),
_forwardIndexConfig.getEncodingType().name());
@@ -402,13 +379,6 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
}
- private void trackMaxRowLengthInBytes(Dictionary dictionary, int[] maxRowLengthInBytes, int docId, int dictId) {
- int curSizeOfRow = getInt(_forwardIndexMaxSizeBuffer, docId);
- int newSizeOfEntry = dictionary.getValueSize(dictId) + curSizeOfRow;
- putInt(_forwardIndexMaxSizeBuffer, docId, newSizeOfEntry);
- maxRowLengthInBytes[0] = Math.max(maxRowLengthInBytes[0], newSizeOfEntry);
- }
-
private void writeToForwardIndex(Dictionary dictionary, IndexCreationContext context)
throws IOException {
try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, _forwardIndexConfig)) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
index 7c05a3fa4c5..d95f509ef32 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/bloomfilter/BloomFilterHandler.java
@@ -114,13 +114,7 @@ public class BloomFilterHandler extends BaseIndexHandler {
private void createAndSealBloomFilterForDictionaryColumn(File indexDir, ColumnMetadata columnMetadata,
BloomFilterConfig bloomFilterConfig, SegmentDirectory.Writer segmentWriter)
throws Exception {
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
try (BloomFilterCreator bloomFilterCreator =
StandardIndexes.bloomFilter().createIndexCreator(context, bloomFilterConfig);
Dictionary dictionary = getDictionaryReader(columnMetadata, segmentWriter)) {
@@ -136,13 +130,7 @@ public class BloomFilterHandler extends BaseIndexHandler {
BloomFilterConfig bloomFilterConfig, SegmentDirectory.Writer segmentWriter)
throws Exception {
int numDocs = columnMetadata.getTotalDocs();
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
try (BloomFilterCreator bloomFilterCreator = StandardIndexes.bloomFilter()
.createIndexCreator(context, bloomFilterConfig);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index 3b7eaeba82f..e41c80dc8b8 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -1098,7 +1098,7 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
boolean isSingleValue = fieldSpec.isSingleValueField();
try (
- ForwardIndexCreator forwardIndexCreator = getForwardIndexCreator(fieldSpec, columnStatistics, numDocs, column,
+ ForwardIndexCreator forwardIndexCreator = getForwardIndexCreator(columnStatistics, column,
true)) {
if (isSingleValue) {
for (Object outputValue : outputValues) {
@@ -1128,7 +1128,7 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
boolean isSingleValue = fieldSpec.isSingleValueField();
try (ForwardIndexCreator forwardIndexCreator
- = getForwardIndexCreator(fieldSpec, columnStatistics, numDocs, column, false)) {
+ = getForwardIndexCreator(columnStatistics, column, false)) {
if (isSingleValue) {
for (Object outputValue : outputValues) {
switch (fieldSpec.getDataType().getStoredType()) {
@@ -1199,20 +1199,11 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
0, FieldConfig.EncodingType.RAW, true);
}
- private ForwardIndexCreator getForwardIndexCreator(FieldSpec fieldSpec, ColumnStatistics columnStatistics,
- int numDocs, String column, boolean hasDictionary)
+ private ForwardIndexCreator getForwardIndexCreator(ColumnStatistics columnStatistics,
+ String column, boolean hasDictionary)
throws Exception {
-
- IndexCreationContext indexCreationContext = IndexCreationContext.builder()
- .withIndexDir(_indexDir)
- .withFieldSpec(fieldSpec)
- .withColumnStatistics(columnStatistics)
- .withTotalDocs(numDocs)
- .withDictionary(hasDictionary)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext indexCreationContext =
+ new IndexCreationContext.Builder(_indexDir, _tableConfig, columnStatistics, hasDictionary).build();
ForwardIndexConfig forwardIndexConfig = null;
FieldIndexConfigs fieldIndexConfig = _indexLoadingConfig.getFieldIndexConfig(column);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
index fc12690fb85..199195b4239 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatistics.java
@@ -44,11 +44,11 @@ public class DefaultColumnStatistics implements ColumnStatistics {
_fieldSpec = fieldSpec;
_numDocs = numDocs;
- DataType valueType = fieldSpec.getDataType().getStoredType();
+ DataType storedType = fieldSpec.getDataType().getStoredType();
Object value = fieldSpec.getDefaultNullValue();
int length = 0;
boolean isAscii = false;
- switch (valueType) {
+ switch (storedType) {
case INT:
_valueSet = new int[]{(Integer) value};
break;
@@ -79,11 +79,10 @@ public class DefaultColumnStatistics implements ColumnStatistics {
break;
// TODO: Support MAP
default:
- throw new IllegalArgumentException(
- "Unsupported value type: " + valueType + " for column: " + fieldSpec.getName());
+ throw new IllegalStateException("Unsupported stored type: " + storedType);
}
_value = (Comparable) value;
- _valueLength = valueType.isFixedWidth() ? valueType.size() : length;
+ _valueLength = storedType.isFixedWidth() ? storedType.size() : length;
_isAscii = isAscii;
}
@@ -93,12 +92,17 @@ public class DefaultColumnStatistics implements ColumnStatistics {
}
@Override
- public Comparable getMinValue() {
+ public int getTotalDocs() {
+ return _numDocs;
+ }
+
+ @Override
+ public Comparable<?> getMinValue() {
return _value;
}
@Override
- public Comparable getMaxValue() {
+ public Comparable<?> getMaxValue() {
return _value;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
index a63351161de..a9cfb004a8a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/FSTIndexHandler.java
@@ -198,13 +198,7 @@ public class FSTIndexHandler extends BaseIndexHandler {
LOGGER.info("Creating new FST index for column: {} in segment: {}, cardinality: {}", columnName, segmentName,
columnMetadata.getCardinality());
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
FstIndexConfig config = _fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.fst());
try (FSTIndexCreator fstIndexCreator = StandardIndexes.fst().createIndexCreator(context, config);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
index 32d39f6f263..d81e4d9cc1d 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/H3IndexHandler.java
@@ -203,13 +203,7 @@ public class H3IndexHandler extends BaseIndexHandler {
FieldIndexConfigs colIndexConf = _fieldIndexConfigs.get(columnName);
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
H3IndexConfig config = colIndexConf.getConfig(StandardIndexes.h3());
try (ForwardIndexReader forwardIndexReader = StandardIndexes.forward().getReaderFactory()
@@ -231,13 +225,7 @@ public class H3IndexHandler extends BaseIndexHandler {
throws Exception {
File indexDir = _segmentDirectory.getSegmentMetadata().getIndexDir();
String columnName = columnMetadata.getColumnName();
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
H3IndexConfig config = _fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.h3());
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
try (ForwardIndexReader forwardIndexReader = readerFactory.createIndexReader(segmentWriter,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/IFSTIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/IFSTIndexHandler.java
index 4c7bc3400c2..34a9e406e26 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/IFSTIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/IFSTIndexHandler.java
@@ -164,8 +164,7 @@ public class IFSTIndexHandler extends BaseIndexHandler {
}
LOGGER.info("Creating new IFST index for column: {} in segment: {}, cardinality: {}", columnName, segmentName,
columnMetadata.getCardinality());
- IndexCreationContext context =
- IndexCreationContext.builder().withIndexDir(indexDir).withColumnMetadata(columnMetadata).build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
FstIndexConfig config = _fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.ifst());
try (FSTIndexCreator ifstIndexCreator = StandardIndexes.ifst().createIndexCreator(context, config);
Dictionary dictionary = DictionaryIndexType.read(segmentWriter, columnMetadata)) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
index 967a23ba076..522ff5bc5aa 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java
@@ -163,13 +163,7 @@ public class InvertedIndexHandler extends BaseIndexHandler {
LOGGER.info("Creating new inverted index for segment: {}, column: {}", segmentName, columnName);
int numDocs = columnMetadata.getTotalDocs();
- IndexCreationContext.Common context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
// Raw-forward columns that require an inverted index must have a shared standalone dictionary created first by
// ForwardIndexHandler (which runs before this handler). If we reach this point without a dictionary, it indicates
// a bug in the handler pipeline — fail fast rather than leave the segment in an inconsistent state.
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
index 7821e76ab48..567aed0a0a7 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/JsonIndexHandler.java
@@ -157,13 +157,7 @@ public class JsonIndexHandler extends BaseIndexHandler {
throws Exception {
File indexDir = _segmentDirectory.getSegmentMetadata().getIndexDir();
String columnName = columnMetadata.getColumnName();
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
JsonIndexConfig config = _jsonIndexConfigs.get(columnName);
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
try (ForwardIndexReader forwardIndexReader = readerFactory.createIndexReader(segmentWriter,
@@ -184,13 +178,7 @@ public class JsonIndexHandler extends BaseIndexHandler {
throws Exception {
File indexDir = _segmentDirectory.getSegmentMetadata().getIndexDir();
String columnName = columnMetadata.getColumnName();
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
JsonIndexConfig config = _jsonIndexConfigs.get(columnName);
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
try (ForwardIndexReader forwardIndexReader = readerFactory.createIndexReader(segmentWriter,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
index 30069a601d2..239317ef835 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandler.java
@@ -296,13 +296,7 @@ public class RangeIndexHandler extends BaseIndexHandler {
private CombinedInvertedIndexCreator newRangeIndexCreator(ColumnMetadata columnMetadata)
throws Exception {
File indexDir = _segmentDirectory.getSegmentMetadata().getIndexDir();
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
RangeIndexConfig config = _fieldIndexConfigs.get(columnMetadata.getColumnName())
.getConfig(StandardIndexes.range());
return StandardIndexes.range().createIndexCreator(context, config);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
index a50663e9a47..f2024a15476 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/TextIndexHandler.java
@@ -228,14 +228,8 @@ public class TextIndexHandler extends BaseIndexHandler {
// segmentDirectory is indicated to us by SegmentDirectoryPaths, we create lucene index there. There is no
// further need to move around the lucene index directory since it is created with correct directory structure
// based on segmentVersion.
-
- IndexCreationContext context = IndexCreationContext.builder()
- .withColumnMetadata(columnMetadata)
- .withIndexDir(segmentDirectory)
+ IndexCreationContext context = new IndexCreationContext.Builder(segmentDirectory, _tableConfig, columnMetadata)
.withTextCommitOnClose(true)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
.build();
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
index 2ba9c05f427..7da94d5cecb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/VectorIndexHandler.java
@@ -185,13 +185,8 @@ public class VectorIndexHandler extends BaseIndexHandler {
FieldIndexConfigs colIndexConf = _fieldIndexConfigs.get(columnName);
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(segmentDirectory)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context =
+ new IndexCreationContext.Builder(segmentDirectory, _tableConfig, columnMetadata).build();
VectorIndexConfig config = colIndexConf.getConfig(StandardIndexes.vector());
try (ForwardIndexReader forwardIndexReader = StandardIndexes.forward().getReaderFactory()
@@ -226,13 +221,8 @@ public class VectorIndexHandler extends BaseIndexHandler {
File segmentDirectory = SegmentDirectoryPaths.segmentDirectoryFor(indexDir,
_segmentDirectory.getSegmentMetadata().getVersion());
- IndexCreationContext context = IndexCreationContext.builder()
- .withIndexDir(segmentDirectory)
- .withColumnMetadata(columnMetadata)
- .withTableNameWithType(_tableConfig.getTableName())
- .withContinueOnError(_tableConfig.getIngestionConfig() != null
- && _tableConfig.getIngestionConfig().isContinueOnError())
- .build();
+ IndexCreationContext context =
+ new IndexCreationContext.Builder(segmentDirectory, _tableConfig, columnMetadata).build();
VectorIndexConfig config = _fieldIndexConfigs.get(columnName).getConfig(StandardIndexes.vector());
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
try (ForwardIndexReader forwardIndexReader = readerFactory.createIndexReader(segmentWriter,
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/SimpleColumnMetadata.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/SimpleColumnMetadata.java
index 067a22d78e8..d62d1fd305e 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/SimpleColumnMetadata.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/SimpleColumnMetadata.java
@@ -23,10 +23,8 @@ import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.index.IndexType;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
-import org.apache.pinot.spi.config.table.FieldConfig;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
// TODO: THIS IS A HACK with all stats UNAVAILABLE. Revisit to fill the actual stats for the column.
@@ -40,10 +38,6 @@ public class SimpleColumnMetadata implements ColumnMetadata {
_totalDocs = totalDocs;
}
- public SimpleColumnMetadata(String columnName, DataType dataType, int totalDocs) {
- this(new DimensionFieldSpec(columnName, dataType, true), totalDocs);
- }
-
@Override
public FieldSpec getFieldSpec() {
return _fieldSpec;
@@ -65,8 +59,8 @@ public class SimpleColumnMetadata implements ColumnMetadata {
}
@Override
- public FieldConfig.EncodingType getForwardIndexEncoding() {
- return FieldConfig.EncodingType.RAW;
+ public EncodingType getForwardIndexEncoding() {
+ return EncodingType.RAW;
}
@Override
@@ -76,13 +70,13 @@ public class SimpleColumnMetadata implements ColumnMetadata {
@Nullable
@Override
- public Comparable getMinValue() {
+ public Comparable<?> getMinValue() {
return null;
}
@Nullable
@Override
- public Comparable getMaxValue() {
+ public Comparable<?> getMaxValue() {
return null;
}
@@ -102,17 +96,17 @@ public class SimpleColumnMetadata implements ColumnMetadata {
}
@Override
- public boolean isAscii() {
- return false;
+ public int getTotalNumberOfEntries() {
+ return UNAVAILABLE;
}
@Override
- public int getTotalNumberOfEntries() {
+ public int getMaxNumberOfMultiValues() {
return UNAVAILABLE;
}
@Override
- public int getMaxNumberOfMultiValues() {
+ public int getMaxRowLengthInBytes() {
return UNAVAILABLE;
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/ConcurrentMapPartitionDedupMetadataManagerWithTTLTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/ConcurrentMapPartitionDedupMetadataManagerWithTTLTest.java
index 23df64d0b14..964b97d7d53 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/ConcurrentMapPartitionDedupMetadataManagerWithTTLTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/ConcurrentMapPartitionDedupMetadataManagerWithTTLTest.java
@@ -43,6 +43,7 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
@@ -448,7 +449,7 @@ public class ConcurrentMapPartitionDedupMetadataManagerWithTTLTest {
when(segmentMetadata.getColumnMetadataMap()).thenReturn(new TreeMap<>() {{
this.put(DEDUP_TIME_COLUMN_NAME, columnMetadata);
}});
- when(columnMetadata.getMaxValue()).thenReturn(System.currentTimeMillis());
+ doReturn(System.currentTimeMillis()).when(columnMetadata).getMaxValue();
when(segment.getSegmentMetadata()).thenReturn(segmentMetadata);
// throws when not stopped
assertThrows(RuntimeException.class, () -> metadataManager.addSegment(segment));
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/DedupWithTimestampColumnTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/DedupWithTimestampColumnTest.java
index 2ca4190ce6a..c48dc99749d 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/DedupWithTimestampColumnTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/dedup/DedupWithTimestampColumnTest.java
@@ -38,6 +38,7 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
@@ -213,7 +214,7 @@ public class DedupWithTimestampColumnTest {
when(segmentMetadata.getColumnMetadataMap()).thenReturn(new TreeMap<>() {{
this.put(DEDUP_TIME_COLUMN_NAME, columnMetadata);
}});
- when(columnMetadata.getMaxValue()).thenReturn(currentTimeMillis);
+ doReturn(currentTimeMillis).when(columnMetadata).getMaxValue();
when(segment.getSegmentMetadata()).thenReturn(segmentMetadata);
metadataManager.stop();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java
index 5709ebce941..f524809d5d1 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java
@@ -390,8 +390,8 @@ public class DictionariesTest implements PinotBuffersAfterMethodCheckRule {
Assert.assertFalse(statsCollector.isSorted());
statsCollector.seal();
Assert.assertEquals(statsCollector.getCardinality(), 6);
- Assert.assertEquals((statsCollector.getMinValue()).toString(), "a");
- Assert.assertEquals((statsCollector.getMaxValue()).toString(), "z");
+ Assert.assertEquals(statsCollector.getMinValue().toString(), "a");
+ Assert.assertEquals(statsCollector.getMaxValue().toString(), "z");
Assert.assertFalse(statsCollector.isSorted());
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java
index 0752de3beee..15da6c9acaf 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/AbstractColumnStatisticsCollectorTest.java
@@ -624,7 +624,7 @@ public class AbstractColumnStatisticsCollectorTest {
// Verify the collector uses the correct stored type
DataType expectedStoredType = dataType.getStoredType();
- assertEquals(collector.getValueType(), expectedStoredType);
+ assertEquals(collector.getStoredType(), expectedStoredType);
}
// Test 8: isAscii Tests
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java
index 0cd1d95a333..1adb98ba0de 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatisticsTest.java
@@ -64,7 +64,7 @@ public class EmptyColumnStatisticsTest {
// Identity
assertEquals(stats.getFieldSpec(), fieldSpec);
- assertEquals(stats.getValueType(), storedType);
+ assertEquals(stats.getStoredType(), storedType);
assertEquals(stats.isSingleValue(), sv);
// Empty: no values
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
index 47bd8feee27..58c205ee55f 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactoryTest.java
@@ -22,12 +22,18 @@ package org.apache.pinot.segment.local.segment.index.forward;
import java.io.File;
import java.nio.file.Files;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.mockito.Mockito;
import org.testng.annotations.Test;
import static org.testng.Assert.assertFalse;
@@ -71,13 +77,16 @@ public class ForwardIndexCreatorFactoryTest {
}
private static IndexCreationContext newContext(File indexDir, boolean hasDictionary) {
- FieldSpec fieldSpec = new DimensionFieldSpec(COLUMN_NAME, FieldSpec.DataType.INT, true);
- return IndexCreationContext.builder()
- .withIndexDir(indexDir)
- .withFieldSpec(fieldSpec)
- .withDictionary(hasDictionary)
- .withCardinality(2)
- .withTotalDocs(1)
- .build();
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ FieldSpec fieldSpec = new DimensionFieldSpec(COLUMN_NAME, DataType.INT, true);
+ ColumnMetadata metadata = Mockito.mock(ColumnMetadata.class);
+ Mockito.when(metadata.getFieldSpec()).thenReturn(fieldSpec);
+ Mockito.when(metadata.getCardinality()).thenReturn(2);
+ Mockito.when(metadata.getTotalDocs()).thenReturn(1);
+ Mockito.when(metadata.hasDictionary()).thenReturn(hasDictionary);
+ Mockito.when(metadata.getLengthOfShortestElement()).thenReturn(Integer.BYTES);
+ Mockito.when(metadata.getLengthOfLongestElement()).thenReturn(Integer.BYTES);
+ Mockito.when(metadata.isFixedLength()).thenReturn(true);
+ return new IndexCreationContext.Builder(indexDir, tableConfig, metadata).build();
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
index a5dc39d6f99..f9b03936f50 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
@@ -32,7 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
-import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.io.FileUtils;
@@ -236,7 +235,7 @@ public class ForwardIndexHandlerTest {
private static final List<CompressionCodec> RAW_COMPRESSION_TYPES = Arrays.stream(CompressionCodec.values())
.filter(CompressionCodec::isApplicableToRawIndex)
- .collect(Collectors.toList());
+ .toList();
//@formatter:off
private static final Schema SCHEMA = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index 1bc242a3a9c..9f1a3ad3017 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -1224,8 +1224,8 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
ColumnMetadata originalColumnMetadata = segmentMetadata.getColumnMetadataFor(COLUMN1_NAME);
assertEquals(columnMetadata.getCardinality(), originalColumnMetadata.getCardinality());
assertEquals(columnMetadata.isSorted(), originalColumnMetadata.isSorted());
- assertEquals(columnMetadata.getMinValue(), (int) originalColumnMetadata.getMinValue() + 1);
- assertEquals(columnMetadata.getMaxValue(), (int) originalColumnMetadata.getMaxValue() + 1);
+ assertEquals(columnMetadata.getMinValue(), (Integer) originalColumnMetadata.getMinValue() + 1);
+ assertEquals(columnMetadata.getMaxValue(), (Integer) originalColumnMetadata.getMaxValue() + 1);
assertEquals(columnMetadata.getBitsPerElement(), originalColumnMetadata.getBitsPerElement());
assertTrue(columnMetadata.isAutoGenerated());
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
index b6340428b65..9ff9db8055f 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/DefaultColumnStatisticsTest.java
@@ -64,9 +64,9 @@ public class DefaultColumnStatisticsTest {
// Assertions follow ColumnStatistics API order
- // getFieldSpec, getValueType, isSingleValue
+ // getFieldSpec, getStoredType, isSingleValue
assertEquals(stats.getFieldSpec(), fieldSpec);
- assertEquals(stats.getValueType(), storedType);
+ assertEquals(stats.getStoredType(), storedType);
assertTrue(stats.isSingleValue());
// getMinValue, getMaxValue — single default value: min == max
@@ -125,9 +125,9 @@ public class DefaultColumnStatisticsTest {
// Assertions follow ColumnStatistics API order
- // getFieldSpec, getValueType, isSingleValue
+ // getFieldSpec, getStoredType, isSingleValue
assertEquals(stats.getFieldSpec(), fieldSpec);
- assertEquals(stats.getValueType(), storedType);
+ assertEquals(stats.getStoredType(), storedType);
assertFalse(stats.isSingleValue());
// getMinValue, getMaxValue — single default value: min == max
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
index e9cd61dace2..bbba657b985 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerTest.java
@@ -1009,7 +1009,7 @@ public class ConcurrentMapPartitionUpsertMetadataManagerTest {
when(segmentMetadata.getColumnMetadataMap()).thenReturn(new TreeMap() {{
this.put(comparisonColumns.get(0), columnMetadata);
}});
- when(columnMetadata.getMaxValue()).thenReturn(endTime);
+ doReturn(endTime).when(columnMetadata).getMaxValue();
if (snapshot != null) {
when(segment.loadDocIdsFromSnapshot(V1Constants.VALID_DOC_IDS_SNAPSHOT_FILE_NAME)).thenReturn(snapshot);
} else {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
index a5ea5c6fb58..940a6bcc80e 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
@@ -20,49 +20,16 @@ package org.apache.pinot.segment.spi;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.Set;
-import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.index.IndexType;
-import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.annotations.InterfaceAudience;
import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
-import org.apache.pinot.spi.data.FieldSpec.FieldType;
/// The `ColumnMetadata` class holds the column level management information and data statistics.
@InterfaceAudience.Private
-@SuppressWarnings("rawtypes")
-public interface ColumnMetadata {
+public interface ColumnMetadata extends ColumnShape {
int UNAVAILABLE = -1;
- FieldSpec getFieldSpec();
-
- default String getColumnName() {
- return getFieldSpec().getName();
- }
-
- default FieldType getFieldType() {
- return getFieldSpec().getFieldType();
- }
-
- default DataType getDataType() {
- return getFieldSpec().getDataType();
- }
-
- default boolean isSingleValue() {
- return getFieldSpec().isSingleValueField();
- }
-
- /// Returns the total number of documents (rows).
- int getTotalDocs();
-
- /// Returns the cardinality of the column.
- /// NOTE: When a realtime segment has no-dictionary columns, the cardinality for those columns will be set to
- /// [Constants#UNKNOWN_CARDINALITY].
- int getCardinality();
-
/// Returns `true` when the column has a dictionary, `false` otherwise.
@JsonProperty("hasDictionary")
boolean hasDictionary();
@@ -72,48 +39,12 @@ public interface ColumnMetadata {
/// derive it from [#hasDictionary()].
EncodingType getForwardIndexEncoding();
- /// Returns `true` when all values in a SV column are sorted in ascending order, `false` otherwise.
- boolean isSorted();
-
- /// Returns the minimum value, or `null` if the value is unavailable.
- @Nullable
- Comparable getMinValue();
-
- /// Returns the maximum value, or `null` if the value is unavailable.
- @Nullable
- Comparable getMaxValue();
-
/// Returns `true` when both min and max value are invalid, and there is no need to regenerate them.
boolean isMinMaxValueInvalid();
- /// Returns the minimum serialized byte length across all elements. For fixed-width types, returns the fixed size.
- int getLengthOfShortestElement();
-
- /// Returns the maximum serialized byte length across all elements. For fixed-width types, returns the fixed size.
- int getLengthOfLongestElement();
-
- /// Returns `true` when all elements of a STRING column contain only ASCII characters.
- boolean isAscii();
-
- /// Returns the total number of entries.
- /// - For SV columns, equals the number of rows
- /// - For MV columns, sums the number of elements across all rows
- int getTotalNumberOfEntries();
-
- /// Returns the maximum number of elements within a single MV value, or `0` for SV columns.
- int getMaxNumberOfMultiValues();
-
/// For dictionary-encoded columns, returns the bits required for each element, [#UNAVAILABLE] otherwise.
int getBitsPerElement();
- /// Returns the [PartitionFunction] for the column.
- @Nullable
- PartitionFunction getPartitionFunction();
-
- /// Returns the partitions within which the values exist.
- @Nullable
- Set<Integer> getPartitions();
-
/// Returns `true` when the column is auto-generated by the default column handler.
boolean isAutoGenerated();
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnShape.java
similarity index 52%
copy from pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java
copy to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnShape.java
index 658539dbbf0..d99f63c7b97 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnShape.java
@@ -16,28 +16,40 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.spi.creator;
+package org.apache.pinot.segment.spi;
-import java.io.Serializable;
-import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.FieldSpec.FieldType;
-/// Statistics for a column, collected before creating indexes.
-/// For MV column, the stats are on individual elements within the MV values.
-@SuppressWarnings("rawtypes")
-public interface ColumnStatistics extends Serializable {
+/// The shape (identity, value distribution, and element layout) of a column.
+public interface ColumnShape {
/// Returns the [FieldSpec] of the column.
FieldSpec getFieldSpec();
+ /// Returns the name of the column.
+ default String getColumnName() {
+ return getFieldSpec().getName();
+ }
+
+ /// Returns the [FieldType] of the column.
+ default FieldType getFieldType() {
+ return getFieldSpec().getFieldType();
+ }
+
+ /// Returns the [DataType] of the column.
+ default DataType getDataType() {
+ return getFieldSpec().getDataType();
+ }
+
/// Returns the stored [DataType] of the column.
- default DataType getValueType() {
- return getFieldSpec().getDataType().getStoredType();
+ default DataType getStoredType() {
+ return getDataType().getStoredType();
}
/// Returns whether the column is single-valued.
@@ -45,109 +57,83 @@ public interface ColumnStatistics extends Serializable {
return getFieldSpec().isSingleValueField();
}
- /// Returns the minimum value, or `null` if the column is empty.
- @Nullable
- Comparable getMinValue();
+ /// Returns the total number of rows (documents).
+ int getTotalDocs();
+
+ /// Returns the cardinality of the column, or [Constants#UNKNOWN_CARDINALITY] if unavailable.
+ int getCardinality();
+
+ /// Returns `true` when all values in a SV column are sorted in ascending order, `false` otherwise.
+ boolean isSorted();
- /// Returns the maximum value, or `null` if the column is empty.
+ /// Returns the minimum value, or `null` if unavailable.
@Nullable
- Comparable getMaxValue();
-
- /// Returns a sorted (ascending) array of all unique values for non-empty dictionary-encoded columns, `null`
- /// otherwise.
- /// - INT: int[]
- /// - LONG: long[]
- /// - FLOAT: float[]
- /// - DOUBLE: double[]
- /// - BIG_DECIMAL: BigDecimal[]
- /// - STRING: String[]
- /// - BYTES: ByteArray[]
+ Comparable<?> getMinValue();
+
+ /// Returns the maximum value, or `null` if unavailable.
@Nullable
- Object getUniqueValuesSet();
-
- /// Returns the cardinality of the column.
- /// - For dictionary-encoded columns, returns the exact cardinality.
- /// - For raw columns:
- /// - When stats are collected from an input file, returns an estimated cardinality
- /// - When stats are derived from a mutable segment, returns
- /// [org.apache.pinot.segment.spi.Constants#UNKNOWN_CARDINALITY]
- // TODO: Consider generating estimated cardinality when converting from a mutable segment
- int getCardinality();
+ Comparable<?> getMaxValue();
/// Returns the minimum serialized byte length across all elements.
/// - For fixed-width types, returns the fixed size
/// - For empty column of var-length types, returns `0`
+ /// - A negative value indicates the length is unavailable (e.g., raw var-width columns from pre-1.6.0 segments)
/// Override it for var-length types.
default int getLengthOfShortestElement() {
- return getValueType().size();
+ return getStoredType().size();
}
/// Returns the maximum serialized byte length across all elements.
/// - For fixed-width types, returns the fixed size
/// - For empty column of var-length types, returns `0`
+ /// - A negative value indicates the length is unavailable (e.g., raw var-width columns from pre-1.6.0 segments)
/// Override it for var-length types.
default int getLengthOfLongestElement() {
- return getValueType().size();
- }
-
- @Deprecated
- default int getLengthOfLargestElement() {
- return getLengthOfLongestElement();
+ return getStoredType().size();
}
/// Returns `true` when all the elements are of the same serialized byte length.
+ /// Returns `false` when either shortest or longest is unavailable.
default boolean isFixedLength() {
- return getLengthOfShortestElement() == getLengthOfLongestElement();
+ int shortest = getLengthOfShortestElement();
+ return shortest >= 0 && shortest == getLengthOfLongestElement();
}
- /// Returns `true` when all elements of a non-empty STRING column contain only ASCII characters, `false` otherwise.
+ /// Returns `true` when all elements of a STRING column contain only ASCII characters, `false` otherwise.
/// Override it for STRING type.
default boolean isAscii() {
return false;
}
- /// Returns `true` when all values in a SV column are sorted in ascending order, `false` otherwise.
- boolean isSorted();
-
/// Returns the total number of entries.
/// - For SV columns, equals the number of rows
/// - For MV columns, sums the number of elements across all rows
int getTotalNumberOfEntries();
- /// Returns the maximum number of elements within a single MV value, or `0` for SV or empty columns.
+ /// Returns the maximum number of elements within a single MV value, or `0` for SV columns.
int getMaxNumberOfMultiValues();
/// Returns the maximum serialized byte length of a single row.
- /// - For SV columns, row length is the same as value length
- /// - For MV columns, row length is total length of all elements within the MV value
+ /// - For SV columns, equals [#getLengthOfLongestElement]
+ /// - For MV columns, equals the sum of element lengths in the longest row
/// - Returns `0` for empty columns
- /// Override it for MV var-length types.
+ /// - A negative value indicates the length is unavailable (e.g., var-width MV columns with varying element
+ /// lengths read from [ColumnMetadata])
+ /// The default formula `maxNumberOfMultiValues * storedType.size()` handles SV and fixed-width MV columns.
+ /// Override it for var-width MV columns.
default int getMaxRowLengthInBytes() {
if (isSingleValue()) {
return getLengthOfLongestElement();
} else {
- return getMaxNumberOfMultiValues() * getValueType().size();
+ return getMaxNumberOfMultiValues() * getStoredType().size();
}
}
- /// Returns the [PartitionFunction] for the column.
+ /// Returns the [PartitionFunction] for the column, or `null` if the column is not partitioned.
@Nullable
PartitionFunction getPartitionFunction();
- /// Returns the partitions within which the values exist.
+ /// Returns the partitions within which the values exist, or `null` if the column is not partitioned.
@Nullable
Set<Integer> getPartitions();
-
- @Deprecated
- default int getNumPartitions() {
- PartitionFunction partitionFunction = getPartitionFunction();
- return partitionFunction != null ? partitionFunction.getNumPartitions() : 0;
- }
-
- @Deprecated
- @Nullable
- default Map<String, String> getPartitionFunctionConfig() {
- PartitionFunction partitionFunction = getPartitionFunction();
- return partitionFunction != null ? partitionFunction.getFunctionConfig() : null;
- }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/EmptyColumnShape.java
similarity index 75%
copy from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java
copy to pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/EmptyColumnShape.java
index a791fa3f039..92a1f47920f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/EmptyColumnStatistics.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/EmptyColumnShape.java
@@ -16,25 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.segment.creator.impl.stats;
+package org.apache.pinot.segment.spi;
import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.pinot.segment.spi.creator.ColumnStatistics;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
-/// Column statistics for an empty column (zero rows).
-@SuppressWarnings("rawtypes")
-public class EmptyColumnStatistics implements ColumnStatistics {
+/// [ColumnShape] for an empty column (zero rows).
+public class EmptyColumnShape implements ColumnShape {
private final FieldSpec _fieldSpec;
private final PartitionFunction _partitionFunction;
private final Set<Integer> _partitions;
// TODO: Revisit if we need to maintain partition info for empty columns
- public EmptyColumnStatistics(FieldSpec fieldSpec, @Nullable PartitionFunction partitionFunction,
+ public EmptyColumnShape(FieldSpec fieldSpec, @Nullable PartitionFunction partitionFunction,
@Nullable Set<Integer> partitions) {
_fieldSpec = fieldSpec;
_partitionFunction = partitionFunction;
@@ -46,44 +44,43 @@ public class EmptyColumnStatistics implements ColumnStatistics {
return _fieldSpec;
}
- @Nullable
@Override
- public Comparable getMinValue() {
- return null;
+ public int getTotalDocs() {
+ return 0;
}
- @Nullable
@Override
- public Comparable getMaxValue() {
- return null;
+ public int getCardinality() {
+ return 0;
+ }
+
+ @Override
+ public boolean isSorted() {
+ return isSingleValue();
}
@Nullable
@Override
- public Object getUniqueValuesSet() {
+ public Comparable<?> getMinValue() {
return null;
}
+ @Nullable
@Override
- public int getCardinality() {
- return 0;
+ public Comparable<?> getMaxValue() {
+ return null;
}
@Override
public int getLengthOfShortestElement() {
- DataType valueType = getValueType();
- return valueType.isFixedWidth() ? valueType.size() : 0;
+ DataType storedType = getStoredType();
+ return storedType.isFixedWidth() ? storedType.size() : 0;
}
@Override
public int getLengthOfLongestElement() {
- DataType valueType = getValueType();
- return valueType.isFixedWidth() ? valueType.size() : 0;
- }
-
- @Override
- public boolean isSorted() {
- return isSingleValue();
+ DataType storedType = getStoredType();
+ return storedType.isFixedWidth() ? storedType.size() : 0;
}
@Override
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java
index 658539dbbf0..2a268e81dd5 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/ColumnStatistics.java
@@ -20,38 +20,15 @@ package org.apache.pinot.segment.spi.creator;
import java.io.Serializable;
import java.util.Map;
-import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.ColumnShape;
import org.apache.pinot.segment.spi.partition.PartitionFunction;
-import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
/// Statistics for a column, collected before creating indexes.
/// For MV column, the stats are on individual elements within the MV values.
-@SuppressWarnings("rawtypes")
-public interface ColumnStatistics extends Serializable {
-
- /// Returns the [FieldSpec] of the column.
- FieldSpec getFieldSpec();
-
- /// Returns the stored [DataType] of the column.
- default DataType getValueType() {
- return getFieldSpec().getDataType().getStoredType();
- }
-
- /// Returns whether the column is single-valued.
- default boolean isSingleValue() {
- return getFieldSpec().isSingleValueField();
- }
-
- /// Returns the minimum value, or `null` if the column is empty.
- @Nullable
- Comparable getMinValue();
-
- /// Returns the maximum value, or `null` if the column is empty.
- @Nullable
- Comparable getMaxValue();
+public interface ColumnStatistics extends ColumnShape, Serializable {
/// Returns a sorted (ascending) array of all unique values for non-empty dictionary-encoded columns, `null`
/// otherwise.
@@ -65,86 +42,23 @@ public interface ColumnStatistics extends Serializable {
@Nullable
Object getUniqueValuesSet();
- /// Returns the cardinality of the column.
- /// - For dictionary-encoded columns, returns the exact cardinality.
- /// - For raw columns:
- /// - When stats are collected from an input file, returns an estimated cardinality
- /// - When stats are derived from a mutable segment, returns
- /// [org.apache.pinot.segment.spi.Constants#UNKNOWN_CARDINALITY]
- // TODO: Consider generating estimated cardinality when converting from a mutable segment
- int getCardinality();
-
- /// Returns the minimum serialized byte length across all elements.
- /// - For fixed-width types, returns the fixed size
- /// - For empty column of var-length types, returns `0`
- /// Override it for var-length types.
- default int getLengthOfShortestElement() {
- return getValueType().size();
- }
-
- /// Returns the maximum serialized byte length across all elements.
- /// - For fixed-width types, returns the fixed size
- /// - For empty column of var-length types, returns `0`
- /// Override it for var-length types.
- default int getLengthOfLongestElement() {
- return getValueType().size();
+ @Deprecated(since = "1.6.0", forRemoval = true)
+ default DataType getValueType() {
+ return getStoredType();
}
- @Deprecated
+ @Deprecated(since = "1.6.0", forRemoval = true)
default int getLengthOfLargestElement() {
return getLengthOfLongestElement();
}
- /// Returns `true` when all the elements are of the same serialized byte length.
- default boolean isFixedLength() {
- return getLengthOfShortestElement() == getLengthOfLongestElement();
- }
-
- /// Returns `true` when all elements of a non-empty STRING column contain only ASCII characters, `false` otherwise.
- /// Override it for STRING type.
- default boolean isAscii() {
- return false;
- }
-
- /// Returns `true` when all values in a SV column are sorted in ascending order, `false` otherwise.
- boolean isSorted();
-
- /// Returns the total number of entries.
- /// - For SV columns, equals the number of rows
- /// - For MV columns, sums the number of elements across all rows
- int getTotalNumberOfEntries();
-
- /// Returns the maximum number of elements within a single MV value, or `0` for SV or empty columns.
- int getMaxNumberOfMultiValues();
-
- /// Returns the maximum serialized byte length of a single row.
- /// - For SV columns, row length is the same as value length
- /// - For MV columns, row length is total length of all elements within the MV value
- /// - Returns `0` for empty columns
- /// Override it for MV var-length types.
- default int getMaxRowLengthInBytes() {
- if (isSingleValue()) {
- return getLengthOfLongestElement();
- } else {
- return getMaxNumberOfMultiValues() * getValueType().size();
- }
- }
-
- /// Returns the [PartitionFunction] for the column.
- @Nullable
- PartitionFunction getPartitionFunction();
-
- /// Returns the partitions within which the values exist.
- @Nullable
- Set<Integer> getPartitions();
-
- @Deprecated
+ @Deprecated(since = "1.6.0", forRemoval = true)
default int getNumPartitions() {
PartitionFunction partitionFunction = getPartitionFunction();
return partitionFunction != null ? partitionFunction.getNumPartitions() : 0;
}
- @Deprecated
+ @Deprecated(since = "1.6.0", forRemoval = true)
@Nullable
default Map<String, String> getPartitionFunctionConfig() {
PartitionFunction partitionFunction = getPartitionFunction();
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
index 7fffc62630d..0eb838934bb 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/IndexCreationContext.java
@@ -19,215 +19,191 @@
package org.apache.pinot.segment.spi.creator;
import java.io.File;
-import java.util.Objects;
+import java.util.Set;
+import javax.annotation.Nullable;
import org.apache.pinot.segment.spi.ColumnMetadata;
-import org.apache.pinot.segment.spi.index.IndexType;
-import org.apache.pinot.spi.config.table.IndexConfig;
-import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.segment.spi.ColumnShape;
+import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.ingestion.IngestionConfig;
import org.apache.pinot.spi.data.FieldSpec;
-/**
- * Provides parameters for constructing indexes via
- * {@link IndexType#createIndexCreator(IndexCreationContext, IndexConfig)}.
- * The responsibility for ensuring that the correct parameters for a particular
- * index type lies with the caller.
- */
-public interface IndexCreationContext {
+/// Context of a column for index creation. Inherits the [ColumnShape] accessors and sources shape values from either
+/// [ColumnStatistics] (segment-creation path) or [ColumnMetadata] (index-handler path) via [Builder].
+public interface IndexCreationContext extends ColumnShape {
- FieldSpec getFieldSpec();
+ // Identity.
+ /// Returns the directory where the index file(s) should be written.
File getIndexDir();
- boolean isOnHeap();
-
- int getLengthOfLongestEntry();
-
- int getMaxNumberOfMultiValueElements();
+ /// Returns the fully-qualified table name (e.g. `myTable_OFFLINE`).
+ String getTableNameWithType();
- int getMaxRowLengthInBytes();
+ /// Returns the source [ColumnStatistics] on the segment-creation path, or `null` on the index-handler path.
+ @Nullable
+ ColumnStatistics getColumnStatistics();
- boolean isSorted();
+ /// Returns the source [ColumnMetadata] on the index-handler path, or `null` on the segment-creation path.
+ @Nullable
+ ColumnMetadata getColumnMetadata();
- int getCardinality();
+ // Overridable column attributes (IndexCreationContext-specific; not on ColumnShape).
- int getTotalNumberOfEntries();
+ /// Returns `true` when the column has a dictionary.
+ boolean hasDictionary();
- int getTotalDocs();
+ // Build-time toggles.
- boolean hasDictionary();
+ /// Returns `true` when index buffers should be allocated on-heap, `false` for off-heap (memory-mapped).
+ boolean isOnHeap();
- Comparable<?> getMinValue();
-
- Comparable<?> getMaxValue();
-
- boolean forwardIndexDisabled();
-
- /**
- * Returns a sorted array with the unique values for the associated column.
- *
- * Primitive types will be stored in an unboxed array (ie if the column contains {@code int}s, this method returns an
- * {@code int[]}).
- *
- * This is an abstraction leak from Text and FST indexes.
- */
- Object getSortedUniqueElementsArray();
-
- /**
- * This could be set for all metrics in {@link IndexingConfig#isOptimizeDictionary()} or only when the field is a
- * metric with {@link IndexingConfig#isOptimizeDictionaryForMetrics()}, in which case this method will only return
- * true if the column is a metric.
- *
- * Therefore the caller code doesn't need to verify the later condition.
- */
+ /// Returns `true` when the dictionary-optimization pass selected this column for raw encoding.
boolean isOptimizeDictionary();
- boolean isFixedLength();
-
- /**
- * This is an abstraction leak from TextIndexType.
- * @return
- */
+ /// Returns `true` when the text-index creator should commit the underlying Lucene index on close.
boolean isTextCommitOnClose();
- ColumnStatistics getColumnStatistics();
- /**
- * This flags whether the index creation is done during realtime segment conversion
- * @return
- */
+ // Realtime-conversion context.
+
+ /// Returns `true` when this index creation is part of a realtime → offline segment conversion.
boolean isRealtimeConversion();
- /**
- * Used in conjunction with isRealtimeConversion, this returns the location of the consumer directory used
- */
+ /// Returns the realtime consumer directory; only meaningful when [#isRealtimeConversion] is `true`.
+ @Nullable
File getConsumerDir();
+ /// Returns `true` when the source mutable segment was compacted prior to conversion; only meaningful when
+ /// [#isRealtimeConversion] is `true`.
boolean isMutableSegmentCompacted();
- /**
- * This contains immutableToMutableIdMap mapping generated in {@link SegmentIndexCreationDriver}
- *
- * This allows for index creation during realtime segment conversion to take advantage of mutable to immutable
- * docId mapping
- * @return
- */
+ /// Returns the mutable-to-immutable docId mapping generated during realtime segment conversion, or `null` when not
+ /// applicable.
+ @Nullable
int[] getMutableToImmutableDocIdMap();
- String getTableNameWithType();
+ // Error handling.
+ /// Returns `true` when index creation should continue past errors instead of aborting the whole segment.
boolean isContinueOnError();
+ @Deprecated(since = "1.6.0", forRemoval = true)
+ default int getLengthOfLongestEntry() {
+ return getLengthOfLongestElement();
+ }
+
+ @Deprecated(since = "1.6.0", forRemoval = true)
+ default int getMaxNumberOfMultiValueElements() {
+ return getMaxNumberOfMultiValues();
+ }
+
+ @Deprecated(since = "1.6.0", forRemoval = true)
+ @Nullable
+ default Object getSortedUniqueElementsArray() {
+ ColumnStatistics columnStatistics = getColumnStatistics();
+ return columnStatistics != null ? columnStatistics.getUniqueValuesSet() : null;
+ }
+
+ @SuppressWarnings("UnusedReturnValue")
final class Builder {
- private ColumnStatistics _columnStatistics;
- private File _indexDir;
- private int _lengthOfLongestEntry;
- private int _maxNumberOfMultiValueElements;
- private int _maxRowLengthInBytes;
- private boolean _onHeap = false;
- private FieldSpec _fieldSpec;
- private boolean _sorted;
- private int _cardinality;
+ // Identity. Non-overridable shape accessors delegate to `_columnShape`.
+ private final File _indexDir;
+ private final String _tableNameWithType;
+ private final ColumnShape _columnShape;
+
+ // Overridable column attributes. Source-derived defaults populated in the constructors below;
+ // handlers may override via `withXxx` setters.
+ private int _lengthOfShortestElement;
+ private int _lengthOfLongestElement;
+ private boolean _isAscii;
private int _totalNumberOfEntries;
- private int _totalDocs;
- private boolean _hasDictionary = true;
- private Comparable<?> _minValue;
- private Comparable<?> _maxValue;
- private boolean _forwardIndexDisabled;
- private Object _sortedUniqueElementsArray;
+ private int _maxNumberOfMultiValues;
+ private int _maxRowLengthInBytes;
+ private boolean _hasDictionary;
+
+ // Build-time toggles.
+ private boolean _onHeap;
private boolean _optimizedDictionary;
- private boolean _fixedLength;
private boolean _textCommitOnClose;
+
+ // Realtime-conversion context (meaningful only when `_realtimeConversion` is true).
private boolean _realtimeConversion;
+ @Nullable
private File _consumerDir;
private boolean _mutableSegmentCompacted;
+ @Nullable
private int[] _mutableToImmutableDocIdMap;
- private String _tableNameWithType;
- private boolean _continueOnError;
-
- public Builder withColumnStatistics(ColumnStatistics columnStatistics) {
- _columnStatistics = columnStatistics;
- _minValue = (Comparable<?>) columnStatistics.getMinValue();
- _maxValue = (Comparable<?>) columnStatistics.getMaxValue();
- _sortedUniqueElementsArray = columnStatistics.getUniqueValuesSet();
- _cardinality = columnStatistics.getCardinality();
- _lengthOfLongestEntry = columnStatistics.getLengthOfLongestElement();
- _fixedLength = columnStatistics.isFixedLength();
- _sorted = columnStatistics.isSorted();
- _totalNumberOfEntries = columnStatistics.getTotalNumberOfEntries();
- _maxNumberOfMultiValueElements = columnStatistics.getMaxNumberOfMultiValues();
- _maxRowLengthInBytes = columnStatistics.getMaxRowLengthInBytes();
- return this;
- }
- public Builder withIndexDir(File indexDir) {
- _indexDir = indexDir;
- return this;
- }
-
- public Builder onHeap(boolean onHeap) {
- _onHeap = onHeap;
- return this;
- }
+ // Error handling.
+ private boolean _continueOnError;
- public Builder withColumnMetadata(ColumnMetadata columnMetadata) {
- return withFieldSpec(columnMetadata.getFieldSpec())
- .sorted(columnMetadata.isSorted())
- .withCardinality(columnMetadata.getCardinality())
- .withTotalNumberOfEntries(columnMetadata.getTotalNumberOfEntries())
- .withTotalDocs(columnMetadata.getTotalDocs())
- .withDictionary(columnMetadata.hasDictionary())
- .withMinValue(columnMetadata.getMinValue())
- .withMaxValue(columnMetadata.getMaxValue())
- .withMaxNumberOfMultiValueElements(columnMetadata.getMaxNumberOfMultiValues());
+ /// Segment-creation path. Shape values come from the freshly-collected [ColumnStatistics]; `hasDictionary` is
+ /// supplied separately because it's a driver decision not carried on [ColumnStatistics]. `_tableNameWithType`
+ /// and `_continueOnError` are derived from `tableConfig`.
+ public Builder(File indexDir, TableConfig tableConfig, ColumnStatistics columnStatistics, boolean hasDictionary) {
+ this(indexDir, tableConfig.getTableName(), columnStatistics, hasDictionary, isContinueOnError(tableConfig));
}
- public Builder withOptimizedDictionary(boolean optimized) {
- _optimizedDictionary = optimized;
- return this;
+ /// Index-handler path. Shape values (including `hasDictionary`) come from [ColumnMetadata]; `_tableNameWithType`
+ /// and `_continueOnError` are derived from `tableConfig`.
+ public Builder(File indexDir, TableConfig tableConfig, ColumnMetadata columnMetadata) {
+ this(indexDir, tableConfig.getTableName(), columnMetadata, columnMetadata.hasDictionary(),
+ isContinueOnError(tableConfig));
}
- public Builder withFixedLength(boolean fixedLength) {
- _fixedLength = fixedLength;
- return this;
+ /// Generic constructor for callers that have a [ColumnShape] but not a [TableConfig] — every value the
+ /// convenience constructors would derive (`tableNameWithType`, `hasDictionary`, `continueOnError`) must be
+ /// supplied explicitly.
+ public Builder(File indexDir, String tableNameWithType, ColumnShape columnShape, boolean hasDictionary,
+ boolean continueOnError) {
+ _indexDir = indexDir;
+ _tableNameWithType = tableNameWithType;
+ _columnShape = columnShape;
+ _lengthOfShortestElement = columnShape.getLengthOfShortestElement();
+ _lengthOfLongestElement = columnShape.getLengthOfLongestElement();
+ _isAscii = columnShape.isAscii();
+ _totalNumberOfEntries = columnShape.getTotalNumberOfEntries();
+ _maxNumberOfMultiValues = columnShape.getMaxNumberOfMultiValues();
+ _maxRowLengthInBytes = columnShape.getMaxRowLengthInBytes();
+ _hasDictionary = hasDictionary;
+ _continueOnError = continueOnError;
}
- public Builder withLengthOfLongestEntry(int lengthOfLongestEntry) {
- _lengthOfLongestEntry = lengthOfLongestEntry;
- return this;
+ private static boolean isContinueOnError(TableConfig tableConfig) {
+ IngestionConfig ingestionConfig = tableConfig.getIngestionConfig();
+ return ingestionConfig != null && ingestionConfig.isContinueOnError();
}
- public Builder withMaxNumberOfMultiValueElements(int maxNumberOfMultiValueElements) {
- _maxNumberOfMultiValueElements = maxNumberOfMultiValueElements;
- return this;
- }
+ // Overridable column-attribute setters.
- public Builder withMaxRowLengthInBytes(int maxRowLengthInBytes) {
- _maxRowLengthInBytes = maxRowLengthInBytes;
+ public Builder withLengthOfShortestElement(int lengthOfShortestElement) {
+ _lengthOfShortestElement = lengthOfShortestElement;
return this;
}
- public Builder withFieldSpec(FieldSpec fieldSpec) {
- _fieldSpec = fieldSpec;
+ public Builder withLengthOfLongestElement(int lengthOfLongestElement) {
+ _lengthOfLongestElement = lengthOfLongestElement;
return this;
}
- public Builder sorted(boolean sorted) {
- _sorted = sorted;
+ public Builder withAscii(boolean isAscii) {
+ _isAscii = isAscii;
return this;
}
- public Builder withCardinality(int cardinality) {
- _cardinality = cardinality;
+ public Builder withTotalNumberOfEntries(int totalNumberOfEntries) {
+ _totalNumberOfEntries = totalNumberOfEntries;
return this;
}
- public Builder withTotalNumberOfEntries(int totalNumberOfEntries) {
- _totalNumberOfEntries = totalNumberOfEntries;
+ public Builder withMaxNumberOfMultiValues(int maxNumberOfMultiValues) {
+ _maxNumberOfMultiValues = maxNumberOfMultiValues;
return this;
}
- public Builder withTotalDocs(int totalDocs) {
- _totalDocs = totalDocs;
+ public Builder withMaxRowLengthInBytes(int maxRowLengthInBytes) {
+ _maxRowLengthInBytes = maxRowLengthInBytes;
return this;
}
@@ -236,18 +212,15 @@ public interface IndexCreationContext {
return this;
}
- public Builder withMinValue(Comparable<?> minValue) {
- _minValue = minValue;
- return this;
- }
+ // Build-time toggle setters.
- public Builder withMaxValue(Comparable<?> maxValue) {
- _maxValue = maxValue;
+ public Builder withOnHeap(boolean onHeap) {
+ _onHeap = onHeap;
return this;
}
- public Builder withForwardIndexDisabled(boolean forwardIndexDisabled) {
- _forwardIndexDisabled = forwardIndexDisabled;
+ public Builder withOptimizedDictionary(boolean optimized) {
+ _optimizedDictionary = optimized;
return this;
}
@@ -256,6 +229,8 @@ public interface IndexCreationContext {
return this;
}
+ // Realtime-conversion context setters.
+
public Builder withRealtimeConversion(boolean realtimeConversion) {
_realtimeConversion = realtimeConversion;
return this;
@@ -276,186 +251,205 @@ public interface IndexCreationContext {
return this;
}
- public Builder withTableNameWithType(String tableNameWithType) {
- _tableNameWithType = tableNameWithType;
- return this;
- }
+ // Error handling setter.
+ /// Overrides the `continueOnError` value supplied at construction. Useful when the
+ /// [TableConfig]-derived default doesn't match the caller's needs (e.g. a [SegmentGeneratorConfig] override).
public Builder withContinueOnError(boolean continueOnError) {
_continueOnError = continueOnError;
return this;
}
public Common build() {
- return new Common(Objects.requireNonNull(_indexDir), _lengthOfLongestEntry, _maxNumberOfMultiValueElements,
- _maxRowLengthInBytes, _onHeap, Objects.requireNonNull(_fieldSpec), _sorted, _cardinality,
- _totalNumberOfEntries, _totalDocs, _hasDictionary, _minValue, _maxValue, _forwardIndexDisabled,
- _sortedUniqueElementsArray, _optimizedDictionary, _fixedLength, _textCommitOnClose, _columnStatistics,
- _realtimeConversion, _consumerDir, _mutableSegmentCompacted, _mutableToImmutableDocIdMap, _tableNameWithType,
- _continueOnError);
+ return new Common(this);
}
-
- public Builder withSortedUniqueElementsArray(Object sortedUniqueElementsArray) {
- _sortedUniqueElementsArray = sortedUniqueElementsArray;
- return this;
- }
- }
-
- static Builder builder() {
- return new Builder();
}
final class Common implements IndexCreationContext {
+ // Identity. Non-overridable shape accessors delegate to `_columnShape`.
+ private final String _tableNameWithType;
private final File _indexDir;
- private final int _lengthOfLongestEntry;
- private final int _maxNumberOfMultiValueElements;
- private final int _maxRowLengthInBytes;
- private final boolean _onHeap;
- private final FieldSpec _fieldSpec;
- private final boolean _sorted;
- private final int _cardinality;
+ private final ColumnShape _columnShape;
+
+ // Overridable column attributes.
+ private final int _lengthOfShortestElement;
+ private final int _lengthOfLongestElement;
+ private final boolean _isAscii;
private final int _totalNumberOfEntries;
- private final int _totalDocs;
+ private final int _maxNumberOfMultiValues;
+ private final int _maxRowLengthInBytes;
private final boolean _hasDictionary;
- private final Comparable<?> _minValue;
- private final Comparable<?> _maxValue;
- private final boolean _forwardIndexDisabled;
- private final Object _sortedUniqueElementsArray;
+
+ // Build-time toggles.
+ private final boolean _onHeap;
private final boolean _optimizeDictionary;
- private final boolean _fixedLength;
private final boolean _textCommitOnClose;
- private final ColumnStatistics _columnStatistics;
+
+ // Realtime-conversion context.
private final boolean _realtimeConversion;
+ @Nullable
private final File _consumerDir;
private final boolean _mutableSegmentCompacted;
+ @Nullable
private final int[] _mutableToImmutableDocIdMap;
- private final String _tableNameWithType;
+
+ // Error handling.
private final boolean _continueOnError;
- private Common(File indexDir, int lengthOfLongestEntry, int maxNumberOfMultiValueElements, int maxRowLengthInBytes,
- boolean onHeap, FieldSpec fieldSpec, boolean sorted, int cardinality, int totalNumberOfEntries, int totalDocs,
- boolean hasDictionary, Comparable<?> minValue, Comparable<?> maxValue, boolean forwardIndexDisabled,
- Object sortedUniqueElementsArray, boolean optimizeDictionary, boolean fixedLength, boolean textCommitOnClose,
- ColumnStatistics columnStatistics, boolean realtimeConversion, File consumerDir,
- boolean mutableSegmentCompacted, int[] mutableToImmutableDocIdMap, String tableNameWithType,
- boolean continueOnError) {
- _indexDir = indexDir;
- _lengthOfLongestEntry = lengthOfLongestEntry;
- _maxNumberOfMultiValueElements = maxNumberOfMultiValueElements;
- _maxRowLengthInBytes = maxRowLengthInBytes;
- _onHeap = onHeap;
- _fieldSpec = fieldSpec;
- _sorted = sorted;
- _cardinality = cardinality;
- _totalNumberOfEntries = totalNumberOfEntries;
- _totalDocs = totalDocs;
- _hasDictionary = hasDictionary;
- _minValue = minValue;
- _maxValue = maxValue;
- _forwardIndexDisabled = forwardIndexDisabled;
- _sortedUniqueElementsArray = sortedUniqueElementsArray;
- _optimizeDictionary = optimizeDictionary;
- _fixedLength = fixedLength;
- _textCommitOnClose = textCommitOnClose;
- _columnStatistics = columnStatistics;
- _realtimeConversion = realtimeConversion;
- _consumerDir = consumerDir;
- _mutableSegmentCompacted = mutableSegmentCompacted;
- _mutableToImmutableDocIdMap = mutableToImmutableDocIdMap;
- _tableNameWithType = tableNameWithType;
- _continueOnError = continueOnError;
- }
+ private Common(Builder builder) {
+ _tableNameWithType = builder._tableNameWithType;
+ _indexDir = builder._indexDir;
+ _columnShape = builder._columnShape;
+ _lengthOfShortestElement = builder._lengthOfShortestElement;
+ _lengthOfLongestElement = builder._lengthOfLongestElement;
+ _isAscii = builder._isAscii;
+ _totalNumberOfEntries = builder._totalNumberOfEntries;
+ _maxNumberOfMultiValues = builder._maxNumberOfMultiValues;
+ _maxRowLengthInBytes = builder._maxRowLengthInBytes;
+ _hasDictionary = builder._hasDictionary;
+ _onHeap = builder._onHeap;
+ _optimizeDictionary = builder._optimizedDictionary;
+ _textCommitOnClose = builder._textCommitOnClose;
+ _realtimeConversion = builder._realtimeConversion;
+ _consumerDir = builder._consumerDir;
+ _mutableSegmentCompacted = builder._mutableSegmentCompacted;
+ _mutableToImmutableDocIdMap = builder._mutableToImmutableDocIdMap;
+ _continueOnError = builder._continueOnError;
+ }
+
+ // Identity accessors.
- public FieldSpec getFieldSpec() {
- return _fieldSpec;
+ @Override
+ public String getTableNameWithType() {
+ return _tableNameWithType;
}
+ @Override
public File getIndexDir() {
return _indexDir;
}
- public boolean isOnHeap() {
- return _onHeap;
+ @Override
+ public FieldSpec getFieldSpec() {
+ return _columnShape.getFieldSpec();
}
- public int getLengthOfLongestEntry() {
- return _lengthOfLongestEntry;
+ @Override
+ @Nullable
+ public ColumnStatistics getColumnStatistics() {
+ return _columnShape instanceof ColumnStatistics ? (ColumnStatistics) _columnShape : null;
}
- public int getMaxNumberOfMultiValueElements() {
- return _maxNumberOfMultiValueElements;
+ @Override
+ @Nullable
+ public ColumnMetadata getColumnMetadata() {
+ return _columnShape instanceof ColumnMetadata ? (ColumnMetadata) _columnShape : null;
}
- public int getMaxRowLengthInBytes() {
- return _maxRowLengthInBytes;
+ // Non-overridable shape accessors delegate to the source [ColumnShape].
+
+ @Override
+ public int getTotalDocs() {
+ return _columnShape.getTotalDocs();
+ }
+
+ @Override
+ public int getCardinality() {
+ return _columnShape.getCardinality();
}
+ @Override
public boolean isSorted() {
- return _sorted;
+ return _columnShape.isSorted();
}
- public int getCardinality() {
- return _cardinality;
+ @Override
+ @Nullable
+ public Comparable<?> getMinValue() {
+ return _columnShape.getMinValue();
}
- public int getTotalNumberOfEntries() {
- return _totalNumberOfEntries;
+ @Override
+ @Nullable
+ public Comparable<?> getMaxValue() {
+ return _columnShape.getMaxValue();
}
- public int getTotalDocs() {
- return _totalDocs;
+ @Override
+ @Nullable
+ public PartitionFunction getPartitionFunction() {
+ return _columnShape.getPartitionFunction();
}
- public boolean hasDictionary() {
- return _hasDictionary;
+ @Override
+ @Nullable
+ public Set<Integer> getPartitions() {
+ return _columnShape.getPartitions();
}
+ // Overridable column-attribute accessors.
+
@Override
- public Comparable<?> getMinValue() {
- return _minValue;
+ public int getLengthOfShortestElement() {
+ return _lengthOfShortestElement;
}
@Override
- public Comparable<?> getMaxValue() {
- return _maxValue;
+ public int getLengthOfLongestElement() {
+ return _lengthOfLongestElement;
}
@Override
- public boolean forwardIndexDisabled() {
- return _forwardIndexDisabled;
+ public boolean isAscii() {
+ return _isAscii;
}
@Override
- public Object getSortedUniqueElementsArray() {
- return _sortedUniqueElementsArray;
+ public int getTotalNumberOfEntries() {
+ return _totalNumberOfEntries;
}
@Override
- public boolean isOptimizeDictionary() {
- return _optimizeDictionary;
+ public int getMaxNumberOfMultiValues() {
+ return _maxNumberOfMultiValues;
}
@Override
- public boolean isFixedLength() {
- return _fixedLength;
+ public int getMaxRowLengthInBytes() {
+ return _maxRowLengthInBytes;
}
@Override
- public boolean isTextCommitOnClose() {
- return _textCommitOnClose;
+ public boolean hasDictionary() {
+ return _hasDictionary;
}
+ // Build-time toggles.
+
@Override
- public ColumnStatistics getColumnStatistics() {
- return _columnStatistics;
+ public boolean isOnHeap() {
+ return _onHeap;
+ }
+
+ @Override
+ public boolean isOptimizeDictionary() {
+ return _optimizeDictionary;
}
+ @Override
+ public boolean isTextCommitOnClose() {
+ return _textCommitOnClose;
+ }
+
+ // Realtime-conversion context.
+
@Override
public boolean isRealtimeConversion() {
return _realtimeConversion;
}
@Override
+ @Nullable
public File getConsumerDir() {
return _consumerDir;
}
@@ -466,14 +460,12 @@ public interface IndexCreationContext {
}
@Override
+ @Nullable
public int[] getMutableToImmutableDocIdMap() {
return _mutableToImmutableDocIdMap;
}
- @Override
- public String getTableNameWithType() {
- return _tableNameWithType;
- }
+ // Error handling.
@Override
public boolean isContinueOnError() {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
index 45c423e66c5..ef00a4563ea 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.spi.index.metadata;
import com.google.common.collect.Maps;
+import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.math.BigDecimal;
import java.util.HashMap;
@@ -70,6 +71,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private final boolean _isAscii;
private final int _totalNumberOfEntries;
private final int _maxNumberOfMultiValues;
+ private final int _maxRowLengthInBytes;
private final int _bitsPerElement;
private final PartitionFunction _partitionFunction;
private final Set<Integer> _partitions;
@@ -85,7 +87,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
private ColumnMetadataImpl(FieldSpec fieldSpec, int totalDocs, int cardinality, boolean hasDictionary,
EncodingType forwardIndexEncoding, boolean sorted, @Nullable Comparable minValue, @Nullable Comparable maxValue,
boolean minMaxValueInvalid, int lengthOfShortestElement, int lengthOfLongestElement, boolean isAscii,
- int totalNumberOfEntries, int maxNumberOfMultiValues, int bitsPerElement,
+ int totalNumberOfEntries, int maxNumberOfMultiValues, int maxRowLengthInBytes, int bitsPerElement,
@Nullable PartitionFunction partitionFunction, @Nullable Set<Integer> partitions, boolean autoGenerated) {
_fieldSpec = fieldSpec;
_totalDocs = totalDocs;
@@ -101,6 +103,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
_isAscii = isAscii;
_bitsPerElement = bitsPerElement;
_maxNumberOfMultiValues = maxNumberOfMultiValues;
+ _maxRowLengthInBytes = maxRowLengthInBytes;
_totalNumberOfEntries = totalNumberOfEntries;
_partitionFunction = partitionFunction;
_partitions = partitions;
@@ -184,6 +187,11 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return _totalNumberOfEntries;
}
+ @Override
+ public int getMaxRowLengthInBytes() {
+ return _maxRowLengthInBytes;
+ }
+
@Nullable
@Override
public PartitionFunction getPartitionFunction() {
@@ -254,41 +262,65 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return false;
}
ColumnMetadataImpl that = (ColumnMetadataImpl) o;
- return _totalDocs == that._totalDocs && _cardinality == that._cardinality && _hasDictionary == that._hasDictionary
- && _forwardIndexEncoding == that._forwardIndexEncoding && _sorted == that._sorted
- && _minMaxValueInvalid == that._minMaxValueInvalid && _lengthOfShortestElement == that._lengthOfShortestElement
- && _lengthOfLongestElement == that._lengthOfLongestElement && _isAscii == that._isAscii
+ return _totalDocs == that._totalDocs
+ && _cardinality == that._cardinality
+ && _hasDictionary == that._hasDictionary
+ && _forwardIndexEncoding == that._forwardIndexEncoding
+ && _sorted == that._sorted
+ && _minMaxValueInvalid == that._minMaxValueInvalid
+ && _lengthOfShortestElement == that._lengthOfShortestElement
+ && _lengthOfLongestElement == that._lengthOfLongestElement
+ && _isAscii == that._isAscii
&& _totalNumberOfEntries == that._totalNumberOfEntries
- && _maxNumberOfMultiValues == that._maxNumberOfMultiValues && _bitsPerElement == that._bitsPerElement
- && _autoGenerated == that._autoGenerated && Objects.equals(_fieldSpec, that._fieldSpec) && Objects.equals(
- _minValue, that._minValue) && Objects.equals(_maxValue, that._maxValue) && Objects.equals(_partitionFunction,
- that._partitionFunction) && Objects.equals(_partitions, that._partitions) && Objects.equals(_indexTypeSizeList,
- that._indexTypeSizeList);
+ && _maxNumberOfMultiValues == that._maxNumberOfMultiValues
+ && _maxRowLengthInBytes == that._maxRowLengthInBytes
+ && _bitsPerElement == that._bitsPerElement
+ && _autoGenerated == that._autoGenerated
+ && Objects.equals(_fieldSpec, that._fieldSpec)
+ && Objects.equals(_minValue, that._minValue)
+ && Objects.equals(_maxValue, that._maxValue)
+ && Objects.equals(_partitionFunction, that._partitionFunction)
+ && Objects.equals(_partitions, that._partitions)
+ && Objects.equals(_indexTypeSizeList, that._indexTypeSizeList);
}
@Override
public int hashCode() {
return Objects.hash(_fieldSpec, _totalDocs, _cardinality, _hasDictionary, _forwardIndexEncoding, _sorted,
_minValue, _maxValue, _minMaxValueInvalid, _lengthOfShortestElement, _lengthOfLongestElement, _isAscii,
- _totalNumberOfEntries, _maxNumberOfMultiValues, _bitsPerElement, _partitionFunction, _partitions,
- _autoGenerated, _indexTypeSizeList);
+ _totalNumberOfEntries, _maxNumberOfMultiValues, _maxRowLengthInBytes, _bitsPerElement, _partitionFunction,
+ _partitions, _autoGenerated, _indexTypeSizeList);
}
@Override
public String toString() {
- return "ColumnMetadataImpl{" + "_fieldSpec=" + _fieldSpec + ", _totalDocs=" + _totalDocs + ", _cardinality="
- + _cardinality + ", _hasDictionary=" + _hasDictionary + ", _forwardIndexEncoding=" + _forwardIndexEncoding
- + ", _sorted=" + _sorted + ", _minValue=" + _minValue + ", _maxValue=" + _maxValue
- + ", _minMaxValueInvalid=" + _minMaxValueInvalid + ", _lengthOfShortestElement=" + _lengthOfShortestElement
- + ", _lengthOfLongestElement=" + _lengthOfLongestElement + ", _isAscii=" + _isAscii
- + ", _totalNumberOfEntries=" + _totalNumberOfEntries + ", _maxNumberOfMultiValues=" + _maxNumberOfMultiValues
- + ", _bitsPerElement=" + _bitsPerElement + ", _partitionFunction=" + _partitionFunction + ", _partitions="
- + _partitions + ", _autoGenerated=" + _autoGenerated + ", _indexTypeSizeList=" + _indexTypeSizeList + '}';
+ return "ColumnMetadataImpl{"
+ + "_fieldSpec=" + _fieldSpec
+ + ", _totalDocs=" + _totalDocs
+ + ", _cardinality=" + _cardinality
+ + ", _hasDictionary=" + _hasDictionary
+ + ", _forwardIndexEncoding=" + _forwardIndexEncoding
+ + ", _sorted=" + _sorted
+ + ", _minValue=" + _minValue
+ + ", _maxValue=" + _maxValue
+ + ", _minMaxValueInvalid=" + _minMaxValueInvalid
+ + ", _lengthOfShortestElement=" + _lengthOfShortestElement
+ + ", _lengthOfLongestElement=" + _lengthOfLongestElement
+ + ", _isAscii=" + _isAscii
+ + ", _totalNumberOfEntries=" + _totalNumberOfEntries
+ + ", _maxNumberOfMultiValues=" + _maxNumberOfMultiValues
+ + ", _maxRowLengthInBytes=" + _maxRowLengthInBytes
+ + ", _bitsPerElement=" + _bitsPerElement
+ + ", _partitionFunction=" + _partitionFunction
+ + ", _partitions=" + _partitions
+ + ", _autoGenerated=" + _autoGenerated
+ + ", _indexTypeSizeList=" + _indexTypeSizeList
+ + '}';
}
public static ColumnMetadataImpl fromPropertiesConfiguration(PropertiesConfiguration config, int totalDocs,
String column) {
- FieldSpec fieldSpec = generateFieldSpec(column, config);
+ FieldSpec fieldSpec = extractFieldSpec(column, config);
boolean hasDictionary = config.getBoolean(Column.getKeyFor(column, Column.HAS_DICTIONARY), true);
// Old segments may not have FORWARD_INDEX_ENCODING — derive from HAS_DICTIONARY in that case.
EncodingType forwardIndexEncoding = config.getEnum(Column.getKeyFor(column, Column.FORWARD_INDEX_ENCODING),
@@ -337,37 +369,16 @@ public class ColumnMetadataImpl implements ColumnMetadata {
}
// Set partition function
- String partitionFunctionName = config.getString(Column.getKeyFor(column, Column.PARTITION_FUNCTION), null);
- if (partitionFunctionName != null) {
- int numPartitions = config.getInt(Column.getKeyFor(column, Column.NUM_PARTITIONS));
- Configuration partitionFunctionConfig = config.subset(Column.getKeyFor(column, Column.PARTITION_FUNCTION_CONFIG));
- Map<String, String> partitionFunctionConfigMap;
- if (!partitionFunctionConfig.isEmpty()) {
- partitionFunctionConfigMap = new HashMap<>();
- partitionFunctionConfig.forEach((k, v) -> {
- // NOTE:
- // A partition function config value can have comma and this value is read as a List from
- // PropertiesConfiguration.getProperty, hence we need to rebuild original comma separated string value from
- // this list of values.
- partitionFunctionConfigMap.put(k,
- v instanceof List ? String.join(",", (List) v)
- : v.toString());
- });
- } else {
- partitionFunctionConfigMap = null;
- }
- PartitionFunction partitionFunction =
- PartitionFunctionFactory.getPartitionFunction(partitionFunctionName, numPartitions,
- partitionFunctionConfigMap);
+ PartitionFunction partitionFunction = extractPartitionFunction(column, config);
+ if (partitionFunction != null) {
builder.setPartitionFunction(partitionFunction);
- builder.setPartitions(
- ColumnPartitionMetadata.extractPartitions(config.getList(Column.getKeyFor(column, Column.PARTITION_VALUES))));
+ builder.setPartitions(extractPartitions(column, config));
}
return builder.build();
}
- public static FieldSpec generateFieldSpec(String column, PropertiesConfiguration config) {
+ public static FieldSpec extractFieldSpec(String column, PropertiesConfiguration config) {
String fieldName = config.getString(Column.getKeyFor(column, Column.COLUMN_NAME), column);
FieldType fieldType = config.getEnum(Column.getKeyFor(column, Column.COLUMN_TYPE), FieldType.class);
DataType dataType = config.getEnum(Column.getKeyFor(column, Column.DATA_TYPE), DataType.class);
@@ -400,7 +411,7 @@ public class ColumnMetadataImpl implements ColumnMetadata {
Map<String, FieldSpec> childFieldSpecs = new HashMap<>();
for (String childField : childFieldNames) {
childFieldSpecs.put(childField,
- generateFieldSpec(ComplexFieldSpec.getFullChildName(column, childField), config));
+ extractFieldSpec(ComplexFieldSpec.getFullChildName(column, childField), config));
}
return new ComplexFieldSpec(fieldName, dataType, true, childFieldSpecs);
default:
@@ -408,6 +419,35 @@ public class ColumnMetadataImpl implements ColumnMetadata {
}
}
+ @Nullable
+ public static PartitionFunction extractPartitionFunction(String column, PropertiesConfiguration config) {
+ String partitionFunctionName = config.getString(Column.getKeyFor(column, Column.PARTITION_FUNCTION), null);
+ if (partitionFunctionName == null) {
+ return null;
+ }
+ int numPartitions = config.getInt(Column.getKeyFor(column, Column.NUM_PARTITIONS));
+ Configuration partitionFunctionConfig = config.subset(Column.getKeyFor(column, Column.PARTITION_FUNCTION_CONFIG));
+ Map<String, String> partitionFunctionConfigMap;
+ if (!partitionFunctionConfig.isEmpty()) {
+ partitionFunctionConfigMap = new HashMap<>();
+ partitionFunctionConfig.forEach((k, v) -> {
+ // NOTE:
+ // A partition function config value can have comma and this value is read as a List from
+ // PropertiesConfiguration.getProperty, hence we need to rebuild original comma separated string value from
+ // this list of values.
+ partitionFunctionConfigMap.put(k, v instanceof List ? String.join(",", (List) v) : v.toString());
+ });
+ } else {
+ partitionFunctionConfigMap = null;
+ }
+ return PartitionFunctionFactory.getPartitionFunction(partitionFunctionName, numPartitions,
+ partitionFunctionConfigMap);
+ }
+
+ public static IntSet extractPartitions(String column, PropertiesConfiguration config) {
+ return ColumnPartitionMetadata.extractPartitions(config.getList(Column.getKeyFor(column, Column.PARTITION_VALUES)));
+ }
+
private static Comparable parseValue(DataType storedType, String column, String valueString) {
switch (storedType) {
case INT:
@@ -581,9 +621,17 @@ public class ColumnMetadataImpl implements ColumnMetadata {
}
// Canonicalize MV related fields
+ int maxRowLengthInBytes;
if (_fieldSpec.isSingleValueField()) {
_totalNumberOfEntries = _totalDocs;
_maxNumberOfMultiValues = 0;
+ maxRowLengthInBytes = _lengthOfLongestElement;
+ } else {
+ if (_lengthOfShortestElement >= 0 && _lengthOfShortestElement == _lengthOfLongestElement) {
+ maxRowLengthInBytes = _maxNumberOfMultiValues * _lengthOfLongestElement;
+ } else {
+ maxRowLengthInBytes = UNAVAILABLE;
+ }
}
// Canonicalize bits per element
@@ -597,8 +645,8 @@ public class ColumnMetadataImpl implements ColumnMetadata {
return new ColumnMetadataImpl(_fieldSpec, _totalDocs, _cardinality, _hasDictionary, _forwardIndexEncoding,
_sorted, _minValue, _maxValue, _minMaxValueInvalid, _lengthOfShortestElement, _lengthOfLongestElement,
- _isAscii, _totalNumberOfEntries, _maxNumberOfMultiValues, _bitsPerElement, _partitionFunction, _partitions,
- _autoGenerated);
+ _isAscii, _totalNumberOfEntries, _maxNumberOfMultiValues, maxRowLengthInBytes, _bitsPerElement,
+ _partitionFunction, _partitions, _autoGenerated);
}
}
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/EmptyColumnMetadata.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/EmptyColumnMetadata.java
new file mode 100644
index 00000000000..241c9e22649
--- /dev/null
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/EmptyColumnMetadata.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.spi.index.metadata;
+
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.EmptyColumnShape;
+import org.apache.pinot.segment.spi.index.IndexType;
+import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+/// [ColumnMetadata] for an empty column (zero rows).
+public class EmptyColumnMetadata extends EmptyColumnShape implements ColumnMetadata {
+
+ public EmptyColumnMetadata(FieldSpec fieldSpec, @Nullable PartitionFunction partitionFunction,
+ @Nullable Set<Integer> partitions) {
+ super(fieldSpec, partitionFunction, partitions);
+ }
+
+ /// Constructs an [EmptyColumnMetadata] for `column` by reading the field spec and partition info from `config`.
+ public static EmptyColumnMetadata fromPropertiesConfiguration(PropertiesConfiguration config, String column) {
+ FieldSpec fieldSpec = ColumnMetadataImpl.extractFieldSpec(column, config);
+ PartitionFunction partitionFunction = ColumnMetadataImpl.extractPartitionFunction(column, config);
+ Set<Integer> partitions =
+ partitionFunction != null ? ColumnMetadataImpl.extractPartitions(column, config) : null;
+ return new EmptyColumnMetadata(fieldSpec, partitionFunction, partitions);
+ }
+
+ @Override
+ public boolean hasDictionary() {
+ return false;
+ }
+
+ @Override
+ public EncodingType getForwardIndexEncoding() {
+ return EncodingType.RAW;
+ }
+
+ @Override
+ public boolean isMinMaxValueInvalid() {
+ return true;
+ }
+
+ @Override
+ public int getBitsPerElement() {
+ return UNAVAILABLE;
+ }
+
+ @Override
+ public boolean isAutoGenerated() {
+ return false;
+ }
+
+ @Override
+ public int getNumIndexes() {
+ return 0;
+ }
+
+ @Override
+ public short getIndexType(int position) {
+ throw new IndexOutOfBoundsException(position);
+ }
+
+ @Override
+ public long getIndexSize(int position) {
+ throw new IndexOutOfBoundsException(position);
+ }
+
+ @Override
+ public long getIndexSizeFor(IndexType type) {
+ return UNAVAILABLE;
+ }
+
+ // NOTE: This method is only meant to retain compatibility of serialization for endpoint:
+ // `/tables/{tableName}/segments/{segmentName}/metadata`
+ @SuppressWarnings("unused")
+ public Map<IndexType<?, ?, ?>, Long> getIndexSizeMap() {
+ return Map.of();
+ }
+}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
index 6064ab39f1e..6c67b8f629b 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
@@ -74,7 +74,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
private static final Logger LOGGER = LoggerFactory.getLogger(SegmentMetadataImpl.class);
private final File _indexDir;
- private final TreeMap<String, ColumnMetadataImpl> _columnMetadataMap;
+ private final TreeMap<String, ColumnMetadata> _columnMetadataMap;
private final Schema _schema;
private String _segmentName;
private int _totalDocs;
@@ -238,34 +238,44 @@ public class SegmentMetadataImpl implements SegmentMetadata {
addPhysicalColumns(segmentMetadata.getList(Segment.DATETIME_COLUMNS), physicalColumns);
addPhysicalColumns(segmentMetadata.getList(Segment.COMPLEX_COLUMNS), physicalColumns);
- // Build column metadata map and schema.
- for (String column : physicalColumns) {
- ColumnMetadataImpl columnMetadata =
- ColumnMetadataImpl.fromPropertiesConfiguration(segmentMetadata, _totalDocs, column);
- _columnMetadataMap.put(column, columnMetadata);
- _schema.addField(columnMetadata.getFieldSpec());
- }
+ // Build column metadata map and schema. Empty segments use a stripped-down [EmptyColumnMetadata] since the
+ // shape stats (cardinality, element lengths, etc.) are meaningless when there are no rows.
+ if (_totalDocs > 0) {
+ for (String column : physicalColumns) {
+ ColumnMetadata columnMetadata =
+ ColumnMetadataImpl.fromPropertiesConfiguration(segmentMetadata, _totalDocs, column);
+ _columnMetadataMap.put(column, columnMetadata);
+ _schema.addField(columnMetadata.getFieldSpec());
+ }
- // Load index metadata
- // Support V3 (e.g. SingleFileIndexDirectory only)
- if (_segmentVersion == SegmentVersion.v3) {
- File indexMapFile = new File(_indexDir, "v3" + File.separator + V1Constants.INDEX_MAP_FILE_NAME);
- if (indexMapFile.exists()) {
- IndexService indexService = IndexService.getInstance();
-
- PropertiesConfiguration mapConfig = CommonsConfigurationUtils.fromFile(indexMapFile);
- for (String key : CommonsConfigurationUtils.getKeys(mapConfig)) {
- try {
- String[] parsedKeys = ColumnIndexUtils.parseIndexMapKeys(key, _indexDir.getPath());
- if (parsedKeys[2].equals(ColumnIndexUtils.MAP_KEY_NAME_SIZE)) {
- short indexType = indexService.getNumericId(parsedKeys[1]);
- _columnMetadataMap.get(parsedKeys[0]).addIndexSize(indexType, mapConfig.getLong(key));
+ // Load index metadata
+ // Support V3 (e.g. SingleFileIndexDirectory only). Skip for empty segments — there is no payload to size up,
+ // and [EmptyColumnMetadata] does not support `addIndexSize`.
+ if (_segmentVersion == SegmentVersion.v3) {
+ File indexMapFile = new File(_indexDir, "v3" + File.separator + V1Constants.INDEX_MAP_FILE_NAME);
+ if (indexMapFile.exists()) {
+ IndexService indexService = IndexService.getInstance();
+ PropertiesConfiguration mapConfig = CommonsConfigurationUtils.fromFile(indexMapFile);
+ for (String key : CommonsConfigurationUtils.getKeys(mapConfig)) {
+ try {
+ String[] parsedKeys = ColumnIndexUtils.parseIndexMapKeys(key, _indexDir.getPath());
+ if (parsedKeys[2].equals(ColumnIndexUtils.MAP_KEY_NAME_SIZE)) {
+ short indexType = indexService.getNumericId(parsedKeys[1]);
+ ((ColumnMetadataImpl) _columnMetadataMap.get(parsedKeys[0])).addIndexSize(indexType,
+ mapConfig.getLong(key));
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Unable to load index metadata in {} for {}!", indexMapFile, key, e);
}
- } catch (Exception e) {
- LOGGER.debug("Unable to load index metadata in {} for {}!", indexMapFile, key, e);
}
}
}
+ } else {
+ for (String column : physicalColumns) {
+ ColumnMetadata columnMetadata = EmptyColumnMetadata.fromPropertiesConfiguration(segmentMetadata, column);
+ _columnMetadataMap.put(column, columnMetadata);
+ _schema.addField(columnMetadata.getFieldSpec());
+ }
}
// Build star-tree v2 metadata
@@ -485,7 +495,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
@Override
public TreeMap<String, ColumnMetadata> getColumnMetadataMap() {
- return (TreeMap<String, ColumnMetadata>) (TreeMap<String, ?>) _columnMetadataMap;
+ return _columnMetadataMap;
}
@Override
@@ -540,7 +550,7 @@ public class SegmentMetadataImpl implements SegmentMetadata {
if (_columnMetadataMap != null) {
ArrayNode columnsMetadata = JsonUtils.newArrayNode();
- for (Map.Entry<String, ColumnMetadataImpl> entry : _columnMetadataMap.entrySet()) {
+ for (Map.Entry<String, ColumnMetadata> entry : _columnMetadataMap.entrySet()) {
if (columnFilter == null || columnFilter.contains(entry.getKey())) {
columnsMetadata.add(JsonUtils.objectToJsonNode(entry.getValue()));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org